2024-08-17T17:02:37.868 INFO:root:teuthology version: 1.2.0 2024-08-17T17:02:37.894 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-08-17T17:02:37.991 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858336 branch: main description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]'' && ceph orch upgrade status | jq ''.in_progress'' | grep true && ! ceph orch upgrade status | jq ''.message'' | grep Error; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' first_in_suite: false job_id: '7858336' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'librbd C and C++ api tests ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(POOL_APP_NOT_ENABLED\) - OSD_DOWN - mons down - mon down - MON_DOWN - out of quorum - PG_DEGRADED - Reduced data availability - Degraded data redundancy - OSDMAP_FLAGS - PG_AVAILABILITY sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: main sha1: 91f88e4d132ac08dc53355fa3d9717a53236a3a9 owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 17 sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 11/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 91f88e4d132ac08dc53355fa3d9717a53236a3a9 targets: smithi130.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIRH5rFRy+pv8dQkKRz5DNmCdP25cGUDRKUKproAYVzJPKP9JPulEO5Zd7B0jv9SkYLRXVGFdY8dWVl4c86sZSY= smithi179.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBRW4uvlvXC3x65EZj+m9HRUFQUPs43ZA41kCSjF/9H0SXkC4u+mEiwPD7qZFr/+2qub2L73RRVOPeL5N7bmnGM= tasks: - pexec: all: - sudo dnf install runc nvmetcli nvme-cli -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 67db276914d29505c9707d04f3a3b8fb9d8aee4b timestamp: 2024-08-17_03:08:15 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-08-17T17:02:37.992 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa; will attempt to use it 2024-08-17T17:02:37.992 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks 2024-08-17T17:02:37.993 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-08-17T17:02:37.994 INFO:teuthology.task.internal:Checking packages... 2024-08-17T17:02:38.015 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453' 2024-08-17T17:02:38.015 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T17:02:38.015 INFO:teuthology.packaging:ref: None 2024-08-17T17:02:38.015 INFO:teuthology.packaging:tag: None 2024-08-17T17:02:38.015 INFO:teuthology.packaging:branch: main 2024-08-17T17:02:38.015 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:02:38.015 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-08-17T17:02:38.248 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-5842.g4887e20f 2024-08-17T17:02:38.250 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-08-17T17:02:38.257 INFO:teuthology.task.internal:no buildpackages task found 2024-08-17T17:02:38.257 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-08-17T17:02:38.263 INFO:teuthology.task.internal:Saving configuration 2024-08-17T17:02:38.285 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-08-17T17:02:38.292 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-08-17T17:02:38.320 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi130.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858336', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-17 16:55:53.596375', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIRH5rFRy+pv8dQkKRz5DNmCdP25cGUDRKUKproAYVzJPKP9JPulEO5Zd7B0jv9SkYLRXVGFdY8dWVl4c86sZSY='} 2024-08-17T17:02:38.352 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi179.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858336', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-17 16:55:53.598278', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBRW4uvlvXC3x65EZj+m9HRUFQUPs43ZA41kCSjF/9H0SXkC4u+mEiwPD7qZFr/+2qub2L73RRVOPeL5N7bmnGM='} 2024-08-17T17:02:38.352 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-08-17T17:02:38.359 INFO:teuthology.task.internal:roles: ubuntu@smithi130.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-08-17T17:02:38.359 INFO:teuthology.task.internal:roles: ubuntu@smithi179.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-08-17T17:02:38.359 INFO:teuthology.run_tasks:Running task console_log... 2024-08-17T17:02:38.437 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f39e282fac0>, signals=[15]) 2024-08-17T17:02:38.437 INFO:teuthology.run_tasks:Running task internal.connect... 2024-08-17T17:02:38.443 INFO:teuthology.task.internal:Opening connections... 2024-08-17T17:02:38.443 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi130.front.sepia.ceph.com 2024-08-17T17:02:38.444 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T17:02:38.526 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi179.front.sepia.ceph.com 2024-08-17T17:02:38.527 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T17:02:38.624 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-08-17T17:02:38.631 DEBUG:teuthology.orchestra.run.smithi130:> uname -m 2024-08-17T17:02:38.649 INFO:teuthology.orchestra.run.smithi130.stdout:x86_64 2024-08-17T17:02:38.649 DEBUG:teuthology.orchestra.run.smithi130:> cat /etc/os-release 2024-08-17T17:02:38.706 INFO:teuthology.orchestra.run.smithi130.stdout:NAME="CentOS Stream" 2024-08-17T17:02:38.706 INFO:teuthology.orchestra.run.smithi130.stdout:VERSION="9" 2024-08-17T17:02:38.707 INFO:teuthology.orchestra.run.smithi130.stdout:ID="centos" 2024-08-17T17:02:38.707 INFO:teuthology.orchestra.run.smithi130.stdout:ID_LIKE="rhel fedora" 2024-08-17T17:02:38.707 INFO:teuthology.orchestra.run.smithi130.stdout:VERSION_ID="9" 2024-08-17T17:02:38.707 INFO:teuthology.orchestra.run.smithi130.stdout:PLATFORM_ID="platform:el9" 2024-08-17T17:02:38.707 INFO:teuthology.orchestra.run.smithi130.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-17T17:02:38.707 INFO:teuthology.orchestra.run.smithi130.stdout:ANSI_COLOR="0;31" 2024-08-17T17:02:38.707 INFO:teuthology.orchestra.run.smithi130.stdout:LOGO="fedora-logo-icon" 2024-08-17T17:02:38.707 INFO:teuthology.orchestra.run.smithi130.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-17T17:02:38.707 INFO:teuthology.orchestra.run.smithi130.stdout:HOME_URL="https://centos.org/" 2024-08-17T17:02:38.707 INFO:teuthology.orchestra.run.smithi130.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-08-17T17:02:38.707 INFO:teuthology.orchestra.run.smithi130.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-17T17:02:38.707 INFO:teuthology.orchestra.run.smithi130.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-17T17:02:38.708 INFO:teuthology.lock.ops:Updating smithi130.front.sepia.ceph.com on lock server 2024-08-17T17:02:38.734 DEBUG:teuthology.orchestra.run.smithi179:> uname -m 2024-08-17T17:02:38.751 INFO:teuthology.orchestra.run.smithi179.stdout:x86_64 2024-08-17T17:02:38.752 DEBUG:teuthology.orchestra.run.smithi179:> cat /etc/os-release 2024-08-17T17:02:38.809 INFO:teuthology.orchestra.run.smithi179.stdout:NAME="CentOS Stream" 2024-08-17T17:02:38.809 INFO:teuthology.orchestra.run.smithi179.stdout:VERSION="9" 2024-08-17T17:02:38.809 INFO:teuthology.orchestra.run.smithi179.stdout:ID="centos" 2024-08-17T17:02:38.809 INFO:teuthology.orchestra.run.smithi179.stdout:ID_LIKE="rhel fedora" 2024-08-17T17:02:38.809 INFO:teuthology.orchestra.run.smithi179.stdout:VERSION_ID="9" 2024-08-17T17:02:38.809 INFO:teuthology.orchestra.run.smithi179.stdout:PLATFORM_ID="platform:el9" 2024-08-17T17:02:38.809 INFO:teuthology.orchestra.run.smithi179.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-17T17:02:38.809 INFO:teuthology.orchestra.run.smithi179.stdout:ANSI_COLOR="0;31" 2024-08-17T17:02:38.809 INFO:teuthology.orchestra.run.smithi179.stdout:LOGO="fedora-logo-icon" 2024-08-17T17:02:38.809 INFO:teuthology.orchestra.run.smithi179.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-17T17:02:38.809 INFO:teuthology.orchestra.run.smithi179.stdout:HOME_URL="https://centos.org/" 2024-08-17T17:02:38.810 INFO:teuthology.orchestra.run.smithi179.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-08-17T17:02:38.810 INFO:teuthology.orchestra.run.smithi179.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-17T17:02:38.810 INFO:teuthology.orchestra.run.smithi179.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-17T17:02:38.810 INFO:teuthology.lock.ops:Updating smithi179.front.sepia.ceph.com on lock server 2024-08-17T17:02:38.842 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-08-17T17:02:38.851 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-08-17T17:02:38.871 INFO:teuthology.task.internal:Checking for old test directory... 2024-08-17T17:02:38.871 DEBUG:teuthology.orchestra.run.smithi130:> test '!' -e /home/ubuntu/cephtest 2024-08-17T17:02:38.874 DEBUG:teuthology.orchestra.run.smithi179:> test '!' -e /home/ubuntu/cephtest 2024-08-17T17:02:38.889 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-08-17T17:02:38.895 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-08-17T17:02:38.896 DEBUG:teuthology.orchestra.run.smithi130:> test -z $(ls -A /var/lib/ceph) 2024-08-17T17:02:38.930 DEBUG:teuthology.orchestra.run.smithi179:> test -z $(ls -A /var/lib/ceph) 2024-08-17T17:02:38.955 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-08-17T17:02:39.024 INFO:teuthology.run_tasks:Running task kernel... 2024-08-17T17:02:39.038 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-08-17T17:02:39.038 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-08-17T17:02:39.038 DEBUG:teuthology.orchestra.run.smithi130:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-17T17:02:39.039 DEBUG:teuthology.orchestra.run.smithi179:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-17T17:02:39.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:02:39.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:02:39.055 DEBUG:teuthology.orchestra.run.smithi179:> uname -r 2024-08-17T17:02:39.056 DEBUG:teuthology.orchestra.run.smithi130:> uname -r 2024-08-17T17:02:39.112 INFO:teuthology.orchestra.run.smithi179.stdout:5.14.0-437.el9.x86_64 2024-08-17T17:02:39.112 INFO:teuthology.task.kernel:Running kernel on smithi179: 5.14.0-437.el9.x86_64 2024-08-17T17:02:39.113 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum install -y kernel 2024-08-17T17:02:39.113 INFO:teuthology.orchestra.run.smithi130.stdout:5.14.0-437.el9.x86_64 2024-08-17T17:02:39.114 INFO:teuthology.task.kernel:Running kernel on smithi130: 5.14.0-437.el9.x86_64 2024-08-17T17:02:39.114 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum install -y kernel 2024-08-17T17:02:42.009 INFO:teuthology.orchestra.run.smithi179.stdout:CentOS Stream 9 - BaseOS 6.3 MB/s | 8.2 MB 00:01 2024-08-17T17:02:42.588 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - BaseOS 4.5 MB/s | 8.2 MB 00:01 2024-08-17T17:02:48.044 INFO:teuthology.orchestra.run.smithi179.stdout:CentOS Stream 9 - AppStream 4.1 MB/s | 20 MB 00:04 2024-08-17T17:02:50.322 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - AppStream 3.0 MB/s | 20 MB 00:06 2024-08-17T17:02:52.361 INFO:teuthology.orchestra.run.smithi179.stdout:CentOS Stream 9 - CRB 13 MB/s | 6.6 MB 00:00 2024-08-17T17:02:53.945 INFO:teuthology.orchestra.run.smithi179.stdout:CentOS Stream 9 - Extras packages 77 kB/s | 18 kB 00:00 2024-08-17T17:02:54.756 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.6 MB 00:00 2024-08-17T17:02:55.100 INFO:teuthology.orchestra.run.smithi179.stdout:Extra Packages for Enterprise Linux 22 MB/s | 22 MB 00:01 2024-08-17T17:02:56.382 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - Extras packages 69 kB/s | 18 kB 00:00 2024-08-17T17:02:57.944 INFO:teuthology.orchestra.run.smithi130.stdout:Extra Packages for Enterprise Linux 16 MB/s | 22 MB 00:01 2024-08-17T17:03:00.439 INFO:teuthology.orchestra.run.smithi179.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-08-17T17:03:02.446 INFO:teuthology.orchestra.run.smithi179.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-17T17:03:02.446 INFO:teuthology.orchestra.run.smithi179.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-08-17T17:03:02.545 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:03:02.551 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:03:02.552 INFO:teuthology.orchestra.run.smithi179.stdout: Package Architecture Version Repository Size 2024-08-17T17:03:02.552 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:03:02.552 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:03:02.552 INFO:teuthology.orchestra.run.smithi179.stdout: kernel x86_64 5.14.0-480.el9 baseos 1.5 M 2024-08-17T17:03:02.552 INFO:teuthology.orchestra.run.smithi179.stdout:Installing dependencies: 2024-08-17T17:03:02.552 INFO:teuthology.orchestra.run.smithi179.stdout: kernel-core x86_64 5.14.0-480.el9 baseos 17 M 2024-08-17T17:03:02.552 INFO:teuthology.orchestra.run.smithi179.stdout: kernel-modules x86_64 5.14.0-480.el9 baseos 36 M 2024-08-17T17:03:02.552 INFO:teuthology.orchestra.run.smithi179.stdout: kernel-modules-core x86_64 5.14.0-480.el9 baseos 30 M 2024-08-17T17:03:02.552 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:03:02.552 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:03:02.552 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:03:02.552 INFO:teuthology.orchestra.run.smithi179.stdout:Install 4 Packages 2024-08-17T17:03:02.553 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:03:02.553 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 84 M 2024-08-17T17:03:02.553 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 126 M 2024-08-17T17:03:02.553 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:03:02.924 INFO:teuthology.orchestra.run.smithi179.stdout:(1/4): kernel-5.14.0-480.el9.x86_64.rpm 6.5 MB/s | 1.5 MB 00:00 2024-08-17T17:03:03.216 INFO:teuthology.orchestra.run.smithi179.stdout:(2/4): kernel-core-5.14.0-480.el9.x86_64.rpm 33 MB/s | 17 MB 00:00 2024-08-17T17:03:03.391 INFO:teuthology.orchestra.run.smithi130.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-08-17T17:03:03.616 INFO:teuthology.orchestra.run.smithi179.stdout:(3/4): kernel-modules-core-5.14.0-480.el9.x86_6 43 MB/s | 30 MB 00:00 2024-08-17T17:03:03.933 INFO:teuthology.orchestra.run.smithi179.stdout:(4/4): kernel-modules-5.14.0-480.el9.x86_64.rpm 29 MB/s | 36 MB 00:01 2024-08-17T17:03:03.934 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:03:03.934 INFO:teuthology.orchestra.run.smithi179.stdout:Total 61 MB/s | 84 MB 00:01 2024-08-17T17:03:04.669 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:03:05.016 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:03:05.016 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:03:05.373 INFO:teuthology.orchestra.run.smithi130.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-17T17:03:05.374 INFO:teuthology.orchestra.run.smithi130.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-08-17T17:03:05.472 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:03:05.484 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:03:05.484 INFO:teuthology.orchestra.run.smithi130.stdout: Package Architecture Version Repository Size 2024-08-17T17:03:05.484 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:03:05.484 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:03:05.484 INFO:teuthology.orchestra.run.smithi130.stdout: kernel x86_64 5.14.0-480.el9 baseos 1.5 M 2024-08-17T17:03:05.484 INFO:teuthology.orchestra.run.smithi130.stdout:Installing dependencies: 2024-08-17T17:03:05.484 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-core x86_64 5.14.0-480.el9 baseos 17 M 2024-08-17T17:03:05.484 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-modules x86_64 5.14.0-480.el9 baseos 36 M 2024-08-17T17:03:05.484 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-modules-core x86_64 5.14.0-480.el9 baseos 30 M 2024-08-17T17:03:05.484 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:03:05.485 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:03:05.485 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:03:05.485 INFO:teuthology.orchestra.run.smithi130.stdout:Install 4 Packages 2024-08-17T17:03:05.485 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:03:05.485 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 84 M 2024-08-17T17:03:05.485 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 126 M 2024-08-17T17:03:05.485 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:03:05.852 INFO:teuthology.orchestra.run.smithi130.stdout:(1/4): kernel-5.14.0-480.el9.x86_64.rpm 4.6 MB/s | 1.5 MB 00:00 2024-08-17T17:03:06.004 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:03:06.004 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:03:08.473 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:03:09.135 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : kernel-modules-core-5.14.0-480.el9.x86_64 1/4 2024-08-17T17:03:09.229 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T17:03:09.868 INFO:teuthology.orchestra.run.smithi130.stdout:(2/4): kernel-core-5.14.0-480.el9.x86_64.rpm 3.9 MB/s | 17 MB 00:04 2024-08-17T17:03:10.393 INFO:teuthology.orchestra.run.smithi130.stdout:(3/4): kernel-modules-5.14.0-480.el9.x86_64.rpm 7.4 MB/s | 36 MB 00:04 2024-08-17T17:03:10.860 INFO:teuthology.orchestra.run.smithi130.stdout:(4/4): kernel-modules-core-5.14.0-480.el9.x86_6 6.0 MB/s | 30 MB 00:05 2024-08-17T17:03:10.860 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:03:10.860 INFO:teuthology.orchestra.run.smithi130.stdout:Total 16 MB/s | 84 MB 00:05 2024-08-17T17:03:11.596 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:03:11.878 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T17:03:11.942 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:03:11.942 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:03:12.021 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T17:03:12.943 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:03:12.944 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:03:15.389 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:03:16.061 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : kernel-modules-core-5.14.0-480.el9.x86_64 1/4 2024-08-17T17:03:16.161 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T17:03:16.833 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T17:03:16.914 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T17:03:18.813 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T17:03:18.952 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T17:03:21.764 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T17:03:23.786 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T17:03:23.860 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T17:03:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T17:03:51.586 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T17:03:51.626 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 4/4 2024-08-17T17:03:52.038 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T17:03:52.038 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : kernel-5.14.0-480.el9.x86_64 1/4 2024-08-17T17:03:52.038 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T17:03:52.038 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T17:03:52.841 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T17:03:52.842 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:03:52.842 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:03:52.842 INFO:teuthology.orchestra.run.smithi179.stdout: kernel-5.14.0-480.el9.x86_64 2024-08-17T17:03:52.842 INFO:teuthology.orchestra.run.smithi179.stdout: kernel-core-5.14.0-480.el9.x86_64 2024-08-17T17:03:52.842 INFO:teuthology.orchestra.run.smithi179.stdout: kernel-modules-5.14.0-480.el9.x86_64 2024-08-17T17:03:52.842 INFO:teuthology.orchestra.run.smithi179.stdout: kernel-modules-core-5.14.0-480.el9.x86_64 2024-08-17T17:03:52.842 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:03:52.842 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:03:53.256 DEBUG:teuthology.orchestra.run.smithi179:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T17:03:53.334 INFO:teuthology.orchestra.run.smithi179.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T17:03:53.334 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T17:03:53.334 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-480.el9.x86_64 2024-08-17T17:03:53.334 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-08-17T17:03:53.335 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-08-17T17:03:53.335 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-08-17T17:03:53.335 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi179.front.sepia.ceph.com, path=None, version=distro) 2024-08-17T17:03:53.335 DEBUG:teuthology.orchestra.run.smithi179:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T17:03:53.391 INFO:teuthology.orchestra.run.smithi179.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T17:03:53.391 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T17:03:53.392 DEBUG:teuthology.orchestra.run.smithi179:> sudo rpm -qi grub2-tools 2024-08-17T17:03:53.422 INFO:teuthology.orchestra.run.smithi179.stdout:Name : grub2-tools 2024-08-17T17:03:53.422 INFO:teuthology.orchestra.run.smithi179.stdout:Epoch : 1 2024-08-17T17:03:53.422 INFO:teuthology.orchestra.run.smithi179.stdout:Version : 2.06 2024-08-17T17:03:53.422 INFO:teuthology.orchestra.run.smithi179.stdout:Release : 68.el9 2024-08-17T17:03:53.422 INFO:teuthology.orchestra.run.smithi179.stdout:Architecture: x86_64 2024-08-17T17:03:53.422 INFO:teuthology.orchestra.run.smithi179.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-08-17T17:03:53.422 INFO:teuthology.orchestra.run.smithi179.stdout:Group : Unspecified 2024-08-17T17:03:53.423 INFO:teuthology.orchestra.run.smithi179.stdout:Size : 8264254 2024-08-17T17:03:53.423 INFO:teuthology.orchestra.run.smithi179.stdout:License : GPLv3+ 2024-08-17T17:03:53.423 INFO:teuthology.orchestra.run.smithi179.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-08-17T17:03:53.423 INFO:teuthology.orchestra.run.smithi179.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-08-17T17:03:53.423 INFO:teuthology.orchestra.run.smithi179.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-08-17T17:03:53.423 INFO:teuthology.orchestra.run.smithi179.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-08-17T17:03:53.423 INFO:teuthology.orchestra.run.smithi179.stdout:Packager : builder@centos.org 2024-08-17T17:03:53.423 INFO:teuthology.orchestra.run.smithi179.stdout:Vendor : CentOS 2024-08-17T17:03:53.423 INFO:teuthology.orchestra.run.smithi179.stdout:URL : http://www.gnu.org/software/grub/ 2024-08-17T17:03:53.423 INFO:teuthology.orchestra.run.smithi179.stdout:Summary : Support tools for GRUB. 2024-08-17T17:03:53.423 INFO:teuthology.orchestra.run.smithi179.stdout:Description : 2024-08-17T17:03:53.423 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:03:53.423 INFO:teuthology.orchestra.run.smithi179.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-08-17T17:03:53.424 INFO:teuthology.orchestra.run.smithi179.stdout:customizable bootloader with modular architecture. It supports a rich 2024-08-17T17:03:53.424 INFO:teuthology.orchestra.run.smithi179.stdout:variety of kernel formats, file systems, computer architectures and 2024-08-17T17:03:53.424 INFO:teuthology.orchestra.run.smithi179.stdout:hardware devices. 2024-08-17T17:03:53.424 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:03:53.424 INFO:teuthology.orchestra.run.smithi179.stdout:This subpackage provides tools for support of all platforms. 2024-08-17T17:03:53.424 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-08-17T17:03:53.425 INFO:teuthology.task.kernel:Updating grub on smithi179 to boot 5.14.0-480.el9.x86_64 2024-08-17T17:03:53.425 DEBUG:teuthology.orchestra.run.smithi179:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-08-17T17:03:54.037 INFO:teuthology.orchestra.run.smithi179.stderr:Generating grub configuration file ... 2024-08-17T17:03:55.432 INFO:teuthology.orchestra.run.smithi179.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-08-17T17:03:55.452 INFO:teuthology.orchestra.run.smithi179.stderr:done 2024-08-17T17:03:55.454 DEBUG:teuthology.orchestra.run.smithi179:> mktemp 2024-08-17T17:03:55.469 INFO:teuthology.orchestra.run.smithi179.stdout:/tmp/tmp.Y8SnFYpcGu 2024-08-17T17:03:55.469 DEBUG:teuthology.orchestra.run.smithi179:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.Y8SnFYpcGu 2024-08-17T17:03:55.534 DEBUG:teuthology.orchestra.run.smithi179:> sudo chmod 0666 /tmp/tmp.Y8SnFYpcGu 2024-08-17T17:03:55.668 DEBUG:teuthology.orchestra.remote:smithi179:/tmp/tmp.Y8SnFYpcGu is 6KB 2024-08-17T17:03:55.716 DEBUG:teuthology.orchestra.run.smithi179:> rm -fr /tmp/tmp.Y8SnFYpcGu 2024-08-17T17:03:55.730 DEBUG:teuthology.orchestra.run.smithi179:> sudo /bin/ls /boot/loader/entries || true 2024-08-17T17:03:55.800 INFO:teuthology.orchestra.run.smithi179.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-08-17T17:03:55.800 INFO:teuthology.orchestra.run.smithi179.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-08-17T17:03:55.801 INFO:teuthology.orchestra.run.smithi179.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-08-17T17:03:55.801 INFO:teuthology.orchestra.run.smithi179.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64.conf 2024-08-17T17:03:55.802 DEBUG:teuthology.orchestra.run.smithi179:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64 2024-08-17T17:03:55.959 DEBUG:teuthology.orchestra.run.smithi179:> sudo shutdown -r now 2024-08-17T17:03:58.242 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T17:03:58.284 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 4/4 2024-08-17T17:03:58.612 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T17:03:58.612 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : kernel-5.14.0-480.el9.x86_64 1/4 2024-08-17T17:03:58.612 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T17:03:58.612 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T17:03:59.453 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T17:03:59.453 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:03:59.453 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:03:59.453 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-5.14.0-480.el9.x86_64 2024-08-17T17:03:59.453 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-core-5.14.0-480.el9.x86_64 2024-08-17T17:03:59.453 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-modules-5.14.0-480.el9.x86_64 2024-08-17T17:03:59.454 INFO:teuthology.orchestra.run.smithi130.stdout: kernel-modules-core-5.14.0-480.el9.x86_64 2024-08-17T17:03:59.454 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:03:59.454 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:03:59.803 DEBUG:teuthology.orchestra.run.smithi130:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T17:03:59.934 INFO:teuthology.orchestra.run.smithi130.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T17:03:59.935 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T17:03:59.935 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-480.el9.x86_64 2024-08-17T17:03:59.935 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-08-17T17:03:59.935 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-08-17T17:03:59.935 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-08-17T17:03:59.935 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi130.front.sepia.ceph.com, path=None, version=distro) 2024-08-17T17:03:59.935 DEBUG:teuthology.orchestra.run.smithi130:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T17:03:59.992 INFO:teuthology.orchestra.run.smithi130.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T17:03:59.993 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T17:03:59.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo rpm -qi grub2-tools 2024-08-17T17:04:00.031 INFO:teuthology.orchestra.run.smithi130.stdout:Name : grub2-tools 2024-08-17T17:04:00.031 INFO:teuthology.orchestra.run.smithi130.stdout:Epoch : 1 2024-08-17T17:04:00.031 INFO:teuthology.orchestra.run.smithi130.stdout:Version : 2.06 2024-08-17T17:04:00.031 INFO:teuthology.orchestra.run.smithi130.stdout:Release : 68.el9 2024-08-17T17:04:00.031 INFO:teuthology.orchestra.run.smithi130.stdout:Architecture: x86_64 2024-08-17T17:04:00.031 INFO:teuthology.orchestra.run.smithi130.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-08-17T17:04:00.032 INFO:teuthology.orchestra.run.smithi130.stdout:Group : Unspecified 2024-08-17T17:04:00.032 INFO:teuthology.orchestra.run.smithi130.stdout:Size : 8264254 2024-08-17T17:04:00.032 INFO:teuthology.orchestra.run.smithi130.stdout:License : GPLv3+ 2024-08-17T17:04:00.032 INFO:teuthology.orchestra.run.smithi130.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-08-17T17:04:00.032 INFO:teuthology.orchestra.run.smithi130.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-08-17T17:04:00.032 INFO:teuthology.orchestra.run.smithi130.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-08-17T17:04:00.032 INFO:teuthology.orchestra.run.smithi130.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-08-17T17:04:00.032 INFO:teuthology.orchestra.run.smithi130.stdout:Packager : builder@centos.org 2024-08-17T17:04:00.032 INFO:teuthology.orchestra.run.smithi130.stdout:Vendor : CentOS 2024-08-17T17:04:00.032 INFO:teuthology.orchestra.run.smithi130.stdout:URL : http://www.gnu.org/software/grub/ 2024-08-17T17:04:00.032 INFO:teuthology.orchestra.run.smithi130.stdout:Summary : Support tools for GRUB. 2024-08-17T17:04:00.032 INFO:teuthology.orchestra.run.smithi130.stdout:Description : 2024-08-17T17:04:00.033 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:04:00.033 INFO:teuthology.orchestra.run.smithi130.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-08-17T17:04:00.033 INFO:teuthology.orchestra.run.smithi130.stdout:customizable bootloader with modular architecture. It supports a rich 2024-08-17T17:04:00.033 INFO:teuthology.orchestra.run.smithi130.stdout:variety of kernel formats, file systems, computer architectures and 2024-08-17T17:04:00.033 INFO:teuthology.orchestra.run.smithi130.stdout:hardware devices. 2024-08-17T17:04:00.033 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:04:00.033 INFO:teuthology.orchestra.run.smithi130.stdout:This subpackage provides tools for support of all platforms. 2024-08-17T17:04:00.034 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-08-17T17:04:00.034 INFO:teuthology.task.kernel:Updating grub on smithi130 to boot 5.14.0-480.el9.x86_64 2024-08-17T17:04:00.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-08-17T17:04:00.616 INFO:teuthology.orchestra.run.smithi130.stderr:Generating grub configuration file ... 2024-08-17T17:04:02.020 INFO:teuthology.orchestra.run.smithi130.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-08-17T17:04:02.051 INFO:teuthology.orchestra.run.smithi130.stderr:done 2024-08-17T17:04:02.061 DEBUG:teuthology.orchestra.run.smithi130:> mktemp 2024-08-17T17:04:02.076 INFO:teuthology.orchestra.run.smithi130.stdout:/tmp/tmp.sPsfecLKzP 2024-08-17T17:04:02.076 DEBUG:teuthology.orchestra.run.smithi130:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.sPsfecLKzP 2024-08-17T17:04:02.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod 0666 /tmp/tmp.sPsfecLKzP 2024-08-17T17:04:02.276 DEBUG:teuthology.orchestra.remote:smithi130:/tmp/tmp.sPsfecLKzP is 6KB 2024-08-17T17:04:02.324 DEBUG:teuthology.orchestra.run.smithi130:> rm -fr /tmp/tmp.sPsfecLKzP 2024-08-17T17:04:02.339 DEBUG:teuthology.orchestra.run.smithi130:> sudo /bin/ls /boot/loader/entries || true 2024-08-17T17:04:02.401 INFO:teuthology.orchestra.run.smithi130.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-08-17T17:04:02.401 INFO:teuthology.orchestra.run.smithi130.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-08-17T17:04:02.401 INFO:teuthology.orchestra.run.smithi130.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-08-17T17:04:02.401 INFO:teuthology.orchestra.run.smithi130.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64.conf 2024-08-17T17:04:02.403 DEBUG:teuthology.orchestra.run.smithi130:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64 2024-08-17T17:04:02.559 DEBUG:teuthology.orchestra.run.smithi130:> sudo shutdown -r now 2024-08-17T17:04:25.976 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-08-17T17:04:25.977 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi179.front.sepia.ceph.com' 2024-08-17T17:04:25.977 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T17:04:32.568 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-08-17T17:04:32.568 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-08-17T17:04:32.569 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T17:04:41.511 ERROR:teuthology.orchestra.connection:Error authenticating with smithi179.front.sepia.ceph.com: AuthenticationException('Authentication failed.') 2024-08-17T17:04:45.592 ERROR:teuthology.orchestra.connection:Error authenticating with smithi179.front.sepia.ceph.com: AuthenticationException('Authentication failed.') 2024-08-17T17:04:50.911 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.130 2024-08-17T17:04:52.726 ERROR:teuthology.orchestra.connection:Error authenticating with smithi179.front.sepia.ceph.com: AuthenticationException('Authentication failed.') 2024-08-17T17:04:59.911 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-08-17T17:04:59.912 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T17:05:03.006 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.130 2024-08-17T17:05:15.007 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-08-17T17:05:15.008 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T17:05:18.120 ERROR:teuthology.orchestra.connection:Error authenticating with smithi179.front.sepia.ceph.com: AuthenticationException('Authentication failed.') 2024-08-17T17:05:31.240 ERROR:teuthology.orchestra.connection:Error authenticating with smithi179.front.sepia.ceph.com: AuthenticationException('Authentication failed.') 2024-08-17T17:05:33.406 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.130 2024-08-17T17:05:47.297 ERROR:teuthology.orchestra.connection:Error authenticating with smithi179.front.sepia.ceph.com: BadHostKeyException('smithi179.front.sepia.ceph.com', PKey(alg=ECDSA, bits=256, fp=SHA256:UHFdjrAueozlb5UIN1CAlob7hdeviWgT6jpwJ4yJVvE), PKey(alg=ED25519, bits=256, fp=SHA256:W+6Y/LgnRby4510vqEHFE/6fAReT8MdM2qNnjG5ecjQ)) (No SSH private key found!) Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/connection.py", line 109, in connect ssh.connect(**connect_args) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/virtualenv/lib/python3.10/site-packages/paramiko/client.py", line 468, in connect raise BadHostKeyException(hostname, server_key, our_key) paramiko.ssh_exception.BadHostKeyException: Host key for server 'smithi179.front.sepia.ceph.com' does not match: got 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBRW4uvlvXC3x65EZj+m9HRUFQUPs43ZA41kCSjF/9H0SXkC4u+mEiwPD7qZFr/+2qub2L73RRVOPeL5N7bmnGM=', expected 'AAAAC3NzaC1lZDI1NTE5AAAAIDnVB65LQ2ZrorCmRDXT/imYcIqttYXGdAR7PyxJCGep' 2024-08-17T17:05:48.407 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-08-17T17:05:48.408 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T17:05:48.923 DEBUG:teuthology.orchestra.run.smithi130:> true 2024-08-17T17:05:49.524 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-08-17T17:05:49.525 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-480.el9.x86_64"... 2024-08-17T17:05:49.525 DEBUG:teuthology.orchestra.run.smithi130:> uname -r 2024-08-17T17:05:49.567 INFO:teuthology.orchestra.run.smithi130.stdout:5.14.0-480.el9.x86_64 2024-08-17T17:05:49.567 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-480.el9.x86_64 vs 5.14.0-480.el9.x86_64 2024-08-17T17:05:49.568 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-08-17T17:05:49.568 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-08-17T17:05:50.568 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-08-17T17:05:50.568 DEBUG:teuthology.orchestra.run.smithi130:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-08-17T17:05:50.695 INFO:teuthology.orchestra.run.smithi130.stdout:ttyS1 2024-08-17T17:05:50.730 DEBUG:teuthology.parallel:result is None 2024-08-17T17:06:06.325 ERROR:teuthology.orchestra.connection:Error authenticating with smithi179.front.sepia.ceph.com: BadHostKeyException('smithi179.front.sepia.ceph.com', PKey(alg=ECDSA, bits=256, fp=SHA256:UHFdjrAueozlb5UIN1CAlob7hdeviWgT6jpwJ4yJVvE), PKey(alg=ED25519, bits=256, fp=SHA256:W+6Y/LgnRby4510vqEHFE/6fAReT8MdM2qNnjG5ecjQ)) (No SSH private key found!) Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/connection.py", line 109, in connect ssh.connect(**connect_args) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/virtualenv/lib/python3.10/site-packages/paramiko/client.py", line 468, in connect raise BadHostKeyException(hostname, server_key, our_key) paramiko.ssh_exception.BadHostKeyException: Host key for server 'smithi179.front.sepia.ceph.com' does not match: got 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBRW4uvlvXC3x65EZj+m9HRUFQUPs43ZA41kCSjF/9H0SXkC4u+mEiwPD7qZFr/+2qub2L73RRVOPeL5N7bmnGM=', expected 'AAAAC3NzaC1lZDI1NTE5AAAAIDnVB65LQ2ZrorCmRDXT/imYcIqttYXGdAR7PyxJCGep' 2024-08-17T17:06:28.352 ERROR:teuthology.orchestra.connection:Error authenticating with smithi179.front.sepia.ceph.com: BadHostKeyException('smithi179.front.sepia.ceph.com', PKey(alg=ECDSA, bits=256, fp=SHA256:UHFdjrAueozlb5UIN1CAlob7hdeviWgT6jpwJ4yJVvE), PKey(alg=ED25519, bits=256, fp=SHA256:W+6Y/LgnRby4510vqEHFE/6fAReT8MdM2qNnjG5ecjQ)) (No SSH private key found!) Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/connection.py", line 109, in connect ssh.connect(**connect_args) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/virtualenv/lib/python3.10/site-packages/paramiko/client.py", line 468, in connect raise BadHostKeyException(hostname, server_key, our_key) paramiko.ssh_exception.BadHostKeyException: Host key for server 'smithi179.front.sepia.ceph.com' does not match: got 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBRW4uvlvXC3x65EZj+m9HRUFQUPs43ZA41kCSjF/9H0SXkC4u+mEiwPD7qZFr/+2qub2L73RRVOPeL5N7bmnGM=', expected 'AAAAC3NzaC1lZDI1NTE5AAAAIDnVB65LQ2ZrorCmRDXT/imYcIqttYXGdAR7PyxJCGep' 2024-08-17T17:06:53.379 ERROR:teuthology.orchestra.connection:Error authenticating with smithi179.front.sepia.ceph.com: BadHostKeyException('smithi179.front.sepia.ceph.com', PKey(alg=ECDSA, bits=256, fp=SHA256:UHFdjrAueozlb5UIN1CAlob7hdeviWgT6jpwJ4yJVvE), PKey(alg=ED25519, bits=256, fp=SHA256:W+6Y/LgnRby4510vqEHFE/6fAReT8MdM2qNnjG5ecjQ)) (No SSH private key found!) Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/connection.py", line 109, in connect ssh.connect(**connect_args) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/virtualenv/lib/python3.10/site-packages/paramiko/client.py", line 468, in connect raise BadHostKeyException(hostname, server_key, our_key) paramiko.ssh_exception.BadHostKeyException: Host key for server 'smithi179.front.sepia.ceph.com' does not match: got 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBRW4uvlvXC3x65EZj+m9HRUFQUPs43ZA41kCSjF/9H0SXkC4u+mEiwPD7qZFr/+2qub2L73RRVOPeL5N7bmnGM=', expected 'AAAAC3NzaC1lZDI1NTE5AAAAIDnVB65LQ2ZrorCmRDXT/imYcIqttYXGdAR7PyxJCGep' 2024-08-17T17:07:21.448 ERROR:teuthology.orchestra.connection:Error authenticating with smithi179.front.sepia.ceph.com: BadHostKeyException('smithi179.front.sepia.ceph.com', PKey(alg=ECDSA, bits=256, fp=SHA256:UHFdjrAueozlb5UIN1CAlob7hdeviWgT6jpwJ4yJVvE), PKey(alg=ED25519, bits=256, fp=SHA256:W+6Y/LgnRby4510vqEHFE/6fAReT8MdM2qNnjG5ecjQ)) (No SSH private key found!) Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/connection.py", line 109, in connect ssh.connect(**connect_args) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/virtualenv/lib/python3.10/site-packages/paramiko/client.py", line 468, in connect raise BadHostKeyException(hostname, server_key, our_key) paramiko.ssh_exception.BadHostKeyException: Host key for server 'smithi179.front.sepia.ceph.com' does not match: got 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBRW4uvlvXC3x65EZj+m9HRUFQUPs43ZA41kCSjF/9H0SXkC4u+mEiwPD7qZFr/+2qub2L73RRVOPeL5N7bmnGM=', expected 'AAAAC3NzaC1lZDI1NTE5AAAAIDnVB65LQ2ZrorCmRDXT/imYcIqttYXGdAR7PyxJCGep' 2024-08-17T17:07:21.448 DEBUG:teuthology.orchestra.remote:'connect to smithi179.front.sepia.ceph.com' reached maximum tries (10) after waiting for 145 seconds 2024-08-17T17:07:30.449 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi179.front.sepia.ceph.com' 2024-08-17T17:07:30.450 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T17:07:30.850 DEBUG:teuthology.orchestra.run.smithi179:> true 2024-08-17T17:07:31.249 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi179.front.sepia.ceph.com' 2024-08-17T17:07:31.249 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-480.el9.x86_64"... 2024-08-17T17:07:31.249 DEBUG:teuthology.orchestra.run.smithi179:> uname -r 2024-08-17T17:07:31.266 INFO:teuthology.orchestra.run.smithi179.stdout:5.14.0-480.el9.x86_64 2024-08-17T17:07:31.266 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-480.el9.x86_64 vs 5.14.0-480.el9.x86_64 2024-08-17T17:07:31.266 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-08-17T17:07:31.266 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-08-17T17:07:32.267 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-08-17T17:07:32.268 DEBUG:teuthology.orchestra.run.smithi179:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-08-17T17:07:32.662 INFO:teuthology.orchestra.run.smithi179.stdout:ttyS1 2024-08-17T17:07:32.696 DEBUG:teuthology.parallel:result is None 2024-08-17T17:07:32.697 INFO:teuthology.run_tasks:Running task internal.base... 2024-08-17T17:07:32.704 INFO:teuthology.task.internal:Creating test directory... 2024-08-17T17:07:32.704 DEBUG:teuthology.orchestra.run.smithi130:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-17T17:07:32.707 DEBUG:teuthology.orchestra.run.smithi179:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-17T17:07:32.725 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-08-17T17:07:32.732 INFO:teuthology.run_tasks:Running task internal.archive... 2024-08-17T17:07:32.743 INFO:teuthology.task.internal:Creating archive directory... 2024-08-17T17:07:32.743 DEBUG:teuthology.orchestra.run.smithi130:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-17T17:07:32.766 DEBUG:teuthology.orchestra.run.smithi179:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-17T17:07:32.803 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-08-17T17:07:32.809 INFO:teuthology.task.internal:Enabling coredump saving... 2024-08-17T17:07:32.809 DEBUG:teuthology.orchestra.run.smithi130:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-17T17:07:32.841 DEBUG:teuthology.orchestra.run.smithi179:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-17T17:07:32.876 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T17:07:32.878 INFO:teuthology.orchestra.run.smithi179.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T17:07:32.888 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T17:07:32.890 INFO:teuthology.orchestra.run.smithi179.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T17:07:32.892 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-08-17T17:07:32.898 INFO:teuthology.task.internal:Configuring sudo... 2024-08-17T17:07:32.898 DEBUG:teuthology.orchestra.run.smithi130:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-17T17:07:32.932 DEBUG:teuthology.orchestra.run.smithi179:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-17T17:07:32.961 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-08-17T17:07:32.970 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-08-17T17:07:32.970 DEBUG:teuthology.orchestra.run.smithi130:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-17T17:07:33.000 DEBUG:teuthology.orchestra.run.smithi179:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-17T17:07:33.019 DEBUG:teuthology.orchestra.run.smithi130:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T17:07:33.104 DEBUG:teuthology.orchestra.run.smithi130:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T17:07:33.147 DEBUG:teuthology.orchestra.run.smithi130:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T17:07:33.235 DEBUG:teuthology.orchestra.run.smithi130:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T17:07:33.272 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:07:33.272 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-17T17:07:33.337 DEBUG:teuthology.orchestra.run.smithi179:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T17:07:33.390 DEBUG:teuthology.orchestra.run.smithi179:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T17:07:33.451 DEBUG:teuthology.orchestra.run.smithi179:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T17:07:33.504 DEBUG:teuthology.orchestra.run.smithi179:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T17:07:33.538 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-08-17T17:07:33.538 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-17T17:07:33.605 DEBUG:teuthology.orchestra.run.smithi130:> sudo service rsyslog restart 2024-08-17T17:07:33.607 DEBUG:teuthology.orchestra.run.smithi179:> sudo service rsyslog restart 2024-08-17T17:07:33.654 INFO:teuthology.orchestra.run.smithi130.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T17:07:33.679 INFO:teuthology.orchestra.run.smithi179.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T17:07:33.955 INFO:teuthology.run_tasks:Running task internal.timer... 2024-08-17T17:07:33.962 INFO:teuthology.task.internal:Starting timer... 2024-08-17T17:07:33.962 INFO:teuthology.run_tasks:Running task pcp... 2024-08-17T17:07:33.972 INFO:teuthology.run_tasks:Running task selinux... 2024-08-17T17:07:33.980 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-08-17T17:07:33.980 DEBUG:teuthology.orchestra.run.smithi130:> sudo service auditd rotate 2024-08-17T17:07:34.062 INFO:teuthology.orchestra.run.smithi130.stdout:Rotating logs: 2024-08-17T17:07:34.064 DEBUG:teuthology.orchestra.run.smithi179:> sudo service auditd rotate 2024-08-17T17:07:34.157 INFO:teuthology.orchestra.run.smithi179.stdout:Rotating logs: 2024-08-17T17:07:34.158 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-08-17T17:07:34.159 DEBUG:teuthology.orchestra.run.smithi130:> /usr/sbin/getenforce 2024-08-17T17:07:34.188 INFO:teuthology.orchestra.run.smithi130.stdout:Permissive 2024-08-17T17:07:34.189 DEBUG:teuthology.orchestra.run.smithi179:> /usr/sbin/getenforce 2024-08-17T17:07:34.211 INFO:teuthology.orchestra.run.smithi179.stdout:Permissive 2024-08-17T17:07:34.211 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi130.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi179.front.sepia.ceph.com': 'permissive'} 2024-08-17T17:07:34.212 DEBUG:teuthology.orchestra.run.smithi130:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-08-17T17:07:34.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:07:34.255 DEBUG:teuthology.orchestra.run.smithi179:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-08-17T17:07:34.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:07:34.282 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-08-17T17:07:34.282 DEBUG:teuthology.orchestra.run.smithi130:> sudo /usr/sbin/setenforce permissive 2024-08-17T17:07:34.320 DEBUG:teuthology.orchestra.run.smithi179:> sudo /usr/sbin/setenforce permissive 2024-08-17T17:07:34.356 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-08-17T17:07:34.365 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-08-17T17:07:34.367 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-08-17T17:07:34.480 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-08-17T17:07:34.490 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-08-17T17:07:34.492 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi130.front.sepia.ceph.com,smithi179.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-08-17T17:12:09.122 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi130.front.sepia.ceph.com'), Remote(name='ubuntu@smithi179.front.sepia.ceph.com')] 2024-08-17T17:12:09.123 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-08-17T17:12:09.125 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi130.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T17:12:09.209 DEBUG:teuthology.orchestra.run.smithi130:> true 2024-08-17T17:12:09.290 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi130.front.sepia.ceph.com' 2024-08-17T17:12:09.290 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi179.front.sepia.ceph.com' 2024-08-17T17:12:09.291 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi179.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T17:12:09.375 DEBUG:teuthology.orchestra.run.smithi179:> true 2024-08-17T17:12:09.454 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi179.front.sepia.ceph.com' 2024-08-17T17:12:09.454 INFO:teuthology.run_tasks:Running task clock... 2024-08-17T17:12:09.474 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-08-17T17:12:09.474 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-17T17:12:09.475 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T17:12:09.478 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-17T17:12:09.478 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T17:12:09.505 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-17T17:12:09.520 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-17T17:12:09.525 INFO:teuthology.orchestra.run.smithi179.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-17T17:12:09.539 INFO:teuthology.orchestra.run.smithi179.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-17T17:12:09.543 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: ntpd: command not found 2024-08-17T17:12:09.555 INFO:teuthology.orchestra.run.smithi130.stdout:506 Cannot talk to daemon 2024-08-17T17:12:09.564 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: ntpd: command not found 2024-08-17T17:12:09.569 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-17T17:12:09.575 INFO:teuthology.orchestra.run.smithi179.stdout:506 Cannot talk to daemon 2024-08-17T17:12:09.582 INFO:teuthology.orchestra.run.smithi130.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-17T17:12:09.590 INFO:teuthology.orchestra.run.smithi179.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-17T17:12:09.603 INFO:teuthology.orchestra.run.smithi179.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-17T17:12:09.648 INFO:teuthology.orchestra.run.smithi130.stderr:bash: line 1: ntpq: command not found 2024-08-17T17:12:09.651 INFO:teuthology.orchestra.run.smithi130.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T17:12:09.652 INFO:teuthology.orchestra.run.smithi130.stdout:=============================================================================== 2024-08-17T17:12:09.652 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T17:12:09.653 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T17:12:09.653 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T17:12:09.653 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T17:12:09.656 INFO:teuthology.orchestra.run.smithi179.stderr:bash: line 1: ntpq: command not found 2024-08-17T17:12:09.661 INFO:teuthology.orchestra.run.smithi179.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T17:12:09.661 INFO:teuthology.orchestra.run.smithi179.stdout:=============================================================================== 2024-08-17T17:12:09.661 INFO:teuthology.orchestra.run.smithi179.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T17:12:09.661 INFO:teuthology.orchestra.run.smithi179.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T17:12:09.661 INFO:teuthology.orchestra.run.smithi179.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T17:12:09.661 INFO:teuthology.orchestra.run.smithi179.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T17:12:09.662 INFO:teuthology.run_tasks:Running task pexec... 2024-08-17T17:12:09.671 INFO:teuthology.task.pexec:Executing custom commands... 2024-08-17T17:12:09.671 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi130.front.sepia.ceph.com 2024-08-17T17:12:09.672 DEBUG:teuthology.orchestra.run.smithi130:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-08-17T17:12:09.672 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi179.front.sepia.ceph.com 2024-08-17T17:12:09.672 DEBUG:teuthology.orchestra.run.smithi179:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-08-17T17:12:10.257 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:12 ago on Sat 17 Aug 2024 05:10:58 PM UTC. 2024-08-17T17:12:10.270 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:11 ago on Sat 17 Aug 2024 05:10:59 PM UTC. 2024-08-17T17:12:10.343 INFO:teuthology.orchestra.run.smithi130.stdout:Package nvme-cli-2.6-5.el9.x86_64 is already installed. 2024-08-17T17:12:10.357 INFO:teuthology.orchestra.run.smithi179.stdout:Package nvme-cli-2.6-5.el9.x86_64 is already installed. 2024-08-17T17:12:10.384 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:12:10.386 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:12:10.386 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repository Size 2024-08-17T17:12:10.386 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:12:10.386 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:12:10.386 INFO:teuthology.orchestra.run.smithi130.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-08-17T17:12:10.386 INFO:teuthology.orchestra.run.smithi130.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-08-17T17:12:10.387 INFO:teuthology.orchestra.run.smithi130.stdout:Upgrading: 2024-08-17T17:12:10.387 INFO:teuthology.orchestra.run.smithi130.stdout: libnvme x86_64 1.9-2.el9 baseos 106 k 2024-08-17T17:12:10.387 INFO:teuthology.orchestra.run.smithi130.stdout: nvme-cli x86_64 2.9.1-3.el9 baseos 903 k 2024-08-17T17:12:10.387 INFO:teuthology.orchestra.run.smithi130.stdout:Installing dependencies: 2024-08-17T17:12:10.387 INFO:teuthology.orchestra.run.smithi130.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-08-17T17:12:10.387 INFO:teuthology.orchestra.run.smithi130.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-08-17T17:12:10.387 INFO:teuthology.orchestra.run.smithi130.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-08-17T17:12:10.387 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:12:10.387 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:12:10.387 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:12:10.387 INFO:teuthology.orchestra.run.smithi130.stdout:Install 5 Packages 2024-08-17T17:12:10.387 INFO:teuthology.orchestra.run.smithi130.stdout:Upgrade 2 Packages 2024-08-17T17:12:10.387 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:12:10.399 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:12:10.401 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 5.2 M 2024-08-17T17:12:10.401 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:12:10.414 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:12:10.414 INFO:teuthology.orchestra.run.smithi179.stdout: Package Arch Version Repository Size 2024-08-17T17:12:10.414 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:12:10.414 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:12:10.414 INFO:teuthology.orchestra.run.smithi179.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-08-17T17:12:10.414 INFO:teuthology.orchestra.run.smithi179.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-08-17T17:12:10.414 INFO:teuthology.orchestra.run.smithi179.stdout:Upgrading: 2024-08-17T17:12:10.415 INFO:teuthology.orchestra.run.smithi179.stdout: libnvme x86_64 1.9-2.el9 baseos 106 k 2024-08-17T17:12:10.415 INFO:teuthology.orchestra.run.smithi179.stdout: nvme-cli x86_64 2.9.1-3.el9 baseos 903 k 2024-08-17T17:12:10.415 INFO:teuthology.orchestra.run.smithi179.stdout:Installing dependencies: 2024-08-17T17:12:10.415 INFO:teuthology.orchestra.run.smithi179.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-08-17T17:12:10.415 INFO:teuthology.orchestra.run.smithi179.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-08-17T17:12:10.415 INFO:teuthology.orchestra.run.smithi179.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-08-17T17:12:10.415 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:12:10.415 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:12:10.415 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:12:10.415 INFO:teuthology.orchestra.run.smithi179.stdout:Install 5 Packages 2024-08-17T17:12:10.415 INFO:teuthology.orchestra.run.smithi179.stdout:Upgrade 2 Packages 2024-08-17T17:12:10.415 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:12:10.425 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 5.2 M 2024-08-17T17:12:10.425 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:12:10.831 INFO:teuthology.orchestra.run.smithi130.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 380 kB/s | 44 kB 00:00 2024-08-17T17:12:10.857 INFO:teuthology.orchestra.run.smithi130.stdout:(2/7): python3-configshell-1.1.30-1.el9.noarch. 504 kB/s | 72 kB 00:00 2024-08-17T17:12:10.882 INFO:teuthology.orchestra.run.smithi130.stdout:(3/7): python3-kmod-0.9-32.el9.x86_64.rpm 503 kB/s | 84 kB 00:00 2024-08-17T17:12:10.957 INFO:teuthology.orchestra.run.smithi130.stdout:(4/7): libnvme-1.9-2.el9.x86_64.rpm 1.4 MB/s | 106 kB 00:00 2024-08-17T17:12:11.066 INFO:teuthology.orchestra.run.smithi130.stdout:(5/7): runc-1.1.13-4.el9.x86_64.rpm 15 MB/s | 3.2 MB 00:00 2024-08-17T17:12:11.091 INFO:teuthology.orchestra.run.smithi130.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 3.2 MB/s | 837 kB 00:00 2024-08-17T17:12:11.233 INFO:teuthology.orchestra.run.smithi130.stdout:(7/7): nvme-cli-2.9.1-3.el9.x86_64.rpm 3.2 MB/s | 903 kB 00:00 2024-08-17T17:12:11.233 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:12:11.233 INFO:teuthology.orchestra.run.smithi130.stdout:Total 6.2 MB/s | 5.2 MB 00:00 2024-08-17T17:12:11.285 INFO:teuthology.orchestra.run.smithi179.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 73 kB/s | 44 kB 00:00 2024-08-17T17:12:11.341 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:12:11.343 INFO:teuthology.orchestra.run.smithi179.stdout:(2/7): python3-configshell-1.1.30-1.el9.noarch. 108 kB/s | 72 kB 00:00 2024-08-17T17:12:11.358 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:12:11.358 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:12:11.402 INFO:teuthology.orchestra.run.smithi179.stdout:(3/7): python3-kmod-0.9-32.el9.x86_64.rpm 116 kB/s | 84 kB 00:00 2024-08-17T17:12:11.595 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:12:11.596 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:12:11.852 INFO:teuthology.orchestra.run.smithi179.stdout:(4/7): runc-1.1.13-4.el9.x86_64.rpm 6.2 MB/s | 3.2 MB 00:00 2024-08-17T17:12:12.003 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:12:12.086 INFO:teuthology.orchestra.run.smithi179.stdout:(5/7): libnvme-1.9-2.el9.x86_64.rpm 155 kB/s | 106 kB 00:00 2024-08-17T17:12:12.232 INFO:teuthology.orchestra.run.smithi130.stdout: Upgrading : libnvme-1.9-2.el9.x86_64 1/9 2024-08-17T17:12:12.244 INFO:teuthology.orchestra.run.smithi179.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 873 kB/s | 837 kB 00:00 2024-08-17T17:12:12.288 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 2/9 2024-08-17T17:12:12.323 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 3/9 2024-08-17T17:12:12.356 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 4/9 2024-08-17T17:12:12.381 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : nvmetcli-0.7-3.el9.noarch 5/9 2024-08-17T17:12:12.643 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 5/9 2024-08-17T17:12:12.709 INFO:teuthology.orchestra.run.smithi130.stdout: Upgrading : nvme-cli-2.9.1-3.el9.x86_64 6/9 2024-08-17T17:12:12.812 INFO:teuthology.orchestra.run.smithi179.stdout:(7/7): nvme-cli-2.9.1-3.el9.x86_64.rpm 943 kB/s | 903 kB 00:00 2024-08-17T17:12:12.812 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:12:12.812 INFO:teuthology.orchestra.run.smithi179.stdout:Total 2.2 MB/s | 5.2 MB 00:02 2024-08-17T17:12:12.928 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:12:12.946 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:12:12.946 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:12:13.098 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: nvme-cli-2.9.1-3.el9.x86_64 6/9 2024-08-17T17:12:13.138 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/9 2024-08-17T17:12:13.167 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:12:13.167 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:12:13.170 INFO:teuthology.orchestra.run.smithi130.stdout: Cleanup : nvme-cli-2.6-5.el9.x86_64 8/9 2024-08-17T17:12:13.230 INFO:teuthology.orchestra.run.smithi130.stdout: Cleanup : libnvme-1.6-1.el9.x86_64 9/9 2024-08-17T17:12:13.571 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:12:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: Upgrading : libnvme-1.9-2.el9.x86_64 1/9 2024-08-17T17:12:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 2/9 2024-08-17T17:12:13.868 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 3/9 2024-08-17T17:12:13.901 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 4/9 2024-08-17T17:12:13.927 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : nvmetcli-0.7-3.el9.noarch 5/9 2024-08-17T17:12:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: libnvme-1.6-1.el9.x86_64 9/9 2024-08-17T17:12:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 1/9 2024-08-17T17:12:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 2/9 2024-08-17T17:12:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 3/9 2024-08-17T17:12:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 4/9 2024-08-17T17:12:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 5/9 2024-08-17T17:12:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libnvme-1.9-2.el9.x86_64 6/9 2024-08-17T17:12:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libnvme-1.6-1.el9.x86_64 7/9 2024-08-17T17:12:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : nvme-cli-2.9.1-3.el9.x86_64 8/9 2024-08-17T17:12:14.209 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 5/9 2024-08-17T17:12:14.264 INFO:teuthology.orchestra.run.smithi179.stdout: Upgrading : nvme-cli-2.9.1-3.el9.x86_64 6/9 2024-08-17T17:12:14.566 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : nvme-cli-2.6-5.el9.x86_64 9/9 2024-08-17T17:12:14.567 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:12:14.567 INFO:teuthology.orchestra.run.smithi130.stdout:Upgraded: 2024-08-17T17:12:14.567 INFO:teuthology.orchestra.run.smithi130.stdout: libnvme-1.9-2.el9.x86_64 nvme-cli-2.9.1-3.el9.x86_64 2024-08-17T17:12:14.567 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:12:14.567 INFO:teuthology.orchestra.run.smithi130.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-08-17T17:12:14.567 INFO:teuthology.orchestra.run.smithi130.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-08-17T17:12:14.567 INFO:teuthology.orchestra.run.smithi130.stdout: runc-4:1.1.13-4.el9.x86_64 2024-08-17T17:12:14.567 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:12:14.567 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:12:14.644 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: nvme-cli-2.9.1-3.el9.x86_64 6/9 2024-08-17T17:12:14.684 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/9 2024-08-17T17:12:14.717 INFO:teuthology.orchestra.run.smithi179.stdout: Cleanup : nvme-cli-2.6-5.el9.x86_64 8/9 2024-08-17T17:12:14.780 INFO:teuthology.orchestra.run.smithi179.stdout: Cleanup : libnvme-1.6-1.el9.x86_64 9/9 2024-08-17T17:12:14.971 DEBUG:teuthology.parallel:result is None 2024-08-17T17:12:15.731 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: libnvme-1.6-1.el9.x86_64 9/9 2024-08-17T17:12:15.731 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 1/9 2024-08-17T17:12:15.731 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 2/9 2024-08-17T17:12:15.731 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 3/9 2024-08-17T17:12:15.731 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 4/9 2024-08-17T17:12:15.731 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 5/9 2024-08-17T17:12:15.731 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : libnvme-1.9-2.el9.x86_64 6/9 2024-08-17T17:12:15.731 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : libnvme-1.6-1.el9.x86_64 7/9 2024-08-17T17:12:15.731 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : nvme-cli-2.9.1-3.el9.x86_64 8/9 2024-08-17T17:12:16.086 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : nvme-cli-2.6-5.el9.x86_64 9/9 2024-08-17T17:12:16.086 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:12:16.086 INFO:teuthology.orchestra.run.smithi179.stdout:Upgraded: 2024-08-17T17:12:16.086 INFO:teuthology.orchestra.run.smithi179.stdout: libnvme-1.9-2.el9.x86_64 nvme-cli-2.9.1-3.el9.x86_64 2024-08-17T17:12:16.086 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:12:16.086 INFO:teuthology.orchestra.run.smithi179.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-08-17T17:12:16.087 INFO:teuthology.orchestra.run.smithi179.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-08-17T17:12:16.087 INFO:teuthology.orchestra.run.smithi179.stdout: runc-4:1.1.13-4.el9.x86_64 2024-08-17T17:12:16.087 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:12:16.087 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:12:16.439 DEBUG:teuthology.parallel:result is None 2024-08-17T17:12:16.440 INFO:teuthology.run_tasks:Running task install... 2024-08-17T17:12:16.448 DEBUG:teuthology.task.install:project ceph 2024-08-17T17:12:16.448 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'}} 2024-08-17T17:12:16.448 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'} 2024-08-17T17:12:16.448 INFO:teuthology.task.install:Using flavor: default 2024-08-17T17:12:16.454 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-08-17T17:12:16.454 INFO:teuthology.task.install:extra packages: [] 2024-08-17T17:12:16.454 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453', 'tag': None, 'wait_for_package': False} 2024-08-17T17:12:16.454 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T17:12:16.454 INFO:teuthology.packaging:ref: None 2024-08-17T17:12:16.455 INFO:teuthology.packaging:tag: None 2024-08-17T17:12:16.455 INFO:teuthology.packaging:branch: quincy 2024-08-17T17:12:16.455 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:12:16.455 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-08-17T17:12:16.457 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453', 'tag': None, 'wait_for_package': False} 2024-08-17T17:12:16.457 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T17:12:16.457 INFO:teuthology.packaging:ref: None 2024-08-17T17:12:16.457 INFO:teuthology.packaging:tag: None 2024-08-17T17:12:16.457 INFO:teuthology.packaging:branch: quincy 2024-08-17T17:12:16.457 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:12:16.457 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-08-17T17:12:16.698 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/ 2024-08-17T17:12:16.699 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1404.gf2c39d4d 2024-08-17T17:12:16.716 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/ 2024-08-17T17:12:16.716 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1404.gf2c39d4d 2024-08-17T17:12:16.821 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-08-17T17:12:16.822 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-08-17T17:12:16.822 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-08-17T17:12:16.833 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/quincy/f2c39d4d5abc084f975be26cc0141421e55727d8/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-08-17T17:12:16.833 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:12:16.833 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-08-17T17:12:16.847 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-08-17T17:12:16.847 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T17:12:16.847 INFO:teuthology.packaging:ref: None 2024-08-17T17:12:16.847 INFO:teuthology.packaging:tag: None 2024-08-17T17:12:16.847 INFO:teuthology.packaging:branch: quincy 2024-08-17T17:12:16.848 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:12:16.848 DEBUG:teuthology.orchestra.run.smithi179:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-08-17T17:12:16.860 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-08-17T17:12:16.860 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T17:12:16.860 INFO:teuthology.packaging:ref: None 2024-08-17T17:12:16.860 INFO:teuthology.packaging:tag: None 2024-08-17T17:12:16.860 INFO:teuthology.packaging:branch: quincy 2024-08-17T17:12:16.861 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:12:16.861 DEBUG:teuthology.orchestra.run.smithi130:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-08-17T17:12:16.913 DEBUG:teuthology.orchestra.run.smithi179:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-08-17T17:12:16.927 DEBUG:teuthology.orchestra.run.smithi130:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-08-17T17:12:16.989 DEBUG:teuthology.orchestra.run.smithi179:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-08-17T17:12:17.002 DEBUG:teuthology.orchestra.run.smithi130:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-08-17T17:12:17.053 INFO:teuthology.orchestra.run.smithi179.stdout:check_obsoletes = 1 2024-08-17T17:12:17.055 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum clean all 2024-08-17T17:12:17.065 INFO:teuthology.orchestra.run.smithi130.stdout:check_obsoletes = 1 2024-08-17T17:12:17.067 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum clean all 2024-08-17T17:12:17.283 INFO:teuthology.orchestra.run.smithi179.stdout:45 files removed 2024-08-17T17:12:17.301 INFO:teuthology.orchestra.run.smithi130.stdout:45 files removed 2024-08-17T17:12:17.310 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install ceph-radosgw 2024-08-17T17:12:17.327 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install ceph-radosgw 2024-08-17T17:12:17.920 INFO:teuthology.orchestra.run.smithi179.stdout:ceph packages for x86_64 232 kB/s | 77 kB 00:00 2024-08-17T17:12:17.929 INFO:teuthology.orchestra.run.smithi130.stdout:ceph packages for x86_64 238 kB/s | 77 kB 00:00 2024-08-17T17:12:18.246 INFO:teuthology.orchestra.run.smithi179.stdout:ceph noarch packages 39 kB/s | 11 kB 00:00 2024-08-17T17:12:18.270 INFO:teuthology.orchestra.run.smithi130.stdout:ceph noarch packages 37 kB/s | 11 kB 00:00 2024-08-17T17:12:18.580 INFO:teuthology.orchestra.run.smithi179.stdout:ceph source packages 6.2 kB/s | 1.8 kB 00:00 2024-08-17T17:12:18.596 INFO:teuthology.orchestra.run.smithi130.stdout:ceph source packages 6.4 kB/s | 1.8 kB 00:00 2024-08-17T17:12:19.454 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - BaseOS 10 MB/s | 8.2 MB 00:00 2024-08-17T17:12:20.163 INFO:teuthology.orchestra.run.smithi179.stdout:CentOS Stream 9 - BaseOS 5.3 MB/s | 8.2 MB 00:01 2024-08-17T17:12:24.931 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - AppStream 4.6 MB/s | 20 MB 00:04 2024-08-17T17:12:25.014 INFO:teuthology.orchestra.run.smithi179.stdout:CentOS Stream 9 - AppStream 5.3 MB/s | 20 MB 00:03 2024-08-17T17:12:29.715 INFO:teuthology.orchestra.run.smithi179.stdout:CentOS Stream 9 - CRB 7.1 MB/s | 6.6 MB 00:00 2024-08-17T17:12:30.548 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - CRB 3.5 MB/s | 6.6 MB 00:01 2024-08-17T17:12:31.582 INFO:teuthology.orchestra.run.smithi179.stdout:CentOS Stream 9 - Extras packages 35 kB/s | 18 kB 00:00 2024-08-17T17:12:32.231 INFO:teuthology.orchestra.run.smithi130.stdout:CentOS Stream 9 - Extras packages 54 kB/s | 18 kB 00:00 2024-08-17T17:12:32.937 INFO:teuthology.orchestra.run.smithi179.stdout:Extra Packages for Enterprise Linux 18 MB/s | 22 MB 00:01 2024-08-17T17:12:33.903 INFO:teuthology.orchestra.run.smithi130.stdout:Extra Packages for Enterprise Linux 15 MB/s | 22 MB 00:01 2024-08-17T17:12:38.309 INFO:teuthology.orchestra.run.smithi179.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-08-17T17:12:39.309 INFO:teuthology.orchestra.run.smithi130.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-08-17T17:12:40.613 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:12:40.614 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:12:40.615 INFO:teuthology.orchestra.run.smithi179.stdout: Package Arch Version Repository Size 2024-08-17T17:12:40.615 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:12:40.615 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:12:40.615 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-radosgw x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 14 M 2024-08-17T17:12:40.615 INFO:teuthology.orchestra.run.smithi179.stdout:Upgrading: 2024-08-17T17:12:40.615 INFO:teuthology.orchestra.run.smithi179.stdout: librados2 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 3.1 M 2024-08-17T17:12:40.615 INFO:teuthology.orchestra.run.smithi179.stdout: librbd1 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 2.9 M 2024-08-17T17:12:40.615 INFO:teuthology.orchestra.run.smithi179.stdout:Installing dependencies: 2024-08-17T17:12:40.615 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-base x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 5.0 M 2024-08-17T17:12:40.616 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-common x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 22 M 2024-08-17T17:12:40.616 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-selinux x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 25 k 2024-08-17T17:12:40.616 INFO:teuthology.orchestra.run.smithi179.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-08-17T17:12:40.616 INFO:teuthology.orchestra.run.smithi179.stdout: libcephfs2 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 637 k 2024-08-17T17:12:40.616 INFO:teuthology.orchestra.run.smithi179.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-08-17T17:12:40.616 INFO:teuthology.orchestra.run.smithi179.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-08-17T17:12:40.616 INFO:teuthology.orchestra.run.smithi179.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-08-17T17:12:40.616 INFO:teuthology.orchestra.run.smithi179.stdout: libradosstriper1 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 457 k 2024-08-17T17:12:40.616 INFO:teuthology.orchestra.run.smithi179.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-08-17T17:12:40.616 INFO:teuthology.orchestra.run.smithi179.stdout: librgw2 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 4.9 M 2024-08-17T17:12:40.616 INFO:teuthology.orchestra.run.smithi179.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-08-17T17:12:40.616 INFO:teuthology.orchestra.run.smithi179.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-08-17T17:12:40.616 INFO:teuthology.orchestra.run.smithi179.stdout: python3-ceph-argparse x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 45 k 2024-08-17T17:12:40.617 INFO:teuthology.orchestra.run.smithi179.stdout: python3-ceph-common x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 120 k 2024-08-17T17:12:40.617 INFO:teuthology.orchestra.run.smithi179.stdout: python3-cephfs x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 182 k 2024-08-17T17:12:40.617 INFO:teuthology.orchestra.run.smithi179.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-08-17T17:12:40.617 INFO:teuthology.orchestra.run.smithi179.stdout: python3-rados x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 368 k 2024-08-17T17:12:40.617 INFO:teuthology.orchestra.run.smithi179.stdout: python3-rbd x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 343 k 2024-08-17T17:12:40.617 INFO:teuthology.orchestra.run.smithi179.stdout: python3-rgw x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 116 k 2024-08-17T17:12:40.617 INFO:teuthology.orchestra.run.smithi179.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-08-17T17:12:40.617 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:12:40.617 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:12:40.617 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:12:40.617 INFO:teuthology.orchestra.run.smithi179.stdout:Install 22 Packages 2024-08-17T17:12:40.617 INFO:teuthology.orchestra.run.smithi179.stdout:Upgrade 2 Packages 2024-08-17T17:12:40.618 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:12:40.618 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 58 M 2024-08-17T17:12:40.618 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:12:41.368 INFO:teuthology.orchestra.run.smithi179.stdout:(1/24): ceph-base-17.2.7-1404.gf2c39d4d.el9.x86 8.5 MB/s | 5.0 MB 00:00 2024-08-17T17:12:41.444 INFO:teuthology.orchestra.run.smithi179.stdout:(2/24): ceph-selinux-17.2.7-1404.gf2c39d4d.el9. 336 kB/s | 25 kB 00:00 2024-08-17T17:12:41.552 INFO:teuthology.orchestra.run.smithi179.stdout:(3/24): libcephfs2-17.2.7-1404.gf2c39d4d.el9.x8 5.8 MB/s | 637 kB 00:00 2024-08-17T17:12:41.575 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:12:41.577 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:12:41.578 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repository Size 2024-08-17T17:12:41.578 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:12:41.578 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:12:41.578 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-radosgw x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 14 M 2024-08-17T17:12:41.578 INFO:teuthology.orchestra.run.smithi130.stdout:Upgrading: 2024-08-17T17:12:41.578 INFO:teuthology.orchestra.run.smithi130.stdout: librados2 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 3.1 M 2024-08-17T17:12:41.578 INFO:teuthology.orchestra.run.smithi130.stdout: librbd1 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 2.9 M 2024-08-17T17:12:41.578 INFO:teuthology.orchestra.run.smithi130.stdout:Installing dependencies: 2024-08-17T17:12:41.578 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-base x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 5.0 M 2024-08-17T17:12:41.578 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-common x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 22 M 2024-08-17T17:12:41.578 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-selinux x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 25 k 2024-08-17T17:12:41.578 INFO:teuthology.orchestra.run.smithi130.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-08-17T17:12:41.579 INFO:teuthology.orchestra.run.smithi130.stdout: libcephfs2 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 637 k 2024-08-17T17:12:41.579 INFO:teuthology.orchestra.run.smithi130.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-08-17T17:12:41.579 INFO:teuthology.orchestra.run.smithi130.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-08-17T17:12:41.579 INFO:teuthology.orchestra.run.smithi130.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-08-17T17:12:41.579 INFO:teuthology.orchestra.run.smithi130.stdout: libradosstriper1 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 457 k 2024-08-17T17:12:41.579 INFO:teuthology.orchestra.run.smithi130.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-08-17T17:12:41.579 INFO:teuthology.orchestra.run.smithi130.stdout: librgw2 x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 4.9 M 2024-08-17T17:12:41.579 INFO:teuthology.orchestra.run.smithi130.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-08-17T17:12:41.579 INFO:teuthology.orchestra.run.smithi130.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-08-17T17:12:41.579 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ceph-argparse x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 45 k 2024-08-17T17:12:41.579 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ceph-common x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 120 k 2024-08-17T17:12:41.579 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cephfs x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 182 k 2024-08-17T17:12:41.579 INFO:teuthology.orchestra.run.smithi130.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-08-17T17:12:41.580 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rados x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 368 k 2024-08-17T17:12:41.580 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rbd x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 343 k 2024-08-17T17:12:41.580 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rgw x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 116 k 2024-08-17T17:12:41.580 INFO:teuthology.orchestra.run.smithi130.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-08-17T17:12:41.580 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:12:41.580 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:12:41.580 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:12:41.580 INFO:teuthology.orchestra.run.smithi130.stdout:Install 22 Packages 2024-08-17T17:12:41.580 INFO:teuthology.orchestra.run.smithi130.stdout:Upgrade 2 Packages 2024-08-17T17:12:41.580 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:12:41.581 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 58 M 2024-08-17T17:12:41.581 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:12:41.652 INFO:teuthology.orchestra.run.smithi179.stdout:(4/24): libradosstriper1-17.2.7-1404.gf2c39d4d. 4.5 MB/s | 457 kB 00:00 2024-08-17T17:12:41.928 INFO:teuthology.orchestra.run.smithi179.stdout:(5/24): ceph-common-17.2.7-1404.gf2c39d4d.el9.x 20 MB/s | 22 MB 00:01 2024-08-17T17:12:41.970 INFO:teuthology.orchestra.run.smithi179.stdout:(6/24): python3-ceph-argparse-17.2.7-1404.gf2c3 1.1 MB/s | 45 kB 00:00 2024-08-17T17:12:42.112 INFO:teuthology.orchestra.run.smithi179.stdout:(7/24): ceph-radosgw-17.2.7-1404.gf2c39d4d.el9. 11 MB/s | 14 MB 00:01 2024-08-17T17:12:42.137 INFO:teuthology.orchestra.run.smithi179.stdout:(8/24): python3-ceph-common-17.2.7-1404.gf2c39d 721 kB/s | 120 kB 00:00 2024-08-17T17:12:42.212 INFO:teuthology.orchestra.run.smithi179.stdout:(9/24): librgw2-17.2.7-1404.gf2c39d4d.el9.x86_6 8.8 MB/s | 4.9 MB 00:00 2024-08-17T17:12:42.237 INFO:teuthology.orchestra.run.smithi179.stdout:(10/24): python3-cephfs-17.2.7-1404.gf2c39d4d.e 1.4 MB/s | 182 kB 00:00 2024-08-17T17:12:42.262 INFO:teuthology.orchestra.run.smithi179.stdout:(11/24): python3-rados-17.2.7-1404.gf2c39d4d.el 2.9 MB/s | 368 kB 00:00 2024-08-17T17:12:42.296 INFO:teuthology.orchestra.run.smithi179.stdout:(12/24): python3-rbd-17.2.7-1404.gf2c39d4d.el9. 4.0 MB/s | 343 kB 00:00 2024-08-17T17:12:42.321 INFO:teuthology.orchestra.run.smithi179.stdout:(13/24): python3-rgw-17.2.7-1404.gf2c39d4d.el9. 1.4 MB/s | 116 kB 00:00 2024-08-17T17:12:42.405 INFO:teuthology.orchestra.run.smithi179.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 418 kB/s | 45 kB 00:00 2024-08-17T17:12:42.430 INFO:teuthology.orchestra.run.smithi179.stdout:(15/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 956 kB/s | 160 kB 00:00 2024-08-17T17:12:42.481 INFO:teuthology.orchestra.run.smithi179.stdout:(16/24): python3-prettytable-0.7.2-27.el9.noarc 845 kB/s | 42 kB 00:00 2024-08-17T17:12:42.539 INFO:teuthology.orchestra.run.smithi179.stdout:(17/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.1 MB/s | 292 kB 00:00 2024-08-17T17:12:42.598 INFO:teuthology.orchestra.run.smithi179.stdout:(18/24): librdkafka-1.6.1-102.el9.x86_64.rpm 2.3 MB/s | 662 kB 00:00 2024-08-17T17:12:42.631 INFO:teuthology.orchestra.run.smithi179.stdout:(19/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 2.0 MB/s | 309 kB 00:00 2024-08-17T17:12:42.682 INFO:teuthology.orchestra.run.smithi179.stdout:(20/24): liboath-2.6.7-2.el9.x86_64.rpm 347 kB/s | 49 kB 00:00 2024-08-17T17:12:42.724 INFO:teuthology.orchestra.run.smithi179.stdout:(21/24): libunwind-1.6.2-1.el9.x86_64.rpm 546 kB/s | 67 kB 00:00 2024-08-17T17:12:42.808 INFO:teuthology.orchestra.run.smithi179.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 10 MB/s | 1.6 MB 00:00 2024-08-17T17:12:42.883 INFO:teuthology.orchestra.run.smithi179.stdout:(23/24): librados2-17.2.7-1404.gf2c39d4d.el9.x8 16 MB/s | 3.1 MB 00:00 2024-08-17T17:12:42.909 INFO:teuthology.orchestra.run.smithi130.stdout:(1/24): ceph-base-17.2.7-1404.gf2c39d4d.el9.x86 4.6 MB/s | 5.0 MB 00:01 2024-08-17T17:12:42.933 INFO:teuthology.orchestra.run.smithi179.stdout:(24/24): librbd1-17.2.7-1404.gf2c39d4d.el9.x86_ 14 MB/s | 2.9 MB 00:00 2024-08-17T17:12:42.938 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:12:42.938 INFO:teuthology.orchestra.run.smithi179.stdout:Total 25 MB/s | 58 MB 00:02 2024-08-17T17:12:42.943 INFO:teuthology.orchestra.run.smithi130.stdout:(2/24): ceph-selinux-17.2.7-1404.gf2c39d4d.el9. 760 kB/s | 25 kB 00:00 2024-08-17T17:12:42.992 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:12:43.024 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:12:43.024 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:12:43.193 INFO:teuthology.orchestra.run.smithi130.stdout:(3/24): ceph-common-17.2.7-1404.gf2c39d4d.el9.x 17 MB/s | 22 MB 00:01 2024-08-17T17:12:43.335 INFO:teuthology.orchestra.run.smithi130.stdout:(4/24): ceph-radosgw-17.2.7-1404.gf2c39d4d.el9. 9.6 MB/s | 14 MB 00:01 2024-08-17T17:12:43.369 INFO:teuthology.orchestra.run.smithi130.stdout:(5/24): libradosstriper1-17.2.7-1404.gf2c39d4d. 2.6 MB/s | 457 kB 00:00 2024-08-17T17:12:43.394 INFO:teuthology.orchestra.run.smithi130.stdout:(6/24): libcephfs2-17.2.7-1404.gf2c39d4d.el9.x8 1.4 MB/s | 637 kB 00:00 2024-08-17T17:12:43.420 INFO:teuthology.orchestra.run.smithi130.stdout:(7/24): python3-ceph-argparse-17.2.7-1404.gf2c3 898 kB/s | 45 kB 00:00 2024-08-17T17:12:43.445 INFO:teuthology.orchestra.run.smithi130.stdout:(8/24): python3-ceph-common-17.2.7-1404.gf2c39d 2.3 MB/s | 120 kB 00:00 2024-08-17T17:12:43.470 INFO:teuthology.orchestra.run.smithi130.stdout:(9/24): python3-cephfs-17.2.7-1404.gf2c39d4d.el 3.5 MB/s | 182 kB 00:00 2024-08-17T17:12:43.473 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:12:43.473 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:12:43.554 INFO:teuthology.orchestra.run.smithi130.stdout:(10/24): librgw2-17.2.7-1404.gf2c39d4d.el9.x86_ 23 MB/s | 4.9 MB 00:00 2024-08-17T17:12:43.579 INFO:teuthology.orchestra.run.smithi130.stdout:(11/24): python3-rados-17.2.7-1404.gf2c39d4d.el 2.7 MB/s | 368 kB 00:00 2024-08-17T17:12:43.605 INFO:teuthology.orchestra.run.smithi130.stdout:(12/24): python3-rgw-17.2.7-1404.gf2c39d4d.el9. 2.3 MB/s | 116 kB 00:00 2024-08-17T17:12:43.638 INFO:teuthology.orchestra.run.smithi130.stdout:(13/24): python3-rbd-17.2.7-1404.gf2c39d4d.el9. 2.0 MB/s | 343 kB 00:00 2024-08-17T17:12:43.755 INFO:teuthology.orchestra.run.smithi130.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 301 kB/s | 45 kB 00:00 2024-08-17T17:12:43.780 INFO:teuthology.orchestra.run.smithi130.stdout:(15/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 798 kB/s | 160 kB 00:00 2024-08-17T17:12:43.839 INFO:teuthology.orchestra.run.smithi130.stdout:(16/24): python3-prettytable-0.7.2-27.el9.noarc 723 kB/s | 42 kB 00:00 2024-08-17T17:12:43.915 INFO:teuthology.orchestra.run.smithi130.stdout:(17/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 4.0 MB/s | 309 kB 00:00 2024-08-17T17:12:43.948 INFO:teuthology.orchestra.run.smithi130.stdout:(18/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 1.5 MB/s | 292 kB 00:00 2024-08-17T17:12:43.973 INFO:teuthology.orchestra.run.smithi130.stdout:(19/24): librdkafka-1.6.1-102.el9.x86_64.rpm 1.9 MB/s | 662 kB 00:00 2024-08-17T17:12:44.024 INFO:teuthology.orchestra.run.smithi130.stdout:(20/24): liboath-2.6.7-2.el9.x86_64.rpm 446 kB/s | 49 kB 00:00 2024-08-17T17:12:44.074 INFO:teuthology.orchestra.run.smithi130.stdout:(21/24): libunwind-1.6.2-1.el9.x86_64.rpm 537 kB/s | 67 kB 00:00 2024-08-17T17:12:44.149 INFO:teuthology.orchestra.run.smithi130.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 11 MB/s | 1.6 MB 00:00 2024-08-17T17:12:44.308 INFO:teuthology.orchestra.run.smithi130.stdout:(23/24): librbd1-17.2.7-1404.gf2c39d4d.el9.x86_ 13 MB/s | 2.9 MB 00:00 2024-08-17T17:12:44.358 INFO:teuthology.orchestra.run.smithi130.stdout:(24/24): librados2-17.2.7-1404.gf2c39d4d.el9.x8 10 MB/s | 3.1 MB 00:00 2024-08-17T17:12:44.363 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:12:44.363 INFO:teuthology.orchestra.run.smithi130.stdout:Total 21 MB/s | 58 MB 00:02 2024-08-17T17:12:44.424 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:12:44.456 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:12:44.456 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:12:44.657 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:12:44.904 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:12:44.905 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:12:44.978 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-08-17T17:12:45.085 INFO:teuthology.orchestra.run.smithi179.stdout: Upgrading : librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/26 2024-08-17T17:12:45.186 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/26 2024-08-17T17:12:45.272 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/26 2024-08-17T17:12:45.303 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/26 2024-08-17T17:12:45.394 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/26 2024-08-17T17:12:45.437 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-08-17T17:12:45.466 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-08-17T17:12:45.903 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-08-17T17:12:45.931 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 8/26 2024-08-17T17:12:45.994 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 8/26 2024-08-17T17:12:46.031 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-08-17T17:12:46.074 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:12:46.076 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-ceph-argparse-2:17.2.7-1404.gf2c39d4d.el9. 10/26 2024-08-17T17:12:46.112 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 11/26 2024-08-17T17:12:46.173 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 12/26 2024-08-17T17:12:46.201 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_6 13/26 2024-08-17T17:12:46.274 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_6 13/26 2024-08-17T17:12:46.315 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-08-17T17:12:46.351 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-08-17T17:12:46.395 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-08-17T17:12:46.399 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-08-17T17:12:46.502 INFO:teuthology.orchestra.run.smithi130.stdout: Upgrading : librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/26 2024-08-17T17:12:46.606 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/26 2024-08-17T17:12:46.688 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/26 2024-08-17T17:12:46.701 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-08-17T17:12:46.720 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/26 2024-08-17T17:12:46.754 INFO:teuthology.orchestra.run.smithi179.stdout: Upgrading : librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 18/26 2024-08-17T17:12:46.803 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/26 2024-08-17T17:12:46.840 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 18/26 2024-08-17T17:12:46.851 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-08-17T17:12:46.884 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-08-17T17:12:46.909 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 19/26 2024-08-17T17:12:46.940 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x8 20/26 2024-08-17T17:12:47.323 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-08-17T17:12:47.348 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 8/26 2024-08-17T17:12:47.411 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 8/26 2024-08-17T17:12:47.448 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-08-17T17:12:47.493 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-ceph-argparse-2:17.2.7-1404.gf2c39d4d.el9. 10/26 2024-08-17T17:12:47.529 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 11/26 2024-08-17T17:12:47.590 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 12/26 2024-08-17T17:12:47.617 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_6 13/26 2024-08-17T17:12:47.667 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_6 13/26 2024-08-17T17:12:47.707 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-08-17T17:12:47.743 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-08-17T17:12:47.775 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-08-17T17:12:48.088 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-08-17T17:12:48.138 INFO:teuthology.orchestra.run.smithi130.stdout: Upgrading : librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 18/26 2024-08-17T17:12:48.233 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 18/26 2024-08-17T17:12:48.304 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 19/26 2024-08-17T17:12:48.334 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x8 20/26 2024-08-17T17:12:49.414 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 21/26 2024-08-17T17:12:49.457 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 21/26 2024-08-17T17:12:49.994 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 21/26 2024-08-17T17:12:50.035 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 22/26 2024-08-17T17:12:50.121 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 22/26 2024-08-17T17:12:50.121 INFO:teuthology.orchestra.run.smithi179.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-08-17T17:12:50.121 INFO:teuthology.orchestra.run.smithi179.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-08-17T17:12:50.122 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:12:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 23/26 2024-08-17T17:12:50.759 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 21/26 2024-08-17T17:12:50.798 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 21/26 2024-08-17T17:12:51.314 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 21/26 2024-08-17T17:12:51.352 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 22/26 2024-08-17T17:12:51.449 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 22/26 2024-08-17T17:12:51.449 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-08-17T17:12:51.450 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-08-17T17:12:51.450 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:12:51.486 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 23/26 2024-08-17T17:12:59.874 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 23/26 2024-08-17T17:12:59.875 INFO:teuthology.orchestra.run.smithi179.stdout:skipping the directory /sys 2024-08-17T17:12:59.875 INFO:teuthology.orchestra.run.smithi179.stdout:skipping the directory /proc 2024-08-17T17:12:59.875 INFO:teuthology.orchestra.run.smithi179.stdout:skipping the directory /mnt 2024-08-17T17:12:59.875 INFO:teuthology.orchestra.run.smithi179.stdout:skipping the directory /var/tmp 2024-08-17T17:12:59.875 INFO:teuthology.orchestra.run.smithi179.stdout:skipping the directory /home 2024-08-17T17:12:59.875 INFO:teuthology.orchestra.run.smithi179.stdout:skipping the directory /root 2024-08-17T17:12:59.875 INFO:teuthology.orchestra.run.smithi179.stdout:skipping the directory /tmp 2024-08-17T17:12:59.875 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:00.971 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 23/26 2024-08-17T17:13:00.971 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /sys 2024-08-17T17:13:00.971 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /proc 2024-08-17T17:13:00.971 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /mnt 2024-08-17T17:13:00.971 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /var/tmp 2024-08-17T17:13:00.971 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /home 2024-08-17T17:13:00.971 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /root 2024-08-17T17:13:00.971 INFO:teuthology.orchestra.run.smithi130.stdout:skipping the directory /tmp 2024-08-17T17:13:00.971 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:01.152 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 24/26 2024-08-17T17:13:01.224 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 24/26 2024-08-17T17:13:01.224 INFO:teuthology.orchestra.run.smithi179.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T17:13:01.224 INFO:teuthology.orchestra.run.smithi179.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-08-17T17:13:01.224 INFO:teuthology.orchestra.run.smithi179.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T17:13:01.225 INFO:teuthology.orchestra.run.smithi179.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T17:13:01.225 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:01.226 INFO:teuthology.orchestra.run.smithi179.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-08-17T17:13:01.313 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-08-17T17:13:01.314 INFO:teuthology.orchestra.run.smithi179.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-08-17T17:13:02.289 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 24/26 2024-08-17T17:13:02.367 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 24/26 2024-08-17T17:13:02.367 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T17:13:02.367 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-08-17T17:13:02.367 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T17:13:02.367 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T17:13:02.367 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:02.368 INFO:teuthology.orchestra.run.smithi130.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-08-17T17:13:02.458 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-08-17T17:13:02.458 INFO:teuthology.orchestra.run.smithi130.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-08-17T17:13:02.634 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-08-17T17:13:02.634 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/26 2024-08-17T17:13:02.634 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/26 2024-08-17T17:13:02.634 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/26 2024-08-17T17:13:02.634 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/26 2024-08-17T17:13:02.634 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 5/26 2024-08-17T17:13:02.634 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_6 6/26 2024-08-17T17:13:02.635 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 7/26 2024-08-17T17:13:02.635 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1404.gf2c39d4d.el9. 8/26 2024-08-17T17:13:02.635 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x8 9/26 2024-08-17T17:13:02.635 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 10/26 2024-08-17T17:13:02.635 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 11/26 2024-08-17T17:13:02.635 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 12/26 2024-08-17T17:13:02.635 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 13/26 2024-08-17T17:13:02.635 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-08-17T17:13:02.635 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-08-17T17:13:02.635 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-08-17T17:13:02.635 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-08-17T17:13:02.635 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-08-17T17:13:02.636 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-08-17T17:13:02.636 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-08-17T17:13:02.636 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-08-17T17:13:02.636 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-08-17T17:13:02.636 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 23/26 2024-08-17T17:13:02.636 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-08-17T17:13:02.636 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 25/26 2024-08-17T17:13:03.374 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-08-17T17:13:03.375 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:03.375 INFO:teuthology.orchestra.run.smithi179.stdout:Upgraded: 2024-08-17T17:13:03.375 INFO:teuthology.orchestra.run.smithi179.stdout: librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.375 INFO:teuthology.orchestra.run.smithi179.stdout: librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.375 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:13:03.375 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.375 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.375 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.375 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.375 INFO:teuthology.orchestra.run.smithi179.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-08-17T17:13:03.375 INFO:teuthology.orchestra.run.smithi179.stdout: libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.376 INFO:teuthology.orchestra.run.smithi179.stdout: liboath-2.6.7-2.el9.x86_64 2024-08-17T17:13:03.376 INFO:teuthology.orchestra.run.smithi179.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-08-17T17:13:03.376 INFO:teuthology.orchestra.run.smithi179.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-08-17T17:13:03.376 INFO:teuthology.orchestra.run.smithi179.stdout: libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.376 INFO:teuthology.orchestra.run.smithi179.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-08-17T17:13:03.376 INFO:teuthology.orchestra.run.smithi179.stdout: librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.376 INFO:teuthology.orchestra.run.smithi179.stdout: libunwind-1.6.2-1.el9.x86_64 2024-08-17T17:13:03.376 INFO:teuthology.orchestra.run.smithi179.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-08-17T17:13:03.376 INFO:teuthology.orchestra.run.smithi179.stdout: python3-ceph-argparse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.376 INFO:teuthology.orchestra.run.smithi179.stdout: python3-ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.376 INFO:teuthology.orchestra.run.smithi179.stdout: python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.377 INFO:teuthology.orchestra.run.smithi179.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-08-17T17:13:03.377 INFO:teuthology.orchestra.run.smithi179.stdout: python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.377 INFO:teuthology.orchestra.run.smithi179.stdout: python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.377 INFO:teuthology.orchestra.run.smithi179.stdout: python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:03.377 INFO:teuthology.orchestra.run.smithi179.stdout: thrift-0.15.0-2.el9.x86_64 2024-08-17T17:13:03.377 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:03.377 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:13:03.624 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-08-17T17:13:03.625 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/26 2024-08-17T17:13:03.625 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/26 2024-08-17T17:13:03.625 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/26 2024-08-17T17:13:03.625 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/26 2024-08-17T17:13:03.625 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 5/26 2024-08-17T17:13:03.625 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_6 6/26 2024-08-17T17:13:03.625 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 7/26 2024-08-17T17:13:03.625 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1404.gf2c39d4d.el9. 8/26 2024-08-17T17:13:03.625 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x8 9/26 2024-08-17T17:13:03.625 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 10/26 2024-08-17T17:13:03.625 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 11/26 2024-08-17T17:13:03.626 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 12/26 2024-08-17T17:13:03.626 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 13/26 2024-08-17T17:13:03.626 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-08-17T17:13:03.626 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-08-17T17:13:03.626 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-08-17T17:13:03.626 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-08-17T17:13:03.626 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-08-17T17:13:03.626 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-08-17T17:13:03.626 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-08-17T17:13:03.626 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-08-17T17:13:03.626 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-08-17T17:13:03.626 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 23/26 2024-08-17T17:13:03.626 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-08-17T17:13:03.627 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 25/26 2024-08-17T17:13:03.713 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install ceph-test 2024-08-17T17:13:04.270 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:00:26 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:13:04.296 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-08-17T17:13:04.296 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:04.296 INFO:teuthology.orchestra.run.smithi130.stdout:Upgraded: 2024-08-17T17:13:04.296 INFO:teuthology.orchestra.run.smithi130.stdout: librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.296 INFO:teuthology.orchestra.run.smithi130.stdout: librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.296 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:13:04.296 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.296 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.296 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-radosgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.296 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-selinux-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.297 INFO:teuthology.orchestra.run.smithi130.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-08-17T17:13:04.297 INFO:teuthology.orchestra.run.smithi130.stdout: libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.297 INFO:teuthology.orchestra.run.smithi130.stdout: liboath-2.6.7-2.el9.x86_64 2024-08-17T17:13:04.297 INFO:teuthology.orchestra.run.smithi130.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-08-17T17:13:04.297 INFO:teuthology.orchestra.run.smithi130.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-08-17T17:13:04.297 INFO:teuthology.orchestra.run.smithi130.stdout: libradosstriper1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.297 INFO:teuthology.orchestra.run.smithi130.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-08-17T17:13:04.297 INFO:teuthology.orchestra.run.smithi130.stdout: librgw2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.297 INFO:teuthology.orchestra.run.smithi130.stdout: libunwind-1.6.2-1.el9.x86_64 2024-08-17T17:13:04.297 INFO:teuthology.orchestra.run.smithi130.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-08-17T17:13:04.297 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ceph-argparse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.297 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ceph-common-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.297 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.298 INFO:teuthology.orchestra.run.smithi130.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-08-17T17:13:04.298 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.298 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.298 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:04.298 INFO:teuthology.orchestra.run.smithi130.stdout: thrift-0.15.0-2.el9.x86_64 2024-08-17T17:13:04.298 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:04.298 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:13:04.395 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:13:04.397 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:04.397 INFO:teuthology.orchestra.run.smithi179.stdout: Package Arch Version Repository Size 2024-08-17T17:13:04.397 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:04.397 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:13:04.397 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-test x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 48 M 2024-08-17T17:13:04.397 INFO:teuthology.orchestra.run.smithi179.stdout:Installing dependencies: 2024-08-17T17:13:04.397 INFO:teuthology.orchestra.run.smithi179.stdout: libcephsqlite x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 162 k 2024-08-17T17:13:04.397 INFO:teuthology.orchestra.run.smithi179.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-08-17T17:13:04.397 INFO:teuthology.orchestra.run.smithi179.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-08-17T17:13:04.398 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:04.398 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:13:04.398 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:04.398 INFO:teuthology.orchestra.run.smithi179.stdout:Install 4 Packages 2024-08-17T17:13:04.398 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:04.398 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 48 M 2024-08-17T17:13:04.398 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 193 M 2024-08-17T17:13:04.398 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:13:04.657 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install ceph-test 2024-08-17T17:13:04.686 INFO:teuthology.orchestra.run.smithi179.stdout:(1/4): libcephsqlite-17.2.7-1404.gf2c39d4d.el9. 655 kB/s | 162 kB 00:00 2024-08-17T17:13:04.712 INFO:teuthology.orchestra.run.smithi179.stdout:(2/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.1 MB/s | 304 kB 00:00 2024-08-17T17:13:04.828 INFO:teuthology.orchestra.run.smithi179.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 449 kB/s | 64 kB 00:00 2024-08-17T17:13:05.207 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:00:26 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:13:05.333 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:13:05.333 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:05.334 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repository Size 2024-08-17T17:13:05.334 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:05.334 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:13:05.334 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-test x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 48 M 2024-08-17T17:13:05.334 INFO:teuthology.orchestra.run.smithi130.stdout:Installing dependencies: 2024-08-17T17:13:05.334 INFO:teuthology.orchestra.run.smithi130.stdout: libcephsqlite x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 162 k 2024-08-17T17:13:05.334 INFO:teuthology.orchestra.run.smithi130.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-08-17T17:13:05.334 INFO:teuthology.orchestra.run.smithi130.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-08-17T17:13:05.334 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:05.334 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:13:05.334 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:05.335 INFO:teuthology.orchestra.run.smithi130.stdout:Install 4 Packages 2024-08-17T17:13:05.335 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:05.335 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 48 M 2024-08-17T17:13:05.335 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 193 M 2024-08-17T17:13:05.335 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:13:05.770 INFO:teuthology.orchestra.run.smithi130.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.9 MB/s | 304 kB 00:00 2024-08-17T17:13:05.812 INFO:teuthology.orchestra.run.smithi130.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 64 kB 00:00 2024-08-17T17:13:05.879 INFO:teuthology.orchestra.run.smithi130.stdout:(3/4): libcephsqlite-17.2.7-1404.gf2c39d4d.el9. 605 kB/s | 162 kB 00:00 2024-08-17T17:13:05.929 INFO:teuthology.orchestra.run.smithi179.stdout:(4/4): ceph-test-17.2.7-1404.gf2c39d4d.el9.x86_ 32 MB/s | 48 MB 00:01 2024-08-17T17:13:05.931 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:05.931 INFO:teuthology.orchestra.run.smithi179.stdout:Total 32 MB/s | 48 MB 00:01 2024-08-17T17:13:05.958 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:13:05.978 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:13:05.978 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:13:06.342 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:13:06.342 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:13:06.858 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:13:06.937 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-08-17T17:13:06.981 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-08-17T17:13:07.005 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/4 2024-08-17T17:13:07.129 INFO:teuthology.orchestra.run.smithi130.stdout:(4/4): ceph-test-17.2.7-1404.gf2c39d4d.el9.x86_ 32 MB/s | 48 MB 00:01 2024-08-17T17:13:07.131 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:07.131 INFO:teuthology.orchestra.run.smithi130.stdout:Total 27 MB/s | 48 MB 00:01 2024-08-17T17:13:07.159 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:13:07.179 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:13:07.179 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:13:07.543 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:13:07.544 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:13:08.063 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:13:08.130 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-08-17T17:13:08.180 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-08-17T17:13:08.206 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/4 2024-08-17T17:13:11.376 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/4 2024-08-17T17:13:11.454 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/4 2024-08-17T17:13:12.638 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/4 2024-08-17T17:13:12.713 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/4 2024-08-17T17:13:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/4 2024-08-17T17:13:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/4 2024-08-17T17:13:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/4 2024-08-17T17:13:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-08-17T17:13:14.102 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-08-17T17:13:14.103 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:14.103 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:13:14.103 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:14.103 INFO:teuthology.orchestra.run.smithi179.stdout: libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:14.103 INFO:teuthology.orchestra.run.smithi179.stdout: socat-1.7.4.1-6.el9.x86_64 2024-08-17T17:13:14.103 INFO:teuthology.orchestra.run.smithi179.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-08-17T17:13:14.103 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:14.103 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:13:14.390 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install ceph 2024-08-17T17:13:14.945 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:00:36 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:13:15.003 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/4 2024-08-17T17:13:15.004 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/4 2024-08-17T17:13:15.004 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/4 2024-08-17T17:13:15.004 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-08-17T17:13:15.074 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:13:15.077 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: Package Arch Version Repository Size 2024-08-17T17:13:15.077 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:15.077 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:13:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: ceph x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 6.5 k 2024-08-17T17:13:15.077 INFO:teuthology.orchestra.run.smithi179.stdout:Installing dependencies: 2024-08-17T17:13:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mds x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 2.1 M 2024-08-17T17:13:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mgr x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 1.4 M 2024-08-17T17:13:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 240 k 2024-08-17T17:13:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mon x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 3.9 M 2024-08-17T17:13:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-osd x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 15 M 2024-08-17T17:13:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-08-17T17:13:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-08-17T17:13:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-08-17T17:13:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-08-17T17:13:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-08-17T17:13:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-08-17T17:13:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-08-17T17:13:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jaraco-collections 2024-08-17T17:13:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-08-17T17:13:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-08-17T17:13:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-08-17T17:13:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-08-17T17:13:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-08-17T17:13:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-08-17T17:13:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-08-17T17:13:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-08-17T17:13:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-08-17T17:13:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-08-17T17:13:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-08-17T17:13:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-08-17T17:13:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-08-17T17:13:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-08-17T17:13:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-08-17T17:13:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-08-17T17:13:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-08-17T17:13:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-08-17T17:13:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-08-17T17:13:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-08-17T17:13:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-08-17T17:13:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:15.080 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:13:15.080 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:15.080 INFO:teuthology.orchestra.run.smithi179.stdout:Install 34 Packages 2024-08-17T17:13:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:15.081 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 27 M 2024-08-17T17:13:15.081 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 96 M 2024-08-17T17:13:15.081 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:13:15.272 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-08-17T17:13:15.272 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:15.272 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:13:15.273 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-test-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:15.273 INFO:teuthology.orchestra.run.smithi130.stdout: libcephsqlite-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:15.273 INFO:teuthology.orchestra.run.smithi130.stdout: socat-1.7.4.1-6.el9.x86_64 2024-08-17T17:13:15.273 INFO:teuthology.orchestra.run.smithi130.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-08-17T17:13:15.273 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:15.273 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:13:15.538 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install ceph 2024-08-17T17:13:15.839 INFO:teuthology.orchestra.run.smithi179.stdout:(1/34): ceph-17.2.7-1404.gf2c39d4d.el9.x86_64.r 34 kB/s | 6.5 kB 00:00 2024-08-17T17:13:15.997 INFO:teuthology.orchestra.run.smithi179.stdout:(2/34): ceph-mds-17.2.7-1404.gf2c39d4d.el9.x86_ 5.9 MB/s | 2.1 MB 00:00 2024-08-17T17:13:16.039 INFO:teuthology.orchestra.run.smithi179.stdout:(3/34): ceph-mgr-17.2.7-1404.gf2c39d4d.el9.x86_ 3.6 MB/s | 1.4 MB 00:00 2024-08-17T17:13:16.084 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:00:37 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:13:16.131 INFO:teuthology.orchestra.run.smithi179.stdout:(4/34): ceph-mgr-modules-core-17.2.7-1404.gf2c3 2.6 MB/s | 240 kB 00:00 2024-08-17T17:13:16.198 INFO:teuthology.orchestra.run.smithi179.stdout:(5/34): ceph-mon-17.2.7-1404.gf2c39d4d.el9.x86_ 11 MB/s | 3.9 MB 00:00 2024-08-17T17:13:16.216 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:13:16.219 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:16.219 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repository Size 2024-08-17T17:13:16.219 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:16.219 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:13:16.219 INFO:teuthology.orchestra.run.smithi130.stdout: ceph x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 6.5 k 2024-08-17T17:13:16.219 INFO:teuthology.orchestra.run.smithi130.stdout:Installing dependencies: 2024-08-17T17:13:16.219 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mds x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 2.1 M 2024-08-17T17:13:16.219 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 1.4 M 2024-08-17T17:13:16.219 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 240 k 2024-08-17T17:13:16.220 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mon x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 3.9 M 2024-08-17T17:13:16.220 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-osd x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 15 M 2024-08-17T17:13:16.220 INFO:teuthology.orchestra.run.smithi130.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-08-17T17:13:16.220 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-08-17T17:13:16.220 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-08-17T17:13:16.220 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-08-17T17:13:16.220 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-08-17T17:13:16.220 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-08-17T17:13:16.220 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-08-17T17:13:16.220 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-collections 2024-08-17T17:13:16.220 INFO:teuthology.orchestra.run.smithi130.stdout: noarch 3.0.0-8.el9 epel 23 k 2024-08-17T17:13:16.220 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-08-17T17:13:16.220 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-08-17T17:13:16.221 INFO:teuthology.orchestra.run.smithi130.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-08-17T17:13:16.221 INFO:teuthology.orchestra.run.smithi130.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-08-17T17:13:16.221 INFO:teuthology.orchestra.run.smithi130.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-08-17T17:13:16.221 INFO:teuthology.orchestra.run.smithi130.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-08-17T17:13:16.221 INFO:teuthology.orchestra.run.smithi130.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-08-17T17:13:16.221 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-08-17T17:13:16.221 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-08-17T17:13:16.221 INFO:teuthology.orchestra.run.smithi130.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-08-17T17:13:16.221 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-08-17T17:13:16.221 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-08-17T17:13:16.221 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-08-17T17:13:16.221 INFO:teuthology.orchestra.run.smithi130.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-08-17T17:13:16.221 INFO:teuthology.orchestra.run.smithi130.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-08-17T17:13:16.222 INFO:teuthology.orchestra.run.smithi130.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-08-17T17:13:16.222 INFO:teuthology.orchestra.run.smithi130.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-08-17T17:13:16.222 INFO:teuthology.orchestra.run.smithi130.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-08-17T17:13:16.222 INFO:teuthology.orchestra.run.smithi130.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-08-17T17:13:16.222 INFO:teuthology.orchestra.run.smithi130.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-08-17T17:13:16.222 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:16.222 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:13:16.222 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:16.222 INFO:teuthology.orchestra.run.smithi130.stdout:Install 34 Packages 2024-08-17T17:13:16.222 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:16.223 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 27 M 2024-08-17T17:13:16.223 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 96 M 2024-08-17T17:13:16.223 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:13:16.348 INFO:teuthology.orchestra.run.smithi179.stdout:(6/34): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.1 MB/s | 253 kB 00:00 2024-08-17T17:13:16.507 INFO:teuthology.orchestra.run.smithi179.stdout:(7/34): ceph-osd-17.2.7-1404.gf2c39d4d.el9.x86_ 30 MB/s | 15 MB 00:00 2024-08-17T17:13:16.532 INFO:teuthology.orchestra.run.smithi179.stdout:(8/34): python3-ply-3.11-14.el9.noarch.rpm 580 kB/s | 106 kB 00:00 2024-08-17T17:13:16.574 INFO:teuthology.orchestra.run.smithi179.stdout:(9/34): python3-cryptography-36.0.1-4.el9.x86_6 3.3 MB/s | 1.2 MB 00:00 2024-08-17T17:13:16.599 INFO:teuthology.orchestra.run.smithi179.stdout:(10/34): python3-requests-2.25.1-8.el9.noarch.r 1.8 MB/s | 125 kB 00:00 2024-08-17T17:13:16.622 INFO:teuthology.orchestra.run.smithi130.stdout:(1/34): ceph-17.2.7-1404.gf2c39d4d.el9.x86_64.r 35 kB/s | 6.5 kB 00:00 2024-08-17T17:13:16.624 INFO:teuthology.orchestra.run.smithi179.stdout:(11/34): python3-urllib3-1.26.5-6.el9.noarch.rp 4.2 MB/s | 215 kB 00:00 2024-08-17T17:13:16.658 INFO:teuthology.orchestra.run.smithi179.stdout:(12/34): python3-pycparser-2.20-6.el9.noarch.rp 896 kB/s | 135 kB 00:00 2024-08-17T17:13:16.789 INFO:teuthology.orchestra.run.smithi130.stdout:(2/34): ceph-mds-17.2.7-1404.gf2c39d4d.el9.x86_ 5.8 MB/s | 2.1 MB 00:00 2024-08-17T17:13:16.831 INFO:teuthology.orchestra.run.smithi130.stdout:(3/34): ceph-mgr-17.2.7-1404.gf2c39d4d.el9.x86_ 3.5 MB/s | 1.4 MB 00:00 2024-08-17T17:13:16.907 INFO:teuthology.orchestra.run.smithi130.stdout:(4/34): ceph-mgr-modules-core-17.2.7-1404.gf2c3 3.1 MB/s | 240 kB 00:00 2024-08-17T17:13:17.032 INFO:teuthology.orchestra.run.smithi130.stdout:(5/34): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.0 MB/s | 253 kB 00:00 2024-08-17T17:13:17.116 INFO:teuthology.orchestra.run.smithi130.stdout:(6/34): python3-cryptography-36.0.1-4.el9.x86_6 15 MB/s | 1.2 MB 00:00 2024-08-17T17:13:17.250 INFO:teuthology.orchestra.run.smithi179.stdout:(13/34): python3-markupsafe-1.1.1-12.el9.x86_64 56 kB/s | 35 kB 00:00 2024-08-17T17:13:17.291 INFO:teuthology.orchestra.run.smithi130.stdout:(7/34): ceph-osd-17.2.7-1404.gf2c39d4d.el9.x86_ 31 MB/s | 15 MB 00:00 2024-08-17T17:13:17.309 INFO:teuthology.orchestra.run.smithi179.stdout:(14/34): python3-packaging-20.9-5.el9.noarch.rp 119 kB/s | 77 kB 00:00 2024-08-17T17:13:17.316 INFO:teuthology.orchestra.run.smithi130.stdout:(8/34): python3-ply-3.11-14.el9.noarch.rpm 532 kB/s | 106 kB 00:00 2024-08-17T17:13:17.359 INFO:teuthology.orchestra.run.smithi179.stdout:(15/34): python3-mako-1.1.4-6.el9.noarch.rpm 227 kB/s | 172 kB 00:00 2024-08-17T17:13:17.391 INFO:teuthology.orchestra.run.smithi130.stdout:(9/34): ceph-mon-17.2.7-1404.gf2c39d4d.el9.x86_ 5.1 MB/s | 3.9 MB 00:00 2024-08-17T17:13:17.417 INFO:teuthology.orchestra.run.smithi130.stdout:(10/34): python3-requests-2.25.1-8.el9.noarch.r 1.2 MB/s | 125 kB 00:00 2024-08-17T17:13:17.417 INFO:teuthology.orchestra.run.smithi179.stdout:(16/34): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 748 kB/s | 43 kB 00:00 2024-08-17T17:13:17.442 INFO:teuthology.orchestra.run.smithi130.stdout:(11/34): python3-pycparser-2.20-6.el9.noarch.rp 898 kB/s | 135 kB 00:00 2024-08-17T17:13:17.451 INFO:teuthology.orchestra.run.smithi179.stdout:(17/34): python3-cheroot-8.6.0-4.el9.noarch.rpm 5.0 MB/s | 172 kB 00:00 2024-08-17T17:13:17.517 INFO:teuthology.orchestra.run.smithi130.stdout:(12/34): python3-urllib3-1.26.5-6.el9.noarch.rp 1.7 MB/s | 215 kB 00:00 2024-08-17T17:13:17.518 INFO:teuthology.orchestra.run.smithi179.stdout:(18/34): python3-cherrypy-18.6.1-2.el9.noarch.r 5.3 MB/s | 358 kB 00:00 2024-08-17T17:13:17.535 INFO:teuthology.orchestra.run.smithi179.stdout:(19/34): python3-jaraco-8.2.1-3.el9.noarch.rpm 643 kB/s | 11 kB 00:00 2024-08-17T17:13:17.576 INFO:teuthology.orchestra.run.smithi130.stdout:(13/34): python3-markupsafe-1.1.1-12.el9.x86_64 260 kB/s | 35 kB 00:00 2024-08-17T17:13:17.577 INFO:teuthology.orchestra.run.smithi179.stdout:(20/34): python3-jaraco-classes-3.2.1-5.el9.noa 985 kB/s | 18 kB 00:00 2024-08-17T17:13:17.602 INFO:teuthology.orchestra.run.smithi179.stdout:(21/34): python3-jaraco-collections-3.0.0-8.el9 954 kB/s | 23 kB 00:00 2024-08-17T17:13:17.619 INFO:teuthology.orchestra.run.smithi179.stdout:(22/34): python3-jaraco-functools-3.5.0-2.el9.n 1.1 MB/s | 19 kB 00:00 2024-08-17T17:13:17.634 INFO:teuthology.orchestra.run.smithi130.stdout:(14/34): python3-mako-1.1.4-6.el9.noarch.rpm 792 kB/s | 172 kB 00:00 2024-08-17T17:13:17.636 INFO:teuthology.orchestra.run.smithi179.stdout:(23/34): python3-jaraco-text-3.2.0-6.el9.noarch 1.1 MB/s | 20 kB 00:00 2024-08-17T17:13:17.669 INFO:teuthology.orchestra.run.smithi179.stdout:(24/34): python3-logutils-0.3.5-21.el9.noarch.r 1.4 MB/s | 46 kB 00:00 2024-08-17T17:13:17.676 INFO:teuthology.orchestra.run.smithi130.stdout:(15/34): python3-pytz-2021.1-5.el9.noarch.rpm 509 kB/s | 51 kB 00:00 2024-08-17T17:13:17.701 INFO:teuthology.orchestra.run.smithi130.stdout:(16/34): python3-toml-0.10.2-6.el9.noarch.rpm 624 kB/s | 42 kB 00:00 2024-08-17T17:13:17.703 INFO:teuthology.orchestra.run.smithi179.stdout:(25/34): python3-more-itertools-8.12.0-2.el9.no 2.3 MB/s | 79 kB 00:00 2024-08-17T17:13:17.727 INFO:teuthology.orchestra.run.smithi130.stdout:(17/34): python3-packaging-20.9-5.el9.noarch.rp 369 kB/s | 77 kB 00:00 2024-08-17T17:13:17.737 INFO:teuthology.orchestra.run.smithi179.stdout:(26/34): python3-pecan-1.4.2-3.el9.noarch.rpm 8.0 MB/s | 272 kB 00:00 2024-08-17T17:13:17.762 INFO:teuthology.orchestra.run.smithi179.stdout:(27/34): python3-portend-3.1.0-2.el9.noarch.rpm 656 kB/s | 16 kB 00:00 2024-08-17T17:13:17.795 INFO:teuthology.orchestra.run.smithi179.stdout:(28/34): python3-pyOpenSSL-21.0.0-1.el9.noarch. 2.6 MB/s | 90 kB 00:00 2024-08-17T17:13:17.821 INFO:teuthology.orchestra.run.smithi179.stdout:(29/34): python3-tempora-5.0.0-2.el9.noarch.rpm 1.4 MB/s | 36 kB 00:00 2024-08-17T17:13:17.827 INFO:teuthology.orchestra.run.smithi130.stdout:(18/34): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 288 kB/s | 43 kB 00:00 2024-08-17T17:13:17.852 INFO:teuthology.orchestra.run.smithi130.stdout:(19/34): python3-jaraco-8.2.1-3.el9.noarch.rpm 470 kB/s | 11 kB 00:00 2024-08-17T17:13:17.863 INFO:teuthology.orchestra.run.smithi179.stdout:(30/34): python3-webob-1.8.7-6.el9.noarch.rpm 5.4 MB/s | 230 kB 00:00 2024-08-17T17:13:17.877 INFO:teuthology.orchestra.run.smithi130.stdout:(20/34): python3-cheroot-8.6.0-4.el9.noarch.rpm 977 kB/s | 172 kB 00:00 2024-08-17T17:13:17.903 INFO:teuthology.orchestra.run.smithi130.stdout:(21/34): python3-jaraco-classes-3.2.1-5.el9.noa 353 kB/s | 18 kB 00:00 2024-08-17T17:13:17.913 INFO:teuthology.orchestra.run.smithi179.stdout:(31/34): python3-werkzeug-2.0.3-3.el9.1.noarch. 8.3 MB/s | 427 kB 00:00 2024-08-17T17:13:17.928 INFO:teuthology.orchestra.run.smithi130.stdout:(22/34): python3-jaraco-collections-3.0.0-8.el9 460 kB/s | 23 kB 00:00 2024-08-17T17:13:17.939 INFO:teuthology.orchestra.run.smithi179.stdout:(32/34): python3-pytz-2021.1-5.el9.noarch.rpm 74 kB/s | 51 kB 00:00 2024-08-17T17:13:17.953 INFO:teuthology.orchestra.run.smithi130.stdout:(23/34): python3-jaraco-functools-3.5.0-2.el9.n 387 kB/s | 19 kB 00:00 2024-08-17T17:13:17.964 INFO:teuthology.orchestra.run.smithi179.stdout:(33/34): python3-toml-0.10.2-6.el9.noarch.rpm 64 kB/s | 42 kB 00:00 2024-08-17T17:13:17.978 INFO:teuthology.orchestra.run.smithi130.stdout:(24/34): python3-jaraco-text-3.2.0-6.el9.noarch 392 kB/s | 20 kB 00:00 2024-08-17T17:13:17.989 INFO:teuthology.orchestra.run.smithi179.stdout:(34/34): python3-zc-lockfile-2.0-10.el9.noarch. 270 kB/s | 20 kB 00:00 2024-08-17T17:13:17.992 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:17.992 INFO:teuthology.orchestra.run.smithi179.stdout:Total 9.3 MB/s | 27 MB 00:02 2024-08-17T17:13:18.004 INFO:teuthology.orchestra.run.smithi130.stdout:(25/34): python3-logutils-0.3.5-21.el9.noarch.r 921 kB/s | 46 kB 00:00 2024-08-17T17:13:18.029 INFO:teuthology.orchestra.run.smithi130.stdout:(26/34): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-08-17T17:13:18.054 INFO:teuthology.orchestra.run.smithi130.stdout:(27/34): python3-cherrypy-18.6.1-2.el9.noarch.r 1.3 MB/s | 358 kB 00:00 2024-08-17T17:13:18.080 INFO:teuthology.orchestra.run.smithi130.stdout:(28/34): python3-portend-3.1.0-2.el9.noarch.rpm 327 kB/s | 16 kB 00:00 2024-08-17T17:13:18.105 INFO:teuthology.orchestra.run.smithi130.stdout:(29/34): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.8 MB/s | 90 kB 00:00 2024-08-17T17:13:18.116 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:13:18.139 INFO:teuthology.orchestra.run.smithi130.stdout:(30/34): python3-tempora-5.0.0-2.el9.noarch.rpm 609 kB/s | 36 kB 00:00 2024-08-17T17:13:18.153 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:13:18.153 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:13:18.164 INFO:teuthology.orchestra.run.smithi130.stdout:(31/34): python3-pecan-1.4.2-3.el9.noarch.rpm 1.7 MB/s | 272 kB 00:00 2024-08-17T17:13:18.189 INFO:teuthology.orchestra.run.smithi130.stdout:(32/34): python3-webob-1.8.7-6.el9.noarch.rpm 2.7 MB/s | 230 kB 00:00 2024-08-17T17:13:18.215 INFO:teuthology.orchestra.run.smithi130.stdout:(33/34): python3-zc-lockfile-2.0-10.el9.noarch. 397 kB/s | 20 kB 00:00 2024-08-17T17:13:18.240 INFO:teuthology.orchestra.run.smithi130.stdout:(34/34): python3-werkzeug-2.0.3-3.el9.1.noarch. 4.1 MB/s | 427 kB 00:00 2024-08-17T17:13:18.244 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:18.244 INFO:teuthology.orchestra.run.smithi130.stdout:Total 13 MB/s | 27 MB 00:02 2024-08-17T17:13:18.366 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:13:18.403 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:13:18.403 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:13:18.508 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:13:18.508 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:13:18.766 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:13:18.766 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:13:19.703 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:13:19.750 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/34 2024-08-17T17:13:19.782 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/34 2024-08-17T17:13:19.815 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/34 2024-08-17T17:13:19.907 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/34 2024-08-17T17:13:19.947 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:13:19.985 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/34 2024-08-17T17:13:20.027 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/34 2024-08-17T17:13:20.027 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/34 2024-08-17T17:13:20.061 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/34 2024-08-17T17:13:20.066 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/34 2024-08-17T17:13:20.093 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/34 2024-08-17T17:13:20.099 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/34 2024-08-17T17:13:20.125 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/34 2024-08-17T17:13:20.159 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/34 2024-08-17T17:13:20.190 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/34 2024-08-17T17:13:20.199 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/34 2024-08-17T17:13:20.234 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/34 2024-08-17T17:13:20.269 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/34 2024-08-17T17:13:20.277 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/34 2024-08-17T17:13:20.310 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/34 2024-08-17T17:13:20.325 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/34 2024-08-17T17:13:20.345 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/34 2024-08-17T17:13:20.358 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-packaging-20.9-5.el9.noarch 15/34 2024-08-17T17:13:20.375 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/34 2024-08-17T17:13:20.410 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/34 2024-08-17T17:13:20.430 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 16/34 2024-08-17T17:13:20.443 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/34 2024-08-17T17:13:20.483 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/34 2024-08-17T17:13:20.527 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/34 2024-08-17T17:13:20.569 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 17/34 2024-08-17T17:13:20.570 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/34 2024-08-17T17:13:20.618 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/34 2024-08-17T17:13:20.654 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 18/34 2024-08-17T17:13:20.658 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-packaging-20.9-5.el9.noarch 15/34 2024-08-17T17:13:20.710 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 19/34 2024-08-17T17:13:20.731 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 16/34 2024-08-17T17:13:20.749 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 20/34 2024-08-17T17:13:20.798 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-ply-3.11-14.el9.noarch 21/34 2024-08-17T17:13:20.853 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 22/34 2024-08-17T17:13:20.869 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 17/34 2024-08-17T17:13:20.950 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 18/34 2024-08-17T17:13:21.003 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 19/34 2024-08-17T17:13:21.042 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 20/34 2024-08-17T17:13:21.045 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 23/34 2024-08-17T17:13:21.095 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 24/34 2024-08-17T17:13:21.098 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-ply-3.11-14.el9.noarch 21/34 2024-08-17T17:13:21.155 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 22/34 2024-08-17T17:13:21.164 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 25/34 2024-08-17T17:13:21.253 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 26/34 2024-08-17T17:13:21.294 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 27/34 2024-08-17T17:13:21.354 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 23/34 2024-08-17T17:13:21.404 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 24/34 2024-08-17T17:13:21.409 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 28/34 2024-08-17T17:13:21.482 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 25/34 2024-08-17T17:13:21.575 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1404.gf2c39d4d.el9. 29/34 2024-08-17T17:13:21.578 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 26/34 2024-08-17T17:13:21.604 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 30/34 2024-08-17T17:13:21.620 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 27/34 2024-08-17T17:13:21.629 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 30/34 2024-08-17T17:13:21.629 INFO:teuthology.orchestra.run.smithi179.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T17:13:21.630 INFO:teuthology.orchestra.run.smithi179.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-08-17T17:13:21.630 INFO:teuthology.orchestra.run.smithi179.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T17:13:21.630 INFO:teuthology.orchestra.run.smithi179.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T17:13:21.630 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:21.735 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 28/34 2024-08-17T17:13:21.892 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1404.gf2c39d4d.el9. 29/34 2024-08-17T17:13:21.922 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 30/34 2024-08-17T17:13:21.947 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 30/34 2024-08-17T17:13:21.947 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T17:13:21.947 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-08-17T17:13:21.947 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T17:13:21.947 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T17:13:21.947 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:23.023 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 31/34 2024-08-17T17:13:23.052 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 31/34 2024-08-17T17:13:23.052 INFO:teuthology.orchestra.run.smithi179.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T17:13:23.052 INFO:teuthology.orchestra.run.smithi179.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-08-17T17:13:23.052 INFO:teuthology.orchestra.run.smithi179.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T17:13:23.052 INFO:teuthology.orchestra.run.smithi179.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T17:13:23.052 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:23.290 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 31/34 2024-08-17T17:13:23.319 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 31/34 2024-08-17T17:13:23.319 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T17:13:23.319 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-08-17T17:13:23.319 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T17:13:23.319 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T17:13:23.319 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:23.424 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 32/34 2024-08-17T17:13:23.447 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 32/34 2024-08-17T17:13:23.447 INFO:teuthology.orchestra.run.smithi179.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T17:13:23.447 INFO:teuthology.orchestra.run.smithi179.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-08-17T17:13:23.448 INFO:teuthology.orchestra.run.smithi179.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T17:13:23.448 INFO:teuthology.orchestra.run.smithi179.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T17:13:23.448 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:23.658 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 33/34 2024-08-17T17:13:23.681 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 33/34 2024-08-17T17:13:23.681 INFO:teuthology.orchestra.run.smithi179.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T17:13:23.681 INFO:teuthology.orchestra.run.smithi179.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-08-17T17:13:23.682 INFO:teuthology.orchestra.run.smithi179.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T17:13:23.682 INFO:teuthology.orchestra.run.smithi179.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T17:13:23.682 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 32/34 2024-08-17T17:13:23.731 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 32/34 2024-08-17T17:13:23.732 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T17:13:23.732 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-08-17T17:13:23.732 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T17:13:23.732 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T17:13:23.732 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:23.747 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 34/34 2024-08-17T17:13:23.950 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 33/34 2024-08-17T17:13:23.973 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 33/34 2024-08-17T17:13:23.973 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T17:13:23.973 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-08-17T17:13:23.973 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T17:13:23.973 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T17:13:23.974 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:24.039 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 34/34 2024-08-17T17:13:25.419 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 34/34 2024-08-17T17:13:25.419 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/34 2024-08-17T17:13:25.420 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/34 2024-08-17T17:13:25.420 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/34 2024-08-17T17:13:25.420 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/34 2024-08-17T17:13:25.420 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 5/34 2024-08-17T17:13:25.420 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1404.gf2c39d4d.el9. 6/34 2024-08-17T17:13:25.420 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/34 2024-08-17T17:13:25.420 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/34 2024-08-17T17:13:25.420 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/34 2024-08-17T17:13:25.420 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/34 2024-08-17T17:13:25.420 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/34 2024-08-17T17:13:25.420 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/34 2024-08-17T17:13:25.420 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/34 2024-08-17T17:13:25.420 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/34 2024-08-17T17:13:25.421 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 15/34 2024-08-17T17:13:25.421 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 16/34 2024-08-17T17:13:25.421 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 17/34 2024-08-17T17:13:25.421 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 18/34 2024-08-17T17:13:25.421 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 19/34 2024-08-17T17:13:25.421 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 20/34 2024-08-17T17:13:25.421 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 21/34 2024-08-17T17:13:25.421 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 22/34 2024-08-17T17:13:25.421 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 23/34 2024-08-17T17:13:25.421 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 24/34 2024-08-17T17:13:25.421 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 25/34 2024-08-17T17:13:25.422 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 26/34 2024-08-17T17:13:25.422 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 27/34 2024-08-17T17:13:25.422 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 28/34 2024-08-17T17:13:25.423 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 29/34 2024-08-17T17:13:25.423 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 30/34 2024-08-17T17:13:25.423 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 31/34 2024-08-17T17:13:25.423 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 32/34 2024-08-17T17:13:25.423 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 33/34 2024-08-17T17:13:25.706 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 34/34 2024-08-17T17:13:25.706 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/34 2024-08-17T17:13:25.707 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/34 2024-08-17T17:13:25.707 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 3/34 2024-08-17T17:13:25.707 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 4/34 2024-08-17T17:13:25.707 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 5/34 2024-08-17T17:13:25.707 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1404.gf2c39d4d.el9. 6/34 2024-08-17T17:13:25.707 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/34 2024-08-17T17:13:25.707 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/34 2024-08-17T17:13:25.707 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/34 2024-08-17T17:13:25.707 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/34 2024-08-17T17:13:25.707 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/34 2024-08-17T17:13:25.707 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/34 2024-08-17T17:13:25.707 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/34 2024-08-17T17:13:25.707 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/34 2024-08-17T17:13:25.708 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 15/34 2024-08-17T17:13:25.708 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 16/34 2024-08-17T17:13:25.708 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 17/34 2024-08-17T17:13:25.708 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 18/34 2024-08-17T17:13:25.708 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 19/34 2024-08-17T17:13:25.708 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 20/34 2024-08-17T17:13:25.708 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 21/34 2024-08-17T17:13:25.708 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 22/34 2024-08-17T17:13:25.708 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 23/34 2024-08-17T17:13:25.708 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 24/34 2024-08-17T17:13:25.708 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 25/34 2024-08-17T17:13:25.708 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 26/34 2024-08-17T17:13:25.708 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 27/34 2024-08-17T17:13:25.708 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 28/34 2024-08-17T17:13:25.709 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 29/34 2024-08-17T17:13:25.709 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 30/34 2024-08-17T17:13:25.709 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 31/34 2024-08-17T17:13:25.709 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 32/34 2024-08-17T17:13:25.710 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 33/34 2024-08-17T17:13:26.456 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 34/34 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mgr-modules-core-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi179.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi179.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi179.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi179.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi179.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi179.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi179.stdout: python3-mako-1.1.4-6.el9.noarch 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi179.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi179.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi179.stdout: python3-packaging-20.9-5.el9.noarch 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi179.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi179.stdout: python3-ply-3.11-14.el9.noarch 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi179.stdout: python3-portend-3.1.0-2.el9.noarch 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi179.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi179.stdout: python3-pycparser-2.20-6.el9.noarch 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi179.stdout: python3-pytz-2021.1-5.el9.noarch 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi179.stdout: python3-requests-2.25.1-8.el9.noarch 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi179.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi179.stdout: python3-toml-0.10.2-6.el9.noarch 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi179.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi179.stdout: python3-webob-1.8.7-6.el9.noarch 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi179.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi179.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:13:26.550 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 34/34 2024-08-17T17:13:26.550 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mds-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-modules-core-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mon-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-osd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi130.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi130.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi130.stdout: python3-mako-1.1.4-6.el9.noarch 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi130.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi130.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi130.stdout: python3-packaging-20.9-5.el9.noarch 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi130.stdout: python3-ply-3.11-14.el9.noarch 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi130.stdout: python3-portend-3.1.0-2.el9.noarch 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pycparser-2.20-6.el9.noarch 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pytz-2021.1-5.el9.noarch 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi130.stdout: python3-requests-2.25.1-8.el9.noarch 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi130.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi130.stdout: python3-toml-0.10.2-6.el9.noarch 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi130.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi130.stdout: python3-webob-1.8.7-6.el9.noarch 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi130.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi130.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-08-17T17:13:26.554 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:26.554 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:13:26.831 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install ceph-base 2024-08-17T17:13:26.917 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install ceph-base 2024-08-17T17:13:27.390 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:00:49 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:13:27.475 INFO:teuthology.orchestra.run.smithi179.stdout:Package ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:13:27.483 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:00:48 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:13:27.517 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:13:27.518 INFO:teuthology.orchestra.run.smithi179.stdout:Nothing to do. 2024-08-17T17:13:27.518 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:13:27.567 INFO:teuthology.orchestra.run.smithi130.stdout:Package ceph-base-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:13:27.585 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install cephadm 2024-08-17T17:13:27.609 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:13:27.610 INFO:teuthology.orchestra.run.smithi130.stdout:Nothing to do. 2024-08-17T17:13:27.610 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:13:27.685 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install cephadm 2024-08-17T17:13:28.138 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:00:50 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:13:28.241 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:00:49 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:13:28.265 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:13:28.266 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:28.266 INFO:teuthology.orchestra.run.smithi179.stdout: Package Arch Version Repository Size 2024-08-17T17:13:28.266 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:28.266 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:13:28.266 INFO:teuthology.orchestra.run.smithi179.stdout: cephadm noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 83 k 2024-08-17T17:13:28.266 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:28.266 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:13:28.266 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:28.266 INFO:teuthology.orchestra.run.smithi179.stdout:Install 1 Package 2024-08-17T17:13:28.266 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:28.267 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 83 k 2024-08-17T17:13:28.267 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 375 k 2024-08-17T17:13:28.267 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:13:28.368 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:13:28.369 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repository Size 2024-08-17T17:13:28.369 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:28.369 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:13:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: cephadm noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 83 k 2024-08-17T17:13:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:28.369 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:13:28.369 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:28.369 INFO:teuthology.orchestra.run.smithi130.stdout:Install 1 Package 2024-08-17T17:13:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:28.370 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 83 k 2024-08-17T17:13:28.370 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 375 k 2024-08-17T17:13:28.370 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:13:28.450 INFO:teuthology.orchestra.run.smithi179.stdout:cephadm-17.2.7-1404.gf2c39d4d.el9.noarch.rpm 452 kB/s | 83 kB 00:00 2024-08-17T17:13:28.450 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:28.451 INFO:teuthology.orchestra.run.smithi179.stdout:Total 448 kB/s | 83 kB 00:00 2024-08-17T17:13:28.451 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:13:28.456 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:13:28.456 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:13:28.458 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:13:28.458 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:13:28.566 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:13:28.575 INFO:teuthology.orchestra.run.smithi130.stdout:cephadm-17.2.7-1404.gf2c39d4d.el9.noarch.rpm 402 kB/s | 83 kB 00:00 2024-08-17T17:13:28.576 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:28.576 INFO:teuthology.orchestra.run.smithi130.stdout:Total 399 kB/s | 83 kB 00:00 2024-08-17T17:13:28.576 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:13:28.582 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:13:28.582 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:13:28.583 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:13:28.584 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:13:28.710 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:13:29.008 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T17:13:29.095 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T17:13:29.141 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T17:13:29.242 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T17:13:29.464 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T17:13:29.594 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T17:13:29.725 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T17:13:29.725 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:29.725 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:13:29.725 INFO:teuthology.orchestra.run.smithi179.stdout: cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:13:29.726 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:29.726 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:13:29.821 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/1 2024-08-17T17:13:29.821 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:29.821 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:13:29.821 INFO:teuthology.orchestra.run.smithi130.stdout: cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:13:29.822 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:29.822 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:13:29.955 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install ceph-immutable-object-cache 2024-08-17T17:13:30.035 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install ceph-immutable-object-cache 2024-08-17T17:13:30.524 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:00:52 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:13:30.595 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:00:51 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:13:30.654 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:13:30.654 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:30.655 INFO:teuthology.orchestra.run.smithi179.stdout: Package Arch Version Repo Size 2024-08-17T17:13:30.655 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:30.655 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:13:30.655 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 137 k 2024-08-17T17:13:30.655 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:30.655 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:13:30.655 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:30.655 INFO:teuthology.orchestra.run.smithi179.stdout:Install 1 Package 2024-08-17T17:13:30.655 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:30.655 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 137 k 2024-08-17T17:13:30.655 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 379 k 2024-08-17T17:13:30.655 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:13:30.725 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:13:30.726 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:30.726 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repo Size 2024-08-17T17:13:30.726 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:30.726 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:13:30.727 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 137 k 2024-08-17T17:13:30.727 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:30.727 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:13:30.727 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:30.727 INFO:teuthology.orchestra.run.smithi130.stdout:Install 1 Package 2024-08-17T17:13:30.727 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:30.727 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 137 k 2024-08-17T17:13:30.727 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 379 k 2024-08-17T17:13:30.727 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:13:30.881 INFO:teuthology.orchestra.run.smithi179.stdout:ceph-immutable-object-cache-17.2.7-1404.gf2c39d 608 kB/s | 137 kB 00:00 2024-08-17T17:13:30.882 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:30.883 INFO:teuthology.orchestra.run.smithi179.stdout:Total 604 kB/s | 137 kB 00:00 2024-08-17T17:13:30.883 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:13:30.890 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:13:30.890 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:13:30.947 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:13:30.947 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:13:30.952 INFO:teuthology.orchestra.run.smithi130.stdout:ceph-immutable-object-cache-17.2.7-1404.gf2c39d 611 kB/s | 137 kB 00:00 2024-08-17T17:13:30.953 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:30.953 INFO:teuthology.orchestra.run.smithi130.stdout:Total 606 kB/s | 137 kB 00:00 2024-08-17T17:13:30.953 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:13:30.961 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:13:30.961 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:13:31.017 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:13:31.018 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:13:31.094 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:13:31.173 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:13:31.183 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d. 1/1 2024-08-17T17:13:31.208 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d. 1/1 2024-08-17T17:13:31.209 INFO:teuthology.orchestra.run.smithi179.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T17:13:31.209 INFO:teuthology.orchestra.run.smithi179.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-08-17T17:13:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:31.245 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d. 1/1 2024-08-17T17:13:31.270 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d. 1/1 2024-08-17T17:13:31.270 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T17:13:31.270 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-08-17T17:13:31.270 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:32.159 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d. 1/1 2024-08-17T17:13:32.160 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:32.160 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:13:32.160 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:32.160 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:32.160 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:13:32.196 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d. 1/1 2024-08-17T17:13:32.196 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:32.196 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:13:32.196 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-immutable-object-cache-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:13:32.196 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:32.196 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:13:32.386 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install ceph-mgr 2024-08-17T17:13:32.441 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install ceph-mgr 2024-08-17T17:13:32.981 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:00:54 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:13:33.008 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:00:54 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:13:33.069 INFO:teuthology.orchestra.run.smithi179.stdout:Package ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:13:33.109 INFO:teuthology.orchestra.run.smithi130.stdout:Package ceph-mgr-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:13:33.112 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:13:33.113 INFO:teuthology.orchestra.run.smithi179.stdout:Nothing to do. 2024-08-17T17:13:33.113 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:13:33.151 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:13:33.152 INFO:teuthology.orchestra.run.smithi130.stdout:Nothing to do. 2024-08-17T17:13:33.153 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:13:33.173 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install ceph-mgr-dashboard 2024-08-17T17:13:33.215 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install ceph-mgr-dashboard 2024-08-17T17:13:33.718 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:00:55 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:13:33.767 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:00:54 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:13:33.845 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:13:33.845 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:33.845 INFO:teuthology.orchestra.run.smithi179.stdout: Package Arch Version Repository Size 2024-08-17T17:13:33.846 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:33.846 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:13:33.846 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 3.4 M 2024-08-17T17:13:33.846 INFO:teuthology.orchestra.run.smithi179.stdout:Installing dependencies: 2024-08-17T17:13:33.846 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 24 k 2024-08-17T17:13:33.846 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 15 k 2024-08-17T17:13:33.846 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-08-17T17:13:33.846 INFO:teuthology.orchestra.run.smithi179.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-08-17T17:13:33.846 INFO:teuthology.orchestra.run.smithi179.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-08-17T17:13:33.846 INFO:teuthology.orchestra.run.smithi179.stdout:Installing weak dependencies: 2024-08-17T17:13:33.846 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-08-17T17:13:33.847 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:33.847 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:13:33.847 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:33.847 INFO:teuthology.orchestra.run.smithi179.stdout:Install 7 Packages 2024-08-17T17:13:33.847 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:33.847 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 3.7 M 2024-08-17T17:13:33.847 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 80 M 2024-08-17T17:13:33.848 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:13:33.895 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:13:33.895 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:33.895 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repository Size 2024-08-17T17:13:33.896 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:33.896 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:13:33.896 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 3.4 M 2024-08-17T17:13:33.896 INFO:teuthology.orchestra.run.smithi130.stdout:Installing dependencies: 2024-08-17T17:13:33.896 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 24 k 2024-08-17T17:13:33.896 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 15 k 2024-08-17T17:13:33.896 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-08-17T17:13:33.896 INFO:teuthology.orchestra.run.smithi130.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-08-17T17:13:33.896 INFO:teuthology.orchestra.run.smithi130.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-08-17T17:13:33.896 INFO:teuthology.orchestra.run.smithi130.stdout:Installing weak dependencies: 2024-08-17T17:13:33.896 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-08-17T17:13:33.896 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:33.896 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:13:33.897 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:33.897 INFO:teuthology.orchestra.run.smithi130.stdout:Install 7 Packages 2024-08-17T17:13:33.897 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:33.897 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 3.7 M 2024-08-17T17:13:33.897 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 80 M 2024-08-17T17:13:33.897 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:13:34.094 INFO:teuthology.orchestra.run.smithi179.stdout:(1/7): ceph-grafana-dashboards-17.2.7-1404.gf2c 124 kB/s | 24 kB 00:00 2024-08-17T17:13:34.111 INFO:teuthology.orchestra.run.smithi179.stdout:(2/7): ceph-prometheus-alerts-17.2.7-1404.gf2c3 68 kB/s | 15 kB 00:00 2024-08-17T17:13:34.164 INFO:teuthology.orchestra.run.smithi130.stdout:(1/7): ceph-prometheus-alerts-17.2.7-1404.gf2c3 77 kB/s | 15 kB 00:00 2024-08-17T17:13:34.181 INFO:teuthology.orchestra.run.smithi130.stdout:(2/7): ceph-grafana-dashboards-17.2.7-1404.gf2c 118 kB/s | 24 kB 00:00 2024-08-17T17:13:34.186 INFO:teuthology.orchestra.run.smithi179.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 99 kB/s | 9.0 kB 00:00 2024-08-17T17:13:34.211 INFO:teuthology.orchestra.run.smithi179.stdout:(4/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 1.3 MB/s | 31 kB 00:00 2024-08-17T17:13:34.228 INFO:teuthology.orchestra.run.smithi179.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 349 kB/s | 41 kB 00:00 2024-08-17T17:13:34.253 INFO:teuthology.orchestra.run.smithi179.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 4.4 MB/s | 188 kB 00:00 2024-08-17T17:13:34.257 INFO:teuthology.orchestra.run.smithi130.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 98 kB/s | 9.0 kB 00:00 2024-08-17T17:13:34.273 INFO:teuthology.orchestra.run.smithi130.stdout:(4/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 2.1 MB/s | 31 kB 00:00 2024-08-17T17:13:34.290 INFO:teuthology.orchestra.run.smithi130.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 376 kB/s | 41 kB 00:00 2024-08-17T17:13:34.324 INFO:teuthology.orchestra.run.smithi130.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 3.7 MB/s | 188 kB 00:00 2024-08-17T17:13:34.370 INFO:teuthology.orchestra.run.smithi179.stdout:(7/7): ceph-mgr-dashboard-17.2.7-1404.gf2c39d4d 7.1 MB/s | 3.4 MB 00:00 2024-08-17T17:13:34.373 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:34.374 INFO:teuthology.orchestra.run.smithi179.stdout:Total 6.9 MB/s | 3.7 MB 00:00 2024-08-17T17:13:34.374 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:13:34.382 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:13:34.382 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:13:34.408 INFO:teuthology.orchestra.run.smithi130.stdout:(7/7): ceph-mgr-dashboard-17.2.7-1404.gf2c39d4d 7.7 MB/s | 3.4 MB 00:00 2024-08-17T17:13:34.414 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:34.414 INFO:teuthology.orchestra.run.smithi130.stdout:Total 7.1 MB/s | 3.7 MB 00:00 2024-08-17T17:13:34.414 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:13:34.419 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:13:34.419 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:13:34.444 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:13:34.445 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:13:34.482 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:13:34.482 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:13:34.750 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:13:34.787 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:13:34.813 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-08-17T17:13:34.851 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-08-17T17:13:34.856 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-08-17T17:13:34.885 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-08-17T17:13:34.902 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-08-17T17:13:34.934 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-08-17T17:13:34.939 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-08-17T17:13:34.963 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1404.gf2c39d4d.el9.n 5/7 2024-08-17T17:13:34.971 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-08-17T17:13:35.000 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1404.gf2c39d4d.el9.n 5/7 2024-08-17T17:13:35.952 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1404.gf2c39d4d.el9. 6/7 2024-08-17T17:13:35.990 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1404.gf2c39d4d.el9. 6/7 2024-08-17T17:13:35.991 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarc 7/7 2024-08-17T17:13:36.028 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarc 7/7 2024-08-17T17:13:37.055 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarc 7/7 2024-08-17T17:13:37.056 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1404.gf2c39d4d.el9. 1/7 2024-08-17T17:13:37.056 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarc 2/7 2024-08-17T17:13:37.056 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1404.gf2c39d4d.el9.n 3/7 2024-08-17T17:13:37.056 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-08-17T17:13:37.056 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-08-17T17:13:37.056 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-08-17T17:13:37.062 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarc 7/7 2024-08-17T17:13:37.062 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1404.gf2c39d4d.el9. 1/7 2024-08-17T17:13:37.062 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarc 2/7 2024-08-17T17:13:37.062 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1404.gf2c39d4d.el9.n 3/7 2024-08-17T17:13:37.062 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-08-17T17:13:37.062 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-08-17T17:13:37.062 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-08-17T17:13:37.319 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-08-17T17:13:37.319 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:37.319 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:13:37.319 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-grafana-dashboards-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:13:37.319 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:13:37.319 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-prometheus-alerts-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:13:37.319 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-08-17T17:13:37.319 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-08-17T17:13:37.320 INFO:teuthology.orchestra.run.smithi130.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-08-17T17:13:37.320 INFO:teuthology.orchestra.run.smithi130.stdout: python3-routes-2.5.1-5.el9.noarch 2024-08-17T17:13:37.320 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:37.320 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:13:37.325 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-08-17T17:13:37.325 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:37.325 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:13:37.325 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-grafana-dashboards-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:13:37.325 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mgr-dashboard-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:13:37.325 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-prometheus-alerts-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:13:37.325 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-08-17T17:13:37.325 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-08-17T17:13:37.326 INFO:teuthology.orchestra.run.smithi179.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-08-17T17:13:37.326 INFO:teuthology.orchestra.run.smithi179.stdout: python3-routes-2.5.1-5.el9.noarch 2024-08-17T17:13:37.326 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:37.326 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:13:37.528 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install ceph-mgr-diskprediction-local 2024-08-17T17:13:37.538 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install ceph-mgr-diskprediction-local 2024-08-17T17:13:38.072 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:00:59 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:13:38.083 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:00 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:13:38.200 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:13:38.201 INFO:teuthology.orchestra.run.smithi130.stdout:======================================================================================= 2024-08-17T17:13:38.201 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repository Size 2024-08-17T17:13:38.201 INFO:teuthology.orchestra.run.smithi130.stdout:======================================================================================= 2024-08-17T17:13:38.201 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:13:38.201 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 7.4 M 2024-08-17T17:13:38.201 INFO:teuthology.orchestra.run.smithi130.stdout:Installing dependencies: 2024-08-17T17:13:38.201 INFO:teuthology.orchestra.run.smithi130.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-08-17T17:13:38.201 INFO:teuthology.orchestra.run.smithi130.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-08-17T17:13:38.201 INFO:teuthology.orchestra.run.smithi130.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-08-17T17:13:38.202 INFO:teuthology.orchestra.run.smithi130.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-08-17T17:13:38.202 INFO:teuthology.orchestra.run.smithi130.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-08-17T17:13:38.202 INFO:teuthology.orchestra.run.smithi130.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-08-17T17:13:38.202 INFO:teuthology.orchestra.run.smithi130.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-08-17T17:13:38.202 INFO:teuthology.orchestra.run.smithi130.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-08-17T17:13:38.202 INFO:teuthology.orchestra.run.smithi130.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-08-17T17:13:38.202 INFO:teuthology.orchestra.run.smithi130.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-08-17T17:13:38.202 INFO:teuthology.orchestra.run.smithi130.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-08-17T17:13:38.202 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:38.202 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:13:38.202 INFO:teuthology.orchestra.run.smithi130.stdout:======================================================================================= 2024-08-17T17:13:38.202 INFO:teuthology.orchestra.run.smithi130.stdout:Install 12 Packages 2024-08-17T17:13:38.203 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:38.203 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 38 M 2024-08-17T17:13:38.203 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 209 M 2024-08-17T17:13:38.203 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:13:38.211 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:13:38.212 INFO:teuthology.orchestra.run.smithi179.stdout:======================================================================================= 2024-08-17T17:13:38.212 INFO:teuthology.orchestra.run.smithi179.stdout: Package Arch Version Repository Size 2024-08-17T17:13:38.212 INFO:teuthology.orchestra.run.smithi179.stdout:======================================================================================= 2024-08-17T17:13:38.212 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:13:38.212 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 7.4 M 2024-08-17T17:13:38.213 INFO:teuthology.orchestra.run.smithi179.stdout:Installing dependencies: 2024-08-17T17:13:38.213 INFO:teuthology.orchestra.run.smithi179.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-08-17T17:13:38.213 INFO:teuthology.orchestra.run.smithi179.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-08-17T17:13:38.213 INFO:teuthology.orchestra.run.smithi179.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-08-17T17:13:38.213 INFO:teuthology.orchestra.run.smithi179.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-08-17T17:13:38.213 INFO:teuthology.orchestra.run.smithi179.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-08-17T17:13:38.213 INFO:teuthology.orchestra.run.smithi179.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-08-17T17:13:38.213 INFO:teuthology.orchestra.run.smithi179.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-08-17T17:13:38.213 INFO:teuthology.orchestra.run.smithi179.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-08-17T17:13:38.213 INFO:teuthology.orchestra.run.smithi179.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-08-17T17:13:38.213 INFO:teuthology.orchestra.run.smithi179.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-08-17T17:13:38.213 INFO:teuthology.orchestra.run.smithi179.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-08-17T17:13:38.213 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:38.214 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:13:38.214 INFO:teuthology.orchestra.run.smithi179.stdout:======================================================================================= 2024-08-17T17:13:38.214 INFO:teuthology.orchestra.run.smithi179.stdout:Install 12 Packages 2024-08-17T17:13:38.214 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:38.214 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 38 M 2024-08-17T17:13:38.214 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 209 M 2024-08-17T17:13:38.214 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:13:38.582 INFO:teuthology.orchestra.run.smithi130.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 1.4 MB/s | 195 kB 00:00 2024-08-17T17:13:38.690 INFO:teuthology.orchestra.run.smithi130.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 3.2 MB/s | 803 kB 00:00 2024-08-17T17:13:38.949 INFO:teuthology.orchestra.run.smithi130.stdout:(3/12): ceph-mgr-diskprediction-local-17.2.7-14 15 MB/s | 7.4 MB 00:00 2024-08-17T17:13:39.045 INFO:teuthology.orchestra.run.smithi179.stdout:(1/12): ceph-mgr-diskprediction-local-17.2.7-14 13 MB/s | 7.4 MB 00:00 2024-08-17T17:13:39.234 INFO:teuthology.orchestra.run.smithi130.stdout:(4/12): flexiblas-3.0.4-8.el9.x86_64.rpm 50 kB/s | 33 kB 00:00 2024-08-17T17:13:39.303 INFO:teuthology.orchestra.run.smithi179.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 127 kB/s | 33 kB 00:00 2024-08-17T17:13:39.329 INFO:teuthology.orchestra.run.smithi179.stdout:(3/12): libquadmath-11.4.1-3.el9.x86_64.rpm 232 kB/s | 195 kB 00:00 2024-08-17T17:13:39.500 INFO:teuthology.orchestra.run.smithi130.stdout:(5/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 35 kB/s | 18 kB 00:00 2024-08-17T17:13:39.504 INFO:teuthology.orchestra.run.smithi179.stdout:(4/12): libgfortran-11.4.1-3.el9.x86_64.rpm 789 kB/s | 803 kB 00:01 2024-08-17T17:13:39.596 INFO:teuthology.orchestra.run.smithi179.stdout:(5/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 68 kB/s | 18 kB 00:00 2024-08-17T17:13:39.780 INFO:teuthology.orchestra.run.smithi179.stdout:(6/12): openblas-0.3.21-2.el9.x86_64.rpm 128 kB/s | 35 kB 00:00 2024-08-17T17:13:39.808 INFO:teuthology.orchestra.run.smithi130.stdout:(6/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 2.7 MB/s | 3.0 MB 00:01 2024-08-17T17:13:39.833 INFO:teuthology.orchestra.run.smithi130.stdout:(7/12): openblas-0.3.21-2.el9.x86_64.rpm 59 kB/s | 35 kB 00:00 2024-08-17T17:13:40.005 INFO:teuthology.orchestra.run.smithi179.stdout:(7/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 4.3 MB/s | 3.0 MB 00:00 2024-08-17T17:13:40.038 INFO:teuthology.orchestra.run.smithi179.stdout:(8/12): python3-devel-3.9.18-3.el9.x86_64.rpm 944 kB/s | 244 kB 00:00 2024-08-17T17:13:40.147 INFO:teuthology.orchestra.run.smithi179.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.3 MB/s | 250 kB 00:00 2024-08-17T17:13:40.264 INFO:teuthology.orchestra.run.smithi179.stdout:(10/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 20 MB/s | 5.1 MB 00:00 2024-08-17T17:13:40.422 INFO:teuthology.orchestra.run.smithi179.stdout:(11/12): openblas-openmp-0.3.21-2.el9.x86_64.rp 5.7 MB/s | 4.7 MB 00:00 2024-08-17T17:13:40.617 INFO:teuthology.orchestra.run.smithi130.stdout:(8/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 4.2 MB/s | 4.7 MB 00:01 2024-08-17T17:13:40.642 INFO:teuthology.orchestra.run.smithi130.stdout:(9/12): python3-devel-3.9.18-3.el9.x86_64.rpm 293 kB/s | 244 kB 00:00 2024-08-17T17:13:41.042 INFO:teuthology.orchestra.run.smithi130.stdout:(10/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 4.2 MB/s | 5.1 MB 00:01 2024-08-17T17:13:41.434 INFO:teuthology.orchestra.run.smithi130.stdout:(11/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 307 kB/s | 250 kB 00:00 2024-08-17T17:13:42.101 INFO:teuthology.orchestra.run.smithi130.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 11 MB/s | 16 MB 00:01 2024-08-17T17:13:42.102 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:42.103 INFO:teuthology.orchestra.run.smithi130.stdout:Total 9.8 MB/s | 38 MB 00:03 2024-08-17T17:13:42.453 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:13:42.480 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:13:42.480 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:13:42.664 INFO:teuthology.orchestra.run.smithi179.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 6.5 MB/s | 16 MB 00:02 2024-08-17T17:13:42.665 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:42.665 INFO:teuthology.orchestra.run.smithi179.stdout:Total 8.5 MB/s | 38 MB 00:04 2024-08-17T17:13:42.998 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:13:42.999 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:13:43.016 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:13:43.043 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:13:43.043 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:13:43.572 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:13:43.572 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:13:43.748 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:13:43.820 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-08-17T17:13:43.867 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-08-17T17:13:43.964 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-08-17T17:13:44.008 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-08-17T17:13:44.185 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-08-17T17:13:44.215 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-08-17T17:13:44.309 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-08-17T17:13:44.317 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:13:44.389 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-08-17T17:13:44.434 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-08-17T17:13:44.527 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-08-17T17:13:44.572 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-08-17T17:13:44.747 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-08-17T17:13:44.779 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-08-17T17:13:44.870 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-08-17T17:13:44.949 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-08-17T17:13:45.077 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-08-17T17:13:45.510 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-08-17T17:13:45.624 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-08-17T17:13:46.308 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T17:13:46.868 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T17:13:47.326 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-08-17T17:13:47.347 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39 12/12 2024-08-17T17:13:47.888 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-08-17T17:13:47.911 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39 12/12 2024-08-17T17:13:49.816 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39 12/12 2024-08-17T17:13:49.817 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39 1/12 2024-08-17T17:13:49.817 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-08-17T17:13:49.817 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-08-17T17:13:49.817 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-08-17T17:13:49.817 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-08-17T17:13:49.817 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-08-17T17:13:49.817 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-08-17T17:13:49.817 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-08-17T17:13:49.817 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-08-17T17:13:49.817 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T17:13:49.817 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-08-17T17:13:50.187 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-08-17T17:13:50.187 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:50.187 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:13:50.187 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:13:50.187 INFO:teuthology.orchestra.run.smithi130.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-08-17T17:13:50.188 INFO:teuthology.orchestra.run.smithi130.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-08-17T17:13:50.188 INFO:teuthology.orchestra.run.smithi130.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-08-17T17:13:50.188 INFO:teuthology.orchestra.run.smithi130.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-08-17T17:13:50.188 INFO:teuthology.orchestra.run.smithi130.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-08-17T17:13:50.188 INFO:teuthology.orchestra.run.smithi130.stdout: openblas-0.3.21-2.el9.x86_64 2024-08-17T17:13:50.188 INFO:teuthology.orchestra.run.smithi130.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-08-17T17:13:50.188 INFO:teuthology.orchestra.run.smithi130.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-08-17T17:13:50.188 INFO:teuthology.orchestra.run.smithi130.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-08-17T17:13:50.188 INFO:teuthology.orchestra.run.smithi130.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-08-17T17:13:50.188 INFO:teuthology.orchestra.run.smithi130.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-08-17T17:13:50.188 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:50.188 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:13:50.411 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39 12/12 2024-08-17T17:13:50.411 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39 1/12 2024-08-17T17:13:50.412 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-08-17T17:13:50.412 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-08-17T17:13:50.412 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-08-17T17:13:50.412 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-08-17T17:13:50.412 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-08-17T17:13:50.412 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-08-17T17:13:50.412 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-08-17T17:13:50.412 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-08-17T17:13:50.412 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T17:13:50.412 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-08-17T17:13:50.587 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install ceph-mgr-rook 2024-08-17T17:13:50.826 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-08-17T17:13:50.826 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:50.827 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:13:50.827 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:13:50.827 INFO:teuthology.orchestra.run.smithi179.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-08-17T17:13:50.827 INFO:teuthology.orchestra.run.smithi179.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-08-17T17:13:50.827 INFO:teuthology.orchestra.run.smithi179.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-08-17T17:13:50.827 INFO:teuthology.orchestra.run.smithi179.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-08-17T17:13:50.827 INFO:teuthology.orchestra.run.smithi179.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-08-17T17:13:50.827 INFO:teuthology.orchestra.run.smithi179.stdout: openblas-0.3.21-2.el9.x86_64 2024-08-17T17:13:50.827 INFO:teuthology.orchestra.run.smithi179.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-08-17T17:13:50.827 INFO:teuthology.orchestra.run.smithi179.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-08-17T17:13:50.827 INFO:teuthology.orchestra.run.smithi179.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-08-17T17:13:50.828 INFO:teuthology.orchestra.run.smithi179.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-08-17T17:13:50.828 INFO:teuthology.orchestra.run.smithi179.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-08-17T17:13:50.828 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:50.828 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:13:51.147 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:12 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:13:51.206 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install ceph-mgr-rook 2024-08-17T17:13:51.274 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:13:51.275 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:51.276 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repository Size 2024-08-17T17:13:51.276 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:51.276 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:13:51.276 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-rook noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 47 k 2024-08-17T17:13:51.276 INFO:teuthology.orchestra.run.smithi130.stdout:Installing dependencies: 2024-08-17T17:13:51.276 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-08-17T17:13:51.276 INFO:teuthology.orchestra.run.smithi130.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-08-17T17:13:51.276 INFO:teuthology.orchestra.run.smithi130.stdout: python3-google-auth noarch 1:2.33.0-1.el9 epel 220 k 2024-08-17T17:13:51.276 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-08-17T17:13:51.276 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-08-17T17:13:51.276 INFO:teuthology.orchestra.run.smithi130.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-08-17T17:13:51.276 INFO:teuthology.orchestra.run.smithi130.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-08-17T17:13:51.277 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-08-17T17:13:51.277 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-08-17T17:13:51.277 INFO:teuthology.orchestra.run.smithi130.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-08-17T17:13:51.277 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-08-17T17:13:51.277 INFO:teuthology.orchestra.run.smithi130.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-08-17T17:13:51.277 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:51.277 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:13:51.277 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:51.277 INFO:teuthology.orchestra.run.smithi130.stdout:Install 13 Packages 2024-08-17T17:13:51.277 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:51.278 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 2.2 M 2024-08-17T17:13:51.278 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 26 M 2024-08-17T17:13:51.278 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:13:51.770 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:13 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:13:51.903 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:13:51.904 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:51.904 INFO:teuthology.orchestra.run.smithi179.stdout: Package Arch Version Repository Size 2024-08-17T17:13:51.904 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:51.904 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:13:51.904 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mgr-rook noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 47 k 2024-08-17T17:13:51.905 INFO:teuthology.orchestra.run.smithi179.stdout:Installing dependencies: 2024-08-17T17:13:51.905 INFO:teuthology.orchestra.run.smithi179.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-08-17T17:13:51.905 INFO:teuthology.orchestra.run.smithi179.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-08-17T17:13:51.905 INFO:teuthology.orchestra.run.smithi179.stdout: python3-google-auth noarch 1:2.33.0-1.el9 epel 220 k 2024-08-17T17:13:51.905 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-08-17T17:13:51.905 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-08-17T17:13:51.905 INFO:teuthology.orchestra.run.smithi179.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-08-17T17:13:51.905 INFO:teuthology.orchestra.run.smithi179.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-08-17T17:13:51.905 INFO:teuthology.orchestra.run.smithi179.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-08-17T17:13:51.905 INFO:teuthology.orchestra.run.smithi179.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-08-17T17:13:51.905 INFO:teuthology.orchestra.run.smithi179.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-08-17T17:13:51.905 INFO:teuthology.orchestra.run.smithi179.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-08-17T17:13:51.905 INFO:teuthology.orchestra.run.smithi179.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-08-17T17:13:51.906 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:51.906 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:13:51.906 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:51.906 INFO:teuthology.orchestra.run.smithi179.stdout:Install 13 Packages 2024-08-17T17:13:51.906 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:51.906 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 2.2 M 2024-08-17T17:13:51.907 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 26 M 2024-08-17T17:13:51.907 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:13:51.954 INFO:teuthology.orchestra.run.smithi130.stdout:(1/13): ceph-mgr-rook-17.2.7-1404.gf2c39d4d.el9 258 kB/s | 47 kB 00:00 2024-08-17T17:13:52.037 INFO:teuthology.orchestra.run.smithi130.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 96 kB/s | 26 kB 00:00 2024-08-17T17:13:52.063 INFO:teuthology.orchestra.run.smithi130.stdout:(3/13): python3-jsonpointer-2.0-4.el9.noarch.rp 65 kB/s | 19 kB 00:00 2024-08-17T17:13:52.405 INFO:teuthology.orchestra.run.smithi130.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 464 kB/s | 159 kB 00:00 2024-08-17T17:13:52.462 INFO:teuthology.orchestra.run.smithi179.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 312 kB/s | 26 kB 00:00 2024-08-17T17:13:52.487 INFO:teuthology.orchestra.run.smithi179.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 179 kB/s | 19 kB 00:00 2024-08-17T17:13:52.496 INFO:teuthology.orchestra.run.smithi130.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 643 kB/s | 279 kB 00:00 2024-08-17T17:13:52.522 INFO:teuthology.orchestra.run.smithi130.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 459 kB/s | 54 kB 00:00 2024-08-17T17:13:52.562 INFO:teuthology.orchestra.run.smithi179.stdout:(3/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 2.1 MB/s | 159 kB 00:00 2024-08-17T17:13:52.588 INFO:teuthology.orchestra.run.smithi179.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.2 MB/s | 222 kB 00:00 2024-08-17T17:13:52.597 INFO:teuthology.orchestra.run.smithi130.stdout:(7/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 346 kB/s | 222 kB 00:00 2024-08-17T17:13:52.613 INFO:teuthology.orchestra.run.smithi179.stdout:(5/13): ceph-mgr-rook-17.2.7-1404.gf2c39d4d.el9 204 kB/s | 47 kB 00:00 2024-08-17T17:13:52.622 INFO:teuthology.orchestra.run.smithi130.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 257 kB/s | 32 kB 00:00 2024-08-17T17:13:52.638 INFO:teuthology.orchestra.run.smithi179.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 1.0 MB/s | 54 kB 00:00 2024-08-17T17:13:52.663 INFO:teuthology.orchestra.run.smithi179.stdout:(7/13): python3-pyasn1-modules-0.4.8-6.el9.noar 2.7 MB/s | 279 kB 00:00 2024-08-17T17:13:52.672 INFO:teuthology.orchestra.run.smithi130.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 94 kB/s | 14 kB 00:00 2024-08-17T17:13:52.706 INFO:teuthology.orchestra.run.smithi130.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 1.9 MB/s | 59 kB 00:00 2024-08-17T17:13:52.739 INFO:teuthology.orchestra.run.smithi130.stdout:(11/13): python3-websocket-client-1.2.3-2.el9.n 2.7 MB/s | 90 kB 00:00 2024-08-17T17:13:52.764 INFO:teuthology.orchestra.run.smithi130.stdout:(12/13): python3-google-auth-2.33.0-1.el9.noarc 1.3 MB/s | 220 kB 00:00 2024-08-17T17:13:52.772 INFO:teuthology.orchestra.run.smithi179.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 202 kB/s | 32 kB 00:00 2024-08-17T17:13:52.806 INFO:teuthology.orchestra.run.smithi179.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 84 kB/s | 14 kB 00:00 2024-08-17T17:13:52.806 INFO:teuthology.orchestra.run.smithi130.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 6.4 MB/s | 1.0 MB 00:00 2024-08-17T17:13:52.809 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:52.809 INFO:teuthology.orchestra.run.smithi130.stdout:Total 1.4 MB/s | 2.2 MB 00:01 2024-08-17T17:13:52.839 INFO:teuthology.orchestra.run.smithi179.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 1.7 MB/s | 59 kB 00:00 2024-08-17T17:13:52.865 INFO:teuthology.orchestra.run.smithi179.stdout:(11/13): python3-google-auth-2.33.0-1.el9.noarc 1.4 MB/s | 220 kB 00:00 2024-08-17T17:13:52.872 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:13:52.886 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:13:52.886 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:13:52.890 INFO:teuthology.orchestra.run.smithi179.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.8 MB/s | 90 kB 00:00 2024-08-17T17:13:52.932 INFO:teuthology.orchestra.run.smithi179.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 6.5 MB/s | 1.0 MB 00:00 2024-08-17T17:13:52.934 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:52.935 INFO:teuthology.orchestra.run.smithi179.stdout:Total 2.2 MB/s | 2.2 MB 00:01 2024-08-17T17:13:52.971 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:13:52.972 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:13:52.997 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:13:53.011 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:13:53.012 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:13:53.098 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:13:53.100 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:13:53.488 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:13:53.616 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:13:53.660 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-08-17T17:13:53.715 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-08-17T17:13:53.770 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-08-17T17:13:53.778 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-08-17T17:13:53.808 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-08-17T17:13:53.841 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-08-17T17:13:53.849 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-08-17T17:13:53.896 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-08-17T17:13:53.933 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-08-17T17:13:53.951 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-08-17T17:13:53.974 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-08-17T17:13:54.052 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-google-auth-1:2.33.0-1.el9.noarch 7/13 2024-08-17T17:13:54.084 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-08-17T17:13:54.107 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-08-17T17:13:54.190 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-google-auth-1:2.33.0-1.el9.noarch 7/13 2024-08-17T17:13:54.248 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-08-17T17:13:54.826 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-08-17T17:13:54.876 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-08-17T17:13:54.909 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-08-17T17:13:54.962 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-08-17T17:13:54.963 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-08-17T17:13:54.987 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 13/13 2024-08-17T17:13:55.012 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-08-17T17:13:55.046 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-08-17T17:13:55.092 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-08-17T17:13:55.116 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 13/13 2024-08-17T17:13:55.789 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 13/13 2024-08-17T17:13:55.789 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/13 2024-08-17T17:13:55.789 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-08-17T17:13:55.789 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-08-17T17:13:55.789 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-08-17T17:13:55.789 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-08-17T17:13:55.789 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-08-17T17:13:55.789 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-08-17T17:13:55.790 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-08-17T17:13:55.790 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-08-17T17:13:55.790 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-google-auth-1:2.33.0-1.el9.noarch 10/13 2024-08-17T17:13:55.790 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-08-17T17:13:55.790 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-08-17T17:13:55.906 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 13/13 2024-08-17T17:13:55.907 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/13 2024-08-17T17:13:55.907 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-08-17T17:13:55.907 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-08-17T17:13:55.907 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-08-17T17:13:55.907 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-08-17T17:13:55.907 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-08-17T17:13:55.907 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-08-17T17:13:55.907 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-08-17T17:13:55.907 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-08-17T17:13:55.907 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-google-auth-1:2.33.0-1.el9.noarch 10/13 2024-08-17T17:13:55.907 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-08-17T17:13:55.908 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-08-17T17:13:56.100 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-08-17T17:13:56.100 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:56.100 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:13:56.100 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:13:56.100 INFO:teuthology.orchestra.run.smithi130.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-08-17T17:13:56.100 INFO:teuthology.orchestra.run.smithi130.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-08-17T17:13:56.100 INFO:teuthology.orchestra.run.smithi130.stdout: python3-google-auth-1:2.33.0-1.el9.noarch 2024-08-17T17:13:56.100 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-08-17T17:13:56.101 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-08-17T17:13:56.101 INFO:teuthology.orchestra.run.smithi130.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-08-17T17:13:56.101 INFO:teuthology.orchestra.run.smithi130.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-08-17T17:13:56.101 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-08-17T17:13:56.101 INFO:teuthology.orchestra.run.smithi130.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-08-17T17:13:56.101 INFO:teuthology.orchestra.run.smithi130.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-08-17T17:13:56.101 INFO:teuthology.orchestra.run.smithi130.stdout: python3-rsa-4.9-2.el9.noarch 2024-08-17T17:13:56.101 INFO:teuthology.orchestra.run.smithi130.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-08-17T17:13:56.101 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:56.101 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:13:56.230 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-08-17T17:13:56.230 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:56.230 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:13:56.230 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mgr-rook-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:13:56.230 INFO:teuthology.orchestra.run.smithi179.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-08-17T17:13:56.231 INFO:teuthology.orchestra.run.smithi179.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-08-17T17:13:56.231 INFO:teuthology.orchestra.run.smithi179.stdout: python3-google-auth-1:2.33.0-1.el9.noarch 2024-08-17T17:13:56.231 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-08-17T17:13:56.231 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-08-17T17:13:56.231 INFO:teuthology.orchestra.run.smithi179.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-08-17T17:13:56.231 INFO:teuthology.orchestra.run.smithi179.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-08-17T17:13:56.231 INFO:teuthology.orchestra.run.smithi179.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-08-17T17:13:56.231 INFO:teuthology.orchestra.run.smithi179.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-08-17T17:13:56.231 INFO:teuthology.orchestra.run.smithi179.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-08-17T17:13:56.231 INFO:teuthology.orchestra.run.smithi179.stdout: python3-rsa-4.9-2.el9.noarch 2024-08-17T17:13:56.231 INFO:teuthology.orchestra.run.smithi179.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-08-17T17:13:56.231 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:56.232 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:13:56.350 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install ceph-mgr-cephadm 2024-08-17T17:13:56.495 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install ceph-mgr-cephadm 2024-08-17T17:13:56.902 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:17 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:13:57.031 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:13:57.032 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:57.032 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repository Size 2024-08-17T17:13:57.032 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:57.032 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:13:57.032 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 124 k 2024-08-17T17:13:57.032 INFO:teuthology.orchestra.run.smithi130.stdout:Installing dependencies: 2024-08-17T17:13:57.032 INFO:teuthology.orchestra.run.smithi130.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-08-17T17:13:57.032 INFO:teuthology.orchestra.run.smithi130.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-08-17T17:13:57.032 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-08-17T17:13:57.033 INFO:teuthology.orchestra.run.smithi130.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-08-17T17:13:57.033 INFO:teuthology.orchestra.run.smithi130.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-08-17T17:13:57.033 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:57.033 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:13:57.033 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:13:57.033 INFO:teuthology.orchestra.run.smithi130.stdout:Install 6 Packages 2024-08-17T17:13:57.033 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:13:57.034 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 7.0 M 2024-08-17T17:13:57.034 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 33 M 2024-08-17T17:13:57.034 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:13:57.045 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:19 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:13:57.175 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:13:57.175 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:57.175 INFO:teuthology.orchestra.run.smithi179.stdout: Package Arch Version Repository Size 2024-08-17T17:13:57.175 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:57.175 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:13:57.175 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1404.gf2c39d4d.el9 ceph-noarch 124 k 2024-08-17T17:13:57.176 INFO:teuthology.orchestra.run.smithi179.stdout:Installing dependencies: 2024-08-17T17:13:57.176 INFO:teuthology.orchestra.run.smithi179.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-08-17T17:13:57.176 INFO:teuthology.orchestra.run.smithi179.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-08-17T17:13:57.176 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-08-17T17:13:57.176 INFO:teuthology.orchestra.run.smithi179.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-08-17T17:13:57.176 INFO:teuthology.orchestra.run.smithi179.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-08-17T17:13:57.176 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:57.176 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:13:57.176 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:13:57.176 INFO:teuthology.orchestra.run.smithi179.stdout:Install 6 Packages 2024-08-17T17:13:57.176 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:13:57.177 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 7.0 M 2024-08-17T17:13:57.177 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 33 M 2024-08-17T17:13:57.177 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:13:57.445 INFO:teuthology.orchestra.run.smithi130.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.2 MB/s | 248 kB 00:00 2024-08-17T17:13:57.470 INFO:teuthology.orchestra.run.smithi130.stdout:(2/6): ceph-mgr-cephadm-17.2.7-1404.gf2c39d4d.e 531 kB/s | 124 kB 00:00 2024-08-17T17:13:57.523 INFO:teuthology.orchestra.run.smithi179.stdout:(1/6): ceph-mgr-cephadm-17.2.7-1404.gf2c39d4d.e 605 kB/s | 124 kB 00:00 2024-08-17T17:13:57.549 INFO:teuthology.orchestra.run.smithi179.stdout:(2/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.1 MB/s | 248 kB 00:00 2024-08-17T17:13:57.563 INFO:teuthology.orchestra.run.smithi130.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 625 kB/s | 58 kB 00:00 2024-08-17T17:13:57.588 INFO:teuthology.orchestra.run.smithi130.stdout:(4/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 3.8 MB/s | 548 kB 00:00 2024-08-17T17:13:57.613 INFO:teuthology.orchestra.run.smithi130.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 1.5 MB/s | 74 kB 00:00 2024-08-17T17:13:57.640 INFO:teuthology.orchestra.run.smithi179.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 630 kB/s | 58 kB 00:00 2024-08-17T17:13:57.670 INFO:teuthology.orchestra.run.smithi179.stdout:(4/6): python3-typing-extensions-4.12.2-2.el9.n 2.5 MB/s | 74 kB 00:00 2024-08-17T17:13:57.704 INFO:teuthology.orchestra.run.smithi179.stdout:(5/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 3.0 MB/s | 548 kB 00:00 2024-08-17T17:13:57.812 INFO:teuthology.orchestra.run.smithi179.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 12 MB/s | 6.0 MB 00:00 2024-08-17T17:13:57.814 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:57.815 INFO:teuthology.orchestra.run.smithi179.stdout:Total 11 MB/s | 7.0 MB 00:00 2024-08-17T17:13:57.885 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:13:57.896 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:13:57.896 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:13:57.993 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:13:57.994 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:13:58.305 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:13:58.494 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-08-17T17:13:58.547 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-08-17T17:13:58.755 INFO:teuthology.orchestra.run.smithi130.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 3.9 MB/s | 6.0 MB 00:01 2024-08-17T17:13:58.757 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:13:58.758 INFO:teuthology.orchestra.run.smithi130.stdout:Total 4.1 MB/s | 7.0 MB 00:01 2024-08-17T17:13:58.828 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:13:58.839 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:13:58.839 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:13:58.938 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:13:58.939 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:13:59.069 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-08-17T17:13:59.158 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-08-17T17:13:59.214 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-08-17T17:13:59.243 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 6/6 2024-08-17T17:13:59.244 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:13:59.437 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-08-17T17:13:59.490 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-08-17T17:14:00.017 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-08-17T17:14:00.092 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 6/6 2024-08-17T17:14:00.092 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/6 2024-08-17T17:14:00.092 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-08-17T17:14:00.092 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-08-17T17:14:00.093 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-08-17T17:14:00.093 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-08-17T17:14:00.108 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-08-17T17:14:00.164 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-08-17T17:14:00.193 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 6/6 2024-08-17T17:14:00.355 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-08-17T17:14:00.355 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:00.355 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:14:00.355 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:14:00.355 INFO:teuthology.orchestra.run.smithi179.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-08-17T17:14:00.355 INFO:teuthology.orchestra.run.smithi179.stdout: python3-babel-2.9.1-2.el9.noarch 2024-08-17T17:14:00.355 INFO:teuthology.orchestra.run.smithi179.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-08-17T17:14:00.355 INFO:teuthology.orchestra.run.smithi179.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-08-17T17:14:00.356 INFO:teuthology.orchestra.run.smithi179.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-08-17T17:14:00.356 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:00.356 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:14:00.595 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install ceph-fuse 2024-08-17T17:14:00.985 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 6/6 2024-08-17T17:14:00.985 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 1/6 2024-08-17T17:14:00.985 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-08-17T17:14:00.985 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-08-17T17:14:00.985 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-08-17T17:14:00.985 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-08-17T17:14:01.142 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:23 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:14:01.249 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-08-17T17:14:01.249 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:01.249 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:14:01.249 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-mgr-cephadm-2:17.2.7-1404.gf2c39d4d.el9.noarch 2024-08-17T17:14:01.249 INFO:teuthology.orchestra.run.smithi130.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-08-17T17:14:01.249 INFO:teuthology.orchestra.run.smithi130.stdout: python3-babel-2.9.1-2.el9.noarch 2024-08-17T17:14:01.250 INFO:teuthology.orchestra.run.smithi130.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-08-17T17:14:01.250 INFO:teuthology.orchestra.run.smithi130.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-08-17T17:14:01.250 INFO:teuthology.orchestra.run.smithi130.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-08-17T17:14:01.250 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:01.250 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:14:01.269 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:14:01.270 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:01.270 INFO:teuthology.orchestra.run.smithi179.stdout: Package Arch Version Repository Size 2024-08-17T17:14:01.270 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:01.270 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:14:01.270 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-fuse x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 813 k 2024-08-17T17:14:01.270 INFO:teuthology.orchestra.run.smithi179.stdout:Installing dependencies: 2024-08-17T17:14:01.270 INFO:teuthology.orchestra.run.smithi179.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-08-17T17:14:01.270 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:01.270 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:14:01.270 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:01.271 INFO:teuthology.orchestra.run.smithi179.stdout:Install 2 Packages 2024-08-17T17:14:01.271 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:01.271 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 893 k 2024-08-17T17:14:01.271 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 2.6 M 2024-08-17T17:14:01.271 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:14:01.466 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install ceph-fuse 2024-08-17T17:14:01.482 INFO:teuthology.orchestra.run.smithi179.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 538 kB/s | 80 kB 00:00 2024-08-17T17:14:01.640 INFO:teuthology.orchestra.run.smithi179.stdout:(2/2): ceph-fuse-17.2.7-1404.gf2c39d4d.el9.x86_ 2.6 MB/s | 813 kB 00:00 2024-08-17T17:14:01.641 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:14:01.641 INFO:teuthology.orchestra.run.smithi179.stdout:Total 2.4 MB/s | 893 kB 00:00 2024-08-17T17:14:01.652 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:14:01.673 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:14:01.673 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:14:01.750 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:14:01.751 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:14:01.930 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:14:02.015 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:23 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:14:02.077 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-08-17T17:14:02.139 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/2 2024-08-17T17:14:02.144 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:14:02.144 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:02.144 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repository Size 2024-08-17T17:14:02.144 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:02.145 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:14:02.145 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-fuse x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 813 k 2024-08-17T17:14:02.145 INFO:teuthology.orchestra.run.smithi130.stdout:Installing dependencies: 2024-08-17T17:14:02.145 INFO:teuthology.orchestra.run.smithi130.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-08-17T17:14:02.145 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:02.145 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:14:02.145 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:02.145 INFO:teuthology.orchestra.run.smithi130.stdout:Install 2 Packages 2024-08-17T17:14:02.145 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:02.146 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 893 k 2024-08-17T17:14:02.146 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 2.6 M 2024-08-17T17:14:02.146 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:14:02.322 INFO:teuthology.orchestra.run.smithi130.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 573 kB/s | 80 kB 00:00 2024-08-17T17:14:02.489 INFO:teuthology.orchestra.run.smithi130.stdout:(2/2): ceph-fuse-17.2.7-1404.gf2c39d4d.el9.x86_ 2.6 MB/s | 813 kB 00:00 2024-08-17T17:14:02.490 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:14:02.490 INFO:teuthology.orchestra.run.smithi130.stdout:Total 2.5 MB/s | 893 kB 00:00 2024-08-17T17:14:02.501 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:14:02.523 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:14:02.523 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:14:02.600 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:14:02.600 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:14:02.786 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:14:02.810 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/2 2024-08-17T17:14:02.810 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/2 2024-08-17T17:14:02.910 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-08-17T17:14:02.972 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/2 2024-08-17T17:14:03.083 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-08-17T17:14:03.083 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:03.083 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:14:03.083 INFO:teuthology.orchestra.run.smithi179.stdout: ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-08-17T17:14:03.083 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:03.084 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:14:03.354 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install librados-devel 2024-08-17T17:14:03.662 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2/2 2024-08-17T17:14:03.662 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/2 2024-08-17T17:14:03.925 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:25 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:14:03.941 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-08-17T17:14:03.941 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:03.941 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:14:03.941 INFO:teuthology.orchestra.run.smithi130.stdout: ceph-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-08-17T17:14:03.941 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:03.941 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:14:04.060 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:14:04.061 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:04.061 INFO:teuthology.orchestra.run.smithi179.stdout: Package Arch Version Repo Size 2024-08-17T17:14:04.061 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:04.061 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:14:04.061 INFO:teuthology.orchestra.run.smithi179.stdout: librados-devel x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 126 k 2024-08-17T17:14:04.061 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:04.062 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:14:04.062 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:04.062 INFO:teuthology.orchestra.run.smithi179.stdout:Install 1 Package 2024-08-17T17:14:04.062 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:04.062 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 126 k 2024-08-17T17:14:04.062 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 448 k 2024-08-17T17:14:04.062 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:14:04.193 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install librados-devel 2024-08-17T17:14:04.266 INFO:teuthology.orchestra.run.smithi179.stdout:librados-devel-17.2.7-1404.gf2c39d4d.el9.x86_64 620 kB/s | 126 kB 00:00 2024-08-17T17:14:04.267 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:14:04.267 INFO:teuthology.orchestra.run.smithi179.stdout:Total 614 kB/s | 126 kB 00:00 2024-08-17T17:14:04.267 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:14:04.272 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:14:04.273 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:14:04.335 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:14:04.335 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:14:04.490 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:14:04.589 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:25 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:14:04.888 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:14:04.888 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:04.889 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repo Size 2024-08-17T17:14:04.889 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:04.889 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:14:04.889 INFO:teuthology.orchestra.run.smithi130.stdout: librados-devel x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 126 k 2024-08-17T17:14:04.889 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:04.889 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:14:04.889 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:04.889 INFO:teuthology.orchestra.run.smithi130.stdout:Install 1 Package 2024-08-17T17:14:04.889 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:04.889 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 126 k 2024-08-17T17:14:04.889 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 448 k 2024-08-17T17:14:04.889 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:14:05.016 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:05.111 INFO:teuthology.orchestra.run.smithi130.stdout:librados-devel-17.2.7-1404.gf2c39d4d.el9.x86_64 570 kB/s | 126 kB 00:00 2024-08-17T17:14:05.112 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:14:05.112 INFO:teuthology.orchestra.run.smithi130.stdout:Total 565 kB/s | 126 kB 00:00 2024-08-17T17:14:05.112 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:14:05.118 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:14:05.118 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:14:05.181 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:14:05.181 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:14:05.272 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:05.272 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:05.272 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:14:05.273 INFO:teuthology.orchestra.run.smithi179.stdout: librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:14:05.273 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:05.273 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:14:05.335 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:14:05.435 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:05.530 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install libcephfs2 2024-08-17T17:14:05.855 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:06.090 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:28 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:14:06.095 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:06.095 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:06.095 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:14:06.095 INFO:teuthology.orchestra.run.smithi130.stdout: librados-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:14:06.096 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:06.096 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:14:06.177 INFO:teuthology.orchestra.run.smithi179.stdout:Package libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:14:06.219 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:14:06.220 INFO:teuthology.orchestra.run.smithi179.stdout:Nothing to do. 2024-08-17T17:14:06.221 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:14:06.285 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install libcephfs-devel 2024-08-17T17:14:06.391 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install libcephfs2 2024-08-17T17:14:06.843 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:28 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:14:06.955 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:27 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:14:06.969 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:14:06.970 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:06.970 INFO:teuthology.orchestra.run.smithi179.stdout: Package Arch Version Repo Size 2024-08-17T17:14:06.970 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:06.970 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:14:06.970 INFO:teuthology.orchestra.run.smithi179.stdout: libcephfs-devel x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 25 k 2024-08-17T17:14:06.970 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:06.970 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:14:06.971 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:06.971 INFO:teuthology.orchestra.run.smithi179.stdout:Install 1 Package 2024-08-17T17:14:06.971 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:06.971 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 25 k 2024-08-17T17:14:06.971 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 110 k 2024-08-17T17:14:06.971 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:14:07.044 INFO:teuthology.orchestra.run.smithi130.stdout:Package libcephfs2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:14:07.089 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:14:07.090 INFO:teuthology.orchestra.run.smithi130.stdout:Nothing to do. 2024-08-17T17:14:07.090 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:14:07.140 INFO:teuthology.orchestra.run.smithi179.stdout:libcephfs-devel-17.2.7-1404.gf2c39d4d.el9.x86_6 148 kB/s | 25 kB 00:00 2024-08-17T17:14:07.140 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:14:07.140 INFO:teuthology.orchestra.run.smithi179.stdout:Total 147 kB/s | 25 kB 00:00 2024-08-17T17:14:07.140 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:14:07.144 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:14:07.144 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:14:07.145 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:14:07.146 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:14:07.155 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install libcephfs-devel 2024-08-17T17:14:07.227 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:14:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:07.708 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:28 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:14:07.710 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:07.837 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:14:07.837 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:07.837 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repo Size 2024-08-17T17:14:07.837 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:07.837 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:14:07.837 INFO:teuthology.orchestra.run.smithi130.stdout: libcephfs-devel x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 25 k 2024-08-17T17:14:07.837 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:07.838 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:14:07.838 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:07.838 INFO:teuthology.orchestra.run.smithi130.stdout:Install 1 Package 2024-08-17T17:14:07.838 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:07.838 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 25 k 2024-08-17T17:14:07.838 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 110 k 2024-08-17T17:14:07.838 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:14:07.928 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:07.928 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:07.928 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:14:07.929 INFO:teuthology.orchestra.run.smithi179.stdout: libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:14:07.929 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:07.929 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:14:08.003 INFO:teuthology.orchestra.run.smithi130.stdout:libcephfs-devel-17.2.7-1404.gf2c39d4d.el9.x86_6 152 kB/s | 25 kB 00:00 2024-08-17T17:14:08.003 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:14:08.004 INFO:teuthology.orchestra.run.smithi130.stdout:Total 151 kB/s | 25 kB 00:00 2024-08-17T17:14:08.004 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:14:08.007 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:14:08.007 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:14:08.009 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:14:08.009 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:14:08.089 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:14:08.105 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install librados2 2024-08-17T17:14:08.199 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:08.550 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:08.658 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:30 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:14:08.744 INFO:teuthology.orchestra.run.smithi179.stdout:Package librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:14:08.763 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:08.764 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:08.764 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:14:08.764 INFO:teuthology.orchestra.run.smithi130.stdout: libcephfs-devel-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:14:08.764 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:08.764 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:14:08.786 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:14:08.788 INFO:teuthology.orchestra.run.smithi179.stdout:Nothing to do. 2024-08-17T17:14:08.788 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:14:08.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install librbd1 2024-08-17T17:14:08.956 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install librados2 2024-08-17T17:14:09.407 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:31 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:14:09.493 INFO:teuthology.orchestra.run.smithi179.stdout:Package librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:14:09.505 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:30 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:14:09.536 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:14:09.537 INFO:teuthology.orchestra.run.smithi179.stdout:Nothing to do. 2024-08-17T17:14:09.537 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:14:09.591 INFO:teuthology.orchestra.run.smithi130.stdout:Package librados2-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:14:09.613 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install python3-rados 2024-08-17T17:14:09.634 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:14:09.635 INFO:teuthology.orchestra.run.smithi130.stdout:Nothing to do. 2024-08-17T17:14:09.635 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:14:09.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install librbd1 2024-08-17T17:14:10.161 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:32 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:14:10.247 INFO:teuthology.orchestra.run.smithi179.stdout:Package python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:14:10.250 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:31 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:14:10.290 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:14:10.291 INFO:teuthology.orchestra.run.smithi179.stdout:Nothing to do. 2024-08-17T17:14:10.291 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:14:10.336 INFO:teuthology.orchestra.run.smithi130.stdout:Package librbd1-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:14:10.351 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install python3-rgw 2024-08-17T17:14:10.378 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:14:10.379 INFO:teuthology.orchestra.run.smithi130.stdout:Nothing to do. 2024-08-17T17:14:10.380 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:14:10.450 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install python3-rados 2024-08-17T17:14:10.898 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:32 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:14:10.984 INFO:teuthology.orchestra.run.smithi179.stdout:Package python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:14:10.997 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:31 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:14:11.026 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:14:11.027 INFO:teuthology.orchestra.run.smithi179.stdout:Nothing to do. 2024-08-17T17:14:11.027 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:14:11.083 INFO:teuthology.orchestra.run.smithi130.stdout:Package python3-rados-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:14:11.096 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install python3-cephfs 2024-08-17T17:14:11.126 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:14:11.126 INFO:teuthology.orchestra.run.smithi130.stdout:Nothing to do. 2024-08-17T17:14:11.126 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:14:11.192 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install python3-rgw 2024-08-17T17:14:11.647 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:33 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:14:11.733 INFO:teuthology.orchestra.run.smithi179.stdout:Package python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:14:11.742 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:32 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:14:11.775 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:14:11.776 INFO:teuthology.orchestra.run.smithi179.stdout:Nothing to do. 2024-08-17T17:14:11.776 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:14:11.827 INFO:teuthology.orchestra.run.smithi130.stdout:Package python3-rgw-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:14:11.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install python3-rbd 2024-08-17T17:14:11.871 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:14:11.872 INFO:teuthology.orchestra.run.smithi130.stdout:Nothing to do. 2024-08-17T17:14:11.872 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:14:11.946 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install python3-cephfs 2024-08-17T17:14:12.405 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:34 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:14:12.491 INFO:teuthology.orchestra.run.smithi179.stdout:Package python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:14:12.498 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:33 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:14:12.532 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:14:12.533 INFO:teuthology.orchestra.run.smithi179.stdout:Nothing to do. 2024-08-17T17:14:12.533 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:14:12.584 INFO:teuthology.orchestra.run.smithi130.stdout:Package python3-cephfs-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:14:12.596 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install rbd-fuse 2024-08-17T17:14:12.626 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:14:12.627 INFO:teuthology.orchestra.run.smithi130.stdout:Nothing to do. 2024-08-17T17:14:12.627 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:14:12.695 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install python3-rbd 2024-08-17T17:14:13.148 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:35 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:14:13.243 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:34 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:14:13.276 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:14:13.277 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:13.277 INFO:teuthology.orchestra.run.smithi179.stdout: Package Architecture Version Repository Size 2024-08-17T17:14:13.277 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:13.277 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:14:13.277 INFO:teuthology.orchestra.run.smithi179.stdout: rbd-fuse x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 82 k 2024-08-17T17:14:13.278 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:13.278 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:14:13.278 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:13.278 INFO:teuthology.orchestra.run.smithi179.stdout:Install 1 Package 2024-08-17T17:14:13.278 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:13.278 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 82 k 2024-08-17T17:14:13.278 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 222 k 2024-08-17T17:14:13.278 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:14:13.328 INFO:teuthology.orchestra.run.smithi130.stdout:Package python3-rbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 is already installed. 2024-08-17T17:14:13.370 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:14:13.374 INFO:teuthology.orchestra.run.smithi130.stdout:Nothing to do. 2024-08-17T17:14:13.374 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:14:13.438 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install rbd-fuse 2024-08-17T17:14:13.626 INFO:teuthology.orchestra.run.smithi179.stdout:rbd-fuse-17.2.7-1404.gf2c39d4d.el9.x86_64.rpm 235 kB/s | 82 kB 00:00 2024-08-17T17:14:13.627 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:14:13.627 INFO:teuthology.orchestra.run.smithi179.stdout:Total 234 kB/s | 82 kB 00:00 2024-08-17T17:14:13.627 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:14:13.634 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:14:13.634 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:14:13.697 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:14:13.697 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:14:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:14:13.947 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:13.983 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:34 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:14:14.110 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:14:14.111 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:14.111 INFO:teuthology.orchestra.run.smithi130.stdout: Package Architecture Version Repository Size 2024-08-17T17:14:14.111 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:14.111 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:14:14.111 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-fuse x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 82 k 2024-08-17T17:14:14.112 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:14.112 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:14:14.112 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:14.112 INFO:teuthology.orchestra.run.smithi130.stdout:Install 1 Package 2024-08-17T17:14:14.112 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:14.112 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 82 k 2024-08-17T17:14:14.112 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 222 k 2024-08-17T17:14:14.112 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:14:14.316 INFO:teuthology.orchestra.run.smithi130.stdout:rbd-fuse-17.2.7-1404.gf2c39d4d.el9.x86_64.rpm 401 kB/s | 82 kB 00:00 2024-08-17T17:14:14.316 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:14:14.317 INFO:teuthology.orchestra.run.smithi130.stdout:Total 398 kB/s | 82 kB 00:00 2024-08-17T17:14:14.317 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:14:14.323 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:14:14.324 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:14:14.328 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:14.386 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:14:14.386 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:14:14.544 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:14:14.586 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:14.586 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:14.586 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:14:14.586 INFO:teuthology.orchestra.run.smithi179.stdout: rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:14:14.586 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:14.586 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:14:14.637 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:14.827 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install rbd-mirror 2024-08-17T17:14:14.970 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:15.213 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:15.213 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:15.213 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:14:15.214 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-fuse-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:14:15.214 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:15.214 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:14:15.394 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:37 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:14:15.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install rbd-mirror 2024-08-17T17:14:15.528 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:14:15.528 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:15.529 INFO:teuthology.orchestra.run.smithi179.stdout: Package Arch Version Repo Size 2024-08-17T17:14:15.529 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:15.529 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:14:15.529 INFO:teuthology.orchestra.run.smithi179.stdout: rbd-mirror x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 2.9 M 2024-08-17T17:14:15.529 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:15.529 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:14:15.529 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:15.529 INFO:teuthology.orchestra.run.smithi179.stdout:Install 1 Package 2024-08-17T17:14:15.529 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:15.529 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 2.9 M 2024-08-17T17:14:15.529 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 12 M 2024-08-17T17:14:15.530 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:14:15.898 INFO:teuthology.orchestra.run.smithi179.stdout:rbd-mirror-17.2.7-1404.gf2c39d4d.el9.x86_64.rpm 8.0 MB/s | 2.9 MB 00:00 2024-08-17T17:14:15.898 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:14:15.899 INFO:teuthology.orchestra.run.smithi179.stdout:Total 7.9 MB/s | 2.9 MB 00:00 2024-08-17T17:14:15.899 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:14:15.908 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:14:15.908 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:14:15.987 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:14:15.988 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:14:16.005 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:37 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:14:16.135 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:14:16.136 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:16.136 INFO:teuthology.orchestra.run.smithi130.stdout: Package Arch Version Repo Size 2024-08-17T17:14:16.136 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:16.136 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:14:16.136 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-mirror x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 2.9 M 2024-08-17T17:14:16.136 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:16.136 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:14:16.137 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:16.137 INFO:teuthology.orchestra.run.smithi130.stdout:Install 1 Package 2024-08-17T17:14:16.137 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:16.137 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 2.9 M 2024-08-17T17:14:16.137 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 12 M 2024-08-17T17:14:16.137 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:14:16.402 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:14:16.463 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:16.486 INFO:teuthology.orchestra.run.smithi130.stdout:rbd-mirror-17.2.7-1404.gf2c39d4d.el9.x86_64.rpm 8.4 MB/s | 2.9 MB 00:00 2024-08-17T17:14:16.487 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:14:16.487 INFO:teuthology.orchestra.run.smithi130.stdout:Total 8.4 MB/s | 2.9 MB 00:00 2024-08-17T17:14:16.487 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:14:16.488 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:16.489 INFO:teuthology.orchestra.run.smithi179.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T17:14:16.489 INFO:teuthology.orchestra.run.smithi179.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-08-17T17:14:16.489 INFO:teuthology.orchestra.run.smithi179.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T17:14:16.489 INFO:teuthology.orchestra.run.smithi179.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T17:14:16.490 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:16.496 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:14:16.496 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:14:16.576 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:14:16.577 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:14:16.988 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:14:17.045 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:17.070 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:17.070 INFO:teuthology.orchestra.run.smithi130.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T17:14:17.070 INFO:teuthology.orchestra.run.smithi130.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-08-17T17:14:17.070 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T17:14:17.070 INFO:teuthology.orchestra.run.smithi130.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T17:14:17.070 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:17.579 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:17.579 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:17.579 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:14:17.580 INFO:teuthology.orchestra.run.smithi179.stdout: rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:14:17.580 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:17.580 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:14:17.821 DEBUG:teuthology.orchestra.run.smithi179:> sudo yum -y install rbd-nbd 2024-08-17T17:14:18.127 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:18.127 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:18.127 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:14:18.127 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-mirror-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:14:18.127 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:18.128 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:14:18.380 DEBUG:teuthology.orchestra.run.smithi130:> sudo yum -y install rbd-nbd 2024-08-17T17:14:18.383 INFO:teuthology.orchestra.run.smithi179.stdout:Last metadata expiration check: 0:01:40 ago on Sat 17 Aug 2024 05:12:38 PM UTC. 2024-08-17T17:14:18.512 INFO:teuthology.orchestra.run.smithi179.stdout:Dependencies resolved. 2024-08-17T17:14:18.512 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:18.513 INFO:teuthology.orchestra.run.smithi179.stdout: Package Architecture Version Repository Size 2024-08-17T17:14:18.513 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:18.513 INFO:teuthology.orchestra.run.smithi179.stdout:Installing: 2024-08-17T17:14:18.513 INFO:teuthology.orchestra.run.smithi179.stdout: rbd-nbd x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 165 k 2024-08-17T17:14:18.513 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:18.513 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction Summary 2024-08-17T17:14:18.513 INFO:teuthology.orchestra.run.smithi179.stdout:================================================================================ 2024-08-17T17:14:18.513 INFO:teuthology.orchestra.run.smithi179.stdout:Install 1 Package 2024-08-17T17:14:18.513 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:18.513 INFO:teuthology.orchestra.run.smithi179.stdout:Total download size: 165 k 2024-08-17T17:14:18.513 INFO:teuthology.orchestra.run.smithi179.stdout:Installed size: 477 k 2024-08-17T17:14:18.513 INFO:teuthology.orchestra.run.smithi179.stdout:Downloading Packages: 2024-08-17T17:14:18.752 INFO:teuthology.orchestra.run.smithi179.stdout:rbd-nbd-17.2.7-1404.gf2c39d4d.el9.x86_64.rpm 690 kB/s | 165 kB 00:00 2024-08-17T17:14:18.753 INFO:teuthology.orchestra.run.smithi179.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:14:18.753 INFO:teuthology.orchestra.run.smithi179.stdout:Total 686 kB/s | 165 kB 00:00 2024-08-17T17:14:18.753 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction check 2024-08-17T17:14:18.761 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction check succeeded. 2024-08-17T17:14:18.761 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction test 2024-08-17T17:14:18.824 INFO:teuthology.orchestra.run.smithi179.stdout:Transaction test succeeded. 2024-08-17T17:14:18.824 INFO:teuthology.orchestra.run.smithi179.stdout:Running transaction 2024-08-17T17:14:18.940 INFO:teuthology.orchestra.run.smithi130.stdout:Last metadata expiration check: 0:01:39 ago on Sat 17 Aug 2024 05:12:39 PM UTC. 2024-08-17T17:14:18.974 INFO:teuthology.orchestra.run.smithi179.stdout: Preparing : 1/1 2024-08-17T17:14:19.071 INFO:teuthology.orchestra.run.smithi130.stdout:Dependencies resolved. 2024-08-17T17:14:19.072 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:19.072 INFO:teuthology.orchestra.run.smithi130.stdout: Package Architecture Version Repository Size 2024-08-17T17:14:19.072 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:19.072 INFO:teuthology.orchestra.run.smithi130.stdout:Installing: 2024-08-17T17:14:19.072 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-nbd x86_64 2:17.2.7-1404.gf2c39d4d.el9 ceph 165 k 2024-08-17T17:14:19.072 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:19.072 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction Summary 2024-08-17T17:14:19.073 INFO:teuthology.orchestra.run.smithi130.stdout:================================================================================ 2024-08-17T17:14:19.073 INFO:teuthology.orchestra.run.smithi130.stdout:Install 1 Package 2024-08-17T17:14:19.073 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:19.073 INFO:teuthology.orchestra.run.smithi130.stdout:Total download size: 165 k 2024-08-17T17:14:19.073 INFO:teuthology.orchestra.run.smithi130.stdout:Installed size: 477 k 2024-08-17T17:14:19.073 INFO:teuthology.orchestra.run.smithi130.stdout:Downloading Packages: 2024-08-17T17:14:19.090 INFO:teuthology.orchestra.run.smithi179.stdout: Installing : rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:19.307 INFO:teuthology.orchestra.run.smithi130.stdout:rbd-nbd-17.2.7-1404.gf2c39d4d.el9.x86_64.rpm 707 kB/s | 165 kB 00:00 2024-08-17T17:14:19.307 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:14:19.308 INFO:teuthology.orchestra.run.smithi130.stdout:Total 702 kB/s | 165 kB 00:00 2024-08-17T17:14:19.308 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction check 2024-08-17T17:14:19.315 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction check succeeded. 2024-08-17T17:14:19.315 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction test 2024-08-17T17:14:19.378 INFO:teuthology.orchestra.run.smithi130.stdout:Transaction test succeeded. 2024-08-17T17:14:19.379 INFO:teuthology.orchestra.run.smithi130.stdout:Running transaction 2024-08-17T17:14:19.515 INFO:teuthology.orchestra.run.smithi179.stdout: Running scriptlet: rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:19.528 INFO:teuthology.orchestra.run.smithi130.stdout: Preparing : 1/1 2024-08-17T17:14:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: Installing : rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:19.777 INFO:teuthology.orchestra.run.smithi179.stdout: Verifying : rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:19.777 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:19.778 INFO:teuthology.orchestra.run.smithi179.stdout:Installed: 2024-08-17T17:14:19.778 INFO:teuthology.orchestra.run.smithi179.stdout: rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:14:19.778 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:14:19.778 INFO:teuthology.orchestra.run.smithi179.stdout:Complete! 2024-08-17T17:14:20.048 INFO:teuthology.orchestra.run.smithi130.stdout: Running scriptlet: rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:20.059 DEBUG:teuthology.parallel:result is None 2024-08-17T17:14:20.296 INFO:teuthology.orchestra.run.smithi130.stdout: Verifying : rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 1/1 2024-08-17T17:14:20.296 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:20.296 INFO:teuthology.orchestra.run.smithi130.stdout:Installed: 2024-08-17T17:14:20.296 INFO:teuthology.orchestra.run.smithi130.stdout: rbd-nbd-2:17.2.7-1404.gf2c39d4d.el9.x86_64 2024-08-17T17:14:20.296 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:14:20.297 INFO:teuthology.orchestra.run.smithi130.stdout:Complete! 2024-08-17T17:14:20.528 DEBUG:teuthology.parallel:result is None 2024-08-17T17:14:20.528 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T17:14:20.528 INFO:teuthology.packaging:ref: None 2024-08-17T17:14:20.528 INFO:teuthology.packaging:tag: None 2024-08-17T17:14:20.528 INFO:teuthology.packaging:branch: quincy 2024-08-17T17:14:20.528 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:14:20.529 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-08-17T17:14:20.755 DEBUG:teuthology.orchestra.run.smithi130:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-08-17T17:14:20.777 INFO:teuthology.orchestra.run.smithi130.stdout:17.2.7-1404.gf2c39d4d.el9 2024-08-17T17:14:20.778 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1404.gf2c39d4d.el9 2024-08-17T17:14:20.778 INFO:teuthology.task.install:The correct ceph version 17.2.7-1404.gf2c39d4d is installed. 2024-08-17T17:14:20.781 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T17:14:20.781 INFO:teuthology.packaging:ref: None 2024-08-17T17:14:20.781 INFO:teuthology.packaging:tag: None 2024-08-17T17:14:20.782 INFO:teuthology.packaging:branch: quincy 2024-08-17T17:14:20.782 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:14:20.782 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-08-17T17:14:21.001 DEBUG:teuthology.orchestra.run.smithi179:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-08-17T17:14:21.025 INFO:teuthology.orchestra.run.smithi179.stdout:17.2.7-1404.gf2c39d4d.el9 2024-08-17T17:14:21.025 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1404.gf2c39d4d.el9 2024-08-17T17:14:21.025 INFO:teuthology.task.install:The correct ceph version 17.2.7-1404.gf2c39d4d is installed. 2024-08-17T17:14:21.027 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-08-17T17:14:21.028 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:14:21.028 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-08-17T17:14:21.053 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-08-17T17:14:21.053 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-08-17T17:14:21.094 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-08-17T17:14:21.094 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:14:21.094 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/usr/bin/daemon-helper 2024-08-17T17:14:21.119 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-08-17T17:14:21.183 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-08-17T17:14:21.183 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd of=/usr/bin/daemon-helper 2024-08-17T17:14:21.208 DEBUG:teuthology.orchestra.run.smithi179:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-08-17T17:14:21.273 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-08-17T17:14:21.273 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:14:21.273 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/usr/bin/adjust-ulimits 2024-08-17T17:14:21.298 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-08-17T17:14:21.364 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-08-17T17:14:21.364 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd of=/usr/bin/adjust-ulimits 2024-08-17T17:14:21.389 DEBUG:teuthology.orchestra.run.smithi179:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-08-17T17:14:21.456 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-08-17T17:14:21.456 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:14:21.457 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/usr/bin/stdin-killer 2024-08-17T17:14:21.481 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-08-17T17:14:21.546 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-08-17T17:14:21.546 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd of=/usr/bin/stdin-killer 2024-08-17T17:14:21.571 DEBUG:teuthology.orchestra.run.smithi179:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-08-17T17:14:21.635 INFO:teuthology.run_tasks:Running task cephadm... 2024-08-17T17:14:21.733 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'PG_AVAILABILITY'], 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'} 2024-08-17T17:14:21.734 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-08-17T17:14:21.734 INFO:tasks.cephadm:Cluster fsid is 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:14:21.734 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-08-17T17:14:21.735 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.130', 'mon.c': '[v2:172.21.15.130:3301,v1:172.21.15.130:6790]', 'mon.b': '172.21.15.179'} 2024-08-17T17:14:21.735 INFO:tasks.cephadm:First mon is mon.a on smithi130 2024-08-17T17:14:21.735 INFO:tasks.cephadm:First mgr is y 2024-08-17T17:14:21.735 INFO:tasks.cephadm:Normalizing hostnames... 2024-08-17T17:14:21.735 DEBUG:teuthology.orchestra.run.smithi130:> sudo hostname $(hostname -s) 2024-08-17T17:14:21.763 DEBUG:teuthology.orchestra.run.smithi179:> sudo hostname $(hostname -s) 2024-08-17T17:14:21.788 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-08-17T17:14:21.788 DEBUG:teuthology.orchestra.run.smithi130:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-08-17T17:14:21.991 INFO:teuthology.orchestra.run.smithi130.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Aug 17 17:14 /home/ubuntu/cephtest/cephadm 2024-08-17T17:14:21.992 DEBUG:teuthology.orchestra.run.smithi179:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-08-17T17:14:22.155 INFO:teuthology.orchestra.run.smithi179.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Aug 17 17:14 /home/ubuntu/cephtest/cephadm 2024-08-17T17:14:22.156 DEBUG:teuthology.orchestra.run.smithi130:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-08-17T17:14:22.174 DEBUG:teuthology.orchestra.run.smithi179:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-08-17T17:14:22.199 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-08-17T17:14:22.199 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-08-17T17:14:22.216 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-08-17T17:14:23.167 INFO:teuthology.orchestra.run.smithi179.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-08-17T17:14:23.170 INFO:teuthology.orchestra.run.smithi130.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-08-17T17:14:51.591 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:14:51.592 INFO:teuthology.orchestra.run.smithi179.stdout: "ceph_version": "ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)", 2024-08-17T17:14:51.592 INFO:teuthology.orchestra.run.smithi179.stdout: "image_id": "f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce", 2024-08-17T17:14:51.592 INFO:teuthology.orchestra.run.smithi179.stdout: "repo_digests": [ 2024-08-17T17:14:51.592 INFO:teuthology.orchestra.run.smithi179.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251" 2024-08-17T17:14:51.592 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:14:51.593 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:14:58.515 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:14:58.515 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph_version": "ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)", 2024-08-17T17:14:58.516 INFO:teuthology.orchestra.run.smithi130.stdout: "image_id": "f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce", 2024-08-17T17:14:58.516 INFO:teuthology.orchestra.run.smithi130.stdout: "repo_digests": [ 2024-08-17T17:14:58.516 INFO:teuthology.orchestra.run.smithi130.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251" 2024-08-17T17:14:58.516 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:14:58.516 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:14:58.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo mkdir -p /etc/ceph 2024-08-17T17:14:58.580 DEBUG:teuthology.orchestra.run.smithi179:> sudo mkdir -p /etc/ceph 2024-08-17T17:14:58.614 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod 777 /etc/ceph 2024-08-17T17:14:58.639 DEBUG:teuthology.orchestra.run.smithi179:> sudo chmod 777 /etc/ceph 2024-08-17T17:14:58.682 INFO:tasks.cephadm:Writing seed config... 2024-08-17T17:14:58.684 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-08-17T17:14:58.684 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-08-17T17:14:58.684 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-08-17T17:14:58.684 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-08-17T17:14:58.684 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-08-17T17:14:58.684 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-08-17T17:14:58.684 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-08-17T17:14:58.684 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-08-17T17:14:58.684 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-08-17T17:14:58.684 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-08-17T17:14:58.684 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-08-17T17:14:58.685 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:14:58.685 DEBUG:teuthology.orchestra.run.smithi130:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-08-17T17:14:58.700 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 25127030-5cbc-11ef-bccf-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-08-17T17:14:58.701 DEBUG:teuthology.orchestra.run.smithi130:mon.a> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.a.service 2024-08-17T17:14:58.743 DEBUG:teuthology.orchestra.run.smithi130:mgr.y> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.y.service 2024-08-17T17:14:58.785 INFO:tasks.cephadm:Bootstrapping... 2024-08-17T17:14:58.786 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 25127030-5cbc-11ef-bccf-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.130 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:14:58.987 INFO:teuthology.orchestra.run.smithi130.stdout:-------------------------------------------------------------------------------- 2024-08-17T17:14:58.987 INFO:teuthology.orchestra.run.smithi130.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '25127030-5cbc-11ef-bccf-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.130', '--skip-admin-label'] 2024-08-17T17:14:59.014 INFO:teuthology.orchestra.run.smithi130.stdout:/bin/podman: stdout 5.1.2 2024-08-17T17:14:59.014 INFO:teuthology.orchestra.run.smithi130.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-08-17T17:14:59.015 INFO:teuthology.orchestra.run.smithi130.stdout:Verifying podman|docker is present... 2024-08-17T17:14:59.039 INFO:teuthology.orchestra.run.smithi130.stdout:/bin/podman: stdout 5.1.2 2024-08-17T17:14:59.039 INFO:teuthology.orchestra.run.smithi130.stdout:Verifying lvm2 is present... 2024-08-17T17:14:59.039 INFO:teuthology.orchestra.run.smithi130.stdout:Verifying time synchronization is in place... 2024-08-17T17:14:59.047 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-08-17T17:14:59.047 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-08-17T17:14:59.054 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-08-17T17:14:59.054 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stdout inactive 2024-08-17T17:14:59.061 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stdout enabled 2024-08-17T17:14:59.070 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stdout active 2024-08-17T17:14:59.070 INFO:teuthology.orchestra.run.smithi130.stdout:Unit chronyd.service is enabled and running 2024-08-17T17:14:59.070 INFO:teuthology.orchestra.run.smithi130.stdout:Repeating the final host check... 2024-08-17T17:14:59.092 INFO:teuthology.orchestra.run.smithi130.stdout:/bin/podman: stdout 5.1.2 2024-08-17T17:14:59.092 INFO:teuthology.orchestra.run.smithi130.stdout:podman (/bin/podman) version 5.1.2 is present 2024-08-17T17:14:59.092 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl is present 2024-08-17T17:14:59.092 INFO:teuthology.orchestra.run.smithi130.stdout:lvcreate is present 2024-08-17T17:14:59.098 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-08-17T17:14:59.099 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-08-17T17:14:59.105 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-08-17T17:14:59.105 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stdout inactive 2024-08-17T17:14:59.112 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stdout enabled 2024-08-17T17:14:59.119 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stdout active 2024-08-17T17:14:59.119 INFO:teuthology.orchestra.run.smithi130.stdout:Unit chronyd.service is enabled and running 2024-08-17T17:14:59.119 INFO:teuthology.orchestra.run.smithi130.stdout:Host looks OK 2024-08-17T17:14:59.119 INFO:teuthology.orchestra.run.smithi130.stdout:Cluster fsid: 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:14:59.119 INFO:teuthology.orchestra.run.smithi130.stdout:Acquiring lock 140425788302192 on /run/cephadm/25127030-5cbc-11ef-bccf-c7b262605968.lock 2024-08-17T17:14:59.119 INFO:teuthology.orchestra.run.smithi130.stdout:Lock 140425788302192 acquired on /run/cephadm/25127030-5cbc-11ef-bccf-c7b262605968.lock 2024-08-17T17:14:59.120 INFO:teuthology.orchestra.run.smithi130.stdout:Verifying IP 172.21.15.130 port 3300 ... 2024-08-17T17:14:59.120 INFO:teuthology.orchestra.run.smithi130.stdout:Verifying IP 172.21.15.130 port 6789 ... 2024-08-17T17:14:59.120 INFO:teuthology.orchestra.run.smithi130.stdout:Base mon IP(s) is [172.21.15.130:3300, 172.21.15.130:6789], mon addrv is [v2:172.21.15.130:3300,v1:172.21.15.130:6789] 2024-08-17T17:14:59.124 INFO:teuthology.orchestra.run.smithi130.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.130 metric 100 2024-08-17T17:14:59.124 INFO:teuthology.orchestra.run.smithi130.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.130 metric 100 2024-08-17T17:14:59.126 INFO:teuthology.orchestra.run.smithi130.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-08-17T17:14:59.127 INFO:teuthology.orchestra.run.smithi130.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-08-17T17:14:59.127 INFO:teuthology.orchestra.run.smithi130.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1792sec hoplimit 64 pref medium 2024-08-17T17:14:59.129 INFO:teuthology.orchestra.run.smithi130.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-08-17T17:14:59.129 INFO:teuthology.orchestra.run.smithi130.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-08-17T17:14:59.129 INFO:teuthology.orchestra.run.smithi130.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-08-17T17:14:59.129 INFO:teuthology.orchestra.run.smithi130.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-08-17T17:14:59.129 INFO:teuthology.orchestra.run.smithi130.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe88:7f37/64 scope link 2024-08-17T17:14:59.130 INFO:teuthology.orchestra.run.smithi130.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-08-17T17:14:59.130 INFO:teuthology.orchestra.run.smithi130.stdout:Mon IP `172.21.15.130` is in CIDR network `172.21.0.0/20` 2024-08-17T17:14:59.130 INFO:teuthology.orchestra.run.smithi130.stdout:Mon IP `172.21.15.130` is in CIDR network `172.21.0.0/20` 2024-08-17T17:14:59.130 INFO:teuthology.orchestra.run.smithi130.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-08-17T17:14:59.131 INFO:teuthology.orchestra.run.smithi130.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-08-17T17:14:59.131 INFO:teuthology.orchestra.run.smithi130.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-08-17T17:15:01.350 INFO:teuthology.orchestra.run.smithi130.stdout:/bin/podman: stdout f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce 2024-08-17T17:15:01.351 INFO:teuthology.orchestra.run.smithi130.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-08-17T17:15:01.351 INFO:teuthology.orchestra.run.smithi130.stdout:/bin/podman: stderr Getting image source signatures 2024-08-17T17:15:01.351 INFO:teuthology.orchestra.run.smithi130.stdout:/bin/podman: stderr Copying blob sha256:11a3dd1ce731c9342fb0d936b067311689a4bd18187bbc27c4f481cc3262ce2c 2024-08-17T17:15:01.351 INFO:teuthology.orchestra.run.smithi130.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-08-17T17:15:01.351 INFO:teuthology.orchestra.run.smithi130.stdout:/bin/podman: stderr Copying config sha256:f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce 2024-08-17T17:15:01.351 INFO:teuthology.orchestra.run.smithi130.stdout:/bin/podman: stderr Writing manifest to image destination 2024-08-17T17:15:02.219 INFO:teuthology.orchestra.run.smithi130.stdout:ceph: stdout ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable) 2024-08-17T17:15:02.220 INFO:teuthology.orchestra.run.smithi130.stdout:Ceph version: ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable) 2024-08-17T17:15:02.220 INFO:teuthology.orchestra.run.smithi130.stdout:Extracting ceph user uid/gid from container image... 2024-08-17T17:15:03.300 INFO:teuthology.orchestra.run.smithi130.stdout:stat: stdout 167 167 2024-08-17T17:15:03.301 INFO:teuthology.orchestra.run.smithi130.stdout:Creating initial keys... 2024-08-17T17:15:04.238 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph-authtool: stdout AQCX2sBm1DUoJxAA33uISoAY4bwNh+ga6J5dIA== 2024-08-17T17:15:05.318 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph-authtool: stdout AQCY2sBmMWh0JBAA/7pBqEJf+kMAelcz1sPv/g== 2024-08-17T17:15:06.248 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph-authtool: stdout AQCZ2sBmgBI3JxAAb5rwp58O/XZdHk3jmQ0d3A== 2024-08-17T17:15:06.249 INFO:teuthology.orchestra.run.smithi130.stdout:Creating initial monmap... 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi130.stdout:monmaptool for a [v2:172.21.15.130:3300,v1:172.21.15.130:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi130.stdout:setting min_mon_release = octopus 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/monmaptool: set fsid to 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi130.stdout:Creating mon... 2024-08-17T17:15:08.234 INFO:teuthology.orchestra.run.smithi130.stdout:create mon.a on 2024-08-17T17:15:08.445 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-08-17T17:15:08.585 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-08-17T17:15:08.731 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-25127030-5cbc-11ef-bccf-c7b262605968.target → /etc/systemd/system/ceph-25127030-5cbc-11ef-bccf-c7b262605968.target. 2024-08-17T17:15:08.731 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-25127030-5cbc-11ef-bccf-c7b262605968.target → /etc/systemd/system/ceph-25127030-5cbc-11ef-bccf-c7b262605968.target. 2024-08-17T17:15:08.894 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.a 2024-08-17T17:15:08.895 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stderr Failed to reset failed state of unit ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.a.service: Unit ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.a.service not loaded. 2024-08-17T17:15:09.054 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-25127030-5cbc-11ef-bccf-c7b262605968.target.wants/ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.a.service → /etc/systemd/system/ceph-25127030-5cbc-11ef-bccf-c7b262605968@.service. 2024-08-17T17:15:09.523 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T17:15:09.523 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stdout disabled 2024-08-17T17:15:09.530 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T17:15:09.530 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stdout inactive 2024-08-17T17:15:09.530 INFO:teuthology.orchestra.run.smithi130.stdout:firewalld.service is not enabled 2024-08-17T17:15:09.530 INFO:teuthology.orchestra.run.smithi130.stdout:Not possible to enable service . firewalld.service is not available 2024-08-17T17:15:09.531 INFO:teuthology.orchestra.run.smithi130.stdout:Waiting for mon to start... 2024-08-17T17:15:09.531 INFO:teuthology.orchestra.run.smithi130.stdout:Waiting for mon... 2024-08-17T17:15:09.655 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:09 smithi130 systemd[1]: Started Ceph mon.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:15:10.640 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:10 smithi130 ceph-mon[30869]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-17T17:15:10.641 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:10 smithi130 ceph-mon[30869]: monmap e1: 1 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T17:15:10.641 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:10 smithi130 ceph-mon[30869]: fsmap 2024-08-17T17:15:10.641 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:10 smithi130 ceph-mon[30869]: osdmap e1: 0 total, 0 up, 0 in 2024-08-17T17:15:10.641 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:10 smithi130 ceph-mon[30869]: mgrmap e1: no daemons active 2024-08-17T17:15:10.641 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:10 smithi130 ceph-mon[30869]: from='client.? 172.21.15.130:0/743209491' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-17T17:15:10.644 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout cluster: 2024-08-17T17:15:10.644 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout id: 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:15:10.644 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-08-17T17:15:10.645 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout 2024-08-17T17:15:10.645 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout services: 2024-08-17T17:15:10.645 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.531506s) 2024-08-17T17:15:10.645 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-08-17T17:15:10.645 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-08-17T17:15:10.645 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout 2024-08-17T17:15:10.645 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout data: 2024-08-17T17:15:10.645 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-08-17T17:15:10.645 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-08-17T17:15:10.645 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-08-17T17:15:10.645 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout pgs: 2024-08-17T17:15:10.645 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout 2024-08-17T17:15:10.645 INFO:teuthology.orchestra.run.smithi130.stdout:mon is available 2024-08-17T17:15:10.645 INFO:teuthology.orchestra.run.smithi130.stdout:Assimilating anything we can from ceph.conf... 2024-08-17T17:15:11.656 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout 2024-08-17T17:15:11.658 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout [global] 2024-08-17T17:15:11.658 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout fsid = 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:15:11.658 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-08-17T17:15:11.658 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.130:3300,v1:172.21.15.130:6789] 2024-08-17T17:15:11.658 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-08-17T17:15:11.658 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-08-17T17:15:11.659 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-08-17T17:15:11.659 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-08-17T17:15:11.659 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout 2024-08-17T17:15:11.659 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout [mgr] 2024-08-17T17:15:11.659 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-08-17T17:15:11.659 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout 2024-08-17T17:15:11.659 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout [osd] 2024-08-17T17:15:11.659 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-08-17T17:15:11.659 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-08-17T17:15:11.659 INFO:teuthology.orchestra.run.smithi130.stdout:Generating new minimal ceph.conf... 2024-08-17T17:15:11.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:11 smithi130 ceph-mon[30869]: from='client.? 172.21.15.130:0/1955703132' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-08-17T17:15:11.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:11 smithi130 ceph-mon[30869]: from='client.? 172.21.15.130:0/1955703132' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-08-17T17:15:12.604 INFO:teuthology.orchestra.run.smithi130.stdout:Restarting the monitor... 2024-08-17T17:15:12.626 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:12 smithi130 ceph-mon[30869]: from='client.? 172.21.15.130:0/1339114919' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:15:12.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:12 smithi130 systemd[1]: Stopping Ceph mon.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:15:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[30846]: 2024-08-17T17:15:12.880+0000 7f3b19808640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:15:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[30846]: 2024-08-17T17:15:12.880+0000 7f3b19808640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-08-17T17:15:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:12 smithi130 podman[31187]: 2024-08-17 17:15:12.921908439 +0000 UTC m=+0.151468921 container died d6f154106bcd4821ffc2878a41ecee2293556079bb1c19c3f07a3f83b289639d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=quincy-f2c39d4, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-08-17T17:15:13.512 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:13 smithi130 podman[31187]: 2024-08-17 17:15:13.294725392 +0000 UTC m=+0.524285862 container cleanup d6f154106bcd4821ffc2878a41ecee2293556079bb1c19c3f07a3f83b289639d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:15:13.512 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:13 smithi130 bash[31187]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a 2024-08-17T17:15:13.860 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:13 smithi130 podman[31205]: 2024-08-17 17:15:13.512118178 +0000 UTC m=+0.585915806 container remove d6f154106bcd4821ffc2878a41ecee2293556079bb1c19c3f07a3f83b289639d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-08-17T17:15:13.860 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:13 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.a.service: Deactivated successfully. 2024-08-17T17:15:13.860 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:13 smithi130 systemd[1]: Stopped Ceph mon.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:15:13.860 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:13 smithi130 systemd[1]: Starting Ceph mon.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:15:14.111 INFO:teuthology.orchestra.run.smithi130.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-08-17T17:15:14.136 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:13 smithi130 podman[31303]: 2024-08-17 17:15:13.762263064 +0000 UTC m=+0.017146558 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph:quincy 2024-08-17T17:15:14.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:13 smithi130 podman[31303]: 2024-08-17 17:15:13.882886822 +0000 UTC m=+0.137770304 container create 7a678d70ca3b84607c7d84ff1e3dfc18986485d46f40df4df678ba61c50cd651 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0) 2024-08-17T17:15:14.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 podman[31303]: 2024-08-17 17:15:14.048683734 +0000 UTC m=+0.303567226 container init 7a678d70ca3b84607c7d84ff1e3dfc18986485d46f40df4df678ba61c50cd651 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, GIT_CLEAN=True) 2024-08-17T17:15:14.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 podman[31303]: 2024-08-17 17:15:14.053105115 +0000 UTC m=+0.307988596 container start 7a678d70ca3b84607c7d84ff1e3dfc18986485d46f40df4df678ba61c50cd651 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240731) 2024-08-17T17:15:14.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:15:14.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable), process ceph-mon, pid 7 2024-08-17T17:15:14.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: pidfile_write: ignore empty --pid-file 2024-08-17T17:15:14.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: load: jerasure load: lrc 2024-08-17T17:15:14.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: RocksDB version: 6.15.5 2024-08-17T17:15:14.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-08-17T17:15:14.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Compile date Aug 17 2024 2024-08-17T17:15:14.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: DB SUMMARY 2024-08-17T17:15:14.138 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: DB Session ID: PKSTUNJUOX8I3MCFGR0Y 2024-08-17T17:15:14.138 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:15:14.138 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:15:14.138 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-08-17T17:15:14.138 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-08-17T17:15:14.138 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83714 ; 2024-08-17T17:15:14.138 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:15:14.138 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:15:14.138 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:15:14.139 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:15:14.139 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.env: 0x557da4802480 2024-08-17T17:15:14.139 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.fs: Posix File System 2024-08-17T17:15:14.139 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.info_log: 0x557da5976f80 2024-08-17T17:15:14.139 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:15:14.139 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.statistics: (nil) 2024-08-17T17:15:14.139 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.use_fsync: 0 2024-08-17T17:15:14.139 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:15:14.140 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:15:14.140 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:15:14.140 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:15:14.140 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:15:14.140 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:15:14.140 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:15:14.140 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:15:14.140 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:15:14.140 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:15:14.140 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:15:14.140 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.db_log_dir: 2024-08-17T17:15:14.140 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-08-17T17:15:14.141 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:15:14.141 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:15:14.141 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:15:14.141 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:15:14.141 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:15:14.141 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:15:14.141 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:15:14.141 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:15:14.141 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.write_buffer_manager: 0x557da5a34720 2024-08-17T17:15:14.141 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:15:14.141 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-08-17T17:15:14.141 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:15:14.142 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:15:14.142 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:15:14.142 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:15:14.142 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:15:14.142 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:15:14.142 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:15:14.142 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.unordered_write: 0 2024-08-17T17:15:14.142 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:15:14.142 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:15:14.142 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:15:14.142 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:15:14.142 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.row_cache: None 2024-08-17T17:15:14.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.wal_filter: None 2024-08-17T17:15:14.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:15:14.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:15:14.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.preserve_deletes: 0 2024-08-17T17:15:14.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:15:14.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:15:14.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:15:14.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:15:14.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:15:14.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:15:14.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:15:14.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:15:14.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:15:14.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:15:14.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:15:14.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:15:14.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:15:14.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:15:14.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:15:14.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:15:14.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:15:14.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:15:14.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:15:14.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:15:14.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:15:14.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:15:14.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:15:14.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:15:14.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_open_files: -1 2024-08-17T17:15:14.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:15:14.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:15:14.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:15:14.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:15:14.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:15:14.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Compression algorithms supported: 2024-08-17T17:15:14.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: kZSTD supported: 0 2024-08-17T17:15:14.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:15:14.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:15:14.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:15:14.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:15:14.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:15:14.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:15:14.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:15:14.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:15:14.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-08-17T17:15:14.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-08-17T17:15:14.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:15:14.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.merge_operator: 2024-08-17T17:15:14.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compaction_filter: None 2024-08-17T17:15:14.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:15:14.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:15:14.147 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:15:14.147 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:15:14.147 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557da591da78) 2024-08-17T17:15:14.147 INFO:journalctl@ceph.mon.a.smithi130.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:15:14.147 INFO:journalctl@ceph.mon.a.smithi130.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:15:14.147 INFO:journalctl@ceph.mon.a.smithi130.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:15:14.147 INFO:journalctl@ceph.mon.a.smithi130.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:15:14.147 INFO:journalctl@ceph.mon.a.smithi130.stdout: index_type: 0 2024-08-17T17:15:14.147 INFO:journalctl@ceph.mon.a.smithi130.stdout: data_block_index_type: 0 2024-08-17T17:15:14.147 INFO:journalctl@ceph.mon.a.smithi130.stdout: index_shortening: 1 2024-08-17T17:15:14.147 INFO:journalctl@ceph.mon.a.smithi130.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:15:14.147 INFO:journalctl@ceph.mon.a.smithi130.stdout: hash_index_allow_collision: 1 2024-08-17T17:15:14.148 INFO:journalctl@ceph.mon.a.smithi130.stdout: checksum: 1 2024-08-17T17:15:14.148 INFO:journalctl@ceph.mon.a.smithi130.stdout: no_block_cache: 0 2024-08-17T17:15:14.148 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache: 0x557da59531f0 2024-08-17T17:15:14.148 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:15:14.148 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache_options: 2024-08-17T17:15:14.148 INFO:journalctl@ceph.mon.a.smithi130.stdout: capacity : 536870912 2024-08-17T17:15:14.148 INFO:journalctl@ceph.mon.a.smithi130.stdout: num_shard_bits : 4 2024-08-17T17:15:14.148 INFO:journalctl@ceph.mon.a.smithi130.stdout: strict_capacity_limit : 0 2024-08-17T17:15:14.148 INFO:journalctl@ceph.mon.a.smithi130.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:15:14.148 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache_compressed: (nil) 2024-08-17T17:15:14.148 INFO:journalctl@ceph.mon.a.smithi130.stdout: persistent_cache: (nil) 2024-08-17T17:15:14.148 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_size: 4096 2024-08-17T17:15:14.149 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_size_deviation: 10 2024-08-17T17:15:14.149 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_restart_interval: 16 2024-08-17T17:15:14.149 INFO:journalctl@ceph.mon.a.smithi130.stdout: index_block_restart_interval: 1 2024-08-17T17:15:14.149 INFO:journalctl@ceph.mon.a.smithi130.stdout: metadata_block_size: 4096 2024-08-17T17:15:14.149 INFO:journalctl@ceph.mon.a.smithi130.stdout: partition_filters: 0 2024-08-17T17:15:14.149 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_delta_encoding: 1 2024-08-17T17:15:14.149 INFO:journalctl@ceph.mon.a.smithi130.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-08-17T17:15:14.149 INFO:journalctl@ceph.mon.a.smithi130.stdout: whole_key_filtering: 1 2024-08-17T17:15:14.150 INFO:journalctl@ceph.mon.a.smithi130.stdout: verify_compression: 0 2024-08-17T17:15:14.150 INFO:journalctl@ceph.mon.a.smithi130.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:15:14.150 INFO:journalctl@ceph.mon.a.smithi130.stdout: format_version: 4 2024-08-17T17:15:14.150 INFO:journalctl@ceph.mon.a.smithi130.stdout: enable_index_compression: 1 2024-08-17T17:15:14.150 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_align: 0 2024-08-17T17:15:14.150 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:15:14.151 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:15:14.151 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compression: NoCompression 2024-08-17T17:15:14.151 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:15:14.151 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:15:14.151 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:15:14.151 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.num_levels: 7 2024-08-17T17:15:14.151 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:15:14.151 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:15:14.151 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:15:14.151 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:15:14.151 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:15:14.151 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:15:14.151 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:15:14.151 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:15:14.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:15:14.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:15:14.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:15:14.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:15:14.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:15:14.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:15:14.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:15:14.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:15:14.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:15:14.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:15:14.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:15:14.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:15:14.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:15:14.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:15:14.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:15:14.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:15:14.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:15:14.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:15:14.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:15:14.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:15:14.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:15:14.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:15:14.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:15:14.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:15:14.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:15:14.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:15:14.154 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.arena_block_size: 4194304 2024-08-17T17:15:14.154 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:15:14.154 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:15:14.154 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-08-17T17:15:14.154 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:15:14.154 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:15:14.154 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:15:14.154 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:15:14.154 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:15:14.154 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:15:14.154 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:15:14.154 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:15:14.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:15:14.156 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.ttl: 2592000 2024-08-17T17:15:14.156 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:15:14.156 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:15:14.156 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:15:14.156 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:15:14.156 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:15:14.156 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:15:14.156 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:15:14.156 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T17:15:14.156 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-08-17T17:15:14.156 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-08-17T17:15:14.156 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914914076518, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-08-17T17:15:14.156 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-08-17T17:15:14.157 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-08-17T17:15:14.157 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914914077229, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80287, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78529, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10130, "raw_average_key_size": 47, "raw_value_size": 72742, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1723914914, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "547788f8-3706-47e8-bc62-4e916c76bb05", "db_session_id": "PKSTUNJUOX8I3MCFGR0Y"}} 2024-08-17T17:15:14.157 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-08-17T17:15:14.157 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914914077551, "job": 1, "event": "recovery_finished"} 2024-08-17T17:15:14.157 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:15:14.157 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x557da5986700 2024-08-17T17:15:14.157 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: DB pointer 0x557da5956000 2024-08-17T17:15:14.157 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-08-17T17:15:14.157 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-08-17T17:15:14.157 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** DB Stats ** 2024-08-17T17:15:14.157 INFO:journalctl@ceph.mon.a.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:15:14.157 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:15:14.158 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:15:14.158 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:15:14.158 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:15:14.158 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-08-17T17:15:14.158 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:15:14.158 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:15:14.158 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** Compaction Stats [default] ** 2024-08-17T17:15:14.158 INFO:journalctl@ceph.mon.a.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T17:15:14.158 INFO:journalctl@ceph.mon.a.smithi130.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:15:14.158 INFO:journalctl@ceph.mon.a.smithi130.stdout: L0 2/0 80.19 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.5 0.00 0.00 1 0.000 0 0 2024-08-17T17:15:14.158 INFO:journalctl@ceph.mon.a.smithi130.stdout: Sum 2/0 80.19 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.5 0.00 0.00 1 0.000 0 0 2024-08-17T17:15:14.158 INFO:journalctl@ceph.mon.a.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.5 0.00 0.00 1 0.000 0 0 2024-08-17T17:15:14.158 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:15:14.159 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** Compaction Stats [default] ** 2024-08-17T17:15:14.159 INFO:journalctl@ceph.mon.a.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T17:15:14.159 INFO:journalctl@ceph.mon.a.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:15:14.159 INFO:journalctl@ceph.mon.a.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 159.5 0.00 0.00 1 0.000 0 0 2024-08-17T17:15:14.159 INFO:journalctl@ceph.mon.a.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:15:14.159 INFO:journalctl@ceph.mon.a.smithi130.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T17:15:14.159 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:15:14.159 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:15:14.159 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:15:14.159 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:15:14.159 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative compaction: 0.00 GB write, 25.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:15:14.159 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval compaction: 0.00 GB write, 25.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:15:14.159 INFO:journalctl@ceph.mon.a.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:15:14.159 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:15:14.160 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:15:14.160 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:15:14.160 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** Compaction Stats [default] ** 2024-08-17T17:15:14.160 INFO:journalctl@ceph.mon.a.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T17:15:14.160 INFO:journalctl@ceph.mon.a.smithi130.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:15:14.160 INFO:journalctl@ceph.mon.a.smithi130.stdout: L0 2/0 80.19 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.5 0.00 0.00 1 0.000 0 0 2024-08-17T17:15:14.160 INFO:journalctl@ceph.mon.a.smithi130.stdout: Sum 2/0 80.19 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 159.5 0.00 0.00 1 0.000 0 0 2024-08-17T17:15:14.160 INFO:journalctl@ceph.mon.a.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-08-17T17:15:14.160 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:15:14.160 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** Compaction Stats [default] ** 2024-08-17T17:15:14.160 INFO:journalctl@ceph.mon.a.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T17:15:14.160 INFO:journalctl@ceph.mon.a.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:15:14.160 INFO:journalctl@ceph.mon.a.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 159.5 0.00 0.00 1 0.000 0 0 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.a.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.a.smithi130.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative compaction: 0.00 GB write, 25.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.a.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: starting mon.a rank 0 at public addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] at bind addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: mon.a@-1(???) e1 preinit fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:15:14.161 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: mon.a@-1(???).mds e1 new map 2024-08-17T17:15:14.162 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: mon.a@-1(???).mds e1 print_map 2024-08-17T17:15:14.162 INFO:journalctl@ceph.mon.a.smithi130.stdout: e1 2024-08-17T17:15:14.162 INFO:journalctl@ceph.mon.a.smithi130.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:15:14.162 INFO:journalctl@ceph.mon.a.smithi130.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:15:14.162 INFO:journalctl@ceph.mon.a.smithi130.stdout: legacy client fscid: -1 2024-08-17T17:15:14.162 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:15:14.162 INFO:journalctl@ceph.mon.a.smithi130.stdout: No filesystems configured 2024-08-17T17:15:14.162 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-08-17T17:15:14.162 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:15:14.162 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:15:14.162 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:15:14.162 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-08-17T17:15:14.162 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 bash[31303]: 7a678d70ca3b84607c7d84ff1e3dfc18986485d46f40df4df678ba61c50cd651 2024-08-17T17:15:14.162 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 systemd[1]: Started Ceph mon.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:15:14.163 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-17T17:15:14.163 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: monmap e1: 1 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T17:15:14.163 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: fsmap 2024-08-17T17:15:14.163 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: osdmap e1: 0 total, 0 up, 0 in 2024-08-17T17:15:14.163 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:14 smithi130 ceph-mon[31342]: mgrmap e1: no daemons active 2024-08-17T17:15:15.333 INFO:teuthology.orchestra.run.smithi130.stdout:Wrote config to /etc/ceph/ceph.conf 2024-08-17T17:15:15.333 INFO:teuthology.orchestra.run.smithi130.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:15:15.333 INFO:teuthology.orchestra.run.smithi130.stdout:Creating mgr... 2024-08-17T17:15:15.333 INFO:teuthology.orchestra.run.smithi130.stdout:Verifying port 0.0.0.0:9283 ... 2024-08-17T17:15:15.334 INFO:teuthology.orchestra.run.smithi130.stdout:Verifying port 0.0.0.0:8765 ... 2024-08-17T17:15:15.512 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.y 2024-08-17T17:15:15.513 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stderr Failed to reset failed state of unit ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.y.service: Unit ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.y.service not loaded. 2024-08-17T17:15:15.670 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-25127030-5cbc-11ef-bccf-c7b262605968.target.wants/ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-25127030-5cbc-11ef-bccf-c7b262605968@.service. 2024-08-17T17:15:15.996 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:15 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/617013763' entity='client.admin' 2024-08-17T17:15:15.997 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:15 smithi130 podman[31579]: 2024-08-17 17:15:15.811761822 +0000 UTC m=+0.017151367 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph:quincy 2024-08-17T17:15:16.165 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T17:15:16.166 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stdout disabled 2024-08-17T17:15:16.174 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T17:15:16.174 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stdout inactive 2024-08-17T17:15:16.174 INFO:teuthology.orchestra.run.smithi130.stdout:firewalld.service is not enabled 2024-08-17T17:15:16.174 INFO:teuthology.orchestra.run.smithi130.stdout:Not possible to enable service . firewalld.service is not available 2024-08-17T17:15:16.184 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T17:15:16.184 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stdout disabled 2024-08-17T17:15:16.193 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T17:15:16.193 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stdout inactive 2024-08-17T17:15:16.193 INFO:teuthology.orchestra.run.smithi130.stdout:firewalld.service is not enabled 2024-08-17T17:15:16.193 INFO:teuthology.orchestra.run.smithi130.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-08-17T17:15:16.193 INFO:teuthology.orchestra.run.smithi130.stdout:Waiting for mgr to start... 2024-08-17T17:15:16.193 INFO:teuthology.orchestra.run.smithi130.stdout:Waiting for mgr... 2024-08-17T17:15:16.347 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:16 smithi130 podman[31579]: 2024-08-17 17:15:16.092849374 +0000 UTC m=+0.298238931 container init 5e2d324fc1a5ad19fec5c03e6b27b911408dda3a94d2f261a31233b44011238d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=quincy-f2c39d4, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-08-17T17:15:16.347 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:16 smithi130 podman[31579]: 2024-08-17 17:15:16.097300819 +0000 UTC m=+0.302690366 container start 5e2d324fc1a5ad19fec5c03e6b27b911408dda3a94d2f261a31233b44011238d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, RELEASE=quincy-f2c39d4, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-08-17T17:15:16.347 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:16 smithi130 bash[31579]: 5e2d324fc1a5ad19fec5c03e6b27b911408dda3a94d2f261a31233b44011238d 2024-08-17T17:15:16.347 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:16 smithi130 systemd[1]: Started Ceph mgr.y for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:15:16.347 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:16 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:16.224+0000 7f5075c9aec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T17:15:16.347 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:16 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:16.345+0000 7f5075c9aec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:15:16.629 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:16 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:16.411+0000 7f5075c9aec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T17:15:16.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:16 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2494762506' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-17T17:15:16.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:16 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:16.660+0000 7f5075c9aec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T17:15:16.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:16 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:16.720+0000 7f5075c9aec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T17:15:17.319 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout 2024-08-17T17:15:17.321 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout { 2024-08-17T17:15:17.321 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "fsid": "25127030-5cbc-11ef-bccf-c7b262605968", 2024-08-17T17:15:17.321 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "health": { 2024-08-17T17:15:17.321 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-08-17T17:15:17.322 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-08-17T17:15:17.322 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-08-17T17:15:17.322 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout }, 2024-08-17T17:15:17.322 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-08-17T17:15:17.322 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-08-17T17:15:17.322 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout 0 2024-08-17T17:15:17.322 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout ], 2024-08-17T17:15:17.322 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-08-17T17:15:17.322 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "a" 2024-08-17T17:15:17.322 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout ], 2024-08-17T17:15:17.322 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-08-17T17:15:17.322 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "monmap": { 2024-08-17T17:15:17.322 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T17:15:17.323 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-08-17T17:15:17.323 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-08-17T17:15:17.323 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout }, 2024-08-17T17:15:17.323 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-08-17T17:15:17.323 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T17:15:17.323 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-08-17T17:15:17.323 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-08-17T17:15:17.323 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-08-17T17:15:17.326 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-08-17T17:15:17.327 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-08-17T17:15:17.327 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-08-17T17:15:17.327 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout }, 2024-08-17T17:15:17.327 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-08-17T17:15:17.327 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-08-17T17:15:17.327 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-08-17T17:15:17.327 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-08-17T17:15:17.327 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-08-17T17:15:17.327 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-08-17T17:15:17.327 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-08-17T17:15:17.327 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-08-17T17:15:17.327 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-08-17T17:15:17.328 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout }, 2024-08-17T17:15:17.328 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-08-17T17:15:17.328 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T17:15:17.328 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-08-17T17:15:17.328 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-08-17T17:15:17.328 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout }, 2024-08-17T17:15:17.328 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-08-17T17:15:17.328 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "available": false, 2024-08-17T17:15:17.328 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-08-17T17:15:17.328 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "modules": [ 2024-08-17T17:15:17.328 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "iostat", 2024-08-17T17:15:17.328 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "nfs", 2024-08-17T17:15:17.328 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "restful" 2024-08-17T17:15:17.328 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout ], 2024-08-17T17:15:17.329 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T17:15:17.329 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout }, 2024-08-17T17:15:17.329 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-08-17T17:15:17.329 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T17:15:17.329 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "modified": "2024-08-17T17:15:09.488422+0000", 2024-08-17T17:15:17.329 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T17:15:17.329 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout }, 2024-08-17T17:15:17.329 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-08-17T17:15:17.329 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout } 2024-08-17T17:15:17.329 INFO:teuthology.orchestra.run.smithi130.stdout:mgr not available, waiting (1/15)... 2024-08-17T17:15:17.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:17 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:17.635+0000 7f5075c9aec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T17:15:17.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:17 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:17.743+0000 7f5075c9aec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T17:15:17.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:17 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:17.796+0000 7f5075c9aec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T17:15:18.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:17 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:17.980+0000 7f5075c9aec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T17:15:18.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:18 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:18.038+0000 7f5075c9aec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T17:15:18.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:18 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:18.101+0000 7f5075c9aec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T17:15:18.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:18 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:18.220+0000 7f5075c9aec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:15:18.906 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:18 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:18.622+0000 7f5075c9aec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T17:15:19.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:18 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:18.905+0000 7f5075c9aec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T17:15:19.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:18 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:18.969+0000 7f5075c9aec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T17:15:19.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:19.092+0000 7f5075c9aec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T17:15:19.649 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:19.345+0000 7f5075c9aec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T17:15:19.649 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:19.407+0000 7f5075c9aec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T17:15:19.650 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:19.587+0000 7f5075c9aec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T17:15:19.650 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:19.648+0000 7f5075c9aec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T17:15:19.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:19.728+0000 7f5075c9aec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T17:15:19.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:19.784+0000 7f5075c9aec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T17:15:20.666 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:20.296+0000 7f5075c9aec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T17:15:20.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:20.430+0000 7f5075c9aec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T17:15:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-mon[31342]: Activating manager daemon y 2024-08-17T17:15:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-mon[31342]: mgrmap e2: y(active, starting, since 0.00287708s) 2024-08-17T17:15:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-mon[31342]: from='mgr.14100 172.21.15.130:0/3420324697' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:15:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-mon[31342]: from='mgr.14100 172.21.15.130:0/3420324697' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:15:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-mon[31342]: from='mgr.14100 172.21.15.130:0/3420324697' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:15:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-mon[31342]: from='mgr.14100 172.21.15.130:0/3420324697' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:15:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-mon[31342]: from='mgr.14100 172.21.15.130:0/3420324697' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:15:20.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-mon[31342]: Manager daemon y is now available 2024-08-17T17:15:20.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-mon[31342]: from='mgr.14100 172.21.15.130:0/3420324697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:15:20.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-mon[31342]: from='mgr.14100 172.21.15.130:0/3420324697' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:15:20.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-mon[31342]: from='mgr.14100 172.21.15.130:0/3420324697' entity='mgr.y' 2024-08-17T17:15:20.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-mon[31342]: from='mgr.14100 172.21.15.130:0/3420324697' entity='mgr.y' 2024-08-17T17:15:20.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:20 smithi130 ceph-mon[31342]: from='mgr.14100 172.21.15.130:0/3420324697' entity='mgr.y' 2024-08-17T17:15:22.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:22 smithi130 ceph-mon[31342]: mgrmap e3: y(active, since 1.00615s) 2024-08-17T17:15:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:23 smithi130 ceph-mon[31342]: mgrmap e4: y(active, since 2s) 2024-08-17T17:15:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:23 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/994080082' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-17T17:15:28.181 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout 2024-08-17T17:15:28.182 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout { 2024-08-17T17:15:28.182 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "fsid": "25127030-5cbc-11ef-bccf-c7b262605968", 2024-08-17T17:15:28.182 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "health": { 2024-08-17T17:15:28.182 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-08-17T17:15:28.182 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-08-17T17:15:28.182 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-08-17T17:15:28.182 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout }, 2024-08-17T17:15:28.182 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-08-17T17:15:28.182 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-08-17T17:15:28.183 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout 0 2024-08-17T17:15:28.183 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout ], 2024-08-17T17:15:28.183 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-08-17T17:15:28.183 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "a" 2024-08-17T17:15:28.183 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout ], 2024-08-17T17:15:28.183 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "quorum_age": 8, 2024-08-17T17:15:28.183 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "monmap": { 2024-08-17T17:15:28.183 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T17:15:28.183 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-08-17T17:15:28.183 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-08-17T17:15:28.183 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout }, 2024-08-17T17:15:28.184 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-08-17T17:15:28.184 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T17:15:28.184 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-08-17T17:15:28.184 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-08-17T17:15:28.184 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-08-17T17:15:28.184 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-08-17T17:15:28.184 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-08-17T17:15:28.184 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-08-17T17:15:28.184 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout }, 2024-08-17T17:15:28.184 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-08-17T17:15:28.184 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-08-17T17:15:28.184 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-08-17T17:15:28.184 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-08-17T17:15:28.185 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-08-17T17:15:28.185 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-08-17T17:15:28.185 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-08-17T17:15:28.185 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-08-17T17:15:28.185 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-08-17T17:15:28.185 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout }, 2024-08-17T17:15:28.185 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-08-17T17:15:28.185 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T17:15:28.185 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-08-17T17:15:28.185 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-08-17T17:15:28.185 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout }, 2024-08-17T17:15:28.185 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-08-17T17:15:28.185 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-17T17:15:28.185 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-08-17T17:15:28.186 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "modules": [ 2024-08-17T17:15:28.186 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "iostat", 2024-08-17T17:15:28.186 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "nfs", 2024-08-17T17:15:28.186 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "restful" 2024-08-17T17:15:28.186 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout ], 2024-08-17T17:15:28.186 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T17:15:28.186 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout }, 2024-08-17T17:15:28.186 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-08-17T17:15:28.186 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T17:15:28.186 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "modified": "2024-08-17T17:15:09.488422+0000", 2024-08-17T17:15:28.186 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T17:15:28.186 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout }, 2024-08-17T17:15:28.186 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-08-17T17:15:28.187 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout } 2024-08-17T17:15:28.187 INFO:teuthology.orchestra.run.smithi130.stdout:mgr is available 2024-08-17T17:15:29.562 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:29 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3370691541' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-08-17T17:15:29.782 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout 2024-08-17T17:15:29.782 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout [global] 2024-08-17T17:15:29.782 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout fsid = 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:15:29.783 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-08-17T17:15:29.783 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-08-17T17:15:29.783 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-08-17T17:15:29.783 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-08-17T17:15:29.783 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-08-17T17:15:29.783 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout 2024-08-17T17:15:29.783 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout [mgr] 2024-08-17T17:15:29.783 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-08-17T17:15:29.783 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout 2024-08-17T17:15:29.783 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout [osd] 2024-08-17T17:15:29.783 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-08-17T17:15:29.783 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-08-17T17:15:29.783 INFO:teuthology.orchestra.run.smithi130.stdout:Enabling cephadm module... 2024-08-17T17:15:30.778 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:30 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2698938119' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-08-17T17:15:30.778 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:30 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ignoring --setuser ceph since I am not root 2024-08-17T17:15:30.778 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:30 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ignoring --setgroup ceph since I am not root 2024-08-17T17:15:30.778 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:30 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:30.589+0000 7fc0b414bec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T17:15:30.778 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:30 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:30.709+0000 7fc0b414bec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:15:30.778 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:30 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:30.776+0000 7fc0b414bec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T17:15:31.067 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:31.015+0000 7fc0b414bec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T17:15:31.327 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:31.073+0000 7fc0b414bec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T17:15:31.606 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:31 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2698938119' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-08-17T17:15:31.606 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:31 smithi130 ceph-mon[31342]: mgrmap e5: y(active, since 10s) 2024-08-17T17:15:32.350 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:32.035+0000 7fc0b414bec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T17:15:32.350 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:32.178+0000 7fc0b414bec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T17:15:32.350 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:32.234+0000 7fc0b414bec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T17:15:32.352 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout { 2024-08-17T17:15:32.353 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-08-17T17:15:32.353 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-17T17:15:32.354 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-08-17T17:15:32.354 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-08-17T17:15:32.354 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout } 2024-08-17T17:15:32.354 INFO:teuthology.orchestra.run.smithi130.stdout:Waiting for the mgr to restart... 2024-08-17T17:15:32.354 INFO:teuthology.orchestra.run.smithi130.stdout:Waiting for mgr epoch 5... 2024-08-17T17:15:32.610 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:32 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2158230714' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-08-17T17:15:32.610 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:32.434+0000 7fc0b414bec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T17:15:32.610 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:32.497+0000 7fc0b414bec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T17:15:32.610 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:32.563+0000 7fc0b414bec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T17:15:32.877 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:32.712+0000 7fc0b414bec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:15:33.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:33.121+0000 7fc0b414bec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T17:15:33.666 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:33.405+0000 7fc0b414bec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T17:15:33.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:33.468+0000 7fc0b414bec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T17:15:33.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:33.591+0000 7fc0b414bec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T17:15:34.139 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:33.839+0000 7fc0b414bec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T17:15:34.139 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:33.897+0000 7fc0b414bec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T17:15:34.139 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:34.076+0000 7fc0b414bec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T17:15:34.416 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:34.138+0000 7fc0b414bec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T17:15:34.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:34.219+0000 7fc0b414bec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T17:15:34.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:34.275+0000 7fc0b414bec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T17:15:35.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:34.791+0000 7fc0b414bec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T17:15:35.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:34.933+0000 7fc0b414bec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T17:15:35.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: Active manager daemon y restarted 2024-08-17T17:15:35.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: Activating manager daemon y 2024-08-17T17:15:35.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: osdmap e2: 0 total, 0 up, 0 in 2024-08-17T17:15:35.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: mgrmap e6: y(active, starting, since 0.00311523s) 2024-08-17T17:15:35.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:15:35.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:15:35.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:15:35.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:15:35.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:15:35.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: Manager daemon y is now available 2024-08-17T17:15:35.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' 2024-08-17T17:15:35.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' 2024-08-17T17:15:35.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:15:35.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:15:35.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:15:35.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:34 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:15:36.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:35 smithi130 ceph-mon[31342]: Found migration_current of "None". Setting to last migration. 2024-08-17T17:15:36.309 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:35 smithi130 ceph-mon[31342]: mgrmap e7: y(active, since 1.00451s) 2024-08-17T17:15:36.604 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout { 2024-08-17T17:15:36.604 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-08-17T17:15:36.604 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "initialized": true 2024-08-17T17:15:36.604 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout } 2024-08-17T17:15:36.604 INFO:teuthology.orchestra.run.smithi130.stdout:mgr epoch 5 is available 2024-08-17T17:15:36.604 INFO:teuthology.orchestra.run.smithi130.stdout:Setting orchestrator backend to cephadm... 2024-08-17T17:15:37.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:36 smithi130 ceph-mon[31342]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-08-17T17:15:37.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:36 smithi130 ceph-mon[31342]: from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-08-17T17:15:37.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:36.999+0000 7fc0664d2640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:17:15:37] ENGINE Error in HTTPServer.serve 2024-08-17T17:15:37.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: Traceback (most recent call last): 2024-08-17T17:15:37.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:15:37.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: self._connections.run(self.expiration_interval) 2024-08-17T17:15:37.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:15:37.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: self._run(expiration_interval) 2024-08-17T17:15:37.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:15:37.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:15:37.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:15:37.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:15:37.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:15:37.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: s = self.context.wrap_socket( 2024-08-17T17:15:37.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:15:37.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: return self.sslsocket_class._create( 2024-08-17T17:15:37.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:15:37.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: self.do_handshake() 2024-08-17T17:15:37.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:15:37.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: self._sslobj.do_handshake() 2024-08-17T17:15:37.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:15:37.169 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:38.082 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-mon[31342]: [17/Aug/2024:17:15:36] ENGINE Bus STARTING 2024-08-17T17:15:38.082 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-mon[31342]: mgrmap e8: y(active, since 2s) 2024-08-17T17:15:38.082 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-mon[31342]: [17/Aug/2024:17:15:36] ENGINE Serving on https://172.21.15.130:7150 2024-08-17T17:15:38.082 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-mon[31342]: [17/Aug/2024:17:15:36] ENGINE Bus STARTED 2024-08-17T17:15:38.082 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-mon[31342]: [17/Aug/2024:17:15:37] ENGINE Error in HTTPServer.serve 2024-08-17T17:15:38.082 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-08-17T17:15:38.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:15:38.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:15:38.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:15:38.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._run(expiration_interval) 2024-08-17T17:15:38.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:15:38.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:15:38.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:15:38.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:15:38.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:15:38.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: s = self.context.wrap_socket( 2024-08-17T17:15:38.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:15:38.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.sslsocket_class._create( 2024-08-17T17:15:38.083 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:15:38.084 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.do_handshake() 2024-08-17T17:15:38.084 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:15:38.084 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._sslobj.do_handshake() 2024-08-17T17:15:38.084 INFO:journalctl@ceph.mon.a.smithi130.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:15:38.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' 2024-08-17T17:15:38.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:15:38.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-mon[31342]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:15:38.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' 2024-08-17T17:15:38.084 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:37 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:15:39.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:38 smithi130 ceph-mon[31342]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:15:39.049 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout value unchanged 2024-08-17T17:15:39.049 INFO:teuthology.orchestra.run.smithi130.stdout:Generating ssh key... 2024-08-17T17:15:40.119 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: Generating public/private rsa key pair. 2024-08-17T17:15:40.119 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: Your identification has been saved in /tmp/tmp8t04fqoo/key 2024-08-17T17:15:40.120 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: Your public key has been saved in /tmp/tmp8t04fqoo/key.pub 2024-08-17T17:15:40.120 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: The key fingerprint is: 2024-08-17T17:15:40.120 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: SHA256:E3a7FcVECRmWI5qrRNIC/qZ1vc8pDK8Wwbjn5ULWR6E ceph-25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:15:40.120 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: The key's randomart image is: 2024-08-17T17:15:40.120 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: +---[RSA 3072]----+ 2024-08-17T17:15:40.120 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: | +O+. | 2024-08-17T17:15:40.120 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: | . o.+.o | 2024-08-17T17:15:40.120 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: | . . + o+.o.. | 2024-08-17T17:15:40.120 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: | . + =.Eo.. . | 2024-08-17T17:15:40.120 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: | . = +So. . | 2024-08-17T17:15:40.120 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: | = O =..o | 2024-08-17T17:15:40.120 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: | + B O o. | 2024-08-17T17:15:40.120 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: | . = *. . | 2024-08-17T17:15:40.121 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: | ..o o+ | 2024-08-17T17:15:40.121 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: +----[SHA256]-----+ 2024-08-17T17:15:41.007 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:40 smithi130 ceph-mon[31342]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:15:41.007 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:40 smithi130 ceph-mon[31342]: Generating ssh key... 2024-08-17T17:15:41.007 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:40 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' 2024-08-17T17:15:41.007 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:40 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' 2024-08-17T17:15:41.628 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9Zb7fijsNDWl/dIHXz2vFDYEIorqWu77XZuNclA1igvvJKYt9NJtruFhQVj2IobSBgyncPfmiJe2dFYzx8edKdLIPZLPwoVwNIowAuQUP6VlFyht8ogDStpTcMnKdWEbsPfLrMNC610XndQ/7huknRfRg/+FTHTHRX3WlGct3BrbxcdJGqSjAMTlMVcKvFWfnJxHFaEg157UeST+JtU9Bb4EaIBFrh2fr7iA5T1rQn8EZ5PlAZ3PwWq8QdfQR3+CTZCykmkpipiS/+lDuAjbG3ji1RTifodyK5c4mVVStEa++ol3nFP5aS6oZ3Rr1GUSRcqMdx+E2yOKAttFFH5xOTZwN0F/mZyrAmwr+Ke37MnSNdCKUuZa8b/jkcnTCj5EBhqCVA2iZzScunv6fx+839rvusEYF8KL0/8d0TgQWaDpmDd/TUhR9+8/gil1kYQ1Ba+h/Q/jtoKE89nsRJzQ2goa3/mtf9l28/jzp9E4WFl14l21uIv07l5IJibaM5UM= ceph-25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:15:41.629 INFO:teuthology.orchestra.run.smithi130.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-08-17T17:15:41.629 INFO:teuthology.orchestra.run.smithi130.stdout:Adding key to root@localhost authorized_keys... 2024-08-17T17:15:41.629 INFO:teuthology.orchestra.run.smithi130.stdout:Adding host smithi130... 2024-08-17T17:15:41.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:41 smithi130 ceph-mon[31342]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:15:43.028 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:42 smithi130 ceph-mon[31342]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi130", "addr": "172.21.15.130", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:15:43.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:43 smithi130 ceph-mon[31342]: Deploying cephadm binary to smithi130 2024-08-17T17:15:45.537 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout Added host 'smithi130' with addr '172.21.15.130' 2024-08-17T17:15:45.537 INFO:teuthology.orchestra.run.smithi130.stdout:Deploying unmanaged mon service... 2024-08-17T17:15:46.116 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:45 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' 2024-08-17T17:15:46.116 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:45 smithi130 ceph-mon[31342]: Added host smithi130 2024-08-17T17:15:46.116 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:45 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:15:47.371 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:47 smithi130 ceph-mon[31342]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:15:47.371 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:47 smithi130 ceph-mon[31342]: Saving service mon spec with placement count:5 2024-08-17T17:15:47.371 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:47 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' 2024-08-17T17:15:47.373 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-08-17T17:15:47.374 INFO:teuthology.orchestra.run.smithi130.stdout:Deploying unmanaged mgr service... 2024-08-17T17:15:48.996 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:48 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' 2024-08-17T17:15:48.997 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:48 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' 2024-08-17T17:15:48.997 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:48 smithi130 ceph-mon[31342]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:15:48.997 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:48 smithi130 ceph-mon[31342]: Saving service mgr spec with placement count:2 2024-08-17T17:15:48.997 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:48 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' 2024-08-17T17:15:48.999 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-08-17T17:15:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:50 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/954420727' entity='client.admin' 2024-08-17T17:15:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:50 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' 2024-08-17T17:15:52.156 INFO:teuthology.orchestra.run.smithi130.stdout:Enabling the dashboard module... 2024-08-17T17:15:52.665 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:52 smithi130 ceph-mon[31342]: from='mgr.14116 172.21.15.130:0/788955183' entity='mgr.y' 2024-08-17T17:15:52.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:52 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/809507268' entity='client.admin' 2024-08-17T17:15:53.733 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:53 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/769615874' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-08-17T17:15:53.733 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:53 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ignoring --setuser ceph since I am not root 2024-08-17T17:15:53.734 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:53 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ignoring --setgroup ceph since I am not root 2024-08-17T17:15:53.734 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:53 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:53.609+0000 7f34bc122ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T17:15:53.734 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:53 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:53.731+0000 7f34bc122ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:15:54.040 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:53 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:53.797+0000 7f34bc122ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T17:15:54.416 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:54 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:54.038+0000 7f34bc122ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T17:15:54.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:54 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:54.096+0000 7f34bc122ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T17:15:54.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:54 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/769615874' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-08-17T17:15:54.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:54 smithi130 ceph-mon[31342]: mgrmap e9: y(active, since 18s) 2024-08-17T17:15:55.115 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:55 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:55.041+0000 7f34bc122ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T17:15:55.403 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:55 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:55.158+0000 7f34bc122ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T17:15:55.404 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:55 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:55.213+0000 7f34bc122ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T17:15:55.404 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:55 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:55.401+0000 7f34bc122ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T17:15:55.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:55 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2723542325' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-08-17T17:15:55.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:55 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:55.461+0000 7f34bc122ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T17:15:55.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:55 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:55.528+0000 7f34bc122ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T17:15:55.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:55 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:55.650+0000 7f34bc122ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:15:55.785 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout { 2024-08-17T17:15:55.785 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-08-17T17:15:55.786 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-17T17:15:55.786 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-08-17T17:15:55.786 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-08-17T17:15:55.786 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout } 2024-08-17T17:15:55.786 INFO:teuthology.orchestra.run.smithi130.stdout:Waiting for the mgr to restart... 2024-08-17T17:15:55.786 INFO:teuthology.orchestra.run.smithi130.stdout:Waiting for mgr epoch 9... 2024-08-17T17:15:56.374 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:56.082+0000 7f34bc122ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T17:15:56.375 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:56.372+0000 7f34bc122ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T17:15:56.666 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:56.436+0000 7f34bc122ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T17:15:56.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:56.560+0000 7f34bc122ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T17:15:57.112 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:56.811+0000 7f34bc122ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T17:15:57.112 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:56.869+0000 7f34bc122ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T17:15:57.112 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:57.048+0000 7f34bc122ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T17:15:57.416 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:57.110+0000 7f34bc122ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T17:15:57.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:57.190+0000 7f34bc122ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T17:15:57.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:57.245+0000 7f34bc122ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T17:15:58.041 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:57.759+0000 7f34bc122ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T17:15:58.041 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:57.891+0000 7f34bc122ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T17:15:58.041 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-mon[31342]: Active manager daemon y restarted 2024-08-17T17:15:58.041 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-mon[31342]: Activating manager daemon y 2024-08-17T17:15:58.041 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-mon[31342]: osdmap e3: 0 total, 0 up, 0 in 2024-08-17T17:15:58.041 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-mon[31342]: mgrmap e10: y(active, starting, since 0.00312759s) 2024-08-17T17:15:58.041 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:15:58.042 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:15:58.042 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:15:58.042 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:15:58.042 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:15:58.042 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-mon[31342]: Manager daemon y is now available 2024-08-17T17:15:58.042 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:15:58.042 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:15:58.042 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:57 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:15:59.750 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:59.469+0000 7f346cc58640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:17:15:59] ENGINE Error in HTTPServer.serve 2024-08-17T17:15:59.751 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: Traceback (most recent call last): 2024-08-17T17:15:59.751 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:15:59.751 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: self._connections.run(self.expiration_interval) 2024-08-17T17:15:59.751 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:15:59.751 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: self._run(expiration_interval) 2024-08-17T17:15:59.751 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:15:59.751 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:15:59.751 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:15:59.751 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:15:59.752 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:15:59.752 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: s = self.context.wrap_socket( 2024-08-17T17:15:59.752 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:15:59.752 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: return self.sslsocket_class._create( 2024-08-17T17:15:59.752 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:15:59.752 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: self.do_handshake() 2024-08-17T17:15:59.752 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:15:59.752 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: self._sslobj.do_handshake() 2024-08-17T17:15:59.753 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:15:59.753 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:15:59.754 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout { 2024-08-17T17:15:59.754 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-08-17T17:15:59.754 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout "initialized": true 2024-08-17T17:15:59.754 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout } 2024-08-17T17:15:59.754 INFO:teuthology.orchestra.run.smithi130.stdout:mgr epoch 9 is available 2024-08-17T17:15:59.754 INFO:teuthology.orchestra.run.smithi130.stdout:Generating a dashboard self-signed certificate... 2024-08-17T17:16:00.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-mon[31342]: mgrmap e11: y(active, since 1.00598s) 2024-08-17T17:16:00.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-mon[31342]: [17/Aug/2024:17:15:59] ENGINE Bus STARTING 2024-08-17T17:16:00.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-mon[31342]: [17/Aug/2024:17:15:59] ENGINE Serving on https://172.21.15.130:7150 2024-08-17T17:16:00.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-mon[31342]: [17/Aug/2024:17:15:59] ENGINE Bus STARTED 2024-08-17T17:16:00.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-mon[31342]: [17/Aug/2024:17:15:59] ENGINE Error in HTTPServer.serve 2024-08-17T17:16:00.018 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-08-17T17:16:00.018 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:16:00.019 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:16:00.019 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:16:00.019 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._run(expiration_interval) 2024-08-17T17:16:00.019 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:16:00.019 INFO:journalctl@ceph.mon.a.smithi130.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:16:00.019 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:16:00.019 INFO:journalctl@ceph.mon.a.smithi130.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:16:00.019 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:16:00.019 INFO:journalctl@ceph.mon.a.smithi130.stdout: s = self.context.wrap_socket( 2024-08-17T17:16:00.019 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:16:00.019 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.sslsocket_class._create( 2024-08-17T17:16:00.019 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:16:00.019 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.do_handshake() 2024-08-17T17:16:00.020 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:16:00.020 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._sslobj.do_handshake() 2024-08-17T17:16:00.020 INFO:journalctl@ceph.mon.a.smithi130.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:16:00.020 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:15:59 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:01.143 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-08-17T17:16:01.143 INFO:teuthology.orchestra.run.smithi130.stdout:Creating initial admin user... 2024-08-17T17:16:01.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:01 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:01.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:01 smithi130 ceph-mon[31342]: from='client.14160 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:16:01.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:01 smithi130 ceph-mon[31342]: mgrmap e12: y(active, since 2s) 2024-08-17T17:16:01.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:01 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:01.143 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:01 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:01.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:01 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:02.887 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$VPQfk1h8pib6Yax4l9fl2e/PCF5/5Zh0l2A/yJPDOqKShmi53Ay5i", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1723914962, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-08-17T17:16:02.887 INFO:teuthology.orchestra.run.smithi130.stdout:Fetching dashboard port number... 2024-08-17T17:16:03.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:03 smithi130 ceph-mon[31342]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:16:03.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:04.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:04 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2814780936' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-08-17T17:16:04.529 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stdout 8443 2024-08-17T17:16:04.541 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T17:16:04.541 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stdout disabled 2024-08-17T17:16:04.550 INFO:teuthology.orchestra.run.smithi130.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T17:16:04.550 INFO:teuthology.orchestra.run.smithi130.stdout:systemctl: stdout inactive 2024-08-17T17:16:04.550 INFO:teuthology.orchestra.run.smithi130.stdout:firewalld.service is not enabled 2024-08-17T17:16:04.550 INFO:teuthology.orchestra.run.smithi130.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-08-17T17:16:04.553 INFO:teuthology.orchestra.run.smithi130.stdout:Ceph Dashboard is now available at: 2024-08-17T17:16:04.553 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:16:04.553 INFO:teuthology.orchestra.run.smithi130.stdout: URL: https://smithi130.front.sepia.ceph.com:8443/ 2024-08-17T17:16:04.553 INFO:teuthology.orchestra.run.smithi130.stdout: User: admin 2024-08-17T17:16:04.553 INFO:teuthology.orchestra.run.smithi130.stdout: Password: 04cl34kwdn 2024-08-17T17:16:04.553 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:16:04.554 INFO:teuthology.orchestra.run.smithi130.stdout:Saving cluster configuration to /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config directory 2024-08-17T17:16:04.554 INFO:teuthology.orchestra.run.smithi130.stdout:Enabling autotune for osd_memory_target 2024-08-17T17:16:05.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:05 smithi130 ceph-mon[31342]: mgrmap e13: y(active, since 6s) 2024-08-17T17:16:07.032 INFO:teuthology.orchestra.run.smithi130.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-08-17T17:16:07.032 INFO:teuthology.orchestra.run.smithi130.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-08-17T17:16:07.033 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:16:07.033 INFO:teuthology.orchestra.run.smithi130.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:16:07.033 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:16:07.033 INFO:teuthology.orchestra.run.smithi130.stdout:Or, if you are only running a single cluster on this host: 2024-08-17T17:16:07.033 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:16:07.033 INFO:teuthology.orchestra.run.smithi130.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-08-17T17:16:07.033 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:16:07.033 INFO:teuthology.orchestra.run.smithi130.stdout:Please consider enabling telemetry to help improve Ceph: 2024-08-17T17:16:07.033 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:16:07.033 INFO:teuthology.orchestra.run.smithi130.stdout: ceph telemetry on 2024-08-17T17:16:07.033 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:16:07.033 INFO:teuthology.orchestra.run.smithi130.stdout:For more information see: 2024-08-17T17:16:07.034 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:16:07.034 INFO:teuthology.orchestra.run.smithi130.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-08-17T17:16:07.034 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:16:07.034 INFO:teuthology.orchestra.run.smithi130.stdout:Bootstrap complete. 2024-08-17T17:16:07.072 INFO:tasks.cephadm:Fetching config... 2024-08-17T17:16:07.072 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:16:07.073 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-08-17T17:16:07.100 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-08-17T17:16:07.100 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:16:07.100 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-08-17T17:16:07.161 INFO:tasks.cephadm:Fetching mon keyring... 2024-08-17T17:16:07.161 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:16:07.161 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/keyring of=/dev/stdout 2024-08-17T17:16:07.232 INFO:tasks.cephadm:Fetching pub ssh key... 2024-08-17T17:16:07.233 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:16:07.233 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-08-17T17:16:07.290 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-08-17T17:16:07.290 DEBUG:teuthology.orchestra.run.smithi130:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9Zb7fijsNDWl/dIHXz2vFDYEIorqWu77XZuNclA1igvvJKYt9NJtruFhQVj2IobSBgyncPfmiJe2dFYzx8edKdLIPZLPwoVwNIowAuQUP6VlFyht8ogDStpTcMnKdWEbsPfLrMNC610XndQ/7huknRfRg/+FTHTHRX3WlGct3BrbxcdJGqSjAMTlMVcKvFWfnJxHFaEg157UeST+JtU9Bb4EaIBFrh2fr7iA5T1rQn8EZ5PlAZ3PwWq8QdfQR3+CTZCykmkpipiS/+lDuAjbG3ji1RTifodyK5c4mVVStEa++ol3nFP5aS6oZ3Rr1GUSRcqMdx+E2yOKAttFFH5xOTZwN0F/mZyrAmwr+Ke37MnSNdCKUuZa8b/jkcnTCj5EBhqCVA2iZzScunv6fx+839rvusEYF8KL0/8d0TgQWaDpmDd/TUhR9+8/gil1kYQ1Ba+h/Q/jtoKE89nsRJzQ2goa3/mtf9l28/jzp9E4WFl14l21uIv07l5IJibaM5UM= ceph-25127030-5cbc-11ef-bccf-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-08-17T17:16:07.390 INFO:teuthology.orchestra.run.smithi130.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9Zb7fijsNDWl/dIHXz2vFDYEIorqWu77XZuNclA1igvvJKYt9NJtruFhQVj2IobSBgyncPfmiJe2dFYzx8edKdLIPZLPwoVwNIowAuQUP6VlFyht8ogDStpTcMnKdWEbsPfLrMNC610XndQ/7huknRfRg/+FTHTHRX3WlGct3BrbxcdJGqSjAMTlMVcKvFWfnJxHFaEg157UeST+JtU9Bb4EaIBFrh2fr7iA5T1rQn8EZ5PlAZ3PwWq8QdfQR3+CTZCykmkpipiS/+lDuAjbG3ji1RTifodyK5c4mVVStEa++ol3nFP5aS6oZ3Rr1GUSRcqMdx+E2yOKAttFFH5xOTZwN0F/mZyrAmwr+Ke37MnSNdCKUuZa8b/jkcnTCj5EBhqCVA2iZzScunv6fx+839rvusEYF8KL0/8d0TgQWaDpmDd/TUhR9+8/gil1kYQ1Ba+h/Q/jtoKE89nsRJzQ2goa3/mtf9l28/jzp9E4WFl14l21uIv07l5IJibaM5UM= ceph-25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:16:07.402 DEBUG:teuthology.orchestra.run.smithi179:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9Zb7fijsNDWl/dIHXz2vFDYEIorqWu77XZuNclA1igvvJKYt9NJtruFhQVj2IobSBgyncPfmiJe2dFYzx8edKdLIPZLPwoVwNIowAuQUP6VlFyht8ogDStpTcMnKdWEbsPfLrMNC610XndQ/7huknRfRg/+FTHTHRX3WlGct3BrbxcdJGqSjAMTlMVcKvFWfnJxHFaEg157UeST+JtU9Bb4EaIBFrh2fr7iA5T1rQn8EZ5PlAZ3PwWq8QdfQR3+CTZCykmkpipiS/+lDuAjbG3ji1RTifodyK5c4mVVStEa++ol3nFP5aS6oZ3Rr1GUSRcqMdx+E2yOKAttFFH5xOTZwN0F/mZyrAmwr+Ke37MnSNdCKUuZa8b/jkcnTCj5EBhqCVA2iZzScunv6fx+839rvusEYF8KL0/8d0TgQWaDpmDd/TUhR9+8/gil1kYQ1Ba+h/Q/jtoKE89nsRJzQ2goa3/mtf9l28/jzp9E4WFl14l21uIv07l5IJibaM5UM= ceph-25127030-5cbc-11ef-bccf-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-08-17T17:16:07.444 INFO:teuthology.orchestra.run.smithi179.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9Zb7fijsNDWl/dIHXz2vFDYEIorqWu77XZuNclA1igvvJKYt9NJtruFhQVj2IobSBgyncPfmiJe2dFYzx8edKdLIPZLPwoVwNIowAuQUP6VlFyht8ogDStpTcMnKdWEbsPfLrMNC610XndQ/7huknRfRg/+FTHTHRX3WlGct3BrbxcdJGqSjAMTlMVcKvFWfnJxHFaEg157UeST+JtU9Bb4EaIBFrh2fr7iA5T1rQn8EZ5PlAZ3PwWq8QdfQR3+CTZCykmkpipiS/+lDuAjbG3ji1RTifodyK5c4mVVStEa++ol3nFP5aS6oZ3Rr1GUSRcqMdx+E2yOKAttFFH5xOTZwN0F/mZyrAmwr+Ke37MnSNdCKUuZa8b/jkcnTCj5EBhqCVA2iZzScunv6fx+839rvusEYF8KL0/8d0TgQWaDpmDd/TUhR9+8/gil1kYQ1Ba+h/Q/jtoKE89nsRJzQ2goa3/mtf9l28/jzp9E4WFl14l21uIv07l5IJibaM5UM= ceph-25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:16:07.456 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-08-17T17:16:07.608 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:07 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3637324437' entity='client.admin' 2024-08-17T17:16:07.710 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:10.537 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-08-17T17:16:10.537 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-08-17T17:16:10.804 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:10 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3849403737' entity='client.admin' 2024-08-17T17:16:10.973 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:13.368 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:13 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:13.368 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:13 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:13.368 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:13 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi130", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:16:13.368 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:13 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:13.368 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:13 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:13.369 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:13 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:14.117 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi179 2024-08-17T17:16:14.118 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-08-17T17:16:14.118 DEBUG:teuthology.orchestra.run.smithi179:> dd of=/etc/ceph/ceph.conf 2024-08-17T17:16:14.138 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-08-17T17:16:14.139 DEBUG:teuthology.orchestra.run.smithi179:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:16:14.197 INFO:tasks.cephadm:Adding host smithi179 to orchestrator... 2024-08-17T17:16:14.197 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch host add smithi179 2024-08-17T17:16:14.444 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:14 smithi130 ceph-mon[31342]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:16:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:14 smithi130 ceph-mon[31342]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:16:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:15.455 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:15 smithi130 ceph-mon[31342]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:16:15.455 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:15 smithi130 ceph-mon[31342]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:16:15.456 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:15 smithi130 ceph-mon[31342]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:16:16.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:16 smithi130 ceph-mon[31342]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi179", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:16:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:17 smithi130 ceph-mon[31342]: Deploying cephadm binary to smithi179 2024-08-17T17:16:18.854 INFO:teuthology.orchestra.run.smithi130.stdout:Added host 'smithi179' with addr '172.21.15.179' 2024-08-17T17:16:19.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:18 smithi130 ceph-mon[31342]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:19.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:18 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:19.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:18 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:19.437 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch host ls --format=json 2024-08-17T17:16:19.682 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:20.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:20 smithi130 ceph-mon[31342]: Added host smithi179 2024-08-17T17:16:20.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:20 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:20.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:20 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:21.231 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:21 smithi130 ceph-mon[31342]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:21.231 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:21 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:21.476 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:16:21.476 INFO:teuthology.orchestra.run.smithi130.stdout:[{"addr": "172.21.15.130", "hostname": "smithi130", "labels": [], "status": ""}, {"addr": "172.21.15.179", "hostname": "smithi179", "labels": [], "status": ""}] 2024-08-17T17:16:22.075 INFO:tasks.cephadm:Setting crush tunables to default 2024-08-17T17:16:22.076 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd crush tunables default 2024-08-17T17:16:22.322 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:22.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:22 smithi130 ceph-mon[31342]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:16:23.507 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:23 smithi130 ceph-mon[31342]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:23.507 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:23.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:23.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:23.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:23.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi179", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:16:23.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:23.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:24.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:24 smithi130 ceph-mon[31342]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:16:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:24 smithi130 ceph-mon[31342]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:16:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:24 smithi130 ceph-mon[31342]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:16:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:24 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:24 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:24 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:24 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3388439489' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-08-17T17:16:24.921 INFO:teuthology.orchestra.run.smithi130.stderr:adjusted tunables profile to default 2024-08-17T17:16:25.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:25 smithi130 ceph-mon[31342]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:16:25.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:25 smithi130 ceph-mon[31342]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:25.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:25 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3388439489' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-08-17T17:16:25.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:25 smithi130 ceph-mon[31342]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:16:25.537 INFO:tasks.cephadm:Adding mon.a on smithi130 2024-08-17T17:16:25.537 INFO:tasks.cephadm:Adding mon.c on smithi130 2024-08-17T17:16:25.537 INFO:tasks.cephadm:Adding mon.b on smithi179 2024-08-17T17:16:25.538 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch apply mon '3;smithi130:172.21.15.130=a;smithi130:[v2:172.21.15.130:3301,v1:172.21.15.130:6790]=c;smithi179:172.21.15.179=b' 2024-08-17T17:16:25.790 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:16:27.478 INFO:teuthology.orchestra.run.smithi179.stdout:Scheduled mon update... 2024-08-17T17:16:27.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:27 smithi130 ceph-mon[31342]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:28.046 DEBUG:teuthology.orchestra.run.smithi130:mon.c> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.c.service 2024-08-17T17:16:28.049 DEBUG:teuthology.orchestra.run.smithi179:mon.b> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.b.service 2024-08-17T17:16:28.051 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-17T17:16:28.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph mon dump -f json 2024-08-17T17:16:28.821 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:16:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:28 smithi130 ceph-mon[31342]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi130:172.21.15.130=a;smithi130:[v2:172.21.15.130:3301,v1:172.21.15.130:6790]=c;smithi179:172.21.15.179=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:16:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:28 smithi130 ceph-mon[31342]: Saving service mon spec with placement smithi130:172.21.15.130=a;smithi130:[v2:172.21.15.130:3301,v1:172.21.15.130:6790]=c;smithi179:172.21.15.179=b;count:3 2024-08-17T17:16:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:28 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:28 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:28 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:28 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:28 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:28 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:28.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:28 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:28.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:28 smithi130 ceph-mon[31342]: Deploying daemon mon.b on smithi179 2024-08-17T17:16:29.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:29 smithi130 ceph-mon[31342]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:31.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:31 smithi130 ceph-mon[31342]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:31 smithi179 systemd[1]: Starting Ceph mon.b for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:16:32.143 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:16:32.143 INFO:teuthology.orchestra.run.smithi179.stdout:{"epoch":1,"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","modified":"2024-08-17T17:15:06.618377Z","created":"2024-08-17T17:15:06.618377Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-08-17T17:16:32.143 INFO:teuthology.orchestra.run.smithi179.stderr:dumped monmap epoch 1 2024-08-17T17:16:32.542 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 podman[34726]: 2024-08-17 17:16:32.044130334 +0000 UTC m=+0.017540473 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:16:32.543 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 podman[34726]: 2024-08-17 17:16:32.143913782 +0000 UTC m=+0.117323912 container create c29492fa52969cad8679f5539b409950bb4ad88fe21ac4a60837d77ef150d1f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-08-17T17:16:32.543 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 podman[34726]: 2024-08-17 17:16:32.539013192 +0000 UTC m=+0.512423333 container init c29492fa52969cad8679f5539b409950bb4ad88fe21ac4a60837d77ef150d1f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-08-17T17:16:32.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 podman[34726]: 2024-08-17 17:16:32.544144155 +0000 UTC m=+0.517554286 container start c29492fa52969cad8679f5539b409950bb4ad88fe21ac4a60837d77ef150d1f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=quincy-f2c39d4, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-08-17T17:16:32.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:16:32.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable), process ceph-mon, pid 7 2024-08-17T17:16:32.801 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: pidfile_write: ignore empty --pid-file 2024-08-17T17:16:32.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: load: jerasure load: lrc 2024-08-17T17:16:32.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: RocksDB version: 6.15.5 2024-08-17T17:16:32.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-08-17T17:16:32.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Compile date Aug 17 2024 2024-08-17T17:16:32.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: DB SUMMARY 2024-08-17T17:16:32.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: DB Session ID: YRZYP9ZC221CDO8RKG50 2024-08-17T17:16:32.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:16:32.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:16:32.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-08-17T17:16:32.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-08-17T17:16:32.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-08-17T17:16:32.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:16:32.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:16:32.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:16:32.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:16:32.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.env: 0x55a0362af480 2024-08-17T17:16:32.803 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.fs: Posix File System 2024-08-17T17:16:32.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.info_log: 0x55a038882f80 2024-08-17T17:16:32.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:16:32.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.statistics: (nil) 2024-08-17T17:16:32.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.use_fsync: 0 2024-08-17T17:16:32.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:16:32.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:16:32.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:16:32.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:16:32.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:16:32.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:16:32.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:16:32.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:16:32.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:16:32.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:16:32.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:16:32.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.db_log_dir: 2024-08-17T17:16:32.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-08-17T17:16:32.806 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:16:32.806 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:16:32.806 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:16:32.806 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:16:32.806 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:16:32.806 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:16:32.806 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:16:32.806 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:16:32.806 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.write_buffer_manager: 0x55a038940720 2024-08-17T17:16:32.806 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:16:32.807 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-08-17T17:16:32.807 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:16:32.807 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:16:32.807 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:16:32.807 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:16:32.807 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:16:32.807 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:16:32.807 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:16:32.807 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.unordered_write: 0 2024-08-17T17:16:32.807 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:16:32.807 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:16:32.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:16:32.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:16:32.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.row_cache: None 2024-08-17T17:16:32.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.wal_filter: None 2024-08-17T17:16:32.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:16:32.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:16:32.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.preserve_deletes: 0 2024-08-17T17:16:32.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:16:32.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:16:32.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:16:32.808 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:16:32.809 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:16:32.809 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:16:32.809 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:16:32.810 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:16:32.810 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:16:32.810 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:16:32.810 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:16:32.811 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:16:32.811 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:16:32.811 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:16:32.811 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:16:32.811 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:16:32.811 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:16:32.811 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:16:32.811 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:16:32.811 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:16:32.811 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:16:32.811 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:16:32.811 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:16:32.812 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:16:32.812 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_open_files: -1 2024-08-17T17:16:32.812 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:16:32.812 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:16:32.812 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:16:32.812 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:16:32.812 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:16:32.812 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Compression algorithms supported: 2024-08-17T17:16:32.812 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: kZSTD supported: 0 2024-08-17T17:16:32.812 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:16:32.812 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:16:32.813 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:16:32.813 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:16:32.813 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:16:32.813 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:16:32.813 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:16:32.813 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:16:32.813 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-08-17T17:16:32.813 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-08-17T17:16:32.813 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:16:32.813 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.merge_operator: 2024-08-17T17:16:32.813 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compaction_filter: None 2024-08-17T17:16:32.813 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:16:32.814 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:16:32.814 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:16:32.814 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:16:32.814 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a038829a78) 2024-08-17T17:16:32.814 INFO:journalctl@ceph.mon.b.smithi179.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:16:32.814 INFO:journalctl@ceph.mon.b.smithi179.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:16:32.814 INFO:journalctl@ceph.mon.b.smithi179.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:16:32.814 INFO:journalctl@ceph.mon.b.smithi179.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:16:32.814 INFO:journalctl@ceph.mon.b.smithi179.stdout: index_type: 0 2024-08-17T17:16:32.814 INFO:journalctl@ceph.mon.b.smithi179.stdout: data_block_index_type: 0 2024-08-17T17:16:32.814 INFO:journalctl@ceph.mon.b.smithi179.stdout: index_shortening: 1 2024-08-17T17:16:32.815 INFO:journalctl@ceph.mon.b.smithi179.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:16:32.815 INFO:journalctl@ceph.mon.b.smithi179.stdout: hash_index_allow_collision: 1 2024-08-17T17:16:32.815 INFO:journalctl@ceph.mon.b.smithi179.stdout: checksum: 1 2024-08-17T17:16:32.815 INFO:journalctl@ceph.mon.b.smithi179.stdout: no_block_cache: 0 2024-08-17T17:16:32.815 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache: 0x55a03885f1f0 2024-08-17T17:16:32.815 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:16:32.815 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache_options: 2024-08-17T17:16:32.815 INFO:journalctl@ceph.mon.b.smithi179.stdout: capacity : 536870912 2024-08-17T17:16:32.815 INFO:journalctl@ceph.mon.b.smithi179.stdout: num_shard_bits : 4 2024-08-17T17:16:32.815 INFO:journalctl@ceph.mon.b.smithi179.stdout: strict_capacity_limit : 0 2024-08-17T17:16:32.815 INFO:journalctl@ceph.mon.b.smithi179.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:16:32.815 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache_compressed: (nil) 2024-08-17T17:16:32.815 INFO:journalctl@ceph.mon.b.smithi179.stdout: persistent_cache: (nil) 2024-08-17T17:16:32.816 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_size: 4096 2024-08-17T17:16:32.816 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_size_deviation: 10 2024-08-17T17:16:32.816 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_restart_interval: 16 2024-08-17T17:16:32.816 INFO:journalctl@ceph.mon.b.smithi179.stdout: index_block_restart_interval: 1 2024-08-17T17:16:32.816 INFO:journalctl@ceph.mon.b.smithi179.stdout: metadata_block_size: 4096 2024-08-17T17:16:32.816 INFO:journalctl@ceph.mon.b.smithi179.stdout: partition_filters: 0 2024-08-17T17:16:32.816 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_delta_encoding: 1 2024-08-17T17:16:32.816 INFO:journalctl@ceph.mon.b.smithi179.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-08-17T17:16:32.816 INFO:journalctl@ceph.mon.b.smithi179.stdout: whole_key_filtering: 1 2024-08-17T17:16:32.816 INFO:journalctl@ceph.mon.b.smithi179.stdout: verify_compression: 0 2024-08-17T17:16:32.816 INFO:journalctl@ceph.mon.b.smithi179.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:16:32.817 INFO:journalctl@ceph.mon.b.smithi179.stdout: format_version: 4 2024-08-17T17:16:32.817 INFO:journalctl@ceph.mon.b.smithi179.stdout: enable_index_compression: 1 2024-08-17T17:16:32.817 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_align: 0 2024-08-17T17:16:32.817 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:16:32.817 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:16:32.817 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compression: NoCompression 2024-08-17T17:16:32.817 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:16:32.817 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:16:32.817 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:16:32.817 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.num_levels: 7 2024-08-17T17:16:32.817 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:16:32.817 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:16:32.818 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:16:32.818 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:16:32.818 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:16:32.818 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:16:32.818 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:16:32.818 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:16:32.818 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:16:32.818 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:16:32.818 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:16:32.818 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:16:32.818 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:16:32.819 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:16:32.819 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:16:32.819 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:16:32.819 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:16:32.819 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:16:32.819 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:16:32.819 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:16:32.819 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:16:32.819 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:16:32.819 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:16:32.819 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:16:32.819 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:16:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:16:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:16:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:16:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:16:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:16:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:16:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:16:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:16:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:16:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.arena_block_size: 4194304 2024-08-17T17:16:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:16:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:16:32.821 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-08-17T17:16:32.821 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:16:32.821 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:16:32.821 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:16:32.821 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:16:32.821 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:16:32.821 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:16:32.821 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:16:32.821 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:16:32.821 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:16:32.821 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:16:32.821 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:16:32.822 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T17:16:32.822 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:16:32.822 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:16:32.822 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:16:32.822 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:16:32.822 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:16:32.822 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:16:32.822 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:16:32.822 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:16:32.822 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:16:32.822 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:16:32.823 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:16:32.823 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.ttl: 2592000 2024-08-17T17:16:32.823 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:16:32.823 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:16:32.823 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:16:32.823 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:16:32.823 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:16:32.823 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:16:32.823 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:16:32.823 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T17:16:32.824 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-08-17T17:16:32.824 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-08-17T17:16:32.824 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914992568325, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-08-17T17:16:32.824 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-08-17T17:16:32.824 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-08-17T17:16:32.824 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914992568626, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1571, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1723914992, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "e7eabaef-696a-42cc-9c46-55891d45f0c8", "db_session_id": "YRZYP9ZC221CDO8RKG50"}} 2024-08-17T17:16:32.824 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-08-17T17:16:32.824 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914992568952, "job": 1, "event": "recovery_finished"} 2024-08-17T17:16:32.824 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:16:32.824 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55a038892700 2024-08-17T17:16:32.824 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: DB pointer 0x55a038862000 2024-08-17T17:16:32.825 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-08-17T17:16:32.825 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-08-17T17:16:32.825 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** DB Stats ** 2024-08-17T17:16:32.825 INFO:journalctl@ceph.mon.b.smithi179.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:16:32.825 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:16:32.825 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:16:32.825 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:16:32.825 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:16:32.825 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-08-17T17:16:32.825 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:16:32.826 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:16:32.826 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** Compaction Stats [default] ** 2024-08-17T17:16:32.826 INFO:journalctl@ceph.mon.b.smithi179.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T17:16:32.826 INFO:journalctl@ceph.mon.b.smithi179.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:16:32.826 INFO:journalctl@ceph.mon.b.smithi179.stdout: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-08-17T17:16:32.827 INFO:journalctl@ceph.mon.b.smithi179.stdout: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-08-17T17:16:32.827 INFO:journalctl@ceph.mon.b.smithi179.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-08-17T17:16:32.827 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:16:32.827 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** Compaction Stats [default] ** 2024-08-17T17:16:32.827 INFO:journalctl@ceph.mon.b.smithi179.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T17:16:32.827 INFO:journalctl@ceph.mon.b.smithi179.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:16:32.827 INFO:journalctl@ceph.mon.b.smithi179.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-08-17T17:16:32.827 INFO:journalctl@ceph.mon.b.smithi179.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:16:32.827 INFO:journalctl@ceph.mon.b.smithi179.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T17:16:32.827 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:16:32.827 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:16:32.827 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:16:32.827 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:16:32.827 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:16:32.828 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:16:32.828 INFO:journalctl@ceph.mon.b.smithi179.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:16:32.828 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:16:32.828 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:16:32.828 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:16:32.828 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** Compaction Stats [default] ** 2024-08-17T17:16:32.828 INFO:journalctl@ceph.mon.b.smithi179.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T17:16:32.828 INFO:journalctl@ceph.mon.b.smithi179.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:16:32.828 INFO:journalctl@ceph.mon.b.smithi179.stdout: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-08-17T17:16:32.828 INFO:journalctl@ceph.mon.b.smithi179.stdout: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-08-17T17:16:32.828 INFO:journalctl@ceph.mon.b.smithi179.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-08-17T17:16:32.828 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:16:32.828 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** Compaction Stats [default] ** 2024-08-17T17:16:32.829 INFO:journalctl@ceph.mon.b.smithi179.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T17:16:32.829 INFO:journalctl@ceph.mon.b.smithi179.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:16:32.829 INFO:journalctl@ceph.mon.b.smithi179.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-08-17T17:16:32.829 INFO:journalctl@ceph.mon.b.smithi179.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:16:32.829 INFO:journalctl@ceph.mon.b.smithi179.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T17:16:32.829 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:16:32.829 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:16:32.829 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:16:32.829 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:16:32.829 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:16:32.829 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:16:32.829 INFO:journalctl@ceph.mon.b.smithi179.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:16:32.830 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:16:32.830 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:16:32.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-08-17T17:16:32.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: using public_addr v2:172.21.15.179:0/0 -> [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] 2024-08-17T17:16:32.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: starting mon.b rank -1 at public addrs [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] at bind addrs [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:16:32.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b@-1(???) e0 preinit fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:16:32.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b@-1(synchronizing).mds e1 new map 2024-08-17T17:16:32.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b@-1(synchronizing).mds e1 print_map 2024-08-17T17:16:32.830 INFO:journalctl@ceph.mon.b.smithi179.stdout: e1 2024-08-17T17:16:32.830 INFO:journalctl@ceph.mon.b.smithi179.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:16:32.830 INFO:journalctl@ceph.mon.b.smithi179.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:16:32.830 INFO:journalctl@ceph.mon.b.smithi179.stdout: legacy client fscid: -1 2024-08-17T17:16:32.830 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:16:32.831 INFO:journalctl@ceph.mon.b.smithi179.stdout: No filesystems configured 2024-08-17T17:16:32.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-08-17T17:16:32.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-08-17T17:16:32.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-08-17T17:16:32.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-08-17T17:16:32.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-08-17T17:16:32.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-08-17T17:16:32.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-08-17T17:16:32.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:16:32.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:16:32.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:16:32.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:16:32.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:32.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3388439489' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-08-17T17:16:32.832 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:16:32.832 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:32.832 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi130:172.21.15.130=a;smithi130:[v2:172.21.15.130:3301,v1:172.21.15.130:6790]=c;smithi179:172.21.15.179=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:16:32.832 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: Saving service mon spec with placement smithi130:172.21.15.130=a;smithi130:[v2:172.21.15.130:3301,v1:172.21.15.130:6790]=c;smithi179:172.21.15.179=b;count:3 2024-08-17T17:16:32.832 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:32.832 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:32.832 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:32.832 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:32.832 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:32.832 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:32.832 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:32.832 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: Deploying daemon mon.b on smithi179 2024-08-17T17:16:32.832 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:32.832 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:32.833 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: from='client.? 172.21.15.179:0/2689909647' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:16:32.833 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 ceph-mon[34775]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-08-17T17:16:32.833 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 bash[34726]: c29492fa52969cad8679f5539b409950bb4ad88fe21ac4a60837d77ef150d1f9 2024-08-17T17:16:32.833 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:32 smithi179 systemd[1]: Started Ceph mon.b for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:16:32.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:32 smithi130 ceph-mon[31342]: from='client.? 172.21.15.179:0/2689909647' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:16:34.210 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-17T17:16:34.211 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph mon dump -f json 2024-08-17T17:16:34.471 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:16:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:16:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: mon.a calling monitor election 2024-08-17T17:16:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: mon.b calling monitor election 2024-08-17T17:16:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:37.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:37.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:37.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-17T17:16:37.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: monmap e2: 2 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T17:16:37.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: fsmap 2024-08-17T17:16:37.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:16:37.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: mgrmap e13: y(active, since 39s) 2024-08-17T17:16:37.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: overall HEALTH_OK 2024-08-17T17:16:37.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:37.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:37.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:37.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:37.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:37 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:16:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: mon.a calling monitor election 2024-08-17T17:16:38.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:38.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:38.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:38.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:38.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: mon.b calling monitor election 2024-08-17T17:16:38.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:38.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:38.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:38.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:38.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-17T17:16:38.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: monmap e2: 2 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T17:16:38.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: fsmap 2024-08-17T17:16:38.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:16:38.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: mgrmap e13: y(active, since 39s) 2024-08-17T17:16:38.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: overall HEALTH_OK 2024-08-17T17:16:38.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:38.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:38.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:38.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:38.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:37 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:39.028 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:38 smithi130 ceph-mon[31342]: Deploying daemon mon.c on smithi130 2024-08-17T17:16:39.028 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:38 smithi130 ceph-mon[31342]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:39.029 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:38 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:38 smithi179 ceph-mon[34775]: Deploying daemon mon.c on smithi130 2024-08-17T17:16:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:38 smithi179 ceph-mon[34775]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:38 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:39.204 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:16:39.205 INFO:teuthology.orchestra.run.smithi179.stdout:{"epoch":2,"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","modified":"2024-08-17T17:16:32.594115Z","created":"2024-08-17T17:15:06.618377Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:3300","nonce":0},{"type":"v1","addr":"172.21.15.179:6789","nonce":0}]},"addr":"172.21.15.179:6789/0","public_addr":"172.21.15.179:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-08-17T17:16:39.205 INFO:teuthology.orchestra.run.smithi179.stderr:dumped monmap epoch 2 2024-08-17T17:16:39.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:39 smithi179 ceph-mon[34775]: from='client.? 172.21.15.179:0/3092316774' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:16:40.093 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:39 smithi130 ceph-mon[31342]: from='client.? 172.21.15.179:0/3092316774' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:16:40.711 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[31342]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:40.829 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-17T17:16:40.829 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph mon dump -f json 2024-08-17T17:16:40.978 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 podman[42711]: 2024-08-17 17:16:40.959587658 +0000 UTC m=+0.396536509 container start 0c43a998b5faa0a92471418f5d859ff0fa86489f8ba3ac635274265e6b51f2fe (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4) 2024-08-17T17:16:41.078 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:16:41.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:40 smithi179 ceph-mon[34775]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:16:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable), process ceph-mon, pid 7 2024-08-17T17:16:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: pidfile_write: ignore empty --pid-file 2024-08-17T17:16:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: load: jerasure load: lrc 2024-08-17T17:16:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: RocksDB version: 6.15.5 2024-08-17T17:16:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-08-17T17:16:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Compile date Aug 17 2024 2024-08-17T17:16:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: DB SUMMARY 2024-08-17T17:16:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: DB Session ID: 4SPDFX2LS1Q9S3VQ1O90 2024-08-17T17:16:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:16:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:16:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-08-17T17:16:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-08-17T17:16:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-08-17T17:16:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:16:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:16:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:16:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:16:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.env: 0x5568a2c0a480 2024-08-17T17:16:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.fs: Posix File System 2024-08-17T17:16:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.info_log: 0x5568a4128f80 2024-08-17T17:16:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.statistics: (nil) 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.use_fsync: 0 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.db_log_dir: 2024-08-17T17:16:41.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-08-17T17:16:41.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:16:41.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:16:41.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:16:41.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:16:41.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:16:41.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:16:41.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:16:41.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:16:41.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.write_buffer_manager: 0x5568a41e6750 2024-08-17T17:16:41.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:16:41.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-08-17T17:16:41.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:16:41.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:16:41.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:16:41.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:16:41.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:16:41.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:16:41.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:16:41.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.unordered_write: 0 2024-08-17T17:16:41.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:16:41.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:16:41.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:16:41.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:16:41.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.row_cache: None 2024-08-17T17:16:41.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.wal_filter: None 2024-08-17T17:16:41.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:16:41.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:16:41.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.preserve_deletes: 0 2024-08-17T17:16:41.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:16:41.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:16:41.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:16:41.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:16:41.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:16:41.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:16:41.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:16:41.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:16:41.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:16:41.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:16:41.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:16:41.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:16:41.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:16:41.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:16:41.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:16:41.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:16:41.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:16:41.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:16:41.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:16:41.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:16:41.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:16:41.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_open_files: -1 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Compression algorithms supported: 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: kZSTD supported: 0 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:16:41.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:16:41.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:16:41.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:16:41.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:16:41.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-08-17T17:16:41.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-08-17T17:16:41.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:16:41.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.merge_operator: 2024-08-17T17:16:41.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compaction_filter: None 2024-08-17T17:16:41.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:16:41.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:16:41.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:16:41.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:16:41.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5568a40cfa78) 2024-08-17T17:16:41.427 INFO:journalctl@ceph.mon.c.smithi130.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:16:41.427 INFO:journalctl@ceph.mon.c.smithi130.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:16:41.427 INFO:journalctl@ceph.mon.c.smithi130.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:16:41.427 INFO:journalctl@ceph.mon.c.smithi130.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:16:41.427 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_type: 0 2024-08-17T17:16:41.427 INFO:journalctl@ceph.mon.c.smithi130.stdout: data_block_index_type: 0 2024-08-17T17:16:41.427 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_shortening: 1 2024-08-17T17:16:41.427 INFO:journalctl@ceph.mon.c.smithi130.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:16:41.427 INFO:journalctl@ceph.mon.c.smithi130.stdout: hash_index_allow_collision: 1 2024-08-17T17:16:41.427 INFO:journalctl@ceph.mon.c.smithi130.stdout: checksum: 1 2024-08-17T17:16:41.427 INFO:journalctl@ceph.mon.c.smithi130.stdout: no_block_cache: 0 2024-08-17T17:16:41.427 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache: 0x5568a41051f0 2024-08-17T17:16:41.427 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:16:41.427 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_options: 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: capacity : 536870912 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: num_shard_bits : 4 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: strict_capacity_limit : 0 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_compressed: (nil) 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: persistent_cache: (nil) 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_size: 4096 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_size_deviation: 10 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_restart_interval: 16 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_block_restart_interval: 1 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: metadata_block_size: 4096 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: partition_filters: 0 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_delta_encoding: 1 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-08-17T17:16:41.428 INFO:journalctl@ceph.mon.c.smithi130.stdout: whole_key_filtering: 1 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: verify_compression: 0 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: format_version: 4 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: enable_index_compression: 1 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_align: 0 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compression: NoCompression 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.num_levels: 7 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:16:41.429 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:16:41.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:16:41.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:16:41.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:16:41.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:16:41.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:16:41.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:16:41.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:16:41.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:16:41.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:16:41.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:16:41.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:16:41.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:16:41.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:16:41.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:16:41.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:16:41.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:16:41.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:16:41.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:16:41.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:16:41.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:16:41.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:16:41.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:16:41.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:16:41.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:16:41.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.arena_block_size: 4194304 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:16:41.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:16:41.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:16:41.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:16:41.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:16:41.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:16:41.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:16:41.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:16:41.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T17:16:41.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:16:41.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:16:41.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:16:41.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:16:41.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:16:41.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:16:41.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.ttl: 2592000 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-08-17T17:16:41.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-08-17T17:16:41.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723915000983010, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-08-17T17:16:41.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-08-17T17:16:41.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-08-17T17:16:41.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723915000983300, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1696, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1723915000, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "bcfc63fa-c974-4628-b7ce-2644f8dc1115", "db_session_id": "4SPDFX2LS1Q9S3VQ1O90"}} 2024-08-17T17:16:41.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-08-17T17:16:41.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723915000983611, "job": 1, "event": "recovery_finished"} 2024-08-17T17:16:41.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:16:41.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5568a4138700 2024-08-17T17:16:41.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: DB pointer 0x5568a4108000 2024-08-17T17:16:41.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-08-17T17:16:41.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-08-17T17:16:41.435 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** DB Stats ** 2024-08-17T17:16:41.435 INFO:journalctl@ceph.mon.c.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:16:41.435 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2024-08-17T17:16:41.436 INFO:journalctl@ceph.mon.c.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:16:41.437 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2024-08-17T17:16:41.438 INFO:journalctl@ceph.mon.c.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T17:16:41.438 INFO:journalctl@ceph.mon.c.smithi130.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:16:41.438 INFO:journalctl@ceph.mon.c.smithi130.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2024-08-17T17:16:41.438 INFO:journalctl@ceph.mon.c.smithi130.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2024-08-17T17:16:41.438 INFO:journalctl@ceph.mon.c.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-08-17T17:16:41.438 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:16:41.438 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2024-08-17T17:16:41.438 INFO:journalctl@ceph.mon.c.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-08-17T17:16:41.438 INFO:journalctl@ceph.mon.c.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:16:41.438 INFO:journalctl@ceph.mon.c.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2024-08-17T17:16:41.438 INFO:journalctl@ceph.mon.c.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:16:41.438 INFO:journalctl@ceph.mon.c.smithi130.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T17:16:41.438 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:16:41.439 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:16:41.439 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:16:41.439 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:16:41.439 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:16:41.439 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:16:41.439 INFO:journalctl@ceph.mon.c.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:16:41.439 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:16:41.439 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:16:41.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-08-17T17:16:41.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: using public_addrv [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] 2024-08-17T17:16:41.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: starting mon.c rank -1 at public addrs [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] at bind addrs [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:16:41.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:40 smithi130 ceph-mon[42749]: mon.c@-1(???) e0 preinit fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:16:41.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.c@-1(synchronizing).mds e1 new map 2024-08-17T17:16:41.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.c@-1(synchronizing).mds e1 print_map 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout: e1 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout: legacy client fscid: -1 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout: No filesystems configured 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:16:41.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:16:41.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T17:16:41.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:16:41.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:41.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3388439489' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-08-17T17:16:41.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:16:41.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:41.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi130:172.21.15.130=a;smithi130:[v2:172.21.15.130:3301,v1:172.21.15.130:6790]=c;smithi179:172.21.15.179=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:16:41.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: Saving service mon spec with placement smithi130:172.21.15.130=a;smithi130:[v2:172.21.15.130:3301,v1:172.21.15.130:6790]=c;smithi179:172.21.15.179=b;count:3 2024-08-17T17:16:41.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:41.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:41.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:41.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: Deploying daemon mon.b on smithi179 2024-08-17T17:16:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='client.? 172.21.15.179:0/2689909647' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:16:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:16:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.a calling monitor election 2024-08-17T17:16:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:41.442 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:41.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.b calling monitor election 2024-08-17T17:16:41.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:41.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:41.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:41.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:41.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-17T17:16:41.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: monmap e2: 2 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T17:16:41.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: fsmap 2024-08-17T17:16:41.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:16:41.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mgrmap e13: y(active, since 39s) 2024-08-17T17:16:41.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: overall HEALTH_OK 2024-08-17T17:16:41.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:41.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:41.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:41.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:41.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:41.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: Deploying daemon mon.c on smithi130 2024-08-17T17:16:41.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:41.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:41.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: from='client.? 172.21.15.179:0/3092316774' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:16:41.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:41.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 ceph-mon[42749]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-08-17T17:16:41.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 bash[42711]: 0c43a998b5faa0a92471418f5d859ff0fa86489f8ba3ac635274265e6b51f2fe 2024-08-17T17:16:41.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:41 smithi130 systemd[1]: Started Ceph mon.c for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:16:46.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:16:46.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: mon.a calling monitor election 2024-08-17T17:16:46.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:46.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: mon.b calling monitor election 2024-08-17T17:16:46.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:46.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: mon.c calling monitor election 2024-08-17T17:16:46.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:46.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:16:46.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: monmap e3: 3 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0],c=[v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T17:16:46.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: fsmap 2024-08-17T17:16:46.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:16:46.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: mgrmap e13: y(active, since 48s) 2024-08-17T17:16:46.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: overall HEALTH_OK 2024-08-17T17:16:46.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:46.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:46.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:46.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:46.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:16:46.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: mon.a calling monitor election 2024-08-17T17:16:46.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:46.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: mon.b calling monitor election 2024-08-17T17:16:46.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:46.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: mon.c calling monitor election 2024-08-17T17:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: monmap e3: 3 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0],c=[v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T17:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: fsmap 2024-08-17T17:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: mgrmap e13: y(active, since 48s) 2024-08-17T17:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: overall HEALTH_OK 2024-08-17T17:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:46.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:46.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:46.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:16:46.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: mon.a calling monitor election 2024-08-17T17:16:46.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:16:46.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: mon.b calling monitor election 2024-08-17T17:16:46.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:46.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: mon.c calling monitor election 2024-08-17T17:16:46.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:46.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:46.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:16:46.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: monmap e3: 3 mons at {a=[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0],b=[v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0],c=[v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T17:16:46.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: fsmap 2024-08-17T17:16:46.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T17:16:46.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: mgrmap e13: y(active, since 48s) 2024-08-17T17:16:46.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: overall HEALTH_OK 2024-08-17T17:16:46.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:46.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:46.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:46.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:46.421 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:16:46.421 INFO:teuthology.orchestra.run.smithi179.stdout:{"epoch":3,"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","modified":"2024-08-17T17:16:41.007482Z","created":"2024-08-17T17:15:06.618377Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3300","nonce":0},{"type":"v1","addr":"172.21.15.130:6789","nonce":0}]},"addr":"172.21.15.130:6789/0","public_addr":"172.21.15.130:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:3300","nonce":0},{"type":"v1","addr":"172.21.15.179:6789","nonce":0}]},"addr":"172.21.15.179:6789/0","public_addr":"172.21.15.179:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:3301","nonce":0},{"type":"v1","addr":"172.21.15.130:6790","nonce":0}]},"addr":"172.21.15.130:6790/0","public_addr":"172.21.15.130:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-08-17T17:16:46.421 INFO:teuthology.orchestra.run.smithi179.stderr:dumped monmap epoch 3 2024-08-17T17:16:47.056 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-08-17T17:16:47.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph config generate-minimal-conf 2024-08-17T17:16:47.321 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:47.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:47 smithi179 ceph-mon[34775]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:47 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:47 smithi179 ceph-mon[34775]: from='client.? 172.21.15.179:0/3227573280' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:16:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:47 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:47.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:47 smithi130 ceph-mon[31342]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:47.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:47 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:47.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:47 smithi130 ceph-mon[31342]: from='client.? 172.21.15.179:0/3227573280' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:16:47.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:47 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:47.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:47 smithi130 ceph-mon[42749]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:47.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:47 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:47.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:47 smithi130 ceph-mon[42749]: from='client.? 172.21.15.179:0/3227573280' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T17:16:47.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:47 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:16:48.527 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.527 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.527 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.527 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.527 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:48.527 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:48.527 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.528 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.528 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.528 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.528 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.528 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:48.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:48.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.529 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.529 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.529 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.529 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:48.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:49.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[31342]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:16:49.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[31342]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:16:49.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[31342]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:49.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[31342]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:16:49.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[31342]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:16:49.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[31342]: Reconfiguring mon.a (unknown last config time)... 2024-08-17T17:16:49.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:49.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:16:49.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:49.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[31342]: Reconfiguring daemon mon.a on smithi130 2024-08-17T17:16:49.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[42749]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:16:49.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[42749]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:16:49.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[42749]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:49.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[42749]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:16:49.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[42749]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:16:49.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[42749]: Reconfiguring mon.a (unknown last config time)... 2024-08-17T17:16:49.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:49.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:16:49.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:49.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:49 smithi130 ceph-mon[42749]: Reconfiguring daemon mon.a on smithi130 2024-08-17T17:16:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:49 smithi179 ceph-mon[34775]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:16:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:49 smithi179 ceph-mon[34775]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:16:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:49 smithi179 ceph-mon[34775]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:49 smithi179 ceph-mon[34775]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:16:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:49 smithi179 ceph-mon[34775]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:16:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:49 smithi179 ceph-mon[34775]: Reconfiguring mon.a (unknown last config time)... 2024-08-17T17:16:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:49 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:49 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:16:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:49 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:49.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:49 smithi179 ceph-mon[34775]: Reconfiguring daemon mon.a on smithi130 2024-08-17T17:16:49.784 INFO:teuthology.orchestra.run.smithi130.stdout:# minimal ceph.conf for 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:16:49.785 INFO:teuthology.orchestra.run.smithi130.stdout:[global] 2024-08-17T17:16:49.785 INFO:teuthology.orchestra.run.smithi130.stdout: fsid = 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:16:49.785 INFO:teuthology.orchestra.run.smithi130.stdout: mon_host = [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] 2024-08-17T17:16:50.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:50 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2396809187' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:50.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:50 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:50.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:50 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:50.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:50 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:50.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:50 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:16:50.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:50 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:50.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:50 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2396809187' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:50.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:50 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:50.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:50 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:50.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:50 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:50.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:50 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:16:50.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:50 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:50 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2396809187' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:50 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:50 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:50 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:50 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:16:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:50 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:50.758 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-08-17T17:16:50.759 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:16:50.759 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.conf 2024-08-17T17:16:50.787 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:16:50.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:16:50.851 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-08-17T17:16:50.851 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd of=/etc/ceph/ceph.conf 2024-08-17T17:16:50.884 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-08-17T17:16:50.884 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:16:50.952 INFO:tasks.cephadm:Adding mgr.y on smithi130 2024-08-17T17:16:50.953 INFO:tasks.cephadm:Adding mgr.x on smithi179 2024-08-17T17:16:50.953 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch apply mgr '2;smithi130=y;smithi179=x' 2024-08-17T17:16:51.238 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:16:51.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:51 smithi179 ceph-mon[34775]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:51.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:51 smithi179 ceph-mon[34775]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:16:51.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:51 smithi179 ceph-mon[34775]: Reconfiguring daemon mon.c on smithi130 2024-08-17T17:16:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:51 smithi130 ceph-mon[31342]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:51 smithi130 ceph-mon[31342]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:16:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:51 smithi130 ceph-mon[31342]: Reconfiguring daemon mon.c on smithi130 2024-08-17T17:16:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:51 smithi130 ceph-mon[42749]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:51 smithi130 ceph-mon[42749]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:16:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:51 smithi130 ceph-mon[42749]: Reconfiguring daemon mon.c on smithi130 2024-08-17T17:16:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:53 smithi179 ceph-mon[34775]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:53 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:53 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:53 smithi179 ceph-mon[34775]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:16:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:53 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:53 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:16:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:53 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:53 smithi179 ceph-mon[34775]: Reconfiguring daemon mon.b on smithi179 2024-08-17T17:16:53.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[31342]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:53.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:53.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:53.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[31342]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:16:53.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:53.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:16:53.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:53.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[31342]: Reconfiguring daemon mon.b on smithi179 2024-08-17T17:16:53.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[42749]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:53.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:53.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:53.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[42749]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:16:53.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:16:53.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:16:53.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:53.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:53 smithi130 ceph-mon[42749]: Reconfiguring daemon mon.b on smithi179 2024-08-17T17:16:53.884 INFO:teuthology.orchestra.run.smithi179.stdout:Scheduled mgr update... 2024-08-17T17:16:54.471 DEBUG:teuthology.orchestra.run.smithi179:mgr.x> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.x.service 2024-08-17T17:16:54.473 DEBUG:tasks.cephadm:set 0 configs 2024-08-17T17:16:54.474 INFO:tasks.cephadm:Deploying OSDs... 2024-08-17T17:16:54.474 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:16:54.474 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/scratch_devs of=/dev/stdout 2024-08-17T17:16:54.493 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-17T17:16:54.493 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_1 2024-08-17T17:16:54.552 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-17T17:16:54.552 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:16:54.552 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 972 Links: 1 2024-08-17T17:16:54.552 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:16:54.552 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:16:54.552 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-08-17 17:16:40.398084827 +0000 2024-08-17T17:16:54.553 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-08-17 17:12:13.121723545 +0000 2024-08-17T17:16:54.553 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-08-17 17:12:13.121723545 +0000 2024-08-17T17:16:54.553 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-08-17 17:12:13.121723545 +0000 2024-08-17T17:16:54.553 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-17T17:16:54.620 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-08-17T17:16:54.621 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-08-17T17:16:54.621 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000222752 s, 2.3 MB/s 2024-08-17T17:16:54.622 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-17T17:16:54.680 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_2 2024-08-17T17:16:54.737 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-17T17:16:54.737 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:16:54.737 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 981 Links: 1 2024-08-17T17:16:54.737 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:16:54.737 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:16:54.737 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-08-17 17:16:40.398084827 +0000 2024-08-17T17:16:54.737 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-08-17 17:12:13.122723528 +0000 2024-08-17T17:16:54.737 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-08-17 17:12:13.122723528 +0000 2024-08-17T17:16:54.738 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-08-17 17:12:13.122723528 +0000 2024-08-17T17:16:54.738 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-17T17:16:54.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:54.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:54.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:54.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi130=y;smithi179=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:16:54.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: Saving service mgr spec with placement smithi130=y;smithi179=x;count:2 2024-08-17T17:16:54.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.779 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:54.780 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:54.780 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:54.780 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.780 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:16:54.780 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-17T17:16:54.780 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:16:54.780 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:54.780 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: Deploying daemon mgr.x on smithi179 2024-08-17T17:16:54.780 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:54 smithi179 ceph-mon[34775]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:54.803 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-08-17T17:16:54.803 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-08-17T17:16:54.803 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000222189 s, 2.3 MB/s 2024-08-17T17:16:54.804 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-17T17:16:54.861 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_3 2024-08-17T17:16:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi130=y;smithi179=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:16:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: Saving service mgr spec with placement smithi130=y;smithi179=x;count:2 2024-08-17T17:16:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:54.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:16:54.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-17T17:16:54.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:16:54.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:54.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: Deploying daemon mgr.x on smithi179 2024-08-17T17:16:54.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[42749]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:54.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:54.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:54.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:54.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi130=y;smithi179=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:16:54.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: Saving service mgr spec with placement smithi130=y;smithi179=x;count:2 2024-08-17T17:16:54.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:54.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:54.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:54.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:54.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:16:54.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-17T17:16:54.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:16:54.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:54.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: Deploying daemon mgr.x on smithi179 2024-08-17T17:16:54.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:54 smithi130 ceph-mon[31342]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:54.920 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-17T17:16:54.920 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:16:54.920 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 986 Links: 1 2024-08-17T17:16:54.920 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:16:54.920 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:16:54.920 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-08-17 17:16:40.399084809 +0000 2024-08-17T17:16:54.921 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-08-17 17:12:13.123723510 +0000 2024-08-17T17:16:54.921 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-08-17 17:12:13.123723510 +0000 2024-08-17T17:16:54.921 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-08-17 17:12:13.123723510 +0000 2024-08-17T17:16:54.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-17T17:16:54.984 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-08-17T17:16:54.985 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-08-17T17:16:54.985 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.00020033 s, 2.6 MB/s 2024-08-17T17:16:54.986 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-17T17:16:55.043 DEBUG:teuthology.orchestra.run.smithi130:> stat /dev/vg_nvme/lv_4 2024-08-17T17:16:55.099 INFO:teuthology.orchestra.run.smithi130.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-17T17:16:55.099 INFO:teuthology.orchestra.run.smithi130.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:16:55.099 INFO:teuthology.orchestra.run.smithi130.stdout:Device: 5h/5d Inode: 976 Links: 1 2024-08-17T17:16:55.099 INFO:teuthology.orchestra.run.smithi130.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:16:55.099 INFO:teuthology.orchestra.run.smithi130.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:16:55.099 INFO:teuthology.orchestra.run.smithi130.stdout:Access: 2024-08-17 17:16:40.399084809 +0000 2024-08-17T17:16:55.099 INFO:teuthology.orchestra.run.smithi130.stdout:Modify: 2024-08-17 17:12:13.121723545 +0000 2024-08-17T17:16:55.099 INFO:teuthology.orchestra.run.smithi130.stdout:Change: 2024-08-17 17:12:13.121723545 +0000 2024-08-17T17:16:55.099 INFO:teuthology.orchestra.run.smithi130.stdout: Birth: 2024-08-17 17:12:13.121723545 +0000 2024-08-17T17:16:55.099 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-17T17:16:55.162 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records in 2024-08-17T17:16:55.162 INFO:teuthology.orchestra.run.smithi130.stderr:1+0 records out 2024-08-17T17:16:55.162 INFO:teuthology.orchestra.run.smithi130.stderr:512 bytes copied, 0.000189681 s, 2.7 MB/s 2024-08-17T17:16:55.163 DEBUG:teuthology.orchestra.run.smithi130:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-17T17:16:55.219 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-08-17T17:16:55.219 DEBUG:teuthology.orchestra.run.smithi179:> dd if=/scratch_devs of=/dev/stdout 2024-08-17T17:16:55.235 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-17T17:16:55.235 DEBUG:teuthology.orchestra.run.smithi179:> stat /dev/vg_nvme/lv_1 2024-08-17T17:16:55.291 INFO:teuthology.orchestra.run.smithi179.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-17T17:16:55.291 INFO:teuthology.orchestra.run.smithi179.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:16:55.291 INFO:teuthology.orchestra.run.smithi179.stdout:Device: 5h/5d Inode: 984 Links: 1 2024-08-17T17:16:55.291 INFO:teuthology.orchestra.run.smithi179.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:16:55.291 INFO:teuthology.orchestra.run.smithi179.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:16:55.292 INFO:teuthology.orchestra.run.smithi179.stdout:Access: 2024-08-17 17:16:31.837239001 +0000 2024-08-17T17:16:55.292 INFO:teuthology.orchestra.run.smithi179.stdout:Modify: 2024-08-17 17:12:14.672166580 +0000 2024-08-17T17:16:55.292 INFO:teuthology.orchestra.run.smithi179.stdout:Change: 2024-08-17 17:12:14.672166580 +0000 2024-08-17T17:16:55.292 INFO:teuthology.orchestra.run.smithi179.stdout: Birth: 2024-08-17 17:12:14.672166580 +0000 2024-08-17T17:16:55.292 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-17T17:16:55.354 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records in 2024-08-17T17:16:55.355 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records out 2024-08-17T17:16:55.355 INFO:teuthology.orchestra.run.smithi179.stderr:512 bytes copied, 0.000268131 s, 1.9 MB/s 2024-08-17T17:16:55.356 DEBUG:teuthology.orchestra.run.smithi179:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-17T17:16:55.413 DEBUG:teuthology.orchestra.run.smithi179:> stat /dev/vg_nvme/lv_2 2024-08-17T17:16:55.467 INFO:teuthology.orchestra.run.smithi179.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-17T17:16:55.467 INFO:teuthology.orchestra.run.smithi179.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:16:55.468 INFO:teuthology.orchestra.run.smithi179.stdout:Device: 5h/5d Inode: 985 Links: 1 2024-08-17T17:16:55.468 INFO:teuthology.orchestra.run.smithi179.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:16:55.468 INFO:teuthology.orchestra.run.smithi179.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:16:55.468 INFO:teuthology.orchestra.run.smithi179.stdout:Access: 2024-08-17 17:16:31.838238977 +0000 2024-08-17T17:16:55.468 INFO:teuthology.orchestra.run.smithi179.stdout:Modify: 2024-08-17 17:12:14.672166580 +0000 2024-08-17T17:16:55.468 INFO:teuthology.orchestra.run.smithi179.stdout:Change: 2024-08-17 17:12:14.672166580 +0000 2024-08-17T17:16:55.468 INFO:teuthology.orchestra.run.smithi179.stdout: Birth: 2024-08-17 17:12:14.672166580 +0000 2024-08-17T17:16:55.468 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-17T17:16:55.530 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records in 2024-08-17T17:16:55.531 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records out 2024-08-17T17:16:55.531 INFO:teuthology.orchestra.run.smithi179.stderr:512 bytes copied, 0.000194709 s, 2.6 MB/s 2024-08-17T17:16:55.532 DEBUG:teuthology.orchestra.run.smithi179:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-17T17:16:55.587 DEBUG:teuthology.orchestra.run.smithi179:> stat /dev/vg_nvme/lv_3 2024-08-17T17:16:55.643 INFO:teuthology.orchestra.run.smithi179.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-17T17:16:55.643 INFO:teuthology.orchestra.run.smithi179.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:16:55.643 INFO:teuthology.orchestra.run.smithi179.stdout:Device: 5h/5d Inode: 982 Links: 1 2024-08-17T17:16:55.643 INFO:teuthology.orchestra.run.smithi179.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:16:55.644 INFO:teuthology.orchestra.run.smithi179.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:16:55.644 INFO:teuthology.orchestra.run.smithi179.stdout:Access: 2024-08-17 17:16:31.838238977 +0000 2024-08-17T17:16:55.644 INFO:teuthology.orchestra.run.smithi179.stdout:Modify: 2024-08-17 17:12:14.672166580 +0000 2024-08-17T17:16:55.644 INFO:teuthology.orchestra.run.smithi179.stdout:Change: 2024-08-17 17:12:14.672166580 +0000 2024-08-17T17:16:55.644 INFO:teuthology.orchestra.run.smithi179.stdout: Birth: 2024-08-17 17:12:14.672166580 +0000 2024-08-17T17:16:55.644 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-17T17:16:55.795 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records in 2024-08-17T17:16:55.796 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records out 2024-08-17T17:16:55.796 INFO:teuthology.orchestra.run.smithi179.stderr:512 bytes copied, 0.000221947 s, 2.3 MB/s 2024-08-17T17:16:55.797 DEBUG:teuthology.orchestra.run.smithi179:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-17T17:16:55.813 DEBUG:teuthology.orchestra.run.smithi179:> stat /dev/vg_nvme/lv_4 2024-08-17T17:16:55.872 INFO:teuthology.orchestra.run.smithi179.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-17T17:16:55.873 INFO:teuthology.orchestra.run.smithi179.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T17:16:55.873 INFO:teuthology.orchestra.run.smithi179.stdout:Device: 5h/5d Inode: 972 Links: 1 2024-08-17T17:16:55.873 INFO:teuthology.orchestra.run.smithi179.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T17:16:55.873 INFO:teuthology.orchestra.run.smithi179.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T17:16:55.873 INFO:teuthology.orchestra.run.smithi179.stdout:Access: 2024-08-17 17:16:55.749688070 +0000 2024-08-17T17:16:55.873 INFO:teuthology.orchestra.run.smithi179.stdout:Modify: 2024-08-17 17:12:14.671166599 +0000 2024-08-17T17:16:55.873 INFO:teuthology.orchestra.run.smithi179.stdout:Change: 2024-08-17 17:12:14.671166599 +0000 2024-08-17T17:16:55.873 INFO:teuthology.orchestra.run.smithi179.stdout: Birth: 2024-08-17 17:12:14.671166599 +0000 2024-08-17T17:16:55.873 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-17T17:16:55.976 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records in 2024-08-17T17:16:55.976 INFO:teuthology.orchestra.run.smithi179.stderr:1+0 records out 2024-08-17T17:16:55.976 INFO:teuthology.orchestra.run.smithi179.stderr:512 bytes copied, 0.000203397 s, 2.5 MB/s 2024-08-17T17:16:55.977 DEBUG:teuthology.orchestra.run.smithi179:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-17T17:16:55.993 INFO:tasks.cephadm:Deploying osd.0 on smithi130 with /dev/vg_nvme/lv_4... 2024-08-17T17:16:55.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-08-17T17:16:56.204 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:55 smithi179 systemd[1]: Starting Ceph mgr.x for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:16:56.243 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:56.498 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:56 smithi179 podman[36860]: 2024-08-17 17:16:56.203302722 +0000 UTC m=+0.108784276 container create fdc68f217f77de01b2694d983459f44aa8bfeee0b75387aad0b61eeccd569a4d (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:16:56.498 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:56 smithi179 podman[36860]: 2024-08-17 17:16:56.111327939 +0000 UTC m=+0.016809507 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T17:16:56.498 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:56 smithi179 podman[36860]: 2024-08-17 17:16:56.397638799 +0000 UTC m=+0.303120364 container init fdc68f217f77de01b2694d983459f44aa8bfeee0b75387aad0b61eeccd569a4d (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:16:56.498 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:56 smithi179 podman[36860]: 2024-08-17 17:16:56.402451913 +0000 UTC m=+0.307933467 container start fdc68f217f77de01b2694d983459f44aa8bfeee0b75387aad0b61eeccd569a4d (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-08-17T17:16:56.498 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:56 smithi179 ceph-mgr[36898]: -- 172.21.15.179:0/400631539 <== mon.1 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b98efca9c0 con 0x55b98efd6c00 2024-08-17T17:16:56.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:56 smithi179 bash[36860]: fdc68f217f77de01b2694d983459f44aa8bfeee0b75387aad0b61eeccd569a4d 2024-08-17T17:16:56.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:56 smithi179 systemd[1]: Started Ceph mgr.x for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:16:56.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:56.518+0000 7f9dbdc5dec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T17:16:56.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:56.647+0000 7f9dbdc5dec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:16:56.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:56.716+0000 7f9dbdc5dec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T17:16:57.275 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:56.973+0000 7f9dbdc5dec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T17:16:57.275 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:57 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:57.039+0000 7f9dbdc5dec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T17:16:57.275 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:56 smithi179 ceph-mon[34775]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:57.275 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:56 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:57.275 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:56 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:57.275 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:56 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:57.276 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:56 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:57.276 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:56 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:57.336 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:56 smithi130 ceph-mon[42749]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:57.337 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:56 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:57.337 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:56 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:57.337 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:56 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:57.337 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:56 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:57.337 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:56 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:57.337 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:56 smithi130 ceph-mon[31342]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:57.337 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:56 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:57.337 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:56 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:57.337 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:56 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:57.337 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:56 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:57.337 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:56 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:16:58.323 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:57 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:57.976+0000 7f9dbdc5dec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T17:16:58.323 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:58.085+0000 7f9dbdc5dec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T17:16:58.323 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:58.138+0000 7f9dbdc5dec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T17:16:58.323 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:58.322+0000 7f9dbdc5dec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T17:16:58.593 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:58.381+0000 7f9dbdc5dec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T17:16:58.593 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:58.444+0000 7f9dbdc5dec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T17:16:58.593 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:58.563+0000 7f9dbdc5dec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:16:58.649 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:16:58.672 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_4 2024-08-17T17:16:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[31342]: Reconfiguring mgr.y (unknown last config time)... 2024-08-17T17:16:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:16:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:16:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[31342]: Reconfiguring daemon mgr.y on smithi130 2024-08-17T17:16:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[31342]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[42749]: Reconfiguring mgr.y (unknown last config time)... 2024-08-17T17:16:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:16:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:16:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[42749]: Reconfiguring daemon mgr.y on smithi130 2024-08-17T17:16:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[42749]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:58.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:16:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:59.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:58.970+0000 7f9dbdc5dec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T17:16:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:16:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-mon[34775]: Reconfiguring mgr.y (unknown last config time)... 2024-08-17T17:16:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:16:59.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:16:59.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:16:59.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-mon[34775]: Reconfiguring daemon mgr.y on smithi130 2024-08-17T17:16:59.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-mon[34775]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:16:59.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:16:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:16:59.405 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:59.593 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:59 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:59.253+0000 7f9dbdc5dec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T17:16:59.593 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:59 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:59.316+0000 7f9dbdc5dec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T17:16:59.593 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:59 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:59.440+0000 7f9dbdc5dec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T17:16:59.989 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:59 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:59.690+0000 7f9dbdc5dec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T17:16:59.989 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:59 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:59.748+0000 7f9dbdc5dec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T17:16:59.989 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:59 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:59.926+0000 7f9dbdc5dec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T17:16:59.989 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:16:59 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:16:59.988+0000 7f9dbdc5dec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T17:17:00.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:17:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:17:00.068+0000 7f9dbdc5dec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T17:17:00.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:17:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:17:00.124+0000 7f9dbdc5dec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T17:17:01.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:17:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:17:00.641+0000 7f9dbdc5dec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T17:17:01.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:17:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:17:00.772+0000 7f9dbdc5dec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T17:17:01.339 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[31342]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:01.340 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:01.340 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:01.340 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:01.340 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:01.340 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:01.340 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:01.340 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[31342]: Standby manager daemon x started 2024-08-17T17:17:01.340 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/659576245' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:17:01.340 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/659576245' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:17:01.340 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/659576245' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:17:01.341 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/659576245' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:17:01.341 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[42749]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:01.341 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:01.341 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:01.341 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:01.341 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:01.341 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:01.341 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:01.341 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[42749]: Standby manager daemon x started 2024-08-17T17:17:01.341 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/659576245' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:17:01.341 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/659576245' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:17:01.341 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/659576245' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:17:01.342 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:01 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/659576245' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:17:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:01 smithi179 ceph-mon[34775]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:01 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:01 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:01 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:01 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:01 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:01 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:01.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:01 smithi179 ceph-mon[34775]: Standby manager daemon x started 2024-08-17T17:17:01.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:01 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/659576245' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:17:01.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:01 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/659576245' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:17:01.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:01 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/659576245' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:17:01.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:01 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/659576245' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:17:02.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:02 smithi130 ceph-mon[42749]: mgrmap e14: y(active, since 63s), standbys: x 2024-08-17T17:17:02.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:02 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:17:02.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:02 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:17:02.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:02 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:17:02.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:02 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:02.288 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:02 smithi130 ceph-mon[31342]: mgrmap e14: y(active, since 63s), standbys: x 2024-08-17T17:17:02.288 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:02 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:17:02.288 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:02 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:17:02.288 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:02 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:17:02.288 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:02 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:02 smithi179 ceph-mon[34775]: mgrmap e14: y(active, since 63s), standbys: x 2024-08-17T17:17:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:02 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:17:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:02 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:17:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:02 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:17:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:02 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:03.413 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:03 smithi130 ceph-mon[31342]: from='client.14205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:03.413 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:03 smithi130 ceph-mon[31342]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:03.413 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:03 smithi130 ceph-mon[42749]: from='client.14205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:03.413 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:03 smithi130 ceph-mon[42749]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:03.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:03 smithi179 ceph-mon[34775]: from='client.14205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:03.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:03 smithi179 ceph-mon[34775]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:04.571 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:04 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1713248918' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30c31adb-2cda-466b-933e-a8e2cb714fab"}]: dispatch 2024-08-17T17:17:04.571 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:04 smithi130 ceph-mon[42749]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30c31adb-2cda-466b-933e-a8e2cb714fab"}]: dispatch 2024-08-17T17:17:04.572 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:04 smithi130 ceph-mon[42749]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "30c31adb-2cda-466b-933e-a8e2cb714fab"}]': finished 2024-08-17T17:17:04.572 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:04 smithi130 ceph-mon[42749]: osdmap e5: 1 total, 0 up, 1 in 2024-08-17T17:17:04.572 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:04 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:04.572 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:04 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1713248918' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30c31adb-2cda-466b-933e-a8e2cb714fab"}]: dispatch 2024-08-17T17:17:04.572 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:04 smithi130 ceph-mon[31342]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30c31adb-2cda-466b-933e-a8e2cb714fab"}]: dispatch 2024-08-17T17:17:04.572 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:04 smithi130 ceph-mon[31342]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "30c31adb-2cda-466b-933e-a8e2cb714fab"}]': finished 2024-08-17T17:17:04.572 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:04 smithi130 ceph-mon[31342]: osdmap e5: 1 total, 0 up, 1 in 2024-08-17T17:17:04.572 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:04 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:04.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:04 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1713248918' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30c31adb-2cda-466b-933e-a8e2cb714fab"}]: dispatch 2024-08-17T17:17:04.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:04 smithi179 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "30c31adb-2cda-466b-933e-a8e2cb714fab"}]: dispatch 2024-08-17T17:17:04.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:04 smithi179 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "30c31adb-2cda-466b-933e-a8e2cb714fab"}]': finished 2024-08-17T17:17:04.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:04 smithi179 ceph-mon[34775]: osdmap e5: 1 total, 0 up, 1 in 2024-08-17T17:17:04.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:04 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:05 smithi179 ceph-mon[34775]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:05 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/867567628' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:17:05.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:05 smithi130 ceph-mon[42749]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:05.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:05 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/867567628' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:17:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:05 smithi130 ceph-mon[31342]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:05 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/867567628' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:17:07.498 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:07 smithi130 ceph-mon[31342]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:07.499 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:07 smithi130 ceph-mon[42749]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:07.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:07 smithi179 ceph-mon[34775]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:09.540 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:09 smithi130 ceph-mon[31342]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:09.541 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:09 smithi130 ceph-mon[42749]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:09 smithi179 ceph-mon[34775]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:10.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:10 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-17T17:17:10.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:10 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:10.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:10 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-17T17:17:10.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:10 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:10.590 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:10 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-17T17:17:10.590 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:10 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:11.503 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:11 smithi130 ceph-mon[31342]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:11.503 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:11 smithi130 ceph-mon[31342]: Deploying daemon osd.0 on smithi130 2024-08-17T17:17:11.504 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:11 smithi130 ceph-mon[42749]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:11.504 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:11 smithi130 ceph-mon[42749]: Deploying daemon osd.0 on smithi130 2024-08-17T17:17:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:11 smithi179 ceph-mon[34775]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:11 smithi179 ceph-mon[34775]: Deploying daemon osd.0 on smithi130 2024-08-17T17:17:13.401 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:13 smithi130 ceph-mon[31342]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:13.401 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:13 smithi130 ceph-mon[42749]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:13 smithi179 ceph-mon[34775]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:15.487 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:15 smithi130 ceph-mon[31342]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:15.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:15 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:15.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:15 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:15.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:15 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:15.488 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:15 smithi130 ceph-mon[42749]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:15.488 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:15 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:15.488 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:15 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:15.488 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:15 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:15.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:15 smithi179 ceph-mon[34775]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:15.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:15 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:15.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:15 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:15.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:15 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:17.323 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:17 smithi130 ceph-mon[31342]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:17.324 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:17 smithi130 ceph-mon[42749]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:17.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:17 smithi179 ceph-mon[34775]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:18.322 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:18 smithi130 ceph-mon[31342]: from='osd.0 [v2:172.21.15.130:6802/2346469001,v1:172.21.15.130:6803/2346469001]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T17:17:18.322 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:18 smithi130 ceph-mon[31342]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T17:17:18.322 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:18 smithi130 ceph-mon[42749]: from='osd.0 [v2:172.21.15.130:6802/2346469001,v1:172.21.15.130:6803/2346469001]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T17:17:18.322 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:18 smithi130 ceph-mon[42749]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T17:17:18.353 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 0 on host 'smithi130' 2024-08-17T17:17:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:18 smithi179 ceph-mon[34775]: from='osd.0 [v2:172.21.15.130:6802/2346469001,v1:172.21.15.130:6803/2346469001]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T17:17:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:18 smithi179 ceph-mon[34775]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T17:17:18.979 DEBUG:teuthology.orchestra.run.smithi130:osd.0> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.0.service 2024-08-17T17:17:18.982 INFO:tasks.cephadm:Deploying osd.1 on smithi130 with /dev/vg_nvme/lv_3... 2024-08-17T17:17:18.982 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-08-17T17:17:19.312 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:19.540 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[47053]: 2024-08-17T17:17:19.224+0000 7f4008bbd640 -1 osd.0 0 waiting for initial osdmap 2024-08-17T17:17:19.540 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[47053]: 2024-08-17T17:17:19.227+0000 7f40059f8640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:17:19.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[31342]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:19.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[31342]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:17:19.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[31342]: osdmap e6: 1 total, 0 up, 1 in 2024-08-17T17:17:19.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:19.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[31342]: from='osd.0 [v2:172.21.15.130:6802/2346469001,v1:172.21.15.130:6803/2346469001]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:17:19.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[31342]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:17:19.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:19.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:19.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:19.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:19.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[42749]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:19.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[42749]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:17:19.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[42749]: osdmap e6: 1 total, 0 up, 1 in 2024-08-17T17:17:19.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:19.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[42749]: from='osd.0 [v2:172.21.15.130:6802/2346469001,v1:172.21.15.130:6803/2346469001]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:17:19.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[42749]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:17:19.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:19.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:19.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:19.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:19 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:19 smithi179 ceph-mon[34775]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:19 smithi179 ceph-mon[34775]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:17:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:19 smithi179 ceph-mon[34775]: osdmap e6: 1 total, 0 up, 1 in 2024-08-17T17:17:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:19 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:19 smithi179 ceph-mon[34775]: from='osd.0 [v2:172.21.15.130:6802/2346469001,v1:172.21.15.130:6803/2346469001]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:17:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:19 smithi179 ceph-mon[34775]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:17:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:19 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:19 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:19.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:19 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:19.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:19 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:20.460 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:20 smithi130 ceph-mon[31342]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-08-17T17:17:20.460 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:20 smithi130 ceph-mon[31342]: osdmap e7: 1 total, 0 up, 1 in 2024-08-17T17:17:20.460 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:20 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:20.460 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:20 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:20.460 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:20 smithi130 ceph-mon[42749]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-08-17T17:17:20.460 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:20 smithi130 ceph-mon[42749]: osdmap e7: 1 total, 0 up, 1 in 2024-08-17T17:17:20.460 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:20 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:20.460 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:20 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:20 smithi179 ceph-mon[34775]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-08-17T17:17:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:20 smithi179 ceph-mon[34775]: osdmap e7: 1 total, 0 up, 1 in 2024-08-17T17:17:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:20 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:20 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:21.439 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:21 smithi130 ceph-mon[31342]: purged_snaps scrub starts 2024-08-17T17:17:21.439 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:21 smithi130 ceph-mon[31342]: purged_snaps scrub ok 2024-08-17T17:17:21.440 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:21 smithi130 ceph-mon[31342]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:21.440 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:21 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:21.440 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:21 smithi130 ceph-mon[31342]: osd.0 [v2:172.21.15.130:6802/2346469001,v1:172.21.15.130:6803/2346469001] boot 2024-08-17T17:17:21.440 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:21 smithi130 ceph-mon[31342]: osdmap e8: 1 total, 1 up, 1 in 2024-08-17T17:17:21.440 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:21 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:21.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:21 smithi130 ceph-mon[42749]: purged_snaps scrub starts 2024-08-17T17:17:21.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:21 smithi130 ceph-mon[42749]: purged_snaps scrub ok 2024-08-17T17:17:21.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:21 smithi130 ceph-mon[42749]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:21.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:21 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:21.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:21 smithi130 ceph-mon[42749]: osd.0 [v2:172.21.15.130:6802/2346469001,v1:172.21.15.130:6803/2346469001] boot 2024-08-17T17:17:21.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:21 smithi130 ceph-mon[42749]: osdmap e8: 1 total, 1 up, 1 in 2024-08-17T17:17:21.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:21 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:21 smithi179 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T17:17:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:21 smithi179 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T17:17:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:21 smithi179 ceph-mon[34775]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T17:17:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:21 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:21 smithi179 ceph-mon[34775]: osd.0 [v2:172.21.15.130:6802/2346469001,v1:172.21.15.130:6803/2346469001] boot 2024-08-17T17:17:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:21 smithi179 ceph-mon[34775]: osdmap e8: 1 total, 1 up, 1 in 2024-08-17T17:17:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:21 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:17:22.548 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:22 smithi130 ceph-mon[31342]: osdmap e9: 1 total, 1 up, 1 in 2024-08-17T17:17:22.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:22 smithi130 ceph-mon[42749]: osdmap e9: 1 total, 1 up, 1 in 2024-08-17T17:17:22.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:22 smithi179 ceph-mon[34775]: osdmap e9: 1 total, 1 up, 1 in 2024-08-17T17:17:23.588 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[42749]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:23.588 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.588 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.588 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:17:23.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:23.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:23.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:23.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:23.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:23.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[31342]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:23.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:17:23.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:23.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:23.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:23.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:23.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:23.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:23 smithi179 ceph-mon[34775]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:23 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:23 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:23 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:17:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:23 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:23 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:23 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:23 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:23 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:23 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:23.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:23 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:23.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:23 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:23.902 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:17:23.923 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_3 2024-08-17T17:17:24.172 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:24.575 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:24 smithi130 ceph-mon[31342]: Detected new or changed devices on smithi130 2024-08-17T17:17:24.575 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:24 smithi130 ceph-mon[31342]: Adjusting osd_memory_target on smithi130 to 16064M 2024-08-17T17:17:24.575 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:24 smithi130 ceph-mon[42749]: Detected new or changed devices on smithi130 2024-08-17T17:17:24.575 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:24 smithi130 ceph-mon[42749]: Adjusting osd_memory_target on smithi130 to 16064M 2024-08-17T17:17:24.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:24 smithi179 ceph-mon[34775]: Detected new or changed devices on smithi130 2024-08-17T17:17:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:24 smithi179 ceph-mon[34775]: Adjusting osd_memory_target on smithi130 to 16064M 2024-08-17T17:17:25.465 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:25 smithi130 ceph-mon[31342]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:25.465 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:25 smithi130 ceph-mon[42749]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:25 smithi179 ceph-mon[34775]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:26.499 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:26 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:17:26.500 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:26 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:17:26.500 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:26 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:26.500 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:26 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:17:26.500 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:26 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:17:26.500 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:26 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:26 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:17:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:26 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:17:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:26 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:27.489 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:27 smithi130 ceph-mon[31342]: from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:27.489 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:27 smithi130 ceph-mon[31342]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:27.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:27 smithi130 ceph-mon[42749]: from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:27.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:27 smithi130 ceph-mon[42749]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:27.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:27 smithi179 ceph-mon[34775]: from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:27 smithi179 ceph-mon[34775]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:28 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1778441472' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55049bb4-b976-42c6-8a5d-8b3e9ae20988"}]: dispatch 2024-08-17T17:17:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:28 smithi179 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55049bb4-b976-42c6-8a5d-8b3e9ae20988"}]: dispatch 2024-08-17T17:17:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:28 smithi179 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55049bb4-b976-42c6-8a5d-8b3e9ae20988"}]': finished 2024-08-17T17:17:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:28 smithi179 ceph-mon[34775]: osdmap e10: 2 total, 1 up, 2 in 2024-08-17T17:17:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:28 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:28.661 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:28 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1778441472' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55049bb4-b976-42c6-8a5d-8b3e9ae20988"}]: dispatch 2024-08-17T17:17:28.661 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:28 smithi130 ceph-mon[31342]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55049bb4-b976-42c6-8a5d-8b3e9ae20988"}]: dispatch 2024-08-17T17:17:28.661 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:28 smithi130 ceph-mon[31342]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55049bb4-b976-42c6-8a5d-8b3e9ae20988"}]': finished 2024-08-17T17:17:28.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:28 smithi130 ceph-mon[31342]: osdmap e10: 2 total, 1 up, 2 in 2024-08-17T17:17:28.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:28 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:28.662 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:28 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1778441472' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55049bb4-b976-42c6-8a5d-8b3e9ae20988"}]: dispatch 2024-08-17T17:17:28.662 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:28 smithi130 ceph-mon[42749]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55049bb4-b976-42c6-8a5d-8b3e9ae20988"}]: dispatch 2024-08-17T17:17:28.662 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:28 smithi130 ceph-mon[42749]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55049bb4-b976-42c6-8a5d-8b3e9ae20988"}]': finished 2024-08-17T17:17:28.662 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:28 smithi130 ceph-mon[42749]: osdmap e10: 2 total, 1 up, 2 in 2024-08-17T17:17:28.662 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:28 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:29 smithi179 ceph-mon[34775]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:29 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/66141096' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:17:29.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:29 smithi130 ceph-mon[31342]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:29 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/66141096' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:17:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:29 smithi130 ceph-mon[42749]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:29 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/66141096' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:17:31.503 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:31 smithi130 ceph-mon[31342]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:31.503 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:31 smithi130 ceph-mon[42749]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:31 smithi179 ceph-mon[34775]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:33.513 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:33 smithi130 ceph-mon[31342]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:33.514 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:33 smithi130 ceph-mon[42749]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:33.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:33 smithi179 ceph-mon[34775]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:34 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-17T17:17:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:34 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:34 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-17T17:17:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:34 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:34.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:34 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-17T17:17:34.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:34 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:35.471 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:35 smithi130 ceph-mon[31342]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:35.472 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:35 smithi130 ceph-mon[31342]: Deploying daemon osd.1 on smithi130 2024-08-17T17:17:35.472 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:35 smithi130 ceph-mon[42749]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:35.472 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:35 smithi130 ceph-mon[42749]: Deploying daemon osd.1 on smithi130 2024-08-17T17:17:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:35 smithi179 ceph-mon[34775]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:35 smithi179 ceph-mon[34775]: Deploying daemon osd.1 on smithi130 2024-08-17T17:17:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:37 smithi130 ceph-mon[42749]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:37.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:37 smithi130 ceph-mon[31342]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:37 smithi179 ceph-mon[34775]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:39 smithi130 ceph-mon[42749]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:39 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:39 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:39 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:39.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:39 smithi130 ceph-mon[31342]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:39.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:39 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:39.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:39 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:39.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:39 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:39 smithi179 ceph-mon[34775]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:39 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:39 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:39 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:41.467 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:41 smithi130 ceph-mon[31342]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:41.467 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:41 smithi130 ceph-mon[42749]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:41.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:41 smithi179 ceph-mon[34775]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:42.172 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 1 on host 'smithi130' 2024-08-17T17:17:42.733 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:42.733 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:42.733 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:42.733 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:42.733 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:42.733 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[31342]: from='osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T17:17:42.733 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:42.733 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:42.733 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:42.733 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:42.733 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:42.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:42.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:42.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:42.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[42749]: from='osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T17:17:42.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:42.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:42.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:42 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:42.757 DEBUG:teuthology.orchestra.run.smithi130:osd.1> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.1.service 2024-08-17T17:17:42.759 INFO:tasks.cephadm:Deploying osd.2 on smithi130 with /dev/vg_nvme/lv_2... 2024-08-17T17:17:42.759 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-08-17T17:17:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:42 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:42 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:42 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:42 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:42 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:42 smithi179 ceph-mon[34775]: from='osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T17:17:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:42 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:17:42.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:42 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:42.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:42 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:43.603 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:43 smithi130 ceph-mon[31342]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:43 smithi130 ceph-mon[31342]: from='osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:17:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:43 smithi130 ceph-mon[31342]: osdmap e11: 2 total, 1 up, 2 in 2024-08-17T17:17:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:43 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:43 smithi130 ceph-mon[31342]: from='osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:17:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:43 smithi130 ceph-mon[42749]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:43 smithi130 ceph-mon[42749]: from='osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:17:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:43 smithi130 ceph-mon[42749]: osdmap e11: 2 total, 1 up, 2 in 2024-08-17T17:17:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:43 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:43 smithi130 ceph-mon[42749]: from='osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:17:43.668 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:17:43 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1[53354]: 2024-08-17T17:17:43.450+0000 7f14f3dbd640 -1 osd.1 0 waiting for initial osdmap 2024-08-17T17:17:43.668 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:17:43 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1[53354]: 2024-08-17T17:17:43.453+0000 7f14efbf6640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:17:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:43 smithi179 ceph-mon[34775]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:43 smithi179 ceph-mon[34775]: from='osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:17:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:43 smithi179 ceph-mon[34775]: osdmap e11: 2 total, 1 up, 2 in 2024-08-17T17:17:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:43 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:43 smithi179 ceph-mon[34775]: from='osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:17:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:44 smithi130 ceph-mon[31342]: from='osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-08-17T17:17:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:44 smithi130 ceph-mon[31342]: osdmap e12: 2 total, 1 up, 2 in 2024-08-17T17:17:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:44 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:44 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:44 smithi130 ceph-mon[42749]: from='osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-08-17T17:17:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:44 smithi130 ceph-mon[42749]: osdmap e12: 2 total, 1 up, 2 in 2024-08-17T17:17:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:44 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:44 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:44 smithi179 ceph-mon[34775]: from='osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-08-17T17:17:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:44 smithi179 ceph-mon[34775]: osdmap e12: 2 total, 1 up, 2 in 2024-08-17T17:17:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:44 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:44 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:45 smithi179 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T17:17:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:45 smithi179 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T17:17:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:45 smithi179 ceph-mon[34775]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:45 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:45 smithi179 ceph-mon[34775]: osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082] boot 2024-08-17T17:17:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:45 smithi179 ceph-mon[34775]: osdmap e13: 2 total, 2 up, 2 in 2024-08-17T17:17:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:45 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:45.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:45 smithi130 ceph-mon[31342]: purged_snaps scrub starts 2024-08-17T17:17:45.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:45 smithi130 ceph-mon[31342]: purged_snaps scrub ok 2024-08-17T17:17:45.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:45 smithi130 ceph-mon[31342]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:45.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:45 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:45.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:45 smithi130 ceph-mon[31342]: osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082] boot 2024-08-17T17:17:45.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:45 smithi130 ceph-mon[31342]: osdmap e13: 2 total, 2 up, 2 in 2024-08-17T17:17:45.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:45 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:45.870 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:45 smithi130 ceph-mon[42749]: purged_snaps scrub starts 2024-08-17T17:17:45.870 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:45 smithi130 ceph-mon[42749]: purged_snaps scrub ok 2024-08-17T17:17:45.870 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:45 smithi130 ceph-mon[42749]: pgmap v55: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-08-17T17:17:45.871 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:45 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:45.871 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:45 smithi130 ceph-mon[42749]: osd.1 [v2:172.21.15.130:6810/2284482082,v1:172.21.15.130:6811/2284482082] boot 2024-08-17T17:17:45.871 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:45 smithi130 ceph-mon[42749]: osdmap e13: 2 total, 2 up, 2 in 2024-08-17T17:17:45.871 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:45 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:17:46.827 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[31342]: osdmap e14: 2 total, 2 up, 2 in 2024-08-17T17:17:46.827 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:46.827 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:46.827 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:17:46.827 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:17:46.827 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:46.827 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:46.827 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:46.827 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:46.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[42749]: osdmap e14: 2 total, 2 up, 2 in 2024-08-17T17:17:46.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:46.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:46.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:17:46.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:17:46.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:46.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:46.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:46.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:46 smithi179 ceph-mon[34775]: osdmap e14: 2 total, 2 up, 2 in 2024-08-17T17:17:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:17:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:17:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:17:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:17:47.520 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:47 smithi130 ceph-mon[31342]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:47.521 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:47 smithi130 ceph-mon[31342]: Detected new or changed devices on smithi130 2024-08-17T17:17:47.521 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:47 smithi130 ceph-mon[31342]: Adjusting osd_memory_target on smithi130 to 8032M 2024-08-17T17:17:47.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:47 smithi130 ceph-mon[42749]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:47.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:47 smithi130 ceph-mon[42749]: Detected new or changed devices on smithi130 2024-08-17T17:17:47.521 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:47 smithi130 ceph-mon[42749]: Adjusting osd_memory_target on smithi130 to 8032M 2024-08-17T17:17:47.526 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:17:47.545 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_2 2024-08-17T17:17:47.797 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:47 smithi179 ceph-mon[34775]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:47 smithi179 ceph-mon[34775]: Detected new or changed devices on smithi130 2024-08-17T17:17:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:47 smithi179 ceph-mon[34775]: Adjusting osd_memory_target on smithi130 to 8032M 2024-08-17T17:17:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:49 smithi130 ceph-mon[42749]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:49.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:49 smithi130 ceph-mon[31342]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:49 smithi179 ceph-mon[34775]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:50.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:50 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:17:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:50 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:17:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:50 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:50 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:17:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:50 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:17:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:50 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:50 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:17:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:50 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:17:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:50 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:51.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:51 smithi130 ceph-mon[31342]: from='client.24164 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:51 smithi130 ceph-mon[31342]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:51 smithi130 ceph-mon[42749]: from='client.24164 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:51 smithi130 ceph-mon[42749]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:51 smithi179 ceph-mon[34775]: from='client.24164 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:17:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:51 smithi179 ceph-mon[34775]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:52 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2119949573' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f40f088-27f7-4450-8f31-b449711365ab"}]: dispatch 2024-08-17T17:17:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:52 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2119949573' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3f40f088-27f7-4450-8f31-b449711365ab"}]': finished 2024-08-17T17:17:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:52 smithi179 ceph-mon[34775]: osdmap e15: 3 total, 2 up, 3 in 2024-08-17T17:17:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:52 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:17:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:52 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2119949573' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f40f088-27f7-4450-8f31-b449711365ab"}]: dispatch 2024-08-17T17:17:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:52 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2119949573' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3f40f088-27f7-4450-8f31-b449711365ab"}]': finished 2024-08-17T17:17:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:52 smithi130 ceph-mon[31342]: osdmap e15: 3 total, 2 up, 3 in 2024-08-17T17:17:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:52 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:17:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:52 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2119949573' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f40f088-27f7-4450-8f31-b449711365ab"}]: dispatch 2024-08-17T17:17:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:52 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2119949573' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3f40f088-27f7-4450-8f31-b449711365ab"}]': finished 2024-08-17T17:17:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:52 smithi130 ceph-mon[42749]: osdmap e15: 3 total, 2 up, 3 in 2024-08-17T17:17:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:52 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:17:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:53 smithi179 ceph-mon[34775]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:53 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/4270969329' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:17:53.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:53 smithi130 ceph-mon[42749]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:53 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/4270969329' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:17:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:53 smithi130 ceph-mon[31342]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:53 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/4270969329' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:17:55.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:55 smithi130 ceph-mon[31342]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:55 smithi130 ceph-mon[42749]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:55 smithi179 ceph-mon[34775]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:57 smithi130 ceph-mon[31342]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:57.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:57 smithi130 ceph-mon[42749]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:57 smithi179 ceph-mon[34775]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:58.787 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-17T17:17:58.787 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:58.787 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-17T17:17:58.787 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-17T17:17:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:17:59.740 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:59 smithi130 ceph-mon[31342]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:59.740 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:17:59 smithi130 ceph-mon[31342]: Deploying daemon osd.2 on smithi130 2024-08-17T17:17:59.740 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:59 smithi130 ceph-mon[42749]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:59.740 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:17:59 smithi130 ceph-mon[42749]: Deploying daemon osd.2 on smithi130 2024-08-17T17:17:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:59 smithi179 ceph-mon[34775]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:17:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:17:59 smithi179 ceph-mon[34775]: Deploying daemon osd.2 on smithi130 2024-08-17T17:18:01.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:01 smithi130 ceph-mon[31342]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:01.791 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:01 smithi130 ceph-mon[42749]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:01 smithi179 ceph-mon[34775]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:03.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:03 smithi130 ceph-mon[42749]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:03.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:03.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:03.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:03.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:03 smithi130 ceph-mon[31342]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:03.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:03.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:03.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:03 smithi179 ceph-mon[34775]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:05 smithi130 ceph-mon[31342]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:05 smithi130 ceph-mon[42749]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:05 smithi179 ceph-mon[34775]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:06.051 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 2 on host 'smithi130' 2024-08-17T17:18:06.630 DEBUG:teuthology.orchestra.run.smithi130:osd.2> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.2.service 2024-08-17T17:18:06.633 INFO:tasks.cephadm:Deploying osd.3 on smithi130 with /dev/vg_nvme/lv_1... 2024-08-17T17:18:06.633 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-08-17T17:18:06.956 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:07.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:07 smithi130 ceph-mon[42749]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:07.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:07 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:07.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:07 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:07.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:07 smithi130 ceph-mon[42749]: from='osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T17:18:07.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:07 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:07.300 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:07 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:07.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:07 smithi130 ceph-mon[31342]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:07.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:07 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:07.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:07 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:07.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:07 smithi130 ceph-mon[31342]: from='osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T17:18:07.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:07 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:07.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:07 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:07 smithi179 ceph-mon[34775]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:07 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:07 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:07 smithi179 ceph-mon[34775]: from='osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T17:18:07.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:07 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:07.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:07 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:08.275 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:08 smithi130 ceph-mon[31342]: from='osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:18:08.276 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:08 smithi130 ceph-mon[31342]: osdmap e16: 3 total, 2 up, 3 in 2024-08-17T17:18:08.276 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:08 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:18:08.276 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:08 smithi130 ceph-mon[31342]: from='osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:18:08.276 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:08 smithi130 ceph-mon[42749]: from='osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:18:08.276 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:08 smithi130 ceph-mon[42749]: osdmap e16: 3 total, 2 up, 3 in 2024-08-17T17:18:08.276 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:08 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:18:08.276 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:08 smithi130 ceph-mon[42749]: from='osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:18:08.276 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:18:08 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[59596]: 2024-08-17T17:18:08.057+0000 7f0d763ff640 -1 osd.2 0 waiting for initial osdmap 2024-08-17T17:18:08.277 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:18:08 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[59596]: 2024-08-17T17:18:08.061+0000 7f0d6fa33640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:18:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:08 smithi179 ceph-mon[34775]: from='osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:18:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:08 smithi179 ceph-mon[34775]: osdmap e16: 3 total, 2 up, 3 in 2024-08-17T17:18:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:08 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:18:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:08 smithi179 ceph-mon[34775]: from='osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:18:09.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:09 smithi130 ceph-mon[31342]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:09.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:09 smithi130 ceph-mon[31342]: from='osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-08-17T17:18:09.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:09 smithi130 ceph-mon[31342]: osdmap e17: 3 total, 2 up, 3 in 2024-08-17T17:18:09.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:09 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:18:09.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:09 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:18:09.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:09 smithi130 ceph-mon[42749]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:09.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:09 smithi130 ceph-mon[42749]: from='osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-08-17T17:18:09.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:09 smithi130 ceph-mon[42749]: osdmap e17: 3 total, 2 up, 3 in 2024-08-17T17:18:09.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:09 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:18:09.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:09 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:18:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:09 smithi179 ceph-mon[34775]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:09 smithi179 ceph-mon[34775]: from='osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-08-17T17:18:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:09 smithi179 ceph-mon[34775]: osdmap e17: 3 total, 2 up, 3 in 2024-08-17T17:18:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:09 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:18:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:09 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:18:10.465 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:10 smithi130 ceph-mon[31342]: purged_snaps scrub starts 2024-08-17T17:18:10.465 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:10 smithi130 ceph-mon[31342]: purged_snaps scrub ok 2024-08-17T17:18:10.465 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:10 smithi130 ceph-mon[31342]: osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313] boot 2024-08-17T17:18:10.465 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:10 smithi130 ceph-mon[31342]: osdmap e18: 3 total, 3 up, 3 in 2024-08-17T17:18:10.465 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:10 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:18:10.465 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:10 smithi130 ceph-mon[42749]: purged_snaps scrub starts 2024-08-17T17:18:10.465 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:10 smithi130 ceph-mon[42749]: purged_snaps scrub ok 2024-08-17T17:18:10.465 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:10 smithi130 ceph-mon[42749]: osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313] boot 2024-08-17T17:18:10.466 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:10 smithi130 ceph-mon[42749]: osdmap e18: 3 total, 3 up, 3 in 2024-08-17T17:18:10.466 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:10 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:18:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:10 smithi179 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T17:18:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:10 smithi179 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T17:18:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:10 smithi179 ceph-mon[34775]: osd.2 [v2:172.21.15.130:6818/160441313,v1:172.21.15.130:6819/160441313] boot 2024-08-17T17:18:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:10 smithi179 ceph-mon[34775]: osdmap e18: 3 total, 3 up, 3 in 2024-08-17T17:18:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:10 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:18:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: pgmap v74: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-08-17T17:18:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-08-17T17:18:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: osdmap e19: 3 total, 3 up, 3 in 2024-08-17T17:18:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:18:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: Detected new or changed devices on smithi130 2024-08-17T17:18:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:11.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:11.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:11.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: Adjusting osd_memory_target on smithi130 to 5354M 2024-08-17T17:18:11.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:11.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:11.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:11.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:11.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:11.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: pgmap v74: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:11.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-08-17T17:18:11.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-08-17T17:18:11.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: osdmap e19: 3 total, 3 up, 3 in 2024-08-17T17:18:11.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:18:11.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: Detected new or changed devices on smithi130 2024-08-17T17:18:11.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:11.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:11.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:11.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: Adjusting osd_memory_target on smithi130 to 5354M 2024-08-17T17:18:11.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:11.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:11.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:11.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:11.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:11.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: pgmap v74: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-08-17T17:18:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-08-17T17:18:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-08-17T17:18:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: osdmap e19: 3 total, 3 up, 3 in 2024-08-17T17:18:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:18:11.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: Detected new or changed devices on smithi130 2024-08-17T17:18:11.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:11.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:11.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:11.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: Adjusting osd_memory_target on smithi130 to 5354M 2024-08-17T17:18:11.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:11.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:11.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:11.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:11.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:11.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:11.808 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:18:11.828 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi130:vg_nvme/lv_1 2024-08-17T17:18:12.105 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:12.498 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:12 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:18:12.498 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:12 smithi130 ceph-mon[31342]: osdmap e20: 3 total, 3 up, 3 in 2024-08-17T17:18:12.498 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:12 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:18:12.499 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:12 smithi130 ceph-mon[42749]: osdmap e20: 3 total, 3 up, 3 in 2024-08-17T17:18:12.499 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:18:12 smithi130 sudo[63630]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T17:18:12.499 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:18:12 smithi130 sudo[63630]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:18:12.499 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:18:12 smithi130 sudo[63630]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:18:12.499 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:18:12 smithi130 sudo[63630]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:18:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:12 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:18:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:12 smithi179 ceph-mon[34775]: osdmap e20: 3 total, 3 up, 3 in 2024-08-17T17:18:12.762 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:18:12 smithi130 sudo[63669]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T17:18:12.762 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:18:12 smithi130 sudo[63669]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:18:12.762 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:18:12 smithi130 sudo[63669]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:18:12.762 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:18:12 smithi130 sudo[63669]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:18:13.019 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:18:12 smithi130 sudo[63672]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T17:18:13.020 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:18:12 smithi130 sudo[63672]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:18:13.020 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:18:12 smithi130 sudo[63672]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:18:13.020 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:18:12 smithi130 sudo[63672]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:18:13.283 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:13 smithi130 ceph-mon[31342]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:13.283 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:13 smithi130 ceph-mon[31342]: osdmap e21: 3 total, 3 up, 3 in 2024-08-17T17:18:13.283 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:13 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:13.284 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:13 smithi130 ceph-mon[42749]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:13.284 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:13 smithi130 ceph-mon[42749]: osdmap e21: 3 total, 3 up, 3 in 2024-08-17T17:18:13.284 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:13 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:13.284 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63685]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T17:18:13.284 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63685]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:18:13.284 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63685]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:18:13.284 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63685]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:18:13.285 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63708]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T17:18:13.285 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63708]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:18:13.285 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63708]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:18:13.285 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63708]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:18:13.533 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63730]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T17:18:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:13 smithi179 ceph-mon[34775]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:13 smithi179 ceph-mon[34775]: osdmap e21: 3 total, 3 up, 3 in 2024-08-17T17:18:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:13 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:13.791 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63730]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:18:13.791 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63730]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:18:13.791 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63730]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:18:14.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63760]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T17:18:14.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63760]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:18:14.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63760]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:18:14.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:13 smithi130 sudo[63760]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:18:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 sudo[63764]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T17:18:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 sudo[63764]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:18:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 sudo[63764]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:18:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 sudo[63764]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:18:14.550 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 sudo[37568]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T17:18:14.550 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 sudo[37568]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:18:14.550 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 sudo[37568]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:18:14.550 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 sudo[37568]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:18:14.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[42749]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:18:14.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[42749]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:18:14.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:18:14.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:18:14.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:18:14.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:18:14.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:18:14.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:18:14.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:18:14.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:18:14.791 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:14.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[31342]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:18:14.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[31342]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:18:14.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:18:14.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:18:14.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:18:14.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:18:14.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:18:14.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:18:14.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:18:14.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:18:14.792 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 sudo[37571]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T17:18:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 sudo[37571]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:18:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 sudo[37571]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:18:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 sudo[37571]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:18:14.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 ceph-mon[34775]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:18:14.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 ceph-mon[34775]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:18:14.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:18:14.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:18:14.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:18:14.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:18:14.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:18:14.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:18:14.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:18:14.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:18:14.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:15.086 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 sudo[63925]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T17:18:15.086 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 sudo[63925]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:18:15.086 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 sudo[63925]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:18:15.086 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:14 smithi130 sudo[63925]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:18:15.351 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 sudo[63934]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T17:18:15.351 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 sudo[63934]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T17:18:15.351 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 sudo[63934]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T17:18:15.351 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 sudo[63934]: pam_unix(sudo:session): session closed for user root 2024-08-17T17:18:15.661 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[31342]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:15.661 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[31342]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:15.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[31342]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:18:15.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[31342]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:18:15.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[31342]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-17T17:18:15.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:15.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[31342]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:18:15.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:18:15.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:18:15.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:18:15.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[31342]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:18:15.662 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[42749]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:15.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[42749]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:15.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[42749]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:18:15.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[42749]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:18:15.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[42749]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-17T17:18:15.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:15.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[42749]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:18:15.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:18:15.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:18:15.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:18:15.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:15 smithi130 ceph-mon[42749]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:18:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:15 smithi179 ceph-mon[34775]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi130:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:15 smithi179 ceph-mon[34775]: pgmap v79: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:15 smithi179 ceph-mon[34775]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:18:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:15 smithi179 ceph-mon[34775]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:18:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:15 smithi179 ceph-mon[34775]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-17T17:18:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:15 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:15 smithi179 ceph-mon[34775]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T17:18:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:15 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:18:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:15 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:18:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:15 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:18:15.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:15 smithi179 ceph-mon[34775]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T17:18:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:16 smithi179 ceph-mon[34775]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:16 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/45709768' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7684c401-462a-4e35-bb9c-1be23dfe573b"}]: dispatch 2024-08-17T17:18:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:16 smithi179 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7684c401-462a-4e35-bb9c-1be23dfe573b"}]: dispatch 2024-08-17T17:18:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:16 smithi179 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7684c401-462a-4e35-bb9c-1be23dfe573b"}]': finished 2024-08-17T17:18:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:16 smithi179 ceph-mon[34775]: osdmap e22: 4 total, 3 up, 4 in 2024-08-17T17:18:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:16 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:16.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:16 smithi130 ceph-mon[42749]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:16.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:16 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/45709768' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7684c401-462a-4e35-bb9c-1be23dfe573b"}]: dispatch 2024-08-17T17:18:16.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:16 smithi130 ceph-mon[42749]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7684c401-462a-4e35-bb9c-1be23dfe573b"}]: dispatch 2024-08-17T17:18:16.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:16 smithi130 ceph-mon[42749]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7684c401-462a-4e35-bb9c-1be23dfe573b"}]': finished 2024-08-17T17:18:16.874 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:16 smithi130 ceph-mon[42749]: osdmap e22: 4 total, 3 up, 4 in 2024-08-17T17:18:16.874 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:16 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:16.874 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:16 smithi130 ceph-mon[31342]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:16.874 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:16 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/45709768' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7684c401-462a-4e35-bb9c-1be23dfe573b"}]: dispatch 2024-08-17T17:18:16.874 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:16 smithi130 ceph-mon[31342]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7684c401-462a-4e35-bb9c-1be23dfe573b"}]: dispatch 2024-08-17T17:18:16.874 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:16 smithi130 ceph-mon[31342]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7684c401-462a-4e35-bb9c-1be23dfe573b"}]': finished 2024-08-17T17:18:16.874 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:16 smithi130 ceph-mon[31342]: osdmap e22: 4 total, 3 up, 4 in 2024-08-17T17:18:16.874 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:16 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:17 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1320555799' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:18:17.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:17 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1320555799' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:18:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:17 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1320555799' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:18:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:18 smithi179 ceph-mon[34775]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:18.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:18 smithi130 ceph-mon[42749]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:18.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:18 smithi130 ceph-mon[31342]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:21.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:20 smithi130 ceph-mon[31342]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:21.074 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:20 smithi130 ceph-mon[42749]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:20 smithi179 ceph-mon[34775]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:23.075 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:22 smithi130 ceph-mon[42749]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:23.075 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:22 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-17T17:18:23.075 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:22 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:23.075 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:22 smithi130 ceph-mon[42749]: Deploying daemon osd.3 on smithi130 2024-08-17T17:18:23.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:22 smithi130 ceph-mon[31342]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:23.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:22 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-17T17:18:23.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:22 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:23.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:22 smithi130 ceph-mon[31342]: Deploying daemon osd.3 on smithi130 2024-08-17T17:18:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:22 smithi179 ceph-mon[34775]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:22 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-17T17:18:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:22 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:22 smithi179 ceph-mon[34775]: Deploying daemon osd.3 on smithi130 2024-08-17T17:18:25.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:24 smithi130 ceph-mon[31342]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:25.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:24 smithi130 ceph-mon[42749]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:25.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:24 smithi179 ceph-mon[34775]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:27.058 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:26 smithi130 ceph-mon[31342]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:27.058 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:26 smithi130 ceph-mon[42749]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:26 smithi179 ceph-mon[34775]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:28.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:27 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:28.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:27 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:28.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:27 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:28.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:27 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:28.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:27 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:28.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:27 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:27 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:27 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:27 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:29.288 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:29 smithi130 ceph-mon[31342]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:29.289 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:29 smithi130 ceph-mon[42749]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:29.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:29 smithi179 ceph-mon[34775]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:30.332 INFO:teuthology.orchestra.run.smithi130.stdout:Created osd(s) 3 on host 'smithi130' 2024-08-17T17:18:30.995 DEBUG:teuthology.orchestra.run.smithi130:osd.3> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.3.service 2024-08-17T17:18:30.997 INFO:tasks.cephadm:Deploying osd.4 on smithi179 with /dev/vg_nvme/lv_4... 2024-08-17T17:18:30.998 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-08-17T17:18:31.257 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:18:31.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:31 smithi130 ceph-mon[31342]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:31.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:31 smithi130 ceph-mon[31342]: from='osd.3 [v2:172.21.15.130:6826/775049303,v1:172.21.15.130:6827/775049303]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T17:18:31.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:31 smithi130 ceph-mon[31342]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T17:18:31.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:31 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:31.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:31 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:31.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:31 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:31.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:31 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:31.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:31 smithi130 ceph-mon[42749]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:31.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:31 smithi130 ceph-mon[42749]: from='osd.3 [v2:172.21.15.130:6826/775049303,v1:172.21.15.130:6827/775049303]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T17:18:31.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:31 smithi130 ceph-mon[42749]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T17:18:31.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:31 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:31.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:31 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:31.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:31 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:31.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:31 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:31 smithi179 ceph-mon[34775]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:31 smithi179 ceph-mon[34775]: from='osd.3 [v2:172.21.15.130:6826/775049303,v1:172.21.15.130:6827/775049303]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T17:18:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:31 smithi179 ceph-mon[34775]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T17:18:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:31 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:31 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:31 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:31 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:32.308 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:32 smithi179 ceph-mon[34775]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:18:32.309 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:32 smithi179 ceph-mon[34775]: osdmap e23: 4 total, 3 up, 4 in 2024-08-17T17:18:32.309 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:32 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:32.309 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:32 smithi179 ceph-mon[34775]: from='osd.3 [v2:172.21.15.130:6826/775049303,v1:172.21.15.130:6827/775049303]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:18:32.309 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:32 smithi179 ceph-mon[34775]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:18:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:32 smithi130 ceph-mon[42749]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:18:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:32 smithi130 ceph-mon[42749]: osdmap e23: 4 total, 3 up, 4 in 2024-08-17T17:18:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:32 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:32 smithi130 ceph-mon[42749]: from='osd.3 [v2:172.21.15.130:6826/775049303,v1:172.21.15.130:6827/775049303]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:18:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:32 smithi130 ceph-mon[42749]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:18:32.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:32 smithi130 ceph-mon[31342]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:18:32.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:32 smithi130 ceph-mon[31342]: osdmap e23: 4 total, 3 up, 4 in 2024-08-17T17:18:32.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:32 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:32.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:32 smithi130 ceph-mon[31342]: from='osd.3 [v2:172.21.15.130:6826/775049303,v1:172.21.15.130:6827/775049303]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:18:32.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:32 smithi130 ceph-mon[31342]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]: dispatch 2024-08-17T17:18:32.418 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:18:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[65809]: 2024-08-17T17:18:32.209+0000 7f16bee9a640 -1 osd.3 0 waiting for initial osdmap 2024-08-17T17:18:32.418 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:18:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[65809]: 2024-08-17T17:18:32.213+0000 7f16b6ca9640 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:18:33.590 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:33 smithi179 ceph-mon[34775]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:33.590 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:33 smithi179 ceph-mon[34775]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-08-17T17:18:33.590 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:33 smithi179 ceph-mon[34775]: osdmap e24: 4 total, 3 up, 4 in 2024-08-17T17:18:33.591 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:33 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:33.591 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:33 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:33.595 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:18:33.613 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi179:vg_nvme/lv_4 2024-08-17T17:18:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:33 smithi130 ceph-mon[31342]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:33 smithi130 ceph-mon[31342]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-08-17T17:18:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:33 smithi130 ceph-mon[31342]: osdmap e24: 4 total, 3 up, 4 in 2024-08-17T17:18:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:33 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:33 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:33 smithi130 ceph-mon[42749]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:33 smithi130 ceph-mon[42749]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]}]': finished 2024-08-17T17:18:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:33 smithi130 ceph-mon[42749]: osdmap e24: 4 total, 3 up, 4 in 2024-08-17T17:18:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:33 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:33 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:33.860 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:18:34.268 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:34 smithi179 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T17:18:34.268 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:34 smithi179 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T17:18:34.268 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:34 smithi179 ceph-mon[34775]: osd.3 [v2:172.21.15.130:6826/775049303,v1:172.21.15.130:6827/775049303] boot 2024-08-17T17:18:34.268 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:34 smithi179 ceph-mon[34775]: osdmap e25: 4 total, 4 up, 4 in 2024-08-17T17:18:34.268 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:34 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:34.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:34 smithi130 ceph-mon[31342]: purged_snaps scrub starts 2024-08-17T17:18:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:34 smithi130 ceph-mon[31342]: purged_snaps scrub ok 2024-08-17T17:18:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:34 smithi130 ceph-mon[31342]: osd.3 [v2:172.21.15.130:6826/775049303,v1:172.21.15.130:6827/775049303] boot 2024-08-17T17:18:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:34 smithi130 ceph-mon[31342]: osdmap e25: 4 total, 4 up, 4 in 2024-08-17T17:18:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:34 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:34 smithi130 ceph-mon[42749]: purged_snaps scrub starts 2024-08-17T17:18:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:34 smithi130 ceph-mon[42749]: purged_snaps scrub ok 2024-08-17T17:18:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:34 smithi130 ceph-mon[42749]: osd.3 [v2:172.21.15.130:6826/775049303,v1:172.21.15.130:6827/775049303] boot 2024-08-17T17:18:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:34 smithi130 ceph-mon[42749]: osdmap e25: 4 total, 4 up, 4 in 2024-08-17T17:18:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:34 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:18:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: osdmap e26: 4 total, 4 up, 4 in 2024-08-17T17:18:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: Detected new or changed devices on smithi130 2024-08-17T17:18:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:35.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:35.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: Adjusting osd_memory_target on smithi130 to 4016M 2024-08-17T17:18:35.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:35.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:35.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:35.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:35.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:35.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:35.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:35.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:35 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: osdmap e26: 4 total, 4 up, 4 in 2024-08-17T17:18:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: Detected new or changed devices on smithi130 2024-08-17T17:18:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:35.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:35.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: Adjusting osd_memory_target on smithi130 to 4016M 2024-08-17T17:18:35.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:35.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:35.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:35.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:35.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:35.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:35.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:35.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:35.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-08-17T17:18:35.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: osdmap e26: 4 total, 4 up, 4 in 2024-08-17T17:18:35.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: Detected new or changed devices on smithi130 2024-08-17T17:18:35.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:35.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:35.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:35.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:35.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:35.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:35.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: Adjusting osd_memory_target on smithi130 to 4016M 2024-08-17T17:18:35.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:35.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:35.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:35.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:35.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:35.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:35.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:35.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:35 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:36.513 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:36 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:18:36.513 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:36 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:18:36.513 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:36 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:36 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:18:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:36 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:18:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:36 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:36.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:36 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:18:36.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:36 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:18:36.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:36 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:37.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:37 smithi179 ceph-mon[34775]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:37 smithi179 ceph-mon[34775]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:37.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:37 smithi130 ceph-mon[31342]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:37 smithi130 ceph-mon[31342]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:37 smithi130 ceph-mon[42749]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:37 smithi130 ceph-mon[42749]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:38 smithi179 ceph-mon[34775]: from='client.? 172.21.15.179:0/3622981435' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a3a331f-533a-49de-910d-5989f2f84b65"}]: dispatch 2024-08-17T17:18:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:38 smithi179 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a3a331f-533a-49de-910d-5989f2f84b65"}]: dispatch 2024-08-17T17:18:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:38 smithi179 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7a3a331f-533a-49de-910d-5989f2f84b65"}]': finished 2024-08-17T17:18:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:38 smithi179 ceph-mon[34775]: osdmap e27: 5 total, 4 up, 5 in 2024-08-17T17:18:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:38 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:38 smithi179 ceph-mon[34775]: from='client.? 172.21.15.179:0/3688862312' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:18:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:38 smithi130 ceph-mon[31342]: from='client.? 172.21.15.179:0/3622981435' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a3a331f-533a-49de-910d-5989f2f84b65"}]: dispatch 2024-08-17T17:18:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:38 smithi130 ceph-mon[31342]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a3a331f-533a-49de-910d-5989f2f84b65"}]: dispatch 2024-08-17T17:18:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:38 smithi130 ceph-mon[31342]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7a3a331f-533a-49de-910d-5989f2f84b65"}]': finished 2024-08-17T17:18:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:38 smithi130 ceph-mon[31342]: osdmap e27: 5 total, 4 up, 5 in 2024-08-17T17:18:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:38 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:38 smithi130 ceph-mon[31342]: from='client.? 172.21.15.179:0/3688862312' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:18:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:38 smithi130 ceph-mon[42749]: from='client.? 172.21.15.179:0/3622981435' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a3a331f-533a-49de-910d-5989f2f84b65"}]: dispatch 2024-08-17T17:18:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:38 smithi130 ceph-mon[42749]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a3a331f-533a-49de-910d-5989f2f84b65"}]: dispatch 2024-08-17T17:18:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:38 smithi130 ceph-mon[42749]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7a3a331f-533a-49de-910d-5989f2f84b65"}]': finished 2024-08-17T17:18:38.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:38 smithi130 ceph-mon[42749]: osdmap e27: 5 total, 4 up, 5 in 2024-08-17T17:18:38.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:38 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:38.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:38 smithi130 ceph-mon[42749]: from='client.? 172.21.15.179:0/3688862312' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:18:39.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:39 smithi179 ceph-mon[34775]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:39.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:39 smithi130 ceph-mon[31342]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:39 smithi130 ceph-mon[42749]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:41.495 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:41 smithi179 ceph-mon[34775]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:41 smithi130 ceph-mon[31342]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:41 smithi130 ceph-mon[42749]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:43 smithi179 ceph-mon[34775]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:43 smithi130 ceph-mon[31342]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:43 smithi130 ceph-mon[42749]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:44.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:44 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-17T17:18:44.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:44 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:44.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:44 smithi179 ceph-mon[34775]: Deploying daemon osd.4 on smithi179 2024-08-17T17:18:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:44 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-17T17:18:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:44 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:44 smithi130 ceph-mon[31342]: Deploying daemon osd.4 on smithi179 2024-08-17T17:18:44.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:44 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-17T17:18:44.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:44 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:44.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:44 smithi130 ceph-mon[42749]: Deploying daemon osd.4 on smithi179 2024-08-17T17:18:45.271 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:45 smithi179 ceph-mon[34775]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:45.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:45 smithi130 ceph-mon[31342]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:45 smithi130 ceph-mon[42749]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:47.512 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:47 smithi179 ceph-mon[34775]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:47.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:47 smithi130 ceph-mon[31342]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:47 smithi130 ceph-mon[42749]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:48.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:48.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:48.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:48 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:48 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:48.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:48.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:48.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:48 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:49 smithi179 ceph-mon[34775]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:49.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:49 smithi130 ceph-mon[31342]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:49 smithi130 ceph-mon[42749]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:50.942 INFO:teuthology.orchestra.run.smithi179.stdout:Created osd(s) 4 on host 'smithi179' 2024-08-17T17:18:51.525 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:51 smithi179 ceph-mon[34775]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:51.525 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:51 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:51.525 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:51 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:51.525 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:51 smithi179 ceph-mon[34775]: from='osd.4 [v2:172.21.15.179:6800/2107797845,v1:172.21.15.179:6801/2107797845]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T17:18:51.525 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:51 smithi179 ceph-mon[34775]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T17:18:51.549 DEBUG:teuthology.orchestra.run.smithi179:osd.4> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.4.service 2024-08-17T17:18:51.553 INFO:tasks.cephadm:Deploying osd.5 on smithi179 with /dev/vg_nvme/lv_3... 2024-08-17T17:18:51.554 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-08-17T17:18:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:51 smithi130 ceph-mon[31342]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:51 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:51 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:51 smithi130 ceph-mon[31342]: from='osd.4 [v2:172.21.15.179:6800/2107797845,v1:172.21.15.179:6801/2107797845]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T17:18:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:51 smithi130 ceph-mon[31342]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T17:18:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:51 smithi130 ceph-mon[42749]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:51 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:51.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:51 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:51.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:51 smithi130 ceph-mon[42749]: from='osd.4 [v2:172.21.15.179:6800/2107797845,v1:172.21.15.179:6801/2107797845]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T17:18:51.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:51 smithi130 ceph-mon[42749]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T17:18:51.868 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:18:52.883 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:52 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:52.883 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:52 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:52.883 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:52 smithi179 ceph-mon[34775]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:52.883 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:52 smithi179 ceph-mon[34775]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-17T17:18:52.883 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:52 smithi179 ceph-mon[34775]: osdmap e28: 5 total, 4 up, 5 in 2024-08-17T17:18:52.883 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:52 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:52.883 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:52 smithi179 ceph-mon[34775]: from='osd.4 [v2:172.21.15.179:6800/2107797845,v1:172.21.15.179:6801/2107797845]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:18:52.883 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:52 smithi179 ceph-mon[34775]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:18:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[31342]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[31342]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-17T17:18:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[31342]: osdmap e28: 5 total, 4 up, 5 in 2024-08-17T17:18:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[31342]: from='osd.4 [v2:172.21.15.179:6800/2107797845,v1:172.21.15.179:6801/2107797845]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:18:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[31342]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:18:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[42749]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[42749]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-17T17:18:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[42749]: osdmap e28: 5 total, 4 up, 5 in 2024-08-17T17:18:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[42749]: from='osd.4 [v2:172.21.15.179:6800/2107797845,v1:172.21.15.179:6801/2107797845]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:18:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:52 smithi130 ceph-mon[42749]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:18:53.184 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 17:18:52 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4[40234]: 2024-08-17T17:18:52.946+0000 7f543f6eb640 -1 osd.4 0 waiting for initial osdmap 2024-08-17T17:18:53.184 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 17:18:52 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4[40234]: 2024-08-17T17:18:52.950+0000 7f543a522640 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:18:54.013 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:53 smithi179 ceph-mon[34775]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-08-17T17:18:54.013 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:53 smithi179 ceph-mon[34775]: osdmap e29: 5 total, 4 up, 5 in 2024-08-17T17:18:54.013 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:53 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:54.013 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:53 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:54.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:53 smithi130 ceph-mon[31342]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-08-17T17:18:54.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:53 smithi130 ceph-mon[31342]: osdmap e29: 5 total, 4 up, 5 in 2024-08-17T17:18:54.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:53 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:54.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:53 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:54.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:53 smithi130 ceph-mon[42749]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-08-17T17:18:54.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:53 smithi130 ceph-mon[42749]: osdmap e29: 5 total, 4 up, 5 in 2024-08-17T17:18:54.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:53 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:54.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:53 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:54.966 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:18:54.985 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:54 smithi179 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T17:18:54.985 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:54 smithi179 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T17:18:54.985 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:54 smithi179 ceph-mon[34775]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:54.985 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:54 smithi179 ceph-mon[34775]: osd.4 [v2:172.21.15.179:6800/2107797845,v1:172.21.15.179:6801/2107797845] boot 2024-08-17T17:18:54.985 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:54 smithi179 ceph-mon[34775]: osdmap e30: 5 total, 5 up, 5 in 2024-08-17T17:18:54.985 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:54 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:54.985 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:54 smithi179 ceph-mon[34775]: osdmap e31: 5 total, 5 up, 5 in 2024-08-17T17:18:54.986 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi179:vg_nvme/lv_3 2024-08-17T17:18:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:54 smithi130 ceph-mon[31342]: purged_snaps scrub starts 2024-08-17T17:18:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:54 smithi130 ceph-mon[31342]: purged_snaps scrub ok 2024-08-17T17:18:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:54 smithi130 ceph-mon[31342]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:54 smithi130 ceph-mon[31342]: osd.4 [v2:172.21.15.179:6800/2107797845,v1:172.21.15.179:6801/2107797845] boot 2024-08-17T17:18:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:54 smithi130 ceph-mon[31342]: osdmap e30: 5 total, 5 up, 5 in 2024-08-17T17:18:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:54 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:54 smithi130 ceph-mon[31342]: osdmap e31: 5 total, 5 up, 5 in 2024-08-17T17:18:55.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:54 smithi130 ceph-mon[42749]: purged_snaps scrub starts 2024-08-17T17:18:55.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:54 smithi130 ceph-mon[42749]: purged_snaps scrub ok 2024-08-17T17:18:55.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:54 smithi130 ceph-mon[42749]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-08-17T17:18:55.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:54 smithi130 ceph-mon[42749]: osd.4 [v2:172.21.15.179:6800/2107797845,v1:172.21.15.179:6801/2107797845] boot 2024-08-17T17:18:55.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:54 smithi130 ceph-mon[42749]: osdmap e30: 5 total, 5 up, 5 in 2024-08-17T17:18:55.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:54 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:18:55.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:54 smithi130 ceph-mon[42749]: osdmap e31: 5 total, 5 up, 5 in 2024-08-17T17:18:55.522 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:18:56.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:56 smithi179 ceph-mon[34775]: osdmap e32: 5 total, 5 up, 5 in 2024-08-17T17:18:56.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:56 smithi130 ceph-mon[31342]: osdmap e32: 5 total, 5 up, 5 in 2024-08-17T17:18:56.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:56 smithi130 ceph-mon[42749]: osdmap e32: 5 total, 5 up, 5 in 2024-08-17T17:18:57.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:57 smithi130 ceph-mon[31342]: pgmap v111: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 145 KiB/s, 0 objects/s recovering 2024-08-17T17:18:57.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:57 smithi130 ceph-mon[31342]: osdmap e33: 5 total, 5 up, 5 in 2024-08-17T17:18:57.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:57 smithi130 ceph-mon[42749]: pgmap v111: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 145 KiB/s, 0 objects/s recovering 2024-08-17T17:18:57.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:57 smithi130 ceph-mon[42749]: osdmap e33: 5 total, 5 up, 5 in 2024-08-17T17:18:57.485 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:57 smithi179 ceph-mon[34775]: pgmap v111: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 145 KiB/s, 0 objects/s recovering 2024-08-17T17:18:57.485 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:57 smithi179 ceph-mon[34775]: osdmap e33: 5 total, 5 up, 5 in 2024-08-17T17:18:58.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:18:57 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:18:57.909+0000 7f349ae02640 -1 calc_pg_upmaps abort due to max <= 0 2024-08-17T17:18:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: osdmap e34: 5 total, 5 up, 5 in 2024-08-17T17:18:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: Detected new or changed devices on smithi179 2024-08-17T17:18:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: Adjusting osd_memory_target on smithi179 to 17088M 2024-08-17T17:18:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:18:58.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:18:58.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:58.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: osdmap e34: 5 total, 5 up, 5 in 2024-08-17T17:18:58.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: Detected new or changed devices on smithi179 2024-08-17T17:18:58.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:58.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: Adjusting osd_memory_target on smithi179 to 17088M 2024-08-17T17:18:58.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:58.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:58.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:58.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:58.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:58.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:18:58.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:18:58.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:58 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:58.396 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: osdmap e34: 5 total, 5 up, 5 in 2024-08-17T17:18:58.396 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: Detected new or changed devices on smithi179 2024-08-17T17:18:58.396 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.396 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:18:58.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: Adjusting osd_memory_target on smithi179 to 17088M 2024-08-17T17:18:58.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:58.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:58.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:18:58.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:58.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:18:58.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:18:58.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:18:58.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:18:58.398 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:58 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:18:59.269 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:59 smithi179 ceph-mon[34775]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:59.269 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:18:59 smithi179 ceph-mon[34775]: pgmap v114: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 145 KiB/s, 0 objects/s recovering 2024-08-17T17:18:59.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:59 smithi130 ceph-mon[31342]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:59.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:18:59 smithi130 ceph-mon[31342]: pgmap v114: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 145 KiB/s, 0 objects/s recovering 2024-08-17T17:18:59.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:59 smithi130 ceph-mon[42749]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:18:59.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:18:59 smithi130 ceph-mon[42749]: pgmap v114: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 145 KiB/s, 0 objects/s recovering 2024-08-17T17:19:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:01 smithi130 ceph-mon[31342]: pgmap v115: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-08-17T17:19:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:01 smithi130 ceph-mon[31342]: from='client.? 172.21.15.179:0/2077985949' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d53d65ee-bbb2-45ec-b557-07cc62a45b05"}]: dispatch 2024-08-17T17:19:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:01 smithi130 ceph-mon[31342]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d53d65ee-bbb2-45ec-b557-07cc62a45b05"}]: dispatch 2024-08-17T17:19:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:01 smithi130 ceph-mon[31342]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d53d65ee-bbb2-45ec-b557-07cc62a45b05"}]': finished 2024-08-17T17:19:01.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:01 smithi130 ceph-mon[31342]: osdmap e35: 6 total, 5 up, 6 in 2024-08-17T17:19:01.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:01 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:01.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:01 smithi130 ceph-mon[31342]: from='client.? 172.21.15.179:0/3167968250' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:19:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:01 smithi130 ceph-mon[42749]: pgmap v115: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-08-17T17:19:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:01 smithi130 ceph-mon[42749]: from='client.? 172.21.15.179:0/2077985949' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d53d65ee-bbb2-45ec-b557-07cc62a45b05"}]: dispatch 2024-08-17T17:19:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:01 smithi130 ceph-mon[42749]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d53d65ee-bbb2-45ec-b557-07cc62a45b05"}]: dispatch 2024-08-17T17:19:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:01 smithi130 ceph-mon[42749]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d53d65ee-bbb2-45ec-b557-07cc62a45b05"}]': finished 2024-08-17T17:19:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:01 smithi130 ceph-mon[42749]: osdmap e35: 6 total, 5 up, 6 in 2024-08-17T17:19:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:01 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:01 smithi130 ceph-mon[42749]: from='client.? 172.21.15.179:0/3167968250' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:19:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:01 smithi179 ceph-mon[34775]: pgmap v115: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-08-17T17:19:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:01 smithi179 ceph-mon[34775]: from='client.? 172.21.15.179:0/2077985949' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d53d65ee-bbb2-45ec-b557-07cc62a45b05"}]: dispatch 2024-08-17T17:19:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:01 smithi179 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d53d65ee-bbb2-45ec-b557-07cc62a45b05"}]: dispatch 2024-08-17T17:19:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:01 smithi179 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d53d65ee-bbb2-45ec-b557-07cc62a45b05"}]': finished 2024-08-17T17:19:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:01 smithi179 ceph-mon[34775]: osdmap e35: 6 total, 5 up, 6 in 2024-08-17T17:19:01.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:01 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:01.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:01 smithi179 ceph-mon[34775]: from='client.? 172.21.15.179:0/3167968250' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:19:03.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:03 smithi130 ceph-mon[31342]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:03.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:03 smithi130 ceph-mon[42749]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:03.470 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:03 smithi179 ceph-mon[34775]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:05.312 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:05 smithi179 ceph-mon[34775]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:05.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:05 smithi130 ceph-mon[31342]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:05.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:05 smithi130 ceph-mon[42749]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:07.304 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:07 smithi179 ceph-mon[34775]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:07.305 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:07 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-17T17:19:07.305 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:07 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:07.305 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:07 smithi179 ceph-mon[34775]: Deploying daemon osd.5 on smithi179 2024-08-17T17:19:07.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:07 smithi130 ceph-mon[31342]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:07.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:07 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-17T17:19:07.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:07 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:07.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:07 smithi130 ceph-mon[31342]: Deploying daemon osd.5 on smithi179 2024-08-17T17:19:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:07 smithi130 ceph-mon[42749]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:07 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-17T17:19:07.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:07 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:07.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:07 smithi130 ceph-mon[42749]: Deploying daemon osd.5 on smithi179 2024-08-17T17:19:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:09 smithi179 ceph-mon[34775]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:09.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:09 smithi130 ceph-mon[31342]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:09 smithi130 ceph-mon[42749]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:11 smithi130 ceph-mon[31342]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:11 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:11.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:11 smithi130 ceph-mon[42749]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:11.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:11.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:11.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:11 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:11.490 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:11 smithi179 ceph-mon[34775]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:11.490 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:11.490 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:11.490 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:11 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:13.341 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:13 smithi179 ceph-mon[34775]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:13.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:13 smithi130 ceph-mon[31342]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:13 smithi130 ceph-mon[42749]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:13.915 INFO:teuthology.orchestra.run.smithi179.stdout:Created osd(s) 5 on host 'smithi179' 2024-08-17T17:19:14.555 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.556 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.556 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:14.556 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:19:14.556 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.556 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:14.556 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.556 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:14 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.556 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:14 smithi179 ceph-mon[34775]: from='osd.5 [v2:172.21.15.179:6808/2392026049,v1:172.21.15.179:6809/2392026049]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T17:19:14.556 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:14 smithi179 ceph-mon[34775]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T17:19:14.557 DEBUG:teuthology.orchestra.run.smithi179:osd.5> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.5.service 2024-08-17T17:19:14.560 INFO:tasks.cephadm:Deploying osd.6 on smithi179 with /dev/vg_nvme/lv_2... 2024-08-17T17:19:14.560 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-08-17T17:19:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:19:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[31342]: from='osd.5 [v2:172.21.15.179:6808/2392026049,v1:172.21.15.179:6809/2392026049]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T17:19:14.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[31342]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T17:19:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:19:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[42749]: from='osd.5 [v2:172.21.15.179:6808/2392026049,v1:172.21.15.179:6809/2392026049]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T17:19:14.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:14 smithi130 ceph-mon[42749]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T17:19:15.371 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:19:15.481 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:15 smithi179 ceph-mon[34775]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:15.481 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:15 smithi179 ceph-mon[34775]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T17:19:15.481 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:15 smithi179 ceph-mon[34775]: osdmap e36: 6 total, 5 up, 6 in 2024-08-17T17:19:15.482 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:15 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:15.482 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:15 smithi179 ceph-mon[34775]: from='osd.5 [v2:172.21.15.179:6808/2392026049,v1:172.21.15.179:6809/2392026049]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:19:15.482 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:15 smithi179 ceph-mon[34775]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:19:15.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:15 smithi130 ceph-mon[31342]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:15.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:15 smithi130 ceph-mon[31342]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T17:19:15.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:15 smithi130 ceph-mon[31342]: osdmap e36: 6 total, 5 up, 6 in 2024-08-17T17:19:15.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:15 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:15.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:15 smithi130 ceph-mon[31342]: from='osd.5 [v2:172.21.15.179:6808/2392026049,v1:172.21.15.179:6809/2392026049]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:19:15.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:15 smithi130 ceph-mon[31342]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:19:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:15 smithi130 ceph-mon[42749]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:15 smithi130 ceph-mon[42749]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T17:19:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:15 smithi130 ceph-mon[42749]: osdmap e36: 6 total, 5 up, 6 in 2024-08-17T17:19:15.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:15 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:15.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:15 smithi130 ceph-mon[42749]: from='osd.5 [v2:172.21.15.179:6808/2392026049,v1:172.21.15.179:6809/2392026049]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:19:15.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:15 smithi130 ceph-mon[42749]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:19:16.136 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 17:19:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T17:19:15.926+0000 7efc5848c640 -1 osd.5 0 waiting for initial osdmap 2024-08-17T17:19:16.136 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 17:19:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T17:19:15.931+0000 7efc53ac4640 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:19:17.228 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:16 smithi179 ceph-mon[34775]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-08-17T17:19:17.228 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:16 smithi179 ceph-mon[34775]: osdmap e37: 6 total, 5 up, 6 in 2024-08-17T17:19:17.228 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:16 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:17.229 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:16 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:17.229 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:16 smithi179 ceph-mon[34775]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:17.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:16 smithi130 ceph-mon[31342]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-08-17T17:19:17.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:16 smithi130 ceph-mon[31342]: osdmap e37: 6 total, 5 up, 6 in 2024-08-17T17:19:17.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:16 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:17.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:16 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:17.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:16 smithi130 ceph-mon[31342]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:17.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:16 smithi130 ceph-mon[42749]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-08-17T17:19:17.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:16 smithi130 ceph-mon[42749]: osdmap e37: 6 total, 5 up, 6 in 2024-08-17T17:19:17.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:16 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:17.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:16 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:17.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:16 smithi130 ceph-mon[42749]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:18.194 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:17 smithi179 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T17:19:18.194 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:17 smithi179 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T17:19:18.195 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:17 smithi179 ceph-mon[34775]: osd.5 [v2:172.21.15.179:6808/2392026049,v1:172.21.15.179:6809/2392026049] boot 2024-08-17T17:19:18.195 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:17 smithi179 ceph-mon[34775]: osdmap e38: 6 total, 6 up, 6 in 2024-08-17T17:19:18.195 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:17 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:17 smithi130 ceph-mon[31342]: purged_snaps scrub starts 2024-08-17T17:19:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:17 smithi130 ceph-mon[31342]: purged_snaps scrub ok 2024-08-17T17:19:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:17 smithi130 ceph-mon[31342]: osd.5 [v2:172.21.15.179:6808/2392026049,v1:172.21.15.179:6809/2392026049] boot 2024-08-17T17:19:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:17 smithi130 ceph-mon[31342]: osdmap e38: 6 total, 6 up, 6 in 2024-08-17T17:19:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:17 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:17 smithi130 ceph-mon[42749]: purged_snaps scrub starts 2024-08-17T17:19:18.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:17 smithi130 ceph-mon[42749]: purged_snaps scrub ok 2024-08-17T17:19:18.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:17 smithi130 ceph-mon[42749]: osd.5 [v2:172.21.15.179:6808/2392026049,v1:172.21.15.179:6809/2392026049] boot 2024-08-17T17:19:18.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:17 smithi130 ceph-mon[42749]: osdmap e38: 6 total, 6 up, 6 in 2024-08-17T17:19:18.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:17 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:19:19.166 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:19:19.180 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:18 smithi179 ceph-mon[34775]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:19.181 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:18 smithi179 ceph-mon[34775]: osdmap e39: 6 total, 6 up, 6 in 2024-08-17T17:19:19.181 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:18 smithi179 ceph-mon[34775]: Detected new or changed devices on smithi179 2024-08-17T17:19:19.181 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:18 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:19.181 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:18 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:19.181 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:18 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:19.181 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:18 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:19.181 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:18 smithi179 ceph-mon[34775]: Adjusting osd_memory_target on smithi179 to 8544M 2024-08-17T17:19:19.181 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:18 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:19.181 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:18 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:19.181 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:18 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:19:19.181 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:18 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:19.182 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:18 smithi179 ceph-mon[34775]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:19:19.182 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:18 smithi179 ceph-mon[34775]: osdmap e40: 6 total, 6 up, 6 in 2024-08-17T17:19:19.183 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi179:vg_nvme/lv_2 2024-08-17T17:19:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[31342]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[31342]: osdmap e39: 6 total, 6 up, 6 in 2024-08-17T17:19:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[31342]: Detected new or changed devices on smithi179 2024-08-17T17:19:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[31342]: Adjusting osd_memory_target on smithi179 to 8544M 2024-08-17T17:19:19.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:19.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:19.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:19:19.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:19.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[31342]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:19:19.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[31342]: osdmap e40: 6 total, 6 up, 6 in 2024-08-17T17:19:19.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[42749]: pgmap v128: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-08-17T17:19:19.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[42749]: osdmap e39: 6 total, 6 up, 6 in 2024-08-17T17:19:19.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[42749]: Detected new or changed devices on smithi179 2024-08-17T17:19:19.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:19.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:19.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:19.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:19.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[42749]: Adjusting osd_memory_target on smithi179 to 8544M 2024-08-17T17:19:19.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:19.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:19.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:19:19.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:19.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[42749]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:19:19.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:18 smithi130 ceph-mon[42749]: osdmap e40: 6 total, 6 up, 6 in 2024-08-17T17:19:19.433 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:19:20.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:20 smithi179 ceph-mon[34775]: osdmap e41: 6 total, 6 up, 6 in 2024-08-17T17:19:20.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:20 smithi130 ceph-mon[31342]: osdmap e41: 6 total, 6 up, 6 in 2024-08-17T17:19:20.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:20 smithi130 ceph-mon[42749]: osdmap e41: 6 total, 6 up, 6 in 2024-08-17T17:19:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:21 smithi179 ceph-mon[34775]: pgmap v132: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:21 smithi179 ceph-mon[34775]: osdmap e42: 6 total, 6 up, 6 in 2024-08-17T17:19:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:21 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:19:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:21 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:19:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:21 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:21.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:21 smithi130 ceph-mon[31342]: pgmap v132: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:21 smithi130 ceph-mon[31342]: osdmap e42: 6 total, 6 up, 6 in 2024-08-17T17:19:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:21 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:19:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:21 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:19:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:21 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:21 smithi130 ceph-mon[42749]: pgmap v132: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:21 smithi130 ceph-mon[42749]: osdmap e42: 6 total, 6 up, 6 in 2024-08-17T17:19:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:21 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:19:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:21 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:19:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:21 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:22.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:22 smithi179 ceph-mon[34775]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:19:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:22 smithi130 ceph-mon[31342]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:19:22.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:22 smithi130 ceph-mon[42749]: from='client.24232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:19:23.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:23 smithi130 ceph-mon[31342]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:23 smithi130 ceph-mon[42749]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:23.501 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:23 smithi179 ceph-mon[34775]: pgmap v134: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:24 smithi130 ceph-mon[31342]: from='client.? 172.21.15.179:0/2693157423' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b84229ad-2c88-4064-a070-33b98714ef96"}]: dispatch 2024-08-17T17:19:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:24 smithi130 ceph-mon[31342]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b84229ad-2c88-4064-a070-33b98714ef96"}]: dispatch 2024-08-17T17:19:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:24 smithi130 ceph-mon[31342]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b84229ad-2c88-4064-a070-33b98714ef96"}]': finished 2024-08-17T17:19:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:24 smithi130 ceph-mon[31342]: osdmap e43: 7 total, 6 up, 7 in 2024-08-17T17:19:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:24 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:24 smithi130 ceph-mon[31342]: from='client.? 172.21.15.179:0/2421816346' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:19:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:24 smithi130 ceph-mon[42749]: from='client.? 172.21.15.179:0/2693157423' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b84229ad-2c88-4064-a070-33b98714ef96"}]: dispatch 2024-08-17T17:19:24.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:24 smithi130 ceph-mon[42749]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b84229ad-2c88-4064-a070-33b98714ef96"}]: dispatch 2024-08-17T17:19:24.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:24 smithi130 ceph-mon[42749]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b84229ad-2c88-4064-a070-33b98714ef96"}]': finished 2024-08-17T17:19:24.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:24 smithi130 ceph-mon[42749]: osdmap e43: 7 total, 6 up, 7 in 2024-08-17T17:19:24.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:24 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:24.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:24 smithi130 ceph-mon[42749]: from='client.? 172.21.15.179:0/2421816346' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:19:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:24 smithi179 ceph-mon[34775]: from='client.? 172.21.15.179:0/2693157423' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b84229ad-2c88-4064-a070-33b98714ef96"}]: dispatch 2024-08-17T17:19:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:24 smithi179 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b84229ad-2c88-4064-a070-33b98714ef96"}]: dispatch 2024-08-17T17:19:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:24 smithi179 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b84229ad-2c88-4064-a070-33b98714ef96"}]': finished 2024-08-17T17:19:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:24 smithi179 ceph-mon[34775]: osdmap e43: 7 total, 6 up, 7 in 2024-08-17T17:19:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:24 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:24 smithi179 ceph-mon[34775]: from='client.? 172.21.15.179:0/2421816346' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:19:25.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:25 smithi130 ceph-mon[31342]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-08-17T17:19:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:25 smithi130 ceph-mon[31342]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:19:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:25 smithi130 ceph-mon[31342]: Cluster is now healthy 2024-08-17T17:19:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:25 smithi130 ceph-mon[42749]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-08-17T17:19:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:25 smithi130 ceph-mon[42749]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:19:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:25 smithi130 ceph-mon[42749]: Cluster is now healthy 2024-08-17T17:19:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:25 smithi179 ceph-mon[34775]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-08-17T17:19:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:25 smithi179 ceph-mon[34775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:19:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:25 smithi179 ceph-mon[34775]: Cluster is now healthy 2024-08-17T17:19:27.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:27 smithi179 ceph-mon[34775]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 85 KiB/s, 0 objects/s recovering 2024-08-17T17:19:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:27 smithi130 ceph-mon[31342]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 85 KiB/s, 0 objects/s recovering 2024-08-17T17:19:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:27 smithi130 ceph-mon[42749]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 85 KiB/s, 0 objects/s recovering 2024-08-17T17:19:29.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:29 smithi179 ceph-mon[34775]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T17:19:29.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:29 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-17T17:19:29.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:29 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:29 smithi130 ceph-mon[31342]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T17:19:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:29 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-17T17:19:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:29 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:29.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:29 smithi130 ceph-mon[42749]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T17:19:29.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:29 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-17T17:19:29.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:29 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:30.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:30 smithi130 ceph-mon[31342]: Deploying daemon osd.6 on smithi179 2024-08-17T17:19:30.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:30 smithi130 ceph-mon[42749]: Deploying daemon osd.6 on smithi179 2024-08-17T17:19:30.555 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:30 smithi179 ceph-mon[34775]: Deploying daemon osd.6 on smithi179 2024-08-17T17:19:31.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:31 smithi130 ceph-mon[31342]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-08-17T17:19:31.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:31 smithi130 ceph-mon[42749]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-08-17T17:19:31.431 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:31 smithi179 ceph-mon[34775]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-08-17T17:19:33.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:33 smithi130 ceph-mon[31342]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:19:33.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:33 smithi130 ceph-mon[42749]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:19:33.421 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:33 smithi179 ceph-mon[34775]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:19:34.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:34 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:34.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:34 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:34.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:34 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:34.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:34 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:34 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:34 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:34 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:34 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:34 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:35 smithi130 ceph-mon[31342]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:35 smithi130 ceph-mon[42749]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:35.565 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:35 smithi179 ceph-mon[34775]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:36.672 INFO:teuthology.orchestra.run.smithi179.stdout:Created osd(s) 6 on host 'smithi179' 2024-08-17T17:19:37.243 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:37 smithi179 ceph-mon[34775]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:37.244 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:37 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:37.244 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:37 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:37.244 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:37 smithi179 ceph-mon[34775]: from='osd.6 [v2:172.21.15.179:6816/139895003,v1:172.21.15.179:6817/139895003]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T17:19:37.244 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:37 smithi179 ceph-mon[34775]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T17:19:37.268 DEBUG:teuthology.orchestra.run.smithi179:osd.6> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.6.service 2024-08-17T17:19:37.270 INFO:tasks.cephadm:Deploying osd.7 on smithi179 with /dev/vg_nvme/lv_1... 2024-08-17T17:19:37.271 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-08-17T17:19:37.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:37 smithi130 ceph-mon[31342]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:37.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:37 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:37.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:37 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:37.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:37 smithi130 ceph-mon[31342]: from='osd.6 [v2:172.21.15.179:6816/139895003,v1:172.21.15.179:6817/139895003]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T17:19:37.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:37 smithi130 ceph-mon[31342]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T17:19:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:37 smithi130 ceph-mon[42749]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:37 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:37 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:37 smithi130 ceph-mon[42749]: from='osd.6 [v2:172.21.15.179:6816/139895003,v1:172.21.15.179:6817/139895003]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T17:19:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:37 smithi130 ceph-mon[42749]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T17:19:37.587 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:19:38.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:38 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:38.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:38 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:38.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:38 smithi179 ceph-mon[34775]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:19:38.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:38 smithi179 ceph-mon[34775]: osdmap e44: 7 total, 6 up, 7 in 2024-08-17T17:19:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:38 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:38 smithi179 ceph-mon[34775]: from='osd.6 [v2:172.21.15.179:6816/139895003,v1:172.21.15.179:6817/139895003]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:19:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:38 smithi179 ceph-mon[34775]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:19:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:38 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:38 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:38 smithi130 ceph-mon[31342]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:19:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:38 smithi130 ceph-mon[31342]: osdmap e44: 7 total, 6 up, 7 in 2024-08-17T17:19:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:38 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:38 smithi130 ceph-mon[31342]: from='osd.6 [v2:172.21.15.179:6816/139895003,v1:172.21.15.179:6817/139895003]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:19:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:38 smithi130 ceph-mon[31342]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:19:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:38 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:38.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:38 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:38.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:38 smithi130 ceph-mon[42749]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:19:38.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:38 smithi130 ceph-mon[42749]: osdmap e44: 7 total, 6 up, 7 in 2024-08-17T17:19:38.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:38 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:38.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:38 smithi130 ceph-mon[42749]: from='osd.6 [v2:172.21.15.179:6816/139895003,v1:172.21.15.179:6817/139895003]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:19:38.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:38 smithi130 ceph-mon[42749]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:19:38.843 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 17:19:38 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T17:19:38.683+0000 7f91149d1640 -1 osd.6 0 waiting for initial osdmap 2024-08-17T17:19:38.843 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 17:19:38 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T17:19:38.689+0000 7f911180c640 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:19:39.437 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:39 smithi179 ceph-mon[34775]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:39.437 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:39 smithi179 ceph-mon[34775]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-08-17T17:19:39.437 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:39 smithi179 ceph-mon[34775]: osdmap e45: 7 total, 6 up, 7 in 2024-08-17T17:19:39.438 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:39 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:39.438 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:39 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:39.438 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:39 smithi179 ceph-mon[34775]: osd.6 [v2:172.21.15.179:6816/139895003,v1:172.21.15.179:6817/139895003] boot 2024-08-17T17:19:39.438 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:39 smithi179 ceph-mon[34775]: osdmap e46: 7 total, 7 up, 7 in 2024-08-17T17:19:39.438 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:39 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[31342]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[31342]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-08-17T17:19:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[31342]: osdmap e45: 7 total, 6 up, 7 in 2024-08-17T17:19:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[31342]: osd.6 [v2:172.21.15.179:6816/139895003,v1:172.21.15.179:6817/139895003] boot 2024-08-17T17:19:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[31342]: osdmap e46: 7 total, 7 up, 7 in 2024-08-17T17:19:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[42749]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:19:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[42749]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-08-17T17:19:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[42749]: osdmap e45: 7 total, 6 up, 7 in 2024-08-17T17:19:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[42749]: osd.6 [v2:172.21.15.179:6816/139895003,v1:172.21.15.179:6817/139895003] boot 2024-08-17T17:19:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[42749]: osdmap e46: 7 total, 7 up, 7 in 2024-08-17T17:19:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:39 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:19:40.063 INFO:teuthology.orchestra.run.smithi179.stdout: 2024-08-17T17:19:40.083 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi179:vg_nvme/lv_1 2024-08-17T17:19:40.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:40 smithi179 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T17:19:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:40 smithi179 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T17:19:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:40 smithi179 ceph-mon[34775]: osdmap e47: 7 total, 7 up, 7 in 2024-08-17T17:19:40.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:40 smithi130 ceph-mon[31342]: purged_snaps scrub starts 2024-08-17T17:19:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:40 smithi130 ceph-mon[31342]: purged_snaps scrub ok 2024-08-17T17:19:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:40 smithi130 ceph-mon[31342]: osdmap e47: 7 total, 7 up, 7 in 2024-08-17T17:19:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:40 smithi130 ceph-mon[42749]: purged_snaps scrub starts 2024-08-17T17:19:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:40 smithi130 ceph-mon[42749]: purged_snaps scrub ok 2024-08-17T17:19:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:40 smithi130 ceph-mon[42749]: osdmap e47: 7 total, 7 up, 7 in 2024-08-17T17:19:41.468 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:19:41.591 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:41 smithi179 ceph-mon[34775]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:19:41.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:41 smithi179 ceph-mon[34775]: osdmap e48: 7 total, 7 up, 7 in 2024-08-17T17:19:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:41 smithi130 ceph-mon[31342]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:19:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:41 smithi130 ceph-mon[31342]: osdmap e48: 7 total, 7 up, 7 in 2024-08-17T17:19:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:41 smithi130 ceph-mon[42749]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:19:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:41 smithi130 ceph-mon[42749]: osdmap e48: 7 total, 7 up, 7 in 2024-08-17T17:19:42.752 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:42 smithi179 ceph-mon[34775]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:42.752 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:42 smithi179 ceph-mon[34775]: osdmap e49: 7 total, 7 up, 7 in 2024-08-17T17:19:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:42 smithi130 ceph-mon[31342]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:42 smithi130 ceph-mon[31342]: osdmap e49: 7 total, 7 up, 7 in 2024-08-17T17:19:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:42 smithi130 ceph-mon[42749]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:19:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:42 smithi130 ceph-mon[42749]: osdmap e49: 7 total, 7 up, 7 in 2024-08-17T17:19:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: pgmap v150: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-08-17T17:19:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:19:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:19:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:43 smithi179 ceph-mon[34775]: osdmap e50: 7 total, 7 up, 7 in 2024-08-17T17:19:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: pgmap v150: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-08-17T17:19:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:19:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:43.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:43.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:19:43.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[31342]: osdmap e50: 7 total, 7 up, 7 in 2024-08-17T17:19:43.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: pgmap v150: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-08-17T17:19:43.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:43.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:43.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:19:43.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:43.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:19:43.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:43.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:43.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:19:43.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:43.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:43 smithi130 ceph-mon[42749]: osdmap e50: 7 total, 7 up, 7 in 2024-08-17T17:19:44.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:44 smithi179 ceph-mon[34775]: Detected new or changed devices on smithi179 2024-08-17T17:19:44.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:44 smithi179 ceph-mon[34775]: Adjusting osd_memory_target on smithi179 to 5696M 2024-08-17T17:19:44.495 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:44 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:19:44.495 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:44 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:19:44.495 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:44 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:44 smithi130 ceph-mon[31342]: Detected new or changed devices on smithi179 2024-08-17T17:19:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:44 smithi130 ceph-mon[31342]: Adjusting osd_memory_target on smithi179 to 5696M 2024-08-17T17:19:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:44 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:19:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:44 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:19:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:44 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:44 smithi130 ceph-mon[42749]: Detected new or changed devices on smithi179 2024-08-17T17:19:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:44 smithi130 ceph-mon[42749]: Adjusting osd_memory_target on smithi179 to 5696M 2024-08-17T17:19:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:44 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T17:19:44.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:44 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T17:19:44.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:44 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:45.581 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:45 smithi179 ceph-mon[34775]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:19:45.581 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:45 smithi179 ceph-mon[34775]: pgmap v153: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-08-17T17:19:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:45 smithi130 ceph-mon[31342]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:19:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:45 smithi130 ceph-mon[31342]: pgmap v153: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-08-17T17:19:45.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:45 smithi130 ceph-mon[42749]: from='client.24259 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi179:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:19:45.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:45 smithi130 ceph-mon[42749]: pgmap v153: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-08-17T17:19:46.669 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:46 smithi179 ceph-mon[34775]: from='client.? 172.21.15.179:0/434557884' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bba7978f-8b19-40e6-86bd-692a0e282429"}]: dispatch 2024-08-17T17:19:46.669 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:46 smithi179 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bba7978f-8b19-40e6-86bd-692a0e282429"}]: dispatch 2024-08-17T17:19:46.669 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:46 smithi179 ceph-mon[34775]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bba7978f-8b19-40e6-86bd-692a0e282429"}]': finished 2024-08-17T17:19:46.669 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:46 smithi179 ceph-mon[34775]: osdmap e51: 8 total, 7 up, 8 in 2024-08-17T17:19:46.669 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:46 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:19:46.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:46 smithi130 ceph-mon[31342]: from='client.? 172.21.15.179:0/434557884' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bba7978f-8b19-40e6-86bd-692a0e282429"}]: dispatch 2024-08-17T17:19:46.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:46 smithi130 ceph-mon[31342]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bba7978f-8b19-40e6-86bd-692a0e282429"}]: dispatch 2024-08-17T17:19:46.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:46 smithi130 ceph-mon[31342]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bba7978f-8b19-40e6-86bd-692a0e282429"}]': finished 2024-08-17T17:19:46.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:46 smithi130 ceph-mon[31342]: osdmap e51: 8 total, 7 up, 8 in 2024-08-17T17:19:46.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:46 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:19:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:46 smithi130 ceph-mon[42749]: from='client.? 172.21.15.179:0/434557884' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bba7978f-8b19-40e6-86bd-692a0e282429"}]: dispatch 2024-08-17T17:19:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:46 smithi130 ceph-mon[42749]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bba7978f-8b19-40e6-86bd-692a0e282429"}]: dispatch 2024-08-17T17:19:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:46 smithi130 ceph-mon[42749]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bba7978f-8b19-40e6-86bd-692a0e282429"}]': finished 2024-08-17T17:19:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:46 smithi130 ceph-mon[42749]: osdmap e51: 8 total, 7 up, 8 in 2024-08-17T17:19:46.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:46 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:19:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:47 smithi179 ceph-mon[34775]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-08-17T17:19:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:47 smithi179 ceph-mon[34775]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-08-17T17:19:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:47 smithi179 ceph-mon[34775]: Cluster is now healthy 2024-08-17T17:19:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:47 smithi179 ceph-mon[34775]: from='client.? 172.21.15.179:0/3320362975' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:19:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:47 smithi130 ceph-mon[31342]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-08-17T17:19:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:47 smithi130 ceph-mon[31342]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-08-17T17:19:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:47 smithi130 ceph-mon[31342]: Cluster is now healthy 2024-08-17T17:19:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:47 smithi130 ceph-mon[31342]: from='client.? 172.21.15.179:0/3320362975' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:19:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:47 smithi130 ceph-mon[42749]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-08-17T17:19:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:47 smithi130 ceph-mon[42749]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-08-17T17:19:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:47 smithi130 ceph-mon[42749]: Cluster is now healthy 2024-08-17T17:19:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:47 smithi130 ceph-mon[42749]: from='client.? 172.21.15.179:0/3320362975' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T17:19:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:49 smithi179 ceph-mon[34775]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:19:49.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:49 smithi130 ceph-mon[31342]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:19:49.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:49 smithi130 ceph-mon[42749]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:19:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:51 smithi179 ceph-mon[34775]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-08-17T17:19:51.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:51 smithi130 ceph-mon[31342]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-08-17T17:19:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:51 smithi130 ceph-mon[42749]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-08-17T17:19:52.679 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:52 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-17T17:19:52.679 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:52 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:52 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-17T17:19:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:52 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:52 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-17T17:19:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:52 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:19:53.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:53 smithi179 ceph-mon[34775]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-08-17T17:19:53.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:53 smithi179 ceph-mon[34775]: Deploying daemon osd.7 on smithi179 2024-08-17T17:19:53.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:53 smithi130 ceph-mon[31342]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-08-17T17:19:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:53 smithi130 ceph-mon[31342]: Deploying daemon osd.7 on smithi179 2024-08-17T17:19:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:53 smithi130 ceph-mon[42749]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-08-17T17:19:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:53 smithi130 ceph-mon[42749]: Deploying daemon osd.7 on smithi179 2024-08-17T17:19:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:55 smithi179 ceph-mon[34775]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:19:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:55 smithi130 ceph-mon[31342]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:19:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:55 smithi130 ceph-mon[42749]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:19:57.772 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:57 smithi179 ceph-mon[34775]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:19:57.772 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:57 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:57.772 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:57 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:57.772 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:57 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:57 smithi130 ceph-mon[31342]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:19:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:57 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:57 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:57 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:57 smithi130 ceph-mon[42749]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:19:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:57 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:19:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:57 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:57 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:19:59.774 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:19:59 smithi179 ceph-mon[34775]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:19:59.814 INFO:teuthology.orchestra.run.smithi179.stdout:Created osd(s) 7 on host 'smithi179' 2024-08-17T17:19:59.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:19:59 smithi130 ceph-mon[31342]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:19:59.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:19:59 smithi130 ceph-mon[42749]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:20:00.343 DEBUG:teuthology.orchestra.run.smithi179:osd.7> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.7.service 2024-08-17T17:20:00.346 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-08-17T17:20:00.346 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd stat -f json 2024-08-17T17:20:00.614 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:01.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:01.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:01.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:01.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:01.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[42749]: from='osd.7 [v2:172.21.15.179:6824/2533639201,v1:172.21.15.179:6825/2533639201]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T17:20:01.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[42749]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T17:20:01.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[42749]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:20:01.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[42749]: overall HEALTH_OK 2024-08-17T17:20:01.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:01.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:01.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:01.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:01.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[31342]: from='osd.7 [v2:172.21.15.179:6824/2533639201,v1:172.21.15.179:6825/2533639201]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T17:20:01.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[31342]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T17:20:01.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[31342]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:20:01.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:00 smithi130 ceph-mon[31342]: overall HEALTH_OK 2024-08-17T17:20:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:00 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:00 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:00 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:00 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:00 smithi179 ceph-mon[34775]: from='osd.7 [v2:172.21.15.179:6824/2533639201,v1:172.21.15.179:6825/2533639201]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T17:20:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:00 smithi179 ceph-mon[34775]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T17:20:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:00 smithi179 ceph-mon[34775]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:20:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:00 smithi179 ceph-mon[34775]: overall HEALTH_OK 2024-08-17T17:20:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:01 smithi179 ceph-mon[34775]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:20:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:01 smithi179 ceph-mon[34775]: osdmap e52: 8 total, 7 up, 8 in 2024-08-17T17:20:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:01 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:01 smithi179 ceph-mon[34775]: from='osd.7 [v2:172.21.15.179:6824/2533639201,v1:172.21.15.179:6825/2533639201]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:20:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:01 smithi179 ceph-mon[34775]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:20:02.094 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 17:20:01 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T17:20:01.838+0000 7f9cede44640 -1 osd.7 0 waiting for initial osdmap 2024-08-17T17:20:02.094 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 17:20:01 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T17:20:01.844+0000 7f9ceac7f640 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:20:02.161 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:01 smithi130 ceph-mon[31342]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:20:02.161 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:01 smithi130 ceph-mon[31342]: osdmap e52: 8 total, 7 up, 8 in 2024-08-17T17:20:02.161 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:01 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:02.161 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:01 smithi130 ceph-mon[31342]: from='osd.7 [v2:172.21.15.179:6824/2533639201,v1:172.21.15.179:6825/2533639201]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:20:02.161 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:01 smithi130 ceph-mon[31342]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:20:02.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:01 smithi130 ceph-mon[42749]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:20:02.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:01 smithi130 ceph-mon[42749]: osdmap e52: 8 total, 7 up, 8 in 2024-08-17T17:20:02.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:01 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:02.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:01 smithi130 ceph-mon[42749]: from='osd.7 [v2:172.21.15.179:6824/2533639201,v1:172.21.15.179:6825/2533639201]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:20:02.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:01 smithi130 ceph-mon[42749]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]: dispatch 2024-08-17T17:20:02.565 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:20:03.074 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:02 smithi179 ceph-mon[34775]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-08-17T17:20:03.075 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:02 smithi179 ceph-mon[34775]: osdmap e53: 8 total, 7 up, 8 in 2024-08-17T17:20:03.075 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:02 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:03.075 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:02 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:03.075 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:02 smithi179 ceph-mon[34775]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:20:03.075 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:02 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3552701328' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T17:20:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:02 smithi130 ceph-mon[42749]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-08-17T17:20:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:02 smithi130 ceph-mon[42749]: osdmap e53: 8 total, 7 up, 8 in 2024-08-17T17:20:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:02 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:02 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:02 smithi130 ceph-mon[42749]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:20:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:02 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3552701328' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T17:20:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:02 smithi130 ceph-mon[31342]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi179", "root=default"]}]': finished 2024-08-17T17:20:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:02 smithi130 ceph-mon[31342]: osdmap e53: 8 total, 7 up, 8 in 2024-08-17T17:20:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:02 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:03.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:02 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:03.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:02 smithi130 ceph-mon[31342]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-08-17T17:20:03.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:02 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3552701328' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T17:20:03.324 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":53,"num_osds":8,"num_up_osds":7,"osd_up_since":1723915179,"num_in_osds":8,"osd_in_since":1723915186,"num_remapped_pgs":0} 2024-08-17T17:20:04.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: purged_snaps scrub starts 2024-08-17T17:20:04.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: purged_snaps scrub ok 2024-08-17T17:20:04.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:04.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: osd.7 [v2:172.21.15.179:6824/2533639201,v1:172.21.15.179:6825/2533639201] boot 2024-08-17T17:20:04.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: osdmap e54: 8 total, 8 up, 8 in 2024-08-17T17:20:04.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:04.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: Detected new or changed devices on smithi179 2024-08-17T17:20:04.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:04.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:04.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:04.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:04.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: Adjusting osd_memory_target on smithi179 to 4272M 2024-08-17T17:20:04.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:20:04.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:20:04.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:20:04.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:20:04.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:20:04.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:03 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: purged_snaps scrub starts 2024-08-17T17:20:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: purged_snaps scrub ok 2024-08-17T17:20:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: osd.7 [v2:172.21.15.179:6824/2533639201,v1:172.21.15.179:6825/2533639201] boot 2024-08-17T17:20:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: osdmap e54: 8 total, 8 up, 8 in 2024-08-17T17:20:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: Detected new or changed devices on smithi179 2024-08-17T17:20:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:04.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:04.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:04.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:04.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: Adjusting osd_memory_target on smithi179 to 4272M 2024-08-17T17:20:04.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:20:04.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:20:04.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:20:04.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:20:04.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:20:04.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: purged_snaps scrub starts 2024-08-17T17:20:04.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: purged_snaps scrub ok 2024-08-17T17:20:04.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:04.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: osd.7 [v2:172.21.15.179:6824/2533639201,v1:172.21.15.179:6825/2533639201] boot 2024-08-17T17:20:04.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: osdmap e54: 8 total, 8 up, 8 in 2024-08-17T17:20:04.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:04.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: Detected new or changed devices on smithi179 2024-08-17T17:20:04.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:04.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:04.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:04.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:04.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: Adjusting osd_memory_target on smithi179 to 4272M 2024-08-17T17:20:04.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:20:04.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:20:04.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:20:04.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:20:04.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:20:04.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:03 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:04.326 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd stat -f json 2024-08-17T17:20:04.582 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:04.959 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:04 smithi130 ceph-mon[31342]: osdmap e55: 8 total, 8 up, 8 in 2024-08-17T17:20:04.960 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:04 smithi130 ceph-mon[31342]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:04.960 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:04 smithi130 ceph-mon[42749]: osdmap e55: 8 total, 8 up, 8 in 2024-08-17T17:20:04.960 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:04 smithi130 ceph-mon[42749]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:05.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:04 smithi179 ceph-mon[34775]: osdmap e55: 8 total, 8 up, 8 in 2024-08-17T17:20:05.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:04 smithi179 ceph-mon[34775]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:05.985 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:05 smithi130 ceph-mon[31342]: osdmap e56: 8 total, 8 up, 8 in 2024-08-17T17:20:05.985 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:05 smithi130 ceph-mon[42749]: osdmap e56: 8 total, 8 up, 8 in 2024-08-17T17:20:06.326 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:20:06.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:05 smithi179 ceph-mon[34775]: osdmap e56: 8 total, 8 up, 8 in 2024-08-17T17:20:06.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:06 smithi130 ceph-mon[31342]: osdmap e57: 8 total, 8 up, 8 in 2024-08-17T17:20:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:06 smithi130 ceph-mon[31342]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:06 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/469684141' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T17:20:06.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:06 smithi130 ceph-mon[42749]: osdmap e57: 8 total, 8 up, 8 in 2024-08-17T17:20:06.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:06 smithi130 ceph-mon[42749]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:06.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:06 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/469684141' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T17:20:07.047 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1723915202,"num_in_osds":8,"osd_in_since":1723915186,"num_remapped_pgs":0} 2024-08-17T17:20:07.047 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:20:07.297 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:06 smithi179 ceph-mon[34775]: osdmap e57: 8 total, 8 up, 8 in 2024-08-17T17:20:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:06 smithi179 ceph-mon[34775]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:06 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/469684141' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T17:20:08.086 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:07 smithi130 ceph-mon[31342]: osdmap e58: 8 total, 8 up, 8 in 2024-08-17T17:20:08.086 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:07 smithi130 ceph-mon[42749]: osdmap e58: 8 total, 8 up, 8 in 2024-08-17T17:20:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:07 smithi179 ceph-mon[34775]: osdmap e58: 8 total, 8 up, 8 in 2024-08-17T17:20:09.158 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:20:09.158 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":58,"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","created":"2024-08-17T17:15:09.488012+0000","modified":"2024-08-17T17:20:06.862016+0000","last_up_change":"2024-08-17T17:20:02.834333+0000","last_in_change":"2024-08-17T17:19:46.093156+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:18:10.192831+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"30c31adb-2cda-466b-933e-a8e2cb714fab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6803","nonce":2346469001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6805","nonce":2346469001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6809","nonce":2346469001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6807","nonce":2346469001}]},"public_addr":"172.21.15.130:6803/2346469001","cluster_addr":"172.21.15.130:6805/2346469001","heartbeat_back_addr":"172.21.15.130:6809/2346469001","heartbeat_front_addr":"172.21.15.130:6807/2346469001","state":["exists","up"]},{"osd":1,"uuid":"55049bb4-b976-42c6-8a5d-8b3e9ae20988","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6811","nonce":2284482082}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6813","nonce":2284482082}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6817","nonce":2284482082}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6815","nonce":2284482082}]},"public_addr":"172.21.15.130:6811/2284482082","cluster_addr":"172.21.15.130:6813/2284482082","heartbeat_back_addr":"172.21.15.130:6817/2284482082","heartbeat_front_addr":"172.21.15.130:6815/2284482082","state":["exists","up"]},{"osd":2,"uuid":"3f40f088-27f7-4450-8f31-b449711365ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6819","nonce":160441313}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6821","nonce":160441313}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6825","nonce":160441313}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6823","nonce":160441313}]},"public_addr":"172.21.15.130:6819/160441313","cluster_addr":"172.21.15.130:6821/160441313","heartbeat_back_addr":"172.21.15.130:6825/160441313","heartbeat_front_addr":"172.21.15.130:6823/160441313","state":["exists","up"]},{"osd":3,"uuid":"7684c401-462a-4e35-bb9c-1be23dfe573b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6827","nonce":775049303}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6829","nonce":775049303}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6833","nonce":775049303}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6831","nonce":775049303}]},"public_addr":"172.21.15.130:6827/775049303","cluster_addr":"172.21.15.130:6829/775049303","heartbeat_back_addr":"172.21.15.130:6833/775049303","heartbeat_front_addr":"172.21.15.130:6831/775049303","state":["exists","up"]},{"osd":4,"uuid":"7a3a331f-533a-49de-910d-5989f2f84b65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6800","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6801","nonce":2107797845}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6802","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6803","nonce":2107797845}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6806","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6807","nonce":2107797845}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6804","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6805","nonce":2107797845}]},"public_addr":"172.21.15.179:6801/2107797845","cluster_addr":"172.21.15.179:6803/2107797845","heartbeat_back_addr":"172.21.15.179:6807/2107797845","heartbeat_front_addr":"172.21.15.179:6805/2107797845","state":["exists","up"]},{"osd":5,"uuid":"d53d65ee-bbb2-45ec-b557-07cc62a45b05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6808","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6809","nonce":2392026049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6810","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6811","nonce":2392026049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6814","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6815","nonce":2392026049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6812","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6813","nonce":2392026049}]},"public_addr":"172.21.15.179:6809/2392026049","cluster_addr":"172.21.15.179:6811/2392026049","heartbeat_back_addr":"172.21.15.179:6815/2392026049","heartbeat_front_addr":"172.21.15.179:6813/2392026049","state":["exists","up"]},{"osd":6,"uuid":"b84229ad-2c88-4064-a070-33b98714ef96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6816","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6817","nonce":139895003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6818","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6819","nonce":139895003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6822","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6823","nonce":139895003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6820","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6821","nonce":139895003}]},"public_addr":"172.21.15.179:6817/139895003","cluster_addr":"172.21.15.179:6819/139895003","heartbeat_back_addr":"172.21.15.179:6823/139895003","heartbeat_front_addr":"172.21.15.179:6821/139895003","state":["exists","up"]},{"osd":7,"uuid":"bba7978f-8b19-40e6-86bd-692a0e282429","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6824","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6825","nonce":2533639201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6826","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6827","nonce":2533639201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6830","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6831","nonce":2533639201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6828","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6829","nonce":2533639201}]},"public_addr":"172.21.15.179:6825/2533639201","cluster_addr":"172.21.15.179:6827/2533639201","heartbeat_back_addr":"172.21.15.179:6831/2533639201","heartbeat_front_addr":"172.21.15.179:6829/2533639201","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:19.220776+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:42.876683+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:07.028539+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:31.259509+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:52.172235+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:15.112109+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:37.781476+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:20:00.893216+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/1247136467":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/3471387839":"2024-08-18T17:15:57.894717+0000","172.21.15.130:6800/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/88320168":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6801/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6800/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6801/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2999329360":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/3071443649":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/1690071627":"2024-08-18T17:15:34.935061+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:20:09.159 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:08 smithi130 ceph-mon[31342]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:09.159 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:08 smithi130 ceph-mon[42749]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:09.244 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:08 smithi179 ceph-mon[34775]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:09.752 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-08-17T17:18:10.192831+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-08-17T17:20:09.752 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-08-17T17:20:10.004 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:10.127 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:09 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/4236688227' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:20:10.127 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:09 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/4236688227' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:20:10.263 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:09 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/4236688227' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:20:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:10 smithi130 ceph-mon[31342]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:20:11.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:10 smithi130 ceph-mon[42749]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:20:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:10 smithi179 ceph-mon[34775]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T17:20:11.903 INFO:teuthology.orchestra.run.smithi130.stdout:pg_num: 1 2024-08-17T17:20:12.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:11 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/4113557612' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-17T17:20:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:11 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/4113557612' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-17T17:20:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:11 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/4113557612' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-17T17:20:12.626 INFO:tasks.cephadm:Adding prometheus.a on smithi179 2024-08-17T17:20:12.626 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch apply prometheus '1;smithi179=a' 2024-08-17T17:20:12.887 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:13.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:12 smithi179 ceph-mon[34775]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-08-17T17:20:13.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:12 smithi130 ceph-mon[31342]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-08-17T17:20:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:12 smithi130 ceph-mon[42749]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-08-17T17:20:14.502 INFO:teuthology.orchestra.run.smithi179.stdout:Scheduled prometheus update... 2024-08-17T17:20:15.127 DEBUG:teuthology.orchestra.run.smithi179:prometheus.a> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@prometheus.a.service 2024-08-17T17:20:15.130 INFO:tasks.cephadm:Adding node-exporter.a on smithi130 2024-08-17T17:20:15.130 INFO:tasks.cephadm:Adding node-exporter.b on smithi179 2024-08-17T17:20:15.131 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch apply node-exporter '2;smithi130=a;smithi179=b' 2024-08-17T17:20:15.444 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:15.773 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:15 smithi179 ceph-mon[34775]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-08-17T17:20:15.774 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:15 smithi179 ceph-mon[34775]: from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi179=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:20:15.774 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:15 smithi179 ceph-mon[34775]: Saving service prometheus spec with placement smithi179=a;count:1 2024-08-17T17:20:15.774 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:15 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:15.774 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:15 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:20:15.774 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:15 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:20:15.774 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:15 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:20:15.774 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:15 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:15.774 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:15 smithi179 ceph-mon[34775]: Deploying daemon prometheus.a on smithi179 2024-08-17T17:20:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[31342]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-08-17T17:20:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[31342]: from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi179=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:20:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[31342]: Saving service prometheus spec with placement smithi179=a;count:1 2024-08-17T17:20:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:20:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:20:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:20:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[31342]: Deploying daemon prometheus.a on smithi179 2024-08-17T17:20:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[42749]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-08-17T17:20:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[42749]: from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi179=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:20:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[42749]: Saving service prometheus spec with placement smithi179=a;count:1 2024-08-17T17:20:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:20:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:20:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:20:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:15 smithi130 ceph-mon[42749]: Deploying daemon prometheus.a on smithi179 2024-08-17T17:20:17.536 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:17 smithi179 ceph-mon[34775]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:20:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:17 smithi130 ceph-mon[31342]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:20:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:17 smithi130 ceph-mon[42749]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:20:18.113 INFO:teuthology.orchestra.run.smithi179.stdout:Scheduled node-exporter update... 2024-08-17T17:20:19.066 DEBUG:teuthology.orchestra.run.smithi130:node-exporter.a> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@node-exporter.a.service 2024-08-17T17:20:19.069 DEBUG:teuthology.orchestra.run.smithi179:node-exporter.b> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@node-exporter.b.service 2024-08-17T17:20:19.071 INFO:tasks.cephadm:Adding alertmanager.a on smithi130 2024-08-17T17:20:19.072 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch apply alertmanager '1;smithi130=a' 2024-08-17T17:20:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:19 smithi179 ceph-mon[34775]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-08-17T17:20:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:19 smithi179 ceph-mon[34775]: from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi130=a;smithi179=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:20:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:19 smithi179 ceph-mon[34775]: Saving service node-exporter spec with placement smithi130=a;smithi179=b;count:2 2024-08-17T17:20:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:19 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:19.361 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:19 smithi130 ceph-mon[42749]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-08-17T17:20:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:19 smithi130 ceph-mon[42749]: from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi130=a;smithi179=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:20:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:19 smithi130 ceph-mon[42749]: Saving service node-exporter spec with placement smithi130=a;smithi179=b;count:2 2024-08-17T17:20:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:19 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:19 smithi130 ceph-mon[31342]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-08-17T17:20:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:19 smithi130 ceph-mon[31342]: from='client.24298 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi130=a;smithi179=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:20:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:19 smithi130 ceph-mon[31342]: Saving service node-exporter spec with placement smithi130=a;smithi179=b;count:2 2024-08-17T17:20:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:19 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:21.301 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:21 smithi179 ceph-mon[34775]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-08-17T17:20:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:21 smithi130 ceph-mon[42749]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-08-17T17:20:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:21 smithi130 ceph-mon[31342]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-08-17T17:20:23.025 INFO:teuthology.orchestra.run.smithi179.stdout:Scheduled alertmanager update... 2024-08-17T17:20:23.324 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:23 smithi179 ceph-mon[34775]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:23.324 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:23 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:23.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:23 smithi130 ceph-mon[31342]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:23.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:23 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:23 smithi130 ceph-mon[42749]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:23 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:23.945 DEBUG:teuthology.orchestra.run.smithi130:alertmanager.a> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@alertmanager.a.service 2024-08-17T17:20:23.948 INFO:tasks.cephadm:Adding grafana.a on smithi179 2024-08-17T17:20:23.948 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph orch apply grafana '1;smithi179=a' 2024-08-17T17:20:24.207 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:24 smithi179 ceph-mon[34775]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi130=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:20:24.207 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:24 smithi179 ceph-mon[34775]: Saving service alertmanager spec with placement smithi130=a;count:1 2024-08-17T17:20:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:24 smithi130 ceph-mon[31342]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi130=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:20:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:24 smithi130 ceph-mon[31342]: Saving service alertmanager spec with placement smithi130=a;count:1 2024-08-17T17:20:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:24 smithi130 ceph-mon[42749]: from='client.24304 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi130=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:20:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:24 smithi130 ceph-mon[42749]: Saving service alertmanager spec with placement smithi130=a;count:1 2024-08-17T17:20:25.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:25 smithi130 ceph-mon[31342]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:25 smithi130 ceph-mon[42749]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:25.440 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:25 smithi179 ceph-mon[34775]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:25.642 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:26.843 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:26 smithi179 systemd[1]: Starting Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:20:27.301 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-mon[34775]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:27.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:27 smithi130 ceph-mon[31342]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:27 smithi130 ceph-mon[42749]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:27.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 podman[63914]: 2024-08-17 17:20:27.217643019 +0000 UTC m=+0.024878363 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-17T17:20:27.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 podman[63914]: 2024-08-17 17:20:27.522336404 +0000 UTC m=+0.329571730 container create 56bc2272bf9552218a4aa8bc13c5046cf55bd8256c36be174451c37696eb9c6b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:20:28.002 INFO:teuthology.orchestra.run.smithi179.stdout:Scheduled grafana update... 2024-08-17T17:20:28.003 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 podman[63914]: 2024-08-17 17:20:27.799750146 +0000 UTC m=+0.606985482 container init 56bc2272bf9552218a4aa8bc13c5046cf55bd8256c36be174451c37696eb9c6b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:20:28.003 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 podman[63914]: 2024-08-17 17:20:27.80505749 +0000 UTC m=+0.612292819 container start 56bc2272bf9552218a4aa8bc13c5046cf55bd8256c36be174451c37696eb9c6b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:20:28.003 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.840Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-17T17:20:28.003 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.840Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-17T17:20:28.003 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.840Z caller=main.go:570 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi179 (none))" 2024-08-17T17:20:28.003 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.840Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T17:20:28.003 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.840Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T17:20:28.003 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.842Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T17:20:28.003 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.842Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-17T17:20:28.003 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.843Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T17:20:28.003 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.843Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T17:20:28.004 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.846Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T17:20:28.004 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.846Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.211µs 2024-08-17T17:20:28.004 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.846Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T17:20:28.004 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.846Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-08-17T17:20:28.004 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.846Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=17.31µs wal_replay_duration=214.121µs wbl_replay_duration=110ns total_replay_duration=250.103µs 2024-08-17T17:20:28.004 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.847Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T17:20:28.004 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.847Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-17T17:20:28.004 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.847Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T17:20:28.004 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.861Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.970387ms db_storage=739ns remote_storage=1.075µs web_handler=362ns query_engine=598ns scrape=388.627µs scrape_sd=16.609µs notify=966ns notify_sd=1.315µs rules=13.371148ms tracing=8.695µs 2024-08-17T17:20:28.004 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.861Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-17T17:20:28.004 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:20:27.861Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T17:20:28.004 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 bash[63914]: 56bc2272bf9552218a4aa8bc13c5046cf55bd8256c36be174451c37696eb9c6b 2024-08-17T17:20:28.004 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:20:27 smithi179 systemd[1]: Started Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:20:28.562 DEBUG:teuthology.orchestra.run.smithi179:grafana.a> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@grafana.a.service 2024-08-17T17:20:28.564 INFO:tasks.cephadm:Setting up client nodes... 2024-08-17T17:20:28.565 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-08-17T17:20:28.834 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:29.268 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[31342]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:29.269 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[31342]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi179=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:20:29.269 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[31342]: Saving service grafana spec with placement smithi179=a;count:1 2024-08-17T17:20:29.269 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:29.269 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:29.269 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:29.269 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:29.269 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-17T17:20:29.269 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ignoring --setuser ceph since I am not root 2024-08-17T17:20:29.270 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ignoring --setgroup ceph since I am not root 2024-08-17T17:20:29.270 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mgr[31617]: -- 172.21.15.130:0/3728744543 <== mon.0 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56124220e9c0 con 0x56124221a800 2024-08-17T17:20:29.270 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:29.186+0000 7f4131477ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T17:20:29.270 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[42749]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:29.270 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[42749]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi179=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:20:29.270 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[42749]: Saving service grafana spec with placement smithi179=a;count:1 2024-08-17T17:20:29.270 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:29.270 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:29.270 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:29.270 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:29.271 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-17T17:20:29.301 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-mon[34775]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:29.301 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-mon[34775]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi179=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:20:29.301 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-mon[34775]: Saving service grafana spec with placement smithi179=a;count:1 2024-08-17T17:20:29.301 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:29.301 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:29.302 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:29.302 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' 2024-08-17T17:20:29.302 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-17T17:20:29.302 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ignoring --setuser ceph since I am not root 2024-08-17T17:20:29.302 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ignoring --setgroup ceph since I am not root 2024-08-17T17:20:29.302 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-mgr[36898]: -- 172.21.15.179:0/1321577306 <== mon.1 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560301eba9c0 con 0x560301ec7000 2024-08-17T17:20:29.302 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:29.181+0000 7f4ca632eec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T17:20:29.302 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:29.300+0000 7f4ca632eec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:20:29.593 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:29.365+0000 7f4ca632eec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T17:20:29.614 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:29.312+0000 7f4131477ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:20:29.615 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:29.377+0000 7f4131477ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T17:20:29.615 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:29.613+0000 7f4131477ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T17:20:29.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:29.675+0000 7f4131477ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T17:20:30.043 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:29.604+0000 7f4ca632eec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T17:20:30.043 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:29 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:29.661+0000 7f4ca632eec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T17:20:30.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:30 smithi179 ceph-mon[34775]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-17T17:20:30.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:30 smithi179 ceph-mon[34775]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-17T17:20:30.348 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:30 smithi130 ceph-mon[31342]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-17T17:20:30.348 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:30 smithi130 ceph-mon[31342]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-17T17:20:30.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:30 smithi130 ceph-mon[42749]: from='mgr.14148 172.21.15.130:0/249147594' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-17T17:20:30.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:30 smithi130 ceph-mon[42749]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-17T17:20:30.798 INFO:teuthology.orchestra.run.smithi130.stdout:[client.0] 2024-08-17T17:20:30.798 INFO:teuthology.orchestra.run.smithi130.stdout: key = AQDe28BmGSopLxAAKRslB6aZDwWLFALJaZC+tw== 2024-08-17T17:20:30.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:30.554+0000 7f4ca632eec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T17:20:30.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:30.665+0000 7f4ca632eec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T17:20:30.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:30.718+0000 7f4ca632eec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T17:20:30.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:30 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:30.606+0000 7f4131477ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T17:20:30.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:30 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:30.720+0000 7f4131477ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T17:20:30.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:30 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:30.776+0000 7f4131477ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T17:20:31.226 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:31 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2074170670' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:20:31.226 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:31 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:20:31.226 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:31 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T17:20:31.226 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:31 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2074170670' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:20:31.226 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:31 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:20:31.226 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:31 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T17:20:31.227 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:30 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:30.970+0000 7f4131477ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T17:20:31.227 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:31.034+0000 7f4131477ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T17:20:31.227 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:31.099+0000 7f4131477ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T17:20:31.227 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:31.224+0000 7f4131477ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:20:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:31 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2074170670' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:20:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:31 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:20:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:31 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T17:20:31.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:30.903+0000 7f4ca632eec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T17:20:31.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:30.962+0000 7f4ca632eec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T17:20:31.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:31 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:31.026+0000 7f4ca632eec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T17:20:31.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:31 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:31.148+0000 7f4ca632eec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:20:31.513 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:20:31.513 DEBUG:teuthology.orchestra.run.smithi130:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-08-17T17:20:31.513 DEBUG:teuthology.orchestra.run.smithi130:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-08-17T17:20:31.552 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-08-17T17:20:31.787 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:31.647+0000 7f4131477ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T17:20:31.810 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:31.842 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:31 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:31.552+0000 7f4ca632eec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T17:20:32.150 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:31.950+0000 7f4131477ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T17:20:32.150 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:32.018+0000 7f4131477ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T17:20:32.174 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:31 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:31.859+0000 7f4ca632eec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T17:20:32.174 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:31 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:31.924+0000 7f4ca632eec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T17:20:32.174 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:32 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:32.053+0000 7f4ca632eec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T17:20:32.409 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:32.148+0000 7f4131477ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T17:20:32.555 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:32 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:32.314+0000 7f4ca632eec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T17:20:32.555 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:32 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:32.373+0000 7f4ca632eec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T17:20:32.555 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:32 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:32.554+0000 7f4ca632eec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T17:20:32.666 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:32.407+0000 7f4131477ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T17:20:32.666 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:32.468+0000 7f4131477ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T17:20:32.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:32.653+0000 7f4131477ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T17:20:32.842 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:32 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:32.616+0000 7f4ca632eec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T17:20:32.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:32 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:32.698+0000 7f4ca632eec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T17:20:32.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:32 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:32.755+0000 7f4ca632eec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T17:20:33.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:32.716+0000 7f4131477ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T17:20:33.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:32.799+0000 7f4131477ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T17:20:33.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:32.855+0000 7f4131477ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T17:20:33.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:33 smithi179 ceph-mon[34775]: Standby manager daemon x restarted 2024-08-17T17:20:33.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:33 smithi179 ceph-mon[34775]: Standby manager daemon x started 2024-08-17T17:20:33.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:33 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/757478577' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:20:33.537 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:33 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/757478577' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:20:33.538 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:33 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/757478577' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:20:33.538 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:33 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/757478577' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:20:33.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:33 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:33.292+0000 7f4ca632eec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T17:20:33.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:33 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:33.429+0000 7f4ca632eec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T17:20:33.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:33 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: [17/Aug/2024:17:20:33] ENGINE Bus STARTING 2024-08-17T17:20:33.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:33 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: CherryPy Checker: 2024-08-17T17:20:33.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:33 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: The Application mounted at '' has an empty config. 2024-08-17T17:20:33.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:33 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: 2024-08-17T17:20:33.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:33 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: [17/Aug/2024:17:20:33] ENGINE Serving on http://:::9283 2024-08-17T17:20:33.538 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:33 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: [17/Aug/2024:17:20:33] ENGINE Bus STARTED 2024-08-17T17:20:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-mon[31342]: Standby manager daemon x restarted 2024-08-17T17:20:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-mon[31342]: Standby manager daemon x started 2024-08-17T17:20:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/757478577' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:20:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/757478577' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:20:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/757478577' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:20:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/757478577' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:20:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-mon[42749]: Standby manager daemon x restarted 2024-08-17T17:20:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-mon[42749]: Standby manager daemon x started 2024-08-17T17:20:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/757478577' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:20:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/757478577' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:20:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/757478577' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:20:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/757478577' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:20:33.668 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:33.367+0000 7f4131477ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T17:20:33.668 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:33.504+0000 7f4131477ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T17:20:33.668 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:20:33] ENGINE Bus STARTING 2024-08-17T17:20:33.668 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: CherryPy Checker: 2024-08-17T17:20:33.668 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: The Application mounted at '' has an empty config. 2024-08-17T17:20:33.668 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:33.689 INFO:teuthology.orchestra.run.smithi179.stdout:[client.1] 2024-08-17T17:20:33.689 INFO:teuthology.orchestra.run.smithi179.stdout: key = AQDh28BmWs7TKBAAVReUsqfw6ERalaeOAVwVVg== 2024-08-17T17:20:34.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:20:33] ENGINE Serving on http://:::9283 2024-08-17T17:20:34.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:20:33] ENGINE Bus STARTED 2024-08-17T17:20:34.275 DEBUG:teuthology.orchestra.run.smithi179:> set -ex 2024-08-17T17:20:34.275 DEBUG:teuthology.orchestra.run.smithi179:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-08-17T17:20:34.275 DEBUG:teuthology.orchestra.run.smithi179:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-08-17T17:20:34.315 INFO:tasks.cephadm:Creating RBD pool 2024-08-17T17:20:34.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool create rbd 8 2024-08-17T17:20:34.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: mgrmap e17: y(active, since 4m), standbys: x 2024-08-17T17:20:34.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: Active manager daemon y restarted 2024-08-17T17:20:34.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: Activating manager daemon y 2024-08-17T17:20:34.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: osdmap e59: 8 total, 8 up, 8 in 2024-08-17T17:20:34.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: mgrmap e18: y(active, starting, since 0.00584867s), standbys: x 2024-08-17T17:20:34.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:20:34.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:20:34.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:20:34.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:20:34.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:20:34.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:20:34.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:20:34.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:20:34.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:20:34.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:20:34.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:20:34.585 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:20:34.585 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:34.585 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:20:34.585 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:20:34.585 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:20:34.585 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: Manager daemon y is now available 2024-08-17T17:20:34.585 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:20:34.585 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:34.585 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:20:34.585 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:20:34.585 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:20:34.585 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:20:34.585 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:20:34.585 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='client.? 172.21.15.179:0/81312071' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:20:34.586 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:20:34.586 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T17:20:34.619 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: mgrmap e17: y(active, since 4m), standbys: x 2024-08-17T17:20:34.619 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: Active manager daemon y restarted 2024-08-17T17:20:34.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: Activating manager daemon y 2024-08-17T17:20:34.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: osdmap e59: 8 total, 8 up, 8 in 2024-08-17T17:20:34.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: mgrmap e18: y(active, starting, since 0.00584867s), standbys: x 2024-08-17T17:20:34.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:20:34.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:20:34.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:20:34.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:20:34.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:20:34.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:20:34.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:20:34.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:20:34.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:20:34.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:20:34.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:20:34.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:20:34.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:34.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:20:34.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:20:34.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:20:34.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: Manager daemon y is now available 2024-08-17T17:20:34.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:20:34.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:34.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:20:34.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:20:34.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:20:34.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:20:34.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:20:34.622 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='client.? 172.21.15.179:0/81312071' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:20:34.622 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:20:34.622 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T17:20:34.622 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: mgrmap e17: y(active, since 4m), standbys: x 2024-08-17T17:20:34.622 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: Active manager daemon y restarted 2024-08-17T17:20:34.622 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: Activating manager daemon y 2024-08-17T17:20:34.622 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: osdmap e59: 8 total, 8 up, 8 in 2024-08-17T17:20:34.622 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: mgrmap e18: y(active, starting, since 0.00584867s), standbys: x 2024-08-17T17:20:34.623 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:20:34.623 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:20:34.623 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:20:34.623 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:20:34.623 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:20:34.623 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:20:34.623 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:20:34.623 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:20:34.623 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:20:34.623 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:20:34.623 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:20:34.623 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:20:34.623 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:20:34.623 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:20:34.624 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:20:34.624 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:20:34.624 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: Manager daemon y is now available 2024-08-17T17:20:34.624 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:20:34.624 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:34.624 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:20:34.624 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:20:34.624 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:20:34.624 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:20:34.624 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:20:34.624 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='client.? 172.21.15.179:0/81312071' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:20:34.624 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T17:20:34.625 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:34 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T17:20:34.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:34 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:20:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:20:34.952 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:35.334 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:35.185+0000 7f40d41d2640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:17:20:35] ENGINE Error in HTTPServer.serve 2024-08-17T17:20:35.334 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: Traceback (most recent call last): 2024-08-17T17:20:35.334 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:20:35.334 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: self._connections.run(self.expiration_interval) 2024-08-17T17:20:35.334 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:20:35.334 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: self._run(expiration_interval) 2024-08-17T17:20:35.335 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:20:35.335 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:20:35.335 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:20:35.335 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:20:35.335 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:20:35.335 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: s = self.context.wrap_socket( 2024-08-17T17:20:35.335 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:20:35.335 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: return self.sslsocket_class._create( 2024-08-17T17:20:35.335 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:20:35.335 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: self.do_handshake() 2024-08-17T17:20:35.335 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:20:35.335 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: self._sslobj.do_handshake() 2024-08-17T17:20:35.335 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:20:35.335 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:20:35.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-mon[31342]: mgrmap e19: y(active, since 1.01s), standbys: x 2024-08-17T17:20:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-mon[31342]: [17/Aug/2024:17:20:35] ENGINE Bus STARTING 2024-08-17T17:20:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-mon[42749]: mgrmap e19: y(active, since 1.01s), standbys: x 2024-08-17T17:20:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-mon[42749]: [17/Aug/2024:17:20:35] ENGINE Bus STARTING 2024-08-17T17:20:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:35 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:35 smithi179 ceph-mon[34775]: mgrmap e19: y(active, since 1.01s), standbys: x 2024-08-17T17:20:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:35 smithi179 ceph-mon[34775]: [17/Aug/2024:17:20:35] ENGINE Bus STARTING 2024-08-17T17:20:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:35 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: [17/Aug/2024:17:20:35] ENGINE Serving on https://172.21.15.130:7150 2024-08-17T17:20:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: [17/Aug/2024:17:20:35] ENGINE Bus STARTED 2024-08-17T17:20:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: [17/Aug/2024:17:20:35] ENGINE Error in HTTPServer.serve 2024-08-17T17:20:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-08-17T17:20:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:20:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:20:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:20:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout: self._run(expiration_interval) 2024-08-17T17:20:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:20:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:20:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:20:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:20:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:20:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout: s = self.context.wrap_socket( 2024-08-17T17:20:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:20:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.sslsocket_class._create( 2024-08-17T17:20:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:20:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.do_handshake() 2024-08-17T17:20:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:20:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout: self._sslobj.do_handshake() 2024-08-17T17:20:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:20:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: mgrmap e20: y(active, since 2s), standbys: x 2024-08-17T17:20:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: [17/Aug/2024:17:20:35] ENGINE Serving on https://172.21.15.130:7150 2024-08-17T17:20:37.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: [17/Aug/2024:17:20:35] ENGINE Bus STARTED 2024-08-17T17:20:37.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: [17/Aug/2024:17:20:35] ENGINE Error in HTTPServer.serve 2024-08-17T17:20:37.420 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-08-17T17:20:37.420 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:20:37.420 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:20:37.420 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:20:37.421 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._run(expiration_interval) 2024-08-17T17:20:37.421 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:20:37.421 INFO:journalctl@ceph.mon.a.smithi130.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:20:37.421 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:20:37.421 INFO:journalctl@ceph.mon.a.smithi130.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:20:37.421 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:20:37.421 INFO:journalctl@ceph.mon.a.smithi130.stdout: s = self.context.wrap_socket( 2024-08-17T17:20:37.421 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:20:37.421 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.sslsocket_class._create( 2024-08-17T17:20:37.421 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:20:37.421 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.do_handshake() 2024-08-17T17:20:37.421 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:20:37.421 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._sslobj.do_handshake() 2024-08-17T17:20:37.421 INFO:journalctl@ceph.mon.a.smithi130.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:20:37.422 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:37.422 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.422 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.422 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: mgrmap e20: y(active, since 2s), standbys: x 2024-08-17T17:20:37.422 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.422 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.422 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.422 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.422 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.422 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.422 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.422 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.423 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.423 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.423 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.423 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.423 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:37 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: [17/Aug/2024:17:20:35] ENGINE Serving on https://172.21.15.130:7150 2024-08-17T17:20:37.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: [17/Aug/2024:17:20:35] ENGINE Bus STARTED 2024-08-17T17:20:37.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: [17/Aug/2024:17:20:35] ENGINE Error in HTTPServer.serve 2024-08-17T17:20:37.594 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-08-17T17:20:37.594 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:20:37.595 INFO:journalctl@ceph.mon.b.smithi179.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:20:37.595 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:20:37.595 INFO:journalctl@ceph.mon.b.smithi179.stdout: self._run(expiration_interval) 2024-08-17T17:20:37.595 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:20:37.595 INFO:journalctl@ceph.mon.b.smithi179.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:20:37.595 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:20:37.596 INFO:journalctl@ceph.mon.b.smithi179.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:20:37.596 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:20:37.596 INFO:journalctl@ceph.mon.b.smithi179.stdout: s = self.context.wrap_socket( 2024-08-17T17:20:37.596 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:20:37.596 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.sslsocket_class._create( 2024-08-17T17:20:37.597 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:20:37.597 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.do_handshake() 2024-08-17T17:20:37.597 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:20:37.597 INFO:journalctl@ceph.mon.b.smithi179.stdout: self._sslobj.do_handshake() 2024-08-17T17:20:37.597 INFO:journalctl@ceph.mon.b.smithi179.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:20:37.597 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:37.598 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.598 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.598 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: mgrmap e20: y(active, since 2s), standbys: x 2024-08-17T17:20:37.598 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.598 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.599 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.599 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.599 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.599 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.599 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.599 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.600 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.600 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.600 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.600 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:20:37.600 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:37 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:37.651 INFO:teuthology.orchestra.run.smithi130.stderr:pool 'rbd' created 2024-08-17T17:20:38.384 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: Adjusting osd_memory_target on smithi179 to 4016M 2024-08-17T17:20:38.384 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2518782532' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T17:20:38.384 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T17:20:38.384 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.384 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.384 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:20:38.384 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:20:38.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:20:38.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:20:38.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:20:38.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:20:38.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:38.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:20:38.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:20:38.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-17T17:20:38.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: osdmap e60: 8 total, 8 up, 8 in 2024-08-17T17:20:38.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:20:38.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:20:38.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.386 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.386 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.386 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[31342]: Deploying daemon alertmanager.a on smithi130 2024-08-17T17:20:38.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: Adjusting osd_memory_target on smithi179 to 4016M 2024-08-17T17:20:38.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2518782532' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T17:20:38.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T17:20:38.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:20:38.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:20:38.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:20:38.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:20:38.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:20:38.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:20:38.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:38.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:20:38.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:20:38.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-17T17:20:38.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: osdmap e60: 8 total, 8 up, 8 in 2024-08-17T17:20:38.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:20:38.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:20:38.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:38 smithi130 ceph-mon[42749]: Deploying daemon alertmanager.a on smithi130 2024-08-17T17:20:38.407 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- rbd pool init rbd 2024-08-17T17:20:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: Adjusting osd_memory_target on smithi179 to 4016M 2024-08-17T17:20:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2518782532' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T17:20:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T17:20:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:20:38.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:20:38.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:20:38.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:20:38.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:20:38.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:20:38.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:38.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:20:38.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:20:38.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-17T17:20:38.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: osdmap e60: 8 total, 8 up, 8 in 2024-08-17T17:20:38.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:20:38.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:20:38.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:38.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:38 smithi179 ceph-mon[34775]: Deploying daemon alertmanager.a on smithi130 2024-08-17T17:20:38.708 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:39.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:39 smithi130 ceph-mon[31342]: mgrmap e21: y(active, since 4s), standbys: x 2024-08-17T17:20:39.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:39 smithi130 ceph-mon[31342]: osdmap e61: 8 total, 8 up, 8 in 2024-08-17T17:20:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:39 smithi130 ceph-mon[42749]: mgrmap e21: y(active, since 4s), standbys: x 2024-08-17T17:20:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:39 smithi130 ceph-mon[42749]: osdmap e61: 8 total, 8 up, 8 in 2024-08-17T17:20:39.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:39 smithi179 ceph-mon[34775]: mgrmap e21: y(active, since 4s), standbys: x 2024-08-17T17:20:39.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:39 smithi179 ceph-mon[34775]: osdmap e61: 8 total, 8 up, 8 in 2024-08-17T17:20:40.824 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:40 smithi130 ceph-mon[31342]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:40.824 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:40 smithi130 ceph-mon[31342]: osdmap e62: 8 total, 8 up, 8 in 2024-08-17T17:20:40.824 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:40 smithi130 ceph-mon[31342]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:20:40.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:40 smithi130 ceph-mon[42749]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:40.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:40 smithi130 ceph-mon[42749]: osdmap e62: 8 total, 8 up, 8 in 2024-08-17T17:20:40.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:40 smithi130 ceph-mon[42749]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:20:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:40 smithi179 ceph-mon[34775]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:40 smithi179 ceph-mon[34775]: osdmap e62: 8 total, 8 up, 8 in 2024-08-17T17:20:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:40 smithi179 ceph-mon[34775]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:20:41.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:41 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3354162148' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T17:20:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:41 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3354162148' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T17:20:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:41 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3354162148' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T17:20:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:42 smithi130 ceph-mon[42749]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-08-17T17:20:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:42 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3354162148' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-17T17:20:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:42 smithi130 ceph-mon[42749]: osdmap e63: 8 total, 8 up, 8 in 2024-08-17T17:20:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:42 smithi130 ceph-mon[31342]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-08-17T17:20:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:42 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3354162148' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-17T17:20:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:42 smithi130 ceph-mon[31342]: osdmap e63: 8 total, 8 up, 8 in 2024-08-17T17:20:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:42 smithi179 ceph-mon[34775]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-08-17T17:20:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:42 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3354162148' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-17T17:20:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:42 smithi179 ceph-mon[34775]: osdmap e63: 8 total, 8 up, 8 in 2024-08-17T17:20:43.960 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:43 smithi130 ceph-mon[31342]: osdmap e64: 8 total, 8 up, 8 in 2024-08-17T17:20:43.960 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:43 smithi130 ceph-mon[42749]: osdmap e64: 8 total, 8 up, 8 in 2024-08-17T17:20:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:43 smithi179 ceph-mon[34775]: osdmap e64: 8 total, 8 up, 8 in 2024-08-17T17:20:44.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:20:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:20:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:44 smithi179 ceph-mon[34775]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 23 op/s 2024-08-17T17:20:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:44 smithi179 ceph-mon[34775]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:20:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:44 smithi179 ceph-mon[34775]: Cluster is now healthy 2024-08-17T17:20:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:44 smithi179 ceph-mon[34775]: osdmap e65: 8 total, 8 up, 8 in 2024-08-17T17:20:44.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:44 smithi130 ceph-mon[31342]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 23 op/s 2024-08-17T17:20:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:44 smithi130 ceph-mon[31342]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:20:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:44 smithi130 ceph-mon[31342]: Cluster is now healthy 2024-08-17T17:20:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:44 smithi130 ceph-mon[31342]: osdmap e65: 8 total, 8 up, 8 in 2024-08-17T17:20:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:44 smithi130 ceph-mon[42749]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 23 op/s 2024-08-17T17:20:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:44 smithi130 ceph-mon[42749]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:20:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:44 smithi130 ceph-mon[42749]: Cluster is now healthy 2024-08-17T17:20:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:44 smithi130 ceph-mon[42749]: osdmap e65: 8 total, 8 up, 8 in 2024-08-17T17:20:45.301 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph config log 1 --format=json 2024-08-17T17:20:46.089 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:46.902 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:46 smithi130 ceph-mon[31342]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 348 B/s wr, 0 op/s 2024-08-17T17:20:46.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:46 smithi130 ceph-mon[42749]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 348 B/s wr, 0 op/s 2024-08-17T17:20:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:46 smithi179 ceph-mon[34775]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 348 B/s wr, 0 op/s 2024-08-17T17:20:47.928 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:20:48.167 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:47 smithi130 podman[73417]: 2024-08-17 17:20:47.997412622 +0000 UTC m=+0.315795658 container create be07dfea12b0dc320c18b5ca97accc47ff8438d0a3b07084e354e3d63ac4bdd8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:20:48.612 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:48 smithi130 podman[73417]: 2024-08-17 17:20:48.308401419 +0000 UTC m=+0.626784462 container init be07dfea12b0dc320c18b5ca97accc47ff8438d0a3b07084e354e3d63ac4bdd8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:20:48.612 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:48 smithi130 podman[73417]: 2024-08-17 17:20:48.313309359 +0000 UTC m=+0.631692394 container start be07dfea12b0dc320c18b5ca97accc47ff8438d0a3b07084e354e3d63ac4bdd8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:20:48.612 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[73466]: ts=2024-08-17T17:20:48.325Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-08-17T17:20:48.612 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[73466]: ts=2024-08-17T17:20:48.325Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-08-17T17:20:48.612 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[73466]: ts=2024-08-17T17:20:48.326Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.130 port=9094 2024-08-17T17:20:48.612 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[73466]: ts=2024-08-17T17:20:48.327Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-17T17:20:48.613 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[73466]: ts=2024-08-17T17:20:48.350Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T17:20:48.613 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[73466]: ts=2024-08-17T17:20:48.350Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T17:20:48.613 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[73466]: ts=2024-08-17T17:20:48.353Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-08-17T17:20:48.613 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[73466]: ts=2024-08-17T17:20:48.353Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-17T17:20:48.613 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:48 smithi130 bash[73417]: be07dfea12b0dc320c18b5ca97accc47ff8438d0a3b07084e354e3d63ac4bdd8 2024-08-17T17:20:48.613 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:48 smithi130 systemd[1]: Started Ceph alertmanager.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:20:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[31342]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-08-17T17:20:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1913051732' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-17T17:20:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:20:48.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T17:20:48.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.918 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:20:48] ENGINE Bus STOPPING 2024-08-17T17:20:48.918 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:20:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:20:48.918 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:20:48] ENGINE Bus STOPPED 2024-08-17T17:20:48.918 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:20:48] ENGINE Bus STARTING 2024-08-17T17:20:48.919 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:20:48] ENGINE Serving on http://:::9283 2024-08-17T17:20:48.919 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:20:48] ENGINE Bus STARTED 2024-08-17T17:20:48.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[42749]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-08-17T17:20:48.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1913051732' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-17T17:20:48.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:20:48.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:48.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T17:20:48.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:48 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:48 smithi179 ceph-mon[34775]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-08-17T17:20:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:48 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1913051732' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-17T17:20:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:48 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:48 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:48 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:48 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:48 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:48 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:20:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:48 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:48 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:48 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T17:20:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:48 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:20:49.104 INFO:teuthology.orchestra.run.smithi130.stdout:[{"version":18,"timestamp":"2024-08-17T17:20:36.641133+0000","name":"","changes":[{"name":"osd/host:smithi179/osd_memory_target","previous_value":"4479546163","new_value":"4211110707"}]}] 2024-08-17T17:20:49.105 INFO:tasks.ceph_manager:config epoch is 18 2024-08-17T17:20:49.105 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-17T17:20:49.105 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-17T17:20:49.105 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph mgr dump --format=json 2024-08-17T17:20:49.360 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:49.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:49 smithi130 ceph-mon[31342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T17:20:49.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:49 smithi130 ceph-mon[31342]: Deploying daemon grafana.a on smithi179 2024-08-17T17:20:49.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:49 smithi130 ceph-mon[31342]: osdmap e66: 8 total, 8 up, 8 in 2024-08-17T17:20:49.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:49 smithi130 ceph-mon[42749]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T17:20:49.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:49 smithi130 ceph-mon[42749]: Deploying daemon grafana.a on smithi179 2024-08-17T17:20:49.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:49 smithi130 ceph-mon[42749]: osdmap e66: 8 total, 8 up, 8 in 2024-08-17T17:20:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:49 smithi179 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T17:20:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:49 smithi179 ceph-mon[34775]: Deploying daemon grafana.a on smithi179 2024-08-17T17:20:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:49 smithi179 ceph-mon[34775]: osdmap e66: 8 total, 8 up, 8 in 2024-08-17T17:20:50.667 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:50 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[73466]: ts=2024-08-17T17:20:50.327Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000191107s 2024-08-17T17:20:50.980 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:50 smithi130 ceph-mon[31342]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 298 B/s wr, 0 op/s 2024-08-17T17:20:50.980 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:50 smithi130 ceph-mon[42749]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 298 B/s wr, 0 op/s 2024-08-17T17:20:51.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:50 smithi179 ceph-mon[34775]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 298 B/s wr, 0 op/s 2024-08-17T17:20:51.519 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:20:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:51 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2031181834' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T17:20:52.161 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:51 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2031181834' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T17:20:52.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:51 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2031181834' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T17:20:52.186 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":21,"flags":0,"active_gid":24338,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":824081166},{"type":"v1","addr":"172.21.15.130:6801","nonce":824081166}]},"active_addr":"172.21.15.130:6801/824081166","active_change":"2024-08-17T17:20:33.508124+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24322,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":1457627030}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":1299294781}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":1843620043}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":854839742}]}]} 2024-08-17T17:20:52.190 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-17T17:20:52.190 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T17:20:52.190 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:20:52.442 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:52.815 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:52 smithi130 ceph-mon[31342]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:20:52.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:52 smithi130 ceph-mon[42749]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:20:53.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:52 smithi179 ceph-mon[34775]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:20:54.292 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:20:54.292 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":66,"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","created":"2024-08-17T17:15:09.488012+0000","modified":"2024-08-17T17:20:49.130548+0000","last_up_change":"2024-08-17T17:20:02.834333+0000","last_in_change":"2024-08-17T17:19:46.093156+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:18:10.192831+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:20:37.124673+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"30c31adb-2cda-466b-933e-a8e2cb714fab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6803","nonce":2346469001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6805","nonce":2346469001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6809","nonce":2346469001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6807","nonce":2346469001}]},"public_addr":"172.21.15.130:6803/2346469001","cluster_addr":"172.21.15.130:6805/2346469001","heartbeat_back_addr":"172.21.15.130:6809/2346469001","heartbeat_front_addr":"172.21.15.130:6807/2346469001","state":["exists","up"]},{"osd":1,"uuid":"55049bb4-b976-42c6-8a5d-8b3e9ae20988","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6811","nonce":2284482082}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6813","nonce":2284482082}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6817","nonce":2284482082}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6815","nonce":2284482082}]},"public_addr":"172.21.15.130:6811/2284482082","cluster_addr":"172.21.15.130:6813/2284482082","heartbeat_back_addr":"172.21.15.130:6817/2284482082","heartbeat_front_addr":"172.21.15.130:6815/2284482082","state":["exists","up"]},{"osd":2,"uuid":"3f40f088-27f7-4450-8f31-b449711365ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6819","nonce":160441313}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6821","nonce":160441313}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6825","nonce":160441313}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6823","nonce":160441313}]},"public_addr":"172.21.15.130:6819/160441313","cluster_addr":"172.21.15.130:6821/160441313","heartbeat_back_addr":"172.21.15.130:6825/160441313","heartbeat_front_addr":"172.21.15.130:6823/160441313","state":["exists","up"]},{"osd":3,"uuid":"7684c401-462a-4e35-bb9c-1be23dfe573b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6827","nonce":775049303}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6829","nonce":775049303}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6833","nonce":775049303}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6831","nonce":775049303}]},"public_addr":"172.21.15.130:6827/775049303","cluster_addr":"172.21.15.130:6829/775049303","heartbeat_back_addr":"172.21.15.130:6833/775049303","heartbeat_front_addr":"172.21.15.130:6831/775049303","state":["exists","up"]},{"osd":4,"uuid":"7a3a331f-533a-49de-910d-5989f2f84b65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6800","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6801","nonce":2107797845}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6802","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6803","nonce":2107797845}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6806","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6807","nonce":2107797845}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6804","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6805","nonce":2107797845}]},"public_addr":"172.21.15.179:6801/2107797845","cluster_addr":"172.21.15.179:6803/2107797845","heartbeat_back_addr":"172.21.15.179:6807/2107797845","heartbeat_front_addr":"172.21.15.179:6805/2107797845","state":["exists","up"]},{"osd":5,"uuid":"d53d65ee-bbb2-45ec-b557-07cc62a45b05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6808","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6809","nonce":2392026049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6810","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6811","nonce":2392026049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6814","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6815","nonce":2392026049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6812","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6813","nonce":2392026049}]},"public_addr":"172.21.15.179:6809/2392026049","cluster_addr":"172.21.15.179:6811/2392026049","heartbeat_back_addr":"172.21.15.179:6815/2392026049","heartbeat_front_addr":"172.21.15.179:6813/2392026049","state":["exists","up"]},{"osd":6,"uuid":"b84229ad-2c88-4064-a070-33b98714ef96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6816","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6817","nonce":139895003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6818","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6819","nonce":139895003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6822","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6823","nonce":139895003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6820","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6821","nonce":139895003}]},"public_addr":"172.21.15.179:6817/139895003","cluster_addr":"172.21.15.179:6819/139895003","heartbeat_back_addr":"172.21.15.179:6823/139895003","heartbeat_front_addr":"172.21.15.179:6821/139895003","state":["exists","up"]},{"osd":7,"uuid":"bba7978f-8b19-40e6-86bd-692a0e282429","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6824","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6825","nonce":2533639201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6826","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6827","nonce":2533639201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6830","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6831","nonce":2533639201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6828","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6829","nonce":2533639201}]},"public_addr":"172.21.15.179:6825/2533639201","cluster_addr":"172.21.15.179:6827/2533639201","heartbeat_back_addr":"172.21.15.179:6831/2533639201","heartbeat_front_addr":"172.21.15.179:6829/2533639201","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:19.220776+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:42.876683+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:07.028539+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:31.259509+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:52.172235+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:15.112109+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:37.781476+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:20:00.893216+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/851371227":"2024-08-18T17:20:33.507974+0000","172.21.15.130:6801/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/1690071627":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6800/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/2999329360":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/3071443649":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/368845317":"2024-08-18T17:20:33.507974+0000","172.21.15.130:6800/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6801/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/88320168":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/3471387839":"2024-08-18T17:15:57.894717+0000","172.21.15.130:6800/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/1247136467":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2589122337":"2024-08-18T17:20:33.507974+0000","172.21.15.130:6801/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2578011752":"2024-08-18T17:20:33.507974+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:20:54.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:20:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:20:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:20:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:54 smithi179 ceph-mon[34775]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 207 B/s wr, 0 op/s 2024-08-17T17:20:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:54 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/582045202' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:20:54.882 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:54 smithi130 ceph-mon[42749]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 207 B/s wr, 0 op/s 2024-08-17T17:20:54.882 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:54 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/582045202' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:20:54.883 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:54 smithi130 ceph-mon[31342]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 207 B/s wr, 0 op/s 2024-08-17T17:20:54.883 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:54 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/582045202' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:20:54.907 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T17:20:54.907 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:20:55.159 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:56.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:56 smithi130 ceph-mon[42749]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:56.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:56 smithi130 ceph-mon[31342]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:57.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:56 smithi179 ceph-mon[34775]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:57.278 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:20:57.278 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":66,"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","created":"2024-08-17T17:15:09.488012+0000","modified":"2024-08-17T17:20:49.130548+0000","last_up_change":"2024-08-17T17:20:02.834333+0000","last_in_change":"2024-08-17T17:19:46.093156+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:18:10.192831+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:20:37.124673+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"30c31adb-2cda-466b-933e-a8e2cb714fab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6803","nonce":2346469001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6805","nonce":2346469001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6809","nonce":2346469001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6807","nonce":2346469001}]},"public_addr":"172.21.15.130:6803/2346469001","cluster_addr":"172.21.15.130:6805/2346469001","heartbeat_back_addr":"172.21.15.130:6809/2346469001","heartbeat_front_addr":"172.21.15.130:6807/2346469001","state":["exists","up"]},{"osd":1,"uuid":"55049bb4-b976-42c6-8a5d-8b3e9ae20988","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6811","nonce":2284482082}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6813","nonce":2284482082}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6817","nonce":2284482082}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6815","nonce":2284482082}]},"public_addr":"172.21.15.130:6811/2284482082","cluster_addr":"172.21.15.130:6813/2284482082","heartbeat_back_addr":"172.21.15.130:6817/2284482082","heartbeat_front_addr":"172.21.15.130:6815/2284482082","state":["exists","up"]},{"osd":2,"uuid":"3f40f088-27f7-4450-8f31-b449711365ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6819","nonce":160441313}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6821","nonce":160441313}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6825","nonce":160441313}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6823","nonce":160441313}]},"public_addr":"172.21.15.130:6819/160441313","cluster_addr":"172.21.15.130:6821/160441313","heartbeat_back_addr":"172.21.15.130:6825/160441313","heartbeat_front_addr":"172.21.15.130:6823/160441313","state":["exists","up"]},{"osd":3,"uuid":"7684c401-462a-4e35-bb9c-1be23dfe573b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6827","nonce":775049303}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6829","nonce":775049303}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6833","nonce":775049303}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6831","nonce":775049303}]},"public_addr":"172.21.15.130:6827/775049303","cluster_addr":"172.21.15.130:6829/775049303","heartbeat_back_addr":"172.21.15.130:6833/775049303","heartbeat_front_addr":"172.21.15.130:6831/775049303","state":["exists","up"]},{"osd":4,"uuid":"7a3a331f-533a-49de-910d-5989f2f84b65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6800","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6801","nonce":2107797845}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6802","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6803","nonce":2107797845}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6806","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6807","nonce":2107797845}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6804","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6805","nonce":2107797845}]},"public_addr":"172.21.15.179:6801/2107797845","cluster_addr":"172.21.15.179:6803/2107797845","heartbeat_back_addr":"172.21.15.179:6807/2107797845","heartbeat_front_addr":"172.21.15.179:6805/2107797845","state":["exists","up"]},{"osd":5,"uuid":"d53d65ee-bbb2-45ec-b557-07cc62a45b05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6808","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6809","nonce":2392026049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6810","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6811","nonce":2392026049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6814","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6815","nonce":2392026049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6812","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6813","nonce":2392026049}]},"public_addr":"172.21.15.179:6809/2392026049","cluster_addr":"172.21.15.179:6811/2392026049","heartbeat_back_addr":"172.21.15.179:6815/2392026049","heartbeat_front_addr":"172.21.15.179:6813/2392026049","state":["exists","up"]},{"osd":6,"uuid":"b84229ad-2c88-4064-a070-33b98714ef96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6816","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6817","nonce":139895003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6818","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6819","nonce":139895003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6822","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6823","nonce":139895003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6820","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6821","nonce":139895003}]},"public_addr":"172.21.15.179:6817/139895003","cluster_addr":"172.21.15.179:6819/139895003","heartbeat_back_addr":"172.21.15.179:6823/139895003","heartbeat_front_addr":"172.21.15.179:6821/139895003","state":["exists","up"]},{"osd":7,"uuid":"bba7978f-8b19-40e6-86bd-692a0e282429","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6824","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6825","nonce":2533639201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6826","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6827","nonce":2533639201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6830","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6831","nonce":2533639201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6828","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6829","nonce":2533639201}]},"public_addr":"172.21.15.179:6825/2533639201","cluster_addr":"172.21.15.179:6827/2533639201","heartbeat_back_addr":"172.21.15.179:6831/2533639201","heartbeat_front_addr":"172.21.15.179:6829/2533639201","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:19.220776+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:42.876683+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:07.028539+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:31.259509+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:52.172235+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:15.112109+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:37.781476+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:20:00.893216+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/851371227":"2024-08-18T17:20:33.507974+0000","172.21.15.130:6801/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/1690071627":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6800/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/2999329360":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/3071443649":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/368845317":"2024-08-18T17:20:33.507974+0000","172.21.15.130:6800/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6801/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/88320168":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/3471387839":"2024-08-18T17:15:57.894717+0000","172.21.15.130:6800/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/1247136467":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2589122337":"2024-08-18T17:20:33.507974+0000","172.21.15.130:6801/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2578011752":"2024-08-18T17:20:33.507974+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:20:57.829 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T17:20:57.830 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T17:20:57.830 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T17:20:57.831 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T17:20:57.831 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T17:20:57.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T17:20:57.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T17:20:57.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T17:20:57.868 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:57 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/311795742' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:20:57.868 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:57 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/311795742' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:20:58.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:57 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/311795742' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:20:58.377 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:58.378 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:58.379 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:58.382 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:58.384 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:58.624 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:20:58 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[73466]: ts=2024-08-17T17:20:58.329Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001821416s 2024-08-17T17:20:58.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:20:58 smithi130 ceph-mon[31342]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:58.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:20:58 smithi130 ceph-mon[42749]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:20:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:20:58 smithi179 ceph-mon[34775]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:00.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:00 smithi130 ceph-mon[31342]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:00.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:00 smithi130 ceph-mon[42749]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:01.017 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:00 smithi179 ceph-mon[34775]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:02.277 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 podman[66097]: 2024-08-17 17:21:02.083650576 +0000 UTC m=+0.016606354 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-08-17T17:21:02.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 podman[66097]: 2024-08-17 17:21:02.399852782 +0000 UTC m=+0.332808561 container init 8ed57a40b09e83625d1afd3efc7d82d827799b789cd1733bf490dabb3c809eb2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a, vendor=Red Hat, Inc., maintainer=Paul Cuzner , distribution-scope=public, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git) 2024-08-17T17:21:02.651 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 podman[66097]: 2024-08-17 17:21:02.405485876 +0000 UTC m=+0.338441645 container start 8ed57a40b09e83625d1afd3efc7d82d827799b789cd1733bf490dabb3c809eb2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a, name=ubi8, description=Ceph Grafana Container, architecture=x86_64, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, vcs-type=git, vendor=Red Hat, Inc., io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public) 2024-08-17T17:21:02.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=settings t=2024-08-17T17:21:02.554001586Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-08-17T17:21:02.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=settings t=2024-08-17T17:21:02.554209411Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-08-17T17:21:02.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=settings t=2024-08-17T17:21:02.554227777Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-08-17T17:21:02.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=settings t=2024-08-17T17:21:02.554234484Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-08-17T17:21:02.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=settings t=2024-08-17T17:21:02.554240452Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-08-17T17:21:02.652 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=settings t=2024-08-17T17:21:02.554246093Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-08-17T17:21:02.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=settings t=2024-08-17T17:21:02.554251513Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-08-17T17:21:02.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=settings t=2024-08-17T17:21:02.554256998Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-08-17T17:21:02.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=settings t=2024-08-17T17:21:02.554262934Z level=info msg="Path Home" path=/usr/share/grafana 2024-08-17T17:21:02.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=settings t=2024-08-17T17:21:02.554268331Z level=info msg="Path Data" path=/var/lib/grafana 2024-08-17T17:21:02.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=settings t=2024-08-17T17:21:02.554273532Z level=info msg="Path Logs" path=/var/log/grafana 2024-08-17T17:21:02.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=settings t=2024-08-17T17:21:02.554279515Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-08-17T17:21:02.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=settings t=2024-08-17T17:21:02.55428491Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-08-17T17:21:02.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=settings t=2024-08-17T17:21:02.55429066Z level=info msg="App mode production" 2024-08-17T17:21:02.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=sqlstore t=2024-08-17T17:21:02.554355939Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-08-17T17:21:02.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=sqlstore t=2024-08-17T17:21:02.554371549Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-08-17T17:21:02.653 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.555218012Z level=info msg="Starting DB migrations" 2024-08-17T17:21:02.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.555292115Z level=info msg="Executing migration" id="create migration_log table" 2024-08-17T17:21:02.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.55582254Z level=info msg="Executing migration" id="create user table" 2024-08-17T17:21:02.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.556264082Z level=info msg="Executing migration" id="add unique index user.login" 2024-08-17T17:21:02.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.55894843Z level=info msg="Executing migration" id="add unique index user.email" 2024-08-17T17:21:02.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.561746817Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-08-17T17:21:02.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.564556157Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-08-17T17:21:02.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.567512056Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-08-17T17:21:02.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.570331758Z level=info msg="Executing migration" id="create user table v2" 2024-08-17T17:21:02.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.573196012Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-08-17T17:21:02.654 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 bash[66097]: 8ed57a40b09e83625d1afd3efc7d82d827799b789cd1733bf490dabb3c809eb2 2024-08-17T17:21:02.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.575836327Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-08-17T17:21:02.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 systemd[1]: Started Ceph grafana.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:21:02.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.578651434Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-08-17T17:21:02.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.581329954Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-08-17T17:21:02.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.581709542Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-08-17T17:21:02.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.584512143Z level=info msg="Executing migration" id="Update user table charset" 2024-08-17T17:21:02.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.58728254Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-08-17T17:21:02.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.590005242Z level=info msg="Executing migration" id="Add missing user data" 2024-08-17T17:21:02.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.590393131Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-08-17T17:21:02.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.593179546Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-08-17T17:21:02.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.593642075Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-08-17T17:21:02.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.596195755Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-08-17T17:21:02.655 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.598633829Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-08-17T17:21:02.656 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.599147333Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-08-17T17:21:02.656 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.601688149Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-08-17T17:21:02.656 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.604283927Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-08-17T17:21:02.656 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.604698415Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-08-17T17:21:02.656 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.607315733Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-08-17T17:21:02.656 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.609905588Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-08-17T17:21:02.656 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.612466425Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-08-17T17:21:02.657 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.614993327Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-08-17T17:21:02.657 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.617557731Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-08-17T17:21:02.657 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.620334849Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-08-17T17:21:02.657 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.623039185Z level=info msg="Executing migration" id="create temp_user v2" 2024-08-17T17:21:02.657 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.625532597Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-08-17T17:21:02.657 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.62808401Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-08-17T17:21:02.657 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.630676711Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-08-17T17:21:02.657 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.633249464Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-08-17T17:21:02.657 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.636017859Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-08-17T17:21:02.657 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.636605785Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-08-17T17:21:02.658 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.637472519Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-08-17T17:21:02.658 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.638356622Z level=info msg="Executing migration" id="create star table" 2024-08-17T17:21:02.658 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.639143548Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-08-17T17:21:02.658 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.639581142Z level=info msg="Executing migration" id="create org table v1" 2024-08-17T17:21:02.658 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.639997774Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-08-17T17:21:02.658 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.640417814Z level=info msg="Executing migration" id="create org_user table v1" 2024-08-17T17:21:02.658 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.64083315Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-08-17T17:21:02.658 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.641244734Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-08-17T17:21:02.658 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.641772346Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-08-17T17:21:02.658 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.64219903Z level=info msg="Executing migration" id="Update org table charset" 2024-08-17T17:21:02.658 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.642472295Z level=info msg="Executing migration" id="Update org_user table charset" 2024-08-17T17:21:02.658 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.642745386Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-08-17T17:21:02.658 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.643072894Z level=info msg="Executing migration" id="create dashboard table" 2024-08-17T17:21:02.659 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.643460936Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-08-17T17:21:02.659 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.643873815Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-08-17T17:21:02.659 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.644294427Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-08-17T17:21:02.659 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.645121379Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-08-17T17:21:02.659 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.645576362Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-08-17T17:21:02.659 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.646432483Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-08-17T17:21:02.659 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.647301073Z level=info msg="Executing migration" id="create dashboard v2" 2024-08-17T17:21:02.659 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.648173598Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-08-17T17:21:02.659 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.649027549Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-08-17T17:21:02.659 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.649998287Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-08-17T17:21:02.659 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.650900351Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-08-17T17:21:02.659 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.651552037Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-08-17T17:21:02.659 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.651862777Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-08-17T17:21:02.659 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.652737924Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-08-17T17:21:02.901 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-mon[34775]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:02.901 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:02.901 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:02.901 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:02.901 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:02.902 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.655471479Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-08-17T17:21:02.902 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.658393189Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-08-17T17:21:02.902 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.661141341Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-08-17T17:21:02.902 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.664015352Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-08-17T17:21:02.903 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.666888923Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-08-17T17:21:02.903 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.669657846Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-08-17T17:21:02.903 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.672274069Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-08-17T17:21:02.903 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.672595884Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-08-17T17:21:02.903 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.673130607Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-08-17T17:21:02.903 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.676603923Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-08-17T17:21:02.903 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.679392072Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-08-17T17:21:02.903 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.682113797Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-08-17T17:21:02.903 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.685068628Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-08-17T17:21:02.903 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.688004756Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-08-17T17:21:02.903 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.690769546Z level=info msg="Executing migration" id="Update dashboard title length" 2024-08-17T17:21:02.903 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.693659548Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-08-17T17:21:02.903 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.696540709Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.699353367Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.702246852Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.705003332Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.707550906Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.710039165Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.71277797Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.715554693Z level=info msg="Executing migration" id="Add check_sum column" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.716281056Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.719020389Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.721649245Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.724350602Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.726951596Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.729612163Z level=info msg="Executing migration" id="create data_source table" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.732401305Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-08-17T17:21:02.904 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.734832966Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-08-17T17:21:02.905 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.735259792Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-08-17T17:21:02.905 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.737903899Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-08-17T17:21:02.905 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.738345504Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-08-17T17:21:02.905 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.740806271Z level=info msg="Executing migration" id="create data_source table v2" 2024-08-17T17:21:02.905 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.74323999Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-08-17T17:21:02.905 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.745800958Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-08-17T17:21:02.905 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.746483863Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-08-17T17:21:02.905 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.747285923Z level=info msg="Executing migration" id="Add column with_credentials" 2024-08-17T17:21:02.905 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.747871736Z level=info msg="Executing migration" id="Add secure json data column" 2024-08-17T17:21:02.905 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.748436695Z level=info msg="Executing migration" id="Update data_source table charset" 2024-08-17T17:21:02.905 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.749292407Z level=info msg="Executing migration" id="Update initial version to 1" 2024-08-17T17:21:02.905 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.749640565Z level=info msg="Executing migration" id="Add read_only data column" 2024-08-17T17:21:02.905 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.750176535Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-08-17T17:21:02.906 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.751026211Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-08-17T17:21:02.906 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.752006489Z level=info msg="Executing migration" id="Add uid column" 2024-08-17T17:21:02.906 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.752750785Z level=info msg="Executing migration" id="Update uid value" 2024-08-17T17:21:02.906 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.753735764Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-08-17T17:21:02.906 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.754419178Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-08-17T17:21:02.906 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.755414389Z level=info msg="Executing migration" id="create api_key table" 2024-08-17T17:21:02.906 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.756359166Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-08-17T17:21:02.906 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.757028154Z level=info msg="Executing migration" id="add index api_key.key" 2024-08-17T17:21:02.906 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.757876566Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-08-17T17:21:02.906 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.75888946Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-08-17T17:21:02.906 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.759605349Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-08-17T17:21:02.906 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.760382164Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-08-17T17:21:02.906 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.76077415Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-08-17T17:21:02.907 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.761814746Z level=info msg="Executing migration" id="create api_key table v2" 2024-08-17T17:21:02.907 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.762697851Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-08-17T17:21:02.907 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.763579487Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-08-17T17:21:02.907 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.764002497Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-08-17T17:21:02.907 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.765030637Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-08-17T17:21:02.907 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.765688209Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-08-17T17:21:02.907 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.766044046Z level=info msg="Executing migration" id="Update api_key table charset" 2024-08-17T17:21:02.907 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.766334792Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-08-17T17:21:02.907 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.76728553Z level=info msg="Executing migration" id="Add service account foreign key" 2024-08-17T17:21:02.907 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.767977718Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-08-17T17:21:02.907 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.768289631Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-08-17T17:21:02.907 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.769066588Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-08-17T17:21:02.907 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.769653995Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-08-17T17:21:02.907 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.770049844Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-08-17T17:21:02.908 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.770911614Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-08-17T17:21:02.908 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.771751031Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-08-17T17:21:02.908 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.774638472Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-08-17T17:21:02.908 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.775087054Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-08-17T17:21:02.908 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.778018311Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-08-17T17:21:02.908 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.78084003Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-08-17T17:21:02.909 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.783553214Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-08-17T17:21:02.909 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.784252891Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-08-17T17:21:02.909 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.787169379Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-08-17T17:21:02.909 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.789837895Z level=info msg="Executing migration" id="create quota table v1" 2024-08-17T17:21:02.909 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.792593268Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-08-17T17:21:02.909 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.793070899Z level=info msg="Executing migration" id="Update quota table charset" 2024-08-17T17:21:02.909 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.795879225Z level=info msg="Executing migration" id="create plugin_setting table" 2024-08-17T17:21:02.909 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.79633807Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-08-17T17:21:02.909 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.799113634Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-08-17T17:21:02.909 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.79977875Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-08-17T17:21:02.909 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.802655691Z level=info msg="Executing migration" id="create session table" 2024-08-17T17:21:02.909 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.803191117Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-08-17T17:21:02.909 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.803515842Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-08-17T17:21:02.910 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.803811006Z level=info msg="Executing migration" id="create playlist table v2" 2024-08-17T17:21:02.910 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.80612549Z level=info msg="Executing migration" id="create playlist item table v2" 2024-08-17T17:21:02.910 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.808850374Z level=info msg="Executing migration" id="Update playlist table charset" 2024-08-17T17:21:02.910 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.811497565Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-08-17T17:21:02.910 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.811945901Z level=info msg="Executing migration" id="drop preferences table v2" 2024-08-17T17:21:02.910 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.812251104Z level=info msg="Executing migration" id="drop preferences table v3" 2024-08-17T17:21:02.910 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.814832799Z level=info msg="Executing migration" id="create preferences table v3" 2024-08-17T17:21:02.910 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.815398878Z level=info msg="Executing migration" id="Update preferences table charset" 2024-08-17T17:21:02.910 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.817422233Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-08-17T17:21:02.910 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.818316045Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-08-17T17:21:02.910 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.820096211Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-08-17T17:21:02.910 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.820763708Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-08-17T17:21:02.910 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.823280288Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-08-17T17:21:02.910 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.826047086Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-08-17T17:21:02.911 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.82869273Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-08-17T17:21:02.911 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.829223035Z level=info msg="Executing migration" id="create alert table v1" 2024-08-17T17:21:02.911 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.831714069Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-08-17T17:21:02.911 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.834295944Z level=info msg="Executing migration" id="add index alert state" 2024-08-17T17:21:02.911 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.836971461Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-08-17T17:21:02.911 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.837575396Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-08-17T17:21:02.911 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.840270983Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-08-17T17:21:02.911 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.840980566Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-08-17T17:21:02.911 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.841846404Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-08-17T17:21:02.911 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.843276502Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-08-17T17:21:02.911 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.844104038Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-08-17T17:21:02.911 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.845078647Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-08-17T17:21:02.912 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.845770462Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-08-17T17:21:02.912 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.846129649Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-08-17T17:21:02.912 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.846992047Z level=info msg="Executing migration" id="Add column is_default" 2024-08-17T17:21:02.912 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.847778989Z level=info msg="Executing migration" id="Add column frequency" 2024-08-17T17:21:02.912 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.848435211Z level=info msg="Executing migration" id="Add column send_reminder" 2024-08-17T17:21:02.912 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.849294307Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-08-17T17:21:02.912 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.849958615Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-08-17T17:21:02.912 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.850825197Z level=info msg="Executing migration" id="Update alert table charset" 2024-08-17T17:21:02.912 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.851711706Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-08-17T17:21:02.912 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.852463908Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-08-17T17:21:02.912 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.852875324Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-08-17T17:21:02.912 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.853940097Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-08-17T17:21:02.912 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.854614713Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-08-17T17:21:02.913 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.855578003Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-08-17T17:21:02.913 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.856156657Z level=info msg="Executing migration" id="Add for to alert table" 2024-08-17T17:21:02.913 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.857032444Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-08-17T17:21:02.913 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.857912249Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-08-17T17:21:02.913 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.858933067Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-08-17T17:21:02.913 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.859477437Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-08-17T17:21:02.913 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.860312968Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-08-17T17:21:02.913 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.860958427Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-08-17T17:21:02.913 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.861255516Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-08-17T17:21:02.913 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.862095163Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-08-17T17:21:02.913 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.862523106Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-08-17T17:21:02.913 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.862799305Z level=info msg="Executing migration" id="create annotation table v5" 2024-08-17T17:21:02.913 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.863276286Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-08-17T17:21:02.914 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.863734939Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-08-17T17:21:02.914 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.86460176Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-08-17T17:21:02.914 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.865455864Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-08-17T17:21:02.914 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.86593727Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-08-17T17:21:02.914 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.868697224Z level=info msg="Executing migration" id="Update annotation table charset" 2024-08-17T17:21:02.914 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.871486004Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-08-17T17:21:02.914 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.874124071Z level=info msg="Executing migration" id="Drop category_id index" 2024-08-17T17:21:02.914 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.87458684Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-08-17T17:21:02.914 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.877649773Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-08-17T17:21:02.914 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.880309584Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-08-17T17:21:02.914 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.88300683Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-08-17T17:21:02.914 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.883435209Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-08-17T17:21:02.914 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.88610071Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-08-17T17:21:02.915 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.888895623Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-08-17T17:21:02.915 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.891622061Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-08-17T17:21:02.915 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.892004312Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-08-17T17:21:02.915 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.894767846Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-08-17T17:21:02.915 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.897584835Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-08-17T17:21:02.915 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.90035583Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-08-17T17:21:02.915 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.901136165Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-08-17T17:21:02.915 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.904014051Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-08-17T17:21:03.126 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:02 smithi130 ceph-mon[31342]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:03.126 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:02 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:03.127 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:02 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:03.127 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:02 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:03.127 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:02 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:03.127 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:02 smithi130 ceph-mon[42749]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:03.127 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:02 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:03.127 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:02 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:03.127 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:02 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:03.127 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:02 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:03.153 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.906873855Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-08-17T17:21:03.153 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.909441605Z level=info msg="Executing migration" id="Add epoch_end column" 2024-08-17T17:21:03.153 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.910183448Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-08-17T17:21:03.153 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.912795202Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-08-17T17:21:03.153 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.915548552Z level=info msg="Executing migration" id="Move region to single row" 2024-08-17T17:21:03.153 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.917960643Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-08-17T17:21:03.153 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.91840481Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-08-17T17:21:03.154 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.920985374Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-08-17T17:21:03.154 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.92143977Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-08-17T17:21:03.154 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.923846107Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-08-17T17:21:03.154 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.924279932Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-08-17T17:21:03.154 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.926451743Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-08-17T17:21:03.154 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.92686748Z level=info msg="Executing migration" id="create test_data table" 2024-08-17T17:21:03.154 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.927291175Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-08-17T17:21:03.154 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.929870197Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-08-17T17:21:03.154 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.932603581Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-08-17T17:21:03.154 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.935087135Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-08-17T17:21:03.154 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.93554089Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.936359208Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.936702707Z level=info msg="Executing migration" id="create team table" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.937232218Z level=info msg="Executing migration" id="add index team.org_id" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.938097131Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.938571782Z level=info msg="Executing migration" id="create team member table" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.93941012Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.940380799Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.94104855Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.941514916Z level=info msg="Executing migration" id="Add column email to team table" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.942384552Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.943193332Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.944219349Z level=info msg="Executing migration" id="create dashboard acl table" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.944871859Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.945645279Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-08-17T17:21:03.155 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.946504795Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-08-17T17:21:03.156 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.947394477Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-08-17T17:21:03.156 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.948024666Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-08-17T17:21:03.156 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.948979704Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-08-17T17:21:03.156 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.949705002Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-08-17T17:21:03.157 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.950587201Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-08-17T17:21:03.157 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.95154764Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-08-17T17:21:03.157 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.952312847Z level=info msg="Executing migration" id="create tag table" 2024-08-17T17:21:03.157 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.953264883Z level=info msg="Executing migration" id="add index tag.key_value" 2024-08-17T17:21:03.157 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.954032997Z level=info msg="Executing migration" id="create login attempt table" 2024-08-17T17:21:03.157 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.954920971Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-08-17T17:21:03.157 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.956018698Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-08-17T17:21:03.158 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.956651348Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-08-17T17:21:03.158 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.958570964Z level=info msg="Executing migration" id="create login_attempt v2" 2024-08-17T17:21:03.158 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.959443626Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-08-17T17:21:03.158 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.960478389Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-08-17T17:21:03.158 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.961200302Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-08-17T17:21:03.158 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.962147879Z level=info msg="Executing migration" id="create user auth table" 2024-08-17T17:21:03.158 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.96302426Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-08-17T17:21:03.158 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.96380134Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-08-17T17:21:03.158 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.964820679Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-08-17T17:21:03.158 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.967469242Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-08-17T17:21:03.158 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.970224868Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-08-17T17:21:03.158 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.972997036Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-08-17T17:21:03.158 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.975903985Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-08-17T17:21:03.159 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.978596955Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-08-17T17:21:03.159 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.981529609Z level=info msg="Executing migration" id="create server_lock table" 2024-08-17T17:21:03.159 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.984477366Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-08-17T17:21:03.159 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.987272482Z level=info msg="Executing migration" id="create user auth token table" 2024-08-17T17:21:03.159 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.989967845Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-08-17T17:21:03.159 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.992852116Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-08-17T17:21:03.159 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.995834994Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-08-17T17:21:03.159 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:02.998561327Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-08-17T17:21:03.159 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.00131421Z level=info msg="Executing migration" id="create cache_data table" 2024-08-17T17:21:03.159 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.004117741Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-08-17T17:21:03.159 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.006932946Z level=info msg="Executing migration" id="create short_url table v1" 2024-08-17T17:21:03.159 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.009631795Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-08-17T17:21:03.159 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.012571745Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-08-17T17:21:03.159 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.013019051Z level=info msg="Executing migration" id="delete alert_definition table" 2024-08-17T17:21:03.160 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.013306622Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-08-17T17:21:03.160 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.013777003Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-08-17T17:21:03.160 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.01455302Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-08-17T17:21:03.160 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.015026057Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-08-17T17:21:03.160 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.015325105Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-08-17T17:21:03.160 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.01572559Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-08-17T17:21:03.160 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.016177257Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-08-17T17:21:03.160 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.016615701Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-08-17T17:21:03.160 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.017058302Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-08-17T17:21:03.160 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.017863019Z level=info msg="Executing migration" id="drop alert_definition table" 2024-08-17T17:21:03.160 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.018324912Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-08-17T17:21:03.160 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.018610081Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-08-17T17:21:03.161 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.019145725Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-08-17T17:21:03.161 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.019590191Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-08-17T17:21:03.161 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.020045971Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-08-17T17:21:03.161 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.020353968Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-08-17T17:21:03.161 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.020773479Z level=info msg="Executing migration" id="create alert_instance table" 2024-08-17T17:21:03.161 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.021239121Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-08-17T17:21:03.161 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.02164419Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-08-17T17:21:03.161 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.022086289Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-08-17T17:21:03.161 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.022871263Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-08-17T17:21:03.161 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.023302257Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-08-17T17:21:03.161 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.02370131Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-08-17T17:21:03.161 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.02708511Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-08-17T17:21:03.161 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.030460655Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-08-17T17:21:03.161 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.030915601Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.03139477Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.032245021Z level=info msg="Executing migration" id="create alert_rule table" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.032681143Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.033174509Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.033610275Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.034063373Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.034362136Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.035187873Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.036024545Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.036842898Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.037303096Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.03770986Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.038547979Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.039384683Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-08-17T17:21:03.162 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.039841047Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-08-17T17:21:03.163 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.040692498Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-08-17T17:21:03.163 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.041535267Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-08-17T17:21:03.163 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.041875844Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-08-17T17:21:03.163 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.042359051Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-08-17T17:21:03.163 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.042808889Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-08-17T17:21:03.163 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.043278975Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-08-17T17:21:03.163 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.043564016Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-08-17T17:21:03.163 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.044461302Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-08-17T17:21:03.163 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.04534735Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-08-17T17:21:03.163 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.046214722Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-08-17T17:21:03.163 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.04709753Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-08-17T17:21:03.163 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.047945276Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-08-17T17:21:03.163 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.048840244Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-08-17T17:21:03.163 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.049718372Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.050579202Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.051343747Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.052264Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.052739908Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.053591369Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.054440629Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.055296127Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.056222458Z level=info msg="Executing migration" id="create provenance_type table" 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.057141307Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.058032364Z level=info msg="Executing migration" id="create alert_image table" 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.058978818Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.059672112Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.060720902Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.061365023Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-08-17T17:21:03.164 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.061777473Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.061866936Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.062153966Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.062621783Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.063090966Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.06322919Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.063356349Z level=info msg="alerts found to migrate" alerts=0 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.063928904Z level=info msg="Executing migration" id="create library_element table v1" 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.064810109Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.065664069Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.068614746Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.071282394Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.07425408Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.076990218Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.079841088Z level=info msg="Executing migration" id="create data_keys table" 2024-08-17T17:21:03.165 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.082738737Z level=info msg="Executing migration" id="create secrets table" 2024-08-17T17:21:03.166 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.085313803Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-08-17T17:21:03.166 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.089512457Z level=info msg="Executing migration" id="add name column into data_keys" 2024-08-17T17:21:03.166 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.092300539Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-08-17T17:21:03.166 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.095151652Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-08-17T17:21:03.166 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.099432699Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-08-17T17:21:03.166 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.103636203Z level=info msg="Executing migration" id="create kv_store table v1" 2024-08-17T17:21:03.166 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.106513491Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-08-17T17:21:03.166 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.109437644Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-08-17T17:21:03.166 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.112314083Z level=info msg="Executing migration" id="create permission table" 2024-08-17T17:21:03.166 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.115234423Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-08-17T17:21:03.166 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.118066613Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-08-17T17:21:03.166 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.120689865Z level=info msg="Executing migration" id="create role table" 2024-08-17T17:21:03.166 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.123517363Z level=info msg="Executing migration" id="add column display_name" 2024-08-17T17:21:03.166 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.126040522Z level=info msg="Executing migration" id="add column group_name" 2024-08-17T17:21:03.167 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.128635197Z level=info msg="Executing migration" id="add index role.org_id" 2024-08-17T17:21:03.167 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.131428246Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-08-17T17:21:03.167 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.134216762Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-08-17T17:21:03.167 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.13685179Z level=info msg="Executing migration" id="create team role table" 2024-08-17T17:21:03.167 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.139622967Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-08-17T17:21:03.167 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.142186437Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-08-17T17:21:03.167 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.14486496Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-08-17T17:21:03.167 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.147332126Z level=info msg="Executing migration" id="create user role table" 2024-08-17T17:21:03.167 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.150261993Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-08-17T17:21:03.167 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.152594874Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-08-17T17:21:03.167 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.155205874Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-08-17T17:21:03.593 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.15794569Z level=info msg="Executing migration" id="create builtin role table" 2024-08-17T17:21:03.593 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.1608059Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-08-17T17:21:03.593 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.163358118Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-08-17T17:21:03.593 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.165909679Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-08-17T17:21:03.594 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.166902646Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-08-17T17:21:03.594 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.167421883Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-08-17T17:21:03.594 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.167876513Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-08-17T17:21:03.594 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.168286984Z level=info msg="Executing migration" id="add unique index role.uid" 2024-08-17T17:21:03.594 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.168683516Z level=info msg="Executing migration" id="create seed assignment table" 2024-08-17T17:21:03.594 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.169073309Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-08-17T17:21:03.594 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.16952836Z level=info msg="Executing migration" id="add column hidden to role table" 2024-08-17T17:21:03.594 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.170492852Z level=info msg="Executing migration" id="create query_history table v1" 2024-08-17T17:21:03.594 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.170886305Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-08-17T17:21:03.594 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.171309477Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-08-17T17:21:03.594 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.171575001Z level=info msg="Executing migration" id="teams permissions migration" 2024-08-17T17:21:03.594 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.171990027Z level=info msg="Executing migration" id="dashboard permissions" 2024-08-17T17:21:03.595 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.172424981Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-08-17T17:21:03.595 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.172761386Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-08-17T17:21:03.595 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.173068604Z level=info msg="Executing migration" id="alerting notification permissions" 2024-08-17T17:21:03.595 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.173384144Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-08-17T17:21:03.595 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.173762545Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-08-17T17:21:03.595 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.174197148Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-08-17T17:21:03.595 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.175170926Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-08-17T17:21:03.595 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.175465831Z level=info msg="Executing migration" id="create correlation table v1" 2024-08-17T17:21:03.595 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.175887172Z level=info msg="Executing migration" id="add index correlations.uid" 2024-08-17T17:21:03.595 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.176330018Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-08-17T17:21:03.595 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.176744025Z level=info msg="Executing migration" id="add correlation config column" 2024-08-17T17:21:03.595 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.177710134Z level=info msg="Executing migration" id="create entity_events table" 2024-08-17T17:21:03.595 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.178129015Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-08-17T17:21:03.596 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.178869889Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-08-17T17:21:03.596 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.178950387Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-08-17T17:21:03.596 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.179254021Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-17T17:21:03.596 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.17936133Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-17T17:21:03.596 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.179647769Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-08-17T17:21:03.596 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.180112504Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-08-17T17:21:03.596 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.180549056Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-08-17T17:21:03.596 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.181023306Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.181476623Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.184173325Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.186913875Z level=info msg="Executing migration" id="Drop public config table" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.189570979Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.192226163Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.194918252Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.197568711Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.200181707Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.203206062Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.206040513Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.208286585Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.210983837Z level=info msg="Executing migration" id="add share column" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.213592243Z level=info msg="Executing migration" id="create default alerting folders" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.216292427Z level=info msg="Executing migration" id="create file table" 2024-08-17T17:21:03.597 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.218767729Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.219617388Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.220542543Z level=info msg="Executing migration" id="create file_meta table" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.221246638Z level=info msg="Executing migration" id="file table idx: path key" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.22213728Z level=info msg="Executing migration" id="set path collation in file table" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.222898115Z level=info msg="Executing migration" id="managed permissions migration" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.223839571Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.224571353Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.225387576Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.2264569Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.227322555Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.228286809Z level=info msg="Executing migration" id="update group index for alert rules" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.229076915Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.229886744Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.230793131Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-08-17T17:21:03.598 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.231833923Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.23288025Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.233893596Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.243842491Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.244702856Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.245557455Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.248845382Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.249756429Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.250633406Z level=info msg="Executing migration" id="create folder table" 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.25143729Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.252435964Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.253319184Z level=info msg="Executing migration" id="Update folder title length" 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.254032923Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=migrator t=2024-08-17T17:21:03.254901863Z level=info msg="migrations completed" performed=485 skipped=0 duration=699.62892ms 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=sqlstore t=2024-08-17T17:21:03.255326905Z level=info msg="Created default organization" 2024-08-17T17:21:03.599 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=plugin.loader t=2024-08-17T17:21:03.311695504Z level=info msg="Plugin registered" pluginID=input 2024-08-17T17:21:03.600 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=plugin.loader t=2024-08-17T17:21:03.339092624Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-08-17T17:21:03.600 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=plugin.loader t=2024-08-17T17:21:03.339111654Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-08-17T17:21:03.600 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=secrets t=2024-08-17T17:21:03.33914361Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-08-17T17:21:03.600 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=query_data t=2024-08-17T17:21:03.340753461Z level=info msg="Query Service initialization" 2024-08-17T17:21:03.600 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=live.push_http t=2024-08-17T17:21:03.343013522Z level=info msg="Live Push Gateway initialization" 2024-08-17T17:21:03.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:03 smithi130 ceph-mon[31342]: Deploying daemon node-exporter.a on smithi130 2024-08-17T17:21:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:03 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:03 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:21:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:03 smithi130 ceph-mon[42749]: Deploying daemon node-exporter.a on smithi130 2024-08-17T17:21:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:03 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:03 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:21:04.051 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-mon[34775]: Deploying daemon node-exporter.a on smithi130 2024-08-17T17:21:04.051 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:04.051 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:03 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:21:04.343 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=infra.usagestats.collector t=2024-08-17T17:21:04.051037884Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-08-17T17:21:04.343 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=provisioning.datasources t=2024-08-17T17:21:04.051428808Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-08-17T17:21:04.343 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=provisioning.datasources t=2024-08-17T17:21:04.085033309Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-08-17T17:21:04.343 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=provisioning.datasources t=2024-08-17T17:21:04.116744607Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-08-17T17:21:04.344 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=provisioning.alerting t=2024-08-17T17:21:04.138212833Z level=info msg="starting to provision alerting" 2024-08-17T17:21:04.344 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=provisioning.alerting t=2024-08-17T17:21:04.138232661Z level=info msg="finished to provision alerting" 2024-08-17T17:21:04.344 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=ngalert.state.manager t=2024-08-17T17:21:04.13884986Z level=info msg="Warming state cache for startup" 2024-08-17T17:21:04.344 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=grafanaStorageLogger t=2024-08-17T17:21:04.139257096Z level=info msg="storage starting" 2024-08-17T17:21:04.344 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=http.server t=2024-08-17T17:21:04.144064807Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-08-17T17:21:04.344 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=ngalert.state.manager t=2024-08-17T17:21:04.150283648Z level=info msg="State cache has been initialized" states=0 duration=11.432474ms 2024-08-17T17:21:04.344 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=ticker t=2024-08-17T17:21:04.150342447Z level=info msg=starting first_tick=2024-08-17T17:21:10Z 2024-08-17T17:21:04.344 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=ngalert.multiorg.alertmanager t=2024-08-17T17:21:04.150343431Z level=info msg="starting MultiOrg Alertmanager" 2024-08-17T17:21:04.344 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=sqlstore.transactions t=2024-08-17T17:21:04.183283107Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T17:21:04.344 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=sqlstore.transactions t=2024-08-17T17:21:04.223813668Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T17:21:04.344 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[66111]: logger=sqlstore.transactions t=2024-08-17T17:21:04.235035995Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-08-17T17:21:04.726 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:04 smithi130 ceph-mon[31342]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:04.727 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:04 smithi130 ceph-mon[42749]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-mon[34775]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:04.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:21:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:21:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:21:07.015 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:06 smithi130 ceph-mon[31342]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:07.015 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:06 smithi130 ceph-mon[42749]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:07.044 INFO:teuthology.orchestra.run.smithi130.stdout:128849018908 2024-08-17T17:21:07.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T17:21:07.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:06 smithi179 ceph-mon[34775]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:09.003 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:08 smithi130 ceph-mon[31342]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:09.003 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:08 smithi130 ceph-mon[42749]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:08 smithi179 ceph-mon[34775]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:10.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:10 smithi130 ceph-mon[31342]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:10 smithi130 ceph-mon[42749]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:10 smithi179 ceph-mon[34775]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:11.428 INFO:teuthology.orchestra.run.smithi130.stdout:55834574892 2024-08-17T17:21:11.428 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T17:21:12.049 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:11 smithi130 systemd[1]: Starting Ceph node-exporter.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:21:12.071 INFO:teuthology.orchestra.run.smithi130.stdout:197568495635 2024-08-17T17:21:12.072 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T17:21:12.584 INFO:teuthology.orchestra.run.smithi130.stdout:163208757273 2024-08-17T17:21:12.585 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T17:21:12.912 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:12 smithi130 ceph-mon[31342]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:12.912 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:12 smithi130 ceph-mon[42749]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:12 smithi179 ceph-mon[34775]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:13.647 INFO:teuthology.orchestra.run.smithi130.stdout:34359738417 2024-08-17T17:21:13.648 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T17:21:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:14 smithi179 ceph-mon[34775]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:14.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:21:14 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:21:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:21:14.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:14 smithi130 ceph-mon[31342]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:14.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:14 smithi130 ceph-mon[42749]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:15.035 INFO:teuthology.orchestra.run.smithi130.stdout:231928234001 2024-08-17T17:21:15.035 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T17:21:15.230 INFO:teuthology.orchestra.run.smithi130.stdout:77309411366 2024-08-17T17:21:15.230 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T17:21:16.013 INFO:teuthology.orchestra.run.smithi130.stdout:107374182435 2024-08-17T17:21:16.013 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T17:21:16.087 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:16.099 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:16.100 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:16.104 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:16.327 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:16 smithi130 ceph-mon[31342]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:16 smithi130 ceph-mon[42749]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:17.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:16 smithi179 ceph-mon[34775]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:19.042 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:18 smithi130 ceph-mon[31342]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:19.042 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:18 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:21:19.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:18 smithi130 ceph-mon[42749]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:19.043 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:18 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:21:19.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:18 smithi179 ceph-mon[34775]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:18 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:21:20.008 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:19 smithi130 bash[76618]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-08-17T17:21:20.261 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:20 smithi130 bash[76618]: Getting image source signatures 2024-08-17T17:21:20.262 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:20 smithi130 bash[76618]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-08-17T17:21:20.262 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:20 smithi130 bash[76618]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-08-17T17:21:20.262 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:20 smithi130 bash[76618]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-08-17T17:21:20.372 INFO:teuthology.orchestra.run.smithi130.stdout:128849018911 2024-08-17T17:21:20.718 INFO:teuthology.orchestra.run.smithi130.stdout:163208757276 2024-08-17T17:21:20.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:20 smithi130 ceph-mon[31342]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:20.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:20 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2017058996' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T17:21:20.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:20 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2383521921' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T17:21:20.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:20 smithi130 ceph-mon[42749]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:20.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:20 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2017058996' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T17:21:20.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:20 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2383521921' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T17:21:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:20 smithi179 ceph-mon[34775]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:20 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2017058996' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T17:21:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:20 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2383521921' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T17:21:22.195 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:22.592 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:22 smithi130 bash[76618]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-08-17T17:21:22.593 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:22 smithi130 bash[76618]: Writing manifest to image destination 2024-08-17T17:21:22.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:22 smithi130 ceph-mon[31342]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:22.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:22 smithi130 ceph-mon[42749]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:23.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:22 smithi179 ceph-mon[34775]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:23.416 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 podman[76618]: 2024-08-17 17:21:23.092354226 +0000 UTC m=+4.689959811 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-08-17T17:21:23.535 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757273 got 163208757276 for osd.5 2024-08-17T17:21:23.535 DEBUG:teuthology.parallel:result is None 2024-08-17T17:21:23.745 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:23.829 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 podman[76618]: 2024-08-17 17:21:23.627045794 +0000 UTC m=+5.224651437 container create a24247d0fd94a2b5031a84685e17a8a6cd83e11e79e74a54ea3dcf2c53ca376f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:24.108 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 podman[76618]: 2024-08-17 17:21:23.913729056 +0000 UTC m=+5.511334641 container init a24247d0fd94a2b5031a84685e17a8a6cd83e11e79e74a54ea3dcf2c53ca376f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:24.109 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 podman[76618]: 2024-08-17 17:21:23.918674597 +0000 UTC m=+5.516280172 container start a24247d0fd94a2b5031a84685e17a8a6cd83e11e79e74a54ea3dcf2c53ca376f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:24.109 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.922Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-08-17T17:21:24.109 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.922Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-08-17T17:21:24.109 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T17:21:24.109 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T17:21:24.109 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T17:21:24.109 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T17:21:24.109 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T17:21:24.109 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T17:21:24.109 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T17:21:24.109 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T17:21:24.110 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T17:21:24.110 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T17:21:24.110 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T17:21:24.110 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T17:21:24.110 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T17:21:24.110 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T17:21:24.110 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T17:21:24.110 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T17:21:24.111 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T17:21:24.111 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T17:21:24.111 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T17:21:24.111 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T17:21:24.111 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T17:21:24.111 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T17:21:24.111 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T17:21:24.111 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T17:21:24.111 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T17:21:24.111 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T17:21:24.111 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T17:21:24.111 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T17:21:24.111 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T17:21:24.112 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T17:21:24.112 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T17:21:24.112 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T17:21:24.112 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T17:21:24.112 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T17:21:24.112 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T17:21:24.112 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T17:21:24.112 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T17:21:24.112 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T17:21:24.112 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T17:21:24.112 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T17:21:24.112 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T17:21:24.112 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T17:21:24.112 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T17:21:24.113 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T17:21:24.113 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T17:21:24.113 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T17:21:24.113 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T17:21:24.113 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T17:21:24.113 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.923Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T17:21:24.113 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.924Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-08-17T17:21:24.113 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[76966]: ts=2024-08-17T17:21:23.924Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T17:21:24.113 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:23 smithi130 bash[76618]: a24247d0fd94a2b5031a84685e17a8a6cd83e11e79e74a54ea3dcf2c53ca376f 2024-08-17T17:21:24.113 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:21:24 smithi130 systemd[1]: Started Ceph node-exporter.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:21:24.553 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018908 got 128849018911 for osd.4 2024-08-17T17:21:24.553 DEBUG:teuthology.parallel:result is None 2024-08-17T17:21:24.769 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:21:24 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:21:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:21:25.060 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:24 smithi179 bash[66393]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-08-17T17:21:25.149 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:25 smithi130 ceph-mon[31342]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:25.149 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:25 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:25.149 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:25 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:25.149 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:25 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:25.149 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:25 smithi130 ceph-mon[31342]: Deploying daemon node-exporter.b on smithi179 2024-08-17T17:21:25.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:25 smithi130 ceph-mon[42749]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:25.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:25 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:25.149 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:25 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:25.150 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:25 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:25.150 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:25 smithi130 ceph-mon[42749]: Deploying daemon node-exporter.b on smithi179 2024-08-17T17:21:25.316 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:25 smithi179 ceph-mon[34775]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:25.317 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:25 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:25.317 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:25 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:25.317 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:25 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:25.317 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:25 smithi179 ceph-mon[34775]: Deploying daemon node-exporter.b on smithi179 2024-08-17T17:21:25.593 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:25 smithi179 bash[66393]: Getting image source signatures 2024-08-17T17:21:25.593 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:25 smithi179 bash[66393]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-08-17T17:21:25.593 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:25 smithi179 bash[66393]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-08-17T17:21:25.593 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:25 smithi179 bash[66393]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-08-17T17:21:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:26 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/284122696' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T17:21:26.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:26 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/284122696' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T17:21:26.338 INFO:teuthology.orchestra.run.smithi130.stdout:77309411368 2024-08-17T17:21:26.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:26 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/284122696' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T17:21:26.957 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411366 got 77309411368 for osd.2 2024-08-17T17:21:26.957 DEBUG:teuthology.parallel:result is None 2024-08-17T17:21:27.067 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:26 smithi179 bash[66393]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-08-17T17:21:27.067 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:26 smithi179 bash[66393]: Writing manifest to image destination 2024-08-17T17:21:27.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:27 smithi130 ceph-mon[31342]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:27.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:27 smithi130 ceph-mon[42749]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:27.328 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-mon[34775]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:27.328 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 podman[66393]: 2024-08-17 17:21:27.15393621 +0000 UTC m=+2.282933595 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-08-17T17:21:27.328 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 podman[66393]: 2024-08-17 17:21:27.235999522 +0000 UTC m=+2.364996947 container create ed661fd8fe9402a13befb1fcf40f18787ca9472e6f8380f76ccdcc80903dadba (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:21:27.843 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 podman[66393]: 2024-08-17 17:21:27.454703353 +0000 UTC m=+2.583700751 container init ed661fd8fe9402a13befb1fcf40f18787ca9472e6f8380f76ccdcc80903dadba (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:21:27.844 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 podman[66393]: 2024-08-17 17:21:27.459551921 +0000 UTC m=+2.588549304 container start ed661fd8fe9402a13befb1fcf40f18787ca9472e6f8380f76ccdcc80903dadba (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:21:27.844 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.463Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-08-17T17:21:27.844 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.463Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-08-17T17:21:27.844 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.463Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T17:21:27.844 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.463Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T17:21:27.844 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T17:21:27.844 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T17:21:27.844 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T17:21:27.844 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T17:21:27.844 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T17:21:27.845 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T17:21:27.845 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T17:21:27.845 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T17:21:27.845 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T17:21:27.845 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T17:21:27.845 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T17:21:27.845 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T17:21:27.845 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T17:21:27.846 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T17:21:27.846 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T17:21:27.846 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T17:21:27.846 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T17:21:27.846 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T17:21:27.846 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T17:21:27.846 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T17:21:27.846 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T17:21:27.846 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T17:21:27.846 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T17:21:27.846 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T17:21:27.847 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T17:21:27.847 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T17:21:27.847 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T17:21:27.847 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T17:21:27.847 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T17:21:27.847 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T17:21:27.847 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T17:21:27.847 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T17:21:27.847 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T17:21:27.847 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T17:21:27.847 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.464Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T17:21:27.847 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.465Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T17:21:27.847 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.465Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T17:21:27.848 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.465Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T17:21:27.848 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.465Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T17:21:27.848 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.465Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T17:21:27.848 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.465Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T17:21:27.848 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.465Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T17:21:27.848 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.465Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T17:21:27.848 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.465Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T17:21:27.848 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.465Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T17:21:27.848 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.465Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T17:21:27.848 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.465Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T17:21:27.848 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.465Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-08-17T17:21:27.848 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[66445]: ts=2024-08-17T17:21:27.465Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T17:21:27.849 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 bash[66393]: ed661fd8fe9402a13befb1fcf40f18787ca9472e6f8380f76ccdcc80903dadba 2024-08-17T17:21:27.849 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:21:27 smithi179 systemd[1]: Started Ceph node-exporter.b for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:21:28.731 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[31342]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:28.731 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.731 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.731 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.731 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.731 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.731 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:21:28.731 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[42749]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:28.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:21:28.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:28 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:28 smithi179 ceph-mon[34775]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:28 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:28 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:28 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:28 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:28 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:28 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:21:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:28 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:29.506 INFO:teuthology.orchestra.run.smithi130.stdout:34359738420 2024-08-17T17:21:29.714 INFO:teuthology.orchestra.run.smithi130.stdout:197568495640 2024-08-17T17:21:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:29 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2253699279' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T17:21:29.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:29 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2253699279' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T17:21:29.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:29 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2253699279' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T17:21:29.935 INFO:teuthology.orchestra.run.smithi130.stdout:55834574896 2024-08-17T17:21:30.502 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495635 got 197568495640 for osd.6 2024-08-17T17:21:30.502 DEBUG:teuthology.parallel:result is None 2024-08-17T17:21:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:30 smithi179 ceph-mon[34775]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:30 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2542282682' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T17:21:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:30 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/833762189' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T17:21:30.880 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:30 smithi130 ceph-mon[31342]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:30.880 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:30 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2542282682' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T17:21:30.880 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:30 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/833762189' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T17:21:30.880 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:30 smithi130 ceph-mon[42749]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:30.880 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:30 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2542282682' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T17:21:30.881 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:30 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/833762189' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T17:21:30.980 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:31.133 INFO:teuthology.orchestra.run.smithi130.stdout:231928234004 2024-08-17T17:21:31.518 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574896 for osd.1 2024-08-17T17:21:31.519 DEBUG:teuthology.parallel:result is None 2024-08-17T17:21:31.877 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:31 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/416372010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T17:21:31.878 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:31 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:31.878 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:31 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:31.878 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:31 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/416372010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T17:21:31.878 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:31 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:31.878 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:31 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:31.904 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738417 got 34359738420 for osd.0 2024-08-17T17:21:31.905 DEBUG:teuthology.parallel:result is None 2024-08-17T17:21:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:31 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/416372010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T17:21:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:31 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:31 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:32 smithi130 ceph-mon[42749]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:32.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:32 smithi130 ceph-mon[31342]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:32.720 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234001 got 231928234004 for osd.7 2024-08-17T17:21:32.720 DEBUG:teuthology.parallel:result is None 2024-08-17T17:21:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:32 smithi179 ceph-mon[34775]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:33.780 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:33 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:21:33.780 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:33 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:21:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:33 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:21:34.751 INFO:teuthology.orchestra.run.smithi130.stdout:107374182438 2024-08-17T17:21:34.762 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:34 smithi130 ceph-mon[31342]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:34.762 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:34 smithi130 ceph-mon[42749]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:34 smithi179 ceph-mon[34775]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:34.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:21:34 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:21:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:21:35.492 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182435 got 107374182438 for osd.3 2024-08-17T17:21:35.492 DEBUG:teuthology.parallel:result is None 2024-08-17T17:21:35.493 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T17:21:35.493 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:21:35.788 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:35 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1279885046' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T17:21:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:35 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1279885046' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T17:21:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:35 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1279885046' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T17:21:36.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:36 smithi130 ceph-mon[31342]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:36.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:36 smithi130 ceph-mon[42749]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:36 smithi179 ceph-mon[34775]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:38.318 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:21:38.319 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:21:38.887 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:38 smithi130 ceph-mon[31342]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:38.887 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:38 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:38.887 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:38 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:38.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:38 smithi130 ceph-mon[42749]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:38.888 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:38 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:38.888 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:38 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:38 smithi179 ceph-mon[34775]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:38 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:38 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:39.290 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":42,"stamp":"2024-08-17T17:21:37.528615+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385792,"kb_used_data":6944,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575424,"statfs":{"total":767960285184,"available":765517234176,"internally_reserved":0,"allocated":7110656,"data_stored":4027485,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002081"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.145554+0000","last_change":"2024-08-17T17:20:43.677752+0000","last_active":"2024-08-17T17:20:49.145554+0000","last_peered":"2024-08-17T17:20:49.145554+0000","last_clean":"2024-08-17T17:20:49.145554+0000","last_became_active":"2024-08-17T17:20:38.652246+0000","last_became_peered":"2024-08-17T17:20:38.652246+0000","last_unstale":"2024-08-17T17:20:49.145554+0000","last_undegraded":"2024-08-17T17:20:49.145554+0000","last_fullsized":"2024-08-17T17:20:49.145554+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:52.620577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.139327+0000","last_change":"2024-08-17T17:20:43.677369+0000","last_active":"2024-08-17T17:20:49.139327+0000","last_peered":"2024-08-17T17:20:49.139327+0000","last_clean":"2024-08-17T17:20:49.139327+0000","last_became_active":"2024-08-17T17:20:38.782865+0000","last_became_peered":"2024-08-17T17:20:38.782865+0000","last_unstale":"2024-08-17T17:20:49.139327+0000","last_undegraded":"2024-08-17T17:20:49.139327+0000","last_fullsized":"2024-08-17T17:20:49.139327+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:15:56.927535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021889699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.145958+0000","last_change":"2024-08-17T17:20:43.716798+0000","last_active":"2024-08-17T17:20:49.145958+0000","last_peered":"2024-08-17T17:20:49.145958+0000","last_clean":"2024-08-17T17:20:49.145958+0000","last_became_active":"2024-08-17T17:20:38.782941+0000","last_became_peered":"2024-08-17T17:20:38.782941+0000","last_unstale":"2024-08-17T17:20:49.145958+0000","last_undegraded":"2024-08-17T17:20:49.145958+0000","last_fullsized":"2024-08-17T17:20:49.145958+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:35.726637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.139305+0000","last_change":"2024-08-17T17:20:43.677386+0000","last_active":"2024-08-17T17:20:49.139305+0000","last_peered":"2024-08-17T17:20:49.139305+0000","last_clean":"2024-08-17T17:20:49.139305+0000","last_became_active":"2024-08-17T17:20:38.653370+0000","last_became_peered":"2024-08-17T17:20:38.653370+0000","last_unstale":"2024-08-17T17:20:49.139305+0000","last_undegraded":"2024-08-17T17:20:49.139305+0000","last_fullsized":"2024-08-17T17:20:49.139305+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:40.082296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.145608+0000","last_change":"2024-08-17T17:20:43.676769+0000","last_active":"2024-08-17T17:20:49.145608+0000","last_peered":"2024-08-17T17:20:49.145608+0000","last_clean":"2024-08-17T17:20:49.145608+0000","last_became_active":"2024-08-17T17:20:38.651624+0000","last_became_peered":"2024-08-17T17:20:38.651624+0000","last_unstale":"2024-08-17T17:20:49.145608+0000","last_undegraded":"2024-08-17T17:20:49.145608+0000","last_fullsized":"2024-08-17T17:20:49.145608+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:58:44.833201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0008386,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.145399+0000","last_change":"2024-08-17T17:20:43.672739+0000","last_active":"2024-08-17T17:20:49.145399+0000","last_peered":"2024-08-17T17:20:49.145399+0000","last_clean":"2024-08-17T17:20:49.145399+0000","last_became_active":"2024-08-17T17:20:38.783090+0000","last_became_peered":"2024-08-17T17:20:38.783090+0000","last_unstale":"2024-08-17T17:20:49.145399+0000","last_undegraded":"2024-08-17T17:20:49.145399+0000","last_fullsized":"2024-08-17T17:20:49.145399+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:23.536205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.145638+0000","last_change":"2024-08-17T17:20:43.716700+0000","last_active":"2024-08-17T17:20:49.145638+0000","last_peered":"2024-08-17T17:20:49.145638+0000","last_clean":"2024-08-17T17:20:49.145638+0000","last_became_active":"2024-08-17T17:20:38.653957+0000","last_became_peered":"2024-08-17T17:20:38.653957+0000","last_unstale":"2024-08-17T17:20:49.145638+0000","last_undegraded":"2024-08-17T17:20:49.145638+0000","last_fullsized":"2024-08-17T17:20:49.145638+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:43:42.214017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.145796+0000","last_change":"2024-08-17T17:20:43.676802+0000","last_active":"2024-08-17T17:20:49.145796+0000","last_peered":"2024-08-17T17:20:49.145796+0000","last_clean":"2024-08-17T17:20:49.145796+0000","last_became_active":"2024-08-17T17:20:38.651573+0000","last_became_peered":"2024-08-17T17:20:38.651573+0000","last_unstale":"2024-08-17T17:20:49.145796+0000","last_undegraded":"2024-08-17T17:20:49.145796+0000","last_fullsized":"2024-08-17T17:20:49.145796+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:20:15.923334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085985900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":273,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-08-17T17:20:43.716450+0000","last_change":"2024-08-17T17:20:06.878678+0000","last_active":"2024-08-17T17:20:43.716450+0000","last_peered":"2024-08-17T17:20:43.716450+0000","last_clean":"2024-08-17T17:20:43.716450+0000","last_became_active":"2024-08-17T17:20:06.878356+0000","last_became_peered":"2024-08-17T17:20:06.878356+0000","last_unstale":"2024-08-17T17:20:43.716450+0000","last_undegraded":"2024-08-17T17:20:43.716450+0000","last_fullsized":"2024-08-17T17:20:43.716450+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:02:43.355026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234006,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298584,"kb_used_data":1228,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446568,"statfs":{"total":95995035648,"available":95689285632,"internally_reserved":0,"allocated":1257472,"data_stored":872392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.53800000000000003}]},{"osd":1,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.503},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.48199999999999998}]},{"osd":2,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.44},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.52800000000000002}]},{"osd":3,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.47399999999999998}]},{"osd":4,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.35799999999999998}]},{"osd":5,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.41299999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495641,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298584,"kb_used_data":1228,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446568,"statfs":{"total":95995035648,"available":95689285632,"internally_reserved":0,"allocated":1257472,"data_stored":872096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:20:39 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.56100000000000005}]},{"osd":1,"last update":"Sat Aug 17 17:20:39 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.504},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.621}]},{"osd":2,"last update":"Sat Aug 17 17:20:39 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.60699999999999998}]},{"osd":3,"last update":"Sat Aug 17 17:20:39 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.53000000000000003}]},{"osd":4,"last update":"Sat Aug 17 17:20:39 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.40699999999999997}]},{"osd":5,"last update":"Sat Aug 17 17:20:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.63500000000000001}]},{"osd":7,"last update":"Sat Aug 17 17:21:06 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.64800000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757279,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":667648,"data_stored":282051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.51400000000000001,"15min":0.51600000000000001},"min":{"1min":0.35699999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64900000000000002,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.51000000000000001,"15min":0.51600000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.60599999999999998,"5min":0.75,"15min":0.75},"last":0.51900000000000002}]},{"osd":1,"last update":"Sat Aug 17 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.51100000000000001,"15min":0.51300000000000001},"min":{"1min":0.41599999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.65100000000000002,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.51500000000000001,"15min":0.52100000000000002},"min":{"1min":0.36199999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.59199999999999997,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.46500000000000002}]},{"osd":2,"last update":"Sat Aug 17 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.65000000000000002,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.623},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.51800000000000002,"15min":0.52200000000000002},"min":{"1min":0.34300000000000003,"5min":0.316,"15min":0.316},"max":{"1min":0.60299999999999998,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.61399999999999999}]},{"osd":3,"last update":"Sat Aug 17 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.51600000000000001,"15min":0.52400000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.64000000000000001,"5min":0.876,"15min":0.876},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.52800000000000002,"15min":0.53400000000000003},"min":{"1min":0.40300000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.628,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.58199999999999996}]},{"osd":4,"last update":"Sat Aug 17 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.41099999999999998,"15min":0.41299999999999998},"min":{"1min":0.30199999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.63400000000000001,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.434,"15min":0.436},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.70999999999999996}]},{"osd":6,"last update":"Sat Aug 17 17:20:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.59899999999999998}]},{"osd":7,"last update":"Sat Aug 17 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.67800000000000005}]}]},{"osd":4,"up_from":30,"seq":128849018914,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298004,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447148,"statfs":{"total":95995035648,"available":95689879552,"internally_reserved":0,"allocated":663552,"data_stored":282032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.51900000000000002,"15min":0.52200000000000002},"min":{"1min":0.317,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68600000000000005,"5min":0.748,"15min":0.748},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.52700000000000002,"15min":0.53700000000000003},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.67600000000000005,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.45700000000000002}]},{"osd":1,"last update":"Sat Aug 17 17:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.505,"15min":0.50900000000000001},"min":{"1min":0.34399999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.63300000000000001,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.51200000000000001,"15min":0.51800000000000002},"min":{"1min":0.35399999999999998,"5min":0.316,"15min":0.316},"max":{"1min":0.63700000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.497}]},{"osd":2,"last update":"Sat Aug 17 17:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.51900000000000002,"15min":0.52600000000000002},"min":{"1min":0.38900000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.58699999999999997,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.498,"5min":0.52000000000000002,"15min":0.52400000000000002},"min":{"1min":0.35499999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.65300000000000002,"5min":0.748,"15min":0.748},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Aug 17 17:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.52300000000000002,"15min":0.52800000000000002},"min":{"1min":0.36499999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.63,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.50900000000000001,"15min":0.51400000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.60199999999999998,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.46800000000000003}]},{"osd":5,"last update":"Sat Aug 17 17:21:20 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.40100000000000002,"15min":0.38900000000000001},"min":{"1min":0.32300000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.437},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.42699999999999999,"15min":0.41299999999999998},"min":{"1min":0.373,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.375}]},{"osd":6,"last update":"Sat Aug 17 17:20:43 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.40899999999999997}]},{"osd":7,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.35399999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182439,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":667648,"data_stored":282297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:20:37 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.36299999999999999,"15min":0.36599999999999999},"min":{"1min":0.251,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.502,"5min":0.50900000000000001,"15min":0.50900000000000001},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.379,"15min":0.373},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.32300000000000001}]},{"osd":1,"last update":"Sat Aug 17 17:20:37 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.28299999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.40100000000000002,"15min":0.40300000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.56000000000000005,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.46200000000000002}]},{"osd":2,"last update":"Sat Aug 17 17:20:37 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.496},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.38600000000000001,"15min":0.38500000000000001},"min":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.66400000000000003}]},{"osd":4,"last update":"Sat Aug 17 17:20:57 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.628},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53600000000000003,"15min":0.53700000000000003},"min":{"1min":0.39300000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.54500000000000004}]},{"osd":5,"last update":"Sat Aug 17 17:21:22 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.54400000000000004,"15min":0.54900000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.72099999999999997,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.55900000000000005,"15min":0.56699999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.83099999999999996,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.59399999999999997}]},{"osd":6,"last update":"Sat Aug 17 17:20:40 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.67300000000000004}]},{"osd":7,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.68700000000000006}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298004,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447148,"statfs":{"total":95995035648,"available":95689879552,"internally_reserved":0,"allocated":663552,"data_stored":281636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:21:11 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.375,"15min":0.36899999999999999},"min":{"1min":0.23400000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.52600000000000002,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.378,"5min":0.40600000000000003,"15min":0.40400000000000003},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.53000000000000003,"5min":0.62,"15min":0.62},"last":0.36499999999999999}]},{"osd":1,"last update":"Sat Aug 17 17:21:11 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.36699999999999999,"15min":0.35999999999999999},"min":{"1min":0.23899999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.51400000000000001,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38600000000000001,"15min":0.379},"min":{"1min":0.28799999999999998,"5min":0.222,"15min":0.222},"max":{"1min":0.50900000000000001,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.37}]},{"osd":3,"last update":"Sat Aug 17 17:20:36 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.45600000000000002,"15min":0.45800000000000002},"min":{"1min":0.33400000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.433},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.436,"15min":0.442},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.32900000000000001}]},{"osd":4,"last update":"Sat Aug 17 17:20:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.76900000000000002,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.622},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.56599999999999995,"15min":0.57099999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.70799999999999996,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.70499999999999996}]},{"osd":5,"last update":"Sat Aug 17 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.42699999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.54400000000000004,"15min":0.54800000000000004},"min":{"1min":0.41599999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.60399999999999998}]},{"osd":6,"last update":"Sat Aug 17 17:20:41 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.59399999999999997}]},{"osd":7,"last update":"Sat Aug 17 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.74299999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298012,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447140,"statfs":{"total":95995035648,"available":95689871360,"internally_reserved":0,"allocated":671744,"data_stored":282316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:20:46 2024","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.34499999999999997,"15min":0.33500000000000002},"min":{"1min":0.255,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.373,"15min":0.35699999999999998},"min":{"1min":0.308,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.54900000000000004,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.42199999999999999}]},{"osd":2,"last update":"Sat Aug 17 17:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.36299999999999999,"15min":0.35799999999999998},"min":{"1min":0.23899999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.499,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.44},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41599999999999998,"15min":0.40600000000000003},"min":{"1min":0.253,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.45800000000000002}]},{"osd":3,"last update":"Sat Aug 17 17:20:35 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.442,"15min":0.438},"min":{"1min":0.375,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.52700000000000002,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.53300000000000003}]},{"osd":4,"last update":"Sat Aug 17 17:20:56 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.57899999999999996,"15min":0.57299999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.55300000000000005,"15min":0.54700000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.57799999999999996}]},{"osd":5,"last update":"Sat Aug 17 17:21:21 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.54700000000000004,"15min":0.54900000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.621,"5min":0.60499999999999998,"15min":0.60199999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.0429999999999999,"5min":1.0429999999999999,"15min":1.0429999999999999},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Aug 17 17:20:41 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.628}]},{"osd":7,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.59799999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298588,"kb_used_data":1232,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446564,"statfs":{"total":95995035648,"available":95689281536,"internally_reserved":0,"allocated":1261568,"data_stored":872665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 17:20:50 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.33500000000000002,"15min":0.316},"min":{"1min":0.26300000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.623,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.496},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.34999999999999998,"15min":0.31900000000000001},"min":{"1min":0.23400000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.63400000000000001,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Aug 17 17:21:15 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.39200000000000002,"15min":0.38500000000000001},"min":{"1min":0.27400000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.38400000000000001,"15min":0.373},"min":{"1min":0.28399999999999997,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.46000000000000002,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.32900000000000001}]},{"osd":3,"last update":"Sat Aug 17 17:20:34 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.42399999999999999,"15min":0.43099999999999999},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.52100000000000002}]},{"osd":4,"last update":"Sat Aug 17 17:20:59 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.54300000000000004,"15min":0.55200000000000005},"min":{"1min":0.40899999999999997,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.65100000000000002,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.55300000000000005,"15min":0.55900000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.77700000000000002,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.442}]},{"osd":5,"last update":"Sat Aug 17 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.57099999999999995,"15min":0.57499999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.745,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.505},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.53000000000000003,"15min":0.53400000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.67200000000000004,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Aug 17 17:20:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.48499999999999999}]},{"osd":7,"last update":"Sat Aug 17 17:21:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.43099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:21:39.292 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:21:39.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:39 smithi130 ceph-mon[31342]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:21:39.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:39 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:21:39.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:39 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:21:39.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:39 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:39.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:39 smithi130 ceph-mon[31342]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T17:21:39.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:39 smithi130 ceph-mon[31342]: Reconfiguring daemon alertmanager.a on smithi130 2024-08-17T17:21:39.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:39 smithi130 ceph-mon[42749]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:21:39.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:39 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:21:39.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:39 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:21:39.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:39 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:39.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:39 smithi130 ceph-mon[42749]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T17:21:39.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:39 smithi130 ceph-mon[42749]: Reconfiguring daemon alertmanager.a on smithi130 2024-08-17T17:21:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:39 smithi179 ceph-mon[34775]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:21:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:39 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:21:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:39 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:21:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:39 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:39 smithi179 ceph-mon[34775]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T17:21:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:39 smithi179 ceph-mon[34775]: Reconfiguring daemon alertmanager.a on smithi130 2024-08-17T17:21:40.180 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:41.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:40 smithi130 ceph-mon[31342]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:40 smithi130 ceph-mon[42749]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:41.257 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:40 smithi179 ceph-mon[34775]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:43.088 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:42 smithi130 ceph-mon[31342]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:43.088 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:42 smithi130 ceph-mon[42749]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:43.088 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:42 smithi130 systemd[1]: Stopping Ceph alertmanager.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:21:43.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:42 smithi179 ceph-mon[34775]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:44.060 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:21:44.061 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:21:44.329 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:44 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[73466]: ts=2024-08-17T17:21:44.038Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:21:44.329 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:44 smithi130 podman[78953]: 2024-08-17 17:21:44.058619308 +0000 UTC m=+0.139115432 container died be07dfea12b0dc320c18b5ca97accc47ff8438d0a3b07084e354e3d63ac4bdd8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:44.667 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:44 smithi130 podman[78953]: 2024-08-17 17:21:44.419587131 +0000 UTC m=+0.500083242 container cleanup be07dfea12b0dc320c18b5ca97accc47ff8438d0a3b07084e354e3d63ac4bdd8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:44.667 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:44 smithi130 bash[78953]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a 2024-08-17T17:21:44.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:21:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:21:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:21:45.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:44 smithi130 ceph-mon[31342]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:45.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:44 smithi130 ceph-mon[31342]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:21:45.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:44 smithi130 ceph-mon[42749]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:45.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:44 smithi130 ceph-mon[42749]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:21:45.079 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":45,"stamp":"2024-08-17T17:21:43.529652+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385792,"kb_used_data":6944,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575424,"statfs":{"total":767960285184,"available":765517234176,"internally_reserved":0,"allocated":7110656,"data_stored":4027485,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001976"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.145554+0000","last_change":"2024-08-17T17:20:43.677752+0000","last_active":"2024-08-17T17:20:49.145554+0000","last_peered":"2024-08-17T17:20:49.145554+0000","last_clean":"2024-08-17T17:20:49.145554+0000","last_became_active":"2024-08-17T17:20:38.652246+0000","last_became_peered":"2024-08-17T17:20:38.652246+0000","last_unstale":"2024-08-17T17:20:49.145554+0000","last_undegraded":"2024-08-17T17:20:49.145554+0000","last_fullsized":"2024-08-17T17:20:49.145554+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:52.620577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.139327+0000","last_change":"2024-08-17T17:20:43.677369+0000","last_active":"2024-08-17T17:20:49.139327+0000","last_peered":"2024-08-17T17:20:49.139327+0000","last_clean":"2024-08-17T17:20:49.139327+0000","last_became_active":"2024-08-17T17:20:38.782865+0000","last_became_peered":"2024-08-17T17:20:38.782865+0000","last_unstale":"2024-08-17T17:20:49.139327+0000","last_undegraded":"2024-08-17T17:20:49.139327+0000","last_fullsized":"2024-08-17T17:20:49.139327+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:15:56.927535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021889699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.145958+0000","last_change":"2024-08-17T17:20:43.716798+0000","last_active":"2024-08-17T17:20:49.145958+0000","last_peered":"2024-08-17T17:20:49.145958+0000","last_clean":"2024-08-17T17:20:49.145958+0000","last_became_active":"2024-08-17T17:20:38.782941+0000","last_became_peered":"2024-08-17T17:20:38.782941+0000","last_unstale":"2024-08-17T17:20:49.145958+0000","last_undegraded":"2024-08-17T17:20:49.145958+0000","last_fullsized":"2024-08-17T17:20:49.145958+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:35.726637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.139305+0000","last_change":"2024-08-17T17:20:43.677386+0000","last_active":"2024-08-17T17:20:49.139305+0000","last_peered":"2024-08-17T17:20:49.139305+0000","last_clean":"2024-08-17T17:20:49.139305+0000","last_became_active":"2024-08-17T17:20:38.653370+0000","last_became_peered":"2024-08-17T17:20:38.653370+0000","last_unstale":"2024-08-17T17:20:49.139305+0000","last_undegraded":"2024-08-17T17:20:49.139305+0000","last_fullsized":"2024-08-17T17:20:49.139305+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:40.082296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.145608+0000","last_change":"2024-08-17T17:20:43.676769+0000","last_active":"2024-08-17T17:20:49.145608+0000","last_peered":"2024-08-17T17:20:49.145608+0000","last_clean":"2024-08-17T17:20:49.145608+0000","last_became_active":"2024-08-17T17:20:38.651624+0000","last_became_peered":"2024-08-17T17:20:38.651624+0000","last_unstale":"2024-08-17T17:20:49.145608+0000","last_undegraded":"2024-08-17T17:20:49.145608+0000","last_fullsized":"2024-08-17T17:20:49.145608+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:58:44.833201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0008386,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.145399+0000","last_change":"2024-08-17T17:20:43.672739+0000","last_active":"2024-08-17T17:20:49.145399+0000","last_peered":"2024-08-17T17:20:49.145399+0000","last_clean":"2024-08-17T17:20:49.145399+0000","last_became_active":"2024-08-17T17:20:38.783090+0000","last_became_peered":"2024-08-17T17:20:38.783090+0000","last_unstale":"2024-08-17T17:20:49.145399+0000","last_undegraded":"2024-08-17T17:20:49.145399+0000","last_fullsized":"2024-08-17T17:20:49.145399+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:23.536205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.145638+0000","last_change":"2024-08-17T17:20:43.716700+0000","last_active":"2024-08-17T17:20:49.145638+0000","last_peered":"2024-08-17T17:20:49.145638+0000","last_clean":"2024-08-17T17:20:49.145638+0000","last_became_active":"2024-08-17T17:20:38.653957+0000","last_became_peered":"2024-08-17T17:20:38.653957+0000","last_unstale":"2024-08-17T17:20:49.145638+0000","last_undegraded":"2024-08-17T17:20:49.145638+0000","last_fullsized":"2024-08-17T17:20:49.145638+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:43:42.214017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T17:20:49.145796+0000","last_change":"2024-08-17T17:20:43.676802+0000","last_active":"2024-08-17T17:20:49.145796+0000","last_peered":"2024-08-17T17:20:49.145796+0000","last_clean":"2024-08-17T17:20:49.145796+0000","last_became_active":"2024-08-17T17:20:38.651573+0000","last_became_peered":"2024-08-17T17:20:38.651573+0000","last_unstale":"2024-08-17T17:20:49.145796+0000","last_undegraded":"2024-08-17T17:20:49.145796+0000","last_fullsized":"2024-08-17T17:20:49.145796+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:20:15.923334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085985900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":273,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-08-17T17:20:43.716450+0000","last_change":"2024-08-17T17:20:06.878678+0000","last_active":"2024-08-17T17:20:43.716450+0000","last_peered":"2024-08-17T17:20:43.716450+0000","last_clean":"2024-08-17T17:20:43.716450+0000","last_became_active":"2024-08-17T17:20:06.878356+0000","last_became_peered":"2024-08-17T17:20:06.878356+0000","last_unstale":"2024-08-17T17:20:43.716450+0000","last_undegraded":"2024-08-17T17:20:43.716450+0000","last_fullsized":"2024-08-17T17:20:43.716450+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:02:43.355026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234007,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298584,"kb_used_data":1228,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446568,"statfs":{"total":95995035648,"available":95689285632,"internally_reserved":0,"allocated":1257472,"data_stored":872392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.61599999999999999}]},{"osd":1,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.65600000000000003}]},{"osd":2,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.66200000000000003}]},{"osd":3,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.76200000000000001}]},{"osd":4,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.58999999999999997}]},{"osd":5,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.56899999999999995}]},{"osd":6,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.68700000000000006}]}]},{"osd":6,"up_from":46,"seq":197568495642,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298584,"kb_used_data":1228,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446568,"statfs":{"total":95995035648,"available":95689285632,"internally_reserved":0,"allocated":1257472,"data_stored":872096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:20:39 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.52100000000000002}]},{"osd":1,"last update":"Sat Aug 17 17:20:39 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.438}]},{"osd":2,"last update":"Sat Aug 17 17:20:39 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.47799999999999998}]},{"osd":3,"last update":"Sat Aug 17 17:20:39 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.57299999999999995}]},{"osd":4,"last update":"Sat Aug 17 17:20:39 2024","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.45200000000000001}]},{"osd":5,"last update":"Sat Aug 17 17:20:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.38300000000000001}]},{"osd":7,"last update":"Sat Aug 17 17:21:06 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.39600000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757280,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":667648,"data_stored":282051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.51400000000000001,"15min":0.51600000000000001},"min":{"1min":0.35699999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.64900000000000002,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.51000000000000001,"15min":0.51600000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.60599999999999998,"5min":0.75,"15min":0.75},"last":0.53000000000000003}]},{"osd":1,"last update":"Sat Aug 17 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.51100000000000001,"15min":0.51300000000000001},"min":{"1min":0.41599999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.65100000000000002,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.501},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.51500000000000001,"15min":0.52100000000000002},"min":{"1min":0.36199999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.59199999999999997,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.59099999999999997}]},{"osd":2,"last update":"Sat Aug 17 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.65000000000000002,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.51800000000000002,"15min":0.52200000000000002},"min":{"1min":0.34300000000000003,"5min":0.316,"15min":0.316},"max":{"1min":0.60299999999999998,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.64300000000000002}]},{"osd":3,"last update":"Sat Aug 17 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.51600000000000001,"15min":0.52400000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.64000000000000001,"5min":0.876,"15min":0.876},"last":0.627},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.52800000000000002,"15min":0.53400000000000003},"min":{"1min":0.40300000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.628,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.56599999999999995}]},{"osd":4,"last update":"Sat Aug 17 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.41099999999999998,"15min":0.41299999999999998},"min":{"1min":0.30199999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.63400000000000001,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.434,"15min":0.436},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.51800000000000002}]},{"osd":6,"last update":"Sat Aug 17 17:20:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.68799999999999994}]},{"osd":7,"last update":"Sat Aug 17 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.67400000000000004}]}]},{"osd":4,"up_from":30,"seq":128849018915,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298004,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447148,"statfs":{"total":95995035648,"available":95689879552,"internally_reserved":0,"allocated":663552,"data_stored":282032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.51900000000000002,"15min":0.52200000000000002},"min":{"1min":0.317,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68600000000000005,"5min":0.748,"15min":0.748},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.52700000000000002,"15min":0.53700000000000003},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.67600000000000005,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.55900000000000005}]},{"osd":1,"last update":"Sat Aug 17 17:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.505,"15min":0.50900000000000001},"min":{"1min":0.34399999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.63300000000000001,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.51200000000000001,"15min":0.51800000000000002},"min":{"1min":0.35399999999999998,"5min":0.316,"15min":0.316},"max":{"1min":0.63700000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.42399999999999999}]},{"osd":2,"last update":"Sat Aug 17 17:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.51900000000000002,"15min":0.52600000000000002},"min":{"1min":0.38900000000000001,"5min":0.376,"15min":0.376},"max":{"1min":0.58699999999999997,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.498,"5min":0.52000000000000002,"15min":0.52400000000000002},"min":{"1min":0.35499999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.65300000000000002,"5min":0.748,"15min":0.748},"last":0.624}]},{"osd":3,"last update":"Sat Aug 17 17:20:54 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.52300000000000002,"15min":0.52800000000000002},"min":{"1min":0.36499999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.63,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.50900000000000001,"15min":0.51400000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.60199999999999998,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.63900000000000001}]},{"osd":5,"last update":"Sat Aug 17 17:21:20 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.40100000000000002,"15min":0.38900000000000001},"min":{"1min":0.32300000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.747},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.42699999999999999,"15min":0.41299999999999998},"min":{"1min":0.373,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.65500000000000003}]},{"osd":6,"last update":"Sat Aug 17 17:20:43 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.70999999999999996}]},{"osd":7,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.77600000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182440,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":667648,"data_stored":282297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:21:38 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.35699999999999998,"15min":0.36399999999999999},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.374},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.374,"15min":0.372},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.48799999999999999,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.23300000000000001}]},{"osd":1,"last update":"Sat Aug 17 17:21:38 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.39600000000000002,"15min":0.40400000000000003},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.38900000000000001,"15min":0.39900000000000002},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.28999999999999998}]},{"osd":2,"last update":"Sat Aug 17 17:21:38 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.42099999999999999,"15min":0.42699999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.36399999999999999,"5min":0.38200000000000001,"15min":0.38400000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.245}]},{"osd":4,"last update":"Sat Aug 17 17:20:57 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.504},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53600000000000003,"15min":0.53700000000000003},"min":{"1min":0.39300000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.52000000000000002}]},{"osd":5,"last update":"Sat Aug 17 17:21:22 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.54400000000000004,"15min":0.54900000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.72099999999999997,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.55900000000000005,"15min":0.56699999999999995},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.83099999999999996,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Aug 17 17:20:40 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.51000000000000001}]},{"osd":7,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.55800000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411372,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298004,"kb_used_data":648,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447148,"statfs":{"total":95995035648,"available":95689879552,"internally_reserved":0,"allocated":663552,"data_stored":281636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:21:11 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.375,"15min":0.36899999999999999},"min":{"1min":0.23400000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.52600000000000002,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.378,"5min":0.40600000000000003,"15min":0.40400000000000003},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.53000000000000003,"5min":0.62,"15min":0.62},"last":0.316}]},{"osd":1,"last update":"Sat Aug 17 17:21:11 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.36699999999999999,"15min":0.35999999999999999},"min":{"1min":0.23899999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.51400000000000001,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38600000000000001,"15min":0.379},"min":{"1min":0.28799999999999998,"5min":0.222,"15min":0.222},"max":{"1min":0.50900000000000001,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.28000000000000003}]},{"osd":3,"last update":"Sat Aug 17 17:21:36 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.44500000000000001,"15min":0.45400000000000001},"min":{"1min":0.30599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.54600000000000004,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.372},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.41799999999999998,"15min":0.436},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.52000000000000002,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.40000000000000002}]},{"osd":4,"last update":"Sat Aug 17 17:20:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54900000000000004,"15min":0.55500000000000005},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.76900000000000002,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.56599999999999995,"15min":0.57099999999999995},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.70799999999999996,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Aug 17 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.42699999999999999,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.54400000000000004,"15min":0.54800000000000004},"min":{"1min":0.41599999999999998,"5min":0.38,"15min":0.38},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.501}]},{"osd":6,"last update":"Sat Aug 17 17:20:41 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.47599999999999998}]},{"osd":7,"last update":"Sat Aug 17 17:21:05 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.53900000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298012,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447140,"statfs":{"total":95995035648,"available":95689871360,"internally_reserved":0,"allocated":671744,"data_stored":282316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:20:46 2024","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.34499999999999997,"15min":0.33500000000000002},"min":{"1min":0.255,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.373,"15min":0.35699999999999998},"min":{"1min":0.308,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.54900000000000004,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.26600000000000001}]},{"osd":2,"last update":"Sat Aug 17 17:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.36299999999999999,"15min":0.35799999999999998},"min":{"1min":0.23899999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.499,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41599999999999998,"15min":0.40600000000000003},"min":{"1min":0.253,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.315}]},{"osd":3,"last update":"Sat Aug 17 17:21:36 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.435,"15min":0.435},"min":{"1min":0.33000000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.51100000000000001,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.371,"5min":0.40400000000000003,"15min":0.40999999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.53300000000000003,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.34899999999999998}]},{"osd":4,"last update":"Sat Aug 17 17:20:56 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.57899999999999996,"15min":0.57299999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.55300000000000005,"15min":0.54700000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.59799999999999998}]},{"osd":5,"last update":"Sat Aug 17 17:21:21 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.54700000000000004,"15min":0.54900000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.621,"5min":0.60499999999999998,"15min":0.60199999999999998},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.0429999999999999,"5min":1.0429999999999999,"15min":1.0429999999999999},"last":0.64700000000000002}]},{"osd":6,"last update":"Sat Aug 17 17:20:41 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.68899999999999995}]},{"osd":7,"last update":"Sat Aug 17 17:21:04 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.72699999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298588,"kb_used_data":1232,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446564,"statfs":{"total":95995035648,"available":95689281536,"internally_reserved":0,"allocated":1261568,"data_stored":872665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 17:20:50 2024","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.33500000000000002,"15min":0.316},"min":{"1min":0.26300000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.623,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.34999999999999998,"15min":0.31900000000000001},"min":{"1min":0.23400000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.63400000000000001,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.434}]},{"osd":2,"last update":"Sat Aug 17 17:21:15 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.39200000000000002,"15min":0.38500000000000001},"min":{"1min":0.27400000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.38400000000000001,"15min":0.373},"min":{"1min":0.28399999999999997,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.46000000000000002,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Aug 17 17:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.40699999999999997,"15min":0.42499999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.42799999999999999,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.378,"5min":0.44500000000000001,"15min":0.45600000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.52100000000000002,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.374}]},{"osd":4,"last update":"Sat Aug 17 17:20:59 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.54300000000000004,"15min":0.55200000000000005},"min":{"1min":0.40899999999999997,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.65100000000000002,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.55300000000000005,"15min":0.55900000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.77700000000000002,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.55300000000000005}]},{"osd":5,"last update":"Sat Aug 17 17:21:19 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.57099999999999995,"15min":0.57499999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.745,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.53000000000000003,"15min":0.53400000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.67200000000000004,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.60199999999999998}]},{"osd":6,"last update":"Sat Aug 17 17:20:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.66000000000000003}]},{"osd":7,"last update":"Sat Aug 17 17:21:02 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.623}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:21:45.081 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T17:21:45.081 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-17T17:21:45.081 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-17T17:21:45.081 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph health --format=json 2024-08-17T17:21:45.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:44 smithi179 ceph-mon[34775]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:45.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:44 smithi179 ceph-mon[34775]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:21:45.375 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:45 smithi130 bash[79039]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a 2024-08-17T17:21:45.580 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:45.666 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:45 smithi130 podman[78972]: 2024-08-17 17:21:45.375261377 +0000 UTC m=+1.314353536 container remove be07dfea12b0dc320c18b5ca97accc47ff8438d0a3b07084e354e3d63ac4bdd8 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:45.667 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:45 smithi130 podman[78972]: 2024-08-17 17:21:45.459217608 +0000 UTC m=+1.398309767 volume remove 32ad83081a4689587326f8129d969a0d5b80280437502eb510456d840998a0ad 2024-08-17T17:21:45.667 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:45 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-17T17:21:45.667 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:45 smithi130 systemd[1]: Stopped Ceph alertmanager.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:21:45.667 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:45 smithi130 systemd[1]: Starting Ceph alertmanager.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:21:45.996 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:45 smithi130 podman[79159]: 2024-08-17 17:21:45.693486649 +0000 UTC m=+0.108477691 volume create 90c509b0ab2a09655d9619ed5c375ecd2c2e208ba40e86b2296e712c0e744382 2024-08-17T17:21:45.996 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:45 smithi130 podman[79159]: 2024-08-17 17:21:45.60453675 +0000 UTC m=+0.019527808 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-08-17T17:21:45.997 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:45 smithi130 podman[79159]: 2024-08-17 17:21:45.768822861 +0000 UTC m=+0.183813904 container create 434453bf091074c28a27e8b6b31733d9c7cac2d8e47b99f726fefcd7e6a7a82a (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:45.997 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:45 smithi130 podman[79159]: 2024-08-17 17:21:45.996418893 +0000 UTC m=+0.411409936 container init 434453bf091074c28a27e8b6b31733d9c7cac2d8e47b99f726fefcd7e6a7a82a (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:46.348 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:46 smithi130 podman[79159]: 2024-08-17 17:21:46.001096277 +0000 UTC m=+0.416087321 container start 434453bf091074c28a27e8b6b31733d9c7cac2d8e47b99f726fefcd7e6a7a82a (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:46.348 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:46 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[79188]: ts=2024-08-17T17:21:46.012Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-08-17T17:21:46.348 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:46 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[79188]: ts=2024-08-17T17:21:46.013Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-08-17T17:21:46.348 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:46 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[79188]: ts=2024-08-17T17:21:46.014Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.130 port=9094 2024-08-17T17:21:46.348 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:46 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[79188]: ts=2024-08-17T17:21:46.015Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-17T17:21:46.348 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:46 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[79188]: ts=2024-08-17T17:21:46.038Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T17:21:46.348 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:46 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[79188]: ts=2024-08-17T17:21:46.039Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T17:21:46.348 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:46 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[79188]: ts=2024-08-17T17:21:46.041Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-08-17T17:21:46.348 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:46 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[79188]: ts=2024-08-17T17:21:46.041Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-17T17:21:46.348 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:46 smithi130 bash[79159]: 434453bf091074c28a27e8b6b31733d9c7cac2d8e47b99f726fefcd7e6a7a82a 2024-08-17T17:21:46.348 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:46 smithi130 systemd[1]: Started Ceph alertmanager.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:21:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:46 smithi179 ceph-mon[34775]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:46 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:46 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:47.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:46 smithi130 ceph-mon[31342]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:47.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:46 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:47.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:46 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:47.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:46 smithi130 ceph-mon[42749]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:47.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:46 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:47.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:46 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:47.828 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:21:47.828 INFO:teuthology.orchestra.run.smithi130.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-17T17:21:48.147 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:47 smithi179 ceph-mon[34775]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:21:48.147 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:47 smithi179 ceph-mon[34775]: Reconfiguring daemon prometheus.a on smithi179 2024-08-17T17:21:48.147 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:47 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3445634236' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T17:21:48.166 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[79188]: ts=2024-08-17T17:21:48.016Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00089068s 2024-08-17T17:21:48.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:47 smithi130 ceph-mon[31342]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:21:48.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:47 smithi130 ceph-mon[31342]: Reconfiguring daemon prometheus.a on smithi179 2024-08-17T17:21:48.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:47 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3445634236' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T17:21:48.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:47 smithi130 ceph-mon[42749]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:21:48.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:47 smithi130 ceph-mon[42749]: Reconfiguring daemon prometheus.a on smithi179 2024-08-17T17:21:48.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:47 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3445634236' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T17:21:48.656 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-17T17:21:48.656 INFO:tasks.cephadm:Setup complete, yielding 2024-08-17T17:21:48.657 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-08-17T17:21:48.666 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi130.front.sepia.ceph.com 2024-08-17T17:21:48.666 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-08-17T17:21:48.923 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:49.228 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:48 smithi179 ceph-mon[34775]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:49.228 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:48 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:21:49.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:48 smithi130 ceph-mon[31342]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:49.305 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:48 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:21:49.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:48 smithi130 ceph-mon[42749]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:49.305 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:48 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:21:49.559 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 systemd[1]: Stopping Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:21:49.833 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:21:49.558Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:21:49.833 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:21:49.558Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-17T17:21:49.833 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:21:49.558Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-17T17:21:49.833 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:21:49.558Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T17:21:49.833 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:21:49.558Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-17T17:21:49.833 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:21:49.558Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-17T17:21:49.834 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:21:49.558Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T17:21:49.834 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:21:49.558Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-17T17:21:49.834 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:21:49.559Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-17T17:21:49.834 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:21:49.559Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T17:21:49.834 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:21:49.559Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-17T17:21:49.834 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[63957]: ts=2024-08-17T17:21:49.559Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-17T17:21:49.834 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 podman[67329]: 2024-08-17 17:21:49.569111786 +0000 UTC m=+0.123541990 container died 56bc2272bf9552218a4aa8bc13c5046cf55bd8256c36be174451c37696eb9c6b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:50.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 podman[67329]: 2024-08-17 17:21:49.93128141 +0000 UTC m=+0.485711613 container cleanup 56bc2272bf9552218a4aa8bc13c5046cf55bd8256c36be174451c37696eb9c6b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:50.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:49 smithi179 bash[67329]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T17:21:50.566 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 podman[67346]: 2024-08-17 17:21:50.199041982 +0000 UTC m=+0.628477468 container remove 56bc2272bf9552218a4aa8bc13c5046cf55bd8256c36be174451c37696eb9c6b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:50.567 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-17T17:21:50.567 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 systemd[1]: Stopped Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:21:50.567 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@prometheus.a.service: Consumed 1.129s CPU time. 2024-08-17T17:21:50.567 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 systemd[1]: Starting Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:21:50.828 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 podman[67444]: 2024-08-17 17:21:50.469565857 +0000 UTC m=+0.018109375 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-17T17:21:50.829 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 podman[67444]: 2024-08-17 17:21:50.567327666 +0000 UTC m=+0.115871228 container create ac95b2aef275041c8a7b0a327f9d9d57ca240fdf72ccd11468ec3c255c8e1e0f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:50.829 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 podman[67444]: 2024-08-17 17:21:50.777817276 +0000 UTC m=+0.326360809 container init ac95b2aef275041c8a7b0a327f9d9d57ca240fdf72ccd11468ec3c255c8e1e0f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:50.829 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 podman[67444]: 2024-08-17 17:21:50.782492258 +0000 UTC m=+0.331035775 container start ac95b2aef275041c8a7b0a327f9d9d57ca240fdf72ccd11468ec3c255c8e1e0f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:21:50.829 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.806Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-17T17:21:50.829 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.806Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-17T17:21:50.829 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.806Z caller=main.go:570 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi179 (none))" 2024-08-17T17:21:50.829 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.806Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T17:21:50.829 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.806Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T17:21:50.829 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.807Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T17:21:50.829 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.808Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-17T17:21:50.829 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.810Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T17:21:50.830 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.810Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T17:21:50.830 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.811Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T17:21:50.830 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.811Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.301µs 2024-08-17T17:21:50.830 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.811Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T17:21:50.830 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.811Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-08-17T17:21:50.830 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.811Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-08-17T17:21:50.830 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.811Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.522µs wal_replay_duration=524.784µs wbl_replay_duration=122ns total_replay_duration=565.243µs 2024-08-17T17:21:50.830 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.812Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T17:21:50.830 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.812Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-17T17:21:50.830 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.812Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T17:21:50.831 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.828Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.522419ms db_storage=741ns remote_storage=1.177µs web_handler=690ns query_engine=562ns scrape=372.69µs scrape_sd=25.802µs notify=24.585µs notify_sd=6.761µs rules=14.812547ms tracing=6.089µs 2024-08-17T17:21:50.831 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.828Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-17T17:21:50.831 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:21:50.828Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T17:21:51.083 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 bash[67444]: ac95b2aef275041c8a7b0a327f9d9d57ca240fdf72ccd11468ec3c255c8e1e0f 2024-08-17T17:21:51.083 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:21:50 smithi179 systemd[1]: Started Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:21:51.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:51 smithi179 ceph-mon[34775]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:51.398 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:51 smithi130 ceph-mon[31342]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:51.398 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:51 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:51] ENGINE Bus STOPPING 2024-08-17T17:21:51.398 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:51 smithi130 ceph-mon[42749]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:51.666 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:51 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:21:51.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:51 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:51] ENGINE Bus STOPPED 2024-08-17T17:21:51.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:51 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:51] ENGINE Bus STARTING 2024-08-17T17:21:51.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:51 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:51] ENGINE Serving on http://:::9283 2024-08-17T17:21:51.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:51 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:51] ENGINE Bus STARTED 2024-08-17T17:21:51.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:51 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:51] ENGINE Bus STOPPING 2024-08-17T17:21:52.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:21:52.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T17:21:52.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T17:21:52.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:21:52.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:21:52.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T17:21:52.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T17:21:52.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:21:52.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi179.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T17:21:52.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-17T17:21:52.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: osdmap e67: 8 total, 8 up, 8 in 2024-08-17T17:21:52.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:21:52.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T17:21:52.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T17:21:52.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi179.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T17:21:52.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:52 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:21:52.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:21:52.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T17:21:52.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T17:21:52.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:21:52.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:21:52.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T17:21:52.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T17:21:52.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:21:52.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi179.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T17:21:52.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-17T17:21:52.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: osdmap e67: 8 total, 8 up, 8 in 2024-08-17T17:21:52.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:21:52.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T17:21:52.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T17:21:52.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi179.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T17:21:52.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:21:52.419 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:52] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:21:52.419 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:52] ENGINE Bus STOPPED 2024-08-17T17:21:52.419 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:52] ENGINE Bus STARTING 2024-08-17T17:21:52.419 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:52] ENGINE Serving on http://:::9283 2024-08-17T17:21:52.419 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:52] ENGINE Bus STARTED 2024-08-17T17:21:52.419 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:52] ENGINE Bus STOPPING 2024-08-17T17:21:52.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:21:52.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T17:21:52.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T17:21:52.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:21:52.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi130.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi179.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: osdmap e67: 8 total, 8 up, 8 in 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi179.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:21:52.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[31319]: 2024-08-17T17:21:52.145+0000 7fc637e10640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T17:21:52.836 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:52] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:21:52.836 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:52] ENGINE Bus STOPPED 2024-08-17T17:21:52.836 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:52] ENGINE Bus STARTING 2024-08-17T17:21:52.836 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:52] ENGINE Serving on http://:::9283 2024-08-17T17:21:52.836 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:21:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:21:52] ENGINE Bus STARTED 2024-08-17T17:21:53.086 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:21:53.087 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi179.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T17:21:53.087 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:21:53.087 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi179.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T17:21:53.087 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: pgmap v50: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:53.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:21:53.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi179.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T17:21:53.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:21:53.088 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi179.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T17:21:53.088 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: pgmap v50: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:53.088 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-17T17:21:53.088 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: osdmap e68: 8 total, 8 up, 8 in 2024-08-17T17:21:53.088 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T17:21:53.088 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T17:21:53.088 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T17:21:53.088 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-17T17:21:53.088 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: osdmap e69: 8 total, 8 up, 8 in 2024-08-17T17:21:53.088 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-17T17:21:53.088 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: fsmap foo:0 2024-08-17T17:21:53.088 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:53.089 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[31342]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:21:53.274 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-08-17T17:21:53.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:21:53.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi179.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T17:21:53.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:21:53.347 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi179.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T17:21:53.347 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: pgmap v50: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:53.347 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-17T17:21:53.347 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: osdmap e68: 8 total, 8 up, 8 in 2024-08-17T17:21:53.347 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T17:21:53.347 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T17:21:53.347 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T17:21:53.347 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-17T17:21:53.347 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: osdmap e69: 8 total, 8 up, 8 in 2024-08-17T17:21:53.347 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-17T17:21:53.347 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: fsmap foo:0 2024-08-17T17:21:53.347 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:53.347 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:53 smithi179 ceph-mon[34775]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:21:53.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-17T17:21:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: osdmap e68: 8 total, 8 up, 8 in 2024-08-17T17:21:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T17:21:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T17:21:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T17:21:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-17T17:21:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: osdmap e69: 8 total, 8 up, 8 in 2024-08-17T17:21:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-17T17:21:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: fsmap foo:0 2024-08-17T17:21:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:53 smithi130 ceph-mon[42749]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:21:53.852 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:54.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:54 smithi179 ceph-mon[34775]: Saving service mds.foo spec with placement count:2 2024-08-17T17:21:54.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:54 smithi179 ceph-mon[34775]: osdmap e70: 8 total, 8 up, 8 in 2024-08-17T17:21:54.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:54 smithi130 ceph-mon[31342]: Saving service mds.foo spec with placement count:2 2024-08-17T17:21:54.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:54 smithi130 ceph-mon[31342]: osdmap e70: 8 total, 8 up, 8 in 2024-08-17T17:21:54.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:54 smithi130 ceph-mon[42749]: Saving service mds.foo spec with placement count:2 2024-08-17T17:21:54.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:54 smithi130 ceph-mon[42749]: osdmap e70: 8 total, 8 up, 8 in 2024-08-17T17:21:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:55 smithi179 ceph-mon[34775]: pgmap v54: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:55 smithi179 ceph-mon[34775]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:21:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:55 smithi179 ceph-mon[34775]: osdmap e71: 8 total, 8 up, 8 in 2024-08-17T17:21:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:55 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:55 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:55 smithi130 ceph-mon[42749]: pgmap v54: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:55 smithi130 ceph-mon[42749]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:21:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:55 smithi130 ceph-mon[42749]: osdmap e71: 8 total, 8 up, 8 in 2024-08-17T17:21:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:55 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:55 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:55 smithi130 ceph-mon[31342]: pgmap v54: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:55 smithi130 ceph-mon[31342]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:21:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:55 smithi130 ceph-mon[31342]: osdmap e71: 8 total, 8 up, 8 in 2024-08-17T17:21:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:55 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:55.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:55 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:56.111 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:21:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[79188]: ts=2024-08-17T17:21:56.018Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003269631s 2024-08-17T17:21:57.033 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-08-17T17:21:57.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[42749]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:57.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:57.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:57.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:21:57.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:21:57.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:57.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:21:57.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:21:57.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T17:21:57.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:21:57.299 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[31342]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:57.299 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:57.299 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:57.299 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:21:57.299 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:21:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:21:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:21:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T17:21:57.300 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:57 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:21:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:57 smithi179 ceph-mon[34775]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:57 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:57 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:57 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:21:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:57 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:21:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:57 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:21:57.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:57 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:21:57.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:57 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:21:57.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:57 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T17:21:57.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:57 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:21:57.883 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:58.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:58 smithi130 ceph-mon[31342]: Deploying daemon mds.foo.smithi130.jmaidg on smithi130 2024-08-17T17:21:58.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:58 smithi130 ceph-mon[42749]: Deploying daemon mds.foo.smithi130.jmaidg on smithi130 2024-08-17T17:21:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:58 smithi179 ceph-mon[34775]: Deploying daemon mds.foo.smithi130.jmaidg on smithi130 2024-08-17T17:21:59.236 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:21:59 smithi130 ceph-mon[31342]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:59.237 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:21:59 smithi130 ceph-mon[42749]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:21:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:21:59 smithi179 ceph-mon[34775]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:01.364 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:01 smithi179 ceph-mon[34775]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:01.364 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:01 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:01.365 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:01 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:01.365 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:01 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:01.365 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:01 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:22:01.365 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:01 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:22:01.365 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:01 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T17:22:01.365 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:01 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:22:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[31342]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:22:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:22:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T17:22:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:22:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[42749]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:22:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T17:22:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T17:22:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:01 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:22:01.521 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-08-17T17:22:01.534 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-17T17:22:01.534 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-17T17:22:01.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph mgr dump --format=json 2024-08-17T17:22:01.803 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:02.311 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:02 smithi179 ceph-mon[34775]: Deploying daemon mds.foo.smithi179.zkkvno on smithi179 2024-08-17T17:22:02.311 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:02 smithi179 ceph-mon[34775]: mds.? [v2:172.21.15.130:6834/1214210479,v1:172.21.15.130:6835/1214210479] up:boot 2024-08-17T17:22:02.311 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:02 smithi179 ceph-mon[34775]: daemon mds.foo.smithi130.jmaidg assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-17T17:22:02.311 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:02 smithi179 ceph-mon[34775]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-17T17:22:02.311 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:02 smithi179 ceph-mon[34775]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-17T17:22:02.311 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:02 smithi179 ceph-mon[34775]: Cluster is now healthy 2024-08-17T17:22:02.311 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:02 smithi179 ceph-mon[34775]: fsmap foo:0 1 up:standby 2024-08-17T17:22:02.311 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:02 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.jmaidg"}]: dispatch 2024-08-17T17:22:02.312 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:02 smithi179 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:creating} 2024-08-17T17:22:02.312 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:02 smithi179 ceph-mon[34775]: daemon mds.foo.smithi130.jmaidg is now active in filesystem foo as rank 0 2024-08-17T17:22:02.312 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:02 smithi179 ceph-mon[34775]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:02.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[31342]: Deploying daemon mds.foo.smithi179.zkkvno on smithi179 2024-08-17T17:22:02.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[31342]: mds.? [v2:172.21.15.130:6834/1214210479,v1:172.21.15.130:6835/1214210479] up:boot 2024-08-17T17:22:02.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[31342]: daemon mds.foo.smithi130.jmaidg assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-17T17:22:02.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[31342]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-17T17:22:02.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[31342]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-17T17:22:02.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[31342]: Cluster is now healthy 2024-08-17T17:22:02.316 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[31342]: fsmap foo:0 1 up:standby 2024-08-17T17:22:02.317 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.jmaidg"}]: dispatch 2024-08-17T17:22:02.317 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[31342]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:creating} 2024-08-17T17:22:02.317 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[31342]: daemon mds.foo.smithi130.jmaidg is now active in filesystem foo as rank 0 2024-08-17T17:22:02.317 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[31342]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:02.317 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[42749]: Deploying daemon mds.foo.smithi179.zkkvno on smithi179 2024-08-17T17:22:02.317 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[42749]: mds.? [v2:172.21.15.130:6834/1214210479,v1:172.21.15.130:6835/1214210479] up:boot 2024-08-17T17:22:02.317 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[42749]: daemon mds.foo.smithi130.jmaidg assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-17T17:22:02.317 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[42749]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-17T17:22:02.317 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[42749]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-17T17:22:02.317 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[42749]: Cluster is now healthy 2024-08-17T17:22:02.317 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[42749]: fsmap foo:0 1 up:standby 2024-08-17T17:22:02.317 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.jmaidg"}]: dispatch 2024-08-17T17:22:02.318 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[42749]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:creating} 2024-08-17T17:22:02.318 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[42749]: daemon mds.foo.smithi130.jmaidg is now active in filesystem foo as rank 0 2024-08-17T17:22:02.318 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:02 smithi130 ceph-mon[42749]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:03.265 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:03 smithi130 ceph-mon[31342]: mds.? [v2:172.21.15.130:6834/1214210479,v1:172.21.15.130:6835/1214210479] up:active 2024-08-17T17:22:03.265 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:03 smithi130 ceph-mon[31342]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 2024-08-17T17:22:03.266 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:03 smithi130 ceph-mon[42749]: mds.? [v2:172.21.15.130:6834/1214210479,v1:172.21.15.130:6835/1214210479] up:active 2024-08-17T17:22:03.266 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:03 smithi130 ceph-mon[42749]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 2024-08-17T17:22:03.273 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:03 smithi179 ceph-mon[34775]: mds.? [v2:172.21.15.130:6834/1214210479,v1:172.21.15.130:6835/1214210479] up:active 2024-08-17T17:22:03.273 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:03 smithi179 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 2024-08-17T17:22:03.857 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:22:04.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:22:04.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[42749]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:04.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:22:04.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/284805435' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T17:22:04.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[42749]: mds.? [v2:172.21.15.179:6832/449182576,v1:172.21.15.179:6833/449182576] up:boot 2024-08-17T17:22:04.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[42749]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:22:04.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi179.zkkvno"}]: dispatch 2024-08-17T17:22:04.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[42749]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:22:04.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:22:04.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[31342]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:04.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:22:04.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/284805435' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T17:22:04.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[31342]: mds.? [v2:172.21.15.179:6832/449182576,v1:172.21.15.179:6833/449182576] up:boot 2024-08-17T17:22:04.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[31342]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:22:04.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi179.zkkvno"}]: dispatch 2024-08-17T17:22:04.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-mon[31342]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:22:04.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:22:04.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-mon[34775]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:04.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:22:04.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:04.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/284805435' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T17:22:04.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-mon[34775]: mds.? [v2:172.21.15.179:6832/449182576,v1:172.21.15.179:6833/449182576] up:boot 2024-08-17T17:22:04.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:22:04.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi179.zkkvno"}]: dispatch 2024-08-17T17:22:04.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:22:04.808 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:22:04 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:22:04] "GET /metrics HTTP/1.1" 200 208924 "" "Prometheus/2.43.0" 2024-08-17T17:22:04.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:22:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:22:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:22:05.082 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":21,"flags":0,"active_gid":24338,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":824081166},{"type":"v1","addr":"172.21.15.130:6801","nonce":824081166}]},"active_addr":"172.21.15.130:6801/824081166","active_change":"2024-08-17T17:20:33.508124+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24322,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":1457627030}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":1299294781}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":1843620043}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":854839742}]}]} 2024-08-17T17:22:05.086 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-17T17:22:05.086 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T17:22:05.087 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:22:05.373 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:06.636 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:06 smithi179 ceph-mon[34775]: mds.? [v2:172.21.15.130:6834/1214210479,v1:172.21.15.130:6835/1214210479] up:active 2024-08-17T17:22:06.636 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:06 smithi179 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:22:06.636 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:06 smithi179 ceph-mon[34775]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:06.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:06 smithi130 ceph-mon[31342]: mds.? [v2:172.21.15.130:6834/1214210479,v1:172.21.15.130:6835/1214210479] up:active 2024-08-17T17:22:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:06 smithi130 ceph-mon[31342]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:22:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:06 smithi130 ceph-mon[31342]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:06 smithi130 ceph-mon[42749]: mds.? [v2:172.21.15.130:6834/1214210479,v1:172.21.15.130:6835/1214210479] up:active 2024-08-17T17:22:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:06 smithi130 ceph-mon[42749]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:22:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:06 smithi130 ceph-mon[42749]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:07.474 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:22:07.475 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":71,"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","created":"2024-08-17T17:15:09.488012+0000","modified":"2024-08-17T17:21:54.150173+0000","last_up_change":"2024-08-17T17:20:02.834333+0000","last_in_change":"2024-08-17T17:19:46.093156+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:18:10.192831+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:20:37.124673+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T17:21:51.081217+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T17:21:51.379206+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"30c31adb-2cda-466b-933e-a8e2cb714fab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6803","nonce":2346469001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6805","nonce":2346469001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6809","nonce":2346469001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6807","nonce":2346469001}]},"public_addr":"172.21.15.130:6803/2346469001","cluster_addr":"172.21.15.130:6805/2346469001","heartbeat_back_addr":"172.21.15.130:6809/2346469001","heartbeat_front_addr":"172.21.15.130:6807/2346469001","state":["exists","up"]},{"osd":1,"uuid":"55049bb4-b976-42c6-8a5d-8b3e9ae20988","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6811","nonce":2284482082}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6813","nonce":2284482082}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6817","nonce":2284482082}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6815","nonce":2284482082}]},"public_addr":"172.21.15.130:6811/2284482082","cluster_addr":"172.21.15.130:6813/2284482082","heartbeat_back_addr":"172.21.15.130:6817/2284482082","heartbeat_front_addr":"172.21.15.130:6815/2284482082","state":["exists","up"]},{"osd":2,"uuid":"3f40f088-27f7-4450-8f31-b449711365ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6819","nonce":160441313}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6821","nonce":160441313}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6825","nonce":160441313}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6823","nonce":160441313}]},"public_addr":"172.21.15.130:6819/160441313","cluster_addr":"172.21.15.130:6821/160441313","heartbeat_back_addr":"172.21.15.130:6825/160441313","heartbeat_front_addr":"172.21.15.130:6823/160441313","state":["exists","up"]},{"osd":3,"uuid":"7684c401-462a-4e35-bb9c-1be23dfe573b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6827","nonce":775049303}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6829","nonce":775049303}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6833","nonce":775049303}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6831","nonce":775049303}]},"public_addr":"172.21.15.130:6827/775049303","cluster_addr":"172.21.15.130:6829/775049303","heartbeat_back_addr":"172.21.15.130:6833/775049303","heartbeat_front_addr":"172.21.15.130:6831/775049303","state":["exists","up"]},{"osd":4,"uuid":"7a3a331f-533a-49de-910d-5989f2f84b65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6800","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6801","nonce":2107797845}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6802","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6803","nonce":2107797845}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6806","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6807","nonce":2107797845}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6804","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6805","nonce":2107797845}]},"public_addr":"172.21.15.179:6801/2107797845","cluster_addr":"172.21.15.179:6803/2107797845","heartbeat_back_addr":"172.21.15.179:6807/2107797845","heartbeat_front_addr":"172.21.15.179:6805/2107797845","state":["exists","up"]},{"osd":5,"uuid":"d53d65ee-bbb2-45ec-b557-07cc62a45b05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6808","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6809","nonce":2392026049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6810","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6811","nonce":2392026049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6814","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6815","nonce":2392026049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6812","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6813","nonce":2392026049}]},"public_addr":"172.21.15.179:6809/2392026049","cluster_addr":"172.21.15.179:6811/2392026049","heartbeat_back_addr":"172.21.15.179:6815/2392026049","heartbeat_front_addr":"172.21.15.179:6813/2392026049","state":["exists","up"]},{"osd":6,"uuid":"b84229ad-2c88-4064-a070-33b98714ef96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6816","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6817","nonce":139895003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6818","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6819","nonce":139895003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6822","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6823","nonce":139895003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6820","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6821","nonce":139895003}]},"public_addr":"172.21.15.179:6817/139895003","cluster_addr":"172.21.15.179:6819/139895003","heartbeat_back_addr":"172.21.15.179:6823/139895003","heartbeat_front_addr":"172.21.15.179:6821/139895003","state":["exists","up"]},{"osd":7,"uuid":"bba7978f-8b19-40e6-86bd-692a0e282429","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6824","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6825","nonce":2533639201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6826","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6827","nonce":2533639201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6830","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6831","nonce":2533639201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6828","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6829","nonce":2533639201}]},"public_addr":"172.21.15.179:6825/2533639201","cluster_addr":"172.21.15.179:6827/2533639201","heartbeat_back_addr":"172.21.15.179:6831/2533639201","heartbeat_front_addr":"172.21.15.179:6829/2533639201","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:19.220776+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:42.876683+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:07.028539+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:31.259509+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:52.172235+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:15.112109+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:37.781476+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:20:00.893216+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/851371227":"2024-08-18T17:20:33.507974+0000","172.21.15.130:6801/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/1690071627":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6800/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/2999329360":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/3071443649":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/368845317":"2024-08-18T17:20:33.507974+0000","172.21.15.130:6800/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6801/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/88320168":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/3471387839":"2024-08-18T17:15:57.894717+0000","172.21.15.130:6800/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/1247136467":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2589122337":"2024-08-18T17:20:33.507974+0000","172.21.15.130:6801/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2578011752":"2024-08-18T17:20:33.507974+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:22:08.376 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:08 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:08.377 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:08 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:08.377 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:08 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2466011357' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:22:08.377 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:08 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:08.377 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:08 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:08.377 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:08 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2466011357' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:22:08.407 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T17:22:08.407 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:22:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:08 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:08 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:08 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2466011357' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:22:09.195 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:09.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:09 smithi130 ceph-mon[31342]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:09.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:09 smithi130 ceph-mon[31342]: mds.? [v2:172.21.15.179:6832/449182576,v1:172.21.15.179:6833/449182576] up:standby 2024-08-17T17:22:09.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:09 smithi130 ceph-mon[31342]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:22:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:09 smithi130 ceph-mon[42749]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:09 smithi130 ceph-mon[42749]: mds.? [v2:172.21.15.179:6832/449182576,v1:172.21.15.179:6833/449182576] up:standby 2024-08-17T17:22:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:09 smithi130 ceph-mon[42749]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:22:09.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:09 smithi179 ceph-mon[34775]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:09 smithi179 ceph-mon[34775]: mds.? [v2:172.21.15.179:6832/449182576,v1:172.21.15.179:6833/449182576] up:standby 2024-08-17T17:22:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:09 smithi179 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:22:10.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:10 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:10.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:10 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:10.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:10 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:22:10.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:10 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:22:10.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:10 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:10.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:10 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:22:10.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:10 smithi130 ceph-mon[31342]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:10.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:10 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:10.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:10 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:10.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:10 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:22:10.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:10 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:22:10.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:10 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:10.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:10 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:22:10.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:10 smithi130 ceph-mon[42749]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:10.457 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:10 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:10.457 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:10 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:10.458 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:10 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:22:10.458 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:10 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:22:10.458 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:10 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:10.458 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:10 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:22:10.458 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:10 smithi179 ceph-mon[34775]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:11.292 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:22:11.292 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":71,"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","created":"2024-08-17T17:15:09.488012+0000","modified":"2024-08-17T17:21:54.150173+0000","last_up_change":"2024-08-17T17:20:02.834333+0000","last_in_change":"2024-08-17T17:19:46.093156+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:18:10.192831+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:20:37.124673+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T17:21:51.081217+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T17:21:51.379206+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"30c31adb-2cda-466b-933e-a8e2cb714fab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6803","nonce":2346469001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6805","nonce":2346469001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6809","nonce":2346469001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":2346469001},{"type":"v1","addr":"172.21.15.130:6807","nonce":2346469001}]},"public_addr":"172.21.15.130:6803/2346469001","cluster_addr":"172.21.15.130:6805/2346469001","heartbeat_back_addr":"172.21.15.130:6809/2346469001","heartbeat_front_addr":"172.21.15.130:6807/2346469001","state":["exists","up"]},{"osd":1,"uuid":"55049bb4-b976-42c6-8a5d-8b3e9ae20988","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6811","nonce":2284482082}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6813","nonce":2284482082}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6817","nonce":2284482082}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":2284482082},{"type":"v1","addr":"172.21.15.130:6815","nonce":2284482082}]},"public_addr":"172.21.15.130:6811/2284482082","cluster_addr":"172.21.15.130:6813/2284482082","heartbeat_back_addr":"172.21.15.130:6817/2284482082","heartbeat_front_addr":"172.21.15.130:6815/2284482082","state":["exists","up"]},{"osd":2,"uuid":"3f40f088-27f7-4450-8f31-b449711365ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6819","nonce":160441313}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6821","nonce":160441313}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6825","nonce":160441313}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":160441313},{"type":"v1","addr":"172.21.15.130:6823","nonce":160441313}]},"public_addr":"172.21.15.130:6819/160441313","cluster_addr":"172.21.15.130:6821/160441313","heartbeat_back_addr":"172.21.15.130:6825/160441313","heartbeat_front_addr":"172.21.15.130:6823/160441313","state":["exists","up"]},{"osd":3,"uuid":"7684c401-462a-4e35-bb9c-1be23dfe573b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6827","nonce":775049303}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6829","nonce":775049303}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6833","nonce":775049303}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":775049303},{"type":"v1","addr":"172.21.15.130:6831","nonce":775049303}]},"public_addr":"172.21.15.130:6827/775049303","cluster_addr":"172.21.15.130:6829/775049303","heartbeat_back_addr":"172.21.15.130:6833/775049303","heartbeat_front_addr":"172.21.15.130:6831/775049303","state":["exists","up"]},{"osd":4,"uuid":"7a3a331f-533a-49de-910d-5989f2f84b65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6800","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6801","nonce":2107797845}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6802","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6803","nonce":2107797845}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6806","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6807","nonce":2107797845}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6804","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6805","nonce":2107797845}]},"public_addr":"172.21.15.179:6801/2107797845","cluster_addr":"172.21.15.179:6803/2107797845","heartbeat_back_addr":"172.21.15.179:6807/2107797845","heartbeat_front_addr":"172.21.15.179:6805/2107797845","state":["exists","up"]},{"osd":5,"uuid":"d53d65ee-bbb2-45ec-b557-07cc62a45b05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6808","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6809","nonce":2392026049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6810","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6811","nonce":2392026049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6814","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6815","nonce":2392026049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6812","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6813","nonce":2392026049}]},"public_addr":"172.21.15.179:6809/2392026049","cluster_addr":"172.21.15.179:6811/2392026049","heartbeat_back_addr":"172.21.15.179:6815/2392026049","heartbeat_front_addr":"172.21.15.179:6813/2392026049","state":["exists","up"]},{"osd":6,"uuid":"b84229ad-2c88-4064-a070-33b98714ef96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6816","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6817","nonce":139895003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6818","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6819","nonce":139895003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6822","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6823","nonce":139895003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6820","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6821","nonce":139895003}]},"public_addr":"172.21.15.179:6817/139895003","cluster_addr":"172.21.15.179:6819/139895003","heartbeat_back_addr":"172.21.15.179:6823/139895003","heartbeat_front_addr":"172.21.15.179:6821/139895003","state":["exists","up"]},{"osd":7,"uuid":"bba7978f-8b19-40e6-86bd-692a0e282429","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6824","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6825","nonce":2533639201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6826","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6827","nonce":2533639201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6830","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6831","nonce":2533639201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6828","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6829","nonce":2533639201}]},"public_addr":"172.21.15.179:6825/2533639201","cluster_addr":"172.21.15.179:6827/2533639201","heartbeat_back_addr":"172.21.15.179:6831/2533639201","heartbeat_front_addr":"172.21.15.179:6829/2533639201","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:19.220776+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:42.876683+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:07.028539+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:31.259509+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:52.172235+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:15.112109+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:37.781476+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:20:00.893216+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/851371227":"2024-08-18T17:20:33.507974+0000","172.21.15.130:6801/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/1690071627":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6800/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/2999329360":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/3071443649":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/368845317":"2024-08-18T17:20:33.507974+0000","172.21.15.130:6800/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6801/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/88320168":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/3471387839":"2024-08-18T17:15:57.894717+0000","172.21.15.130:6800/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/1247136467":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2589122337":"2024-08-18T17:20:33.507974+0000","172.21.15.130:6801/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2578011752":"2024-08-18T17:20:33.507974+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:22:11.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:11 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1368440488' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:22:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:11 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1368440488' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:22:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:11 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1368440488' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T17:22:12.086 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T17:22:12.086 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T17:22:12.087 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T17:22:12.087 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T17:22:12.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T17:22:12.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T17:22:12.089 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T17:22:12.089 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T17:22:12.358 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:12 smithi130 ceph-mon[42749]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:12 smithi179 ceph-mon[34775]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:12.658 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:12.662 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:12.662 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:12.664 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:12.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:12 smithi130 ceph-mon[31342]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:12.669 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:12.670 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:12.670 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:13.776 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:13 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:13.776 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:13 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:13.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:13 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:13.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:13 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:13 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:13 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:14 smithi130 ceph-mon[31342]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:14.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:22:14 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:22:14] "GET /metrics HTTP/1.1" 200 208924 "" "Prometheus/2.43.0" 2024-08-17T17:22:14.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:14 smithi130 ceph-mon[42749]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:14 smithi179 ceph-mon[34775]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:14.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:22:14 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:22:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:22:16.638 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:16 smithi130 ceph-mon[31342]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:16.638 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:16 smithi130 ceph-mon[42749]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:17.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:16 smithi179 ceph-mon[34775]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-08-17T17:22:18.851 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:18 smithi130 ceph-mon[31342]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:22:18.851 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:18 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:22:18.851 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:18 smithi130 ceph-mon[42749]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:22:18.851 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:18 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:22:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:18 smithi179 ceph-mon[34775]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:22:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:18 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:22:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:20 smithi130 ceph-mon[31342]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:22:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:20 smithi130 ceph-mon[42749]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:22:20.922 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:20 smithi179 ceph-mon[34775]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:22:22.198 INFO:teuthology.orchestra.run.smithi130.stdout:34359738431 2024-08-17T17:22:22.198 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T17:22:22.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:22 smithi130 ceph-mon[31342]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:22.802 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:22 smithi130 ceph-mon[42749]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:23.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:22 smithi179 ceph-mon[34775]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:23.665 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:24.353 INFO:teuthology.orchestra.run.smithi130.stdout:231928234016 2024-08-17T17:22:24.353 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T17:22:24.621 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:22:24 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:22:24] "GET /metrics HTTP/1.1" 200 239641 "" "Prometheus/2.43.0" 2024-08-17T17:22:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:24 smithi179 ceph-mon[34775]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:24.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:22:24 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:22:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:22:24.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:24 smithi130 ceph-mon[31342]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:24 smithi130 ceph-mon[42749]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:24.951 INFO:teuthology.orchestra.run.smithi130.stdout:197568495651 2024-08-17T17:22:24.951 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T17:22:26.788 INFO:teuthology.orchestra.run.smithi130.stdout:107374182450 2024-08-17T17:22:26.788 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T17:22:26.915 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:26 smithi130 ceph-mon[31342]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:26.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:26 smithi130 ceph-mon[42749]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:27.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:26 smithi179 ceph-mon[34775]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:28.248 INFO:teuthology.orchestra.run.smithi130.stdout:55834574908 2024-08-17T17:22:28.248 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T17:22:28.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:28 smithi130 ceph-mon[31342]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:28.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:28 smithi130 ceph-mon[42749]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:29.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:28 smithi179 ceph-mon[34775]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:29.273 INFO:teuthology.orchestra.run.smithi130.stdout:128849018925 2024-08-17T17:22:29.273 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T17:22:29.767 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:30.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:29 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:30.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:29 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:30.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:29 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:22:30.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:29 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:22:30.050 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:29 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:30.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:29 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:30.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:29 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:30.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:29 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:22:30.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:29 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:22:30.051 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:29 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:30.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:29 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:30.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:29 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:30.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:29 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:22:30.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:29 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:22:30.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:29 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:22:30.574 INFO:teuthology.orchestra.run.smithi130.stdout:77309411382 2024-08-17T17:22:30.574 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T17:22:30.791 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:30.791 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:30.792 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:31.256 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:31.328 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:30 smithi130 ceph-mon[31342]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:31.328 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:30 smithi130 ceph-mon[42749]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:30 smithi179 ceph-mon[34775]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:32.008 INFO:teuthology.orchestra.run.smithi130.stdout:163208757291 2024-08-17T17:22:32.008 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T17:22:32.286 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:33.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:32 smithi130 ceph-mon[31342]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:33.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:32 smithi130 ceph-mon[42749]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:33.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:32 smithi179 ceph-mon[34775]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:34.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:33 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-08-17T17:22:34.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:33 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-08-17T17:22:34.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:33 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-08-17T17:22:34.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:33 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-08-17T17:22:34.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:33 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-08-17T17:22:34.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:33 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-08-17T17:22:34.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:33 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:22:34.061 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:33 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-08-17T17:22:34.061 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:33 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-08-17T17:22:34.062 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:33 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-08-17T17:22:34.062 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:33 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-08-17T17:22:34.062 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:33 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-08-17T17:22:34.062 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:33 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-08-17T17:22:34.062 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:33 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:22:34.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:33 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-08-17T17:22:34.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:33 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-08-17T17:22:34.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:33 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-08-17T17:22:34.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:33 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-08-17T17:22:34.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:33 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-08-17T17:22:34.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:33 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-08-17T17:22:34.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:33 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:22:34.666 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:22:34 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:22:34] "GET /metrics HTTP/1.1" 200 239615 "" "Prometheus/2.43.0" 2024-08-17T17:22:34.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:22:34 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:22:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:22:35.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:34 smithi179 ceph-mon[34775]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:35.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:34 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-08-17T17:22:35.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:34 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-08-17T17:22:35.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:34 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-08-17T17:22:35.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:34 smithi179 ceph-mon[34775]: osdmap e72: 8 total, 8 up, 8 in 2024-08-17T17:22:35.379 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:34 smithi130 ceph-mon[42749]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:35.380 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:34 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-08-17T17:22:35.380 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:34 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-08-17T17:22:35.380 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:34 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-08-17T17:22:35.380 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:34 smithi130 ceph-mon[42749]: osdmap e72: 8 total, 8 up, 8 in 2024-08-17T17:22:35.380 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:34 smithi130 ceph-mon[31342]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:35.380 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:34 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-08-17T17:22:35.380 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:34 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-08-17T17:22:35.380 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:34 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-08-17T17:22:35.380 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:34 smithi130 ceph-mon[31342]: osdmap e72: 8 total, 8 up, 8 in 2024-08-17T17:22:36.206 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:35 smithi130 ceph-mon[31342]: osdmap e73: 8 total, 8 up, 8 in 2024-08-17T17:22:36.206 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:35 smithi130 ceph-mon[42749]: osdmap e73: 8 total, 8 up, 8 in 2024-08-17T17:22:36.225 INFO:teuthology.orchestra.run.smithi130.stdout:107374182452 2024-08-17T17:22:36.291 INFO:teuthology.orchestra.run.smithi130.stdout:231928234019 2024-08-17T17:22:36.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:35 smithi179 ceph-mon[34775]: osdmap e73: 8 total, 8 up, 8 in 2024-08-17T17:22:36.924 INFO:teuthology.orchestra.run.smithi130.stdout:55834574910 2024-08-17T17:22:37.131 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:37.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:36 smithi179 ceph-mon[34775]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:37.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:36 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/569604696' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T17:22:37.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:36 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/136741010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T17:22:37.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:36 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/643966601' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T17:22:37.344 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:36 smithi130 ceph-mon[31342]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:37.344 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:36 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/569604696' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T17:22:37.344 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:36 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/136741010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T17:22:37.344 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:36 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/643966601' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T17:22:37.344 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:36 smithi130 ceph-mon[42749]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:37.344 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:36 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/569604696' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T17:22:37.344 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:36 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/136741010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T17:22:37.344 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:36 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/643966601' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T17:22:38.464 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234016 got 231928234019 for osd.7 2024-08-17T17:22:38.464 DEBUG:teuthology.parallel:result is None 2024-08-17T17:22:38.712 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574908 got 55834574910 for osd.1 2024-08-17T17:22:38.712 DEBUG:teuthology.parallel:result is None 2024-08-17T17:22:39.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:38 smithi179 ceph-mon[34775]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:39.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:38 smithi130 ceph-mon[31342]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:38 smithi130 ceph-mon[42749]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:40.787 INFO:teuthology.orchestra.run.smithi130.stdout:197568495655 2024-08-17T17:22:41.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:40 smithi130 ceph-mon[31342]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:41.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:40 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1940396269' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T17:22:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:40 smithi130 ceph-mon[42749]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:40 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1940396269' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T17:22:41.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:40 smithi179 ceph-mon[34775]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:41.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:40 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1940396269' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T17:22:41.771 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495651 got 197568495655 for osd.6 2024-08-17T17:22:41.771 DEBUG:teuthology.parallel:result is None 2024-08-17T17:22:42.915 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:42.941 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182450 got 107374182452 for osd.3 2024-08-17T17:22:42.942 DEBUG:teuthology.parallel:result is None 2024-08-17T17:22:43.159 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:42 smithi130 ceph-mon[31342]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:43.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:42 smithi130 ceph-mon[42749]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:43.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:42 smithi179 ceph-mon[34775]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:43.405 INFO:teuthology.orchestra.run.smithi130.stdout:128849018928 2024-08-17T17:22:44.039 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:43 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1037347560' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T17:22:44.039 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:43 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1037347560' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T17:22:44.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:43 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1037347560' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T17:22:44.484 INFO:teuthology.orchestra.run.smithi130.stdout:34359738436 2024-08-17T17:22:44.621 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:22:44 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:22:44] "GET /metrics HTTP/1.1" 200 239615 "" "Prometheus/2.43.0" 2024-08-17T17:22:44.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:22:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:22:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:22:45.091 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018925 got 128849018928 for osd.4 2024-08-17T17:22:45.091 DEBUG:teuthology.parallel:result is None 2024-08-17T17:22:45.250 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:44 smithi130 ceph-mon[31342]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:45.250 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:44 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1008506629' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T17:22:45.250 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:44 smithi130 ceph-mon[42749]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:45.250 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:44 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1008506629' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T17:22:45.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:44 smithi179 ceph-mon[34775]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:45.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:44 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1008506629' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T17:22:45.610 INFO:teuthology.orchestra.run.smithi130.stdout:77309411385 2024-08-17T17:22:46.114 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:45 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3728693706' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T17:22:46.114 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:45 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3728693706' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T17:22:46.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:45 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3728693706' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T17:22:46.482 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738431 got 34359738436 for osd.0 2024-08-17T17:22:46.482 DEBUG:teuthology.parallel:result is None 2024-08-17T17:22:47.104 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:46 smithi130 ceph-mon[42749]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:47.104 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:46 smithi130 ceph-mon[31342]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:46 smithi179 ceph-mon[34775]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:47.413 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411382 got 77309411385 for osd.2 2024-08-17T17:22:47.413 DEBUG:teuthology.parallel:result is None 2024-08-17T17:22:48.001 INFO:teuthology.orchestra.run.smithi130.stdout:163208757294 2024-08-17T17:22:48.662 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757291 got 163208757294 for osd.5 2024-08-17T17:22:48.662 DEBUG:teuthology.parallel:result is None 2024-08-17T17:22:48.662 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T17:22:48.663 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:22:48.915 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:49.283 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:48 smithi130 ceph-mon[31342]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:49.283 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:48 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/948974058' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T17:22:49.283 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:48 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:22:49.283 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:49 smithi130 ceph-mon[42749]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:49.284 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:49 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/948974058' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T17:22:49.284 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:49 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:22:49.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:48 smithi179 ceph-mon[34775]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:49.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:48 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/948974058' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T17:22:49.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:48 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:22:50.652 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:22:50.653 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:22:51.327 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:51 smithi130 ceph-mon[31342]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:51.327 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:51 smithi130 ceph-mon[42749]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:51.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:51 smithi179 ceph-mon[34775]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:51.355 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":85,"stamp":"2024-08-17T17:22:49.543356+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592725,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386580,"kb_used_data":7712,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574636,"statfs":{"total":767960285184,"available":765516427264,"internally_reserved":0,"allocated":7897088,"data_stored":4437915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002673"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.963712+0000","last_change":"2024-08-17T17:21:52.937978+0000","last_active":"2024-08-17T17:22:33.963712+0000","last_peered":"2024-08-17T17:22:33.963712+0000","last_clean":"2024-08-17T17:22:33.963712+0000","last_became_active":"2024-08-17T17:21:52.937666+0000","last_became_peered":"2024-08-17T17:21:52.937666+0000","last_unstale":"2024-08-17T17:22:33.963712+0000","last_undegraded":"2024-08-17T17:22:33.963712+0000","last_fullsized":"2024-08-17T17:22:33.963712+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:45.299257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357614+0000","last_change":"2024-08-17T17:21:52.150893+0000","last_active":"2024-08-17T17:22:34.357614+0000","last_peered":"2024-08-17T17:22:34.357614+0000","last_clean":"2024-08-17T17:22:34.357614+0000","last_became_active":"2024-08-17T17:21:52.150629+0000","last_became_peered":"2024-08-17T17:21:52.150629+0000","last_unstale":"2024-08-17T17:22:34.357614+0000","last_undegraded":"2024-08-17T17:22:34.357614+0000","last_fullsized":"2024-08-17T17:22:34.357614+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:07:49.243622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965289+0000","last_change":"2024-08-17T17:21:53.159733+0000","last_active":"2024-08-17T17:22:33.965289+0000","last_peered":"2024-08-17T17:22:33.965289+0000","last_clean":"2024-08-17T17:22:33.965289+0000","last_became_active":"2024-08-17T17:21:53.159457+0000","last_became_peered":"2024-08-17T17:21:53.159457+0000","last_unstale":"2024-08-17T17:22:33.965289+0000","last_undegraded":"2024-08-17T17:22:33.965289+0000","last_fullsized":"2024-08-17T17:22:33.965289+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:29.543094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965529+0000","last_change":"2024-08-17T17:21:52.156592+0000","last_active":"2024-08-17T17:22:33.965529+0000","last_peered":"2024-08-17T17:22:33.965529+0000","last_clean":"2024-08-17T17:22:33.965529+0000","last_became_active":"2024-08-17T17:21:52.156315+0000","last_became_peered":"2024-08-17T17:21:52.156315+0000","last_unstale":"2024-08-17T17:22:33.965529+0000","last_undegraded":"2024-08-17T17:22:33.965529+0000","last_fullsized":"2024-08-17T17:22:33.965529+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:17.075192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.966503+0000","last_change":"2024-08-17T17:21:52.938939+0000","last_active":"2024-08-17T17:22:33.966503+0000","last_peered":"2024-08-17T17:22:33.966503+0000","last_clean":"2024-08-17T17:22:33.966503+0000","last_became_active":"2024-08-17T17:21:52.938664+0000","last_became_peered":"2024-08-17T17:21:52.938664+0000","last_unstale":"2024-08-17T17:22:33.966503+0000","last_undegraded":"2024-08-17T17:22:33.966503+0000","last_fullsized":"2024-08-17T17:22:33.966503+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:59.476765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357531+0000","last_change":"2024-08-17T17:21:52.149957+0000","last_active":"2024-08-17T17:22:34.357531+0000","last_peered":"2024-08-17T17:22:34.357531+0000","last_clean":"2024-08-17T17:22:34.357531+0000","last_became_active":"2024-08-17T17:21:52.149673+0000","last_became_peered":"2024-08-17T17:21:52.149673+0000","last_unstale":"2024-08-17T17:22:34.357531+0000","last_undegraded":"2024-08-17T17:22:34.357531+0000","last_fullsized":"2024-08-17T17:22:34.357531+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:01:16.473279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.964485+0000","last_change":"2024-08-17T17:21:52.938008+0000","last_active":"2024-08-17T17:22:33.964485+0000","last_peered":"2024-08-17T17:22:33.964485+0000","last_clean":"2024-08-17T17:22:33.964485+0000","last_became_active":"2024-08-17T17:21:52.937626+0000","last_became_peered":"2024-08-17T17:21:52.937626+0000","last_unstale":"2024-08-17T17:22:33.964485+0000","last_undegraded":"2024-08-17T17:22:33.964485+0000","last_fullsized":"2024-08-17T17:22:33.964485+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:23:23.508933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.966568+0000","last_change":"2024-08-17T17:21:52.152506+0000","last_active":"2024-08-17T17:22:33.966568+0000","last_peered":"2024-08-17T17:22:33.966568+0000","last_clean":"2024-08-17T17:22:33.966568+0000","last_became_active":"2024-08-17T17:21:52.152240+0000","last_became_peered":"2024-08-17T17:21:52.152240+0000","last_unstale":"2024-08-17T17:22:33.966568+0000","last_undegraded":"2024-08-17T17:22:33.966568+0000","last_fullsized":"2024-08-17T17:22:33.966568+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:27:29.140467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357887+0000","last_change":"2024-08-17T17:21:52.154979+0000","last_active":"2024-08-17T17:22:34.357887+0000","last_peered":"2024-08-17T17:22:34.357887+0000","last_clean":"2024-08-17T17:22:34.357887+0000","last_became_active":"2024-08-17T17:21:52.154661+0000","last_became_peered":"2024-08-17T17:21:52.154661+0000","last_unstale":"2024-08-17T17:22:34.357887+0000","last_undegraded":"2024-08-17T17:22:34.357887+0000","last_fullsized":"2024-08-17T17:22:34.357887+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:02.042791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965040+0000","last_change":"2024-08-17T17:21:53.161767+0000","last_active":"2024-08-17T17:22:33.965040+0000","last_peered":"2024-08-17T17:22:33.965040+0000","last_clean":"2024-08-17T17:22:33.965040+0000","last_became_active":"2024-08-17T17:21:53.161514+0000","last_became_peered":"2024-08-17T17:21:53.161514+0000","last_unstale":"2024-08-17T17:22:33.965040+0000","last_undegraded":"2024-08-17T17:22:33.965040+0000","last_fullsized":"2024-08-17T17:22:33.965040+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:59.837673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357891+0000","last_change":"2024-08-17T17:21:52.150559+0000","last_active":"2024-08-17T17:22:34.357891+0000","last_peered":"2024-08-17T17:22:34.357891+0000","last_clean":"2024-08-17T17:22:34.357891+0000","last_became_active":"2024-08-17T17:21:52.150318+0000","last_became_peered":"2024-08-17T17:21:52.150318+0000","last_unstale":"2024-08-17T17:22:34.357891+0000","last_undegraded":"2024-08-17T17:22:34.357891+0000","last_fullsized":"2024-08-17T17:22:34.357891+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:36.509676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961749+0000","last_change":"2024-08-17T17:21:53.159700+0000","last_active":"2024-08-17T17:22:33.961749+0000","last_peered":"2024-08-17T17:22:33.961749+0000","last_clean":"2024-08-17T17:22:33.961749+0000","last_became_active":"2024-08-17T17:21:53.159384+0000","last_became_peered":"2024-08-17T17:21:53.159384+0000","last_unstale":"2024-08-17T17:22:33.961749+0000","last_undegraded":"2024-08-17T17:22:33.961749+0000","last_fullsized":"2024-08-17T17:22:33.961749+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:43.798412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357759+0000","last_change":"2024-08-17T17:21:52.151540+0000","last_active":"2024-08-17T17:22:34.357759+0000","last_peered":"2024-08-17T17:22:34.357759+0000","last_clean":"2024-08-17T17:22:34.357759+0000","last_became_active":"2024-08-17T17:21:52.151294+0000","last_became_peered":"2024-08-17T17:21:52.151294+0000","last_unstale":"2024-08-17T17:22:34.357759+0000","last_undegraded":"2024-08-17T17:22:34.357759+0000","last_fullsized":"2024-08-17T17:22:34.357759+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:03.533252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965988+0000","last_change":"2024-08-17T17:21:53.163473+0000","last_active":"2024-08-17T17:22:33.965988+0000","last_peered":"2024-08-17T17:22:33.965988+0000","last_clean":"2024-08-17T17:22:33.965988+0000","last_became_active":"2024-08-17T17:21:53.163215+0000","last_became_peered":"2024-08-17T17:21:53.163215+0000","last_unstale":"2024-08-17T17:22:33.965988+0000","last_undegraded":"2024-08-17T17:22:33.965988+0000","last_fullsized":"2024-08-17T17:22:33.965988+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:55:15.816408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965337+0000","last_change":"2024-08-17T17:21:52.148375+0000","last_active":"2024-08-17T17:22:33.965337+0000","last_peered":"2024-08-17T17:22:33.965337+0000","last_clean":"2024-08-17T17:22:33.965337+0000","last_became_active":"2024-08-17T17:21:52.148079+0000","last_became_peered":"2024-08-17T17:21:52.148079+0000","last_unstale":"2024-08-17T17:22:33.965337+0000","last_undegraded":"2024-08-17T17:22:33.965337+0000","last_fullsized":"2024-08-17T17:22:33.965337+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:16.638141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965695+0000","last_change":"2024-08-17T17:21:52.165264+0000","last_active":"2024-08-17T17:22:33.965695+0000","last_peered":"2024-08-17T17:22:33.965695+0000","last_clean":"2024-08-17T17:22:33.965695+0000","last_became_active":"2024-08-17T17:21:52.164991+0000","last_became_peered":"2024-08-17T17:21:52.164991+0000","last_unstale":"2024-08-17T17:22:33.965695+0000","last_undegraded":"2024-08-17T17:22:33.965695+0000","last_fullsized":"2024-08-17T17:22:33.965695+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:36:34.605579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.963566+0000","last_change":"2024-08-17T17:21:52.148708+0000","last_active":"2024-08-17T17:22:33.963566+0000","last_peered":"2024-08-17T17:22:33.963566+0000","last_clean":"2024-08-17T17:22:33.963566+0000","last_became_active":"2024-08-17T17:21:52.148403+0000","last_became_peered":"2024-08-17T17:21:52.148403+0000","last_unstale":"2024-08-17T17:22:33.963566+0000","last_undegraded":"2024-08-17T17:22:33.963566+0000","last_fullsized":"2024-08-17T17:22:33.963566+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:15:13.267832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962037+0000","last_change":"2024-08-17T17:21:53.161163+0000","last_active":"2024-08-17T17:22:33.962037+0000","last_peered":"2024-08-17T17:22:33.962037+0000","last_clean":"2024-08-17T17:22:33.962037+0000","last_became_active":"2024-08-17T17:21:53.160870+0000","last_became_peered":"2024-08-17T17:21:53.160870+0000","last_unstale":"2024-08-17T17:22:33.962037+0000","last_undegraded":"2024-08-17T17:22:33.962037+0000","last_fullsized":"2024-08-17T17:22:33.962037+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:05:16.331923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.966148+0000","last_change":"2024-08-17T17:21:52.150404+0000","last_active":"2024-08-17T17:22:33.966148+0000","last_peered":"2024-08-17T17:22:33.966148+0000","last_clean":"2024-08-17T17:22:33.966148+0000","last_became_active":"2024-08-17T17:21:52.150146+0000","last_became_peered":"2024-08-17T17:21:52.150146+0000","last_unstale":"2024-08-17T17:22:33.966148+0000","last_undegraded":"2024-08-17T17:22:33.966148+0000","last_fullsized":"2024-08-17T17:22:33.966148+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:08.479783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962296+0000","last_change":"2024-08-17T17:21:53.294813+0000","last_active":"2024-08-17T17:22:33.962296+0000","last_peered":"2024-08-17T17:22:33.962296+0000","last_clean":"2024-08-17T17:22:33.962296+0000","last_became_active":"2024-08-17T17:21:53.294591+0000","last_became_peered":"2024-08-17T17:21:53.294591+0000","last_unstale":"2024-08-17T17:22:33.962296+0000","last_undegraded":"2024-08-17T17:22:33.962296+0000","last_fullsized":"2024-08-17T17:22:33.962296+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:05:10.508717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961998+0000","last_change":"2024-08-17T17:21:52.152251+0000","last_active":"2024-08-17T17:22:33.961998+0000","last_peered":"2024-08-17T17:22:33.961998+0000","last_clean":"2024-08-17T17:22:33.961998+0000","last_became_active":"2024-08-17T17:21:52.151941+0000","last_became_peered":"2024-08-17T17:21:52.151941+0000","last_unstale":"2024-08-17T17:22:33.961998+0000","last_undegraded":"2024-08-17T17:22:33.961998+0000","last_fullsized":"2024-08-17T17:22:33.961998+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:36.846304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962158+0000","last_change":"2024-08-17T17:21:53.162744+0000","last_active":"2024-08-17T17:22:33.962158+0000","last_peered":"2024-08-17T17:22:33.962158+0000","last_clean":"2024-08-17T17:22:33.962158+0000","last_became_active":"2024-08-17T17:21:53.161996+0000","last_became_peered":"2024-08-17T17:21:53.161996+0000","last_unstale":"2024-08-17T17:22:33.962158+0000","last_undegraded":"2024-08-17T17:22:33.962158+0000","last_fullsized":"2024-08-17T17:22:33.962158+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:46.913168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.964388+0000","last_change":"2024-08-17T17:21:52.149433+0000","last_active":"2024-08-17T17:22:33.964388+0000","last_peered":"2024-08-17T17:22:33.964388+0000","last_clean":"2024-08-17T17:22:33.964388+0000","last_became_active":"2024-08-17T17:21:52.149194+0000","last_became_peered":"2024-08-17T17:21:52.149194+0000","last_unstale":"2024-08-17T17:22:33.964388+0000","last_undegraded":"2024-08-17T17:22:33.964388+0000","last_fullsized":"2024-08-17T17:22:33.964388+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:19.168680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962506+0000","last_change":"2024-08-17T17:21:53.159869+0000","last_active":"2024-08-17T17:22:33.962506+0000","last_peered":"2024-08-17T17:22:33.962506+0000","last_clean":"2024-08-17T17:22:33.962506+0000","last_became_active":"2024-08-17T17:21:53.159536+0000","last_became_peered":"2024-08-17T17:21:53.159536+0000","last_unstale":"2024-08-17T17:22:33.962506+0000","last_undegraded":"2024-08-17T17:22:33.962506+0000","last_fullsized":"2024-08-17T17:22:33.962506+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:04.307205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":280,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357616+0000","last_change":"2024-08-17T17:20:06.878678+0000","last_active":"2024-08-17T17:22:34.357616+0000","last_peered":"2024-08-17T17:22:34.357616+0000","last_clean":"2024-08-17T17:22:34.357616+0000","last_became_active":"2024-08-17T17:20:06.878356+0000","last_became_peered":"2024-08-17T17:20:06.878356+0000","last_unstale":"2024-08-17T17:22:34.357616+0000","last_undegraded":"2024-08-17T17:22:34.357616+0000","last_fullsized":"2024-08-17T17:22:34.357616+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:02:43.355026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":27,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965082+0000","last_change":"2024-08-17T17:20:43.676802+0000","last_active":"2024-08-17T17:22:33.965082+0000","last_peered":"2024-08-17T17:22:33.965082+0000","last_clean":"2024-08-17T17:22:33.965082+0000","last_became_active":"2024-08-17T17:20:38.651573+0000","last_became_peered":"2024-08-17T17:20:38.651573+0000","last_unstale":"2024-08-17T17:22:33.965082+0000","last_undegraded":"2024-08-17T17:22:33.965082+0000","last_fullsized":"2024-08-17T17:22:33.965082+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:20:15.923334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085985900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.963601+0000","last_change":"2024-08-17T17:21:52.149270+0000","last_active":"2024-08-17T17:22:33.963601+0000","last_peered":"2024-08-17T17:22:33.963601+0000","last_clean":"2024-08-17T17:22:33.963601+0000","last_became_active":"2024-08-17T17:21:52.149026+0000","last_became_peered":"2024-08-17T17:21:52.149026+0000","last_unstale":"2024-08-17T17:22:33.963601+0000","last_undegraded":"2024-08-17T17:22:33.963601+0000","last_fullsized":"2024-08-17T17:22:33.963601+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:30.037565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.966355+0000","last_change":"2024-08-17T17:21:53.164672+0000","last_active":"2024-08-17T17:22:33.966355+0000","last_peered":"2024-08-17T17:22:33.966355+0000","last_clean":"2024-08-17T17:22:33.966355+0000","last_became_active":"2024-08-17T17:21:53.164381+0000","last_became_peered":"2024-08-17T17:21:53.164381+0000","last_unstale":"2024-08-17T17:22:33.966355+0000","last_undegraded":"2024-08-17T17:22:33.966355+0000","last_fullsized":"2024-08-17T17:22:33.966355+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:43:36.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357552+0000","last_change":"2024-08-17T17:20:43.716700+0000","last_active":"2024-08-17T17:22:34.357552+0000","last_peered":"2024-08-17T17:22:34.357552+0000","last_clean":"2024-08-17T17:22:34.357552+0000","last_became_active":"2024-08-17T17:20:38.653957+0000","last_became_peered":"2024-08-17T17:20:38.653957+0000","last_unstale":"2024-08-17T17:22:34.357552+0000","last_undegraded":"2024-08-17T17:22:34.357552+0000","last_fullsized":"2024-08-17T17:22:34.357552+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:43:42.214017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965160+0000","last_change":"2024-08-17T17:21:52.158478+0000","last_active":"2024-08-17T17:22:33.965160+0000","last_peered":"2024-08-17T17:22:33.965160+0000","last_clean":"2024-08-17T17:22:33.965160+0000","last_became_active":"2024-08-17T17:21:52.158147+0000","last_became_peered":"2024-08-17T17:21:52.158147+0000","last_unstale":"2024-08-17T17:22:33.965160+0000","last_undegraded":"2024-08-17T17:22:33.965160+0000","last_fullsized":"2024-08-17T17:22:33.965160+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:50.137734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:22:35.230913+0000","last_change":"2024-08-17T17:22:35.230913+0000","last_active":"2024-08-17T17:22:35.230913+0000","last_peered":"2024-08-17T17:22:35.230913+0000","last_clean":"2024-08-17T17:22:35.230913+0000","last_became_active":"2024-08-17T17:22:35.230556+0000","last_became_peered":"2024-08-17T17:22:35.230556+0000","last_unstale":"2024-08-17T17:22:35.230913+0000","last_undegraded":"2024-08-17T17:22:35.230913+0000","last_fullsized":"2024-08-17T17:22:35.230913+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:55.819618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.963994+0000","last_change":"2024-08-17T17:20:43.672739+0000","last_active":"2024-08-17T17:22:33.963994+0000","last_peered":"2024-08-17T17:22:33.963994+0000","last_clean":"2024-08-17T17:22:33.963994+0000","last_became_active":"2024-08-17T17:20:38.783090+0000","last_became_peered":"2024-08-17T17:20:38.783090+0000","last_unstale":"2024-08-17T17:22:33.963994+0000","last_undegraded":"2024-08-17T17:22:33.963994+0000","last_fullsized":"2024-08-17T17:22:33.963994+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:23.536205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961169+0000","last_change":"2024-08-17T17:21:52.150340+0000","last_active":"2024-08-17T17:22:33.961169+0000","last_peered":"2024-08-17T17:22:33.961169+0000","last_clean":"2024-08-17T17:22:33.961169+0000","last_became_active":"2024-08-17T17:21:52.150086+0000","last_became_peered":"2024-08-17T17:21:52.150086+0000","last_unstale":"2024-08-17T17:22:33.961169+0000","last_undegraded":"2024-08-17T17:22:33.961169+0000","last_fullsized":"2024-08-17T17:22:33.961169+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:21.964979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962575+0000","last_change":"2024-08-17T17:21:53.206453+0000","last_active":"2024-08-17T17:22:33.962575+0000","last_peered":"2024-08-17T17:22:33.962575+0000","last_clean":"2024-08-17T17:22:33.962575+0000","last_became_active":"2024-08-17T17:21:53.206096+0000","last_became_peered":"2024-08-17T17:21:53.206096+0000","last_unstale":"2024-08-17T17:22:33.962575+0000","last_undegraded":"2024-08-17T17:22:33.962575+0000","last_fullsized":"2024-08-17T17:22:33.962575+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:03.531301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":28,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965699+0000","last_change":"2024-08-17T17:20:43.676769+0000","last_active":"2024-08-17T17:22:33.965699+0000","last_peered":"2024-08-17T17:22:33.965699+0000","last_clean":"2024-08-17T17:22:33.965699+0000","last_became_active":"2024-08-17T17:20:38.651624+0000","last_became_peered":"2024-08-17T17:20:38.651624+0000","last_unstale":"2024-08-17T17:22:33.965699+0000","last_undegraded":"2024-08-17T17:22:33.965699+0000","last_fullsized":"2024-08-17T17:22:33.965699+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:58:44.833201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0008386,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962257+0000","last_change":"2024-08-17T17:21:52.157986+0000","last_active":"2024-08-17T17:22:33.962257+0000","last_peered":"2024-08-17T17:22:33.962257+0000","last_clean":"2024-08-17T17:22:33.962257+0000","last_became_active":"2024-08-17T17:21:52.157699+0000","last_became_peered":"2024-08-17T17:21:52.157699+0000","last_unstale":"2024-08-17T17:22:33.962257+0000","last_undegraded":"2024-08-17T17:22:33.962257+0000","last_fullsized":"2024-08-17T17:22:33.962257+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:38.733835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961752+0000","last_change":"2024-08-17T17:21:53.295569+0000","last_active":"2024-08-17T17:22:33.961752+0000","last_peered":"2024-08-17T17:22:33.961752+0000","last_clean":"2024-08-17T17:22:33.961752+0000","last_became_active":"2024-08-17T17:21:53.295131+0000","last_became_peered":"2024-08-17T17:21:53.295131+0000","last_unstale":"2024-08-17T17:22:33.961752+0000","last_undegraded":"2024-08-17T17:22:33.961752+0000","last_fullsized":"2024-08-17T17:22:33.961752+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:58:26.289880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965539+0000","last_change":"2024-08-17T17:21:52.151870+0000","last_active":"2024-08-17T17:22:33.965539+0000","last_peered":"2024-08-17T17:22:33.965539+0000","last_clean":"2024-08-17T17:22:33.965539+0000","last_became_active":"2024-08-17T17:21:52.151441+0000","last_became_peered":"2024-08-17T17:21:52.151441+0000","last_unstale":"2024-08-17T17:22:33.965539+0000","last_undegraded":"2024-08-17T17:22:33.965539+0000","last_fullsized":"2024-08-17T17:22:33.965539+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:07.731078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961947+0000","last_change":"2024-08-17T17:20:43.677386+0000","last_active":"2024-08-17T17:22:33.961947+0000","last_peered":"2024-08-17T17:22:33.961947+0000","last_clean":"2024-08-17T17:22:33.961947+0000","last_became_active":"2024-08-17T17:20:38.653370+0000","last_became_peered":"2024-08-17T17:20:38.653370+0000","last_unstale":"2024-08-17T17:22:33.961947+0000","last_undegraded":"2024-08-17T17:22:33.961947+0000","last_fullsized":"2024-08-17T17:22:33.961947+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:40.082296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:22:35.231291+0000","last_change":"2024-08-17T17:22:35.231291+0000","last_active":"2024-08-17T17:22:35.231291+0000","last_peered":"2024-08-17T17:22:35.231291+0000","last_clean":"2024-08-17T17:22:35.231291+0000","last_became_active":"2024-08-17T17:22:35.231021+0000","last_became_peered":"2024-08-17T17:22:35.231021+0000","last_unstale":"2024-08-17T17:22:35.231291+0000","last_undegraded":"2024-08-17T17:22:35.231291+0000","last_fullsized":"2024-08-17T17:22:35.231291+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:19:18.292988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961838+0000","last_change":"2024-08-17T17:21:52.151729+0000","last_active":"2024-08-17T17:22:33.961838+0000","last_peered":"2024-08-17T17:22:33.961838+0000","last_clean":"2024-08-17T17:22:33.961838+0000","last_became_active":"2024-08-17T17:21:52.151470+0000","last_became_peered":"2024-08-17T17:21:52.151470+0000","last_unstale":"2024-08-17T17:22:33.961838+0000","last_undegraded":"2024-08-17T17:22:33.961838+0000","last_fullsized":"2024-08-17T17:22:33.961838+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:04.706991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357766+0000","last_change":"2024-08-17T17:20:43.716798+0000","last_active":"2024-08-17T17:22:34.357766+0000","last_peered":"2024-08-17T17:22:34.357766+0000","last_clean":"2024-08-17T17:22:34.357766+0000","last_became_active":"2024-08-17T17:20:38.782941+0000","last_became_peered":"2024-08-17T17:20:38.782941+0000","last_unstale":"2024-08-17T17:22:34.357766+0000","last_undegraded":"2024-08-17T17:22:34.357766+0000","last_fullsized":"2024-08-17T17:22:34.357766+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:35.726637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965417+0000","last_change":"2024-08-17T17:21:52.937973+0000","last_active":"2024-08-17T17:22:33.965417+0000","last_peered":"2024-08-17T17:22:33.965417+0000","last_clean":"2024-08-17T17:22:33.965417+0000","last_became_active":"2024-08-17T17:21:52.937636+0000","last_became_peered":"2024-08-17T17:21:52.937636+0000","last_unstale":"2024-08-17T17:22:33.965417+0000","last_undegraded":"2024-08-17T17:22:33.965417+0000","last_fullsized":"2024-08-17T17:22:33.965417+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:56:05.025801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.964453+0000","last_change":"2024-08-17T17:21:52.147358+0000","last_active":"2024-08-17T17:22:33.964453+0000","last_peered":"2024-08-17T17:22:33.964453+0000","last_clean":"2024-08-17T17:22:33.964453+0000","last_became_active":"2024-08-17T17:21:52.147112+0000","last_became_peered":"2024-08-17T17:21:52.147112+0000","last_unstale":"2024-08-17T17:22:33.964453+0000","last_undegraded":"2024-08-17T17:22:33.964453+0000","last_fullsized":"2024-08-17T17:22:33.964453+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:02:49.646927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962421+0000","last_change":"2024-08-17T17:20:43.677369+0000","last_active":"2024-08-17T17:22:33.962421+0000","last_peered":"2024-08-17T17:22:33.962421+0000","last_clean":"2024-08-17T17:22:33.962421+0000","last_became_active":"2024-08-17T17:20:38.782865+0000","last_became_peered":"2024-08-17T17:20:38.782865+0000","last_unstale":"2024-08-17T17:22:33.962421+0000","last_undegraded":"2024-08-17T17:22:33.962421+0000","last_fullsized":"2024-08-17T17:22:33.962421+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:15:56.927535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021889699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.964227+0000","last_change":"2024-08-17T17:21:52.938072+0000","last_active":"2024-08-17T17:22:33.964227+0000","last_peered":"2024-08-17T17:22:33.964227+0000","last_clean":"2024-08-17T17:22:33.964227+0000","last_became_active":"2024-08-17T17:21:52.937627+0000","last_became_peered":"2024-08-17T17:21:52.937627+0000","last_unstale":"2024-08-17T17:22:33.964227+0000","last_undegraded":"2024-08-17T17:22:33.964227+0000","last_fullsized":"2024-08-17T17:22:33.964227+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:52.163926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.966409+0000","last_change":"2024-08-17T17:21:52.157729+0000","last_active":"2024-08-17T17:22:33.966409+0000","last_peered":"2024-08-17T17:22:33.966409+0000","last_clean":"2024-08-17T17:22:33.966409+0000","last_became_active":"2024-08-17T17:21:52.157026+0000","last_became_peered":"2024-08-17T17:21:52.157026+0000","last_unstale":"2024-08-17T17:22:33.966409+0000","last_undegraded":"2024-08-17T17:22:33.966409+0000","last_fullsized":"2024-08-17T17:22:33.966409+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:24:51.873449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":25,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.966429+0000","last_change":"2024-08-17T17:20:43.677752+0000","last_active":"2024-08-17T17:22:33.966429+0000","last_peered":"2024-08-17T17:22:33.966429+0000","last_clean":"2024-08-17T17:22:33.966429+0000","last_became_active":"2024-08-17T17:20:38.652246+0000","last_became_peered":"2024-08-17T17:20:38.652246+0000","last_unstale":"2024-08-17T17:22:33.966429+0000","last_undegraded":"2024-08-17T17:22:33.966429+0000","last_fullsized":"2024-08-17T17:22:33.966429+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:52.620577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961640+0000","last_change":"2024-08-17T17:21:53.161517+0000","last_active":"2024-08-17T17:22:33.961640+0000","last_peered":"2024-08-17T17:22:33.961640+0000","last_clean":"2024-08-17T17:22:33.961640+0000","last_became_active":"2024-08-17T17:21:53.161203+0000","last_became_peered":"2024-08-17T17:21:53.161203+0000","last_unstale":"2024-08-17T17:22:33.961640+0000","last_undegraded":"2024-08-17T17:22:33.961640+0000","last_fullsized":"2024-08-17T17:22:33.961640+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:46.059237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962125+0000","last_change":"2024-08-17T17:21:53.163858+0000","last_active":"2024-08-17T17:22:33.962125+0000","last_peered":"2024-08-17T17:22:33.962125+0000","last_clean":"2024-08-17T17:22:33.962125+0000","last_became_active":"2024-08-17T17:21:53.163018+0000","last_became_peered":"2024-08-17T17:21:53.163018+0000","last_unstale":"2024-08-17T17:22:33.962125+0000","last_undegraded":"2024-08-17T17:22:33.962125+0000","last_fullsized":"2024-08-17T17:22:33.962125+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:04:36.044906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"71'6","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962119+0000","last_change":"2024-08-17T17:21:52.156855+0000","last_active":"2024-08-17T17:22:33.962119+0000","last_peered":"2024-08-17T17:22:33.962119+0000","last_clean":"2024-08-17T17:22:33.962119+0000","last_became_active":"2024-08-17T17:21:52.156377+0000","last_became_peered":"2024-08-17T17:21:52.156377+0000","last_unstale":"2024-08-17T17:22:33.962119+0000","last_undegraded":"2024-08-17T17:22:33.962119+0000","last_fullsized":"2024-08-17T17:22:33.962119+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:29.248851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961333+0000","last_change":"2024-08-17T17:21:53.295126+0000","last_active":"2024-08-17T17:22:33.961333+0000","last_peered":"2024-08-17T17:22:33.961333+0000","last_clean":"2024-08-17T17:22:33.961333+0000","last_became_active":"2024-08-17T17:21:53.294792+0000","last_became_peered":"2024-08-17T17:21:53.294792+0000","last_unstale":"2024-08-17T17:22:33.961333+0000","last_undegraded":"2024-08-17T17:22:33.961333+0000","last_fullsized":"2024-08-17T17:22:33.961333+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:09.378025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357639+0000","last_change":"2024-08-17T17:21:52.149310+0000","last_active":"2024-08-17T17:22:34.357639+0000","last_peered":"2024-08-17T17:22:34.357639+0000","last_clean":"2024-08-17T17:22:34.357639+0000","last_became_active":"2024-08-17T17:21:52.148996+0000","last_became_peered":"2024-08-17T17:21:52.148996+0000","last_unstale":"2024-08-17T17:22:34.357639+0000","last_undegraded":"2024-08-17T17:22:34.357639+0000","last_fullsized":"2024-08-17T17:22:34.357639+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:26.830642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965563+0000","last_change":"2024-08-17T17:21:52.152923+0000","last_active":"2024-08-17T17:22:33.965563+0000","last_peered":"2024-08-17T17:22:33.965563+0000","last_clean":"2024-08-17T17:22:33.965563+0000","last_became_active":"2024-08-17T17:21:52.152588+0000","last_became_peered":"2024-08-17T17:21:52.152588+0000","last_unstale":"2024-08-17T17:22:33.965563+0000","last_undegraded":"2024-08-17T17:22:33.965563+0000","last_fullsized":"2024-08-17T17:22:33.965563+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:11:27.075241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.963432+0000","last_change":"2024-08-17T17:21:53.161868+0000","last_active":"2024-08-17T17:22:33.963432+0000","last_peered":"2024-08-17T17:22:33.963432+0000","last_clean":"2024-08-17T17:22:33.963432+0000","last_became_active":"2024-08-17T17:21:53.161100+0000","last_became_peered":"2024-08-17T17:21:53.161100+0000","last_unstale":"2024-08-17T17:22:33.963432+0000","last_undegraded":"2024-08-17T17:22:33.963432+0000","last_fullsized":"2024-08-17T17:22:33.963432+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:11:07.175330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965689+0000","last_change":"2024-08-17T17:21:52.151043+0000","last_active":"2024-08-17T17:22:33.965689+0000","last_peered":"2024-08-17T17:22:33.965689+0000","last_clean":"2024-08-17T17:22:33.965689+0000","last_became_active":"2024-08-17T17:21:52.150614+0000","last_became_peered":"2024-08-17T17:21:52.150614+0000","last_unstale":"2024-08-17T17:22:33.965689+0000","last_undegraded":"2024-08-17T17:22:33.965689+0000","last_fullsized":"2024-08-17T17:22:33.965689+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:01.681256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.965386+0000","last_change":"2024-08-17T17:22:34.965386+0000","last_active":"2024-08-17T17:22:34.965386+0000","last_peered":"2024-08-17T17:22:34.965386+0000","last_clean":"2024-08-17T17:22:34.965386+0000","last_became_active":"2024-08-17T17:22:34.964930+0000","last_became_peered":"2024-08-17T17:22:34.964930+0000","last_unstale":"2024-08-17T17:22:34.965386+0000","last_undegraded":"2024-08-17T17:22:34.965386+0000","last_fullsized":"2024-08-17T17:22:34.965386+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:24:23.306640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.966404+0000","last_change":"2024-08-17T17:21:53.162596+0000","last_active":"2024-08-17T17:22:33.966404+0000","last_peered":"2024-08-17T17:22:33.966404+0000","last_clean":"2024-08-17T17:22:33.966404+0000","last_became_active":"2024-08-17T17:21:53.162339+0000","last_became_peered":"2024-08-17T17:21:53.162339+0000","last_unstale":"2024-08-17T17:22:33.966404+0000","last_undegraded":"2024-08-17T17:22:33.966404+0000","last_fullsized":"2024-08-17T17:22:33.966404+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:21:00.795345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.964316+0000","last_change":"2024-08-17T17:21:52.149351+0000","last_active":"2024-08-17T17:22:33.964316+0000","last_peered":"2024-08-17T17:22:33.964316+0000","last_clean":"2024-08-17T17:22:33.964316+0000","last_became_active":"2024-08-17T17:21:52.149100+0000","last_became_peered":"2024-08-17T17:21:52.149100+0000","last_unstale":"2024-08-17T17:22:33.964316+0000","last_undegraded":"2024-08-17T17:22:33.964316+0000","last_fullsized":"2024-08-17T17:22:33.964316+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:26:05.195805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965619+0000","last_change":"2024-08-17T17:21:52.938947+0000","last_active":"2024-08-17T17:22:33.965619+0000","last_peered":"2024-08-17T17:22:33.965619+0000","last_clean":"2024-08-17T17:22:33.965619+0000","last_became_active":"2024-08-17T17:21:52.938697+0000","last_became_peered":"2024-08-17T17:21:52.938697+0000","last_unstale":"2024-08-17T17:22:33.965619+0000","last_undegraded":"2024-08-17T17:22:33.965619+0000","last_fullsized":"2024-08-17T17:22:33.965619+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:40:05.765657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961080+0000","last_change":"2024-08-17T17:21:52.154388+0000","last_active":"2024-08-17T17:22:33.961080+0000","last_peered":"2024-08-17T17:22:33.961080+0000","last_clean":"2024-08-17T17:22:33.961080+0000","last_became_active":"2024-08-17T17:21:52.153910+0000","last_became_peered":"2024-08-17T17:21:52.153910+0000","last_unstale":"2024-08-17T17:22:33.961080+0000","last_undegraded":"2024-08-17T17:22:33.961080+0000","last_fullsized":"2024-08-17T17:22:33.961080+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:24:24.483938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.963990+0000","last_change":"2024-08-17T17:21:52.166598+0000","last_active":"2024-08-17T17:22:33.963990+0000","last_peered":"2024-08-17T17:22:33.963990+0000","last_clean":"2024-08-17T17:22:33.963990+0000","last_became_active":"2024-08-17T17:21:52.166339+0000","last_became_peered":"2024-08-17T17:21:52.166339+0000","last_unstale":"2024-08-17T17:22:33.963990+0000","last_undegraded":"2024-08-17T17:22:33.963990+0000","last_fullsized":"2024-08-17T17:22:33.963990+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:45.264989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961976+0000","last_change":"2024-08-17T17:21:52.157102+0000","last_active":"2024-08-17T17:22:33.961976+0000","last_peered":"2024-08-17T17:22:33.961976+0000","last_clean":"2024-08-17T17:22:33.961976+0000","last_became_active":"2024-08-17T17:21:52.156813+0000","last_became_peered":"2024-08-17T17:21:52.156813+0000","last_unstale":"2024-08-17T17:22:33.961976+0000","last_undegraded":"2024-08-17T17:22:33.961976+0000","last_fullsized":"2024-08-17T17:22:33.961976+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:44:32.159655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.964656+0000","last_change":"2024-08-17T17:21:52.165947+0000","last_active":"2024-08-17T17:22:33.964656+0000","last_peered":"2024-08-17T17:22:33.964656+0000","last_clean":"2024-08-17T17:22:33.964656+0000","last_became_active":"2024-08-17T17:21:52.165655+0000","last_became_peered":"2024-08-17T17:21:52.165655+0000","last_unstale":"2024-08-17T17:22:33.964656+0000","last_undegraded":"2024-08-17T17:22:33.964656+0000","last_fullsized":"2024-08-17T17:22:33.964656+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:43:22.658721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965570+0000","last_change":"2024-08-17T17:21:52.157236+0000","last_active":"2024-08-17T17:22:33.965570+0000","last_peered":"2024-08-17T17:22:33.965570+0000","last_clean":"2024-08-17T17:22:33.965570+0000","last_became_active":"2024-08-17T17:21:52.156966+0000","last_became_peered":"2024-08-17T17:21:52.156966+0000","last_unstale":"2024-08-17T17:22:33.965570+0000","last_undegraded":"2024-08-17T17:22:33.965570+0000","last_fullsized":"2024-08-17T17:22:33.965570+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:10.648147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961913+0000","last_change":"2024-08-17T17:21:53.162727+0000","last_active":"2024-08-17T17:22:33.961913+0000","last_peered":"2024-08-17T17:22:33.961913+0000","last_clean":"2024-08-17T17:22:33.961913+0000","last_became_active":"2024-08-17T17:21:53.162503+0000","last_became_peered":"2024-08-17T17:21:53.162503+0000","last_unstale":"2024-08-17T17:22:33.961913+0000","last_undegraded":"2024-08-17T17:22:33.961913+0000","last_fullsized":"2024-08-17T17:22:33.961913+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:27:17.073240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965338+0000","last_change":"2024-08-17T17:21:52.159030+0000","last_active":"2024-08-17T17:22:33.965338+0000","last_peered":"2024-08-17T17:22:33.965338+0000","last_clean":"2024-08-17T17:22:33.965338+0000","last_became_active":"2024-08-17T17:21:52.158599+0000","last_became_peered":"2024-08-17T17:21:52.158599+0000","last_unstale":"2024-08-17T17:22:33.965338+0000","last_undegraded":"2024-08-17T17:22:33.965338+0000","last_fullsized":"2024-08-17T17:22:33.965338+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:22.522063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961731+0000","last_change":"2024-08-17T17:21:53.206411+0000","last_active":"2024-08-17T17:22:33.961731+0000","last_peered":"2024-08-17T17:22:33.961731+0000","last_clean":"2024-08-17T17:22:33.961731+0000","last_became_active":"2024-08-17T17:21:53.206139+0000","last_became_peered":"2024-08-17T17:21:53.206139+0000","last_unstale":"2024-08-17T17:22:33.961731+0000","last_undegraded":"2024-08-17T17:22:33.961731+0000","last_fullsized":"2024-08-17T17:22:33.961731+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:37:56.943926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965390+0000","last_change":"2024-08-17T17:21:52.157731+0000","last_active":"2024-08-17T17:22:33.965390+0000","last_peered":"2024-08-17T17:22:33.965390+0000","last_clean":"2024-08-17T17:22:33.965390+0000","last_became_active":"2024-08-17T17:21:52.157453+0000","last_became_peered":"2024-08-17T17:21:52.157453+0000","last_unstale":"2024-08-17T17:22:33.965390+0000","last_undegraded":"2024-08-17T17:22:33.965390+0000","last_fullsized":"2024-08-17T17:22:33.965390+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:30:09.106088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.963517+0000","last_change":"2024-08-17T17:21:52.166049+0000","last_active":"2024-08-17T17:22:33.963517+0000","last_peered":"2024-08-17T17:22:33.963517+0000","last_clean":"2024-08-17T17:22:33.963517+0000","last_became_active":"2024-08-17T17:21:52.165300+0000","last_became_peered":"2024-08-17T17:21:52.165300+0000","last_unstale":"2024-08-17T17:22:33.963517+0000","last_undegraded":"2024-08-17T17:22:33.963517+0000","last_fullsized":"2024-08-17T17:22:33.963517+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:22:34.677517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.964335+0000","last_change":"2024-08-17T17:21:52.148393+0000","last_active":"2024-08-17T17:22:33.964335+0000","last_peered":"2024-08-17T17:22:33.964335+0000","last_clean":"2024-08-17T17:22:33.964335+0000","last_became_active":"2024-08-17T17:21:52.148098+0000","last_became_peered":"2024-08-17T17:21:52.148098+0000","last_unstale":"2024-08-17T17:22:33.964335+0000","last_undegraded":"2024-08-17T17:22:33.964335+0000","last_fullsized":"2024-08-17T17:22:33.964335+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:52:44.434667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961651+0000","last_change":"2024-08-17T17:21:53.160925+0000","last_active":"2024-08-17T17:22:33.961651+0000","last_peered":"2024-08-17T17:22:33.961651+0000","last_clean":"2024-08-17T17:22:33.961651+0000","last_became_active":"2024-08-17T17:21:53.160641+0000","last_became_peered":"2024-08-17T17:21:53.160641+0000","last_unstale":"2024-08-17T17:22:33.961651+0000","last_undegraded":"2024-08-17T17:22:33.961651+0000","last_fullsized":"2024-08-17T17:22:33.961651+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:26.697016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965632+0000","last_change":"2024-08-17T17:21:52.159652+0000","last_active":"2024-08-17T17:22:33.965632+0000","last_peered":"2024-08-17T17:22:33.965632+0000","last_clean":"2024-08-17T17:22:33.965632+0000","last_became_active":"2024-08-17T17:21:52.159348+0000","last_became_peered":"2024-08-17T17:21:52.159348+0000","last_unstale":"2024-08-17T17:22:33.965632+0000","last_undegraded":"2024-08-17T17:22:33.965632+0000","last_fullsized":"2024-08-17T17:22:33.965632+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:42:47.650336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234022,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298700,"kb_used_data":1340,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446452,"statfs":{"total":95995035648,"available":95689166848,"internally_reserved":0,"allocated":1372160,"data_stored":924587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.49099999999999999,"15min":0.48499999999999999},"min":{"1min":0.378,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.503,"15min":0.499},"min":{"1min":0.40799999999999997,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.65900000000000003,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.54100000000000004}]},{"osd":1,"last update":"Sat Aug 17 17:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.48999999999999999,"15min":0.48299999999999998},"min":{"1min":0.39100000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.70599999999999996,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.51300000000000001,"15min":0.51400000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.51400000000000001}]},{"osd":2,"last update":"Sat Aug 17 17:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.496,"15min":0.49099999999999999},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.49199999999999999,"15min":0.48799999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.66700000000000004,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.441}]},{"osd":3,"last update":"Sat Aug 17 17:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.46700000000000003,"15min":0.45500000000000002},"min":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.48899999999999999,"15min":0.48099999999999998},"min":{"1min":0.38600000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.501}]},{"osd":4,"last update":"Sat Aug 17 17:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.40899999999999997,"15min":0.40500000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.40999999999999998,"15min":0.40400000000000003},"min":{"1min":0.31900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.42099999999999999}]},{"osd":5,"last update":"Sat Aug 17 17:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.41699999999999998,"15min":0.41099999999999998},"min":{"1min":0.32400000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.498,"5min":0.45500000000000002,"15min":0.44800000000000001},"min":{"1min":0.35999999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.746,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.44700000000000001}]},{"osd":6,"last update":"Sat Aug 17 17:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.48099999999999998,"15min":0.47699999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.46500000000000002,"15min":0.46100000000000002},"min":{"1min":0.34899999999999998,"5min":0.316,"15min":0.316},"max":{"1min":0.76800000000000002,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.47299999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495657,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298684,"kb_used_data":1328,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446468,"statfs":{"total":95995035648,"available":95689183232,"internally_reserved":0,"allocated":1359872,"data_stored":924145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.50800000000000001,"15min":0.505},"min":{"1min":0.39000000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.41499999999999998,"5min":0.373,"15min":0.373},"max":{"1min":0.70099999999999996,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.46200000000000002}]},{"osd":1,"last update":"Sat Aug 17 17:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.35799999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.77500000000000002,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53600000000000003,"15min":0.53500000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.44900000000000001}]},{"osd":2,"last update":"Sat Aug 17 17:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53100000000000003,"15min":0.53000000000000003},"min":{"1min":0.42599999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.50700000000000001,"15min":0.499},"min":{"1min":0.32800000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.41899999999999998}]},{"osd":3,"last update":"Sat Aug 17 17:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.51700000000000002,"15min":0.504},"min":{"1min":0.38600000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.432},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.54800000000000004,"15min":0.54100000000000004},"min":{"1min":0.40699999999999997,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.48099999999999998}]},{"osd":4,"last update":"Sat Aug 17 17:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.42999999999999999,"15min":0.40600000000000003},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.439,"15min":0.41899999999999998},"min":{"1min":0.32600000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Aug 17 17:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.44500000000000001,"15min":0.432},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69199999999999995,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.497},{"interface":"front","average":{"1min":0.502,"5min":0.45900000000000002,"15min":0.433},"min":{"1min":0.32900000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.752,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.371}]},{"osd":7,"last update":"Sat Aug 17 17:22:07 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.50900000000000001,"15min":0.505},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.74399999999999999,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.51100000000000001,"15min":0.504},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.753,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.54700000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757295,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298104,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447048,"statfs":{"total":95995035648,"available":95689777152,"internally_reserved":0,"allocated":761856,"data_stored":333118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:22:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.51600000000000001,"15min":0.51700000000000002},"min":{"1min":0.38300000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.69799999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.50800000000000001,"15min":0.51600000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.68500000000000005,"5min":0.75,"15min":0.75},"last":0.62}]},{"osd":1,"last update":"Sat Aug 17 17:22:22 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.505,"15min":0.51100000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.58699999999999997,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.503},{"interface":"front","average":{"1min":0.504,"5min":0.51100000000000001,"15min":0.52000000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.63600000000000001,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.54400000000000004}]},{"osd":2,"last update":"Sat Aug 17 17:22:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.50800000000000001,"15min":0.50600000000000001},"min":{"1min":0.35199999999999998,"5min":0.311,"15min":0.311},"max":{"1min":0.68999999999999995,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.51700000000000002,"15min":0.52200000000000002},"min":{"1min":0.35999999999999999,"5min":0.316,"15min":0.316},"max":{"1min":0.65700000000000003,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.60899999999999999}]},{"osd":3,"last update":"Sat Aug 17 17:22:22 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.52200000000000002,"15min":0.52600000000000002},"min":{"1min":0.38200000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.67200000000000004,"5min":0.876,"15min":0.876},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52500000000000002,"15min":0.53300000000000003},"min":{"1min":0.34000000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.66700000000000004,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.57899999999999996}]},{"osd":4,"last update":"Sat Aug 17 17:22:22 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.42499999999999999,"15min":0.41799999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.68300000000000005,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.437},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.44800000000000001,"15min":0.441},"min":{"1min":0.34399999999999997,"5min":0.251,"15min":0.251},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Aug 17 17:22:42 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.45200000000000001,"15min":0.436},"min":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.67900000000000005,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.47599999999999998,"15min":0.46200000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Aug 17 17:22:07 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.46400000000000002,"15min":0.45400000000000001},"min":{"1min":0.312,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.48499999999999999,"15min":0.47499999999999998},"min":{"1min":0.36499999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.71499999999999997,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.69199999999999995}]}]},{"osd":4,"up_from":30,"seq":128849018930,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298116,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447036,"statfs":{"total":95995035648,"available":95689764864,"internally_reserved":0,"allocated":774144,"data_stored":334157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:21:56 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.51500000000000001,"15min":0.52100000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.51500000000000001,"15min":0.53300000000000003},"min":{"1min":0.35899999999999999,"5min":0.313,"15min":0.313},"max":{"1min":0.60699999999999998,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.59699999999999998}]},{"osd":1,"last update":"Sat Aug 17 17:21:56 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.50600000000000001,"15min":0.50900000000000001},"min":{"1min":0.34000000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.5,"5min":0.50800000000000001,"15min":0.51700000000000002},"min":{"1min":0.34999999999999998,"5min":0.316,"15min":0.316},"max":{"1min":0.65400000000000003,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.371}]},{"osd":2,"last update":"Sat Aug 17 17:21:56 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.51300000000000001,"15min":0.52400000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.72899999999999998,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.52300000000000002},"min":{"1min":0.40100000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.624,"5min":0.748,"15min":0.748},"last":0.65700000000000003}]},{"osd":3,"last update":"Sat Aug 17 17:21:56 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51900000000000002,"15min":0.52700000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.72199999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.51200000000000001,"15min":0.51500000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.74299999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.39600000000000002}]},{"osd":5,"last update":"Sat Aug 17 17:22:22 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.41599999999999998,"15min":0.39400000000000002},"min":{"1min":0.33600000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.44,"15min":0.41699999999999998},"min":{"1min":0.26300000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.75900000000000001,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.441}]},{"osd":6,"last update":"Sat Aug 17 17:22:48 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.41499999999999998,"15min":0.39500000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.72999999999999998,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.439,"15min":0.42699999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.63,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.432}]},{"osd":7,"last update":"Sat Aug 17 17:22:06 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.45900000000000002,"15min":0.45700000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.47799999999999998,"15min":0.47199999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.77000000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182455,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298100,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447052,"statfs":{"total":95995035648,"available":95689781248,"internally_reserved":0,"allocated":757760,"data_stored":332814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.309,"5min":0.34499999999999997,"15min":0.35999999999999999},"min":{"1min":0.20799999999999999,"5min":0.183,"15min":0.183},"max":{"1min":0.46000000000000002,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.34100000000000003,"5min":0.36799999999999999,"15min":0.37},"min":{"1min":0.22700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.46800000000000003,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.45600000000000002}]},{"osd":1,"last update":"Sat Aug 17 17:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.38300000000000001,"15min":0.39900000000000002},"min":{"1min":0.24099999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.48899999999999999,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.32400000000000001,"5min":0.373,"15min":0.39400000000000002},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.439,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.47499999999999998}]},{"osd":2,"last update":"Sat Aug 17 17:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.40300000000000002,"15min":0.42099999999999999},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.47899999999999998,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.439},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.377,"15min":0.38200000000000001},"min":{"1min":0.27500000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.55000000000000004,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.497}]},{"osd":4,"last update":"Sat Aug 17 17:21:59 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.502,"5min":0.52900000000000003,"15min":0.53500000000000003},"min":{"1min":0.38700000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.65700000000000003,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.65000000000000002}]},{"osd":5,"last update":"Sat Aug 17 17:22:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.54600000000000004,"15min":0.55000000000000004},"min":{"1min":0.42999999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73899999999999999,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55300000000000005,"15min":0.56499999999999995},"min":{"1min":0.41799999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.77000000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.70099999999999996}]},{"osd":6,"last update":"Sat Aug 17 17:22:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.55300000000000005,"15min":0.55800000000000005},"min":{"1min":0.42999999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.746,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.62},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.54700000000000004,"15min":0.54900000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.70499999999999996,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.67400000000000004}]},{"osd":7,"last update":"Sat Aug 17 17:22:07 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.78300000000000003,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55600000000000005,"15min":0.55800000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.73599999999999999,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.68799999999999994}]}]},{"osd":2,"up_from":18,"seq":77309411387,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298096,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447056,"statfs":{"total":95995035648,"available":95689785344,"internally_reserved":0,"allocated":757760,"data_stored":332667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:22:15 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.37,"15min":0.36699999999999999},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.47999999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.39900000000000002,"15min":0.40200000000000002},"min":{"1min":0.26200000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.49399999999999999,"5min":0.62,"15min":0.62},"last":0.56100000000000005}]},{"osd":1,"last update":"Sat Aug 17 17:22:15 2024","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.36199999999999999,"15min":0.35799999999999998},"min":{"1min":0.22,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.49099999999999999,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.38100000000000001,"15min":0.377},"min":{"1min":0.248,"5min":0.222,"15min":0.222},"max":{"1min":0.47399999999999998,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.497}]},{"osd":3,"last update":"Sat Aug 17 17:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.433,"15min":0.45000000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.40000000000000002,"15min":0.42999999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.53200000000000003}]},{"osd":4,"last update":"Sat Aug 17 17:21:56 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.55000000000000004,"15min":0.55500000000000005},"min":{"1min":0.42299999999999999,"5min":0.379,"15min":0.379},"max":{"1min":0.77400000000000002,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56699999999999995,"15min":0.57099999999999995},"min":{"1min":0.40200000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.74299999999999999,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.79700000000000004}]},{"osd":5,"last update":"Sat Aug 17 17:22:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77800000000000002,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.64600000000000002}]},{"osd":6,"last update":"Sat Aug 17 17:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.56299999999999994,"15min":0.55300000000000005},"min":{"1min":0.39300000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.1779999999999999,"5min":1.1779999999999999,"15min":1.1779999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.76300000000000001}]},{"osd":7,"last update":"Sat Aug 17 17:22:08 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.56899999999999995,"15min":0.56399999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.79200000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.60299999999999998,"15min":0.60099999999999998},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.74299999999999999,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.78200000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298096,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447056,"statfs":{"total":95995035648,"available":95689785344,"internally_reserved":0,"allocated":757760,"data_stored":332743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:21:50 2024","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.34399999999999997,"15min":0.33500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.371,"15min":0.35599999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.50800000000000001,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.56599999999999995}]},{"osd":2,"last update":"Sat Aug 17 17:22:17 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.36799999999999999,"15min":0.35999999999999999},"min":{"1min":0.251,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.41499999999999998,"15min":0.40500000000000003},"min":{"1min":0.28399999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.72399999999999998,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.51600000000000001}]},{"osd":3,"last update":"Sat Aug 17 17:22:41 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.43099999999999999,"15min":0.434},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.39200000000000002,"15min":0.40600000000000003},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.50600000000000001,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.53300000000000003}]},{"osd":4,"last update":"Sat Aug 17 17:21:57 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.57599999999999996,"15min":0.57199999999999995},"min":{"1min":0.39800000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.72799999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55300000000000005,"15min":0.54700000000000004},"min":{"1min":0.41399999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.66600000000000004,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.66500000000000004}]},{"osd":5,"last update":"Sat Aug 17 17:22:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54900000000000004},"min":{"1min":0.40899999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81399999999999995,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.59499999999999997,"15min":0.59899999999999998},"min":{"1min":0.42599999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.67100000000000004,"5min":1.0429999999999999,"15min":1.0429999999999999},"last":0.70499999999999996}]},{"osd":6,"last update":"Sat Aug 17 17:22:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.58999999999999997,"15min":0.61099999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.69499999999999995,"5min":0.879,"15min":0.879},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.59499999999999997,"15min":0.61599999999999999},"min":{"1min":0.41299999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.78700000000000003,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.75600000000000001}]},{"osd":7,"last update":"Sat Aug 17 17:22:07 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.60199999999999998,"15min":0.60499999999999998},"min":{"1min":0.434,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.745,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.58799999999999997,"15min":0.59699999999999998},"min":{"1min":0.39400000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.72699999999999998,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.76600000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298684,"kb_used_data":1324,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446468,"statfs":{"total":95995035648,"available":95689183232,"internally_reserved":0,"allocated":1355776,"data_stored":923684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 17:21:50 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.34699999999999998,"15min":0.32000000000000001},"min":{"1min":0.245,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.496,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.34899999999999998,"5min":0.35899999999999999,"15min":0.32200000000000001},"min":{"1min":0.253,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.45900000000000002,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.23300000000000001}]},{"osd":2,"last update":"Sat Aug 17 17:22:15 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.39700000000000002,"15min":0.38700000000000001},"min":{"1min":0.247,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.247},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.38900000000000001,"15min":0.375},"min":{"1min":0.27000000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.378}]},{"osd":3,"last update":"Sat Aug 17 17:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.39200000000000002,"15min":0.41999999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.433,"15min":0.45200000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Aug 17 17:22:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.54400000000000004,"15min":0.55200000000000005},"min":{"1min":0.45700000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.72299999999999998,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55300000000000005,"15min":0.55900000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.52100000000000002}]},{"osd":5,"last update":"Sat Aug 17 17:22:21 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57099999999999995,"15min":0.57499999999999996},"min":{"1min":0.45400000000000001,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.40500000000000003,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.55900000000000005}]},{"osd":6,"last update":"Sat Aug 17 17:22:45 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.40600000000000003,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58199999999999996,"15min":0.57999999999999996},"min":{"1min":0.41599999999999998,"5min":0.374,"15min":0.374},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.54200000000000004}]},{"osd":7,"last update":"Sat Aug 17 17:22:06 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.52600000000000002,"15min":0.51800000000000002},"min":{"1min":0.43099999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.58599999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:22:51.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:22:51.619 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:52.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:52 smithi179 ceph-mon[34775]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:22:52.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:52 smithi130 ceph-mon[31342]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:22:52.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:52 smithi130 ceph-mon[42749]: from='client.14769 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:22:53.325 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:53 smithi130 ceph-mon[31342]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:53.325 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:53 smithi130 ceph-mon[42749]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:53 smithi179 ceph-mon[34775]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:53.466 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:22:53.467 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:22:54.044 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":86,"stamp":"2024-08-17T17:22:51.543876+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592725,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2431,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386580,"kb_used_data":7712,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574636,"statfs":{"total":767960285184,"available":765516427264,"internally_reserved":0,"allocated":7897088,"data_stored":4437915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002818"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.963712+0000","last_change":"2024-08-17T17:21:52.937978+0000","last_active":"2024-08-17T17:22:33.963712+0000","last_peered":"2024-08-17T17:22:33.963712+0000","last_clean":"2024-08-17T17:22:33.963712+0000","last_became_active":"2024-08-17T17:21:52.937666+0000","last_became_peered":"2024-08-17T17:21:52.937666+0000","last_unstale":"2024-08-17T17:22:33.963712+0000","last_undegraded":"2024-08-17T17:22:33.963712+0000","last_fullsized":"2024-08-17T17:22:33.963712+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:45.299257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357614+0000","last_change":"2024-08-17T17:21:52.150893+0000","last_active":"2024-08-17T17:22:34.357614+0000","last_peered":"2024-08-17T17:22:34.357614+0000","last_clean":"2024-08-17T17:22:34.357614+0000","last_became_active":"2024-08-17T17:21:52.150629+0000","last_became_peered":"2024-08-17T17:21:52.150629+0000","last_unstale":"2024-08-17T17:22:34.357614+0000","last_undegraded":"2024-08-17T17:22:34.357614+0000","last_fullsized":"2024-08-17T17:22:34.357614+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:07:49.243622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965289+0000","last_change":"2024-08-17T17:21:53.159733+0000","last_active":"2024-08-17T17:22:33.965289+0000","last_peered":"2024-08-17T17:22:33.965289+0000","last_clean":"2024-08-17T17:22:33.965289+0000","last_became_active":"2024-08-17T17:21:53.159457+0000","last_became_peered":"2024-08-17T17:21:53.159457+0000","last_unstale":"2024-08-17T17:22:33.965289+0000","last_undegraded":"2024-08-17T17:22:33.965289+0000","last_fullsized":"2024-08-17T17:22:33.965289+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:29.543094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965529+0000","last_change":"2024-08-17T17:21:52.156592+0000","last_active":"2024-08-17T17:22:33.965529+0000","last_peered":"2024-08-17T17:22:33.965529+0000","last_clean":"2024-08-17T17:22:33.965529+0000","last_became_active":"2024-08-17T17:21:52.156315+0000","last_became_peered":"2024-08-17T17:21:52.156315+0000","last_unstale":"2024-08-17T17:22:33.965529+0000","last_undegraded":"2024-08-17T17:22:33.965529+0000","last_fullsized":"2024-08-17T17:22:33.965529+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:17.075192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.966503+0000","last_change":"2024-08-17T17:21:52.938939+0000","last_active":"2024-08-17T17:22:33.966503+0000","last_peered":"2024-08-17T17:22:33.966503+0000","last_clean":"2024-08-17T17:22:33.966503+0000","last_became_active":"2024-08-17T17:21:52.938664+0000","last_became_peered":"2024-08-17T17:21:52.938664+0000","last_unstale":"2024-08-17T17:22:33.966503+0000","last_undegraded":"2024-08-17T17:22:33.966503+0000","last_fullsized":"2024-08-17T17:22:33.966503+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:59.476765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357531+0000","last_change":"2024-08-17T17:21:52.149957+0000","last_active":"2024-08-17T17:22:34.357531+0000","last_peered":"2024-08-17T17:22:34.357531+0000","last_clean":"2024-08-17T17:22:34.357531+0000","last_became_active":"2024-08-17T17:21:52.149673+0000","last_became_peered":"2024-08-17T17:21:52.149673+0000","last_unstale":"2024-08-17T17:22:34.357531+0000","last_undegraded":"2024-08-17T17:22:34.357531+0000","last_fullsized":"2024-08-17T17:22:34.357531+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:01:16.473279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.964485+0000","last_change":"2024-08-17T17:21:52.938008+0000","last_active":"2024-08-17T17:22:33.964485+0000","last_peered":"2024-08-17T17:22:33.964485+0000","last_clean":"2024-08-17T17:22:33.964485+0000","last_became_active":"2024-08-17T17:21:52.937626+0000","last_became_peered":"2024-08-17T17:21:52.937626+0000","last_unstale":"2024-08-17T17:22:33.964485+0000","last_undegraded":"2024-08-17T17:22:33.964485+0000","last_fullsized":"2024-08-17T17:22:33.964485+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:23:23.508933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.966568+0000","last_change":"2024-08-17T17:21:52.152506+0000","last_active":"2024-08-17T17:22:33.966568+0000","last_peered":"2024-08-17T17:22:33.966568+0000","last_clean":"2024-08-17T17:22:33.966568+0000","last_became_active":"2024-08-17T17:21:52.152240+0000","last_became_peered":"2024-08-17T17:21:52.152240+0000","last_unstale":"2024-08-17T17:22:33.966568+0000","last_undegraded":"2024-08-17T17:22:33.966568+0000","last_fullsized":"2024-08-17T17:22:33.966568+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:27:29.140467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357887+0000","last_change":"2024-08-17T17:21:52.154979+0000","last_active":"2024-08-17T17:22:34.357887+0000","last_peered":"2024-08-17T17:22:34.357887+0000","last_clean":"2024-08-17T17:22:34.357887+0000","last_became_active":"2024-08-17T17:21:52.154661+0000","last_became_peered":"2024-08-17T17:21:52.154661+0000","last_unstale":"2024-08-17T17:22:34.357887+0000","last_undegraded":"2024-08-17T17:22:34.357887+0000","last_fullsized":"2024-08-17T17:22:34.357887+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:02.042791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965040+0000","last_change":"2024-08-17T17:21:53.161767+0000","last_active":"2024-08-17T17:22:33.965040+0000","last_peered":"2024-08-17T17:22:33.965040+0000","last_clean":"2024-08-17T17:22:33.965040+0000","last_became_active":"2024-08-17T17:21:53.161514+0000","last_became_peered":"2024-08-17T17:21:53.161514+0000","last_unstale":"2024-08-17T17:22:33.965040+0000","last_undegraded":"2024-08-17T17:22:33.965040+0000","last_fullsized":"2024-08-17T17:22:33.965040+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:59.837673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357891+0000","last_change":"2024-08-17T17:21:52.150559+0000","last_active":"2024-08-17T17:22:34.357891+0000","last_peered":"2024-08-17T17:22:34.357891+0000","last_clean":"2024-08-17T17:22:34.357891+0000","last_became_active":"2024-08-17T17:21:52.150318+0000","last_became_peered":"2024-08-17T17:21:52.150318+0000","last_unstale":"2024-08-17T17:22:34.357891+0000","last_undegraded":"2024-08-17T17:22:34.357891+0000","last_fullsized":"2024-08-17T17:22:34.357891+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:36.509676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961749+0000","last_change":"2024-08-17T17:21:53.159700+0000","last_active":"2024-08-17T17:22:33.961749+0000","last_peered":"2024-08-17T17:22:33.961749+0000","last_clean":"2024-08-17T17:22:33.961749+0000","last_became_active":"2024-08-17T17:21:53.159384+0000","last_became_peered":"2024-08-17T17:21:53.159384+0000","last_unstale":"2024-08-17T17:22:33.961749+0000","last_undegraded":"2024-08-17T17:22:33.961749+0000","last_fullsized":"2024-08-17T17:22:33.961749+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:43.798412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357759+0000","last_change":"2024-08-17T17:21:52.151540+0000","last_active":"2024-08-17T17:22:34.357759+0000","last_peered":"2024-08-17T17:22:34.357759+0000","last_clean":"2024-08-17T17:22:34.357759+0000","last_became_active":"2024-08-17T17:21:52.151294+0000","last_became_peered":"2024-08-17T17:21:52.151294+0000","last_unstale":"2024-08-17T17:22:34.357759+0000","last_undegraded":"2024-08-17T17:22:34.357759+0000","last_fullsized":"2024-08-17T17:22:34.357759+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:03.533252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965988+0000","last_change":"2024-08-17T17:21:53.163473+0000","last_active":"2024-08-17T17:22:33.965988+0000","last_peered":"2024-08-17T17:22:33.965988+0000","last_clean":"2024-08-17T17:22:33.965988+0000","last_became_active":"2024-08-17T17:21:53.163215+0000","last_became_peered":"2024-08-17T17:21:53.163215+0000","last_unstale":"2024-08-17T17:22:33.965988+0000","last_undegraded":"2024-08-17T17:22:33.965988+0000","last_fullsized":"2024-08-17T17:22:33.965988+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:55:15.816408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965337+0000","last_change":"2024-08-17T17:21:52.148375+0000","last_active":"2024-08-17T17:22:33.965337+0000","last_peered":"2024-08-17T17:22:33.965337+0000","last_clean":"2024-08-17T17:22:33.965337+0000","last_became_active":"2024-08-17T17:21:52.148079+0000","last_became_peered":"2024-08-17T17:21:52.148079+0000","last_unstale":"2024-08-17T17:22:33.965337+0000","last_undegraded":"2024-08-17T17:22:33.965337+0000","last_fullsized":"2024-08-17T17:22:33.965337+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:16.638141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965695+0000","last_change":"2024-08-17T17:21:52.165264+0000","last_active":"2024-08-17T17:22:33.965695+0000","last_peered":"2024-08-17T17:22:33.965695+0000","last_clean":"2024-08-17T17:22:33.965695+0000","last_became_active":"2024-08-17T17:21:52.164991+0000","last_became_peered":"2024-08-17T17:21:52.164991+0000","last_unstale":"2024-08-17T17:22:33.965695+0000","last_undegraded":"2024-08-17T17:22:33.965695+0000","last_fullsized":"2024-08-17T17:22:33.965695+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:36:34.605579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.963566+0000","last_change":"2024-08-17T17:21:52.148708+0000","last_active":"2024-08-17T17:22:33.963566+0000","last_peered":"2024-08-17T17:22:33.963566+0000","last_clean":"2024-08-17T17:22:33.963566+0000","last_became_active":"2024-08-17T17:21:52.148403+0000","last_became_peered":"2024-08-17T17:21:52.148403+0000","last_unstale":"2024-08-17T17:22:33.963566+0000","last_undegraded":"2024-08-17T17:22:33.963566+0000","last_fullsized":"2024-08-17T17:22:33.963566+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:15:13.267832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962037+0000","last_change":"2024-08-17T17:21:53.161163+0000","last_active":"2024-08-17T17:22:33.962037+0000","last_peered":"2024-08-17T17:22:33.962037+0000","last_clean":"2024-08-17T17:22:33.962037+0000","last_became_active":"2024-08-17T17:21:53.160870+0000","last_became_peered":"2024-08-17T17:21:53.160870+0000","last_unstale":"2024-08-17T17:22:33.962037+0000","last_undegraded":"2024-08-17T17:22:33.962037+0000","last_fullsized":"2024-08-17T17:22:33.962037+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:05:16.331923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.966148+0000","last_change":"2024-08-17T17:21:52.150404+0000","last_active":"2024-08-17T17:22:33.966148+0000","last_peered":"2024-08-17T17:22:33.966148+0000","last_clean":"2024-08-17T17:22:33.966148+0000","last_became_active":"2024-08-17T17:21:52.150146+0000","last_became_peered":"2024-08-17T17:21:52.150146+0000","last_unstale":"2024-08-17T17:22:33.966148+0000","last_undegraded":"2024-08-17T17:22:33.966148+0000","last_fullsized":"2024-08-17T17:22:33.966148+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:08.479783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962296+0000","last_change":"2024-08-17T17:21:53.294813+0000","last_active":"2024-08-17T17:22:33.962296+0000","last_peered":"2024-08-17T17:22:33.962296+0000","last_clean":"2024-08-17T17:22:33.962296+0000","last_became_active":"2024-08-17T17:21:53.294591+0000","last_became_peered":"2024-08-17T17:21:53.294591+0000","last_unstale":"2024-08-17T17:22:33.962296+0000","last_undegraded":"2024-08-17T17:22:33.962296+0000","last_fullsized":"2024-08-17T17:22:33.962296+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:05:10.508717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961998+0000","last_change":"2024-08-17T17:21:52.152251+0000","last_active":"2024-08-17T17:22:33.961998+0000","last_peered":"2024-08-17T17:22:33.961998+0000","last_clean":"2024-08-17T17:22:33.961998+0000","last_became_active":"2024-08-17T17:21:52.151941+0000","last_became_peered":"2024-08-17T17:21:52.151941+0000","last_unstale":"2024-08-17T17:22:33.961998+0000","last_undegraded":"2024-08-17T17:22:33.961998+0000","last_fullsized":"2024-08-17T17:22:33.961998+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:36.846304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962158+0000","last_change":"2024-08-17T17:21:53.162744+0000","last_active":"2024-08-17T17:22:33.962158+0000","last_peered":"2024-08-17T17:22:33.962158+0000","last_clean":"2024-08-17T17:22:33.962158+0000","last_became_active":"2024-08-17T17:21:53.161996+0000","last_became_peered":"2024-08-17T17:21:53.161996+0000","last_unstale":"2024-08-17T17:22:33.962158+0000","last_undegraded":"2024-08-17T17:22:33.962158+0000","last_fullsized":"2024-08-17T17:22:33.962158+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:46.913168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.964388+0000","last_change":"2024-08-17T17:21:52.149433+0000","last_active":"2024-08-17T17:22:33.964388+0000","last_peered":"2024-08-17T17:22:33.964388+0000","last_clean":"2024-08-17T17:22:33.964388+0000","last_became_active":"2024-08-17T17:21:52.149194+0000","last_became_peered":"2024-08-17T17:21:52.149194+0000","last_unstale":"2024-08-17T17:22:33.964388+0000","last_undegraded":"2024-08-17T17:22:33.964388+0000","last_fullsized":"2024-08-17T17:22:33.964388+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:19.168680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962506+0000","last_change":"2024-08-17T17:21:53.159869+0000","last_active":"2024-08-17T17:22:33.962506+0000","last_peered":"2024-08-17T17:22:33.962506+0000","last_clean":"2024-08-17T17:22:33.962506+0000","last_became_active":"2024-08-17T17:21:53.159536+0000","last_became_peered":"2024-08-17T17:21:53.159536+0000","last_unstale":"2024-08-17T17:22:33.962506+0000","last_undegraded":"2024-08-17T17:22:33.962506+0000","last_fullsized":"2024-08-17T17:22:33.962506+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:04.307205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":280,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357616+0000","last_change":"2024-08-17T17:20:06.878678+0000","last_active":"2024-08-17T17:22:34.357616+0000","last_peered":"2024-08-17T17:22:34.357616+0000","last_clean":"2024-08-17T17:22:34.357616+0000","last_became_active":"2024-08-17T17:20:06.878356+0000","last_became_peered":"2024-08-17T17:20:06.878356+0000","last_unstale":"2024-08-17T17:22:34.357616+0000","last_undegraded":"2024-08-17T17:22:34.357616+0000","last_fullsized":"2024-08-17T17:22:34.357616+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:02:43.355026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":27,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965082+0000","last_change":"2024-08-17T17:20:43.676802+0000","last_active":"2024-08-17T17:22:33.965082+0000","last_peered":"2024-08-17T17:22:33.965082+0000","last_clean":"2024-08-17T17:22:33.965082+0000","last_became_active":"2024-08-17T17:20:38.651573+0000","last_became_peered":"2024-08-17T17:20:38.651573+0000","last_unstale":"2024-08-17T17:22:33.965082+0000","last_undegraded":"2024-08-17T17:22:33.965082+0000","last_fullsized":"2024-08-17T17:22:33.965082+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:20:15.923334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085985900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.963601+0000","last_change":"2024-08-17T17:21:52.149270+0000","last_active":"2024-08-17T17:22:33.963601+0000","last_peered":"2024-08-17T17:22:33.963601+0000","last_clean":"2024-08-17T17:22:33.963601+0000","last_became_active":"2024-08-17T17:21:52.149026+0000","last_became_peered":"2024-08-17T17:21:52.149026+0000","last_unstale":"2024-08-17T17:22:33.963601+0000","last_undegraded":"2024-08-17T17:22:33.963601+0000","last_fullsized":"2024-08-17T17:22:33.963601+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:30.037565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.966355+0000","last_change":"2024-08-17T17:21:53.164672+0000","last_active":"2024-08-17T17:22:33.966355+0000","last_peered":"2024-08-17T17:22:33.966355+0000","last_clean":"2024-08-17T17:22:33.966355+0000","last_became_active":"2024-08-17T17:21:53.164381+0000","last_became_peered":"2024-08-17T17:21:53.164381+0000","last_unstale":"2024-08-17T17:22:33.966355+0000","last_undegraded":"2024-08-17T17:22:33.966355+0000","last_fullsized":"2024-08-17T17:22:33.966355+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:43:36.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357552+0000","last_change":"2024-08-17T17:20:43.716700+0000","last_active":"2024-08-17T17:22:34.357552+0000","last_peered":"2024-08-17T17:22:34.357552+0000","last_clean":"2024-08-17T17:22:34.357552+0000","last_became_active":"2024-08-17T17:20:38.653957+0000","last_became_peered":"2024-08-17T17:20:38.653957+0000","last_unstale":"2024-08-17T17:22:34.357552+0000","last_undegraded":"2024-08-17T17:22:34.357552+0000","last_fullsized":"2024-08-17T17:22:34.357552+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:43:42.214017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965160+0000","last_change":"2024-08-17T17:21:52.158478+0000","last_active":"2024-08-17T17:22:33.965160+0000","last_peered":"2024-08-17T17:22:33.965160+0000","last_clean":"2024-08-17T17:22:33.965160+0000","last_became_active":"2024-08-17T17:21:52.158147+0000","last_became_peered":"2024-08-17T17:21:52.158147+0000","last_unstale":"2024-08-17T17:22:33.965160+0000","last_undegraded":"2024-08-17T17:22:33.965160+0000","last_fullsized":"2024-08-17T17:22:33.965160+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:50.137734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:22:35.230913+0000","last_change":"2024-08-17T17:22:35.230913+0000","last_active":"2024-08-17T17:22:35.230913+0000","last_peered":"2024-08-17T17:22:35.230913+0000","last_clean":"2024-08-17T17:22:35.230913+0000","last_became_active":"2024-08-17T17:22:35.230556+0000","last_became_peered":"2024-08-17T17:22:35.230556+0000","last_unstale":"2024-08-17T17:22:35.230913+0000","last_undegraded":"2024-08-17T17:22:35.230913+0000","last_fullsized":"2024-08-17T17:22:35.230913+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:55.819618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.963994+0000","last_change":"2024-08-17T17:20:43.672739+0000","last_active":"2024-08-17T17:22:33.963994+0000","last_peered":"2024-08-17T17:22:33.963994+0000","last_clean":"2024-08-17T17:22:33.963994+0000","last_became_active":"2024-08-17T17:20:38.783090+0000","last_became_peered":"2024-08-17T17:20:38.783090+0000","last_unstale":"2024-08-17T17:22:33.963994+0000","last_undegraded":"2024-08-17T17:22:33.963994+0000","last_fullsized":"2024-08-17T17:22:33.963994+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:23.536205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961169+0000","last_change":"2024-08-17T17:21:52.150340+0000","last_active":"2024-08-17T17:22:33.961169+0000","last_peered":"2024-08-17T17:22:33.961169+0000","last_clean":"2024-08-17T17:22:33.961169+0000","last_became_active":"2024-08-17T17:21:52.150086+0000","last_became_peered":"2024-08-17T17:21:52.150086+0000","last_unstale":"2024-08-17T17:22:33.961169+0000","last_undegraded":"2024-08-17T17:22:33.961169+0000","last_fullsized":"2024-08-17T17:22:33.961169+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:45:21.964979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962575+0000","last_change":"2024-08-17T17:21:53.206453+0000","last_active":"2024-08-17T17:22:33.962575+0000","last_peered":"2024-08-17T17:22:33.962575+0000","last_clean":"2024-08-17T17:22:33.962575+0000","last_became_active":"2024-08-17T17:21:53.206096+0000","last_became_peered":"2024-08-17T17:21:53.206096+0000","last_unstale":"2024-08-17T17:22:33.962575+0000","last_undegraded":"2024-08-17T17:22:33.962575+0000","last_fullsized":"2024-08-17T17:22:33.962575+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:03.531301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":28,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965699+0000","last_change":"2024-08-17T17:20:43.676769+0000","last_active":"2024-08-17T17:22:33.965699+0000","last_peered":"2024-08-17T17:22:33.965699+0000","last_clean":"2024-08-17T17:22:33.965699+0000","last_became_active":"2024-08-17T17:20:38.651624+0000","last_became_peered":"2024-08-17T17:20:38.651624+0000","last_unstale":"2024-08-17T17:22:33.965699+0000","last_undegraded":"2024-08-17T17:22:33.965699+0000","last_fullsized":"2024-08-17T17:22:33.965699+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:58:44.833201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0008386,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962257+0000","last_change":"2024-08-17T17:21:52.157986+0000","last_active":"2024-08-17T17:22:33.962257+0000","last_peered":"2024-08-17T17:22:33.962257+0000","last_clean":"2024-08-17T17:22:33.962257+0000","last_became_active":"2024-08-17T17:21:52.157699+0000","last_became_peered":"2024-08-17T17:21:52.157699+0000","last_unstale":"2024-08-17T17:22:33.962257+0000","last_undegraded":"2024-08-17T17:22:33.962257+0000","last_fullsized":"2024-08-17T17:22:33.962257+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:38.733835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961752+0000","last_change":"2024-08-17T17:21:53.295569+0000","last_active":"2024-08-17T17:22:33.961752+0000","last_peered":"2024-08-17T17:22:33.961752+0000","last_clean":"2024-08-17T17:22:33.961752+0000","last_became_active":"2024-08-17T17:21:53.295131+0000","last_became_peered":"2024-08-17T17:21:53.295131+0000","last_unstale":"2024-08-17T17:22:33.961752+0000","last_undegraded":"2024-08-17T17:22:33.961752+0000","last_fullsized":"2024-08-17T17:22:33.961752+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:58:26.289880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965539+0000","last_change":"2024-08-17T17:21:52.151870+0000","last_active":"2024-08-17T17:22:33.965539+0000","last_peered":"2024-08-17T17:22:33.965539+0000","last_clean":"2024-08-17T17:22:33.965539+0000","last_became_active":"2024-08-17T17:21:52.151441+0000","last_became_peered":"2024-08-17T17:21:52.151441+0000","last_unstale":"2024-08-17T17:22:33.965539+0000","last_undegraded":"2024-08-17T17:22:33.965539+0000","last_fullsized":"2024-08-17T17:22:33.965539+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:07.731078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961947+0000","last_change":"2024-08-17T17:20:43.677386+0000","last_active":"2024-08-17T17:22:33.961947+0000","last_peered":"2024-08-17T17:22:33.961947+0000","last_clean":"2024-08-17T17:22:33.961947+0000","last_became_active":"2024-08-17T17:20:38.653370+0000","last_became_peered":"2024-08-17T17:20:38.653370+0000","last_unstale":"2024-08-17T17:22:33.961947+0000","last_undegraded":"2024-08-17T17:22:33.961947+0000","last_fullsized":"2024-08-17T17:22:33.961947+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:40.082296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:22:35.231291+0000","last_change":"2024-08-17T17:22:35.231291+0000","last_active":"2024-08-17T17:22:35.231291+0000","last_peered":"2024-08-17T17:22:35.231291+0000","last_clean":"2024-08-17T17:22:35.231291+0000","last_became_active":"2024-08-17T17:22:35.231021+0000","last_became_peered":"2024-08-17T17:22:35.231021+0000","last_unstale":"2024-08-17T17:22:35.231291+0000","last_undegraded":"2024-08-17T17:22:35.231291+0000","last_fullsized":"2024-08-17T17:22:35.231291+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:19:18.292988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961838+0000","last_change":"2024-08-17T17:21:52.151729+0000","last_active":"2024-08-17T17:22:33.961838+0000","last_peered":"2024-08-17T17:22:33.961838+0000","last_clean":"2024-08-17T17:22:33.961838+0000","last_became_active":"2024-08-17T17:21:52.151470+0000","last_became_peered":"2024-08-17T17:21:52.151470+0000","last_unstale":"2024-08-17T17:22:33.961838+0000","last_undegraded":"2024-08-17T17:22:33.961838+0000","last_fullsized":"2024-08-17T17:22:33.961838+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:04.706991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357766+0000","last_change":"2024-08-17T17:20:43.716798+0000","last_active":"2024-08-17T17:22:34.357766+0000","last_peered":"2024-08-17T17:22:34.357766+0000","last_clean":"2024-08-17T17:22:34.357766+0000","last_became_active":"2024-08-17T17:20:38.782941+0000","last_became_peered":"2024-08-17T17:20:38.782941+0000","last_unstale":"2024-08-17T17:22:34.357766+0000","last_undegraded":"2024-08-17T17:22:34.357766+0000","last_fullsized":"2024-08-17T17:22:34.357766+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:35.726637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965417+0000","last_change":"2024-08-17T17:21:52.937973+0000","last_active":"2024-08-17T17:22:33.965417+0000","last_peered":"2024-08-17T17:22:33.965417+0000","last_clean":"2024-08-17T17:22:33.965417+0000","last_became_active":"2024-08-17T17:21:52.937636+0000","last_became_peered":"2024-08-17T17:21:52.937636+0000","last_unstale":"2024-08-17T17:22:33.965417+0000","last_undegraded":"2024-08-17T17:22:33.965417+0000","last_fullsized":"2024-08-17T17:22:33.965417+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:56:05.025801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.964453+0000","last_change":"2024-08-17T17:21:52.147358+0000","last_active":"2024-08-17T17:22:33.964453+0000","last_peered":"2024-08-17T17:22:33.964453+0000","last_clean":"2024-08-17T17:22:33.964453+0000","last_became_active":"2024-08-17T17:21:52.147112+0000","last_became_peered":"2024-08-17T17:21:52.147112+0000","last_unstale":"2024-08-17T17:22:33.964453+0000","last_undegraded":"2024-08-17T17:22:33.964453+0000","last_fullsized":"2024-08-17T17:22:33.964453+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:02:49.646927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962421+0000","last_change":"2024-08-17T17:20:43.677369+0000","last_active":"2024-08-17T17:22:33.962421+0000","last_peered":"2024-08-17T17:22:33.962421+0000","last_clean":"2024-08-17T17:22:33.962421+0000","last_became_active":"2024-08-17T17:20:38.782865+0000","last_became_peered":"2024-08-17T17:20:38.782865+0000","last_unstale":"2024-08-17T17:22:33.962421+0000","last_undegraded":"2024-08-17T17:22:33.962421+0000","last_fullsized":"2024-08-17T17:22:33.962421+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:15:56.927535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021889699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.964227+0000","last_change":"2024-08-17T17:21:52.938072+0000","last_active":"2024-08-17T17:22:33.964227+0000","last_peered":"2024-08-17T17:22:33.964227+0000","last_clean":"2024-08-17T17:22:33.964227+0000","last_became_active":"2024-08-17T17:21:52.937627+0000","last_became_peered":"2024-08-17T17:21:52.937627+0000","last_unstale":"2024-08-17T17:22:33.964227+0000","last_undegraded":"2024-08-17T17:22:33.964227+0000","last_fullsized":"2024-08-17T17:22:33.964227+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:52.163926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.966409+0000","last_change":"2024-08-17T17:21:52.157729+0000","last_active":"2024-08-17T17:22:33.966409+0000","last_peered":"2024-08-17T17:22:33.966409+0000","last_clean":"2024-08-17T17:22:33.966409+0000","last_became_active":"2024-08-17T17:21:52.157026+0000","last_became_peered":"2024-08-17T17:21:52.157026+0000","last_unstale":"2024-08-17T17:22:33.966409+0000","last_undegraded":"2024-08-17T17:22:33.966409+0000","last_fullsized":"2024-08-17T17:22:33.966409+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:24:51.873449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":25,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.966429+0000","last_change":"2024-08-17T17:20:43.677752+0000","last_active":"2024-08-17T17:22:33.966429+0000","last_peered":"2024-08-17T17:22:33.966429+0000","last_clean":"2024-08-17T17:22:33.966429+0000","last_became_active":"2024-08-17T17:20:38.652246+0000","last_became_peered":"2024-08-17T17:20:38.652246+0000","last_unstale":"2024-08-17T17:22:33.966429+0000","last_undegraded":"2024-08-17T17:22:33.966429+0000","last_fullsized":"2024-08-17T17:22:33.966429+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:52.620577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961640+0000","last_change":"2024-08-17T17:21:53.161517+0000","last_active":"2024-08-17T17:22:33.961640+0000","last_peered":"2024-08-17T17:22:33.961640+0000","last_clean":"2024-08-17T17:22:33.961640+0000","last_became_active":"2024-08-17T17:21:53.161203+0000","last_became_peered":"2024-08-17T17:21:53.161203+0000","last_unstale":"2024-08-17T17:22:33.961640+0000","last_undegraded":"2024-08-17T17:22:33.961640+0000","last_fullsized":"2024-08-17T17:22:33.961640+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:46.059237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962125+0000","last_change":"2024-08-17T17:21:53.163858+0000","last_active":"2024-08-17T17:22:33.962125+0000","last_peered":"2024-08-17T17:22:33.962125+0000","last_clean":"2024-08-17T17:22:33.962125+0000","last_became_active":"2024-08-17T17:21:53.163018+0000","last_became_peered":"2024-08-17T17:21:53.163018+0000","last_unstale":"2024-08-17T17:22:33.962125+0000","last_undegraded":"2024-08-17T17:22:33.962125+0000","last_fullsized":"2024-08-17T17:22:33.962125+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:04:36.044906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"71'6","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.962119+0000","last_change":"2024-08-17T17:21:52.156855+0000","last_active":"2024-08-17T17:22:33.962119+0000","last_peered":"2024-08-17T17:22:33.962119+0000","last_clean":"2024-08-17T17:22:33.962119+0000","last_became_active":"2024-08-17T17:21:52.156377+0000","last_became_peered":"2024-08-17T17:21:52.156377+0000","last_unstale":"2024-08-17T17:22:33.962119+0000","last_undegraded":"2024-08-17T17:22:33.962119+0000","last_fullsized":"2024-08-17T17:22:33.962119+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:29.248851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961333+0000","last_change":"2024-08-17T17:21:53.295126+0000","last_active":"2024-08-17T17:22:33.961333+0000","last_peered":"2024-08-17T17:22:33.961333+0000","last_clean":"2024-08-17T17:22:33.961333+0000","last_became_active":"2024-08-17T17:21:53.294792+0000","last_became_peered":"2024-08-17T17:21:53.294792+0000","last_unstale":"2024-08-17T17:22:33.961333+0000","last_undegraded":"2024-08-17T17:22:33.961333+0000","last_fullsized":"2024-08-17T17:22:33.961333+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:09.378025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.357639+0000","last_change":"2024-08-17T17:21:52.149310+0000","last_active":"2024-08-17T17:22:34.357639+0000","last_peered":"2024-08-17T17:22:34.357639+0000","last_clean":"2024-08-17T17:22:34.357639+0000","last_became_active":"2024-08-17T17:21:52.148996+0000","last_became_peered":"2024-08-17T17:21:52.148996+0000","last_unstale":"2024-08-17T17:22:34.357639+0000","last_undegraded":"2024-08-17T17:22:34.357639+0000","last_fullsized":"2024-08-17T17:22:34.357639+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:26.830642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965563+0000","last_change":"2024-08-17T17:21:52.152923+0000","last_active":"2024-08-17T17:22:33.965563+0000","last_peered":"2024-08-17T17:22:33.965563+0000","last_clean":"2024-08-17T17:22:33.965563+0000","last_became_active":"2024-08-17T17:21:52.152588+0000","last_became_peered":"2024-08-17T17:21:52.152588+0000","last_unstale":"2024-08-17T17:22:33.965563+0000","last_undegraded":"2024-08-17T17:22:33.965563+0000","last_fullsized":"2024-08-17T17:22:33.965563+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:11:27.075241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.963432+0000","last_change":"2024-08-17T17:21:53.161868+0000","last_active":"2024-08-17T17:22:33.963432+0000","last_peered":"2024-08-17T17:22:33.963432+0000","last_clean":"2024-08-17T17:22:33.963432+0000","last_became_active":"2024-08-17T17:21:53.161100+0000","last_became_peered":"2024-08-17T17:21:53.161100+0000","last_unstale":"2024-08-17T17:22:33.963432+0000","last_undegraded":"2024-08-17T17:22:33.963432+0000","last_fullsized":"2024-08-17T17:22:33.963432+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:11:07.175330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965689+0000","last_change":"2024-08-17T17:21:52.151043+0000","last_active":"2024-08-17T17:22:33.965689+0000","last_peered":"2024-08-17T17:22:33.965689+0000","last_clean":"2024-08-17T17:22:33.965689+0000","last_became_active":"2024-08-17T17:21:52.150614+0000","last_became_peered":"2024-08-17T17:21:52.150614+0000","last_unstale":"2024-08-17T17:22:33.965689+0000","last_undegraded":"2024-08-17T17:22:33.965689+0000","last_fullsized":"2024-08-17T17:22:33.965689+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:01.681256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T17:22:34.965386+0000","last_change":"2024-08-17T17:22:34.965386+0000","last_active":"2024-08-17T17:22:34.965386+0000","last_peered":"2024-08-17T17:22:34.965386+0000","last_clean":"2024-08-17T17:22:34.965386+0000","last_became_active":"2024-08-17T17:22:34.964930+0000","last_became_peered":"2024-08-17T17:22:34.964930+0000","last_unstale":"2024-08-17T17:22:34.965386+0000","last_undegraded":"2024-08-17T17:22:34.965386+0000","last_fullsized":"2024-08-17T17:22:34.965386+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:24:23.306640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.966404+0000","last_change":"2024-08-17T17:21:53.162596+0000","last_active":"2024-08-17T17:22:33.966404+0000","last_peered":"2024-08-17T17:22:33.966404+0000","last_clean":"2024-08-17T17:22:33.966404+0000","last_became_active":"2024-08-17T17:21:53.162339+0000","last_became_peered":"2024-08-17T17:21:53.162339+0000","last_unstale":"2024-08-17T17:22:33.966404+0000","last_undegraded":"2024-08-17T17:22:33.966404+0000","last_fullsized":"2024-08-17T17:22:33.966404+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:21:00.795345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.964316+0000","last_change":"2024-08-17T17:21:52.149351+0000","last_active":"2024-08-17T17:22:33.964316+0000","last_peered":"2024-08-17T17:22:33.964316+0000","last_clean":"2024-08-17T17:22:33.964316+0000","last_became_active":"2024-08-17T17:21:52.149100+0000","last_became_peered":"2024-08-17T17:21:52.149100+0000","last_unstale":"2024-08-17T17:22:33.964316+0000","last_undegraded":"2024-08-17T17:22:33.964316+0000","last_fullsized":"2024-08-17T17:22:33.964316+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:26:05.195805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965619+0000","last_change":"2024-08-17T17:21:52.938947+0000","last_active":"2024-08-17T17:22:33.965619+0000","last_peered":"2024-08-17T17:22:33.965619+0000","last_clean":"2024-08-17T17:22:33.965619+0000","last_became_active":"2024-08-17T17:21:52.938697+0000","last_became_peered":"2024-08-17T17:21:52.938697+0000","last_unstale":"2024-08-17T17:22:33.965619+0000","last_undegraded":"2024-08-17T17:22:33.965619+0000","last_fullsized":"2024-08-17T17:22:33.965619+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:40:05.765657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961080+0000","last_change":"2024-08-17T17:21:52.154388+0000","last_active":"2024-08-17T17:22:33.961080+0000","last_peered":"2024-08-17T17:22:33.961080+0000","last_clean":"2024-08-17T17:22:33.961080+0000","last_became_active":"2024-08-17T17:21:52.153910+0000","last_became_peered":"2024-08-17T17:21:52.153910+0000","last_unstale":"2024-08-17T17:22:33.961080+0000","last_undegraded":"2024-08-17T17:22:33.961080+0000","last_fullsized":"2024-08-17T17:22:33.961080+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:24:24.483938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.963990+0000","last_change":"2024-08-17T17:21:52.166598+0000","last_active":"2024-08-17T17:22:33.963990+0000","last_peered":"2024-08-17T17:22:33.963990+0000","last_clean":"2024-08-17T17:22:33.963990+0000","last_became_active":"2024-08-17T17:21:52.166339+0000","last_became_peered":"2024-08-17T17:21:52.166339+0000","last_unstale":"2024-08-17T17:22:33.963990+0000","last_undegraded":"2024-08-17T17:22:33.963990+0000","last_fullsized":"2024-08-17T17:22:33.963990+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:45.264989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961976+0000","last_change":"2024-08-17T17:21:52.157102+0000","last_active":"2024-08-17T17:22:33.961976+0000","last_peered":"2024-08-17T17:22:33.961976+0000","last_clean":"2024-08-17T17:22:33.961976+0000","last_became_active":"2024-08-17T17:21:52.156813+0000","last_became_peered":"2024-08-17T17:21:52.156813+0000","last_unstale":"2024-08-17T17:22:33.961976+0000","last_undegraded":"2024-08-17T17:22:33.961976+0000","last_fullsized":"2024-08-17T17:22:33.961976+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:44:32.159655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.964656+0000","last_change":"2024-08-17T17:21:52.165947+0000","last_active":"2024-08-17T17:22:33.964656+0000","last_peered":"2024-08-17T17:22:33.964656+0000","last_clean":"2024-08-17T17:22:33.964656+0000","last_became_active":"2024-08-17T17:21:52.165655+0000","last_became_peered":"2024-08-17T17:21:52.165655+0000","last_unstale":"2024-08-17T17:22:33.964656+0000","last_undegraded":"2024-08-17T17:22:33.964656+0000","last_fullsized":"2024-08-17T17:22:33.964656+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:43:22.658721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965570+0000","last_change":"2024-08-17T17:21:52.157236+0000","last_active":"2024-08-17T17:22:33.965570+0000","last_peered":"2024-08-17T17:22:33.965570+0000","last_clean":"2024-08-17T17:22:33.965570+0000","last_became_active":"2024-08-17T17:21:52.156966+0000","last_became_peered":"2024-08-17T17:21:52.156966+0000","last_unstale":"2024-08-17T17:22:33.965570+0000","last_undegraded":"2024-08-17T17:22:33.965570+0000","last_fullsized":"2024-08-17T17:22:33.965570+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:10.648147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961913+0000","last_change":"2024-08-17T17:21:53.162727+0000","last_active":"2024-08-17T17:22:33.961913+0000","last_peered":"2024-08-17T17:22:33.961913+0000","last_clean":"2024-08-17T17:22:33.961913+0000","last_became_active":"2024-08-17T17:21:53.162503+0000","last_became_peered":"2024-08-17T17:21:53.162503+0000","last_unstale":"2024-08-17T17:22:33.961913+0000","last_undegraded":"2024-08-17T17:22:33.961913+0000","last_fullsized":"2024-08-17T17:22:33.961913+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:27:17.073240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965338+0000","last_change":"2024-08-17T17:21:52.159030+0000","last_active":"2024-08-17T17:22:33.965338+0000","last_peered":"2024-08-17T17:22:33.965338+0000","last_clean":"2024-08-17T17:22:33.965338+0000","last_became_active":"2024-08-17T17:21:52.158599+0000","last_became_peered":"2024-08-17T17:21:52.158599+0000","last_unstale":"2024-08-17T17:22:33.965338+0000","last_undegraded":"2024-08-17T17:22:33.965338+0000","last_fullsized":"2024-08-17T17:22:33.965338+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:22.522063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961731+0000","last_change":"2024-08-17T17:21:53.206411+0000","last_active":"2024-08-17T17:22:33.961731+0000","last_peered":"2024-08-17T17:22:33.961731+0000","last_clean":"2024-08-17T17:22:33.961731+0000","last_became_active":"2024-08-17T17:21:53.206139+0000","last_became_peered":"2024-08-17T17:21:53.206139+0000","last_unstale":"2024-08-17T17:22:33.961731+0000","last_undegraded":"2024-08-17T17:22:33.961731+0000","last_fullsized":"2024-08-17T17:22:33.961731+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:37:56.943926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965390+0000","last_change":"2024-08-17T17:21:52.157731+0000","last_active":"2024-08-17T17:22:33.965390+0000","last_peered":"2024-08-17T17:22:33.965390+0000","last_clean":"2024-08-17T17:22:33.965390+0000","last_became_active":"2024-08-17T17:21:52.157453+0000","last_became_peered":"2024-08-17T17:21:52.157453+0000","last_unstale":"2024-08-17T17:22:33.965390+0000","last_undegraded":"2024-08-17T17:22:33.965390+0000","last_fullsized":"2024-08-17T17:22:33.965390+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:30:09.106088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.963517+0000","last_change":"2024-08-17T17:21:52.166049+0000","last_active":"2024-08-17T17:22:33.963517+0000","last_peered":"2024-08-17T17:22:33.963517+0000","last_clean":"2024-08-17T17:22:33.963517+0000","last_became_active":"2024-08-17T17:21:52.165300+0000","last_became_peered":"2024-08-17T17:21:52.165300+0000","last_unstale":"2024-08-17T17:22:33.963517+0000","last_undegraded":"2024-08-17T17:22:33.963517+0000","last_fullsized":"2024-08-17T17:22:33.963517+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:22:34.677517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.964335+0000","last_change":"2024-08-17T17:21:52.148393+0000","last_active":"2024-08-17T17:22:33.964335+0000","last_peered":"2024-08-17T17:22:33.964335+0000","last_clean":"2024-08-17T17:22:33.964335+0000","last_became_active":"2024-08-17T17:21:52.148098+0000","last_became_peered":"2024-08-17T17:21:52.148098+0000","last_unstale":"2024-08-17T17:22:33.964335+0000","last_undegraded":"2024-08-17T17:22:33.964335+0000","last_fullsized":"2024-08-17T17:22:33.964335+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:52:44.434667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.961651+0000","last_change":"2024-08-17T17:21:53.160925+0000","last_active":"2024-08-17T17:22:33.961651+0000","last_peered":"2024-08-17T17:22:33.961651+0000","last_clean":"2024-08-17T17:22:33.961651+0000","last_became_active":"2024-08-17T17:21:53.160641+0000","last_became_peered":"2024-08-17T17:21:53.160641+0000","last_unstale":"2024-08-17T17:22:33.961651+0000","last_undegraded":"2024-08-17T17:22:33.961651+0000","last_fullsized":"2024-08-17T17:22:33.961651+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:26.697016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-08-17T17:22:33.965632+0000","last_change":"2024-08-17T17:21:52.159652+0000","last_active":"2024-08-17T17:22:33.965632+0000","last_peered":"2024-08-17T17:22:33.965632+0000","last_clean":"2024-08-17T17:22:33.965632+0000","last_became_active":"2024-08-17T17:21:52.159348+0000","last_became_peered":"2024-08-17T17:21:52.159348+0000","last_unstale":"2024-08-17T17:22:33.965632+0000","last_undegraded":"2024-08-17T17:22:33.965632+0000","last_fullsized":"2024-08-17T17:22:33.965632+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:42:47.650336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234022,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298700,"kb_used_data":1340,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446452,"statfs":{"total":95995035648,"available":95689166848,"internally_reserved":0,"allocated":1372160,"data_stored":924587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.49099999999999999,"15min":0.48499999999999999},"min":{"1min":0.378,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.503,"15min":0.499},"min":{"1min":0.40799999999999997,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.65900000000000003,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.54100000000000004}]},{"osd":1,"last update":"Sat Aug 17 17:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.48999999999999999,"15min":0.48299999999999998},"min":{"1min":0.39100000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.70599999999999996,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.51300000000000001,"15min":0.51400000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.51400000000000001}]},{"osd":2,"last update":"Sat Aug 17 17:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.496,"15min":0.49099999999999999},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.49199999999999999,"15min":0.48799999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.66700000000000004,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.441}]},{"osd":3,"last update":"Sat Aug 17 17:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.46700000000000003,"15min":0.45500000000000002},"min":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.48899999999999999,"15min":0.48099999999999998},"min":{"1min":0.38600000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.501}]},{"osd":4,"last update":"Sat Aug 17 17:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.40899999999999997,"15min":0.40500000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.40999999999999998,"15min":0.40400000000000003},"min":{"1min":0.31900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.42099999999999999}]},{"osd":5,"last update":"Sat Aug 17 17:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.41699999999999998,"15min":0.41099999999999998},"min":{"1min":0.32400000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.498,"5min":0.45500000000000002,"15min":0.44800000000000001},"min":{"1min":0.35999999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.746,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.44700000000000001}]},{"osd":6,"last update":"Sat Aug 17 17:22:04 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.48099999999999998,"15min":0.47699999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.46500000000000002,"15min":0.46100000000000002},"min":{"1min":0.34899999999999998,"5min":0.316,"15min":0.316},"max":{"1min":0.76800000000000002,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.47299999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495657,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298684,"kb_used_data":1328,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446468,"statfs":{"total":95995035648,"available":95689183232,"internally_reserved":0,"allocated":1359872,"data_stored":924145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.50800000000000001,"15min":0.505},"min":{"1min":0.39000000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.41499999999999998,"5min":0.373,"15min":0.373},"max":{"1min":0.70099999999999996,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.46200000000000002}]},{"osd":1,"last update":"Sat Aug 17 17:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.35799999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.77500000000000002,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53600000000000003,"15min":0.53500000000000003},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.44900000000000001}]},{"osd":2,"last update":"Sat Aug 17 17:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53100000000000003,"15min":0.53000000000000003},"min":{"1min":0.42599999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.50700000000000001,"15min":0.499},"min":{"1min":0.32800000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.41899999999999998}]},{"osd":3,"last update":"Sat Aug 17 17:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.51700000000000002,"15min":0.504},"min":{"1min":0.38600000000000001,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.432},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.54800000000000004,"15min":0.54100000000000004},"min":{"1min":0.40699999999999997,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.48099999999999998}]},{"osd":4,"last update":"Sat Aug 17 17:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.42999999999999999,"15min":0.40600000000000003},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.439,"15min":0.41899999999999998},"min":{"1min":0.32600000000000001,"5min":0.308,"15min":0.308},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Aug 17 17:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.44500000000000001,"15min":0.432},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69199999999999995,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.497},{"interface":"front","average":{"1min":0.502,"5min":0.45900000000000002,"15min":0.433},"min":{"1min":0.32900000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.752,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.371}]},{"osd":7,"last update":"Sat Aug 17 17:22:07 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.50900000000000001,"15min":0.505},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.74399999999999999,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.51100000000000001,"15min":0.504},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.753,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.54700000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757295,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298104,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447048,"statfs":{"total":95995035648,"available":95689777152,"internally_reserved":0,"allocated":761856,"data_stored":333118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:22:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.51600000000000001,"15min":0.51700000000000002},"min":{"1min":0.38300000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.69799999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.50800000000000001,"15min":0.51600000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.68500000000000005,"5min":0.75,"15min":0.75},"last":0.62}]},{"osd":1,"last update":"Sat Aug 17 17:22:22 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.505,"15min":0.51100000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.58699999999999997,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.503},{"interface":"front","average":{"1min":0.504,"5min":0.51100000000000001,"15min":0.52000000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.63600000000000001,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.54400000000000004}]},{"osd":2,"last update":"Sat Aug 17 17:22:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.50800000000000001,"15min":0.50600000000000001},"min":{"1min":0.35199999999999998,"5min":0.311,"15min":0.311},"max":{"1min":0.68999999999999995,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.51700000000000002,"15min":0.52200000000000002},"min":{"1min":0.35999999999999999,"5min":0.316,"15min":0.316},"max":{"1min":0.65700000000000003,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.60899999999999999}]},{"osd":3,"last update":"Sat Aug 17 17:22:22 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.52200000000000002,"15min":0.52600000000000002},"min":{"1min":0.38200000000000001,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.67200000000000004,"5min":0.876,"15min":0.876},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52500000000000002,"15min":0.53300000000000003},"min":{"1min":0.34000000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.66700000000000004,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.57899999999999996}]},{"osd":4,"last update":"Sat Aug 17 17:22:22 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.42499999999999999,"15min":0.41799999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.68300000000000005,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.437},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.44800000000000001,"15min":0.441},"min":{"1min":0.34399999999999997,"5min":0.251,"15min":0.251},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Aug 17 17:22:42 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.45200000000000001,"15min":0.436},"min":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.67900000000000005,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.47599999999999998,"15min":0.46200000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Aug 17 17:22:07 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.46400000000000002,"15min":0.45400000000000001},"min":{"1min":0.312,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.48499999999999999,"15min":0.47499999999999998},"min":{"1min":0.36499999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.71499999999999997,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.69199999999999995}]}]},{"osd":4,"up_from":30,"seq":128849018930,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298116,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447036,"statfs":{"total":95995035648,"available":95689764864,"internally_reserved":0,"allocated":774144,"data_stored":334157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:21:56 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.51500000000000001,"15min":0.52100000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.51500000000000001,"15min":0.53300000000000003},"min":{"1min":0.35899999999999999,"5min":0.313,"15min":0.313},"max":{"1min":0.60699999999999998,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.59699999999999998}]},{"osd":1,"last update":"Sat Aug 17 17:21:56 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.50600000000000001,"15min":0.50900000000000001},"min":{"1min":0.34000000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.5,"5min":0.50800000000000001,"15min":0.51700000000000002},"min":{"1min":0.34999999999999998,"5min":0.316,"15min":0.316},"max":{"1min":0.65400000000000003,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.371}]},{"osd":2,"last update":"Sat Aug 17 17:21:56 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.51300000000000001,"15min":0.52400000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.72899999999999998,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.52300000000000002},"min":{"1min":0.40100000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.624,"5min":0.748,"15min":0.748},"last":0.65700000000000003}]},{"osd":3,"last update":"Sat Aug 17 17:21:56 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51900000000000002,"15min":0.52700000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.72199999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.51200000000000001,"15min":0.51500000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.74299999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.39600000000000002}]},{"osd":5,"last update":"Sat Aug 17 17:22:22 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.41599999999999998,"15min":0.39400000000000002},"min":{"1min":0.33600000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.44,"15min":0.41699999999999998},"min":{"1min":0.26300000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.75900000000000001,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.441}]},{"osd":6,"last update":"Sat Aug 17 17:22:48 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.41499999999999998,"15min":0.39500000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.72999999999999998,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.439,"15min":0.42699999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.63,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.432}]},{"osd":7,"last update":"Sat Aug 17 17:22:06 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.45900000000000002,"15min":0.45700000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.47799999999999998,"15min":0.47199999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.77000000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182455,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298100,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447052,"statfs":{"total":95995035648,"available":95689781248,"internally_reserved":0,"allocated":757760,"data_stored":332814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.309,"5min":0.34499999999999997,"15min":0.35999999999999999},"min":{"1min":0.20799999999999999,"5min":0.183,"15min":0.183},"max":{"1min":0.46000000000000002,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.34100000000000003,"5min":0.36799999999999999,"15min":0.37},"min":{"1min":0.22700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.46800000000000003,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.45600000000000002}]},{"osd":1,"last update":"Sat Aug 17 17:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.38300000000000001,"15min":0.39900000000000002},"min":{"1min":0.24099999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.48899999999999999,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.32400000000000001,"5min":0.373,"15min":0.39400000000000002},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.439,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.47499999999999998}]},{"osd":2,"last update":"Sat Aug 17 17:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.40300000000000002,"15min":0.42099999999999999},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.47899999999999998,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.439},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.377,"15min":0.38200000000000001},"min":{"1min":0.27500000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.55000000000000004,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.497}]},{"osd":4,"last update":"Sat Aug 17 17:21:59 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.502,"5min":0.52900000000000003,"15min":0.53500000000000003},"min":{"1min":0.38700000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.65700000000000003,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.65000000000000002}]},{"osd":5,"last update":"Sat Aug 17 17:22:25 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.54600000000000004,"15min":0.55000000000000004},"min":{"1min":0.42999999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73899999999999999,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55300000000000005,"15min":0.56499999999999995},"min":{"1min":0.41799999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.77000000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.70099999999999996}]},{"osd":6,"last update":"Sat Aug 17 17:22:44 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.55300000000000005,"15min":0.55800000000000005},"min":{"1min":0.42999999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.746,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.62},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.54700000000000004,"15min":0.54900000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.70499999999999996,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.67400000000000004}]},{"osd":7,"last update":"Sat Aug 17 17:22:07 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.78300000000000003,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55600000000000005,"15min":0.55800000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.73599999999999999,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.68799999999999994}]}]},{"osd":2,"up_from":18,"seq":77309411387,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298096,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447056,"statfs":{"total":95995035648,"available":95689785344,"internally_reserved":0,"allocated":757760,"data_stored":332667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:22:15 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.37,"15min":0.36699999999999999},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.47999999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.39900000000000002,"15min":0.40200000000000002},"min":{"1min":0.26200000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.49399999999999999,"5min":0.62,"15min":0.62},"last":0.56100000000000005}]},{"osd":1,"last update":"Sat Aug 17 17:22:15 2024","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.36199999999999999,"15min":0.35799999999999998},"min":{"1min":0.22,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.49099999999999999,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.38100000000000001,"15min":0.377},"min":{"1min":0.248,"5min":0.222,"15min":0.222},"max":{"1min":0.47399999999999998,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.497}]},{"osd":3,"last update":"Sat Aug 17 17:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.433,"15min":0.45000000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":1.0149999999999999,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.40000000000000002,"15min":0.42999999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.53200000000000003}]},{"osd":4,"last update":"Sat Aug 17 17:21:56 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.55000000000000004,"15min":0.55500000000000005},"min":{"1min":0.42299999999999999,"5min":0.379,"15min":0.379},"max":{"1min":0.77400000000000002,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56699999999999995,"15min":0.57099999999999995},"min":{"1min":0.40200000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.74299999999999999,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.79700000000000004}]},{"osd":5,"last update":"Sat Aug 17 17:22:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77800000000000002,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.64600000000000002}]},{"osd":6,"last update":"Sat Aug 17 17:22:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.56299999999999994,"15min":0.55300000000000005},"min":{"1min":0.39300000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.1779999999999999,"5min":1.1779999999999999,"15min":1.1779999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.76300000000000001}]},{"osd":7,"last update":"Sat Aug 17 17:22:08 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.56899999999999995,"15min":0.56399999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.79200000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.60299999999999998,"15min":0.60099999999999998},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.74299999999999999,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.78200000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298096,"kb_used_data":740,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447056,"statfs":{"total":95995035648,"available":95689785344,"internally_reserved":0,"allocated":757760,"data_stored":332743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 17:21:50 2024","interfaces":[{"interface":"back","average":{"1min":0.32400000000000001,"5min":0.34399999999999997,"15min":0.33500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.33800000000000002,"5min":0.371,"15min":0.35599999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.50800000000000001,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.56599999999999995}]},{"osd":2,"last update":"Sat Aug 17 17:22:17 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.36799999999999999,"15min":0.35999999999999999},"min":{"1min":0.251,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.41499999999999998,"15min":0.40500000000000003},"min":{"1min":0.28399999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.72399999999999998,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.51600000000000001}]},{"osd":3,"last update":"Sat Aug 17 17:22:41 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.43099999999999999,"15min":0.434},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.0549999999999999,"5min":1.0549999999999999,"15min":1.0549999999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.39200000000000002,"15min":0.40600000000000003},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.50600000000000001,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.53300000000000003}]},{"osd":4,"last update":"Sat Aug 17 17:21:57 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.57599999999999996,"15min":0.57199999999999995},"min":{"1min":0.39800000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.72799999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55300000000000005,"15min":0.54700000000000004},"min":{"1min":0.41399999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.66600000000000004,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.66500000000000004}]},{"osd":5,"last update":"Sat Aug 17 17:22:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54900000000000004},"min":{"1min":0.40899999999999997,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81399999999999995,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.59499999999999997,"15min":0.59899999999999998},"min":{"1min":0.42599999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.67100000000000004,"5min":1.0429999999999999,"15min":1.0429999999999999},"last":0.70499999999999996}]},{"osd":6,"last update":"Sat Aug 17 17:22:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.58999999999999997,"15min":0.61099999999999999},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.69499999999999995,"5min":0.879,"15min":0.879},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.59499999999999997,"15min":0.61599999999999999},"min":{"1min":0.41299999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.78700000000000003,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.75600000000000001}]},{"osd":7,"last update":"Sat Aug 17 17:22:07 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.60199999999999998,"15min":0.60499999999999998},"min":{"1min":0.434,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.745,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.58799999999999997,"15min":0.59699999999999998},"min":{"1min":0.39400000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.72699999999999998,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.76600000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298684,"kb_used_data":1324,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446468,"statfs":{"total":95995035648,"available":95689183232,"internally_reserved":0,"allocated":1355776,"data_stored":923684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 17:21:50 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.34699999999999998,"15min":0.32000000000000001},"min":{"1min":0.245,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.496,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.34899999999999998,"5min":0.35899999999999999,"15min":0.32200000000000001},"min":{"1min":0.253,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.45900000000000002,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.23300000000000001}]},{"osd":2,"last update":"Sat Aug 17 17:22:15 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.39700000000000002,"15min":0.38700000000000001},"min":{"1min":0.247,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.247},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.38900000000000001,"15min":0.375},"min":{"1min":0.27000000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.378}]},{"osd":3,"last update":"Sat Aug 17 17:22:38 2024","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.39200000000000002,"15min":0.41999999999999998},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.433,"15min":0.45200000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Aug 17 17:22:00 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.54400000000000004,"15min":0.55200000000000005},"min":{"1min":0.45700000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.72299999999999998,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55300000000000005,"15min":0.55900000000000005},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.52100000000000002}]},{"osd":5,"last update":"Sat Aug 17 17:22:21 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57099999999999995,"15min":0.57499999999999996},"min":{"1min":0.45400000000000001,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.40500000000000003,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.55900000000000005}]},{"osd":6,"last update":"Sat Aug 17 17:22:45 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.40600000000000003,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58199999999999996,"15min":0.57999999999999996},"min":{"1min":0.41599999999999998,"5min":0.374,"15min":0.374},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.54200000000000004}]},{"osd":7,"last update":"Sat Aug 17 17:22:06 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.52600000000000002,"15min":0.51800000000000002},"min":{"1min":0.43099999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.58599999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:22:54.049 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T17:22:54.049 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-17T17:22:54.049 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-17T17:22:54.049 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph health --format=json 2024-08-17T17:22:54.304 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:54.689 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:22:54 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:22:54] "GET /metrics HTTP/1.1" 200 239679 "" "Prometheus/2.43.0" 2024-08-17T17:22:54.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:22:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:22:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:22:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:55 smithi179 ceph-mon[34775]: from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:22:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:55 smithi179 ceph-mon[34775]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:55 smithi130 ceph-mon[31342]: from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:22:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:55 smithi130 ceph-mon[31342]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:55 smithi130 ceph-mon[42749]: from='client.24677 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:22:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:55 smithi130 ceph-mon[42749]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:56.203 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:22:56.203 INFO:teuthology.orchestra.run.smithi130.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-17T17:22:56.886 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-17T17:22:56.886 INFO:teuthology.run_tasks:Running task print... 2024-08-17T17:22:56.898 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-08-17T17:22:56.898 INFO:teuthology.run_tasks:Running task parallel... 2024-08-17T17:22:56.911 INFO:teuthology.task.parallel:starting parallel... 2024-08-17T17:22:56.911 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-17T17:22:56.912 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-08-17T17:22:56.917 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-08-17T17:22:56.917 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-08-17T17:22:56.917 DEBUG:teuthology.orchestra.run.smithi130:> stat -- /home/ubuntu/cephtest/mnt.0 2024-08-17T17:22:56.918 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-17T17:22:56.918 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-08-17T17:22:56.918 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi130.front.sepia.ceph.com 2024-08-17T17:22:56.919 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-08-17T17:22:56.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.935 INFO:teuthology.orchestra.run.smithi130.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-08-17T17:22:56.935 DEBUG:teuthology.orchestra.run.smithi130:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-08-17T17:22:56.990 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-08-17T17:22:56.990 DEBUG:teuthology.orchestra.run.smithi130:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-08-17T17:22:57.046 INFO:tasks.workunit:timeout=3h 2024-08-17T17:22:57.046 INFO:tasks.workunit:cleanup=True 2024-08-17T17:22:57.047 DEBUG:teuthology.orchestra.run.smithi130:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-08-17T17:22:57.103 INFO:tasks.workunit.client.0.smithi130.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-08-17T17:22:57.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:57 smithi130 ceph-mon[31342]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:57.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:57 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3069165738' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T17:22:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:57 smithi130 ceph-mon[42749]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:57 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3069165738' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T17:22:57.183 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:57 smithi179 ceph-mon[34775]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:57 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3069165738' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T17:22:59.267 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:59 smithi130 ceph-mon[31342]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:59.267 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:59 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1887017670' entity='client.admin' 2024-08-17T17:22:59.267 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:22:59 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:22:59.267 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:22:58 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:22:58] ENGINE Bus STOPPING 2024-08-17T17:22:59.268 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:59 smithi130 ceph-mon[42749]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:59.268 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:59 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1887017670' entity='client.admin' 2024-08-17T17:22:59.268 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:22:59 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:22:59.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:59 smithi179 ceph-mon[34775]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:22:59.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:59 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1887017670' entity='client.admin' 2024-08-17T17:22:59.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:22:59 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:22:59.529 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:22:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:22:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:22:59.529 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:22:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:22:59] ENGINE Bus STOPPED 2024-08-17T17:22:59.529 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:22:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:22:59] ENGINE Bus STARTING 2024-08-17T17:22:59.530 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:22:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:22:59] ENGINE Serving on http://:::9283 2024-08-17T17:22:59.530 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:22:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:22:59] ENGINE Bus STARTED 2024-08-17T17:22:59.693 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph config set global log_to_journald false --force' 2024-08-17T17:23:00.003 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:00.281 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:00 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:23:00.281 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:00 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:23:00.281 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:00 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:00.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:00 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:23:00.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:00 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:23:00.282 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:00 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:00.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:00 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:23:00.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:00 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:23:00.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:00 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:01 smithi130 ceph-mon[31342]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:01 smithi130 ceph-mon[42749]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:01 smithi179 ceph-mon[34775]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:02.677 DEBUG:teuthology.orchestra.run.smithi130:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-08-17T17:23:02.723 INFO:tasks.workunit.client.0.smithi130.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-08-17T17:23:02.725 INFO:tasks.workunit.client.0.smithi130.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-17T17:23:02.725 INFO:tasks.workunit.client.0.smithi130.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-08-17T17:23:03.058 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-08-17T17:23:03.314 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:03 smithi179 ceph-mon[34775]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:03.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:03 smithi130 ceph-mon[42749]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:03.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:03 smithi130 ceph-mon[31342]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:04.284 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:04 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:23:04.284 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:04 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:23:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:04 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:23:04.448 INFO:tasks.workunit.client.0.smithi130.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-08-17T17:23:04.549 INFO:tasks.workunit.client.0.smithi130.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-08-17T17:23:04.590 INFO:tasks.workunit.client.0.smithi130.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-17T17:23:04.591 INFO:tasks.workunit.client.0.smithi130.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-17T17:23:04.591 INFO:tasks.workunit.client.0.smithi130.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-08-17T17:23:04.631 INFO:tasks.workunit.client.0.smithi130.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-17T17:23:04.634 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T17:23:04.634 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-08-17T17:23:04.650 INFO:tasks.workunit:Running workunits matching rbd/test_librbd.sh on client.0... 2024-08-17T17:23:04.652 INFO:tasks.workunit:Running workunit rbd/test_librbd.sh... 2024-08-17T17:23:04.652 DEBUG:teuthology.orchestra.run.smithi130:workunit test rbd/test_librbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd.sh 2024-08-17T17:23:04.666 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:23:04 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:23:04] "GET /metrics HTTP/1.1" 200 239661 "" "Prometheus/2.43.0" 2024-08-17T17:23:04.788 INFO:tasks.workunit.client.0.smithi130.stderr:warning: failed to enable lockdep 2024-08-17T17:23:04.788 INFO:tasks.workunit.client.0.smithi130.stdout:seed 87148 2024-08-17T17:23:04.788 INFO:tasks.workunit.client.0.smithi130.stdout:[==========] Running 425 tests from 16 test suites. 2024-08-17T17:23:04.788 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] Global test environment set-up. 2024-08-17T17:23:04.789 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 1 test from TestJournalStress 2024-08-17T17:23:04.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:23:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:23:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:23:05.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:05 smithi179 ceph-mon[34775]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:05.384 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:05 smithi130 ceph-mon[31342]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:05.385 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:05 smithi130 ceph-mon[42749]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:05.535 INFO:teuthology.orchestra.run.smithi130.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:23:06.131 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalStress.DiscardWithPruneWriteOverlap 2024-08-17T17:23:06.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[42749]: osdmap e74: 8 total, 8 up, 8 in 2024-08-17T17:23:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1552278495' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:23:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:23:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:23:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[31342]: osdmap e74: 8 total, 8 up, 8 in 2024-08-17T17:23:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1552278495' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:06.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:23:06.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:23:06.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:23:06.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:06 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:06.291 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch ps' 2024-08-17T17:23:06.551 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:06 smithi179 ceph-mon[34775]: osdmap e74: 8 total, 8 up, 8 in 2024-08-17T17:23:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:06 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1552278495' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:06 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:06 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:06.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:06 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:23:06.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:06 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:23:06.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:06 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:23:06.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:06 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[42749]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[42749]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:23:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[42749]: pgmap v94: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[42749]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:23:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[42749]: osdmap e75: 8 total, 8 up, 8 in 2024-08-17T17:23:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[42749]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[42749]: osdmap e76: 8 total, 8 up, 8 in 2024-08-17T17:23:07.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[31342]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:07.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[31342]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:23:07.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[31342]: pgmap v94: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:07.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[31342]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:23:07.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:07.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[31342]: osdmap e75: 8 total, 8 up, 8 in 2024-08-17T17:23:07.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[31342]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:07.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:07 smithi130 ceph-mon[31342]: osdmap e76: 8 total, 8 up, 8 in 2024-08-17T17:23:07.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:07 smithi179 ceph-mon[34775]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:07.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:07 smithi179 ceph-mon[34775]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:23:07.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:07 smithi179 ceph-mon[34775]: pgmap v94: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:07.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:07 smithi179 ceph-mon[34775]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:23:07.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:07 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:07.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:07 smithi179 ceph-mon[34775]: osdmap e75: 8 total, 8 up, 8 in 2024-08-17T17:23:07.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:07 smithi179 ceph-mon[34775]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:07.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:07 smithi179 ceph-mon[34775]: osdmap e76: 8 total, 8 up, 8 in 2024-08-17T17:23:08.140 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:08.140 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalStress.DiscardWithPruneWriteOverlap (2009 ms) 2024-08-17T17:23:08.370 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:08 smithi130 ceph-mon[31342]: pgmap v97: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:08.370 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:08 smithi130 ceph-mon[31342]: osdmap e77: 8 total, 8 up, 8 in 2024-08-17T17:23:08.370 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:08 smithi130 ceph-mon[42749]: pgmap v97: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:08.370 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:08 smithi130 ceph-mon[42749]: osdmap e77: 8 total, 8 up, 8 in 2024-08-17T17:23:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:08 smithi179 ceph-mon[34775]: pgmap v97: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:08 smithi179 ceph-mon[34775]: osdmap e77: 8 total, 8 up, 8 in 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (83s) 39s ago 2m 15.6M - 0.25.0 c8568f914cd2 434453bf0910 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi179 *:3000 running (2m) 56s ago 2m 81.1M - 9.4.7 954c08fa6188 8ed57a40b09e 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foo.smithi130.jmaidg smithi130 running (68s) 39s ago 67s 15.1M - 17.2.7-1404-gf2c39d4d f69471c56189 91d7d1762094 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foo.smithi179.zkkvno smithi179 running (65s) 56s ago 65s 11.5M - 17.2.7-1404-gf2c39d4d f69471c56189 0486517d425a 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi179 *:8443 running (6m) 56s ago 6m 399M - 17.2.7-1404-gf2c39d4d f69471c56189 fdc68f217f77 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:9283,8765 running (7m) 39s ago 7m 443M - 17.2.7-1404-gf2c39d4d f69471c56189 5e2d324fc1a5 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (7m) 39s ago 7m 42.8M 2048M 17.2.7-1404-gf2c39d4d f69471c56189 7a678d70ca3b 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi179 running (6m) 56s ago 6m 36.9M 2048M 17.2.7-1404-gf2c39d4d f69471c56189 c29492fa5296 2024-08-17T17:23:08.637 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (6m) 39s ago 6m 36.7M 2048M 17.2.7-1404-gf2c39d4d f69471c56189 0c43a998b5fa 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (105s) 39s ago 104s 14.1M - 1.5.0 0da6a335fe13 a24247d0fd94 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi179 *:9100 running (101s) 56s ago 101s 8472k - 1.5.0 0da6a335fe13 ed661fd8fe94 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (5m) 39s ago 5m 43.2M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 293583e22b78 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (5m) 39s ago 5m 42.8M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 1dc0b4634e15 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (5m) 39s ago 5m 40.2M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 d72289cb8b72 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (4m) 39s ago 4m 41.5M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 1509c2840516 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi179 running (4m) 56s ago 4m 41.0M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 254dc0763420 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi179 running (3m) 56s ago 3m 40.1M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 078704aadcd7 2024-08-17T17:23:08.638 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi179 running (3m) 56s ago 3m 39.3M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 ab570ba67ae7 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi179 running (3m) 56s ago 3m 43.5M 4016M 17.2.7-1404-gf2c39d4d f69471c56189 99ae6a861fa0 2024-08-17T17:23:08.639 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi179 *:9095 running (78s) 56s ago 2m 32.4M - 2.43.0 a07b618ecd1d ac95b2aef275 2024-08-17T17:23:09.147 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 1 test from TestJournalStress (2009 ms total) 2024-08-17T17:23:09.147 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:23:09.147 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 16 tests from TestJournalReplay 2024-08-17T17:23:09.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:09 smithi130 ceph-mon[31342]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:23:09.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:09 smithi130 ceph-mon[31342]: Cluster is now healthy 2024-08-17T17:23:09.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:09 smithi130 ceph-mon[31342]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:09 smithi130 ceph-mon[42749]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:23:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:09 smithi130 ceph-mon[42749]: Cluster is now healthy 2024-08-17T17:23:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:09 smithi130 ceph-mon[42749]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:09.470 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for minority of mons to upgrade' 2024-08-17T17:23:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:09 smithi179 ceph-mon[34775]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:23:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:09 smithi179 ceph-mon[34775]: Cluster is now healthy 2024-08-17T17:23:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:09 smithi179 ceph-mon[34775]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:09.751 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:10.395 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:10 smithi130 ceph-mon[31342]: osdmap e78: 8 total, 8 up, 8 in 2024-08-17T17:23:10.396 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:10 smithi130 ceph-mon[31342]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:10.396 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:10 smithi130 ceph-mon[42749]: osdmap e78: 8 total, 8 up, 8 in 2024-08-17T17:23:10.396 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:10 smithi130 ceph-mon[42749]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:10 smithi179 ceph-mon[34775]: osdmap e78: 8 total, 8 up, 8 in 2024-08-17T17:23:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:10 smithi179 ceph-mon[34775]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:11.166 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.AioDiscardEvent 2024-08-17T17:23:11.330 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:11 smithi130 ceph-mon[31342]: osdmap e79: 8 total, 8 up, 8 in 2024-08-17T17:23:11.330 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:11 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1054648230' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:11.330 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:11 smithi130 ceph-mon[42749]: osdmap e79: 8 total, 8 up, 8 in 2024-08-17T17:23:11.330 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:11 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1054648230' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:11.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:11 smithi179 ceph-mon[34775]: osdmap e79: 8 total, 8 up, 8 in 2024-08-17T17:23:11.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:11 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1054648230' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:11.562 INFO:teuthology.orchestra.run.smithi130.stdout:wait for minority of mons to upgrade 2024-08-17T17:23:12.326 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:12 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1054648230' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:12.326 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:12 smithi130 ceph-mon[31342]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T17:23:12.326 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:12 smithi130 ceph-mon[31342]: pgmap v103: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:12.326 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:12 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1054648230' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:12.326 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:12 smithi130 ceph-mon[42749]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T17:23:12.326 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:12 smithi130 ceph-mon[42749]: pgmap v103: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:12.350 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ! ceph mon versions | grep $sha1 && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-08-17T17:23:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:12 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1054648230' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:12 smithi179 ceph-mon[34775]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T17:23:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:12 smithi179 ceph-mon[34775]: pgmap v103: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:12.640 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:13.186 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.186 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.AioDiscardEvent (2020 ms) 2024-08-17T17:23:13.186 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.AioWriteEvent 2024-08-17T17:23:13.195 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.195 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.AioWriteEvent (10 ms) 2024-08-17T17:23:13.195 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.AioFlushEvent 2024-08-17T17:23:13.203 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.203 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.AioFlushEvent (7 ms) 2024-08-17T17:23:13.203 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.SnapCreate 2024-08-17T17:23:13.211 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.212 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.SnapCreate (9 ms) 2024-08-17T17:23:13.212 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.SnapProtect 2024-08-17T17:23:13.219 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.219 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.SnapProtect (7 ms) 2024-08-17T17:23:13.219 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.SnapUnprotect 2024-08-17T17:23:13.226 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.226 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.SnapUnprotect (7 ms) 2024-08-17T17:23:13.226 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.SnapRename 2024-08-17T17:23:13.233 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.233 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.SnapRename (8 ms) 2024-08-17T17:23:13.233 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.SnapRollback 2024-08-17T17:23:13.240 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.240 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.SnapRollback (7 ms) 2024-08-17T17:23:13.241 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.SnapRemove 2024-08-17T17:23:13.247 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.247 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.SnapRemove (7 ms) 2024-08-17T17:23:13.248 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.Rename 2024-08-17T17:23:13.254 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.254 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.Rename (7 ms) 2024-08-17T17:23:13.254 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.Resize 2024-08-17T17:23:13.262 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.262 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.Resize (7 ms) 2024-08-17T17:23:13.262 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.Flatten 2024-08-17T17:23:13.269 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.269 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.Flatten (7 ms) 2024-08-17T17:23:13.269 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.UpdateFeatures 2024-08-17T17:23:13.276 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.276 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.UpdateFeatures (7 ms) 2024-08-17T17:23:13.276 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.MetadataSet 2024-08-17T17:23:13.283 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.283 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.MetadataSet (7 ms) 2024-08-17T17:23:13.283 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.MetadataRemove 2024-08-17T17:23:13.290 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.290 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.MetadataRemove (7 ms) 2024-08-17T17:23:13.290 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalReplay.ObjectPosition 2024-08-17T17:23:13.297 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:13.297 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalReplay.ObjectPosition (7 ms) 2024-08-17T17:23:13.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:13 smithi130 ceph-mon[31342]: osdmap e81: 8 total, 8 up, 8 in 2024-08-17T17:23:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:13 smithi130 ceph-mon[42749]: osdmap e81: 8 total, 8 up, 8 in 2024-08-17T17:23:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:13 smithi179 ceph-mon[34775]: osdmap e81: 8 total, 8 up, 8 in 2024-08-17T17:23:14.185 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 16 tests from TestJournalReplay (2131 ms total) 2024-08-17T17:23:14.186 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:23:14.186 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 4 tests from TestJournalEntries 2024-08-17T17:23:14.480 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:14 smithi130 ceph-mon[31342]: osdmap e82: 8 total, 8 up, 8 in 2024-08-17T17:23:14.480 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:14 smithi130 ceph-mon[31342]: pgmap v106: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:14.481 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:23:14 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:23:14] "GET /metrics HTTP/1.1" 200 239661 "" "Prometheus/2.43.0" 2024-08-17T17:23:14.481 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:14 smithi130 ceph-mon[42749]: osdmap e82: 8 total, 8 up, 8 in 2024-08-17T17:23:14.481 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:14 smithi130 ceph-mon[42749]: pgmap v106: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:14.582 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:14 smithi179 ceph-mon[34775]: osdmap e82: 8 total, 8 up, 8 in 2024-08-17T17:23:14.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:14 smithi179 ceph-mon[34775]: pgmap v106: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:14.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:23:14 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:23:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:23:15.131 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:15.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:15 smithi130 ceph-mon[31342]: osdmap e83: 8 total, 8 up, 8 in 2024-08-17T17:23:15.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:15 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/4185503947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:15.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:15 smithi130 ceph-mon[42749]: osdmap e83: 8 total, 8 up, 8 in 2024-08-17T17:23:15.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:15 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/4185503947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:15.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:15 smithi179 ceph-mon[34775]: osdmap e83: 8 total, 8 up, 8 in 2024-08-17T17:23:15.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:15 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/4185503947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:16.201 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalEntries.AioWrite 2024-08-17T17:23:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:16 smithi179 ceph-mon[34775]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:16 smithi179 ceph-mon[34775]: osdmap e84: 8 total, 8 up, 8 in 2024-08-17T17:23:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:16 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1464748964' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:16 smithi179 ceph-mon[34775]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:16 smithi179 ceph-mon[34775]: pgmap v109: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:16 smithi130 ceph-mon[31342]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:16 smithi130 ceph-mon[31342]: osdmap e84: 8 total, 8 up, 8 in 2024-08-17T17:23:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:16 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1464748964' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:16 smithi130 ceph-mon[31342]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:16 smithi130 ceph-mon[31342]: pgmap v109: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:16 smithi130 ceph-mon[42749]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:16 smithi130 ceph-mon[42749]: osdmap e84: 8 total, 8 up, 8 in 2024-08-17T17:23:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:16 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1464748964' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:16 smithi130 ceph-mon[42749]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:16 smithi130 ceph-mon[42749]: pgmap v109: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:17 smithi179 ceph-mon[34775]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:17 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1464748964' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:17 smithi179 ceph-mon[34775]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:23:17.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:17 smithi130 ceph-mon[31342]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:17 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1464748964' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:17 smithi130 ceph-mon[31342]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:23:17.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:17 smithi130 ceph-mon[42749]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:17.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:17 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1464748964' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:17.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:17 smithi130 ceph-mon[42749]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T17:23:18.070 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:18.219 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:18.220 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalEntries.AioWrite (2018 ms) 2024-08-17T17:23:18.220 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalEntries.AioDiscard 2024-08-17T17:23:18.228 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:18.228 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalEntries.AioDiscard (9 ms) 2024-08-17T17:23:18.228 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalEntries.AioDiscardWithPrune 2024-08-17T17:23:18.237 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:18.237 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalEntries.AioDiscardWithPrune (9 ms) 2024-08-17T17:23:18.237 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestJournalEntries.AioFlush 2024-08-17T17:23:18.244 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:18.244 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestJournalEntries.AioFlush (8 ms) 2024-08-17T17:23:18.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:18 smithi130 ceph-mon[31342]: osdmap e86: 8 total, 8 up, 8 in 2024-08-17T17:23:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:18 smithi130 ceph-mon[31342]: pgmap v112: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:18 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1727063768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:18 smithi130 ceph-mon[31342]: from='client.24779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:18 smithi130 ceph-mon[42749]: osdmap e86: 8 total, 8 up, 8 in 2024-08-17T17:23:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:18 smithi130 ceph-mon[42749]: pgmap v112: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:18 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1727063768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:18 smithi130 ceph-mon[42749]: from='client.24779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:18 smithi179 ceph-mon[34775]: osdmap e86: 8 total, 8 up, 8 in 2024-08-17T17:23:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:18 smithi179 ceph-mon[34775]: pgmap v112: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:18 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1727063768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:18 smithi179 ceph-mon[34775]: from='client.24779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:19.220 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 4 tests from TestJournalEntries (2044 ms total) 2024-08-17T17:23:19.220 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:23:19.220 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 2 tests from TestTrash 2024-08-17T17:23:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:19 smithi179 ceph-mon[34775]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:23:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:19 smithi179 ceph-mon[34775]: Cluster is now healthy 2024-08-17T17:23:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:19 smithi179 ceph-mon[34775]: osdmap e87: 8 total, 8 up, 8 in 2024-08-17T17:23:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:19 smithi179 ceph-mon[34775]: from='client.24785 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:19 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:23:19.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:19 smithi130 ceph-mon[31342]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:23:19.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:19 smithi130 ceph-mon[31342]: Cluster is now healthy 2024-08-17T17:23:19.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:19 smithi130 ceph-mon[31342]: osdmap e87: 8 total, 8 up, 8 in 2024-08-17T17:23:19.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:19 smithi130 ceph-mon[31342]: from='client.24785 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:19.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:19 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:23:19.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:19 smithi130 ceph-mon[42749]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:23:19.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:19 smithi130 ceph-mon[42749]: Cluster is now healthy 2024-08-17T17:23:19.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:19 smithi130 ceph-mon[42749]: osdmap e87: 8 total, 8 up, 8 in 2024-08-17T17:23:19.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:19 smithi130 ceph-mon[42749]: from='client.24785 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:19.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:19 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:23:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:20 smithi179 ceph-mon[34775]: osdmap e88: 8 total, 8 up, 8 in 2024-08-17T17:23:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:20 smithi179 ceph-mon[34775]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:20.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:20 smithi130 ceph-mon[31342]: osdmap e88: 8 total, 8 up, 8 in 2024-08-17T17:23:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:20 smithi130 ceph-mon[31342]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:20.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:20 smithi130 ceph-mon[42749]: osdmap e88: 8 total, 8 up, 8 in 2024-08-17T17:23:20.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:20 smithi130 ceph-mon[42749]: pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:21.056 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:21.240 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestTrash.UserRemovingSource 2024-08-17T17:23:21.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:21 smithi130 ceph-mon[31342]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T17:23:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:21 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1647643211' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:21 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1277161510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:21 smithi130 ceph-mon[31342]: from='client.24809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:21 smithi130 ceph-mon[42749]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T17:23:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:21 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1647643211' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:21 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1277161510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:21 smithi130 ceph-mon[42749]: from='client.24809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:21 smithi179 ceph-mon[34775]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T17:23:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:21 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1647643211' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:21 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1277161510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:21 smithi179 ceph-mon[34775]: from='client.24809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:22.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:22 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1647643211' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:22.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:22 smithi179 ceph-mon[34775]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:23:22.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:22 smithi179 ceph-mon[34775]: from='client.14913 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:22.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:22 smithi179 ceph-mon[34775]: pgmap v118: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:22.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:22 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1647643211' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:22.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:22 smithi130 ceph-mon[42749]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:23:22.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:22 smithi130 ceph-mon[42749]: from='client.14913 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:22.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:22 smithi130 ceph-mon[42749]: pgmap v118: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:22.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:22 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1647643211' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:22.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:22 smithi130 ceph-mon[31342]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T17:23:22.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:22 smithi130 ceph-mon[31342]: from='client.14913 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:22.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:22 smithi130 ceph-mon[31342]: pgmap v118: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:23.311 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestTrash.UserRemovingSource (2071 ms) 2024-08-17T17:23:23.311 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestTrash.RestoreMirroringSource 2024-08-17T17:23:23.348 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestTrash.RestoreMirroringSource (37 ms) 2024-08-17T17:23:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:23 smithi179 ceph-mon[34775]: osdmap e91: 8 total, 8 up, 8 in 2024-08-17T17:23:23.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:23 smithi130 ceph-mon[31342]: osdmap e91: 8 total, 8 up, 8 in 2024-08-17T17:23:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:23 smithi130 ceph-mon[42749]: osdmap e91: 8 total, 8 up, 8 in 2024-08-17T17:23:24.001 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:24.254 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 2 tests from TestTrash (2108 ms total) 2024-08-17T17:23:24.254 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:23:24.254 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 1 test from TestOperations 2024-08-17T17:23:24.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:24 smithi130 ceph-mon[31342]: osdmap e92: 8 total, 8 up, 8 in 2024-08-17T17:23:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:24 smithi130 ceph-mon[31342]: pgmap v121: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:24 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2141942478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:24 smithi130 ceph-mon[31342]: from='client.14925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:24 smithi130 ceph-mon[42749]: osdmap e92: 8 total, 8 up, 8 in 2024-08-17T17:23:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:24 smithi130 ceph-mon[42749]: pgmap v121: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:24 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2141942478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:24 smithi130 ceph-mon[42749]: from='client.14925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:24.582 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:24 smithi179 ceph-mon[34775]: osdmap e92: 8 total, 8 up, 8 in 2024-08-17T17:23:24.582 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:24 smithi179 ceph-mon[34775]: pgmap v121: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:24.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:24 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2141942478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:24.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:24 smithi179 ceph-mon[34775]: from='client.14925 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:24.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:23:24 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:23:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:23:24.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:23:24 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:23:24] "GET /metrics HTTP/1.1" 200 242414 "" "Prometheus/2.43.0" 2024-08-17T17:23:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:25 smithi179 ceph-mon[34775]: osdmap e93: 8 total, 8 up, 8 in 2024-08-17T17:23:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:25 smithi179 ceph-mon[34775]: from='client.14931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:25.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:25 smithi130 ceph-mon[31342]: osdmap e93: 8 total, 8 up, 8 in 2024-08-17T17:23:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:25 smithi130 ceph-mon[31342]: from='client.14931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:25 smithi130 ceph-mon[42749]: osdmap e93: 8 total, 8 up, 8 in 2024-08-17T17:23:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:25 smithi130 ceph-mon[42749]: from='client.14931 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:26.267 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestOperations.DisableJournalingCorrupt 2024-08-17T17:23:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:26 smithi179 ceph-mon[34775]: osdmap e94: 8 total, 8 up, 8 in 2024-08-17T17:23:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:26 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/284193885' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:26 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:26 smithi179 ceph-mon[34775]: pgmap v124: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:26.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:26 smithi130 ceph-mon[42749]: osdmap e94: 8 total, 8 up, 8 in 2024-08-17T17:23:26.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:26 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/284193885' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:26.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:26 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:26.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:26 smithi130 ceph-mon[42749]: pgmap v124: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:26.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:26 smithi130 ceph-mon[31342]: osdmap e94: 8 total, 8 up, 8 in 2024-08-17T17:23:26.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:26 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/284193885' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:26.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:26 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:26.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:26 smithi130 ceph-mon[31342]: pgmap v124: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:26.904 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:27 smithi179 ceph-mon[34775]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:27 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:27 smithi179 ceph-mon[34775]: osdmap e95: 8 total, 8 up, 8 in 2024-08-17T17:23:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:27 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2004336846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:27 smithi179 ceph-mon[34775]: from='client.24857 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:27.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:27 smithi130 ceph-mon[31342]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:27 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:27 smithi130 ceph-mon[31342]: osdmap e95: 8 total, 8 up, 8 in 2024-08-17T17:23:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:27 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2004336846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:27 smithi130 ceph-mon[31342]: from='client.24857 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:27 smithi130 ceph-mon[42749]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:27 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:27 smithi130 ceph-mon[42749]: osdmap e95: 8 total, 8 up, 8 in 2024-08-17T17:23:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:27 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2004336846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:27.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:27 smithi130 ceph-mon[42749]: from='client.24857 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:28.283 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:28.283 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestOperations.DisableJournalingCorrupt (2016 ms) 2024-08-17T17:23:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:28 smithi179 ceph-mon[34775]: from='client.24863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:28 smithi179 ceph-mon[34775]: osdmap e96: 8 total, 8 up, 8 in 2024-08-17T17:23:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:28 smithi179 ceph-mon[34775]: pgmap v127: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:28.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:28 smithi130 ceph-mon[42749]: from='client.24863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:28 smithi130 ceph-mon[42749]: osdmap e96: 8 total, 8 up, 8 in 2024-08-17T17:23:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:28 smithi130 ceph-mon[42749]: pgmap v127: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:28 smithi130 ceph-mon[31342]: from='client.24863 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:28 smithi130 ceph-mon[31342]: osdmap e96: 8 total, 8 up, 8 in 2024-08-17T17:23:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:28 smithi130 ceph-mon[31342]: pgmap v127: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:29.276 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 1 test from TestOperations (2016 ms total) 2024-08-17T17:23:29.276 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:23:29.277 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 4 tests from TestObjectMap 2024-08-17T17:23:29.567 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:29 smithi130 ceph-mon[31342]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:23:29.567 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:29 smithi130 ceph-mon[31342]: Cluster is now healthy 2024-08-17T17:23:29.568 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:29 smithi130 ceph-mon[31342]: osdmap e97: 8 total, 8 up, 8 in 2024-08-17T17:23:29.568 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:29 smithi130 ceph-mon[42749]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:23:29.568 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:29 smithi130 ceph-mon[42749]: Cluster is now healthy 2024-08-17T17:23:29.568 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:29 smithi130 ceph-mon[42749]: osdmap e97: 8 total, 8 up, 8 in 2024-08-17T17:23:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:29 smithi179 ceph-mon[34775]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:23:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:29 smithi179 ceph-mon[34775]: Cluster is now healthy 2024-08-17T17:23:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:29 smithi179 ceph-mon[34775]: osdmap e97: 8 total, 8 up, 8 in 2024-08-17T17:23:29.922 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:30.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:30 smithi130 ceph-mon[31342]: osdmap e98: 8 total, 8 up, 8 in 2024-08-17T17:23:30.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:30 smithi130 ceph-mon[31342]: pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:30.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:30 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3463471715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:30.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:30 smithi130 ceph-mon[31342]: from='client.24887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:30.578 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:30 smithi130 ceph-mon[42749]: osdmap e98: 8 total, 8 up, 8 in 2024-08-17T17:23:30.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:30 smithi130 ceph-mon[42749]: pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:30.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:30 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3463471715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:30.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:30 smithi130 ceph-mon[42749]: from='client.24887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:30 smithi179 ceph-mon[34775]: osdmap e98: 8 total, 8 up, 8 in 2024-08-17T17:23:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:30 smithi179 ceph-mon[34775]: pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:30 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3463471715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:30 smithi179 ceph-mon[34775]: from='client.24887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:31.314 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenCorrupt 2024-08-17T17:23:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:31 smithi179 ceph-mon[34775]: from='client.24893 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:31 smithi179 ceph-mon[34775]: osdmap e99: 8 total, 8 up, 8 in 2024-08-17T17:23:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:31 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2958192542' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:31 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:31 smithi130 ceph-mon[31342]: from='client.24893 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:31 smithi130 ceph-mon[31342]: osdmap e99: 8 total, 8 up, 8 in 2024-08-17T17:23:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:31 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2958192542' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:31 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:31 smithi130 ceph-mon[42749]: from='client.24893 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:31 smithi130 ceph-mon[42749]: osdmap e99: 8 total, 8 up, 8 in 2024-08-17T17:23:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:31 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2958192542' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:31.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:31 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:32.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:32 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:32.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:32 smithi130 ceph-mon[31342]: osdmap e100: 8 total, 8 up, 8 in 2024-08-17T17:23:32.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:32 smithi130 ceph-mon[31342]: pgmap v133: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:32 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:32 smithi179 ceph-mon[34775]: osdmap e100: 8 total, 8 up, 8 in 2024-08-17T17:23:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:32 smithi179 ceph-mon[34775]: pgmap v133: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:32.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:32 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:32 smithi130 ceph-mon[42749]: osdmap e100: 8 total, 8 up, 8 in 2024-08-17T17:23:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:32 smithi130 ceph-mon[42749]: pgmap v133: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:32.869 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:33.339 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenCorrupt (2026 ms) 2024-08-17T17:23:33.339 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenTooSmall 2024-08-17T17:23:33.373 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenTooSmall (34 ms) 2024-08-17T17:23:33.373 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestObjectMap.InvalidateFlagOnDisk 2024-08-17T17:23:33.410 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestObjectMap.InvalidateFlagOnDisk (37 ms) 2024-08-17T17:23:33.410 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall 2024-08-17T17:23:33.440 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall (30 ms) 2024-08-17T17:23:33.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:33 smithi179 ceph-mon[34775]: osdmap e101: 8 total, 8 up, 8 in 2024-08-17T17:23:33.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:33 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2238369443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:33.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:33 smithi179 ceph-mon[34775]: from='client.24905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:33.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:33 smithi130 ceph-mon[31342]: osdmap e101: 8 total, 8 up, 8 in 2024-08-17T17:23:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:33 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2238369443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:33 smithi130 ceph-mon[31342]: from='client.24905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:33 smithi130 ceph-mon[42749]: osdmap e101: 8 total, 8 up, 8 in 2024-08-17T17:23:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:33 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2238369443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:33 smithi130 ceph-mon[42749]: from='client.24905 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:34.321 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 4 tests from TestObjectMap (2127 ms total) 2024-08-17T17:23:34.321 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:23:34.321 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 2 tests from TestMirroringWatcher 2024-08-17T17:23:34.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:34 smithi179 ceph-mon[34775]: from='client.15009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:34.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:34 smithi179 ceph-mon[34775]: osdmap e102: 8 total, 8 up, 8 in 2024-08-17T17:23:34.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:34 smithi179 ceph-mon[34775]: pgmap v136: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:34.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:34 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:23:34.666 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:23:34 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:23:34] "GET /metrics HTTP/1.1" 200 242671 "" "Prometheus/2.43.0" 2024-08-17T17:23:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:34 smithi130 ceph-mon[42749]: from='client.15009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:34 smithi130 ceph-mon[42749]: osdmap e102: 8 total, 8 up, 8 in 2024-08-17T17:23:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:34 smithi130 ceph-mon[42749]: pgmap v136: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:34 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:23:34.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:34 smithi130 ceph-mon[31342]: from='client.15009 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:34.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:34 smithi130 ceph-mon[31342]: osdmap e102: 8 total, 8 up, 8 in 2024-08-17T17:23:34.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:34 smithi130 ceph-mon[31342]: pgmap v136: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:34.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:34 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:23:34.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:23:34 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:23:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:23:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:35 smithi179 ceph-mon[34775]: osdmap e103: 8 total, 8 up, 8 in 2024-08-17T17:23:35.641 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:35 smithi130 ceph-mon[31342]: osdmap e103: 8 total, 8 up, 8 in 2024-08-17T17:23:35.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:35 smithi130 ceph-mon[42749]: osdmap e103: 8 total, 8 up, 8 in 2024-08-17T17:23:35.768 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:36.329 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroringWatcher.ModeUpdated 2024-08-17T17:23:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:36 smithi179 ceph-mon[34775]: osdmap e104: 8 total, 8 up, 8 in 2024-08-17T17:23:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:36 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/381854872' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:36 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:36 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2304464358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:36 smithi179 ceph-mon[34775]: pgmap v139: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:36 smithi179 ceph-mon[34775]: from='client.15033 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:36 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:36 smithi179 ceph-mon[34775]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (unknown) 2024-08-17T17:23:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:36 smithi179 ceph-mon[34775]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:23:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:36 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:23:36.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:36 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:36.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:36 smithi179 ceph-mon[34775]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-17T17:23:36.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:36 smithi179 ceph-mon[34775]: from='client.24941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[31342]: osdmap e104: 8 total, 8 up, 8 in 2024-08-17T17:23:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/381854872' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2304464358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[31342]: pgmap v139: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[31342]: from='client.15033 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[31342]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (unknown) 2024-08-17T17:23:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[31342]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:23:36.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:23:36.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:36.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[31342]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-17T17:23:36.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[31342]: from='client.24941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:36.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[42749]: osdmap e104: 8 total, 8 up, 8 in 2024-08-17T17:23:36.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/381854872' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:36.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:36.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2304464358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:36.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[42749]: pgmap v139: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:36.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[42749]: from='client.15033 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:36.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:36.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[42749]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (unknown) 2024-08-17T17:23:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[42749]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:23:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:23:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[42749]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-17T17:23:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:36 smithi130 ceph-mon[42749]: from='client.24941 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:37 smithi179 ceph-mon[34775]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d on smithi179 2024-08-17T17:23:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:37 smithi179 ceph-mon[34775]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:37 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:37 smithi179 ceph-mon[34775]: osdmap e105: 8 total, 8 up, 8 in 2024-08-17T17:23:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:37 smithi130 ceph-mon[31342]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d on smithi179 2024-08-17T17:23:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:37 smithi130 ceph-mon[31342]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:37 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:37 smithi130 ceph-mon[31342]: osdmap e105: 8 total, 8 up, 8 in 2024-08-17T17:23:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:37 smithi130 ceph-mon[42749]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d on smithi179 2024-08-17T17:23:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:37 smithi130 ceph-mon[42749]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:37 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:37 smithi130 ceph-mon[42749]: osdmap e105: 8 total, 8 up, 8 in 2024-08-17T17:23:38.361 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroringWatcher.ModeUpdated (2032 ms) 2024-08-17T17:23:38.361 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroringWatcher.ImageStatusUpdated 2024-08-17T17:23:38.381 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroringWatcher.ImageStatusUpdated (20 ms) 2024-08-17T17:23:38.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:38 smithi179 ceph-mon[34775]: osdmap e106: 8 total, 8 up, 8 in 2024-08-17T17:23:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:38 smithi179 ceph-mon[34775]: pgmap v142: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:38.619 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:38.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:38 smithi130 ceph-mon[31342]: osdmap e106: 8 total, 8 up, 8 in 2024-08-17T17:23:38.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:38 smithi130 ceph-mon[31342]: pgmap v142: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:38.620 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:38 smithi130 ceph-mon[42749]: osdmap e106: 8 total, 8 up, 8 in 2024-08-17T17:23:38.620 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:38 smithi130 ceph-mon[42749]: pgmap v142: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:39.351 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 2 tests from TestMirroringWatcher (2052 ms total) 2024-08-17T17:23:39.351 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:23:39.352 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 57 tests from TestMigration 2024-08-17T17:23:39.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:39 smithi179 ceph-mon[34775]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:23:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:39 smithi179 ceph-mon[34775]: Cluster is now healthy 2024-08-17T17:23:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:39 smithi179 ceph-mon[34775]: osdmap e107: 8 total, 8 up, 8 in 2024-08-17T17:23:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:39 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1570303077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:39 smithi179 ceph-mon[34775]: from='client.15051 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:39 smithi179 ceph-mon[34775]: from='client.24959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:39.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:39 smithi130 ceph-mon[31342]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:23:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:39 smithi130 ceph-mon[31342]: Cluster is now healthy 2024-08-17T17:23:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:39 smithi130 ceph-mon[31342]: osdmap e107: 8 total, 8 up, 8 in 2024-08-17T17:23:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:39 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1570303077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:39 smithi130 ceph-mon[31342]: from='client.15051 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:39 smithi130 ceph-mon[31342]: from='client.24959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:39 smithi130 ceph-mon[42749]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:23:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:39 smithi130 ceph-mon[42749]: Cluster is now healthy 2024-08-17T17:23:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:39 smithi130 ceph-mon[42749]: osdmap e107: 8 total, 8 up, 8 in 2024-08-17T17:23:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:39 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1570303077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:39 smithi130 ceph-mon[42749]: from='client.15051 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:39 smithi130 ceph-mon[42749]: from='client.24959 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:40 smithi130 ceph-mon[31342]: osdmap e108: 8 total, 8 up, 8 in 2024-08-17T17:23:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:40 smithi130 ceph-mon[31342]: pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:40 smithi130 ceph-mon[42749]: osdmap e108: 8 total, 8 up, 8 in 2024-08-17T17:23:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:40 smithi130 ceph-mon[42749]: pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:40.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:40 smithi179 ceph-mon[34775]: osdmap e108: 8 total, 8 up, 8 in 2024-08-17T17:23:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:40 smithi179 ceph-mon[34775]: pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:41.498 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:41.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:41 smithi130 ceph-mon[31342]: osdmap e109: 8 total, 8 up, 8 in 2024-08-17T17:23:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:41 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1613397583' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:41 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:41 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/4248420053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:41 smithi130 ceph-mon[42749]: osdmap e109: 8 total, 8 up, 8 in 2024-08-17T17:23:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:41 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1613397583' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:41 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:41 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/4248420053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:41.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:41 smithi179 ceph-mon[34775]: osdmap e109: 8 total, 8 up, 8 in 2024-08-17T17:23:41.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:41 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1613397583' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:41.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:41 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-08-17T17:23:41.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:41 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/4248420053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:42.366 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.Empty 2024-08-17T17:23:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:42 smithi130 ceph-mon[31342]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:42 smithi130 ceph-mon[31342]: osdmap e110: 8 total, 8 up, 8 in 2024-08-17T17:23:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:42 smithi130 ceph-mon[31342]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:42 smithi130 ceph-mon[31342]: pgmap v148: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:42 smithi130 ceph-mon[31342]: from='client.24989 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:42 smithi130 ceph-mon[42749]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:42 smithi130 ceph-mon[42749]: osdmap e110: 8 total, 8 up, 8 in 2024-08-17T17:23:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:42 smithi130 ceph-mon[42749]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:42 smithi130 ceph-mon[42749]: pgmap v148: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:42.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:42 smithi130 ceph-mon[42749]: from='client.24989 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:42 smithi179 ceph-mon[34775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:23:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:42 smithi179 ceph-mon[34775]: osdmap e110: 8 total, 8 up, 8 in 2024-08-17T17:23:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:42 smithi179 ceph-mon[34775]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:42 smithi179 ceph-mon[34775]: pgmap v148: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:42 smithi179 ceph-mon[34775]: from='client.24989 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:43 smithi130 ceph-mon[31342]: osdmap e111: 8 total, 8 up, 8 in 2024-08-17T17:23:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:43 smithi130 ceph-mon[42749]: osdmap e111: 8 total, 8 up, 8 in 2024-08-17T17:23:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:43 smithi179 ceph-mon[34775]: osdmap e111: 8 total, 8 up, 8 in 2024-08-17T17:23:44.336 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:44.418 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:44.428 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:44.561 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:44.623 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:23:44.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:44 smithi130 ceph-mon[31342]: osdmap e112: 8 total, 8 up, 8 in 2024-08-17T17:23:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:44 smithi130 ceph-mon[31342]: pgmap v151: 137 pgs: 64 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:44 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/671490055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:44.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:23:44 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:23:44] "GET /metrics HTTP/1.1" 200 242671 "" "Prometheus/2.43.0" 2024-08-17T17:23:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:44 smithi130 ceph-mon[42749]: osdmap e112: 8 total, 8 up, 8 in 2024-08-17T17:23:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:44 smithi130 ceph-mon[42749]: pgmap v151: 137 pgs: 64 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:44 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/671490055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:44.700 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.Empty (2334 ms) 2024-08-17T17:23:44.700 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.OtherName 2024-08-17T17:23:44.727 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:44.737 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:44 smithi179 ceph-mon[34775]: osdmap e112: 8 total, 8 up, 8 in 2024-08-17T17:23:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:44 smithi179 ceph-mon[34775]: pgmap v151: 137 pgs: 64 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:23:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:44 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/671490055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:44.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:23:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:23:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:23:44.857 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:44.917 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:23:44.993 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.OtherName (292 ms) 2024-08-17T17:23:44.993 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.OtherPool 2024-08-17T17:23:45.018 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:45.028 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:45.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:45 smithi130 ceph-mon[31342]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:45.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:45 smithi130 ceph-mon[31342]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:45.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:45 smithi130 ceph-mon[31342]: osdmap e113: 8 total, 8 up, 8 in 2024-08-17T17:23:45.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:45 smithi130 ceph-mon[31342]: from='client.25007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:45 smithi130 ceph-mon[42749]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:45 smithi130 ceph-mon[42749]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:45 smithi130 ceph-mon[42749]: osdmap e113: 8 total, 8 up, 8 in 2024-08-17T17:23:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:45 smithi130 ceph-mon[42749]: from='client.25007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:45.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:45 smithi179 ceph-mon[34775]: from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:45 smithi179 ceph-mon[34775]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:23:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:45 smithi179 ceph-mon[34775]: osdmap e113: 8 total, 8 up, 8 in 2024-08-17T17:23:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:45 smithi179 ceph-mon[34775]: from='client.25007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:46.505 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:46.561 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:23:46.635 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.OtherPool (1643 ms) 2024-08-17T17:23:46.636 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.OtherNamespace 2024-08-17T17:23:46.663 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:46.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:46 smithi130 ceph-mon[31342]: osdmap e114: 8 total, 8 up, 8 in 2024-08-17T17:23:46.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:46 smithi130 ceph-mon[31342]: pgmap v154: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 6.5 KiB/s wr, 87 op/s 2024-08-17T17:23:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:46 smithi130 ceph-mon[42749]: osdmap e114: 8 total, 8 up, 8 in 2024-08-17T17:23:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:46 smithi130 ceph-mon[42749]: pgmap v154: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 6.5 KiB/s wr, 87 op/s 2024-08-17T17:23:46.676 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:46.802 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:46 smithi179 ceph-mon[34775]: osdmap e114: 8 total, 8 up, 8 in 2024-08-17T17:23:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:46 smithi179 ceph-mon[34775]: pgmap v154: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 6.5 KiB/s wr, 87 op/s 2024-08-17T17:23:46.862 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:23:46.940 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.OtherNamespace (305 ms) 2024-08-17T17:23:46.940 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.DataPool 2024-08-17T17:23:46.966 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:46.975 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:47.119 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:47.190 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:23:47.266 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.DataPool (326 ms) 2024-08-17T17:23:47.266 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.AbortAfterPrepare 2024-08-17T17:23:47.294 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:47.295 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:47.306 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:47.446 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:23:47.541 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.AbortAfterPrepare (275 ms) 2024-08-17T17:23:47.541 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.AbortAfterFailedPrepare 2024-08-17T17:23:47.567 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:47.577 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:47.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:47 smithi130 ceph-mon[42749]: osdmap e115: 8 total, 8 up, 8 in 2024-08-17T17:23:47.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:47 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1271432622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:47.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:47 smithi130 ceph-mon[31342]: osdmap e115: 8 total, 8 up, 8 in 2024-08-17T17:23:47.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:47 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1271432622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:47.669 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.AbortAfterFailedPrepare (129 ms) 2024-08-17T17:23:47.669 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.AbortAfterExecute 2024-08-17T17:23:47.700 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:47.709 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:47.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:47 smithi179 ceph-mon[34775]: osdmap e115: 8 total, 8 up, 8 in 2024-08-17T17:23:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:47 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1271432622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:47.845 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:47.913 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:23:48.009 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.AbortAfterExecute (339 ms) 2024-08-17T17:23:48.009 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.OtherPoolAbortAfterExecute 2024-08-17T17:23:48.034 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:48.044 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:48.157 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:48.216 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:23:48.308 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.OtherPoolAbortAfterExecute (299 ms) 2024-08-17T17:23:48.308 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.OtherNamespaceAbortAfterExecute 2024-08-17T17:23:48.336 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:48.349 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:48.477 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:48.541 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:23:48.648 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.OtherNamespaceAbortAfterExecute (340 ms) 2024-08-17T17:23:48.648 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.MirroringSamePool 2024-08-17T17:23:48.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:48 smithi130 ceph-mon[31342]: from='client.24787 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:48.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:48 smithi130 ceph-mon[31342]: pgmap v156: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s rd, 6.2 KiB/s wr, 83 op/s 2024-08-17T17:23:48.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:48 smithi130 ceph-mon[31342]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:48 smithi130 ceph-mon[42749]: from='client.24787 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:48 smithi130 ceph-mon[42749]: pgmap v156: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s rd, 6.2 KiB/s wr, 83 op/s 2024-08-17T17:23:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:48 smithi130 ceph-mon[42749]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:48.673 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:48.683 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:48.689 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.MirroringSamePool (41 ms) 2024-08-17T17:23:48.689 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.MirroringAbort 2024-08-17T17:23:48.714 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:48.722 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:48.728 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.MirroringAbort (40 ms) 2024-08-17T17:23:48.729 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.MirroringOtherPoolDisabled 2024-08-17T17:23:48.754 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:48.764 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:48.770 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.MirroringOtherPoolDisabled (41 ms) 2024-08-17T17:23:48.770 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.MirroringOtherPoolEnabled 2024-08-17T17:23:48.797 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:48.809 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:48.815 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.MirroringOtherPoolEnabled (46 ms) 2024-08-17T17:23:48.815 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.MirroringPool 2024-08-17T17:23:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:48 smithi179 ceph-mon[34775]: from='client.24787 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:48 smithi179 ceph-mon[34775]: pgmap v156: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s rd, 6.2 KiB/s wr, 83 op/s 2024-08-17T17:23:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:48 smithi179 ceph-mon[34775]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:48.843 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:48.852 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:23:48.858 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.MirroringPool (42 ms) 2024-08-17T17:23:48.858 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.Group 2024-08-17T17:23:48.885 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:48.902 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:49.031 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:49.094 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:23:49.190 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.Group (332 ms) 2024-08-17T17:23:49.190 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.GroupAbort 2024-08-17T17:23:49.220 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:49.236 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:49.360 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:23:49.462 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.GroupAbort (272 ms) 2024-08-17T17:23:49.462 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.NoSnaps 2024-08-17T17:23:49.489 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:49.498 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:23:49.553 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 16777218~8388609 2024-08-17T17:23:49.606 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:49.776 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:49 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:49 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:23:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:49 smithi179 ceph-mon[34775]: osdmap e116: 8 total, 8 up, 8 in 2024-08-17T17:23:49.885 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:23:49.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:49 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:49.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:49 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:23:49.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:49 smithi130 ceph-mon[31342]: osdmap e116: 8 total, 8 up, 8 in 2024-08-17T17:23:49.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:49 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:49.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:49 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:23:49.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:49 smithi130 ceph-mon[42749]: osdmap e116: 8 total, 8 up, 8 in 2024-08-17T17:23:49.995 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.NoSnaps (533 ms) 2024-08-17T17:23:49.996 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.NoSnapsOtherPool 2024-08-17T17:23:50.024 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:50.034 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:23:50.095 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 16777218~8388609 2024-08-17T17:23:50.150 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:23:50.155 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 16777218~8388609 2024-08-17T17:23:50.228 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:50.303 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:50.393 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:50.495 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:23:50.608 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.NoSnapsOtherPool (613 ms) 2024-08-17T17:23:50.609 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.NoSnapsDataPool 2024-08-17T17:23:50.637 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:50.647 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:23:50.710 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 16777218~8388609 2024-08-17T17:23:50.767 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:50 smithi179 ceph-mon[34775]: pgmap v158: 137 pgs: 137 active+clean; 8.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.6 MiB/s wr, 1.91k op/s 2024-08-17T17:23:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:50 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2081324146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:50 smithi130 ceph-mon[31342]: pgmap v158: 137 pgs: 137 active+clean; 8.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.6 MiB/s wr, 1.91k op/s 2024-08-17T17:23:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:50 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2081324146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:50 smithi130 ceph-mon[42749]: pgmap v158: 137 pgs: 137 active+clean; 8.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.6 MiB/s wr, 1.91k op/s 2024-08-17T17:23:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:50 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2081324146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:50.946 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:51.053 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:23:51.167 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.NoSnapsDataPool (559 ms) 2024-08-17T17:23:51.167 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.NoSnapsShrinkAfterPrepare 2024-08-17T17:23:51.195 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:51.207 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:23:51.268 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 16777218~8388609 2024-08-17T17:23:51.318 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:51.494 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 41943040 2024-08-17T17:23:51.501 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:51.608 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:23:51.704 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.NoSnapsShrinkAfterPrepare (537 ms) 2024-08-17T17:23:51.704 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroBeforePrepare 2024-08-17T17:23:51.731 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:51.740 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:23:51.799 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 16777218~8388609 2024-08-17T17:23:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:51 smithi179 ceph-mon[34775]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:51 smithi179 ceph-mon[34775]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:51.853 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 0 2024-08-17T17:23:51.866 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:51.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:51 smithi130 ceph-mon[31342]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:51.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:51 smithi130 ceph-mon[31342]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:51 smithi130 ceph-mon[42749]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:51 smithi130 ceph-mon[42749]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:51.965 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:52.077 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:23:52.120 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroBeforePrepare (416 ms) 2024-08-17T17:23:52.120 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroAfterPrepare 2024-08-17T17:23:52.145 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:52.156 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:23:52.218 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 16777218~8388609 2024-08-17T17:23:52.271 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:52.438 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 0 2024-08-17T17:23:52.448 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:52.507 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:23:52.557 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroAfterPrepare (438 ms) 2024-08-17T17:23:52.558 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.NoSnapsExpandAfterPrepare 2024-08-17T17:23:52.583 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:52.594 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:23:52.645 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 16777218~8388609 2024-08-17T17:23:52.700 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:52.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:52 smithi179 ceph-mon[34775]: pgmap v159: 137 pgs: 137 active+clean; 8.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.4 MiB/s wr, 1.55k op/s 2024-08-17T17:23:52.878 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 167772160 2024-08-17T17:23:52.886 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:52.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:52 smithi130 ceph-mon[31342]: pgmap v159: 137 pgs: 137 active+clean; 8.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.4 MiB/s wr, 1.55k op/s 2024-08-17T17:23:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:52 smithi130 ceph-mon[42749]: pgmap v159: 137 pgs: 137 active+clean; 8.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.4 MiB/s wr, 1.55k op/s 2024-08-17T17:23:52.992 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:23:53.126 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.NoSnapsExpandAfterPrepare (568 ms) 2024-08-17T17:23:53.126 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.NoSnapsSnapAfterPrepare 2024-08-17T17:23:53.152 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:53.161 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:23:53.214 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 16777218~8388609 2024-08-17T17:23:53.257 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:23:53.278 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:53.431 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: after_prepare_snap 2024-08-17T17:23:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:53 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3650031378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:53 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3650031378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:53 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3650031378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:54.175 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 41943040 2024-08-17T17:23:54.195 INFO:tasks.workunit.client.0.smithi130.stdout:write: * 0~1000 2024-08-17T17:23:54.201 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:23:54.315 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:23:54.464 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.NoSnapsSnapAfterPrepare (1338 ms) 2024-08-17T17:23:54.464 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.Snaps 2024-08-17T17:23:54.492 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:23:54.501 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:23:54.556 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:23:54.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:23:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:23:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:23:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:54 smithi179 ceph-mon[34775]: from='client.25055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:54 smithi179 ceph-mon[34775]: from='client.15159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:54 smithi179 ceph-mon[34775]: pgmap v160: 137 pgs: 137 active+clean; 8.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 926 KiB/s rd, 1.1 MiB/s wr, 1.19k op/s 2024-08-17T17:23:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:54 smithi179 ceph-mon[34775]: osdmap e117: 8 total, 8 up, 8 in 2024-08-17T17:23:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:54 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:54 smithi179 ceph-mon[34775]: osdmap e118: 8 total, 8 up, 8 in 2024-08-17T17:23:54.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:23:54 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:23:54] "GET /metrics HTTP/1.1" 200 245386 "" "Prometheus/2.43.0" 2024-08-17T17:23:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:54 smithi130 ceph-mon[42749]: from='client.25055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:54 smithi130 ceph-mon[42749]: from='client.15159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:54 smithi130 ceph-mon[42749]: pgmap v160: 137 pgs: 137 active+clean; 8.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 926 KiB/s rd, 1.1 MiB/s wr, 1.19k op/s 2024-08-17T17:23:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:54 smithi130 ceph-mon[42749]: osdmap e117: 8 total, 8 up, 8 in 2024-08-17T17:23:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:54 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:54 smithi130 ceph-mon[42749]: osdmap e118: 8 total, 8 up, 8 in 2024-08-17T17:23:54.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:54 smithi130 ceph-mon[31342]: from='client.25055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:54.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:54 smithi130 ceph-mon[31342]: from='client.15159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:54.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:54 smithi130 ceph-mon[31342]: pgmap v160: 137 pgs: 137 active+clean; 8.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 926 KiB/s rd, 1.1 MiB/s wr, 1.19k op/s 2024-08-17T17:23:54.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:54 smithi130 ceph-mon[31342]: osdmap e117: 8 total, 8 up, 8 in 2024-08-17T17:23:54.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:54 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:23:54.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:54 smithi130 ceph-mon[31342]: osdmap e118: 8 total, 8 up, 8 in 2024-08-17T17:23:56.130 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:56.185 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 8388609~8388609 2024-08-17T17:23:56.198 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 0~1000 2024-08-17T17:23:56.199 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 1010~1000 2024-08-17T17:23:56.199 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:23:56.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:56 smithi130 ceph-mon[31342]: osdmap e119: 8 total, 8 up, 8 in 2024-08-17T17:23:56.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:56 smithi130 ceph-mon[31342]: pgmap v164: 137 pgs: 137 active+clean; 201 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 92 MiB/s rd, 64 MiB/s wr, 1.53k op/s 2024-08-17T17:23:56.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:56 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1080548925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:56.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:56 smithi130 ceph-mon[31342]: from='client.15171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:56.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:56 smithi130 ceph-mon[42749]: osdmap e119: 8 total, 8 up, 8 in 2024-08-17T17:23:56.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:56 smithi130 ceph-mon[42749]: pgmap v164: 137 pgs: 137 active+clean; 201 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 92 MiB/s rd, 64 MiB/s wr, 1.53k op/s 2024-08-17T17:23:56.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:56 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1080548925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:56.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:56 smithi130 ceph-mon[42749]: from='client.15171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:56.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:56 smithi179 ceph-mon[34775]: osdmap e119: 8 total, 8 up, 8 in 2024-08-17T17:23:56.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:56 smithi179 ceph-mon[34775]: pgmap v164: 137 pgs: 137 active+clean; 201 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 92 MiB/s rd, 64 MiB/s wr, 1.53k op/s 2024-08-17T17:23:56.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:56 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1080548925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:56.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:56 smithi179 ceph-mon[34775]: from='client.15171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:57 smithi179 ceph-mon[34775]: osdmap e120: 8 total, 8 up, 8 in 2024-08-17T17:23:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:57 smithi179 ceph-mon[34775]: from='client.25079 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:57.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:57 smithi130 ceph-mon[31342]: osdmap e120: 8 total, 8 up, 8 in 2024-08-17T17:23:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:57 smithi130 ceph-mon[31342]: from='client.25079 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:57.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:57 smithi130 ceph-mon[42749]: osdmap e120: 8 total, 8 up, 8 in 2024-08-17T17:23:57.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:57 smithi130 ceph-mon[42749]: from='client.25079 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:58.206 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 8388609~1000 2024-08-17T17:23:58.212 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 16777228~1000 2024-08-17T17:23:58.212 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 167772160 2024-08-17T17:23:58.221 INFO:tasks.workunit.client.0.smithi130.stdout:write: 2 83886079~8388609 2024-08-17T17:23:58.236 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:23:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:58 smithi179 ceph-mon[34775]: osdmap e121: 8 total, 8 up, 8 in 2024-08-17T17:23:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:58 smithi179 ceph-mon[34775]: pgmap v167: 137 pgs: 137 active+clean; 201 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 139 MiB/s rd, 97 MiB/s wr, 2.32k op/s 2024-08-17T17:23:58.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:58 smithi130 ceph-mon[31342]: osdmap e121: 8 total, 8 up, 8 in 2024-08-17T17:23:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:58 smithi130 ceph-mon[31342]: pgmap v167: 137 pgs: 137 active+clean; 201 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 139 MiB/s rd, 97 MiB/s wr, 2.32k op/s 2024-08-17T17:23:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:58 smithi130 ceph-mon[42749]: osdmap e121: 8 total, 8 up, 8 in 2024-08-17T17:23:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:58 smithi130 ceph-mon[42749]: pgmap v167: 137 pgs: 137 active+clean; 201 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 139 MiB/s rd, 97 MiB/s wr, 2.32k op/s 2024-08-17T17:23:58.967 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:23:59.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:59 smithi130 ceph-mon[42749]: osdmap e122: 8 total, 8 up, 8 in 2024-08-17T17:23:59.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:59 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/4096852352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:59.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:23:59 smithi130 ceph-mon[42749]: from='client.25091 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:59.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:59 smithi130 ceph-mon[31342]: osdmap e122: 8 total, 8 up, 8 in 2024-08-17T17:23:59.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:59 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/4096852352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:59.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:23:59 smithi130 ceph-mon[31342]: from='client.25091 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:23:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:59 smithi179 ceph-mon[34775]: osdmap e122: 8 total, 8 up, 8 in 2024-08-17T17:23:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:59 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/4096852352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:23:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:23:59 smithi179 ceph-mon[34775]: from='client.25091 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:00.220 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:24:00.252 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 83886079~1 2024-08-17T17:24:00.259 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:24:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:00 smithi179 ceph-mon[34775]: osdmap e123: 8 total, 8 up, 8 in 2024-08-17T17:24:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:00 smithi179 ceph-mon[34775]: from='client.25097 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:00 smithi179 ceph-mon[34775]: pgmap v170: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 70 KiB/s rd, 10 MiB/s wr, 104 op/s 2024-08-17T17:24:00.625 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:00 smithi130 ceph-mon[31342]: osdmap e123: 8 total, 8 up, 8 in 2024-08-17T17:24:00.626 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:00 smithi130 ceph-mon[31342]: from='client.25097 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:00.626 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:00 smithi130 ceph-mon[31342]: pgmap v170: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 70 KiB/s rd, 10 MiB/s wr, 104 op/s 2024-08-17T17:24:00.626 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:00 smithi130 ceph-mon[42749]: osdmap e123: 8 total, 8 up, 8 in 2024-08-17T17:24:00.626 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:00 smithi130 ceph-mon[42749]: from='client.25097 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:00.626 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:00 smithi130 ceph-mon[42749]: pgmap v170: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 70 KiB/s rd, 10 MiB/s wr, 104 op/s 2024-08-17T17:24:01.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:01 smithi179 ceph-mon[34775]: osdmap e124: 8 total, 8 up, 8 in 2024-08-17T17:24:01.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:01 smithi130 ceph-mon[31342]: osdmap e124: 8 total, 8 up, 8 in 2024-08-17T17:24:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:01 smithi130 ceph-mon[42749]: osdmap e124: 8 total, 8 up, 8 in 2024-08-17T17:24:01.799 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:02 smithi179 ceph-mon[34775]: osdmap e125: 8 total, 8 up, 8 in 2024-08-17T17:24:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:02 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1387401993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:02 smithi179 ceph-mon[34775]: pgmap v173: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 70 KiB/s rd, 10 MiB/s wr, 104 op/s 2024-08-17T17:24:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:02 smithi179 ceph-mon[34775]: from='client.15207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:02 smithi179 ceph-mon[34775]: from='client.15213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:02 smithi130 ceph-mon[31342]: osdmap e125: 8 total, 8 up, 8 in 2024-08-17T17:24:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:02 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1387401993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:02 smithi130 ceph-mon[31342]: pgmap v173: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 70 KiB/s rd, 10 MiB/s wr, 104 op/s 2024-08-17T17:24:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:02 smithi130 ceph-mon[31342]: from='client.15207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:02 smithi130 ceph-mon[31342]: from='client.15213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:02 smithi130 ceph-mon[42749]: osdmap e125: 8 total, 8 up, 8 in 2024-08-17T17:24:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:02 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1387401993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:02 smithi130 ceph-mon[42749]: pgmap v173: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 70 KiB/s rd, 10 MiB/s wr, 104 op/s 2024-08-17T17:24:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:02 smithi130 ceph-mon[42749]: from='client.15207 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:02 smithi130 ceph-mon[42749]: from='client.15213 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:03.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:03 smithi179 ceph-mon[34775]: osdmap e126: 8 total, 8 up, 8 in 2024-08-17T17:24:03.636 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:24:03.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:03 smithi130 ceph-mon[31342]: osdmap e126: 8 total, 8 up, 8 in 2024-08-17T17:24:03.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:03 smithi130 ceph-mon[42749]: osdmap e126: 8 total, 8 up, 8 in 2024-08-17T17:24:03.773 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:24:04.582 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:04 smithi179 ceph-mon[34775]: osdmap e127: 8 total, 8 up, 8 in 2024-08-17T17:24:04.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:04 smithi179 ceph-mon[34775]: pgmap v176: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:04.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:04 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:04.643 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:04 smithi130 ceph-mon[31342]: osdmap e127: 8 total, 8 up, 8 in 2024-08-17T17:24:04.643 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:04 smithi130 ceph-mon[31342]: pgmap v176: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:04.643 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:04 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:04.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:04 smithi130 ceph-mon[42749]: osdmap e127: 8 total, 8 up, 8 in 2024-08-17T17:24:04.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:04 smithi130 ceph-mon[42749]: pgmap v176: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:04.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:04 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:04.644 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:04 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:24:04] "GET /metrics HTTP/1.1" 200 246094 "" "Prometheus/2.43.0" 2024-08-17T17:24:04.644 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:04.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[36875]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:24:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:24:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:05 smithi179 ceph-mon[34775]: osdmap e128: 8 total, 8 up, 8 in 2024-08-17T17:24:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:05 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/4082848753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:05 smithi179 ceph-mon[34775]: from='client.15225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:05 smithi179 ceph-mon[34775]: from='client.15231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:05 smithi130 ceph-mon[31342]: osdmap e128: 8 total, 8 up, 8 in 2024-08-17T17:24:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:05 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/4082848753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:05 smithi130 ceph-mon[31342]: from='client.15225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:05 smithi130 ceph-mon[31342]: from='client.15231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:05 smithi130 ceph-mon[42749]: osdmap e128: 8 total, 8 up, 8 in 2024-08-17T17:24:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:05 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/4082848753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:05 smithi130 ceph-mon[42749]: from='client.15225 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:05 smithi130 ceph-mon[42749]: from='client.15231 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:06.249 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:06 smithi179 ceph-mon[34775]: osdmap e129: 8 total, 8 up, 8 in 2024-08-17T17:24:06.249 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:06 smithi179 ceph-mon[34775]: pgmap v179: 137 pgs: 137 active+clean; 269 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 499 op/s 2024-08-17T17:24:06.249 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:06 smithi179 ceph-mon[34775]: Upgrade: Updating mgr.x 2024-08-17T17:24:06.249 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:06 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:06.509 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.Snaps (12045 ms) 2024-08-17T17:24:06.509 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.SnapsOtherPool 2024-08-17T17:24:06.526 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:06 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:24:06.526 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:06 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:24:06.526 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:06 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:24:06.526 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:06 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:24:06.526 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:06 smithi179 ceph-mon[34775]: Deploying daemon mgr.x on smithi179 2024-08-17T17:24:06.538 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:24:06.547 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:24:06.607 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:24:06.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[31342]: osdmap e129: 8 total, 8 up, 8 in 2024-08-17T17:24:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[31342]: pgmap v179: 137 pgs: 137 active+clean; 269 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 499 op/s 2024-08-17T17:24:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[31342]: Upgrade: Updating mgr.x 2024-08-17T17:24:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:24:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:24:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:24:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:24:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[31342]: Deploying daemon mgr.x on smithi179 2024-08-17T17:24:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[42749]: osdmap e129: 8 total, 8 up, 8 in 2024-08-17T17:24:06.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[42749]: pgmap v179: 137 pgs: 137 active+clean; 269 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 499 op/s 2024-08-17T17:24:06.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[42749]: Upgrade: Updating mgr.x 2024-08-17T17:24:06.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:06.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:24:06.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:24:06.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:24:06.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:24:06.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:06 smithi130 ceph-mon[42749]: Deploying daemon mgr.x on smithi179 2024-08-17T17:24:07.422 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:07 smithi179 ceph-mon[34775]: osdmap e130: 8 total, 8 up, 8 in 2024-08-17T17:24:07.422 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:07 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1405013719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:07.474 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:07.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:07 smithi130 ceph-mon[31342]: osdmap e130: 8 total, 8 up, 8 in 2024-08-17T17:24:07.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:07 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1405013719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:07 smithi130 ceph-mon[42749]: osdmap e130: 8 total, 8 up, 8 in 2024-08-17T17:24:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:07 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1405013719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:07.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:07 smithi179 systemd[1]: Stopping Ceph mgr.x for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:24:08.266 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:07 smithi179 podman[70324]: 2024-08-17 17:24:07.882003593 +0000 UTC m=+0.114457217 container died fdc68f217f77de01b2694d983459f44aa8bfeee0b75387aad0b61eeccd569a4d (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=quincy-f2c39d4, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-08-17T17:24:08.276 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 8388609~8388609 2024-08-17T17:24:08.290 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 0~1000 2024-08-17T17:24:08.290 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 1010~1000 2024-08-17T17:24:08.290 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:24:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:08 smithi179 ceph-mon[34775]: osdmap e131: 8 total, 8 up, 8 in 2024-08-17T17:24:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:08 smithi179 ceph-mon[34775]: from='client.15243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:08 smithi179 ceph-mon[34775]: pgmap v182: 137 pgs: 137 active+clean; 269 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 499 op/s 2024-08-17T17:24:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:08 smithi179 ceph-mon[34775]: from='client.15249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:08.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:08 smithi130 ceph-mon[31342]: osdmap e131: 8 total, 8 up, 8 in 2024-08-17T17:24:08.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:08 smithi130 ceph-mon[31342]: from='client.15243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:08.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:08 smithi130 ceph-mon[31342]: pgmap v182: 137 pgs: 137 active+clean; 269 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 499 op/s 2024-08-17T17:24:08.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:08 smithi130 ceph-mon[31342]: from='client.15249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:08 smithi130 ceph-mon[42749]: osdmap e131: 8 total, 8 up, 8 in 2024-08-17T17:24:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:08 smithi130 ceph-mon[42749]: from='client.15243 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:08 smithi130 ceph-mon[42749]: pgmap v182: 137 pgs: 137 active+clean; 269 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 499 op/s 2024-08-17T17:24:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:08 smithi130 ceph-mon[42749]: from='client.15249 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:09.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:08 smithi179 podman[70324]: 2024-08-17 17:24:08.626024219 +0000 UTC m=+0.858477881 container cleanup fdc68f217f77de01b2694d983459f44aa8bfeee0b75387aad0b61eeccd569a4d (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-08-17T17:24:09.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:08 smithi179 bash[70324]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x 2024-08-17T17:24:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:09 smithi179 ceph-mon[34775]: osdmap e132: 8 total, 8 up, 8 in 2024-08-17T17:24:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:09 smithi179 ceph-mon[34775]: osdmap e133: 8 total, 8 up, 8 in 2024-08-17T17:24:09.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:09 smithi130 ceph-mon[31342]: osdmap e132: 8 total, 8 up, 8 in 2024-08-17T17:24:09.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:09 smithi130 ceph-mon[31342]: osdmap e133: 8 total, 8 up, 8 in 2024-08-17T17:24:09.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:09 smithi130 ceph-mon[42749]: osdmap e132: 8 total, 8 up, 8 in 2024-08-17T17:24:09.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:09 smithi130 ceph-mon[42749]: osdmap e133: 8 total, 8 up, 8 in 2024-08-17T17:24:10.179 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 8388609~1000 2024-08-17T17:24:10.185 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 16777228~1000 2024-08-17T17:24:10.185 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 167772160 2024-08-17T17:24:10.192 INFO:tasks.workunit.client.0.smithi130.stdout:write: 2 83886079~8388609 2024-08-17T17:24:10.210 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:24:10.333 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:10.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:10 smithi130 ceph-mon[31342]: pgmap v185: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 134 op/s 2024-08-17T17:24:10.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:10 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1534108979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:10.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:10 smithi130 ceph-mon[31342]: osdmap e134: 8 total, 8 up, 8 in 2024-08-17T17:24:10.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:10 smithi130 ceph-mon[42749]: pgmap v185: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 134 op/s 2024-08-17T17:24:10.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:10 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1534108979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:10.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:10 smithi130 ceph-mon[42749]: osdmap e134: 8 total, 8 up, 8 in 2024-08-17T17:24:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:10 smithi179 ceph-mon[34775]: pgmap v185: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 134 op/s 2024-08-17T17:24:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:10 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1534108979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:10 smithi179 ceph-mon[34775]: osdmap e134: 8 total, 8 up, 8 in 2024-08-17T17:24:11.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:10 smithi179 podman[70344]: 2024-08-17 17:24:10.676433605 +0000 UTC m=+2.789259522 container remove fdc68f217f77de01b2694d983459f44aa8bfeee0b75387aad0b61eeccd569a4d (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-08-17T17:24:11.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:10 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:24:11.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:10 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-17T17:24:11.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:10 smithi179 systemd[1]: Stopped Ceph mgr.x for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:24:11.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:10 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.x.service: Consumed 10.116s CPU time. 2024-08-17T17:24:11.501 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:11 smithi179 systemd[1]: Starting Ceph mgr.x for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:24:11.501 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:11 smithi179 ceph-mon[34775]: from='client.15261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:11.501 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:11 smithi179 ceph-mon[34775]: from='client.15267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:11.502 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:11 smithi179 ceph-mon[34775]: osdmap e135: 8 total, 8 up, 8 in 2024-08-17T17:24:11.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:11 smithi130 ceph-mon[42749]: from='client.15261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:11 smithi130 ceph-mon[42749]: from='client.15267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:11 smithi130 ceph-mon[42749]: osdmap e135: 8 total, 8 up, 8 in 2024-08-17T17:24:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:11 smithi130 ceph-mon[31342]: from='client.15261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:11 smithi130 ceph-mon[31342]: from='client.15267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:11 smithi130 ceph-mon[31342]: osdmap e135: 8 total, 8 up, 8 in 2024-08-17T17:24:11.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:11 smithi179 podman[70476]: 2024-08-17 17:24:11.403493411 +0000 UTC m=+0.017730740 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:24:12.181 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:24:12.204 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 83886079~1 2024-08-17T17:24:12.211 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:24:12.220 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 16777218~8388609 2024-08-17T17:24:12.285 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:24:12.582 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:12 smithi179 ceph-mon[34775]: pgmap v188: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 134 op/s 2024-08-17T17:24:12.582 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:12 smithi179 ceph-mon[34775]: osdmap e136: 8 total, 8 up, 8 in 2024-08-17T17:24:12.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:12 smithi130 ceph-mon[31342]: pgmap v188: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 134 op/s 2024-08-17T17:24:12.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:12 smithi130 ceph-mon[31342]: osdmap e136: 8 total, 8 up, 8 in 2024-08-17T17:24:12.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:12 smithi130 ceph-mon[42749]: pgmap v188: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 134 op/s 2024-08-17T17:24:12.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:12 smithi130 ceph-mon[42749]: osdmap e136: 8 total, 8 up, 8 in 2024-08-17T17:24:12.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:12 smithi179 podman[70476]: 2024-08-17 17:24:12.581852929 +0000 UTC m=+1.196090247 container create d129c8e500cc233d5d7bdf5117a11d96eaf90ace5db1c9449bd1727355fc88a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:24:13.152 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:13.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:13 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1270376334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:13.464 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:13 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1270376334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:13 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1270376334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:14 smithi179 ceph-mon[34775]: from='client.25181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:14 smithi179 ceph-mon[34775]: osdmap e137: 8 total, 8 up, 8 in 2024-08-17T17:24:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:14 smithi179 ceph-mon[34775]: from='client.15285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:14 smithi179 ceph-mon[34775]: pgmap v191: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:14 smithi179 ceph-mon[34775]: osdmap e138: 8 total, 8 up, 8 in 2024-08-17T17:24:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:14 smithi130 ceph-mon[31342]: from='client.25181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:14 smithi130 ceph-mon[31342]: osdmap e137: 8 total, 8 up, 8 in 2024-08-17T17:24:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:14 smithi130 ceph-mon[31342]: from='client.15285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:14 smithi130 ceph-mon[31342]: pgmap v191: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:14 smithi130 ceph-mon[31342]: osdmap e138: 8 total, 8 up, 8 in 2024-08-17T17:24:14.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:14 smithi130 ceph-mon[42749]: from='client.25181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:14.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:14 smithi130 ceph-mon[42749]: osdmap e137: 8 total, 8 up, 8 in 2024-08-17T17:24:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:14 smithi130 ceph-mon[42749]: from='client.15285 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:14 smithi130 ceph-mon[42749]: pgmap v191: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:14 smithi130 ceph-mon[42749]: osdmap e138: 8 total, 8 up, 8 in 2024-08-17T17:24:14.668 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:14 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:24:14] "GET /metrics HTTP/1.1" 200 246094 "" "Prometheus/2.43.0" 2024-08-17T17:24:15.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:14 smithi179 podman[70476]: 2024-08-17 17:24:14.697395392 +0000 UTC m=+3.311632729 container init d129c8e500cc233d5d7bdf5117a11d96eaf90ace5db1c9449bd1727355fc88a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:24:15.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:14 smithi179 podman[70476]: 2024-08-17 17:24:14.704289082 +0000 UTC m=+3.318526401 container start d129c8e500cc233d5d7bdf5117a11d96eaf90ace5db1c9449bd1727355fc88a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-cd8bdbe, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-08-17T17:24:15.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:14 smithi179 ceph-mgr[70526]: -- 172.21.15.179:0/1928781184 <== mon.1 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56012d8e50e0 con 0x56012d8c2800 2024-08-17T17:24:15.610 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:24:15.785 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:24:15.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:15 smithi179 bash[70476]: d129c8e500cc233d5d7bdf5117a11d96eaf90ace5db1c9449bd1727355fc88a9 2024-08-17T17:24:15.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:15 smithi179 systemd[1]: Started Ceph mgr.x for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:24:16.087 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:16.406 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:16 smithi130 ceph-mon[31342]: osdmap e139: 8 total, 8 up, 8 in 2024-08-17T17:24:16.407 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:16 smithi130 ceph-mon[31342]: pgmap v194: 137 pgs: 137 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 134 KiB/s rd, 12 MiB/s wr, 196 op/s 2024-08-17T17:24:16.407 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:16 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:16.407 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:16 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:16.407 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:16 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:24:16.407 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:16 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/556178016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:16.407 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:16 smithi130 ceph-mon[31342]: from='client.25205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:16.407 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:16 smithi130 ceph-mon[42749]: osdmap e139: 8 total, 8 up, 8 in 2024-08-17T17:24:16.407 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:16 smithi130 ceph-mon[42749]: pgmap v194: 137 pgs: 137 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 134 KiB/s rd, 12 MiB/s wr, 196 op/s 2024-08-17T17:24:16.407 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:16 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:16.408 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:16 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:16.408 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:16 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:24:16.408 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:16 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/556178016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:16.408 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:16 smithi130 ceph-mon[42749]: from='client.25205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:16 smithi179 ceph-mon[34775]: osdmap e139: 8 total, 8 up, 8 in 2024-08-17T17:24:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:16 smithi179 ceph-mon[34775]: pgmap v194: 137 pgs: 137 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 134 KiB/s rd, 12 MiB/s wr, 196 op/s 2024-08-17T17:24:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:16 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:16 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:16 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:24:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:16 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/556178016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:16 smithi179 ceph-mon[34775]: from='client.25205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:18.019 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:17 smithi179 ceph-mon[34775]: from='client.15309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:18.020 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:17 smithi179 ceph-mon[34775]: osdmap e140: 8 total, 8 up, 8 in 2024-08-17T17:24:18.064 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:17 smithi130 ceph-mon[31342]: from='client.15309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:18.064 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:17 smithi130 ceph-mon[31342]: osdmap e140: 8 total, 8 up, 8 in 2024-08-17T17:24:18.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:17 smithi130 ceph-mon[42749]: from='client.15309 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:18.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:17 smithi130 ceph-mon[42749]: osdmap e140: 8 total, 8 up, 8 in 2024-08-17T17:24:18.958 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:18 smithi179 ceph-mon[34775]: pgmap v196: 137 pgs: 137 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 11 MiB/s wr, 184 op/s 2024-08-17T17:24:18.958 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:18 smithi179 ceph-mon[34775]: osdmap e141: 8 total, 8 up, 8 in 2024-08-17T17:24:18.958 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:18 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:18.958 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:18 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:18.958 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:18 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:18.958 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:18 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2084115740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:18.987 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:18.987 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:18 smithi130 ceph-mon[42749]: pgmap v196: 137 pgs: 137 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 11 MiB/s wr, 184 op/s 2024-08-17T17:24:18.988 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:18 smithi130 ceph-mon[42749]: osdmap e141: 8 total, 8 up, 8 in 2024-08-17T17:24:18.988 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:18 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:18.988 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:18 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:18.988 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:18 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:18.988 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:18 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2084115740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:18.988 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:18 smithi130 ceph-mon[31342]: pgmap v196: 137 pgs: 137 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 11 MiB/s wr, 184 op/s 2024-08-17T17:24:18.988 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:18 smithi130 ceph-mon[31342]: osdmap e141: 8 total, 8 up, 8 in 2024-08-17T17:24:18.988 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:18 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:18.989 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:18 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:18.989 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:18 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:18.989 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:18 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2084115740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:19.011 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.SnapsOtherPool (12502 ms) 2024-08-17T17:24:19.012 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.SnapsDataPool 2024-08-17T17:24:19.045 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:24:19.057 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:24:19.116 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:24:20.070 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:19 smithi179 ceph-mon[34775]: osdmap e142: 8 total, 8 up, 8 in 2024-08-17T17:24:20.071 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:19 smithi179 ceph-mon[34775]: from='client.25223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:20.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:19 smithi130 ceph-mon[31342]: osdmap e142: 8 total, 8 up, 8 in 2024-08-17T17:24:20.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:19 smithi130 ceph-mon[31342]: from='client.25223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:20.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:19 smithi130 ceph-mon[42749]: osdmap e142: 8 total, 8 up, 8 in 2024-08-17T17:24:20.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:19 smithi130 ceph-mon[42749]: from='client.25223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:20.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: [17/Aug/2024:17:24:20] ENGINE Bus STARTING 2024-08-17T17:24:20.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: CherryPy Checker: 2024-08-17T17:24:20.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: The Application mounted at '' has an empty config. 2024-08-17T17:24:20.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:24:20.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: [17/Aug/2024:17:24:20] ENGINE Serving on http://:::9283 2024-08-17T17:24:20.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: [17/Aug/2024:17:24:20] ENGINE Bus STARTED 2024-08-17T17:24:20.704 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 8388609~8388609 2024-08-17T17:24:20.717 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 0~1000 2024-08-17T17:24:20.718 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 1010~1000 2024-08-17T17:24:20.718 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:24:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-mon[34775]: from='client.15327 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-mon[34775]: pgmap v199: 137 pgs: 137 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 75 MiB/s rd, 13 MiB/s wr, 418 op/s 2024-08-17T17:24:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-mon[34775]: osdmap e143: 8 total, 8 up, 8 in 2024-08-17T17:24:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-mon[34775]: Standby manager daemon x restarted 2024-08-17T17:24:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-mon[34775]: Standby manager daemon x started 2024-08-17T17:24:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:24:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:24:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:24:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:20 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:24:21.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[31342]: from='client.15327 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:21.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[31342]: pgmap v199: 137 pgs: 137 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 75 MiB/s rd, 13 MiB/s wr, 418 op/s 2024-08-17T17:24:21.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[31342]: osdmap e143: 8 total, 8 up, 8 in 2024-08-17T17:24:21.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[31342]: Standby manager daemon x restarted 2024-08-17T17:24:21.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[31342]: Standby manager daemon x started 2024-08-17T17:24:21.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:24:21.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:24:21.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:24:21.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:24:21.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[42749]: from='client.15327 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:21.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[42749]: pgmap v199: 137 pgs: 137 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 75 MiB/s rd, 13 MiB/s wr, 418 op/s 2024-08-17T17:24:21.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[42749]: osdmap e143: 8 total, 8 up, 8 in 2024-08-17T17:24:21.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[42749]: Standby manager daemon x restarted 2024-08-17T17:24:21.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[42749]: Standby manager daemon x started 2024-08-17T17:24:21.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:24:21.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:24:21.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:24:21.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:20 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:24:21.851 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:21.965 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:21 smithi179 ceph-mon[34775]: osdmap e144: 8 total, 8 up, 8 in 2024-08-17T17:24:21.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:21 smithi179 ceph-mon[34775]: mgrmap e22: y(active, since 3m), standbys: x 2024-08-17T17:24:21.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:21 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:21.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:21 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:21.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:21 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:21.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:21 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:21.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:21 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3982277422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:22.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:21 smithi130 ceph-mon[31342]: osdmap e144: 8 total, 8 up, 8 in 2024-08-17T17:24:22.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:21 smithi130 ceph-mon[31342]: mgrmap e22: y(active, since 3m), standbys: x 2024-08-17T17:24:22.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:21 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:22.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:21 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:22.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:21 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:22.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:21 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:22.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:21 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3982277422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:22.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:21 smithi130 ceph-mon[42749]: osdmap e144: 8 total, 8 up, 8 in 2024-08-17T17:24:22.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:21 smithi130 ceph-mon[42749]: mgrmap e22: y(active, since 3m), standbys: x 2024-08-17T17:24:22.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:21 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:22.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:21 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:22.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:21 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:22.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:21 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:22.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:21 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3982277422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:22.715 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 8388609~1000 2024-08-17T17:24:22.719 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 16777228~1000 2024-08-17T17:24:22.720 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 167772160 2024-08-17T17:24:22.726 INFO:tasks.workunit.client.0.smithi130.stdout:write: 2 83886079~8388609 2024-08-17T17:24:22.745 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:24:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:22 smithi179 ceph-mon[34775]: pgmap v202: 137 pgs: 137 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 82 MiB/s rd, 14 MiB/s wr, 459 op/s 2024-08-17T17:24:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:22 smithi179 ceph-mon[34775]: osdmap e145: 8 total, 8 up, 8 in 2024-08-17T17:24:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:22 smithi179 ceph-mon[34775]: from='client.15339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:22 smithi179 ceph-mon[34775]: from='client.15345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:23.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:22 smithi130 ceph-mon[31342]: pgmap v202: 137 pgs: 137 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 82 MiB/s rd, 14 MiB/s wr, 459 op/s 2024-08-17T17:24:23.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:22 smithi130 ceph-mon[31342]: osdmap e145: 8 total, 8 up, 8 in 2024-08-17T17:24:23.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:22 smithi130 ceph-mon[31342]: from='client.15339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:23.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:22 smithi130 ceph-mon[31342]: from='client.15345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:23.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:22 smithi130 ceph-mon[42749]: pgmap v202: 137 pgs: 137 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 82 MiB/s rd, 14 MiB/s wr, 459 op/s 2024-08-17T17:24:23.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:22 smithi130 ceph-mon[42749]: osdmap e145: 8 total, 8 up, 8 in 2024-08-17T17:24:23.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:22 smithi130 ceph-mon[42749]: from='client.15339 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:23.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:22 smithi130 ceph-mon[42749]: from='client.15345 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:23.966 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:23 smithi179 ceph-mon[34775]: osdmap e146: 8 total, 8 up, 8 in 2024-08-17T17:24:24.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:23 smithi130 ceph-mon[31342]: osdmap e146: 8 total, 8 up, 8 in 2024-08-17T17:24:24.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:23 smithi130 ceph-mon[42749]: osdmap e146: 8 total, 8 up, 8 in 2024-08-17T17:24:24.182 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:24:24.203 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 83886079~1 2024-08-17T17:24:24.211 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:24:24.712 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:24.712 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:24:24] "GET /metrics HTTP/1.1" 200 246138 "" "Prometheus/2.43.0" 2024-08-17T17:24:24.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:24 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:24:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:24:24.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:24 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: [17/Aug/2024:17:24:24] ENGINE Bus STOPPING 2024-08-17T17:24:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:24 smithi179 ceph-mon[34775]: pgmap v205: 137 pgs: 137 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:24 smithi179 ceph-mon[34775]: osdmap e147: 8 total, 8 up, 8 in 2024-08-17T17:24:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:24 smithi179 ceph-mon[34775]: osdmap e148: 8 total, 8 up, 8 in 2024-08-17T17:24:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:24 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1726887494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:24.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:24 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:24.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:24 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:24.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:24 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:24:24.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:24 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:24:24.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:24 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:25.022 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[31342]: pgmap v205: 137 pgs: 137 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:25.022 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[31342]: osdmap e147: 8 total, 8 up, 8 in 2024-08-17T17:24:25.022 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[31342]: osdmap e148: 8 total, 8 up, 8 in 2024-08-17T17:24:25.022 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1726887494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:25.022 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:25.023 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:25.023 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:24:25.023 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:24:25.023 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:25.023 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[42749]: pgmap v205: 137 pgs: 137 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:25.023 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[42749]: osdmap e147: 8 total, 8 up, 8 in 2024-08-17T17:24:25.023 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[42749]: osdmap e148: 8 total, 8 up, 8 in 2024-08-17T17:24:25.023 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1726887494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:25.023 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:25.023 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:25.023 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:24:25.024 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:24:25.024 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' 2024-08-17T17:24:25.024 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ignoring --setuser ceph since I am not root 2024-08-17T17:24:25.024 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ignoring --setgroup ceph since I am not root 2024-08-17T17:24:25.024 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-mgr[31617]: -- 172.21.15.130:0/2944435703 <== mon.2 v2:172.21.15.130:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55609ead09c0 con 0x55609eadcc00 2024-08-17T17:24:25.024 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:24 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:24.895+0000 7f8bee9a9ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T17:24:25.326 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:25.019+0000 7f8bee9a9ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:24:25.326 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:25.085+0000 7f8bee9a9ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T17:24:25.326 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:25.324+0000 7f8bee9a9ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T17:24:25.342 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: [17/Aug/2024:17:24:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:24:25.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: [17/Aug/2024:17:24:25] ENGINE Bus STOPPED 2024-08-17T17:24:25.666 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:25.380+0000 7f8bee9a9ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T17:24:25.748 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENOTSUP: Warning: due to ceph-mgr restart, some PG states may not be up to date 2024-08-17T17:24:25.748 INFO:teuthology.orchestra.run.smithi130.stderr:Module 'orchestrator' is not enabled/loaded (required by command 'orch upgrade status'): use `ceph mgr module enable orchestrator` to enable it 2024-08-17T17:24:26.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:24:25.747+0000 7f3eb3ebb640 -1 mgr.server reply reply (95) Operation not supported Warning: due to ceph-mgr restart, some PG states may not be up to date 2024-08-17T17:24:26.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: Module 'orchestrator' is not enabled/loaded (required by command 'orch upgrade status'): use `ceph mgr module enable orchestrator` to enable it 2024-08-17T17:24:26.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:24:26.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T17:24:26.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T17:24:26.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: Activating manager daemon x 2024-08-17T17:24:26.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:24:26.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:24:26.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:24:26.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi179.zkkvno"}]: dispatch 2024-08-17T17:24:26.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.jmaidg"}]: dispatch 2024-08-17T17:24:26.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:24:26.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:24:26.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:24:26.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:24:26.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:24:26.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:24:26.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:24:26.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:24:26.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: osdmap e149: 8 total, 8 up, 8 in 2024-08-17T17:24:26.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:24:26.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-17T17:24:26.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: mgrmap e23: x(active, starting, since 0.0129774s) 2024-08-17T17:24:26.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:24:26.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:24:26.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:24:26.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:25 smithi179 ceph-mon[34775]: Manager daemon x is now available 2024-08-17T17:24:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:24:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T17:24:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T17:24:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: Activating manager daemon x 2024-08-17T17:24:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:24:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:24:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:24:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi179.zkkvno"}]: dispatch 2024-08-17T17:24:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.jmaidg"}]: dispatch 2024-08-17T17:24:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:24:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:24:26.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:24:26.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:24:26.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:24:26.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:24:26.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:24:26.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:24:26.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: osdmap e149: 8 total, 8 up, 8 in 2024-08-17T17:24:26.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:24:26.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-17T17:24:26.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: mgrmap e23: x(active, starting, since 0.0129774s) 2024-08-17T17:24:26.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:24:26.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:24:26.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:24:26.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[31342]: Manager daemon x is now available 2024-08-17T17:24:26.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:24:26.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24338 172.21.15.130:0/3136272662' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T17:24:26.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T17:24:26.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: Activating manager daemon x 2024-08-17T17:24:26.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:24:26.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:24:26.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:24:26.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi179.zkkvno"}]: dispatch 2024-08-17T17:24:26.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.jmaidg"}]: dispatch 2024-08-17T17:24:26.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:24:26.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:24:26.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:24:26.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:24:26.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:24:26.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:24:26.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:24:26.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:24:26.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: osdmap e149: 8 total, 8 up, 8 in 2024-08-17T17:24:26.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:24:26.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24338 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-17T17:24:26.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: mgrmap e23: x(active, starting, since 0.0129774s) 2024-08-17T17:24:26.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:24:26.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:24:26.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:24:26.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:25 smithi130 ceph-mon[42749]: Manager daemon x is now available 2024-08-17T17:24:26.637 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:26.280+0000 7f8bee9a9ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T17:24:26.637 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:26.389+0000 7f8bee9a9ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T17:24:26.637 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:26.443+0000 7f8bee9a9ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T17:24:26.638 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:26.635+0000 7f8bee9a9ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T17:24:26.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-mon[34775]: osdmap e150: 8 total, 8 up, 8 in 2024-08-17T17:24:26.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-mon[34775]: mgrmap e24: x(active, since 1.01991s) 2024-08-17T17:24:26.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:24:26.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:26.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:24:26.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:24:26.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:24:26.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:24:26.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:24:26.861 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: [17/Aug/2024:17:24:26] ENGINE Bus STARTING 2024-08-17T17:24:26.861 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:24:26.647+0000 7f3e89e67640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:24:26.861 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:24:26.647+0000 7f3e89e67640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:24:26.861 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:24:26.647+0000 7f3e89e67640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:24:26.861 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:24:26.647+0000 7f3e89e67640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:24:26.861 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:24:26.647+0000 7f3e89e67640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:24:26.861 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:24:26.647+0000 7f3e86660640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:24:26.861 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:24:26.647+0000 7f3e86660640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:24:26.861 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:24:26.647+0000 7f3e86660640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:24:26.862 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:24:26.648+0000 7f3e86660640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:24:26.862 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:24:26.648+0000 7f3e86660640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:24:26.862 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: [17/Aug/2024:17:24:26] ENGINE Serving on http://:::9283 2024-08-17T17:24:26.862 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: [17/Aug/2024:17:24:26] ENGINE Bus STARTED 2024-08-17T17:24:26.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:26.695+0000 7f8bee9a9ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T17:24:26.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:26.768+0000 7f8bee9a9ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T17:24:26.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[31342]: osdmap e150: 8 total, 8 up, 8 in 2024-08-17T17:24:26.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[31342]: mgrmap e24: x(active, since 1.01991s) 2024-08-17T17:24:26.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:24:26.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:26.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:24:26.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:24:26.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:24:26.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:24:26.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:24:26.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[42749]: osdmap e150: 8 total, 8 up, 8 in 2024-08-17T17:24:26.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[42749]: mgrmap e24: x(active, since 1.01991s) 2024-08-17T17:24:26.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:26.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:24:26.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:26.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:24:26.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:24:26.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:24:26.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:24:26.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T17:24:27.157 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:24:27.314 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:24:27.416 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:26 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:26.918+0000 7f8bee9a9ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T17:24:27.701 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:27 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:27.416+0000 7f8bee9a9ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T17:24:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-mon[34775]: Checking for cert/key for grafana.a 2024-08-17T17:24:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-mon[34775]: Migrating grafana.a cert to cert store 2024-08-17T17:24:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-mon[34775]: Migrating grafana.a key to cert store 2024-08-17T17:24:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:24:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-mon[34775]: osdmap e151: 8 total, 8 up, 8 in 2024-08-17T17:24:27.844 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:24:27.530+0000 7f3e6e373640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:17:24:27] ENGINE Error in HTTPServer.serve 2024-08-17T17:24:27.844 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: Traceback (most recent call last): 2024-08-17T17:24:27.844 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:24:27.844 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: self._connections.run(self.expiration_interval) 2024-08-17T17:24:27.844 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:24:27.844 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: self._run(expiration_interval) 2024-08-17T17:24:27.844 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:24:27.844 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:24:27.844 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:24:27.844 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:24:27.844 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:24:27.844 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: s = self.context.wrap_socket( 2024-08-17T17:24:27.845 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:24:27.845 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: return self.sslsocket_class._create( 2024-08-17T17:24:27.845 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:24:27.845 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: self.do_handshake() 2024-08-17T17:24:27.845 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:24:27.845 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: self._sslobj.do_handshake() 2024-08-17T17:24:27.845 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:24:27.845 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:27 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:24:28.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:27 smithi130 ceph-mon[31342]: Checking for cert/key for grafana.a 2024-08-17T17:24:28.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:27 smithi130 ceph-mon[31342]: Migrating grafana.a cert to cert store 2024-08-17T17:24:28.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:27 smithi130 ceph-mon[31342]: Migrating grafana.a key to cert store 2024-08-17T17:24:28.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:27 smithi130 ceph-mon[31342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:24:28.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:27 smithi130 ceph-mon[31342]: osdmap e151: 8 total, 8 up, 8 in 2024-08-17T17:24:28.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:27 smithi130 ceph-mon[42749]: Checking for cert/key for grafana.a 2024-08-17T17:24:28.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:27 smithi130 ceph-mon[42749]: Migrating grafana.a cert to cert store 2024-08-17T17:24:28.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:27 smithi130 ceph-mon[42749]: Migrating grafana.a key to cert store 2024-08-17T17:24:28.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:27 smithi130 ceph-mon[42749]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:24:28.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:27 smithi130 ceph-mon[42749]: osdmap e151: 8 total, 8 up, 8 in 2024-08-17T17:24:28.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:27 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:27.699+0000 7f8bee9a9ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T17:24:28.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:27 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:27.773+0000 7f8bee9a9ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T17:24:28.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:27 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:27.906+0000 7f8bee9a9ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T17:24:28.402 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:28.492 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:28.171+0000 7f8bee9a9ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T17:24:28.492 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:28.234+0000 7f8bee9a9ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T17:24:28.492 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:28.420+0000 7f8bee9a9ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T17:24:28.814 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:28 smithi179 ceph-mon[34775]: pgmap v5: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:28.884 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:28 smithi179 ceph-mon[34775]: Deploying cephadm binary to smithi179 2024-08-17T17:24:28.885 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:28 smithi179 ceph-mon[34775]: Deploying cephadm binary to smithi130 2024-08-17T17:24:28.885 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:28 smithi179 ceph-mon[34775]: [17/Aug/2024:17:24:27] ENGINE Bus STARTING 2024-08-17T17:24:28.885 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:28 smithi179 ceph-mon[34775]: [17/Aug/2024:17:24:27] ENGINE Serving on http://172.21.15.179:8765 2024-08-17T17:24:28.885 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:28 smithi179 ceph-mon[34775]: [17/Aug/2024:17:24:27] ENGINE Serving on https://172.21.15.179:7150 2024-08-17T17:24:28.885 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:28 smithi179 ceph-mon[34775]: [17/Aug/2024:17:24:27] ENGINE Bus STARTED 2024-08-17T17:24:28.885 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:28 smithi179 ceph-mon[34775]: [17/Aug/2024:17:24:27] ENGINE Error in HTTPServer.serve 2024-08-17T17:24:28.885 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-08-17T17:24:28.885 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:24:28.885 INFO:journalctl@ceph.mon.b.smithi179.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:24:28.885 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:24:28.885 INFO:journalctl@ceph.mon.b.smithi179.stdout: self._run(expiration_interval) 2024-08-17T17:24:28.886 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:24:28.886 INFO:journalctl@ceph.mon.b.smithi179.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:24:28.886 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:24:28.886 INFO:journalctl@ceph.mon.b.smithi179.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:24:28.886 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:24:28.886 INFO:journalctl@ceph.mon.b.smithi179.stdout: s = self.context.wrap_socket( 2024-08-17T17:24:28.886 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:24:28.886 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.sslsocket_class._create( 2024-08-17T17:24:28.886 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:24:28.886 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.do_handshake() 2024-08-17T17:24:28.886 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:24:28.886 INFO:journalctl@ceph.mon.b.smithi179.stdout: self._sslobj.do_handshake() 2024-08-17T17:24:28.886 INFO:journalctl@ceph.mon.b.smithi179.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:24:28.887 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:28 smithi179 ceph-mon[34775]: osdmap e152: 8 total, 8 up, 8 in 2024-08-17T17:24:28.887 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:28 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3802882938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:28.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:28.490+0000 7f8bee9a9ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T17:24:28.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:28.575+0000 7f8bee9a9ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T17:24:28.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:28.635+0000 7f8bee9a9ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T17:24:28.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[31342]: pgmap v5: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:28.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[31342]: Deploying cephadm binary to smithi179 2024-08-17T17:24:28.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[31342]: Deploying cephadm binary to smithi130 2024-08-17T17:24:28.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[31342]: [17/Aug/2024:17:24:27] ENGINE Bus STARTING 2024-08-17T17:24:28.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[31342]: [17/Aug/2024:17:24:27] ENGINE Serving on http://172.21.15.179:8765 2024-08-17T17:24:28.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[31342]: [17/Aug/2024:17:24:27] ENGINE Serving on https://172.21.15.179:7150 2024-08-17T17:24:28.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[31342]: [17/Aug/2024:17:24:27] ENGINE Bus STARTED 2024-08-17T17:24:28.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[31342]: [17/Aug/2024:17:24:27] ENGINE Error in HTTPServer.serve 2024-08-17T17:24:28.922 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-08-17T17:24:28.922 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:24:28.922 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:24:28.923 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:24:28.923 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._run(expiration_interval) 2024-08-17T17:24:28.923 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:24:28.923 INFO:journalctl@ceph.mon.a.smithi130.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:24:28.923 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:24:28.924 INFO:journalctl@ceph.mon.a.smithi130.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:24:28.924 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:24:28.924 INFO:journalctl@ceph.mon.a.smithi130.stdout: s = self.context.wrap_socket( 2024-08-17T17:24:28.924 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:24:28.924 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.sslsocket_class._create( 2024-08-17T17:24:28.925 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:24:28.925 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.do_handshake() 2024-08-17T17:24:28.925 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:24:28.925 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._sslobj.do_handshake() 2024-08-17T17:24:28.925 INFO:journalctl@ceph.mon.a.smithi130.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:24:28.925 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[31342]: osdmap e152: 8 total, 8 up, 8 in 2024-08-17T17:24:28.925 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3802882938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:28.925 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[42749]: pgmap v5: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:28.925 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[42749]: Deploying cephadm binary to smithi179 2024-08-17T17:24:28.925 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[42749]: Deploying cephadm binary to smithi130 2024-08-17T17:24:28.926 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[42749]: [17/Aug/2024:17:24:27] ENGINE Bus STARTING 2024-08-17T17:24:28.926 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[42749]: [17/Aug/2024:17:24:27] ENGINE Serving on http://172.21.15.179:8765 2024-08-17T17:24:28.926 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[42749]: [17/Aug/2024:17:24:27] ENGINE Serving on https://172.21.15.179:7150 2024-08-17T17:24:28.926 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[42749]: [17/Aug/2024:17:24:27] ENGINE Bus STARTED 2024-08-17T17:24:28.926 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[42749]: [17/Aug/2024:17:24:27] ENGINE Error in HTTPServer.serve 2024-08-17T17:24:28.926 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-08-17T17:24:28.926 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:24:28.926 INFO:journalctl@ceph.mon.c.smithi130.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:24:28.926 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:24:28.926 INFO:journalctl@ceph.mon.c.smithi130.stdout: self._run(expiration_interval) 2024-08-17T17:24:28.926 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:24:28.926 INFO:journalctl@ceph.mon.c.smithi130.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:24:28.926 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:24:28.927 INFO:journalctl@ceph.mon.c.smithi130.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:24:28.927 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:24:28.927 INFO:journalctl@ceph.mon.c.smithi130.stdout: s = self.context.wrap_socket( 2024-08-17T17:24:28.927 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:24:28.927 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.sslsocket_class._create( 2024-08-17T17:24:28.927 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:24:28.927 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.do_handshake() 2024-08-17T17:24:28.927 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:24:28.927 INFO:journalctl@ceph.mon.c.smithi130.stdout: self._sslobj.do_handshake() 2024-08-17T17:24:28.927 INFO:journalctl@ceph.mon.c.smithi130.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:24:28.927 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[42749]: osdmap e152: 8 total, 8 up, 8 in 2024-08-17T17:24:28.927 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:28 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3802882938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:29.529 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:29.207+0000 7f8bee9a9ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T17:24:29.530 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:29.343+0000 7f8bee9a9ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T17:24:29.530 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:24:29] ENGINE Bus STARTING 2024-08-17T17:24:29.530 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: CherryPy Checker: 2024-08-17T17:24:29.530 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: The Application mounted at '' has an empty config. 2024-08-17T17:24:29.530 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: 2024-08-17T17:24:29.530 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:24:29] ENGINE Serving on http://:::9283 2024-08-17T17:24:29.530 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: [17/Aug/2024:17:24:29] ENGINE Bus STARTED 2024-08-17T17:24:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[31342]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[31342]: from='client.15417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[31342]: osdmap e153: 8 total, 8 up, 8 in 2024-08-17T17:24:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[31342]: mgrmap e25: x(active, since 4s) 2024-08-17T17:24:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[31342]: Standby manager daemon y started 2024-08-17T17:24:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.130:0/1532861655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:24:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.130:0/1532861655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:24:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.130:0/1532861655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:24:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.130:0/1532861655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:24:29.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[42749]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:29.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[42749]: from='client.15417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:29.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[42749]: osdmap e153: 8 total, 8 up, 8 in 2024-08-17T17:24:29.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[42749]: mgrmap e25: x(active, since 4s) 2024-08-17T17:24:29.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[42749]: Standby manager daemon y started 2024-08-17T17:24:29.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.130:0/1532861655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:24:29.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.130:0/1532861655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:24:29.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.130:0/1532861655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:24:29.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:29 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.130:0/1532861655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:24:30.083 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.SnapsDataPool (11071 ms) 2024-08-17T17:24:30.083 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.SnapsShrinkAfterPrepare 2024-08-17T17:24:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:29 smithi179 ceph-mon[34775]: from='client.15411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:29 smithi179 ceph-mon[34775]: from='client.15417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:29 smithi179 ceph-mon[34775]: osdmap e153: 8 total, 8 up, 8 in 2024-08-17T17:24:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:29 smithi179 ceph-mon[34775]: mgrmap e25: x(active, since 4s) 2024-08-17T17:24:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:29 smithi179 ceph-mon[34775]: Standby manager daemon y started 2024-08-17T17:24:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:29 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.130:0/1532861655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:24:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:29 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.130:0/1532861655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:24:30.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:29 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.130:0/1532861655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:24:30.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:29 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.130:0/1532861655' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:24:30.117 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:24:30.128 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:24:30.201 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:24:30.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:30 smithi130 ceph-mon[31342]: pgmap v8: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:30.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:30 smithi130 ceph-mon[31342]: osdmap e154: 8 total, 8 up, 8 in 2024-08-17T17:24:30.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:30 smithi130 ceph-mon[31342]: mgrmap e26: x(active, since 5s), standbys: y 2024-08-17T17:24:30.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:30 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:24:30.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:30 smithi130 ceph-mon[42749]: pgmap v8: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:30.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:30 smithi130 ceph-mon[42749]: osdmap e154: 8 total, 8 up, 8 in 2024-08-17T17:24:30.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:30 smithi130 ceph-mon[42749]: mgrmap e26: x(active, since 5s), standbys: y 2024-08-17T17:24:30.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:30 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:24:31.090 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:30 smithi179 ceph-mon[34775]: pgmap v8: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:31.090 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:30 smithi179 ceph-mon[34775]: osdmap e154: 8 total, 8 up, 8 in 2024-08-17T17:24:31.090 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:30 smithi179 ceph-mon[34775]: mgrmap e26: x(active, since 5s), standbys: y 2024-08-17T17:24:31.091 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:30 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:24:31.375 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:31.816 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 8388609~8388609 2024-08-17T17:24:31.833 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 0~1000 2024-08-17T17:24:31.833 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 1010~1000 2024-08-17T17:24:31.833 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:24:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:31 smithi179 ceph-mon[34775]: pgmap v11: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:31 smithi179 ceph-mon[34775]: osdmap e155: 8 total, 8 up, 8 in 2024-08-17T17:24:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:31 smithi179 ceph-mon[34775]: mgrmap e27: x(active, since 6s), standbys: y 2024-08-17T17:24:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:31 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2034267643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:31 smithi179 ceph-mon[34775]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:31 smithi179 ceph-mon[34775]: from='client.25343 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:31 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:31 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:32.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[31342]: pgmap v11: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:32.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[31342]: osdmap e155: 8 total, 8 up, 8 in 2024-08-17T17:24:32.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[31342]: mgrmap e27: x(active, since 6s), standbys: y 2024-08-17T17:24:32.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2034267643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:32.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[31342]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:32.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[31342]: from='client.25343 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:32.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:32.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:32.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[42749]: pgmap v11: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:32.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[42749]: osdmap e155: 8 total, 8 up, 8 in 2024-08-17T17:24:32.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[42749]: mgrmap e27: x(active, since 6s), standbys: y 2024-08-17T17:24:32.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2034267643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:32.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[42749]: from='client.15435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:32.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[42749]: from='client.25343 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:32.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:32.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:31 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: osdmap e156: 8 total, 8 up, 8 in 2024-08-17T17:24:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: Adjusting osd_memory_target on smithi130 to 2480M 2024-08-17T17:24:33.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:32 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: osdmap e156: 8 total, 8 up, 8 in 2024-08-17T17:24:33.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: Adjusting osd_memory_target on smithi130 to 2480M 2024-08-17T17:24:33.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: osdmap e156: 8 total, 8 up, 8 in 2024-08-17T17:24:33.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:33.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: Adjusting osd_memory_target on smithi130 to 2480M 2024-08-17T17:24:33.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:32 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:33.833 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 8388609~1000 2024-08-17T17:24:33.838 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 16777228~1000 2024-08-17T17:24:33.838 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 167772160 2024-08-17T17:24:33.845 INFO:tasks.workunit.client.0.smithi130.stdout:write: 2 83886079~8388609 2024-08-17T17:24:33.858 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:24:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:33 smithi179 ceph-mon[34775]: pgmap v14: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 500 op/s 2024-08-17T17:24:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:33 smithi179 ceph-mon[34775]: osdmap e157: 8 total, 8 up, 8 in 2024-08-17T17:24:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:33 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:33 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:34.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:33 smithi130 ceph-mon[31342]: pgmap v14: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 500 op/s 2024-08-17T17:24:34.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:33 smithi130 ceph-mon[31342]: osdmap e157: 8 total, 8 up, 8 in 2024-08-17T17:24:34.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:33 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:34.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:33 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:34.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:33 smithi130 ceph-mon[42749]: pgmap v14: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 500 op/s 2024-08-17T17:24:34.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:33 smithi130 ceph-mon[42749]: osdmap e157: 8 total, 8 up, 8 in 2024-08-17T17:24:34.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:33 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:34.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:33 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:34.321 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:34.666 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:34 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:24:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:24:34.690 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:34 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:24:34] "GET /metrics HTTP/1.1" 200 38357 "" "Prometheus/2.43.0" 2024-08-17T17:24:35.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:34 smithi179 ceph-mon[34775]: osdmap e158: 8 total, 8 up, 8 in 2024-08-17T17:24:35.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:34 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1315907718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:35.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:34 smithi179 ceph-mon[34775]: osdmap e159: 8 total, 8 up, 8 in 2024-08-17T17:24:35.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:34 smithi179 ceph-mon[34775]: from='client.25355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:35.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:34 smithi179 ceph-mon[34775]: from='client.25361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:35.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:34 smithi130 ceph-mon[31342]: osdmap e158: 8 total, 8 up, 8 in 2024-08-17T17:24:35.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:34 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1315907718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:35.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:34 smithi130 ceph-mon[31342]: osdmap e159: 8 total, 8 up, 8 in 2024-08-17T17:24:35.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:34 smithi130 ceph-mon[31342]: from='client.25355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:35.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:34 smithi130 ceph-mon[31342]: from='client.25361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:35.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:34 smithi130 ceph-mon[42749]: osdmap e158: 8 total, 8 up, 8 in 2024-08-17T17:24:35.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:34 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1315907718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:35.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:34 smithi130 ceph-mon[42749]: osdmap e159: 8 total, 8 up, 8 in 2024-08-17T17:24:35.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:34 smithi130 ceph-mon[42749]: from='client.25355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:35.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:34 smithi130 ceph-mon[42749]: from='client.25361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:35.188 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:24:35.206 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 83886079~1 2024-08-17T17:24:35.213 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:24:36.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: pgmap v18: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:36.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: osdmap e160: 8 total, 8 up, 8 in 2024-08-17T17:24:36.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:36.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:36.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:36.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:24:36.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:36 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:24:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: pgmap v18: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: osdmap e160: 8 total, 8 up, 8 in 2024-08-17T17:24:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:36.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:24:36.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:24:36.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: pgmap v18: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:36.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: osdmap e160: 8 total, 8 up, 8 in 2024-08-17T17:24:36.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:36.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T17:24:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:24:36.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:36 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:24:37.252 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: Adjusting osd_memory_target on smithi179 to 2480M 2024-08-17T17:24:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:24:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:24:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: osdmap e161: 8 total, 8 up, 8 in 2024-08-17T17:24:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:24:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:24:37.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:24:37.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:24:37.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:24:37.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:24:37.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/749938475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:37.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:37 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: Adjusting osd_memory_target on smithi179 to 2480M 2024-08-17T17:24:37.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:24:37.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:24:37.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: osdmap e161: 8 total, 8 up, 8 in 2024-08-17T17:24:37.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:24:37.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:24:37.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:24:37.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:24:37.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:24:37.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:24:37.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/749938475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:37.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: Adjusting osd_memory_target on smithi179 to 2480M 2024-08-17T17:24:37.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:24:37.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:24:37.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: osdmap e161: 8 total, 8 up, 8 in 2024-08-17T17:24:37.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:24:37.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:24:37.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:24:37.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:24:37.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:24:37.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:24:37.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/749938475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:37.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:37.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:37 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:38 smithi179 ceph-mon[34775]: pgmap v21: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 145 KiB/s rd, 8.2 MiB/s wr, 209 op/s 2024-08-17T17:24:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:38 smithi179 ceph-mon[34775]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T17:24:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:38 smithi179 ceph-mon[34775]: Deploying daemon alertmanager.a on smithi130 2024-08-17T17:24:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:38 smithi179 ceph-mon[34775]: osdmap e162: 8 total, 8 up, 8 in 2024-08-17T17:24:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:38 smithi179 ceph-mon[34775]: from='client.25373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:38 smithi179 ceph-mon[34775]: from='client.25379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:38.631 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 41943040 2024-08-17T17:24:38.654 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:24:38.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:38 smithi130 ceph-mon[31342]: pgmap v21: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 145 KiB/s rd, 8.2 MiB/s wr, 209 op/s 2024-08-17T17:24:38.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:38 smithi130 ceph-mon[31342]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T17:24:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:38 smithi130 ceph-mon[31342]: Deploying daemon alertmanager.a on smithi130 2024-08-17T17:24:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:38 smithi130 ceph-mon[31342]: osdmap e162: 8 total, 8 up, 8 in 2024-08-17T17:24:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:38 smithi130 ceph-mon[31342]: from='client.25373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:38 smithi130 ceph-mon[31342]: from='client.25379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:38 smithi130 ceph-mon[42749]: pgmap v21: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 145 KiB/s rd, 8.2 MiB/s wr, 209 op/s 2024-08-17T17:24:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:38 smithi130 ceph-mon[42749]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T17:24:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:38 smithi130 ceph-mon[42749]: Deploying daemon alertmanager.a on smithi130 2024-08-17T17:24:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:38 smithi130 ceph-mon[42749]: osdmap e162: 8 total, 8 up, 8 in 2024-08-17T17:24:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:38 smithi130 ceph-mon[42749]: from='client.25373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:38.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:38 smithi130 ceph-mon[42749]: from='client.25379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:38.793 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:24:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:39 smithi179 ceph-mon[34775]: osdmap e163: 8 total, 8 up, 8 in 2024-08-17T17:24:39.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:39 smithi130 ceph-mon[31342]: osdmap e163: 8 total, 8 up, 8 in 2024-08-17T17:24:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:39 smithi130 ceph-mon[42749]: osdmap e163: 8 total, 8 up, 8 in 2024-08-17T17:24:40.229 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:40 smithi179 ceph-mon[34775]: pgmap v24: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 142 KiB/s rd, 8.0 MiB/s wr, 205 op/s 2024-08-17T17:24:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:40 smithi179 ceph-mon[34775]: osdmap e164: 8 total, 8 up, 8 in 2024-08-17T17:24:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:40 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/108601648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:40.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:40 smithi130 ceph-mon[31342]: pgmap v24: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 142 KiB/s rd, 8.0 MiB/s wr, 205 op/s 2024-08-17T17:24:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:40 smithi130 ceph-mon[31342]: osdmap e164: 8 total, 8 up, 8 in 2024-08-17T17:24:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:40 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/108601648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:40 smithi130 ceph-mon[42749]: pgmap v24: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 142 KiB/s rd, 8.0 MiB/s wr, 205 op/s 2024-08-17T17:24:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:40 smithi130 ceph-mon[42749]: osdmap e164: 8 total, 8 up, 8 in 2024-08-17T17:24:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:40 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/108601648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:41.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:41 smithi179 ceph-mon[34775]: from='client.25391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:41.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:41 smithi179 ceph-mon[34775]: osdmap e165: 8 total, 8 up, 8 in 2024-08-17T17:24:41.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:41 smithi179 ceph-mon[34775]: from='client.25397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:41.514 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.SnapsShrinkAfterPrepare (11431 ms) 2024-08-17T17:24:41.514 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroBeforePrepare 2024-08-17T17:24:41.542 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:24:41.552 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:24:41.622 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:24:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:41 smithi130 ceph-mon[31342]: from='client.25391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:41 smithi130 ceph-mon[31342]: osdmap e165: 8 total, 8 up, 8 in 2024-08-17T17:24:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:41 smithi130 ceph-mon[31342]: from='client.25397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:41 smithi130 ceph-mon[42749]: from='client.25391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:41 smithi130 ceph-mon[42749]: osdmap e165: 8 total, 8 up, 8 in 2024-08-17T17:24:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:41 smithi130 ceph-mon[42749]: from='client.25397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:42.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:42 smithi130 ceph-mon[31342]: pgmap v27: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:42.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:42 smithi130 ceph-mon[31342]: osdmap e166: 8 total, 8 up, 8 in 2024-08-17T17:24:42.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:42 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:42.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:42 smithi130 ceph-mon[42749]: pgmap v27: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:42.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:42 smithi130 ceph-mon[42749]: osdmap e166: 8 total, 8 up, 8 in 2024-08-17T17:24:42.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:42 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:42.506 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:42 smithi179 ceph-mon[34775]: pgmap v27: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:42.506 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:42 smithi179 ceph-mon[34775]: osdmap e166: 8 total, 8 up, 8 in 2024-08-17T17:24:42.506 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:42 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:43.143 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:43.242 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 8388609~8388609 2024-08-17T17:24:43.252 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 0~1000 2024-08-17T17:24:43.252 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 1010~1000 2024-08-17T17:24:43.252 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:24:43.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:43 smithi130 ceph-mon[31342]: osdmap e167: 8 total, 8 up, 8 in 2024-08-17T17:24:43.314 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:43 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2253623776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:43.314 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:43 smithi130 ceph-mon[42749]: osdmap e167: 8 total, 8 up, 8 in 2024-08-17T17:24:43.315 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:43 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2253623776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:43 smithi179 ceph-mon[34775]: osdmap e167: 8 total, 8 up, 8 in 2024-08-17T17:24:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:43 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2253623776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:44.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:44 smithi179 ceph-mon[34775]: pgmap v30: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 436 op/s 2024-08-17T17:24:44.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:44 smithi179 ceph-mon[34775]: from='client.25409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:44.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:44 smithi179 ceph-mon[34775]: osdmap e168: 8 total, 8 up, 8 in 2024-08-17T17:24:44.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:44 smithi179 ceph-mon[34775]: from='client.25415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:44.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:44 smithi179 ceph-mon[34775]: osdmap e169: 8 total, 8 up, 8 in 2024-08-17T17:24:44.588 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:44 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:24:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:24:44.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:44 smithi130 ceph-mon[31342]: pgmap v30: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 436 op/s 2024-08-17T17:24:44.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:44 smithi130 ceph-mon[31342]: from='client.25409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:44.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:44 smithi130 ceph-mon[31342]: osdmap e168: 8 total, 8 up, 8 in 2024-08-17T17:24:44.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:44 smithi130 ceph-mon[31342]: from='client.25415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:44.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:44 smithi130 ceph-mon[31342]: osdmap e169: 8 total, 8 up, 8 in 2024-08-17T17:24:44.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:44 smithi130 ceph-mon[42749]: pgmap v30: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 436 op/s 2024-08-17T17:24:44.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:44 smithi130 ceph-mon[42749]: from='client.25409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:44.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:44 smithi130 ceph-mon[42749]: osdmap e168: 8 total, 8 up, 8 in 2024-08-17T17:24:44.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:44 smithi130 ceph-mon[42749]: from='client.25415 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:44.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:44 smithi130 ceph-mon[42749]: osdmap e169: 8 total, 8 up, 8 in 2024-08-17T17:24:44.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:24:44] "GET /metrics HTTP/1.1" 200 38776 "" "Prometheus/2.43.0" 2024-08-17T17:24:45.190 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 8388609~1000 2024-08-17T17:24:45.195 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 16777228~1000 2024-08-17T17:24:45.195 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 167772160 2024-08-17T17:24:45.203 INFO:tasks.workunit.client.0.smithi130.stdout:write: 2 83886079~8388609 2024-08-17T17:24:45.225 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:24:45.242 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:44 smithi130 systemd[1]: Stopping Ceph alertmanager.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:24:45.561 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:45 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[79188]: ts=2024-08-17T17:24:45.240Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:24:45.561 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:45 smithi130 podman[93299]: 2024-08-17 17:24:45.282778606 +0000 UTC m=+0.162292724 container died 434453bf091074c28a27e8b6b31733d9c7cac2d8e47b99f726fefcd7e6a7a82a (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:24:45.917 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:45 smithi130 podman[93299]: 2024-08-17 17:24:45.651522127 +0000 UTC m=+0.531036236 container cleanup 434453bf091074c28a27e8b6b31733d9c7cac2d8e47b99f726fefcd7e6a7a82a (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:24:45.917 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:45 smithi130 bash[93299]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a 2024-08-17T17:24:46.076 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:46.230 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:46 smithi130 ceph-mon[31342]: pgmap v33: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 436 op/s 2024-08-17T17:24:46.230 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:46 smithi130 ceph-mon[31342]: osdmap e170: 8 total, 8 up, 8 in 2024-08-17T17:24:46.230 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:46 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1387138476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:46.230 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:46 smithi130 ceph-mon[42749]: pgmap v33: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 436 op/s 2024-08-17T17:24:46.231 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:46 smithi130 ceph-mon[42749]: osdmap e170: 8 total, 8 up, 8 in 2024-08-17T17:24:46.231 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:46 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1387138476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:46.231 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:45 smithi130 podman[93317]: 2024-08-17 17:24:45.927975396 +0000 UTC m=+0.643290808 container remove 434453bf091074c28a27e8b6b31733d9c7cac2d8e47b99f726fefcd7e6a7a82a (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:24:46.231 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:46 smithi130 podman[93317]: 2024-08-17 17:24:46.028585392 +0000 UTC m=+0.743900801 volume remove 90c509b0ab2a09655d9619ed5c375ecd2c2e208ba40e86b2296e712c0e744382 2024-08-17T17:24:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:46 smithi179 ceph-mon[34775]: pgmap v33: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 436 op/s 2024-08-17T17:24:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:46 smithi179 ceph-mon[34775]: osdmap e170: 8 total, 8 up, 8 in 2024-08-17T17:24:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:46 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1387138476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:46.666 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:46 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-17T17:24:46.667 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:46 smithi130 systemd[1]: Stopped Ceph alertmanager.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:24:46.667 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:46 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@alertmanager.a.service: Consumed 1.167s CPU time. 2024-08-17T17:24:46.667 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:46 smithi130 systemd[1]: Starting Ceph alertmanager.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:24:46.955 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:46 smithi130 podman[93518]: 2024-08-17 17:24:46.570150595 +0000 UTC m=+0.017684197 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-08-17T17:24:46.955 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:46 smithi130 podman[93518]: 2024-08-17 17:24:46.671165439 +0000 UTC m=+0.118699064 volume create 1a974e46d1a594913a189ebbcebde784db90732e8d8be8a5f4d9580f03fa0bcf 2024-08-17T17:24:46.956 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:46 smithi130 podman[93518]: 2024-08-17 17:24:46.955448796 +0000 UTC m=+0.402982378 container create 73ea4ebcf68b62371b6f3a9b602f4da8a4a5d8caf3bf52b41612736d8edb2256 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:24:47.257 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:24:47.279 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 83886079~1 2024-08-17T17:24:47.286 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 0 2024-08-17T17:24:47.301 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:24:47.500 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:47 smithi130 ceph-mon[31342]: from='client.25427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:47.500 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:47 smithi130 ceph-mon[31342]: osdmap e171: 8 total, 8 up, 8 in 2024-08-17T17:24:47.500 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:47 smithi130 ceph-mon[31342]: from='client.25433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:47.500 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:47 smithi130 ceph-mon[42749]: from='client.25427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:47.500 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:47 smithi130 ceph-mon[42749]: osdmap e171: 8 total, 8 up, 8 in 2024-08-17T17:24:47.501 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:47 smithi130 ceph-mon[42749]: from='client.25433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:47.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:47 smithi179 ceph-mon[34775]: from='client.25427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:47.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:47 smithi179 ceph-mon[34775]: osdmap e171: 8 total, 8 up, 8 in 2024-08-17T17:24:47.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:47 smithi179 ceph-mon[34775]: from='client.25433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:47.817 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:47 smithi130 podman[93518]: 2024-08-17 17:24:47.500010871 +0000 UTC m=+0.947544463 container init 73ea4ebcf68b62371b6f3a9b602f4da8a4a5d8caf3bf52b41612736d8edb2256 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:24:47.818 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:47 smithi130 podman[93518]: 2024-08-17 17:24:47.505020189 +0000 UTC m=+0.952553777 container start 73ea4ebcf68b62371b6f3a9b602f4da8a4a5d8caf3bf52b41612736d8edb2256 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:24:47.818 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:47 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[93533]: ts=2024-08-17T17:24:47.515Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-08-17T17:24:47.818 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:47 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[93533]: ts=2024-08-17T17:24:47.515Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-08-17T17:24:47.818 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:47 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[93533]: ts=2024-08-17T17:24:47.516Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.130 port=9094 2024-08-17T17:24:47.818 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:47 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[93533]: ts=2024-08-17T17:24:47.518Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-17T17:24:47.818 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:47 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[93533]: ts=2024-08-17T17:24:47.539Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T17:24:47.818 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:47 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[93533]: ts=2024-08-17T17:24:47.539Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T17:24:47.818 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:47 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[93533]: ts=2024-08-17T17:24:47.541Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-08-17T17:24:47.818 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:47 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[93533]: ts=2024-08-17T17:24:47.541Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-17T17:24:47.818 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:47 smithi130 bash[93518]: 73ea4ebcf68b62371b6f3a9b602f4da8a4a5d8caf3bf52b41612736d8edb2256 2024-08-17T17:24:48.166 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:47 smithi130 systemd[1]: Started Ceph alertmanager.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:24:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:48 smithi179 ceph-mon[34775]: pgmap v36: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 MiB/s wr, 193 op/s 2024-08-17T17:24:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:48 smithi179 ceph-mon[34775]: osdmap e172: 8 total, 8 up, 8 in 2024-08-17T17:24:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:48 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:48 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:48.667 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:48 smithi130 systemd[1]: Stopping Ceph node-exporter.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:24:48.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:48 smithi130 ceph-mon[31342]: pgmap v36: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 MiB/s wr, 193 op/s 2024-08-17T17:24:48.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:48 smithi130 ceph-mon[31342]: osdmap e172: 8 total, 8 up, 8 in 2024-08-17T17:24:48.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:48 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:48.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:48 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:48.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:48 smithi130 ceph-mon[42749]: pgmap v36: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 MiB/s wr, 193 op/s 2024-08-17T17:24:48.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:48 smithi130 ceph-mon[42749]: osdmap e172: 8 total, 8 up, 8 in 2024-08-17T17:24:48.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:48 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:48.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:48 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:48.955 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:48 smithi130 podman[93735]: 2024-08-17 17:24:48.674553762 +0000 UTC m=+0.113594269 container died a24247d0fd94a2b5031a84685e17a8a6cd83e11e79e74a54ea3dcf2c53ca376f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:24:49.049 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:49.214 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:49 smithi130 podman[93735]: 2024-08-17 17:24:49.037235316 +0000 UTC m=+0.476275822 container cleanup a24247d0fd94a2b5031a84685e17a8a6cd83e11e79e74a54ea3dcf2c53ca376f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:24:49.214 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:49 smithi130 bash[93735]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a 2024-08-17T17:24:49.475 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:49 smithi130 podman[93751]: 2024-08-17 17:24:49.213920448 +0000 UTC m=+0.543247281 container remove a24247d0fd94a2b5031a84685e17a8a6cd83e11e79e74a54ea3dcf2c53ca376f (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:24:49.475 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:49 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:24:49.475 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:49 smithi130 ceph-mon[31342]: Reconfiguring node-exporter.a (dependencies changed)... 2024-08-17T17:24:49.475 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:49 smithi130 ceph-mon[31342]: Deploying daemon node-exporter.a on smithi130 2024-08-17T17:24:49.475 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:49 smithi130 ceph-mon[31342]: osdmap e173: 8 total, 8 up, 8 in 2024-08-17T17:24:49.476 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:49 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1661684660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:49.476 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:49 smithi130 ceph-mon[31342]: osdmap e174: 8 total, 8 up, 8 in 2024-08-17T17:24:49.476 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:49 smithi130 ceph-mon[42749]: Reconfiguring node-exporter.a (dependencies changed)... 2024-08-17T17:24:49.476 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:49 smithi130 ceph-mon[42749]: Deploying daemon node-exporter.a on smithi130 2024-08-17T17:24:49.476 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:49 smithi130 ceph-mon[42749]: osdmap e173: 8 total, 8 up, 8 in 2024-08-17T17:24:49.476 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:49 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1661684660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:49.476 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:49 smithi130 ceph-mon[42749]: osdmap e174: 8 total, 8 up, 8 in 2024-08-17T17:24:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:49 smithi179 ceph-mon[34775]: Reconfiguring node-exporter.a (dependencies changed)... 2024-08-17T17:24:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:49 smithi179 ceph-mon[34775]: Deploying daemon node-exporter.a on smithi130 2024-08-17T17:24:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:49 smithi179 ceph-mon[34775]: osdmap e173: 8 total, 8 up, 8 in 2024-08-17T17:24:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:49 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1661684660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:49 smithi179 ceph-mon[34775]: osdmap e174: 8 total, 8 up, 8 in 2024-08-17T17:24:49.881 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[93533]: ts=2024-08-17T17:24:49.518Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000030579s 2024-08-17T17:24:49.881 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:49 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-08-17T17:24:49.881 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:49 smithi130 systemd[1]: Stopped Ceph node-exporter.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:24:49.882 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:49 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@node-exporter.a.service: Consumed 2.253s CPU time. 2024-08-17T17:24:49.882 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:49 smithi130 systemd[1]: Starting Ceph node-exporter.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:24:50.167 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:49 smithi130 bash[93934]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-08-17T17:24:50.550 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:24:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:50 smithi179 ceph-mon[34775]: pgmap v39: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 MiB/s wr, 193 op/s 2024-08-17T17:24:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:50 smithi179 ceph-mon[34775]: from='client.25445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:50 smithi179 ceph-mon[34775]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:50 smithi179 ceph-mon[34775]: osdmap e175: 8 total, 8 up, 8 in 2024-08-17T17:24:50.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:50 smithi130 ceph-mon[31342]: pgmap v39: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 MiB/s wr, 193 op/s 2024-08-17T17:24:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:50 smithi130 ceph-mon[31342]: from='client.25445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:50 smithi130 ceph-mon[31342]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:50 smithi130 ceph-mon[31342]: osdmap e175: 8 total, 8 up, 8 in 2024-08-17T17:24:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:50 smithi130 ceph-mon[42749]: pgmap v39: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 MiB/s wr, 193 op/s 2024-08-17T17:24:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:50 smithi130 ceph-mon[42749]: from='client.25445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:50 smithi130 ceph-mon[42749]: from='client.15549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:50 smithi130 ceph-mon[42749]: osdmap e175: 8 total, 8 up, 8 in 2024-08-17T17:24:50.668 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:50 smithi130 bash[93934]: Getting image source signatures 2024-08-17T17:24:50.668 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:50 smithi130 bash[93934]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-08-17T17:24:50.668 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:50 smithi130 bash[93934]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-08-17T17:24:50.668 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:50 smithi130 bash[93934]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-08-17T17:24:50.693 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:24:51.992 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:52.166 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:51 smithi130 bash[93934]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-08-17T17:24:52.167 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:51 smithi130 bash[93934]: Writing manifest to image destination 2024-08-17T17:24:52.585 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-mon[31342]: pgmap v42: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:52.586 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-mon[31342]: osdmap e176: 8 total, 8 up, 8 in 2024-08-17T17:24:52.586 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1231063258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:52.586 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-mon[42749]: pgmap v42: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:52.586 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-mon[42749]: osdmap e176: 8 total, 8 up, 8 in 2024-08-17T17:24:52.586 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1231063258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:52.586 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 podman[93934]: 2024-08-17 17:24:52.234638393 +0000 UTC m=+2.448345798 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-08-17T17:24:52.587 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 podman[93934]: 2024-08-17 17:24:52.350034363 +0000 UTC m=+2.563741758 container create fd7392d87597789984f045c0c3d7a948955a4b151d8280d745ff2da3cbe23f5a (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:24:52.587 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 podman[93934]: 2024-08-17 17:24:52.585455658 +0000 UTC m=+2.799163067 container init fd7392d87597789984f045c0c3d7a948955a4b151d8280d745ff2da3cbe23f5a (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:24:52.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:52 smithi179 ceph-mon[34775]: pgmap v42: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:24:52.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:52 smithi179 ceph-mon[34775]: osdmap e176: 8 total, 8 up, 8 in 2024-08-17T17:24:52.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:52 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1231063258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:52.917 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 podman[93934]: 2024-08-17 17:24:52.590735975 +0000 UTC m=+2.804443370 container start fd7392d87597789984f045c0c3d7a948955a4b151d8280d745ff2da3cbe23f5a (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:24:52.917 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.594Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-08-17T17:24:52.917 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.594Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-08-17T17:24:52.917 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.594Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T17:24:52.917 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.594Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T17:24:52.918 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T17:24:52.918 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T17:24:52.918 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T17:24:52.918 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T17:24:52.918 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T17:24:52.918 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T17:24:52.918 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T17:24:52.918 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T17:24:52.918 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T17:24:52.918 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T17:24:52.918 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T17:24:52.918 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T17:24:52.919 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T17:24:52.919 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T17:24:52.919 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T17:24:52.919 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T17:24:52.919 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T17:24:52.919 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T17:24:52.919 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T17:24:52.919 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T17:24:52.919 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T17:24:52.919 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T17:24:52.919 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T17:24:52.919 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T17:24:52.920 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T17:24:52.920 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T17:24:52.920 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T17:24:52.920 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T17:24:52.920 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T17:24:52.920 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T17:24:52.920 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T17:24:52.920 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T17:24:52.920 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T17:24:52.920 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T17:24:52.920 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T17:24:52.921 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T17:24:52.921 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T17:24:52.921 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T17:24:52.921 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T17:24:52.921 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T17:24:52.921 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T17:24:52.921 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T17:24:52.921 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T17:24:52.921 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T17:24:52.921 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T17:24:52.921 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T17:24:52.921 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T17:24:52.922 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-08-17T17:24:52.922 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a[94055]: ts=2024-08-17T17:24:52.595Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T17:24:52.922 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 bash[93934]: fd7392d87597789984f045c0c3d7a948955a4b151d8280d745ff2da3cbe23f5a 2024-08-17T17:24:52.922 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 17:24:52 smithi130 systemd[1]: Started Ceph node-exporter.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:24:53.512 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.SnapsShrinkToZeroBeforePrepare (11998 ms) 2024-08-17T17:24:53.512 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroAfterPrepare 2024-08-17T17:24:53.541 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:24:53.551 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:24:53.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:53 smithi179 ceph-mon[34775]: from='client.25463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:53.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:53 smithi179 ceph-mon[34775]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:53.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:53 smithi179 ceph-mon[34775]: osdmap e177: 8 total, 8 up, 8 in 2024-08-17T17:24:53.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:53 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:53.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:53 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:53.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:53 smithi179 ceph-mon[34775]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:24:53.601 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:24:53.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:53 smithi130 ceph-mon[31342]: from='client.25463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:53.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:53 smithi130 ceph-mon[31342]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:53.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:53 smithi130 ceph-mon[31342]: osdmap e177: 8 total, 8 up, 8 in 2024-08-17T17:24:53.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:53 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:53.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:53 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:53.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:53 smithi130 ceph-mon[31342]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:24:53.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:53 smithi130 ceph-mon[42749]: from='client.25463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:53.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:53 smithi130 ceph-mon[42749]: from='client.15567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:53.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:53 smithi130 ceph-mon[42749]: osdmap e177: 8 total, 8 up, 8 in 2024-08-17T17:24:53.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:53 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:53.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:53 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:24:53.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:53 smithi130 ceph-mon[42749]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:24:54.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:54 smithi179 ceph-mon[34775]: pgmap v45: 137 pgs: 137 active+clean; 533 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 500 op/s 2024-08-17T17:24:54.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:54 smithi179 ceph-mon[34775]: Deploying daemon prometheus.a on smithi179 2024-08-17T17:24:54.583 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:54 smithi179 ceph-mon[34775]: osdmap e178: 8 total, 8 up, 8 in 2024-08-17T17:24:54.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:54 smithi130 ceph-mon[31342]: pgmap v45: 137 pgs: 137 active+clean; 533 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 500 op/s 2024-08-17T17:24:54.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:54 smithi130 ceph-mon[31342]: Deploying daemon prometheus.a on smithi179 2024-08-17T17:24:54.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:54 smithi130 ceph-mon[31342]: osdmap e178: 8 total, 8 up, 8 in 2024-08-17T17:24:54.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:24:54 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:24:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:24:54.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:54 smithi130 ceph-mon[42749]: pgmap v45: 137 pgs: 137 active+clean; 533 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 500 op/s 2024-08-17T17:24:54.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:54 smithi130 ceph-mon[42749]: Deploying daemon prometheus.a on smithi179 2024-08-17T17:24:54.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:54 smithi130 ceph-mon[42749]: osdmap e178: 8 total, 8 up, 8 in 2024-08-17T17:24:54.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:24:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:24:54] "GET /metrics HTTP/1.1" 200 38776 "" "Prometheus/2.43.0" 2024-08-17T17:24:54.902 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:55.315 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 8388609~8388609 2024-08-17T17:24:55.333 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 0~1000 2024-08-17T17:24:55.333 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 1010~1000 2024-08-17T17:24:55.333 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:24:55.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:55 smithi179 ceph-mon[34775]: osdmap e179: 8 total, 8 up, 8 in 2024-08-17T17:24:55.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:55 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/389363704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:55.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:55 smithi130 ceph-mon[31342]: osdmap e179: 8 total, 8 up, 8 in 2024-08-17T17:24:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:55 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/389363704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:55 smithi130 ceph-mon[42749]: osdmap e179: 8 total, 8 up, 8 in 2024-08-17T17:24:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:55 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/389363704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:56.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:56 smithi179 ceph-mon[34775]: pgmap v48: 137 pgs: 137 active+clean; 533 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 500 op/s 2024-08-17T17:24:56.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:56 smithi179 ceph-mon[34775]: from='client.15579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:56.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:56 smithi179 ceph-mon[34775]: from='client.15585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:56.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:56 smithi179 ceph-mon[34775]: osdmap e180: 8 total, 8 up, 8 in 2024-08-17T17:24:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:56 smithi130 ceph-mon[31342]: pgmap v48: 137 pgs: 137 active+clean; 533 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 500 op/s 2024-08-17T17:24:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:56 smithi130 ceph-mon[31342]: from='client.15579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:56.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:56 smithi130 ceph-mon[31342]: from='client.15585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:56.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:56 smithi130 ceph-mon[31342]: osdmap e180: 8 total, 8 up, 8 in 2024-08-17T17:24:56.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:56 smithi130 ceph-mon[42749]: pgmap v48: 137 pgs: 137 active+clean; 533 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 500 op/s 2024-08-17T17:24:56.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:56 smithi130 ceph-mon[42749]: from='client.15579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:56.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:56 smithi130 ceph-mon[42749]: from='client.15585 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:56.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:56 smithi130 ceph-mon[42749]: osdmap e180: 8 total, 8 up, 8 in 2024-08-17T17:24:57.329 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 8388609~1000 2024-08-17T17:24:57.336 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 16777228~1000 2024-08-17T17:24:57.336 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 167772160 2024-08-17T17:24:57.345 INFO:tasks.workunit.client.0.smithi130.stdout:write: 2 83886079~8388609 2024-08-17T17:24:57.358 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:24:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:57 smithi179 ceph-mon[34775]: osdmap e181: 8 total, 8 up, 8 in 2024-08-17T17:24:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:57 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:57 smithi130 ceph-mon[31342]: osdmap e181: 8 total, 8 up, 8 in 2024-08-17T17:24:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:57 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:57.667 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 17:24:57 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[93533]: ts=2024-08-17T17:24:57.520Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002550018s 2024-08-17T17:24:57.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:57 smithi130 ceph-mon[42749]: osdmap e181: 8 total, 8 up, 8 in 2024-08-17T17:24:57.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:57 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:24:57.819 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:24:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:58 smithi179 ceph-mon[34775]: pgmap v51: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-08-17T17:24:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:58 smithi179 ceph-mon[34775]: osdmap e182: 8 total, 8 up, 8 in 2024-08-17T17:24:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:58 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2999753891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:58 smithi130 ceph-mon[42749]: pgmap v51: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-08-17T17:24:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:58 smithi130 ceph-mon[42749]: osdmap e182: 8 total, 8 up, 8 in 2024-08-17T17:24:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:58 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2999753891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:58 smithi130 ceph-mon[31342]: pgmap v51: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-08-17T17:24:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:58 smithi130 ceph-mon[31342]: osdmap e182: 8 total, 8 up, 8 in 2024-08-17T17:24:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:58 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2999753891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:24:59.195 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:24:59.228 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 83886079~1 2024-08-17T17:24:59.236 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:24:59.584 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:59 smithi130 ceph-mon[31342]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:59.584 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:59 smithi130 ceph-mon[31342]: from='client.15603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:59.584 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:59 smithi130 ceph-mon[31342]: osdmap e183: 8 total, 8 up, 8 in 2024-08-17T17:24:59.584 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:24:59 smithi130 ceph-mon[31342]: osdmap e184: 8 total, 8 up, 8 in 2024-08-17T17:24:59.584 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:59 smithi130 ceph-mon[42749]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:59.584 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:59 smithi130 ceph-mon[42749]: from='client.15603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:59.585 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:59 smithi130 ceph-mon[42749]: osdmap e183: 8 total, 8 up, 8 in 2024-08-17T17:24:59.585 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:24:59 smithi130 ceph-mon[42749]: osdmap e184: 8 total, 8 up, 8 in 2024-08-17T17:24:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:59 smithi179 ceph-mon[34775]: from='client.15597 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:59 smithi179 ceph-mon[34775]: from='client.15603 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:24:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:59 smithi179 ceph-mon[34775]: osdmap e183: 8 total, 8 up, 8 in 2024-08-17T17:24:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:24:59 smithi179 ceph-mon[34775]: osdmap e184: 8 total, 8 up, 8 in 2024-08-17T17:25:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:00 smithi179 ceph-mon[34775]: pgmap v54: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-08-17T17:25:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:00 smithi179 ceph-mon[34775]: osdmap e185: 8 total, 8 up, 8 in 2024-08-17T17:25:00.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:00 smithi130 ceph-mon[31342]: pgmap v54: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-08-17T17:25:00.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:00 smithi130 ceph-mon[31342]: osdmap e185: 8 total, 8 up, 8 in 2024-08-17T17:25:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:00 smithi130 ceph-mon[42749]: pgmap v54: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 135 op/s 2024-08-17T17:25:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:00 smithi130 ceph-mon[42749]: osdmap e185: 8 total, 8 up, 8 in 2024-08-17T17:25:00.726 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:01 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1337172225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:01 smithi179 ceph-mon[34775]: from='client.25517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:01 smithi179 ceph-mon[34775]: osdmap e186: 8 total, 8 up, 8 in 2024-08-17T17:25:01.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:01 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1337172225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:01 smithi130 ceph-mon[31342]: from='client.25517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:01 smithi130 ceph-mon[31342]: osdmap e186: 8 total, 8 up, 8 in 2024-08-17T17:25:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:01 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1337172225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:01 smithi130 ceph-mon[42749]: from='client.25517 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:01 smithi130 ceph-mon[42749]: osdmap e186: 8 total, 8 up, 8 in 2024-08-17T17:25:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:02 smithi179 ceph-mon[34775]: pgmap v57: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:02 smithi179 ceph-mon[34775]: from='client.25523 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:02 smithi179 ceph-mon[34775]: osdmap e187: 8 total, 8 up, 8 in 2024-08-17T17:25:02.627 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 0 2024-08-17T17:25:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:02 smithi130 ceph-mon[31342]: pgmap v57: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:02 smithi130 ceph-mon[31342]: from='client.25523 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:02 smithi130 ceph-mon[31342]: osdmap e187: 8 total, 8 up, 8 in 2024-08-17T17:25:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:02 smithi130 ceph-mon[42749]: pgmap v57: 137 pgs: 137 active+clean; 537 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:02 smithi130 ceph-mon[42749]: from='client.25523 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:02 smithi130 ceph-mon[42749]: osdmap e187: 8 total, 8 up, 8 in 2024-08-17T17:25:02.713 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:25:02.832 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:25:03.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:03 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2232293791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:03.651 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:03.651 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:03 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2232293791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:03.651 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:03 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2232293791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:04.584 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:04 smithi179 systemd[1]: Stopping Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:25:04.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:04 smithi179 ceph-mon[34775]: pgmap v60: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-08-17T17:25:04.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:04 smithi179 ceph-mon[34775]: osdmap e188: 8 total, 8 up, 8 in 2024-08-17T17:25:04.584 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:04 smithi179 ceph-mon[34775]: from='client.15633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:04.666 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:04 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[31594]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:25:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T17:25:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:04 smithi130 ceph-mon[31342]: pgmap v60: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-08-17T17:25:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:04 smithi130 ceph-mon[31342]: osdmap e188: 8 total, 8 up, 8 in 2024-08-17T17:25:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:04 smithi130 ceph-mon[31342]: from='client.15633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:04 smithi130 ceph-mon[42749]: pgmap v60: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-08-17T17:25:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:04 smithi130 ceph-mon[42749]: osdmap e188: 8 total, 8 up, 8 in 2024-08-17T17:25:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:04 smithi130 ceph-mon[42749]: from='client.15633 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:04.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:25:04] "GET /metrics HTTP/1.1" 200 38776 "" "Prometheus/2.43.0" 2024-08-17T17:25:05.585 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.SnapsShrinkToZeroAfterPrepare (12072 ms) 2024-08-17T17:25:05.585 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare 2024-08-17T17:25:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:05 smithi179 ceph-mon[34775]: from='client.25541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:05 smithi179 ceph-mon[34775]: osdmap e189: 8 total, 8 up, 8 in 2024-08-17T17:25:05.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:25:05.326Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:25:05.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:25:05.327Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-17T17:25:05.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:25:05.327Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-17T17:25:05.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:25:05.327Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T17:25:05.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:25:05.327Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-17T17:25:05.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:25:05.327Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T17:25:05.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:25:05.327Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-17T17:25:05.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:25:05.327Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-17T17:25:05.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:25:05.327Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-17T17:25:05.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:25:05.327Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T17:25:05.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:25:05.327Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-17T17:25:05.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[67460]: ts=2024-08-17T17:25:05.327Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-17T17:25:05.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:05 smithi179 podman[79426]: 2024-08-17 17:25:05.337517407 +0000 UTC m=+1.049868292 container died ac95b2aef275041c8a7b0a327f9d9d57ca240fdf72ccd11468ec3c255c8e1e0f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:25:05.617 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:25:05.626 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:25:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:05 smithi130 ceph-mon[31342]: from='client.25541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:05 smithi130 ceph-mon[31342]: osdmap e189: 8 total, 8 up, 8 in 2024-08-17T17:25:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:05 smithi130 ceph-mon[42749]: from='client.25541 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:05 smithi130 ceph-mon[42749]: osdmap e189: 8 total, 8 up, 8 in 2024-08-17T17:25:05.688 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:25:06.542 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:06.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:06 smithi130 ceph-mon[42749]: pgmap v63: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-08-17T17:25:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:06 smithi130 ceph-mon[42749]: osdmap e190: 8 total, 8 up, 8 in 2024-08-17T17:25:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:06 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2772101759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:06 smithi130 ceph-mon[31342]: pgmap v63: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-08-17T17:25:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:06 smithi130 ceph-mon[31342]: osdmap e190: 8 total, 8 up, 8 in 2024-08-17T17:25:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:06 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2772101759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:06.676 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:06 smithi179 ceph-mon[34775]: pgmap v63: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-08-17T17:25:06.676 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:06 smithi179 ceph-mon[34775]: osdmap e190: 8 total, 8 up, 8 in 2024-08-17T17:25:06.676 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:06 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2772101759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:06.676 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:06 smithi179 podman[79426]: 2024-08-17 17:25:06.441125201 +0000 UTC m=+2.153476075 container cleanup ac95b2aef275041c8a7b0a327f9d9d57ca240fdf72ccd11468ec3c255c8e1e0f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:25:06.676 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:06 smithi179 bash[79426]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T17:25:07.021 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:06 smithi179 podman[79446]: 2024-08-17 17:25:06.675676027 +0000 UTC m=+1.339658183 container remove ac95b2aef275041c8a7b0a327f9d9d57ca240fdf72ccd11468ec3c255c8e1e0f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:25:07.021 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:06 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-17T17:25:07.021 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:06 smithi179 systemd[1]: Stopped Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:25:07.021 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:06 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@prometheus.a.service: Consumed 1.762s CPU time. 2024-08-17T17:25:07.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 systemd[1]: Starting Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:25:07.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 podman[79579]: 2024-08-17 17:25:07.138858615 +0000 UTC m=+0.018344287 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-08-17T17:25:07.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 podman[79579]: 2024-08-17 17:25:07.251931088 +0000 UTC m=+0.131416749 container create fb1a2c10ce69f7c022d338076f2168bb506c296c54873457f567b09936ff79b9 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:25:07.371 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 8388609~8388609 2024-08-17T17:25:07.396 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 0~1000 2024-08-17T17:25:07.396 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 1010~1000 2024-08-17T17:25:07.396 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:25:07.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:07 smithi130 ceph-mon[31342]: osdmap e191: 8 total, 8 up, 8 in 2024-08-17T17:25:07.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:07 smithi130 ceph-mon[31342]: from='client.25553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:07 smithi130 ceph-mon[42749]: osdmap e191: 8 total, 8 up, 8 in 2024-08-17T17:25:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:07 smithi130 ceph-mon[42749]: from='client.25553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-mon[34775]: osdmap e191: 8 total, 8 up, 8 in 2024-08-17T17:25:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-mon[34775]: from='client.25553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:07.843 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 podman[79579]: 2024-08-17 17:25:07.479366437 +0000 UTC m=+0.358852109 container init fb1a2c10ce69f7c022d338076f2168bb506c296c54873457f567b09936ff79b9 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:25:07.844 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 podman[79579]: 2024-08-17 17:25:07.48425729 +0000 UTC m=+0.363742956 container start fb1a2c10ce69f7c022d338076f2168bb506c296c54873457f567b09936ff79b9 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:25:07.844 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.503Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-08-17T17:25:07.844 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.503Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-08-17T17:25:07.844 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.503Z caller=main.go:623 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi179 (none))" 2024-08-17T17:25:07.844 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.503Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T17:25:07.844 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.503Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T17:25:07.844 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.505Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T17:25:07.844 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.505Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-08-17T17:25:07.844 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.508Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T17:25:07.844 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.508Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T17:25:07.844 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.509Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T17:25:07.845 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.509Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.76µs 2024-08-17T17:25:07.845 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.509Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T17:25:07.845 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.510Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-08-17T17:25:07.845 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.521Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-08-17T17:25:07.845 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.521Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-08-17T17:25:07.845 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.521Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=23.22µs wal_replay_duration=12.065097ms wbl_replay_duration=105ns total_replay_duration=12.1056ms 2024-08-17T17:25:07.845 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.524Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T17:25:07.845 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.524Z caller=main.go:1153 level=info msg="TSDB started" 2024-08-17T17:25:07.845 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.524Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T17:25:07.845 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.541Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.159021ms db_storage=909ns remote_storage=1.132µs web_handler=350ns query_engine=659ns scrape=442.645µs scrape_sd=286.125µs notify=10.25µs notify_sd=8.956µs rules=16.061439ms tracing=6.091µs 2024-08-17T17:25:07.845 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.541Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-08-17T17:25:07.845 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:07.541Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T17:25:07.845 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 bash[79579]: fb1a2c10ce69f7c022d338076f2168bb506c296c54873457f567b09936ff79b9 2024-08-17T17:25:07.846 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:07 smithi179 systemd[1]: Started Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:25:08.542 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:08 smithi179 ceph-mon[34775]: pgmap v66: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 538 op/s 2024-08-17T17:25:08.542 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:08 smithi179 ceph-mon[34775]: from='client.25559 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:08.542 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:08 smithi179 ceph-mon[34775]: osdmap e192: 8 total, 8 up, 8 in 2024-08-17T17:25:08.542 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:08 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:08.542 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:08 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:08.542 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:08 smithi179 ceph-mon[34775]: Reconfiguring grafana.a (dependencies changed)... 2024-08-17T17:25:08.543 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:08 smithi179 ceph-mon[34775]: Reconfiguring daemon grafana.a on smithi179 2024-08-17T17:25:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:08 smithi130 ceph-mon[42749]: pgmap v66: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 538 op/s 2024-08-17T17:25:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:08 smithi130 ceph-mon[42749]: from='client.25559 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:08 smithi130 ceph-mon[42749]: osdmap e192: 8 total, 8 up, 8 in 2024-08-17T17:25:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:08 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:08 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:08 smithi130 ceph-mon[42749]: Reconfiguring grafana.a (dependencies changed)... 2024-08-17T17:25:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:08 smithi130 ceph-mon[42749]: Reconfiguring daemon grafana.a on smithi179 2024-08-17T17:25:08.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:08 smithi130 ceph-mon[31342]: pgmap v66: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 538 op/s 2024-08-17T17:25:08.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:08 smithi130 ceph-mon[31342]: from='client.25559 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:08.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:08 smithi130 ceph-mon[31342]: osdmap e192: 8 total, 8 up, 8 in 2024-08-17T17:25:08.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:08 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:08.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:08 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:08.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:08 smithi130 ceph-mon[31342]: Reconfiguring grafana.a (dependencies changed)... 2024-08-17T17:25:08.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:08 smithi130 ceph-mon[31342]: Reconfiguring daemon grafana.a on smithi179 2024-08-17T17:25:09.194 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 8388609~1000 2024-08-17T17:25:09.202 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 16777228~1000 2024-08-17T17:25:09.202 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 167772160 2024-08-17T17:25:09.209 INFO:tasks.workunit.client.0.smithi130.stdout:write: 2 83886079~8388609 2024-08-17T17:25:09.229 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:25:09.463 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:09.569 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:09 smithi179 ceph-mon[34775]: osdmap e193: 8 total, 8 up, 8 in 2024-08-17T17:25:09.569 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:09 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3574600470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:09.569 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:09 smithi179 ceph-mon[34775]: osdmap e194: 8 total, 8 up, 8 in 2024-08-17T17:25:09.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:09 smithi130 ceph-mon[31342]: osdmap e193: 8 total, 8 up, 8 in 2024-08-17T17:25:09.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:09 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3574600470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:09.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:09 smithi130 ceph-mon[31342]: osdmap e194: 8 total, 8 up, 8 in 2024-08-17T17:25:09.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:09 smithi130 ceph-mon[42749]: osdmap e193: 8 total, 8 up, 8 in 2024-08-17T17:25:09.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:09 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3574600470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:09.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:09 smithi130 ceph-mon[42749]: osdmap e194: 8 total, 8 up, 8 in 2024-08-17T17:25:10.609 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:10 smithi179 systemd[1]: Stopping Ceph grafana.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:25:10.610 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:10 smithi179 ceph-mon[34775]: pgmap v69: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 538 op/s 2024-08-17T17:25:10.610 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:10 smithi179 ceph-mon[34775]: from='client.15669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:10.610 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:10 smithi179 ceph-mon[34775]: from='client.15675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:10.610 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:10 smithi179 ceph-mon[34775]: osdmap e195: 8 total, 8 up, 8 in 2024-08-17T17:25:10.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:10 smithi130 ceph-mon[31342]: pgmap v69: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 538 op/s 2024-08-17T17:25:10.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:10 smithi130 ceph-mon[31342]: from='client.15669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:10.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:10 smithi130 ceph-mon[31342]: from='client.15675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:10.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:10 smithi130 ceph-mon[31342]: osdmap e195: 8 total, 8 up, 8 in 2024-08-17T17:25:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:10 smithi130 ceph-mon[42749]: pgmap v69: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 66 MiB/s rd, 10 MiB/s wr, 538 op/s 2024-08-17T17:25:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:10 smithi130 ceph-mon[42749]: from='client.15669 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:10 smithi130 ceph-mon[42749]: from='client.15675 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:10 smithi130 ceph-mon[42749]: osdmap e195: 8 total, 8 up, 8 in 2024-08-17T17:25:10.889 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:10 smithi179 podman[79873]: 2024-08-17 17:25:10.60888052 +0000 UTC m=+0.115006982 container died 8ed57a40b09e83625d1afd3efc7d82d827799b789cd1733bf490dabb3c809eb2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, version=8.5, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, io.buildah.version=1.32.0, maintainer=Paul Cuzner , release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664) 2024-08-17T17:25:11.199 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:25:11.220 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 83886079~1 2024-08-17T17:25:11.228 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:25:11.240 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:10 smithi179 podman[79873]: 2024-08-17 17:25:10.971634063 +0000 UTC m=+0.477760516 container cleanup 8ed57a40b09e83625d1afd3efc7d82d827799b789cd1733bf490dabb3c809eb2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, distribution-scope=public, io.buildah.version=1.32.0, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, release=239.1651231664, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, vendor=Red Hat, Inc., version=8.5) 2024-08-17T17:25:11.240 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:10 smithi179 bash[79873]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a 2024-08-17T17:25:11.593 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 podman[79890]: 2024-08-17 17:25:11.239729248 +0000 UTC m=+0.633262922 container remove 8ed57a40b09e83625d1afd3efc7d82d827799b789cd1733bf490dabb3c809eb2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a, distribution-scope=public, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, version=8.5, io.openshift.expose-services=, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0) 2024-08-17T17:25:11.593 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:25:11.593 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-08-17T17:25:11.593 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 systemd[1]: Stopped Ceph grafana.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:25:11.593 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@grafana.a.service: Consumed 5.304s CPU time. 2024-08-17T17:25:11.594 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 systemd[1]: Starting Ceph grafana.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:25:11.919 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 podman[79988]: 2024-08-17 17:25:11.511604421 +0000 UTC m=+0.016393632 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-08-17T17:25:11.919 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 podman[79988]: 2024-08-17 17:25:11.619998333 +0000 UTC m=+0.124787533 container create 399f89cc0037a839cab9284c7043df281fd15a93122b75ef9312b690844c92c2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , version=8.5, vendor=Red Hat, Inc., description=Ceph Grafana Container, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, vcs-type=git, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=) 2024-08-17T17:25:11.919 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 podman[79988]: 2024-08-17 17:25:11.818276304 +0000 UTC m=+0.323065501 container init 399f89cc0037a839cab9284c7043df281fd15a93122b75ef9312b690844c92c2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.openshift.expose-services=, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, vendor=Red Hat, Inc., vcs-type=git, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-08-17T17:25:11.919 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 podman[79988]: 2024-08-17 17:25:11.82321286 +0000 UTC m=+0.328002061 container start 399f89cc0037a839cab9284c7043df281fd15a93122b75ef9312b690844c92c2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, io.openshift.expose-services=, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64) 2024-08-17T17:25:11.919 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 bash[79988]: 399f89cc0037a839cab9284c7043df281fd15a93122b75ef9312b690844c92c2 2024-08-17T17:25:12.184 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 systemd[1]: Started Ceph grafana.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:25:12.184 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=settings t=2024-08-17T17:25:11.976869419Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-08-17T17:25:12.184 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=settings t=2024-08-17T17:25:11.977093883Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-08-17T17:25:12.184 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=settings t=2024-08-17T17:25:11.977117203Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-08-17T17:25:12.184 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=settings t=2024-08-17T17:25:11.977126933Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-08-17T17:25:12.184 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=settings t=2024-08-17T17:25:11.977135163Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-08-17T17:25:12.185 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=settings t=2024-08-17T17:25:11.977144385Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-08-17T17:25:12.185 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=settings t=2024-08-17T17:25:11.977152364Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-08-17T17:25:12.185 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=settings t=2024-08-17T17:25:11.977160472Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-08-17T17:25:12.185 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=settings t=2024-08-17T17:25:11.977168642Z level=info msg="Path Home" path=/usr/share/grafana 2024-08-17T17:25:12.185 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=settings t=2024-08-17T17:25:11.977177398Z level=info msg="Path Data" path=/var/lib/grafana 2024-08-17T17:25:12.185 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=settings t=2024-08-17T17:25:11.97718692Z level=info msg="Path Logs" path=/var/log/grafana 2024-08-17T17:25:12.185 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=settings t=2024-08-17T17:25:11.977195107Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-08-17T17:25:12.185 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=settings t=2024-08-17T17:25:11.977203128Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-08-17T17:25:12.186 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=settings t=2024-08-17T17:25:11.977211193Z level=info msg="App mode production" 2024-08-17T17:25:12.186 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=sqlstore t=2024-08-17T17:25:11.977268837Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-08-17T17:25:12.186 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=sqlstore t=2024-08-17T17:25:11.977288627Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-08-17T17:25:12.186 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=migrator t=2024-08-17T17:25:11.991358946Z level=info msg="Starting DB migrations" 2024-08-17T17:25:12.186 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=migrator t=2024-08-17T17:25:11.996367007Z level=info msg="migrations completed" performed=0 skipped=481 duration=528.968µs 2024-08-17T17:25:12.186 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=sqlstore t=2024-08-17T17:25:11.9967608Z level=info msg="Created default organization" 2024-08-17T17:25:12.186 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:12 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=plugin.loader t=2024-08-17T17:25:12.05728874Z level=info msg="Plugin registered" pluginID=input 2024-08-17T17:25:12.186 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:12 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=plugin.loader t=2024-08-17T17:25:12.087266446Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-08-17T17:25:12.186 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:12 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=plugin.loader t=2024-08-17T17:25:12.087303656Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-08-17T17:25:12.186 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:12 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=secrets t=2024-08-17T17:25:12.087357741Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-08-17T17:25:12.186 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:12 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=query_data t=2024-08-17T17:25:12.089140223Z level=info msg="Query Service initialization" 2024-08-17T17:25:12.186 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:12 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=live.push_http t=2024-08-17T17:25:12.091611554Z level=info msg="Live Push Gateway initialization" 2024-08-17T17:25:12.369 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:12.531 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:12 smithi179 ceph-mon[34775]: pgmap v72: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:12.531 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:12 smithi179 ceph-mon[34775]: osdmap e196: 8 total, 8 up, 8 in 2024-08-17T17:25:12.531 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:12 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:25:12.531 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:12 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:12.531 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:12 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:12.531 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:12 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/4204455385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:12.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:12 smithi130 ceph-mon[42749]: pgmap v72: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:12.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:12 smithi130 ceph-mon[42749]: osdmap e196: 8 total, 8 up, 8 in 2024-08-17T17:25:12.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:12 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:25:12.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:12 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:12.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:12 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:12.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:12 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/4204455385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:12.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:12 smithi130 ceph-mon[31342]: pgmap v72: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:12.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:12 smithi130 ceph-mon[31342]: osdmap e196: 8 total, 8 up, 8 in 2024-08-17T17:25:12.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:12 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:25:12.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:12 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:12.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:12 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:12.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:12 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/4204455385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:12.843 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:12 smithi179 systemd[1]: Stopping Ceph node-exporter.b for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:25:13.193 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:13 smithi179 podman[80190]: 2024-08-17 17:25:13.069115003 +0000 UTC m=+0.347845119 container died ed661fd8fe9402a13befb1fcf40f18787ca9472e6f8380f76ccdcc80903dadba (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:25:13.194 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:12 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=infra.usagestats.collector t=2024-08-17T17:25:12.903042069Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-08-17T17:25:13.194 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=provisioning.datasources t=2024-08-17T17:25:13.028773322Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-08-17T17:25:13.194 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=provisioning.datasources t=2024-08-17T17:25:13.028807224Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-08-17T17:25:13.194 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=provisioning.datasources t=2024-08-17T17:25:13.038952679Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-08-17T17:25:13.194 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=provisioning.alerting t=2024-08-17T17:25:13.120670623Z level=info msg="starting to provision alerting" 2024-08-17T17:25:13.194 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=provisioning.alerting t=2024-08-17T17:25:13.120716277Z level=info msg="finished to provision alerting" 2024-08-17T17:25:13.194 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=grafanaStorageLogger t=2024-08-17T17:25:13.120884199Z level=info msg="storage starting" 2024-08-17T17:25:13.194 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=ngalert.state.manager t=2024-08-17T17:25:13.121120739Z level=info msg="Warming state cache for startup" 2024-08-17T17:25:13.194 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=http.server t=2024-08-17T17:25:13.123340613Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-08-17T17:25:13.194 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=ngalert.state.manager t=2024-08-17T17:25:13.131821738Z level=info msg="State cache has been initialized" states=0 duration=10.700582ms 2024-08-17T17:25:13.194 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=ticker t=2024-08-17T17:25:13.131879957Z level=info msg=starting first_tick=2024-08-17T17:25:20Z 2024-08-17T17:25:13.195 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=ngalert.multiorg.alertmanager t=2024-08-17T17:25:13.131868526Z level=info msg="starting MultiOrg Alertmanager" 2024-08-17T17:25:13.483 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-mon[34775]: Reconfiguring node-exporter.b (dependencies changed)... 2024-08-17T17:25:13.483 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-mon[34775]: Deploying daemon node-exporter.b on smithi179 2024-08-17T17:25:13.483 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-mon[34775]: osdmap e197: 8 total, 8 up, 8 in 2024-08-17T17:25:13.483 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-mon[34775]: from='client.25589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:13.483 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:13 smithi179 ceph-mon[34775]: from='client.25595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:13.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:13 smithi130 ceph-mon[31342]: Reconfiguring node-exporter.b (dependencies changed)... 2024-08-17T17:25:13.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:13 smithi130 ceph-mon[31342]: Deploying daemon node-exporter.b on smithi179 2024-08-17T17:25:13.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:13 smithi130 ceph-mon[31342]: osdmap e197: 8 total, 8 up, 8 in 2024-08-17T17:25:13.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:13 smithi130 ceph-mon[31342]: from='client.25589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:13.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:13 smithi130 ceph-mon[31342]: from='client.25595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:13.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:13 smithi130 ceph-mon[42749]: Reconfiguring node-exporter.b (dependencies changed)... 2024-08-17T17:25:13.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:13 smithi130 ceph-mon[42749]: Deploying daemon node-exporter.b on smithi179 2024-08-17T17:25:13.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:13 smithi130 ceph-mon[42749]: osdmap e197: 8 total, 8 up, 8 in 2024-08-17T17:25:13.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:13 smithi130 ceph-mon[42749]: from='client.25589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:13.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:13 smithi130 ceph-mon[42749]: from='client.25595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:13.843 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:13 smithi179 podman[80190]: 2024-08-17 17:25:13.482656031 +0000 UTC m=+0.761386145 container cleanup ed661fd8fe9402a13befb1fcf40f18787ca9472e6f8380f76ccdcc80903dadba (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:25:13.843 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:13 smithi179 bash[80190]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b 2024-08-17T17:25:13.843 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:13 smithi179 podman[80209]: 2024-08-17 17:25:13.650746278 +0000 UTC m=+0.585181253 container remove ed661fd8fe9402a13befb1fcf40f18787ca9472e6f8380f76ccdcc80903dadba (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:25:13.843 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:13 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:25:14.205 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:13 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-08-17T17:25:14.205 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:13 smithi179 systemd[1]: Stopped Ceph node-exporter.b for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:25:14.205 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:13 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@node-exporter.b.service: Consumed 2.226s CPU time. 2024-08-17T17:25:14.205 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:14 smithi179 systemd[1]: Starting Ceph node-exporter.b for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:25:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:14 smithi179 ceph-mon[34775]: pgmap v75: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-08-17T17:25:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:14 smithi179 ceph-mon[34775]: osdmap e198: 8 total, 8 up, 8 in 2024-08-17T17:25:14.593 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:14 smithi179 bash[80341]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-08-17T17:25:14.631 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 167772160 2024-08-17T17:25:14.641 INFO:tasks.workunit.client.0.smithi130.stdout:write: * 83886080~1000 2024-08-17T17:25:14.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:14 smithi130 ceph-mon[31342]: pgmap v75: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-08-17T17:25:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:14 smithi130 ceph-mon[31342]: osdmap e198: 8 total, 8 up, 8 in 2024-08-17T17:25:14.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:14 smithi130 ceph-mon[42749]: pgmap v75: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-08-17T17:25:14.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:14 smithi130 ceph-mon[42749]: osdmap e198: 8 total, 8 up, 8 in 2024-08-17T17:25:14.679 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:25:14.816 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:25:15.093 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:14 smithi179 bash[80341]: Getting image source signatures 2024-08-17T17:25:15.093 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:14 smithi179 bash[80341]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-08-17T17:25:15.093 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:14 smithi179 bash[80341]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-08-17T17:25:15.094 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:14 smithi179 bash[80341]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-08-17T17:25:15.299 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:15.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:15 smithi179 ceph-mon[34775]: osdmap e199: 8 total, 8 up, 8 in 2024-08-17T17:25:15.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:15 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3140940608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:15.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:15 smithi130 ceph-mon[31342]: osdmap e199: 8 total, 8 up, 8 in 2024-08-17T17:25:15.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:15 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3140940608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:15 smithi130 ceph-mon[42749]: osdmap e199: 8 total, 8 up, 8 in 2024-08-17T17:25:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:15 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3140940608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:16.222 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:15 smithi179 bash[80341]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-08-17T17:25:16.222 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 bash[80341]: Writing manifest to image destination 2024-08-17T17:25:16.579 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-mon[34775]: pgmap v78: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-08-17T17:25:16.579 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-mon[34775]: osdmap e200: 8 total, 8 up, 8 in 2024-08-17T17:25:16.580 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-mon[34775]: from='client.15705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:16.580 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-mon[34775]: from='client.25613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:16.580 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 podman[80341]: 2024-08-17 17:25:16.371248838 +0000 UTC m=+2.163251539 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-08-17T17:25:16.580 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 podman[80341]: 2024-08-17 17:25:16.46123663 +0000 UTC m=+2.253239316 container create c10588f17ed83aab23e10a7e4dabc369e00fe54a6f76dd2b3fbfb21b4526ff9f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:25:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:16 smithi130 ceph-mon[31342]: pgmap v78: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-08-17T17:25:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:16 smithi130 ceph-mon[31342]: osdmap e200: 8 total, 8 up, 8 in 2024-08-17T17:25:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:16 smithi130 ceph-mon[31342]: from='client.15705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:16 smithi130 ceph-mon[31342]: from='client.25613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:16 smithi130 ceph-mon[42749]: pgmap v78: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-08-17T17:25:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:16 smithi130 ceph-mon[42749]: osdmap e200: 8 total, 8 up, 8 in 2024-08-17T17:25:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:16 smithi130 ceph-mon[42749]: from='client.15705 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:16 smithi130 ceph-mon[42749]: from='client.25613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:17.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:17.094 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 podman[80341]: 2024-08-17 17:25:16.705061621 +0000 UTC m=+2.497064325 container init c10588f17ed83aab23e10a7e4dabc369e00fe54a6f76dd2b3fbfb21b4526ff9f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:25:17.094 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 podman[80341]: 2024-08-17 17:25:16.709895109 +0000 UTC m=+2.501897796 container start c10588f17ed83aab23e10a7e4dabc369e00fe54a6f76dd2b3fbfb21b4526ff9f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:25:17.094 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.713Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-08-17T17:25:17.095 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.713Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-08-17T17:25:17.095 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.714Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T17:25:17.095 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.714Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T17:25:17.095 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.714Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T17:25:17.095 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.714Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T17:25:17.095 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.714Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T17:25:17.096 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.714Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T17:25:17.096 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.714Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T17:25:17.096 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.714Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T17:25:17.096 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.714Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T17:25:17.096 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.714Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T17:25:17.096 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.714Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T17:25:17.097 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T17:25:17.097 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T17:25:17.097 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T17:25:17.097 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T17:25:17.097 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T17:25:17.097 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T17:25:17.098 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T17:25:17.098 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T17:25:17.098 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T17:25:17.098 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T17:25:17.098 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T17:25:17.099 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T17:25:17.099 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T17:25:17.099 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T17:25:17.099 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T17:25:17.099 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T17:25:17.099 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T17:25:17.100 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T17:25:17.100 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T17:25:17.100 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T17:25:17.100 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T17:25:17.100 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T17:25:17.100 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T17:25:17.101 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T17:25:17.101 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T17:25:17.101 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T17:25:17.101 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T17:25:17.102 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T17:25:17.102 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T17:25:17.102 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T17:25:17.102 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T17:25:17.102 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T17:25:17.103 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T17:25:17.103 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T17:25:17.103 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T17:25:17.103 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T17:25:17.104 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T17:25:17.104 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T17:25:17.104 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-08-17T17:25:17.104 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b[80391]: ts=2024-08-17T17:25:16.715Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T17:25:17.104 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 bash[80341]: c10588f17ed83aab23e10a7e4dabc369e00fe54a6f76dd2b3fbfb21b4526ff9f 2024-08-17T17:25:17.105 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 17:25:16 smithi179 systemd[1]: Started Ceph node-exporter.b for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:25:17.547 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare (11963 ms) 2024-08-17T17:25:17.548 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare2 2024-08-17T17:25:17.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:17 smithi130 ceph-mon[31342]: osdmap e201: 8 total, 8 up, 8 in 2024-08-17T17:25:17.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:17 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:17.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:17 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:17.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:17 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:25:17.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:17 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:25:17.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:17 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:25:17.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:17 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:17.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:17 smithi130 ceph-mon[42749]: osdmap e201: 8 total, 8 up, 8 in 2024-08-17T17:25:17.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:17 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:17.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:17 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:17.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:17 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:25:17.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:17 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:25:17.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:17 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:25:17.559 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:17 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:17.579 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:25:17.591 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 41943040~10 2024-08-17T17:25:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:17 smithi179 ceph-mon[34775]: osdmap e201: 8 total, 8 up, 8 in 2024-08-17T17:25:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:17 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:17 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:17.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:17 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:25:17.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:17 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:25:17.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:17 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:25:17.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:17 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:17.599 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:25:18.200 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:18.327 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[31342]: pgmap v81: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 528 op/s 2024-08-17T17:25:18.327 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[31342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:25:18.327 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[31342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:25:18.328 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[31342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:25:18.328 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[31342]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:25:18.328 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[31342]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:25:18.328 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[31342]: osdmap e202: 8 total, 8 up, 8 in 2024-08-17T17:25:18.328 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/885808588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:18.328 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[42749]: pgmap v81: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 528 op/s 2024-08-17T17:25:18.328 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[42749]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:25:18.328 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[42749]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:25:18.328 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[42749]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:25:18.328 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[42749]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:25:18.328 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[42749]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:25:18.329 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[42749]: osdmap e202: 8 total, 8 up, 8 in 2024-08-17T17:25:18.329 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:18 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/885808588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:18 smithi179 ceph-mon[34775]: pgmap v81: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 528 op/s 2024-08-17T17:25:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:18 smithi179 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T17:25:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:18 smithi179 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:25:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:18 smithi179 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T17:25:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:18 smithi179 ceph-mon[34775]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:25:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:18 smithi179 ceph-mon[34775]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T17:25:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:18 smithi179 ceph-mon[34775]: osdmap e202: 8 total, 8 up, 8 in 2024-08-17T17:25:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:18 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/885808588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:19.200 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 41943040 2024-08-17T17:25:19.218 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:25:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[42749]: from='client.25625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[42749]: osdmap e203: 8 total, 8 up, 8 in 2024-08-17T17:25:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[42749]: Upgrade: Updating mgr.y 2024-08-17T17:25:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:25:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:25:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:25:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:25:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[42749]: Deploying daemon mgr.y on smithi130 2024-08-17T17:25:19.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[42749]: from='client.25631 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:19.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[42749]: osdmap e204: 8 total, 8 up, 8 in 2024-08-17T17:25:19.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[31342]: from='client.25625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:19.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[31342]: osdmap e203: 8 total, 8 up, 8 in 2024-08-17T17:25:19.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[31342]: Upgrade: Updating mgr.y 2024-08-17T17:25:19.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:19.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:25:19.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:25:19.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:25:19.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:25:19.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[31342]: Deploying daemon mgr.y on smithi130 2024-08-17T17:25:19.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[31342]: from='client.25631 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:19.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:19 smithi130 ceph-mon[31342]: osdmap e204: 8 total, 8 up, 8 in 2024-08-17T17:25:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:19 smithi179 ceph-mon[34775]: from='client.25625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:19 smithi179 ceph-mon[34775]: osdmap e203: 8 total, 8 up, 8 in 2024-08-17T17:25:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:19 smithi179 ceph-mon[34775]: Upgrade: Updating mgr.y 2024-08-17T17:25:19.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:19 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:19.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:19 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:25:19.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:19 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:25:19.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:19 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:25:19.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:19 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:25:19.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:19 smithi179 ceph-mon[34775]: Deploying daemon mgr.y on smithi130 2024-08-17T17:25:19.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:19 smithi179 ceph-mon[34775]: from='client.25631 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:19.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:19 smithi179 ceph-mon[34775]: osdmap e204: 8 total, 8 up, 8 in 2024-08-17T17:25:20.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:25:20] "GET /metrics HTTP/1.1" 200 38771 "" "Prometheus/2.51.0" 2024-08-17T17:25:20.396 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:25:20.405 INFO:tasks.workunit.client.0.smithi130.stdout:write: Y 41943040~5 2024-08-17T17:25:20.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:20 smithi130 ceph-mon[31342]: pgmap v84: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 528 op/s 2024-08-17T17:25:20.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:20 smithi130 ceph-mon[31342]: osdmap e205: 8 total, 8 up, 8 in 2024-08-17T17:25:20.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:20 smithi130 systemd[1]: Stopping Ceph mgr.y for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:25:20.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:20 smithi130 ceph-mon[42749]: pgmap v84: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 528 op/s 2024-08-17T17:25:20.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:20 smithi130 ceph-mon[42749]: osdmap e205: 8 total, 8 up, 8 in 2024-08-17T17:25:20.518 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:25:20.595 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:25:20.749 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:20 smithi130 podman[95105]: 2024-08-17 17:25:20.439007596 +0000 UTC m=+0.158099387 container died 5e2d324fc1a5ad19fec5c03e6b27b911408dda3a94d2f261a31233b44011238d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-08-17T17:25:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:20 smithi179 ceph-mon[34775]: pgmap v84: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 528 op/s 2024-08-17T17:25:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:20 smithi179 ceph-mon[34775]: osdmap e205: 8 total, 8 up, 8 in 2024-08-17T17:25:21.057 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:20 smithi130 podman[95105]: 2024-08-17 17:25:20.855307268 +0000 UTC m=+0.574399057 container cleanup 5e2d324fc1a5ad19fec5c03e6b27b911408dda3a94d2f261a31233b44011238d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-08-17T17:25:21.057 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:20 smithi130 bash[95105]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y 2024-08-17T17:25:21.147 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:21.317 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:21 smithi130 podman[95123]: 2024-08-17 17:25:21.056864998 +0000 UTC m=+0.618463344 container remove 5e2d324fc1a5ad19fec5c03e6b27b911408dda3a94d2f261a31233b44011238d (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-08-17T17:25:21.317 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:21 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:25:21.486 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare2 (3938 ms) 2024-08-17T17:25:21.486 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.SnapsSnapAfterPrepare 2024-08-17T17:25:21.520 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:25:21.531 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:25:21.603 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:25:21.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:21 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2209591664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:21 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2209591664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:21.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:21 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-08-17T17:25:21.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:21 smithi130 systemd[1]: Stopped Ceph mgr.y for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:25:21.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:21 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.y.service: Consumed 40.217s CPU time. 2024-08-17T17:25:21.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:21 smithi130 systemd[1]: Starting Ceph mgr.y for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:25:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:21 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2209591664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:22.027 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:21 smithi130 podman[95330]: 2024-08-17 17:25:21.677876327 +0000 UTC m=+0.017015485 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:25:22.028 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:21 smithi130 podman[95330]: 2024-08-17 17:25:21.799976467 +0000 UTC m=+0.139115600 container create 584426b3875b678837089bbc161553d2e12545075efb0e11cffda8c96414307c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y, io.buildah.version=1.36.0, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-08-17T17:25:22.311 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:22 smithi130 podman[95330]: 2024-08-17 17:25:22.027461787 +0000 UTC m=+0.366600937 container init 584426b3875b678837089bbc161553d2e12545075efb0e11cffda8c96414307c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe) 2024-08-17T17:25:22.311 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:22 smithi130 podman[95330]: 2024-08-17 17:25:22.032196566 +0000 UTC m=+0.371335699 container start 584426b3875b678837089bbc161553d2e12545075efb0e11cffda8c96414307c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe) 2024-08-17T17:25:22.311 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:22 smithi130 ceph-mgr[95367]: -- 172.21.15.130:0/3154303420 <== mon.0 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5612643e30e0 con 0x5612643c0800 2024-08-17T17:25:22.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:22 smithi130 ceph-mon[31342]: pgmap v87: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:22.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:22 smithi130 ceph-mon[31342]: from='client.25643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:22.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:22 smithi130 ceph-mon[31342]: osdmap e206: 8 total, 8 up, 8 in 2024-08-17T17:25:22.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:22 smithi130 ceph-mon[31342]: from='client.25649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:22.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:22 smithi130 bash[95330]: 584426b3875b678837089bbc161553d2e12545075efb0e11cffda8c96414307c 2024-08-17T17:25:22.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:22 smithi130 systemd[1]: Started Ceph mgr.y for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:25:22.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:22 smithi130 ceph-mon[42749]: pgmap v87: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:22.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:22 smithi130 ceph-mon[42749]: from='client.25643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:22.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:22 smithi130 ceph-mon[42749]: osdmap e206: 8 total, 8 up, 8 in 2024-08-17T17:25:22.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:22 smithi130 ceph-mon[42749]: from='client.25649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:22.829 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:22 smithi179 ceph-mon[34775]: pgmap v87: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:22.829 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:22 smithi179 ceph-mon[34775]: from='client.25643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:22.829 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:22 smithi179 ceph-mon[34775]: osdmap e206: 8 total, 8 up, 8 in 2024-08-17T17:25:22.829 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:22 smithi179 ceph-mon[34775]: from='client.25649 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:23.419 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 8388609~8388609 2024-08-17T17:25:23.436 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 0~1000 2024-08-17T17:25:23.436 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 1010~1000 2024-08-17T17:25:23.436 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:25:23.444 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:23 smithi179 ceph-mon[34775]: osdmap e207: 8 total, 8 up, 8 in 2024-08-17T17:25:23.444 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:23 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:23.444 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:23 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:23.444 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:23 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:23.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:23 smithi130 ceph-mon[31342]: osdmap e207: 8 total, 8 up, 8 in 2024-08-17T17:25:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:23 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:23 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:23 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:23 smithi130 ceph-mon[42749]: osdmap e207: 8 total, 8 up, 8 in 2024-08-17T17:25:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:23 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:23 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:23 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:24.152 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:24.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:24 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:24.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:24.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:24 smithi130 ceph-mon[31342]: pgmap v90: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 22 KiB/s wr, 417 op/s 2024-08-17T17:25:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:24 smithi130 ceph-mon[31342]: osdmap e208: 8 total, 8 up, 8 in 2024-08-17T17:25:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:24 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1190917234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:24 smithi130 ceph-mon[31342]: osdmap e209: 8 total, 8 up, 8 in 2024-08-17T17:25:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:24 smithi130 ceph-mon[42749]: pgmap v90: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 22 KiB/s wr, 417 op/s 2024-08-17T17:25:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:24 smithi130 ceph-mon[42749]: osdmap e208: 8 total, 8 up, 8 in 2024-08-17T17:25:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:24 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1190917234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:24 smithi130 ceph-mon[42749]: osdmap e209: 8 total, 8 up, 8 in 2024-08-17T17:25:24.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:24 smithi179 ceph-mon[34775]: pgmap v90: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 22 KiB/s wr, 417 op/s 2024-08-17T17:25:24.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:24 smithi179 ceph-mon[34775]: osdmap e208: 8 total, 8 up, 8 in 2024-08-17T17:25:24.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:24 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1190917234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:24 smithi179 ceph-mon[34775]: osdmap e209: 8 total, 8 up, 8 in 2024-08-17T17:25:25.196 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 8388609~1000 2024-08-17T17:25:25.201 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 16777228~1000 2024-08-17T17:25:25.201 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 167772160 2024-08-17T17:25:25.210 INFO:tasks.workunit.client.0.smithi130.stdout:write: 2 83886079~8388609 2024-08-17T17:25:25.223 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:25:25.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:25 smithi130 ceph-mon[31342]: from='client.15765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:25 smithi130 ceph-mon[31342]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:25 smithi130 ceph-mon[31342]: osdmap e210: 8 total, 8 up, 8 in 2024-08-17T17:25:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:25 smithi130 ceph-mon[42749]: from='client.15765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:25 smithi130 ceph-mon[42749]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:25 smithi130 ceph-mon[42749]: osdmap e210: 8 total, 8 up, 8 in 2024-08-17T17:25:25.740 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:25 smithi179 ceph-mon[34775]: from='client.15765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:25.740 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:25 smithi179 ceph-mon[34775]: from='client.15771 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:25.740 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:25 smithi179 ceph-mon[34775]: osdmap e210: 8 total, 8 up, 8 in 2024-08-17T17:25:26.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:25 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:26.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:25 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:26.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:26 smithi130 ceph-mon[31342]: pgmap v93: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 22 KiB/s wr, 417 op/s 2024-08-17T17:25:26.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:26 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:26.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:26 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:26.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:26 smithi130 ceph-mon[31342]: osdmap e211: 8 total, 8 up, 8 in 2024-08-17T17:25:26.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:26 smithi130 ceph-mon[42749]: pgmap v93: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 22 KiB/s wr, 417 op/s 2024-08-17T17:25:26.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:26 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:26.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:26 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:26.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:26 smithi130 ceph-mon[42749]: osdmap e211: 8 total, 8 up, 8 in 2024-08-17T17:25:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:26 smithi179 ceph-mon[34775]: pgmap v93: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 22 KiB/s wr, 417 op/s 2024-08-17T17:25:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:26 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:26 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:26 smithi179 ceph-mon[34775]: osdmap e211: 8 total, 8 up, 8 in 2024-08-17T17:25:27.135 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:27.205 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:25:27.229 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 83886079~1 2024-08-17T17:25:27.236 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:25:27.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:25:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2369109483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:27.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:27.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-mon[42749]: osdmap e212: 8 total, 8 up, 8 in 2024-08-17T17:25:27.668 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:25:27] ENGINE Bus STARTING 2024-08-17T17:25:27.668 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: CherryPy Checker: 2024-08-17T17:25:27.668 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: The Application mounted at '' has an empty config. 2024-08-17T17:25:27.668 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:25:27.668 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:25:27] ENGINE Serving on http://:::9283 2024-08-17T17:25:27.668 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:25:27] ENGINE Bus STARTED 2024-08-17T17:25:27.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:25:27.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:27.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:27.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2369109483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:27.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:27.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:27.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:27 smithi130 ceph-mon[31342]: osdmap e212: 8 total, 8 up, 8 in 2024-08-17T17:25:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:27 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:25:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:27 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:27 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:27 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2369109483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:27.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:27 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:27.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:27 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:27.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:27 smithi179 ceph-mon[34775]: osdmap e212: 8 total, 8 up, 8 in 2024-08-17T17:25:28.577 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[31342]: pgmap v96: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 108 KiB/s rd, 12 MiB/s wr, 161 op/s 2024-08-17T17:25:28.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[31342]: from='client.25685 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:28.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[31342]: from='client.25312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:28.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[31342]: Standby manager daemon y restarted 2024-08-17T17:25:28.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[31342]: Standby manager daemon y started 2024-08-17T17:25:28.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:25:28.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:25:28.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:25:28.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:25:28.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[31342]: osdmap e213: 8 total, 8 up, 8 in 2024-08-17T17:25:28.578 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[42749]: pgmap v96: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 108 KiB/s rd, 12 MiB/s wr, 161 op/s 2024-08-17T17:25:28.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[42749]: from='client.25685 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:28.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[42749]: from='client.25312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:28.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[42749]: Standby manager daemon y restarted 2024-08-17T17:25:28.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[42749]: Standby manager daemon y started 2024-08-17T17:25:28.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:25:28.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:25:28.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:25:28.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:25:28.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:28 smithi130 ceph-mon[42749]: osdmap e213: 8 total, 8 up, 8 in 2024-08-17T17:25:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:28 smithi179 ceph-mon[34775]: pgmap v96: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 108 KiB/s rd, 12 MiB/s wr, 161 op/s 2024-08-17T17:25:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:28 smithi179 ceph-mon[34775]: from='client.25685 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:28 smithi179 ceph-mon[34775]: from='client.25312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:28 smithi179 ceph-mon[34775]: Standby manager daemon y restarted 2024-08-17T17:25:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:28 smithi179 ceph-mon[34775]: Standby manager daemon y started 2024-08-17T17:25:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:28 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T17:25:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:28 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:25:28.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:28 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T17:25:28.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:28 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:25:28.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:28 smithi179 ceph-mon[34775]: osdmap e213: 8 total, 8 up, 8 in 2024-08-17T17:25:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:29 smithi130 ceph-mon[31342]: mgrmap e28: x(active, since 63s), standbys: y 2024-08-17T17:25:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:29 smithi130 ceph-mon[31342]: osdmap e214: 8 total, 8 up, 8 in 2024-08-17T17:25:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:29 smithi130 ceph-mon[42749]: mgrmap e28: x(active, since 63s), standbys: y 2024-08-17T17:25:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:29 smithi130 ceph-mon[42749]: osdmap e214: 8 total, 8 up, 8 in 2024-08-17T17:25:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:29 smithi179 ceph-mon[34775]: mgrmap e28: x(active, since 63s), standbys: y 2024-08-17T17:25:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:29 smithi179 ceph-mon[34775]: osdmap e214: 8 total, 8 up, 8 in 2024-08-17T17:25:30.064 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:30.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:25:30] "GET /metrics HTTP/1.1" 200 38772 "" "Prometheus/2.51.0" 2024-08-17T17:25:30.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:30.612 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:25:30] ENGINE Bus STOPPING 2024-08-17T17:25:30.613 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: pgmap v99: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 108 KiB/s rd, 12 MiB/s wr, 161 op/s 2024-08-17T17:25:30.613 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3139910467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:30.613 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: osdmap e215: 8 total, 8 up, 8 in 2024-08-17T17:25:30.613 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:30.613 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:30.613 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:25:30.613 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:25:30.613 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:30.613 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:30.613 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:25:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:25:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: Activating manager daemon y 2024-08-17T17:25:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: osdmap e216: 8 total, 8 up, 8 in 2024-08-17T17:25:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.24937 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-17T17:25:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: mgrmap e29: y(active, starting, since 0.0121355s) 2024-08-17T17:25:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:25:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:25:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:25:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi179.zkkvno"}]: dispatch 2024-08-17T17:25:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.jmaidg"}]: dispatch 2024-08-17T17:25:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:25:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:25:30.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:25:30.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:25:30.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:25:30.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:25:30.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:25:30.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:25:30.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:25:30.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:25:30.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:25:30.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:25:30.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: pgmap v99: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 108 KiB/s rd, 12 MiB/s wr, 161 op/s 2024-08-17T17:25:30.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3139910467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:30.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: osdmap e215: 8 total, 8 up, 8 in 2024-08-17T17:25:30.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:30.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:30.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:25:30.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:25:30.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:30.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:30.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:25:30.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:25:30.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: Activating manager daemon y 2024-08-17T17:25:30.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: osdmap e216: 8 total, 8 up, 8 in 2024-08-17T17:25:30.617 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.24937 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-17T17:25:30.617 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: mgrmap e29: y(active, starting, since 0.0121355s) 2024-08-17T17:25:30.617 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:25:30.617 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:25:30.617 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:25:30.617 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi179.zkkvno"}]: dispatch 2024-08-17T17:25:30.617 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.jmaidg"}]: dispatch 2024-08-17T17:25:30.617 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:25:30.617 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:25:30.617 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:25:30.617 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:25:30.617 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:25:30.617 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:25:30.618 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:25:30.618 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:25:30.618 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:25:30.618 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:25:30.618 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:25:30.618 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:25:30.779 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: after_prepare_snap 2024-08-17T17:25:30.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: ignoring --setuser ceph since I am not root 2024-08-17T17:25:30.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: ignoring --setgroup ceph since I am not root 2024-08-17T17:25:30.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mgr[70526]: -- 172.21.15.179:0/1188452138 <== mon.1 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55caf3fbd0e0 con 0x55caf3f9b000 2024-08-17T17:25:30.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: pgmap v99: 137 pgs: 137 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 108 KiB/s rd, 12 MiB/s wr, 161 op/s 2024-08-17T17:25:30.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3139910467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:30.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: osdmap e215: 8 total, 8 up, 8 in 2024-08-17T17:25:30.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:30.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:30.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:25:30.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:25:30.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' 2024-08-17T17:25:30.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:30.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.24937 172.21.15.179:0/1435333251' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:25:30.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T17:25:30.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: Activating manager daemon y 2024-08-17T17:25:30.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: osdmap e216: 8 total, 8 up, 8 in 2024-08-17T17:25:30.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.24937 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-17T17:25:30.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: mgrmap e29: y(active, starting, since 0.0121355s) 2024-08-17T17:25:30.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:25:30.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:25:30.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:25:30.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi179.zkkvno"}]: dispatch 2024-08-17T17:25:30.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.jmaidg"}]: dispatch 2024-08-17T17:25:30.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:25:30.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:25:30.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:25:30.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:25:30.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:25:30.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:25:30.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:25:30.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:25:30.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:25:30.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:25:30.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:25:30.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:30 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:25:30.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:25:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:25:30.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:30 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:25:30] ENGINE Bus STOPPED 2024-08-17T17:25:31.291 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:25:31.031+0000 7fe15ac78640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:25:31.292 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:25:31.031+0000 7fe15ac78640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:25:31.292 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:25:31.031+0000 7fe15ac78640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:25:31.292 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:25:31.031+0000 7fe15ac78640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:25:31.292 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:25:31.031+0000 7fe15ac78640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:25:31.292 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:25:31.032+0000 7fe15cc7c640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:25:31.292 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:25:31.032+0000 7fe15cc7c640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:25:31.292 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:25:31.032+0000 7fe15cc7c640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:25:31.292 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:25:31.032+0000 7fe15cc7c640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:25:31.292 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:25:31.032+0000 7fe15cc7c640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:25:31.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[31342]: Manager daemon y is now available 2024-08-17T17:25:31.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:31.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:25:31.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:25:31.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:25:31.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:25:31.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:25:31.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:25:31.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[42749]: Manager daemon y is now available 2024-08-17T17:25:31.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:31.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:25:31.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:25:31.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:25:31.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:25:31.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:25:31.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:25:31.557 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:25:31] ENGINE Bus STARTING 2024-08-17T17:25:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:31 smithi179 ceph-mon[34775]: Manager daemon y is now available 2024-08-17T17:25:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:31 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:31 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:25:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:31 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:25:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:31 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:25:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:31 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:25:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:31 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:25:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:31 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:25:31.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:25:31] ENGINE Serving on http://:::9283 2024-08-17T17:25:31.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:31 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:25:31] ENGINE Bus STARTED 2024-08-17T17:25:32.451 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 41943040 2024-08-17T17:25:32.481 INFO:tasks.workunit.client.0.smithi130.stdout:write: * 0~1000 2024-08-17T17:25:32.487 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:25:32.653 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:25:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:32 smithi179 ceph-mon[34775]: osdmap e217: 8 total, 8 up, 8 in 2024-08-17T17:25:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:32 smithi179 ceph-mon[34775]: mgrmap e30: y(active, since 1.09576s) 2024-08-17T17:25:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:32 smithi179 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:25:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:32 smithi179 ceph-mon[34775]: from='client.25709 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:32.820 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:32 smithi179 ceph-mon[34775]: pgmap v4: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 111 GiB/s rd, 40 GiB/s wr, 1.85M op/s; 84 MiB/s, 298 objects/s recovering 2024-08-17T17:25:32.880 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-mon[31342]: osdmap e217: 8 total, 8 up, 8 in 2024-08-17T17:25:32.880 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-mon[31342]: mgrmap e30: y(active, since 1.09576s) 2024-08-17T17:25:32.880 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-mon[31342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:25:32.881 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-mon[31342]: from='client.25709 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:32.881 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-mon[31342]: pgmap v4: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 111 GiB/s rd, 40 GiB/s wr, 1.85M op/s; 84 MiB/s, 298 objects/s recovering 2024-08-17T17:25:32.881 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-mon[42749]: osdmap e217: 8 total, 8 up, 8 in 2024-08-17T17:25:32.881 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-mon[42749]: mgrmap e30: y(active, since 1.09576s) 2024-08-17T17:25:32.881 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-mon[42749]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:25:32.881 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-mon[42749]: from='client.25709 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:32.881 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-mon[42749]: pgmap v4: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 111 GiB/s rd, 40 GiB/s wr, 1.85M op/s; 84 MiB/s, 298 objects/s recovering 2024-08-17T17:25:33.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:25:32.850+0000 7fe13b97d640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:17:25:32] ENGINE Error in HTTPServer.serve 2024-08-17T17:25:33.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: Traceback (most recent call last): 2024-08-17T17:25:33.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:25:33.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: self._connections.run(self.expiration_interval) 2024-08-17T17:25:33.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:25:33.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: self._run(expiration_interval) 2024-08-17T17:25:33.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:25:33.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:25:33.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:25:33.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:25:33.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:25:33.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: s = self.context.wrap_socket( 2024-08-17T17:25:33.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:25:33.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: return self.sslsocket_class._create( 2024-08-17T17:25:33.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:25:33.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: self.do_handshake() 2024-08-17T17:25:33.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:25:33.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: self._sslobj.do_handshake() 2024-08-17T17:25:33.169 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:25:33.169 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:25:33.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:33 smithi179 ceph-mon[34775]: osdmap e218: 8 total, 8 up, 8 in 2024-08-17T17:25:33.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:33 smithi179 ceph-mon[34775]: mgrmap e31: y(active, since 2s) 2024-08-17T17:25:33.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:33 smithi179 ceph-mon[34775]: [17/Aug/2024:17:25:32] ENGINE Bus STARTING 2024-08-17T17:25:33.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:33 smithi179 ceph-mon[34775]: [17/Aug/2024:17:25:32] ENGINE Serving on http://172.21.15.130:8765 2024-08-17T17:25:33.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:33 smithi179 ceph-mon[34775]: [17/Aug/2024:17:25:32] ENGINE Serving on https://172.21.15.130:7150 2024-08-17T17:25:33.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:33 smithi179 ceph-mon[34775]: [17/Aug/2024:17:25:32] ENGINE Bus STARTED 2024-08-17T17:25:33.733 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:33 smithi179 ceph-mon[34775]: [17/Aug/2024:17:25:32] ENGINE Error in HTTPServer.serve 2024-08-17T17:25:33.733 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-08-17T17:25:33.733 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:25:33.733 INFO:journalctl@ceph.mon.b.smithi179.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:25:33.733 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:25:33.733 INFO:journalctl@ceph.mon.b.smithi179.stdout: self._run(expiration_interval) 2024-08-17T17:25:33.733 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:25:33.733 INFO:journalctl@ceph.mon.b.smithi179.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:25:33.733 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:25:33.733 INFO:journalctl@ceph.mon.b.smithi179.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:25:33.733 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:25:33.733 INFO:journalctl@ceph.mon.b.smithi179.stdout: s = self.context.wrap_socket( 2024-08-17T17:25:33.733 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:25:33.733 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.sslsocket_class._create( 2024-08-17T17:25:33.734 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:25:33.734 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.do_handshake() 2024-08-17T17:25:33.734 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:25:33.734 INFO:journalctl@ceph.mon.b.smithi179.stdout: self._sslobj.do_handshake() 2024-08-17T17:25:33.734 INFO:journalctl@ceph.mon.b.smithi179.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:25:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:33 smithi130 ceph-mon[42749]: osdmap e218: 8 total, 8 up, 8 in 2024-08-17T17:25:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:33 smithi130 ceph-mon[42749]: mgrmap e31: y(active, since 2s) 2024-08-17T17:25:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:33 smithi130 ceph-mon[42749]: [17/Aug/2024:17:25:32] ENGINE Bus STARTING 2024-08-17T17:25:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:33 smithi130 ceph-mon[42749]: [17/Aug/2024:17:25:32] ENGINE Serving on http://172.21.15.130:8765 2024-08-17T17:25:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:33 smithi130 ceph-mon[42749]: [17/Aug/2024:17:25:32] ENGINE Serving on https://172.21.15.130:7150 2024-08-17T17:25:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:33 smithi130 ceph-mon[42749]: [17/Aug/2024:17:25:32] ENGINE Bus STARTED 2024-08-17T17:25:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:33 smithi130 ceph-mon[42749]: [17/Aug/2024:17:25:32] ENGINE Error in HTTPServer.serve 2024-08-17T17:25:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-08-17T17:25:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:25:33.918 INFO:journalctl@ceph.mon.c.smithi130.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:25:33.918 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:25:33.918 INFO:journalctl@ceph.mon.c.smithi130.stdout: self._run(expiration_interval) 2024-08-17T17:25:33.918 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:25:33.918 INFO:journalctl@ceph.mon.c.smithi130.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:25:33.918 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:25:33.918 INFO:journalctl@ceph.mon.c.smithi130.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:25:33.918 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:25:33.918 INFO:journalctl@ceph.mon.c.smithi130.stdout: s = self.context.wrap_socket( 2024-08-17T17:25:33.918 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:25:33.918 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.sslsocket_class._create( 2024-08-17T17:25:33.918 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:25:33.918 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.do_handshake() 2024-08-17T17:25:33.919 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:25:33.919 INFO:journalctl@ceph.mon.c.smithi130.stdout: self._sslobj.do_handshake() 2024-08-17T17:25:33.919 INFO:journalctl@ceph.mon.c.smithi130.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:25:33.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:33 smithi130 ceph-mon[31342]: osdmap e218: 8 total, 8 up, 8 in 2024-08-17T17:25:33.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:33 smithi130 ceph-mon[31342]: mgrmap e31: y(active, since 2s) 2024-08-17T17:25:33.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:33 smithi130 ceph-mon[31342]: [17/Aug/2024:17:25:32] ENGINE Bus STARTING 2024-08-17T17:25:33.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:33 smithi130 ceph-mon[31342]: [17/Aug/2024:17:25:32] ENGINE Serving on http://172.21.15.130:8765 2024-08-17T17:25:33.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:33 smithi130 ceph-mon[31342]: [17/Aug/2024:17:25:32] ENGINE Serving on https://172.21.15.130:7150 2024-08-17T17:25:33.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:33 smithi130 ceph-mon[31342]: [17/Aug/2024:17:25:32] ENGINE Bus STARTED 2024-08-17T17:25:33.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:33 smithi130 ceph-mon[31342]: [17/Aug/2024:17:25:32] ENGINE Error in HTTPServer.serve 2024-08-17T17:25:33.919 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-08-17T17:25:33.920 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:25:33.920 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:25:33.920 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:25:33.920 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._run(expiration_interval) 2024-08-17T17:25:33.920 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:25:33.920 INFO:journalctl@ceph.mon.a.smithi130.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:25:33.920 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:25:33.920 INFO:journalctl@ceph.mon.a.smithi130.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:25:33.920 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:25:33.920 INFO:journalctl@ceph.mon.a.smithi130.stdout: s = self.context.wrap_socket( 2024-08-17T17:25:33.920 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:25:33.920 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.sslsocket_class._create( 2024-08-17T17:25:33.920 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:25:33.921 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.do_handshake() 2024-08-17T17:25:33.921 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:25:33.921 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._sslobj.do_handshake() 2024-08-17T17:25:33.921 INFO:journalctl@ceph.mon.a.smithi130.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:25:34.114 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:34.446 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:34 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:34.814 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:34 smithi130 ceph-mon[31342]: osdmap e219: 8 total, 8 up, 8 in 2024-08-17T17:25:34.815 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:34 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/4274298643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:34.815 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:34 smithi130 ceph-mon[31342]: from='client.15852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:34.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:34 smithi130 ceph-mon[42749]: osdmap e219: 8 total, 8 up, 8 in 2024-08-17T17:25:34.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:34 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/4274298643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:34.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:34 smithi130 ceph-mon[42749]: from='client.15852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:34.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:34 smithi179 ceph-mon[34775]: osdmap e219: 8 total, 8 up, 8 in 2024-08-17T17:25:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:34 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/4274298643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:34 smithi179 ceph-mon[34775]: from='client.15852 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:35.740 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:35 smithi179 ceph-mon[34775]: pgmap v8: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 781 MiB/s rd, 278 MiB/s wr, 12.75k op/s; 592 KiB/s, 2 objects/s recovering 2024-08-17T17:25:35.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:35 smithi179 ceph-mon[34775]: from='client.15858 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:35.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:35 smithi179 ceph-mon[34775]: osdmap e220: 8 total, 8 up, 8 in 2024-08-17T17:25:35.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:35 smithi179 ceph-mon[34775]: mgrmap e32: y(active, since 4s) 2024-08-17T17:25:35.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:35 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:35.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:35 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:35.741 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: [17/Aug/2024:17:25:35] ENGINE Bus STARTING 2024-08-17T17:25:35.741 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: CherryPy Checker: 2024-08-17T17:25:35.741 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: The Application mounted at '' has an empty config. 2024-08-17T17:25:35.741 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: 2024-08-17T17:25:35.772 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.SnapsSnapAfterPrepare (14287 ms) 2024-08-17T17:25:35.772 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.SnapsSnapExpandAfterPrepare 2024-08-17T17:25:35.803 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:25:35.814 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:25:35.870 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:25:35.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:35 smithi130 ceph-mon[42749]: pgmap v8: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 781 MiB/s rd, 278 MiB/s wr, 12.75k op/s; 592 KiB/s, 2 objects/s recovering 2024-08-17T17:25:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:35 smithi130 ceph-mon[42749]: from='client.15858 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:35 smithi130 ceph-mon[42749]: osdmap e220: 8 total, 8 up, 8 in 2024-08-17T17:25:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:35 smithi130 ceph-mon[42749]: mgrmap e32: y(active, since 4s) 2024-08-17T17:25:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:35 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:35 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:35 smithi130 ceph-mon[31342]: pgmap v8: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 781 MiB/s rd, 278 MiB/s wr, 12.75k op/s; 592 KiB/s, 2 objects/s recovering 2024-08-17T17:25:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:35 smithi130 ceph-mon[31342]: from='client.15858 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:35 smithi130 ceph-mon[31342]: osdmap e220: 8 total, 8 up, 8 in 2024-08-17T17:25:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:35 smithi130 ceph-mon[31342]: mgrmap e32: y(active, since 4s) 2024-08-17T17:25:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:35 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:35.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:35 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: [17/Aug/2024:17:25:35] ENGINE Serving on http://:::9283 2024-08-17T17:25:36.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: [17/Aug/2024:17:25:35] ENGINE Bus STARTED 2024-08-17T17:25:36.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:36.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:36.724 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[31342]: osdmap e221: 8 total, 8 up, 8 in 2024-08-17T17:25:36.724 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[31342]: Standby manager daemon x started 2024-08-17T17:25:36.724 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/1913967930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:25:36.725 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/1913967930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:25:36.725 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/1913967930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:25:36.725 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/1913967930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:25:36.725 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.725 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.725 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.725 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.725 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.725 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.725 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[42749]: osdmap e221: 8 total, 8 up, 8 in 2024-08-17T17:25:36.725 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[42749]: Standby manager daemon x started 2024-08-17T17:25:36.725 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/1913967930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:25:36.726 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/1913967930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:25:36.726 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/1913967930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:25:36.726 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/1913967930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:25:36.726 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.726 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.726 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.726 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.726 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.726 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:36 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:36 smithi179 ceph-mon[34775]: osdmap e221: 8 total, 8 up, 8 in 2024-08-17T17:25:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:36 smithi179 ceph-mon[34775]: Standby manager daemon x started 2024-08-17T17:25:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:36 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/1913967930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:25:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:36 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/1913967930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:25:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:36 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/1913967930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:25:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:36 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/1913967930' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:25:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:36 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:36 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:36 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:36 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:36 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:36.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:36 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:37.050 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:37.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:36 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:37.698 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 8388609~8388609 2024-08-17T17:25:37.710 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 0~1000 2024-08-17T17:25:37.711 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 1010~1000 2024-08-17T17:25:37.711 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:25:37.733 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:37 smithi130 ceph-mon[31342]: pgmap v11: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:37.733 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:37 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:25:37.733 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:37 smithi130 ceph-mon[31342]: mgrmap e33: y(active, since 6s), standbys: x 2024-08-17T17:25:37.734 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:37 smithi130 ceph-mon[31342]: osdmap e222: 8 total, 8 up, 8 in 2024-08-17T17:25:37.734 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:37 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/348565762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:37.734 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:37 smithi130 ceph-mon[31342]: from='client.15867 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:37.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:37 smithi130 ceph-mon[42749]: pgmap v11: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:37.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:37 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:25:37.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:37 smithi130 ceph-mon[42749]: mgrmap e33: y(active, since 6s), standbys: x 2024-08-17T17:25:37.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:37 smithi130 ceph-mon[42749]: osdmap e222: 8 total, 8 up, 8 in 2024-08-17T17:25:37.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:37 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/348565762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:37.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:37 smithi130 ceph-mon[42749]: from='client.15867 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:37 smithi179 ceph-mon[34775]: pgmap v11: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:37 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:25:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:37 smithi179 ceph-mon[34775]: mgrmap e33: y(active, since 6s), standbys: x 2024-08-17T17:25:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:37 smithi179 ceph-mon[34775]: osdmap e222: 8 total, 8 up, 8 in 2024-08-17T17:25:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:37 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/348565762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:37 smithi179 ceph-mon[34775]: from='client.15867 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:38 smithi179 ceph-mon[34775]: from='client.25784 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:38 smithi179 ceph-mon[34775]: osdmap e223: 8 total, 8 up, 8 in 2024-08-17T17:25:39.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:38 smithi130 ceph-mon[42749]: from='client.25784 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:39.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:38 smithi130 ceph-mon[42749]: osdmap e223: 8 total, 8 up, 8 in 2024-08-17T17:25:39.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:38 smithi130 ceph-mon[31342]: from='client.25784 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:39.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:38 smithi130 ceph-mon[31342]: osdmap e223: 8 total, 8 up, 8 in 2024-08-17T17:25:39.201 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 8388609~1000 2024-08-17T17:25:39.206 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 16777228~1000 2024-08-17T17:25:39.206 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 167772160 2024-08-17T17:25:39.216 INFO:tasks.workunit.client.0.smithi130.stdout:write: 2 83886079~8388609 2024-08-17T17:25:39.230 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:25:39.971 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:39 smithi130 ceph-mon[31342]: pgmap v14: 137 pgs: 137 active+clean; 689 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 442 op/s 2024-08-17T17:25:39.971 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:39 smithi130 ceph-mon[31342]: osdmap e224: 8 total, 8 up, 8 in 2024-08-17T17:25:39.971 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:39 smithi130 ceph-mon[31342]: osdmap e225: 8 total, 8 up, 8 in 2024-08-17T17:25:39.971 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:39 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1323138222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:39.971 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:39 smithi130 ceph-mon[42749]: pgmap v14: 137 pgs: 137 active+clean; 689 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 442 op/s 2024-08-17T17:25:39.971 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:39 smithi130 ceph-mon[42749]: osdmap e224: 8 total, 8 up, 8 in 2024-08-17T17:25:39.972 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:39 smithi130 ceph-mon[42749]: osdmap e225: 8 total, 8 up, 8 in 2024-08-17T17:25:39.972 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:39 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1323138222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:39.972 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:40.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:39 smithi179 ceph-mon[34775]: pgmap v14: 137 pgs: 137 active+clean; 689 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 442 op/s 2024-08-17T17:25:40.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:39 smithi179 ceph-mon[34775]: osdmap e224: 8 total, 8 up, 8 in 2024-08-17T17:25:40.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:39 smithi179 ceph-mon[34775]: osdmap e225: 8 total, 8 up, 8 in 2024-08-17T17:25:40.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:39 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1323138222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:40.343 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[70503]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:25:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.51.0" 2024-08-17T17:25:40.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:25:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:25:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:25:41.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:25:41.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: from='client.15885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:41.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:25:41.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:25:41.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: osdmap e226: 8 total, 8 up, 8 in 2024-08-17T17:25:41.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:40 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:25:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:25:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:25:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:25:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: from='client.15885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:25:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:25:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: osdmap e226: 8 total, 8 up, 8 in 2024-08-17T17:25:41.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:25:41.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:25:41.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:25:41.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:25:41.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: from='client.15885 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:41.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:25:41.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:25:41.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: osdmap e226: 8 total, 8 up, 8 in 2024-08-17T17:25:41.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:40 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:41.204 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:25:41.228 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 83886079~1 2024-08-17T17:25:41.236 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:25:41.919 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:41 smithi179 ceph-mon[34775]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:25:41.919 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:41 smithi179 ceph-mon[34775]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:25:41.919 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:41 smithi179 ceph-mon[34775]: from='client.15891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:41.919 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:41 smithi179 ceph-mon[34775]: pgmap v18: 137 pgs: 137 active+clean; 689 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:41.919 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:41 smithi179 ceph-mon[34775]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:25:41.919 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:41 smithi179 ceph-mon[34775]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:25:41.920 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:41 smithi179 ceph-mon[34775]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:25:41.920 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:41 smithi179 ceph-mon[34775]: Reconfiguring daemon prometheus.a on smithi179 2024-08-17T17:25:41.920 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:41 smithi179 ceph-mon[34775]: osdmap e227: 8 total, 8 up, 8 in 2024-08-17T17:25:42.023 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[42749]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:25:42.023 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[42749]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:25:42.023 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[42749]: from='client.15891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:42.023 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[42749]: pgmap v18: 137 pgs: 137 active+clean; 689 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:42.023 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[42749]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:25:42.023 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[42749]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:25:42.024 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[42749]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:25:42.024 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[42749]: Reconfiguring daemon prometheus.a on smithi179 2024-08-17T17:25:42.024 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[42749]: osdmap e227: 8 total, 8 up, 8 in 2024-08-17T17:25:42.024 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[31342]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:25:42.024 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[31342]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:25:42.024 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[31342]: from='client.15891 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:42.024 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[31342]: pgmap v18: 137 pgs: 137 active+clean; 689 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:42.024 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[31342]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:25:42.024 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[31342]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:25:42.024 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[31342]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T17:25:42.024 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[31342]: Reconfiguring daemon prometheus.a on smithi179 2024-08-17T17:25:42.025 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:41 smithi130 ceph-mon[31342]: osdmap e227: 8 total, 8 up, 8 in 2024-08-17T17:25:42.886 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:43.546 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:43 smithi179 ceph-mon[34775]: osdmap e228: 8 total, 8 up, 8 in 2024-08-17T17:25:43.546 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:43 smithi179 ceph-mon[34775]: pgmap v21: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 156 KiB/s rd, 8.8 MiB/s wr, 224 op/s 2024-08-17T17:25:43.546 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:43 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2232378648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:43.546 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:43 smithi179 ceph-mon[34775]: from='client.25814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:43 smithi130 ceph-mon[42749]: osdmap e228: 8 total, 8 up, 8 in 2024-08-17T17:25:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:43 smithi130 ceph-mon[42749]: pgmap v21: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 156 KiB/s rd, 8.8 MiB/s wr, 224 op/s 2024-08-17T17:25:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:43 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2232378648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:43 smithi130 ceph-mon[42749]: from='client.25814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:43 smithi130 ceph-mon[31342]: osdmap e228: 8 total, 8 up, 8 in 2024-08-17T17:25:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:43 smithi130 ceph-mon[31342]: pgmap v21: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 156 KiB/s rd, 8.8 MiB/s wr, 224 op/s 2024-08-17T17:25:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:43 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2232378648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:43 smithi130 ceph-mon[31342]: from='client.25814 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:44.271 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:44 smithi179 ceph-mon[34775]: from='client.15909 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:44.271 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:44 smithi179 ceph-mon[34775]: osdmap e229: 8 total, 8 up, 8 in 2024-08-17T17:25:44.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:44 smithi179 systemd[1]: Stopping Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:25:44.652 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: after_prepare_snap 2024-08-17T17:25:44.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:44 smithi130 ceph-mon[31342]: from='client.15909 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:44 smithi130 ceph-mon[31342]: osdmap e229: 8 total, 8 up, 8 in 2024-08-17T17:25:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:44 smithi130 ceph-mon[42749]: from='client.15909 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:44 smithi130 ceph-mon[42749]: osdmap e229: 8 total, 8 up, 8 in 2024-08-17T17:25:44.998 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:44.690Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:25:44.998 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:44.690Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-08-17T17:25:44.998 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:44.690Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-08-17T17:25:44.998 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:44.690Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T17:25:44.998 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:44.690Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-08-17T17:25:44.998 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:44.690Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-08-17T17:25:44.999 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:44.690Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T17:25:44.999 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:44.690Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-08-17T17:25:44.999 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:44.691Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-08-17T17:25:44.999 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:44.692Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T17:25:44.999 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:44.692Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-08-17T17:25:44.999 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[79593]: ts=2024-08-17T17:25:44.692Z caller=main.go:1273 level=info msg="See you next time!" 2024-08-17T17:25:44.999 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:44 smithi179 podman[83001]: 2024-08-17 17:25:44.700995587 +0000 UTC m=+0.124714653 container died fb1a2c10ce69f7c022d338076f2168bb506c296c54873457f567b09936ff79b9 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:25:45.264 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:45 smithi179 ceph-mon[34775]: osdmap e230: 8 total, 8 up, 8 in 2024-08-17T17:25:45.264 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:45 smithi179 ceph-mon[34775]: pgmap v24: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 142 KiB/s rd, 8.0 MiB/s wr, 205 op/s 2024-08-17T17:25:45.532 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:45 smithi179 podman[83001]: 2024-08-17 17:25:45.263957369 +0000 UTC m=+0.687676432 container cleanup fb1a2c10ce69f7c022d338076f2168bb506c296c54873457f567b09936ff79b9 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:25:45.532 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:45 smithi179 bash[83001]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T17:25:45.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:45 smithi130 ceph-mon[31342]: osdmap e230: 8 total, 8 up, 8 in 2024-08-17T17:25:45.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:45 smithi130 ceph-mon[31342]: pgmap v24: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 142 KiB/s rd, 8.0 MiB/s wr, 205 op/s 2024-08-17T17:25:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:45 smithi130 ceph-mon[42749]: osdmap e230: 8 total, 8 up, 8 in 2024-08-17T17:25:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:45 smithi130 ceph-mon[42749]: pgmap v24: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 142 KiB/s rd, 8.0 MiB/s wr, 205 op/s 2024-08-17T17:25:45.781 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:45.843 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:45 smithi179 podman[83017]: 2024-08-17 17:25:45.532220494 +0000 UTC m=+0.832386957 container remove fb1a2c10ce69f7c022d338076f2168bb506c296c54873457f567b09936ff79b9 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:25:45.843 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:45 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-17T17:25:45.843 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:45 smithi179 systemd[1]: Stopped Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:25:45.843 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:45 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@prometheus.a.service: Consumed 1.147s CPU time. 2024-08-17T17:25:45.843 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:45 smithi179 systemd[1]: Starting Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:25:46.225 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:45 smithi179 podman[83117]: 2024-08-17 17:25:45.778963262 +0000 UTC m=+0.018927328 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-08-17T17:25:46.226 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:45 smithi179 podman[83117]: 2024-08-17 17:25:45.883495763 +0000 UTC m=+0.123459882 container create 145f77b6fc933e25c44dde4b744667240605594338e857c8de6b5dac9f44f7d0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:25:46.244 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 167772160 2024-08-17T17:25:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:write: * 83886080~1000 2024-08-17T17:25:46.261 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:25:46.406 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:25:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-mon[34775]: osdmap e231: 8 total, 8 up, 8 in 2024-08-17T17:25:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3388452734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-mon[34775]: from='client.15921 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:25:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-mon[34775]: from='client.25838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:46.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 podman[83117]: 2024-08-17 17:25:46.252911899 +0000 UTC m=+0.492875959 container init 145f77b6fc933e25c44dde4b744667240605594338e857c8de6b5dac9f44f7d0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:25:46.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 podman[83117]: 2024-08-17 17:25:46.258106544 +0000 UTC m=+0.498070608 container start 145f77b6fc933e25c44dde4b744667240605594338e857c8de6b5dac9f44f7d0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:25:46.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.279Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-08-17T17:25:46.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.279Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-08-17T17:25:46.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.279Z caller=main.go:623 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi179 (none))" 2024-08-17T17:25:46.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.279Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T17:25:46.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.279Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T17:25:46.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.284Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T17:25:46.595 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.284Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-08-17T17:25:46.595 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.286Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T17:25:46.595 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.286Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T17:25:46.595 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.288Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T17:25:46.595 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.288Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.843µs 2024-08-17T17:25:46.595 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.288Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T17:25:46.595 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.289Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-08-17T17:25:46.595 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.301Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-08-17T17:25:46.595 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.311Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-08-17T17:25:46.595 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.311Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-08-17T17:25:46.595 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.312Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=33.987µs wal_replay_duration=23.065692ms wbl_replay_duration=137ns total_replay_duration=23.11788ms 2024-08-17T17:25:46.595 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.317Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T17:25:46.595 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.317Z caller=main.go:1153 level=info msg="TSDB started" 2024-08-17T17:25:46.596 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.317Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T17:25:46.596 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.337Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.484756ms db_storage=998ns remote_storage=1.141µs web_handler=325ns query_engine=1.062µs scrape=466.466µs scrape_sd=97.639µs notify=9.978µs notify_sd=13.877µs rules=18.469287ms tracing=7.32µs 2024-08-17T17:25:46.596 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.337Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-08-17T17:25:46.596 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.337Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T17:25:46.596 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 bash[83117]: 145f77b6fc933e25c44dde4b744667240605594338e857c8de6b5dac9f44f7d0 2024-08-17T17:25:46.596 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 systemd[1]: Started Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:25:46.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:46 smithi130 ceph-mon[31342]: osdmap e231: 8 total, 8 up, 8 in 2024-08-17T17:25:46.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:46 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/3388452734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:46.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:46 smithi130 ceph-mon[31342]: from='client.15921 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:46.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:46 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:25:46.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:46 smithi130 ceph-mon[31342]: from='client.25838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:46 smithi130 ceph-mon[42749]: osdmap e231: 8 total, 8 up, 8 in 2024-08-17T17:25:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:46 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3388452734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:46 smithi130 ceph-mon[42749]: from='client.15921 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:46 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:25:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:46 smithi130 ceph-mon[42749]: from='client.25838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:47.090 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:47 smithi179 ceph-mon[34775]: osdmap e232: 8 total, 8 up, 8 in 2024-08-17T17:25:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:47 smithi179 ceph-mon[34775]: pgmap v27: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:47 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:47 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:47 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:25:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:47 smithi179 ceph-mon[34775]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:25:47.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:47 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:47 smithi130 ceph-mon[42749]: osdmap e232: 8 total, 8 up, 8 in 2024-08-17T17:25:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:47 smithi130 ceph-mon[42749]: pgmap v27: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:47 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:47 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:47 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:25:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:47 smithi130 ceph-mon[42749]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:25:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:47 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:47.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:47 smithi130 ceph-mon[31342]: osdmap e232: 8 total, 8 up, 8 in 2024-08-17T17:25:47.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:47 smithi130 ceph-mon[31342]: pgmap v27: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:47.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:47 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:47.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:47 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:47.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:47 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:25:47.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:47 smithi130 ceph-mon[31342]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T17:25:47.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:47 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:48.507 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:48 smithi179 ceph-mon[34775]: osdmap e233: 8 total, 8 up, 8 in 2024-08-17T17:25:48.507 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:48 smithi179 ceph-mon[34775]: Upgrade: Updating mgr.x 2024-08-17T17:25:48.508 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:48 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:25:48.508 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:48 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:25:48.508 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:48 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:25:48.508 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:48 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:25:48.508 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:48 smithi179 ceph-mon[34775]: Deploying daemon mgr.x on smithi179 2024-08-17T17:25:48.508 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:48 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1872789947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:48.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[42749]: osdmap e233: 8 total, 8 up, 8 in 2024-08-17T17:25:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[42749]: Upgrade: Updating mgr.x 2024-08-17T17:25:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:25:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:25:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:25:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:25:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[42749]: Deploying daemon mgr.x on smithi179 2024-08-17T17:25:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1872789947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:48.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[31342]: osdmap e233: 8 total, 8 up, 8 in 2024-08-17T17:25:48.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[31342]: Upgrade: Updating mgr.x 2024-08-17T17:25:48.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:25:48.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T17:25:48.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T17:25:48.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:25:48.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[31342]: Deploying daemon mgr.x on smithi179 2024-08-17T17:25:48.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:48 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1872789947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:48.676 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:49.411 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:49 smithi179 systemd[1]: Stopping Ceph mgr.x for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:25:49.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:49 smithi130 ceph-mon[42749]: pgmap v29: 137 pgs: 137 active+clean; 757 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 5.8 MiB/s wr, 458 op/s 2024-08-17T17:25:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:49 smithi130 ceph-mon[42749]: osdmap e234: 8 total, 8 up, 8 in 2024-08-17T17:25:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:49 smithi130 ceph-mon[42749]: from='client.25850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:49 smithi130 ceph-mon[42749]: from='client.25856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:49.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:49 smithi130 ceph-mon[31342]: pgmap v29: 137 pgs: 137 active+clean; 757 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 5.8 MiB/s wr, 458 op/s 2024-08-17T17:25:49.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:49 smithi130 ceph-mon[31342]: osdmap e234: 8 total, 8 up, 8 in 2024-08-17T17:25:49.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:49 smithi130 ceph-mon[31342]: from='client.25850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:49.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:49 smithi130 ceph-mon[31342]: from='client.25856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:49.798 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.SnapsSnapExpandAfterPrepare (14026 ms) 2024-08-17T17:25:49.799 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.Clone 2024-08-17T17:25:49.828 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:25:49.838 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 0~8388609 2024-08-17T17:25:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:49 smithi179 ceph-mon[34775]: pgmap v29: 137 pgs: 137 active+clean; 757 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 5.8 MiB/s wr, 458 op/s 2024-08-17T17:25:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:49 smithi179 ceph-mon[34775]: osdmap e234: 8 total, 8 up, 8 in 2024-08-17T17:25:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:49 smithi179 ceph-mon[34775]: from='client.25850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:49 smithi179 ceph-mon[34775]: from='client.25856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:49.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:49 smithi179 podman[83523]: 2024-08-17 17:25:49.657038212 +0000 UTC m=+0.154187771 container died d129c8e500cc233d5d7bdf5117a11d96eaf90ace5db1c9449bd1727355fc88a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:25:49.895 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 16777218~8388609 2024-08-17T17:25:49.954 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap 2024-08-17T17:25:50.212 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:50 smithi179 podman[83523]: 2024-08-17 17:25:50.033578045 +0000 UTC m=+0.530727592 container cleanup d129c8e500cc233d5d7bdf5117a11d96eaf90ace5db1c9449bd1727355fc88a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-08-17T17:25:50.213 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:50 smithi179 bash[83523]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x 2024-08-17T17:25:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:50 smithi179 ceph-mon[34775]: osdmap e235: 8 total, 8 up, 8 in 2024-08-17T17:25:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:50.593 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:50 smithi179 podman[83542]: 2024-08-17 17:25:50.301546675 +0000 UTC m=+0.642921139 container remove d129c8e500cc233d5d7bdf5117a11d96eaf90ace5db1c9449bd1727355fc88a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-08-17T17:25:50.593 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:50 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:25:50.594 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:50 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-17T17:25:50.594 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:50 smithi179 systemd[1]: Stopped Ceph mgr.x for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:25:50.594 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:50 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.x.service: Consumed 18.809s CPU time. 2024-08-17T17:25:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:50 smithi130 ceph-mon[31342]: osdmap e235: 8 total, 8 up, 8 in 2024-08-17T17:25:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:50 smithi130 ceph-mon[42749]: osdmap e235: 8 total, 8 up, 8 in 2024-08-17T17:25:50.919 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:50 smithi179 systemd[1]: Starting Ceph mgr.x for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:25:50.920 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:50 smithi179 podman[83670]: 2024-08-17 17:25:50.750458536 +0000 UTC m=+0.017483738 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:25:51.173 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:50 smithi179 podman[83670]: 2024-08-17 17:25:50.920009191 +0000 UTC m=+0.187034387 container create cdd6dde7c7af3a41df5f582d18829d798ea8c6a7e1ae45b9befcfb7280d2e13f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-17T17:25:51.425 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:51 smithi179 podman[83670]: 2024-08-17 17:25:51.172958601 +0000 UTC m=+0.439983798 container init cdd6dde7c7af3a41df5f582d18829d798ea8c6a7e1ae45b9befcfb7280d2e13f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:25:51.426 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:51 smithi179 podman[83670]: 2024-08-17 17:25:51.177998866 +0000 UTC m=+0.445024066 container start cdd6dde7c7af3a41df5f582d18829d798ea8c6a7e1ae45b9befcfb7280d2e13f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD) 2024-08-17T17:25:51.426 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:51 smithi179 ceph-mgr[83708]: -- 172.21.15.179:0/610725874 <== mon.1 v2:172.21.15.179:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f8962e30e0 con 0x55f8962c0800 2024-08-17T17:25:51.426 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:51 smithi179 bash[83670]: cdd6dde7c7af3a41df5f582d18829d798ea8c6a7e1ae45b9befcfb7280d2e13f 2024-08-17T17:25:51.426 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:51 smithi179 systemd[1]: Started Ceph mgr.x for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:25:51.448 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap 2024-08-17T17:25:51.461 INFO:tasks.workunit.client.0.smithi130.stdout:clone image105 -> image108 2024-08-17T17:25:51.566 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 0~1000 2024-08-17T17:25:51.616 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 1010~1000 2024-08-17T17:25:51.617 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:51.618 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap 2024-08-17T17:25:51.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:51 smithi179 ceph-mon[34775]: pgmap v32: 137 pgs: 137 active+clean; 757 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 5.8 MiB/s wr, 461 op/s 2024-08-17T17:25:51.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:51 smithi179 ceph-mon[34775]: osdmap e236: 8 total, 8 up, 8 in 2024-08-17T17:25:51.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:51 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/323161508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:51.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:51 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:51.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:51 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:51.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:51 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:51 smithi130 ceph-mon[31342]: pgmap v32: 137 pgs: 137 active+clean; 757 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 5.8 MiB/s wr, 461 op/s 2024-08-17T17:25:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:51 smithi130 ceph-mon[31342]: osdmap e236: 8 total, 8 up, 8 in 2024-08-17T17:25:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:51 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/323161508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:51 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:51 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:51 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:51.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:51 smithi130 ceph-mon[42749]: pgmap v32: 137 pgs: 137 active+clean; 757 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 37 MiB/s rd, 5.8 MiB/s wr, 461 op/s 2024-08-17T17:25:51.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:51 smithi130 ceph-mon[42749]: osdmap e236: 8 total, 8 up, 8 in 2024-08-17T17:25:51.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:51 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/323161508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:51.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:51 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:51.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:51 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:51.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:51 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:25:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:52 smithi179 ceph-mon[34775]: osdmap e237: 8 total, 8 up, 8 in 2024-08-17T17:25:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:52 smithi179 ceph-mon[34775]: from='client.15963 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:52 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:52 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:52 smithi179 ceph-mon[34775]: from='client.15969 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:52 smithi130 ceph-mon[31342]: osdmap e237: 8 total, 8 up, 8 in 2024-08-17T17:25:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:52 smithi130 ceph-mon[31342]: from='client.15963 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:52 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:52 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:52 smithi130 ceph-mon[31342]: from='client.15969 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:52 smithi130 ceph-mon[42749]: osdmap e237: 8 total, 8 up, 8 in 2024-08-17T17:25:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:52 smithi130 ceph-mon[42749]: from='client.15963 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:52 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:52 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:52 smithi130 ceph-mon[42749]: from='client.15969 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:53.464 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap 2024-08-17T17:25:53.475 INFO:tasks.workunit.client.0.smithi130.stdout:clone image108 -> image110 2024-08-17T17:25:53.594 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 1000~1000 2024-08-17T17:25:53.653 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 2010~1000 2024-08-17T17:25:53.655 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:25:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:53 smithi179 ceph-mon[34775]: pgmap v35: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 138 op/s 2024-08-17T17:25:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:53 smithi179 ceph-mon[34775]: osdmap e238: 8 total, 8 up, 8 in 2024-08-17T17:25:53.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:53 smithi130 ceph-mon[31342]: pgmap v35: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 138 op/s 2024-08-17T17:25:53.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:53 smithi130 ceph-mon[31342]: osdmap e238: 8 total, 8 up, 8 in 2024-08-17T17:25:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:53 smithi130 ceph-mon[42749]: pgmap v35: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 138 op/s 2024-08-17T17:25:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:53 smithi130 ceph-mon[42749]: osdmap e238: 8 total, 8 up, 8 in 2024-08-17T17:25:53.933 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:25:54.099 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:25:54.272 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.Clone (4474 ms) 2024-08-17T17:25:54.272 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.CloneParent 2024-08-17T17:25:54.305 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:25:54.315 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap 2024-08-17T17:25:54.576 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:54.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:54 smithi179 ceph-mon[34775]: osdmap e239: 8 total, 8 up, 8 in 2024-08-17T17:25:54.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:54 smithi179 ceph-mon[34775]: osdmap e240: 8 total, 8 up, 8 in 2024-08-17T17:25:54.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:54 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2353321900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:54 smithi130 ceph-mon[31342]: osdmap e239: 8 total, 8 up, 8 in 2024-08-17T17:25:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:54 smithi130 ceph-mon[31342]: osdmap e240: 8 total, 8 up, 8 in 2024-08-17T17:25:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:54 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/2353321900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:54 smithi130 ceph-mon[42749]: osdmap e239: 8 total, 8 up, 8 in 2024-08-17T17:25:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:54 smithi130 ceph-mon[42749]: osdmap e240: 8 total, 8 up, 8 in 2024-08-17T17:25:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:54 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/2353321900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:55 smithi179 ceph-mon[34775]: pgmap v39: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 41 MiB/s rd, 8.1 MiB/s wr, 140 op/s 2024-08-17T17:25:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:55 smithi179 ceph-mon[34775]: from='client.15981 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:55 smithi179 ceph-mon[34775]: from='client.25898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:55 smithi179 ceph-mon[34775]: osdmap e241: 8 total, 8 up, 8 in 2024-08-17T17:25:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:55 smithi130 ceph-mon[31342]: pgmap v39: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 41 MiB/s rd, 8.1 MiB/s wr, 140 op/s 2024-08-17T17:25:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:55 smithi130 ceph-mon[31342]: from='client.15981 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:55 smithi130 ceph-mon[31342]: from='client.25898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:55 smithi130 ceph-mon[31342]: osdmap e241: 8 total, 8 up, 8 in 2024-08-17T17:25:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:55 smithi130 ceph-mon[42749]: pgmap v39: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 41 MiB/s rd, 8.1 MiB/s wr, 140 op/s 2024-08-17T17:25:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:55 smithi130 ceph-mon[42749]: from='client.15981 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:55 smithi130 ceph-mon[42749]: from='client.25898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:55 smithi130 ceph-mon[42749]: osdmap e241: 8 total, 8 up, 8 in 2024-08-17T17:25:56.209 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap 2024-08-17T17:25:56.221 INFO:tasks.workunit.client.0.smithi130.stdout:clone image111 -> image114 2024-08-17T17:25:56.319 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:25:56.464 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:25:56.592 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.CloneParent (2320 ms) 2024-08-17T17:25:56.592 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.CloneUpdateAfterPrepare 2024-08-17T17:25:56.623 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:25:56.632 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 0~10 2024-08-17T17:25:56.640 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap 2024-08-17T17:25:56.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[83685]: [17/Aug/2024:17:25:56] ENGINE Bus STARTING 2024-08-17T17:25:56.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[83685]: CherryPy Checker: 2024-08-17T17:25:56.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[83685]: The Application mounted at '' has an empty config. 2024-08-17T17:25:56.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[83685]: 2024-08-17T17:25:56.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[83685]: [17/Aug/2024:17:25:56] ENGINE Serving on http://:::9283 2024-08-17T17:25:56.843 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[83685]: [17/Aug/2024:17:25:56] ENGINE Bus STARTED 2024-08-17T17:25:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[42749]: osdmap e242: 8 total, 8 up, 8 in 2024-08-17T17:25:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[42749]: Standby manager daemon x restarted 2024-08-17T17:25:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[42749]: Standby manager daemon x started 2024-08-17T17:25:57.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/2615933071' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:25:57.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/2615933071' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:25:57.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/2615933071' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:25:57.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/2615933071' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:25:57.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:57.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:57.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[31342]: osdmap e242: 8 total, 8 up, 8 in 2024-08-17T17:25:57.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:57.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:57.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[31342]: Standby manager daemon x restarted 2024-08-17T17:25:57.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[31342]: Standby manager daemon x started 2024-08-17T17:25:57.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/2615933071' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:25:57.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/2615933071' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:25:57.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/2615933071' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:25:57.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:56 smithi130 ceph-mon[31342]: from='mgr.? 172.21.15.179:0/2615933071' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:25:57.228 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:25:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:25:57.228 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:57.228 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:57.228 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-mon[34775]: osdmap e242: 8 total, 8 up, 8 in 2024-08-17T17:25:57.228 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:57.229 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:25:57.229 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-mon[34775]: Standby manager daemon x restarted 2024-08-17T17:25:57.229 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-mon[34775]: Standby manager daemon x started 2024-08-17T17:25:57.229 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/2615933071' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:25:57.229 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/2615933071' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:25:57.229 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/2615933071' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:25:57.229 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:56 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/2615933071' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:25:57.489 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:25:57.957 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:57 smithi179 ceph-mon[34775]: pgmap v42: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:57.958 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:57 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/1270268697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:57.958 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:57 smithi179 ceph-mon[34775]: mgrmap e34: y(active, since 27s), standbys: x 2024-08-17T17:25:57.958 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:57 smithi179 ceph-mon[34775]: osdmap e243: 8 total, 8 up, 8 in 2024-08-17T17:25:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:57 smithi130 ceph-mon[31342]: pgmap v42: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:57 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/1270268697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:57 smithi130 ceph-mon[31342]: mgrmap e34: y(active, since 27s), standbys: x 2024-08-17T17:25:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:57 smithi130 ceph-mon[31342]: osdmap e243: 8 total, 8 up, 8 in 2024-08-17T17:25:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:57 smithi130 ceph-mon[42749]: pgmap v42: 137 pgs: 137 active+clean; 761 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:25:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:57 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/1270268697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:25:58.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:57 smithi130 ceph-mon[42749]: mgrmap e34: y(active, since 27s), standbys: x 2024-08-17T17:25:58.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:57 smithi130 ceph-mon[42749]: osdmap e243: 8 total, 8 up, 8 in 2024-08-17T17:25:58.561 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap 2024-08-17T17:25:58.571 INFO:tasks.workunit.client.0.smithi130.stdout:clone image115 -> image118 2024-08-17T17:25:58.659 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:25:58.797 INFO:tasks.workunit.client.0.smithi130.stdout:write: Y 0~1 2024-08-17T17:25:58.824 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:25:58.889 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:25:58.987 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.CloneUpdateAfterPrepare (2395 ms) 2024-08-17T17:25:58.987 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.TriggerAssertSnapcSeq 2024-08-17T17:25:59.015 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:25:59.026 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 41943040~10 2024-08-17T17:25:59.035 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:25:59.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:58 smithi130 ceph-mon[31342]: from='client.25910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:59.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:58 smithi130 ceph-mon[31342]: from='client.16005 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:59.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:58 smithi130 ceph-mon[31342]: osdmap e244: 8 total, 8 up, 8 in 2024-08-17T17:25:59.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:58 smithi130 ceph-mon[42749]: from='client.25910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:59.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:58 smithi130 ceph-mon[42749]: from='client.16005 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:59.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:58 smithi130 ceph-mon[42749]: osdmap e244: 8 total, 8 up, 8 in 2024-08-17T17:25:59.299 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:58 smithi179 ceph-mon[34775]: from='client.25910 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:59.299 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:58 smithi179 ceph-mon[34775]: from='client.16005 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:25:59.299 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:58 smithi179 ceph-mon[34775]: osdmap e244: 8 total, 8 up, 8 in 2024-08-17T17:26:00.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: pgmap v44: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 4.8 MiB/s wr, 992 op/s 2024-08-17T17:26:00.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:00.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:00.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:26:00.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:26:00.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:00.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:26:00.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:26:00.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:00.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:26:00.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:26:00.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-17T17:26:00.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:26:00.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:26:00.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-17T17:26:00.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:26:00.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:25:59 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-17T17:26:00.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:25:59] "GET /metrics HTTP/1.1" 200 38777 "" "Prometheus/2.51.0" 2024-08-17T17:26:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: pgmap v44: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 4.8 MiB/s wr, 992 op/s 2024-08-17T17:26:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:26:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:26:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:26:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:26:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:26:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:26:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-17T17:26:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:26:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:26:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-17T17:26:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:26:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-17T17:26:00.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: pgmap v44: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 4.8 MiB/s wr, 992 op/s 2024-08-17T17:26:00.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:26:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:26:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:26:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:26:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:26:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T17:26:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-17T17:26:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:26:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T17:26:00.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-17T17:26:00.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:26:00.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:25:59 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-17T17:26:00.433 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:26:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:00.888 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 41943041~10 2024-08-17T17:26:00.894 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:26:01.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:00 smithi130 ceph-mon[31342]: Upgrade: Setting container_image for all mgr 2024-08-17T17:26:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:00 smithi130 ceph-mon[31342]: Upgrade: It appears safe to stop mon.a 2024-08-17T17:26:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:00 smithi130 ceph-mon[31342]: osdmap e245: 8 total, 8 up, 8 in 2024-08-17T17:26:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:00 smithi130 ceph-mon[31342]: from='client.? 172.21.15.130:0/403958397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:26:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:00 smithi130 ceph-mon[42749]: Upgrade: Setting container_image for all mgr 2024-08-17T17:26:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:00 smithi130 ceph-mon[42749]: Upgrade: It appears safe to stop mon.a 2024-08-17T17:26:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:00 smithi130 ceph-mon[42749]: osdmap e245: 8 total, 8 up, 8 in 2024-08-17T17:26:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:00 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/403958397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:26:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:00 smithi179 ceph-mon[34775]: Upgrade: Setting container_image for all mgr 2024-08-17T17:26:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:00 smithi179 ceph-mon[34775]: Upgrade: It appears safe to stop mon.a 2024-08-17T17:26:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:00 smithi179 ceph-mon[34775]: osdmap e245: 8 total, 8 up, 8 in 2024-08-17T17:26:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:00 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/403958397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:26:02.024 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 41943042~10 2024-08-17T17:26:02.036 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:26:02.068 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[42749]: pgmap v47: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 4.8 MiB/s wr, 993 op/s 2024-08-17T17:26:02.068 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[42749]: from='client.16017 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:02.068 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[42749]: from='client.25934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:02.068 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[42749]: osdmap e246: 8 total, 8 up, 8 in 2024-08-17T17:26:02.068 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:02.068 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:02.069 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:26:02.069 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:26:02.069 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[42749]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:26:02.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[31342]: pgmap v47: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 4.8 MiB/s wr, 993 op/s 2024-08-17T17:26:02.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[31342]: from='client.16017 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:02.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[31342]: from='client.25934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:02.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[31342]: osdmap e246: 8 total, 8 up, 8 in 2024-08-17T17:26:02.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:02.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[31342]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:02.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:26:02.069 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:26:02.070 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:01 smithi130 ceph-mon[31342]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:26:02.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:01 smithi179 ceph-mon[34775]: pgmap v47: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 4.8 MiB/s wr, 993 op/s 2024-08-17T17:26:02.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:01 smithi179 ceph-mon[34775]: from='client.16017 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:02.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:01 smithi179 ceph-mon[34775]: from='client.25934 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:02.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:01 smithi179 ceph-mon[34775]: osdmap e246: 8 total, 8 up, 8 in 2024-08-17T17:26:02.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:01 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:02.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:01 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='mgr.y' 2024-08-17T17:26:02.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:01 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:26:02.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:01 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:26:02.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:01 smithi179 ceph-mon[34775]: from='mgr.15753 172.21.15.130:0/1890451470' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:26:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:02 smithi130 ceph-mon[31342]: Upgrade: Updating mon.a 2024-08-17T17:26:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:02 smithi130 ceph-mon[31342]: Deploying daemon mon.a on smithi130 2024-08-17T17:26:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:02 smithi130 ceph-mon[31342]: osdmap e247: 8 total, 8 up, 8 in 2024-08-17T17:26:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:02 smithi130 ceph-mon[42749]: Upgrade: Updating mon.a 2024-08-17T17:26:03.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:02 smithi130 ceph-mon[42749]: Deploying daemon mon.a on smithi130 2024-08-17T17:26:03.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:02 smithi130 ceph-mon[42749]: osdmap e247: 8 total, 8 up, 8 in 2024-08-17T17:26:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:02 smithi179 ceph-mon[34775]: Upgrade: Updating mon.a 2024-08-17T17:26:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:02 smithi179 ceph-mon[34775]: Deploying daemon mon.a on smithi130 2024-08-17T17:26:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:02 smithi179 ceph-mon[34775]: osdmap e247: 8 total, 8 up, 8 in 2024-08-17T17:26:03.392 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:26:03.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:03 smithi130 systemd[1]: Stopping Ceph mon.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:26:03.894 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:03 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[31319]: 2024-08-17T17:26:03.486+0000 7fc63d61b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:26:03.895 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:03 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[31319]: 2024-08-17T17:26:03.486+0000 7fc63d61b640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-08-17T17:26:03.895 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:03 smithi130 podman[105303]: 2024-08-17 17:26:03.619363472 +0000 UTC m=+0.253847617 container died 7a678d70ca3b84607c7d84ff1e3dfc18986485d46f40df4df678ba61c50cd651 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=quincy-f2c39d4, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-08-17T17:26:04.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:04 smithi130 podman[105303]: 2024-08-17 17:26:04.084790513 +0000 UTC m=+0.719274658 container cleanup 7a678d70ca3b84607c7d84ff1e3dfc18986485d46f40df4df678ba61c50cd651 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.build-date=20240731) 2024-08-17T17:26:04.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:04 smithi130 bash[105303]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a 2024-08-17T17:26:04.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:04.883 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:04 smithi130 podman[105346]: 2024-08-17 17:26:04.461708804 +0000 UTC m=+0.837351096 container remove 7a678d70ca3b84607c7d84ff1e3dfc18986485d46f40df4df678ba61c50cd651 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-08-17T17:26:04.883 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:04 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.a.service: Deactivated successfully. 2024-08-17T17:26:04.883 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:04 smithi130 systemd[1]: Stopped Ceph mon.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:26:04.883 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:04 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.a.service: Consumed 15.231s CPU time. 2024-08-17T17:26:05.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:04 smithi130 systemd[1]: Starting Ceph mon.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:26:05.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 podman[105484]: 2024-08-17 17:26:05.079622514 +0000 UTC m=+0.094575516 container create 79749aebcefa9827c6fada1f615d739fe273e8768e0aff218f6e74616b5bd5e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:26:05.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 podman[105484]: 2024-08-17 17:26:05.002449889 +0000 UTC m=+0.017402842 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:26:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 podman[105484]: 2024-08-17 17:26:05.256549195 +0000 UTC m=+0.271502146 container init 79749aebcefa9827c6fada1f615d739fe273e8768e0aff218f6e74616b5bd5e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:26:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 podman[105484]: 2024-08-17 17:26:05.260826601 +0000 UTC m=+0.275779542 container start 79749aebcefa9827c6fada1f615d739fe273e8768e0aff218f6e74616b5bd5e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-08-17T17:26:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:26:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev), process ceph-mon, pid 7 2024-08-17T17:26:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: pidfile_write: ignore empty --pid-file 2024-08-17T17:26:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: load: jerasure load: lrc 2024-08-17T17:26:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: RocksDB version: 7.9.2 2024-08-17T17:26:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Git sha 0 2024-08-17T17:26:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Compile date 2024-08-15 20:43:31 2024-08-17T17:26:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: DB SUMMARY 2024-08-17T17:26:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: DB Session ID: EJKUEVPB6CDG21YX86ON 2024-08-17T17:26:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:26:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:26:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1173 Bytes 2024-08-17T17:26:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-08-17T17:26:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 3156318 ; 2024-08-17T17:26:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:26:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:26:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:26:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T17:26:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:26:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T17:26:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.env: 0x56034be45be0 2024-08-17T17:26:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T17:26:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.info_log: 0x56034c8b1700 2024-08-17T17:26:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:26:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.statistics: (nil) 2024-08-17T17:26:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.use_fsync: 0 2024-08-17T17:26:05.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:26:05.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:26:05.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:26:05.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:26:05.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:26:05.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:26:05.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:26:05.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:26:05.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:26:05.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:26:05.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:26:05.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.db_log_dir: 2024-08-17T17:26:05.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.wal_dir: 2024-08-17T17:26:05.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:26:05.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:26:05.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:26:05.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:26:05.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:26:05.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:26:05.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:26:05.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:26:05.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.write_buffer_manager: 0x56034c8b4960 2024-08-17T17:26:05.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:26:05.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:26:05.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:26:05.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:26:05.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:26:05.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:26:05.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:26:05.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:26:05.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.unordered_write: 0 2024-08-17T17:26:05.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:26:05.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:26:05.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:26:05.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:26:05.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.row_cache: None 2024-08-17T17:26:05.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.wal_filter: None 2024-08-17T17:26:05.674 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:26:05.674 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:26:05.674 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:26:05.674 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:26:05.674 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.wal_compression: 0 2024-08-17T17:26:05.674 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:26:05.674 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:26:05.674 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:26:05.674 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:26:05.675 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:26:05.675 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:26:05.675 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:26:05.675 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:26:05.675 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:26:05.675 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:26:05.675 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:26:05.675 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T17:26:05.675 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:26:05.675 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:26:05.675 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:26:05.675 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:26:05.675 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:26:05.675 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:26:05.676 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:26:05.676 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:26:05.676 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:26:05.676 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:26:05.676 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:26:05.676 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_open_files: -1 2024-08-17T17:26:05.676 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:26:05.676 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:26:05.676 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:26:05.676 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:26:05.676 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:26:05.676 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Compression algorithms supported: 2024-08-17T17:26:05.676 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: kZSTD supported: 0 2024-08-17T17:26:05.677 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:26:05.677 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:26:05.677 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:26:05.677 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:26:05.677 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:26:05.677 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:26:05.677 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:26:05.677 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:26:05.677 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T17:26:05.677 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-08-17T17:26:05.677 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T17:26:05.677 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:26:05.678 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.merge_operator: 2024-08-17T17:26:05.678 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compaction_filter: None 2024-08-17T17:26:05.678 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:26:05.678 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:26:05.678 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:26:05.678 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:26:05.678 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56034c8b1380) 2024-08-17T17:26:05.678 INFO:journalctl@ceph.mon.a.smithi130.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:26:05.678 INFO:journalctl@ceph.mon.a.smithi130.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:26:05.678 INFO:journalctl@ceph.mon.a.smithi130.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:26:05.678 INFO:journalctl@ceph.mon.a.smithi130.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:26:05.678 INFO:journalctl@ceph.mon.a.smithi130.stdout: index_type: 0 2024-08-17T17:26:05.678 INFO:journalctl@ceph.mon.a.smithi130.stdout: data_block_index_type: 0 2024-08-17T17:26:05.679 INFO:journalctl@ceph.mon.a.smithi130.stdout: index_shortening: 1 2024-08-17T17:26:05.679 INFO:journalctl@ceph.mon.a.smithi130.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:26:05.679 INFO:journalctl@ceph.mon.a.smithi130.stdout: checksum: 4 2024-08-17T17:26:05.679 INFO:journalctl@ceph.mon.a.smithi130.stdout: no_block_cache: 0 2024-08-17T17:26:05.679 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache: 0x56034c8a5610 2024-08-17T17:26:05.679 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:26:05.679 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache_options: 2024-08-17T17:26:05.679 INFO:journalctl@ceph.mon.a.smithi130.stdout: capacity : 536870912 2024-08-17T17:26:05.679 INFO:journalctl@ceph.mon.a.smithi130.stdout: num_shard_bits : 4 2024-08-17T17:26:05.679 INFO:journalctl@ceph.mon.a.smithi130.stdout: strict_capacity_limit : 0 2024-08-17T17:26:05.679 INFO:journalctl@ceph.mon.a.smithi130.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:26:05.679 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_cache_compressed: (nil) 2024-08-17T17:26:05.679 INFO:journalctl@ceph.mon.a.smithi130.stdout: persistent_cache: (nil) 2024-08-17T17:26:05.679 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_size: 4096 2024-08-17T17:26:05.680 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_size_deviation: 10 2024-08-17T17:26:05.680 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_restart_interval: 16 2024-08-17T17:26:05.680 INFO:journalctl@ceph.mon.a.smithi130.stdout: index_block_restart_interval: 1 2024-08-17T17:26:05.680 INFO:journalctl@ceph.mon.a.smithi130.stdout: metadata_block_size: 4096 2024-08-17T17:26:05.680 INFO:journalctl@ceph.mon.a.smithi130.stdout: partition_filters: 0 2024-08-17T17:26:05.680 INFO:journalctl@ceph.mon.a.smithi130.stdout: use_delta_encoding: 1 2024-08-17T17:26:05.680 INFO:journalctl@ceph.mon.a.smithi130.stdout: filter_policy: bloomfilter 2024-08-17T17:26:05.680 INFO:journalctl@ceph.mon.a.smithi130.stdout: whole_key_filtering: 1 2024-08-17T17:26:05.680 INFO:journalctl@ceph.mon.a.smithi130.stdout: verify_compression: 0 2024-08-17T17:26:05.680 INFO:journalctl@ceph.mon.a.smithi130.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:26:05.680 INFO:journalctl@ceph.mon.a.smithi130.stdout: format_version: 5 2024-08-17T17:26:05.680 INFO:journalctl@ceph.mon.a.smithi130.stdout: enable_index_compression: 1 2024-08-17T17:26:05.680 INFO:journalctl@ceph.mon.a.smithi130.stdout: block_align: 0 2024-08-17T17:26:05.680 INFO:journalctl@ceph.mon.a.smithi130.stdout: max_auto_readahead_size: 262144 2024-08-17T17:26:05.681 INFO:journalctl@ceph.mon.a.smithi130.stdout: prepopulate_block_cache: 0 2024-08-17T17:26:05.681 INFO:journalctl@ceph.mon.a.smithi130.stdout: initial_auto_readahead_size: 8192 2024-08-17T17:26:05.681 INFO:journalctl@ceph.mon.a.smithi130.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T17:26:05.681 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:26:05.681 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:26:05.681 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compression: NoCompression 2024-08-17T17:26:05.681 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:26:05.681 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:26:05.681 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:26:05.681 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.num_levels: 7 2024-08-17T17:26:05.681 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:26:05.681 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:26:05.681 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:26:05.682 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:26:05.682 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:26:05.682 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:26:05.682 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:26:05.682 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:26:05.682 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:26:05.682 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:26:05.682 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:26:05.682 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:26:05.682 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:26:05.682 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:26:05.682 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:26:05.682 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:26:05.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:26:05.684 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:26:05.685 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:26:05.685 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:26:05.685 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:26:05.685 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:26:05.685 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:26:05.685 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:26:05.685 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:26:05.685 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T17:26:05.685 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:26:05.685 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:26:05.685 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:26:05.685 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:26:05.685 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:26:05.685 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:26:05.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:26:05.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:26:05.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:26:05.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:26:05.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:26:05.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.ttl: 2592000 2024-08-17T17:26:05.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:26:05.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T17:26:05.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T17:26:05.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:26:05.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:26:05.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:26:05.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:26:05.687 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:26:05.687 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:26:05.687 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T17:26:05.687 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T17:26:05.687 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T17:26:05.687 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T17:26:05.687 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-08-17T17:26:05.687 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:26:05.687 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12885, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T17:26:05.687 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-08-17T17:26:05.687 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 547788f8-3706-47e8-bc62-4e916c76bb05 2024-08-17T17:26:05.687 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723915565288056, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-08-17T17:26:05.687 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-08-17T17:26:05.687 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723915565298751, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 2975590, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12879, "largest_seqno": 14116, "table_properties": {"data_size": 2968409, "index_size": 4664, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 13738, "raw_average_key_size": 23, "raw_value_size": 2955331, "raw_average_value_size": 5148, "num_data_blocks": 205, "num_entries": 574, "num_filter_entries": 574, "num_deletions": 6, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723915565, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "547788f8-3706-47e8-bc62-4e916c76bb05", "db_session_id": "EJKUEVPB6CDG21YX86ON", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-08-17T17:26:05.688 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723915565298889, "job": 1, "event": "recovery_finished"} 2024-08-17T17:26:05.688 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-08-17T17:26:05.688 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:26:05.688 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:26:05.688 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56034c8d4e00 2024-08-17T17:26:05.688 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: DB pointer 0x56034c9d2000 2024-08-17T17:26:05.688 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T17:26:05.688 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T17:26:05.688 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** DB Stats ** 2024-08-17T17:26:05.688 INFO:journalctl@ceph.mon.a.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:26:05.688 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:26:05.688 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:26:05.688 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:26:05.689 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:26:05.689 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:26:05.689 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:26:05.689 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:26:05.689 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** Compaction Stats [default] ** 2024-08-17T17:26:05.689 INFO:journalctl@ceph.mon.a.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:26:05.689 INFO:journalctl@ceph.mon.a.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T17:26:05.689 INFO:journalctl@ceph.mon.a.smithi130.stdout: L0 1/0 2.84 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 556.6 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-08-17T17:26:05.689 INFO:journalctl@ceph.mon.a.smithi130.stdout: L6 1/0 8.07 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-17T17:26:05.689 INFO:journalctl@ceph.mon.a.smithi130.stdout: Sum 2/0 10.91 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 556.6 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-08-17T17:26:05.689 INFO:journalctl@ceph.mon.a.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 556.6 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-08-17T17:26:05.689 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:26:05.689 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** Compaction Stats [default] ** 2024-08-17T17:26:05.690 INFO:journalctl@ceph.mon.a.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:26:05.690 INFO:journalctl@ceph.mon.a.smithi130.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:26:05.690 INFO:journalctl@ceph.mon.a.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 556.6 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-08-17T17:26:05.690 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:26:05.690 INFO:journalctl@ceph.mon.a.smithi130.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T17:26:05.690 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:26:05.690 INFO:journalctl@ceph.mon.a.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:26:05.690 INFO:journalctl@ceph.mon.a.smithi130.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-08-17T17:26:05.690 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:26:05.690 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:26:05.690 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:26:05.690 INFO:journalctl@ceph.mon.a.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:26:05.691 INFO:journalctl@ceph.mon.a.smithi130.stdout: Cumulative compaction: 0.00 GB write, 212.98 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:26:05.691 INFO:journalctl@ceph.mon.a.smithi130.stdout: Interval compaction: 0.00 GB write, 212.98 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:26:05.691 INFO:journalctl@ceph.mon.a.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:26:05.691 INFO:journalctl@ceph.mon.a.smithi130.stdout: Block cache BinnedLRUCache@0x56034c8a5610#7 capacity: 512.00 MB usage: 6.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-08-17T17:26:05.691 INFO:journalctl@ceph.mon.a.smithi130.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-08-17T17:26:05.691 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:26:05.691 INFO:journalctl@ceph.mon.a.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:26:05.691 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: starting mon.a rank 0 at public addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] at bind addrs [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:26:05.691 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: mon.a@-1(???) e3 preinit fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:26:05.691 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: mon.a@-1(???).mds e9 new map 2024-08-17T17:26:05.691 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: mon.a@-1(???).mds e9 print_map 2024-08-17T17:26:05.691 INFO:journalctl@ceph.mon.a.smithi130.stdout: e9 2024-08-17T17:26:05.691 INFO:journalctl@ceph.mon.a.smithi130.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-17T17:26:05.692 INFO:journalctl@ceph.mon.a.smithi130.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:26:05.692 INFO:journalctl@ceph.mon.a.smithi130.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:26:05.692 INFO:journalctl@ceph.mon.a.smithi130.stdout: legacy client fscid: 1 2024-08-17T17:26:05.692 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:26:05.692 INFO:journalctl@ceph.mon.a.smithi130.stdout: Filesystem 'foo' (1) 2024-08-17T17:26:05.692 INFO:journalctl@ceph.mon.a.smithi130.stdout: fs_name foo 2024-08-17T17:26:05.692 INFO:journalctl@ceph.mon.a.smithi130.stdout: epoch 8 2024-08-17T17:26:05.692 INFO:journalctl@ceph.mon.a.smithi130.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-17T17:26:05.692 INFO:journalctl@ceph.mon.a.smithi130.stdout: created 2024-08-17T17:21:52.146143+0000 2024-08-17T17:26:05.692 INFO:journalctl@ceph.mon.a.smithi130.stdout: modified 2024-08-17T17:22:05.340164+0000 2024-08-17T17:26:05.692 INFO:journalctl@ceph.mon.a.smithi130.stdout: tableserver 0 2024-08-17T17:26:05.692 INFO:journalctl@ceph.mon.a.smithi130.stdout: root 0 2024-08-17T17:26:05.693 INFO:journalctl@ceph.mon.a.smithi130.stdout: session_timeout 60 2024-08-17T17:26:05.693 INFO:journalctl@ceph.mon.a.smithi130.stdout: session_autoclose 300 2024-08-17T17:26:05.693 INFO:journalctl@ceph.mon.a.smithi130.stdout: max_file_size 1099511627776 2024-08-17T17:26:05.693 INFO:journalctl@ceph.mon.a.smithi130.stdout: max_xattr_size 65536 2024-08-17T17:26:05.693 INFO:journalctl@ceph.mon.a.smithi130.stdout: required_client_features {} 2024-08-17T17:26:05.693 INFO:journalctl@ceph.mon.a.smithi130.stdout: last_failure 0 2024-08-17T17:26:05.693 INFO:journalctl@ceph.mon.a.smithi130.stdout: last_failure_osd_epoch 0 2024-08-17T17:26:05.693 INFO:journalctl@ceph.mon.a.smithi130.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:26:05.693 INFO:journalctl@ceph.mon.a.smithi130.stdout: max_mds 1 2024-08-17T17:26:05.693 INFO:journalctl@ceph.mon.a.smithi130.stdout: in 0 2024-08-17T17:26:05.693 INFO:journalctl@ceph.mon.a.smithi130.stdout: up {0=24548} 2024-08-17T17:26:05.693 INFO:journalctl@ceph.mon.a.smithi130.stdout: failed 2024-08-17T17:26:05.693 INFO:journalctl@ceph.mon.a.smithi130.stdout: damaged 2024-08-17T17:26:05.693 INFO:journalctl@ceph.mon.a.smithi130.stdout: stopped 2024-08-17T17:26:05.694 INFO:journalctl@ceph.mon.a.smithi130.stdout: data_pools [4] 2024-08-17T17:26:05.694 INFO:journalctl@ceph.mon.a.smithi130.stdout: metadata_pool 3 2024-08-17T17:26:05.694 INFO:journalctl@ceph.mon.a.smithi130.stdout: inline_data disabled 2024-08-17T17:26:05.694 INFO:journalctl@ceph.mon.a.smithi130.stdout: balancer 2024-08-17T17:26:05.694 INFO:journalctl@ceph.mon.a.smithi130.stdout: bal_rank_mask -1 2024-08-17T17:26:05.694 INFO:journalctl@ceph.mon.a.smithi130.stdout: standby_count_wanted 1 2024-08-17T17:26:05.694 INFO:journalctl@ceph.mon.a.smithi130.stdout: qdb_cluster leader: 0 members: 2024-08-17T17:26:05.694 INFO:journalctl@ceph.mon.a.smithi130.stdout: [mds.foo.smithi130.jmaidg{0:24548} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.130:6834/1214210479,v1:172.21.15.130:6835/1214210479] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:26:05.694 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:26:05.694 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:26:05.695 INFO:journalctl@ceph.mon.a.smithi130.stdout: Standby daemons: 2024-08-17T17:26:05.695 INFO:journalctl@ceph.mon.a.smithi130.stdout: 2024-08-17T17:26:05.695 INFO:journalctl@ceph.mon.a.smithi130.stdout: [mds.foo.smithi179.zkkvno{-1:24466} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.179:6832/449182576,v1:172.21.15.179:6833/449182576] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:26:05.695 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: mon.a@-1(???).osd e248 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-17T17:26:05.695 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: mon.a@-1(???).osd e248 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:26:05.695 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: mon.a@-1(???).osd e248 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:26:05.695 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: mon.a@-1(???).osd e248 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:26:05.695 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 ceph-mon[105521]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2024-08-17T17:26:05.695 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 bash[105484]: 79749aebcefa9827c6fada1f615d739fe273e8768e0aff218f6e74616b5bd5e3 2024-08-17T17:26:05.695 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:05 smithi130 systemd[1]: Started Ceph mon.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:26:06.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:06.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:06.287 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:26:06.791 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 41943040 2024-08-17T17:26:06.822 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:26:06.877 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:26:07.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:07.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[83685]: ignoring --setuser ceph since I am not root 2024-08-17T17:26:07.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[83685]: ignoring --setgroup ceph since I am not root 2024-08-17T17:26:07.093 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mgr[83708]: -- 172.21.15.179:0/2764585607 <== mon.2 v2:172.21.15.130:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5619135250e0 con 0x561913502800 2024-08-17T17:26:07.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: from='client.16035 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:07.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: from='client.16041 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:07.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: pgmap v52: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 939 KiB/s wr, 522 op/s 2024-08-17T17:26:07.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/3282097300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:26:07.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: mon.a calling monitor election 2024-08-17T17:26:07.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:26:07.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: monmap epoch 3 2024-08-17T17:26:07.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:26:07.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: last_changed 2024-08-17T17:16:41.007482+0000 2024-08-17T17:26:07.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: created 2024-08-17T17:15:06.618377+0000 2024-08-17T17:26:07.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: min_mon_release 17 (quincy) 2024-08-17T17:26:07.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: election_strategy: 1 2024-08-17T17:26:07.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-08-17T17:26:07.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: 1: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-08-17T17:26:07.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: 2: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-08-17T17:26:07.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:26:07.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: osdmap e248: 8 total, 8 up, 8 in 2024-08-17T17:26:07.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: mgrmap e34: y(active, since 36s), standbys: x 2024-08-17T17:26:07.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-08-17T17:26:07.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:26:07.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: application not enabled on pool 'test-librbd-smithi130-87148-9' 2024-08-17T17:26:07.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:26:07.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: from='mgr.15753 ' entity='' 2024-08-17T17:26:07.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:06 smithi179 ceph-mon[34775]: mgrmap e35: y(active, since 36s), standbys: x 2024-08-17T17:26:07.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ignoring --setuser ceph since I am not root 2024-08-17T17:26:07.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ignoring --setgroup ceph since I am not root 2024-08-17T17:26:07.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mgr[95367]: -- 172.21.15.130:0/1924100631 <== mon.0 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55aff28af0e0 con 0x55aff288c800 2024-08-17T17:26:07.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: from='client.16035 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:07.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: from='client.16041 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:07.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: pgmap v52: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 939 KiB/s wr, 522 op/s 2024-08-17T17:26:07.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3282097300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:26:07.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: mon.a calling monitor election 2024-08-17T17:26:07.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:26:07.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: monmap epoch 3 2024-08-17T17:26:07.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:26:07.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: last_changed 2024-08-17T17:16:41.007482+0000 2024-08-17T17:26:07.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: created 2024-08-17T17:15:06.618377+0000 2024-08-17T17:26:07.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: min_mon_release 17 (quincy) 2024-08-17T17:26:07.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: election_strategy: 1 2024-08-17T17:26:07.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-08-17T17:26:07.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: 1: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-08-17T17:26:07.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: 2: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-08-17T17:26:07.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:26:07.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: osdmap e248: 8 total, 8 up, 8 in 2024-08-17T17:26:07.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: mgrmap e34: y(active, since 36s), standbys: x 2024-08-17T17:26:07.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-08-17T17:26:07.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:26:07.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: application not enabled on pool 'test-librbd-smithi130-87148-9' 2024-08-17T17:26:07.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:26:07.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: from='mgr.15753 ' entity='' 2024-08-17T17:26:07.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[105521]: mgrmap e35: y(active, since 36s), standbys: x 2024-08-17T17:26:07.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: from='client.16035 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:07.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: from='client.16041 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:07.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: pgmap v52: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 1.3 MiB/s rd, 939 KiB/s wr, 522 op/s 2024-08-17T17:26:07.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/3282097300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-08-17T17:26:07.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: mon.a calling monitor election 2024-08-17T17:26:07.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:26:07.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: monmap epoch 3 2024-08-17T17:26:07.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:26:07.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: last_changed 2024-08-17T17:16:41.007482+0000 2024-08-17T17:26:07.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: created 2024-08-17T17:15:06.618377+0000 2024-08-17T17:26:07.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: min_mon_release 17 (quincy) 2024-08-17T17:26:07.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: election_strategy: 1 2024-08-17T17:26:07.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-08-17T17:26:07.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: 1: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-08-17T17:26:07.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: 2: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-08-17T17:26:07.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:26:07.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: osdmap e248: 8 total, 8 up, 8 in 2024-08-17T17:26:07.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: mgrmap e34: y(active, since 36s), standbys: x 2024-08-17T17:26:07.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-08-17T17:26:07.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:26:07.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: application not enabled on pool 'test-librbd-smithi130-87148-9' 2024-08-17T17:26:07.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:26:07.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: from='mgr.15753 ' entity='' 2024-08-17T17:26:07.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:06 smithi130 ceph-mon[42749]: mgrmap e35: y(active, since 36s), standbys: x 2024-08-17T17:26:07.900 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.TriggerAssertSnapcSeq (8913 ms) 2024-08-17T17:26:07.900 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.SnapTrimBeforePrepare 2024-08-17T17:26:07.929 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:26:07.942 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 41943040~10 2024-08-17T17:26:07.950 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:26:08.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:07 smithi179 ceph-mon[34775]: osdmap e249: 8 total, 8 up, 8 in 2024-08-17T17:26:08.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:07 smithi130 ceph-mon[105521]: osdmap e249: 8 total, 8 up, 8 in 2024-08-17T17:26:08.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:07 smithi130 ceph-mon[42749]: osdmap e249: 8 total, 8 up, 8 in 2024-08-17T17:26:08.919 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 1 2024-08-17T17:26:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:08 smithi179 ceph-mon[34775]: osdmap e250: 8 total, 8 up, 8 in 2024-08-17T17:26:09.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:08 smithi130 ceph-mon[105521]: osdmap e250: 8 total, 8 up, 8 in 2024-08-17T17:26:09.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:08 smithi130 ceph-mon[42749]: osdmap e250: 8 total, 8 up, 8 in 2024-08-17T17:26:09.640 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch ps' 2024-08-17T17:26:09.817 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 41943040 2024-08-17T17:26:09.836 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:26:09.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:09 smithi130 ceph-mon[105521]: osdmap e251: 8 total, 8 up, 8 in 2024-08-17T17:26:09.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:09 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4167337139' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:26:09.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:09 smithi130 ceph-mon[42749]: osdmap e251: 8 total, 8 up, 8 in 2024-08-17T17:26:09.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:09 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/4167337139' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:26:09.930 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:09 smithi179 ceph-mon[34775]: osdmap e251: 8 total, 8 up, 8 in 2024-08-17T17:26:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:09 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/4167337139' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:26:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:10.934 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:26:10.944 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:26:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:10 smithi179 ceph-mon[34775]: osdmap e252: 8 total, 8 up, 8 in 2024-08-17T17:26:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:10 smithi130 ceph-mon[105521]: osdmap e252: 8 total, 8 up, 8 in 2024-08-17T17:26:11.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:10 smithi130 ceph-mon[42749]: osdmap e252: 8 total, 8 up, 8 in 2024-08-17T17:26:11.964 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 41943040~10 2024-08-17T17:26:11.971 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap4 2024-08-17T17:26:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:11 smithi179 ceph-mon[34775]: osdmap e253: 8 total, 8 up, 8 in 2024-08-17T17:26:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:11 smithi179 ceph-mon[34775]: Standby manager daemon x restarted 2024-08-17T17:26:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:11 smithi179 ceph-mon[34775]: Standby manager daemon x started 2024-08-17T17:26:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:11 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/3869284207' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:26:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:11 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/3869284207' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:26:12.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:11 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/3869284207' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:26:12.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:11 smithi179 ceph-mon[34775]: from='mgr.? 172.21.15.179:0/3869284207' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:26:12.094 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:26:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[83685]: [17/Aug/2024:17:26:11] ENGINE Bus STARTING 2024-08-17T17:26:12.094 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:26:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[83685]: CherryPy Checker: 2024-08-17T17:26:12.094 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:26:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[83685]: The Application mounted at '' has an empty config. 2024-08-17T17:26:12.094 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:26:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[83685]: 2024-08-17T17:26:12.094 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:26:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[83685]: [17/Aug/2024:17:26:11] ENGINE Serving on http://:::9283 2024-08-17T17:26:12.094 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 17:26:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x[83685]: [17/Aug/2024:17:26:11] ENGINE Bus STARTED 2024-08-17T17:26:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:11 smithi130 ceph-mon[105521]: osdmap e253: 8 total, 8 up, 8 in 2024-08-17T17:26:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:11 smithi130 ceph-mon[105521]: Standby manager daemon x restarted 2024-08-17T17:26:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:11 smithi130 ceph-mon[105521]: Standby manager daemon x started 2024-08-17T17:26:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:11 smithi130 ceph-mon[105521]: from='mgr.? 172.21.15.179:0/3869284207' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:26:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:11 smithi130 ceph-mon[105521]: from='mgr.? 172.21.15.179:0/3869284207' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:26:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:11 smithi130 ceph-mon[105521]: from='mgr.? 172.21.15.179:0/3869284207' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:26:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:11 smithi130 ceph-mon[105521]: from='mgr.? 172.21.15.179:0/3869284207' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:26:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:11 smithi130 ceph-mon[42749]: osdmap e253: 8 total, 8 up, 8 in 2024-08-17T17:26:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:11 smithi130 ceph-mon[42749]: Standby manager daemon x restarted 2024-08-17T17:26:12.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:11 smithi130 ceph-mon[42749]: Standby manager daemon x started 2024-08-17T17:26:12.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:11 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/3869284207' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T17:26:12.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:11 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/3869284207' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T17:26:12.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:11 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/3869284207' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T17:26:12.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:11 smithi130 ceph-mon[42749]: from='mgr.? 172.21.15.179:0/3869284207' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T17:26:12.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:26:12.298+0000 7f8e38f43640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:26:12.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:26:12.298+0000 7f8e38f43640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:26:12.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:26:12.298+0000 7f8e38f43640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:26:12.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:26:12.298+0000 7f8e38f43640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:26:12.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:26:12.298+0000 7f8e38f43640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:26:12.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:26:12.299+0000 7f8e3bf49640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:26:12.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:26:12.299+0000 7f8e3bf49640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:26:12.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:26:12.299+0000 7f8e3bf49640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:26:12.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:26:12.299+0000 7f8e3bf49640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:26:12.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:26:12.299+0000 7f8e3bf49640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T17:26:12.979 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:26:12.979 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (86s) 37s ago 5m 17.2M - 0.27.0 11f11916f8cd 73ea4ebcf68b 2024-08-17T17:26:12.979 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi179 *:3000 running (61s) 17s ago 5m 77.9M - 9.4.7 954c08fa6188 399f89cc0037 2024-08-17T17:26:12.979 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foo.smithi130.jmaidg smithi130 running (4m) 37s ago 4m 17.8M - 17.2.7-1404-gf2c39d4d f69471c56189 91d7d1762094 2024-08-17T17:26:12.979 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foo.smithi179.zkkvno smithi179 running (4m) 17s ago 4m 13.3M - 17.2.7-1404-gf2c39d4d f69471c56189 0486517d425a 2024-08-17T17:26:12.979 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi179 *:8443,9283,8765 running (22s) 17s ago 9m 63.0M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb cdd6dde7c7af 2024-08-17T17:26:12.979 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (51s) 37s ago 10m 491M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 584426b3875b 2024-08-17T17:26:12.979 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (10m) 37s ago 11m 54.0M 2048M 17.2.7-1404-gf2c39d4d f69471c56189 7a678d70ca3b 2024-08-17T17:26:12.979 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi179 running (9m) 17s ago 9m 49.0M 2048M 17.2.7-1404-gf2c39d4d f69471c56189 c29492fa5296 2024-08-17T17:26:12.980 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (9m) 37s ago 9m 48.1M 2048M 17.2.7-1404-gf2c39d4d f69471c56189 0c43a998b5fa 2024-08-17T17:26:12.980 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (80s) 37s ago 4m 7600k - 1.7.0 72c9c2088986 fd7392d87597 2024-08-17T17:26:12.980 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi179 *:9100 running (56s) 17s ago 4m 9.96M - 1.7.0 72c9c2088986 c10588f17ed8 2024-08-17T17:26:12.980 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (8m) 37s ago 8m 112M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 293583e22b78 2024-08-17T17:26:12.980 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (8m) 37s ago 8m 112M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 1dc0b4634e15 2024-08-17T17:26:12.980 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (8m) 37s ago 8m 90.3M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 d72289cb8b72 2024-08-17T17:26:12.980 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (7m) 37s ago 7m 102M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 1509c2840516 2024-08-17T17:26:12.980 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi179 running (7m) 17s ago 7m 93.5M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 254dc0763420 2024-08-17T17:26:12.980 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi179 running (7m) 17s ago 7m 91.8M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 078704aadcd7 2024-08-17T17:26:12.980 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi179 running (6m) 17s ago 6m 118M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 ab570ba67ae7 2024-08-17T17:26:12.980 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi179 running (6m) 17s ago 6m 91.9M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 99ae6a861fa0 2024-08-17T17:26:12.980 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi179 *:9095 running (27s) 17s ago 5m 41.6M - 2.51.0 1d3b7f56885b 145f77b6fc93 2024-08-17T17:26:13.084 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:26:12.950+0000 7f8e6b27b640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-08-17T17:26:13.084 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:26:12] ENGINE Bus STARTING 2024-08-17T17:26:13.084 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: CherryPy Checker: 2024-08-17T17:26:13.085 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: The Application mounted at '' has an empty config. 2024-08-17T17:26:13.085 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:26:13.085 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:26:13] ENGINE Serving on http://:::9283 2024-08-17T17:26:13.085 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:26:13] ENGINE Bus STARTED 2024-08-17T17:26:13.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: osdmap e254: 8 total, 8 up, 8 in 2024-08-17T17:26:13.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: mgrmap e36: y(active, since 41s), standbys: x 2024-08-17T17:26:13.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: Active manager daemon y restarted 2024-08-17T17:26:13.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: Activating manager daemon y 2024-08-17T17:26:13.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: osdmap e255: 8 total, 8 up, 8 in 2024-08-17T17:26:13.086 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: mgrmap e37: y(active, starting, since 0.00601573s), standbys: x 2024-08-17T17:26:13.086 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:26:13.086 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:26:13.086 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:26:13.086 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi179.zkkvno"}]: dispatch 2024-08-17T17:26:13.086 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.jmaidg"}]: dispatch 2024-08-17T17:26:13.086 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:26:13.086 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:26:13.086 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:26:13.086 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:26:13.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:26:13.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:26:13.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:26:13.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:26:13.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:26:13.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:26:13.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:26:13.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:26:13.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:26:13.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: Manager daemon y is now available 2024-08-17T17:26:13.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:26:13.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:13.088 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:26:13.088 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: osdmap e254: 8 total, 8 up, 8 in 2024-08-17T17:26:13.088 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: mgrmap e36: y(active, since 41s), standbys: x 2024-08-17T17:26:13.088 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: Active manager daemon y restarted 2024-08-17T17:26:13.088 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: Activating manager daemon y 2024-08-17T17:26:13.088 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: osdmap e255: 8 total, 8 up, 8 in 2024-08-17T17:26:13.088 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: mgrmap e37: y(active, starting, since 0.00601573s), standbys: x 2024-08-17T17:26:13.088 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:26:13.088 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:26:13.088 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:26:13.089 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi179.zkkvno"}]: dispatch 2024-08-17T17:26:13.089 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.jmaidg"}]: dispatch 2024-08-17T17:26:13.089 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:26:13.089 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:26:13.089 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:26:13.089 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:26:13.089 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:26:13.089 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:26:13.089 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:26:13.090 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:26:13.090 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:26:13.090 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:26:13.090 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:26:13.090 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:26:13.090 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:26:13.090 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: Manager daemon y is now available 2024-08-17T17:26:13.090 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:26:13.090 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:13.090 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:26:13.091 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:26:13.091 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:26:13.091 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-17T17:26:13.091 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:26:13.091 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-17T17:26:13.091 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-17T17:26:13.091 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:26:13.091 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:12 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-17T17:26:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: osdmap e254: 8 total, 8 up, 8 in 2024-08-17T17:26:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: mgrmap e36: y(active, since 41s), standbys: x 2024-08-17T17:26:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: Active manager daemon y restarted 2024-08-17T17:26:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: Activating manager daemon y 2024-08-17T17:26:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: osdmap e255: 8 total, 8 up, 8 in 2024-08-17T17:26:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: mgrmap e37: y(active, starting, since 0.00601573s), standbys: x 2024-08-17T17:26:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T17:26:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T17:26:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T17:26:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi179.zkkvno"}]: dispatch 2024-08-17T17:26:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi130.jmaidg"}]: dispatch 2024-08-17T17:26:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T17:26:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T17:26:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T17:26:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T17:26:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T17:26:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T17:26:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T17:26:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T17:26:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T17:26:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T17:26:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T17:26:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T17:26:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T17:26:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: Manager daemon y is now available 2024-08-17T17:26:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:26:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:26:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T17:26:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-17T17:26:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T17:26:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:12 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-17T17:26:13.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:26:13.300+0000 7f8e1ed95640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:17:26:13] ENGINE Error in HTTPServer.serve 2024-08-17T17:26:13.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: Traceback (most recent call last): 2024-08-17T17:26:13.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:26:13.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: self._connections.run(self.expiration_interval) 2024-08-17T17:26:13.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:26:13.418 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: self._run(expiration_interval) 2024-08-17T17:26:13.418 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:26:13.418 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:26:13.418 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:26:13.418 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:26:13.418 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:26:13.418 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: s = self.context.wrap_socket( 2024-08-17T17:26:13.418 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:26:13.418 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: return self.sslsocket_class._create( 2024-08-17T17:26:13.418 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:26:13.418 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: self.do_handshake() 2024-08-17T17:26:13.418 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:26:13.419 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: self._sslobj.do_handshake() 2024-08-17T17:26:13.419 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:26:13.419 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:26:13.988 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 41943040 2024-08-17T17:26:14.005 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:26:14.057 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:26:14.200 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:13 smithi179 ceph-mon[34775]: mgrmap e38: y(active, since 1.0106s), standbys: x 2024-08-17T17:26:14.200 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:13 smithi179 ceph-mon[34775]: osdmap e256: 8 total, 8 up, 8 in 2024-08-17T17:26:14.200 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:13 smithi179 ceph-mon[34775]: [17/Aug/2024:17:26:13] ENGINE Bus STARTING 2024-08-17T17:26:14.201 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:13 smithi179 ceph-mon[34775]: [17/Aug/2024:17:26:13] ENGINE Serving on https://172.21.15.130:7150 2024-08-17T17:26:14.201 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:13 smithi179 ceph-mon[34775]: [17/Aug/2024:17:26:13] ENGINE Error in HTTPServer.serve 2024-08-17T17:26:14.201 INFO:journalctl@ceph.mon.b.smithi179.stdout: Traceback (most recent call last): 2024-08-17T17:26:14.201 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:26:14.201 INFO:journalctl@ceph.mon.b.smithi179.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:26:14.201 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:26:14.201 INFO:journalctl@ceph.mon.b.smithi179.stdout: self._run(expiration_interval) 2024-08-17T17:26:14.201 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:26:14.201 INFO:journalctl@ceph.mon.b.smithi179.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:26:14.201 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:26:14.201 INFO:journalctl@ceph.mon.b.smithi179.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:26:14.201 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:26:14.202 INFO:journalctl@ceph.mon.b.smithi179.stdout: s = self.context.wrap_socket( 2024-08-17T17:26:14.202 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:26:14.202 INFO:journalctl@ceph.mon.b.smithi179.stdout: return self.sslsocket_class._create( 2024-08-17T17:26:14.202 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:26:14.202 INFO:journalctl@ceph.mon.b.smithi179.stdout: self.do_handshake() 2024-08-17T17:26:14.202 INFO:journalctl@ceph.mon.b.smithi179.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:26:14.202 INFO:journalctl@ceph.mon.b.smithi179.stdout: self._sslobj.do_handshake() 2024-08-17T17:26:14.202 INFO:journalctl@ceph.mon.b.smithi179.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:26:14.202 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:13 smithi179 ceph-mon[34775]: [17/Aug/2024:17:26:13] ENGINE Serving on http://172.21.15.130:8765 2024-08-17T17:26:14.202 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:13 smithi179 ceph-mon[34775]: [17/Aug/2024:17:26:13] ENGINE Bus STARTED 2024-08-17T17:26:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-mon[105521]: mgrmap e38: y(active, since 1.0106s), standbys: x 2024-08-17T17:26:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-mon[105521]: osdmap e256: 8 total, 8 up, 8 in 2024-08-17T17:26:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-mon[105521]: [17/Aug/2024:17:26:13] ENGINE Bus STARTING 2024-08-17T17:26:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-mon[105521]: [17/Aug/2024:17:26:13] ENGINE Serving on https://172.21.15.130:7150 2024-08-17T17:26:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-mon[105521]: [17/Aug/2024:17:26:13] ENGINE Error in HTTPServer.serve 2024-08-17T17:26:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout: Traceback (most recent call last): 2024-08-17T17:26:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:26:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:26:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:26:14.418 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._run(expiration_interval) 2024-08-17T17:26:14.418 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:26:14.418 INFO:journalctl@ceph.mon.a.smithi130.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:26:14.418 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:26:14.418 INFO:journalctl@ceph.mon.a.smithi130.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:26:14.418 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:26:14.418 INFO:journalctl@ceph.mon.a.smithi130.stdout: s = self.context.wrap_socket( 2024-08-17T17:26:14.418 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:26:14.418 INFO:journalctl@ceph.mon.a.smithi130.stdout: return self.sslsocket_class._create( 2024-08-17T17:26:14.418 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:26:14.418 INFO:journalctl@ceph.mon.a.smithi130.stdout: self.do_handshake() 2024-08-17T17:26:14.418 INFO:journalctl@ceph.mon.a.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:26:14.418 INFO:journalctl@ceph.mon.a.smithi130.stdout: self._sslobj.do_handshake() 2024-08-17T17:26:14.419 INFO:journalctl@ceph.mon.a.smithi130.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:26:14.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-mon[105521]: [17/Aug/2024:17:26:13] ENGINE Serving on http://172.21.15.130:8765 2024-08-17T17:26:14.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-mon[105521]: [17/Aug/2024:17:26:13] ENGINE Bus STARTED 2024-08-17T17:26:14.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-mon[42749]: mgrmap e38: y(active, since 1.0106s), standbys: x 2024-08-17T17:26:14.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-mon[42749]: osdmap e256: 8 total, 8 up, 8 in 2024-08-17T17:26:14.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-mon[42749]: [17/Aug/2024:17:26:13] ENGINE Bus STARTING 2024-08-17T17:26:14.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-mon[42749]: [17/Aug/2024:17:26:13] ENGINE Serving on https://172.21.15.130:7150 2024-08-17T17:26:14.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-mon[42749]: [17/Aug/2024:17:26:13] ENGINE Error in HTTPServer.serve 2024-08-17T17:26:14.419 INFO:journalctl@ceph.mon.c.smithi130.stdout: Traceback (most recent call last): 2024-08-17T17:26:14.419 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T17:26:14.419 INFO:journalctl@ceph.mon.c.smithi130.stdout: self._connections.run(self.expiration_interval) 2024-08-17T17:26:14.420 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T17:26:14.420 INFO:journalctl@ceph.mon.c.smithi130.stdout: self._run(expiration_interval) 2024-08-17T17:26:14.420 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T17:26:14.420 INFO:journalctl@ceph.mon.c.smithi130.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T17:26:14.420 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T17:26:14.420 INFO:journalctl@ceph.mon.c.smithi130.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T17:26:14.420 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T17:26:14.420 INFO:journalctl@ceph.mon.c.smithi130.stdout: s = self.context.wrap_socket( 2024-08-17T17:26:14.420 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T17:26:14.420 INFO:journalctl@ceph.mon.c.smithi130.stdout: return self.sslsocket_class._create( 2024-08-17T17:26:14.420 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T17:26:14.420 INFO:journalctl@ceph.mon.c.smithi130.stdout: self.do_handshake() 2024-08-17T17:26:14.420 INFO:journalctl@ceph.mon.c.smithi130.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T17:26:14.420 INFO:journalctl@ceph.mon.c.smithi130.stdout: self._sslobj.do_handshake() 2024-08-17T17:26:14.421 INFO:journalctl@ceph.mon.c.smithi130.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T17:26:14.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-mon[42749]: [17/Aug/2024:17:26:13] ENGINE Serving on http://172.21.15.130:8765 2024-08-17T17:26:14.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:13 smithi130 ceph-mon[42749]: [17/Aug/2024:17:26:13] ENGINE Bus STARTED 2024-08-17T17:26:14.482 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade pause' 2024-08-17T17:26:15.063 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:14 smithi130 ceph-mon[105521]: pgmap v5: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 2.8 GiB/s rd, 1.0 GiB/s wr, 45.34k op/s; 1.7 MiB/s, 6 objects/s recovering 2024-08-17T17:26:15.063 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:14 smithi130 ceph-mon[105521]: osdmap e257: 8 total, 8 up, 8 in 2024-08-17T17:26:15.063 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:14 smithi130 ceph-mon[105521]: mgrmap e39: y(active, since 2s), standbys: x 2024-08-17T17:26:15.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:14 smithi130 ceph-mon[42749]: pgmap v5: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 2.8 GiB/s rd, 1.0 GiB/s wr, 45.34k op/s; 1.7 MiB/s, 6 objects/s recovering 2024-08-17T17:26:15.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:14 smithi130 ceph-mon[42749]: osdmap e257: 8 total, 8 up, 8 in 2024-08-17T17:26:15.064 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:14 smithi130 ceph-mon[42749]: mgrmap e39: y(active, since 2s), standbys: x 2024-08-17T17:26:15.088 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.SnapTrimBeforePrepare (7188 ms) 2024-08-17T17:26:15.088 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.AbortInUseImage 2024-08-17T17:26:15.119 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:26:15.129 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:26:15.263 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:15.302 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.AbortInUseImage (214 ms) 2024-08-17T17:26:15.302 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.AbortWithoutSnapshots 2024-08-17T17:26:15.341 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:26:15.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:14 smithi179 ceph-mon[34775]: pgmap v5: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 2.8 GiB/s rd, 1.0 GiB/s wr, 45.34k op/s; 1.7 MiB/s, 6 objects/s recovering 2024-08-17T17:26:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:14 smithi179 ceph-mon[34775]: osdmap e257: 8 total, 8 up, 8 in 2024-08-17T17:26:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:14 smithi179 ceph-mon[34775]: mgrmap e39: y(active, since 2s), standbys: x 2024-08-17T17:26:15.351 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:26:15.417 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 16777218~8388609 2024-08-17T17:26:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:26:15.656 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:26:15.708 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 16777218~8388609 2024-08-17T17:26:15.763 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:26:15.952 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.AbortWithoutSnapshots (650 ms) 2024-08-17T17:26:15.952 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.AbortWithSnapshots 2024-08-17T17:26:15.982 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:26:15.995 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:26:16.060 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:26:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:15 smithi179 ceph-mon[34775]: osdmap e258: 8 total, 8 up, 8 in 2024-08-17T17:26:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:15 smithi179 ceph-mon[34775]: osdmap e259: 8 total, 8 up, 8 in 2024-08-17T17:26:16.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:15 smithi130 ceph-mon[105521]: osdmap e258: 8 total, 8 up, 8 in 2024-08-17T17:26:16.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:15 smithi130 ceph-mon[105521]: osdmap e259: 8 total, 8 up, 8 in 2024-08-17T17:26:16.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:15 smithi130 ceph-mon[42749]: osdmap e258: 8 total, 8 up, 8 in 2024-08-17T17:26:16.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:15 smithi130 ceph-mon[42749]: osdmap e259: 8 total, 8 up, 8 in 2024-08-17T17:26:17.317 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:17.350 INFO:teuthology.orchestra.run.smithi130.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:26:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:17 smithi179 ceph-mon[34775]: pgmap v9: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 937 MiB/s rd, 340 MiB/s wr, 15.04k op/s; 578 KiB/s, 2 objects/s recovering 2024-08-17T17:26:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:17 smithi179 ceph-mon[34775]: mgrmap e40: y(active, since 4s), standbys: x 2024-08-17T17:26:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:17 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:17 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:17 smithi179 ceph-mon[34775]: osdmap e260: 8 total, 8 up, 8 in 2024-08-17T17:26:17.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:17 smithi130 ceph-mon[105521]: pgmap v9: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 937 MiB/s rd, 340 MiB/s wr, 15.04k op/s; 578 KiB/s, 2 objects/s recovering 2024-08-17T17:26:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:17 smithi130 ceph-mon[105521]: mgrmap e40: y(active, since 4s), standbys: x 2024-08-17T17:26:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:17 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:17 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:17 smithi130 ceph-mon[105521]: osdmap e260: 8 total, 8 up, 8 in 2024-08-17T17:26:17.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:17 smithi130 ceph-mon[42749]: pgmap v9: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 937 MiB/s rd, 340 MiB/s wr, 15.04k op/s; 578 KiB/s, 2 objects/s recovering 2024-08-17T17:26:17.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:17 smithi130 ceph-mon[42749]: mgrmap e40: y(active, since 4s), standbys: x 2024-08-17T17:26:17.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:17 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:17.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:17 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:17.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:17 smithi130 ceph-mon[42749]: osdmap e260: 8 total, 8 up, 8 in 2024-08-17T17:26:17.991 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 8388609~8388609 2024-08-17T17:26:18.004 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 0~1000 2024-08-17T17:26:18.006 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 1010~1000 2024-08-17T17:26:18.006 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:26:18.157 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'sleep 60' 2024-08-17T17:26:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:18 smithi130 ceph-mon[105521]: from='client.26030 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:18 smithi130 ceph-mon[105521]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:26:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:18 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:18 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:18 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:18 smithi130 ceph-mon[105521]: osdmap e261: 8 total, 8 up, 8 in 2024-08-17T17:26:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:18 smithi130 ceph-mon[42749]: from='client.26030 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:18 smithi130 ceph-mon[42749]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:26:18.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:18 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:18.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:18 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:18.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:18 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:18.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:18 smithi130 ceph-mon[42749]: osdmap e261: 8 total, 8 up, 8 in 2024-08-17T17:26:18.684 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:18 smithi179 ceph-mon[34775]: from='client.26030 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:26:18.684 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:18 smithi179 ceph-mon[34775]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:26:18.684 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:18 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:18.684 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:18 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:18.684 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:18 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:18.684 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:18 smithi179 ceph-mon[34775]: osdmap e261: 8 total, 8 up, 8 in 2024-08-17T17:26:19.582 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:19 smithi179 ceph-mon[34775]: pgmap v11: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:26:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:19 smithi179 ceph-mon[34775]: osdmap e262: 8 total, 8 up, 8 in 2024-08-17T17:26:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:19 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:19 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:19.858 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:19 smithi130 ceph-mon[105521]: pgmap v11: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:26:19.858 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:19 smithi130 ceph-mon[105521]: osdmap e262: 8 total, 8 up, 8 in 2024-08-17T17:26:19.858 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:19 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:19.858 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:19 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:19.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:26:19] "GET /metrics HTTP/1.1" 200 35930 "" "Prometheus/2.51.0" 2024-08-17T17:26:19.859 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:19 smithi130 ceph-mon[42749]: pgmap v11: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:26:19.859 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:19 smithi130 ceph-mon[42749]: osdmap e262: 8 total, 8 up, 8 in 2024-08-17T17:26:19.859 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:19 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:19.859 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:19 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:19.993 INFO:tasks.workunit.client.0.smithi130.stdout:write: X 8388609~1000 2024-08-17T17:26:19.999 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 16777228~1000 2024-08-17T17:26:19.999 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 167772160 2024-08-17T17:26:20.009 INFO:tasks.workunit.client.0.smithi130.stdout:write: 2 83886079~8388609 2024-08-17T17:26:20.033 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:26:20.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:21.071 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[42749]: pgmap v14: 137 pgs: 137 active+clean; 829 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 22 MiB/s wr, 545 op/s 2024-08-17T17:26:21.071 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[42749]: osdmap e263: 8 total, 8 up, 8 in 2024-08-17T17:26:21.071 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:21.071 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:21.071 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:21.071 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:21.072 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:26:21.072 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:26:21.072 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[42749]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:26:21.072 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[42749]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:26:21.072 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[105521]: pgmap v14: 137 pgs: 137 active+clean; 829 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 22 MiB/s wr, 545 op/s 2024-08-17T17:26:21.072 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[105521]: osdmap e263: 8 total, 8 up, 8 in 2024-08-17T17:26:21.072 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:21.072 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:21.072 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:21.072 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:21.072 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:26:21.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:26:21.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[105521]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:26:21.073 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:20 smithi130 ceph-mon[105521]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:26:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:20 smithi179 ceph-mon[34775]: pgmap v14: 137 pgs: 137 active+clean; 829 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s rd, 22 MiB/s wr, 545 op/s 2024-08-17T17:26:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:20 smithi179 ceph-mon[34775]: osdmap e263: 8 total, 8 up, 8 in 2024-08-17T17:26:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:20 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:20 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:20 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:20 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:20 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:26:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:20 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:26:21.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:20 smithi179 ceph-mon[34775]: Updating smithi130:/etc/ceph/ceph.conf 2024-08-17T17:26:21.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:20 smithi179 ceph-mon[34775]: Updating smithi179:/etc/ceph/ceph.conf 2024-08-17T17:26:22.002 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:26:22.023 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 83886079~1 2024-08-17T17:26:22.031 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:26:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:21 smithi179 ceph-mon[34775]: osdmap e264: 8 total, 8 up, 8 in 2024-08-17T17:26:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:21 smithi179 ceph-mon[34775]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:26:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:21 smithi179 ceph-mon[34775]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:26:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:21 smithi179 ceph-mon[34775]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:26:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:21 smithi179 ceph-mon[34775]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:26:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:21 smithi179 ceph-mon[34775]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:26:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:21 smithi179 ceph-mon[34775]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:26:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:21 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:21 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:21 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:21 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:21 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:21 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:26:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[105521]: osdmap e264: 8 total, 8 up, 8 in 2024-08-17T17:26:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[105521]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:26:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[105521]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:26:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[105521]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:26:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[105521]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:26:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[105521]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:26:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[105521]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:26:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:26:22.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[42749]: osdmap e264: 8 total, 8 up, 8 in 2024-08-17T17:26:22.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[42749]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:26:22.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[42749]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T17:26:22.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[42749]: Updating smithi179:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:26:22.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[42749]: Updating smithi130:/etc/ceph/ceph.client.admin.keyring 2024-08-17T17:26:22.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[42749]: Updating smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:26:22.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[42749]: Updating smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T17:26:22.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:22.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:21 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:26:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:22 smithi179 ceph-mon[34775]: pgmap v17: 137 pgs: 137 active+clean; 829 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 25 MiB/s wr, 634 op/s 2024-08-17T17:26:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:22 smithi179 ceph-mon[34775]: osdmap e265: 8 total, 8 up, 8 in 2024-08-17T17:26:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:22 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:22 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:22 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:26:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:22 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:26:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:22 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:23.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:22 smithi130 ceph-mon[105521]: pgmap v17: 137 pgs: 137 active+clean; 829 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 25 MiB/s wr, 634 op/s 2024-08-17T17:26:23.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:22 smithi130 ceph-mon[105521]: osdmap e265: 8 total, 8 up, 8 in 2024-08-17T17:26:23.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:22 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:23.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:22 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:23.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:22 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:26:23.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:22 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:26:23.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:22 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:23.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:22 smithi130 ceph-mon[42749]: pgmap v17: 137 pgs: 137 active+clean; 829 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 25 MiB/s wr, 634 op/s 2024-08-17T17:26:23.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:23 smithi130 ceph-mon[42749]: osdmap e265: 8 total, 8 up, 8 in 2024-08-17T17:26:23.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:23 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:23.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:23 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:23.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:23 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:26:23.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:23 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:26:23.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:23 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:26:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:24 smithi130 ceph-mon[105521]: osdmap e266: 8 total, 8 up, 8 in 2024-08-17T17:26:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:24 smithi130 ceph-mon[42749]: osdmap e266: 8 total, 8 up, 8 in 2024-08-17T17:26:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:24 smithi179 ceph-mon[34775]: osdmap e266: 8 total, 8 up, 8 in 2024-08-17T17:26:24.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:24 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:25.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:25 smithi130 ceph-mon[105521]: pgmap v20: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-08-17T17:26:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:25 smithi130 ceph-mon[105521]: osdmap e267: 8 total, 8 up, 8 in 2024-08-17T17:26:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:25 smithi130 ceph-mon[42749]: pgmap v20: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-08-17T17:26:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:25 smithi130 ceph-mon[42749]: osdmap e267: 8 total, 8 up, 8 in 2024-08-17T17:26:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:25 smithi179 ceph-mon[34775]: pgmap v20: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-08-17T17:26:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:25 smithi179 ceph-mon[34775]: osdmap e267: 8 total, 8 up, 8 in 2024-08-17T17:26:25.595 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:26:25.708 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 16777218~8388609 2024-08-17T17:26:26.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:25 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:26.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:25 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:26.327 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 0~8388609 2024-08-17T17:26:26.346 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 16777218~8388609 2024-08-17T17:26:26.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:26 smithi130 ceph-mon[105521]: osdmap e268: 8 total, 8 up, 8 in 2024-08-17T17:26:26.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:26 smithi130 ceph-mon[105521]: osdmap e269: 8 total, 8 up, 8 in 2024-08-17T17:26:26.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:26 smithi130 ceph-mon[42749]: osdmap e268: 8 total, 8 up, 8 in 2024-08-17T17:26:26.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:26 smithi130 ceph-mon[42749]: osdmap e269: 8 total, 8 up, 8 in 2024-08-17T17:26:26.433 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:26:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:26 smithi179 ceph-mon[34775]: osdmap e268: 8 total, 8 up, 8 in 2024-08-17T17:26:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:26 smithi179 ceph-mon[34775]: osdmap e269: 8 total, 8 up, 8 in 2024-08-17T17:26:27.317 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:27 smithi179 ceph-mon[34775]: pgmap v24: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 141 KiB/s rd, 8.1 MiB/s wr, 203 op/s 2024-08-17T17:26:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:27 smithi179 ceph-mon[34775]: osdmap e270: 8 total, 8 up, 8 in 2024-08-17T17:26:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:27 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:27 smithi130 ceph-mon[105521]: pgmap v24: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 141 KiB/s rd, 8.1 MiB/s wr, 203 op/s 2024-08-17T17:26:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:27 smithi130 ceph-mon[105521]: osdmap e270: 8 total, 8 up, 8 in 2024-08-17T17:26:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:27 smithi130 ceph-mon[42749]: pgmap v24: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 141 KiB/s rd, 8.1 MiB/s wr, 203 op/s 2024-08-17T17:26:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:27 smithi130 ceph-mon[42749]: osdmap e270: 8 total, 8 up, 8 in 2024-08-17T17:26:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:27 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:28 smithi179 ceph-mon[34775]: osdmap e271: 8 total, 8 up, 8 in 2024-08-17T17:26:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:28 smithi130 ceph-mon[105521]: osdmap e271: 8 total, 8 up, 8 in 2024-08-17T17:26:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:28 smithi130 ceph-mon[42749]: osdmap e271: 8 total, 8 up, 8 in 2024-08-17T17:26:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:29 smithi179 ceph-mon[34775]: pgmap v27: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:26:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:29 smithi179 ceph-mon[34775]: osdmap e272: 8 total, 8 up, 8 in 2024-08-17T17:26:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:29 smithi130 ceph-mon[105521]: pgmap v27: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:26:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:29 smithi130 ceph-mon[105521]: osdmap e272: 8 total, 8 up, 8 in 2024-08-17T17:26:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:29 smithi130 ceph-mon[42749]: pgmap v27: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-08-17T17:26:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:29 smithi130 ceph-mon[42749]: osdmap e272: 8 total, 8 up, 8 in 2024-08-17T17:26:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:26:29] "GET /metrics HTTP/1.1" 200 38769 "" "Prometheus/2.51.0" 2024-08-17T17:26:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:30 smithi179 ceph-mon[34775]: osdmap e273: 8 total, 8 up, 8 in 2024-08-17T17:26:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:30 smithi130 ceph-mon[105521]: osdmap e273: 8 total, 8 up, 8 in 2024-08-17T17:26:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:30 smithi130 ceph-mon[42749]: osdmap e273: 8 total, 8 up, 8 in 2024-08-17T17:26:30.710 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.AbortWithSnapshots (14759 ms) 2024-08-17T17:26:30.711 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.CloneV1Parent 2024-08-17T17:26:30.740 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:26:30.749 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 0~10 2024-08-17T17:26:30.757 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:26:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:31 smithi130 ceph-mon[105521]: pgmap v30: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 377 op/s 2024-08-17T17:26:31.696 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:31 smithi130 ceph-mon[105521]: osdmap e274: 8 total, 8 up, 8 in 2024-08-17T17:26:31.696 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:31 smithi130 ceph-mon[42749]: pgmap v30: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 377 op/s 2024-08-17T17:26:31.697 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:31 smithi130 ceph-mon[42749]: osdmap e274: 8 total, 8 up, 8 in 2024-08-17T17:26:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:31 smithi179 ceph-mon[34775]: pgmap v30: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 377 op/s 2024-08-17T17:26:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:31 smithi179 ceph-mon[34775]: osdmap e274: 8 total, 8 up, 8 in 2024-08-17T17:26:32.371 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap1 2024-08-17T17:26:32.432 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:26:32.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:32 smithi130 ceph-mon[105521]: osdmap e275: 8 total, 8 up, 8 in 2024-08-17T17:26:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:32 smithi130 ceph-mon[42749]: osdmap e275: 8 total, 8 up, 8 in 2024-08-17T17:26:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:32 smithi179 ceph-mon[34775]: osdmap e275: 8 total, 8 up, 8 in 2024-08-17T17:26:33.578 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:26:33.658 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:26:33.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:33 smithi130 ceph-mon[105521]: pgmap v33: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 377 op/s 2024-08-17T17:26:33.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:33 smithi130 ceph-mon[105521]: osdmap e276: 8 total, 8 up, 8 in 2024-08-17T17:26:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:33 smithi130 ceph-mon[42749]: pgmap v33: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 377 op/s 2024-08-17T17:26:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:33 smithi130 ceph-mon[42749]: osdmap e276: 8 total, 8 up, 8 in 2024-08-17T17:26:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:33 smithi179 ceph-mon[34775]: pgmap v33: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 377 op/s 2024-08-17T17:26:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:33 smithi179 ceph-mon[34775]: osdmap e276: 8 total, 8 up, 8 in 2024-08-17T17:26:34.481 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.CloneV1Parent (3770 ms) 2024-08-17T17:26:34.481 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.CloneV2Parent 2024-08-17T17:26:34.516 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:26:34.527 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 0~10 2024-08-17T17:26:34.535 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:26:34.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:34 smithi179 ceph-mon[34775]: osdmap e277: 8 total, 8 up, 8 in 2024-08-17T17:26:34.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:34 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:34.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:34 smithi130 ceph-mon[105521]: osdmap e277: 8 total, 8 up, 8 in 2024-08-17T17:26:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:34 smithi130 ceph-mon[42749]: osdmap e277: 8 total, 8 up, 8 in 2024-08-17T17:26:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:35 smithi130 ceph-mon[105521]: pgmap v36: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 15 KiB/s wr, 224 op/s 2024-08-17T17:26:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:35 smithi130 ceph-mon[105521]: osdmap e278: 8 total, 8 up, 8 in 2024-08-17T17:26:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:35 smithi130 ceph-mon[105521]: osdmap e279: 8 total, 8 up, 8 in 2024-08-17T17:26:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:35 smithi130 ceph-mon[42749]: pgmap v36: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 15 KiB/s wr, 224 op/s 2024-08-17T17:26:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:35 smithi130 ceph-mon[42749]: osdmap e278: 8 total, 8 up, 8 in 2024-08-17T17:26:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:35 smithi130 ceph-mon[42749]: osdmap e279: 8 total, 8 up, 8 in 2024-08-17T17:26:35.740 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:35 smithi179 ceph-mon[34775]: pgmap v36: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 15 KiB/s wr, 224 op/s 2024-08-17T17:26:35.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:35 smithi179 ceph-mon[34775]: osdmap e278: 8 total, 8 up, 8 in 2024-08-17T17:26:35.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:35 smithi179 ceph-mon[34775]: osdmap e279: 8 total, 8 up, 8 in 2024-08-17T17:26:36.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:36.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:36.332 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap1 2024-08-17T17:26:36.388 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:26:37.317 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:36 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:37.574 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:26:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:37 smithi179 ceph-mon[34775]: pgmap v39: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 15 KiB/s wr, 224 op/s 2024-08-17T17:26:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:37 smithi179 ceph-mon[34775]: osdmap e280: 8 total, 8 up, 8 in 2024-08-17T17:26:37.648 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:26:37.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:37 smithi130 ceph-mon[105521]: pgmap v39: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 15 KiB/s wr, 224 op/s 2024-08-17T17:26:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:37 smithi130 ceph-mon[105521]: osdmap e280: 8 total, 8 up, 8 in 2024-08-17T17:26:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:37 smithi130 ceph-mon[42749]: pgmap v39: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 15 KiB/s wr, 224 op/s 2024-08-17T17:26:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:37 smithi130 ceph-mon[42749]: osdmap e280: 8 total, 8 up, 8 in 2024-08-17T17:26:38.496 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.CloneV2Parent (4015 ms) 2024-08-17T17:26:38.496 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.CloneV1ParentAbort 2024-08-17T17:26:38.531 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:26:38.541 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 0~10 2024-08-17T17:26:38.549 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:26:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:38 smithi130 ceph-mon[105521]: osdmap e281: 8 total, 8 up, 8 in 2024-08-17T17:26:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:38 smithi130 ceph-mon[42749]: osdmap e281: 8 total, 8 up, 8 in 2024-08-17T17:26:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:38 smithi179 ceph-mon[34775]: osdmap e281: 8 total, 8 up, 8 in 2024-08-17T17:26:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:39 smithi130 ceph-mon[105521]: pgmap v42: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:26:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:39 smithi130 ceph-mon[105521]: osdmap e282: 8 total, 8 up, 8 in 2024-08-17T17:26:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:39 smithi130 ceph-mon[42749]: pgmap v42: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:26:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:39 smithi130 ceph-mon[42749]: osdmap e282: 8 total, 8 up, 8 in 2024-08-17T17:26:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:39 smithi179 ceph-mon[34775]: pgmap v42: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:26:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:39 smithi179 ceph-mon[34775]: osdmap e282: 8 total, 8 up, 8 in 2024-08-17T17:26:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:26:39] "GET /metrics HTTP/1.1" 200 38769 "" "Prometheus/2.51.0" 2024-08-17T17:26:40.404 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap1 2024-08-17T17:26:40.475 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:26:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:40 smithi179 ceph-mon[34775]: osdmap e283: 8 total, 8 up, 8 in 2024-08-17T17:26:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:40 smithi130 ceph-mon[42749]: osdmap e283: 8 total, 8 up, 8 in 2024-08-17T17:26:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:40 smithi130 ceph-mon[105521]: osdmap e283: 8 total, 8 up, 8 in 2024-08-17T17:26:41.580 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:26:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:41 smithi130 ceph-mon[42749]: pgmap v45: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 538 KiB/s rd, 29 KiB/s wr, 669 op/s 2024-08-17T17:26:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:41 smithi130 ceph-mon[42749]: osdmap e284: 8 total, 8 up, 8 in 2024-08-17T17:26:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:41 smithi130 ceph-mon[105521]: pgmap v45: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 538 KiB/s rd, 29 KiB/s wr, 669 op/s 2024-08-17T17:26:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:41 smithi130 ceph-mon[105521]: osdmap e284: 8 total, 8 up, 8 in 2024-08-17T17:26:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:41 smithi179 ceph-mon[34775]: pgmap v45: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 538 KiB/s rd, 29 KiB/s wr, 669 op/s 2024-08-17T17:26:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:41 smithi179 ceph-mon[34775]: osdmap e284: 8 total, 8 up, 8 in 2024-08-17T17:26:42.534 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.CloneV1ParentAbort (4038 ms) 2024-08-17T17:26:42.534 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.CloneV2ParentAbort 2024-08-17T17:26:42.563 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:26:42.572 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 0~10 2024-08-17T17:26:42.580 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:26:42.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:42 smithi130 ceph-mon[42749]: osdmap e285: 8 total, 8 up, 8 in 2024-08-17T17:26:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:42 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:42 smithi130 ceph-mon[105521]: osdmap e285: 8 total, 8 up, 8 in 2024-08-17T17:26:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:42.761 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:42 smithi179 ceph-mon[34775]: osdmap e285: 8 total, 8 up, 8 in 2024-08-17T17:26:42.761 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:42 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:43 smithi130 ceph-mon[105521]: pgmap v48: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 538 KiB/s rd, 29 KiB/s wr, 669 op/s 2024-08-17T17:26:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:43 smithi130 ceph-mon[105521]: osdmap e286: 8 total, 8 up, 8 in 2024-08-17T17:26:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:43 smithi130 ceph-mon[42749]: pgmap v48: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 538 KiB/s rd, 29 KiB/s wr, 669 op/s 2024-08-17T17:26:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:43 smithi130 ceph-mon[42749]: osdmap e286: 8 total, 8 up, 8 in 2024-08-17T17:26:43.691 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:43 smithi179 ceph-mon[34775]: pgmap v48: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 538 KiB/s rd, 29 KiB/s wr, 669 op/s 2024-08-17T17:26:43.692 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:43 smithi179 ceph-mon[34775]: osdmap e286: 8 total, 8 up, 8 in 2024-08-17T17:26:44.412 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:44.428 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap1 2024-08-17T17:26:44.485 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:26:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:44 smithi130 ceph-mon[105521]: osdmap e287: 8 total, 8 up, 8 in 2024-08-17T17:26:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:44 smithi130 ceph-mon[42749]: osdmap e287: 8 total, 8 up, 8 in 2024-08-17T17:26:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:44 smithi179 ceph-mon[34775]: osdmap e287: 8 total, 8 up, 8 in 2024-08-17T17:26:45.490 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:26:45.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:45 smithi179 ceph-mon[34775]: pgmap v51: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 308 KiB/s rd, 22 KiB/s wr, 403 op/s 2024-08-17T17:26:45.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:45 smithi179 ceph-mon[34775]: osdmap e288: 8 total, 8 up, 8 in 2024-08-17T17:26:45.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:45 smithi179 ceph-mon[34775]: osdmap e289: 8 total, 8 up, 8 in 2024-08-17T17:26:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:45 smithi130 ceph-mon[105521]: pgmap v51: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 308 KiB/s rd, 22 KiB/s wr, 403 op/s 2024-08-17T17:26:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:45 smithi130 ceph-mon[105521]: osdmap e288: 8 total, 8 up, 8 in 2024-08-17T17:26:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:45 smithi130 ceph-mon[105521]: osdmap e289: 8 total, 8 up, 8 in 2024-08-17T17:26:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:45 smithi130 ceph-mon[42749]: pgmap v51: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 308 KiB/s rd, 22 KiB/s wr, 403 op/s 2024-08-17T17:26:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:45 smithi130 ceph-mon[42749]: osdmap e288: 8 total, 8 up, 8 in 2024-08-17T17:26:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:45 smithi130 ceph-mon[42749]: osdmap e289: 8 total, 8 up, 8 in 2024-08-17T17:26:46.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:45 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:46.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:45 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:46.550 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.CloneV2ParentAbort (4016 ms) 2024-08-17T17:26:46.551 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach 2024-08-17T17:26:46.581 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:26:46.590 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 0~10 2024-08-17T17:26:46.600 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:26:47.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:47 smithi179 ceph-mon[34775]: pgmap v54: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 308 KiB/s rd, 22 KiB/s wr, 403 op/s 2024-08-17T17:26:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:47 smithi179 ceph-mon[34775]: osdmap e290: 8 total, 8 up, 8 in 2024-08-17T17:26:47.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:47 smithi130 ceph-mon[105521]: pgmap v54: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 308 KiB/s rd, 22 KiB/s wr, 403 op/s 2024-08-17T17:26:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:47 smithi130 ceph-mon[105521]: osdmap e290: 8 total, 8 up, 8 in 2024-08-17T17:26:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:47 smithi130 ceph-mon[42749]: pgmap v54: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 308 KiB/s rd, 22 KiB/s wr, 403 op/s 2024-08-17T17:26:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:47 smithi130 ceph-mon[42749]: osdmap e290: 8 total, 8 up, 8 in 2024-08-17T17:26:48.457 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap1 2024-08-17T17:26:48.519 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:26:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:48 smithi179 ceph-mon[34775]: osdmap e291: 8 total, 8 up, 8 in 2024-08-17T17:26:48.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:48 smithi130 ceph-mon[105521]: osdmap e291: 8 total, 8 up, 8 in 2024-08-17T17:26:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:48 smithi130 ceph-mon[42749]: osdmap e291: 8 total, 8 up, 8 in 2024-08-17T17:26:49.640 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:26:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:49 smithi130 ceph-mon[105521]: pgmap v57: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 42 KiB/s rd, 3.0 KiB/s wr, 54 op/s 2024-08-17T17:26:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:49 smithi130 ceph-mon[105521]: osdmap e292: 8 total, 8 up, 8 in 2024-08-17T17:26:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:49 smithi130 ceph-mon[42749]: pgmap v57: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 42 KiB/s rd, 3.0 KiB/s wr, 54 op/s 2024-08-17T17:26:49.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:49 smithi130 ceph-mon[42749]: osdmap e292: 8 total, 8 up, 8 in 2024-08-17T17:26:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:49 smithi179 ceph-mon[34775]: pgmap v57: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 42 KiB/s rd, 3.0 KiB/s wr, 54 op/s 2024-08-17T17:26:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:49 smithi179 ceph-mon[34775]: osdmap e292: 8 total, 8 up, 8 in 2024-08-17T17:26:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:26:49] "GET /metrics HTTP/1.1" 200 38769 "" "Prometheus/2.51.0" 2024-08-17T17:26:50.583 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach (4033 ms) 2024-08-17T17:26:50.583 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixParentReattach 2024-08-17T17:26:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:50 smithi179 ceph-mon[34775]: osdmap e293: 8 total, 8 up, 8 in 2024-08-17T17:26:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:50.613 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:26:50.623 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 0~10 2024-08-17T17:26:50.631 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:26:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:50 smithi130 ceph-mon[105521]: osdmap e293: 8 total, 8 up, 8 in 2024-08-17T17:26:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:50 smithi130 ceph-mon[42749]: osdmap e293: 8 total, 8 up, 8 in 2024-08-17T17:26:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:51 smithi179 ceph-mon[34775]: pgmap v60: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 239 KiB/s rd, 21 KiB/s wr, 313 op/s 2024-08-17T17:26:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:51 smithi179 ceph-mon[34775]: osdmap e294: 8 total, 8 up, 8 in 2024-08-17T17:26:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:51 smithi130 ceph-mon[105521]: pgmap v60: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 239 KiB/s rd, 21 KiB/s wr, 313 op/s 2024-08-17T17:26:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:51 smithi130 ceph-mon[105521]: osdmap e294: 8 total, 8 up, 8 in 2024-08-17T17:26:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:51 smithi130 ceph-mon[42749]: pgmap v60: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 239 KiB/s rd, 21 KiB/s wr, 313 op/s 2024-08-17T17:26:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:51 smithi130 ceph-mon[42749]: osdmap e294: 8 total, 8 up, 8 in 2024-08-17T17:26:52.485 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap1 2024-08-17T17:26:52.547 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:26:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:52 smithi179 ceph-mon[34775]: osdmap e295: 8 total, 8 up, 8 in 2024-08-17T17:26:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:52 smithi130 ceph-mon[105521]: osdmap e295: 8 total, 8 up, 8 in 2024-08-17T17:26:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:52 smithi130 ceph-mon[42749]: osdmap e295: 8 total, 8 up, 8 in 2024-08-17T17:26:53.672 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:26:53.760 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:53 smithi179 ceph-mon[34775]: pgmap v63: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 197 KiB/s rd, 18 KiB/s wr, 259 op/s 2024-08-17T17:26:53.760 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:53 smithi179 ceph-mon[34775]: osdmap e296: 8 total, 8 up, 8 in 2024-08-17T17:26:53.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:53 smithi130 ceph-mon[105521]: pgmap v63: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 197 KiB/s rd, 18 KiB/s wr, 259 op/s 2024-08-17T17:26:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:53 smithi130 ceph-mon[105521]: osdmap e296: 8 total, 8 up, 8 in 2024-08-17T17:26:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:53 smithi130 ceph-mon[42749]: pgmap v63: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 197 KiB/s rd, 18 KiB/s wr, 259 op/s 2024-08-17T17:26:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:53 smithi130 ceph-mon[42749]: osdmap e296: 8 total, 8 up, 8 in 2024-08-17T17:26:54.487 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:54.603 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixParentReattach (4020 ms) 2024-08-17T17:26:54.603 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.CloneV1ParentAbortRelinkNotNeeded 2024-08-17T17:26:54.631 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:26:54.642 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 0~10 2024-08-17T17:26:54.650 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:26:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:54 smithi179 ceph-mon[34775]: osdmap e297: 8 total, 8 up, 8 in 2024-08-17T17:26:54.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:54 smithi130 ceph-mon[105521]: osdmap e297: 8 total, 8 up, 8 in 2024-08-17T17:26:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:54 smithi130 ceph-mon[42749]: osdmap e297: 8 total, 8 up, 8 in 2024-08-17T17:26:55.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:55 smithi179 ceph-mon[34775]: pgmap v66: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 364 KiB/s rd, 21 KiB/s wr, 464 op/s 2024-08-17T17:26:55.742 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:55 smithi179 ceph-mon[34775]: osdmap e298: 8 total, 8 up, 8 in 2024-08-17T17:26:55.742 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:55 smithi179 ceph-mon[34775]: osdmap e299: 8 total, 8 up, 8 in 2024-08-17T17:26:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:55 smithi130 ceph-mon[105521]: pgmap v66: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 364 KiB/s rd, 21 KiB/s wr, 464 op/s 2024-08-17T17:26:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:55 smithi130 ceph-mon[105521]: osdmap e298: 8 total, 8 up, 8 in 2024-08-17T17:26:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:55 smithi130 ceph-mon[105521]: osdmap e299: 8 total, 8 up, 8 in 2024-08-17T17:26:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:55 smithi130 ceph-mon[42749]: pgmap v66: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 364 KiB/s rd, 21 KiB/s wr, 464 op/s 2024-08-17T17:26:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:55 smithi130 ceph-mon[42749]: osdmap e298: 8 total, 8 up, 8 in 2024-08-17T17:26:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:55 smithi130 ceph-mon[42749]: osdmap e299: 8 total, 8 up, 8 in 2024-08-17T17:26:56.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:55 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:56.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:55 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:56.338 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap1 2024-08-17T17:26:56.394 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:26:57.327 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:26:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:26:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:26:57.511 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:26:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:57 smithi179 ceph-mon[34775]: pgmap v69: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 364 KiB/s rd, 21 KiB/s wr, 464 op/s 2024-08-17T17:26:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:57 smithi179 ceph-mon[34775]: osdmap e300: 8 total, 8 up, 8 in 2024-08-17T17:26:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:57 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:57 smithi130 ceph-mon[105521]: pgmap v69: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 364 KiB/s rd, 21 KiB/s wr, 464 op/s 2024-08-17T17:26:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:57 smithi130 ceph-mon[105521]: osdmap e300: 8 total, 8 up, 8 in 2024-08-17T17:26:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:57.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:57 smithi130 ceph-mon[42749]: pgmap v69: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 364 KiB/s rd, 21 KiB/s wr, 464 op/s 2024-08-17T17:26:57.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:57 smithi130 ceph-mon[42749]: osdmap e300: 8 total, 8 up, 8 in 2024-08-17T17:26:57.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:57 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:26:58.467 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.CloneV1ParentAbortRelinkNotNeeded (3864 ms) 2024-08-17T17:26:58.467 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach 2024-08-17T17:26:58.496 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:26:58.507 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 0~10 2024-08-17T17:26:58.515 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:26:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:58 smithi179 ceph-mon[34775]: osdmap e301: 8 total, 8 up, 8 in 2024-08-17T17:26:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:58 smithi130 ceph-mon[105521]: osdmap e301: 8 total, 8 up, 8 in 2024-08-17T17:26:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:58 smithi130 ceph-mon[42749]: osdmap e301: 8 total, 8 up, 8 in 2024-08-17T17:26:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:59 smithi179 ceph-mon[34775]: pgmap v72: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 336 KiB/s rd, 15 KiB/s wr, 412 op/s 2024-08-17T17:26:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:26:59 smithi179 ceph-mon[34775]: osdmap e302: 8 total, 8 up, 8 in 2024-08-17T17:26:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:59 smithi130 ceph-mon[42749]: pgmap v72: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 336 KiB/s rd, 15 KiB/s wr, 412 op/s 2024-08-17T17:26:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:26:59 smithi130 ceph-mon[42749]: osdmap e302: 8 total, 8 up, 8 in 2024-08-17T17:26:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:59 smithi130 ceph-mon[105521]: pgmap v72: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 336 KiB/s rd, 15 KiB/s wr, 412 op/s 2024-08-17T17:26:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:26:59 smithi130 ceph-mon[105521]: osdmap e302: 8 total, 8 up, 8 in 2024-08-17T17:27:00.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:26:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:26:59] "GET /metrics HTTP/1.1" 200 38770 "" "Prometheus/2.51.0" 2024-08-17T17:27:00.368 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap1 2024-08-17T17:27:00.419 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:27:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:00 smithi179 ceph-mon[34775]: osdmap e303: 8 total, 8 up, 8 in 2024-08-17T17:27:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:00.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:00 smithi130 ceph-mon[105521]: osdmap e303: 8 total, 8 up, 8 in 2024-08-17T17:27:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:00 smithi130 ceph-mon[42749]: osdmap e303: 8 total, 8 up, 8 in 2024-08-17T17:27:01.537 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:27:01.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:01 smithi130 ceph-mon[105521]: pgmap v75: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 398 KiB/s rd, 21 KiB/s wr, 501 op/s 2024-08-17T17:27:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:01 smithi130 ceph-mon[105521]: osdmap e304: 8 total, 8 up, 8 in 2024-08-17T17:27:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:01 smithi130 ceph-mon[42749]: pgmap v75: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 398 KiB/s rd, 21 KiB/s wr, 501 op/s 2024-08-17T17:27:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:01 smithi130 ceph-mon[42749]: osdmap e304: 8 total, 8 up, 8 in 2024-08-17T17:27:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:01 smithi179 ceph-mon[34775]: pgmap v75: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 398 KiB/s rd, 21 KiB/s wr, 501 op/s 2024-08-17T17:27:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:01 smithi179 ceph-mon[34775]: osdmap e304: 8 total, 8 up, 8 in 2024-08-17T17:27:02.489 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach (4022 ms) 2024-08-17T17:27:02.489 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixParentReattach 2024-08-17T17:27:02.520 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:27:02.530 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 0~10 2024-08-17T17:27:02.538 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:27:02.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:02 smithi130 ceph-mon[105521]: osdmap e305: 8 total, 8 up, 8 in 2024-08-17T17:27:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:02 smithi130 ceph-mon[42749]: osdmap e305: 8 total, 8 up, 8 in 2024-08-17T17:27:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:02 smithi179 ceph-mon[34775]: osdmap e305: 8 total, 8 up, 8 in 2024-08-17T17:27:03.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:03 smithi130 ceph-mon[42749]: pgmap v78: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 62 KiB/s rd, 6.2 KiB/s wr, 88 op/s 2024-08-17T17:27:03.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:03 smithi130 ceph-mon[42749]: osdmap e306: 8 total, 8 up, 8 in 2024-08-17T17:27:03.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:03 smithi130 ceph-mon[105521]: pgmap v78: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 62 KiB/s rd, 6.2 KiB/s wr, 88 op/s 2024-08-17T17:27:03.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:03 smithi130 ceph-mon[105521]: osdmap e306: 8 total, 8 up, 8 in 2024-08-17T17:27:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:03 smithi179 ceph-mon[34775]: pgmap v78: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 62 KiB/s rd, 6.2 KiB/s wr, 88 op/s 2024-08-17T17:27:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:03 smithi179 ceph-mon[34775]: osdmap e306: 8 total, 8 up, 8 in 2024-08-17T17:27:04.392 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap1 2024-08-17T17:27:04.445 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:27:04.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:04 smithi179 ceph-mon[34775]: osdmap e307: 8 total, 8 up, 8 in 2024-08-17T17:27:04.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:04.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:04 smithi130 ceph-mon[105521]: osdmap e307: 8 total, 8 up, 8 in 2024-08-17T17:27:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:04 smithi130 ceph-mon[42749]: osdmap e307: 8 total, 8 up, 8 in 2024-08-17T17:27:05.500 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:27:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:05 smithi130 ceph-mon[105521]: pgmap v81: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 294 KiB/s rd, 26 KiB/s wr, 389 op/s 2024-08-17T17:27:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:05 smithi130 ceph-mon[105521]: osdmap e308: 8 total, 8 up, 8 in 2024-08-17T17:27:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:05 smithi130 ceph-mon[105521]: osdmap e309: 8 total, 8 up, 8 in 2024-08-17T17:27:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:05 smithi130 ceph-mon[42749]: pgmap v81: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 294 KiB/s rd, 26 KiB/s wr, 389 op/s 2024-08-17T17:27:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:05 smithi130 ceph-mon[42749]: osdmap e308: 8 total, 8 up, 8 in 2024-08-17T17:27:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:05 smithi130 ceph-mon[42749]: osdmap e309: 8 total, 8 up, 8 in 2024-08-17T17:27:05.740 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:05 smithi179 ceph-mon[34775]: pgmap v81: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 294 KiB/s rd, 26 KiB/s wr, 389 op/s 2024-08-17T17:27:05.740 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:05 smithi179 ceph-mon[34775]: osdmap e308: 8 total, 8 up, 8 in 2024-08-17T17:27:05.740 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:05 smithi179 ceph-mon[34775]: osdmap e309: 8 total, 8 up, 8 in 2024-08-17T17:27:06.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:06.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:06.518 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixParentReattach (4029 ms) 2024-08-17T17:27:06.518 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.CloneV2ParentAbortRelinkNotNeeded 2024-08-17T17:27:06.550 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:27:06.561 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 0~10 2024-08-17T17:27:06.569 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:27:07.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:06 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:07.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:07 smithi130 ceph-mon[105521]: pgmap v84: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 294 KiB/s rd, 26 KiB/s wr, 389 op/s 2024-08-17T17:27:07.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:07 smithi130 ceph-mon[105521]: osdmap e310: 8 total, 8 up, 8 in 2024-08-17T17:27:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:07 smithi130 ceph-mon[42749]: pgmap v84: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 294 KiB/s rd, 26 KiB/s wr, 389 op/s 2024-08-17T17:27:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:07 smithi130 ceph-mon[42749]: osdmap e310: 8 total, 8 up, 8 in 2024-08-17T17:27:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:07 smithi179 ceph-mon[34775]: pgmap v84: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 294 KiB/s rd, 26 KiB/s wr, 389 op/s 2024-08-17T17:27:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:07 smithi179 ceph-mon[34775]: osdmap e310: 8 total, 8 up, 8 in 2024-08-17T17:27:08.411 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap1 2024-08-17T17:27:08.475 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:27:08.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:08 smithi130 ceph-mon[105521]: osdmap e311: 8 total, 8 up, 8 in 2024-08-17T17:27:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:08 smithi130 ceph-mon[42749]: osdmap e311: 8 total, 8 up, 8 in 2024-08-17T17:27:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:08 smithi179 ceph-mon[34775]: osdmap e311: 8 total, 8 up, 8 in 2024-08-17T17:27:09.608 INFO:tasks.workunit.client.0.smithi130.stdout:migration_abort 2024-08-17T17:27:09.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:09 smithi130 ceph-mon[105521]: pgmap v87: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 245 KiB/s rd, 21 KiB/s wr, 318 op/s 2024-08-17T17:27:09.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:09 smithi130 ceph-mon[105521]: osdmap e312: 8 total, 8 up, 8 in 2024-08-17T17:27:09.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:09 smithi130 ceph-mon[42749]: pgmap v87: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 245 KiB/s rd, 21 KiB/s wr, 318 op/s 2024-08-17T17:27:09.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:09 smithi130 ceph-mon[42749]: osdmap e312: 8 total, 8 up, 8 in 2024-08-17T17:27:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:09 smithi179 ceph-mon[34775]: pgmap v87: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 245 KiB/s rd, 21 KiB/s wr, 318 op/s 2024-08-17T17:27:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:09 smithi179 ceph-mon[34775]: osdmap e312: 8 total, 8 up, 8 in 2024-08-17T17:27:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:27:09] "GET /metrics HTTP/1.1" 200 38770 "" "Prometheus/2.51.0" 2024-08-17T17:27:10.526 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.CloneV2ParentAbortRelinkNotNeeded (4008 ms) 2024-08-17T17:27:10.526 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.StressNoMigrate 2024-08-17T17:27:10.555 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:27:10.565 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 6339550~3005899 2024-08-17T17:27:10.587 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 49107104~1298200 2024-08-17T17:27:10.587 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 107311~8136619 2024-08-17T17:27:10.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:10 smithi179 ceph-mon[34775]: osdmap e313: 8 total, 8 up, 8 in 2024-08-17T17:27:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:10.637 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 20104938~5694676 2024-08-17T17:27:10.637 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 22193843~2176727 2024-08-17T17:27:10.659 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 53574498~483640 2024-08-17T17:27:10.659 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 50036195~4760000 2024-08-17T17:27:10.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:10 smithi130 ceph-mon[105521]: osdmap e313: 8 total, 8 up, 8 in 2024-08-17T17:27:10.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:10 smithi130 ceph-mon[42749]: osdmap e313: 8 total, 8 up, 8 in 2024-08-17T17:27:10.693 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 59198997~7221659 2024-08-17T17:27:10.693 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap0 2024-08-17T17:27:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:11 smithi130 ceph-mon[105521]: pgmap v90: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 246 KiB/s rd, 21 KiB/s wr, 321 op/s 2024-08-17T17:27:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:11 smithi130 ceph-mon[105521]: osdmap e314: 8 total, 8 up, 8 in 2024-08-17T17:27:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:11 smithi130 ceph-mon[42749]: pgmap v90: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 246 KiB/s rd, 21 KiB/s wr, 321 op/s 2024-08-17T17:27:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:11 smithi130 ceph-mon[42749]: osdmap e314: 8 total, 8 up, 8 in 2024-08-17T17:27:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:11 smithi179 ceph-mon[34775]: pgmap v90: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 246 KiB/s rd, 21 KiB/s wr, 321 op/s 2024-08-17T17:27:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:11 smithi179 ceph-mon[34775]: osdmap e314: 8 total, 8 up, 8 in 2024-08-17T17:27:12.434 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap0 2024-08-17T17:27:12.445 INFO:tasks.workunit.client.0.smithi130.stdout:clone image159 -> image162 2024-08-17T17:27:12.534 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 121131406 2024-08-17T17:27:12.545 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 45888363~1925316 2024-08-17T17:27:12.567 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 80407131~854340 2024-08-17T17:27:12.572 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 34976101~4244260 2024-08-17T17:27:12.599 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 90320268~4474112 2024-08-17T17:27:12.599 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 81663503~8272274 2024-08-17T17:27:12.643 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 60717492~3096339 2024-08-17T17:27:12.652 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 14133603~1117370 2024-08-17T17:27:12.662 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 52238114~945411 2024-08-17T17:27:12.703 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:27:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:12 smithi179 ceph-mon[34775]: osdmap e315: 8 total, 8 up, 8 in 2024-08-17T17:27:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:12 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:27:12.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:12 smithi130 ceph-mon[105521]: osdmap e315: 8 total, 8 up, 8 in 2024-08-17T17:27:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:27:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:12 smithi130 ceph-mon[42749]: osdmap e315: 8 total, 8 up, 8 in 2024-08-17T17:27:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:12 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:27:13.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:13 smithi179 ceph-mon[34775]: pgmap v93: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 1.5 KiB/s rd, 511 B/s wr, 2 op/s 2024-08-17T17:27:13.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:13 smithi179 ceph-mon[34775]: osdmap e316: 8 total, 8 up, 8 in 2024-08-17T17:27:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:13 smithi130 ceph-mon[105521]: pgmap v93: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 1.5 KiB/s rd, 511 B/s wr, 2 op/s 2024-08-17T17:27:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:13 smithi130 ceph-mon[105521]: osdmap e316: 8 total, 8 up, 8 in 2024-08-17T17:27:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:13 smithi130 ceph-mon[42749]: pgmap v93: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 1.5 KiB/s rd, 511 B/s wr, 2 op/s 2024-08-17T17:27:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:13 smithi130 ceph-mon[42749]: osdmap e316: 8 total, 8 up, 8 in 2024-08-17T17:27:14.429 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:14 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:14.453 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap1 2024-08-17T17:27:14.466 INFO:tasks.workunit.client.0.smithi130.stdout:clone image162 -> image164 2024-08-17T17:27:14.582 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 62843271~5587502 2024-08-17T17:27:14.628 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 29124551~3229450 2024-08-17T17:27:14.636 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 103781088~4606655 2024-08-17T17:27:14.662 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 112022239~7375522 2024-08-17T17:27:14.670 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 5170189~5305352 2024-08-17T17:27:14.718 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 8510175~4232342 2024-08-17T17:27:14.733 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 72596088~3156549 2024-08-17T17:27:14.759 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 96621964~2723684 2024-08-17T17:27:14.764 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:27:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:14 smithi179 ceph-mon[34775]: osdmap e317: 8 total, 8 up, 8 in 2024-08-17T17:27:14.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:14 smithi130 ceph-mon[105521]: osdmap e317: 8 total, 8 up, 8 in 2024-08-17T17:27:14.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:14 smithi130 ceph-mon[42749]: osdmap e317: 8 total, 8 up, 8 in 2024-08-17T17:27:15.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:15 smithi179 ceph-mon[34775]: pgmap v96: 137 pgs: 137 active+clean; 1005 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 18 MiB/s wr, 528 op/s 2024-08-17T17:27:15.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:15 smithi179 ceph-mon[34775]: osdmap e318: 8 total, 8 up, 8 in 2024-08-17T17:27:15.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:15 smithi179 ceph-mon[34775]: osdmap e319: 8 total, 8 up, 8 in 2024-08-17T17:27:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:15 smithi130 ceph-mon[105521]: pgmap v96: 137 pgs: 137 active+clean; 1005 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 18 MiB/s wr, 528 op/s 2024-08-17T17:27:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:15 smithi130 ceph-mon[105521]: osdmap e318: 8 total, 8 up, 8 in 2024-08-17T17:27:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:15 smithi130 ceph-mon[105521]: osdmap e319: 8 total, 8 up, 8 in 2024-08-17T17:27:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:15 smithi130 ceph-mon[42749]: pgmap v96: 137 pgs: 137 active+clean; 1005 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 18 MiB/s wr, 528 op/s 2024-08-17T17:27:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:15 smithi130 ceph-mon[42749]: osdmap e318: 8 total, 8 up, 8 in 2024-08-17T17:27:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:15 smithi130 ceph-mon[42749]: osdmap e319: 8 total, 8 up, 8 in 2024-08-17T17:27:16.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:16.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:16.338 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap2 2024-08-17T17:27:16.349 INFO:tasks.workunit.client.0.smithi130.stdout:clone image164 -> image166 2024-08-17T17:27:16.496 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 12857356~7780247 2024-08-17T17:27:16.552 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 113770194~7361212 2024-08-17T17:27:16.557 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 53407411~4257473 2024-08-17T17:27:16.610 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 111742673~7311124 2024-08-17T17:27:16.615 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 100197175~3116224 2024-08-17T17:27:16.648 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 58270445~5267669 2024-08-17T17:27:16.688 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 77629988~920650 2024-08-17T17:27:16.703 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 20256882~8034532 2024-08-17T17:27:16.709 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:27:17.327 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:17 smithi179 ceph-mon[34775]: pgmap v99: 137 pgs: 137 active+clean; 1005 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 18 MiB/s wr, 528 op/s 2024-08-17T17:27:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:17 smithi179 ceph-mon[34775]: osdmap e320: 8 total, 8 up, 8 in 2024-08-17T17:27:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:17 smithi130 ceph-mon[105521]: pgmap v99: 137 pgs: 137 active+clean; 1005 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 18 MiB/s wr, 528 op/s 2024-08-17T17:27:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:17 smithi130 ceph-mon[105521]: osdmap e320: 8 total, 8 up, 8 in 2024-08-17T17:27:17.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:17 smithi130 ceph-mon[42749]: pgmap v99: 137 pgs: 137 active+clean; 1005 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.5 MiB/s rd, 18 MiB/s wr, 528 op/s 2024-08-17T17:27:17.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:17 smithi130 ceph-mon[42749]: osdmap e320: 8 total, 8 up, 8 in 2024-08-17T17:27:18.463 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap3 2024-08-17T17:27:18.476 INFO:tasks.workunit.client.0.smithi130.stdout:clone image166 -> image168 2024-08-17T17:27:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:18 smithi179 ceph-mon[34775]: osdmap e321: 8 total, 8 up, 8 in 2024-08-17T17:27:18.877 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.StressNoMigrate (8351 ms) 2024-08-17T17:27:18.877 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.Stress 2024-08-17T17:27:18.907 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:27:18.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:18 smithi130 ceph-mon[105521]: osdmap e321: 8 total, 8 up, 8 in 2024-08-17T17:27:18.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:18 smithi130 ceph-mon[42749]: osdmap e321: 8 total, 8 up, 8 in 2024-08-17T17:27:18.917 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 29234823~2233792 2024-08-17T17:27:18.939 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 41093576~175175 2024-08-17T17:27:18.939 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 54739165~3534655 2024-08-17T17:27:18.969 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 77568575~6317505 2024-08-17T17:27:18.969 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 34207013~2176567 2024-08-17T17:27:18.993 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 70957321~2313486 2024-08-17T17:27:18.993 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 35259558~7620525 2024-08-17T17:27:19.042 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 82090601~1605851 2024-08-17T17:27:19.042 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap0 2024-08-17T17:27:19.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:19 smithi130 ceph-mon[105521]: pgmap v102: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8.2 MiB/s rd, 21 MiB/s wr, 501 op/s 2024-08-17T17:27:19.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:19 smithi130 ceph-mon[105521]: osdmap e322: 8 total, 8 up, 8 in 2024-08-17T17:27:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:19 smithi130 ceph-mon[42749]: pgmap v102: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8.2 MiB/s rd, 21 MiB/s wr, 501 op/s 2024-08-17T17:27:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:19 smithi130 ceph-mon[42749]: osdmap e322: 8 total, 8 up, 8 in 2024-08-17T17:27:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:19 smithi179 ceph-mon[34775]: pgmap v102: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8.2 MiB/s rd, 21 MiB/s wr, 501 op/s 2024-08-17T17:27:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:19 smithi179 ceph-mon[34775]: osdmap e322: 8 total, 8 up, 8 in 2024-08-17T17:27:20.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:27:19] "GET /metrics HTTP/1.1" 200 38765 "" "Prometheus/2.51.0" 2024-08-17T17:27:20.478 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 41845988~273464 2024-08-17T17:27:20.484 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 53721337~2849559 2024-08-17T17:27:20.484 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 16505837~8159484 2024-08-17T17:27:20.492 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 42634566~7301206 2024-08-17T17:27:20.492 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 23724347~2195185 2024-08-17T17:27:20.523 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 57608918~4841626 2024-08-17T17:27:20.523 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 23852634~2811103 2024-08-17T17:27:20.535 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 78483750~2957662 2024-08-17T17:27:20.535 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:27:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:20 smithi179 ceph-mon[34775]: osdmap e323: 8 total, 8 up, 8 in 2024-08-17T17:27:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:20 smithi130 ceph-mon[105521]: osdmap e323: 8 total, 8 up, 8 in 2024-08-17T17:27:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:20 smithi130 ceph-mon[42749]: osdmap e323: 8 total, 8 up, 8 in 2024-08-17T17:27:21.648 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:21 smithi130 ceph-mon[105521]: pgmap v105: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8.2 MiB/s rd, 21 MiB/s wr, 501 op/s 2024-08-17T17:27:21.648 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:21 smithi130 ceph-mon[105521]: osdmap e324: 8 total, 8 up, 8 in 2024-08-17T17:27:21.648 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:21 smithi130 ceph-mon[42749]: pgmap v105: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8.2 MiB/s rd, 21 MiB/s wr, 501 op/s 2024-08-17T17:27:21.648 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:21 smithi130 ceph-mon[42749]: osdmap e324: 8 total, 8 up, 8 in 2024-08-17T17:27:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:21 smithi179 ceph-mon[34775]: pgmap v105: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8.2 MiB/s rd, 21 MiB/s wr, 501 op/s 2024-08-17T17:27:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:21 smithi179 ceph-mon[34775]: osdmap e324: 8 total, 8 up, 8 in 2024-08-17T17:27:22.077 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade resume' 2024-08-17T17:27:22.344 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:22.491 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap1 2024-08-17T17:27:22.500 INFO:tasks.workunit.client.0.smithi130.stdout:clone image169 -> image172 2024-08-17T17:27:22.591 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 166777347 2024-08-17T17:27:22.601 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 106022918~4154852 2024-08-17T17:27:22.633 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 38361866~1181754 2024-08-17T17:27:22.670 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 159668947~7108400 2024-08-17T17:27:22.715 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 48336836~4039351 2024-08-17T17:27:22.721 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 119684385~6636063 2024-08-17T17:27:22.728 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:22 smithi130 ceph-mon[105521]: osdmap e325: 8 total, 8 up, 8 in 2024-08-17T17:27:22.728 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:22 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:27:22.728 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:22 smithi130 ceph-mon[42749]: osdmap e325: 8 total, 8 up, 8 in 2024-08-17T17:27:22.728 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:22 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:27:22.770 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 57991576~1405350 2024-08-17T17:27:22.784 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 125712501~2517337 2024-08-17T17:27:22.800 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 141684276~6960339 2024-08-17T17:27:22.800 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:27:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:22 smithi179 ceph-mon[34775]: osdmap e325: 8 total, 8 up, 8 in 2024-08-17T17:27:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:22 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:27:23.735 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:23 smithi179 ceph-mon[34775]: pgmap v108: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:23.736 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:23 smithi179 ceph-mon[34775]: osdmap e326: 8 total, 8 up, 8 in 2024-08-17T17:27:23.749 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:23 smithi130 ceph-mon[105521]: pgmap v108: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:23.749 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:23 smithi130 ceph-mon[105521]: osdmap e326: 8 total, 8 up, 8 in 2024-08-17T17:27:23.750 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:23 smithi130 ceph-mon[42749]: pgmap v108: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:23.750 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:23 smithi130 ceph-mon[42749]: osdmap e326: 8 total, 8 up, 8 in 2024-08-17T17:27:24.357 INFO:teuthology.orchestra.run.smithi130.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:27:24.439 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:24 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:24.506 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap2 2024-08-17T17:27:24.513 INFO:tasks.workunit.client.0.smithi130.stdout:clone image172 -> image174 2024-08-17T17:27:24.624 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 48687805~4819625 2024-08-17T17:27:24.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:24 smithi130 ceph-mon[105521]: osdmap e327: 8 total, 8 up, 8 in 2024-08-17T17:27:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:24 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:24.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:24 smithi130 ceph-mon[42749]: osdmap e327: 8 total, 8 up, 8 in 2024-08-17T17:27:24.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:24 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:24.667 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 132191707~495537 2024-08-17T17:27:24.673 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 3363452~6612326 2024-08-17T17:27:24.713 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 125724088~6189251 2024-08-17T17:27:24.765 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 136564171~2341862 2024-08-17T17:27:24.782 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 141626990~3136484 2024-08-17T17:27:24.787 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 119892388~8180716 2024-08-17T17:27:24.828 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 76471334~5240484 2024-08-17T17:27:24.835 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:27:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:24 smithi179 ceph-mon[34775]: osdmap e327: 8 total, 8 up, 8 in 2024-08-17T17:27:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:24 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:25.059 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for majority of mons to upgrade' 2024-08-17T17:27:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:25 smithi179 ceph-mon[34775]: pgmap v111: 137 pgs: 137 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 29 MiB/s rd, 26 MiB/s wr, 551 op/s 2024-08-17T17:27:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:25 smithi179 ceph-mon[34775]: from='client.26036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:25 smithi179 ceph-mon[34775]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:27:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:25 smithi179 ceph-mon[34775]: osdmap e328: 8 total, 8 up, 8 in 2024-08-17T17:27:25.651 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:25.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:25 smithi130 ceph-mon[42749]: pgmap v111: 137 pgs: 137 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 29 MiB/s rd, 26 MiB/s wr, 551 op/s 2024-08-17T17:27:25.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:25 smithi130 ceph-mon[42749]: from='client.26036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:25 smithi130 ceph-mon[42749]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:27:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:25 smithi130 ceph-mon[42749]: osdmap e328: 8 total, 8 up, 8 in 2024-08-17T17:27:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:25 smithi130 ceph-mon[105521]: pgmap v111: 137 pgs: 137 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 29 MiB/s rd, 26 MiB/s wr, 551 op/s 2024-08-17T17:27:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:25 smithi130 ceph-mon[105521]: from='client.26036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:25 smithi130 ceph-mon[105521]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:27:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:25 smithi130 ceph-mon[105521]: osdmap e328: 8 total, 8 up, 8 in 2024-08-17T17:27:26.015 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:25 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:26.016 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:25 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:25.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:26.513 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap3 2024-08-17T17:27:26.523 INFO:tasks.workunit.client.0.smithi130.stdout:clone image174 -> image176 2024-08-17T17:27:26.675 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:27:26.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:26 smithi130 ceph-mon[42749]: osdmap e329: 8 total, 8 up, 8 in 2024-08-17T17:27:26.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:26 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:26.828 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:26 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:26.828 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:26 smithi130 ceph-mon[105521]: osdmap e329: 8 total, 8 up, 8 in 2024-08-17T17:27:26.828 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:26 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:26.828 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:26 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:26 smithi179 ceph-mon[34775]: osdmap e329: 8 total, 8 up, 8 in 2024-08-17T17:27:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:26 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:26 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:27.182 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:27:27.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:27.363 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:27:27.676 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.Stress (8799 ms) 2024-08-17T17:27:27.677 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.Stress2 2024-08-17T17:27:27.708 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:27:27.719 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 16138122~6004074 2024-08-17T17:27:27.767 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 21028746~4385194 2024-08-17T17:27:27.770 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 56122192~1604779 2024-08-17T17:27:27.786 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 75523770~8362310 2024-08-17T17:27:27.786 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 23711257~7098843 2024-08-17T17:27:27.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:27 smithi130 ceph-mon[105521]: pgmap v114: 137 pgs: 137 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 29 MiB/s rd, 26 MiB/s wr, 551 op/s 2024-08-17T17:27:27.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:27 smithi130 ceph-mon[105521]: osdmap e330: 8 total, 8 up, 8 in 2024-08-17T17:27:27.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:27:27.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:27 smithi130 ceph-mon[42749]: pgmap v114: 137 pgs: 137 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 29 MiB/s rd, 26 MiB/s wr, 551 op/s 2024-08-17T17:27:27.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:27 smithi130 ceph-mon[42749]: osdmap e330: 8 total, 8 up, 8 in 2024-08-17T17:27:27.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:27 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:27:27.831 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 41732449~1640535 2024-08-17T17:27:27.831 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 65180416~5234394 2024-08-17T17:27:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:27 smithi179 ceph-mon[34775]: pgmap v114: 137 pgs: 137 active+clean; 1.2 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 29 MiB/s rd, 26 MiB/s wr, 551 op/s 2024-08-17T17:27:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:27 smithi179 ceph-mon[34775]: osdmap e330: 8 total, 8 up, 8 in 2024-08-17T17:27:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:27 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:27:27.873 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 22507147~1560110 2024-08-17T17:27:27.875 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap0 2024-08-17T17:27:28.196 INFO:teuthology.orchestra.run.smithi130.stdout:wait for majority of mons to upgrade 2024-08-17T17:27:28.951 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-08-17T17:27:29.524 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 24416167~5114131 2024-08-17T17:27:29.531 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 57787641~5082965 2024-08-17T17:27:29.531 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 24996665~37264 2024-08-17T17:27:29.541 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 8217980~7901564 2024-08-17T17:27:29.542 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 71862305~7122178 2024-08-17T17:27:29.550 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 10832340~803234 2024-08-17T17:27:29.550 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 65527565~325109 2024-08-17T17:27:29.550 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 71819168~1804638 2024-08-17T17:27:29.583 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:27:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:29 smithi130 ceph-mon[42749]: pgmap v116: 137 pgs: 137 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 49 MiB/s rd, 20 MiB/s wr, 1.06k op/s 2024-08-17T17:27:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:29 smithi130 ceph-mon[42749]: osdmap e331: 8 total, 8 up, 8 in 2024-08-17T17:27:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:29 smithi130 ceph-mon[105521]: pgmap v116: 137 pgs: 137 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 49 MiB/s rd, 20 MiB/s wr, 1.06k op/s 2024-08-17T17:27:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:29 smithi130 ceph-mon[105521]: osdmap e331: 8 total, 8 up, 8 in 2024-08-17T17:27:29.760 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:29 smithi179 ceph-mon[34775]: pgmap v116: 137 pgs: 137 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 49 MiB/s rd, 20 MiB/s wr, 1.06k op/s 2024-08-17T17:27:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:29 smithi179 ceph-mon[34775]: osdmap e331: 8 total, 8 up, 8 in 2024-08-17T17:27:30.099 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:27:29] "GET /metrics HTTP/1.1" 200 38768 "" "Prometheus/2.51.0" 2024-08-17T17:27:30.515 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:30 smithi130 ceph-mon[105521]: osdmap e332: 8 total, 8 up, 8 in 2024-08-17T17:27:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:30 smithi130 ceph-mon[42749]: osdmap e332: 8 total, 8 up, 8 in 2024-08-17T17:27:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:30 smithi179 ceph-mon[34775]: osdmap e332: 8 total, 8 up, 8 in 2024-08-17T17:27:31.545 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap1 2024-08-17T17:27:31.555 INFO:tasks.workunit.client.0.smithi130.stdout:clone image177 -> image180 2024-08-17T17:27:31.645 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 161495792 2024-08-17T17:27:31.657 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 24188046~2643933 2024-08-17T17:27:31.716 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 110241270~822475 2024-08-17T17:27:31.717 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 61263325~783461 2024-08-17T17:27:31.727 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 122451189~6423760 2024-08-17T17:27:31.727 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 95919068~2171995 2024-08-17T17:27:31.742 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 140337679~304713 2024-08-17T17:27:31.742 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 119047456~490607 2024-08-17T17:27:31.750 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 114886265~7038789 2024-08-17T17:27:31.758 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:27:31.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:31 smithi130 ceph-mon[105521]: pgmap v119: 137 pgs: 137 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 50 MiB/s rd, 20 MiB/s wr, 1.06k op/s 2024-08-17T17:27:31.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:31 smithi130 ceph-mon[105521]: osdmap e333: 8 total, 8 up, 8 in 2024-08-17T17:27:31.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:31 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:31.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:31 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:31.791 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:31 smithi130 ceph-mon[42749]: pgmap v119: 137 pgs: 137 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 50 MiB/s rd, 20 MiB/s wr, 1.06k op/s 2024-08-17T17:27:31.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:31 smithi130 ceph-mon[42749]: osdmap e333: 8 total, 8 up, 8 in 2024-08-17T17:27:31.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:31 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:31.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:31 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:31 smithi179 ceph-mon[34775]: pgmap v119: 137 pgs: 137 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 50 MiB/s rd, 20 MiB/s wr, 1.06k op/s 2024-08-17T17:27:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:31 smithi179 ceph-mon[34775]: osdmap e333: 8 total, 8 up, 8 in 2024-08-17T17:27:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:31 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:31 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:32.615 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[105521]: osdmap e334: 8 total, 8 up, 8 in 2024-08-17T17:27:32.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:27:32.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:27:32.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:32.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:27:32.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:27:32.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:27:32.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-08-17T17:27:32.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/738898680' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:27:32.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[42749]: osdmap e334: 8 total, 8 up, 8 in 2024-08-17T17:27:32.617 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:27:32.617 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:27:32.617 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:32.617 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:27:32.617 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:27:32.617 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:27:32.617 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-08-17T17:27:32.617 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:32 smithi130 ceph-mon[42749]: from='client.? 172.21.15.130:0/738898680' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:27:32.733 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:27:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:32 smithi179 ceph-mon[34775]: osdmap e334: 8 total, 8 up, 8 in 2024-08-17T17:27:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:32 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:27:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:32 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:27:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:32 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:32 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:27:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:32 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:27:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:32 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:27:32.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:32 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-08-17T17:27:32.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:32 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/738898680' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:27:33.563 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 85299308~7459557 2024-08-17T17:27:33.582 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 39430710~5603200 2024-08-17T17:27:33.582 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 133755349~224926 2024-08-17T17:27:33.582 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 4859643~4149500 2024-08-17T17:27:33.582 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 29680444~1462963 2024-08-17T17:27:33.583 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 13200637~1313858 2024-08-17T17:27:33.583 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 12346399~2583436 2024-08-17T17:27:33.622 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 146012248~4015307 2024-08-17T17:27:33.623 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:27:33.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[105521]: Upgrade: It appears safe to stop mon.c 2024-08-17T17:27:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[105521]: pgmap v122: 137 pgs: 137 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[105521]: osdmap e335: 8 total, 8 up, 8 in 2024-08-17T17:27:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[105521]: from='client.26048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:27:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:27:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:27:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[42749]: Upgrade: It appears safe to stop mon.c 2024-08-17T17:27:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[42749]: pgmap v122: 137 pgs: 137 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[42749]: osdmap e335: 8 total, 8 up, 8 in 2024-08-17T17:27:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[42749]: from='client.26048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[42749]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:27:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:27:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:33 smithi130 ceph-mon[42749]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:27:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:33 smithi179 ceph-mon[34775]: Upgrade: It appears safe to stop mon.c 2024-08-17T17:27:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:33 smithi179 ceph-mon[34775]: pgmap v122: 137 pgs: 137 active+clean; 1.4 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-08-17T17:27:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:33 smithi179 ceph-mon[34775]: osdmap e335: 8 total, 8 up, 8 in 2024-08-17T17:27:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:33 smithi179 ceph-mon[34775]: from='client.26048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:33 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:33 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:27:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:33 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:27:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:33 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:27:34.563 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:34 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:34.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:34 smithi130 ceph-mon[105521]: from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:34.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:34 smithi130 ceph-mon[105521]: Upgrade: Updating mon.c 2024-08-17T17:27:34.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:34 smithi130 ceph-mon[105521]: Deploying daemon mon.c on smithi130 2024-08-17T17:27:34.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:34 smithi130 ceph-mon[105521]: osdmap e336: 8 total, 8 up, 8 in 2024-08-17T17:27:34.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:34 smithi130 ceph-mon[42749]: from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:34.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:34 smithi130 ceph-mon[42749]: Upgrade: Updating mon.c 2024-08-17T17:27:34.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:34 smithi130 ceph-mon[42749]: Deploying daemon mon.c on smithi130 2024-08-17T17:27:34.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:34 smithi130 ceph-mon[42749]: osdmap e336: 8 total, 8 up, 8 in 2024-08-17T17:27:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:34 smithi179 ceph-mon[34775]: from='client.34179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:34 smithi179 ceph-mon[34775]: Upgrade: Updating mon.c 2024-08-17T17:27:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:34 smithi179 ceph-mon[34775]: Deploying daemon mon.c on smithi130 2024-08-17T17:27:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:34 smithi179 ceph-mon[34775]: osdmap e336: 8 total, 8 up, 8 in 2024-08-17T17:27:35.107 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:35 smithi130 systemd[1]: Stopping Ceph mon.c for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:27:35.717 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c[42726]: 2024-08-17T17:27:35.424+0000 7f040115a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:27:35.718 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c[42726]: 2024-08-17T17:27:35.424+0000 7f040115a640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-08-17T17:27:35.718 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:35 smithi130 podman[109829]: 2024-08-17 17:27:35.577355495 +0000 UTC m=+0.286151389 container died 0c43a998b5faa0a92471418f5d859ff0fa86489f8ba3ac635274265e6b51f2fe (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:27:35.718 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:27:35] ENGINE Bus STOPPING 2024-08-17T17:27:35.718 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:27:36.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:36.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:36.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:27:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:27:36.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:27:35] ENGINE Bus STOPPED 2024-08-17T17:27:36.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:35 smithi130 podman[109829]: 2024-08-17 17:27:35.930670934 +0000 UTC m=+0.639466810 container cleanup 0c43a998b5faa0a92471418f5d859ff0fa86489f8ba3ac635274265e6b51f2fe (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0) 2024-08-17T17:27:36.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:35 smithi130 bash[109829]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c 2024-08-17T17:27:36.634 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:36 smithi130 podman[109870]: 2024-08-17 17:27:36.26588869 +0000 UTC m=+0.689562887 container remove 0c43a998b5faa0a92471418f5d859ff0fa86489f8ba3ac635274265e6b51f2fe (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.build-date=20240731) 2024-08-17T17:27:36.634 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:36 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.c.service: Deactivated successfully. 2024-08-17T17:27:36.634 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:36 smithi130 systemd[1]: Stopped Ceph mon.c for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:27:36.634 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:36 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.c.service: Consumed 9.363s CPU time. 2024-08-17T17:27:36.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:36 smithi130 systemd[1]: Starting Ceph mon.c for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:27:36.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:36 smithi130 podman[110030]: 2024-08-17 17:27:36.752479476 +0000 UTC m=+0.017309415 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:27:36.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:36 smithi130 podman[110030]: 2024-08-17 17:27:36.883729322 +0000 UTC m=+0.148559285 container create 4161e48dac506efae59e4ed2c12cfb1a94313f82cd32cac24d9a43b58a586648 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-08-17T17:27:37.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:36 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 podman[110030]: 2024-08-17 17:27:37.094767372 +0000 UTC m=+0.359597303 container init 4161e48dac506efae59e4ed2c12cfb1a94313f82cd32cac24d9a43b58a586648 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe) 2024-08-17T17:27:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 podman[110030]: 2024-08-17 17:27:37.099303098 +0000 UTC m=+0.364133021 container start 4161e48dac506efae59e4ed2c12cfb1a94313f82cd32cac24d9a43b58a586648 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-08-17T17:27:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:27:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev), process ceph-mon, pid 7 2024-08-17T17:27:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: pidfile_write: ignore empty --pid-file 2024-08-17T17:27:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: load: jerasure load: lrc 2024-08-17T17:27:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: RocksDB version: 7.9.2 2024-08-17T17:27:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Git sha 0 2024-08-17T17:27:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Compile date 2024-08-15 20:43:31 2024-08-17T17:27:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: DB SUMMARY 2024-08-17T17:27:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: DB Session ID: 6YOG1CBFJ6LG38HEBMGR 2024-08-17T17:27:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:27:37.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:27:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1098 Bytes 2024-08-17T17:27:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-08-17T17:27:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 11628011 ; 2024-08-17T17:27:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:27:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:27:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:27:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T17:27:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:27:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T17:27:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.env: 0x557de4f98be0 2024-08-17T17:27:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T17:27:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.info_log: 0x557de69d3700 2024-08-17T17:27:37.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:27:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.statistics: (nil) 2024-08-17T17:27:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.use_fsync: 0 2024-08-17T17:27:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:27:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:27:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:27:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:27:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:27:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:27:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:27:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:27:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:27:37.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:27:37.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:27:37.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.db_log_dir: 2024-08-17T17:27:37.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.wal_dir: 2024-08-17T17:27:37.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:27:37.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:27:37.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:27:37.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:27:37.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:27:37.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:27:37.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:27:37.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:27:37.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.write_buffer_manager: 0x557de69d6960 2024-08-17T17:27:37.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:27:37.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:27:37.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:27:37.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:27:37.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:27:37.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:27:37.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:27:37.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:27:37.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.unordered_write: 0 2024-08-17T17:27:37.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:27:37.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:27:37.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:27:37.422 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:27:37.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.row_cache: None 2024-08-17T17:27:37.423 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.wal_filter: None 2024-08-17T17:27:37.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:27:37.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:27:37.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:27:37.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:27:37.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.wal_compression: 0 2024-08-17T17:27:37.424 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:27:37.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:27:37.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:27:37.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:27:37.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:27:37.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:27:37.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:27:37.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:27:37.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:27:37.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:27:37.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:27:37.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T17:27:37.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:27:37.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:27:37.425 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:27:37.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:27:37.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:27:37.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:27:37.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:27:37.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:27:37.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:27:37.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:27:37.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:27:37.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_open_files: -1 2024-08-17T17:27:37.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:27:37.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:27:37.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:27:37.426 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:27:37.427 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:27:37.427 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Compression algorithms supported: 2024-08-17T17:27:37.427 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: kZSTD supported: 0 2024-08-17T17:27:37.427 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:27:37.427 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:27:37.427 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:27:37.427 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:27:37.427 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:27:37.427 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:27:37.428 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:27:37.428 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:27:37.428 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T17:27:37.428 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-08-17T17:27:37.428 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T17:27:37.428 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:27:37.428 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.merge_operator: 2024-08-17T17:27:37.428 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compaction_filter: None 2024-08-17T17:27:37.428 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:27:37.428 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:27:37.428 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:27:37.428 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:27:37.428 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557de69d3380) 2024-08-17T17:27:37.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:27:37.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:27:37.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:27:37.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:27:37.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_type: 0 2024-08-17T17:27:37.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: data_block_index_type: 0 2024-08-17T17:27:37.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_shortening: 1 2024-08-17T17:27:37.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:27:37.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: checksum: 4 2024-08-17T17:27:37.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: no_block_cache: 0 2024-08-17T17:27:37.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache: 0x557de69c7610 2024-08-17T17:27:37.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:27:37.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_options: 2024-08-17T17:27:37.429 INFO:journalctl@ceph.mon.c.smithi130.stdout: capacity : 536870912 2024-08-17T17:27:37.430 INFO:journalctl@ceph.mon.c.smithi130.stdout: num_shard_bits : 4 2024-08-17T17:27:37.430 INFO:journalctl@ceph.mon.c.smithi130.stdout: strict_capacity_limit : 0 2024-08-17T17:27:37.430 INFO:journalctl@ceph.mon.c.smithi130.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:27:37.430 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_cache_compressed: (nil) 2024-08-17T17:27:37.430 INFO:journalctl@ceph.mon.c.smithi130.stdout: persistent_cache: (nil) 2024-08-17T17:27:37.430 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_size: 4096 2024-08-17T17:27:37.430 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_size_deviation: 10 2024-08-17T17:27:37.430 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_restart_interval: 16 2024-08-17T17:27:37.430 INFO:journalctl@ceph.mon.c.smithi130.stdout: index_block_restart_interval: 1 2024-08-17T17:27:37.430 INFO:journalctl@ceph.mon.c.smithi130.stdout: metadata_block_size: 4096 2024-08-17T17:27:37.430 INFO:journalctl@ceph.mon.c.smithi130.stdout: partition_filters: 0 2024-08-17T17:27:37.430 INFO:journalctl@ceph.mon.c.smithi130.stdout: use_delta_encoding: 1 2024-08-17T17:27:37.430 INFO:journalctl@ceph.mon.c.smithi130.stdout: filter_policy: bloomfilter 2024-08-17T17:27:37.431 INFO:journalctl@ceph.mon.c.smithi130.stdout: whole_key_filtering: 1 2024-08-17T17:27:37.431 INFO:journalctl@ceph.mon.c.smithi130.stdout: verify_compression: 0 2024-08-17T17:27:37.431 INFO:journalctl@ceph.mon.c.smithi130.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:27:37.431 INFO:journalctl@ceph.mon.c.smithi130.stdout: format_version: 5 2024-08-17T17:27:37.431 INFO:journalctl@ceph.mon.c.smithi130.stdout: enable_index_compression: 1 2024-08-17T17:27:37.431 INFO:journalctl@ceph.mon.c.smithi130.stdout: block_align: 0 2024-08-17T17:27:37.431 INFO:journalctl@ceph.mon.c.smithi130.stdout: max_auto_readahead_size: 262144 2024-08-17T17:27:37.431 INFO:journalctl@ceph.mon.c.smithi130.stdout: prepopulate_block_cache: 0 2024-08-17T17:27:37.431 INFO:journalctl@ceph.mon.c.smithi130.stdout: initial_auto_readahead_size: 8192 2024-08-17T17:27:37.431 INFO:journalctl@ceph.mon.c.smithi130.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T17:27:37.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:27:37.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:27:37.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compression: NoCompression 2024-08-17T17:27:37.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:27:37.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:27:37.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:27:37.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.num_levels: 7 2024-08-17T17:27:37.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:27:37.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:27:37.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:27:37.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:27:37.432 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:27:37.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:27:37.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:27:37.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:27:37.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:27:37.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:27:37.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:27:37.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:27:37.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:27:37.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:27:37.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:27:37.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:27:37.433 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:27:37.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:27:37.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:27:37.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:27:37.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:27:37.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:27:37.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:27:37.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:27:37.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:27:37.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:27:37.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:27:37.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:27:37.434 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:27:37.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:27:37.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:27:37.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:27:37.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:27:37.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:27:37.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:27:37.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:27:37.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:27:37.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:27:37.435 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T17:27:37.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T17:27:37.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:27:37.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:27:37.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:27:37.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:27:37.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:27:37.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:27:37.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:27:37.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:27:37.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:27:37.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:27:37.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:27:37.436 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:27:37.437 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:27:37.437 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T17:27:37.437 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:27:37.437 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:27:37.437 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:27:37.437 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:27:37.437 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:27:37.437 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:27:37.437 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:27:37.437 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:27:37.437 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:27:37.437 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:27:37.437 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:27:37.438 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.ttl: 2592000 2024-08-17T17:27:37.438 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:27:37.438 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T17:27:37.438 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T17:27:37.438 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:27:37.438 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:27:37.438 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:27:37.438 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:27:37.438 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:27:37.438 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:27:37.438 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T17:27:37.438 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T17:27:37.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T17:27:37.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T17:27:37.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-08-17T17:27:37.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:27:37.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13495, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T17:27:37.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-08-17T17:27:37.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: bcfc63fa-c974-4628-b7ce-2644f8dc1115 2024-08-17T17:27:37.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723915657126491, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-08-17T17:27:37.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-08-17T17:27:37.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723915657152296, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 7287789, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13492, "largest_seqno": 17409, "table_properties": {"data_size": 7269815, "index_size": 12703, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4229, "raw_key_size": 36865, "raw_average_key_size": 22, "raw_value_size": 7233963, "raw_average_value_size": 4329, "num_data_blocks": 547, "num_entries": 1671, "num_filter_entries": 1671, "num_deletions": 257, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723915657, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "bcfc63fa-c974-4628-b7ce-2644f8dc1115", "db_session_id": "6YOG1CBFJ6LG38HEBMGR", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-08-17T17:27:37.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723915657152439, "job": 1, "event": "recovery_finished"} 2024-08-17T17:27:37.439 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-08-17T17:27:37.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:27:37.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:27:37.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x557de69f6e00 2024-08-17T17:27:37.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: DB pointer 0x557de6af4000 2024-08-17T17:27:37.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T17:27:37.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T17:27:37.440 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** DB Stats ** 2024-08-17T17:27:37.440 INFO:journalctl@ceph.mon.c.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:27:37.440 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:27:37.440 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:27:37.440 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:27:37.440 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:27:37.440 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:27:37.440 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:27:37.441 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:27:37.441 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2024-08-17T17:27:37.441 INFO:journalctl@ceph.mon.c.smithi130.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:27:37.441 INFO:journalctl@ceph.mon.c.smithi130.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T17:27:37.441 INFO:journalctl@ceph.mon.c.smithi130.stdout: L0 1/0 6.95 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 634.4 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-08-17T17:27:37.441 INFO:journalctl@ceph.mon.c.smithi130.stdout: L6 1/0 8.07 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-17T17:27:37.441 INFO:journalctl@ceph.mon.c.smithi130.stdout: Sum 2/0 15.02 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 634.4 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-08-17T17:27:37.441 INFO:journalctl@ceph.mon.c.smithi130.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 634.4 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-08-17T17:27:37.441 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:27:37.441 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** Compaction Stats [default] ** 2024-08-17T17:27:37.441 INFO:journalctl@ceph.mon.c.smithi130.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:27:37.441 INFO:journalctl@ceph.mon.c.smithi130.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:27:37.441 INFO:journalctl@ceph.mon.c.smithi130.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 634.4 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-08-17T17:27:37.442 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:27:37.442 INFO:journalctl@ceph.mon.c.smithi130.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T17:27:37.442 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:27:37.442 INFO:journalctl@ceph.mon.c.smithi130.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:27:37.442 INFO:journalctl@ceph.mon.c.smithi130.stdout: Flush(GB): cumulative 0.007, interval 0.007 2024-08-17T17:27:37.442 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:27:37.442 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:27:37.442 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:27:37.442 INFO:journalctl@ceph.mon.c.smithi130.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:27:37.442 INFO:journalctl@ceph.mon.c.smithi130.stdout: Cumulative compaction: 0.01 GB write, 223.86 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:27:37.442 INFO:journalctl@ceph.mon.c.smithi130.stdout: Interval compaction: 0.01 GB write, 223.86 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:27:37.442 INFO:journalctl@ceph.mon.c.smithi130.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:27:37.442 INFO:journalctl@ceph.mon.c.smithi130.stdout: Block cache BinnedLRUCache@0x557de69c7610#7 capacity: 512.00 MB usage: 17.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-08-17T17:27:37.443 INFO:journalctl@ceph.mon.c.smithi130.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,13.08 KB,0.00249445%) Misc(1,0.00 KB,0%) 2024-08-17T17:27:37.443 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:27:37.443 INFO:journalctl@ceph.mon.c.smithi130.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:27:37.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: starting mon.c rank 2 at public addrs [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] at bind addrs [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:27:37.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: mon.c@-1(???) e3 preinit fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:27:37.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: mon.c@-1(???).mds e9 new map 2024-08-17T17:27:37.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: mon.c@-1(???).mds e9 print_map 2024-08-17T17:27:37.443 INFO:journalctl@ceph.mon.c.smithi130.stdout: e9 2024-08-17T17:27:37.443 INFO:journalctl@ceph.mon.c.smithi130.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-17T17:27:37.443 INFO:journalctl@ceph.mon.c.smithi130.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:27:37.443 INFO:journalctl@ceph.mon.c.smithi130.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:27:37.443 INFO:journalctl@ceph.mon.c.smithi130.stdout: legacy client fscid: 1 2024-08-17T17:27:37.443 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:27:37.443 INFO:journalctl@ceph.mon.c.smithi130.stdout: Filesystem 'foo' (1) 2024-08-17T17:27:37.444 INFO:journalctl@ceph.mon.c.smithi130.stdout: fs_name foo 2024-08-17T17:27:37.444 INFO:journalctl@ceph.mon.c.smithi130.stdout: epoch 8 2024-08-17T17:27:37.444 INFO:journalctl@ceph.mon.c.smithi130.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-17T17:27:37.444 INFO:journalctl@ceph.mon.c.smithi130.stdout: created 2024-08-17T17:21:52.146143+0000 2024-08-17T17:27:37.444 INFO:journalctl@ceph.mon.c.smithi130.stdout: modified 2024-08-17T17:22:05.340164+0000 2024-08-17T17:27:37.444 INFO:journalctl@ceph.mon.c.smithi130.stdout: tableserver 0 2024-08-17T17:27:37.444 INFO:journalctl@ceph.mon.c.smithi130.stdout: root 0 2024-08-17T17:27:37.444 INFO:journalctl@ceph.mon.c.smithi130.stdout: session_timeout 60 2024-08-17T17:27:37.444 INFO:journalctl@ceph.mon.c.smithi130.stdout: session_autoclose 300 2024-08-17T17:27:37.444 INFO:journalctl@ceph.mon.c.smithi130.stdout: max_file_size 1099511627776 2024-08-17T17:27:37.444 INFO:journalctl@ceph.mon.c.smithi130.stdout: max_xattr_size 65536 2024-08-17T17:27:37.444 INFO:journalctl@ceph.mon.c.smithi130.stdout: required_client_features {} 2024-08-17T17:27:37.444 INFO:journalctl@ceph.mon.c.smithi130.stdout: last_failure 0 2024-08-17T17:27:37.444 INFO:journalctl@ceph.mon.c.smithi130.stdout: last_failure_osd_epoch 0 2024-08-17T17:27:37.445 INFO:journalctl@ceph.mon.c.smithi130.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:27:37.445 INFO:journalctl@ceph.mon.c.smithi130.stdout: max_mds 1 2024-08-17T17:27:37.445 INFO:journalctl@ceph.mon.c.smithi130.stdout: in 0 2024-08-17T17:27:37.445 INFO:journalctl@ceph.mon.c.smithi130.stdout: up {0=24548} 2024-08-17T17:27:37.445 INFO:journalctl@ceph.mon.c.smithi130.stdout: failed 2024-08-17T17:27:37.445 INFO:journalctl@ceph.mon.c.smithi130.stdout: damaged 2024-08-17T17:27:37.445 INFO:journalctl@ceph.mon.c.smithi130.stdout: stopped 2024-08-17T17:27:37.445 INFO:journalctl@ceph.mon.c.smithi130.stdout: data_pools [4] 2024-08-17T17:27:37.445 INFO:journalctl@ceph.mon.c.smithi130.stdout: metadata_pool 3 2024-08-17T17:27:37.445 INFO:journalctl@ceph.mon.c.smithi130.stdout: inline_data disabled 2024-08-17T17:27:37.445 INFO:journalctl@ceph.mon.c.smithi130.stdout: balancer 2024-08-17T17:27:37.445 INFO:journalctl@ceph.mon.c.smithi130.stdout: bal_rank_mask -1 2024-08-17T17:27:37.445 INFO:journalctl@ceph.mon.c.smithi130.stdout: standby_count_wanted 1 2024-08-17T17:27:37.445 INFO:journalctl@ceph.mon.c.smithi130.stdout: qdb_cluster leader: 0 members: 2024-08-17T17:27:37.446 INFO:journalctl@ceph.mon.c.smithi130.stdout: [mds.foo.smithi130.jmaidg{0:24548} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.130:6834/1214210479,v1:172.21.15.130:6835/1214210479] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:27:37.446 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:27:37.446 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:27:37.446 INFO:journalctl@ceph.mon.c.smithi130.stdout: Standby daemons: 2024-08-17T17:27:37.446 INFO:journalctl@ceph.mon.c.smithi130.stdout: 2024-08-17T17:27:37.446 INFO:journalctl@ceph.mon.c.smithi130.stdout: [mds.foo.smithi179.zkkvno{-1:24466} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.179:6832/449182576,v1:172.21.15.179:6833/449182576] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:27:37.446 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: mon.c@-1(???).osd e337 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-17T17:27:37.446 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: mon.c@-1(???).osd e337 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:27:37.446 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: mon.c@-1(???).osd e337 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:27:37.446 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: mon.c@-1(???).osd e337 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:27:37.446 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 ceph-mon[110068]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-08-17T17:27:37.446 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 bash[110030]: 4161e48dac506efae59e4ed2c12cfb1a94313f82cd32cac24d9a43b58a586648 2024-08-17T17:27:37.447 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:37 smithi130 systemd[1]: Started Ceph mon.c for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:27:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:42.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:42 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:27:42] ENGINE Bus STARTING 2024-08-17T17:27:42.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:42 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:27:42] ENGINE Serving on http://:::9283 2024-08-17T17:27:42.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:42 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:27:42] ENGINE Bus STARTED 2024-08-17T17:27:43.105 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:27:43.423 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap3 2024-08-17T17:27:43.509 INFO:tasks.workunit.client.0.smithi130.stdout:clone image180 -> image182 2024-08-17T17:27:43.559 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:27:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: mon.c calling monitor election 2024-08-17T17:27:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: mon.b calling monitor election 2024-08-17T17:27:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: mon.a calling monitor election 2024-08-17T17:27:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:27:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: monmap epoch 3 2024-08-17T17:27:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:27:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: last_changed 2024-08-17T17:16:41.007482+0000 2024-08-17T17:27:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: created 2024-08-17T17:15:06.618377+0000 2024-08-17T17:27:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: min_mon_release 17 (quincy) 2024-08-17T17:27:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: election_strategy: 1 2024-08-17T17:27:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-08-17T17:27:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: 1: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-08-17T17:27:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: 2: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-08-17T17:27:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:27:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: osdmap e338: 8 total, 8 up, 8 in 2024-08-17T17:27:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: mgrmap e40: y(active, since 91s), standbys: x 2024-08-17T17:27:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-08-17T17:27:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-08-17T17:27:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:27:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: application not enabled on pool 'test-librbd-smithi130-87148-9' 2024-08-17T17:27:43.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[110068]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:27:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: mon.c calling monitor election 2024-08-17T17:27:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: mon.b calling monitor election 2024-08-17T17:27:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: mon.a calling monitor election 2024-08-17T17:27:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:27:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: monmap epoch 3 2024-08-17T17:27:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:27:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: last_changed 2024-08-17T17:16:41.007482+0000 2024-08-17T17:27:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: created 2024-08-17T17:15:06.618377+0000 2024-08-17T17:27:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: min_mon_release 17 (quincy) 2024-08-17T17:27:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: election_strategy: 1 2024-08-17T17:27:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-08-17T17:27:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: 1: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-08-17T17:27:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: 2: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-08-17T17:27:43.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:27:43.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: osdmap e338: 8 total, 8 up, 8 in 2024-08-17T17:27:43.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: mgrmap e40: y(active, since 91s), standbys: x 2024-08-17T17:27:43.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-08-17T17:27:43.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-08-17T17:27:43.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:27:43.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: application not enabled on pool 'test-librbd-smithi130-87148-9' 2024-08-17T17:27:43.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:43 smithi130 ceph-mon[105521]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:27:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: mon.c calling monitor election 2024-08-17T17:27:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: mon.b calling monitor election 2024-08-17T17:27:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: mon.a calling monitor election 2024-08-17T17:27:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:27:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: monmap epoch 3 2024-08-17T17:27:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:27:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: last_changed 2024-08-17T17:16:41.007482+0000 2024-08-17T17:27:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: created 2024-08-17T17:15:06.618377+0000 2024-08-17T17:27:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: min_mon_release 17 (quincy) 2024-08-17T17:27:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: election_strategy: 1 2024-08-17T17:27:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-08-17T17:27:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: 1: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-08-17T17:27:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: 2: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-08-17T17:27:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:27:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: osdmap e338: 8 total, 8 up, 8 in 2024-08-17T17:27:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: mgrmap e40: y(active, since 91s), standbys: x 2024-08-17T17:27:43.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-08-17T17:27:43.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-08-17T17:27:43.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:27:43.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: application not enabled on pool 'test-librbd-smithi130-87148-9' 2024-08-17T17:27:43.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:43 smithi179 ceph-mon[34775]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:27:43.962 INFO:tasks.workunit.client.0.smithi130.stdout:write: a 3121958~4017797 2024-08-17T17:27:44.028 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 134070800~3653781 2024-08-17T17:27:44.060 INFO:tasks.workunit.client.0.smithi130.stdout:write: a 85671982~6947394 2024-08-17T17:27:44.118 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 15766258~8145615 2024-08-17T17:27:44.154 INFO:tasks.workunit.client.0.smithi130.stdout:write: a 75835844~3513550 2024-08-17T17:27:44.202 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 59713886~6995995 2024-08-17T17:27:44.231 INFO:tasks.workunit.client.0.smithi130.stdout:write: a 89771968~5276819 2024-08-17T17:27:44.259 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 138640323~2527562 2024-08-17T17:27:44.274 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: user0 2024-08-17T17:27:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:44 smithi179 ceph-mon[34775]: from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:44 smithi179 ceph-mon[34775]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:44 smithi179 ceph-mon[34775]: osdmap e339: 8 total, 8 up, 8 in 2024-08-17T17:27:44.848 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:44 smithi130 ceph-mon[105521]: from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:44.848 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:44 smithi130 ceph-mon[105521]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:44.848 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:44 smithi130 ceph-mon[105521]: osdmap e339: 8 total, 8 up, 8 in 2024-08-17T17:27:44.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:44 smithi130 ceph-mon[110068]: from='client.34209 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:44.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:44 smithi130 ceph-mon[110068]: from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:44.849 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:44 smithi130 ceph-mon[110068]: osdmap e339: 8 total, 8 up, 8 in 2024-08-17T17:27:45.441 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 279516200 2024-08-17T17:27:45.450 INFO:tasks.workunit.client.0.smithi130.stdout:write: b 47595267~2149736 2024-08-17T17:27:45.457 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 85118844~1396116 2024-08-17T17:27:45.457 INFO:tasks.workunit.client.0.smithi130.stdout:write: b 47657350~6123423 2024-08-17T17:27:45.510 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 210033451~96473 2024-08-17T17:27:45.510 INFO:tasks.workunit.client.0.smithi130.stdout:write: b 13124788~6667766 2024-08-17T17:27:45.515 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 182591420~3724163 2024-08-17T17:27:45.515 INFO:tasks.workunit.client.0.smithi130.stdout:write: b 192291336~521940 2024-08-17T17:27:45.516 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 239681145~6608626 2024-08-17T17:27:45.516 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: user1 2024-08-17T17:27:45.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:45 smithi179 ceph-mon[34775]: pgmap v133: 137 pgs: 137 active+clean; 1.5 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 532 KiB/s rd, 3.3 MiB/s wr, 38 op/s 2024-08-17T17:27:45.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:45 smithi179 ceph-mon[34775]: osdmap e340: 8 total, 8 up, 8 in 2024-08-17T17:27:45.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:45 smithi179 ceph-mon[34775]: mgrmap e41: y(active, since 92s), standbys: x 2024-08-17T17:27:45.759 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:45 smithi130 ceph-mon[105521]: pgmap v133: 137 pgs: 137 active+clean; 1.5 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 532 KiB/s rd, 3.3 MiB/s wr, 38 op/s 2024-08-17T17:27:45.759 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:45 smithi130 ceph-mon[105521]: osdmap e340: 8 total, 8 up, 8 in 2024-08-17T17:27:45.759 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:45 smithi130 ceph-mon[105521]: mgrmap e41: y(active, since 92s), standbys: x 2024-08-17T17:27:45.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:45 smithi130 ceph-mon[110068]: pgmap v133: 137 pgs: 137 active+clean; 1.5 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 532 KiB/s rd, 3.3 MiB/s wr, 38 op/s 2024-08-17T17:27:45.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:45 smithi130 ceph-mon[110068]: osdmap e340: 8 total, 8 up, 8 in 2024-08-17T17:27:45.759 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:45 smithi130 ceph-mon[110068]: mgrmap e41: y(active, since 92s), standbys: x 2024-08-17T17:27:45.790 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:27:46.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:46 smithi130 ceph-mon[105521]: osdmap e341: 8 total, 8 up, 8 in 2024-08-17T17:27:46.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:46 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2904621635' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:27:46.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:46 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:46.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:46 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:46 smithi130 ceph-mon[110068]: osdmap e341: 8 total, 8 up, 8 in 2024-08-17T17:27:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:46 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2904621635' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:27:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:46 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:46 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:46.842 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch ps' 2024-08-17T17:27:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:46 smithi179 ceph-mon[34775]: osdmap e341: 8 total, 8 up, 8 in 2024-08-17T17:27:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:46 smithi179 ceph-mon[34775]: from='client.? 172.21.15.130:0/2904621635' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:27:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:46 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:46 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:47.117 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:47.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:47.437 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:47 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:27:47.165+0000 7f8e6b27b640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-08-17T17:27:47.446 INFO:tasks.workunit.client.0.smithi130.stdout:write: c 103413239~5167413 2024-08-17T17:27:47.459 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 264888055~5300152 2024-08-17T17:27:47.459 INFO:tasks.workunit.client.0.smithi130.stdout:write: c 199961860~3259157 2024-08-17T17:27:47.462 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 232857443~2824194 2024-08-17T17:27:47.463 INFO:tasks.workunit.client.0.smithi130.stdout:write: c 53838514~595551 2024-08-17T17:27:47.463 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 81588771~503981 2024-08-17T17:27:47.464 INFO:tasks.workunit.client.0.smithi130.stdout:write: c 168336493~2005257 2024-08-17T17:27:47.466 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 226073492~5236699 2024-08-17T17:27:47.466 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: user2 2024-08-17T17:27:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:47 smithi179 ceph-mon[34775]: pgmap v136: 137 pgs: 137 active+clean; 1.5 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 767 B/s wr, 0 op/s 2024-08-17T17:27:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:47 smithi179 ceph-mon[34775]: osdmap e342: 8 total, 8 up, 8 in 2024-08-17T17:27:47.909 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:47 smithi130 ceph-mon[105521]: pgmap v136: 137 pgs: 137 active+clean; 1.5 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 767 B/s wr, 0 op/s 2024-08-17T17:27:47.909 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:47 smithi130 ceph-mon[105521]: osdmap e342: 8 total, 8 up, 8 in 2024-08-17T17:27:47.909 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:47 smithi130 ceph-mon[110068]: pgmap v136: 137 pgs: 137 active+clean; 1.5 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 767 B/s wr, 0 op/s 2024-08-17T17:27:47.909 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:47 smithi130 ceph-mon[110068]: osdmap e342: 8 total, 8 up, 8 in 2024-08-17T17:27:48.613 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:48 smithi130 ceph-mon[105521]: osdmap e343: 8 total, 8 up, 8 in 2024-08-17T17:27:48.613 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:48 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:48.613 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:48 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:48.613 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:48 smithi130 ceph-mon[110068]: osdmap e343: 8 total, 8 up, 8 in 2024-08-17T17:27:48.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:48 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:48.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:48 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:48 smithi179 ceph-mon[34775]: osdmap e343: 8 total, 8 up, 8 in 2024-08-17T17:27:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:48 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:48 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:49.040 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:27:49.040 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (3m) 2s ago 7m 20.8M - 0.27.0 11f11916f8cd 73ea4ebcf68b 2024-08-17T17:27:49.040 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi179 *:3000 running (2m) 22s ago 6m 75.1M - 9.4.7 954c08fa6188 399f89cc0037 2024-08-17T17:27:49.040 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foo.smithi130.jmaidg smithi130 running (5m) 2s ago 5m 19.4M - 17.2.7-1404-gf2c39d4d f69471c56189 91d7d1762094 2024-08-17T17:27:49.041 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foo.smithi179.zkkvno smithi179 running (5m) 22s ago 5m 13.7M - 17.2.7-1404-gf2c39d4d f69471c56189 0486517d425a 2024-08-17T17:27:49.041 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi179 *:8443,9283,8765 running (118s) 22s ago 10m 452M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb cdd6dde7c7af 2024-08-17T17:27:49.041 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (2m) 2s ago 12m 508M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 584426b3875b 2024-08-17T17:27:49.041 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (104s) 2s ago 12m 45.5M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 79749aebcefa 2024-08-17T17:27:49.041 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi179 running (11m) 22s ago 11m 60.7M 2048M 17.2.7-1404-gf2c39d4d f69471c56189 c29492fa5296 2024-08-17T17:27:49.041 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (12s) 2s ago 11m 28.4M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 4161e48dac50 2024-08-17T17:27:49.041 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (2m) 2s ago 6m 11.0M - 1.7.0 72c9c2088986 fd7392d87597 2024-08-17T17:27:49.041 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi179 *:9100 running (2m) 22s ago 6m 10.1M - 1.7.0 72c9c2088986 c10588f17ed8 2024-08-17T17:27:49.041 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (10m) 2s ago 10m 140M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 293583e22b78 2024-08-17T17:27:49.041 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (10m) 2s ago 10m 134M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 1dc0b4634e15 2024-08-17T17:27:49.041 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (9m) 2s ago 9m 112M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 d72289cb8b72 2024-08-17T17:27:49.042 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (9m) 2s ago 9m 124M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 1509c2840516 2024-08-17T17:27:49.042 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi179 running (9m) 22s ago 9m 115M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 254dc0763420 2024-08-17T17:27:49.042 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi179 running (8m) 22s ago 8m 92.2M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 078704aadcd7 2024-08-17T17:27:49.042 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi179 running (8m) 22s ago 8m 141M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 ab570ba67ae7 2024-08-17T17:27:49.042 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi179 running (7m) 22s ago 7m 104M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 99ae6a861fa0 2024-08-17T17:27:49.042 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi179 *:9095 running (2m) 22s ago 7m 46.3M - 2.51.0 1d3b7f56885b 145f77b6fc93 2024-08-17T17:27:49.477 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 212600431 2024-08-17T17:27:49.486 INFO:tasks.workunit.client.0.smithi130.stdout:write: d 7095475~5392299 2024-08-17T17:27:49.497 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 181604234~557264 2024-08-17T17:27:49.498 INFO:tasks.workunit.client.0.smithi130.stdout:write: d 147275380~5641718 2024-08-17T17:27:49.504 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 92152859~1046106 2024-08-17T17:27:49.504 INFO:tasks.workunit.client.0.smithi130.stdout:write: d 92139977~5190984 2024-08-17T17:27:49.562 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 81711861~7288052 2024-08-17T17:27:49.563 INFO:tasks.workunit.client.0.smithi130.stdout:write: d 168953786~5942603 2024-08-17T17:27:49.603 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 10441500~723704 2024-08-17T17:27:49.603 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: user3 2024-08-17T17:27:49.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:49 smithi130 ceph-mon[105521]: pgmap v139: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 28 MiB/s rd, 40 MiB/s wr, 663 op/s 2024-08-17T17:27:49.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:49 smithi130 ceph-mon[105521]: osdmap e344: 8 total, 8 up, 8 in 2024-08-17T17:27:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:49 smithi130 ceph-mon[110068]: pgmap v139: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 28 MiB/s rd, 40 MiB/s wr, 663 op/s 2024-08-17T17:27:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:49 smithi130 ceph-mon[110068]: osdmap e344: 8 total, 8 up, 8 in 2024-08-17T17:27:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:49 smithi179 ceph-mon[34775]: pgmap v139: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 28 MiB/s rd, 40 MiB/s wr, 663 op/s 2024-08-17T17:27:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:49 smithi179 ceph-mon[34775]: osdmap e344: 8 total, 8 up, 8 in 2024-08-17T17:27:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:27:49] "GET /metrics HTTP/1.1" 200 38767 "" "Prometheus/2.51.0" 2024-08-17T17:27:50.472 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:50.585 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:50 smithi130 ceph-mon[105521]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:50.585 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:50 smithi130 ceph-mon[105521]: osdmap e345: 8 total, 8 up, 8 in 2024-08-17T17:27:50.585 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:50 smithi130 ceph-mon[110068]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:50.585 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:50 smithi130 ceph-mon[110068]: osdmap e345: 8 total, 8 up, 8 in 2024-08-17T17:27:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:50 smithi179 ceph-mon[34775]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:50 smithi179 ceph-mon[34775]: osdmap e345: 8 total, 8 up, 8 in 2024-08-17T17:27:51.258 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade pause' 2024-08-17T17:27:51.493 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 21260043~5315010 2024-08-17T17:27:51.498 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 22588795~2657505 2024-08-17T17:27:51.556 INFO:tasks.workunit.client.0.smithi130.stdout:write: 2 42520086~5315010 2024-08-17T17:27:51.557 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 43848838~2657505 2024-08-17T17:27:51.600 INFO:tasks.workunit.client.0.smithi130.stdout:write: 3 63780129~5315010 2024-08-17T17:27:51.602 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 65108881~2657505 2024-08-17T17:27:51.663 INFO:tasks.workunit.client.0.smithi130.stdout:write: 4 85040172~5315010 2024-08-17T17:27:51.664 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 86368924~2657505 2024-08-17T17:27:51.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:51 smithi130 ceph-mon[110068]: pgmap v142: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 28 MiB/s rd, 40 MiB/s wr, 663 op/s 2024-08-17T17:27:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:51 smithi130 ceph-mon[110068]: osdmap e346: 8 total, 8 up, 8 in 2024-08-17T17:27:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:51 smithi130 ceph-mon[105521]: pgmap v142: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 28 MiB/s rd, 40 MiB/s wr, 663 op/s 2024-08-17T17:27:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:51 smithi130 ceph-mon[105521]: osdmap e346: 8 total, 8 up, 8 in 2024-08-17T17:27:51.700 INFO:tasks.workunit.client.0.smithi130.stdout:write: 5 106300215~5315010 2024-08-17T17:27:51.701 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 107628967~2657505 2024-08-17T17:27:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:51 smithi179 ceph-mon[34775]: pgmap v142: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 28 MiB/s rd, 40 MiB/s wr, 663 op/s 2024-08-17T17:27:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:51 smithi179 ceph-mon[34775]: osdmap e346: 8 total, 8 up, 8 in 2024-08-17T17:27:52.168 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:52.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[105521]: osdmap e347: 8 total, 8 up, 8 in 2024-08-17T17:27:52.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:52.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:52.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:27:52.673 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:27:52.673 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:52.673 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:27:52.673 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:27:52.673 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:27:52.673 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-17T17:27:52.673 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[110068]: osdmap e347: 8 total, 8 up, 8 in 2024-08-17T17:27:52.673 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:52.673 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:52.673 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:27:52.673 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:27:52.674 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:52.674 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:27:52.674 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:27:52.674 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:27:52.674 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:52 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-17T17:27:52.791 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:52 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:52.641Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.b\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.b\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", hostname=\"smithi179\", instance=\"ceph_cluster\", job=\"ceph\", public_addr=\"172.21.15.179\", rank=\"1\"}, {ceph_daemon=\"mon.b\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", hostname=\"smithi179\", instance=\"ceph_cluster\", job=\"ceph\", public_addr=\"172.21.15.179\", rank=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-08-17T17:27:52.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:52 smithi179 ceph-mon[34775]: osdmap e347: 8 total, 8 up, 8 in 2024-08-17T17:27:52.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:52 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:52.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:52 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:52.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:52 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:27:52.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:52 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:27:52.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:52 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:52.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:52 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:27:52.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:52 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:27:52.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:52 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:27:52.792 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:52 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-17T17:27:53.164 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:27:53.317 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:27:53.786 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:53 smithi130 ceph-mon[105521]: pgmap v145: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:27:53.786 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:53 smithi130 ceph-mon[105521]: Upgrade: It appears safe to stop mon.b 2024-08-17T17:27:53.786 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:53 smithi130 ceph-mon[110068]: pgmap v145: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:27:53.786 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:53 smithi130 ceph-mon[110068]: Upgrade: It appears safe to stop mon.b 2024-08-17T17:27:53.828 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:53 smithi179 ceph-mon[34775]: pgmap v145: 137 pgs: 137 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:27:53.828 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:53 smithi179 ceph-mon[34775]: Upgrade: It appears safe to stop mon.b 2024-08-17T17:27:54.300 INFO:teuthology.orchestra.run.smithi130.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:27:54.413 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:54.745 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.Stress2 (27069 ms) 2024-08-17T17:27:54.745 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMigration.StressLive 2024-08-17T17:27:54.775 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83886080 2024-08-17T17:27:54.785 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 31727312~3982861 2024-08-17T17:27:54.816 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 77456752~2719250 2024-08-17T17:27:54.816 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 11610389~7157039 2024-08-17T17:27:54.869 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 68964258~2463261 2024-08-17T17:27:54.870 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 47662771~5227038 2024-08-17T17:27:54.905 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 29661739~7092094 2024-08-17T17:27:54.907 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 57884533~1983708 2024-08-17T17:27:54.921 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 9346268~1499606 2024-08-17T17:27:54.923 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap0 2024-08-17T17:27:55.104 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[110068]: Upgrade: Updating mon.b 2024-08-17T17:27:55.104 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:55.104 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:27:55.104 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:27:55.105 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:27:55.105 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[110068]: Deploying daemon mon.b on smithi179 2024-08-17T17:27:55.105 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[110068]: pgmap v146: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 126 MiB/s rd, 24 MiB/s wr, 307 op/s 2024-08-17T17:27:55.105 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[110068]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:55.105 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[110068]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:27:55.105 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:55.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[105521]: Upgrade: Updating mon.b 2024-08-17T17:27:55.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:55.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:27:55.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:27:55.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:27:55.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[105521]: Deploying daemon mon.b on smithi179 2024-08-17T17:27:55.106 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[105521]: pgmap v146: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 126 MiB/s rd, 24 MiB/s wr, 307 op/s 2024-08-17T17:27:55.106 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[105521]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:55.106 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[105521]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:27:55.106 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:54 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:55.129 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'sleep 60' 2024-08-17T17:27:55.254 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:54 smithi179 ceph-mon[34775]: Upgrade: Updating mon.b 2024-08-17T17:27:55.254 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:54 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:55.254 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:54 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:27:55.255 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:54 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:27:55.255 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:54 smithi179 ceph-mon[34775]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:27:55.255 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:54 smithi179 ceph-mon[34775]: Deploying daemon mon.b on smithi179 2024-08-17T17:27:55.255 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:54 smithi179 ceph-mon[34775]: pgmap v146: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 126 MiB/s rd, 24 MiB/s wr, 307 op/s 2024-08-17T17:27:55.255 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:54 smithi179 ceph-mon[34775]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:27:55.255 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:54 smithi179 ceph-mon[34775]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:27:55.255 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:54 smithi179 ceph-mon[34775]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:27:55.382 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:55.819 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:55 smithi179 systemd[1]: Stopping Ceph mon.b for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:27:55.819 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:55 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:55.819 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:55 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:55 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b[34752]: 2024-08-17T17:27:55.817+0000 7effc13e6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:27:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:55 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b[34752]: 2024-08-17T17:27:55.817+0000 7effc13e6640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-08-17T17:27:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:55 smithi179 podman[93328]: 2024-08-17 17:27:55.941326942 +0000 UTC m=+0.255852968 container died c29492fa52969cad8679f5539b409950bb4ad88fe21ac4a60837d77ef150d1f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:27:56.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:55 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:27:55] ENGINE Bus STOPPING 2024-08-17T17:27:56.452 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:56 smithi179 podman[93328]: 2024-08-17 17:27:56.274714452 +0000 UTC m=+0.589240479 container cleanup c29492fa52969cad8679f5539b409950bb4ad88fe21ac4a60837d77ef150d1f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-08-17T17:27:56.452 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:56 smithi179 bash[93328]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b 2024-08-17T17:27:56.580 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:27:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:27:56.580 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:27:56] ENGINE Bus STOPPED 2024-08-17T17:27:56.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:56 smithi179 podman[93347]: 2024-08-17 17:27:56.451474938 +0000 UTC m=+0.506371692 container remove c29492fa52969cad8679f5539b409950bb4ad88fe21ac4a60837d77ef150d1f9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, ceph=True) 2024-08-17T17:27:56.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:56 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.b.service: Deactivated successfully. 2024-08-17T17:27:56.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:56 smithi179 systemd[1]: Stopped Ceph mon.b for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:27:56.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:56 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.b.service: Consumed 9.603s CPU time. 2024-08-17T17:27:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:56 smithi179 systemd[1]: Starting Ceph mon.b for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:27:57.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:27:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:27:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:27:57.373 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 podman[93481]: 2024-08-17 17:27:57.023167164 +0000 UTC m=+0.017346459 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:27:57.373 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 podman[93481]: 2024-08-17 17:27:57.161451228 +0000 UTC m=+0.155630509 container create d85c3c5c065e6a5ae87a14062c054cbf2b655411d64b7d52406952c31d908d64 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:27:57.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 podman[93481]: 2024-08-17 17:27:57.372537178 +0000 UTC m=+0.366716461 container init d85c3c5c065e6a5ae87a14062c054cbf2b655411d64b7d52406952c31d908d64 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe) 2024-08-17T17:27:57.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 podman[93481]: 2024-08-17 17:27:57.378140903 +0000 UTC m=+0.372320186 container start d85c3c5c065e6a5ae87a14062c054cbf2b655411d64b7d52406952c31d908d64 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True) 2024-08-17T17:27:57.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:27:57.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev), process ceph-mon, pid 7 2024-08-17T17:27:57.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: pidfile_write: ignore empty --pid-file 2024-08-17T17:27:57.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: load: jerasure load: lrc 2024-08-17T17:27:57.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: RocksDB version: 7.9.2 2024-08-17T17:27:57.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Git sha 0 2024-08-17T17:27:57.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Compile date 2024-08-15 20:43:31 2024-08-17T17:27:57.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: DB SUMMARY 2024-08-17T17:27:57.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: DB Session ID: 0UXZQZTX42OTM8IFXA57 2024-08-17T17:27:57.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:27:57.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:27:57.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1098 Bytes 2024-08-17T17:27:57.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-08-17T17:27:57.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 12785339 ; 2024-08-17T17:27:57.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:27:57.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:27:57.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:27:57.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T17:27:57.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:27:57.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T17:27:57.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.env: 0x564bd4fb8be0 2024-08-17T17:27:57.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T17:27:57.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.info_log: 0x564bd6d3f700 2024-08-17T17:27:57.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:27:57.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.statistics: (nil) 2024-08-17T17:27:57.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.use_fsync: 0 2024-08-17T17:27:57.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:27:57.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:27:57.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:27:57.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:27:57.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:27:57.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:27:57.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:27:57.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:27:57.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:27:57.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:27:57.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:27:57.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.db_log_dir: 2024-08-17T17:27:57.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.wal_dir: 2024-08-17T17:27:57.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:27:57.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:27:57.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:27:57.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:27:57.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:27:57.847 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:27:57.848 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:27:57.848 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:27:57.848 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.write_buffer_manager: 0x564bd6d42960 2024-08-17T17:27:57.848 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:27:57.848 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:27:57.848 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:27:57.848 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:27:57.848 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:27:57.848 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:27:57.848 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:27:57.848 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:27:57.848 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.unordered_write: 0 2024-08-17T17:27:57.848 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:27:57.849 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:27:57.849 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:27:57.849 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:27:57.849 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.row_cache: None 2024-08-17T17:27:57.849 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.wal_filter: None 2024-08-17T17:27:57.849 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:27:57.850 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:27:57.850 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:27:57.850 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:27:57.850 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.wal_compression: 0 2024-08-17T17:27:57.850 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:27:57.850 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:27:57.851 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:27:57.851 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:27:57.851 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:27:57.851 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:27:57.851 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:27:57.851 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:27:57.851 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:27:57.851 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:27:57.851 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:27:57.851 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T17:27:57.851 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:27:57.851 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:27:57.851 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:27:57.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:27:57.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:27:57.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:27:57.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:27:57.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:27:57.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:27:57.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:27:57.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:27:57.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_open_files: -1 2024-08-17T17:27:57.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:27:57.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:27:57.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:27:57.852 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:27:57.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:27:57.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Compression algorithms supported: 2024-08-17T17:27:57.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: kZSTD supported: 0 2024-08-17T17:27:57.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:27:57.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:27:57.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:27:57.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:27:57.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:27:57.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:27:57.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:27:57.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:27:57.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T17:27:57.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-08-17T17:27:57.853 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T17:27:57.854 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:27:57.854 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.merge_operator: 2024-08-17T17:27:57.854 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compaction_filter: None 2024-08-17T17:27:57.854 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:27:57.854 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:27:57.854 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:27:57.854 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:27:57.854 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564bd6d3f380) 2024-08-17T17:27:57.854 INFO:journalctl@ceph.mon.b.smithi179.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:27:57.854 INFO:journalctl@ceph.mon.b.smithi179.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:27:57.854 INFO:journalctl@ceph.mon.b.smithi179.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:27:57.854 INFO:journalctl@ceph.mon.b.smithi179.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:27:57.854 INFO:journalctl@ceph.mon.b.smithi179.stdout: index_type: 0 2024-08-17T17:27:57.855 INFO:journalctl@ceph.mon.b.smithi179.stdout: data_block_index_type: 0 2024-08-17T17:27:57.855 INFO:journalctl@ceph.mon.b.smithi179.stdout: index_shortening: 1 2024-08-17T17:27:57.855 INFO:journalctl@ceph.mon.b.smithi179.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:27:57.855 INFO:journalctl@ceph.mon.b.smithi179.stdout: checksum: 4 2024-08-17T17:27:57.855 INFO:journalctl@ceph.mon.b.smithi179.stdout: no_block_cache: 0 2024-08-17T17:27:57.855 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache: 0x564bd6d33610 2024-08-17T17:27:57.855 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:27:57.855 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache_options: 2024-08-17T17:27:57.855 INFO:journalctl@ceph.mon.b.smithi179.stdout: capacity : 536870912 2024-08-17T17:27:57.855 INFO:journalctl@ceph.mon.b.smithi179.stdout: num_shard_bits : 4 2024-08-17T17:27:57.855 INFO:journalctl@ceph.mon.b.smithi179.stdout: strict_capacity_limit : 0 2024-08-17T17:27:57.855 INFO:journalctl@ceph.mon.b.smithi179.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:27:57.855 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_cache_compressed: (nil) 2024-08-17T17:27:57.855 INFO:journalctl@ceph.mon.b.smithi179.stdout: persistent_cache: (nil) 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_size: 4096 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_size_deviation: 10 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_restart_interval: 16 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: index_block_restart_interval: 1 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: metadata_block_size: 4096 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: partition_filters: 0 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: use_delta_encoding: 1 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: filter_policy: bloomfilter 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: whole_key_filtering: 1 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: verify_compression: 0 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: format_version: 5 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: enable_index_compression: 1 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: block_align: 0 2024-08-17T17:27:57.856 INFO:journalctl@ceph.mon.b.smithi179.stdout: max_auto_readahead_size: 262144 2024-08-17T17:27:57.857 INFO:journalctl@ceph.mon.b.smithi179.stdout: prepopulate_block_cache: 0 2024-08-17T17:27:57.857 INFO:journalctl@ceph.mon.b.smithi179.stdout: initial_auto_readahead_size: 8192 2024-08-17T17:27:57.857 INFO:journalctl@ceph.mon.b.smithi179.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T17:27:57.857 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:27:57.857 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:27:57.857 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compression: NoCompression 2024-08-17T17:27:57.857 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:27:57.857 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:27:57.857 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:27:57.857 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.num_levels: 7 2024-08-17T17:27:57.857 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:27:57.857 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:27:57.857 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:27:57.857 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:27:57.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:27:57.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:27:57.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:27:57.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:27:57.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:27:57.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:27:57.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:27:57.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:27:57.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:27:57.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:27:57.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:27:57.858 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:27:57.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:27:57.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:27:57.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:27:57.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:27:57.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:27:57.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:27:57.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:27:57.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:27:57.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:27:57.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:27:57.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:27:57.859 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:27:57.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:27:57.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:27:57.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:27:57.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:27:57.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:27:57.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:27:57.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:27:57.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:27:57.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:27:57.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:27:57.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T17:27:57.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T17:27:57.860 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:27:57.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:27:57.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:27:57.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:27:57.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:27:57.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:27:57.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:27:57.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:27:57.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:27:57.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:27:57.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:27:57.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:27:57.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:27:57.861 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T17:27:57.862 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:27:57.862 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:27:57.862 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:27:57.862 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:27:57.862 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:27:57.862 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:27:57.862 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:27:57.862 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:27:57.862 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:27:57.862 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:27:57.862 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:27:57.863 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.ttl: 2592000 2024-08-17T17:27:57.863 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:27:57.863 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T17:27:57.863 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T17:27:57.863 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:27:57.863 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:27:57.863 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:27:57.863 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:27:57.863 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:27:57.863 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:27:57.863 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T17:27:57.863 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T17:27:57.863 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T17:27:57.864 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T17:27:57.864 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-08-17T17:27:57.864 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:27:57.864 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13572, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T17:27:57.864 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-08-17T17:27:57.864 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e7eabaef-696a-42cc-9c46-55891d45f0c8 2024-08-17T17:27:57.864 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723915677405202, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-08-17T17:27:57.864 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-08-17T17:27:57.864 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723915677437294, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 7986249, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13569, "largest_seqno": 17932, "table_properties": {"data_size": 7966564, "index_size": 14030, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4613, "raw_key_size": 40543, "raw_average_key_size": 22, "raw_value_size": 7927116, "raw_average_value_size": 4324, "num_data_blocks": 604, "num_entries": 1833, "num_filter_entries": 1833, "num_deletions": 257, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723915677, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e7eabaef-696a-42cc-9c46-55891d45f0c8", "db_session_id": "0UXZQZTX42OTM8IFXA57", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-08-17T17:27:57.864 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723915677437460, "job": 1, "event": "recovery_finished"} 2024-08-17T17:27:57.864 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-08-17T17:27:57.865 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:27:57.865 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:27:57.865 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x564bd6d62e00 2024-08-17T17:27:57.865 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: DB pointer 0x564bd6e60000 2024-08-17T17:27:57.865 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T17:27:57.865 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T17:27:57.865 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** DB Stats ** 2024-08-17T17:27:57.865 INFO:journalctl@ceph.mon.b.smithi179.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:27:57.865 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:27:57.865 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:27:57.865 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:27:57.866 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:27:57.866 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:27:57.866 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:27:57.866 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:27:57.866 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** Compaction Stats [default] ** 2024-08-17T17:27:57.866 INFO:journalctl@ceph.mon.b.smithi179.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:27:57.866 INFO:journalctl@ceph.mon.b.smithi179.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T17:27:57.866 INFO:journalctl@ceph.mon.b.smithi179.stdout: L0 1/0 7.62 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 547.3 0.01 0.00 1 0.014 0 0 0.0 0.0 2024-08-17T17:27:57.866 INFO:journalctl@ceph.mon.b.smithi179.stdout: L6 1/0 8.07 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-17T17:27:57.866 INFO:journalctl@ceph.mon.b.smithi179.stdout: Sum 2/0 15.69 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 547.3 0.01 0.00 1 0.014 0 0 0.0 0.0 2024-08-17T17:27:57.866 INFO:journalctl@ceph.mon.b.smithi179.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 547.3 0.01 0.00 1 0.014 0 0 0.0 0.0 2024-08-17T17:27:57.866 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:27:57.866 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** Compaction Stats [default] ** 2024-08-17T17:27:57.867 INFO:journalctl@ceph.mon.b.smithi179.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:27:57.867 INFO:journalctl@ceph.mon.b.smithi179.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:27:57.867 INFO:journalctl@ceph.mon.b.smithi179.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 547.3 0.01 0.00 1 0.014 0 0 0.0 0.0 2024-08-17T17:27:57.867 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:27:57.867 INFO:journalctl@ceph.mon.b.smithi179.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T17:27:57.867 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:27:57.867 INFO:journalctl@ceph.mon.b.smithi179.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:27:57.867 INFO:journalctl@ceph.mon.b.smithi179.stdout: Flush(GB): cumulative 0.007, interval 0.007 2024-08-17T17:27:57.867 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:27:57.867 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:27:57.867 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:27:57.867 INFO:journalctl@ceph.mon.b.smithi179.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:27:57.867 INFO:journalctl@ceph.mon.b.smithi179.stdout: Cumulative compaction: 0.01 GB write, 214.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:27:57.867 INFO:journalctl@ceph.mon.b.smithi179.stdout: Interval compaction: 0.01 GB write, 214.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:27:57.868 INFO:journalctl@ceph.mon.b.smithi179.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:27:57.868 INFO:journalctl@ceph.mon.b.smithi179.stdout: Block cache BinnedLRUCache@0x564bd6d33610#7 capacity: 512.00 MB usage: 21.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-08-17T17:27:57.868 INFO:journalctl@ceph.mon.b.smithi179.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,5.03 KB,0.000959635%) IndexBlock(1,16.08 KB,0.00306666%) Misc(1,0.00 KB,0%) 2024-08-17T17:27:57.868 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:27:57.868 INFO:journalctl@ceph.mon.b.smithi179.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:27:57.868 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: starting mon.b rank 1 at public addrs [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] at bind addrs [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:27:57.868 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: mon.b@-1(???) e3 preinit fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:27:57.868 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: mon.b@-1(???).mds e9 new map 2024-08-17T17:27:57.868 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: mon.b@-1(???).mds e9 print_map 2024-08-17T17:27:57.868 INFO:journalctl@ceph.mon.b.smithi179.stdout: e9 2024-08-17T17:27:57.868 INFO:journalctl@ceph.mon.b.smithi179.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-17T17:27:57.868 INFO:journalctl@ceph.mon.b.smithi179.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:27:57.869 INFO:journalctl@ceph.mon.b.smithi179.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:27:57.869 INFO:journalctl@ceph.mon.b.smithi179.stdout: legacy client fscid: 1 2024-08-17T17:27:57.869 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:27:57.869 INFO:journalctl@ceph.mon.b.smithi179.stdout: Filesystem 'foo' (1) 2024-08-17T17:27:57.869 INFO:journalctl@ceph.mon.b.smithi179.stdout: fs_name foo 2024-08-17T17:27:57.869 INFO:journalctl@ceph.mon.b.smithi179.stdout: epoch 8 2024-08-17T17:27:57.869 INFO:journalctl@ceph.mon.b.smithi179.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-17T17:27:57.869 INFO:journalctl@ceph.mon.b.smithi179.stdout: created 2024-08-17T17:21:52.146143+0000 2024-08-17T17:27:57.869 INFO:journalctl@ceph.mon.b.smithi179.stdout: modified 2024-08-17T17:22:05.340164+0000 2024-08-17T17:27:57.869 INFO:journalctl@ceph.mon.b.smithi179.stdout: tableserver 0 2024-08-17T17:27:57.869 INFO:journalctl@ceph.mon.b.smithi179.stdout: root 0 2024-08-17T17:27:57.869 INFO:journalctl@ceph.mon.b.smithi179.stdout: session_timeout 60 2024-08-17T17:27:57.869 INFO:journalctl@ceph.mon.b.smithi179.stdout: session_autoclose 300 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: max_file_size 1099511627776 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: max_xattr_size 65536 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: required_client_features {} 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: last_failure 0 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: last_failure_osd_epoch 0 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: max_mds 1 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: in 0 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: up {0=24548} 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: failed 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: damaged 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: stopped 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: data_pools [4] 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: metadata_pool 3 2024-08-17T17:27:57.870 INFO:journalctl@ceph.mon.b.smithi179.stdout: inline_data disabled 2024-08-17T17:27:57.871 INFO:journalctl@ceph.mon.b.smithi179.stdout: balancer 2024-08-17T17:27:57.871 INFO:journalctl@ceph.mon.b.smithi179.stdout: bal_rank_mask -1 2024-08-17T17:27:57.871 INFO:journalctl@ceph.mon.b.smithi179.stdout: standby_count_wanted 1 2024-08-17T17:27:57.871 INFO:journalctl@ceph.mon.b.smithi179.stdout: qdb_cluster leader: 0 members: 2024-08-17T17:27:57.871 INFO:journalctl@ceph.mon.b.smithi179.stdout: [mds.foo.smithi130.jmaidg{0:24548} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.130:6834/1214210479,v1:172.21.15.130:6835/1214210479] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:27:57.871 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:27:57.871 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:27:57.871 INFO:journalctl@ceph.mon.b.smithi179.stdout: Standby daemons: 2024-08-17T17:27:57.871 INFO:journalctl@ceph.mon.b.smithi179.stdout: 2024-08-17T17:27:57.871 INFO:journalctl@ceph.mon.b.smithi179.stdout: [mds.foo.smithi179.zkkvno{-1:24466} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.179:6832/449182576,v1:172.21.15.179:6833/449182576] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:27:57.871 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: mon.b@-1(???).osd e347 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-17T17:27:57.871 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: mon.b@-1(???).osd e347 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:27:57.871 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: mon.b@-1(???).osd e347 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:27:57.871 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: mon.b@-1(???).osd e347 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:27:57.872 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 ceph-mon[93519]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-08-17T17:27:57.872 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 bash[93481]: d85c3c5c065e6a5ae87a14062c054cbf2b655411d64b7d52406952c31d908d64 2024-08-17T17:27:57.872 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:57 smithi179 systemd[1]: Started Ceph mon.b for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:27:59.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:58 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:27:58] ENGINE Bus STARTING 2024-08-17T17:27:59.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:58 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:27:58] ENGINE Serving on http://:::9283 2024-08-17T17:27:59.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:58 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:27:58] ENGINE Bus STARTED 2024-08-17T17:28:00.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:27:59] "GET /metrics HTTP/1.1" 200 38691 "" "Prometheus/2.51.0" 2024-08-17T17:28:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: pgmap v147: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 107 MiB/s rd, 21 MiB/s wr, 260 op/s 2024-08-17T17:28:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:28:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: pgmap v148: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 196 MiB/s rd, 21 MiB/s wr, 504 op/s 2024-08-17T17:28:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: osdmap e348: 8 total, 8 up, 8 in 2024-08-17T17:28:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: mgrmap e41: y(active, since 106s), standbys: x 2024-08-17T17:28:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-08-17T17:28:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:28:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: application not enabled on pool 'test-librbd-smithi130-87148-9' 2024-08-17T17:28:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:28:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:28:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: mon.b calling monitor election 2024-08-17T17:28:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: mon.c calling monitor election 2024-08-17T17:28:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: mon.a calling monitor election 2024-08-17T17:28:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: monmap epoch 4 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: last_changed 2024-08-17T17:27:58.893364+0000 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: created 2024-08-17T17:15:06.618377+0000 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: min_mon_release 19 (squid) 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: election_strategy: 1 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: 1: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: 2: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: osdmap e348: 8 total, 8 up, 8 in 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: mgrmap e41: y(active, since 106s), standbys: x 2024-08-17T17:28:00.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-08-17T17:28:00.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:28:00.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: application not enabled on pool 'test-librbd-smithi130-87148-9' 2024-08-17T17:28:00.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:28:00.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='' 2024-08-17T17:28:00.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:00.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:28:00.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: pgmap v147: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 107 MiB/s rd, 21 MiB/s wr, 260 op/s 2024-08-17T17:28:00.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:28:00.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: pgmap v148: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 196 MiB/s rd, 21 MiB/s wr, 504 op/s 2024-08-17T17:28:00.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: osdmap e348: 8 total, 8 up, 8 in 2024-08-17T17:28:00.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: mgrmap e41: y(active, since 106s), standbys: x 2024-08-17T17:28:00.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-08-17T17:28:00.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:28:00.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: application not enabled on pool 'test-librbd-smithi130-87148-9' 2024-08-17T17:28:00.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:28:00.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:28:00.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: mon.b calling monitor election 2024-08-17T17:28:00.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: mon.c calling monitor election 2024-08-17T17:28:00.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: mon.a calling monitor election 2024-08-17T17:28:00.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:28:00.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:28:00.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:28:00.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: monmap epoch 4 2024-08-17T17:28:00.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:28:00.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: last_changed 2024-08-17T17:27:58.893364+0000 2024-08-17T17:28:00.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: created 2024-08-17T17:15:06.618377+0000 2024-08-17T17:28:00.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: min_mon_release 19 (squid) 2024-08-17T17:28:00.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: election_strategy: 1 2024-08-17T17:28:00.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-08-17T17:28:00.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: 1: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-08-17T17:28:00.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: 2: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-08-17T17:28:00.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:28:00.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: osdmap e348: 8 total, 8 up, 8 in 2024-08-17T17:28:00.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: mgrmap e41: y(active, since 106s), standbys: x 2024-08-17T17:28:00.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-08-17T17:28:00.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:28:00.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: application not enabled on pool 'test-librbd-smithi130-87148-9' 2024-08-17T17:28:00.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:28:00.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='' 2024-08-17T17:28:00.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:00.173 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:27:59 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:28:00.214 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: pgmap v147: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 107 MiB/s rd, 21 MiB/s wr, 260 op/s 2024-08-17T17:28:00.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:28:00.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: pgmap v148: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 196 MiB/s rd, 21 MiB/s wr, 504 op/s 2024-08-17T17:28:00.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: osdmap e348: 8 total, 8 up, 8 in 2024-08-17T17:28:00.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: mgrmap e41: y(active, since 106s), standbys: x 2024-08-17T17:28:00.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-08-17T17:28:00.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:28:00.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: application not enabled on pool 'test-librbd-smithi130-87148-9' 2024-08-17T17:28:00.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:28:00.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:28:00.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: mon.b calling monitor election 2024-08-17T17:28:00.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: mon.c calling monitor election 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: mon.a calling monitor election 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: monmap epoch 4 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: last_changed 2024-08-17T17:27:58.893364+0000 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: created 2024-08-17T17:15:06.618377+0000 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: min_mon_release 19 (squid) 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: election_strategy: 1 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: 0: [v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] mon.a 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: 1: [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] mon.b 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: 2: [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0] mon.c 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: fsmap foo:1 {0=foo.smithi130.jmaidg=up:active} 1 up:standby 2024-08-17T17:28:00.216 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: osdmap e348: 8 total, 8 up, 8 in 2024-08-17T17:28:00.217 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: mgrmap e41: y(active, since 106s), standbys: x 2024-08-17T17:28:00.217 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-08-17T17:28:00.217 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-08-17T17:28:00.217 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: application not enabled on pool 'test-librbd-smithi130-87148-9' 2024-08-17T17:28:00.217 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-08-17T17:28:00.217 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='' 2024-08-17T17:28:00.217 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:00.217 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:27:59 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:28:00.557 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:00.919 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap0 2024-08-17T17:28:00.932 INFO:tasks.workunit.client.0.smithi130.stdout:clone image183 -> image186 2024-08-17T17:28:01.022 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 98874066 2024-08-17T17:28:01.033 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 33863026~4390586 2024-08-17T17:28:01.069 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 91595262~7278804 2024-08-17T17:28:01.070 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 73844655~267115 2024-08-17T17:28:01.076 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 76222896~2171060 2024-08-17T17:28:01.081 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 78275633~1856498 2024-08-17T17:28:01.092 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 38210301~675258 2024-08-17T17:28:01.094 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 67746232~6877626 2024-08-17T17:28:01.132 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 19572287~4470805 2024-08-17T17:28:01.158 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap1 2024-08-17T17:28:01.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:00 smithi130 ceph-mon[105521]: osdmap e349: 8 total, 8 up, 8 in 2024-08-17T17:28:01.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:00 smithi130 ceph-mon[105521]: pgmap v151: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 196 MiB/s rd, 21 MiB/s wr, 504 op/s 2024-08-17T17:28:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:00 smithi130 ceph-mon[110068]: osdmap e349: 8 total, 8 up, 8 in 2024-08-17T17:28:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:00 smithi130 ceph-mon[110068]: pgmap v151: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 196 MiB/s rd, 21 MiB/s wr, 504 op/s 2024-08-17T17:28:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:00 smithi179 ceph-mon[93519]: osdmap e349: 8 total, 8 up, 8 in 2024-08-17T17:28:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:00 smithi179 ceph-mon[93519]: pgmap v151: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 196 MiB/s rd, 21 MiB/s wr, 504 op/s 2024-08-17T17:28:02.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:01 smithi130 ceph-mon[105521]: osdmap e350: 8 total, 8 up, 8 in 2024-08-17T17:28:02.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:01 smithi130 ceph-mon[110068]: osdmap e350: 8 total, 8 up, 8 in 2024-08-17T17:28:02.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:01 smithi179 ceph-mon[93519]: osdmap e350: 8 total, 8 up, 8 in 2024-08-17T17:28:02.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:28:02 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:28:02.448+0000 7f8e6b27b640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-08-17T17:28:02.939 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap1 2024-08-17T17:28:02.950 INFO:tasks.workunit.client.0.smithi130.stdout:clone image186 -> image188 2024-08-17T17:28:03.044 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:02 smithi179 ceph-mon[93519]: osdmap e351: 8 total, 8 up, 8 in 2024-08-17T17:28:03.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:02 smithi179 ceph-mon[93519]: pgmap v154: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:28:03.064 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 17364264~2911560 2024-08-17T17:28:03.105 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 54036187~4531017 2024-08-17T17:28:03.144 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 20319854~6564058 2024-08-17T17:28:03.185 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 8075107~2226392 2024-08-17T17:28:03.211 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 46530~2137930 2024-08-17T17:28:03.234 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 81708038~5449877 2024-08-17T17:28:03.245 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 97305004~1394823 2024-08-17T17:28:03.258 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 66162195~827136 2024-08-17T17:28:03.264 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap2 2024-08-17T17:28:03.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:02 smithi130 ceph-mon[105521]: osdmap e351: 8 total, 8 up, 8 in 2024-08-17T17:28:03.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:02 smithi130 ceph-mon[105521]: pgmap v154: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:28:03.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:02 smithi130 ceph-mon[110068]: osdmap e351: 8 total, 8 up, 8 in 2024-08-17T17:28:03.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:02 smithi130 ceph-mon[110068]: pgmap v154: 137 pgs: 137 active+clean; 1.8 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:28:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:03 smithi179 ceph-mon[93519]: osdmap e352: 8 total, 8 up, 8 in 2024-08-17T17:28:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:03 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:03 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:04.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:04.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:03 smithi130 ceph-mon[105521]: osdmap e352: 8 total, 8 up, 8 in 2024-08-17T17:28:04.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:03 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:04.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:03 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:04.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:03 smithi130 ceph-mon[110068]: osdmap e352: 8 total, 8 up, 8 in 2024-08-17T17:28:04.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:03 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:04.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:03 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:04.962 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap2 2024-08-17T17:28:04.974 INFO:tasks.workunit.client.0.smithi130.stdout:clone image188 -> image190 2024-08-17T17:28:05.114 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 125018864 2024-08-17T17:28:05.125 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 51391729~4815879 2024-08-17T17:28:05.181 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 61283316~8319767 2024-08-17T17:28:05.217 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 82851978~7979227 2024-08-17T17:28:05.271 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 10878645~6199663 2024-08-17T17:28:05.309 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 95109530~6153519 2024-08-17T17:28:05.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:04 smithi179 ceph-mon[93519]: osdmap e353: 8 total, 8 up, 8 in 2024-08-17T17:28:05.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:04 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:05.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:04 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:05.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:04 smithi179 ceph-mon[93519]: pgmap v157: 137 pgs: 137 active+clean; 1.9 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.4 MiB/s wr, 219 op/s 2024-08-17T17:28:05.346 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 106120794~4443935 2024-08-17T17:28:05.346 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 32846955~1170914 2024-08-17T17:28:05.385 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 35492238~1599907 2024-08-17T17:28:05.387 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: snap3 2024-08-17T17:28:05.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:04 smithi130 ceph-mon[105521]: osdmap e353: 8 total, 8 up, 8 in 2024-08-17T17:28:05.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:04 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:05.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:04 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:05.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:04 smithi130 ceph-mon[105521]: pgmap v157: 137 pgs: 137 active+clean; 1.9 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.4 MiB/s wr, 219 op/s 2024-08-17T17:28:05.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:04 smithi130 ceph-mon[110068]: osdmap e353: 8 total, 8 up, 8 in 2024-08-17T17:28:05.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:04 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:05.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:04 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:05.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:04 smithi130 ceph-mon[110068]: pgmap v157: 137 pgs: 137 active+clean; 1.9 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.4 MiB/s wr, 219 op/s 2024-08-17T17:28:05.808 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:05.808 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:06.078 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:05 smithi179 ceph-mon[93519]: osdmap e354: 8 total, 8 up, 8 in 2024-08-17T17:28:06.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:05 smithi130 ceph-mon[105521]: osdmap e354: 8 total, 8 up, 8 in 2024-08-17T17:28:06.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:05 smithi130 ceph-mon[110068]: osdmap e354: 8 total, 8 up, 8 in 2024-08-17T17:28:06.971 INFO:tasks.workunit.client.0.smithi130.stdout:snap_protect: snap3 2024-08-17T17:28:06.983 INFO:tasks.workunit.client.0.smithi130.stdout:clone image190 -> image192 2024-08-17T17:28:07.144 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 183041782 2024-08-17T17:28:07.156 INFO:tasks.workunit.client.0.smithi130.stdout:migration_prepare 2024-08-17T17:28:07.265 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:06 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:07.265 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:06 smithi179 ceph-mon[93519]: osdmap e355: 8 total, 8 up, 8 in 2024-08-17T17:28:07.265 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:06 smithi179 ceph-mon[93519]: pgmap v160: 137 pgs: 137 active+clean; 1.9 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.4 MiB/s wr, 219 op/s 2024-08-17T17:28:07.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:06 smithi130 ceph-mon[105521]: osdmap e355: 8 total, 8 up, 8 in 2024-08-17T17:28:07.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:06 smithi130 ceph-mon[105521]: pgmap v160: 137 pgs: 137 active+clean; 1.9 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.4 MiB/s wr, 219 op/s 2024-08-17T17:28:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:06 smithi130 ceph-mon[110068]: osdmap e355: 8 total, 8 up, 8 in 2024-08-17T17:28:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:06 smithi130 ceph-mon[110068]: pgmap v160: 137 pgs: 137 active+clean; 1.9 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 1.1 MiB/s rd, 7.4 MiB/s wr, 219 op/s 2024-08-17T17:28:07.678 INFO:tasks.workunit.client.0.smithi130.stdout:migration_execute 2024-08-17T17:28:07.678 INFO:tasks.workunit.client.0.smithi130.stdout:write: a 61395141~7819684 2024-08-17T17:28:07.758 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 21658170~6824042 2024-08-17T17:28:07.797 INFO:tasks.workunit.client.0.smithi130.stdout:write: a 176647981~5335641 2024-08-17T17:28:07.837 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 154799489~6332810 2024-08-17T17:28:07.840 INFO:tasks.workunit.client.0.smithi130.stdout:write: a 47711799~115277 2024-08-17T17:28:07.868 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 69821583~7257028 2024-08-17T17:28:07.898 INFO:tasks.workunit.client.0.smithi130.stdout:write: a 13710360~5404900 2024-08-17T17:28:07.956 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 74000351~6509704 2024-08-17T17:28:07.996 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: user0 2024-08-17T17:28:08.104 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[105521]: osdmap e356: 8 total, 8 up, 8 in 2024-08-17T17:28:08.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:08.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:08.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:08.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:28:08.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:08.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[105521]: Reconfiguring mon.a (monmap changed)... 2024-08-17T17:28:08.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:28:08.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:28:08.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:08.105 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[105521]: Reconfiguring daemon mon.a on smithi130 2024-08-17T17:28:08.106 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[110068]: osdmap e356: 8 total, 8 up, 8 in 2024-08-17T17:28:08.106 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:08.106 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:08.106 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:08.106 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:28:08.106 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:08.106 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[110068]: Reconfiguring mon.a (monmap changed)... 2024-08-17T17:28:08.106 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:28:08.106 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:28:08.106 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:08.106 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:07 smithi130 ceph-mon[110068]: Reconfiguring daemon mon.a on smithi130 2024-08-17T17:28:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:07 smithi179 ceph-mon[93519]: osdmap e356: 8 total, 8 up, 8 in 2024-08-17T17:28:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:07 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:07 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:07 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:07 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:28:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:07 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:07 smithi179 ceph-mon[93519]: Reconfiguring mon.a (monmap changed)... 2024-08-17T17:28:08.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:07 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:28:08.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:07 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:28:08.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:07 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:08.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:07 smithi179 ceph-mon[93519]: Reconfiguring daemon mon.a on smithi130 2024-08-17T17:28:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:08 smithi179 ceph-mon[93519]: pgmap v162: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 45 MiB/s rd, 45 MiB/s wr, 1.27k op/s 2024-08-17T17:28:09.392 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:08 smithi130 ceph-mon[105521]: pgmap v162: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 45 MiB/s rd, 45 MiB/s wr, 1.27k op/s 2024-08-17T17:28:09.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:08 smithi130 ceph-mon[110068]: pgmap v162: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 45 MiB/s rd, 45 MiB/s wr, 1.27k op/s 2024-08-17T17:28:09.986 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 310839262 2024-08-17T17:28:09.998 INFO:tasks.workunit.client.0.smithi130.stdout:write: b 169209331~2537992 2024-08-17T17:28:10.004 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 16653030~5957330 2024-08-17T17:28:10.004 INFO:tasks.workunit.client.0.smithi130.stdout:write: b 212741420~257094 2024-08-17T17:28:10.004 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 238593251~1968260 2024-08-17T17:28:10.004 INFO:tasks.workunit.client.0.smithi130.stdout:write: b 269513484~4626127 2024-08-17T17:28:10.007 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 215658026~6264342 2024-08-17T17:28:10.007 INFO:tasks.workunit.client.0.smithi130.stdout:write: b 146458690~7021848 2024-08-17T17:28:10.017 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 97271205~2851911 2024-08-17T17:28:10.019 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: user1 2024-08-17T17:28:10.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:28:09] "GET /metrics HTTP/1.1" 200 38691 "" "Prometheus/2.51.0" 2024-08-17T17:28:10.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[105521]: osdmap e357: 8 total, 8 up, 8 in 2024-08-17T17:28:10.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:10.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:10.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[105521]: Reconfiguring mgr.y (monmap changed)... 2024-08-17T17:28:10.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:28:10.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:28:10.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:28:10.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:10.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[105521]: Reconfiguring daemon mgr.y on smithi130 2024-08-17T17:28:10.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[110068]: osdmap e357: 8 total, 8 up, 8 in 2024-08-17T17:28:10.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:10.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:10.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[110068]: Reconfiguring mgr.y (monmap changed)... 2024-08-17T17:28:10.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:28:10.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:28:10.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:28:10.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:10.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:09 smithi130 ceph-mon[110068]: Reconfiguring daemon mgr.y on smithi130 2024-08-17T17:28:10.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:10.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:09 smithi179 ceph-mon[93519]: osdmap e357: 8 total, 8 up, 8 in 2024-08-17T17:28:10.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:09 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:10.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:09 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:10.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:09 smithi179 ceph-mon[93519]: Reconfiguring mgr.y (monmap changed)... 2024-08-17T17:28:10.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:09 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:28:10.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:09 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:28:10.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:09 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:28:10.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:09 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:10.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:09 smithi179 ceph-mon[93519]: Reconfiguring daemon mgr.y on smithi130 2024-08-17T17:28:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:10 smithi130 ceph-mon[105521]: osdmap e358: 8 total, 8 up, 8 in 2024-08-17T17:28:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:10 smithi130 ceph-mon[105521]: pgmap v165: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 45 MiB/s rd, 46 MiB/s wr, 1.28k op/s 2024-08-17T17:28:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:10 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:10 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:10 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:28:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:10 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:28:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:10 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:11.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:10 smithi130 ceph-mon[110068]: osdmap e358: 8 total, 8 up, 8 in 2024-08-17T17:28:11.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:10 smithi130 ceph-mon[110068]: pgmap v165: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 45 MiB/s rd, 46 MiB/s wr, 1.28k op/s 2024-08-17T17:28:11.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:10 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:11.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:10 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:11.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:10 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:28:11.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:10 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:28:11.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:10 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:10 smithi179 ceph-mon[93519]: osdmap e358: 8 total, 8 up, 8 in 2024-08-17T17:28:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:10 smithi179 ceph-mon[93519]: pgmap v165: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 45 MiB/s rd, 46 MiB/s wr, 1.28k op/s 2024-08-17T17:28:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:10 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:10 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:10 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:28:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:10 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:28:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:10 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:11.999 INFO:tasks.workunit.client.0.smithi130.stdout:write: c 128812830~8187553 2024-08-17T17:28:12.017 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 206366782~131966 2024-08-17T17:28:12.018 INFO:tasks.workunit.client.0.smithi130.stdout:write: c 32432355~5402817 2024-08-17T17:28:12.025 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 25993983~4830143 2024-08-17T17:28:12.026 INFO:tasks.workunit.client.0.smithi130.stdout:write: c 76729099~6125888 2024-08-17T17:28:12.035 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 34175585~1713851 2024-08-17T17:28:12.115 INFO:tasks.workunit.client.0.smithi130.stdout:write: c 49545480~8376934 2024-08-17T17:28:12.127 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 146796429~5002209 2024-08-17T17:28:12.127 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: user2 2024-08-17T17:28:12.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:11 smithi130 ceph-mon[105521]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:28:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:11 smithi130 ceph-mon[105521]: Reconfiguring daemon mon.c on smithi130 2024-08-17T17:28:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:11 smithi130 ceph-mon[105521]: osdmap e359: 8 total, 8 up, 8 in 2024-08-17T17:28:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:11 smithi130 ceph-mon[110068]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:28:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:11 smithi130 ceph-mon[110068]: Reconfiguring daemon mon.c on smithi130 2024-08-17T17:28:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:11 smithi130 ceph-mon[110068]: osdmap e359: 8 total, 8 up, 8 in 2024-08-17T17:28:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:11 smithi179 ceph-mon[93519]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:28:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:11 smithi179 ceph-mon[93519]: Reconfiguring daemon mon.c on smithi130 2024-08-17T17:28:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:11 smithi179 ceph-mon[93519]: osdmap e359: 8 total, 8 up, 8 in 2024-08-17T17:28:13.256 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:12 smithi179 ceph-mon[93519]: pgmap v167: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 36 MiB/s rd, 37 MiB/s wr, 1.03k op/s 2024-08-17T17:28:13.256 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:12 smithi179 ceph-mon[93519]: osdmap e360: 8 total, 8 up, 8 in 2024-08-17T17:28:13.257 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:13.257 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:13.257 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:13.257 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:12 smithi179 ceph-mon[93519]: Reconfiguring osd.0 (monmap changed)... 2024-08-17T17:28:13.257 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:28:13.257 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:13.257 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:12 smithi179 ceph-mon[93519]: Reconfiguring daemon osd.0 on smithi130 2024-08-17T17:28:13.408 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[105521]: pgmap v167: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 36 MiB/s rd, 37 MiB/s wr, 1.03k op/s 2024-08-17T17:28:13.408 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[105521]: osdmap e360: 8 total, 8 up, 8 in 2024-08-17T17:28:13.408 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:13.408 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:13.408 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:13.408 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[105521]: Reconfiguring osd.0 (monmap changed)... 2024-08-17T17:28:13.408 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:28:13.408 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:13.408 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[105521]: Reconfiguring daemon osd.0 on smithi130 2024-08-17T17:28:13.409 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[110068]: pgmap v167: 137 pgs: 137 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 36 MiB/s rd, 37 MiB/s wr, 1.03k op/s 2024-08-17T17:28:13.409 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[110068]: osdmap e360: 8 total, 8 up, 8 in 2024-08-17T17:28:13.409 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:13.409 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:13.409 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:13.409 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[110068]: Reconfiguring osd.0 (monmap changed)... 2024-08-17T17:28:13.409 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:28:13.409 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:13.409 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:12 smithi130 ceph-mon[110068]: Reconfiguring daemon osd.0 on smithi130 2024-08-17T17:28:14.013 INFO:tasks.workunit.client.0.smithi130.stdout:write: d 3064889~5799555 2024-08-17T17:28:14.028 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 250115915~2112604 2024-08-17T17:28:14.028 INFO:tasks.workunit.client.0.smithi130.stdout:write: d 29139959~231572 2024-08-17T17:28:14.028 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 147719952~2761066 2024-08-17T17:28:14.028 INFO:tasks.workunit.client.0.smithi130.stdout:write: d 201799366~5214481 2024-08-17T17:28:14.037 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 233176510~3802725 2024-08-17T17:28:14.039 INFO:tasks.workunit.client.0.smithi130.stdout:write: d 179847427~6851589 2024-08-17T17:28:14.050 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 145435326~7865337 2024-08-17T17:28:14.148 INFO:tasks.workunit.client.0.smithi130.stdout:snap_create: user3 2024-08-17T17:28:14.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:14 smithi130 ceph-mon[105521]: osdmap e361: 8 total, 8 up, 8 in 2024-08-17T17:28:14.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:14 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:28:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:14 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:14.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:14 smithi130 ceph-mon[110068]: osdmap e361: 8 total, 8 up, 8 in 2024-08-17T17:28:14.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:14.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:14.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:14 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:28:14.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:14 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:14.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:14 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:14.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:14 smithi179 ceph-mon[93519]: osdmap e361: 8 total, 8 up, 8 in 2024-08-17T17:28:14.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:14.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:14.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:14 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:28:14.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:14 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:15.266 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:15 smithi130 ceph-mon[105521]: Reconfiguring osd.1 (monmap changed)... 2024-08-17T17:28:15.266 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:15 smithi130 ceph-mon[105521]: Reconfiguring daemon osd.1 on smithi130 2024-08-17T17:28:15.267 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:15 smithi130 ceph-mon[105521]: pgmap v170: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 7.1 MiB/s rd, 29 MiB/s wr, 255 op/s 2024-08-17T17:28:15.267 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:15 smithi130 ceph-mon[105521]: osdmap e362: 8 total, 8 up, 8 in 2024-08-17T17:28:15.267 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:15 smithi130 ceph-mon[110068]: Reconfiguring osd.1 (monmap changed)... 2024-08-17T17:28:15.267 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:15 smithi130 ceph-mon[110068]: Reconfiguring daemon osd.1 on smithi130 2024-08-17T17:28:15.267 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:15 smithi130 ceph-mon[110068]: pgmap v170: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 7.1 MiB/s rd, 29 MiB/s wr, 255 op/s 2024-08-17T17:28:15.267 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:15 smithi130 ceph-mon[110068]: osdmap e362: 8 total, 8 up, 8 in 2024-08-17T17:28:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:15 smithi179 ceph-mon[93519]: Reconfiguring osd.1 (monmap changed)... 2024-08-17T17:28:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:15 smithi179 ceph-mon[93519]: Reconfiguring daemon osd.1 on smithi130 2024-08-17T17:28:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:15 smithi179 ceph-mon[93519]: pgmap v170: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 7.1 MiB/s rd, 29 MiB/s wr, 255 op/s 2024-08-17T17:28:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:15 smithi179 ceph-mon[93519]: osdmap e362: 8 total, 8 up, 8 in 2024-08-17T17:28:16.013 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:16.014 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:15.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:16.033 INFO:tasks.workunit.client.0.smithi130.stdout:write: 1 31083926~7770981 2024-08-17T17:28:16.041 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 33026671~3885490 2024-08-17T17:28:16.077 INFO:tasks.workunit.client.0.smithi130.stdout:write: 2 62167852~7770981 2024-08-17T17:28:16.086 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 64110597~3885490 2024-08-17T17:28:16.120 INFO:tasks.workunit.client.0.smithi130.stdout:write: 3 93251778~7770981 2024-08-17T17:28:16.130 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 95194523~3885490 2024-08-17T17:28:16.176 INFO:tasks.workunit.client.0.smithi130.stdout:write: 4 124335704~7770981 2024-08-17T17:28:16.185 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 126278449~3885490 2024-08-17T17:28:16.217 INFO:tasks.workunit.client.0.smithi130.stdout:write: 5 155419631~7770981 2024-08-17T17:28:16.226 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 157362376~3885490 2024-08-17T17:28:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:16 smithi179 ceph-mon[93519]: osdmap e363: 8 total, 8 up, 8 in 2024-08-17T17:28:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:16 smithi179 ceph-mon[93519]: Reconfiguring osd.2 (monmap changed)... 2024-08-17T17:28:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:28:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:16 smithi179 ceph-mon[93519]: Reconfiguring daemon osd.2 on smithi130 2024-08-17T17:28:16.375 INFO:tasks.workunit.client.0.smithi130.stdout:migration_commit 2024-08-17T17:28:16.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:16 smithi130 ceph-mon[105521]: osdmap e363: 8 total, 8 up, 8 in 2024-08-17T17:28:16.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:16.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:16.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:16 smithi130 ceph-mon[105521]: Reconfiguring osd.2 (monmap changed)... 2024-08-17T17:28:16.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:28:16.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:16.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:16 smithi130 ceph-mon[105521]: Reconfiguring daemon osd.2 on smithi130 2024-08-17T17:28:16.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:16 smithi130 ceph-mon[110068]: osdmap e363: 8 total, 8 up, 8 in 2024-08-17T17:28:16.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:16.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:16.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:16 smithi130 ceph-mon[110068]: Reconfiguring osd.2 (monmap changed)... 2024-08-17T17:28:16.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:28:16.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:16.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:16 smithi130 ceph-mon[110068]: Reconfiguring daemon osd.2 on smithi130 2024-08-17T17:28:17.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:17 smithi130 ceph-mon[105521]: pgmap v173: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 7.2 MiB/s rd, 29 MiB/s wr, 256 op/s 2024-08-17T17:28:17.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:17 smithi130 ceph-mon[105521]: osdmap e364: 8 total, 8 up, 8 in 2024-08-17T17:28:17.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:17 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:17.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:17 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:17.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:17 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:28:17.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:17 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:17.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:17 smithi130 ceph-mon[110068]: pgmap v173: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 7.2 MiB/s rd, 29 MiB/s wr, 256 op/s 2024-08-17T17:28:17.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:17 smithi130 ceph-mon[110068]: osdmap e364: 8 total, 8 up, 8 in 2024-08-17T17:28:17.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:17 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:17.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:17 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:17.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:17 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:28:17.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:17 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:17.345 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:17.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:17 smithi179 ceph-mon[93519]: pgmap v173: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 7.2 MiB/s rd, 29 MiB/s wr, 256 op/s 2024-08-17T17:28:17.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:17 smithi179 ceph-mon[93519]: osdmap e364: 8 total, 8 up, 8 in 2024-08-17T17:28:17.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:17 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:17.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:17 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:17.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:17 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:28:17.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:17 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:17.993 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMigration.StressLive (23248 ms) 2024-08-17T17:28:18.334 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:18 smithi130 ceph-mon[105521]: Reconfiguring osd.3 (monmap changed)... 2024-08-17T17:28:18.334 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:18 smithi130 ceph-mon[105521]: Reconfiguring daemon osd.3 on smithi130 2024-08-17T17:28:18.334 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:18 smithi130 ceph-mon[110068]: Reconfiguring osd.3 (monmap changed)... 2024-08-17T17:28:18.334 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:18 smithi130 ceph-mon[110068]: Reconfiguring daemon osd.3 on smithi130 2024-08-17T17:28:18.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:18 smithi179 ceph-mon[93519]: Reconfiguring osd.3 (monmap changed)... 2024-08-17T17:28:18.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:18 smithi179 ceph-mon[93519]: Reconfiguring daemon osd.3 on smithi130 2024-08-17T17:28:19.032 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 57 tests from TestMigration (275627 ms total) 2024-08-17T17:28:19.032 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:28:19.032 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 6 tests from TestGroup 2024-08-17T17:28:19.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[105521]: pgmap v175: 137 pgs: 137 active+clean; 2.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 152 MiB/s rd, 24 MiB/s wr, 504 op/s 2024-08-17T17:28:19.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[105521]: osdmap e365: 8 total, 8 up, 8 in 2024-08-17T17:28:19.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:19.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:19.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[105521]: Reconfiguring mds.foo.smithi130.jmaidg (monmap changed)... 2024-08-17T17:28:19.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:28:19.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:28:19.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:19.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[105521]: Reconfiguring daemon mds.foo.smithi130.jmaidg on smithi130 2024-08-17T17:28:19.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[110068]: pgmap v175: 137 pgs: 137 active+clean; 2.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 152 MiB/s rd, 24 MiB/s wr, 504 op/s 2024-08-17T17:28:19.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[110068]: osdmap e365: 8 total, 8 up, 8 in 2024-08-17T17:28:19.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:19.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:19.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[110068]: Reconfiguring mds.foo.smithi130.jmaidg (monmap changed)... 2024-08-17T17:28:19.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:28:19.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:28:19.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:19.056 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-mon[110068]: Reconfiguring daemon mds.foo.smithi130.jmaidg on smithi130 2024-08-17T17:28:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:19 smithi179 ceph-mon[93519]: pgmap v175: 137 pgs: 137 active+clean; 2.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 152 MiB/s rd, 24 MiB/s wr, 504 op/s 2024-08-17T17:28:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:19 smithi179 ceph-mon[93519]: osdmap e365: 8 total, 8 up, 8 in 2024-08-17T17:28:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:19 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:19 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:19 smithi179 ceph-mon[93519]: Reconfiguring mds.foo.smithi130.jmaidg (monmap changed)... 2024-08-17T17:28:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:19 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:28:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:19 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi130.jmaidg", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:28:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:19 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:19.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:19 smithi179 ceph-mon[93519]: Reconfiguring daemon mds.foo.smithi130.jmaidg on smithi130 2024-08-17T17:28:20.033 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:28:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:28:19] "GET /metrics HTTP/1.1" 200 38759 "" "Prometheus/2.51.0" 2024-08-17T17:28:20.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:20.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:20 smithi179 ceph-mon[93519]: osdmap e366: 8 total, 8 up, 8 in 2024-08-17T17:28:20.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:20 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:20.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:20 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:20.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:20 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:28:20.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:20 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:28:20.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:20 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:20.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:20 smithi130 ceph-mon[105521]: osdmap e366: 8 total, 8 up, 8 in 2024-08-17T17:28:20.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:20 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:20.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:20 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:20.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:20 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:28:20.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:20 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:28:20.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:20 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:20.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:20 smithi130 ceph-mon[110068]: osdmap e366: 8 total, 8 up, 8 in 2024-08-17T17:28:20.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:20 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:20.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:20 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:20.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:20 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:28:20.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:20 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:28:20.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:20 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:21.044 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestGroup.group_create 2024-08-17T17:28:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:21 smithi179 ceph-mon[93519]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:28:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:21 smithi179 ceph-mon[93519]: Reconfiguring daemon mon.b on smithi179 2024-08-17T17:28:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:21 smithi179 ceph-mon[93519]: pgmap v178: 73 pgs: 73 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:28:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:21 smithi179 ceph-mon[93519]: osdmap e367: 8 total, 8 up, 8 in 2024-08-17T17:28:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:21 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2987911254' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:28:21.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:21 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:28:21.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:21 smithi179 ceph-mon[93519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:28:21.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:21 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T17:28:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[105521]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:28:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[105521]: Reconfiguring daemon mon.b on smithi179 2024-08-17T17:28:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[105521]: pgmap v178: 73 pgs: 73 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:28:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[105521]: osdmap e367: 8 total, 8 up, 8 in 2024-08-17T17:28:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2987911254' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:28:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:28:21.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[105521]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:28:21.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T17:28:21.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[110068]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:28:21.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[110068]: Reconfiguring daemon mon.b on smithi179 2024-08-17T17:28:21.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[110068]: pgmap v178: 73 pgs: 73 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:28:21.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[110068]: osdmap e367: 8 total, 8 up, 8 in 2024-08-17T17:28:21.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2987911254' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:28:21.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:28:21.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[110068]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:28:21.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:21 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T17:28:22.182 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:22 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:28:22.182 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:22 smithi179 ceph-mon[93519]: osdmap e368: 8 total, 8 up, 8 in 2024-08-17T17:28:22.183 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:22 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:22.183 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:22 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:22.183 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:22 smithi179 ceph-mon[93519]: Reconfiguring mgr.x (monmap changed)... 2024-08-17T17:28:22.183 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:22 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:28:22.183 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:22 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:28:22.183 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:22 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:28:22.183 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:22 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:22.183 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:22 smithi179 ceph-mon[93519]: Reconfiguring daemon mgr.x on smithi179 2024-08-17T17:28:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:28:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[105521]: osdmap e368: 8 total, 8 up, 8 in 2024-08-17T17:28:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:22.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:22.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[105521]: Reconfiguring mgr.x (monmap changed)... 2024-08-17T17:28:22.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:28:22.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:28:22.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:28:22.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:22.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[105521]: Reconfiguring daemon mgr.x on smithi179 2024-08-17T17:28:22.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:28:22.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[110068]: osdmap e368: 8 total, 8 up, 8 in 2024-08-17T17:28:22.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:22.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:22.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[110068]: Reconfiguring mgr.x (monmap changed)... 2024-08-17T17:28:22.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:28:22.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:28:22.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:28:22.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:22.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:22 smithi130 ceph-mon[110068]: Reconfiguring daemon mgr.x on smithi179 2024-08-17T17:28:23.092 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestGroup.group_create (2048 ms) 2024-08-17T17:28:23.093 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestGroup.group_createPP 2024-08-17T17:28:23.109 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestGroup.group_createPP (17 ms) 2024-08-17T17:28:23.109 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestGroup.add_image 2024-08-17T17:28:23.145 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestGroup.add_image (36 ms) 2024-08-17T17:28:23.145 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestGroup.add_imagePP 2024-08-17T17:28:23.178 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestGroup.add_imagePP (33 ms) 2024-08-17T17:28:23.178 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestGroup.add_snapshot 2024-08-17T17:28:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:23 smithi179 ceph-mon[93519]: pgmap v181: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:28:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:23 smithi179 ceph-mon[93519]: osdmap e369: 8 total, 8 up, 8 in 2024-08-17T17:28:23.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:23 smithi130 ceph-mon[105521]: pgmap v181: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:28:23.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:23 smithi130 ceph-mon[105521]: osdmap e369: 8 total, 8 up, 8 in 2024-08-17T17:28:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:23 smithi130 ceph-mon[110068]: pgmap v181: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2024-08-17T17:28:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:23 smithi130 ceph-mon[110068]: osdmap e369: 8 total, 8 up, 8 in 2024-08-17T17:28:24.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:24 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:24.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:24 smithi179 ceph-mon[93519]: osdmap e370: 8 total, 8 up, 8 in 2024-08-17T17:28:24.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:24 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:24.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:24 smithi179 ceph-mon[93519]: Reconfiguring osd.4 (monmap changed)... 2024-08-17T17:28:24.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:24 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-17T17:28:24.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:24 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:24.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:24 smithi179 ceph-mon[93519]: Reconfiguring daemon osd.4 on smithi179 2024-08-17T17:28:24.344 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:24 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:24 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:24 smithi130 ceph-mon[105521]: osdmap e370: 8 total, 8 up, 8 in 2024-08-17T17:28:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:24 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:24 smithi130 ceph-mon[105521]: Reconfiguring osd.4 (monmap changed)... 2024-08-17T17:28:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:24 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-17T17:28:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:24 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:24 smithi130 ceph-mon[105521]: Reconfiguring daemon osd.4 on smithi179 2024-08-17T17:28:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:24 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:24.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:24 smithi130 ceph-mon[110068]: osdmap e370: 8 total, 8 up, 8 in 2024-08-17T17:28:24.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:24 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:24.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:24 smithi130 ceph-mon[110068]: Reconfiguring osd.4 (monmap changed)... 2024-08-17T17:28:24.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:24 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-17T17:28:24.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:24 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:24.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:24 smithi130 ceph-mon[110068]: Reconfiguring daemon osd.4 on smithi179 2024-08-17T17:28:25.265 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:25 smithi179 ceph-mon[93519]: pgmap v184: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:28:25.265 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:25 smithi179 ceph-mon[93519]: osdmap e371: 8 total, 8 up, 8 in 2024-08-17T17:28:25.265 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:25 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:25.265 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:25 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:25.265 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:25 smithi179 ceph-mon[93519]: Reconfiguring osd.5 (monmap changed)... 2024-08-17T17:28:25.265 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:25 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-17T17:28:25.265 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:25 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:25.266 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:25 smithi179 ceph-mon[93519]: Reconfiguring daemon osd.5 on smithi179 2024-08-17T17:28:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[105521]: pgmap v184: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:28:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[105521]: osdmap e371: 8 total, 8 up, 8 in 2024-08-17T17:28:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[105521]: Reconfiguring osd.5 (monmap changed)... 2024-08-17T17:28:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-17T17:28:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[105521]: Reconfiguring daemon osd.5 on smithi179 2024-08-17T17:28:25.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[110068]: pgmap v184: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:28:25.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[110068]: osdmap e371: 8 total, 8 up, 8 in 2024-08-17T17:28:25.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:25.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:25.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[110068]: Reconfiguring osd.5 (monmap changed)... 2024-08-17T17:28:25.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-17T17:28:25.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:25.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:25 smithi130 ceph-mon[110068]: Reconfiguring daemon osd.5 on smithi179 2024-08-17T17:28:26.003 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:25 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:26.003 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:25 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:26.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:26 smithi179 ceph-mon[93519]: osdmap e372: 8 total, 8 up, 8 in 2024-08-17T17:28:26.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:26 smithi130 ceph-mon[105521]: osdmap e372: 8 total, 8 up, 8 in 2024-08-17T17:28:26.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:26 smithi130 ceph-mon[110068]: osdmap e372: 8 total, 8 up, 8 in 2024-08-17T17:28:27.089 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:27 smithi179 ceph-mon[93519]: pgmap v187: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:28:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:27 smithi179 ceph-mon[93519]: osdmap e373: 8 total, 8 up, 8 in 2024-08-17T17:28:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:27 smithi179 ceph-mon[93519]: Reconfiguring osd.6 (monmap changed)... 2024-08-17T17:28:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-17T17:28:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:27 smithi179 ceph-mon[93519]: Reconfiguring daemon osd.6 on smithi179 2024-08-17T17:28:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[105521]: pgmap v187: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:28:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[105521]: osdmap e373: 8 total, 8 up, 8 in 2024-08-17T17:28:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[105521]: Reconfiguring osd.6 (monmap changed)... 2024-08-17T17:28:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-17T17:28:27.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:27.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[105521]: Reconfiguring daemon osd.6 on smithi179 2024-08-17T17:28:27.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[110068]: pgmap v187: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-08-17T17:28:27.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[110068]: osdmap e373: 8 total, 8 up, 8 in 2024-08-17T17:28:27.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:27.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:27.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[110068]: Reconfiguring osd.6 (monmap changed)... 2024-08-17T17:28:27.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-17T17:28:27.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:27.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:27 smithi130 ceph-mon[110068]: Reconfiguring daemon osd.6 on smithi179 2024-08-17T17:28:28.317 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:28 smithi179 ceph-mon[93519]: osdmap e374: 8 total, 8 up, 8 in 2024-08-17T17:28:28.318 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:28.318 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:28.318 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:28.318 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:28.318 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:28 smithi179 ceph-mon[93519]: Reconfiguring osd.7 (monmap changed)... 2024-08-17T17:28:28.318 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-17T17:28:28.318 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:28.318 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:28 smithi179 ceph-mon[93519]: Reconfiguring daemon osd.7 on smithi179 2024-08-17T17:28:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[105521]: osdmap e374: 8 total, 8 up, 8 in 2024-08-17T17:28:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[105521]: Reconfiguring osd.7 (monmap changed)... 2024-08-17T17:28:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-17T17:28:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[105521]: Reconfiguring daemon osd.7 on smithi179 2024-08-17T17:28:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[110068]: osdmap e374: 8 total, 8 up, 8 in 2024-08-17T17:28:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[110068]: Reconfiguring osd.7 (monmap changed)... 2024-08-17T17:28:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-17T17:28:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:28 smithi130 ceph-mon[110068]: Reconfiguring daemon osd.7 on smithi179 2024-08-17T17:28:29.150 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestGroup.add_snapshot (5972 ms) 2024-08-17T17:28:29.150 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestGroup.add_snapshotPP 2024-08-17T17:28:29.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:29 smithi179 ceph-mon[93519]: pgmap v190: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 18 KiB/s wr, 221 op/s 2024-08-17T17:28:29.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:29 smithi179 ceph-mon[93519]: osdmap e375: 8 total, 8 up, 8 in 2024-08-17T17:28:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:29 smithi130 ceph-mon[105521]: pgmap v190: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 18 KiB/s wr, 221 op/s 2024-08-17T17:28:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:29 smithi130 ceph-mon[105521]: osdmap e375: 8 total, 8 up, 8 in 2024-08-17T17:28:29.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:29 smithi130 ceph-mon[110068]: pgmap v190: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 18 KiB/s wr, 221 op/s 2024-08-17T17:28:29.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:29 smithi130 ceph-mon[110068]: osdmap e375: 8 total, 8 up, 8 in 2024-08-17T17:28:29.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:28:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:28:29] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.51.0" 2024-08-17T17:28:30.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[105521]: osdmap e376: 8 total, 8 up, 8 in 2024-08-17T17:28:30.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:30.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:30.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[105521]: Reconfiguring mds.foo.smithi179.zkkvno (monmap changed)... 2024-08-17T17:28:30.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:28:30.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:28:30.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:30.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[105521]: Reconfiguring daemon mds.foo.smithi179.zkkvno on smithi179 2024-08-17T17:28:30.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[110068]: osdmap e376: 8 total, 8 up, 8 in 2024-08-17T17:28:30.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:30.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:30.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[110068]: Reconfiguring mds.foo.smithi179.zkkvno (monmap changed)... 2024-08-17T17:28:30.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:28:30.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:28:30.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:30.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:30 smithi130 ceph-mon[110068]: Reconfiguring daemon mds.foo.smithi179.zkkvno on smithi179 2024-08-17T17:28:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:30 smithi179 ceph-mon[93519]: osdmap e376: 8 total, 8 up, 8 in 2024-08-17T17:28:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:30 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:30 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:30 smithi179 ceph-mon[93519]: Reconfiguring mds.foo.smithi179.zkkvno (monmap changed)... 2024-08-17T17:28:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:30 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:28:30.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:30 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi179.zkkvno", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:28:30.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:30 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:30.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:30 smithi179 ceph-mon[93519]: Reconfiguring daemon mds.foo.smithi179.zkkvno on smithi179 2024-08-17T17:28:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:31 smithi179 ceph-mon[93519]: pgmap v193: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 18 KiB/s wr, 221 op/s 2024-08-17T17:28:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:31 smithi179 ceph-mon[93519]: osdmap e377: 8 total, 8 up, 8 in 2024-08-17T17:28:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:31 smithi179 ceph-mon[93519]: osdmap e378: 8 total, 8 up, 8 in 2024-08-17T17:28:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:31 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:31 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:31 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:28:31.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:31 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:31.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:31 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:28:31.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:31 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[105521]: pgmap v193: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 18 KiB/s wr, 221 op/s 2024-08-17T17:28:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[105521]: osdmap e377: 8 total, 8 up, 8 in 2024-08-17T17:28:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[105521]: osdmap e378: 8 total, 8 up, 8 in 2024-08-17T17:28:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:28:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:28:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[110068]: pgmap v193: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 164 KiB/s rd, 18 KiB/s wr, 221 op/s 2024-08-17T17:28:31.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[110068]: osdmap e377: 8 total, 8 up, 8 in 2024-08-17T17:28:31.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[110068]: osdmap e378: 8 total, 8 up, 8 in 2024-08-17T17:28:31.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:31.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:31.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:28:31.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:28:31.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:28:31.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:31 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:32.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:32 smithi130 ceph-mon[105521]: osdmap e379: 8 total, 8 up, 8 in 2024-08-17T17:28:32.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:32 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:32 smithi130 ceph-mon[110068]: osdmap e379: 8 total, 8 up, 8 in 2024-08-17T17:28:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:32 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:32 smithi179 ceph-mon[93519]: osdmap e379: 8 total, 8 up, 8 in 2024-08-17T17:28:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:32 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:33 smithi130 ceph-mon[105521]: pgmap v197: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:33 smithi130 ceph-mon[105521]: osdmap e380: 8 total, 8 up, 8 in 2024-08-17T17:28:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:33 smithi130 ceph-mon[110068]: pgmap v197: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:33 smithi130 ceph-mon[110068]: osdmap e380: 8 total, 8 up, 8 in 2024-08-17T17:28:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:33 smithi179 ceph-mon[93519]: pgmap v197: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:33 smithi179 ceph-mon[93519]: osdmap e380: 8 total, 8 up, 8 in 2024-08-17T17:28:34.391 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestGroup.add_snapshotPP (5241 ms) 2024-08-17T17:28:34.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:34 smithi179 ceph-mon[93519]: osdmap e381: 8 total, 8 up, 8 in 2024-08-17T17:28:34.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:34 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:34.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:34 smithi130 ceph-mon[105521]: osdmap e381: 8 total, 8 up, 8 in 2024-08-17T17:28:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:34 smithi130 ceph-mon[110068]: osdmap e381: 8 total, 8 up, 8 in 2024-08-17T17:28:35.347 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 6 tests from TestGroup (13347 ms total) 2024-08-17T17:28:35.347 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:28:35.347 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 31 tests from TestDeepCopy 2024-08-17T17:28:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:35 smithi130 ceph-mon[110068]: pgmap v200: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 13 KiB/s wr, 170 op/s 2024-08-17T17:28:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:35 smithi130 ceph-mon[110068]: osdmap e382: 8 total, 8 up, 8 in 2024-08-17T17:28:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:35 smithi130 ceph-mon[110068]: osdmap e383: 8 total, 8 up, 8 in 2024-08-17T17:28:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:35 smithi130 ceph-mon[105521]: pgmap v200: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 13 KiB/s wr, 170 op/s 2024-08-17T17:28:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:35 smithi130 ceph-mon[105521]: osdmap e382: 8 total, 8 up, 8 in 2024-08-17T17:28:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:35 smithi130 ceph-mon[105521]: osdmap e383: 8 total, 8 up, 8 in 2024-08-17T17:28:35.740 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:35 smithi179 ceph-mon[93519]: pgmap v200: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 13 KiB/s wr, 170 op/s 2024-08-17T17:28:35.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:35 smithi179 ceph-mon[93519]: osdmap e382: 8 total, 8 up, 8 in 2024-08-17T17:28:35.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:35 smithi179 ceph-mon[93519]: osdmap e383: 8 total, 8 up, 8 in 2024-08-17T17:28:36.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:36.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:37.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:36 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:37.378 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Empty 2024-08-17T17:28:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:37 smithi130 ceph-mon[110068]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:37 smithi130 ceph-mon[110068]: osdmap e384: 8 total, 8 up, 8 in 2024-08-17T17:28:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:37 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1703614349' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:28:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:37 smithi130 ceph-mon[105521]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:37 smithi130 ceph-mon[105521]: osdmap e384: 8 total, 8 up, 8 in 2024-08-17T17:28:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:37 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1703614349' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:28:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:37 smithi179 ceph-mon[93519]: pgmap v203: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:37 smithi179 ceph-mon[93519]: osdmap e384: 8 total, 8 up, 8 in 2024-08-17T17:28:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:37 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1703614349' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:28:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:38 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1703614349' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:28:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:38 smithi130 ceph-mon[105521]: osdmap e385: 8 total, 8 up, 8 in 2024-08-17T17:28:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:38 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1703614349' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:28:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:38 smithi130 ceph-mon[110068]: osdmap e385: 8 total, 8 up, 8 in 2024-08-17T17:28:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:38 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1703614349' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:28:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:38 smithi179 ceph-mon[93519]: osdmap e385: 8 total, 8 up, 8 in 2024-08-17T17:28:39.511 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Empty (2133 ms) 2024-08-17T17:28:39.512 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.NoSnaps 2024-08-17T17:28:39.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:39 smithi130 ceph-mon[105521]: pgmap v206: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:39 smithi130 ceph-mon[105521]: osdmap e386: 8 total, 8 up, 8 in 2024-08-17T17:28:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:39 smithi130 ceph-mon[110068]: pgmap v206: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:39 smithi130 ceph-mon[110068]: osdmap e386: 8 total, 8 up, 8 in 2024-08-17T17:28:39.765 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.NoSnaps (253 ms) 2024-08-17T17:28:39.765 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Snaps 2024-08-17T17:28:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:39 smithi179 ceph-mon[93519]: pgmap v206: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:39 smithi179 ceph-mon[93519]: osdmap e386: 8 total, 8 up, 8 in 2024-08-17T17:28:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:28:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:28:39] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.51.0" 2024-08-17T17:28:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:40 smithi179 ceph-mon[93519]: osdmap e387: 8 total, 8 up, 8 in 2024-08-17T17:28:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:40 smithi130 ceph-mon[105521]: osdmap e387: 8 total, 8 up, 8 in 2024-08-17T17:28:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:40 smithi130 ceph-mon[110068]: osdmap e387: 8 total, 8 up, 8 in 2024-08-17T17:28:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:41 smithi130 ceph-mon[105521]: pgmap v209: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:41 smithi130 ceph-mon[105521]: osdmap e388: 8 total, 8 up, 8 in 2024-08-17T17:28:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:41 smithi130 ceph-mon[110068]: pgmap v209: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:41 smithi130 ceph-mon[110068]: osdmap e388: 8 total, 8 up, 8 in 2024-08-17T17:28:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:41 smithi179 ceph-mon[93519]: pgmap v209: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:41 smithi179 ceph-mon[93519]: osdmap e388: 8 total, 8 up, 8 in 2024-08-17T17:28:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:42 smithi130 ceph-mon[110068]: osdmap e389: 8 total, 8 up, 8 in 2024-08-17T17:28:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:42 smithi130 ceph-mon[105521]: osdmap e389: 8 total, 8 up, 8 in 2024-08-17T17:28:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:42 smithi179 ceph-mon[93519]: osdmap e389: 8 total, 8 up, 8 in 2024-08-17T17:28:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:43.691 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Snaps (3926 ms) 2024-08-17T17:28:43.691 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.SnapDiscard 2024-08-17T17:28:43.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:43 smithi179 ceph-mon[93519]: pgmap v212: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:43.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:43 smithi179 ceph-mon[93519]: osdmap e390: 8 total, 8 up, 8 in 2024-08-17T17:28:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:43 smithi130 ceph-mon[105521]: pgmap v212: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:43 smithi130 ceph-mon[105521]: osdmap e390: 8 total, 8 up, 8 in 2024-08-17T17:28:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:43 smithi130 ceph-mon[110068]: pgmap v212: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:43 smithi130 ceph-mon[110068]: osdmap e390: 8 total, 8 up, 8 in 2024-08-17T17:28:44.422 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:44.760 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:44 smithi179 ceph-mon[93519]: osdmap e391: 8 total, 8 up, 8 in 2024-08-17T17:28:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:44 smithi130 ceph-mon[105521]: osdmap e391: 8 total, 8 up, 8 in 2024-08-17T17:28:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:44 smithi130 ceph-mon[110068]: osdmap e391: 8 total, 8 up, 8 in 2024-08-17T17:28:45.448 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.SnapDiscard (1757 ms) 2024-08-17T17:28:45.448 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.CloneDiscard 2024-08-17T17:28:45.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:45 smithi179 ceph-mon[93519]: pgmap v215: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-08-17T17:28:45.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:45 smithi179 ceph-mon[93519]: osdmap e392: 8 total, 8 up, 8 in 2024-08-17T17:28:45.741 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:45 smithi179 ceph-mon[93519]: osdmap e393: 8 total, 8 up, 8 in 2024-08-17T17:28:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:45 smithi130 ceph-mon[110068]: pgmap v215: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-08-17T17:28:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:45 smithi130 ceph-mon[110068]: osdmap e392: 8 total, 8 up, 8 in 2024-08-17T17:28:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:45 smithi130 ceph-mon[110068]: osdmap e393: 8 total, 8 up, 8 in 2024-08-17T17:28:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:45 smithi130 ceph-mon[105521]: pgmap v215: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-08-17T17:28:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:45 smithi130 ceph-mon[105521]: osdmap e392: 8 total, 8 up, 8 in 2024-08-17T17:28:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:45 smithi130 ceph-mon[105521]: osdmap e393: 8 total, 8 up, 8 in 2024-08-17T17:28:46.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:45 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:46.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:45 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:46.606 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.CloneDiscard (1158 ms) 2024-08-17T17:28:46.606 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.CloneShrink 2024-08-17T17:28:47.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:47.600 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.CloneShrink (994 ms) 2024-08-17T17:28:47.601 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.CloneExpand 2024-08-17T17:28:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:47 smithi179 ceph-mon[93519]: pgmap v218: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-08-17T17:28:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:47 smithi179 ceph-mon[93519]: osdmap e394: 8 total, 8 up, 8 in 2024-08-17T17:28:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:47 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:47 smithi130 ceph-mon[110068]: pgmap v218: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-08-17T17:28:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:47 smithi130 ceph-mon[110068]: osdmap e394: 8 total, 8 up, 8 in 2024-08-17T17:28:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:47 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:47 smithi130 ceph-mon[105521]: pgmap v218: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 176 op/s 2024-08-17T17:28:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:47 smithi130 ceph-mon[105521]: osdmap e394: 8 total, 8 up, 8 in 2024-08-17T17:28:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:47 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:28:48.667 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.CloneExpand (1067 ms) 2024-08-17T17:28:48.668 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.CloneHideParent 2024-08-17T17:28:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:48 smithi179 ceph-mon[93519]: osdmap e395: 8 total, 8 up, 8 in 2024-08-17T17:28:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:48 smithi130 ceph-mon[105521]: osdmap e395: 8 total, 8 up, 8 in 2024-08-17T17:28:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:48 smithi130 ceph-mon[110068]: osdmap e395: 8 total, 8 up, 8 in 2024-08-17T17:28:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:49 smithi130 ceph-mon[105521]: pgmap v221: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 437 op/s 2024-08-17T17:28:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:49 smithi130 ceph-mon[105521]: osdmap e396: 8 total, 8 up, 8 in 2024-08-17T17:28:49.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:49 smithi130 ceph-mon[110068]: pgmap v221: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 437 op/s 2024-08-17T17:28:49.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:49 smithi130 ceph-mon[110068]: osdmap e396: 8 total, 8 up, 8 in 2024-08-17T17:28:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:49 smithi179 ceph-mon[93519]: pgmap v221: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 437 op/s 2024-08-17T17:28:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:49 smithi179 ceph-mon[93519]: osdmap e396: 8 total, 8 up, 8 in 2024-08-17T17:28:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:28:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:28:49] "GET /metrics HTTP/1.1" 200 36195 "" "Prometheus/2.51.0" 2024-08-17T17:28:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:50 smithi179 ceph-mon[93519]: osdmap e397: 8 total, 8 up, 8 in 2024-08-17T17:28:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:50 smithi130 ceph-mon[105521]: osdmap e397: 8 total, 8 up, 8 in 2024-08-17T17:28:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:50 smithi130 ceph-mon[110068]: osdmap e397: 8 total, 8 up, 8 in 2024-08-17T17:28:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:51 smithi179 ceph-mon[93519]: pgmap v224: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 437 op/s 2024-08-17T17:28:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:51 smithi179 ceph-mon[93519]: osdmap e398: 8 total, 8 up, 8 in 2024-08-17T17:28:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:51 smithi130 ceph-mon[105521]: pgmap v224: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 437 op/s 2024-08-17T17:28:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:51 smithi130 ceph-mon[105521]: osdmap e398: 8 total, 8 up, 8 in 2024-08-17T17:28:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:51 smithi130 ceph-mon[110068]: pgmap v224: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 437 op/s 2024-08-17T17:28:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:51 smithi130 ceph-mon[110068]: osdmap e398: 8 total, 8 up, 8 in 2024-08-17T17:28:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:52 smithi179 ceph-mon[93519]: osdmap e399: 8 total, 8 up, 8 in 2024-08-17T17:28:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:52 smithi130 ceph-mon[105521]: osdmap e399: 8 total, 8 up, 8 in 2024-08-17T17:28:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:52 smithi130 ceph-mon[110068]: osdmap e399: 8 total, 8 up, 8 in 2024-08-17T17:28:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:53 smithi179 ceph-mon[93519]: pgmap v227: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:53 smithi179 ceph-mon[93519]: osdmap e400: 8 total, 8 up, 8 in 2024-08-17T17:28:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:53 smithi130 ceph-mon[105521]: pgmap v227: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:53 smithi130 ceph-mon[105521]: osdmap e400: 8 total, 8 up, 8 in 2024-08-17T17:28:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:53 smithi130 ceph-mon[110068]: pgmap v227: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:28:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:53 smithi130 ceph-mon[110068]: osdmap e400: 8 total, 8 up, 8 in 2024-08-17T17:28:54.486 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:54 smithi179 ceph-mon[93519]: osdmap e401: 8 total, 8 up, 8 in 2024-08-17T17:28:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:54 smithi130 ceph-mon[105521]: osdmap e401: 8 total, 8 up, 8 in 2024-08-17T17:28:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:54 smithi130 ceph-mon[110068]: osdmap e401: 8 total, 8 up, 8 in 2024-08-17T17:28:55.600 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.CloneHideParent (6933 ms) 2024-08-17T17:28:55.600 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Clone 2024-08-17T17:28:55.740 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:55 smithi179 ceph-mon[93519]: pgmap v230: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-08-17T17:28:55.740 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:55 smithi179 ceph-mon[93519]: osdmap e402: 8 total, 8 up, 8 in 2024-08-17T17:28:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:55 smithi130 ceph-mon[105521]: pgmap v230: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-08-17T17:28:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:55 smithi130 ceph-mon[105521]: osdmap e402: 8 total, 8 up, 8 in 2024-08-17T17:28:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:55 smithi130 ceph-mon[110068]: pgmap v230: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-08-17T17:28:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:55 smithi130 ceph-mon[110068]: osdmap e402: 8 total, 8 up, 8 in 2024-08-17T17:28:55.992 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:55 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:55.992 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:55 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:56 smithi179 ceph-mon[93519]: osdmap e403: 8 total, 8 up, 8 in 2024-08-17T17:28:56.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:56 smithi130 ceph-mon[105521]: osdmap e403: 8 total, 8 up, 8 in 2024-08-17T17:28:56.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:56 smithi130 ceph-mon[110068]: osdmap e403: 8 total, 8 up, 8 in 2024-08-17T17:28:57.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:28:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:28:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:28:57.832 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Clone (2232 ms) 2024-08-17T17:28:57.833 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.CloneFlatten 2024-08-17T17:28:57.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:57 smithi179 ceph-mon[93519]: pgmap v233: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-08-17T17:28:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:57 smithi179 ceph-mon[93519]: osdmap e404: 8 total, 8 up, 8 in 2024-08-17T17:28:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:57.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:57 smithi130 ceph-mon[105521]: pgmap v233: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-08-17T17:28:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:57 smithi130 ceph-mon[105521]: osdmap e404: 8 total, 8 up, 8 in 2024-08-17T17:28:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:57 smithi130 ceph-mon[110068]: pgmap v233: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-08-17T17:28:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:57 smithi130 ceph-mon[110068]: osdmap e404: 8 total, 8 up, 8 in 2024-08-17T17:28:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:58.028 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade resume' 2024-08-17T17:28:58.282 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:58.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:58 smithi130 ceph-mon[105521]: osdmap e405: 8 total, 8 up, 8 in 2024-08-17T17:28:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:58 smithi130 ceph-mon[110068]: osdmap e405: 8 total, 8 up, 8 in 2024-08-17T17:28:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:58 smithi179 ceph-mon[93519]: osdmap e405: 8 total, 8 up, 8 in 2024-08-17T17:28:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:59 smithi130 ceph-mon[105521]: pgmap v236: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-08-17T17:28:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:28:59 smithi130 ceph-mon[105521]: osdmap e406: 8 total, 8 up, 8 in 2024-08-17T17:28:59.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:59 smithi130 ceph-mon[110068]: pgmap v236: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-08-17T17:28:59.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:28:59 smithi130 ceph-mon[110068]: osdmap e406: 8 total, 8 up, 8 in 2024-08-17T17:28:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:59 smithi179 ceph-mon[93519]: pgmap v236: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-08-17T17:28:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:28:59 smithi179 ceph-mon[93519]: osdmap e406: 8 total, 8 up, 8 in 2024-08-17T17:28:59.949 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:28:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:28:59] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.51.0" 2024-08-17T17:29:00.013 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.CloneFlatten (2181 ms) 2024-08-17T17:29:00.013 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Stress 2024-08-17T17:29:00.035 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 25930555~5438452 2024-08-17T17:29:00.054 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 1879987~6843443 2024-08-17T17:29:00.055 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 29746547~4357486 2024-08-17T17:29:00.067 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 48014378~4493912 2024-08-17T17:29:00.068 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 46425278~7044940 2024-08-17T17:29:00.088 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 37212973~7589000 2024-08-17T17:29:00.089 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 73784248~283724 2024-08-17T17:29:00.092 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 71839775~1069488 2024-08-17T17:29:00.093 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap0 2024-08-17T17:29:00.190 INFO:teuthology.orchestra.run.smithi130.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:29:00.518 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:00.530 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 15526719~4962484 2024-08-17T17:29:00.534 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 70178654~3209762 2024-08-17T17:29:00.535 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 4604068~3963984 2024-08-17T17:29:00.535 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 37196200~6872515 2024-08-17T17:29:00.535 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 63194653~3067050 2024-08-17T17:29:00.536 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 23594788~3118498 2024-08-17T17:29:00.536 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 8556950~2208245 2024-08-17T17:29:00.548 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 9051688~7582303 2024-08-17T17:29:00.575 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap1 2024-08-17T17:29:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:00 smithi179 ceph-mon[93519]: osdmap e407: 8 total, 8 up, 8 in 2024-08-17T17:29:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:00 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:00 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:00.846 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:00 smithi130 ceph-mon[105521]: osdmap e407: 8 total, 8 up, 8 in 2024-08-17T17:29:00.846 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:00 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:00.846 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:00 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:00.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:00 smithi130 ceph-mon[110068]: osdmap e407: 8 total, 8 up, 8 in 2024-08-17T17:29:00.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:00 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:00.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:00 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:00.880 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for all mons to upgrade' 2024-08-17T17:29:01.539 INFO:tasks.workunit.client.0.smithi130.stdout:clone image238 -> image239 2024-08-17T17:29:01.588 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 78642276~5243804 2024-08-17T17:29:01.606 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 3900136~1233912 2024-08-17T17:29:01.632 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 8822912~2635871 2024-08-17T17:29:01.651 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 19696811~7558678 2024-08-17T17:29:01.664 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:01.673 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 17016774~7109688 2024-08-17T17:29:01.691 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 49452108~7650507 2024-08-17T17:29:01.712 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 4224383~688254 2024-08-17T17:29:01.715 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 45841056~6258571 2024-08-17T17:29:01.717 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap2 2024-08-17T17:29:01.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:01 smithi130 ceph-mon[110068]: pgmap v239: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-08-17T17:29:01.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:01 smithi130 ceph-mon[110068]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:29:01.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:01 smithi130 ceph-mon[110068]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:29:01.795 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:01 smithi130 ceph-mon[110068]: osdmap e408: 8 total, 8 up, 8 in 2024-08-17T17:29:01.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:01 smithi130 ceph-mon[105521]: pgmap v239: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-08-17T17:29:01.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:01 smithi130 ceph-mon[105521]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:29:01.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:01 smithi130 ceph-mon[105521]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:29:01.795 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:01 smithi130 ceph-mon[105521]: osdmap e408: 8 total, 8 up, 8 in 2024-08-17T17:29:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:01 smithi179 ceph-mon[93519]: pgmap v239: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 13 MiB/s wr, 343 op/s 2024-08-17T17:29:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:01 smithi179 ceph-mon[93519]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:29:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:01 smithi179 ceph-mon[93519]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:29:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:01 smithi179 ceph-mon[93519]: osdmap e408: 8 total, 8 up, 8 in 2024-08-17T17:29:02.548 INFO:tasks.workunit.client.0.smithi130.stdout:clone image239 -> image240 2024-08-17T17:29:02.605 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 83999557 2024-08-17T17:29:02.611 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 28145045~5778081 2024-08-17T17:29:02.639 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 59222402~3180527 2024-08-17T17:29:02.642 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 66789246~205605 2024-08-17T17:29:02.663 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 39707016~4072115 2024-08-17T17:29:02.667 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 54272504~803186 2024-08-17T17:29:02.671 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 20677676~2422276 2024-08-17T17:29:02.688 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 67344342~1590391 2024-08-17T17:29:02.695 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 46298798~5588280 2024-08-17T17:29:02.698 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap3 2024-08-17T17:29:02.832 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:02 smithi130 ceph-mon[105521]: osdmap e409: 8 total, 8 up, 8 in 2024-08-17T17:29:02.833 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:02 smithi130 ceph-mon[110068]: osdmap e409: 8 total, 8 up, 8 in 2024-08-17T17:29:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:02 smithi179 ceph-mon[93519]: osdmap e409: 8 total, 8 up, 8 in 2024-08-17T17:29:03.552 INFO:tasks.workunit.client.0.smithi130.stdout:clone image240 -> image241 2024-08-17T17:29:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:03 smithi179 ceph-mon[93519]: pgmap v242: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:29:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:03 smithi179 ceph-mon[93519]: osdmap e410: 8 total, 8 up, 8 in 2024-08-17T17:29:03.847 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Stress (3834 ms) 2024-08-17T17:29:03.847 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.NoSnaps_LargerDstObjSize 2024-08-17T17:29:03.890 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:03 smithi130 ceph-mon[105521]: pgmap v242: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:29:03.890 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:03 smithi130 ceph-mon[105521]: osdmap e410: 8 total, 8 up, 8 in 2024-08-17T17:29:03.890 INFO:teuthology.orchestra.run.smithi130.stdout:wait for all mons to upgrade 2024-08-17T17:29:03.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:03 smithi130 ceph-mon[110068]: pgmap v242: 105 pgs: 105 active+clean; 117 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:29:03.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:03 smithi130 ceph-mon[110068]: osdmap e410: 8 total, 8 up, 8 in 2024-08-17T17:29:04.108 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.NoSnaps_LargerDstObjSize (261 ms) 2024-08-17T17:29:04.108 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Snaps_LargerDstObjSize 2024-08-17T17:29:04.543 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:04 smithi179 ceph-mon[93519]: osdmap e411: 8 total, 8 up, 8 in 2024-08-17T17:29:04.888 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:04 smithi130 ceph-mon[105521]: osdmap e411: 8 total, 8 up, 8 in 2024-08-17T17:29:04.888 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:04 smithi130 ceph-mon[110068]: osdmap e411: 8 total, 8 up, 8 in 2024-08-17T17:29:04.912 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-08-17T17:29:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:05 smithi130 ceph-mon[105521]: pgmap v245: 105 pgs: 105 active+clean; 313 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 42 MiB/s wr, 505 op/s 2024-08-17T17:29:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:05 smithi130 ceph-mon[105521]: osdmap e412: 8 total, 8 up, 8 in 2024-08-17T17:29:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:05 smithi130 ceph-mon[110068]: pgmap v245: 105 pgs: 105 active+clean; 313 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 42 MiB/s wr, 505 op/s 2024-08-17T17:29:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:05 smithi130 ceph-mon[110068]: osdmap e412: 8 total, 8 up, 8 in 2024-08-17T17:29:05.706 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:05 smithi179 ceph-mon[93519]: pgmap v245: 105 pgs: 105 active+clean; 313 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 42 MiB/s wr, 505 op/s 2024-08-17T17:29:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:05 smithi179 ceph-mon[93519]: osdmap e412: 8 total, 8 up, 8 in 2024-08-17T17:29:05.843 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:05.843 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:06.640 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:06 smithi130 ceph-mon[105521]: osdmap e413: 8 total, 8 up, 8 in 2024-08-17T17:29:06.640 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:06 smithi130 ceph-mon[110068]: osdmap e413: 8 total, 8 up, 8 in 2024-08-17T17:29:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:06 smithi179 ceph-mon[93519]: osdmap e413: 8 total, 8 up, 8 in 2024-08-17T17:29:07.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:06 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:07 smithi179 ceph-mon[93519]: pgmap v248: 105 pgs: 105 active+clean; 313 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 42 MiB/s wr, 505 op/s 2024-08-17T17:29:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:07 smithi179 ceph-mon[93519]: osdmap e414: 8 total, 8 up, 8 in 2024-08-17T17:29:07.893 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Snaps_LargerDstObjSize (3785 ms) 2024-08-17T17:29:07.893 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Clone_LargerDstObjSize 2024-08-17T17:29:07.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:07 smithi130 ceph-mon[110068]: pgmap v248: 105 pgs: 105 active+clean; 313 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 42 MiB/s wr, 505 op/s 2024-08-17T17:29:07.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:07 smithi130 ceph-mon[110068]: osdmap e414: 8 total, 8 up, 8 in 2024-08-17T17:29:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:07 smithi130 ceph-mon[105521]: pgmap v248: 105 pgs: 105 active+clean; 313 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 42 MiB/s wr, 505 op/s 2024-08-17T17:29:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:07 smithi130 ceph-mon[105521]: osdmap e414: 8 total, 8 up, 8 in 2024-08-17T17:29:08.447 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:29:08.804 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: osdmap e415: 8 total, 8 up, 8 in 2024-08-17T17:29:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:29:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:29:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:29:08.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:29:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-17T17:29:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:29:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:29:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-17T17:29:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:29:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:29:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-17T17:29:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:29:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-17T17:29:08.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2595676576' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:29:08.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: osdmap e415: 8 total, 8 up, 8 in 2024-08-17T17:29:08.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:08.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:08.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:08.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:29:08.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:29:08.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:29:08.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:29:08.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-17T17:29:08.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:29:08.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:29:08.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-17T17:29:08.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:29:08.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:29:08.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-17T17:29:08.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:29:08.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-17T17:29:08.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:08 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2595676576' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:29:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: osdmap e415: 8 total, 8 up, 8 in 2024-08-17T17:29:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:29:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:29:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:29:08.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:29:08.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-17T17:29:08.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:29:08.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:29:08.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-17T17:29:08.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:29:08.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:29:08.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-17T17:29:08.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:29:08.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:08.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-17T17:29:08.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:08 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2595676576' entity='client.admin' cmd={"prefix": "mon versions"} : dispatch 2024-08-17T17:29:09.188 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch ps' 2024-08-17T17:29:09.897 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:10.013 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:09 smithi130 ceph-mon[105521]: pgmap v251: 105 pgs: 105 active+clean; 385 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s rd, 18 MiB/s wr, 445 op/s 2024-08-17T17:29:10.013 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:09 smithi130 ceph-mon[105521]: Upgrade: Setting container_image for all mon 2024-08-17T17:29:10.013 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:09 smithi130 ceph-mon[105521]: Upgrade: Setting container_image for all crash 2024-08-17T17:29:10.013 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:09 smithi130 ceph-mon[105521]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-17T17:29:10.013 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:09 smithi130 ceph-mon[105521]: Upgrade: osd.0 is safe to restart 2024-08-17T17:29:10.014 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:09 smithi130 ceph-mon[105521]: osdmap e416: 8 total, 8 up, 8 in 2024-08-17T17:29:10.014 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:29:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:29:09] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.51.0" 2024-08-17T17:29:10.014 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:09 smithi130 ceph-mon[110068]: pgmap v251: 105 pgs: 105 active+clean; 385 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s rd, 18 MiB/s wr, 445 op/s 2024-08-17T17:29:10.014 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:09 smithi130 ceph-mon[110068]: Upgrade: Setting container_image for all mon 2024-08-17T17:29:10.014 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:09 smithi130 ceph-mon[110068]: Upgrade: Setting container_image for all crash 2024-08-17T17:29:10.014 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:09 smithi130 ceph-mon[110068]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-17T17:29:10.014 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:09 smithi130 ceph-mon[110068]: Upgrade: osd.0 is safe to restart 2024-08-17T17:29:10.014 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:09 smithi130 ceph-mon[110068]: osdmap e416: 8 total, 8 up, 8 in 2024-08-17T17:29:10.054 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Clone_LargerDstObjSize (2161 ms) 2024-08-17T17:29:10.054 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.CloneFlatten_LargerDstObjSize 2024-08-17T17:29:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:09 smithi179 ceph-mon[93519]: pgmap v251: 105 pgs: 105 active+clean; 385 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s rd, 18 MiB/s wr, 445 op/s 2024-08-17T17:29:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:09 smithi179 ceph-mon[93519]: Upgrade: Setting container_image for all mon 2024-08-17T17:29:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:09 smithi179 ceph-mon[93519]: Upgrade: Setting container_image for all crash 2024-08-17T17:29:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:09 smithi179 ceph-mon[93519]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-17T17:29:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:09 smithi179 ceph-mon[93519]: Upgrade: osd.0 is safe to restart 2024-08-17T17:29:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:09 smithi179 ceph-mon[93519]: osdmap e416: 8 total, 8 up, 8 in 2024-08-17T17:29:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:10 smithi130 ceph-mon[110068]: osdmap e417: 8 total, 8 up, 8 in 2024-08-17T17:29:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:10 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:10 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:29:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:10 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:10 smithi130 ceph-mon[105521]: osdmap e417: 8 total, 8 up, 8 in 2024-08-17T17:29:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:10 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:10 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:29:10.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:10 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:10 smithi179 ceph-mon[93519]: osdmap e417: 8 total, 8 up, 8 in 2024-08-17T17:29:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:10 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:10 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:29:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:10 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:11.782 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:11 smithi130 ceph-mon[105521]: pgmap v254: 105 pgs: 105 active+clean; 385 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s rd, 18 MiB/s wr, 445 op/s 2024-08-17T17:29:11.783 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:11 smithi130 ceph-mon[105521]: Upgrade: Updating osd.0 2024-08-17T17:29:11.783 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:11 smithi130 ceph-mon[105521]: Deploying daemon osd.0 on smithi130 2024-08-17T17:29:11.783 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:11 smithi130 ceph-mon[105521]: osdmap e418: 8 total, 8 up, 8 in 2024-08-17T17:29:11.783 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:11 smithi130 ceph-mon[110068]: pgmap v254: 105 pgs: 105 active+clean; 385 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s rd, 18 MiB/s wr, 445 op/s 2024-08-17T17:29:11.783 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:11 smithi130 ceph-mon[110068]: Upgrade: Updating osd.0 2024-08-17T17:29:11.783 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:11 smithi130 ceph-mon[110068]: Deploying daemon osd.0 on smithi130 2024-08-17T17:29:11.783 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:11 smithi130 ceph-mon[110068]: osdmap e418: 8 total, 8 up, 8 in 2024-08-17T17:29:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:11 smithi179 ceph-mon[93519]: pgmap v254: 105 pgs: 105 active+clean; 385 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 53 MiB/s rd, 18 MiB/s wr, 445 op/s 2024-08-17T17:29:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:11 smithi179 ceph-mon[93519]: Upgrade: Updating osd.0 2024-08-17T17:29:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:11 smithi179 ceph-mon[93519]: Deploying daemon osd.0 on smithi130 2024-08-17T17:29:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:11 smithi179 ceph-mon[93519]: osdmap e418: 8 total, 8 up, 8 in 2024-08-17T17:29:12.227 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.CloneFlatten_LargerDstObjSize (2173 ms) 2024-08-17T17:29:12.227 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Stress_LargerDstObjSize 2024-08-17T17:29:12.250 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 72502139~4900156 2024-08-17T17:29:12.270 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 30754235~3864124 2024-08-17T17:29:12.270 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 10122695~1229050 2024-08-17T17:29:12.275 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 50466264~1097819 2024-08-17T17:29:12.276 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 26377120~3568034 2024-08-17T17:29:12.288 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 67764006~5912697 2024-08-17T17:29:12.290 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 24553783~4391823 2024-08-17T17:29:12.304 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 56332962~860747 2024-08-17T17:29:12.304 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap0 2024-08-17T17:29:12.739 INFO:tasks.workunit.client.0.smithi130.stdout:clone image260 -> image261 2024-08-17T17:29:12.783 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 107467797 2024-08-17T17:29:12.789 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 58272867~8195269 2024-08-17T17:29:12.816 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 80242627~2770261 2024-08-17T17:29:12.819 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 88788223~4232881 2024-08-17T17:29:12.835 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:12 smithi130 ceph-mon[105521]: osdmap e419: 8 total, 8 up, 8 in 2024-08-17T17:29:12.835 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:12.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:12 smithi130 ceph-mon[110068]: osdmap e419: 8 total, 8 up, 8 in 2024-08-17T17:29:12.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:12.839 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 50971809~6887581 2024-08-17T17:29:12.842 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 72968576~5992518 2024-08-17T17:29:12.861 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 96854331~5187430 2024-08-17T17:29:12.863 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 89426280~8059727 2024-08-17T17:29:12.880 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 101256098~6211699 2024-08-17T17:29:12.880 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap1 2024-08-17T17:29:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:12 smithi179 ceph-mon[93519]: osdmap e419: 8 total, 8 up, 8 in 2024-08-17T17:29:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:13.457 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:29:13.458 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (4m) 5s ago 8m 19.3M - 0.27.0 11f11916f8cd 73ea4ebcf68b 2024-08-17T17:29:13.458 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi179 *:3000 running (4m) 70s ago 8m 85.1M - 9.4.7 954c08fa6188 399f89cc0037 2024-08-17T17:29:13.458 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foo.smithi130.jmaidg smithi130 running (7m) 5s ago 7m 20.4M - 17.2.7-1404-gf2c39d4d f69471c56189 91d7d1762094 2024-08-17T17:29:13.458 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foo.smithi179.zkkvno smithi179 running (7m) 70s ago 7m 14.2M - 17.2.7-1404-gf2c39d4d f69471c56189 0486517d425a 2024-08-17T17:29:13.458 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi179 *:8443,9283,8765 running (3m) 70s ago 12m 453M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb cdd6dde7c7af 2024-08-17T17:29:13.458 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (3m) 5s ago 13m 512M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 584426b3875b 2024-08-17T17:29:13.458 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (3m) 5s ago 14m 51.9M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 79749aebcefa 2024-08-17T17:29:13.458 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi179 running (76s) 70s ago 12m 21.6M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb d85c3c5c065e 2024-08-17T17:29:13.458 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (96s) 5s ago 12m 41.7M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 4161e48dac50 2024-08-17T17:29:13.458 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (4m) 5s ago 7m 10.7M - 1.7.0 72c9c2088986 fd7392d87597 2024-08-17T17:29:13.458 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi179 *:9100 running (3m) 70s ago 7m 10.4M - 1.7.0 72c9c2088986 c10588f17ed8 2024-08-17T17:29:13.459 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (11m) 5s ago 11m 124M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 293583e22b78 2024-08-17T17:29:13.459 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (11m) 5s ago 11m 120M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 1dc0b4634e15 2024-08-17T17:29:13.459 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (11m) 5s ago 11m 148M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 d72289cb8b72 2024-08-17T17:29:13.459 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (10m) 5s ago 10m 127M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 1509c2840516 2024-08-17T17:29:13.459 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi179 running (10m) 70s ago 10m 126M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 254dc0763420 2024-08-17T17:29:13.459 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi179 running (10m) 70s ago 10m 97.3M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 078704aadcd7 2024-08-17T17:29:13.459 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi179 running (9m) 70s ago 9m 150M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 ab570ba67ae7 2024-08-17T17:29:13.459 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi179 running (9m) 70s ago 9m 117M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 99ae6a861fa0 2024-08-17T17:29:13.459 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi179 *:9095 running (3m) 70s ago 8m 48.6M - 2.51.0 1d3b7f56885b 145f77b6fc93 2024-08-17T17:29:13.735 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 165740902 2024-08-17T17:29:13.741 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 126659499~3555764 2024-08-17T17:29:13.743 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 97153481~7568646 2024-08-17T17:29:13.744 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 86667214~3780355 2024-08-17T17:29:13.744 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 109366915~467812 2024-08-17T17:29:13.744 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 9758084~1184568 2024-08-17T17:29:13.744 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 24239133~5875798 2024-08-17T17:29:13.745 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 78145739~3734548 2024-08-17T17:29:13.746 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 46283913~2370723 2024-08-17T17:29:13.789 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap2 2024-08-17T17:29:13.994 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:13 smithi130 ceph-mon[105521]: pgmap v257: 105 pgs: 105 active+clean; 385 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:29:13.994 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:13 smithi130 ceph-mon[105521]: osdmap e420: 8 total, 8 up, 8 in 2024-08-17T17:29:13.994 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:13 smithi130 ceph-mon[110068]: pgmap v257: 105 pgs: 105 active+clean; 385 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:29:13.994 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:13 smithi130 ceph-mon[110068]: osdmap e420: 8 total, 8 up, 8 in 2024-08-17T17:29:13.995 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:13 smithi179 ceph-mon[93519]: pgmap v257: 105 pgs: 105 active+clean; 385 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:29:13.995 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:13 smithi179 ceph-mon[93519]: osdmap e420: 8 total, 8 up, 8 in 2024-08-17T17:29:14.293 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade pause' 2024-08-17T17:29:14.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:14 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:14.749 INFO:tasks.workunit.client.0.smithi130.stdout:clone image261 -> image262 2024-08-17T17:29:14.807 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 227828156 2024-08-17T17:29:14.812 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 217086803~4433096 2024-08-17T17:29:14.828 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 107986511~6790145 2024-08-17T17:29:14.832 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 184331940~2292142 2024-08-17T17:29:14.843 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 56538178~3424017 2024-08-17T17:29:14.860 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 138765581~3332187 2024-08-17T17:29:14.876 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 219767135~8061021 2024-08-17T17:29:14.880 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 138725262~2012322 2024-08-17T17:29:14.889 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 55133731~480967 2024-08-17T17:29:14.890 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap3 2024-08-17T17:29:14.898 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:15.006 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:14 smithi130 ceph-mon[105521]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:29:15.006 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:14 smithi130 ceph-mon[105521]: osdmap e421: 8 total, 8 up, 8 in 2024-08-17T17:29:15.006 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:14 smithi130 ceph-mon[110068]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:29:15.006 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:14 smithi130 ceph-mon[110068]: osdmap e421: 8 total, 8 up, 8 in 2024-08-17T17:29:15.006 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:14 smithi179 ceph-mon[93519]: from='client.34275 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:29:15.006 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:14 smithi179 ceph-mon[93519]: osdmap e421: 8 total, 8 up, 8 in 2024-08-17T17:29:15.344 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:15 smithi130 systemd[1]: Stopping Ceph osd.0 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:29:15.752 INFO:tasks.workunit.client.0.smithi130.stdout:clone image262 -> image263 2024-08-17T17:29:15.815 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 269698132 2024-08-17T17:29:16.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:15 smithi179 ceph-mon[93519]: pgmap v260: 105 pgs: 105 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 49 MiB/s rd, 46 MiB/s wr, 598 op/s 2024-08-17T17:29:16.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:15 smithi179 ceph-mon[93519]: osdmap e422: 8 total, 8 up, 8 in 2024-08-17T17:29:16.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:16.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:16.117 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Stress_LargerDstObjSize (3890 ms) 2024-08-17T17:29:16.117 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.NoSnaps_SmallerDstObjSize 2024-08-17T17:29:16.122 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:15 smithi130 ceph-mon[105521]: pgmap v260: 105 pgs: 105 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 49 MiB/s rd, 46 MiB/s wr, 598 op/s 2024-08-17T17:29:16.123 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:15 smithi130 ceph-mon[105521]: osdmap e422: 8 total, 8 up, 8 in 2024-08-17T17:29:16.123 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:15 smithi130 ceph-mon[110068]: pgmap v260: 105 pgs: 105 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 49 MiB/s rd, 46 MiB/s wr, 598 op/s 2024-08-17T17:29:16.123 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:15 smithi130 ceph-mon[110068]: osdmap e422: 8 total, 8 up, 8 in 2024-08-17T17:29:16.336 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.NoSnaps_SmallerDstObjSize (219 ms) 2024-08-17T17:29:16.337 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Snaps_SmallerDstObjSize 2024-08-17T17:29:16.383 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:16 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[47053]: 2024-08-17T17:29:16.270+0000 7f4009be1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:29:16.383 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:16 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[47053]: 2024-08-17T17:29:16.270+0000 7f4009be1640 -1 osd.0 423 *** Got signal Terminated *** 2024-08-17T17:29:16.383 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:16 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[47053]: 2024-08-17T17:29:16.270+0000 7f4009be1640 -1 osd.0 423 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:29:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:16 smithi130 ceph-mon[105521]: osdmap e423: 8 total, 8 up, 8 in 2024-08-17T17:29:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:16 smithi130 ceph-mon[105521]: osd.0 marked itself down and dead 2024-08-17T17:29:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:16 smithi130 ceph-mon[110068]: osdmap e423: 8 total, 8 up, 8 in 2024-08-17T17:29:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:16 smithi130 ceph-mon[110068]: osd.0 marked itself down and dead 2024-08-17T17:29:16.953 INFO:teuthology.orchestra.run.smithi130.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:29:17.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:16 smithi179 ceph-mon[93519]: osdmap e423: 8 total, 8 up, 8 in 2024-08-17T17:29:17.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:16 smithi179 ceph-mon[93519]: osd.0 marked itself down and dead 2024-08-17T17:29:17.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:17.540 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'sleep 60' 2024-08-17T17:29:17.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:17 smithi130 ceph-mon[105521]: pgmap v263: 105 pgs: 105 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 49 MiB/s rd, 46 MiB/s wr, 598 op/s 2024-08-17T17:29:17.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:17 smithi130 ceph-mon[105521]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:29:17.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:17 smithi130 ceph-mon[105521]: osdmap e424: 8 total, 7 up, 8 in 2024-08-17T17:29:17.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:17 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:17.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:17 smithi130 ceph-mon[110068]: pgmap v263: 105 pgs: 105 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 49 MiB/s rd, 46 MiB/s wr, 598 op/s 2024-08-17T17:29:17.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:17 smithi130 ceph-mon[110068]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:29:17.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:17 smithi130 ceph-mon[110068]: osdmap e424: 8 total, 7 up, 8 in 2024-08-17T17:29:17.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:17 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:17.821 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:17 smithi130 podman[117494]: 2024-08-17 17:29:17.520961423 +0000 UTC m=+1.800435774 container died 293583e22b78b5de2d68c95a9bf030296e9c46fcc39a2359f7ccb9e770c54497 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS) 2024-08-17T17:29:18.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:17 smithi179 ceph-mon[93519]: pgmap v263: 105 pgs: 105 active+clean; 574 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 49 MiB/s rd, 46 MiB/s wr, 598 op/s 2024-08-17T17:29:18.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:17 smithi179 ceph-mon[93519]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:29:18.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:17 smithi179 ceph-mon[93519]: osdmap e424: 8 total, 7 up, 8 in 2024-08-17T17:29:18.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:17 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:18.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:17 smithi130 podman[117494]: 2024-08-17 17:29:17.926397194 +0000 UTC m=+2.205871547 container cleanup 293583e22b78b5de2d68c95a9bf030296e9c46fcc39a2359f7ccb9e770c54497 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-f2c39d4, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:29:18.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:17 smithi130 bash[117494]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0 2024-08-17T17:29:18.333 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:18.456 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:18 smithi130 podman[117611]: 2024-08-17 17:29:18.169781486 +0000 UTC m=+0.871666327 container remove 293583e22b78b5de2d68c95a9bf030296e9c46fcc39a2359f7ccb9e770c54497 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=quincy-f2c39d4, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True) 2024-08-17T17:29:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:18 smithi179 ceph-mon[93519]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:29:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:18 smithi179 ceph-mon[93519]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:29:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:18 smithi179 ceph-mon[93519]: osdmap e425: 8 total, 7 up, 8 in 2024-08-17T17:29:19.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:18 smithi130 podman[117689]: 2024-08-17 17:29:18.781697324 +0000 UTC m=+0.491508572 container died 45e0639f66b9010bed21ef7076effe0fe047525c57f14528894698e1ff545efa (image=quay.ceph.io/ceph-ci/ceph:quincy, name=affectionate_keller, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-f2c39d4, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:29:19.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:18 smithi130 ceph-mon[105521]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:29:19.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:18 smithi130 ceph-mon[105521]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:29:19.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:18 smithi130 ceph-mon[105521]: osdmap e425: 8 total, 7 up, 8 in 2024-08-17T17:29:19.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:18 smithi130 ceph-mon[110068]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:29:19.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:18 smithi130 ceph-mon[110068]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:29:19.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:18 smithi130 ceph-mon[110068]: osdmap e425: 8 total, 7 up, 8 in 2024-08-17T17:29:19.760 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:19 smithi130 podman[117831]: 2024-08-17 17:29:19.543546079 +0000 UTC m=+0.022345952 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:29:20.023 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:19 smithi130 ceph-mon[105521]: pgmap v266: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 20 MiB/s wr, 533 op/s; 182/1635 objects degraded (11.131%) 2024-08-17T17:29:20.023 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:19 smithi130 ceph-mon[105521]: Health check failed: Degraded data redundancy: 182/1635 objects degraded (11.131%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:20.023 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:19 smithi130 ceph-mon[105521]: osdmap e426: 8 total, 7 up, 8 in 2024-08-17T17:29:20.024 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:19 smithi130 ceph-mon[110068]: pgmap v266: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 20 MiB/s wr, 533 op/s; 182/1635 objects degraded (11.131%) 2024-08-17T17:29:20.024 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:19 smithi130 ceph-mon[110068]: Health check failed: Degraded data redundancy: 182/1635 objects degraded (11.131%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:20.024 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:19 smithi130 ceph-mon[110068]: osdmap e426: 8 total, 7 up, 8 in 2024-08-17T17:29:20.024 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:29:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:29:19] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.51.0" 2024-08-17T17:29:20.024 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:19 smithi130 podman[117831]: 2024-08-17 17:29:19.89689206 +0000 UTC m=+0.375691912 container create c60179ecb85bc8fd08d28c2ee41ddad936aeb56e084e6bab53f0f1bf0acc91bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-08-17T17:29:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:19 smithi179 ceph-mon[93519]: pgmap v266: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 20 MiB/s wr, 533 op/s; 182/1635 objects degraded (11.131%) 2024-08-17T17:29:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:19 smithi179 ceph-mon[93519]: Health check failed: Degraded data redundancy: 182/1635 objects degraded (11.131%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:19 smithi179 ceph-mon[93519]: osdmap e426: 8 total, 7 up, 8 in 2024-08-17T17:29:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:20.667 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:20 smithi130 podman[117831]: 2024-08-17 17:29:20.292122248 +0000 UTC m=+0.770922109 container init c60179ecb85bc8fd08d28c2ee41ddad936aeb56e084e6bab53f0f1bf0acc91bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe) 2024-08-17T17:29:20.667 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:20 smithi130 podman[117831]: 2024-08-17 17:29:20.297615824 +0000 UTC m=+0.776415678 container start c60179ecb85bc8fd08d28c2ee41ddad936aeb56e084e6bab53f0f1bf0acc91bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-08-17T17:29:20.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:20 smithi130 podman[117831]: 2024-08-17 17:29:20.367060879 +0000 UTC m=+0.845860730 container attach c60179ecb85bc8fd08d28c2ee41ddad936aeb56e084e6bab53f0f1bf0acc91bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-08-17T17:29:20.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:20 smithi130 podman[117831]: 2024-08-17 17:29:20.491924302 +0000 UTC m=+0.970724153 container died c60179ecb85bc8fd08d28c2ee41ddad936aeb56e084e6bab53f0f1bf0acc91bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-08-17T17:29:21.015 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Snaps_SmallerDstObjSize (4679 ms) 2024-08-17T17:29:21.016 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Clone_SmallerDstObjSize 2024-08-17T17:29:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:20 smithi179 ceph-mon[93519]: osdmap e427: 8 total, 7 up, 8 in 2024-08-17T17:29:21.096 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:20 smithi130 ceph-mon[110068]: osdmap e427: 8 total, 7 up, 8 in 2024-08-17T17:29:21.097 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:20 smithi130 ceph-mon[105521]: osdmap e427: 8 total, 7 up, 8 in 2024-08-17T17:29:21.417 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 podman[117911]: 2024-08-17 17:29:21.096568048 +0000 UTC m=+0.598760685 container remove c60179ecb85bc8fd08d28c2ee41ddad936aeb56e084e6bab53f0f1bf0acc91bd (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-08-17T17:29:21.417 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.0.service: Deactivated successfully. 2024-08-17T17:29:21.417 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 systemd[1]: Stopped Ceph osd.0 for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:29:21.417 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.0.service: Consumed 20.364s CPU time. 2024-08-17T17:29:21.417 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 systemd[1]: Starting Ceph osd.0 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:29:21.876 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:21 smithi130 ceph-mon[105521]: pgmap v269: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 20 MiB/s wr, 533 op/s; 182/1635 objects degraded (11.131%) 2024-08-17T17:29:21.876 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:21 smithi130 ceph-mon[105521]: osdmap e428: 8 total, 7 up, 8 in 2024-08-17T17:29:21.876 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:21 smithi130 ceph-mon[110068]: pgmap v269: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 20 MiB/s wr, 533 op/s; 182/1635 objects degraded (11.131%) 2024-08-17T17:29:21.876 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:21 smithi130 ceph-mon[110068]: osdmap e428: 8 total, 7 up, 8 in 2024-08-17T17:29:21.877 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 podman[118016]: 2024-08-17 17:29:21.4364812 +0000 UTC m=+0.017173749 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:29:21.877 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 podman[118016]: 2024-08-17 17:29:21.539822842 +0000 UTC m=+0.120515435 container create 1036816dd7d35ccbac340a9dfe258a9b10aa4abbad68e79c87c5472389abf417 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-08-17T17:29:21.877 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 podman[118016]: 2024-08-17 17:29:21.767207605 +0000 UTC m=+0.347900166 container init 1036816dd7d35ccbac340a9dfe258a9b10aa4abbad68e79c87c5472389abf417 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240731) 2024-08-17T17:29:21.877 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 podman[118016]: 2024-08-17 17:29:21.772177568 +0000 UTC m=+0.352870118 container start 1036816dd7d35ccbac340a9dfe258a9b10aa4abbad68e79c87c5472389abf417 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD) 2024-08-17T17:29:21.877 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 podman[118016]: 2024-08-17 17:29:21.875783824 +0000 UTC m=+0.456476399 container attach 1036816dd7d35ccbac340a9dfe258a9b10aa4abbad68e79c87c5472389abf417 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.build-date=20240731) 2024-08-17T17:29:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:21 smithi179 ceph-mon[93519]: pgmap v269: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 27 MiB/s rd, 20 MiB/s wr, 533 op/s; 182/1635 objects degraded (11.131%) 2024-08-17T17:29:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:21 smithi179 ceph-mon[93519]: osdmap e428: 8 total, 7 up, 8 in 2024-08-17T17:29:22.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[118031]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:29:22.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 bash[118016]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:29:22.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[118031]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:29:22.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 bash[118016]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:29:22.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[118031]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:29:22.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 bash[118016]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:29:22.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[118031]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:29:22.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 bash[118016]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:29:22.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[118031]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:29:22.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:21 smithi130 bash[118016]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:29:22.168 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[118031]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:29:22.168 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 bash[118016]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:29:22.168 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[118031]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-17T17:29:22.168 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 bash[118016]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-17T17:29:22.621 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[118031]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:29:22.622 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 bash[118016]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:29:22.622 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[118031]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:29:22.622 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 bash[118016]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:29:22.622 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[118031]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-17T17:29:22.622 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 bash[118016]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-17T17:29:22.622 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[118031]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:29:22.622 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 bash[118016]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:29:22.622 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[118031]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-17T17:29:22.622 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 bash[118016]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-17T17:29:22.622 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 podman[118016]: 2024-08-17 17:29:22.334951191 +0000 UTC m=+0.915643748 container died 1036816dd7d35ccbac340a9dfe258a9b10aa4abbad68e79c87c5472389abf417 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-08-17T17:29:22.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:22 smithi130 ceph-mon[105521]: osdmap e429: 8 total, 7 up, 8 in 2024-08-17T17:29:22.906 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:22 smithi130 ceph-mon[110068]: osdmap e429: 8 total, 7 up, 8 in 2024-08-17T17:29:23.088 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Clone_SmallerDstObjSize (2073 ms) 2024-08-17T17:29:23.088 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.CloneFlatten_SmallerDstObjSize 2024-08-17T17:29:23.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:22 smithi179 ceph-mon[93519]: osdmap e429: 8 total, 7 up, 8 in 2024-08-17T17:29:23.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:22 smithi130 podman[118157]: 2024-08-17 17:29:22.904860584 +0000 UTC m=+0.560194370 container remove 1036816dd7d35ccbac340a9dfe258a9b10aa4abbad68e79c87c5472389abf417 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0) 2024-08-17T17:29:23.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:23 smithi130 podman[118218]: 2024-08-17 17:29:23.036170625 +0000 UTC m=+0.020588399 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:29:23.442 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:23 smithi130 podman[118218]: 2024-08-17 17:29:23.17299886 +0000 UTC m=+0.157416623 container create 26704f9d719b0d5a8ee12bc7e5da5b61e42a1935ff27ad52ef4dcff342137b17 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-08-17T17:29:23.442 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:23 smithi130 podman[118218]: 2024-08-17 17:29:23.358919547 +0000 UTC m=+0.343337320 container init 26704f9d719b0d5a8ee12bc7e5da5b61e42a1935ff27ad52ef4dcff342137b17 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:29:23.443 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:23 smithi130 podman[118218]: 2024-08-17 17:29:23.364053587 +0000 UTC m=+0.348471344 container start 26704f9d719b0d5a8ee12bc7e5da5b61e42a1935ff27ad52ef4dcff342137b17 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True) 2024-08-17T17:29:23.443 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:23 smithi130 ceph-osd[118256]: -- 172.21.15.130:0/1390479637 <== mon.0 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563258987a40 con 0x563259754000 2024-08-17T17:29:23.443 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:23 smithi130 bash[118218]: 26704f9d719b0d5a8ee12bc7e5da5b61e42a1935ff27ad52ef4dcff342137b17 2024-08-17T17:29:23.786 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:23 smithi130 systemd[1]: Started Ceph osd.0 for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:29:24.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:23 smithi179 ceph-mon[93519]: pgmap v272: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 182/1635 objects degraded (11.131%) 2024-08-17T17:29:24.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:23 smithi179 ceph-mon[93519]: osdmap e430: 8 total, 7 up, 8 in 2024-08-17T17:29:24.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:23 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:24.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:23 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:24.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:23 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:24.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:23 smithi130 ceph-mon[105521]: pgmap v272: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 182/1635 objects degraded (11.131%) 2024-08-17T17:29:24.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:23 smithi130 ceph-mon[105521]: osdmap e430: 8 total, 7 up, 8 in 2024-08-17T17:29:24.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:23 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:24.145 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:23 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:24.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:23 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:24.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:23 smithi130 ceph-mon[110068]: pgmap v272: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 673 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 182/1635 objects degraded (11.131%) 2024-08-17T17:29:24.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:23 smithi130 ceph-mon[110068]: osdmap e430: 8 total, 7 up, 8 in 2024-08-17T17:29:24.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:23 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:24.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:23 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:24.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:23 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:24.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:24 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:25.081 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:24 smithi179 ceph-mon[93519]: osdmap e431: 8 total, 7 up, 8 in 2024-08-17T17:29:25.116 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:24 smithi130 ceph-mon[110068]: osdmap e431: 8 total, 7 up, 8 in 2024-08-17T17:29:25.116 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:24 smithi130 ceph-mon[105521]: osdmap e431: 8 total, 7 up, 8 in 2024-08-17T17:29:25.259 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.CloneFlatten_SmallerDstObjSize (2171 ms) 2024-08-17T17:29:25.259 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Stress_SmallerDstObjSize 2024-08-17T17:29:25.281 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 46780899~317667 2024-08-17T17:29:25.294 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 1461828~2833814 2024-08-17T17:29:25.294 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 16354023~7208582 2024-08-17T17:29:25.308 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 1160631~6838871 2024-08-17T17:29:25.308 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 2883359~3207381 2024-08-17T17:29:25.321 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 60329174~1105199 2024-08-17T17:29:25.321 INFO:tasks.workunit.client.0.smithi130.stdout:write: A 78894813~4115278 2024-08-17T17:29:25.335 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 32705119~8070183 2024-08-17T17:29:25.335 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap0 2024-08-17T17:29:25.416 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[118233]: 2024-08-17T17:29:25.260+0000 7f624b29a640 -1 Falling back to public interface 2024-08-17T17:29:25.819 INFO:tasks.workunit.client.0.smithi130.stdout:clone image282 -> image283 2024-08-17T17:29:25.864 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 45128028~5995853 2024-08-17T17:29:25.886 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 33173999~4314691 2024-08-17T17:29:25.889 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 21572874~5982615 2024-08-17T17:29:25.910 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 4171989~2711742 2024-08-17T17:29:25.919 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 81769202~459880 2024-08-17T17:29:25.940 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 76883614~7002466 2024-08-17T17:29:25.944 INFO:tasks.workunit.client.0.smithi130.stdout:write: B 74428997~5370997 2024-08-17T17:29:25.958 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 16593419~2146529 2024-08-17T17:29:25.978 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap1 2024-08-17T17:29:26.068 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:25 smithi179 ceph-mon[93519]: pgmap v275: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 718 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 28 MiB/s rd, 11 MiB/s wr, 365 op/s; 195/1779 objects degraded (10.961%) 2024-08-17T17:29:26.068 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:25 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 195/1779 objects degraded (10.961%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:26.068 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:25 smithi179 ceph-mon[93519]: osdmap e432: 8 total, 7 up, 8 in 2024-08-17T17:29:26.068 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:25 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:26.068 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:25 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:26.079 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:25 smithi130 ceph-mon[105521]: pgmap v275: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 718 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 28 MiB/s rd, 11 MiB/s wr, 365 op/s; 195/1779 objects degraded (10.961%) 2024-08-17T17:29:26.079 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:25 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 195/1779 objects degraded (10.961%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:26.079 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:25 smithi130 ceph-mon[105521]: osdmap e432: 8 total, 7 up, 8 in 2024-08-17T17:29:26.079 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:25 smithi130 ceph-mon[110068]: pgmap v275: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 718 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 28 MiB/s rd, 11 MiB/s wr, 365 op/s; 195/1779 objects degraded (10.961%) 2024-08-17T17:29:26.080 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:25 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 195/1779 objects degraded (10.961%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:26.080 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:25 smithi130 ceph-mon[110068]: osdmap e432: 8 total, 7 up, 8 in 2024-08-17T17:29:26.827 INFO:tasks.workunit.client.0.smithi130.stdout:clone image283 -> image284 2024-08-17T17:29:26.883 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 49175264~7781619 2024-08-17T17:29:26.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:26 smithi130 ceph-mon[105521]: osdmap e433: 8 total, 7 up, 8 in 2024-08-17T17:29:26.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:26 smithi130 ceph-mon[110068]: osdmap e433: 8 total, 7 up, 8 in 2024-08-17T17:29:26.920 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 52932040~2127135 2024-08-17T17:29:26.922 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 78729806~1726976 2024-08-17T17:29:26.946 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 39396686~4209311 2024-08-17T17:29:26.954 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 54222534~6629676 2024-08-17T17:29:26.968 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 52085291~4992942 2024-08-17T17:29:26.970 INFO:tasks.workunit.client.0.smithi130.stdout:write: C 36154619~6904648 2024-08-17T17:29:26.990 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 78328203~4938632 2024-08-17T17:29:26.991 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap2 2024-08-17T17:29:27.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:26 smithi179 ceph-mon[93519]: osdmap e433: 8 total, 7 up, 8 in 2024-08-17T17:29:27.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:26.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:27.837 INFO:tasks.workunit.client.0.smithi130.stdout:clone image284 -> image285 2024-08-17T17:29:27.886 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:27 smithi179 ceph-mon[93519]: pgmap v278: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 718 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 28 MiB/s rd, 11 MiB/s wr, 365 op/s; 195/1779 objects degraded (10.961%) 2024-08-17T17:29:27.886 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:27 smithi179 ceph-mon[93519]: osdmap e434: 8 total, 7 up, 8 in 2024-08-17T17:29:27.887 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:27.887 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:27.887 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:27.887 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:27.911 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 133455509 2024-08-17T17:29:27.916 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 98979316~4931496 2024-08-17T17:29:27.930 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 125369240~8086269 2024-08-17T17:29:27.930 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 96140990~5318486 2024-08-17T17:29:27.945 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 72883313~8185616 2024-08-17T17:29:27.953 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 118573962~7541469 2024-08-17T17:29:27.978 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 75616837~4643364 2024-08-17T17:29:27.980 INFO:tasks.workunit.client.0.smithi130.stdout:write: D 82033329~7394830 2024-08-17T17:29:27.998 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 54220721~2246500 2024-08-17T17:29:28.009 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap3 2024-08-17T17:29:28.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:27 smithi130 ceph-mon[110068]: pgmap v278: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 718 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 28 MiB/s rd, 11 MiB/s wr, 365 op/s; 195/1779 objects degraded (10.961%) 2024-08-17T17:29:28.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:27 smithi130 ceph-mon[110068]: osdmap e434: 8 total, 7 up, 8 in 2024-08-17T17:29:28.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:28.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:28.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:28.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:28.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:27 smithi130 ceph-mon[105521]: pgmap v278: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 718 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 28 MiB/s rd, 11 MiB/s wr, 365 op/s; 195/1779 objects degraded (10.961%) 2024-08-17T17:29:28.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:27 smithi130 ceph-mon[105521]: osdmap e434: 8 total, 7 up, 8 in 2024-08-17T17:29:28.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:28.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:28.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:28.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:28.888 INFO:tasks.workunit.client.0.smithi130.stdout:clone image285 -> image286 2024-08-17T17:29:28.964 INFO:tasks.workunit.client.0.smithi130.stdout:resize: 108765879 2024-08-17T17:29:29.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:28 smithi179 ceph-mon[93519]: osdmap e435: 8 total, 7 up, 8 in 2024-08-17T17:29:29.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:29.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:29.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:29.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:29.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:28 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[118233]: 2024-08-17T17:29:28.884+0000 7f624b29a640 -1 osd.0 423 log_to_monitors true 2024-08-17T17:29:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:28 smithi130 ceph-mon[110068]: osdmap e435: 8 total, 7 up, 8 in 2024-08-17T17:29:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:29.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:28 smithi130 ceph-mon[105521]: osdmap e435: 8 total, 7 up, 8 in 2024-08-17T17:29:29.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:29.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:29.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:29.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:29.249 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Stress_SmallerDstObjSize (3990 ms) 2024-08-17T17:29:29.249 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize 2024-08-17T17:29:29.276 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:29:29.350 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize (100 ms) 2024-08-17T17:29:29.350 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingLargerDstObjSize 2024-08-17T17:29:29.370 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:29:29.440 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Snaps_StrippingLargerDstObjSize (91 ms) 2024-08-17T17:29:29.440 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Clone_StrippingLargerDstObjSize 2024-08-17T17:29:29.461 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:29:29.531 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Clone_StrippingLargerDstObjSize (91 ms) 2024-08-17T17:29:29.531 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize 2024-08-17T17:29:29.553 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:29:29.629 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize (98 ms) 2024-08-17T17:29:29.629 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Stress_StrippingLargerDstObjSize 2024-08-17T17:29:29.653 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:29:29.728 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Stress_StrippingLargerDstObjSize (98 ms) 2024-08-17T17:29:29.728 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize 2024-08-17T17:29:29.749 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:29:29.824 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize (97 ms) 2024-08-17T17:29:29.824 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize 2024-08-17T17:29:29.849 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:29:29.939 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize (115 ms) 2024-08-17T17:29:29.940 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Clone_StrippingSmallerDstObjSize 2024-08-17T17:29:29.963 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:29:30.037 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Clone_StrippingSmallerDstObjSize (98 ms) 2024-08-17T17:29:30.037 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize 2024-08-17T17:29:30.061 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:29:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:29 smithi179 ceph-mon[93519]: pgmap v281: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 943 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 50 MiB/s wr, 704 op/s; 236/2172 objects degraded (10.866%) 2024-08-17T17:29:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:29 smithi179 ceph-mon[93519]: osdmap e436: 8 total, 7 up, 8 in 2024-08-17T17:29:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:29 smithi179 ceph-mon[93519]: from='osd.0 [v2:172.21.15.130:6802/3901615741,v1:172.21.15.130:6803/3901615741]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:29:30.137 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize (100 ms) 2024-08-17T17:29:30.137 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestDeepCopy.Stress_StrippingSmallerDstObjSize 2024-08-17T17:29:30.159 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:29:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:29:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:29:29] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.51.0" 2024-08-17T17:29:30.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:29 smithi130 ceph-mon[105521]: pgmap v281: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 943 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 50 MiB/s wr, 704 op/s; 236/2172 objects degraded (10.866%) 2024-08-17T17:29:30.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:29 smithi130 ceph-mon[105521]: osdmap e436: 8 total, 7 up, 8 in 2024-08-17T17:29:30.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:29 smithi130 ceph-mon[105521]: from='osd.0 [v2:172.21.15.130:6802/3901615741,v1:172.21.15.130:6803/3901615741]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:29:30.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:29 smithi130 ceph-mon[110068]: pgmap v281: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 943 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 50 MiB/s wr, 704 op/s; 236/2172 objects degraded (10.866%) 2024-08-17T17:29:30.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:29 smithi130 ceph-mon[110068]: osdmap e436: 8 total, 7 up, 8 in 2024-08-17T17:29:30.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:29 smithi130 ceph-mon[110068]: from='osd.0 [v2:172.21.15.130:6802/3901615741,v1:172.21.15.130:6803/3901615741]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:29:30.168 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:29:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[118233]: 2024-08-17T17:29:29.900+0000 7f6243044640 -1 osd.0 423 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:29:30.231 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestDeepCopy.Stress_StrippingSmallerDstObjSize (94 ms) 2024-08-17T17:29:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:30.890 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 31 tests from TestDeepCopy (52853 ms total) 2024-08-17T17:29:30.890 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:29:30.890 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 51 tests from TestMirroring 2024-08-17T17:29:30.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:30 smithi130 ceph-mon[105521]: from='osd.0 [v2:172.21.15.130:6802/3901615741,v1:172.21.15.130:6803/3901615741]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:29:30.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:30 smithi130 ceph-mon[105521]: osdmap e437: 8 total, 7 up, 8 in 2024-08-17T17:29:30.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:30 smithi130 ceph-mon[105521]: from='osd.0 [v2:172.21.15.130:6802/3901615741,v1:172.21.15.130:6803/3901615741]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:29:30.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:30 smithi130 ceph-mon[105521]: pgmap v284: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 943 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 50 MiB/s wr, 704 op/s; 236/2172 objects degraded (10.866%) 2024-08-17T17:29:30.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:30 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 236/2172 objects degraded (10.866%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:30.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:30 smithi130 ceph-mon[110068]: from='osd.0 [v2:172.21.15.130:6802/3901615741,v1:172.21.15.130:6803/3901615741]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:29:30.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:30 smithi130 ceph-mon[110068]: osdmap e437: 8 total, 7 up, 8 in 2024-08-17T17:29:30.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:30 smithi130 ceph-mon[110068]: from='osd.0 [v2:172.21.15.130:6802/3901615741,v1:172.21.15.130:6803/3901615741]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:29:30.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:30 smithi130 ceph-mon[110068]: pgmap v284: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 943 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 50 MiB/s wr, 704 op/s; 236/2172 objects degraded (10.866%) 2024-08-17T17:29:30.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:30 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 236/2172 objects degraded (10.866%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:30 smithi179 ceph-mon[93519]: from='osd.0 [v2:172.21.15.130:6802/3901615741,v1:172.21.15.130:6803/3901615741]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:29:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:30 smithi179 ceph-mon[93519]: osdmap e437: 8 total, 7 up, 8 in 2024-08-17T17:29:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:30 smithi179 ceph-mon[93519]: from='osd.0 [v2:172.21.15.130:6802/3901615741,v1:172.21.15.130:6803/3901615741]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:29:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:30 smithi179 ceph-mon[93519]: pgmap v284: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 943 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 48 MiB/s rd, 50 MiB/s wr, 704 op/s; 236/2172 objects degraded (10.866%) 2024-08-17T17:29:31.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:30 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 236/2172 objects degraded (10.866%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:29:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:29:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: osd.0 [v2:172.21.15.130:6802/3901615741,v1:172.21.15.130:6803/3901615741] boot 2024-08-17T17:29:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: osdmap e438: 8 total, 8 up, 8 in 2024-08-17T17:29:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:31.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:31.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:31.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:31.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:31.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: osdmap e439: 8 total, 8 up, 8 in 2024-08-17T17:29:31.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:29:31.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:29:31.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:29:31.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:29:31.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: osd.0 [v2:172.21.15.130:6802/3901615741,v1:172.21.15.130:6803/3901615741] boot 2024-08-17T17:29:31.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: osdmap e438: 8 total, 8 up, 8 in 2024-08-17T17:29:31.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:31.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:31.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:31.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:31.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:31.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:31.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:31.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:31.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:31.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: osdmap e439: 8 total, 8 up, 8 in 2024-08-17T17:29:31.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:29:31.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:31 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:29:32.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:29:32.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:29:32.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: osd.0 [v2:172.21.15.130:6802/3901615741,v1:172.21.15.130:6803/3901615741] boot 2024-08-17T17:29:32.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: osdmap e438: 8 total, 8 up, 8 in 2024-08-17T17:29:32.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:32.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:32.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:32.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:32.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:32.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:29:32.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:32.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:32.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:32.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: osdmap e439: 8 total, 8 up, 8 in 2024-08-17T17:29:32.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:29:32.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:31 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:29:32.890 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage 2024-08-17T17:29:33.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:32 smithi130 ceph-mon[110068]: pgmap v287: 105 pgs: 32 unknown, 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:29:33.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:32 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:29:33.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:32 smithi130 ceph-mon[110068]: osdmap e440: 8 total, 8 up, 8 in 2024-08-17T17:29:33.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:32 smithi130 ceph-mon[105521]: pgmap v287: 105 pgs: 32 unknown, 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:29:33.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:32 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:29:33.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:32 smithi130 ceph-mon[105521]: osdmap e440: 8 total, 8 up, 8 in 2024-08-17T17:29:33.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:32 smithi179 ceph-mon[93519]: pgmap v287: 105 pgs: 32 unknown, 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:29:33.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:32 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:29:33.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:32 smithi179 ceph-mon[93519]: osdmap e440: 8 total, 8 up, 8 in 2024-08-17T17:29:34.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:33 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:29:34.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:33 smithi130 ceph-mon[105521]: osdmap e441: 8 total, 8 up, 8 in 2024-08-17T17:29:34.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:33 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:29:34.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:33 smithi130 ceph-mon[110068]: osdmap e441: 8 total, 8 up, 8 in 2024-08-17T17:29:34.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:33 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:29:34.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:33 smithi179 ceph-mon[93519]: osdmap e441: 8 total, 8 up, 8 in 2024-08-17T17:29:34.346 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:34 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:34.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-08-17T17:29:34.346 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:34 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:34.993 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage (2104 ms) 2024-08-17T17:29:34.993 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool 2024-08-17T17:29:35.091 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool (98 ms) 2024-08-17T17:29:35.091 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled 2024-08-17T17:29:35.167 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled (76 ms) 2024-08-17T17:29:35.167 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage 2024-08-17T17:29:35.257 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage (90 ms) 2024-08-17T17:29:35.257 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap 2024-08-17T17:29:35.341 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap (83 ms) 2024-08-17T17:29:35.341 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModePool 2024-08-17T17:29:35.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:34 smithi179 ceph-mon[93519]: pgmap v290: 105 pgs: 7 unknown, 9 active+undersized, 2 active+undersized+degraded, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/78 objects degraded (3.846%) 2024-08-17T17:29:35.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:34 smithi179 ceph-mon[93519]: osdmap e442: 8 total, 8 up, 8 in 2024-08-17T17:29:35.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:34 smithi130 ceph-mon[105521]: pgmap v290: 105 pgs: 7 unknown, 9 active+undersized, 2 active+undersized+degraded, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/78 objects degraded (3.846%) 2024-08-17T17:29:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:34 smithi130 ceph-mon[105521]: osdmap e442: 8 total, 8 up, 8 in 2024-08-17T17:29:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:34 smithi130 ceph-mon[110068]: pgmap v290: 105 pgs: 7 unknown, 9 active+undersized, 2 active+undersized+degraded, 87 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/78 objects degraded (3.846%) 2024-08-17T17:29:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:34 smithi130 ceph-mon[110068]: osdmap e442: 8 total, 8 up, 8 in 2024-08-17T17:29:35.426 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModePool (85 ms) 2024-08-17T17:29:35.426 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled 2024-08-17T17:29:35.530 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled (103 ms) 2024-08-17T17:29:35.530 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.DisableImageMirrorWithPeer 2024-08-17T17:29:35.530 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:29:35.530 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.DisableImageMirrorWithPeer (0 ms) 2024-08-17T17:29:35.530 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.DisableJournalingWithPeer 2024-08-17T17:29:35.530 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:29:35.530 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.DisableJournalingWithPeer (0 ms) 2024-08-17T17:29:35.530 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling 2024-08-17T17:29:35.599 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling (70 ms) 2024-08-17T17:29:35.599 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling 2024-08-17T17:29:35.668 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling (69 ms) 2024-08-17T17:29:35.668 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling 2024-08-17T17:29:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:35 smithi179 ceph-mon[93519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:29:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:35 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 3/78 objects degraded (3.846%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:36.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:35.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:36.094 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:35 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:36.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:35 smithi130 ceph-mon[105521]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:29:36.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:35 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 3/78 objects degraded (3.846%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:36.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:35 smithi130 ceph-mon[110068]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:29:36.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:35 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 3/78 objects degraded (3.846%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:36.986 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling (1318 ms) 2024-08-17T17:29:36.986 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock 2024-08-17T17:29:37.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:36 smithi179 ceph-mon[93519]: osdmap e443: 8 total, 8 up, 8 in 2024-08-17T17:29:37.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:36 smithi179 ceph-mon[93519]: pgmap v293: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:29:37.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:36 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:36.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:37.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:36 smithi130 ceph-mon[105521]: osdmap e443: 8 total, 8 up, 8 in 2024-08-17T17:29:37.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:36 smithi130 ceph-mon[105521]: pgmap v293: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:29:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:36 smithi130 ceph-mon[110068]: osdmap e443: 8 total, 8 up, 8 in 2024-08-17T17:29:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:36 smithi130 ceph-mon[110068]: pgmap v293: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:29:38.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:37 smithi179 ceph-mon[93519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/78 objects degraded (3.846%), 2 pgs degraded) 2024-08-17T17:29:38.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:37 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T17:29:38.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:37 smithi179 ceph-mon[93519]: osdmap e444: 8 total, 8 up, 8 in 2024-08-17T17:29:38.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:37 smithi130 ceph-mon[105521]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/78 objects degraded (3.846%), 2 pgs degraded) 2024-08-17T17:29:38.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:37 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T17:29:38.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:37 smithi130 ceph-mon[105521]: osdmap e444: 8 total, 8 up, 8 in 2024-08-17T17:29:38.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:37 smithi130 ceph-mon[110068]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/78 objects degraded (3.846%), 2 pgs degraded) 2024-08-17T17:29:38.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:37 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T17:29:38.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:37 smithi130 ceph-mon[110068]: osdmap e444: 8 total, 8 up, 8 in 2024-08-17T17:29:38.991 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock (2005 ms) 2024-08-17T17:29:38.992 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeDisabled 2024-08-17T17:29:39.077 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeDisabled (86 ms) 2024-08-17T17:29:39.077 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage 2024-08-17T17:29:39.160 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage (83 ms) 2024-08-17T17:29:39.160 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool 2024-08-17T17:29:39.256 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool (96 ms) 2024-08-17T17:29:39.256 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling 2024-08-17T17:29:39.333 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling (77 ms) 2024-08-17T17:29:39.334 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling 2024-08-17T17:29:39.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:38 smithi179 ceph-mon[93519]: osdmap e445: 8 total, 8 up, 8 in 2024-08-17T17:29:39.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:38 smithi179 ceph-mon[93519]: pgmap v296: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 347 KiB/s rd, 55 KiB/s wr, 503 op/s 2024-08-17T17:29:39.405 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling (71 ms) 2024-08-17T17:29:39.405 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeDisabled 2024-08-17T17:29:39.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:38 smithi130 ceph-mon[105521]: osdmap e445: 8 total, 8 up, 8 in 2024-08-17T17:29:39.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:38 smithi130 ceph-mon[105521]: pgmap v296: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 347 KiB/s rd, 55 KiB/s wr, 503 op/s 2024-08-17T17:29:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:38 smithi130 ceph-mon[110068]: osdmap e445: 8 total, 8 up, 8 in 2024-08-17T17:29:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:38 smithi130 ceph-mon[110068]: pgmap v296: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 347 KiB/s rd, 55 KiB/s wr, 503 op/s 2024-08-17T17:29:39.489 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeDisabled (85 ms) 2024-08-17T17:29:39.489 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage 2024-08-17T17:29:39.577 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage (88 ms) 2024-08-17T17:29:39.577 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled 2024-08-17T17:29:40.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:39 smithi130 ceph-mon[105521]: osdmap e446: 8 total, 8 up, 8 in 2024-08-17T17:29:40.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:39 smithi130 ceph-mon[110068]: osdmap e446: 8 total, 8 up, 8 in 2024-08-17T17:29:40.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:29:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:29:39] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.51.0" 2024-08-17T17:29:40.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:39 smithi179 ceph-mon[93519]: osdmap e446: 8 total, 8 up, 8 in 2024-08-17T17:29:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:41.019 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled (1442 ms) 2024-08-17T17:29:41.019 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModePool 2024-08-17T17:29:41.129 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModePool (110 ms) 2024-08-17T17:29:41.129 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModePool 2024-08-17T17:29:41.224 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModePool (95 ms) 2024-08-17T17:29:41.224 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModeImage 2024-08-17T17:29:41.317 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModeImage (93 ms) 2024-08-17T17:29:41.317 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode 2024-08-17T17:29:41.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:40 smithi179 ceph-mon[93519]: osdmap e447: 8 total, 8 up, 8 in 2024-08-17T17:29:41.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:40 smithi179 ceph-mon[93519]: pgmap v299: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 513 KiB/s rd, 58 KiB/s wr, 727 op/s 2024-08-17T17:29:41.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:40 smithi130 ceph-mon[105521]: osdmap e447: 8 total, 8 up, 8 in 2024-08-17T17:29:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:40 smithi130 ceph-mon[105521]: pgmap v299: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 513 KiB/s rd, 58 KiB/s wr, 727 op/s 2024-08-17T17:29:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:40 smithi130 ceph-mon[110068]: osdmap e447: 8 total, 8 up, 8 in 2024-08-17T17:29:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:40 smithi130 ceph-mon[110068]: pgmap v299: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 513 KiB/s rd, 58 KiB/s wr, 727 op/s 2024-08-17T17:29:41.464 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode (147 ms) 2024-08-17T17:29:41.464 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode 2024-08-17T17:29:41.610 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode (146 ms) 2024-08-17T17:29:41.610 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode 2024-08-17T17:29:41.739 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode (129 ms) 2024-08-17T17:29:41.739 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode 2024-08-17T17:29:41.870 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode (131 ms) 2024-08-17T17:29:41.870 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode 2024-08-17T17:29:42.014 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode (144 ms) 2024-08-17T17:29:42.014 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode 2024-08-17T17:29:42.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:41 smithi130 ceph-mon[105521]: osdmap e448: 8 total, 8 up, 8 in 2024-08-17T17:29:42.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:41 smithi130 ceph-mon[110068]: osdmap e448: 8 total, 8 up, 8 in 2024-08-17T17:29:42.168 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode (154 ms) 2024-08-17T17:29:42.168 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageEnabled 2024-08-17T17:29:42.245 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageEnabled (77 ms) 2024-08-17T17:29:42.245 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDisabled 2024-08-17T17:29:42.315 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDisabled (70 ms) 2024-08-17T17:29:42.315 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.RemoveImage_With_ImageWithoutJournal 2024-08-17T17:29:42.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:41 smithi179 ceph-mon[93519]: osdmap e448: 8 total, 8 up, 8 in 2024-08-17T17:29:42.376 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.RemoveImage_With_ImageWithoutJournal (61 ms) 2024-08-17T17:29:42.377 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDemoted 2024-08-17T17:29:42.484 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDemoted (108 ms) 2024-08-17T17:29:42.485 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.TrashMoveRestore_PoolMode 2024-08-17T17:29:42.599 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.TrashMoveRestore_PoolMode (115 ms) 2024-08-17T17:29:42.599 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled 2024-08-17T17:29:42.699 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled (100 ms) 2024-08-17T17:29:42.699 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled 2024-08-17T17:29:42.808 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled (109 ms) 2024-08-17T17:29:42.808 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.MirrorStatusList 2024-08-17T17:29:42.875 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.MirrorStatusList (67 ms) 2024-08-17T17:29:42.875 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.RemoveBootstrapped 2024-08-17T17:29:42.963 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.RemoveBootstrapped (88 ms) 2024-08-17T17:29:42.963 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.AioPromoteDemote 2024-08-17T17:29:43.300 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.AioPromoteDemote (337 ms) 2024-08-17T17:29:43.300 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.AioGetInfo 2024-08-17T17:29:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:43 smithi179 ceph-mon[93519]: pgmap v301: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 505 KiB/s rd, 57 KiB/s wr, 715 op/s 2024-08-17T17:29:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:43.623 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.AioGetInfo (323 ms) 2024-08-17T17:29:43.623 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.AioGetStatus 2024-08-17T17:29:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:43 smithi130 ceph-mon[110068]: pgmap v301: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 505 KiB/s rd, 57 KiB/s wr, 715 op/s 2024-08-17T17:29:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:43 smithi130 ceph-mon[105521]: pgmap v301: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 505 KiB/s rd, 57 KiB/s wr, 715 op/s 2024-08-17T17:29:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:43.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:29:43.827 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.AioGetStatus (204 ms) 2024-08-17T17:29:43.828 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.SiteName 2024-08-17T17:29:43.837 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.SiteName (10 ms) 2024-08-17T17:29:43.837 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.Bootstrap 2024-08-17T17:29:44.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:44.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-08-17T17:29:44.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:44.594 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.Bootstrap (757 ms) 2024-08-17T17:29:44.594 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.PeerDirection 2024-08-17T17:29:44.602 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.PeerDirection (8 ms) 2024-08-17T17:29:44.602 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.Snapshot 2024-08-17T17:29:45.019 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' 2024-08-17T17:29:45.019 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.019 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.019 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.020 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-08-17T17:29:45.020 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.020 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"} : dispatch 2024-08-17T17:29:45.020 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' 2024-08-17T17:29:45.020 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"} : dispatch 2024-08-17T17:29:45.020 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"} : dispatch 2024-08-17T17:29:45.020 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-08-17T17:29:45.020 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.020 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0f7573f4-860c-47df-a955-f6e1ee16fa9d"} : dispatch 2024-08-17T17:29:45.020 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0f7573f4-860c-47df-a955-f6e1ee16fa9d"} : dispatch 2024-08-17T17:29:45.020 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0f7573f4-860c-47df-a955-f6e1ee16fa9d"}]': finished 2024-08-17T17:29:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' 2024-08-17T17:29:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-08-17T17:29:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"} : dispatch 2024-08-17T17:29:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' 2024-08-17T17:29:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"} : dispatch 2024-08-17T17:29:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"} : dispatch 2024-08-17T17:29:45.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-08-17T17:29:45.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0f7573f4-860c-47df-a955-f6e1ee16fa9d"} : dispatch 2024-08-17T17:29:45.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0f7573f4-860c-47df-a955-f6e1ee16fa9d"} : dispatch 2024-08-17T17:29:45.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0f7573f4-860c-47df-a955-f6e1ee16fa9d"}]': finished 2024-08-17T17:29:45.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' 2024-08-17T17:29:45.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-08-17T17:29:45.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"} : dispatch 2024-08-17T17:29:45.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' 2024-08-17T17:29:45.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"} : dispatch 2024-08-17T17:29:45.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"} : dispatch 2024-08-17T17:29:45.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-08-17T17:29:45.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-08-17T17:29:45.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0f7573f4-860c-47df-a955-f6e1ee16fa9d"} : dispatch 2024-08-17T17:29:45.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0f7573f4-860c-47df-a955-f6e1ee16fa9d"} : dispatch 2024-08-17T17:29:45.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/0f7573f4-860c-47df-a955-f6e1ee16fa9d"}]': finished 2024-08-17T17:29:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:45 smithi179 ceph-mon[93519]: pgmap v302: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 783 KiB/s rd, 108 KiB/s wr, 1.11k op/s 2024-08-17T17:29:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:45 smithi179 ceph-mon[93519]: osdmap e449: 8 total, 8 up, 8 in 2024-08-17T17:29:46.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:45 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:46.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:45 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:45 smithi130 ceph-mon[105521]: pgmap v302: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 783 KiB/s rd, 108 KiB/s wr, 1.11k op/s 2024-08-17T17:29:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:45 smithi130 ceph-mon[105521]: osdmap e449: 8 total, 8 up, 8 in 2024-08-17T17:29:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:45 smithi130 ceph-mon[110068]: pgmap v302: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 783 KiB/s rd, 108 KiB/s wr, 1.11k op/s 2024-08-17T17:29:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:45 smithi130 ceph-mon[110068]: osdmap e449: 8 total, 8 up, 8 in 2024-08-17T17:29:47.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:47.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:47 smithi130 ceph-mon[105521]: pgmap v304: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1001 KiB/s rd, 135 KiB/s wr, 1.40k op/s 2024-08-17T17:29:47.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:47 smithi130 ceph-mon[105521]: osdmap e450: 8 total, 8 up, 8 in 2024-08-17T17:29:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:47 smithi130 ceph-mon[110068]: pgmap v304: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1001 KiB/s rd, 135 KiB/s wr, 1.40k op/s 2024-08-17T17:29:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:47 smithi130 ceph-mon[110068]: osdmap e450: 8 total, 8 up, 8 in 2024-08-17T17:29:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:47 smithi179 ceph-mon[93519]: pgmap v304: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1001 KiB/s rd, 135 KiB/s wr, 1.40k op/s 2024-08-17T17:29:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:47 smithi179 ceph-mon[93519]: osdmap e450: 8 total, 8 up, 8 in 2024-08-17T17:29:48.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:48 smithi130 ceph-mon[105521]: osdmap e451: 8 total, 8 up, 8 in 2024-08-17T17:29:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:48 smithi130 ceph-mon[110068]: osdmap e451: 8 total, 8 up, 8 in 2024-08-17T17:29:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:48 smithi179 ceph-mon[93519]: osdmap e451: 8 total, 8 up, 8 in 2024-08-17T17:29:49.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:49 smithi130 ceph-mon[105521]: pgmap v307: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 233 KiB/s wr, 2.15k op/s 2024-08-17T17:29:49.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:49 smithi130 ceph-mon[105521]: osdmap e452: 8 total, 8 up, 8 in 2024-08-17T17:29:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:49 smithi130 ceph-mon[110068]: pgmap v307: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 233 KiB/s wr, 2.15k op/s 2024-08-17T17:29:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:49 smithi130 ceph-mon[110068]: osdmap e452: 8 total, 8 up, 8 in 2024-08-17T17:29:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:49 smithi179 ceph-mon[93519]: pgmap v307: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 233 KiB/s wr, 2.15k op/s 2024-08-17T17:29:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:49 smithi179 ceph-mon[93519]: osdmap e452: 8 total, 8 up, 8 in 2024-08-17T17:29:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:29:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:29:49] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.51.0" 2024-08-17T17:29:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:50 smithi179 ceph-mon[93519]: osdmap e453: 8 total, 8 up, 8 in 2024-08-17T17:29:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:50 smithi179 ceph-mon[93519]: osdmap e454: 8 total, 8 up, 8 in 2024-08-17T17:29:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:50 smithi130 ceph-mon[110068]: osdmap e453: 8 total, 8 up, 8 in 2024-08-17T17:29:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:50 smithi130 ceph-mon[110068]: osdmap e454: 8 total, 8 up, 8 in 2024-08-17T17:29:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:50 smithi130 ceph-mon[105521]: osdmap e453: 8 total, 8 up, 8 in 2024-08-17T17:29:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:50 smithi130 ceph-mon[105521]: osdmap e454: 8 total, 8 up, 8 in 2024-08-17T17:29:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:51 smithi130 ceph-mon[110068]: pgmap v310: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 856 KiB/s rd, 150 KiB/s wr, 1.16k op/s 2024-08-17T17:29:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:51 smithi130 ceph-mon[105521]: pgmap v310: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 856 KiB/s rd, 150 KiB/s wr, 1.16k op/s 2024-08-17T17:29:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:51 smithi179 ceph-mon[93519]: pgmap v310: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 856 KiB/s rd, 150 KiB/s wr, 1.16k op/s 2024-08-17T17:29:52.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:52 smithi130 ceph-mon[105521]: osdmap e455: 8 total, 8 up, 8 in 2024-08-17T17:29:52.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:52 smithi130 ceph-mon[110068]: osdmap e455: 8 total, 8 up, 8 in 2024-08-17T17:29:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:52 smithi179 ceph-mon[93519]: osdmap e455: 8 total, 8 up, 8 in 2024-08-17T17:29:53.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:53 smithi130 ceph-mon[105521]: pgmap v313: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 4.2 KiB/s wr, 51 op/s 2024-08-17T17:29:53.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:53 smithi130 ceph-mon[105521]: osdmap e456: 8 total, 8 up, 8 in 2024-08-17T17:29:53.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:53 smithi130 ceph-mon[110068]: pgmap v313: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 4.2 KiB/s wr, 51 op/s 2024-08-17T17:29:53.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:53 smithi130 ceph-mon[110068]: osdmap e456: 8 total, 8 up, 8 in 2024-08-17T17:29:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:53 smithi179 ceph-mon[93519]: pgmap v313: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 4.2 KiB/s wr, 51 op/s 2024-08-17T17:29:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:53 smithi179 ceph-mon[93519]: osdmap e456: 8 total, 8 up, 8 in 2024-08-17T17:29:54.410 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:54.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:54 smithi130 ceph-mon[105521]: osdmap e457: 8 total, 8 up, 8 in 2024-08-17T17:29:54.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:54 smithi130 ceph-mon[110068]: osdmap e457: 8 total, 8 up, 8 in 2024-08-17T17:29:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:54 smithi179 ceph-mon[93519]: osdmap e457: 8 total, 8 up, 8 in 2024-08-17T17:29:55.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:55 smithi179 ceph-mon[93519]: pgmap v316: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 1.5 KiB/s wr, 12 op/s 2024-08-17T17:29:55.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:55 smithi179 ceph-mon[93519]: osdmap e458: 8 total, 8 up, 8 in 2024-08-17T17:29:55.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:55 smithi179 ceph-mon[93519]: osdmap e459: 8 total, 8 up, 8 in 2024-08-17T17:29:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:55 smithi130 ceph-mon[105521]: pgmap v316: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 1.5 KiB/s wr, 12 op/s 2024-08-17T17:29:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:55 smithi130 ceph-mon[105521]: osdmap e458: 8 total, 8 up, 8 in 2024-08-17T17:29:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:55 smithi130 ceph-mon[105521]: osdmap e459: 8 total, 8 up, 8 in 2024-08-17T17:29:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:55 smithi130 ceph-mon[110068]: pgmap v316: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 1.5 KiB/s wr, 12 op/s 2024-08-17T17:29:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:55 smithi130 ceph-mon[110068]: osdmap e458: 8 total, 8 up, 8 in 2024-08-17T17:29:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:55 smithi130 ceph-mon[110068]: osdmap e459: 8 total, 8 up, 8 in 2024-08-17T17:29:56.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:55 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:56.093 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:55 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:55.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:57.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:29:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:29:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi130\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi130\", job=\"node\", machine=\"x86_64\", nodename=\"smithi130\", release=\"5.14.0-480.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:29:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:57 smithi130 ceph-mon[105521]: pgmap v319: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 7.2 KiB/s wr, 97 op/s 2024-08-17T17:29:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:57 smithi130 ceph-mon[105521]: osdmap e460: 8 total, 8 up, 8 in 2024-08-17T17:29:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:57.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:57 smithi130 ceph-mon[110068]: pgmap v319: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 7.2 KiB/s wr, 97 op/s 2024-08-17T17:29:57.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:57 smithi130 ceph-mon[110068]: osdmap e460: 8 total, 8 up, 8 in 2024-08-17T17:29:57.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:57 smithi179 ceph-mon[93519]: pgmap v319: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 7.2 KiB/s wr, 97 op/s 2024-08-17T17:29:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:57 smithi179 ceph-mon[93519]: osdmap e460: 8 total, 8 up, 8 in 2024-08-17T17:29:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:58 smithi130 ceph-mon[105521]: osdmap e461: 8 total, 8 up, 8 in 2024-08-17T17:29:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:58 smithi130 ceph-mon[110068]: osdmap e461: 8 total, 8 up, 8 in 2024-08-17T17:29:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:58 smithi179 ceph-mon[93519]: osdmap e461: 8 total, 8 up, 8 in 2024-08-17T17:29:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:59 smithi130 ceph-mon[110068]: pgmap v322: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 69 KiB/s rd, 6.0 KiB/s wr, 93 op/s 2024-08-17T17:29:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:29:59 smithi130 ceph-mon[110068]: osdmap e462: 8 total, 8 up, 8 in 2024-08-17T17:29:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:59 smithi130 ceph-mon[105521]: pgmap v322: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 69 KiB/s rd, 6.0 KiB/s wr, 93 op/s 2024-08-17T17:29:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:29:59 smithi130 ceph-mon[105521]: osdmap e462: 8 total, 8 up, 8 in 2024-08-17T17:29:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:59 smithi179 ceph-mon[93519]: pgmap v322: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 69 KiB/s rd, 6.0 KiB/s wr, 93 op/s 2024-08-17T17:29:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:29:59 smithi179 ceph-mon[93519]: osdmap e462: 8 total, 8 up, 8 in 2024-08-17T17:30:00.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:29:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:29:59] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-08-17T17:30:00.448 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.Snapshot (15846 ms) 2024-08-17T17:30:00.448 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.SnapshotRemoveOnDisable 2024-08-17T17:30:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:00 smithi179 ceph-mon[93519]: osdmap e463: 8 total, 8 up, 8 in 2024-08-17T17:30:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:00 smithi179 ceph-mon[93519]: overall HEALTH_OK 2024-08-17T17:30:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:00 smithi179 ceph-mon[93519]: osdmap e464: 8 total, 8 up, 8 in 2024-08-17T17:30:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:30:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:30:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:30:00.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:00 smithi130 ceph-mon[105521]: osdmap e463: 8 total, 8 up, 8 in 2024-08-17T17:30:00.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:00 smithi130 ceph-mon[105521]: overall HEALTH_OK 2024-08-17T17:30:00.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:00 smithi130 ceph-mon[105521]: osdmap e464: 8 total, 8 up, 8 in 2024-08-17T17:30:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:00 smithi130 ceph-mon[110068]: osdmap e463: 8 total, 8 up, 8 in 2024-08-17T17:30:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:00 smithi130 ceph-mon[110068]: overall HEALTH_OK 2024-08-17T17:30:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:00 smithi130 ceph-mon[110068]: osdmap e464: 8 total, 8 up, 8 in 2024-08-17T17:30:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:01 smithi130 ceph-mon[110068]: pgmap v325: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 5.2 KiB/s wr, 75 op/s 2024-08-17T17:30:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:01 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ddcff9d0-1665-49ef-ac10-841f031e7801"} : dispatch 2024-08-17T17:30:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:01 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ddcff9d0-1665-49ef-ac10-841f031e7801"} : dispatch 2024-08-17T17:30:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:01 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ddcff9d0-1665-49ef-ac10-841f031e7801"}]': finished 2024-08-17T17:30:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:01 smithi130 ceph-mon[105521]: pgmap v325: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 5.2 KiB/s wr, 75 op/s 2024-08-17T17:30:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:01 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ddcff9d0-1665-49ef-ac10-841f031e7801"} : dispatch 2024-08-17T17:30:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:01 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ddcff9d0-1665-49ef-ac10-841f031e7801"} : dispatch 2024-08-17T17:30:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:01 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ddcff9d0-1665-49ef-ac10-841f031e7801"}]': finished 2024-08-17T17:30:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:01 smithi179 ceph-mon[93519]: pgmap v325: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s rd, 5.2 KiB/s wr, 75 op/s 2024-08-17T17:30:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:01 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ddcff9d0-1665-49ef-ac10-841f031e7801"} : dispatch 2024-08-17T17:30:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:01 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ddcff9d0-1665-49ef-ac10-841f031e7801"} : dispatch 2024-08-17T17:30:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:01 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ddcff9d0-1665-49ef-ac10-841f031e7801"}]': finished 2024-08-17T17:30:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:02 smithi179 ceph-mon[93519]: osdmap e465: 8 total, 8 up, 8 in 2024-08-17T17:30:02.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:02 smithi130 ceph-mon[105521]: osdmap e465: 8 total, 8 up, 8 in 2024-08-17T17:30:02.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:02 smithi130 ceph-mon[110068]: osdmap e465: 8 total, 8 up, 8 in 2024-08-17T17:30:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:03 smithi179 ceph-mon[93519]: pgmap v328: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 5.0 KiB/s wr, 67 op/s 2024-08-17T17:30:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:03 smithi179 ceph-mon[93519]: osdmap e466: 8 total, 8 up, 8 in 2024-08-17T17:30:03.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:03 smithi130 ceph-mon[105521]: pgmap v328: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 5.0 KiB/s wr, 67 op/s 2024-08-17T17:30:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:03 smithi130 ceph-mon[105521]: osdmap e466: 8 total, 8 up, 8 in 2024-08-17T17:30:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:03 smithi130 ceph-mon[110068]: pgmap v328: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 5.0 KiB/s wr, 67 op/s 2024-08-17T17:30:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:03 smithi130 ceph-mon[110068]: osdmap e466: 8 total, 8 up, 8 in 2024-08-17T17:30:04.460 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:30:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:30:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)\", cluster_addr=\"172.21.15.130\", device_class=\"ssd\", hostname=\"smithi130\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.130\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:30:04.557 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.SnapshotRemoveOnDisable (4108 ms) 2024-08-17T17:30:04.557 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.SnapshotUnlinkPeer 2024-08-17T17:30:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:04 smithi179 ceph-mon[93519]: osdmap e467: 8 total, 8 up, 8 in 2024-08-17T17:30:04.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:04 smithi130 ceph-mon[105521]: osdmap e467: 8 total, 8 up, 8 in 2024-08-17T17:30:04.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:04 smithi130 ceph-mon[110068]: osdmap e467: 8 total, 8 up, 8 in 2024-08-17T17:30:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:05 smithi179 ceph-mon[93519]: pgmap v331: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 7.0 KiB/s wr, 69 op/s 2024-08-17T17:30:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:05 smithi179 ceph-mon[93519]: osdmap e468: 8 total, 8 up, 8 in 2024-08-17T17:30:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:05 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/26d2cc61-41c2-4da9-82cb-656fdccdff02"} : dispatch 2024-08-17T17:30:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:05 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/26d2cc61-41c2-4da9-82cb-656fdccdff02"} : dispatch 2024-08-17T17:30:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:05 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/26d2cc61-41c2-4da9-82cb-656fdccdff02"}]': finished 2024-08-17T17:30:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:05 smithi179 ceph-mon[93519]: osdmap e469: 8 total, 8 up, 8 in 2024-08-17T17:30:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:05 smithi130 ceph-mon[105521]: pgmap v331: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 7.0 KiB/s wr, 69 op/s 2024-08-17T17:30:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:05 smithi130 ceph-mon[105521]: osdmap e468: 8 total, 8 up, 8 in 2024-08-17T17:30:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:05 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/26d2cc61-41c2-4da9-82cb-656fdccdff02"} : dispatch 2024-08-17T17:30:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:05 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/26d2cc61-41c2-4da9-82cb-656fdccdff02"} : dispatch 2024-08-17T17:30:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:05 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/26d2cc61-41c2-4da9-82cb-656fdccdff02"}]': finished 2024-08-17T17:30:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:05 smithi130 ceph-mon[105521]: osdmap e469: 8 total, 8 up, 8 in 2024-08-17T17:30:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:05 smithi130 ceph-mon[110068]: pgmap v331: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 7.0 KiB/s wr, 69 op/s 2024-08-17T17:30:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:05 smithi130 ceph-mon[110068]: osdmap e468: 8 total, 8 up, 8 in 2024-08-17T17:30:05.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:05 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/26d2cc61-41c2-4da9-82cb-656fdccdff02"} : dispatch 2024-08-17T17:30:05.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:05 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/26d2cc61-41c2-4da9-82cb-656fdccdff02"} : dispatch 2024-08-17T17:30:05.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:05 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/26d2cc61-41c2-4da9-82cb-656fdccdff02"}]': finished 2024-08-17T17:30:05.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:05 smithi130 ceph-mon[110068]: osdmap e469: 8 total, 8 up, 8 in 2024-08-17T17:30:07.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:07 smithi130 ceph-mon[105521]: pgmap v334: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 7.5 KiB/s wr, 95 op/s 2024-08-17T17:30:07.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:07 smithi130 ceph-mon[105521]: osdmap e470: 8 total, 8 up, 8 in 2024-08-17T17:30:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:07 smithi130 ceph-mon[110068]: pgmap v334: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 7.5 KiB/s wr, 95 op/s 2024-08-17T17:30:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:07 smithi130 ceph-mon[110068]: osdmap e470: 8 total, 8 up, 8 in 2024-08-17T17:30:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:07 smithi179 ceph-mon[93519]: pgmap v334: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 7.5 KiB/s wr, 95 op/s 2024-08-17T17:30:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:07 smithi179 ceph-mon[93519]: osdmap e470: 8 total, 8 up, 8 in 2024-08-17T17:30:08.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:08 smithi130 ceph-mon[105521]: osdmap e471: 8 total, 8 up, 8 in 2024-08-17T17:30:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:08 smithi130 ceph-mon[110068]: osdmap e471: 8 total, 8 up, 8 in 2024-08-17T17:30:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:08 smithi179 ceph-mon[93519]: osdmap e471: 8 total, 8 up, 8 in 2024-08-17T17:30:09.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:09 smithi130 ceph-mon[105521]: pgmap v337: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 131 KiB/s rd, 10 KiB/s wr, 176 op/s 2024-08-17T17:30:09.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:09 smithi130 ceph-mon[105521]: osdmap e472: 8 total, 8 up, 8 in 2024-08-17T17:30:09.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:09 smithi130 ceph-mon[110068]: pgmap v337: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 131 KiB/s rd, 10 KiB/s wr, 176 op/s 2024-08-17T17:30:09.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:09 smithi130 ceph-mon[110068]: osdmap e472: 8 total, 8 up, 8 in 2024-08-17T17:30:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:09 smithi179 ceph-mon[93519]: pgmap v337: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 131 KiB/s rd, 10 KiB/s wr, 176 op/s 2024-08-17T17:30:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:09 smithi179 ceph-mon[93519]: osdmap e472: 8 total, 8 up, 8 in 2024-08-17T17:30:10.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:30:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:30:09] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-08-17T17:30:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:30:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:30:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:30:10.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:10 smithi179 ceph-mon[93519]: osdmap e473: 8 total, 8 up, 8 in 2024-08-17T17:30:10.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:10 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/855b5130-6e41-471d-95f5-4d2330c5d009"} : dispatch 2024-08-17T17:30:10.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:10 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/855b5130-6e41-471d-95f5-4d2330c5d009"} : dispatch 2024-08-17T17:30:10.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:10 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/855b5130-6e41-471d-95f5-4d2330c5d009"}]': finished 2024-08-17T17:30:10.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:10 smithi179 ceph-mon[93519]: osdmap e474: 8 total, 8 up, 8 in 2024-08-17T17:30:10.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:10 smithi130 ceph-mon[105521]: osdmap e473: 8 total, 8 up, 8 in 2024-08-17T17:30:10.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:10 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/855b5130-6e41-471d-95f5-4d2330c5d009"} : dispatch 2024-08-17T17:30:10.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:10 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/855b5130-6e41-471d-95f5-4d2330c5d009"} : dispatch 2024-08-17T17:30:10.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:10 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/855b5130-6e41-471d-95f5-4d2330c5d009"}]': finished 2024-08-17T17:30:10.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:10 smithi130 ceph-mon[105521]: osdmap e474: 8 total, 8 up, 8 in 2024-08-17T17:30:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:10 smithi130 ceph-mon[110068]: osdmap e473: 8 total, 8 up, 8 in 2024-08-17T17:30:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:10 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/855b5130-6e41-471d-95f5-4d2330c5d009"} : dispatch 2024-08-17T17:30:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:10 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/855b5130-6e41-471d-95f5-4d2330c5d009"} : dispatch 2024-08-17T17:30:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:10 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/855b5130-6e41-471d-95f5-4d2330c5d009"}]': finished 2024-08-17T17:30:10.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:10 smithi130 ceph-mon[110068]: osdmap e474: 8 total, 8 up, 8 in 2024-08-17T17:30:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:11 smithi130 ceph-mon[105521]: pgmap v340: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 11 KiB/s wr, 153 op/s 2024-08-17T17:30:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:11 smithi130 ceph-mon[110068]: pgmap v340: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 11 KiB/s wr, 153 op/s 2024-08-17T17:30:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:11 smithi179 ceph-mon[93519]: pgmap v340: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 11 KiB/s wr, 153 op/s 2024-08-17T17:30:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:12 smithi179 ceph-mon[93519]: osdmap e475: 8 total, 8 up, 8 in 2024-08-17T17:30:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:12 smithi130 ceph-mon[105521]: osdmap e475: 8 total, 8 up, 8 in 2024-08-17T17:30:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:12 smithi130 ceph-mon[110068]: osdmap e475: 8 total, 8 up, 8 in 2024-08-17T17:30:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:13.745 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:13 smithi179 ceph-mon[93519]: pgmap v343: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 767 B/s wr, 3 op/s 2024-08-17T17:30:13.745 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:13 smithi179 ceph-mon[93519]: osdmap e476: 8 total, 8 up, 8 in 2024-08-17T17:30:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:13 smithi130 ceph-mon[105521]: pgmap v343: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 767 B/s wr, 3 op/s 2024-08-17T17:30:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:13 smithi130 ceph-mon[105521]: osdmap e476: 8 total, 8 up, 8 in 2024-08-17T17:30:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:13 smithi130 ceph-mon[110068]: pgmap v343: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 767 B/s wr, 3 op/s 2024-08-17T17:30:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:13 smithi130 ceph-mon[110068]: osdmap e476: 8 total, 8 up, 8 in 2024-08-17T17:30:14.514 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.SnapshotUnlinkPeer (9958 ms) 2024-08-17T17:30:14.514 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.SnapshotImageState 2024-08-17T17:30:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:14 smithi179 ceph-mon[93519]: osdmap e477: 8 total, 8 up, 8 in 2024-08-17T17:30:14.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:14 smithi130 ceph-mon[105521]: osdmap e477: 8 total, 8 up, 8 in 2024-08-17T17:30:14.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:14 smithi130 ceph-mon[110068]: osdmap e477: 8 total, 8 up, 8 in 2024-08-17T17:30:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:15 smithi179 ceph-mon[93519]: pgmap v346: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 166 KiB/s rd, 9.7 KiB/s wr, 219 op/s 2024-08-17T17:30:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:15 smithi179 ceph-mon[93519]: osdmap e478: 8 total, 8 up, 8 in 2024-08-17T17:30:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:15 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/7aebff05-eaf9-4e0c-9004-5405b1709a4e"} : dispatch 2024-08-17T17:30:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:15 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/7aebff05-eaf9-4e0c-9004-5405b1709a4e"} : dispatch 2024-08-17T17:30:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:15 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/7aebff05-eaf9-4e0c-9004-5405b1709a4e"}]': finished 2024-08-17T17:30:15.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:15 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/310bf0dc-c1e7-4b58-ad62-fb622e66db76"} : dispatch 2024-08-17T17:30:15.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:15 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/310bf0dc-c1e7-4b58-ad62-fb622e66db76"} : dispatch 2024-08-17T17:30:15.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:15 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/310bf0dc-c1e7-4b58-ad62-fb622e66db76"}]': finished 2024-08-17T17:30:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[105521]: pgmap v346: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 166 KiB/s rd, 9.7 KiB/s wr, 219 op/s 2024-08-17T17:30:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[105521]: osdmap e478: 8 total, 8 up, 8 in 2024-08-17T17:30:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/7aebff05-eaf9-4e0c-9004-5405b1709a4e"} : dispatch 2024-08-17T17:30:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/7aebff05-eaf9-4e0c-9004-5405b1709a4e"} : dispatch 2024-08-17T17:30:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/7aebff05-eaf9-4e0c-9004-5405b1709a4e"}]': finished 2024-08-17T17:30:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/310bf0dc-c1e7-4b58-ad62-fb622e66db76"} : dispatch 2024-08-17T17:30:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/310bf0dc-c1e7-4b58-ad62-fb622e66db76"} : dispatch 2024-08-17T17:30:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/310bf0dc-c1e7-4b58-ad62-fb622e66db76"}]': finished 2024-08-17T17:30:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[110068]: pgmap v346: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 166 KiB/s rd, 9.7 KiB/s wr, 219 op/s 2024-08-17T17:30:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[110068]: osdmap e478: 8 total, 8 up, 8 in 2024-08-17T17:30:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/7aebff05-eaf9-4e0c-9004-5405b1709a4e"} : dispatch 2024-08-17T17:30:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/7aebff05-eaf9-4e0c-9004-5405b1709a4e"} : dispatch 2024-08-17T17:30:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/7aebff05-eaf9-4e0c-9004-5405b1709a4e"}]': finished 2024-08-17T17:30:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/310bf0dc-c1e7-4b58-ad62-fb622e66db76"} : dispatch 2024-08-17T17:30:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/310bf0dc-c1e7-4b58-ad62-fb622e66db76"} : dispatch 2024-08-17T17:30:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:15 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/310bf0dc-c1e7-4b58-ad62-fb622e66db76"}]': finished 2024-08-17T17:30:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:16 smithi179 ceph-mon[93519]: osdmap e479: 8 total, 8 up, 8 in 2024-08-17T17:30:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:16 smithi130 ceph-mon[105521]: osdmap e479: 8 total, 8 up, 8 in 2024-08-17T17:30:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:16 smithi130 ceph-mon[110068]: osdmap e479: 8 total, 8 up, 8 in 2024-08-17T17:30:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:17 smithi179 ceph-mon[93519]: pgmap v349: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 168 KiB/s rd, 10 KiB/s wr, 223 op/s 2024-08-17T17:30:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:17 smithi179 ceph-mon[93519]: osdmap e480: 8 total, 8 up, 8 in 2024-08-17T17:30:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:17 smithi130 ceph-mon[105521]: pgmap v349: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 168 KiB/s rd, 10 KiB/s wr, 223 op/s 2024-08-17T17:30:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:17 smithi130 ceph-mon[105521]: osdmap e480: 8 total, 8 up, 8 in 2024-08-17T17:30:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:17 smithi130 ceph-mon[110068]: pgmap v349: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 168 KiB/s rd, 10 KiB/s wr, 223 op/s 2024-08-17T17:30:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:17 smithi130 ceph-mon[110068]: osdmap e480: 8 total, 8 up, 8 in 2024-08-17T17:30:18.580 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.SnapshotImageState (4066 ms) 2024-08-17T17:30:18.580 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.SnapshotPromoteDemote 2024-08-17T17:30:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:18 smithi179 ceph-mon[93519]: osdmap e481: 8 total, 8 up, 8 in 2024-08-17T17:30:18.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:18 smithi130 ceph-mon[105521]: osdmap e481: 8 total, 8 up, 8 in 2024-08-17T17:30:18.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:18 smithi130 ceph-mon[110068]: osdmap e481: 8 total, 8 up, 8 in 2024-08-17T17:30:19.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:19 smithi130 ceph-mon[105521]: pgmap v352: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 152 KiB/s rd, 22 KiB/s wr, 217 op/s 2024-08-17T17:30:19.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:19 smithi130 ceph-mon[105521]: osdmap e482: 8 total, 8 up, 8 in 2024-08-17T17:30:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:19 smithi130 ceph-mon[110068]: pgmap v352: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 152 KiB/s rd, 22 KiB/s wr, 217 op/s 2024-08-17T17:30:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:19 smithi130 ceph-mon[110068]: osdmap e482: 8 total, 8 up, 8 in 2024-08-17T17:30:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:19 smithi179 ceph-mon[93519]: pgmap v352: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 152 KiB/s rd, 22 KiB/s wr, 217 op/s 2024-08-17T17:30:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:19 smithi179 ceph-mon[93519]: osdmap e482: 8 total, 8 up, 8 in 2024-08-17T17:30:20.135 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:30:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:30:19] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.51.0" 2024-08-17T17:30:20.543 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:30:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:30:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:30:20.774 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:20 smithi130 ceph-mon[105521]: osdmap e483: 8 total, 8 up, 8 in 2024-08-17T17:30:20.774 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:20 smithi130 ceph-mon[105521]: osdmap e484: 8 total, 8 up, 8 in 2024-08-17T17:30:20.774 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:20 smithi130 ceph-mon[110068]: osdmap e483: 8 total, 8 up, 8 in 2024-08-17T17:30:20.774 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:20 smithi130 ceph-mon[110068]: osdmap e484: 8 total, 8 up, 8 in 2024-08-17T17:30:20.807 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade resume' 2024-08-17T17:30:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:20 smithi179 ceph-mon[93519]: osdmap e483: 8 total, 8 up, 8 in 2024-08-17T17:30:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:20 smithi179 ceph-mon[93519]: osdmap e484: 8 total, 8 up, 8 in 2024-08-17T17:30:21.072 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:21 smithi179 ceph-mon[93519]: pgmap v355: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 175 KiB/s rd, 25 KiB/s wr, 249 op/s 2024-08-17T17:30:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:21 smithi179 ceph-mon[93519]: osdmap e485: 8 total, 8 up, 8 in 2024-08-17T17:30:21.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:21 smithi130 ceph-mon[105521]: pgmap v355: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 175 KiB/s rd, 25 KiB/s wr, 249 op/s 2024-08-17T17:30:21.906 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:21 smithi130 ceph-mon[105521]: osdmap e485: 8 total, 8 up, 8 in 2024-08-17T17:30:21.906 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:21 smithi130 ceph-mon[110068]: pgmap v355: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 175 KiB/s rd, 25 KiB/s wr, 249 op/s 2024-08-17T17:30:21.907 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:21 smithi130 ceph-mon[110068]: osdmap e485: 8 total, 8 up, 8 in 2024-08-17T17:30:22.834 INFO:teuthology.orchestra.run.smithi130.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:30:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[110068]: pgmap v358: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.2 KiB/s wr, 36 op/s 2024-08-17T17:30:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[110068]: osdmap e486: 8 total, 8 up, 8 in 2024-08-17T17:30:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[110068]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[110068]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:30:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:30:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:23.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:23.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:30:23.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[105521]: pgmap v358: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.2 KiB/s wr, 36 op/s 2024-08-17T17:30:23.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[105521]: osdmap e486: 8 total, 8 up, 8 in 2024-08-17T17:30:23.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[105521]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:23.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[105521]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:30:23.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:23.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:23.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:23.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:30:23.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:23.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:23.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:23.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:23.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:23.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:23 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:30:23.458 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for half of osds to upgrade' 2024-08-17T17:30:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:23 smithi179 ceph-mon[93519]: pgmap v358: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.2 KiB/s wr, 36 op/s 2024-08-17T17:30:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:23 smithi179 ceph-mon[93519]: osdmap e486: 8 total, 8 up, 8 in 2024-08-17T17:30:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:23 smithi179 ceph-mon[93519]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:23 smithi179 ceph-mon[93519]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:30:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:23 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:23 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:23.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:23 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:23.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:23 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:30:23.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:23 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:23.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:23 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:23.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:23 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:23.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:23 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:23.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:23 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:23.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:23 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:30:24.404 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:24.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:24 smithi130 ceph-mon[105521]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:30:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:24 smithi130 ceph-mon[105521]: Upgrade: osd.1 is safe to restart 2024-08-17T17:30:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:24 smithi130 ceph-mon[105521]: osdmap e487: 8 total, 8 up, 8 in 2024-08-17T17:30:24.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:24 smithi130 ceph-mon[110068]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:30:24.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:24 smithi130 ceph-mon[110068]: Upgrade: osd.1 is safe to restart 2024-08-17T17:30:24.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:24 smithi130 ceph-mon[110068]: osdmap e487: 8 total, 8 up, 8 in 2024-08-17T17:30:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:24 smithi179 ceph-mon[93519]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:30:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:24 smithi179 ceph-mon[93519]: Upgrade: osd.1 is safe to restart 2024-08-17T17:30:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:24 smithi179 ceph-mon[93519]: osdmap e487: 8 total, 8 up, 8 in 2024-08-17T17:30:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[110068]: pgmap v361: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.7 KiB/s wr, 39 op/s 2024-08-17T17:30:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[110068]: osdmap e488: 8 total, 8 up, 8 in 2024-08-17T17:30:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[110068]: Upgrade: Updating osd.1 2024-08-17T17:30:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:30:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[110068]: Deploying daemon osd.1 on smithi130 2024-08-17T17:30:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[110068]: osdmap e489: 8 total, 8 up, 8 in 2024-08-17T17:30:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[105521]: pgmap v361: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.7 KiB/s wr, 39 op/s 2024-08-17T17:30:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[105521]: osdmap e488: 8 total, 8 up, 8 in 2024-08-17T17:30:25.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[105521]: Upgrade: Updating osd.1 2024-08-17T17:30:25.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:25.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:30:25.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:25.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[105521]: Deploying daemon osd.1 on smithi130 2024-08-17T17:30:25.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:25 smithi130 ceph-mon[105521]: osdmap e489: 8 total, 8 up, 8 in 2024-08-17T17:30:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:25 smithi179 ceph-mon[93519]: pgmap v361: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.7 KiB/s wr, 39 op/s 2024-08-17T17:30:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:25 smithi179 ceph-mon[93519]: osdmap e488: 8 total, 8 up, 8 in 2024-08-17T17:30:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:25 smithi179 ceph-mon[93519]: Upgrade: Updating osd.1 2024-08-17T17:30:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:25 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:25 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:30:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:25 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:25 smithi179 ceph-mon[93519]: Deploying daemon osd.1 on smithi130 2024-08-17T17:30:25.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:25 smithi179 ceph-mon[93519]: osdmap e489: 8 total, 8 up, 8 in 2024-08-17T17:30:27.343 INFO:teuthology.orchestra.run.smithi130.stdout:wait for half of osds to upgrade 2024-08-17T17:30:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:27 smithi130 ceph-mon[105521]: pgmap v364: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 10 KiB/s wr, 163 op/s 2024-08-17T17:30:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:27 smithi130 ceph-mon[105521]: osdmap e490: 8 total, 8 up, 8 in 2024-08-17T17:30:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:27.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:27 smithi130 ceph-mon[110068]: pgmap v364: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 10 KiB/s wr, 163 op/s 2024-08-17T17:30:27.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:27 smithi130 ceph-mon[110068]: osdmap e490: 8 total, 8 up, 8 in 2024-08-17T17:30:27.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:27 smithi179 ceph-mon[93519]: pgmap v364: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 10 KiB/s wr, 163 op/s 2024-08-17T17:30:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:27 smithi179 ceph-mon[93519]: osdmap e490: 8 total, 8 up, 8 in 2024-08-17T17:30:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:28.299 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"' && ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error; do sleep 2 ; done' 2024-08-17T17:30:28.446 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.SnapshotPromoteDemote (9866 ms) 2024-08-17T17:30:28.446 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestMirroring.AioSnapshotCreate 2024-08-17T17:30:28.595 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:28 smithi130 ceph-mon[110068]: osdmap e491: 8 total, 8 up, 8 in 2024-08-17T17:30:28.596 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:28 smithi130 ceph-mon[105521]: osdmap e491: 8 total, 8 up, 8 in 2024-08-17T17:30:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:28 smithi179 ceph-mon[93519]: osdmap e491: 8 total, 8 up, 8 in 2024-08-17T17:30:28.903 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:29.394 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:29 smithi130 systemd[1]: Stopping Ceph osd.1 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:30:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:29 smithi130 ceph-mon[110068]: pgmap v367: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 102 KiB/s rd, 7.0 KiB/s wr, 137 op/s 2024-08-17T17:30:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:29 smithi130 ceph-mon[110068]: osdmap e492: 8 total, 8 up, 8 in 2024-08-17T17:30:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a56ed9d9-6852-4df7-9b9f-5b7f7ff1389e"} : dispatch 2024-08-17T17:30:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:29 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a56ed9d9-6852-4df7-9b9f-5b7f7ff1389e"} : dispatch 2024-08-17T17:30:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:29 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a56ed9d9-6852-4df7-9b9f-5b7f7ff1389e"}]': finished 2024-08-17T17:30:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:29 smithi130 ceph-mon[105521]: pgmap v367: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 102 KiB/s rd, 7.0 KiB/s wr, 137 op/s 2024-08-17T17:30:29.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:29 smithi130 ceph-mon[105521]: osdmap e492: 8 total, 8 up, 8 in 2024-08-17T17:30:29.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a56ed9d9-6852-4df7-9b9f-5b7f7ff1389e"} : dispatch 2024-08-17T17:30:29.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:29 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a56ed9d9-6852-4df7-9b9f-5b7f7ff1389e"} : dispatch 2024-08-17T17:30:29.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:29 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a56ed9d9-6852-4df7-9b9f-5b7f7ff1389e"}]': finished 2024-08-17T17:30:29.668 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:29 smithi130 podman[123922]: 2024-08-17 17:30:29.542143396 +0000 UTC m=+0.234351253 container died 1b047c250076f00650221fd7664021e8963b6f24c08cecac3c0c33e50a6e331e (image=quay.ceph.io/ceph-ci/ceph:quincy, name=inspiring_ramanujan, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-08-17T17:30:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:29 smithi179 ceph-mon[93519]: pgmap v367: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 102 KiB/s rd, 7.0 KiB/s wr, 137 op/s 2024-08-17T17:30:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:29 smithi179 ceph-mon[93519]: osdmap e492: 8 total, 8 up, 8 in 2024-08-17T17:30:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a56ed9d9-6852-4df7-9b9f-5b7f7ff1389e"} : dispatch 2024-08-17T17:30:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:29 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a56ed9d9-6852-4df7-9b9f-5b7f7ff1389e"} : dispatch 2024-08-17T17:30:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:29 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/a56ed9d9-6852-4df7-9b9f-5b7f7ff1389e"}]': finished 2024-08-17T17:30:30.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:30:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:30:29] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-08-17T17:30:30.167 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1[53354]: 2024-08-17T17:30:29.863+0000 7f14f4de1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:30:30.167 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1[53354]: 2024-08-17T17:30:29.863+0000 7f14f4de1640 -1 osd.1 493 *** Got signal Terminated *** 2024-08-17T17:30:30.167 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1[53354]: 2024-08-17T17:30:29.863+0000 7f14f4de1640 -1 osd.1 493 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:30:30.512 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:30 smithi130 ceph-mon[110068]: osdmap e493: 8 total, 8 up, 8 in 2024-08-17T17:30:30.512 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:30 smithi130 ceph-mon[110068]: osd.1 marked itself down and dead 2024-08-17T17:30:30.512 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:30 smithi130 ceph-mon[110068]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:30:30.512 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:30 smithi130 ceph-mon[110068]: osdmap e494: 8 total, 7 up, 8 in 2024-08-17T17:30:30.513 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:30 smithi130 ceph-mon[105521]: osdmap e493: 8 total, 8 up, 8 in 2024-08-17T17:30:30.513 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:30 smithi130 ceph-mon[105521]: osd.1 marked itself down and dead 2024-08-17T17:30:30.513 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:30 smithi130 ceph-mon[105521]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:30:30.513 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:30 smithi130 ceph-mon[105521]: osdmap e494: 8 total, 7 up, 8 in 2024-08-17T17:30:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:30 smithi179 ceph-mon[93519]: osdmap e493: 8 total, 8 up, 8 in 2024-08-17T17:30:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:30 smithi179 ceph-mon[93519]: osd.1 marked itself down and dead 2024-08-17T17:30:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:30 smithi179 ceph-mon[93519]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:30:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:30 smithi179 ceph-mon[93519]: osdmap e494: 8 total, 7 up, 8 in 2024-08-17T17:30:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:30:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:30:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:30:30.917 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:30 smithi130 podman[123982]: 2024-08-17 17:30:30.753044801 +0000 UTC m=+0.996094189 container died 1dc0b4634e1596e910ed8adde696f73734feef0f443faa65a4d46cf9e1db734f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-08-17T17:30:31.224 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:30:31.417 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:31 smithi130 podman[123982]: 2024-08-17 17:30:31.144576413 +0000 UTC m=+1.387625802 container cleanup 1dc0b4634e1596e910ed8adde696f73734feef0f443faa65a4d46cf9e1db734f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-f2c39d4, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-08-17T17:30:31.417 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:31 smithi130 bash[123982]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1 2024-08-17T17:30:31.417 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:31 smithi130 podman[124064]: 2024-08-17 17:30:31.304351115 +0000 UTC m=+0.586593493 container remove 1dc0b4634e1596e910ed8adde696f73734feef0f443faa65a4d46cf9e1db734f (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4) 2024-08-17T17:30:31.714 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:31 smithi130 ceph-mon[110068]: pgmap v370: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 KiB/s wr, 37 op/s 2024-08-17T17:30:31.714 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/451671092' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:31.714 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:31 smithi130 ceph-mon[110068]: osdmap e495: 8 total, 7 up, 8 in 2024-08-17T17:30:31.714 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:31 smithi130 ceph-mon[105521]: pgmap v370: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 KiB/s wr, 37 op/s 2024-08-17T17:30:31.715 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/451671092' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:31.715 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:31 smithi130 ceph-mon[105521]: osdmap e495: 8 total, 7 up, 8 in 2024-08-17T17:30:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:31 smithi179 ceph-mon[93519]: pgmap v370: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 KiB/s wr, 37 op/s 2024-08-17T17:30:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/451671092' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:31 smithi179 ceph-mon[93519]: osdmap e495: 8 total, 7 up, 8 in 2024-08-17T17:30:32.025 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:31 smithi130 podman[124225]: 2024-08-17 17:30:31.713849485 +0000 UTC m=+0.094680812 container create cc4e6a295826ae318fe4b7489a3e02378a688f61079aee27636bbb3a2521ea23 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:30:32.025 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:31 smithi130 podman[124225]: 2024-08-17 17:30:31.636437392 +0000 UTC m=+0.017268688 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:30:32.025 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:31 smithi130 podman[124225]: 2024-08-17 17:30:31.949682042 +0000 UTC m=+0.330513330 container init cc4e6a295826ae318fe4b7489a3e02378a688f61079aee27636bbb3a2521ea23 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-08-17T17:30:32.025 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:31 smithi130 podman[124225]: 2024-08-17 17:30:31.954114993 +0000 UTC m=+0.334946275 container start cc4e6a295826ae318fe4b7489a3e02378a688f61079aee27636bbb3a2521ea23 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0) 2024-08-17T17:30:32.025 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:32 smithi130 podman[124225]: 2024-08-17 17:30:32.024430633 +0000 UTC m=+0.405261927 container attach cc4e6a295826ae318fe4b7489a3e02378a688f61079aee27636bbb3a2521ea23 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731) 2024-08-17T17:30:32.417 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:32 smithi130 podman[124225]: 2024-08-17 17:30:32.149453556 +0000 UTC m=+0.530284837 container died cc4e6a295826ae318fe4b7489a3e02378a688f61079aee27636bbb3a2521ea23 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:30:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:32 smithi179 ceph-mon[93519]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:32 smithi179 ceph-mon[93519]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:32 smithi179 ceph-mon[93519]: osdmap e496: 8 total, 7 up, 8 in 2024-08-17T17:30:32.879 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:32 smithi130 ceph-mon[105521]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:32.879 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:32 smithi130 ceph-mon[105521]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:32.879 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:32 smithi130 ceph-mon[105521]: osdmap e496: 8 total, 7 up, 8 in 2024-08-17T17:30:32.879 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:32 smithi130 ceph-mon[110068]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:32.879 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:32 smithi130 ceph-mon[110068]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:32.880 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:32 smithi130 ceph-mon[110068]: osdmap e496: 8 total, 7 up, 8 in 2024-08-17T17:30:33.167 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:32 smithi130 podman[124268]: 2024-08-17 17:30:32.87889773 +0000 UTC m=+0.720382852 container remove cc4e6a295826ae318fe4b7489a3e02378a688f61079aee27636bbb3a2521ea23 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:30:33.167 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:32 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.1.service: Deactivated successfully. 2024-08-17T17:30:33.167 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:32 smithi130 systemd[1]: Stopped Ceph osd.1 for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:30:33.167 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:32 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.1.service: Consumed 21.355s CPU time. 2024-08-17T17:30:33.167 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 systemd[1]: Starting Ceph osd.1 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:30:33.625 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:33 smithi130 ceph-mon[105521]: pgmap v373: 105 pgs: 12 stale+active+clean, 93 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 3.5 KiB/s wr, 24 op/s 2024-08-17T17:30:33.625 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:33 smithi130 ceph-mon[105521]: osdmap e497: 8 total, 7 up, 8 in 2024-08-17T17:30:33.625 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:33 smithi130 ceph-mon[110068]: pgmap v373: 105 pgs: 12 stale+active+clean, 93 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 3.5 KiB/s wr, 24 op/s 2024-08-17T17:30:33.625 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:33 smithi130 ceph-mon[110068]: osdmap e497: 8 total, 7 up, 8 in 2024-08-17T17:30:33.625 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 podman[124372]: 2024-08-17 17:30:33.21708993 +0000 UTC m=+0.017569519 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:30:33.625 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 podman[124372]: 2024-08-17 17:30:33.338236906 +0000 UTC m=+0.138716483 container create e408630076ffbf687303aa85ae148e0e6e6a7952d972a5c7acaaf9a56f7c7ca2 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, ceph=True) 2024-08-17T17:30:33.626 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 podman[124372]: 2024-08-17 17:30:33.541424804 +0000 UTC m=+0.341904397 container init e408630076ffbf687303aa85ae148e0e6e6a7952d972a5c7acaaf9a56f7c7ca2 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:30:33.626 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 podman[124372]: 2024-08-17 17:30:33.546579567 +0000 UTC m=+0.347059146 container start e408630076ffbf687303aa85ae148e0e6e6a7952d972a5c7acaaf9a56f7c7ca2 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-08-17T17:30:33.626 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 podman[124372]: 2024-08-17 17:30:33.624408171 +0000 UTC m=+0.424887749 container attach e408630076ffbf687303aa85ae148e0e6e6a7952d972a5c7acaaf9a56f7c7ca2 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:30:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:33 smithi179 ceph-mon[93519]: pgmap v373: 105 pgs: 12 stale+active+clean, 93 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 3.5 KiB/s wr, 24 op/s 2024-08-17T17:30:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:33 smithi179 ceph-mon[93519]: osdmap e497: 8 total, 7 up, 8 in 2024-08-17T17:30:33.917 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate[124387]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:30:33.917 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 bash[124372]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:30:33.917 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate[124387]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:30:33.917 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 bash[124372]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:30:33.917 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate[124387]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:30:33.917 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 bash[124372]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:30:33.918 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate[124387]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:30:33.918 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 bash[124372]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:30:33.918 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate[124387]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:30:33.918 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 bash[124372]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:30:33.918 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate[124387]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:30:33.918 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 bash[124372]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:30:33.918 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate[124387]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-08-17T17:30:33.918 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:33 smithi130 bash[124372]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-08-17T17:30:34.116 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:30:34.413 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:34 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate[124387]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:30:34.414 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:34 smithi130 bash[124372]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:30:34.414 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:34 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate[124387]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:30:34.414 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:34 smithi130 bash[124372]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:30:34.414 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:34 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate[124387]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-17T17:30:34.414 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:34 smithi130 bash[124372]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-17T17:30:34.414 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:34 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate[124387]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:30:34.414 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:34 smithi130 bash[124372]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:30:34.414 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:34 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate[124387]: --> ceph-volume lvm activate successful for osd ID: 1 2024-08-17T17:30:34.414 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:34 smithi130 bash[124372]: --> ceph-volume lvm activate successful for osd ID: 1 2024-08-17T17:30:34.414 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:34 smithi130 podman[124372]: 2024-08-17 17:30:34.121816571 +0000 UTC m=+0.922296149 container died e408630076ffbf687303aa85ae148e0e6e6a7952d972a5c7acaaf9a56f7c7ca2 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=) 2024-08-17T17:30:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:34 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2687988297' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:34 smithi130 ceph-mon[110068]: osdmap e498: 8 total, 7 up, 8 in 2024-08-17T17:30:34.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:34 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2687988297' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:34.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:34 smithi130 ceph-mon[105521]: osdmap e498: 8 total, 7 up, 8 in 2024-08-17T17:30:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:34 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2687988297' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:34 smithi179 ceph-mon[93519]: osdmap e498: 8 total, 7 up, 8 in 2024-08-17T17:30:35.007 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:34 smithi130 podman[124579]: 2024-08-17 17:30:34.678831172 +0000 UTC m=+0.550783391 container remove e408630076ffbf687303aa85ae148e0e6e6a7952d972a5c7acaaf9a56f7c7ca2 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-activate, RELEASE=main-cd8bdbe, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-08-17T17:30:35.007 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:34 smithi130 podman[124665]: 2024-08-17 17:30:34.793189741 +0000 UTC m=+0.017200860 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:30:35.007 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:34 smithi130 podman[124665]: 2024-08-17 17:30:34.896362422 +0000 UTC m=+0.120373536 container create 54f6b453c741e71860bf00acb41c5c844fea1150fb2705046969391e5edcf815 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, ceph=True, GIT_CLEAN=True, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-08-17T17:30:35.417 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:35 smithi130 podman[124665]: 2024-08-17 17:30:35.090715677 +0000 UTC m=+0.314726791 container init 54f6b453c741e71860bf00acb41c5c844fea1150fb2705046969391e5edcf815 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-08-17T17:30:35.417 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:35 smithi130 podman[124665]: 2024-08-17 17:30:35.095986343 +0000 UTC m=+0.319997456 container start 54f6b453c741e71860bf00acb41c5c844fea1150fb2705046969391e5edcf815 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0) 2024-08-17T17:30:35.417 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-osd[124703]: -- 172.21.15.130:0/2442939301 <== mon.0 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5646fefdba40 con 0x5646ffda8400 2024-08-17T17:30:35.417 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:35 smithi130 bash[124665]: 54f6b453c741e71860bf00acb41c5c844fea1150fb2705046969391e5edcf815 2024-08-17T17:30:35.417 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:35 smithi130 systemd[1]: Started Ceph osd.1 for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:30:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:35 smithi179 ceph-mon[93519]: pgmap v376: 105 pgs: 27 active+undersized, 1 stale+active+clean, 16 active+undersized+degraded, 61 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 10 KiB/s wr, 95 op/s; 32/354 objects degraded (9.040%) 2024-08-17T17:30:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:35 smithi179 ceph-mon[93519]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:35 smithi179 ceph-mon[93519]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:35 smithi179 ceph-mon[93519]: Health check failed: Degraded data redundancy: 32/354 objects degraded (9.040%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:35 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:35 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:35 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:35 smithi179 ceph-mon[93519]: osdmap e499: 8 total, 7 up, 8 in 2024-08-17T17:30:35.868 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[110068]: pgmap v376: 105 pgs: 27 active+undersized, 1 stale+active+clean, 16 active+undersized+degraded, 61 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 10 KiB/s wr, 95 op/s; 32/354 objects degraded (9.040%) 2024-08-17T17:30:35.868 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[110068]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:35.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[110068]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:35.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[110068]: Health check failed: Degraded data redundancy: 32/354 objects degraded (9.040%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:35.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:35.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:35.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:35.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[110068]: osdmap e499: 8 total, 7 up, 8 in 2024-08-17T17:30:35.869 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[105521]: pgmap v376: 105 pgs: 27 active+undersized, 1 stale+active+clean, 16 active+undersized+degraded, 61 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 10 KiB/s wr, 95 op/s; 32/354 objects degraded (9.040%) 2024-08-17T17:30:35.869 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[105521]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:35.869 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[105521]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:35.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[105521]: Health check failed: Degraded data redundancy: 32/354 objects degraded (9.040%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:35.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:35.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:35.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:35.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:35 smithi130 ceph-mon[105521]: osdmap e499: 8 total, 7 up, 8 in 2024-08-17T17:30:37.002 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:30:37.002 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:36 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1[124680]: 2024-08-17T17:30:36.735+0000 7fd498ce5640 -1 Falling back to public interface 2024-08-17T17:30:37.628 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:37 smithi130 ceph-mon[105521]: pgmap v379: 105 pgs: 29 active+undersized, 20 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 97 op/s; 43/363 objects degraded (11.846%) 2024-08-17T17:30:37.628 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:37 smithi130 ceph-mon[105521]: osdmap e500: 8 total, 7 up, 8 in 2024-08-17T17:30:37.628 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:37 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1280820429' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:37.629 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:37 smithi130 ceph-mon[110068]: pgmap v379: 105 pgs: 29 active+undersized, 20 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 97 op/s; 43/363 objects degraded (11.846%) 2024-08-17T17:30:37.629 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:37 smithi130 ceph-mon[110068]: osdmap e500: 8 total, 7 up, 8 in 2024-08-17T17:30:37.629 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:37 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1280820429' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:37.734 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:37 smithi179 ceph-mon[93519]: pgmap v379: 105 pgs: 29 active+undersized, 20 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 97 op/s; 43/363 objects degraded (11.846%) 2024-08-17T17:30:37.734 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:37 smithi179 ceph-mon[93519]: osdmap e500: 8 total, 7 up, 8 in 2024-08-17T17:30:37.734 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:37 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1280820429' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:38.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:38 smithi130 ceph-mon[105521]: from='client.44237 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:38 smithi130 ceph-mon[105521]: from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:38 smithi130 ceph-mon[105521]: osdmap e501: 8 total, 7 up, 8 in 2024-08-17T17:30:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:38 smithi130 ceph-mon[110068]: from='client.44237 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:38 smithi130 ceph-mon[110068]: from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:38 smithi130 ceph-mon[110068]: osdmap e501: 8 total, 7 up, 8 in 2024-08-17T17:30:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:38 smithi179 ceph-mon[93519]: from='client.44237 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:38 smithi179 ceph-mon[93519]: from='client.34365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:38 smithi179 ceph-mon[93519]: osdmap e501: 8 total, 7 up, 8 in 2024-08-17T17:30:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:39 smithi130 ceph-mon[105521]: pgmap v382: 105 pgs: 29 active+undersized, 20 active+undersized+degraded, 56 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 13 KiB/s wr, 111 op/s; 54/426 objects degraded (12.676%) 2024-08-17T17:30:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:39 smithi130 ceph-mon[105521]: osdmap e502: 8 total, 7 up, 8 in 2024-08-17T17:30:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:39 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:39 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:39 smithi130 ceph-mon[110068]: pgmap v382: 105 pgs: 29 active+undersized, 20 active+undersized+degraded, 56 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 13 KiB/s wr, 111 op/s; 54/426 objects degraded (12.676%) 2024-08-17T17:30:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:39 smithi130 ceph-mon[110068]: osdmap e502: 8 total, 7 up, 8 in 2024-08-17T17:30:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:39 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:39 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:39 smithi179 ceph-mon[93519]: pgmap v382: 105 pgs: 29 active+undersized, 20 active+undersized+degraded, 56 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 13 KiB/s wr, 111 op/s; 54/426 objects degraded (12.676%) 2024-08-17T17:30:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:39 smithi179 ceph-mon[93519]: osdmap e502: 8 total, 7 up, 8 in 2024-08-17T17:30:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:39 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:39 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:39.945 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:30:40.076 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:30:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:30:39] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-08-17T17:30:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:30:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:30:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:30:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:40 smithi179 ceph-mon[93519]: osdmap e503: 8 total, 7 up, 8 in 2024-08-17T17:30:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:40 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:40 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:40 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:40 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:40 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2586409910' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:40 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 54/426 objects degraded (12.676%), 20 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:40.615 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-mon[105521]: osdmap e503: 8 total, 7 up, 8 in 2024-08-17T17:30:40.615 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:40.615 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:40.615 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:40.615 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:40.615 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2586409910' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:40.615 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 54/426 objects degraded (12.676%), 20 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:40.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-mon[110068]: osdmap e503: 8 total, 7 up, 8 in 2024-08-17T17:30:40.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:40.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:40.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:40.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:40.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2586409910' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:40.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 54/426 objects degraded (12.676%), 20 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:40.616 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:40 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1[124680]: 2024-08-17T17:30:40.396+0000 7fd498ce5640 -1 osd.1 493 log_to_monitors true 2024-08-17T17:30:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:41 smithi130 ceph-mon[105521]: from='client.44255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:41 smithi130 ceph-mon[105521]: pgmap v385: 105 pgs: 29 active+undersized, 20 active+undersized+degraded, 56 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s rd, 15 KiB/s wr, 129 op/s; 56/441 objects degraded (12.698%) 2024-08-17T17:30:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:41 smithi130 ceph-mon[105521]: from='client.44261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:41.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:41 smithi130 ceph-mon[105521]: osdmap e504: 8 total, 7 up, 8 in 2024-08-17T17:30:41.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:41 smithi130 ceph-mon[105521]: from='osd.1 [v2:172.21.15.130:6810/790699819,v1:172.21.15.130:6811/790699819]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-17T17:30:41.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:41 smithi130 ceph-mon[110068]: from='client.44255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:41.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:41 smithi130 ceph-mon[110068]: pgmap v385: 105 pgs: 29 active+undersized, 20 active+undersized+degraded, 56 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s rd, 15 KiB/s wr, 129 op/s; 56/441 objects degraded (12.698%) 2024-08-17T17:30:41.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:41 smithi130 ceph-mon[110068]: from='client.44261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:41.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:41 smithi130 ceph-mon[110068]: osdmap e504: 8 total, 7 up, 8 in 2024-08-17T17:30:41.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:41 smithi130 ceph-mon[110068]: from='osd.1 [v2:172.21.15.130:6810/790699819,v1:172.21.15.130:6811/790699819]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-17T17:30:41.669 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 17:30:41 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1[124680]: 2024-08-17T17:30:41.425+0000 7fd490a8f640 -1 osd.1 493 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:30:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:41 smithi179 ceph-mon[93519]: from='client.44255 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:41 smithi179 ceph-mon[93519]: pgmap v385: 105 pgs: 29 active+undersized, 20 active+undersized+degraded, 56 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s rd, 15 KiB/s wr, 129 op/s; 56/441 objects degraded (12.698%) 2024-08-17T17:30:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:41 smithi179 ceph-mon[93519]: from='client.44261 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:41 smithi179 ceph-mon[93519]: osdmap e504: 8 total, 7 up, 8 in 2024-08-17T17:30:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:41 smithi179 ceph-mon[93519]: from='osd.1 [v2:172.21.15.130:6810/790699819,v1:172.21.15.130:6811/790699819]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-17T17:30:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:42 smithi130 ceph-mon[110068]: from='osd.1 [v2:172.21.15.130:6810/790699819,v1:172.21.15.130:6811/790699819]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:30:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:42 smithi130 ceph-mon[110068]: osdmap e505: 8 total, 7 up, 8 in 2024-08-17T17:30:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:42 smithi130 ceph-mon[110068]: from='osd.1 [v2:172.21.15.130:6810/790699819,v1:172.21.15.130:6811/790699819]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:30:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:42 smithi130 ceph-mon[105521]: from='osd.1 [v2:172.21.15.130:6810/790699819,v1:172.21.15.130:6811/790699819]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:30:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:42 smithi130 ceph-mon[105521]: osdmap e505: 8 total, 7 up, 8 in 2024-08-17T17:30:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:42 smithi130 ceph-mon[105521]: from='osd.1 [v2:172.21.15.130:6810/790699819,v1:172.21.15.130:6811/790699819]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:30:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:42.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:42 smithi179 ceph-mon[93519]: from='osd.1 [v2:172.21.15.130:6810/790699819,v1:172.21.15.130:6811/790699819]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:30:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:42 smithi179 ceph-mon[93519]: osdmap e505: 8 total, 7 up, 8 in 2024-08-17T17:30:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:42 smithi179 ceph-mon[93519]: from='osd.1 [v2:172.21.15.130:6810/790699819,v1:172.21.15.130:6811/790699819]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:30:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:42.844 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:30:43.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:30:42 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:30:42.882+0000 7f8e6b27b640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (14 PGs are or would become offline) 2024-08-17T17:30:43.746 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: pgmap v388: 105 pgs: 29 active+undersized, 20 active+undersized+degraded, 56 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 2.0 KiB/s wr, 20 op/s; 56/441 objects degraded (12.698%) 2024-08-17T17:30:43.746 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:30:43.746 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: osd.1 [v2:172.21.15.130:6810/790699819,v1:172.21.15.130:6811/790699819] boot 2024-08-17T17:30:43.746 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: osdmap e506: 8 total, 8 up, 8 in 2024-08-17T17:30:43.747 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:30:43.747 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1662075156' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:43.747 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:43.747 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:43.747 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:43.747 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:30:43.747 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:43.747 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:43.747 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:43.747 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:43.748 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:43.748 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:43.748 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:30:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: pgmap v388: 105 pgs: 29 active+undersized, 20 active+undersized+degraded, 56 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 2.0 KiB/s wr, 20 op/s; 56/441 objects degraded (12.698%) 2024-08-17T17:30:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:30:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: osd.1 [v2:172.21.15.130:6810/790699819,v1:172.21.15.130:6811/790699819] boot 2024-08-17T17:30:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: osdmap e506: 8 total, 8 up, 8 in 2024-08-17T17:30:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:30:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1662075156' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:43.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:30:43.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:43.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:43.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:43.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:43.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:43.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:43.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:30:43.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: pgmap v388: 105 pgs: 29 active+undersized, 20 active+undersized+degraded, 56 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 2.0 KiB/s wr, 20 op/s; 56/441 objects degraded (12.698%) 2024-08-17T17:30:43.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:30:43.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: osd.1 [v2:172.21.15.130:6810/790699819,v1:172.21.15.130:6811/790699819] boot 2024-08-17T17:30:43.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: osdmap e506: 8 total, 8 up, 8 in 2024-08-17T17:30:43.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:30:43.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1662075156' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:43.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:43.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:43.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:43.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:30:43.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:43.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: from='client.44276 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:43.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:43.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:43.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:43.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:30:43.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:30:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:44 smithi179 ceph-mon[93519]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:30:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:44 smithi179 ceph-mon[93519]: Upgrade: unsafe to stop osd(s) at this time (14 PGs are or would become offline) 2024-08-17T17:30:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:44 smithi179 ceph-mon[93519]: from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:44 smithi179 ceph-mon[93519]: osdmap e507: 8 total, 8 up, 8 in 2024-08-17T17:30:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:44 smithi130 ceph-mon[105521]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:30:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:44 smithi130 ceph-mon[105521]: Upgrade: unsafe to stop osd(s) at this time (14 PGs are or would become offline) 2024-08-17T17:30:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:44 smithi130 ceph-mon[105521]: from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:44 smithi130 ceph-mon[105521]: osdmap e507: 8 total, 8 up, 8 in 2024-08-17T17:30:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:44 smithi130 ceph-mon[110068]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:30:44.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:44 smithi130 ceph-mon[110068]: Upgrade: unsafe to stop osd(s) at this time (14 PGs are or would become offline) 2024-08-17T17:30:44.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:44 smithi130 ceph-mon[110068]: from='client.44282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:44.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:44 smithi130 ceph-mon[110068]: osdmap e507: 8 total, 8 up, 8 in 2024-08-17T17:30:45.709 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:45 smithi130 ceph-mon[105521]: pgmap v391: 105 pgs: 32 peering, 10 active+undersized, 7 active+undersized+degraded, 56 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 134 KiB/s rd, 15 KiB/s wr, 177 op/s; 24/486 objects degraded (4.938%) 2024-08-17T17:30:45.709 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:45 smithi130 ceph-mon[105521]: osdmap e508: 8 total, 8 up, 8 in 2024-08-17T17:30:45.709 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:45 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 24/486 objects degraded (4.938%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:45.709 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:45 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2305986959' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:45.710 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:45 smithi130 ceph-mon[110068]: pgmap v391: 105 pgs: 32 peering, 10 active+undersized, 7 active+undersized+degraded, 56 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 134 KiB/s rd, 15 KiB/s wr, 177 op/s; 24/486 objects degraded (4.938%) 2024-08-17T17:30:45.710 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:45 smithi130 ceph-mon[110068]: osdmap e508: 8 total, 8 up, 8 in 2024-08-17T17:30:45.710 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:45 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 24/486 objects degraded (4.938%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:45.711 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:45 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2305986959' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:45.711 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:30:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:45 smithi179 ceph-mon[93519]: pgmap v391: 105 pgs: 32 peering, 10 active+undersized, 7 active+undersized+degraded, 56 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 134 KiB/s rd, 15 KiB/s wr, 177 op/s; 24/486 objects degraded (4.938%) 2024-08-17T17:30:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:45 smithi179 ceph-mon[93519]: osdmap e508: 8 total, 8 up, 8 in 2024-08-17T17:30:45.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:45 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 24/486 objects degraded (4.938%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:45.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:45 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2305986959' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:46 smithi179 ceph-mon[93519]: osdmap e509: 8 total, 8 up, 8 in 2024-08-17T17:30:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:46 smithi179 ceph-mon[93519]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:46 smithi130 ceph-mon[110068]: osdmap e509: 8 total, 8 up, 8 in 2024-08-17T17:30:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:46 smithi130 ceph-mon[110068]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:46.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:46 smithi130 ceph-mon[105521]: osdmap e509: 8 total, 8 up, 8 in 2024-08-17T17:30:46.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:46 smithi130 ceph-mon[105521]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:47 smithi179 ceph-mon[93519]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:47 smithi179 ceph-mon[93519]: pgmap v394: 105 pgs: 32 peering, 9 active+undersized, 3 active+undersized+degraded, 61 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 144 KiB/s rd, 17 KiB/s wr, 190 op/s; 11/489 objects degraded (2.249%); 30 B/s, 2 keys/s, 0 objects/s recovering 2024-08-17T17:30:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:47 smithi179 ceph-mon[93519]: osdmap e510: 8 total, 8 up, 8 in 2024-08-17T17:30:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:47 smithi130 ceph-mon[105521]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:47 smithi130 ceph-mon[105521]: pgmap v394: 105 pgs: 32 peering, 9 active+undersized, 3 active+undersized+degraded, 61 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 144 KiB/s rd, 17 KiB/s wr, 190 op/s; 11/489 objects degraded (2.249%); 30 B/s, 2 keys/s, 0 objects/s recovering 2024-08-17T17:30:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:47 smithi130 ceph-mon[105521]: osdmap e510: 8 total, 8 up, 8 in 2024-08-17T17:30:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:47 smithi130 ceph-mon[110068]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:47 smithi130 ceph-mon[110068]: pgmap v394: 105 pgs: 32 peering, 9 active+undersized, 3 active+undersized+degraded, 61 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 144 KiB/s rd, 17 KiB/s wr, 190 op/s; 11/489 objects degraded (2.249%); 30 B/s, 2 keys/s, 0 objects/s recovering 2024-08-17T17:30:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:47 smithi130 ceph-mon[110068]: osdmap e510: 8 total, 8 up, 8 in 2024-08-17T17:30:48.560 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:30:48.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:48 smithi179 ceph-mon[93519]: osdmap e511: 8 total, 8 up, 8 in 2024-08-17T17:30:48.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:48 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3155586073' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:48.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:48 smithi130 ceph-mon[110068]: osdmap e511: 8 total, 8 up, 8 in 2024-08-17T17:30:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:48 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3155586073' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:48 smithi130 ceph-mon[105521]: osdmap e511: 8 total, 8 up, 8 in 2024-08-17T17:30:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:48 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3155586073' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:49 smithi130 ceph-mon[110068]: pgmap v397: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 11 KiB/s wr, 172 op/s; 250 B/s, 39 keys/s, 8 objects/s recovering 2024-08-17T17:30:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:49 smithi130 ceph-mon[110068]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/489 objects degraded (2.249%), 3 pgs degraded) 2024-08-17T17:30:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:49 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T17:30:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:49 smithi130 ceph-mon[110068]: osdmap e512: 8 total, 8 up, 8 in 2024-08-17T17:30:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:49 smithi130 ceph-mon[110068]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:49 smithi130 ceph-mon[110068]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:49 smithi130 ceph-mon[105521]: pgmap v397: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 11 KiB/s wr, 172 op/s; 250 B/s, 39 keys/s, 8 objects/s recovering 2024-08-17T17:30:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:49 smithi130 ceph-mon[105521]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/489 objects degraded (2.249%), 3 pgs degraded) 2024-08-17T17:30:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:49 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T17:30:49.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:49 smithi130 ceph-mon[105521]: osdmap e512: 8 total, 8 up, 8 in 2024-08-17T17:30:49.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:49 smithi130 ceph-mon[105521]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:49.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:49 smithi130 ceph-mon[105521]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:49 smithi179 ceph-mon[93519]: pgmap v397: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 11 KiB/s wr, 172 op/s; 250 B/s, 39 keys/s, 8 objects/s recovering 2024-08-17T17:30:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:49 smithi179 ceph-mon[93519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/489 objects degraded (2.249%), 3 pgs degraded) 2024-08-17T17:30:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:49 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T17:30:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:49 smithi179 ceph-mon[93519]: osdmap e512: 8 total, 8 up, 8 in 2024-08-17T17:30:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:49 smithi179 ceph-mon[93519]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:49 smithi179 ceph-mon[93519]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:30:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:30:49] "GET /metrics HTTP/1.1" 200 36295 "" "Prometheus/2.51.0" 2024-08-17T17:30:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:50 smithi179 ceph-mon[93519]: osdmap e513: 8 total, 8 up, 8 in 2024-08-17T17:30:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:50 smithi179 ceph-mon[93519]: osdmap e514: 8 total, 8 up, 8 in 2024-08-17T17:30:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:30:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:30:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:30:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:50 smithi130 ceph-mon[105521]: osdmap e513: 8 total, 8 up, 8 in 2024-08-17T17:30:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:50 smithi130 ceph-mon[105521]: osdmap e514: 8 total, 8 up, 8 in 2024-08-17T17:30:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:50 smithi130 ceph-mon[110068]: osdmap e513: 8 total, 8 up, 8 in 2024-08-17T17:30:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:50 smithi130 ceph-mon[110068]: osdmap e514: 8 total, 8 up, 8 in 2024-08-17T17:30:51.415 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:30:51.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:51 smithi130 ceph-mon[105521]: pgmap v400: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 9.5 KiB/s wr, 160 op/s; 220 B/s, 37 keys/s, 7 objects/s recovering 2024-08-17T17:30:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:51 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1607761677' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:51 smithi130 ceph-mon[105521]: osdmap e515: 8 total, 8 up, 8 in 2024-08-17T17:30:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:51 smithi130 ceph-mon[110068]: pgmap v400: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 9.5 KiB/s wr, 160 op/s; 220 B/s, 37 keys/s, 7 objects/s recovering 2024-08-17T17:30:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:51 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1607761677' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:51 smithi130 ceph-mon[110068]: osdmap e515: 8 total, 8 up, 8 in 2024-08-17T17:30:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:51 smithi179 ceph-mon[93519]: pgmap v400: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 9.5 KiB/s wr, 160 op/s; 220 B/s, 37 keys/s, 7 objects/s recovering 2024-08-17T17:30:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:51 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1607761677' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:51 smithi179 ceph-mon[93519]: osdmap e515: 8 total, 8 up, 8 in 2024-08-17T17:30:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:52 smithi179 ceph-mon[93519]: from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:52 smithi179 ceph-mon[93519]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:52 smithi179 ceph-mon[93519]: osdmap e516: 8 total, 8 up, 8 in 2024-08-17T17:30:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:52 smithi130 ceph-mon[105521]: from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:52 smithi130 ceph-mon[105521]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:52 smithi130 ceph-mon[105521]: osdmap e516: 8 total, 8 up, 8 in 2024-08-17T17:30:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:52 smithi130 ceph-mon[110068]: from='client.44330 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:52 smithi130 ceph-mon[110068]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:52 smithi130 ceph-mon[110068]: osdmap e516: 8 total, 8 up, 8 in 2024-08-17T17:30:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:53 smithi179 ceph-mon[93519]: pgmap v403: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-08-17T17:30:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:53 smithi179 ceph-mon[93519]: osdmap e517: 8 total, 8 up, 8 in 2024-08-17T17:30:53.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:53 smithi130 ceph-mon[105521]: pgmap v403: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-08-17T17:30:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:53 smithi130 ceph-mon[105521]: osdmap e517: 8 total, 8 up, 8 in 2024-08-17T17:30:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:53 smithi130 ceph-mon[110068]: pgmap v403: 105 pgs: 105 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-08-17T17:30:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:53 smithi130 ceph-mon[110068]: osdmap e517: 8 total, 8 up, 8 in 2024-08-17T17:30:54.262 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:30:54.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3875371051' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:54.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:54 smithi130 ceph-mon[105521]: osdmap e518: 8 total, 8 up, 8 in 2024-08-17T17:30:54.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3875371051' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:54.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:54 smithi130 ceph-mon[110068]: osdmap e518: 8 total, 8 up, 8 in 2024-08-17T17:30:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3875371051' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:54 smithi179 ceph-mon[93519]: osdmap e518: 8 total, 8 up, 8 in 2024-08-17T17:30:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:55 smithi179 ceph-mon[93519]: pgmap v406: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-08-17T17:30:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:55 smithi179 ceph-mon[93519]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:55 smithi179 ceph-mon[93519]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:55 smithi179 ceph-mon[93519]: osdmap e519: 8 total, 8 up, 8 in 2024-08-17T17:30:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:55 smithi130 ceph-mon[105521]: pgmap v406: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-08-17T17:30:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:55 smithi130 ceph-mon[105521]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:55 smithi130 ceph-mon[105521]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:55 smithi130 ceph-mon[105521]: osdmap e519: 8 total, 8 up, 8 in 2024-08-17T17:30:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:55 smithi130 ceph-mon[110068]: pgmap v406: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-08-17T17:30:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:55 smithi130 ceph-mon[110068]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:55 smithi130 ceph-mon[110068]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:55 smithi130 ceph-mon[110068]: osdmap e519: 8 total, 8 up, 8 in 2024-08-17T17:30:57.115 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:30:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:57 smithi130 ceph-mon[105521]: pgmap v409: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 118 op/s 2024-08-17T17:30:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:57 smithi130 ceph-mon[105521]: osdmap e520: 8 total, 8 up, 8 in 2024-08-17T17:30:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:57 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3076354061' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:57 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:57 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:57.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:57 smithi130 ceph-mon[110068]: pgmap v409: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 118 op/s 2024-08-17T17:30:57.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:57 smithi130 ceph-mon[110068]: osdmap e520: 8 total, 8 up, 8 in 2024-08-17T17:30:57.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:57 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3076354061' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:57.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:57 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:57.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:57.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:57 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:57 smithi179 ceph-mon[93519]: pgmap v409: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 118 op/s 2024-08-17T17:30:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:57 smithi179 ceph-mon[93519]: osdmap e520: 8 total, 8 up, 8 in 2024-08-17T17:30:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:57 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3076354061' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:30:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:57 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:57 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:30:58.629 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:58 smithi130 ceph-mon[110068]: from='client.44366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:58.629 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:58 smithi130 ceph-mon[110068]: osdmap e521: 8 total, 8 up, 8 in 2024-08-17T17:30:58.629 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:58 smithi130 ceph-mon[110068]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:58.629 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:30:58.629 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:58 smithi130 ceph-mon[105521]: from='client.44366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:58.629 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:58 smithi130 ceph-mon[105521]: osdmap e521: 8 total, 8 up, 8 in 2024-08-17T17:30:58.629 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:58 smithi130 ceph-mon[105521]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:58.630 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:30:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:58 smithi179 ceph-mon[93519]: from='client.44366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:58 smithi179 ceph-mon[93519]: osdmap e521: 8 total, 8 up, 8 in 2024-08-17T17:30:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:58 smithi179 ceph-mon[93519]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:30:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:59 smithi130 ceph-mon[105521]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:30:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:59 smithi130 ceph-mon[105521]: Upgrade: osd.2 is safe to restart 2024-08-17T17:30:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:59 smithi130 ceph-mon[105521]: pgmap v412: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s rd, 8.0 KiB/s wr, 132 op/s 2024-08-17T17:30:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:30:59 smithi130 ceph-mon[105521]: osdmap e522: 8 total, 8 up, 8 in 2024-08-17T17:30:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:59 smithi130 ceph-mon[110068]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:30:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:59 smithi130 ceph-mon[110068]: Upgrade: osd.2 is safe to restart 2024-08-17T17:30:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:59 smithi130 ceph-mon[110068]: pgmap v412: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s rd, 8.0 KiB/s wr, 132 op/s 2024-08-17T17:30:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:30:59 smithi130 ceph-mon[110068]: osdmap e522: 8 total, 8 up, 8 in 2024-08-17T17:30:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:59 smithi179 ceph-mon[93519]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:30:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:59 smithi179 ceph-mon[93519]: Upgrade: osd.2 is safe to restart 2024-08-17T17:30:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:59 smithi179 ceph-mon[93519]: pgmap v412: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 91 KiB/s rd, 8.0 KiB/s wr, 132 op/s 2024-08-17T17:30:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:30:59 smithi179 ceph-mon[93519]: osdmap e522: 8 total, 8 up, 8 in 2024-08-17T17:31:00.006 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:00.127 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:30:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:30:59] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.51.0" 2024-08-17T17:31:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:00 smithi179 ceph-mon[93519]: osdmap e523: 8 total, 8 up, 8 in 2024-08-17T17:31:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:00 smithi179 ceph-mon[93519]: Upgrade: Updating osd.2 2024-08-17T17:31:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:00 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:00 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:31:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:00 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:00.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:00 smithi179 ceph-mon[93519]: Deploying daemon osd.2 on smithi130 2024-08-17T17:31:00.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:00 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/733489373' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:00.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:00 smithi179 ceph-mon[93519]: osdmap e524: 8 total, 8 up, 8 in 2024-08-17T17:31:00.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:31:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:31:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:31:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[110068]: osdmap e523: 8 total, 8 up, 8 in 2024-08-17T17:31:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[110068]: Upgrade: Updating osd.2 2024-08-17T17:31:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:31:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[110068]: Deploying daemon osd.2 on smithi130 2024-08-17T17:31:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/733489373' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[110068]: osdmap e524: 8 total, 8 up, 8 in 2024-08-17T17:31:00.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[105521]: osdmap e523: 8 total, 8 up, 8 in 2024-08-17T17:31:00.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[105521]: Upgrade: Updating osd.2 2024-08-17T17:31:00.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:00.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:31:00.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:00.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[105521]: Deploying daemon osd.2 on smithi130 2024-08-17T17:31:00.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/733489373' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:00.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:00 smithi130 ceph-mon[105521]: osdmap e524: 8 total, 8 up, 8 in 2024-08-17T17:31:01.446 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestMirroring.AioSnapshotCreate (33000 ms) 2024-08-17T17:31:01.603 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:01 smithi130 ceph-mon[105521]: from='client.44384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:01.603 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:01 smithi130 ceph-mon[105521]: pgmap v415: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 123 KiB/s rd, 10 KiB/s wr, 176 op/s 2024-08-17T17:31:01.603 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:01 smithi130 ceph-mon[105521]: from='client.44390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:01.603 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:01 smithi130 ceph-mon[105521]: osdmap e525: 8 total, 8 up, 8 in 2024-08-17T17:31:01.603 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:01 smithi130 ceph-mon[110068]: from='client.44384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:01.603 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:01 smithi130 ceph-mon[110068]: pgmap v415: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 123 KiB/s rd, 10 KiB/s wr, 176 op/s 2024-08-17T17:31:01.603 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:01 smithi130 ceph-mon[110068]: from='client.44390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:01.603 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:01 smithi130 ceph-mon[110068]: osdmap e525: 8 total, 8 up, 8 in 2024-08-17T17:31:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:01 smithi179 ceph-mon[93519]: from='client.44384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:01 smithi179 ceph-mon[93519]: pgmap v415: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 123 KiB/s rd, 10 KiB/s wr, 176 op/s 2024-08-17T17:31:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:01 smithi179 ceph-mon[93519]: from='client.44390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:01 smithi179 ceph-mon[93519]: osdmap e525: 8 total, 8 up, 8 in 2024-08-17T17:31:02.452 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 51 tests from TestMirroring (88557 ms total) 2024-08-17T17:31:02.452 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:31:02.452 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 34 tests from TestInternal 2024-08-17T17:31:02.663 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:02 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8425575a-35b4-450b-9727-b62db9a9c433"} : dispatch 2024-08-17T17:31:02.663 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:02 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8425575a-35b4-450b-9727-b62db9a9c433"} : dispatch 2024-08-17T17:31:02.663 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:02 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8425575a-35b4-450b-9727-b62db9a9c433"}]': finished 2024-08-17T17:31:02.664 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:02 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8425575a-35b4-450b-9727-b62db9a9c433"} : dispatch 2024-08-17T17:31:02.664 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:02 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8425575a-35b4-450b-9727-b62db9a9c433"} : dispatch 2024-08-17T17:31:02.664 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:02 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8425575a-35b4-450b-9727-b62db9a9c433"}]': finished 2024-08-17T17:31:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:02 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2235689025' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8425575a-35b4-450b-9727-b62db9a9c433"} : dispatch 2024-08-17T17:31:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:02 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8425575a-35b4-450b-9727-b62db9a9c433"} : dispatch 2024-08-17T17:31:02.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:02 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/8425575a-35b4-450b-9727-b62db9a9c433"}]': finished 2024-08-17T17:31:02.916 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:02 smithi130 systemd[1]: Stopping Ceph osd.2 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:31:02.923 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:03.417 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:02 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[59596]: 2024-08-17T17:31:02.954+0000 7f0d77423640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:31:03.417 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:02 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[59596]: 2024-08-17T17:31:02.954+0000 7f0d77423640 -1 osd.2 526 *** Got signal Terminated *** 2024-08-17T17:31:03.417 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:02 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[59596]: 2024-08-17T17:31:02.954+0000 7f0d77423640 -1 osd.2 526 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:31:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:03 smithi179 ceph-mon[93519]: pgmap v418: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 2.0 KiB/s wr, 47 op/s 2024-08-17T17:31:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:03 smithi179 ceph-mon[93519]: osdmap e526: 8 total, 8 up, 8 in 2024-08-17T17:31:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:03 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2190416652' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:03 smithi179 ceph-mon[93519]: osd.2 marked itself down and dead 2024-08-17T17:31:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:03 smithi130 ceph-mon[110068]: pgmap v418: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 2.0 KiB/s wr, 47 op/s 2024-08-17T17:31:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:03 smithi130 ceph-mon[110068]: osdmap e526: 8 total, 8 up, 8 in 2024-08-17T17:31:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:03 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2190416652' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:03 smithi130 ceph-mon[110068]: osd.2 marked itself down and dead 2024-08-17T17:31:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:03 smithi130 ceph-mon[105521]: pgmap v418: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 2.0 KiB/s wr, 47 op/s 2024-08-17T17:31:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:03 smithi130 ceph-mon[105521]: osdmap e526: 8 total, 8 up, 8 in 2024-08-17T17:31:03.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:03 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2190416652' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:03.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:03 smithi130 ceph-mon[105521]: osd.2 marked itself down and dead 2024-08-17T17:31:04.336 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:03 smithi130 podman[129922]: 2024-08-17 17:31:03.934893504 +0000 UTC m=+1.110347080 container died d72289cb8b729f34361698b378fb1b37a7c03ce7876e4dfbb6a326e827b289c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0) 2024-08-17T17:31:04.464 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.OpenByID 2024-08-17T17:31:04.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:04 smithi130 podman[129922]: 2024-08-17 17:31:04.33664099 +0000 UTC m=+1.512094552 container cleanup d72289cb8b729f34361698b378fb1b37a7c03ce7876e4dfbb6a326e827b289c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, io.buildah.version=1.36.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:31:04.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:04 smithi130 bash[129922]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2 2024-08-17T17:31:04.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:04 smithi130 podman[129971]: 2024-08-17 17:31:04.530044961 +0000 UTC m=+0.594095889 container remove d72289cb8b729f34361698b378fb1b37a7c03ce7876e4dfbb6a326e827b289c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, RELEASE=quincy-f2c39d4, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:31:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:04 smithi130 ceph-mon[105521]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:04 smithi130 ceph-mon[105521]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:04 smithi130 ceph-mon[105521]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:31:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:04 smithi130 ceph-mon[105521]: osdmap e527: 8 total, 7 up, 8 in 2024-08-17T17:31:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:04 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2960262258' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:04.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:04 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:04.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:04 smithi130 ceph-mon[110068]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:04.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:04 smithi130 ceph-mon[110068]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:04.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:04 smithi130 ceph-mon[110068]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:31:04.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:04 smithi130 ceph-mon[110068]: osdmap e527: 8 total, 7 up, 8 in 2024-08-17T17:31:04.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:04 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2960262258' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:04.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:04 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:04 smithi179 ceph-mon[93519]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:04 smithi179 ceph-mon[93519]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:04 smithi179 ceph-mon[93519]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:31:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:04 smithi179 ceph-mon[93519]: osdmap e527: 8 total, 7 up, 8 in 2024-08-17T17:31:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:04 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2960262258' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:04 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:05.226 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:04 smithi130 podman[130061]: 2024-08-17 17:31:04.865193064 +0000 UTC m=+0.017240689 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:31:05.226 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:04 smithi130 podman[130061]: 2024-08-17 17:31:04.972875365 +0000 UTC m=+0.124922976 container create cd9ef84b163e86c0c8e9bf88acf77194f55549f88fd628883ed0189863c8253c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:31:05.226 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:05 smithi130 podman[130061]: 2024-08-17 17:31:05.225650442 +0000 UTC m=+0.377698054 container init cd9ef84b163e86c0c8e9bf88acf77194f55549f88fd628883ed0189863c8253c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:31:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:05 smithi130 ceph-mon[110068]: pgmap v421: 105 pgs: 32 unknown, 3 stale+active+clean, 70 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:31:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:05 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:31:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:05 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:05 smithi130 ceph-mon[110068]: osdmap e528: 8 total, 7 up, 8 in 2024-08-17T17:31:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:05 smithi130 ceph-mon[105521]: pgmap v421: 105 pgs: 32 unknown, 3 stale+active+clean, 70 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:31:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:05 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:31:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:05 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:05 smithi130 ceph-mon[105521]: osdmap e528: 8 total, 7 up, 8 in 2024-08-17T17:31:05.668 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:05 smithi130 podman[130061]: 2024-08-17 17:31:05.230745815 +0000 UTC m=+0.382793427 container start cd9ef84b163e86c0c8e9bf88acf77194f55549f88fd628883ed0189863c8253c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-08-17T17:31:05.668 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:05 smithi130 podman[130061]: 2024-08-17 17:31:05.317497362 +0000 UTC m=+0.469544991 container attach cd9ef84b163e86c0c8e9bf88acf77194f55549f88fd628883ed0189863c8253c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.36.0) 2024-08-17T17:31:05.668 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:05 smithi130 podman[130061]: 2024-08-17 17:31:05.433794347 +0000 UTC m=+0.585841974 container died cd9ef84b163e86c0c8e9bf88acf77194f55549f88fd628883ed0189863c8253c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:31:05.781 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:05 smithi179 ceph-mon[93519]: pgmap v421: 105 pgs: 32 unknown, 3 stale+active+clean, 70 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:31:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:05 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:31:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:05 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:05 smithi179 ceph-mon[93519]: osdmap e528: 8 total, 7 up, 8 in 2024-08-17T17:31:05.955 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:05 smithi130 podman[130126]: 2024-08-17 17:31:05.938126787 +0000 UTC m=+0.498518967 container remove cd9ef84b163e86c0c8e9bf88acf77194f55549f88fd628883ed0189863c8253c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, ceph=True, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-08-17T17:31:05.955 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:05 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.2.service: Deactivated successfully. 2024-08-17T17:31:06.381 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:05 smithi130 systemd[1]: Stopped Ceph osd.2 for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:31:06.382 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:05 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.2.service: Consumed 22.276s CPU time. 2024-08-17T17:31:06.382 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 systemd[1]: Starting Ceph osd.2 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:31:06.502 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.OpenByID (2038 ms) 2024-08-17T17:31:06.502 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.OpenSnapDNE 2024-08-17T17:31:06.527 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.OpenSnapDNE (25 ms) 2024-08-17T17:31:06.527 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.IsExclusiveLockOwner 2024-08-17T17:31:06.552 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.IsExclusiveLockOwner (25 ms) 2024-08-17T17:31:06.552 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.ResizeLocksImage 2024-08-17T17:31:06.578 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.ResizeLocksImage (26 ms) 2024-08-17T17:31:06.578 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.ResizeFailsToLockImage 2024-08-17T17:31:06.594 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.ResizeFailsToLockImage (15 ms) 2024-08-17T17:31:06.594 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.SnapCreateLocksImage 2024-08-17T17:31:06.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:06 smithi130 ceph-mon[105521]: osdmap e529: 8 total, 7 up, 8 in 2024-08-17T17:31:06.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:06 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2858195174' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:06 smithi130 ceph-mon[105521]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:06 smithi130 ceph-mon[110068]: osdmap e529: 8 total, 7 up, 8 in 2024-08-17T17:31:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:06 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2858195174' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:06 smithi130 ceph-mon[110068]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:06.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 podman[130278]: 2024-08-17 17:31:06.380763093 +0000 UTC m=+0.089569875 container create 4a3222bf44d9768a49c9802eb8f6d85d41d4d9205ae86e9f13e6bbbb5704a444 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-08-17T17:31:06.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 podman[130278]: 2024-08-17 17:31:06.309227175 +0000 UTC m=+0.018016973 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:31:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:06 smithi179 ceph-mon[93519]: osdmap e529: 8 total, 7 up, 8 in 2024-08-17T17:31:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:06 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2858195174' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:06 smithi179 ceph-mon[93519]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:07.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 podman[130278]: 2024-08-17 17:31:06.666266055 +0000 UTC m=+0.375055861 container init 4a3222bf44d9768a49c9802eb8f6d85d41d4d9205ae86e9f13e6bbbb5704a444 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:31:07.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 podman[130278]: 2024-08-17 17:31:06.67121391 +0000 UTC m=+0.380003707 container start 4a3222bf44d9768a49c9802eb8f6d85d41d4d9205ae86e9f13e6bbbb5704a444 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe) 2024-08-17T17:31:07.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 podman[130278]: 2024-08-17 17:31:06.792622568 +0000 UTC m=+0.501412368 container attach 4a3222bf44d9768a49c9802eb8f6d85d41d4d9205ae86e9f13e6bbbb5704a444 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe) 2024-08-17T17:31:07.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[130295]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:07.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 bash[130278]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:07.168 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[130295]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:07.168 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 bash[130278]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:07.168 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[130295]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:31:07.168 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 bash[130278]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:31:07.168 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[130295]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:07.168 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 bash[130278]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:07.168 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[130295]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:07.168 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 bash[130278]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:07.168 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[130295]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:31:07.168 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 bash[130278]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:31:07.168 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[130295]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-08-17T17:31:07.168 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:06 smithi130 bash[130278]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-08-17T17:31:07.468 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[130295]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:31:07.468 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:07 smithi130 bash[130278]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:31:07.468 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[130295]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:31:07.468 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:07 smithi130 bash[130278]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:31:07.468 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[130295]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T17:31:07.468 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:07 smithi130 bash[130278]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T17:31:07.468 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[130295]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:31:07.468 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:07 smithi130 bash[130278]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:31:07.468 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[130295]: --> ceph-volume lvm activate successful for osd ID: 2 2024-08-17T17:31:07.468 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:07 smithi130 bash[130278]: --> ceph-volume lvm activate successful for osd ID: 2 2024-08-17T17:31:07.468 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:07 smithi130 podman[130278]: 2024-08-17 17:31:07.223383594 +0000 UTC m=+0.932173392 container died 4a3222bf44d9768a49c9802eb8f6d85d41d4d9205ae86e9f13e6bbbb5704a444 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, RELEASE=main-cd8bdbe, GIT_CLEAN=True) 2024-08-17T17:31:07.730 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-mon[105521]: pgmap v424: 105 pgs: 3 active+undersized, 25 unknown, 3 stale+active+clean, 1 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2/78 objects degraded (2.564%) 2024-08-17T17:31:07.730 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-mon[105521]: from='client.44438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:07.731 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-mon[105521]: Health check failed: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:31:07.731 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-mon[105521]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:31:07.731 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-mon[105521]: osdmap e530: 8 total, 7 up, 8 in 2024-08-17T17:31:07.731 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-mon[110068]: pgmap v424: 105 pgs: 3 active+undersized, 25 unknown, 3 stale+active+clean, 1 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2/78 objects degraded (2.564%) 2024-08-17T17:31:07.731 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-mon[110068]: from='client.44438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:07.731 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-mon[110068]: Health check failed: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:31:07.731 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-mon[110068]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:31:07.731 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:07 smithi130 ceph-mon[110068]: osdmap e530: 8 total, 7 up, 8 in 2024-08-17T17:31:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:07 smithi179 ceph-mon[93519]: pgmap v424: 105 pgs: 3 active+undersized, 25 unknown, 3 stale+active+clean, 1 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2/78 objects degraded (2.564%) 2024-08-17T17:31:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:07 smithi179 ceph-mon[93519]: from='client.44438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:07 smithi179 ceph-mon[93519]: Health check failed: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:31:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:07 smithi179 ceph-mon[93519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:31:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:07 smithi179 ceph-mon[93519]: osdmap e530: 8 total, 7 up, 8 in 2024-08-17T17:31:08.064 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:07 smithi130 podman[130425]: 2024-08-17 17:31:07.730032514 +0000 UTC m=+0.494909236 container remove 4a3222bf44d9768a49c9802eb8f6d85d41d4d9205ae86e9f13e6bbbb5704a444 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-08-17T17:31:08.064 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:07 smithi130 podman[130491]: 2024-08-17 17:31:07.922665688 +0000 UTC m=+0.095712005 container create 05996ffbb485f9350c8059bfa9d5c6bea45433c3fdace90a921b5a04cd3c4614 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0) 2024-08-17T17:31:08.064 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:07 smithi130 podman[130491]: 2024-08-17 17:31:07.844381359 +0000 UTC m=+0.017427679 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:31:08.482 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:08 smithi130 ceph-mon[105521]: osdmap e531: 8 total, 7 up, 8 in 2024-08-17T17:31:08.482 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:08.482 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:08.482 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:08.483 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:08 smithi130 podman[130491]: 2024-08-17 17:31:08.175424796 +0000 UTC m=+0.348471123 container init 05996ffbb485f9350c8059bfa9d5c6bea45433c3fdace90a921b5a04cd3c4614 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-08-17T17:31:08.483 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:08 smithi130 podman[130491]: 2024-08-17 17:31:08.181281914 +0000 UTC m=+0.354328233 container start 05996ffbb485f9350c8059bfa9d5c6bea45433c3fdace90a921b5a04cd3c4614 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-08-17T17:31:08.483 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:08 smithi130 ceph-osd[130550]: -- 172.21.15.130:0/4294718903 <== mon.0 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55df85081a40 con 0x55df85e4e400 2024-08-17T17:31:08.483 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:08 smithi130 bash[130491]: 05996ffbb485f9350c8059bfa9d5c6bea45433c3fdace90a921b5a04cd3c4614 2024-08-17T17:31:08.483 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:08 smithi130 systemd[1]: Started Ceph osd.2 for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:31:08.483 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:08 smithi130 ceph-mon[110068]: osdmap e531: 8 total, 7 up, 8 in 2024-08-17T17:31:08.483 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:08.483 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:08.483 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:08.483 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:08 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3762630861' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:08.499 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.SnapCreateLocksImage (1906 ms) 2024-08-17T17:31:08.499 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.SnapCreateFailsToLockImage 2024-08-17T17:31:08.524 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.SnapCreateFailsToLockImage (25 ms) 2024-08-17T17:31:08.524 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.SnapRollbackLocksImage 2024-08-17T17:31:08.710 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:08 smithi179 ceph-mon[93519]: osdmap e531: 8 total, 7 up, 8 in 2024-08-17T17:31:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:08 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3762630861' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:08 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3762630861' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:09 smithi130 ceph-mon[105521]: pgmap v427: 105 pgs: 22 active+undersized, 10 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 5.7 KiB/s wr, 77 op/s; 16/123 objects degraded (13.008%) 2024-08-17T17:31:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:09 smithi130 ceph-mon[105521]: osdmap e532: 8 total, 7 up, 8 in 2024-08-17T17:31:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:09 smithi130 ceph-mon[105521]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:09 smithi130 ceph-mon[110068]: pgmap v427: 105 pgs: 22 active+undersized, 10 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 5.7 KiB/s wr, 77 op/s; 16/123 objects degraded (13.008%) 2024-08-17T17:31:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:09 smithi130 ceph-mon[110068]: osdmap e532: 8 total, 7 up, 8 in 2024-08-17T17:31:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:09 smithi130 ceph-mon[110068]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:09 smithi179 ceph-mon[93519]: pgmap v427: 105 pgs: 22 active+undersized, 10 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 5.7 KiB/s wr, 77 op/s; 16/123 objects degraded (13.008%) 2024-08-17T17:31:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:09 smithi179 ceph-mon[93519]: osdmap e532: 8 total, 7 up, 8 in 2024-08-17T17:31:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:09 smithi179 ceph-mon[93519]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:10.075 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:31:09] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.51.0" 2024-08-17T17:31:10.416 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:10 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[130505]: 2024-08-17T17:31:10.081+0000 7f325c53c640 -1 Falling back to public interface 2024-08-17T17:31:10.494 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:31:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:31:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:31:10.507 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.SnapRollbackLocksImage (1983 ms) 2024-08-17T17:31:10.507 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.SnapRollbackFailsToLockImage 2024-08-17T17:31:10.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:10 smithi130 ceph-mon[105521]: from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:10.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:10 smithi130 ceph-mon[105521]: osdmap e533: 8 total, 7 up, 8 in 2024-08-17T17:31:10.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:10 smithi130 ceph-mon[110068]: from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:10.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:10 smithi130 ceph-mon[110068]: osdmap e533: 8 total, 7 up, 8 in 2024-08-17T17:31:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:10 smithi179 ceph-mon[93519]: from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:10 smithi179 ceph-mon[93519]: osdmap e533: 8 total, 7 up, 8 in 2024-08-17T17:31:11.632 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:11 smithi179 ceph-mon[93519]: pgmap v430: 105 pgs: 22 active+undersized, 10 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 6.2 KiB/s wr, 84 op/s; 16/126 objects degraded (12.698%) 2024-08-17T17:31:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:11 smithi179 ceph-mon[93519]: osdmap e534: 8 total, 7 up, 8 in 2024-08-17T17:31:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:11 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/204168361' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:11.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:11 smithi130 ceph-mon[105521]: pgmap v430: 105 pgs: 22 active+undersized, 10 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 6.2 KiB/s wr, 84 op/s; 16/126 objects degraded (12.698%) 2024-08-17T17:31:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:11 smithi130 ceph-mon[105521]: osdmap e534: 8 total, 7 up, 8 in 2024-08-17T17:31:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:11 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/204168361' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:11 smithi130 ceph-mon[110068]: pgmap v430: 105 pgs: 22 active+undersized, 10 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 6.2 KiB/s wr, 84 op/s; 16/126 objects degraded (12.698%) 2024-08-17T17:31:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:11 smithi130 ceph-mon[110068]: osdmap e534: 8 total, 7 up, 8 in 2024-08-17T17:31:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:11 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/204168361' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:12.530 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.SnapRollbackFailsToLockImage (2022 ms) 2024-08-17T17:31:12.530 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.SnapSetReleasesLock 2024-08-17T17:31:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:12 smithi179 ceph-mon[93519]: osdmap e535: 8 total, 7 up, 8 in 2024-08-17T17:31:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:12 smithi179 ceph-mon[93519]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:12.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:12 smithi130 ceph-mon[110068]: osdmap e535: 8 total, 7 up, 8 in 2024-08-17T17:31:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:12 smithi130 ceph-mon[110068]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:12 smithi130 ceph-mon[105521]: osdmap e535: 8 total, 7 up, 8 in 2024-08-17T17:31:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:12 smithi130 ceph-mon[105521]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:13.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[130505]: 2024-08-17T17:31:13.482+0000 7f325c53c640 -1 osd.2 526 log_to_monitors true 2024-08-17T17:31:13.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[130505]: 2024-08-17T17:31:13.533+0000 7f32542e6640 -1 osd.2 526 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:31:13.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-mon[110068]: from='client.34599 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:13.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-mon[110068]: pgmap v433: 105 pgs: 20 active+undersized, 12 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.2 KiB/s wr, 30 op/s; 20/144 objects degraded (13.889%) 2024-08-17T17:31:13.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-mon[110068]: osdmap e536: 8 total, 7 up, 8 in 2024-08-17T17:31:13.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:13.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:13.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 20/144 objects degraded (13.889%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:13.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-mon[110068]: from='osd.2 [v2:172.21.15.130:6818/2944523904,v1:172.21.15.130:6819/2944523904]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:31:13.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-mon[105521]: from='client.34599 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:13.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-mon[105521]: pgmap v433: 105 pgs: 20 active+undersized, 12 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.2 KiB/s wr, 30 op/s; 20/144 objects degraded (13.889%) 2024-08-17T17:31:13.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-mon[105521]: osdmap e536: 8 total, 7 up, 8 in 2024-08-17T17:31:13.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:13.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:13.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 20/144 objects degraded (13.889%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:13.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:13 smithi130 ceph-mon[105521]: from='osd.2 [v2:172.21.15.130:6818/2944523904,v1:172.21.15.130:6819/2944523904]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:31:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:13 smithi179 ceph-mon[93519]: from='client.34599 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:13 smithi179 ceph-mon[93519]: pgmap v433: 105 pgs: 20 active+undersized, 12 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.2 KiB/s wr, 30 op/s; 20/144 objects degraded (13.889%) 2024-08-17T17:31:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:13 smithi179 ceph-mon[93519]: osdmap e536: 8 total, 7 up, 8 in 2024-08-17T17:31:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:13 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 20/144 objects degraded (13.889%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:13.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:13 smithi179 ceph-mon[93519]: from='osd.2 [v2:172.21.15.130:6818/2944523904,v1:172.21.15.130:6819/2944523904]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:31:14.513 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:14 smithi130 ceph-mon[105521]: from='osd.2 [v2:172.21.15.130:6818/2944523904,v1:172.21.15.130:6819/2944523904]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:31:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:14 smithi130 ceph-mon[105521]: osdmap e537: 8 total, 7 up, 8 in 2024-08-17T17:31:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:14 smithi130 ceph-mon[105521]: from='osd.2 [v2:172.21.15.130:6818/2944523904,v1:172.21.15.130:6819/2944523904]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:31:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:14 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3234201038' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:14.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:14 smithi130 ceph-mon[110068]: from='osd.2 [v2:172.21.15.130:6818/2944523904,v1:172.21.15.130:6819/2944523904]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:31:14.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:14 smithi130 ceph-mon[110068]: osdmap e537: 8 total, 7 up, 8 in 2024-08-17T17:31:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:14 smithi130 ceph-mon[110068]: from='osd.2 [v2:172.21.15.130:6818/2944523904,v1:172.21.15.130:6819/2944523904]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:31:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:14 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3234201038' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:14 smithi179 ceph-mon[93519]: from='osd.2 [v2:172.21.15.130:6818/2944523904,v1:172.21.15.130:6819/2944523904]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:31:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:14 smithi179 ceph-mon[93519]: osdmap e537: 8 total, 7 up, 8 in 2024-08-17T17:31:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:14 smithi179 ceph-mon[93519]: from='osd.2 [v2:172.21.15.130:6818/2944523904,v1:172.21.15.130:6819/2944523904]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:31:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:14 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3234201038' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:15.561 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.SnapSetReleasesLock (3031 ms) 2024-08-17T17:31:15.561 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.FlattenLocksImage 2024-08-17T17:31:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:15 smithi130 ceph-mon[110068]: pgmap v436: 105 pgs: 18 active+undersized, 14 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 56 KiB/s rd, 7.7 KiB/s wr, 86 op/s; 23/165 objects degraded (13.939%) 2024-08-17T17:31:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:15 smithi130 ceph-mon[110068]: from='client.44492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:15 smithi130 ceph-mon[110068]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:31:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:15 smithi130 ceph-mon[110068]: osd.2 [v2:172.21.15.130:6818/2944523904,v1:172.21.15.130:6819/2944523904] boot 2024-08-17T17:31:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:15 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:31:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:15 smithi130 ceph-mon[110068]: osdmap e538: 8 total, 8 up, 8 in 2024-08-17T17:31:15.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:15 smithi130 ceph-mon[105521]: pgmap v436: 105 pgs: 18 active+undersized, 14 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 56 KiB/s rd, 7.7 KiB/s wr, 86 op/s; 23/165 objects degraded (13.939%) 2024-08-17T17:31:15.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:15 smithi130 ceph-mon[105521]: from='client.44492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:15.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:15 smithi130 ceph-mon[105521]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:31:15.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:15 smithi130 ceph-mon[105521]: osd.2 [v2:172.21.15.130:6818/2944523904,v1:172.21.15.130:6819/2944523904] boot 2024-08-17T17:31:15.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:15 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:31:15.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:15 smithi130 ceph-mon[105521]: osdmap e538: 8 total, 8 up, 8 in 2024-08-17T17:31:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:15 smithi179 ceph-mon[93519]: pgmap v436: 105 pgs: 18 active+undersized, 14 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 56 KiB/s rd, 7.7 KiB/s wr, 86 op/s; 23/165 objects degraded (13.939%) 2024-08-17T17:31:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:15 smithi179 ceph-mon[93519]: from='client.44492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:15 smithi179 ceph-mon[93519]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:31:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:15 smithi179 ceph-mon[93519]: osd.2 [v2:172.21.15.130:6818/2944523904,v1:172.21.15.130:6819/2944523904] boot 2024-08-17T17:31:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:15 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:31:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:15 smithi179 ceph-mon[93519]: osdmap e538: 8 total, 8 up, 8 in 2024-08-17T17:31:16.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:15 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:31:15.954+0000 7f8e6b27b640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-08-17T17:31:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:16 smithi179 ceph-mon[93519]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:16 smithi179 ceph-mon[93519]: osdmap e539: 8 total, 8 up, 8 in 2024-08-17T17:31:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:31:16.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:16.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:16.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:16.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:16.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:16.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:31:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[105521]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[105521]: osdmap e539: 8 total, 8 up, 8 in 2024-08-17T17:31:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:31:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:16.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:16.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:31:16.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[110068]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:16.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[110068]: osdmap e539: 8 total, 8 up, 8 in 2024-08-17T17:31:16.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:16.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:16.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:16.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:31:16.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:16.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:16.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:16.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:16.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:16.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:31:17.413 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:17.562 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.FlattenLocksImage (2001 ms) 2024-08-17T17:31:17.562 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.FlattenFailsToLockImage 2024-08-17T17:31:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:17 smithi130 ceph-mon[105521]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:31:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:17 smithi130 ceph-mon[105521]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-08-17T17:31:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:17 smithi130 ceph-mon[105521]: pgmap v439: 105 pgs: 3 peering, 16 active+undersized, 13 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 6.0 KiB/s wr, 99 op/s; 21/171 objects degraded (12.281%) 2024-08-17T17:31:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:17 smithi130 ceph-mon[105521]: osdmap e540: 8 total, 8 up, 8 in 2024-08-17T17:31:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:17 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2549155947' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:17.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:17 smithi130 ceph-mon[110068]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:31:17.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:17 smithi130 ceph-mon[110068]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-08-17T17:31:17.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:17 smithi130 ceph-mon[110068]: pgmap v439: 105 pgs: 3 peering, 16 active+undersized, 13 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 6.0 KiB/s wr, 99 op/s; 21/171 objects degraded (12.281%) 2024-08-17T17:31:17.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:17 smithi130 ceph-mon[110068]: osdmap e540: 8 total, 8 up, 8 in 2024-08-17T17:31:17.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:17 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2549155947' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:17 smithi179 ceph-mon[93519]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:31:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:17 smithi179 ceph-mon[93519]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-08-17T17:31:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:17 smithi179 ceph-mon[93519]: pgmap v439: 105 pgs: 3 peering, 16 active+undersized, 13 active+undersized+degraded, 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 6.0 KiB/s wr, 99 op/s; 21/171 objects degraded (12.281%) 2024-08-17T17:31:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:17 smithi179 ceph-mon[93519]: osdmap e540: 8 total, 8 up, 8 in 2024-08-17T17:31:17.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:17 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2549155947' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:18 smithi179 ceph-mon[93519]: from='client.44510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:18 smithi179 ceph-mon[93519]: osdmap e541: 8 total, 8 up, 8 in 2024-08-17T17:31:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:18 smithi179 ceph-mon[93519]: from='client.44516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:18.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:18 smithi130 ceph-mon[105521]: from='client.44510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:18.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:18 smithi130 ceph-mon[105521]: osdmap e541: 8 total, 8 up, 8 in 2024-08-17T17:31:18.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:18 smithi130 ceph-mon[105521]: from='client.44516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:18.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:18 smithi130 ceph-mon[110068]: from='client.44510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:18.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:18 smithi130 ceph-mon[110068]: osdmap e541: 8 total, 8 up, 8 in 2024-08-17T17:31:18.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:18 smithi130 ceph-mon[110068]: from='client.44516 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:19.570 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.FlattenFailsToLockImage (2008 ms) 2024-08-17T17:31:19.570 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklisted 2024-08-17T17:31:19.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:19 smithi130 ceph-mon[105521]: pgmap v442: 105 pgs: 3 peering, 4 active+undersized, 98 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 6.2 KiB/s wr, 96 op/s; 66 B/s, 14 keys/s, 5 objects/s recovering 2024-08-17T17:31:19.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:19 smithi130 ceph-mon[105521]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/171 objects degraded (12.281%), 13 pgs degraded) 2024-08-17T17:31:19.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:19 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T17:31:19.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:19 smithi130 ceph-mon[105521]: osdmap e542: 8 total, 8 up, 8 in 2024-08-17T17:31:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:19 smithi130 ceph-mon[110068]: pgmap v442: 105 pgs: 3 peering, 4 active+undersized, 98 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 6.2 KiB/s wr, 96 op/s; 66 B/s, 14 keys/s, 5 objects/s recovering 2024-08-17T17:31:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:19 smithi130 ceph-mon[110068]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/171 objects degraded (12.281%), 13 pgs degraded) 2024-08-17T17:31:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:19 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T17:31:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:19 smithi130 ceph-mon[110068]: osdmap e542: 8 total, 8 up, 8 in 2024-08-17T17:31:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:19 smithi179 ceph-mon[93519]: pgmap v442: 105 pgs: 3 peering, 4 active+undersized, 98 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 6.2 KiB/s wr, 96 op/s; 66 B/s, 14 keys/s, 5 objects/s recovering 2024-08-17T17:31:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:19 smithi179 ceph-mon[93519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/171 objects degraded (12.281%), 13 pgs degraded) 2024-08-17T17:31:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:19 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T17:31:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:19 smithi179 ceph-mon[93519]: osdmap e542: 8 total, 8 up, 8 in 2024-08-17T17:31:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:31:19] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-08-17T17:31:20.265 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:20.562 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:31:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:31:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:31:20.571 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklisted (1001 ms) 2024-08-17T17:31:20.571 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklistedWatch 2024-08-17T17:31:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:20 smithi179 ceph-mon[93519]: osdmap e543: 8 total, 8 up, 8 in 2024-08-17T17:31:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2447795507' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/2447795507"} : dispatch 2024-08-17T17:31:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2789650460' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:20 smithi130 ceph-mon[105521]: osdmap e543: 8 total, 8 up, 8 in 2024-08-17T17:31:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2447795507' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/2447795507"} : dispatch 2024-08-17T17:31:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2789650460' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:20 smithi130 ceph-mon[110068]: osdmap e543: 8 total, 8 up, 8 in 2024-08-17T17:31:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2447795507' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/2447795507"} : dispatch 2024-08-17T17:31:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2789650460' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:21 smithi179 ceph-mon[93519]: pgmap v445: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 120 KiB/s rd, 8.0 KiB/s wr, 162 op/s; 66 B/s, 14 keys/s, 5 objects/s recovering 2024-08-17T17:31:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:21 smithi179 ceph-mon[93519]: from='client.44534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:21 smithi179 ceph-mon[93519]: from='client.44540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:21 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2447795507' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/2447795507"}]': finished 2024-08-17T17:31:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:21 smithi179 ceph-mon[93519]: osdmap e544: 8 total, 8 up, 8 in 2024-08-17T17:31:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:21 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/485089907' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/485089907"} : dispatch 2024-08-17T17:31:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:21 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/485089907"} : dispatch 2024-08-17T17:31:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:21 smithi130 ceph-mon[105521]: pgmap v445: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 120 KiB/s rd, 8.0 KiB/s wr, 162 op/s; 66 B/s, 14 keys/s, 5 objects/s recovering 2024-08-17T17:31:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:21 smithi130 ceph-mon[105521]: from='client.44534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:21 smithi130 ceph-mon[105521]: from='client.44540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:21 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2447795507' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/2447795507"}]': finished 2024-08-17T17:31:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:21 smithi130 ceph-mon[105521]: osdmap e544: 8 total, 8 up, 8 in 2024-08-17T17:31:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:21 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/485089907' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/485089907"} : dispatch 2024-08-17T17:31:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:21 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/485089907"} : dispatch 2024-08-17T17:31:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:21 smithi130 ceph-mon[110068]: pgmap v445: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 120 KiB/s rd, 8.0 KiB/s wr, 162 op/s; 66 B/s, 14 keys/s, 5 objects/s recovering 2024-08-17T17:31:21.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:21 smithi130 ceph-mon[110068]: from='client.44534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:21.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:21 smithi130 ceph-mon[110068]: from='client.44540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:21.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:21 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2447795507' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/2447795507"}]': finished 2024-08-17T17:31:21.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:21 smithi130 ceph-mon[110068]: osdmap e544: 8 total, 8 up, 8 in 2024-08-17T17:31:21.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:21 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/485089907' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/485089907"} : dispatch 2024-08-17T17:31:21.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:21 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/485089907"} : dispatch 2024-08-17T17:31:22.578 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklistedWatch (2007 ms) 2024-08-17T17:31:22.578 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.AioWriteRequestsLock 2024-08-17T17:31:22.609 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.AioWriteRequestsLock (31 ms) 2024-08-17T17:31:22.610 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.AioDiscardRequestsLock 2024-08-17T17:31:22.636 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.AioDiscardRequestsLock (27 ms) 2024-08-17T17:31:22.636 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.CancelAsyncResize 2024-08-17T17:31:22.679 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.CancelAsyncResize (43 ms) 2024-08-17T17:31:22.680 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.MultipleResize 2024-08-17T17:31:22.731 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.MultipleResize (52 ms) 2024-08-17T17:31:22.731 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.Metadata 2024-08-17T17:31:22.773 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.Metadata (42 ms) 2024-08-17T17:31:22.774 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.MetadataConfApply 2024-08-17T17:31:22.806 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.MetadataConfApply (32 ms) 2024-08-17T17:31:22.806 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.SnapshotCopyup 2024-08-17T17:31:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:22 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/485089907"}]': finished 2024-08-17T17:31:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:22 smithi179 ceph-mon[93519]: osdmap e545: 8 total, 8 up, 8 in 2024-08-17T17:31:22.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:22 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/485089907"}]': finished 2024-08-17T17:31:22.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:22 smithi130 ceph-mon[105521]: osdmap e545: 8 total, 8 up, 8 in 2024-08-17T17:31:22.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:22 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/485089907"}]': finished 2024-08-17T17:31:22.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:22 smithi130 ceph-mon[110068]: osdmap e545: 8 total, 8 up, 8 in 2024-08-17T17:31:23.157 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:23 smithi179 ceph-mon[93519]: pgmap v448: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 6.0 KiB/s wr, 119 op/s 2024-08-17T17:31:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:23 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4072002061' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:23.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:23 smithi130 ceph-mon[105521]: pgmap v448: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 6.0 KiB/s wr, 119 op/s 2024-08-17T17:31:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:23 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4072002061' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:23.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:23 smithi130 ceph-mon[110068]: pgmap v448: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 6.0 KiB/s wr, 119 op/s 2024-08-17T17:31:23.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:23 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4072002061' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:24.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:24 smithi130 ceph-mon[105521]: from='client.44558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:24.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:24 smithi130 ceph-mon[105521]: from='client.44564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:24.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:24 smithi130 ceph-mon[105521]: osdmap e546: 8 total, 8 up, 8 in 2024-08-17T17:31:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:24 smithi130 ceph-mon[110068]: from='client.44558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:24 smithi130 ceph-mon[110068]: from='client.44564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:24 smithi130 ceph-mon[110068]: osdmap e546: 8 total, 8 up, 8 in 2024-08-17T17:31:25.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:24 smithi179 ceph-mon[93519]: from='client.44558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:25.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:24 smithi179 ceph-mon[93519]: from='client.44564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:25.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:24 smithi179 ceph-mon[93519]: osdmap e546: 8 total, 8 up, 8 in 2024-08-17T17:31:25.459 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.SnapshotCopyup (2654 ms) 2024-08-17T17:31:25.459 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.SnapshotCopyupZeros 2024-08-17T17:31:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:25 smithi130 ceph-mon[105521]: pgmap v450: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 21 KiB/s wr, 316 op/s 2024-08-17T17:31:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:25 smithi130 ceph-mon[105521]: osdmap e547: 8 total, 8 up, 8 in 2024-08-17T17:31:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:25 smithi130 ceph-mon[105521]: osdmap e548: 8 total, 8 up, 8 in 2024-08-17T17:31:25.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:25 smithi130 ceph-mon[110068]: pgmap v450: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 21 KiB/s wr, 316 op/s 2024-08-17T17:31:25.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:25 smithi130 ceph-mon[110068]: osdmap e547: 8 total, 8 up, 8 in 2024-08-17T17:31:25.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:25 smithi130 ceph-mon[110068]: osdmap e548: 8 total, 8 up, 8 in 2024-08-17T17:31:25.999 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:26.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:25 smithi179 ceph-mon[93519]: pgmap v450: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 21 KiB/s wr, 316 op/s 2024-08-17T17:31:26.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:25 smithi179 ceph-mon[93519]: osdmap e547: 8 total, 8 up, 8 in 2024-08-17T17:31:26.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:25 smithi179 ceph-mon[93519]: osdmap e548: 8 total, 8 up, 8 in 2024-08-17T17:31:26.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:26 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1859518611' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:26.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:26 smithi130 ceph-mon[105521]: osdmap e549: 8 total, 8 up, 8 in 2024-08-17T17:31:26.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:26 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1859518611' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:26.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:26 smithi130 ceph-mon[110068]: osdmap e549: 8 total, 8 up, 8 in 2024-08-17T17:31:27.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:26 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1859518611' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:27.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:26 smithi179 ceph-mon[93519]: osdmap e549: 8 total, 8 up, 8 in 2024-08-17T17:31:27.413 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.SnapshotCopyupZeros (1954 ms) 2024-08-17T17:31:27.413 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.SnapshotCopyupZerosMigration 2024-08-17T17:31:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:27 smithi130 ceph-mon[105521]: from='client.44576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:27 smithi130 ceph-mon[105521]: pgmap v453: 105 pgs: 105 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 22 KiB/s wr, 255 op/s 2024-08-17T17:31:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:27 smithi130 ceph-mon[105521]: from='client.34704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:27 smithi130 ceph-mon[105521]: osdmap e550: 8 total, 8 up, 8 in 2024-08-17T17:31:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:27 smithi130 ceph-mon[110068]: from='client.44576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:27.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:27 smithi130 ceph-mon[110068]: pgmap v453: 105 pgs: 105 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 22 KiB/s wr, 255 op/s 2024-08-17T17:31:27.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:27 smithi130 ceph-mon[110068]: from='client.34704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:27.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:27.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:27.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:27.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:27 smithi130 ceph-mon[110068]: osdmap e550: 8 total, 8 up, 8 in 2024-08-17T17:31:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:27 smithi179 ceph-mon[93519]: from='client.44576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:27 smithi179 ceph-mon[93519]: pgmap v453: 105 pgs: 105 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 22 KiB/s wr, 255 op/s 2024-08-17T17:31:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:27 smithi179 ceph-mon[93519]: from='client.34704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:28.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:28.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:28.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:27 smithi179 ceph-mon[93519]: osdmap e550: 8 total, 8 up, 8 in 2024-08-17T17:31:28.411 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.SnapshotCopyupZerosMigration (998 ms) 2024-08-17T17:31:28.412 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.ResizeCopyup 2024-08-17T17:31:28.846 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:29.439 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:29 smithi130 ceph-mon[105521]: pgmap v456: 105 pgs: 105 active+clean; 587 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 254 KiB/s rd, 22 KiB/s wr, 340 op/s 2024-08-17T17:31:29.440 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:29 smithi130 ceph-mon[105521]: osdmap e551: 8 total, 8 up, 8 in 2024-08-17T17:31:29.440 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3626476543' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:29.440 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:29 smithi130 ceph-mon[105521]: from='client.34716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:29.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:29 smithi130 ceph-mon[110068]: pgmap v456: 105 pgs: 105 active+clean; 587 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 254 KiB/s rd, 22 KiB/s wr, 340 op/s 2024-08-17T17:31:29.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:29 smithi130 ceph-mon[110068]: osdmap e551: 8 total, 8 up, 8 in 2024-08-17T17:31:29.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3626476543' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:29.440 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:29 smithi130 ceph-mon[110068]: from='client.34716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:29 smithi179 ceph-mon[93519]: pgmap v456: 105 pgs: 105 active+clean; 587 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 254 KiB/s rd, 22 KiB/s wr, 340 op/s 2024-08-17T17:31:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:29 smithi179 ceph-mon[93519]: osdmap e551: 8 total, 8 up, 8 in 2024-08-17T17:31:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3626476543' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:29 smithi179 ceph-mon[93519]: from='client.34716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:31:29] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.51.0" 2024-08-17T17:31:30.446 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.ResizeCopyup (2034 ms) 2024-08-17T17:31:30.446 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.DiscardCopyup 2024-08-17T17:31:30.453 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:31:30.453 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.DiscardCopyup (8 ms) 2024-08-17T17:31:30.453 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.ImageOptions 2024-08-17T17:31:30.461 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.ImageOptions (8 ms) 2024-08-17T17:31:30.461 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.WriteFullCopyup 2024-08-17T17:31:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:30 smithi179 ceph-mon[93519]: from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:30 smithi179 ceph-mon[93519]: osdmap e552: 8 total, 8 up, 8 in 2024-08-17T17:31:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:31:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:31:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:31:30.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:30 smithi130 ceph-mon[105521]: from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:30 smithi130 ceph-mon[105521]: osdmap e552: 8 total, 8 up, 8 in 2024-08-17T17:31:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:30 smithi130 ceph-mon[110068]: from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:30 smithi130 ceph-mon[110068]: osdmap e552: 8 total, 8 up, 8 in 2024-08-17T17:31:31.596 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:31 smithi130 ceph-mon[105521]: pgmap v459: 105 pgs: 105 active+clean; 595 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 281 KiB/s rd, 20 KiB/s wr, 372 op/s 2024-08-17T17:31:31.596 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:31 smithi130 ceph-mon[105521]: osdmap e553: 8 total, 8 up, 8 in 2024-08-17T17:31:31.597 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:31 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:31:31.597 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:31 smithi130 ceph-mon[110068]: pgmap v459: 105 pgs: 105 active+clean; 595 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 281 KiB/s rd, 20 KiB/s wr, 372 op/s 2024-08-17T17:31:31.597 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:31 smithi130 ceph-mon[110068]: osdmap e553: 8 total, 8 up, 8 in 2024-08-17T17:31:31.597 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:31 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:31:31.720 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:31 smithi179 ceph-mon[93519]: pgmap v459: 105 pgs: 105 active+clean; 595 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 281 KiB/s rd, 20 KiB/s wr, 372 op/s 2024-08-17T17:31:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:31 smithi179 ceph-mon[93519]: osdmap e553: 8 total, 8 up, 8 in 2024-08-17T17:31:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:31 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:31:32.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:32 smithi130 ceph-mon[105521]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:31:32.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:32 smithi130 ceph-mon[105521]: Upgrade: osd.3 is safe to restart 2024-08-17T17:31:32.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:32 smithi130 ceph-mon[105521]: osdmap e554: 8 total, 8 up, 8 in 2024-08-17T17:31:32.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1931702399' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:32.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:32 smithi130 ceph-mon[105521]: from='client.44612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:32 smithi130 ceph-mon[110068]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:31:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:32 smithi130 ceph-mon[110068]: Upgrade: osd.3 is safe to restart 2024-08-17T17:31:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:32 smithi130 ceph-mon[110068]: osdmap e554: 8 total, 8 up, 8 in 2024-08-17T17:31:32.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1931702399' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:32.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:32 smithi130 ceph-mon[110068]: from='client.44612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:32 smithi179 ceph-mon[93519]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:31:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:32 smithi179 ceph-mon[93519]: Upgrade: osd.3 is safe to restart 2024-08-17T17:31:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:32 smithi179 ceph-mon[93519]: osdmap e554: 8 total, 8 up, 8 in 2024-08-17T17:31:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1931702399' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:32 smithi179 ceph-mon[93519]: from='client.44612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[105521]: from='client.34740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[105521]: pgmap v462: 105 pgs: 105 active+clean; 600 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 9.2 KiB/s wr, 98 op/s 2024-08-17T17:31:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[105521]: osdmap e555: 8 total, 8 up, 8 in 2024-08-17T17:31:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[105521]: Upgrade: Updating osd.3 2024-08-17T17:31:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:31:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[105521]: Deploying daemon osd.3 on smithi130 2024-08-17T17:31:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[110068]: from='client.34740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[110068]: pgmap v462: 105 pgs: 105 active+clean; 600 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 9.2 KiB/s wr, 98 op/s 2024-08-17T17:31:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[110068]: osdmap e555: 8 total, 8 up, 8 in 2024-08-17T17:31:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[110068]: Upgrade: Updating osd.3 2024-08-17T17:31:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:31:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:33.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:33 smithi130 ceph-mon[110068]: Deploying daemon osd.3 on smithi130 2024-08-17T17:31:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:33 smithi179 ceph-mon[93519]: from='client.34740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:33 smithi179 ceph-mon[93519]: pgmap v462: 105 pgs: 105 active+clean; 600 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 9.2 KiB/s wr, 98 op/s 2024-08-17T17:31:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:33 smithi179 ceph-mon[93519]: osdmap e555: 8 total, 8 up, 8 in 2024-08-17T17:31:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:33 smithi179 ceph-mon[93519]: Upgrade: Updating osd.3 2024-08-17T17:31:33.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:33 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:33.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:33 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:31:33.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:33 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:33.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:33 smithi179 ceph-mon[93519]: Deploying daemon osd.3 on smithi130 2024-08-17T17:31:34.491 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.WriteFullCopyup (4030 ms) 2024-08-17T17:31:34.491 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.DiffIterateCloneOverwrite 2024-08-17T17:31:34.589 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:34.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:34 smithi130 ceph-mon[105521]: osdmap e556: 8 total, 8 up, 8 in 2024-08-17T17:31:34.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:34 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1057971298' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:34 smithi130 ceph-mon[110068]: osdmap e556: 8 total, 8 up, 8 in 2024-08-17T17:31:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:34 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1057971298' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:34 smithi179 ceph-mon[93519]: osdmap e556: 8 total, 8 up, 8 in 2024-08-17T17:31:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:34 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1057971298' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:35.417 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:35 smithi130 systemd[1]: Stopping Ceph osd.3 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:31:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:35 smithi179 ceph-mon[93519]: pgmap v465: 105 pgs: 105 active+clean; 612 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 26 KiB/s wr, 205 op/s 2024-08-17T17:31:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:35 smithi179 ceph-mon[93519]: osdmap e557: 8 total, 8 up, 8 in 2024-08-17T17:31:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:35 smithi179 ceph-mon[93519]: from='client.44630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:35.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:35 smithi130 ceph-mon[110068]: pgmap v465: 105 pgs: 105 active+clean; 612 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 26 KiB/s wr, 205 op/s 2024-08-17T17:31:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:35 smithi130 ceph-mon[110068]: osdmap e557: 8 total, 8 up, 8 in 2024-08-17T17:31:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:35 smithi130 ceph-mon[110068]: from='client.44630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:35 smithi130 ceph-mon[105521]: pgmap v465: 105 pgs: 105 active+clean; 612 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 26 KiB/s wr, 205 op/s 2024-08-17T17:31:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:35 smithi130 ceph-mon[105521]: osdmap e557: 8 total, 8 up, 8 in 2024-08-17T17:31:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:35 smithi130 ceph-mon[105521]: from='client.44630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:35.917 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[65809]: 2024-08-17T17:31:35.491+0000 7f16be699640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:31:35.917 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[65809]: 2024-08-17T17:31:35.491+0000 7f16be699640 -1 osd.3 558 *** Got signal Terminated *** 2024-08-17T17:31:35.918 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:35 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[65809]: 2024-08-17T17:31:35.491+0000 7f16be699640 -1 osd.3 558 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:31:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:36 smithi179 ceph-mon[93519]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:36 smithi179 ceph-mon[93519]: osdmap e558: 8 total, 8 up, 8 in 2024-08-17T17:31:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:36 smithi179 ceph-mon[93519]: osd.3 marked itself down and dead 2024-08-17T17:31:36.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:36 smithi130 ceph-mon[105521]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:36.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:36 smithi130 ceph-mon[105521]: osdmap e558: 8 total, 8 up, 8 in 2024-08-17T17:31:36.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:36 smithi130 ceph-mon[105521]: osd.3 marked itself down and dead 2024-08-17T17:31:36.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:36 smithi130 ceph-mon[110068]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:36.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:36 smithi130 ceph-mon[110068]: osdmap e558: 8 total, 8 up, 8 in 2024-08-17T17:31:36.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:36 smithi130 ceph-mon[110068]: osd.3 marked itself down and dead 2024-08-17T17:31:37.355 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:37 smithi130 podman[135474]: 2024-08-17 17:31:37.042263097 +0000 UTC m=+1.673299178 container died 1509c2840516e94bcfae7f73801bbfb74aaa1acdaf0aa321d3301e1761fff705 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, RELEASE=quincy-f2c39d4, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-08-17T17:31:37.447 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:37.523 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.DiffIterateCloneOverwrite (3032 ms) 2024-08-17T17:31:37.523 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.TestCoR 2024-08-17T17:31:37.532 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING due to disabled rbd_copy_on_read 2024-08-17T17:31:37.532 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.TestCoR (9 ms) 2024-08-17T17:31:37.532 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.FlattenNoEmptyObjects 2024-08-17T17:31:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:37 smithi130 ceph-mon[110068]: pgmap v468: 105 pgs: 105 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.0 MiB/s wr, 198 op/s 2024-08-17T17:31:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:37 smithi130 ceph-mon[110068]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:31:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:37 smithi130 ceph-mon[110068]: osdmap e559: 8 total, 7 up, 8 in 2024-08-17T17:31:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:37 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2578044143' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:37 smithi130 ceph-mon[105521]: pgmap v468: 105 pgs: 105 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.0 MiB/s wr, 198 op/s 2024-08-17T17:31:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:37 smithi130 ceph-mon[105521]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:31:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:37 smithi130 ceph-mon[105521]: osdmap e559: 8 total, 7 up, 8 in 2024-08-17T17:31:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:37 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2578044143' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:37.668 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:37 smithi130 podman[135474]: 2024-08-17 17:31:37.57209803 +0000 UTC m=+2.203134105 container cleanup 1509c2840516e94bcfae7f73801bbfb74aaa1acdaf0aa321d3301e1761fff705 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:31:37.668 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:37 smithi130 bash[135474]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3 2024-08-17T17:31:37.743 INFO:tasks.workunit.client.0.smithi130.stdout:made parent image "image397": 4194304K (1024 * 4K) 2024-08-17T17:31:37.743 INFO:tasks.workunit.client.0.smithi130.stdout:generated random write map: 2024-08-17T17:31:37.744 INFO:tasks.workunit.client.0.smithi130.stdout: [2 , 11152 ] 2024-08-17T17:31:37.744 INFO:tasks.workunit.client.0.smithi130.stdout: [10 , 43489 ] 2024-08-17T17:31:37.744 INFO:tasks.workunit.client.0.smithi130.stdout: [14 , 58505 ] 2024-08-17T17:31:37.744 INFO:tasks.workunit.client.0.smithi130.stdout: [19 , 78115 ] 2024-08-17T17:31:37.744 INFO:tasks.workunit.client.0.smithi130.stdout: [23 , 94748 ] 2024-08-17T17:31:37.744 INFO:tasks.workunit.client.0.smithi130.stdout: [26 , 107427 ] 2024-08-17T17:31:37.744 INFO:tasks.workunit.client.0.smithi130.stdout: [31 , 127519 ] 2024-08-17T17:31:37.744 INFO:tasks.workunit.client.0.smithi130.stdout: [37 , 154439 ] 2024-08-17T17:31:37.744 INFO:tasks.workunit.client.0.smithi130.stdout: [68 , 281532 ] 2024-08-17T17:31:37.744 INFO:tasks.workunit.client.0.smithi130.stdout: [76 , 313589 ] 2024-08-17T17:31:37.744 INFO:tasks.workunit.client.0.smithi130.stdout: [92 , 380207 ] 2024-08-17T17:31:37.744 INFO:tasks.workunit.client.0.smithi130.stdout: [151 , 620287 ] 2024-08-17T17:31:37.745 INFO:tasks.workunit.client.0.smithi130.stdout: [162 , 664861 ] 2024-08-17T17:31:37.745 INFO:tasks.workunit.client.0.smithi130.stdout: [174 , 715291 ] 2024-08-17T17:31:37.745 INFO:tasks.workunit.client.0.smithi130.stdout: [188 , 770915 ] 2024-08-17T17:31:37.745 INFO:tasks.workunit.client.0.smithi130.stdout: [207 , 850857 ] 2024-08-17T17:31:37.745 INFO:tasks.workunit.client.0.smithi130.stdout: [222 , 911469 ] 2024-08-17T17:31:37.745 INFO:tasks.workunit.client.0.smithi130.stdout: [231 , 948348 ] 2024-08-17T17:31:37.745 INFO:tasks.workunit.client.0.smithi130.stdout: [250 , 1024273 ] 2024-08-17T17:31:37.745 INFO:tasks.workunit.client.0.smithi130.stdout: [251 , 1031596 ] 2024-08-17T17:31:37.745 INFO:tasks.workunit.client.0.smithi130.stdout: [257 , 1055338 ] 2024-08-17T17:31:37.745 INFO:tasks.workunit.client.0.smithi130.stdout: [270 , 1105995 ] 2024-08-17T17:31:37.745 INFO:tasks.workunit.client.0.smithi130.stdout: [271 , 1112103 ] 2024-08-17T17:31:37.745 INFO:tasks.workunit.client.0.smithi130.stdout: [289 , 1184805 ] 2024-08-17T17:31:37.745 INFO:tasks.workunit.client.0.smithi130.stdout: [290 , 1189195 ] 2024-08-17T17:31:37.746 INFO:tasks.workunit.client.0.smithi130.stdout: [303 , 1243800 ] 2024-08-17T17:31:37.746 INFO:tasks.workunit.client.0.smithi130.stdout: [304 , 1248051 ] 2024-08-17T17:31:37.746 INFO:tasks.workunit.client.0.smithi130.stdout: [310 , 1272239 ] 2024-08-17T17:31:37.746 INFO:tasks.workunit.client.0.smithi130.stdout: [371 , 1522774 ] 2024-08-17T17:31:37.746 INFO:tasks.workunit.client.0.smithi130.stdout: [373 , 1528103 ] 2024-08-17T17:31:37.746 INFO:tasks.workunit.client.0.smithi130.stdout: [377 , 1547385 ] 2024-08-17T17:31:37.746 INFO:tasks.workunit.client.0.smithi130.stdout: [382 , 1565803 ] 2024-08-17T17:31:37.746 INFO:tasks.workunit.client.0.smithi130.stdout: [387 , 1588712 ] 2024-08-17T17:31:37.746 INFO:tasks.workunit.client.0.smithi130.stdout: [395 , 1618084 ] 2024-08-17T17:31:37.746 INFO:tasks.workunit.client.0.smithi130.stdout: [400 , 1640905 ] 2024-08-17T17:31:37.746 INFO:tasks.workunit.client.0.smithi130.stdout: [409 , 1678411 ] 2024-08-17T17:31:37.746 INFO:tasks.workunit.client.0.smithi130.stdout: [411 , 1685239 ] 2024-08-17T17:31:37.747 INFO:tasks.workunit.client.0.smithi130.stdout: [415 , 1701641 ] 2024-08-17T17:31:37.747 INFO:tasks.workunit.client.0.smithi130.stdout: [434 , 1780013 ] 2024-08-17T17:31:37.747 INFO:tasks.workunit.client.0.smithi130.stdout: [441 , 1809731 ] 2024-08-17T17:31:37.747 INFO:tasks.workunit.client.0.smithi130.stdout: [468 , 1920425 ] 2024-08-17T17:31:37.747 INFO:tasks.workunit.client.0.smithi130.stdout: [469 , 1922661 ] 2024-08-17T17:31:37.747 INFO:tasks.workunit.client.0.smithi130.stdout: [470 , 1927627 ] 2024-08-17T17:31:37.747 INFO:tasks.workunit.client.0.smithi130.stdout: [476 , 1950208 ] 2024-08-17T17:31:37.747 INFO:tasks.workunit.client.0.smithi130.stdout: [490 , 2008031 ] 2024-08-17T17:31:37.747 INFO:tasks.workunit.client.0.smithi130.stdout: [522 , 2139573 ] 2024-08-17T17:31:37.747 INFO:tasks.workunit.client.0.smithi130.stdout: [542 , 2221118 ] 2024-08-17T17:31:37.747 INFO:tasks.workunit.client.0.smithi130.stdout: [552 , 2264153 ] 2024-08-17T17:31:37.747 INFO:tasks.workunit.client.0.smithi130.stdout: [553 , 2265734 ] 2024-08-17T17:31:37.747 INFO:tasks.workunit.client.0.smithi130.stdout: [573 , 2349006 ] 2024-08-17T17:31:37.748 INFO:tasks.workunit.client.0.smithi130.stdout: [586 , 2403011 ] 2024-08-17T17:31:37.748 INFO:tasks.workunit.client.0.smithi130.stdout: [590 , 2418744 ] 2024-08-17T17:31:37.748 INFO:tasks.workunit.client.0.smithi130.stdout: [597 , 2445812 ] 2024-08-17T17:31:37.748 INFO:tasks.workunit.client.0.smithi130.stdout: [599 , 2455053 ] 2024-08-17T17:31:37.748 INFO:tasks.workunit.client.0.smithi130.stdout: [611 , 2505947 ] 2024-08-17T17:31:37.748 INFO:tasks.workunit.client.0.smithi130.stdout: [629 , 2577430 ] 2024-08-17T17:31:37.748 INFO:tasks.workunit.client.0.smithi130.stdout: [636 , 2606501 ] 2024-08-17T17:31:37.748 INFO:tasks.workunit.client.0.smithi130.stdout: [640 , 2622110 ] 2024-08-17T17:31:37.748 INFO:tasks.workunit.client.0.smithi130.stdout: [644 , 2638805 ] 2024-08-17T17:31:37.748 INFO:tasks.workunit.client.0.smithi130.stdout: [646 , 2647727 ] 2024-08-17T17:31:37.748 INFO:tasks.workunit.client.0.smithi130.stdout: [647 , 2652057 ] 2024-08-17T17:31:37.748 INFO:tasks.workunit.client.0.smithi130.stdout: [660 , 2705208 ] 2024-08-17T17:31:37.748 INFO:tasks.workunit.client.0.smithi130.stdout: [668 , 2738038 ] 2024-08-17T17:31:37.748 INFO:tasks.workunit.client.0.smithi130.stdout: [688 , 2821067 ] 2024-08-17T17:31:37.749 INFO:tasks.workunit.client.0.smithi130.stdout: [691 , 2833393 ] 2024-08-17T17:31:37.749 INFO:tasks.workunit.client.0.smithi130.stdout: [692 , 2836196 ] 2024-08-17T17:31:37.749 INFO:tasks.workunit.client.0.smithi130.stdout: [695 , 2848562 ] 2024-08-17T17:31:37.749 INFO:tasks.workunit.client.0.smithi130.stdout: [715 , 2931983 ] 2024-08-17T17:31:37.749 INFO:tasks.workunit.client.0.smithi130.stdout: [718 , 2941590 ] 2024-08-17T17:31:37.749 INFO:tasks.workunit.client.0.smithi130.stdout: [723 , 2962513 ] 2024-08-17T17:31:37.749 INFO:tasks.workunit.client.0.smithi130.stdout: [738 , 3025305 ] 2024-08-17T17:31:37.749 INFO:tasks.workunit.client.0.smithi130.stdout: [742 , 3040817 ] 2024-08-17T17:31:37.749 INFO:tasks.workunit.client.0.smithi130.stdout: [743 , 3043999 ] 2024-08-17T17:31:37.749 INFO:tasks.workunit.client.0.smithi130.stdout: [773 , 3168082 ] 2024-08-17T17:31:37.749 INFO:tasks.workunit.client.0.smithi130.stdout: [776 , 3179132 ] 2024-08-17T17:31:37.749 INFO:tasks.workunit.client.0.smithi130.stdout: [781 , 3199890 ] 2024-08-17T17:31:37.749 INFO:tasks.workunit.client.0.smithi130.stdout: [797 , 3266077 ] 2024-08-17T17:31:37.749 INFO:tasks.workunit.client.0.smithi130.stdout: [800 , 3279287 ] 2024-08-17T17:31:37.750 INFO:tasks.workunit.client.0.smithi130.stdout: [801 , 3282181 ] 2024-08-17T17:31:37.750 INFO:tasks.workunit.client.0.smithi130.stdout: [809 , 3314905 ] 2024-08-17T17:31:37.750 INFO:tasks.workunit.client.0.smithi130.stdout: [812 , 3326770 ] 2024-08-17T17:31:37.750 INFO:tasks.workunit.client.0.smithi130.stdout: [818 , 3351387 ] 2024-08-17T17:31:37.750 INFO:tasks.workunit.client.0.smithi130.stdout: [841 , 3446100 ] 2024-08-17T17:31:37.750 INFO:tasks.workunit.client.0.smithi130.stdout: [847 , 3470499 ] 2024-08-17T17:31:37.750 INFO:tasks.workunit.client.0.smithi130.stdout: [859 , 3521040 ] 2024-08-17T17:31:37.750 INFO:tasks.workunit.client.0.smithi130.stdout: [869 , 3561162 ] 2024-08-17T17:31:37.750 INFO:tasks.workunit.client.0.smithi130.stdout: [888 , 3639144 ] 2024-08-17T17:31:37.750 INFO:tasks.workunit.client.0.smithi130.stdout: [922 , 3777365 ] 2024-08-17T17:31:37.750 INFO:tasks.workunit.client.0.smithi130.stdout: [942 , 3859814 ] 2024-08-17T17:31:37.750 INFO:tasks.workunit.client.0.smithi130.stdout: [963 , 3945391 ] 2024-08-17T17:31:37.750 INFO:tasks.workunit.client.0.smithi130.stdout: [979 , 4012268 ] 2024-08-17T17:31:37.750 INFO:tasks.workunit.client.0.smithi130.stdout: [1003 , 4111670 ] 2024-08-17T17:31:37.751 INFO:tasks.workunit.client.0.smithi130.stdout: [1015 , 4160585 ] 2024-08-17T17:31:37.751 INFO:tasks.workunit.client.0.smithi130.stdout: [1021 , 4184306 ] 2024-08-17T17:31:37.751 INFO:tasks.workunit.client.0.smithi130.stdout:write data based on random map 2024-08-17T17:31:37.751 INFO:tasks.workunit.client.0.smithi130.stdout: write object-2 2024-08-17T17:31:37.751 INFO:tasks.workunit.client.0.smithi130.stdout: write object-10 2024-08-17T17:31:37.751 INFO:tasks.workunit.client.0.smithi130.stdout: write object-14 2024-08-17T17:31:37.751 INFO:tasks.workunit.client.0.smithi130.stdout: write object-19 2024-08-17T17:31:37.751 INFO:tasks.workunit.client.0.smithi130.stdout: write object-23 2024-08-17T17:31:37.751 INFO:tasks.workunit.client.0.smithi130.stdout: write object-26 2024-08-17T17:31:37.751 INFO:tasks.workunit.client.0.smithi130.stdout: write object-31 2024-08-17T17:31:37.751 INFO:tasks.workunit.client.0.smithi130.stdout: write object-37 2024-08-17T17:31:37.751 INFO:tasks.workunit.client.0.smithi130.stdout: write object-68 2024-08-17T17:31:37.751 INFO:tasks.workunit.client.0.smithi130.stdout: write object-76 2024-08-17T17:31:37.752 INFO:tasks.workunit.client.0.smithi130.stdout: write object-92 2024-08-17T17:31:37.752 INFO:tasks.workunit.client.0.smithi130.stdout: write object-151 2024-08-17T17:31:37.752 INFO:tasks.workunit.client.0.smithi130.stdout: write object-162 2024-08-17T17:31:37.752 INFO:tasks.workunit.client.0.smithi130.stdout: write object-174 2024-08-17T17:31:37.752 INFO:tasks.workunit.client.0.smithi130.stdout: write object-188 2024-08-17T17:31:37.752 INFO:tasks.workunit.client.0.smithi130.stdout: write object-207 2024-08-17T17:31:37.752 INFO:tasks.workunit.client.0.smithi130.stdout: write object-222 2024-08-17T17:31:37.752 INFO:tasks.workunit.client.0.smithi130.stdout: write object-231 2024-08-17T17:31:37.752 INFO:tasks.workunit.client.0.smithi130.stdout: write object-250 2024-08-17T17:31:37.752 INFO:tasks.workunit.client.0.smithi130.stdout: write object-251 2024-08-17T17:31:37.752 INFO:tasks.workunit.client.0.smithi130.stdout: write object-257 2024-08-17T17:31:37.752 INFO:tasks.workunit.client.0.smithi130.stdout: write object-270 2024-08-17T17:31:37.752 INFO:tasks.workunit.client.0.smithi130.stdout: write object-271 2024-08-17T17:31:37.753 INFO:tasks.workunit.client.0.smithi130.stdout: write object-289 2024-08-17T17:31:37.753 INFO:tasks.workunit.client.0.smithi130.stdout: write object-290 2024-08-17T17:31:37.753 INFO:tasks.workunit.client.0.smithi130.stdout: write object-303 2024-08-17T17:31:37.753 INFO:tasks.workunit.client.0.smithi130.stdout: write object-304 2024-08-17T17:31:37.753 INFO:tasks.workunit.client.0.smithi130.stdout: write object-310 2024-08-17T17:31:37.753 INFO:tasks.workunit.client.0.smithi130.stdout: write object-371 2024-08-17T17:31:37.753 INFO:tasks.workunit.client.0.smithi130.stdout: write object-373 2024-08-17T17:31:37.753 INFO:tasks.workunit.client.0.smithi130.stdout: write object-377 2024-08-17T17:31:37.753 INFO:tasks.workunit.client.0.smithi130.stdout: write object-382 2024-08-17T17:31:37.753 INFO:tasks.workunit.client.0.smithi130.stdout: write object-387 2024-08-17T17:31:37.753 INFO:tasks.workunit.client.0.smithi130.stdout: write object-395 2024-08-17T17:31:37.753 INFO:tasks.workunit.client.0.smithi130.stdout: write object-400 2024-08-17T17:31:37.753 INFO:tasks.workunit.client.0.smithi130.stdout: write object-409 2024-08-17T17:31:37.753 INFO:tasks.workunit.client.0.smithi130.stdout: write object-411 2024-08-17T17:31:37.754 INFO:tasks.workunit.client.0.smithi130.stdout: write object-415 2024-08-17T17:31:37.754 INFO:tasks.workunit.client.0.smithi130.stdout: write object-434 2024-08-17T17:31:37.754 INFO:tasks.workunit.client.0.smithi130.stdout: write object-441 2024-08-17T17:31:37.754 INFO:tasks.workunit.client.0.smithi130.stdout: write object-468 2024-08-17T17:31:37.754 INFO:tasks.workunit.client.0.smithi130.stdout: write object-469 2024-08-17T17:31:37.754 INFO:tasks.workunit.client.0.smithi130.stdout: write object-470 2024-08-17T17:31:37.754 INFO:tasks.workunit.client.0.smithi130.stdout: write object-476 2024-08-17T17:31:37.754 INFO:tasks.workunit.client.0.smithi130.stdout: write object-490 2024-08-17T17:31:37.754 INFO:tasks.workunit.client.0.smithi130.stdout: write object-522 2024-08-17T17:31:37.754 INFO:tasks.workunit.client.0.smithi130.stdout: write object-542 2024-08-17T17:31:37.754 INFO:tasks.workunit.client.0.smithi130.stdout: write object-552 2024-08-17T17:31:37.754 INFO:tasks.workunit.client.0.smithi130.stdout: write object-553 2024-08-17T17:31:37.754 INFO:tasks.workunit.client.0.smithi130.stdout: write object-573 2024-08-17T17:31:37.754 INFO:tasks.workunit.client.0.smithi130.stdout: write object-586 2024-08-17T17:31:37.755 INFO:tasks.workunit.client.0.smithi130.stdout: write object-590 2024-08-17T17:31:37.755 INFO:tasks.workunit.client.0.smithi130.stdout: write object-597 2024-08-17T17:31:37.755 INFO:tasks.workunit.client.0.smithi130.stdout: write object-599 2024-08-17T17:31:37.755 INFO:tasks.workunit.client.0.smithi130.stdout: write object-611 2024-08-17T17:31:37.755 INFO:tasks.workunit.client.0.smithi130.stdout: write object-629 2024-08-17T17:31:37.755 INFO:tasks.workunit.client.0.smithi130.stdout: write object-636 2024-08-17T17:31:37.755 INFO:tasks.workunit.client.0.smithi130.stdout: write object-640 2024-08-17T17:31:37.755 INFO:tasks.workunit.client.0.smithi130.stdout: write object-644 2024-08-17T17:31:37.755 INFO:tasks.workunit.client.0.smithi130.stdout: write object-646 2024-08-17T17:31:37.755 INFO:tasks.workunit.client.0.smithi130.stdout: write object-647 2024-08-17T17:31:37.755 INFO:tasks.workunit.client.0.smithi130.stdout: write object-660 2024-08-17T17:31:37.755 INFO:tasks.workunit.client.0.smithi130.stdout: write object-668 2024-08-17T17:31:37.755 INFO:tasks.workunit.client.0.smithi130.stdout: write object-688 2024-08-17T17:31:37.755 INFO:tasks.workunit.client.0.smithi130.stdout: write object-691 2024-08-17T17:31:37.756 INFO:tasks.workunit.client.0.smithi130.stdout: write object-692 2024-08-17T17:31:37.756 INFO:tasks.workunit.client.0.smithi130.stdout: write object-695 2024-08-17T17:31:37.756 INFO:tasks.workunit.client.0.smithi130.stdout: write object-715 2024-08-17T17:31:37.756 INFO:tasks.workunit.client.0.smithi130.stdout: write object-718 2024-08-17T17:31:37.756 INFO:tasks.workunit.client.0.smithi130.stdout: write object-723 2024-08-17T17:31:37.756 INFO:tasks.workunit.client.0.smithi130.stdout: write object-738 2024-08-17T17:31:37.756 INFO:tasks.workunit.client.0.smithi130.stdout: write object-742 2024-08-17T17:31:37.756 INFO:tasks.workunit.client.0.smithi130.stdout: write object-743 2024-08-17T17:31:37.756 INFO:tasks.workunit.client.0.smithi130.stdout: write object-773 2024-08-17T17:31:37.756 INFO:tasks.workunit.client.0.smithi130.stdout: write object-776 2024-08-17T17:31:37.756 INFO:tasks.workunit.client.0.smithi130.stdout: write object-781 2024-08-17T17:31:37.756 INFO:tasks.workunit.client.0.smithi130.stdout: write object-797 2024-08-17T17:31:37.756 INFO:tasks.workunit.client.0.smithi130.stdout: write object-800 2024-08-17T17:31:37.756 INFO:tasks.workunit.client.0.smithi130.stdout: write object-801 2024-08-17T17:31:37.757 INFO:tasks.workunit.client.0.smithi130.stdout: write object-809 2024-08-17T17:31:37.757 INFO:tasks.workunit.client.0.smithi130.stdout: write object-812 2024-08-17T17:31:37.757 INFO:tasks.workunit.client.0.smithi130.stdout: write object-818 2024-08-17T17:31:37.757 INFO:tasks.workunit.client.0.smithi130.stdout: write object-841 2024-08-17T17:31:37.757 INFO:tasks.workunit.client.0.smithi130.stdout: write object-847 2024-08-17T17:31:37.757 INFO:tasks.workunit.client.0.smithi130.stdout: write object-859 2024-08-17T17:31:37.757 INFO:tasks.workunit.client.0.smithi130.stdout: write object-869 2024-08-17T17:31:37.757 INFO:tasks.workunit.client.0.smithi130.stdout: write object-888 2024-08-17T17:31:37.757 INFO:tasks.workunit.client.0.smithi130.stdout: write object-922 2024-08-17T17:31:37.757 INFO:tasks.workunit.client.0.smithi130.stdout: write object-942 2024-08-17T17:31:37.757 INFO:tasks.workunit.client.0.smithi130.stdout: write object-963 2024-08-17T17:31:37.757 INFO:tasks.workunit.client.0.smithi130.stdout: write object-979 2024-08-17T17:31:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:37 smithi179 ceph-mon[93519]: pgmap v468: 105 pgs: 105 active+clean; 4.6 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.0 MiB/s wr, 198 op/s 2024-08-17T17:31:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:37 smithi179 ceph-mon[93519]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:31:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:37 smithi179 ceph-mon[93519]: osdmap e559: 8 total, 7 up, 8 in 2024-08-17T17:31:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:37 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2578044143' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:38.167 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:37 smithi130 podman[135515]: 2024-08-17 17:31:37.816025478 +0000 UTC m=+0.772995523 container remove 1509c2840516e94bcfae7f73801bbfb74aaa1acdaf0aa321d3301e1761fff705 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-f2c39d4, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-08-17T17:31:38.503 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:38 smithi130 podman[135661]: 2024-08-17 17:31:38.21716365 +0000 UTC m=+0.096163674 container create 887263ccda55ebce279e9ae64b30e67f226dcb661c3f66292c80aa8205261fdb (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True) 2024-08-17T17:31:38.503 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:38 smithi130 podman[135661]: 2024-08-17 17:31:38.138428316 +0000 UTC m=+0.017428313 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:31:38.503 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:38 smithi130 podman[135661]: 2024-08-17 17:31:38.436202696 +0000 UTC m=+0.315202689 container init 887263ccda55ebce279e9ae64b30e67f226dcb661c3f66292c80aa8205261fdb (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:31:38.504 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:38 smithi130 podman[135661]: 2024-08-17 17:31:38.440541603 +0000 UTC m=+0.319541590 container start 887263ccda55ebce279e9ae64b30e67f226dcb661c3f66292c80aa8205261fdb (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0) 2024-08-17T17:31:38.683 INFO:tasks.workunit.client.0.smithi130.stdout: write object-1003 2024-08-17T17:31:38.684 INFO:tasks.workunit.client.0.smithi130.stdout: write object-1015 2024-08-17T17:31:38.684 INFO:tasks.workunit.client.0.smithi130.stdout: write object-1021 2024-08-17T17:31:38.684 INFO:tasks.workunit.client.0.smithi130.stdout:verify written data by reading 2024-08-17T17:31:38.684 INFO:tasks.workunit.client.0.smithi130.stdout: read object-2 2024-08-17T17:31:38.684 INFO:tasks.workunit.client.0.smithi130.stdout:made snapshot "image397@parent_snap" and protect it 2024-08-17T17:31:38.684 INFO:tasks.workunit.client.0.smithi130.stdout:made and opened clone "image398" 2024-08-17T17:31:38.684 INFO:tasks.workunit.client.0.smithi130.stdout:flattening clone: "image398" 2024-08-17T17:31:38.684 INFO:tasks.workunit.client.0.smithi130.stdout:check whether child image has the same set of objects as parent 2024-08-17T17:31:38.684 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.FlattenNoEmptyObjects (1151 ms) 2024-08-17T17:31:38.684 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.PoolMetadataConfApply 2024-08-17T17:31:38.758 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.PoolMetadataConfApply (75 ms) 2024-08-17T17:31:38.759 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.Sparsify 2024-08-17T17:31:38.781 INFO:tasks.workunit.client.0.smithi130.stdout:sparsify_supported=1 2024-08-17T17:31:38.781 INFO:tasks.workunit.client.0.smithi130.stdout:sparse_read_supported=1 2024-08-17T17:31:38.810 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.Sparsify (52 ms) 2024-08-17T17:31:38.810 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.SparsifyClone 2024-08-17T17:31:38.826 INFO:tasks.workunit.client.0.smithi130.stdout:sparsify_supported=1 2024-08-17T17:31:38.835 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:38 smithi130 ceph-mon[105521]: from='client.34770 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:38.835 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:38 smithi130 ceph-mon[105521]: osdmap e560: 8 total, 7 up, 8 in 2024-08-17T17:31:38.836 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:38 smithi130 ceph-mon[105521]: from='client.44654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:38.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:38 smithi130 ceph-mon[110068]: from='client.34770 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:38.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:38 smithi130 ceph-mon[110068]: osdmap e560: 8 total, 7 up, 8 in 2024-08-17T17:31:38.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:38 smithi130 ceph-mon[110068]: from='client.44654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:38.836 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:38 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:31:38] ENGINE Bus STOPPING 2024-08-17T17:31:38.836 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:38 smithi130 podman[135661]: 2024-08-17 17:31:38.545207024 +0000 UTC m=+0.424207021 container attach 887263ccda55ebce279e9ae64b30e67f226dcb661c3f66292c80aa8205261fdb (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-08-17T17:31:38.837 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:38 smithi130 podman[135661]: 2024-08-17 17:31:38.660266431 +0000 UTC m=+0.539266422 container died 887263ccda55ebce279e9ae64b30e67f226dcb661c3f66292c80aa8205261fdb (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-08-17T17:31:38.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:38 smithi179 ceph-mon[93519]: from='client.34770 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:38 smithi179 ceph-mon[93519]: osdmap e560: 8 total, 7 up, 8 in 2024-08-17T17:31:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:38 smithi179 ceph-mon[93519]: from='client.44654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:39.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:38 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:31:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:31:39.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:38 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:31:38] ENGINE Bus STOPPED 2024-08-17T17:31:39.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:38 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:31:38] ENGINE Bus STARTING 2024-08-17T17:31:39.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:38 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:31:38] ENGINE Serving on http://:::9283 2024-08-17T17:31:39.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:38 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:31:38] ENGINE Bus STARTED 2024-08-17T17:31:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:39 smithi130 ceph-mon[105521]: pgmap v471: 105 pgs: 12 active+undersized, 8 stale+active+clean, 11 active+undersized+degraded, 74 active+clean; 4.8 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.1 MiB/s wr, 344 op/s; 65/693 objects degraded (9.380%) 2024-08-17T17:31:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:39 smithi130 ceph-mon[105521]: Health check failed: Degraded data redundancy: 65/693 objects degraded (9.380%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:39 smithi130 ceph-mon[105521]: osdmap e561: 8 total, 7 up, 8 in 2024-08-17T17:31:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:39 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' 2024-08-17T17:31:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:39 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:31:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:39 smithi130 ceph-mon[105521]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:31:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:39 smithi130 ceph-mon[110068]: pgmap v471: 105 pgs: 12 active+undersized, 8 stale+active+clean, 11 active+undersized+degraded, 74 active+clean; 4.8 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.1 MiB/s wr, 344 op/s; 65/693 objects degraded (9.380%) 2024-08-17T17:31:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:39 smithi130 ceph-mon[110068]: Health check failed: Degraded data redundancy: 65/693 objects degraded (9.380%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:39 smithi130 ceph-mon[110068]: osdmap e561: 8 total, 7 up, 8 in 2024-08-17T17:31:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:39 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' 2024-08-17T17:31:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:39 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:31:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:39 smithi130 ceph-mon[110068]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:31:39.668 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:39 smithi130 podman[135704]: 2024-08-17 17:31:39.282388634 +0000 UTC m=+0.613471169 container remove 887263ccda55ebce279e9ae64b30e67f226dcb661c3f66292c80aa8205261fdb (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:31:39.668 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:39 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.3.service: Deactivated successfully. 2024-08-17T17:31:39.669 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:39 smithi130 systemd[1]: Stopped Ceph osd.3 for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:31:39.669 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:39 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.3.service: Consumed 21.291s CPU time. 2024-08-17T17:31:39.669 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:39 smithi130 systemd[1]: Starting Ceph osd.3 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:31:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:39 smithi179 ceph-mon[93519]: pgmap v471: 105 pgs: 12 active+undersized, 8 stale+active+clean, 11 active+undersized+degraded, 74 active+clean; 4.8 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.1 MiB/s wr, 344 op/s; 65/693 objects degraded (9.380%) 2024-08-17T17:31:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:39 smithi179 ceph-mon[93519]: Health check failed: Degraded data redundancy: 65/693 objects degraded (9.380%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:39 smithi179 ceph-mon[93519]: osdmap e561: 8 total, 7 up, 8 in 2024-08-17T17:31:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:39 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' 2024-08-17T17:31:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:39 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:31:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:39 smithi179 ceph-mon[93519]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:31:39.975 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:31:39] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.51.0" 2024-08-17T17:31:39.976 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:39 smithi130 podman[135819]: 2024-08-17 17:31:39.624732052 +0000 UTC m=+0.017587570 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:31:39.976 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:39 smithi130 podman[135819]: 2024-08-17 17:31:39.725338953 +0000 UTC m=+0.118194466 container create 3eeef4d9954c2262345028b1d08afd134280971564a4b3abfe31ff55569485f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0) 2024-08-17T17:31:39.976 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:39 smithi130 podman[135819]: 2024-08-17 17:31:39.97028469 +0000 UTC m=+0.363140223 container init 3eeef4d9954c2262345028b1d08afd134280971564a4b3abfe31ff55569485f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:31:39.976 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:39 smithi130 podman[135819]: 2024-08-17 17:31:39.975241284 +0000 UTC m=+0.368096797 container start 3eeef4d9954c2262345028b1d08afd134280971564a4b3abfe31ff55569485f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-08-17T17:31:40.365 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:40.365 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 podman[135819]: 2024-08-17 17:31:40.0782989 +0000 UTC m=+0.471154414 container attach 3eeef4d9954c2262345028b1d08afd134280971564a4b3abfe31ff55569485f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-08-17T17:31:40.365 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[135838]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:40.365 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 bash[135819]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:40.365 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[135838]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:40.365 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 bash[135819]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:40.365 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[135838]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:31:40.365 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 bash[135819]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:31:40.366 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[135838]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:40.366 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 bash[135819]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:40.366 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[135838]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:40.366 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 bash[135819]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:31:40.366 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[135838]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:31:40.366 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 bash[135819]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:31:40.366 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[135838]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-08-17T17:31:40.366 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 bash[135819]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-08-17T17:31:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:31:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:31:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:31:40.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[135838]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:31:40.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 bash[135819]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:31:40.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[135838]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:31:40.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 bash[135819]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:31:40.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[135838]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:31:40.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 bash[135819]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:31:40.668 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[135838]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:31:40.668 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 bash[135819]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:31:40.668 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[135838]: --> ceph-volume lvm activate successful for osd ID: 3 2024-08-17T17:31:40.668 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 bash[135819]: --> ceph-volume lvm activate successful for osd ID: 3 2024-08-17T17:31:40.668 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:40 smithi130 podman[135819]: 2024-08-17 17:31:40.545417639 +0000 UTC m=+0.938273155 container died 3eeef4d9954c2262345028b1d08afd134280971564a4b3abfe31ff55569485f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True) 2024-08-17T17:31:40.731 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.SparsifyClone (1921 ms) 2024-08-17T17:31:40.731 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestInternal.MissingDataPool 2024-08-17T17:31:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:40 smithi179 ceph-mon[93519]: osdmap e562: 8 total, 7 up, 8 in 2024-08-17T17:31:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:40 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2324341677' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:41.125 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-mon[105521]: osdmap e562: 8 total, 7 up, 8 in 2024-08-17T17:31:41.125 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2324341677' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:41.125 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-mon[110068]: osdmap e562: 8 total, 7 up, 8 in 2024-08-17T17:31:41.125 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:40 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2324341677' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:41.417 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:41 smithi130 podman[136030]: 2024-08-17 17:31:41.124312041 +0000 UTC m=+0.572528044 container remove 3eeef4d9954c2262345028b1d08afd134280971564a4b3abfe31ff55569485f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-08-17T17:31:41.417 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:41 smithi130 podman[136096]: 2024-08-17 17:31:41.325242243 +0000 UTC m=+0.101309585 container create a4c158c50e3528fb0b38cfc7326954183eac2fd6dff97dc1801d76cde28d99e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe) 2024-08-17T17:31:41.417 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:41 smithi130 podman[136096]: 2024-08-17 17:31:41.240935632 +0000 UTC m=+0.017002985 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:31:41.832 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestInternal.MissingDataPool (1100 ms) 2024-08-17T17:31:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-mon[105521]: pgmap v474: 105 pgs: 22 active+undersized, 3 stale+active+clean, 15 active+undersized+degraded, 65 active+clean; 4.8 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 438 op/s; 94/828 objects degraded (11.353%) 2024-08-17T17:31:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-mon[105521]: from='client.44666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-mon[105521]: from='client.44672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-mon[105521]: osdmap e563: 8 total, 7 up, 8 in 2024-08-17T17:31:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:41.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-mon[110068]: pgmap v474: 105 pgs: 22 active+undersized, 3 stale+active+clean, 15 active+undersized+degraded, 65 active+clean; 4.8 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 438 op/s; 94/828 objects degraded (11.353%) 2024-08-17T17:31:41.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-mon[110068]: from='client.44666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:41.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-mon[110068]: from='client.44672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:41.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-mon[110068]: osdmap e563: 8 total, 7 up, 8 in 2024-08-17T17:31:41.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:41.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:41.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:41.918 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:41 smithi130 podman[136096]: 2024-08-17 17:31:41.51919701 +0000 UTC m=+0.295264357 container init a4c158c50e3528fb0b38cfc7326954183eac2fd6dff97dc1801d76cde28d99e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-08-17T17:31:41.918 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:41 smithi130 podman[136096]: 2024-08-17 17:31:41.52376595 +0000 UTC m=+0.299833290 container start a4c158c50e3528fb0b38cfc7326954183eac2fd6dff97dc1801d76cde28d99e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:31:41.919 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:41 smithi130 ceph-osd[136133]: -- 172.21.15.130:0/2975045220 <== mon.0 v2:172.21.15.130:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56118cd33a40 con 0x56118db02400 2024-08-17T17:31:41.919 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:41 smithi130 bash[136096]: a4c158c50e3528fb0b38cfc7326954183eac2fd6dff97dc1801d76cde28d99e3 2024-08-17T17:31:41.919 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:41 smithi130 systemd[1]: Started Ceph osd.3 for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:31:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:41 smithi179 ceph-mon[93519]: pgmap v474: 105 pgs: 22 active+undersized, 3 stale+active+clean, 15 active+undersized+degraded, 65 active+clean; 4.8 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 438 op/s; 94/828 objects degraded (11.353%) 2024-08-17T17:31:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:41 smithi179 ceph-mon[93519]: from='client.44666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:41 smithi179 ceph-mon[93519]: from='client.44672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:41 smithi179 ceph-mon[93519]: osdmap e563: 8 total, 7 up, 8 in 2024-08-17T17:31:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:41 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:41 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:41 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:42.741 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 34 tests from TestInternal (37367 ms total) 2024-08-17T17:31:42.741 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:31:42.741 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 17 tests from TestImageWatcher 2024-08-17T17:31:42.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:42 smithi130 ceph-mon[105521]: osdmap e564: 8 total, 7 up, 8 in 2024-08-17T17:31:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:42 smithi130 ceph-mon[110068]: osdmap e564: 8 total, 7 up, 8 in 2024-08-17T17:31:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:42 smithi179 ceph-mon[93519]: osdmap e564: 8 total, 7 up, 8 in 2024-08-17T17:31:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:43.358 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:43.633 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:43 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[136110]: 2024-08-17T17:31:43.404+0000 7f1c83621640 -1 Falling back to public interface 2024-08-17T17:31:43.743 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:43 smithi179 ceph-mon[93519]: pgmap v477: 105 pgs: 28 active+undersized, 16 active+undersized+degraded, 61 active+clean; 4.9 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 71 KiB/s wr, 344 op/s; 103/891 objects degraded (11.560%) 2024-08-17T17:31:43.743 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:43 smithi179 ceph-mon[93519]: osdmap e565: 8 total, 7 up, 8 in 2024-08-17T17:31:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:43 smithi130 ceph-mon[105521]: pgmap v477: 105 pgs: 28 active+undersized, 16 active+undersized+degraded, 61 active+clean; 4.9 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 71 KiB/s wr, 344 op/s; 103/891 objects degraded (11.560%) 2024-08-17T17:31:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:43 smithi130 ceph-mon[105521]: osdmap e565: 8 total, 7 up, 8 in 2024-08-17T17:31:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:43 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3616674418' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:43 smithi130 ceph-mon[110068]: pgmap v477: 105 pgs: 28 active+undersized, 16 active+undersized+degraded, 61 active+clean; 4.9 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 71 KiB/s wr, 344 op/s; 103/891 objects degraded (11.560%) 2024-08-17T17:31:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:43 smithi130 ceph-mon[110068]: osdmap e565: 8 total, 7 up, 8 in 2024-08-17T17:31:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:43 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3616674418' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:44.034 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:43 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3616674418' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:44.740 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifyHeaderUpdate 2024-08-17T17:31:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:44 smithi179 ceph-mon[93519]: from='client.44699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:44 smithi179 ceph-mon[93519]: from='client.34827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:44 smithi179 ceph-mon[93519]: osdmap e566: 8 total, 7 up, 8 in 2024-08-17T17:31:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:44 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3230294827' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:44 smithi179 ceph-mon[93519]: osdmap e567: 8 total, 7 up, 8 in 2024-08-17T17:31:45.132 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:44 smithi130 ceph-mon[105521]: from='client.44699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:45.132 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:44 smithi130 ceph-mon[105521]: from='client.34827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:45.133 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:44 smithi130 ceph-mon[105521]: osdmap e566: 8 total, 7 up, 8 in 2024-08-17T17:31:45.133 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:44 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3230294827' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:45.133 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:45.133 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:45.133 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:44 smithi130 ceph-mon[105521]: osdmap e567: 8 total, 7 up, 8 in 2024-08-17T17:31:45.133 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:44 smithi130 ceph-mon[110068]: from='client.44699 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:45.133 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:44 smithi130 ceph-mon[110068]: from='client.34827 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:45.133 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:44 smithi130 ceph-mon[110068]: osdmap e566: 8 total, 7 up, 8 in 2024-08-17T17:31:45.133 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:44 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3230294827' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:45.133 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:45.133 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:45.133 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:44 smithi130 ceph-mon[110068]: osdmap e567: 8 total, 7 up, 8 in 2024-08-17T17:31:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:45 smithi179 ceph-mon[93519]: pgmap v480: 105 pgs: 32 unknown, 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:45 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:45 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:31:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:45 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:45 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:45 smithi179 ceph-mon[93519]: osdmap e568: 8 total, 7 up, 8 in 2024-08-17T17:31:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:45 smithi130 ceph-mon[105521]: pgmap v480: 105 pgs: 32 unknown, 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:45 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:45 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:31:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:45 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:45 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:45 smithi130 ceph-mon[105521]: osdmap e568: 8 total, 7 up, 8 in 2024-08-17T17:31:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:45 smithi130 ceph-mon[110068]: pgmap v480: 105 pgs: 32 unknown, 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:45 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded (PG_DEGRADED) 2024-08-17T17:31:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:45 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:31:46.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:45 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:46.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:45 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:46.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:45 smithi130 ceph-mon[110068]: osdmap e568: 8 total, 7 up, 8 in 2024-08-17T17:31:46.273 INFO:teuthology.orchestra.run.smithi130.stdout:true 2024-08-17T17:31:46.785 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifyHeaderUpdate (2046 ms) 2024-08-17T17:31:46.785 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifyFlatten 2024-08-17T17:31:46.807 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifyFlatten (22 ms) 2024-08-17T17:31:46.807 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifyResize 2024-08-17T17:31:46.823 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:46 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3528432509' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:46.823 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:46 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:46.823 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:46 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:46.824 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:46 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:46.824 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:46 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:46.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:46 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3528432509' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:46.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:46 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:46.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:46 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:46.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:46 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:46.824 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:46 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:46.826 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifyResize (19 ms) 2024-08-17T17:31:46.826 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifyRebuildObjectMap 2024-08-17T17:31:46.849 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifyRebuildObjectMap (23 ms) 2024-08-17T17:31:46.849 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifyUpdateFeatures 2024-08-17T17:31:46.871 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifyUpdateFeatures (22 ms) 2024-08-17T17:31:46.871 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifySnapCreate 2024-08-17T17:31:46.889 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifySnapCreate (18 ms) 2024-08-17T17:31:46.889 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifySnapCreateError 2024-08-17T17:31:46.908 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifySnapCreateError (19 ms) 2024-08-17T17:31:46.908 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifySnapRename 2024-08-17T17:31:46.928 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifySnapRename (20 ms) 2024-08-17T17:31:46.928 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifySnapRenameError 2024-08-17T17:31:46.950 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifySnapRenameError (22 ms) 2024-08-17T17:31:46.950 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifySnapRemove 2024-08-17T17:31:46.970 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifySnapRemove (20 ms) 2024-08-17T17:31:46.970 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifySnapProtect 2024-08-17T17:31:46.990 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifySnapProtect (20 ms) 2024-08-17T17:31:46.990 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifySnapUnprotect 2024-08-17T17:31:47.011 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifySnapUnprotect (21 ms) 2024-08-17T17:31:47.011 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifyRename 2024-08-17T17:31:47.033 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifyRename (22 ms) 2024-08-17T17:31:47.034 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifyAsyncTimedOut 2024-08-17T17:31:47.052 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifyAsyncTimedOut (19 ms) 2024-08-17T17:31:47.052 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifyAsyncError 2024-08-17T17:31:47.071 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifyAsyncError (20 ms) 2024-08-17T17:31:47.072 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifyAsyncCompleteError 2024-08-17T17:31:47.090 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifyAsyncCompleteError (18 ms) 2024-08-17T17:31:47.090 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestImageWatcher.NotifyAsyncRequestTimedOut 2024-08-17T17:31:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:46 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3528432509' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:46 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:46 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:46 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:46 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:47.111 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestImageWatcher.NotifyAsyncRequestTimedOut (21 ms) 2024-08-17T17:31:47.417 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[136110]: 2024-08-17T17:31:47.076+0000 7f1c83621640 -1 osd.3 558 log_to_monitors true 2024-08-17T17:31:47.773 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 17 tests from TestImageWatcher (2372 ms total) 2024-08-17T17:31:47.773 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:31:47.773 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 161 tests from TestLibRBD 2024-08-17T17:31:47.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-mon[105521]: pgmap v483: 105 pgs: 27 unknown, 29 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:47.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-mon[105521]: from='client.44717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:47.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-mon[105521]: from='client.44723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:47.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-mon[105521]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:31:47.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-mon[105521]: osdmap e569: 8 total, 7 up, 8 in 2024-08-17T17:31:47.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-mon[105521]: from='osd.3 [v2:172.21.15.130:6826/3086744673,v1:172.21.15.130:6827/3086744673]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:31:47.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-mon[105521]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:31:47.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-mon[110068]: pgmap v483: 105 pgs: 27 unknown, 29 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:47.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-mon[110068]: from='client.44717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:47.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-mon[110068]: from='client.44723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:47.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-mon[110068]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:31:47.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-mon[110068]: osdmap e569: 8 total, 7 up, 8 in 2024-08-17T17:31:47.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-mon[110068]: from='osd.3 [v2:172.21.15.130:6826/3086744673,v1:172.21.15.130:6827/3086744673]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:31:47.792 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:47 smithi130 ceph-mon[110068]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:31:47.816 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.CreateAndStat 2024-08-17T17:31:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:47 smithi179 ceph-mon[93519]: pgmap v483: 105 pgs: 27 unknown, 29 active+undersized, 3 active+undersized+degraded, 46 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:47 smithi179 ceph-mon[93519]: from='client.44717 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:47 smithi179 ceph-mon[93519]: from='client.44723 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:47 smithi179 ceph-mon[93519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:31:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:47 smithi179 ceph-mon[93519]: osdmap e569: 8 total, 7 up, 8 in 2024-08-17T17:31:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:47 smithi179 ceph-mon[93519]: from='osd.3 [v2:172.21.15.130:6826/3086744673,v1:172.21.15.130:6827/3086744673]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:31:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:47 smithi179 ceph-mon[93519]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:31:48.417 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 17:31:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[136110]: 2024-08-17T17:31:48.205+0000 7f1c773c3640 -1 osd.3 558 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:31:48.868 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 4 2024-08-17T17:31:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:48 smithi179 ceph-mon[93519]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:31:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:48 smithi179 ceph-mon[93519]: osdmap e570: 8 total, 7 up, 8 in 2024-08-17T17:31:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:48 smithi179 ceph-mon[93519]: from='osd.3 [v2:172.21.15.130:6826/3086744673,v1:172.21.15.130:6827/3086744673]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:31:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:48 smithi179 ceph-mon[93519]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:31:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:48 smithi179 ceph-mon[93519]: from='osd.3 ' entity='osd.3' 2024-08-17T17:31:49.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:48 smithi130 ceph-mon[110068]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:31:49.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:48 smithi130 ceph-mon[110068]: osdmap e570: 8 total, 7 up, 8 in 2024-08-17T17:31:49.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:48 smithi130 ceph-mon[110068]: from='osd.3 [v2:172.21.15.130:6826/3086744673,v1:172.21.15.130:6827/3086744673]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:31:49.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:48 smithi130 ceph-mon[110068]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:31:49.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:48 smithi130 ceph-mon[110068]: from='osd.3 ' entity='osd.3' 2024-08-17T17:31:49.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:48 smithi130 ceph-mon[105521]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:31:49.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:48 smithi130 ceph-mon[105521]: osdmap e570: 8 total, 7 up, 8 in 2024-08-17T17:31:49.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:48 smithi130 ceph-mon[105521]: from='osd.3 [v2:172.21.15.130:6826/3086744673,v1:172.21.15.130:6827/3086744673]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:31:49.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:48 smithi130 ceph-mon[105521]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:31:49.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:48 smithi130 ceph-mon[105521]: from='osd.3 ' entity='osd.3' 2024-08-17T17:31:49.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:31:49.571+0000 7f8e6b27b640 -1 mgr.server reply reply (11) Resource temporarily unavailable 32 pgs have unknown state; cannot draw any conclusions 2024-08-17T17:31:49.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: 2024-08-17T17:31:49.571+0000 7f8e6b27b640 -1 mgr.server reply reply (16) Device or resource busy 32 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-08-17T17:31:49.796 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:49.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade pause' 2024-08-17T17:31:50.077 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: pgmap v486: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:31:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: osd.3 [v2:172.21.15.130:6826/3086744673,v1:172.21.15.130:6827/3086744673] boot 2024-08-17T17:31:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: osdmap e571: 8 total, 8 up, 8 in 2024-08-17T17:31:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:31:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/88615865' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4237031226' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:50.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:50.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:50.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:50.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:31:50.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:50.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:50.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:50.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:50.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:50.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:49 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-08-17T17:31:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:31:49] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.51.0" 2024-08-17T17:31:50.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: pgmap v486: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:50.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:31:50.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: osd.3 [v2:172.21.15.130:6826/3086744673,v1:172.21.15.130:6827/3086744673] boot 2024-08-17T17:31:50.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: osdmap e571: 8 total, 8 up, 8 in 2024-08-17T17:31:50.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:31:50.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/88615865' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:50.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:50.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4237031226' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:50.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:50.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:50.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:50.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:31:50.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:50.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:50.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:50.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:50.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:50.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-08-17T17:31:50.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: pgmap v486: 73 pgs: 28 active+undersized, 3 active+undersized+degraded, 42 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:50.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:31:50.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: osd.3 [v2:172.21.15.130:6826/3086744673,v1:172.21.15.130:6827/3086744673] boot 2024-08-17T17:31:50.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: osdmap e571: 8 total, 8 up, 8 in 2024-08-17T17:31:50.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:31:50.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/88615865' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:50.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:50.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4237031226' entity='client.admin' cmd={"prefix": "osd versions"} : dispatch 2024-08-17T17:31:50.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:50.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:50.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:31:50.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:31:50.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:50.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:50.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:50.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:50.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:31:50.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:49 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-08-17T17:31:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:31:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:31:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:31:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:50 smithi179 ceph-mon[93519]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-08-17T17:31:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:50 smithi179 ceph-mon[93519]: Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-08-17T17:31:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:50 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:50 smithi179 ceph-mon[93519]: osdmap e572: 8 total, 8 up, 8 in 2024-08-17T17:31:51.131 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:50 smithi130 ceph-mon[110068]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-08-17T17:31:51.131 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:50 smithi130 ceph-mon[110068]: Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-08-17T17:31:51.131 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:50 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:51.131 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:50 smithi130 ceph-mon[110068]: osdmap e572: 8 total, 8 up, 8 in 2024-08-17T17:31:51.131 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:50 smithi130 ceph-mon[105521]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-08-17T17:31:51.132 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:50 smithi130 ceph-mon[105521]: Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-08-17T17:31:51.132 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:50 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:51.132 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:50 smithi130 ceph-mon[105521]: osdmap e572: 8 total, 8 up, 8 in 2024-08-17T17:31:51.812 INFO:teuthology.orchestra.run.smithi130.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:31:51.831 INFO:tasks.workunit.client.0.smithi130.stdout:image has size 2097152 and order 22 2024-08-17T17:31:51.831 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.CreateAndStat (4015 ms) 2024-08-17T17:31:51.831 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.CreateWithSameDataPool 2024-08-17T17:31:51.831 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:51.851 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.CreateWithSameDataPool (20 ms) 2024-08-17T17:31:51.851 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.CreateAndStatPP 2024-08-17T17:31:51.851 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:51.867 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.CreateAndStatPP (16 ms) 2024-08-17T17:31:51.867 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.GetId 2024-08-17T17:31:51.867 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:51.890 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.GetId (23 ms) 2024-08-17T17:31:51.890 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.GetIdPP 2024-08-17T17:31:51.890 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:51.912 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.GetIdPP (22 ms) 2024-08-17T17:31:51.913 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefix 2024-08-17T17:31:51.913 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:51.928 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.GetBlockNamePrefix (16 ms) 2024-08-17T17:31:51.928 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefixPP 2024-08-17T17:31:51.928 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:51.944 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.GetBlockNamePrefixPP (17 ms) 2024-08-17T17:31:51.945 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestGetCreateTimestamp 2024-08-17T17:31:51.945 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:51.962 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestGetCreateTimestamp (17 ms) 2024-08-17T17:31:51.962 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.GetCreateTimestampPP 2024-08-17T17:31:51.962 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:51.978 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.GetCreateTimestampPP (17 ms) 2024-08-17T17:31:51.979 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.OpenAio 2024-08-17T17:31:51.979 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:51.993 INFO:tasks.workunit.client.0.smithi130.stdout:image has size 2097152 and order 22 2024-08-17T17:31:51.993 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.OpenAio (14 ms) 2024-08-17T17:31:51.993 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.OpenAioFail 2024-08-17T17:31:51.995 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.OpenAioFail (3 ms) 2024-08-17T17:31:51.996 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.OpenAioPP 2024-08-17T17:31:51.996 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:52.020 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.OpenAioPP (24 ms) 2024-08-17T17:31:52.020 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.OpenAioFailPP 2024-08-17T17:31:52.022 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.OpenAioFailPP (2 ms) 2024-08-17T17:31:52.023 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ResizeAndStat 2024-08-17T17:31:52.023 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:52.055 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ResizeAndStat (33 ms) 2024-08-17T17:31:52.055 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ResizeAndStatPP 2024-08-17T17:31:52.055 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:52.082 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ResizeAndStatPP (27 ms) 2024-08-17T17:31:52.082 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResize 2024-08-17T17:31:52.082 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:51 smithi179 ceph-mon[93519]: pgmap v489: 105 pgs: 32 unknown, 23 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:51 smithi179 ceph-mon[93519]: osdmap e573: 8 total, 8 up, 8 in 2024-08-17T17:31:52.117 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.UpdateWatchAndResize (35 ms) 2024-08-17T17:31:52.117 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResizePP 2024-08-17T17:31:52.117 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:52.149 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.UpdateWatchAndResizePP (32 ms) 2024-08-17T17:31:52.149 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCreateLsDelete 2024-08-17T17:31:52.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:51 smithi130 ceph-mon[110068]: pgmap v489: 105 pgs: 32 unknown, 23 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:52.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:51 smithi130 ceph-mon[110068]: osdmap e573: 8 total, 8 up, 8 in 2024-08-17T17:31:52.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:51 smithi130 ceph-mon[105521]: pgmap v489: 105 pgs: 32 unknown, 23 active+undersized, 3 active+undersized+degraded, 47 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:52.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:51 smithi130 ceph-mon[105521]: osdmap e573: 8 total, 8 up, 8 in 2024-08-17T17:31:52.505 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch ps' 2024-08-17T17:31:52.756 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:52 smithi179 ceph-mon[93519]: osdmap e574: 8 total, 8 up, 8 in 2024-08-17T17:31:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:52 smithi179 ceph-mon[93519]: from='client.44756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:52 smithi179 ceph-mon[93519]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:31:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:52 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:53.109 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:52 smithi130 ceph-mon[105521]: osdmap e574: 8 total, 8 up, 8 in 2024-08-17T17:31:53.109 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:52 smithi130 ceph-mon[105521]: from='client.44756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:53.109 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:52 smithi130 ceph-mon[105521]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:31:53.110 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:52 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:53.110 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:52 smithi130 ceph-mon[110068]: osdmap e574: 8 total, 8 up, 8 in 2024-08-17T17:31:53.110 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:52 smithi130 ceph-mon[110068]: from='client.44756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:53.110 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:52 smithi130 ceph-mon[110068]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:31:53.110 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:52 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:53.850 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:54.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:53 smithi130 ceph-mon[105521]: pgmap v492: 105 pgs: 26 unknown, 20 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:54.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:53 smithi130 ceph-mon[105521]: osdmap e575: 8 total, 8 up, 8 in 2024-08-17T17:31:54.085 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:53 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3748511623' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:54.086 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:53 smithi130 ceph-mon[110068]: pgmap v492: 105 pgs: 26 unknown, 20 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:54.086 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:53 smithi130 ceph-mon[110068]: osdmap e575: 8 total, 8 up, 8 in 2024-08-17T17:31:54.086 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:53 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3748511623' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:53 smithi179 ceph-mon[93519]: pgmap v492: 105 pgs: 26 unknown, 20 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-08-17T17:31:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:53 smithi179 ceph-mon[93519]: osdmap e575: 8 total, 8 up, 8 in 2024-08-17T17:31:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:53 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3748511623' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:54.503 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:31:54.503 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (7m) 9s ago 11m 19.5M - 0.27.0 11f11916f8cd 73ea4ebcf68b 2024-08-17T17:31:54.503 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi179 *:3000 running (6m) 8s ago 10m 84.4M - 9.4.7 954c08fa6188 399f89cc0037 2024-08-17T17:31:54.503 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foo.smithi130.jmaidg smithi130 running (9m) 9s ago 9m 21.7M - 17.2.7-1404-gf2c39d4d f69471c56189 91d7d1762094 2024-08-17T17:31:54.503 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foo.smithi179.zkkvno smithi179 running (9m) 8s ago 9m 14.9M - 17.2.7-1404-gf2c39d4d f69471c56189 0486517d425a 2024-08-17T17:31:54.504 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi179 *:8443,9283,8765 running (6m) 8s ago 14m 455M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb cdd6dde7c7af 2024-08-17T17:31:54.504 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (6m) 9s ago 16m 520M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 584426b3875b 2024-08-17T17:31:54.504 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (5m) 9s ago 16m 65.6M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 79749aebcefa 2024-08-17T17:31:54.504 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi179 running (3m) 8s ago 15m 47.5M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb d85c3c5c065e 2024-08-17T17:31:54.504 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (4m) 9s ago 15m 55.8M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 4161e48dac50 2024-08-17T17:31:54.504 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (7m) 9s ago 10m 10.5M - 1.7.0 72c9c2088986 fd7392d87597 2024-08-17T17:31:54.504 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi179 *:9100 running (6m) 8s ago 10m 11.2M - 1.7.0 72c9c2088986 c10588f17ed8 2024-08-17T17:31:54.504 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (2m) 9s ago 14m 73.0M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 26704f9d719b 2024-08-17T17:31:54.504 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (79s) 9s ago 14m 62.0M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 54f6b453c741 2024-08-17T17:31:54.504 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (46s) 9s ago 13m 48.8M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 05996ffbb485 2024-08-17T17:31:54.504 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (13s) 9s ago 13m 12.2M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb a4c158c50e35 2024-08-17T17:31:54.504 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi179 running (13m) 8s ago 13m 118M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 254dc0763420 2024-08-17T17:31:54.504 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi179 running (12m) 8s ago 12m 102M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 078704aadcd7 2024-08-17T17:31:54.505 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi179 running (12m) 8s ago 12m 135M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 ab570ba67ae7 2024-08-17T17:31:54.505 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi179 running (11m) 8s ago 11m 116M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 99ae6a861fa0 2024-08-17T17:31:54.505 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi179 *:9095 running (6m) 8s ago 11m 53.8M - 2.51.0 1d3b7f56885b 145f77b6fc93 2024-08-17T17:31:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3748511623' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:54 smithi179 ceph-mon[93519]: osdmap e576: 8 total, 8 up, 8 in 2024-08-17T17:31:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:54 smithi179 ceph-mon[93519]: osdmap e577: 8 total, 8 up, 8 in 2024-08-17T17:31:55.112 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3748511623' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:55.112 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:54 smithi130 ceph-mon[105521]: osdmap e576: 8 total, 8 up, 8 in 2024-08-17T17:31:55.112 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:54 smithi130 ceph-mon[105521]: osdmap e577: 8 total, 8 up, 8 in 2024-08-17T17:31:55.113 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3748511623' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:55.113 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:54 smithi130 ceph-mon[110068]: osdmap e576: 8 total, 8 up, 8 in 2024-08-17T17:31:55.113 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:54 smithi130 ceph-mon[110068]: osdmap e577: 8 total, 8 up, 8 in 2024-08-17T17:31:55.138 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch ps' 2024-08-17T17:31:55.394 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:55.874 INFO:tasks.workunit.client.0.smithi130.stdout:image: image18 2024-08-17T17:31:55.874 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image18 2024-08-17T17:31:55.874 INFO:tasks.workunit.client.0.smithi130.stdout:found image18 2024-08-17T17:31:55.874 INFO:tasks.workunit.client.0.smithi130.stdout:erased image18 2024-08-17T17:31:55.874 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:55.944 INFO:tasks.workunit.client.0.smithi130.stdout:image: image18 2024-08-17T17:31:55.944 INFO:tasks.workunit.client.0.smithi130.stdout:image: image19 2024-08-17T17:31:55.944 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image18 2024-08-17T17:31:55.944 INFO:tasks.workunit.client.0.smithi130.stdout:found image18 2024-08-17T17:31:55.944 INFO:tasks.workunit.client.0.smithi130.stdout:erased image18 2024-08-17T17:31:55.944 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image19 2024-08-17T17:31:55.944 INFO:tasks.workunit.client.0.smithi130.stdout:found image19 2024-08-17T17:31:55.944 INFO:tasks.workunit.client.0.smithi130.stdout:erased image19 2024-08-17T17:31:55.945 INFO:tasks.workunit.client.0.smithi130.stdout:image: image19 2024-08-17T17:31:55.945 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image19 2024-08-17T17:31:55.945 INFO:tasks.workunit.client.0.smithi130.stdout:found image19 2024-08-17T17:31:55.945 INFO:tasks.workunit.client.0.smithi130.stdout:erased image19 2024-08-17T17:31:55.945 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCreateLsDelete (3795 ms) 2024-08-17T17:31:55.945 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCreateLsDeletePP 2024-08-17T17:31:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:55 smithi179 ceph-mon[93519]: pgmap v495: 137 pgs: 25 creating+peering, 7 unknown, 105 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 17 KiB/s wr, 211 op/s 2024-08-17T17:31:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:55 smithi179 ceph-mon[93519]: from='client.44768 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:55 smithi179 ceph-mon[93519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded) 2024-08-17T17:31:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:55 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T17:31:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:55 smithi179 ceph-mon[93519]: osdmap e578: 8 total, 8 up, 8 in 2024-08-17T17:31:56.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:55 smithi130 ceph-mon[105521]: pgmap v495: 137 pgs: 25 creating+peering, 7 unknown, 105 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 17 KiB/s wr, 211 op/s 2024-08-17T17:31:56.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:55 smithi130 ceph-mon[105521]: from='client.44768 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:56.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:55 smithi130 ceph-mon[105521]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded) 2024-08-17T17:31:56.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:55 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T17:31:56.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:55 smithi130 ceph-mon[105521]: osdmap e578: 8 total, 8 up, 8 in 2024-08-17T17:31:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:55 smithi130 ceph-mon[110068]: pgmap v495: 137 pgs: 25 creating+peering, 7 unknown, 105 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 17 KiB/s wr, 211 op/s 2024-08-17T17:31:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:55 smithi130 ceph-mon[110068]: from='client.44768 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:56.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:55 smithi130 ceph-mon[110068]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/78 objects degraded (5.128%), 3 pgs degraded) 2024-08-17T17:31:56.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:55 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T17:31:56.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:55 smithi130 ceph-mon[110068]: osdmap e578: 8 total, 8 up, 8 in 2024-08-17T17:31:57.293 INFO:teuthology.orchestra.run.smithi130.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:31:57.293 INFO:teuthology.orchestra.run.smithi130.stdout:alertmanager.a smithi130 *:9093,9094 running (7m) 11s ago 11m 19.5M - 0.27.0 11f11916f8cd 73ea4ebcf68b 2024-08-17T17:31:57.293 INFO:teuthology.orchestra.run.smithi130.stdout:grafana.a smithi179 *:3000 running (6m) 11s ago 10m 84.4M - 9.4.7 954c08fa6188 399f89cc0037 2024-08-17T17:31:57.293 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foo.smithi130.jmaidg smithi130 running (9m) 11s ago 9m 21.7M - 17.2.7-1404-gf2c39d4d f69471c56189 91d7d1762094 2024-08-17T17:31:57.294 INFO:teuthology.orchestra.run.smithi130.stdout:mds.foo.smithi179.zkkvno smithi179 running (9m) 11s ago 9m 14.9M - 17.2.7-1404-gf2c39d4d f69471c56189 0486517d425a 2024-08-17T17:31:57.294 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.x smithi179 *:8443,9283,8765 running (6m) 11s ago 15m 455M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb cdd6dde7c7af 2024-08-17T17:31:57.294 INFO:teuthology.orchestra.run.smithi130.stdout:mgr.y smithi130 *:8443,9283,8765 running (6m) 11s ago 16m 520M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 584426b3875b 2024-08-17T17:31:57.294 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a smithi130 running (5m) 11s ago 16m 65.6M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 79749aebcefa 2024-08-17T17:31:57.294 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b smithi179 running (4m) 11s ago 15m 47.5M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb d85c3c5c065e 2024-08-17T17:31:57.294 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c smithi130 running (4m) 11s ago 15m 55.8M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 4161e48dac50 2024-08-17T17:31:57.295 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.a smithi130 *:9100 running (7m) 11s ago 10m 10.5M - 1.7.0 72c9c2088986 fd7392d87597 2024-08-17T17:31:57.295 INFO:teuthology.orchestra.run.smithi130.stdout:node-exporter.b smithi179 *:9100 running (6m) 11s ago 10m 11.2M - 1.7.0 72c9c2088986 c10588f17ed8 2024-08-17T17:31:57.295 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 smithi130 running (2m) 11s ago 14m 73.0M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 26704f9d719b 2024-08-17T17:31:57.295 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 smithi130 running (82s) 11s ago 14m 62.0M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 54f6b453c741 2024-08-17T17:31:57.295 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 smithi130 running (49s) 11s ago 13m 48.8M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 05996ffbb485 2024-08-17T17:31:57.295 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 smithi130 running (16s) 11s ago 13m 12.2M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb a4c158c50e35 2024-08-17T17:31:57.295 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 smithi179 running (13m) 11s ago 13m 118M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 254dc0763420 2024-08-17T17:31:57.295 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 smithi179 running (12m) 11s ago 12m 102M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 078704aadcd7 2024-08-17T17:31:57.295 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 smithi179 running (12m) 11s ago 12m 135M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 ab570ba67ae7 2024-08-17T17:31:57.296 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 smithi179 running (12m) 11s ago 12m 116M 2480M 17.2.7-1404-gf2c39d4d f69471c56189 99ae6a861fa0 2024-08-17T17:31:57.296 INFO:teuthology.orchestra.run.smithi130.stdout:prometheus.a smithi179 *:9095 running (6m) 11s ago 11m 53.8M - 2.51.0 1d3b7f56885b 145f77b6fc93 2024-08-17T17:31:57.853 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:57 smithi130 ceph-mon[110068]: pgmap v498: 137 pgs: 25 creating+peering, 4 unknown, 108 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 17 KiB/s wr, 211 op/s 2024-08-17T17:31:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:57 smithi130 ceph-mon[110068]: osdmap e579: 8 total, 8 up, 8 in 2024-08-17T17:31:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:57 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3028214757' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:57 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:57 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:57 smithi130 ceph-mon[105521]: pgmap v498: 137 pgs: 25 creating+peering, 4 unknown, 108 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 17 KiB/s wr, 211 op/s 2024-08-17T17:31:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:57 smithi130 ceph-mon[105521]: osdmap e579: 8 total, 8 up, 8 in 2024-08-17T17:31:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:57 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3028214757' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:57 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:57 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:58.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:58.291 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph versions' 2024-08-17T17:31:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:57 smithi179 ceph-mon[93519]: pgmap v498: 137 pgs: 25 creating+peering, 4 unknown, 108 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 17 KiB/s wr, 211 op/s 2024-08-17T17:31:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:57 smithi179 ceph-mon[93519]: osdmap e579: 8 total, 8 up, 8 in 2024-08-17T17:31:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:57 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3028214757' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:57 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:31:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:57 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:31:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:58.552 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:59.006 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:58 smithi130 ceph-mon[105521]: from='client.34902 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:59.007 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:58 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:59.007 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:58 smithi130 ceph-mon[105521]: osdmap e580: 8 total, 8 up, 8 in 2024-08-17T17:31:59.007 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:58 smithi130 ceph-mon[105521]: pgmap v501: 169 pgs: 9 unknown, 4 creating+peering, 156 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 4.7 KiB/s wr, 54 op/s 2024-08-17T17:31:59.007 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:58 smithi130 ceph-mon[110068]: from='client.34902 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:59.007 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:58 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:59.007 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:58 smithi130 ceph-mon[110068]: osdmap e580: 8 total, 8 up, 8 in 2024-08-17T17:31:59.007 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:58 smithi130 ceph-mon[110068]: pgmap v501: 169 pgs: 9 unknown, 4 creating+peering, 156 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 4.7 KiB/s wr, 54 op/s 2024-08-17T17:31:59.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:58 smithi179 ceph-mon[93519]: from='client.34902 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:59.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:58 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:31:59.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:58 smithi179 ceph-mon[93519]: osdmap e580: 8 total, 8 up, 8 in 2024-08-17T17:31:59.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:58 smithi179 ceph-mon[93519]: pgmap v501: 169 pgs: 9 unknown, 4 creating+peering, 156 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 4.7 KiB/s wr, 54 op/s 2024-08-17T17:31:59.890 INFO:tasks.workunit.client.0.smithi130.stdout:num images is: 1 2024-08-17T17:31:59.890 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:31:59.890 INFO:tasks.workunit.client.0.smithi130.stdout:image: image20 2024-08-17T17:31:59.890 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image20 2024-08-17T17:31:59.890 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:31:59.898 INFO:tasks.workunit.client.0.smithi130.stdout:num images is: 2 2024-08-17T17:31:59.898 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 2 2024-08-17T17:31:59.898 INFO:tasks.workunit.client.0.smithi130.stdout:image: image20 2024-08-17T17:31:59.899 INFO:tasks.workunit.client.0.smithi130.stdout:image: image21 2024-08-17T17:31:59.899 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image20 2024-08-17T17:31:59.899 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image21 2024-08-17T17:31:59.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:31:59 smithi130 ceph-mon[105521]: osdmap e581: 8 total, 8 up, 8 in 2024-08-17T17:31:59.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:31:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:31:59] "GET /metrics HTTP/1.1" 200 41174 "" "Prometheus/2.51.0" 2024-08-17T17:31:59.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:31:59 smithi130 ceph-mon[110068]: osdmap e581: 8 total, 8 up, 8 in 2024-08-17T17:31:59.952 INFO:tasks.workunit.client.0.smithi130.stdout:num images is: 1 2024-08-17T17:31:59.953 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:31:59.953 INFO:tasks.workunit.client.0.smithi130.stdout:image: image21 2024-08-17T17:31:59.953 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image21 2024-08-17T17:31:59.953 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCreateLsDeletePP (4009 ms) 2024-08-17T17:31:59.953 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCopy 2024-08-17T17:32:00.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:31:59 smithi179 ceph-mon[93519]: osdmap e581: 8 total, 8 up, 8 in 2024-08-17T17:32:00.404 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:32:00.404 INFO:teuthology.orchestra.run.smithi130.stdout: "mon": { 2024-08-17T17:32:00.404 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:32:00.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:32:00.404 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr": { 2024-08-17T17:32:00.404 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:32:00.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:32:00.404 INFO:teuthology.orchestra.run.smithi130.stdout: "osd": { 2024-08-17T17:32:00.405 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)": 4, 2024-08-17T17:32:00.405 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 4 2024-08-17T17:32:00.405 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:32:00.405 INFO:teuthology.orchestra.run.smithi130.stdout: "mds": { 2024-08-17T17:32:00.405 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)": 2 2024-08-17T17:32:00.405 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:32:00.405 INFO:teuthology.orchestra.run.smithi130.stdout: "overall": { 2024-08-17T17:32:00.405 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 17.2.7-1404-gf2c39d4d (f2c39d4d5abc084f975be26cc0141421e55727d8) quincy (stable)": 6, 2024-08-17T17:32:00.405 INFO:teuthology.orchestra.run.smithi130.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 9 2024-08-17T17:32:00.405 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:32:00.405 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:32:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:32:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:32:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:32:00.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:00 smithi130 ceph-mon[105521]: osdmap e582: 8 total, 8 up, 8 in 2024-08-17T17:32:00.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:00 smithi130 ceph-mon[105521]: pgmap v504: 169 pgs: 3 unknown, 166 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 5.7 KiB/s wr, 80 op/s 2024-08-17T17:32:00.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:00 smithi130 ceph-mon[105521]: osdmap e583: 8 total, 8 up, 8 in 2024-08-17T17:32:00.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:00 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3355568356' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:00.968 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:00 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:00.969 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:00 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/603520481' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:32:00.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:00 smithi130 ceph-mon[110068]: osdmap e582: 8 total, 8 up, 8 in 2024-08-17T17:32:00.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:00 smithi130 ceph-mon[110068]: pgmap v504: 169 pgs: 3 unknown, 166 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 5.7 KiB/s wr, 80 op/s 2024-08-17T17:32:00.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:00 smithi130 ceph-mon[110068]: osdmap e583: 8 total, 8 up, 8 in 2024-08-17T17:32:00.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:00 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3355568356' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:00.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:00 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:00.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:00 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/603520481' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:32:00.993 DEBUG:teuthology.parallel:result is None 2024-08-17T17:32:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:00 smithi179 ceph-mon[93519]: osdmap e582: 8 total, 8 up, 8 in 2024-08-17T17:32:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:00 smithi179 ceph-mon[93519]: pgmap v504: 169 pgs: 3 unknown, 166 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 5.7 KiB/s wr, 80 op/s 2024-08-17T17:32:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:00 smithi179 ceph-mon[93519]: osdmap e583: 8 total, 8 up, 8 in 2024-08-17T17:32:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:00 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3355568356' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:00 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:01.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:00 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/603520481' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:32:01.398 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:32:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:02 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:02 smithi130 ceph-mon[105521]: osdmap e584: 8 total, 8 up, 8 in 2024-08-17T17:32:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:02 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:02 smithi130 ceph-mon[110068]: osdmap e584: 8 total, 8 up, 8 in 2024-08-17T17:32:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:02 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:02 smithi179 ceph-mon[93519]: osdmap e584: 8 total, 8 up, 8 in 2024-08-17T17:32:03.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:03 smithi130 ceph-mon[105521]: pgmap v507: 201 pgs: 24 unknown, 177 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 1.7 KiB/s wr, 32 op/s 2024-08-17T17:32:03.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:03 smithi130 ceph-mon[105521]: osdmap e585: 8 total, 8 up, 8 in 2024-08-17T17:32:03.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:03 smithi130 ceph-mon[110068]: pgmap v507: 201 pgs: 24 unknown, 177 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 1.7 KiB/s wr, 32 op/s 2024-08-17T17:32:03.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:03 smithi130 ceph-mon[110068]: osdmap e585: 8 total, 8 up, 8 in 2024-08-17T17:32:03.762 INFO:tasks.workunit.client.0.smithi130.stdout:image: image22 2024-08-17T17:32:03.762 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image22 2024-08-17T17:32:03.762 INFO:tasks.workunit.client.0.smithi130.stdout:found image22 2024-08-17T17:32:03.762 INFO:tasks.workunit.client.0.smithi130.stdout:erased image22 2024-08-17T17:32:03.763 INFO:tasks.workunit.client.0.smithi130.stdout:image: image22 2024-08-17T17:32:03.763 INFO:tasks.workunit.client.0.smithi130.stdout:image: image23 2024-08-17T17:32:03.763 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image22 2024-08-17T17:32:03.763 INFO:tasks.workunit.client.0.smithi130.stdout:found image22 2024-08-17T17:32:03.763 INFO:tasks.workunit.client.0.smithi130.stdout:erased image22 2024-08-17T17:32:03.763 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image23 2024-08-17T17:32:03.763 INFO:tasks.workunit.client.0.smithi130.stdout:found image23 2024-08-17T17:32:03.763 INFO:tasks.workunit.client.0.smithi130.stdout:erased image23 2024-08-17T17:32:03.763 INFO:tasks.workunit.client.0.smithi130.stdout:100.00% done 2024-08-17T17:32:03.763 INFO:tasks.workunit.client.0.smithi130.stdout:image: image22 2024-08-17T17:32:03.763 INFO:tasks.workunit.client.0.smithi130.stdout:image: image23 2024-08-17T17:32:03.763 INFO:tasks.workunit.client.0.smithi130.stdout:image: image24 2024-08-17T17:32:03.763 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image22 2024-08-17T17:32:03.764 INFO:tasks.workunit.client.0.smithi130.stdout:found image22 2024-08-17T17:32:03.764 INFO:tasks.workunit.client.0.smithi130.stdout:erased image22 2024-08-17T17:32:03.764 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image23 2024-08-17T17:32:03.764 INFO:tasks.workunit.client.0.smithi130.stdout:found image23 2024-08-17T17:32:03.764 INFO:tasks.workunit.client.0.smithi130.stdout:erased image23 2024-08-17T17:32:03.764 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image24 2024-08-17T17:32:03.764 INFO:tasks.workunit.client.0.smithi130.stdout:found image24 2024-08-17T17:32:03.764 INFO:tasks.workunit.client.0.smithi130.stdout:erased image24 2024-08-17T17:32:03.764 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCopy (3809 ms) 2024-08-17T17:32:03.764 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCopyPP 2024-08-17T17:32:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:03 smithi179 ceph-mon[93519]: pgmap v507: 201 pgs: 24 unknown, 177 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 1.7 KiB/s wr, 32 op/s 2024-08-17T17:32:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:03 smithi179 ceph-mon[93519]: osdmap e585: 8 total, 8 up, 8 in 2024-08-17T17:32:04.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:04 smithi130 ceph-mon[105521]: osdmap e586: 8 total, 8 up, 8 in 2024-08-17T17:32:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:04 smithi130 ceph-mon[110068]: osdmap e586: 8 total, 8 up, 8 in 2024-08-17T17:32:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:04 smithi179 ceph-mon[93519]: osdmap e586: 8 total, 8 up, 8 in 2024-08-17T17:32:05.398 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:32:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[110068]: pgmap v510: 201 pgs: 1 unknown, 200 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 3.2 KiB/s wr, 40 op/s 2024-08-17T17:32:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[110068]: osdmap e587: 8 total, 8 up, 8 in 2024-08-17T17:32:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/784547730' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:32:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:32:05.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:05.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:05.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:05.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:05.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[110068]: osdmap e588: 8 total, 8 up, 8 in 2024-08-17T17:32:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[105521]: pgmap v510: 201 pgs: 1 unknown, 200 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 3.2 KiB/s wr, 40 op/s 2024-08-17T17:32:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[105521]: osdmap e587: 8 total, 8 up, 8 in 2024-08-17T17:32:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/784547730' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:32:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:32:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:05.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:05 smithi130 ceph-mon[105521]: osdmap e588: 8 total, 8 up, 8 in 2024-08-17T17:32:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:05 smithi179 ceph-mon[93519]: pgmap v510: 201 pgs: 1 unknown, 200 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 3.2 KiB/s wr, 40 op/s 2024-08-17T17:32:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:05 smithi179 ceph-mon[93519]: osdmap e587: 8 total, 8 up, 8 in 2024-08-17T17:32:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:05 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/784547730' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:05 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:05 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:32:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:05 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:05 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:05.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:05 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:05.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:05 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:32:05.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:05 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:05.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:05 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:05.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:05 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:05.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:05 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:05.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:05 smithi179 ceph-mon[93519]: osdmap e588: 8 total, 8 up, 8 in 2024-08-17T17:32:07.648 INFO:tasks.workunit.client.0.smithi130.stdout:num images is: 1 2024-08-17T17:32:07.648 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:32:07.648 INFO:tasks.workunit.client.0.smithi130.stdout:image: image25 2024-08-17T17:32:07.648 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image25 2024-08-17T17:32:07.664 INFO:tasks.workunit.client.0.smithi130.stdout:num images is: 2 2024-08-17T17:32:07.664 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 2 2024-08-17T17:32:07.664 INFO:tasks.workunit.client.0.smithi130.stdout:image: image25 2024-08-17T17:32:07.665 INFO:tasks.workunit.client.0.smithi130.stdout:image: image26 2024-08-17T17:32:07.665 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image25 2024-08-17T17:32:07.665 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image26 2024-08-17T17:32:07.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:07 smithi130 ceph-mon[105521]: pgmap v513: 233 pgs: 7 creating+peering, 25 unknown, 201 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 35 op/s 2024-08-17T17:32:07.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:07 smithi130 ceph-mon[105521]: osdmap e589: 8 total, 8 up, 8 in 2024-08-17T17:32:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:07 smithi130 ceph-mon[110068]: pgmap v513: 233 pgs: 7 creating+peering, 25 unknown, 201 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 35 op/s 2024-08-17T17:32:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:07 smithi130 ceph-mon[110068]: osdmap e589: 8 total, 8 up, 8 in 2024-08-17T17:32:07.704 INFO:tasks.workunit.client.0.smithi130.stdout:100.00% done 2024-08-17T17:32:07.704 INFO:tasks.workunit.client.0.smithi130.stdout:num images is: 3 2024-08-17T17:32:07.704 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 3 2024-08-17T17:32:07.704 INFO:tasks.workunit.client.0.smithi130.stdout:image: image25 2024-08-17T17:32:07.704 INFO:tasks.workunit.client.0.smithi130.stdout:image: image26 2024-08-17T17:32:07.704 INFO:tasks.workunit.client.0.smithi130.stdout:image: image27 2024-08-17T17:32:07.704 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image25 2024-08-17T17:32:07.704 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image26 2024-08-17T17:32:07.704 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image27 2024-08-17T17:32:07.731 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCopyPP (3969 ms) 2024-08-17T17:32:07.731 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestDeepCopy 2024-08-17T17:32:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:07 smithi179 ceph-mon[93519]: pgmap v513: 233 pgs: 7 creating+peering, 25 unknown, 201 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.5 KiB/s wr, 35 op/s 2024-08-17T17:32:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:07 smithi179 ceph-mon[93519]: osdmap e589: 8 total, 8 up, 8 in 2024-08-17T17:32:08.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:08 smithi130 ceph-mon[105521]: osdmap e590: 8 total, 8 up, 8 in 2024-08-17T17:32:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:08 smithi130 ceph-mon[110068]: osdmap e590: 8 total, 8 up, 8 in 2024-08-17T17:32:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:08 smithi179 ceph-mon[93519]: osdmap e590: 8 total, 8 up, 8 in 2024-08-17T17:32:09.419 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:32:09.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:09 smithi130 ceph-mon[110068]: pgmap v516: 233 pgs: 7 creating+peering, 4 unknown, 222 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 630 KiB/s rd, 41 KiB/s wr, 738 op/s 2024-08-17T17:32:09.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:09 smithi130 ceph-mon[110068]: osdmap e591: 8 total, 8 up, 8 in 2024-08-17T17:32:09.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:09 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2608445863' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:09.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:09 smithi130 ceph-mon[105521]: pgmap v516: 233 pgs: 7 creating+peering, 4 unknown, 222 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 630 KiB/s rd, 41 KiB/s wr, 738 op/s 2024-08-17T17:32:09.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:09 smithi130 ceph-mon[105521]: osdmap e591: 8 total, 8 up, 8 in 2024-08-17T17:32:09.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:09 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2608445863' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:09 smithi179 ceph-mon[93519]: pgmap v516: 233 pgs: 7 creating+peering, 4 unknown, 222 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 630 KiB/s rd, 41 KiB/s wr, 738 op/s 2024-08-17T17:32:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:09 smithi179 ceph-mon[93519]: osdmap e591: 8 total, 8 up, 8 in 2024-08-17T17:32:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:09 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2608445863' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:10.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:32:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:32:09] "GET /metrics HTTP/1.1" 200 41174 "" "Prometheus/2.51.0" 2024-08-17T17:32:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:10 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2608445863' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:10 smithi179 ceph-mon[93519]: osdmap e592: 8 total, 8 up, 8 in 2024-08-17T17:32:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:10 smithi179 ceph-mon[93519]: osdmap e593: 8 total, 8 up, 8 in 2024-08-17T17:32:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:32:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:32:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:32:10.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:10 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2608445863' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:10.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:10 smithi130 ceph-mon[105521]: osdmap e592: 8 total, 8 up, 8 in 2024-08-17T17:32:10.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:10 smithi130 ceph-mon[105521]: osdmap e593: 8 total, 8 up, 8 in 2024-08-17T17:32:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:10 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2608445863' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:10 smithi130 ceph-mon[110068]: osdmap e592: 8 total, 8 up, 8 in 2024-08-17T17:32:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:10 smithi130 ceph-mon[110068]: osdmap e593: 8 total, 8 up, 8 in 2024-08-17T17:32:11.692 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:11 smithi130 ceph-mon[105521]: pgmap v519: 265 pgs: 6 creating+peering, 23 unknown, 236 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 682 KiB/s rd, 43 KiB/s wr, 794 op/s 2024-08-17T17:32:11.692 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:11 smithi130 ceph-mon[105521]: osdmap e594: 8 total, 8 up, 8 in 2024-08-17T17:32:11.692 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:11 smithi130 ceph-mon[110068]: pgmap v519: 265 pgs: 6 creating+peering, 23 unknown, 236 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 682 KiB/s rd, 43 KiB/s wr, 794 op/s 2024-08-17T17:32:11.692 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:11 smithi130 ceph-mon[110068]: osdmap e594: 8 total, 8 up, 8 in 2024-08-17T17:32:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:11 smithi179 ceph-mon[93519]: pgmap v519: 265 pgs: 6 creating+peering, 23 unknown, 236 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 682 KiB/s rd, 43 KiB/s wr, 794 op/s 2024-08-17T17:32:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:11 smithi179 ceph-mon[93519]: osdmap e594: 8 total, 8 up, 8 in 2024-08-17T17:32:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:12.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:13.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:13 smithi179 ceph-mon[93519]: pgmap v522: 265 pgs: 6 creating+peering, 18 unknown, 241 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 2.7 KiB/s wr, 58 op/s 2024-08-17T17:32:13.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:13 smithi179 ceph-mon[93519]: osdmap e595: 8 total, 8 up, 8 in 2024-08-17T17:32:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:13 smithi130 ceph-mon[105521]: pgmap v522: 265 pgs: 6 creating+peering, 18 unknown, 241 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 2.7 KiB/s wr, 58 op/s 2024-08-17T17:32:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:13 smithi130 ceph-mon[105521]: osdmap e595: 8 total, 8 up, 8 in 2024-08-17T17:32:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:13 smithi130 ceph-mon[110068]: pgmap v522: 265 pgs: 6 creating+peering, 18 unknown, 241 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 2.7 KiB/s wr, 58 op/s 2024-08-17T17:32:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:13 smithi130 ceph-mon[110068]: osdmap e595: 8 total, 8 up, 8 in 2024-08-17T17:32:14.756 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:14 smithi179 ceph-mon[93519]: osdmap e596: 8 total, 8 up, 8 in 2024-08-17T17:32:14.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:14 smithi130 ceph-mon[105521]: osdmap e596: 8 total, 8 up, 8 in 2024-08-17T17:32:14.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:14 smithi130 ceph-mon[110068]: osdmap e596: 8 total, 8 up, 8 in 2024-08-17T17:32:15.474 INFO:tasks.workunit.client.0.smithi130.stdout:image: image28 2024-08-17T17:32:15.474 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image28 2024-08-17T17:32:15.474 INFO:tasks.workunit.client.0.smithi130.stdout:found image28 2024-08-17T17:32:15.474 INFO:tasks.workunit.client.0.smithi130.stdout:erased image28 2024-08-17T17:32:15.474 INFO:tasks.workunit.client.0.smithi130.stdout:image: image28 2024-08-17T17:32:15.474 INFO:tasks.workunit.client.0.smithi130.stdout:image: image29 2024-08-17T17:32:15.474 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image28 2024-08-17T17:32:15.474 INFO:tasks.workunit.client.0.smithi130.stdout:found image28 2024-08-17T17:32:15.474 INFO:tasks.workunit.client.0.smithi130.stdout:erased image28 2024-08-17T17:32:15.474 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image29 2024-08-17T17:32:15.474 INFO:tasks.workunit.client.0.smithi130.stdout:found image29 2024-08-17T17:32:15.474 INFO:tasks.workunit.client.0.smithi130.stdout:erased image29 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:100.00% done 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:image: image28 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:image: image29 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:image: image30 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image28 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:found image28 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:erased image28 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image29 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:found image29 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:erased image29 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image30 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:found image30 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:erased image30 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:image: image28 2024-08-17T17:32:15.475 INFO:tasks.workunit.client.0.smithi130.stdout:image: image29 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:image: image30 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:image: image31 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image28 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:found image28 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:erased image28 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image29 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:found image29 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:erased image29 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image30 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:found image30 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:erased image30 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image31 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:found image31 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:erased image31 2024-08-17T17:32:15.476 INFO:tasks.workunit.client.0.smithi130.stdout:image: image28 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:image: image29 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:image: image30 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:image: image31 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:image: image32 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image28 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:found image28 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:erased image28 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image29 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:found image29 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:erased image29 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image30 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:found image30 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:erased image30 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image31 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:found image31 2024-08-17T17:32:15.477 INFO:tasks.workunit.client.0.smithi130.stdout:erased image31 2024-08-17T17:32:15.478 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image32 2024-08-17T17:32:15.478 INFO:tasks.workunit.client.0.smithi130.stdout:found image32 2024-08-17T17:32:15.478 INFO:tasks.workunit.client.0.smithi130.stdout:erased image32 2024-08-17T17:32:15.478 INFO:tasks.workunit.client.0.smithi130.stdout:100.00% done 2024-08-17T17:32:15.478 INFO:tasks.workunit.client.0.smithi130.stdout:image: image28 2024-08-17T17:32:15.478 INFO:tasks.workunit.client.0.smithi130.stdout:image: image29 2024-08-17T17:32:15.478 INFO:tasks.workunit.client.0.smithi130.stdout:image: image30 2024-08-17T17:32:15.478 INFO:tasks.workunit.client.0.smithi130.stdout:image: image31 2024-08-17T17:32:15.478 INFO:tasks.workunit.client.0.smithi130.stdout:image: image32 2024-08-17T17:32:15.478 INFO:tasks.workunit.client.0.smithi130.stdout:image: image33 2024-08-17T17:32:15.478 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image28 2024-08-17T17:32:15.478 INFO:tasks.workunit.client.0.smithi130.stdout:found image28 2024-08-17T17:32:15.478 INFO:tasks.workunit.client.0.smithi130.stdout:erased image28 2024-08-17T17:32:15.479 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image29 2024-08-17T17:32:15.479 INFO:tasks.workunit.client.0.smithi130.stdout:found image29 2024-08-17T17:32:15.479 INFO:tasks.workunit.client.0.smithi130.stdout:erased image29 2024-08-17T17:32:15.479 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image30 2024-08-17T17:32:15.479 INFO:tasks.workunit.client.0.smithi130.stdout:found image30 2024-08-17T17:32:15.479 INFO:tasks.workunit.client.0.smithi130.stdout:erased image30 2024-08-17T17:32:15.479 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image31 2024-08-17T17:32:15.479 INFO:tasks.workunit.client.0.smithi130.stdout:found image31 2024-08-17T17:32:15.479 INFO:tasks.workunit.client.0.smithi130.stdout:erased image31 2024-08-17T17:32:15.479 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image32 2024-08-17T17:32:15.479 INFO:tasks.workunit.client.0.smithi130.stdout:found image32 2024-08-17T17:32:15.479 INFO:tasks.workunit.client.0.smithi130.stdout:erased image32 2024-08-17T17:32:15.479 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image33 2024-08-17T17:32:15.479 INFO:tasks.workunit.client.0.smithi130.stdout:found image33 2024-08-17T17:32:15.480 INFO:tasks.workunit.client.0.smithi130.stdout:erased image33 2024-08-17T17:32:15.480 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestDeepCopy (7742 ms) 2024-08-17T17:32:15.480 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestDeepCopyPP 2024-08-17T17:32:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:15 smithi179 ceph-mon[93519]: pgmap v525: 265 pgs: 265 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 7.0 KiB/s wr, 110 op/s 2024-08-17T17:32:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:15 smithi179 ceph-mon[93519]: osdmap e597: 8 total, 8 up, 8 in 2024-08-17T17:32:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:15 smithi179 ceph-mon[93519]: osdmap e598: 8 total, 8 up, 8 in 2024-08-17T17:32:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:15 smithi130 ceph-mon[105521]: pgmap v525: 265 pgs: 265 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 7.0 KiB/s wr, 110 op/s 2024-08-17T17:32:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:15 smithi130 ceph-mon[105521]: osdmap e597: 8 total, 8 up, 8 in 2024-08-17T17:32:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:15 smithi130 ceph-mon[105521]: osdmap e598: 8 total, 8 up, 8 in 2024-08-17T17:32:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:15 smithi130 ceph-mon[110068]: pgmap v525: 265 pgs: 265 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 7.0 KiB/s wr, 110 op/s 2024-08-17T17:32:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:15 smithi130 ceph-mon[110068]: osdmap e597: 8 total, 8 up, 8 in 2024-08-17T17:32:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:15 smithi130 ceph-mon[110068]: osdmap e598: 8 total, 8 up, 8 in 2024-08-17T17:32:17.455 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:32:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:17 smithi179 ceph-mon[93519]: pgmap v528: 265 pgs: 265 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 427 KiB/s rd, 25 KiB/s wr, 507 op/s 2024-08-17T17:32:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:17 smithi179 ceph-mon[93519]: osdmap e599: 8 total, 8 up, 8 in 2024-08-17T17:32:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:17 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1189193923' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:17 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:17 smithi130 ceph-mon[105521]: pgmap v528: 265 pgs: 265 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 427 KiB/s rd, 25 KiB/s wr, 507 op/s 2024-08-17T17:32:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:17 smithi130 ceph-mon[105521]: osdmap e599: 8 total, 8 up, 8 in 2024-08-17T17:32:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:17 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1189193923' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:17 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:17 smithi130 ceph-mon[110068]: pgmap v528: 265 pgs: 265 active+clean; 583 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 427 KiB/s rd, 25 KiB/s wr, 507 op/s 2024-08-17T17:32:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:17 smithi130 ceph-mon[110068]: osdmap e599: 8 total, 8 up, 8 in 2024-08-17T17:32:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:17 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1189193923' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:17.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:17 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:18 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:18 smithi179 ceph-mon[93519]: osdmap e600: 8 total, 8 up, 8 in 2024-08-17T17:32:18.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:18 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:18.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:18 smithi130 ceph-mon[105521]: osdmap e600: 8 total, 8 up, 8 in 2024-08-17T17:32:18.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:18 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:18.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:18 smithi130 ceph-mon[110068]: osdmap e600: 8 total, 8 up, 8 in 2024-08-17T17:32:19.727 INFO:tasks.workunit.client.0.smithi130.stdout:num images is: 1 2024-08-17T17:32:19.727 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:32:19.727 INFO:tasks.workunit.client.0.smithi130.stdout:image: image34 2024-08-17T17:32:19.727 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image34 2024-08-17T17:32:19.751 INFO:tasks.workunit.client.0.smithi130.stdout:num images is: 2 2024-08-17T17:32:19.751 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 2 2024-08-17T17:32:19.751 INFO:tasks.workunit.client.0.smithi130.stdout:image: image34 2024-08-17T17:32:19.752 INFO:tasks.workunit.client.0.smithi130.stdout:image: image35 2024-08-17T17:32:19.752 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image34 2024-08-17T17:32:19.752 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image35 2024-08-17T17:32:19.803 INFO:tasks.workunit.client.0.smithi130.stdout:100.00% done 2024-08-17T17:32:19.803 INFO:tasks.workunit.client.0.smithi130.stdout:num images is: 3 2024-08-17T17:32:19.803 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 3 2024-08-17T17:32:19.804 INFO:tasks.workunit.client.0.smithi130.stdout:image: image34 2024-08-17T17:32:19.804 INFO:tasks.workunit.client.0.smithi130.stdout:image: image35 2024-08-17T17:32:19.804 INFO:tasks.workunit.client.0.smithi130.stdout:image: image36 2024-08-17T17:32:19.804 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image34 2024-08-17T17:32:19.804 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image35 2024-08-17T17:32:19.804 INFO:tasks.workunit.client.0.smithi130.stdout:expected = image36 2024-08-17T17:32:19.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:19 smithi130 ceph-mon[105521]: pgmap v531: 297 pgs: 13 unknown, 284 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 428 KiB/s rd, 24 KiB/s wr, 504 op/s 2024-08-17T17:32:19.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:19 smithi130 ceph-mon[105521]: osdmap e601: 8 total, 8 up, 8 in 2024-08-17T17:32:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:19 smithi130 ceph-mon[110068]: pgmap v531: 297 pgs: 13 unknown, 284 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 428 KiB/s rd, 24 KiB/s wr, 504 op/s 2024-08-17T17:32:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:19 smithi130 ceph-mon[110068]: osdmap e601: 8 total, 8 up, 8 in 2024-08-17T17:32:19.833 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestDeepCopyPP (4360 ms) 2024-08-17T17:32:19.833 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnap 2024-08-17T17:32:19.833 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:32:19.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:19 smithi179 ceph-mon[93519]: pgmap v531: 297 pgs: 13 unknown, 284 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 428 KiB/s rd, 24 KiB/s wr, 504 op/s 2024-08-17T17:32:19.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:19 smithi179 ceph-mon[93519]: osdmap e601: 8 total, 8 up, 8 in 2024-08-17T17:32:20.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:32:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:32:19] "GET /metrics HTTP/1.1" 200 48422 "" "Prometheus/2.51.0" 2024-08-17T17:32:20.473 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:32:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:32:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:32:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:20 smithi179 ceph-mon[93519]: osdmap e602: 8 total, 8 up, 8 in 2024-08-17T17:32:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:20 smithi179 ceph-mon[93519]: osdmap e603: 8 total, 8 up, 8 in 2024-08-17T17:32:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:20 smithi130 ceph-mon[105521]: osdmap e602: 8 total, 8 up, 8 in 2024-08-17T17:32:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:20 smithi130 ceph-mon[105521]: osdmap e603: 8 total, 8 up, 8 in 2024-08-17T17:32:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:20 smithi130 ceph-mon[110068]: osdmap e602: 8 total, 8 up, 8 in 2024-08-17T17:32:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:20 smithi130 ceph-mon[110068]: osdmap e603: 8 total, 8 up, 8 in 2024-08-17T17:32:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:21 smithi179 ceph-mon[93519]: pgmap v534: 297 pgs: 2 unknown, 295 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 171 KiB/s rd, 8.2 KiB/s wr, 198 op/s 2024-08-17T17:32:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:21 smithi179 ceph-mon[93519]: osdmap e604: 8 total, 8 up, 8 in 2024-08-17T17:32:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:21 smithi130 ceph-mon[105521]: pgmap v534: 297 pgs: 2 unknown, 295 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 171 KiB/s rd, 8.2 KiB/s wr, 198 op/s 2024-08-17T17:32:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:21 smithi130 ceph-mon[105521]: osdmap e604: 8 total, 8 up, 8 in 2024-08-17T17:32:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:21 smithi130 ceph-mon[110068]: pgmap v534: 297 pgs: 2 unknown, 295 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 171 KiB/s rd, 8.2 KiB/s wr, 198 op/s 2024-08-17T17:32:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:21 smithi130 ceph-mon[110068]: osdmap e604: 8 total, 8 up, 8 in 2024-08-17T17:32:23.420 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 1 2024-08-17T17:32:23.420 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:32:23.421 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap1 2024-08-17T17:32:23.421 INFO:tasks.workunit.client.0.smithi130.stdout:found snap1 with size 2097152 2024-08-17T17:32:23.421 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 2 2024-08-17T17:32:23.421 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 2 2024-08-17T17:32:23.421 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap1 2024-08-17T17:32:23.421 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap2 2024-08-17T17:32:23.421 INFO:tasks.workunit.client.0.smithi130.stdout:found snap1 with size 2097152 2024-08-17T17:32:23.421 INFO:tasks.workunit.client.0.smithi130.stdout:found snap2 with size 4194304 2024-08-17T17:32:23.421 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 1 2024-08-17T17:32:23.421 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:32:23.421 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap2 2024-08-17T17:32:23.421 INFO:tasks.workunit.client.0.smithi130.stdout:found snap2 with size 4194304 2024-08-17T17:32:23.421 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 0 2024-08-17T17:32:23.422 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 0 2024-08-17T17:32:23.422 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnap (3587 ms) 2024-08-17T17:32:23.422 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestGetSnapShotTimeStamp 2024-08-17T17:32:23.422 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:32:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:23 smithi130 ceph-mon[105521]: pgmap v537: 297 pgs: 297 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 3.7 KiB/s wr, 97 op/s 2024-08-17T17:32:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:23 smithi130 ceph-mon[105521]: osdmap e605: 8 total, 8 up, 8 in 2024-08-17T17:32:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:23 smithi130 ceph-mon[110068]: pgmap v537: 297 pgs: 297 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 3.7 KiB/s wr, 97 op/s 2024-08-17T17:32:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:23 smithi130 ceph-mon[110068]: osdmap e605: 8 total, 8 up, 8 in 2024-08-17T17:32:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:23 smithi179 ceph-mon[93519]: pgmap v537: 297 pgs: 297 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 3.7 KiB/s wr, 97 op/s 2024-08-17T17:32:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:23 smithi179 ceph-mon[93519]: osdmap e605: 8 total, 8 up, 8 in 2024-08-17T17:32:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:24 smithi130 ceph-mon[105521]: osdmap e606: 8 total, 8 up, 8 in 2024-08-17T17:32:24.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:24 smithi130 ceph-mon[110068]: osdmap e606: 8 total, 8 up, 8 in 2024-08-17T17:32:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:24 smithi179 ceph-mon[93519]: osdmap e606: 8 total, 8 up, 8 in 2024-08-17T17:32:25.420 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestGetSnapShotTimeStamp (2000 ms) 2024-08-17T17:32:25.420 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnapPP 2024-08-17T17:32:25.420 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:32:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:25 smithi130 ceph-mon[105521]: pgmap v540: 297 pgs: 297 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 332 KiB/s rd, 25 KiB/s wr, 404 op/s 2024-08-17T17:32:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:25 smithi130 ceph-mon[105521]: osdmap e607: 8 total, 8 up, 8 in 2024-08-17T17:32:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:25 smithi130 ceph-mon[105521]: osdmap e608: 8 total, 8 up, 8 in 2024-08-17T17:32:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:25 smithi130 ceph-mon[110068]: pgmap v540: 297 pgs: 297 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 332 KiB/s rd, 25 KiB/s wr, 404 op/s 2024-08-17T17:32:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:25 smithi130 ceph-mon[110068]: osdmap e607: 8 total, 8 up, 8 in 2024-08-17T17:32:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:25 smithi130 ceph-mon[110068]: osdmap e608: 8 total, 8 up, 8 in 2024-08-17T17:32:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:25 smithi179 ceph-mon[93519]: pgmap v540: 297 pgs: 297 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 332 KiB/s rd, 25 KiB/s wr, 404 op/s 2024-08-17T17:32:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:25 smithi179 ceph-mon[93519]: osdmap e607: 8 total, 8 up, 8 in 2024-08-17T17:32:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:25 smithi179 ceph-mon[93519]: osdmap e608: 8 total, 8 up, 8 in 2024-08-17T17:32:26.435 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 1 2024-08-17T17:32:26.435 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:32:26.435 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap1 2024-08-17T17:32:26.435 INFO:tasks.workunit.client.0.smithi130.stdout:found snap1 with size 2097152 2024-08-17T17:32:27.443 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 2 2024-08-17T17:32:27.443 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 2 2024-08-17T17:32:27.444 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap1 2024-08-17T17:32:27.444 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap2 2024-08-17T17:32:27.444 INFO:tasks.workunit.client.0.smithi130.stdout:found snap1 with size 2097152 2024-08-17T17:32:27.444 INFO:tasks.workunit.client.0.smithi130.stdout:found snap2 with size 4194304 2024-08-17T17:32:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:27 smithi130 ceph-mon[105521]: pgmap v543: 297 pgs: 297 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 331 KiB/s rd, 24 KiB/s wr, 400 op/s 2024-08-17T17:32:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:27 smithi130 ceph-mon[105521]: osdmap e609: 8 total, 8 up, 8 in 2024-08-17T17:32:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:27 smithi130 ceph-mon[110068]: pgmap v543: 297 pgs: 297 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 331 KiB/s rd, 24 KiB/s wr, 400 op/s 2024-08-17T17:32:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:27 smithi130 ceph-mon[110068]: osdmap e609: 8 total, 8 up, 8 in 2024-08-17T17:32:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:27 smithi179 ceph-mon[93519]: pgmap v543: 297 pgs: 297 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 331 KiB/s rd, 24 KiB/s wr, 400 op/s 2024-08-17T17:32:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:27 smithi179 ceph-mon[93519]: osdmap e609: 8 total, 8 up, 8 in 2024-08-17T17:32:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:28.447 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 1 2024-08-17T17:32:28.448 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:32:28.448 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap2 2024-08-17T17:32:28.448 INFO:tasks.workunit.client.0.smithi130.stdout:found snap2 with size 4194304 2024-08-17T17:32:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:28 smithi179 ceph-mon[93519]: osdmap e610: 8 total, 8 up, 8 in 2024-08-17T17:32:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:28 smithi130 ceph-mon[105521]: osdmap e610: 8 total, 8 up, 8 in 2024-08-17T17:32:28.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:28 smithi130 ceph-mon[110068]: osdmap e610: 8 total, 8 up, 8 in 2024-08-17T17:32:29.448 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 0 2024-08-17T17:32:29.448 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 0 2024-08-17T17:32:29.454 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnapPP (4034 ms) 2024-08-17T17:32:29.454 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestGetNameIdSnapPP 2024-08-17T17:32:29.454 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:32:29.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:29 smithi130 ceph-mon[105521]: pgmap v546: 297 pgs: 297 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 4.0 KiB/s wr, 49 op/s 2024-08-17T17:32:29.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:29 smithi130 ceph-mon[105521]: osdmap e611: 8 total, 8 up, 8 in 2024-08-17T17:32:29.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:29 smithi130 ceph-mon[110068]: pgmap v546: 297 pgs: 297 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 4.0 KiB/s wr, 49 op/s 2024-08-17T17:32:29.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:29 smithi130 ceph-mon[110068]: osdmap e611: 8 total, 8 up, 8 in 2024-08-17T17:32:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:29 smithi179 ceph-mon[93519]: pgmap v546: 297 pgs: 297 active+clean; 584 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 4.0 KiB/s wr, 49 op/s 2024-08-17T17:32:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:29 smithi179 ceph-mon[93519]: osdmap e611: 8 total, 8 up, 8 in 2024-08-17T17:32:30.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:32:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:32:29] "GET /metrics HTTP/1.1" 200 50856 "" "Prometheus/2.51.0" 2024-08-17T17:32:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:30 smithi179 ceph-mon[93519]: osdmap e612: 8 total, 8 up, 8 in 2024-08-17T17:32:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:30 smithi179 ceph-mon[93519]: osdmap e613: 8 total, 8 up, 8 in 2024-08-17T17:32:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:32:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:32:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:32:30.836 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:30 smithi130 ceph-mon[105521]: osdmap e612: 8 total, 8 up, 8 in 2024-08-17T17:32:30.836 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:30 smithi130 ceph-mon[105521]: osdmap e613: 8 total, 8 up, 8 in 2024-08-17T17:32:30.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:30 smithi130 ceph-mon[110068]: osdmap e612: 8 total, 8 up, 8 in 2024-08-17T17:32:30.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:30 smithi130 ceph-mon[110068]: osdmap e613: 8 total, 8 up, 8 in 2024-08-17T17:32:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:31 smithi179 ceph-mon[93519]: pgmap v549: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 7.2 KiB/s wr, 91 op/s 2024-08-17T17:32:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:31 smithi179 ceph-mon[93519]: osdmap e614: 8 total, 8 up, 8 in 2024-08-17T17:32:31.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:31 smithi130 ceph-mon[105521]: pgmap v549: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 7.2 KiB/s wr, 91 op/s 2024-08-17T17:32:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:31 smithi130 ceph-mon[105521]: osdmap e614: 8 total, 8 up, 8 in 2024-08-17T17:32:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:31 smithi130 ceph-mon[110068]: pgmap v549: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 7.2 KiB/s wr, 91 op/s 2024-08-17T17:32:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:31 smithi130 ceph-mon[110068]: osdmap e614: 8 total, 8 up, 8 in 2024-08-17T17:32:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:33 smithi130 ceph-mon[105521]: pgmap v552: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 3.5 KiB/s wr, 43 op/s 2024-08-17T17:32:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:33 smithi130 ceph-mon[105521]: osdmap e615: 8 total, 8 up, 8 in 2024-08-17T17:32:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:33 smithi130 ceph-mon[110068]: pgmap v552: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 3.5 KiB/s wr, 43 op/s 2024-08-17T17:32:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:33 smithi130 ceph-mon[110068]: osdmap e615: 8 total, 8 up, 8 in 2024-08-17T17:32:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:33 smithi179 ceph-mon[93519]: pgmap v552: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 3.5 KiB/s wr, 43 op/s 2024-08-17T17:32:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:33 smithi179 ceph-mon[93519]: osdmap e615: 8 total, 8 up, 8 in 2024-08-17T17:32:34.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:34 smithi130 ceph-mon[105521]: osdmap e616: 8 total, 8 up, 8 in 2024-08-17T17:32:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:34 smithi130 ceph-mon[110068]: osdmap e616: 8 total, 8 up, 8 in 2024-08-17T17:32:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:34 smithi179 ceph-mon[93519]: osdmap e616: 8 total, 8 up, 8 in 2024-08-17T17:32:35.416 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 0 2024-08-17T17:32:35.416 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 0 2024-08-17T17:32:35.425 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestGetNameIdSnapPP (5971 ms) 2024-08-17T17:32:35.426 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCreateLsRenameSnapPP 2024-08-17T17:32:35.426 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:32:35.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:35 smithi130 ceph-mon[110068]: pgmap v555: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 4.2 KiB/s wr, 57 op/s 2024-08-17T17:32:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:35 smithi130 ceph-mon[110068]: osdmap e617: 8 total, 8 up, 8 in 2024-08-17T17:32:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:35 smithi130 ceph-mon[110068]: osdmap e618: 8 total, 8 up, 8 in 2024-08-17T17:32:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:35 smithi130 ceph-mon[105521]: pgmap v555: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 4.2 KiB/s wr, 57 op/s 2024-08-17T17:32:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:35 smithi130 ceph-mon[105521]: osdmap e617: 8 total, 8 up, 8 in 2024-08-17T17:32:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:35 smithi130 ceph-mon[105521]: osdmap e618: 8 total, 8 up, 8 in 2024-08-17T17:32:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:35 smithi179 ceph-mon[93519]: pgmap v555: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 4.2 KiB/s wr, 57 op/s 2024-08-17T17:32:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:35 smithi179 ceph-mon[93519]: osdmap e617: 8 total, 8 up, 8 in 2024-08-17T17:32:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:35 smithi179 ceph-mon[93519]: osdmap e618: 8 total, 8 up, 8 in 2024-08-17T17:32:36.442 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 1 2024-08-17T17:32:36.442 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:32:36.442 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap1 2024-08-17T17:32:36.442 INFO:tasks.workunit.client.0.smithi130.stdout:found snap1 with size 2097152 2024-08-17T17:32:37.453 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 2 2024-08-17T17:32:37.453 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 2 2024-08-17T17:32:37.453 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap1 2024-08-17T17:32:37.453 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap2 2024-08-17T17:32:37.453 INFO:tasks.workunit.client.0.smithi130.stdout:found snap1 with size 2097152 2024-08-17T17:32:37.453 INFO:tasks.workunit.client.0.smithi130.stdout:found snap2 with size 4194304 2024-08-17T17:32:37.457 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 2 2024-08-17T17:32:37.457 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 2 2024-08-17T17:32:37.457 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap1-rename 2024-08-17T17:32:37.457 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap2 2024-08-17T17:32:37.457 INFO:tasks.workunit.client.0.smithi130.stdout:found snap1-rename with size 2097152 2024-08-17T17:32:37.457 INFO:tasks.workunit.client.0.smithi130.stdout:found snap2 with size 4194304 2024-08-17T17:32:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:37 smithi179 ceph-mon[93519]: pgmap v558: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 4.0 KiB/s wr, 55 op/s 2024-08-17T17:32:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:37 smithi179 ceph-mon[93519]: osdmap e619: 8 total, 8 up, 8 in 2024-08-17T17:32:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:37 smithi130 ceph-mon[105521]: pgmap v558: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 4.0 KiB/s wr, 55 op/s 2024-08-17T17:32:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:37 smithi130 ceph-mon[105521]: osdmap e619: 8 total, 8 up, 8 in 2024-08-17T17:32:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:37 smithi130 ceph-mon[110068]: pgmap v558: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 4.0 KiB/s wr, 55 op/s 2024-08-17T17:32:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:37 smithi130 ceph-mon[110068]: osdmap e619: 8 total, 8 up, 8 in 2024-08-17T17:32:38.455 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 1 2024-08-17T17:32:38.455 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:32:38.455 INFO:tasks.workunit.client.0.smithi130.stdout:snap: snap2-rename 2024-08-17T17:32:38.455 INFO:tasks.workunit.client.0.smithi130.stdout:found snap2-rename with size 4194304 2024-08-17T17:32:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:38 smithi179 ceph-mon[93519]: osdmap e620: 8 total, 8 up, 8 in 2024-08-17T17:32:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:38 smithi130 ceph-mon[105521]: osdmap e620: 8 total, 8 up, 8 in 2024-08-17T17:32:38.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:38 smithi130 ceph-mon[110068]: osdmap e620: 8 total, 8 up, 8 in 2024-08-17T17:32:39.457 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 0 2024-08-17T17:32:39.457 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 0 2024-08-17T17:32:39.468 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCreateLsRenameSnapPP (4042 ms) 2024-08-17T17:32:39.468 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ConcurrentCreatesUnvalidatedPool 2024-08-17T17:32:39.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:39 smithi130 ceph-mon[105521]: pgmap v561: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 5.7 KiB/s wr, 77 op/s 2024-08-17T17:32:39.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:39 smithi130 ceph-mon[105521]: osdmap e621: 8 total, 8 up, 8 in 2024-08-17T17:32:39.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:39 smithi130 ceph-mon[110068]: pgmap v561: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 5.7 KiB/s wr, 77 op/s 2024-08-17T17:32:39.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:39 smithi130 ceph-mon[110068]: osdmap e621: 8 total, 8 up, 8 in 2024-08-17T17:32:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:39 smithi179 ceph-mon[93519]: pgmap v561: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 5.7 KiB/s wr, 77 op/s 2024-08-17T17:32:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:39 smithi179 ceph-mon[93519]: osdmap e621: 8 total, 8 up, 8 in 2024-08-17T17:32:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:32:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:32:39] "GET /metrics HTTP/1.1" 200 50856 "" "Prometheus/2.51.0" 2024-08-17T17:32:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:40 smithi179 ceph-mon[93519]: osdmap e622: 8 total, 8 up, 8 in 2024-08-17T17:32:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:32:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:32:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:32:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:40 smithi130 ceph-mon[105521]: osdmap e622: 8 total, 8 up, 8 in 2024-08-17T17:32:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:40 smithi130 ceph-mon[110068]: osdmap e622: 8 total, 8 up, 8 in 2024-08-17T17:32:41.465 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:32:41.465 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:32:41.465 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:32:41.465 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:32:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:41 smithi179 ceph-mon[93519]: pgmap v564: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 6.0 KiB/s wr, 78 op/s 2024-08-17T17:32:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:41 smithi179 ceph-mon[93519]: osdmap e623: 8 total, 8 up, 8 in 2024-08-17T17:32:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:41 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1505302551' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:41 smithi130 ceph-mon[105521]: pgmap v564: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 6.0 KiB/s wr, 78 op/s 2024-08-17T17:32:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:41 smithi130 ceph-mon[105521]: osdmap e623: 8 total, 8 up, 8 in 2024-08-17T17:32:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:41 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1505302551' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:41 smithi130 ceph-mon[110068]: pgmap v564: 297 pgs: 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 6.0 KiB/s wr, 78 op/s 2024-08-17T17:32:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:41 smithi130 ceph-mon[110068]: osdmap e623: 8 total, 8 up, 8 in 2024-08-17T17:32:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:41 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1505302551' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:42 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1505302551' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:42 smithi179 ceph-mon[93519]: osdmap e624: 8 total, 8 up, 8 in 2024-08-17T17:32:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:42 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1505302551' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:42 smithi130 ceph-mon[105521]: osdmap e624: 8 total, 8 up, 8 in 2024-08-17T17:32:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:42 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1505302551' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:42 smithi130 ceph-mon[110068]: osdmap e624: 8 total, 8 up, 8 in 2024-08-17T17:32:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:43.714 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ConcurrentCreatesUnvalidatedPool (4246 ms) 2024-08-17T17:32:43.715 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.RemoveFullTry 2024-08-17T17:32:43.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:43 smithi179 ceph-mon[93519]: pgmap v567: 329 pgs: 7 creating+peering, 25 unknown, 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-08-17T17:32:43.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:43 smithi179 ceph-mon[93519]: osdmap e625: 8 total, 8 up, 8 in 2024-08-17T17:32:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:43 smithi130 ceph-mon[105521]: pgmap v567: 329 pgs: 7 creating+peering, 25 unknown, 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-08-17T17:32:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:43 smithi130 ceph-mon[105521]: osdmap e625: 8 total, 8 up, 8 in 2024-08-17T17:32:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:43 smithi130 ceph-mon[110068]: pgmap v567: 329 pgs: 7 creating+peering, 25 unknown, 297 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 767 B/s wr, 6 op/s 2024-08-17T17:32:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:43 smithi130 ceph-mon[110068]: osdmap e625: 8 total, 8 up, 8 in 2024-08-17T17:32:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:44 smithi179 ceph-mon[93519]: osdmap e626: 8 total, 8 up, 8 in 2024-08-17T17:32:44.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:44 smithi130 ceph-mon[105521]: osdmap e626: 8 total, 8 up, 8 in 2024-08-17T17:32:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:44 smithi130 ceph-mon[110068]: osdmap e626: 8 total, 8 up, 8 in 2024-08-17T17:32:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:45 smithi179 ceph-mon[93519]: pgmap v570: 329 pgs: 7 creating+peering, 2 unknown, 320 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 KiB/s wr, 28 op/s 2024-08-17T17:32:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:45 smithi179 ceph-mon[93519]: osdmap e627: 8 total, 8 up, 8 in 2024-08-17T17:32:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:45 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3243339668' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:45 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3243339668' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:45 smithi179 ceph-mon[93519]: osdmap e628: 8 total, 8 up, 8 in 2024-08-17T17:32:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:45 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:45 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:45 smithi130 ceph-mon[105521]: pgmap v570: 329 pgs: 7 creating+peering, 2 unknown, 320 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 KiB/s wr, 28 op/s 2024-08-17T17:32:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:45 smithi130 ceph-mon[105521]: osdmap e627: 8 total, 8 up, 8 in 2024-08-17T17:32:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:45 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3243339668' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:45 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3243339668' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:45 smithi130 ceph-mon[105521]: osdmap e628: 8 total, 8 up, 8 in 2024-08-17T17:32:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:45 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:45 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:45 smithi130 ceph-mon[110068]: pgmap v570: 329 pgs: 7 creating+peering, 2 unknown, 320 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 KiB/s wr, 28 op/s 2024-08-17T17:32:45.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:45 smithi130 ceph-mon[110068]: osdmap e627: 8 total, 8 up, 8 in 2024-08-17T17:32:45.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:45 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3243339668' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:45.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:45 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3243339668' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:45.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:45 smithi130 ceph-mon[110068]: osdmap e628: 8 total, 8 up, 8 in 2024-08-17T17:32:45.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:45 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:45.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:45 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:32:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:47 smithi130 ceph-mon[110068]: pgmap v573: 361 pgs: 13 creating+peering, 26 unknown, 322 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 1.7 KiB/s wr, 47 op/s 2024-08-17T17:32:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:47 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:47 smithi130 ceph-mon[110068]: osdmap e629: 8 total, 8 up, 8 in 2024-08-17T17:32:47.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:47 smithi130 ceph-mon[105521]: pgmap v573: 361 pgs: 13 creating+peering, 26 unknown, 322 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 1.7 KiB/s wr, 47 op/s 2024-08-17T17:32:47.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:47 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:47.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:47 smithi130 ceph-mon[105521]: osdmap e629: 8 total, 8 up, 8 in 2024-08-17T17:32:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:47 smithi179 ceph-mon[93519]: pgmap v573: 361 pgs: 13 creating+peering, 26 unknown, 322 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 1.7 KiB/s wr, 47 op/s 2024-08-17T17:32:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:47 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-23","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-08-17T17:32:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:47 smithi179 ceph-mon[93519]: osdmap e629: 8 total, 8 up, 8 in 2024-08-17T17:32:48.441 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:32:48.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:48 smithi130 ceph-mon[105521]: osdmap e630: 8 total, 8 up, 8 in 2024-08-17T17:32:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:48 smithi130 ceph-mon[110068]: osdmap e630: 8 total, 8 up, 8 in 2024-08-17T17:32:48.667 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:32:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:32:48] ENGINE Bus STOPPING 2024-08-17T17:32:48.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:48 smithi179 ceph-mon[93519]: osdmap e630: 8 total, 8 up, 8 in 2024-08-17T17:32:49.144 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:32:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:32:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:32:49.144 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:32:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:32:48] ENGINE Bus STOPPED 2024-08-17T17:32:49.144 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:32:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:32:48] ENGINE Bus STARTING 2024-08-17T17:32:49.144 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:32:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:32:48] ENGINE Serving on http://:::9283 2024-08-17T17:32:49.144 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:32:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:32:48] ENGINE Bus STARTED 2024-08-17T17:32:49.414 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[105521]: pgmap v576: 361 pgs: 6 creating+peering, 5 unknown, 350 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-08-17T17:32:49.414 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[110068]: pgmap v576: 361 pgs: 6 creating+peering, 5 unknown, 350 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-08-17T17:32:49.414 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[110068]: osdmap e631: 8 total, 8 up, 8 in 2024-08-17T17:32:49.414 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' 2024-08-17T17:32:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:32:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-23", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:32:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-23", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:32:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:32:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[110068]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:32:49.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[105521]: osdmap e631: 8 total, 8 up, 8 in 2024-08-17T17:32:49.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' 2024-08-17T17:32:49.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:32:49.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-23", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:32:49.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-23", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:32:49.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:32:49.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-mon[105521]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:32:49.673 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:49 smithi179 ceph-mon[93519]: pgmap v576: 361 pgs: 6 creating+peering, 5 unknown, 350 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-08-17T17:32:49.673 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:49 smithi179 ceph-mon[93519]: osdmap e631: 8 total, 8 up, 8 in 2024-08-17T17:32:49.673 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:49 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' 2024-08-17T17:32:49.673 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:49 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:32:49.674 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:49 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-23", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:32:49.674 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:49 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-23", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:32:49.674 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:49 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:32:49.674 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:49 smithi179 ceph-mon[93519]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:32:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:32:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:32:49] "GET /metrics HTTP/1.1" 200 53218 "" "Prometheus/2.51.0" 2024-08-17T17:32:50.469 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:50 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-08-17T17:32:50.469 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:50 smithi179 ceph-mon[93519]: osdmap e632: 8 total, 8 up, 8 in 2024-08-17T17:32:50.470 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:50 smithi179 ceph-mon[93519]: osdmap e633: 8 total, 8 up, 8 in 2024-08-17T17:32:50.470 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:32:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:32:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:32:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:50 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-08-17T17:32:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:50 smithi130 ceph-mon[110068]: osdmap e632: 8 total, 8 up, 8 in 2024-08-17T17:32:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:50 smithi130 ceph-mon[110068]: osdmap e633: 8 total, 8 up, 8 in 2024-08-17T17:32:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:50 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-08-17T17:32:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:50 smithi130 ceph-mon[105521]: osdmap e632: 8 total, 8 up, 8 in 2024-08-17T17:32:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:50 smithi130 ceph-mon[105521]: osdmap e633: 8 total, 8 up, 8 in 2024-08-17T17:32:51.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:51 smithi179 ceph-mon[93519]: pgmap v579: 361 pgs: 361 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 84 KiB/s rd, 9.7 KiB/s wr, 126 op/s 2024-08-17T17:32:51.707 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:51 smithi130 ceph-mon[105521]: pgmap v579: 361 pgs: 361 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 84 KiB/s rd, 9.7 KiB/s wr, 126 op/s 2024-08-17T17:32:51.707 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:51 smithi130 ceph-mon[110068]: pgmap v579: 361 pgs: 361 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 84 KiB/s rd, 9.7 KiB/s wr, 126 op/s 2024-08-17T17:32:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:52 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:52 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:52 smithi179 ceph-mon[93519]: pgmap v581: 361 pgs: 361 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 6.4 KiB/s wr, 74 op/s 2024-08-17T17:32:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:52 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:52 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:52 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:52 smithi130 ceph-mon[110068]: pgmap v581: 361 pgs: 361 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 6.4 KiB/s wr, 74 op/s 2024-08-17T17:32:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:52 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:53.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:52 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:53.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:52 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:53.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:52 smithi130 ceph-mon[105521]: pgmap v581: 361 pgs: 361 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 6.4 KiB/s wr, 74 op/s 2024-08-17T17:32:53.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:52 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:54.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:54 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:54.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:54 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:54.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:54 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:54.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:54 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:54.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:54 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:54.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:54 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:54.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:54 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:54.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:54 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:54.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:54 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:54.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:54 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:54.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:54 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:54.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:54 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:54.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:54 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:54.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:54 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:54.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:54 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:32:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:55 smithi179 ceph-mon[93519]: pgmap v582: 361 pgs: 361 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 684 KiB/s wr, 10 op/s 2024-08-17T17:32:55.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:55 smithi130 ceph-mon[105521]: pgmap v582: 361 pgs: 361 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 684 KiB/s wr, 10 op/s 2024-08-17T17:32:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:55 smithi130 ceph-mon[110068]: pgmap v582: 361 pgs: 361 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 684 KiB/s wr, 10 op/s 2024-08-17T17:32:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:56 smithi130 ceph-mon[105521]: osdmap e634: 8 total, 8 up, 8 in 2024-08-17T17:32:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:56 smithi130 ceph-mon[110068]: osdmap e634: 8 total, 8 up, 8 in 2024-08-17T17:32:56.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:56 smithi179 ceph-mon[93519]: osdmap e634: 8 total, 8 up, 8 in 2024-08-17T17:32:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:57 smithi130 ceph-mon[105521]: pgmap v584: 361 pgs: 361 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 621 KiB/s wr, 10 op/s 2024-08-17T17:32:57.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:57.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:57 smithi130 ceph-mon[110068]: pgmap v584: 361 pgs: 361 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 621 KiB/s wr, 10 op/s 2024-08-17T17:32:57.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:57 smithi179 ceph-mon[93519]: pgmap v584: 361 pgs: 361 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 621 KiB/s wr, 10 op/s 2024-08-17T17:32:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:32:59 smithi130 ceph-mon[105521]: pgmap v585: 361 pgs: 361 active+clean; 9.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-08-17T17:32:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:32:59 smithi130 ceph-mon[110068]: pgmap v585: 361 pgs: 361 active+clean; 9.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-08-17T17:32:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:32:59 smithi179 ceph-mon[93519]: pgmap v585: 361 pgs: 361 active+clean; 9.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.1 MiB/s wr, 6 op/s 2024-08-17T17:33:00.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:32:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:32:59] "GET /metrics HTTP/1.1" 200 55684 "" "Prometheus/2.51.0" 2024-08-17T17:33:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:33:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:33:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:33:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:01 smithi130 ceph-mon[105521]: pgmap v586: 361 pgs: 361 active+clean; 9.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.4 KiB/s rd, 954 KiB/s wr, 6 op/s 2024-08-17T17:33:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:01 smithi130 ceph-mon[110068]: pgmap v586: 361 pgs: 361 active+clean; 9.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.4 KiB/s rd, 954 KiB/s wr, 6 op/s 2024-08-17T17:33:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:01 smithi179 ceph-mon[93519]: pgmap v586: 361 pgs: 361 active+clean; 9.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.4 KiB/s rd, 954 KiB/s wr, 6 op/s 2024-08-17T17:33:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:03 smithi179 ceph-mon[93519]: pgmap v587: 361 pgs: 361 active+clean; 9.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 922 KiB/s wr, 5 op/s 2024-08-17T17:33:03.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:03 smithi130 ceph-mon[105521]: pgmap v587: 361 pgs: 361 active+clean; 9.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 922 KiB/s wr, 5 op/s 2024-08-17T17:33:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:03 smithi130 ceph-mon[110068]: pgmap v587: 361 pgs: 361 active+clean; 9.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 922 KiB/s wr, 5 op/s 2024-08-17T17:33:05.618 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.RemoveFullTry (21904 ms) 2024-08-17T17:33:05.619 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.RemoveFullTryDataPool 2024-08-17T17:33:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:05 smithi179 ceph-mon[93519]: pgmap v588: 361 pgs: 361 active+clean; 15 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-08-17T17:33:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:05 smithi179 ceph-mon[93519]: pool 'test-librbd-smithi130-87148-23' is full (reached quota's max_bytes: 10 MiB) 2024-08-17T17:33:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:05 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-08-17T17:33:05.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:05 smithi179 ceph-mon[93519]: osdmap e635: 8 total, 8 up, 8 in 2024-08-17T17:33:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:05 smithi130 ceph-mon[110068]: pgmap v588: 361 pgs: 361 active+clean; 15 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-08-17T17:33:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:05 smithi130 ceph-mon[110068]: pool 'test-librbd-smithi130-87148-23' is full (reached quota's max_bytes: 10 MiB) 2024-08-17T17:33:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:05 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-08-17T17:33:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:05 smithi130 ceph-mon[110068]: osdmap e635: 8 total, 8 up, 8 in 2024-08-17T17:33:05.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:05 smithi130 ceph-mon[105521]: pgmap v588: 361 pgs: 361 active+clean; 15 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-08-17T17:33:05.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:05 smithi130 ceph-mon[105521]: pool 'test-librbd-smithi130-87148-23' is full (reached quota's max_bytes: 10 MiB) 2024-08-17T17:33:05.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:05 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-08-17T17:33:05.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:05 smithi130 ceph-mon[105521]: osdmap e635: 8 total, 8 up, 8 in 2024-08-17T17:33:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:07 smithi179 ceph-mon[93519]: pgmap v590: 361 pgs: 361 active+clean; 15 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-08-17T17:33:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:07 smithi179 ceph-mon[93519]: osdmap e636: 8 total, 8 up, 8 in 2024-08-17T17:33:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:07 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3827436431' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:07.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:07 smithi130 ceph-mon[105521]: pgmap v590: 361 pgs: 361 active+clean; 15 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-08-17T17:33:07.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:07 smithi130 ceph-mon[105521]: osdmap e636: 8 total, 8 up, 8 in 2024-08-17T17:33:07.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:07 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3827436431' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:07.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:07 smithi130 ceph-mon[110068]: pgmap v590: 361 pgs: 361 active+clean; 15 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-08-17T17:33:07.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:07 smithi130 ceph-mon[110068]: osdmap e636: 8 total, 8 up, 8 in 2024-08-17T17:33:07.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:07 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3827436431' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:08 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3827436431' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:08 smithi179 ceph-mon[93519]: osdmap e637: 8 total, 8 up, 8 in 2024-08-17T17:33:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:08 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3827436431' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:08 smithi130 ceph-mon[105521]: osdmap e637: 8 total, 8 up, 8 in 2024-08-17T17:33:08.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:08 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3827436431' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:08.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:08 smithi130 ceph-mon[110068]: osdmap e637: 8 total, 8 up, 8 in 2024-08-17T17:33:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:09 smithi130 ceph-mon[105521]: pgmap v593: 393 pgs: 11 unknown, 382 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-08-17T17:33:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:09 smithi130 ceph-mon[105521]: osdmap e638: 8 total, 8 up, 8 in 2024-08-17T17:33:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:09 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3429875021' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:09.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:09 smithi130 ceph-mon[110068]: pgmap v593: 393 pgs: 11 unknown, 382 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-08-17T17:33:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:09 smithi130 ceph-mon[110068]: osdmap e638: 8 total, 8 up, 8 in 2024-08-17T17:33:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:09 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3429875021' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:09 smithi179 ceph-mon[93519]: pgmap v593: 393 pgs: 11 unknown, 382 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-08-17T17:33:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:09 smithi179 ceph-mon[93519]: osdmap e638: 8 total, 8 up, 8 in 2024-08-17T17:33:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:09 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3429875021' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:10.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:33:09] "GET /metrics HTTP/1.1" 200 55684 "" "Prometheus/2.51.0" 2024-08-17T17:33:10.475 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:33:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:33:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:33:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:10 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3429875021' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:10 smithi179 ceph-mon[93519]: osdmap e639: 8 total, 8 up, 8 in 2024-08-17T17:33:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:10 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:10 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:10 smithi179 ceph-mon[93519]: pool 'test-librbd-smithi130-87148-23' no longer out of quota; removing NO_QUOTA flag 2024-08-17T17:33:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:10 smithi179 ceph-mon[93519]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-08-17T17:33:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:10 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T17:33:10.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:10 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:10.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:10 smithi179 ceph-mon[93519]: osdmap e640: 8 total, 8 up, 8 in 2024-08-17T17:33:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3429875021' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[105521]: osdmap e639: 8 total, 8 up, 8 in 2024-08-17T17:33:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[105521]: pool 'test-librbd-smithi130-87148-23' no longer out of quota; removing NO_QUOTA flag 2024-08-17T17:33:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[105521]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-08-17T17:33:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T17:33:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[105521]: osdmap e640: 8 total, 8 up, 8 in 2024-08-17T17:33:10.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3429875021' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:10.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[110068]: osdmap e639: 8 total, 8 up, 8 in 2024-08-17T17:33:10.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:10.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:10.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[110068]: pool 'test-librbd-smithi130-87148-23' no longer out of quota; removing NO_QUOTA flag 2024-08-17T17:33:10.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[110068]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-08-17T17:33:10.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T17:33:10.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-24","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:10.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:10 smithi130 ceph-mon[110068]: osdmap e640: 8 total, 8 up, 8 in 2024-08-17T17:33:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:11 smithi179 ceph-mon[93519]: pgmap v596: 425 pgs: 3 creating+peering, 32 unknown, 390 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-08-17T17:33:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:11 smithi179 ceph-mon[93519]: osdmap e641: 8 total, 8 up, 8 in 2024-08-17T17:33:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:11 smithi130 ceph-mon[105521]: pgmap v596: 425 pgs: 3 creating+peering, 32 unknown, 390 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-08-17T17:33:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:11 smithi130 ceph-mon[105521]: osdmap e641: 8 total, 8 up, 8 in 2024-08-17T17:33:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:11 smithi130 ceph-mon[110068]: pgmap v596: 425 pgs: 3 creating+peering, 32 unknown, 390 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-08-17T17:33:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:11 smithi130 ceph-mon[110068]: osdmap e641: 8 total, 8 up, 8 in 2024-08-17T17:33:12.427 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:33:12.830 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:12] ENGINE Bus STOPPING 2024-08-17T17:33:12.830 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:12.830 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-mon[105521]: osdmap e642: 8 total, 8 up, 8 in 2024-08-17T17:33:12.830 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' 2024-08-17T17:33:12.830 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:12.831 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:33:12.831 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:33:12.831 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:12.831 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:12.831 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-mon[110068]: osdmap e642: 8 total, 8 up, 8 in 2024-08-17T17:33:12.831 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' 2024-08-17T17:33:12.831 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:12.831 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:33:12.831 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:33:12.831 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:12 smithi179 ceph-mon[93519]: osdmap e642: 8 total, 8 up, 8 in 2024-08-17T17:33:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:12 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' 2024-08-17T17:33:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:33:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:33:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:13.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:12] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:33:13.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:12] ENGINE Bus STOPPED 2024-08-17T17:33:13.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:12] ENGINE Bus STARTING 2024-08-17T17:33:13.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:12] ENGINE Serving on http://:::9283 2024-08-17T17:33:13.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:12] ENGINE Bus STARTED 2024-08-17T17:33:13.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:13 smithi179 ceph-mon[93519]: pgmap v599: 425 pgs: 3 creating+peering, 23 unknown, 399 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 1.5 KiB/s wr, 14 op/s 2024-08-17T17:33:13.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:33:13.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:13 smithi179 ceph-mon[93519]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:33:13.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:13 smithi179 ceph-mon[93519]: osdmap e643: 8 total, 8 up, 8 in 2024-08-17T17:33:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:13 smithi130 ceph-mon[105521]: pgmap v599: 425 pgs: 3 creating+peering, 23 unknown, 399 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 1.5 KiB/s wr, 14 op/s 2024-08-17T17:33:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:33:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:13 smithi130 ceph-mon[105521]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:33:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:13 smithi130 ceph-mon[105521]: osdmap e643: 8 total, 8 up, 8 in 2024-08-17T17:33:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:13 smithi130 ceph-mon[110068]: pgmap v599: 425 pgs: 3 creating+peering, 23 unknown, 399 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 1.5 KiB/s wr, 14 op/s 2024-08-17T17:33:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:33:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:13 smithi130 ceph-mon[110068]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:33:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:13 smithi130 ceph-mon[110068]: osdmap e643: 8 total, 8 up, 8 in 2024-08-17T17:33:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:15 smithi179 ceph-mon[93519]: pgmap v602: 425 pgs: 425 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 767 B/s wr, 0 op/s 2024-08-17T17:33:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:15 smithi179 ceph-mon[93519]: osdmap e644: 8 total, 8 up, 8 in 2024-08-17T17:33:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:15 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-25", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:33:15.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:15 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-25", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:33:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:15 smithi130 ceph-mon[110068]: pgmap v602: 425 pgs: 425 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 767 B/s wr, 0 op/s 2024-08-17T17:33:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:15 smithi130 ceph-mon[110068]: osdmap e644: 8 total, 8 up, 8 in 2024-08-17T17:33:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:15 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-25", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:33:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:15 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-25", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:33:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:15 smithi130 ceph-mon[105521]: pgmap v602: 425 pgs: 425 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 767 B/s wr, 0 op/s 2024-08-17T17:33:15.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:15 smithi130 ceph-mon[105521]: osdmap e644: 8 total, 8 up, 8 in 2024-08-17T17:33:15.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:15 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-25", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:33:15.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:15 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-25", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:33:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:16 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-08-17T17:33:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:16 smithi179 ceph-mon[93519]: osdmap e645: 8 total, 8 up, 8 in 2024-08-17T17:33:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:16 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-08-17T17:33:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:16 smithi130 ceph-mon[105521]: osdmap e645: 8 total, 8 up, 8 in 2024-08-17T17:33:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:16 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-08-17T17:33:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:16 smithi130 ceph-mon[110068]: osdmap e645: 8 total, 8 up, 8 in 2024-08-17T17:33:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:17 smithi179 ceph-mon[93519]: pgmap v605: 425 pgs: 425 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 1023 B/s wr, 1 op/s 2024-08-17T17:33:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:17 smithi130 ceph-mon[105521]: pgmap v605: 425 pgs: 425 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 1023 B/s wr, 1 op/s 2024-08-17T17:33:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:17 smithi130 ceph-mon[110068]: pgmap v605: 425 pgs: 425 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 1023 B/s wr, 1 op/s 2024-08-17T17:33:19.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:19 smithi130 ceph-mon[105521]: pgmap v606: 425 pgs: 425 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 544 KiB/s wr, 8 op/s 2024-08-17T17:33:19.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:19 smithi130 ceph-mon[110068]: pgmap v606: 425 pgs: 425 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 544 KiB/s wr, 8 op/s 2024-08-17T17:33:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:19 smithi179 ceph-mon[93519]: pgmap v606: 425 pgs: 425 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 544 KiB/s wr, 8 op/s 2024-08-17T17:33:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:33:19] "GET /metrics HTTP/1.1" 200 60424 "" "Prometheus/2.51.0" 2024-08-17T17:33:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:33:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:33:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:33:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:21 smithi130 ceph-mon[105521]: pgmap v607: 425 pgs: 425 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.6 KiB/s rd, 464 KiB/s wr, 9 op/s 2024-08-17T17:33:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:21 smithi130 ceph-mon[105521]: osdmap e646: 8 total, 8 up, 8 in 2024-08-17T17:33:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:21 smithi130 ceph-mon[110068]: pgmap v607: 425 pgs: 425 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.6 KiB/s rd, 464 KiB/s wr, 9 op/s 2024-08-17T17:33:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:21 smithi130 ceph-mon[110068]: osdmap e646: 8 total, 8 up, 8 in 2024-08-17T17:33:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:21 smithi179 ceph-mon[93519]: pgmap v607: 425 pgs: 425 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.6 KiB/s rd, 464 KiB/s wr, 9 op/s 2024-08-17T17:33:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:21 smithi179 ceph-mon[93519]: osdmap e646: 8 total, 8 up, 8 in 2024-08-17T17:33:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:23 smithi130 ceph-mon[105521]: pgmap v609: 425 pgs: 425 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 403 KiB/s wr, 8 op/s 2024-08-17T17:33:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:23 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:23 smithi130 ceph-mon[110068]: pgmap v609: 425 pgs: 425 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 403 KiB/s wr, 8 op/s 2024-08-17T17:33:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:23 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:23 smithi179 ceph-mon[93519]: pgmap v609: 425 pgs: 425 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 403 KiB/s wr, 8 op/s 2024-08-17T17:33:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:23 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:25 smithi130 ceph-mon[110068]: pgmap v610: 425 pgs: 425 active+clean; 8.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.3 KiB/s rd, 949 KiB/s wr, 7 op/s 2024-08-17T17:33:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:25 smithi130 ceph-mon[105521]: pgmap v610: 425 pgs: 425 active+clean; 8.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.3 KiB/s rd, 949 KiB/s wr, 7 op/s 2024-08-17T17:33:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:25 smithi179 ceph-mon[93519]: pgmap v610: 425 pgs: 425 active+clean; 8.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.3 KiB/s rd, 949 KiB/s wr, 7 op/s 2024-08-17T17:33:26.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:26 smithi130 ceph-mon[105521]: osdmap e647: 8 total, 8 up, 8 in 2024-08-17T17:33:26.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:26 smithi130 ceph-mon[110068]: osdmap e647: 8 total, 8 up, 8 in 2024-08-17T17:33:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:26 smithi179 ceph-mon[93519]: osdmap e647: 8 total, 8 up, 8 in 2024-08-17T17:33:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:27 smithi130 ceph-mon[110068]: pgmap v612: 425 pgs: 425 active+clean; 8.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 640 KiB/s wr, 4 op/s 2024-08-17T17:33:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:27 smithi130 ceph-mon[105521]: pgmap v612: 425 pgs: 425 active+clean; 8.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 640 KiB/s wr, 4 op/s 2024-08-17T17:33:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:27 smithi179 ceph-mon[93519]: pgmap v612: 425 pgs: 425 active+clean; 8.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 640 KiB/s wr, 4 op/s 2024-08-17T17:33:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:29.730 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:29 smithi130 ceph-mon[105521]: pgmap v613: 425 pgs: 425 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 1.2 MiB/s wr, 2 op/s 2024-08-17T17:33:29.730 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:29 smithi130 ceph-mon[110068]: pgmap v613: 425 pgs: 425 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 1.2 MiB/s wr, 2 op/s 2024-08-17T17:33:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:29 smithi179 ceph-mon[93519]: pgmap v613: 425 pgs: 425 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 1.2 MiB/s wr, 2 op/s 2024-08-17T17:33:30.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:33:29] "GET /metrics HTTP/1.1" 200 60490 "" "Prometheus/2.51.0" 2024-08-17T17:33:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:30 smithi179 ceph-mon[93519]: pool 'test-librbd-smithi130-87148-25' is full (reached quota's max_bytes: 10 MiB) 2024-08-17T17:33:30.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:30 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-08-17T17:33:30.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:30 smithi179 ceph-mon[93519]: osdmap e648: 8 total, 8 up, 8 in 2024-08-17T17:33:30.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:33:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:33:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:33:30.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:30 smithi130 ceph-mon[105521]: pool 'test-librbd-smithi130-87148-25' is full (reached quota's max_bytes: 10 MiB) 2024-08-17T17:33:30.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:30 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-08-17T17:33:30.671 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:30 smithi130 ceph-mon[105521]: osdmap e648: 8 total, 8 up, 8 in 2024-08-17T17:33:30.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:30 smithi130 ceph-mon[110068]: pool 'test-librbd-smithi130-87148-25' is full (reached quota's max_bytes: 10 MiB) 2024-08-17T17:33:30.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:30 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-08-17T17:33:30.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:30 smithi130 ceph-mon[110068]: osdmap e648: 8 total, 8 up, 8 in 2024-08-17T17:33:31.428 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:33:31.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:31 smithi179 ceph-mon[93519]: pgmap v614: 425 pgs: 425 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 MiB/s wr, 2 op/s 2024-08-17T17:33:31.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-24", "field": "max_objects", "val": "6"} : dispatch 2024-08-17T17:33:31.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:31 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-24", "field": "max_objects", "val": "6"} : dispatch 2024-08-17T17:33:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:31 smithi130 ceph-mon[110068]: pgmap v614: 425 pgs: 425 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 MiB/s wr, 2 op/s 2024-08-17T17:33:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-24", "field": "max_objects", "val": "6"} : dispatch 2024-08-17T17:33:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:31 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-24", "field": "max_objects", "val": "6"} : dispatch 2024-08-17T17:33:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:31 smithi130 ceph-mon[105521]: pgmap v614: 425 pgs: 425 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 MiB/s wr, 2 op/s 2024-08-17T17:33:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-24", "field": "max_objects", "val": "6"} : dispatch 2024-08-17T17:33:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:31 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-24", "field": "max_objects", "val": "6"} : dispatch 2024-08-17T17:33:32.446 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:33:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:32 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-24", "field": "max_objects", "val": "6"}]': finished 2024-08-17T17:33:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:32 smithi179 ceph-mon[93519]: osdmap e649: 8 total, 8 up, 8 in 2024-08-17T17:33:32.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:32 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-24", "field": "max_objects", "val": "6"}]': finished 2024-08-17T17:33:32.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:32 smithi130 ceph-mon[110068]: osdmap e649: 8 total, 8 up, 8 in 2024-08-17T17:33:32.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:32 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-24", "field": "max_objects", "val": "6"}]': finished 2024-08-17T17:33:32.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:32 smithi130 ceph-mon[105521]: osdmap e649: 8 total, 8 up, 8 in 2024-08-17T17:33:33.457 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:33:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:33 smithi179 ceph-mon[93519]: pgmap v617: 425 pgs: 425 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 921 B/s rd, 768 KiB/s wr, 2 op/s 2024-08-17T17:33:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:33 smithi130 ceph-mon[105521]: pgmap v617: 425 pgs: 425 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 921 B/s rd, 768 KiB/s wr, 2 op/s 2024-08-17T17:33:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:33 smithi130 ceph-mon[110068]: pgmap v617: 425 pgs: 425 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 921 B/s rd, 768 KiB/s wr, 2 op/s 2024-08-17T17:33:34.467 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:33:35.479 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:33:35.548 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.RemoveFullTryDataPool (29930 ms) 2024-08-17T17:33:35.549 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.RemoveFullTryNamespace 2024-08-17T17:33:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:35 smithi179 ceph-mon[93519]: pgmap v618: 425 pgs: 425 active+clean; 16 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 897 KiB/s wr, 4 op/s 2024-08-17T17:33:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:35 smithi179 ceph-mon[93519]: pool 'test-librbd-smithi130-87148-24' is full (reached quota's max_objects: 6) 2024-08-17T17:33:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:35 smithi179 ceph-mon[93519]: Health check update: 2 pool(s) full (POOL_FULL) 2024-08-17T17:33:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:35 smithi179 ceph-mon[93519]: osdmap e650: 8 total, 8 up, 8 in 2024-08-17T17:33:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:35 smithi130 ceph-mon[105521]: pgmap v618: 425 pgs: 425 active+clean; 16 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 897 KiB/s wr, 4 op/s 2024-08-17T17:33:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:35 smithi130 ceph-mon[105521]: pool 'test-librbd-smithi130-87148-24' is full (reached quota's max_objects: 6) 2024-08-17T17:33:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:35 smithi130 ceph-mon[105521]: Health check update: 2 pool(s) full (POOL_FULL) 2024-08-17T17:33:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:35 smithi130 ceph-mon[105521]: osdmap e650: 8 total, 8 up, 8 in 2024-08-17T17:33:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:35 smithi130 ceph-mon[110068]: pgmap v618: 425 pgs: 425 active+clean; 16 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.9 KiB/s rd, 897 KiB/s wr, 4 op/s 2024-08-17T17:33:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:35 smithi130 ceph-mon[110068]: pool 'test-librbd-smithi130-87148-24' is full (reached quota's max_objects: 6) 2024-08-17T17:33:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:35 smithi130 ceph-mon[110068]: Health check update: 2 pool(s) full (POOL_FULL) 2024-08-17T17:33:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:35 smithi130 ceph-mon[110068]: osdmap e650: 8 total, 8 up, 8 in 2024-08-17T17:33:37.498 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:33:37.836 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:37 smithi130 ceph-mon[105521]: pgmap v620: 425 pgs: 425 active+clean; 16 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 342 KiB/s wr, 4 op/s 2024-08-17T17:33:37.837 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:37 smithi130 ceph-mon[105521]: osdmap e651: 8 total, 8 up, 8 in 2024-08-17T17:33:37.837 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:37 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2732646093' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:37.837 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:37] ENGINE Bus STOPPING 2024-08-17T17:33:37.837 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:37 smithi130 ceph-mon[110068]: pgmap v620: 425 pgs: 425 active+clean; 16 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 342 KiB/s wr, 4 op/s 2024-08-17T17:33:37.838 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:37 smithi130 ceph-mon[110068]: osdmap e651: 8 total, 8 up, 8 in 2024-08-17T17:33:37.838 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:37 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2732646093' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:37.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:37 smithi179 ceph-mon[93519]: pgmap v620: 425 pgs: 425 active+clean; 16 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 342 KiB/s wr, 4 op/s 2024-08-17T17:33:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:37 smithi179 ceph-mon[93519]: osdmap e651: 8 total, 8 up, 8 in 2024-08-17T17:33:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:37 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2732646093' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:38.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:37] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:33:38.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:37] ENGINE Bus STOPPED 2024-08-17T17:33:38.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:37] ENGINE Bus STARTING 2024-08-17T17:33:38.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:37] ENGINE Serving on http://:::9283 2024-08-17T17:33:38.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:37] ENGINE Bus STARTED 2024-08-17T17:33:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:38 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2732646093' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:38 smithi179 ceph-mon[93519]: osdmap e652: 8 total, 8 up, 8 in 2024-08-17T17:33:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:38 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' 2024-08-17T17:33:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:38 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:38 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:33:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:38 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:33:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:38 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:38 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:33:38.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:38 smithi179 ceph-mon[93519]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:33:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2732646093' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[105521]: osdmap e652: 8 total, 8 up, 8 in 2024-08-17T17:33:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' 2024-08-17T17:33:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:33:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:33:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:33:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[105521]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:33:38.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2732646093' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:38.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[110068]: osdmap e652: 8 total, 8 up, 8 in 2024-08-17T17:33:38.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' 2024-08-17T17:33:38.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:38.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:33:38.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:33:38.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:38.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:33:38.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:38 smithi130 ceph-mon[110068]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:33:39.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:39 smithi130 ceph-mon[110068]: pgmap v623: 457 pgs: 11 unknown, 446 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 345 KiB/s wr, 30 op/s 2024-08-17T17:33:39.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:39 smithi130 ceph-mon[110068]: osdmap e653: 8 total, 8 up, 8 in 2024-08-17T17:33:39.821 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:39 smithi130 ceph-mon[105521]: pgmap v623: 457 pgs: 11 unknown, 446 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 345 KiB/s wr, 30 op/s 2024-08-17T17:33:39.821 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:39 smithi130 ceph-mon[105521]: osdmap e653: 8 total, 8 up, 8 in 2024-08-17T17:33:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:39 smithi179 ceph-mon[93519]: pgmap v623: 457 pgs: 11 unknown, 446 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 345 KiB/s wr, 30 op/s 2024-08-17T17:33:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:39 smithi179 ceph-mon[93519]: osdmap e653: 8 total, 8 up, 8 in 2024-08-17T17:33:40.158 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:33:39] "GET /metrics HTTP/1.1" 200 60490 "" "Prometheus/2.51.0" 2024-08-17T17:33:40.502 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:33:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:33:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:33:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:40 smithi179 ceph-mon[93519]: osdmap e654: 8 total, 8 up, 8 in 2024-08-17T17:33:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:40 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-26", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:33:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:40 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-26", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:33:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:40 smithi179 ceph-mon[93519]: pool 'test-librbd-smithi130-87148-25' no longer out of quota; removing NO_QUOTA flag 2024-08-17T17:33:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:40 smithi179 ceph-mon[93519]: Health check update: 1 pool(s) full (POOL_FULL) 2024-08-17T17:33:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:40 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-26", "field": "max_bytes", "val": "10485760"}]': finished 2024-08-17T17:33:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:40 smithi179 ceph-mon[93519]: osdmap e655: 8 total, 8 up, 8 in 2024-08-17T17:33:40.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:40 smithi130 ceph-mon[105521]: osdmap e654: 8 total, 8 up, 8 in 2024-08-17T17:33:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:40 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-26", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:33:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:40 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-26", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:33:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:40 smithi130 ceph-mon[105521]: pool 'test-librbd-smithi130-87148-25' no longer out of quota; removing NO_QUOTA flag 2024-08-17T17:33:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:40 smithi130 ceph-mon[105521]: Health check update: 1 pool(s) full (POOL_FULL) 2024-08-17T17:33:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:40 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-26", "field": "max_bytes", "val": "10485760"}]': finished 2024-08-17T17:33:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:40 smithi130 ceph-mon[105521]: osdmap e655: 8 total, 8 up, 8 in 2024-08-17T17:33:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:40 smithi130 ceph-mon[110068]: osdmap e654: 8 total, 8 up, 8 in 2024-08-17T17:33:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:40 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-26", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:33:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:40 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-26", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:33:40.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:40 smithi130 ceph-mon[110068]: pool 'test-librbd-smithi130-87148-25' no longer out of quota; removing NO_QUOTA flag 2024-08-17T17:33:40.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:40 smithi130 ceph-mon[110068]: Health check update: 1 pool(s) full (POOL_FULL) 2024-08-17T17:33:40.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:40 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-26", "field": "max_bytes", "val": "10485760"}]': finished 2024-08-17T17:33:40.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:40 smithi130 ceph-mon[110068]: osdmap e655: 8 total, 8 up, 8 in 2024-08-17T17:33:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:41 smithi179 ceph-mon[93519]: pgmap v626: 457 pgs: 4 unknown, 453 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.7 KiB/s wr, 47 op/s 2024-08-17T17:33:41.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:41 smithi130 ceph-mon[105521]: pgmap v626: 457 pgs: 4 unknown, 453 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.7 KiB/s wr, 47 op/s 2024-08-17T17:33:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:41 smithi130 ceph-mon[110068]: pgmap v626: 457 pgs: 4 unknown, 453 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.7 KiB/s wr, 47 op/s 2024-08-17T17:33:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:43 smithi179 ceph-mon[93519]: pgmap v628: 457 pgs: 457 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.7 KiB/s wr, 20 op/s 2024-08-17T17:33:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:43 smithi130 ceph-mon[105521]: pgmap v628: 457 pgs: 457 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.7 KiB/s wr, 20 op/s 2024-08-17T17:33:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:43 smithi130 ceph-mon[110068]: pgmap v628: 457 pgs: 457 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.7 KiB/s wr, 20 op/s 2024-08-17T17:33:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:45.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:45 smithi179 ceph-mon[93519]: pgmap v629: 457 pgs: 457 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 514 KiB/s wr, 16 op/s 2024-08-17T17:33:45.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:45 smithi130 ceph-mon[105521]: pgmap v629: 457 pgs: 457 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 514 KiB/s wr, 16 op/s 2024-08-17T17:33:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:45 smithi130 ceph-mon[110068]: pgmap v629: 457 pgs: 457 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 514 KiB/s wr, 16 op/s 2024-08-17T17:33:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:46 smithi179 ceph-mon[93519]: osdmap e656: 8 total, 8 up, 8 in 2024-08-17T17:33:46.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:46 smithi130 ceph-mon[105521]: osdmap e656: 8 total, 8 up, 8 in 2024-08-17T17:33:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:46 smithi130 ceph-mon[110068]: osdmap e656: 8 total, 8 up, 8 in 2024-08-17T17:33:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:47 smithi179 ceph-mon[93519]: pgmap v631: 457 pgs: 457 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 467 KiB/s wr, 14 op/s 2024-08-17T17:33:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:47 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:47 smithi130 ceph-mon[105521]: pgmap v631: 457 pgs: 457 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 467 KiB/s wr, 14 op/s 2024-08-17T17:33:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:47 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:47 smithi130 ceph-mon[110068]: pgmap v631: 457 pgs: 457 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 467 KiB/s wr, 14 op/s 2024-08-17T17:33:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:47 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:33:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:49 smithi130 ceph-mon[110068]: pgmap v632: 457 pgs: 457 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.9 KiB/s rd, 769 KiB/s wr, 8 op/s 2024-08-17T17:33:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:49 smithi130 ceph-mon[105521]: pgmap v632: 457 pgs: 457 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.9 KiB/s rd, 769 KiB/s wr, 8 op/s 2024-08-17T17:33:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:49 smithi179 ceph-mon[93519]: pgmap v632: 457 pgs: 457 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.9 KiB/s rd, 769 KiB/s wr, 8 op/s 2024-08-17T17:33:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:33:49] "GET /metrics HTTP/1.1" 200 62950 "" "Prometheus/2.51.0" 2024-08-17T17:33:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:33:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:33:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:33:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:51 smithi179 ceph-mon[93519]: pgmap v633: 457 pgs: 457 active+clean; 8.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.0 KiB/s rd, 848 KiB/s wr, 7 op/s 2024-08-17T17:33:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:51 smithi130 ceph-mon[105521]: pgmap v633: 457 pgs: 457 active+clean; 8.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.0 KiB/s rd, 848 KiB/s wr, 7 op/s 2024-08-17T17:33:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:51 smithi130 ceph-mon[110068]: pgmap v633: 457 pgs: 457 active+clean; 8.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.0 KiB/s rd, 848 KiB/s wr, 7 op/s 2024-08-17T17:33:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:53 smithi179 ceph-mon[93519]: pgmap v634: 457 pgs: 457 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.0 MiB/s wr, 7 op/s 2024-08-17T17:33:53.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:53 smithi130 ceph-mon[105521]: pgmap v634: 457 pgs: 457 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.0 MiB/s wr, 7 op/s 2024-08-17T17:33:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:53 smithi130 ceph-mon[110068]: pgmap v634: 457 pgs: 457 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.0 MiB/s wr, 7 op/s 2024-08-17T17:33:55.593 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.RemoveFullTryNamespace (20045 ms) 2024-08-17T17:33:55.593 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.RemoveFullTryNamespaceDataPool 2024-08-17T17:33:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:55 smithi179 ceph-mon[93519]: pgmap v635: 457 pgs: 457 active+clean; 12 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 819 KiB/s wr, 3 op/s 2024-08-17T17:33:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:55 smithi179 ceph-mon[93519]: pool 'test-librbd-smithi130-87148-26' is full (reached quota's max_bytes: 10 MiB) 2024-08-17T17:33:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:55 smithi179 ceph-mon[93519]: Health check update: 2 pool(s) full (POOL_FULL) 2024-08-17T17:33:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:55 smithi179 ceph-mon[93519]: osdmap e657: 8 total, 8 up, 8 in 2024-08-17T17:33:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:55 smithi130 ceph-mon[105521]: pgmap v635: 457 pgs: 457 active+clean; 12 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 819 KiB/s wr, 3 op/s 2024-08-17T17:33:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:55 smithi130 ceph-mon[105521]: pool 'test-librbd-smithi130-87148-26' is full (reached quota's max_bytes: 10 MiB) 2024-08-17T17:33:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:55 smithi130 ceph-mon[105521]: Health check update: 2 pool(s) full (POOL_FULL) 2024-08-17T17:33:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:55 smithi130 ceph-mon[105521]: osdmap e657: 8 total, 8 up, 8 in 2024-08-17T17:33:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:55 smithi130 ceph-mon[110068]: pgmap v635: 457 pgs: 457 active+clean; 12 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 819 KiB/s wr, 3 op/s 2024-08-17T17:33:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:55 smithi130 ceph-mon[110068]: pool 'test-librbd-smithi130-87148-26' is full (reached quota's max_bytes: 10 MiB) 2024-08-17T17:33:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:55 smithi130 ceph-mon[110068]: Health check update: 2 pool(s) full (POOL_FULL) 2024-08-17T17:33:55.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:55 smithi130 ceph-mon[110068]: osdmap e657: 8 total, 8 up, 8 in 2024-08-17T17:33:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:57 smithi179 ceph-mon[93519]: pgmap v637: 457 pgs: 457 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-08-17T17:33:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:57 smithi179 ceph-mon[93519]: osdmap e658: 8 total, 8 up, 8 in 2024-08-17T17:33:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:57 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/740380133' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:57 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:57 smithi130 ceph-mon[105521]: pgmap v637: 457 pgs: 457 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-08-17T17:33:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:57 smithi130 ceph-mon[105521]: osdmap e658: 8 total, 8 up, 8 in 2024-08-17T17:33:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:57 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/740380133' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:57 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:57 smithi130 ceph-mon[110068]: pgmap v637: 457 pgs: 457 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-08-17T17:33:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:57 smithi130 ceph-mon[110068]: osdmap e658: 8 total, 8 up, 8 in 2024-08-17T17:33:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:57 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/740380133' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:57 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:57.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:58 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:58 smithi179 ceph-mon[93519]: osdmap e659: 8 total, 8 up, 8 in 2024-08-17T17:33:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:58 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:58 smithi130 ceph-mon[105521]: osdmap e659: 8 total, 8 up, 8 in 2024-08-17T17:33:58.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:58 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:33:58.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:58 smithi130 ceph-mon[110068]: osdmap e659: 8 total, 8 up, 8 in 2024-08-17T17:33:59.513 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:33:59.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-mon[110068]: pgmap v640: 489 pgs: 13 unknown, 476 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 854 KiB/s wr, 27 op/s 2024-08-17T17:33:59.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-mon[110068]: osdmap e660: 8 total, 8 up, 8 in 2024-08-17T17:33:59.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/859782416' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:59.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:59.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-mon[105521]: pgmap v640: 489 pgs: 13 unknown, 476 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 854 KiB/s wr, 27 op/s 2024-08-17T17:33:59.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-mon[105521]: osdmap e660: 8 total, 8 up, 8 in 2024-08-17T17:33:59.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/859782416' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:59.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:59.819 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:59] ENGINE Bus STOPPING 2024-08-17T17:33:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:59 smithi179 ceph-mon[93519]: pgmap v640: 489 pgs: 13 unknown, 476 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 854 KiB/s wr, 27 op/s 2024-08-17T17:33:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:59 smithi179 ceph-mon[93519]: osdmap e660: 8 total, 8 up, 8 in 2024-08-17T17:33:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:59 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/859782416' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:33:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:33:59 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:34:00.158 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:33:59] "GET /metrics HTTP/1.1" 200 65341 "" "Prometheus/2.51.0" 2024-08-17T17:34:00.158 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:34:00.158 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:59] ENGINE Bus STOPPED 2024-08-17T17:34:00.158 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:59] ENGINE Bus STARTING 2024-08-17T17:34:00.158 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:59] ENGINE Serving on http://:::9283 2024-08-17T17:34:00.158 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:33:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:33:59] ENGINE Bus STARTED 2024-08-17T17:34:00.476 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:34:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:34:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:34:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:00 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:34:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:00 smithi179 ceph-mon[93519]: osdmap e661: 8 total, 8 up, 8 in 2024-08-17T17:34:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:00 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' 2024-08-17T17:34:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:00 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:34:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:00 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:34:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:00 smithi179 ceph-mon[93519]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:34:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:00 smithi179 ceph-mon[93519]: pool 'test-librbd-smithi130-87148-26' no longer out of quota; removing NO_QUOTA flag 2024-08-17T17:34:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:00 smithi179 ceph-mon[93519]: Health check update: 1 pool(s) full (POOL_FULL) 2024-08-17T17:34:00.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:00 smithi179 ceph-mon[93519]: osdmap e662: 8 total, 8 up, 8 in 2024-08-17T17:34:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:34:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[105521]: osdmap e661: 8 total, 8 up, 8 in 2024-08-17T17:34:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' 2024-08-17T17:34:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:34:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:34:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[105521]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:34:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[105521]: pool 'test-librbd-smithi130-87148-26' no longer out of quota; removing NO_QUOTA flag 2024-08-17T17:34:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[105521]: Health check update: 1 pool(s) full (POOL_FULL) 2024-08-17T17:34:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[105521]: osdmap e662: 8 total, 8 up, 8 in 2024-08-17T17:34:00.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:34:00.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[110068]: osdmap e661: 8 total, 8 up, 8 in 2024-08-17T17:34:00.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' 2024-08-17T17:34:00.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:34:00.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:34:00.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[110068]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:34:00.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[110068]: pool 'test-librbd-smithi130-87148-26' no longer out of quota; removing NO_QUOTA flag 2024-08-17T17:34:00.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[110068]: Health check update: 1 pool(s) full (POOL_FULL) 2024-08-17T17:34:00.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:00 smithi130 ceph-mon[110068]: osdmap e662: 8 total, 8 up, 8 in 2024-08-17T17:34:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:01 smithi179 ceph-mon[93519]: pgmap v643: 521 pgs: 4 creating+peering, 30 unknown, 487 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 514 KiB/s wr, 40 op/s 2024-08-17T17:34:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:01 smithi179 ceph-mon[93519]: osdmap e663: 8 total, 8 up, 8 in 2024-08-17T17:34:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:01 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-28", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:34:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:01 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-28", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:34:01.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:01 smithi130 ceph-mon[105521]: pgmap v643: 521 pgs: 4 creating+peering, 30 unknown, 487 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 514 KiB/s wr, 40 op/s 2024-08-17T17:34:01.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:01 smithi130 ceph-mon[105521]: osdmap e663: 8 total, 8 up, 8 in 2024-08-17T17:34:01.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:01 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-28", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:34:01.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:01 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-28", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:34:01.883 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:01 smithi130 ceph-mon[110068]: pgmap v643: 521 pgs: 4 creating+peering, 30 unknown, 487 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 514 KiB/s wr, 40 op/s 2024-08-17T17:34:01.883 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:01 smithi130 ceph-mon[110068]: osdmap e663: 8 total, 8 up, 8 in 2024-08-17T17:34:01.883 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:01 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-28", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:34:01.883 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:01 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-28", "field": "max_bytes", "val": "10485760"} : dispatch 2024-08-17T17:34:03.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:03 smithi179 ceph-mon[93519]: pgmap v646: 521 pgs: 4 creating+peering, 19 unknown, 498 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-08-17T17:34:03.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:03 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-28", "field": "max_bytes", "val": "10485760"}]': finished 2024-08-17T17:34:03.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:03 smithi179 ceph-mon[93519]: osdmap e664: 8 total, 8 up, 8 in 2024-08-17T17:34:03.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:03 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:03.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:03 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:03 smithi130 ceph-mon[105521]: pgmap v646: 521 pgs: 4 creating+peering, 19 unknown, 498 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-08-17T17:34:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:03 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-28", "field": "max_bytes", "val": "10485760"}]': finished 2024-08-17T17:34:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:03 smithi130 ceph-mon[105521]: osdmap e664: 8 total, 8 up, 8 in 2024-08-17T17:34:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:03 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:03 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:03 smithi130 ceph-mon[110068]: pgmap v646: 521 pgs: 4 creating+peering, 19 unknown, 498 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-08-17T17:34:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:03 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-28", "field": "max_bytes", "val": "10485760"}]': finished 2024-08-17T17:34:03.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:03 smithi130 ceph-mon[110068]: osdmap e664: 8 total, 8 up, 8 in 2024-08-17T17:34:03.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:03 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:03.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:03 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:05.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:04 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:05.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:04 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:05.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:04 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:34:05.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:04 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:34:05.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:04 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:05.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:04 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:05.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:04 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:05.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:04 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:34:05.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:04 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:34:05.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:04 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:05.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:04 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:05.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:04 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:05.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:04 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:34:05.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:04 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:34:05.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:04 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:05 smithi179 ceph-mon[93519]: pgmap v648: 521 pgs: 521 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.0 KiB/s wr, 14 op/s 2024-08-17T17:34:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:05 smithi130 ceph-mon[105521]: pgmap v648: 521 pgs: 521 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.0 KiB/s wr, 14 op/s 2024-08-17T17:34:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:05 smithi130 ceph-mon[110068]: pgmap v648: 521 pgs: 521 active+clean; 584 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.0 KiB/s wr, 14 op/s 2024-08-17T17:34:08.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:07 smithi179 ceph-mon[93519]: pgmap v649: 521 pgs: 521 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.5 KiB/s rd, 513 KiB/s wr, 9 op/s 2024-08-17T17:34:08.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:07 smithi130 ceph-mon[105521]: pgmap v649: 521 pgs: 521 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.5 KiB/s rd, 513 KiB/s wr, 9 op/s 2024-08-17T17:34:08.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:07 smithi130 ceph-mon[110068]: pgmap v649: 521 pgs: 521 active+clean; 3.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.5 KiB/s rd, 513 KiB/s wr, 9 op/s 2024-08-17T17:34:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:09 smithi179 ceph-mon[93519]: pgmap v650: 521 pgs: 521 active+clean; 5.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.1 KiB/s rd, 669 KiB/s wr, 9 op/s 2024-08-17T17:34:10.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:09 smithi130 ceph-mon[105521]: pgmap v650: 521 pgs: 521 active+clean; 5.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.1 KiB/s rd, 669 KiB/s wr, 9 op/s 2024-08-17T17:34:10.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:09 smithi130 ceph-mon[110068]: pgmap v650: 521 pgs: 521 active+clean; 5.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.1 KiB/s rd, 669 KiB/s wr, 9 op/s 2024-08-17T17:34:10.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:34:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:34:09] "GET /metrics HTTP/1.1" 200 65341 "" "Prometheus/2.51.0" 2024-08-17T17:34:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:34:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:34:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:34:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:11 smithi179 ceph-mon[93519]: pgmap v651: 521 pgs: 521 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.5 KiB/s rd, 710 KiB/s wr, 8 op/s 2024-08-17T17:34:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:11 smithi179 ceph-mon[93519]: osdmap e665: 8 total, 8 up, 8 in 2024-08-17T17:34:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:11 smithi130 ceph-mon[105521]: pgmap v651: 521 pgs: 521 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.5 KiB/s rd, 710 KiB/s wr, 8 op/s 2024-08-17T17:34:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:11 smithi130 ceph-mon[105521]: osdmap e665: 8 total, 8 up, 8 in 2024-08-17T17:34:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:11 smithi130 ceph-mon[110068]: pgmap v651: 521 pgs: 521 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.5 KiB/s rd, 710 KiB/s wr, 8 op/s 2024-08-17T17:34:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:11 smithi130 ceph-mon[110068]: osdmap e665: 8 total, 8 up, 8 in 2024-08-17T17:34:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]} : dispatch 2024-08-17T17:34:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]} : dispatch 2024-08-17T17:34:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]} : dispatch 2024-08-17T17:34:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]} : dispatch 2024-08-17T17:34:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]} : dispatch 2024-08-17T17:34:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]} : dispatch 2024-08-17T17:34:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1b", "id": [0, 2]} : dispatch 2024-08-17T17:34:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]} : dispatch 2024-08-17T17:34:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1b", "id": [0, 2]} : dispatch 2024-08-17T17:34:12.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]} : dispatch 2024-08-17T17:34:12.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]} : dispatch 2024-08-17T17:34:12.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]} : dispatch 2024-08-17T17:34:12.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.13", "id": [6, 0]} : dispatch 2024-08-17T17:34:12.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.13", "id": [6, 0]} : dispatch 2024-08-17T17:34:12.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]} : dispatch 2024-08-17T17:34:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]} : dispatch 2024-08-17T17:34:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]} : dispatch 2024-08-17T17:34:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]} : dispatch 2024-08-17T17:34:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]} : dispatch 2024-08-17T17:34:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]} : dispatch 2024-08-17T17:34:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1b", "id": [0, 2]} : dispatch 2024-08-17T17:34:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]} : dispatch 2024-08-17T17:34:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1b", "id": [0, 2]} : dispatch 2024-08-17T17:34:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]} : dispatch 2024-08-17T17:34:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]} : dispatch 2024-08-17T17:34:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]} : dispatch 2024-08-17T17:34:12.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.13", "id": [6, 0]} : dispatch 2024-08-17T17:34:12.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.13", "id": [6, 0]} : dispatch 2024-08-17T17:34:12.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:12.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]} : dispatch 2024-08-17T17:34:12.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]} : dispatch 2024-08-17T17:34:12.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]} : dispatch 2024-08-17T17:34:12.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]} : dispatch 2024-08-17T17:34:12.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]} : dispatch 2024-08-17T17:34:12.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]} : dispatch 2024-08-17T17:34:12.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1b", "id": [0, 2]} : dispatch 2024-08-17T17:34:12.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]} : dispatch 2024-08-17T17:34:12.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]} : dispatch 2024-08-17T17:34:12.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1b", "id": [0, 2]} : dispatch 2024-08-17T17:34:12.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]} : dispatch 2024-08-17T17:34:12.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]} : dispatch 2024-08-17T17:34:12.921 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]} : dispatch 2024-08-17T17:34:12.921 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.13", "id": [6, 0]} : dispatch 2024-08-17T17:34:12.921 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.13", "id": [6, 0]} : dispatch 2024-08-17T17:34:12.921 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:13.745 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:13 smithi179 ceph-mon[93519]: pgmap v653: 521 pgs: 521 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.0 KiB/s rd, 636 KiB/s wr, 7 op/s 2024-08-17T17:34:13.745 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:13.745 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-08-17T17:34:13.745 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]}]': finished 2024-08-17T17:34:13.745 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 0]}]': finished 2024-08-17T17:34:13.745 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]': finished 2024-08-17T17:34:13.745 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]}]': finished 2024-08-17T17:34:13.745 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]}]': finished 2024-08-17T17:34:13.745 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1b", "id": [0, 2]}]': finished 2024-08-17T17:34:13.745 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]': finished 2024-08-17T17:34:13.745 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]}]': finished 2024-08-17T17:34:13.746 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.13", "id": [6, 0]}]': finished 2024-08-17T17:34:13.746 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:13 smithi179 ceph-mon[93519]: osdmap e666: 8 total, 8 up, 8 in 2024-08-17T17:34:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[105521]: pgmap v653: 521 pgs: 521 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.0 KiB/s rd, 636 KiB/s wr, 7 op/s 2024-08-17T17:34:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-08-17T17:34:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]}]': finished 2024-08-17T17:34:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 0]}]': finished 2024-08-17T17:34:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]': finished 2024-08-17T17:34:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]}]': finished 2024-08-17T17:34:13.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]}]': finished 2024-08-17T17:34:13.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1b", "id": [0, 2]}]': finished 2024-08-17T17:34:13.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]': finished 2024-08-17T17:34:13.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]}]': finished 2024-08-17T17:34:13.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.13", "id": [6, 0]}]': finished 2024-08-17T17:34:13.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[105521]: osdmap e666: 8 total, 8 up, 8 in 2024-08-17T17:34:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[110068]: pgmap v653: 521 pgs: 521 active+clean; 6.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.0 KiB/s rd, 636 KiB/s wr, 7 op/s 2024-08-17T17:34:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.9", "id": [7, 4]}]': finished 2024-08-17T17:34:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]}]': finished 2024-08-17T17:34:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1b", "id": [7, 0]}]': finished 2024-08-17T17:34:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1c", "id": [7, 5]}]': finished 2024-08-17T17:34:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.3", "id": [7, 0]}]': finished 2024-08-17T17:34:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.12", "id": [7, 0]}]': finished 2024-08-17T17:34:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1b", "id": [0, 2]}]': finished 2024-08-17T17:34:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.1f", "id": [0, 1]}]': finished 2024-08-17T17:34:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.9", "id": [6, 4]}]': finished 2024-08-17T17:34:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.13", "id": [6, 0]}]': finished 2024-08-17T17:34:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:13 smithi130 ceph-mon[110068]: osdmap e666: 8 total, 8 up, 8 in 2024-08-17T17:34:14.765 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:14 smithi179 ceph-mon[93519]: osdmap e667: 8 total, 8 up, 8 in 2024-08-17T17:34:14.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:14 smithi130 ceph-mon[105521]: osdmap e667: 8 total, 8 up, 8 in 2024-08-17T17:34:14.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:14 smithi130 ceph-mon[110068]: osdmap e667: 8 total, 8 up, 8 in 2024-08-17T17:34:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:15 smithi179 ceph-mon[93519]: pgmap v656: 521 pgs: 7 peering, 514 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 1.2 MiB/s wr, 1 op/s 2024-08-17T17:34:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:15 smithi179 ceph-mon[93519]: pool 'test-librbd-smithi130-87148-28' is full (reached quota's max_bytes: 10 MiB) 2024-08-17T17:34:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:15 smithi179 ceph-mon[93519]: Health check update: 2 pool(s) full (POOL_FULL) 2024-08-17T17:34:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:15 smithi179 ceph-mon[93519]: osdmap e668: 8 total, 8 up, 8 in 2024-08-17T17:34:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:15 smithi130 ceph-mon[105521]: pgmap v656: 521 pgs: 7 peering, 514 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 1.2 MiB/s wr, 1 op/s 2024-08-17T17:34:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:15 smithi130 ceph-mon[105521]: pool 'test-librbd-smithi130-87148-28' is full (reached quota's max_bytes: 10 MiB) 2024-08-17T17:34:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:15 smithi130 ceph-mon[105521]: Health check update: 2 pool(s) full (POOL_FULL) 2024-08-17T17:34:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:15 smithi130 ceph-mon[105521]: osdmap e668: 8 total, 8 up, 8 in 2024-08-17T17:34:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:15 smithi130 ceph-mon[110068]: pgmap v656: 521 pgs: 7 peering, 514 active+clean; 13 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 852 B/s rd, 1.2 MiB/s wr, 1 op/s 2024-08-17T17:34:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:15 smithi130 ceph-mon[110068]: pool 'test-librbd-smithi130-87148-28' is full (reached quota's max_bytes: 10 MiB) 2024-08-17T17:34:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:15 smithi130 ceph-mon[110068]: Health check update: 2 pool(s) full (POOL_FULL) 2024-08-17T17:34:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:15 smithi130 ceph-mon[110068]: osdmap e668: 8 total, 8 up, 8 in 2024-08-17T17:34:16.464 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:16 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-27", "field": "max_objects", "val": "7"} : dispatch 2024-08-17T17:34:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:16 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-27", "field": "max_objects", "val": "7"} : dispatch 2024-08-17T17:34:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:16 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-27", "field": "max_objects", "val": "7"} : dispatch 2024-08-17T17:34:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:16 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-27", "field": "max_objects", "val": "7"} : dispatch 2024-08-17T17:34:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:16 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-27", "field": "max_objects", "val": "7"} : dispatch 2024-08-17T17:34:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:16 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-27", "field": "max_objects", "val": "7"} : dispatch 2024-08-17T17:34:17.483 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:17 smithi179 ceph-mon[93519]: pgmap v658: 521 pgs: 7 peering, 514 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 901 B/s rd, 1.2 MiB/s wr, 1 op/s 2024-08-17T17:34:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:17 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-27", "field": "max_objects", "val": "7"}]': finished 2024-08-17T17:34:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:17 smithi179 ceph-mon[93519]: osdmap e669: 8 total, 8 up, 8 in 2024-08-17T17:34:17.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:17 smithi130 ceph-mon[105521]: pgmap v658: 521 pgs: 7 peering, 514 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 901 B/s rd, 1.2 MiB/s wr, 1 op/s 2024-08-17T17:34:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:17 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-27", "field": "max_objects", "val": "7"}]': finished 2024-08-17T17:34:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:17 smithi130 ceph-mon[105521]: osdmap e669: 8 total, 8 up, 8 in 2024-08-17T17:34:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:17 smithi130 ceph-mon[110068]: pgmap v658: 521 pgs: 7 peering, 514 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 901 B/s rd, 1.2 MiB/s wr, 1 op/s 2024-08-17T17:34:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:17 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi130-87148-27", "field": "max_objects", "val": "7"}]': finished 2024-08-17T17:34:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:17 smithi130 ceph-mon[110068]: osdmap e669: 8 total, 8 up, 8 in 2024-08-17T17:34:18.493 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:19.504 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:19.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:19 smithi130 ceph-mon[105521]: pgmap v660: 521 pgs: 521 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.9 KiB/s rd, 1.2 MiB/s wr, 8 op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:34:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:19 smithi130 ceph-mon[110068]: pgmap v660: 521 pgs: 521 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.9 KiB/s rd, 1.2 MiB/s wr, 8 op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:34:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:19 smithi179 ceph-mon[93519]: pgmap v660: 521 pgs: 521 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.9 KiB/s rd, 1.2 MiB/s wr, 8 op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:34:20.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:34:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:34:19] "GET /metrics HTTP/1.1" 200 67786 "" "Prometheus/2.51.0" 2024-08-17T17:34:20.472 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:34:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:34:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:34:20.515 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:20.585 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.RemoveFullTryNamespaceDataPool (24992 ms) 2024-08-17T17:34:20.585 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestIO 2024-08-17T17:34:20.585 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:20 smithi179 ceph-mon[93519]: pool 'test-librbd-smithi130-87148-27' is full (reached quota's max_objects: 7) 2024-08-17T17:34:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:20 smithi179 ceph-mon[93519]: Health check update: 3 pool(s) full (POOL_FULL) 2024-08-17T17:34:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:20 smithi179 ceph-mon[93519]: osdmap e670: 8 total, 8 up, 8 in 2024-08-17T17:34:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:20 smithi130 ceph-mon[105521]: pool 'test-librbd-smithi130-87148-27' is full (reached quota's max_objects: 7) 2024-08-17T17:34:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:20 smithi130 ceph-mon[105521]: Health check update: 3 pool(s) full (POOL_FULL) 2024-08-17T17:34:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:20 smithi130 ceph-mon[105521]: osdmap e670: 8 total, 8 up, 8 in 2024-08-17T17:34:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:20 smithi130 ceph-mon[110068]: pool 'test-librbd-smithi130-87148-27' is full (reached quota's max_objects: 7) 2024-08-17T17:34:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:20 smithi130 ceph-mon[110068]: Health check update: 3 pool(s) full (POOL_FULL) 2024-08-17T17:34:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:20 smithi130 ceph-mon[110068]: osdmap e670: 8 total, 8 up, 8 in 2024-08-17T17:34:21.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:21 smithi179 ceph-mon[93519]: pgmap v661: 521 pgs: 521 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 770 KiB/s wr, 7 op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:34:21.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:21 smithi130 ceph-mon[105521]: pgmap v661: 521 pgs: 521 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 770 KiB/s wr, 7 op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:34:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:21 smithi130 ceph-mon[110068]: pgmap v661: 521 pgs: 521 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 770 KiB/s wr, 7 op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:34:22.342 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:22.343 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:22.343 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:22.343 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:22.343 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:22.344 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.344 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:22.344 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.344 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.344 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:22.344 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.345 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:22.345 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.345 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.345 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:22.345 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.345 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:22.345 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.345 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.345 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:22.345 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.345 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:22.345 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.346 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.346 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:22.346 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.346 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:22.346 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.347 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.347 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:22.347 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:22.347 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:22.347 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:22.347 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:22.347 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:22.347 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:22.347 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:22.347 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:22.348 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:22.348 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:22.348 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.348 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:22.348 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.348 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.348 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:22.348 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.348 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:22.348 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.348 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.349 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:22.349 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.349 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:22.349 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.349 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.349 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:22.349 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.349 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:22.349 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.349 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.349 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:22.350 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.350 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:22.350 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.350 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.350 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:22.350 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:22.350 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:22.350 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:22.350 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:22.350 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:22.350 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.350 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:22.350 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:22.351 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:22.351 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.351 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:22.351 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:22.351 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:22.351 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.351 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:22.351 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:22.351 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:22.351 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.351 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:22.352 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:22.352 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:22.352 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.352 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:22.352 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:22.352 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:22.352 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 512~512 = 512 2024-08-17T17:34:22.353 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.353 INFO:tasks.workunit.client.0.smithi130.stdout:aio discard: 1536~512 = 0 2024-08-17T17:34:22.353 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:22.353 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:22.353 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:22.354 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:22.354 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:22.354 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 0 2024-08-17T17:34:22.354 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.354 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.354 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 0 2024-08-17T17:34:22.354 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.354 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.355 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 16384 2024-08-17T17:34:22.355 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.355 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.355 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 16384 2024-08-17T17:34:22.355 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.355 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.355 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:22.355 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:22.355 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 49152 2024-08-17T17:34:22.355 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.355 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.355 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 49152 2024-08-17T17:34:22.356 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.356 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.356 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 65536 2024-08-17T17:34:22.356 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.356 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.356 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 65536 2024-08-17T17:34:22.356 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.356 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.356 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:22.356 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:22.356 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 98304 2024-08-17T17:34:22.356 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.357 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.357 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 98304 2024-08-17T17:34:22.357 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.357 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.357 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 114688 2024-08-17T17:34:22.357 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.357 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.357 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 114688 2024-08-17T17:34:22.357 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.357 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.357 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:22.357 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:22.357 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 147456 2024-08-17T17:34:22.358 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.358 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.358 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 147456 2024-08-17T17:34:22.358 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.358 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.358 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 163840 2024-08-17T17:34:22.358 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.358 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.358 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 163840 2024-08-17T17:34:22.358 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.358 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.358 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:22.359 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:22.359 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 196608 2024-08-17T17:34:22.359 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.359 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.359 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 196608 2024-08-17T17:34:22.359 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.359 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.359 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 212992 2024-08-17T17:34:22.359 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.359 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.359 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 212992 2024-08-17T17:34:22.359 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.359 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.359 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:22.360 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:22.360 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.360 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:22.360 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.360 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.360 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:22.360 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.360 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.360 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.360 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:22.361 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.361 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.361 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:22.361 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.361 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.361 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.361 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:22.361 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.361 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.361 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:22.361 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.361 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.361 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.362 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:22.362 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.362 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.362 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:22.362 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.362 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.362 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.362 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:22.362 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:22.362 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.362 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:22.363 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:22.363 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.363 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:22.363 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.363 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.363 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:22.363 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.363 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.363 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:22.363 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:22.363 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:22.363 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:22.363 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:22.364 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:22.364 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:22.364 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:23 smithi179 ceph-mon[93519]: pgmap v663: 521 pgs: 521 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.6 KiB/s rd, 2.2 KiB/s wr, 12 op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:34:23.843 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.843 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:23.843 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:23.844 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:23.844 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:23.844 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:23.844 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.844 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.844 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:23.844 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:23.844 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:23.844 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:23.844 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:23.844 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.844 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.844 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:23.844 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.845 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.845 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:23.845 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.845 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.845 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:23.845 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:23.845 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:23.845 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:23.845 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:23.845 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.845 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.845 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:23.845 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:23.845 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:23.846 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:23.846 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:23.846 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.846 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.846 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:23.846 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:23.846 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:23.846 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:23.846 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:23.846 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.849 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.849 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:23.849 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:23.850 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:23.850 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:23.850 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:23.850 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.850 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.850 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:23.850 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.850 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.850 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:23.850 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.850 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.850 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:23.850 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:23.850 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:23.851 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:23.851 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:23.851 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.851 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.851 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:23.851 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:23.851 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:23.851 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:23.851 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:23.851 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.851 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.851 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:23.851 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:23.852 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:23.852 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:23.852 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:23.852 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.852 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.852 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:23.852 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:23.852 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:23.852 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:23.852 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:23.852 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.852 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.852 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:23.852 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.853 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.853 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:23.853 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.853 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.853 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:23.853 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:23.853 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:23.853 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:23.853 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:23.853 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.853 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.854 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:23.854 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:23.854 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:23.854 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:23.854 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:23.854 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.854 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.854 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:23.854 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:23.854 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:23.854 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:23.854 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:23.855 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.855 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.855 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:23.855 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:23.855 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:23.855 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:23.855 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:23.855 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.855 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.855 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:23.855 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:23.855 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:23.855 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:23.855 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:23.856 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:23.856 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:23.856 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:23.856 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestIO (3258 ms) 2024-08-17T17:34:23.856 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS1 2024-08-17T17:34:23.856 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:23 smithi130 ceph-mon[105521]: pgmap v663: 521 pgs: 521 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.6 KiB/s rd, 2.2 KiB/s wr, 12 op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:34:23.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:23 smithi130 ceph-mon[110068]: pgmap v663: 521 pgs: 521 active+clean; 14 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.6 KiB/s rd, 2.2 KiB/s wr, 12 op/s; 2 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:34:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:25 smithi179 ceph-mon[93519]: pgmap v664: 521 pgs: 521 active+clean; 5.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 494 KiB/s rd, 266 KiB/s wr, 980 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:34:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:25 smithi179 ceph-mon[93519]: pool 'test-librbd-smithi130-87148-28' no longer out of quota; removing NO_QUOTA flag 2024-08-17T17:34:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:25 smithi179 ceph-mon[93519]: Health check update: 2 pool(s) full (POOL_FULL) 2024-08-17T17:34:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:25 smithi179 ceph-mon[93519]: osdmap e671: 8 total, 8 up, 8 in 2024-08-17T17:34:25.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:25 smithi130 ceph-mon[110068]: pgmap v664: 521 pgs: 521 active+clean; 5.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 494 KiB/s rd, 266 KiB/s wr, 980 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:34:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:25 smithi130 ceph-mon[110068]: pool 'test-librbd-smithi130-87148-28' no longer out of quota; removing NO_QUOTA flag 2024-08-17T17:34:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:25 smithi130 ceph-mon[110068]: Health check update: 2 pool(s) full (POOL_FULL) 2024-08-17T17:34:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:25 smithi130 ceph-mon[110068]: osdmap e671: 8 total, 8 up, 8 in 2024-08-17T17:34:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:25 smithi130 ceph-mon[105521]: pgmap v664: 521 pgs: 521 active+clean; 5.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 494 KiB/s rd, 266 KiB/s wr, 980 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:34:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:25 smithi130 ceph-mon[105521]: pool 'test-librbd-smithi130-87148-28' no longer out of quota; removing NO_QUOTA flag 2024-08-17T17:34:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:25 smithi130 ceph-mon[105521]: Health check update: 2 pool(s) full (POOL_FULL) 2024-08-17T17:34:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:25 smithi130 ceph-mon[105521]: osdmap e671: 8 total, 8 up, 8 in 2024-08-17T17:34:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:27 smithi179 ceph-mon[93519]: pgmap v666: 521 pgs: 521 active+clean; 793 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 492 KiB/s rd, 265 KiB/s wr, 976 op/s 2024-08-17T17:34:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:27.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:27 smithi130 ceph-mon[110068]: pgmap v666: 521 pgs: 521 active+clean; 793 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 492 KiB/s rd, 265 KiB/s wr, 976 op/s 2024-08-17T17:34:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:27 smithi130 ceph-mon[105521]: pgmap v666: 521 pgs: 521 active+clean; 793 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 492 KiB/s rd, 265 KiB/s wr, 976 op/s 2024-08-17T17:34:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:34:29.161 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:29.161 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:29.162 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:29.162 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:29.162 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:29.162 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.162 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:29.162 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.162 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.162 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:29.162 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.162 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:29.162 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.162 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.162 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.163 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.164 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:29.165 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.166 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:29.166 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.166 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.166 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:29.166 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:29.166 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:29.166 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:29.166 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:29.166 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:29.166 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.166 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:29.166 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:29.166 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:29.167 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 512~512 = 512 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:aio discard: 1536~512 = 0 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 0 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 0 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 16384 2024-08-17T17:34:29.168 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.169 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.169 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 16384 2024-08-17T17:34:29.169 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.169 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.169 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:29.169 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:29.169 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 49152 2024-08-17T17:34:29.169 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.169 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.169 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 49152 2024-08-17T17:34:29.169 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.169 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.169 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 65536 2024-08-17T17:34:29.169 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.170 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.170 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 65536 2024-08-17T17:34:29.170 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.170 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.170 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:29.170 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:29.170 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 98304 2024-08-17T17:34:29.170 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.170 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.170 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 98304 2024-08-17T17:34:29.170 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.170 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.170 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 114688 2024-08-17T17:34:29.170 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.171 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.171 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 114688 2024-08-17T17:34:29.171 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.171 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.171 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:29.171 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:29.171 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 147456 2024-08-17T17:34:29.171 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.171 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.171 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 147456 2024-08-17T17:34:29.171 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.171 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.171 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 163840 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 163840 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 196608 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 196608 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 212992 2024-08-17T17:34:29.172 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.173 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.173 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 212992 2024-08-17T17:34:29.173 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.173 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.173 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:29.173 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:29.173 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.173 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:29.173 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.173 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.173 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:29.173 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.173 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.173 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.174 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:29.174 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.174 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.174 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:29.174 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.174 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.174 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.174 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:29.174 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.174 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.174 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:29.174 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.174 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:29.175 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.176 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.176 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:29.176 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.176 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.176 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.176 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:29.176 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:29.176 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:29.176 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:29.176 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:29.176 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:29.176 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:29.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:29 smithi130 ceph-mon[105521]: pgmap v667: 521 pgs: 521 active+clean; 4.9 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 697 KiB/s rd, 883 KiB/s wr, 1.19k op/s 2024-08-17T17:34:29.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:29 smithi130 ceph-mon[110068]: pgmap v667: 521 pgs: 521 active+clean; 4.9 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 697 KiB/s rd, 883 KiB/s wr, 1.19k op/s 2024-08-17T17:34:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:29 smithi179 ceph-mon[93519]: pgmap v667: 521 pgs: 521 active+clean; 4.9 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 697 KiB/s rd, 883 KiB/s wr, 1.19k op/s 2024-08-17T17:34:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:34:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:34:29] "GET /metrics HTTP/1.1" 200 67782 "" "Prometheus/2.51.0" 2024-08-17T17:34:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:34:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:34:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:34:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:31 smithi179 ceph-mon[93519]: pgmap v668: 521 pgs: 521 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 619 KiB/s rd, 730 KiB/s wr, 1.07k op/s 2024-08-17T17:34:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:31 smithi130 ceph-mon[105521]: pgmap v668: 521 pgs: 521 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 619 KiB/s rd, 730 KiB/s wr, 1.07k op/s 2024-08-17T17:34:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:31 smithi130 ceph-mon[110068]: pgmap v668: 521 pgs: 521 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 619 KiB/s rd, 730 KiB/s wr, 1.07k op/s 2024-08-17T17:34:32.822 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.822 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:32.822 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:32.822 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:32.822 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:32.822 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:32.822 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.822 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.823 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:32.823 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:32.823 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:32.823 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:32.823 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:32.823 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.823 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.823 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:32.823 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.823 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.823 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:32.823 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.823 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.823 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:32.824 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:32.824 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:32.824 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:32.824 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:32.824 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.824 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.824 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:32.824 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:32.824 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:32.824 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:32.824 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:32.824 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.824 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.824 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:32.825 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:32.825 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:32.825 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:32.825 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:32.825 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.825 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.825 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:32.825 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:32.825 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:32.825 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:32.825 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:32.825 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.825 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.825 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:32.826 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.826 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.826 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:32.826 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.826 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.826 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:32.826 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:32.826 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:32.826 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:32.826 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:32.826 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.826 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.826 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:32.826 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:32.827 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:32.827 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:32.827 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:32.827 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.827 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.827 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:32.827 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:32.827 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:32.827 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:32.827 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:32.827 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.827 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.827 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:32.828 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:32.828 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:32.828 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:32.828 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:32.828 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.828 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.828 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:32.828 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.828 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.828 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:32.828 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.828 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.828 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:32.828 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:32.829 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:32.830 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:32.830 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:32.830 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.830 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.830 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:32.830 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:32.830 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:32.830 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:32.830 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:32.830 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.830 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.832 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:32.832 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:32.832 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:32.832 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:32.832 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:32.832 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:32.832 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:32.832 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:32.832 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 4 2024-08-17T17:34:32.832 INFO:tasks.workunit.client.0.smithi130.stdout:read: 4 2024-08-17T17:34:32.832 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS1 (8979 ms) 2024-08-17T17:34:32.832 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS2 2024-08-17T17:34:32.833 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:33 smithi179 ceph-mon[93519]: pgmap v669: 521 pgs: 521 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 599 KiB/s rd, 706 KiB/s wr, 1.03k op/s 2024-08-17T17:34:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:33 smithi130 ceph-mon[105521]: pgmap v669: 521 pgs: 521 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 599 KiB/s rd, 706 KiB/s wr, 1.03k op/s 2024-08-17T17:34:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:33 smithi130 ceph-mon[110068]: pgmap v669: 521 pgs: 521 active+clean; 6.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 599 KiB/s rd, 706 KiB/s wr, 1.03k op/s 2024-08-17T17:34:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:35 smithi179 ceph-mon[93519]: pgmap v670: 521 pgs: 521 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 918 KiB/s rd, 831 KiB/s wr, 942 op/s 2024-08-17T17:34:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:35 smithi130 ceph-mon[105521]: pgmap v670: 521 pgs: 521 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 918 KiB/s rd, 831 KiB/s wr, 942 op/s 2024-08-17T17:34:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:35 smithi130 ceph-mon[110068]: pgmap v670: 521 pgs: 521 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 918 KiB/s rd, 831 KiB/s wr, 942 op/s 2024-08-17T17:34:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:37 smithi179 ceph-mon[93519]: pgmap v671: 521 pgs: 521 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 860 KiB/s rd, 778 KiB/s wr, 882 op/s 2024-08-17T17:34:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:37 smithi130 ceph-mon[105521]: pgmap v671: 521 pgs: 521 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 860 KiB/s rd, 778 KiB/s wr, 882 op/s 2024-08-17T17:34:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:37 smithi130 ceph-mon[110068]: pgmap v671: 521 pgs: 521 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 860 KiB/s rd, 778 KiB/s wr, 882 op/s 2024-08-17T17:34:39.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:39 smithi130 ceph-mon[105521]: pgmap v672: 521 pgs: 521 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 765 KiB/s rd, 692 KiB/s wr, 785 op/s 2024-08-17T17:34:39.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:39 smithi130 ceph-mon[110068]: pgmap v672: 521 pgs: 521 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 765 KiB/s rd, 692 KiB/s wr, 785 op/s 2024-08-17T17:34:39.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:39 smithi179 ceph-mon[93519]: pgmap v672: 521 pgs: 521 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 765 KiB/s rd, 692 KiB/s wr, 785 op/s 2024-08-17T17:34:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:34:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:34:39] "GET /metrics HTTP/1.1" 200 67782 "" "Prometheus/2.51.0" 2024-08-17T17:34:40.508 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:40.508 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:40.508 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:40.508 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:40.508 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:40.508 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.508 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:40.508 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.509 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.509 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:40.509 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.509 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:40.509 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.509 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.509 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:40.509 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.509 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:40.509 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.509 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.509 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:40.509 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.510 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:40.510 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.510 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.510 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:40.510 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.510 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:40.510 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.510 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.510 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:40.510 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:40.510 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:40.510 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:40.510 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:40.511 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:40.511 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:40.511 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:40.511 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:40.511 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:40.511 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:40.511 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.511 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:40.511 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.511 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.511 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:40.511 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.511 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:40.512 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.513 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.513 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:40.513 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:40.513 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:40.513 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:40.513 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:40.513 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:40.513 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.513 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:40.513 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:40.513 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:40.513 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.513 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:40.513 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:40.514 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:40.514 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.514 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:40.514 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:40.514 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:40.514 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.514 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:40.514 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:40.514 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:40.514 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.514 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:40.514 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:40.514 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:40.514 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 512~512 = 512 2024-08-17T17:34:40.515 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.515 INFO:tasks.workunit.client.0.smithi130.stdout:aio discard: 1536~512 = 0 2024-08-17T17:34:40.515 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:40.515 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:40.515 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:40.515 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:40.515 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:40.515 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 0 2024-08-17T17:34:40.515 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.515 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.515 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 0 2024-08-17T17:34:40.515 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.515 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.515 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 16384 2024-08-17T17:34:40.516 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.516 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.516 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 16384 2024-08-17T17:34:40.516 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.516 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.516 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:40.516 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:40.516 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 49152 2024-08-17T17:34:40.516 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.516 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.516 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 49152 2024-08-17T17:34:40.516 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.516 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.516 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 65536 2024-08-17T17:34:40.517 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.517 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.517 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 65536 2024-08-17T17:34:40.517 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.517 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.517 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:40.517 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:40.517 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 98304 2024-08-17T17:34:40.517 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.517 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.517 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 98304 2024-08-17T17:34:40.517 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.517 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.517 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 114688 2024-08-17T17:34:40.518 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.518 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.518 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 114688 2024-08-17T17:34:40.518 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.518 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.518 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:40.518 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:40.518 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 147456 2024-08-17T17:34:40.518 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.518 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.518 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 147456 2024-08-17T17:34:40.518 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.518 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.518 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 163840 2024-08-17T17:34:40.519 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.519 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.519 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 163840 2024-08-17T17:34:40.519 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.519 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.519 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:40.519 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:40.519 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 196608 2024-08-17T17:34:40.519 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.519 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.519 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 196608 2024-08-17T17:34:40.519 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.519 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.519 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 212992 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 212992 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.520 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.521 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:40.521 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.521 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.521 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:40.521 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.521 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.521 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.521 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:40.521 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.521 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.521 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:40.521 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.521 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.521 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.522 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:40.522 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.522 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.522 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:40.522 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.522 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.522 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.522 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:40.522 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:40.522 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.522 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:40.522 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:40.522 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.523 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:40.523 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.523 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.523 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:40.523 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.523 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.523 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.523 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:40.523 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:40.523 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:40.523 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:40.523 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:40.523 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:40.523 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:34:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:34:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:34:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:41 smithi179 ceph-mon[93519]: pgmap v673: 521 pgs: 521 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 628 KiB/s rd, 281 KiB/s wr, 644 op/s 2024-08-17T17:34:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:41 smithi130 ceph-mon[105521]: pgmap v673: 521 pgs: 521 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 628 KiB/s rd, 281 KiB/s wr, 644 op/s 2024-08-17T17:34:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:41 smithi130 ceph-mon[110068]: pgmap v673: 521 pgs: 521 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 628 KiB/s rd, 281 KiB/s wr, 644 op/s 2024-08-17T17:34:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:43 smithi179 ceph-mon[93519]: pgmap v674: 521 pgs: 521 active+clean; 15 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 594 KiB/s rd, 622 KiB/s wr, 574 op/s 2024-08-17T17:34:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:43 smithi130 ceph-mon[105521]: pgmap v674: 521 pgs: 521 active+clean; 15 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 594 KiB/s rd, 622 KiB/s wr, 574 op/s 2024-08-17T17:34:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:43 smithi130 ceph-mon[110068]: pgmap v674: 521 pgs: 521 active+clean; 15 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 594 KiB/s rd, 622 KiB/s wr, 574 op/s 2024-08-17T17:34:44.435 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.436 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:44.436 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:44.436 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:44.436 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:44.436 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:44.436 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.436 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.436 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:44.436 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:44.436 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:44.436 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:44.436 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:44.437 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.437 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.437 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:44.437 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.437 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.437 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:44.437 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.437 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.437 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:44.437 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:44.437 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:44.437 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:44.437 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:44.437 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.438 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.438 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:44.438 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:44.438 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:44.438 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:44.438 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:44.438 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.438 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.438 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:44.438 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:44.438 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:44.438 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:44.438 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:44.439 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.439 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.439 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:44.439 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:44.439 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:44.439 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:44.439 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:44.439 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.439 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.439 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:44.439 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.439 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.439 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:44.440 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.440 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.440 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:44.440 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:44.440 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:44.440 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:44.440 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:44.440 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.440 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.440 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:44.440 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:44.440 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:44.440 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:44.440 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:44.441 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.441 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.441 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:44.441 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:44.441 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:44.441 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:44.441 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:44.441 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.441 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.441 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:44.441 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:44.441 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:44.441 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:44.441 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:44.442 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.442 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.442 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:44.442 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.442 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.442 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:44.442 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.442 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.442 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:44.442 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:44.442 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:44.442 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:44.442 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:44.443 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.443 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.443 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:44.443 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:44.443 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:44.443 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:44.443 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:44.443 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.443 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.443 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:44.443 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:44.443 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:44.443 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:44.443 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:44.444 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.444 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.444 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:44.444 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:44.444 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:44.444 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:44.444 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:44.444 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.444 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.444 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:44.444 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:44.444 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:44.444 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:44.444 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:44.445 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:44.445 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:44.445 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:44.445 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 4 2024-08-17T17:34:44.445 INFO:tasks.workunit.client.0.smithi130.stdout:read: 4 2024-08-17T17:34:44.445 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS2 (11613 ms) 2024-08-17T17:34:44.445 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestIOWithIOHint 2024-08-17T17:34:44.445 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:45 smithi179 ceph-mon[93519]: pgmap v675: 521 pgs: 521 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.0 MiB/s wr, 1.29k op/s 2024-08-17T17:34:45.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:45 smithi130 ceph-mon[105521]: pgmap v675: 521 pgs: 521 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.0 MiB/s wr, 1.29k op/s 2024-08-17T17:34:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:45 smithi130 ceph-mon[110068]: pgmap v675: 521 pgs: 521 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.0 MiB/s wr, 1.29k op/s 2024-08-17T17:34:46.250 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:46.250 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:46.250 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:46.251 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:46.251 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:46.251 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.251 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:46.251 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.251 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.251 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:46.251 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.251 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:46.251 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.251 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.251 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:46.251 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.251 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:46.252 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.252 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.252 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:46.252 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.252 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:46.252 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.252 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.252 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:46.252 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.252 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:46.252 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.253 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.253 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:46.253 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:46.253 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:46.253 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:46.253 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:46.253 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:46.253 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:46.253 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:46.253 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:46.254 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:46.254 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:46.254 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.254 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:46.254 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.254 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.254 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:46.254 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.254 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:46.254 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.254 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.255 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:46.255 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.255 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:46.255 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.255 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.255 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:46.255 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.255 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:46.255 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.255 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.255 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:46.255 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.255 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:46.255 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:46.256 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:46.257 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:46.257 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.257 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:46.257 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:46.257 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:46.257 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.257 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:46.257 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:46.257 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:46.257 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.257 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:46.257 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:46.257 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:46.258 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 512~512 = 512 2024-08-17T17:34:46.258 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.258 INFO:tasks.workunit.client.0.smithi130.stdout:aio discard: 1536~512 = 0 2024-08-17T17:34:46.258 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:46.258 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:46.258 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:46.258 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:46.258 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:46.258 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 0 2024-08-17T17:34:46.258 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.258 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.258 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 0 2024-08-17T17:34:46.259 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.259 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.259 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 16384 2024-08-17T17:34:46.259 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.259 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.259 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 16384 2024-08-17T17:34:46.259 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.259 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.259 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:46.259 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:46.259 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 49152 2024-08-17T17:34:46.259 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.260 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.260 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 49152 2024-08-17T17:34:46.260 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.260 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.260 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 65536 2024-08-17T17:34:46.260 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.260 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.260 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 65536 2024-08-17T17:34:46.260 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.260 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.260 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:46.260 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:46.261 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 98304 2024-08-17T17:34:46.261 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.261 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.261 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 98304 2024-08-17T17:34:46.261 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.261 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.261 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 114688 2024-08-17T17:34:46.261 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.261 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.261 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 114688 2024-08-17T17:34:46.261 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.262 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.262 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:46.262 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:46.262 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 147456 2024-08-17T17:34:46.262 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.262 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.262 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 147456 2024-08-17T17:34:46.262 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.262 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.262 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 163840 2024-08-17T17:34:46.262 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.262 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.263 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 163840 2024-08-17T17:34:46.263 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.263 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.263 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:46.263 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:46.263 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 196608 2024-08-17T17:34:46.263 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.263 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.263 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 196608 2024-08-17T17:34:46.263 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.263 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.263 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 212992 2024-08-17T17:34:46.263 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.263 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.264 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 212992 2024-08-17T17:34:46.264 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.264 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.264 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:46.264 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:46.264 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.264 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:46.264 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.264 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.264 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:46.264 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.264 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.264 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.265 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:46.265 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.265 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.265 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:46.265 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.265 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.265 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.265 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:46.265 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.265 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.265 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:46.265 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.265 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.265 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.266 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:46.266 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.266 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.266 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:46.266 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.266 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.266 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.266 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:46.266 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:46.266 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.266 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:46.266 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:46.266 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.267 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:46.267 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.267 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.267 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:46.267 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.267 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.267 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:46.267 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:46.267 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:46.267 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:46.267 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:46.267 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:46.267 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:46.267 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.744 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.744 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:47.744 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:47.744 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:47.744 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:47.744 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:47.745 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.745 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.745 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:47.745 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:47.745 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:47.745 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:47.745 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:47.745 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.745 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.745 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:47.745 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.746 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.746 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:47.746 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.746 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.746 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:47.746 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:47.746 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:47.746 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:47.746 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:47.747 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.747 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.747 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:47.747 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:47.747 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:47.747 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:47.747 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:47.747 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.747 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.748 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:47.748 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:47.748 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:47.748 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:47.748 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:47.748 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.748 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.748 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:47.748 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:47.748 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:47.748 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:47.749 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:47.749 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.749 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.749 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:47.749 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.749 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.749 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:47.749 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.749 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.749 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:47.749 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:47.749 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:47.750 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:47.750 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:47.750 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.750 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.750 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:47.750 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:47.750 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:47.750 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:47.750 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:47.750 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.750 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.750 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:47.750 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:47.751 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:47.751 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:47.751 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:47.751 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.751 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.751 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:47.751 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:47.751 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:47.751 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:47.751 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:47.751 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.751 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.751 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:47.752 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.752 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.752 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:47.752 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.752 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.752 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:47.752 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:47.752 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:47.752 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:47.752 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:47.752 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.753 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.753 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:47.753 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:47.753 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:47.753 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:47.753 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:47.753 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.753 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.753 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:47.753 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:47.753 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:47.753 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:47.754 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:47.754 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.754 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.754 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:47.754 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:47.754 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:47.754 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:47.754 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:47.754 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.754 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.754 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:47.755 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:47.755 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:47.755 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:47.755 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:47.755 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:47.755 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:47.755 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestIOWithIOHint (3309 ms) 2024-08-17T17:34:47.755 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestDataPoolIO 2024-08-17T17:34:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:47 smithi179 ceph-mon[93519]: pgmap v676: 521 pgs: 521 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.7 MiB/s wr, 722 op/s 2024-08-17T17:34:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:47 smithi130 ceph-mon[105521]: pgmap v676: 521 pgs: 521 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.7 MiB/s wr, 722 op/s 2024-08-17T17:34:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:47 smithi130 ceph-mon[110068]: pgmap v676: 521 pgs: 521 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.7 MiB/s wr, 722 op/s 2024-08-17T17:34:49.530 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:49 smithi130 ceph-mon[105521]: pgmap v677: 521 pgs: 521 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.7 MiB/s wr, 727 op/s 2024-08-17T17:34:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:49 smithi130 ceph-mon[105521]: osdmap e672: 8 total, 8 up, 8 in 2024-08-17T17:34:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:49 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2226871454' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:34:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:49 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:34:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:49 smithi130 ceph-mon[110068]: pgmap v677: 521 pgs: 521 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.7 MiB/s wr, 727 op/s 2024-08-17T17:34:49.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:49 smithi130 ceph-mon[110068]: osdmap e672: 8 total, 8 up, 8 in 2024-08-17T17:34:49.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:49 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2226871454' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:34:49.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:49 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:34:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:49 smithi179 ceph-mon[93519]: pgmap v677: 521 pgs: 521 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.7 MiB/s wr, 727 op/s 2024-08-17T17:34:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:49 smithi179 ceph-mon[93519]: osdmap e672: 8 total, 8 up, 8 in 2024-08-17T17:34:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:49 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2226871454' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:34:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:49 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:34:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:34:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:34:49] "GET /metrics HTTP/1.1" 200 67784 "" "Prometheus/2.51.0" 2024-08-17T17:34:50.529 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:34:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:34:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:34:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:50 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:34:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:50 smithi179 ceph-mon[93519]: osdmap e673: 8 total, 8 up, 8 in 2024-08-17T17:34:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:50 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:34:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:50 smithi130 ceph-mon[110068]: osdmap e673: 8 total, 8 up, 8 in 2024-08-17T17:34:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:50 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:34:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:50 smithi130 ceph-mon[105521]: osdmap e673: 8 total, 8 up, 8 in 2024-08-17T17:34:51.595 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:51.596 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:51.596 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:51.596 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:51.596 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:51.596 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:51.596 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:51.596 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:51.596 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:51.596 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:51.596 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:51.596 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:51.596 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:51.596 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:51.596 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:51.597 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:51.598 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:51.598 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:51.598 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:51.598 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:51.598 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:51.598 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:51.598 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:51.598 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:51.598 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:51.598 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:51.598 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:51.598 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:51.598 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:51.599 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:51.599 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:51.599 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:51.599 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:51.599 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:51.599 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:51.599 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:51.599 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:51.599 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:51.599 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:51.599 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:51.599 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:51.599 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 512~512 = 512 2024-08-17T17:34:51.600 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:51.600 INFO:tasks.workunit.client.0.smithi130.stdout:aio discard: 1536~512 = 0 2024-08-17T17:34:51.600 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:51.600 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:51.600 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:51.600 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:51.600 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:51.600 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:51.600 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:51.600 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestDataPoolIO (3851 ms) 2024-08-17T17:34:51.600 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatch 2024-08-17T17:34:51.600 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:51.622 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatch (27 ms) 2024-08-17T17:34:51.622 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatch 2024-08-17T17:34:51.622 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:51.649 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatch (27 ms) 2024-08-17T17:34:51.649 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccess 2024-08-17T17:34:51.649 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:51.677 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccess (28 ms) 2024-08-17T17:34:51.677 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccess 2024-08-17T17:34:51.677 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:51.707 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccess (30 ms) 2024-08-17T17:34:51.707 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned 2024-08-17T17:34:51.707 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:51.782 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned (75 ms) 2024-08-17T17:34:51.782 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned 2024-08-17T17:34:51.783 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:51 smithi179 ceph-mon[93519]: pgmap v680: 553 pgs: 8 creating+peering, 22 unknown, 523 active+clean; 33 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.3 MiB/s wr, 2.14k op/s 2024-08-17T17:34:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:51 smithi179 ceph-mon[93519]: osdmap e674: 8 total, 8 up, 8 in 2024-08-17T17:34:51.846 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned (64 ms) 2024-08-17T17:34:51.846 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLarge 2024-08-17T17:34:51.846 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:51.904 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLarge (58 ms) 2024-08-17T17:34:51.904 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLarge 2024-08-17T17:34:51.904 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:51.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:51 smithi130 ceph-mon[105521]: pgmap v680: 553 pgs: 8 creating+peering, 22 unknown, 523 active+clean; 33 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.3 MiB/s wr, 2.14k op/s 2024-08-17T17:34:51.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:51 smithi130 ceph-mon[105521]: osdmap e674: 8 total, 8 up, 8 in 2024-08-17T17:34:51.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:51 smithi130 ceph-mon[110068]: pgmap v680: 553 pgs: 8 creating+peering, 22 unknown, 523 active+clean; 33 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.3 MiB/s wr, 2.14k op/s 2024-08-17T17:34:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:51 smithi130 ceph-mon[110068]: osdmap e674: 8 total, 8 up, 8 in 2024-08-17T17:34:51.972 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLarge (68 ms) 2024-08-17T17:34:51.972 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess 2024-08-17T17:34:51.973 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:52.236 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess (264 ms) 2024-08-17T17:34:52.236 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess 2024-08-17T17:34:52.236 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:52.497 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess (261 ms) 2024-08-17T17:34:52.497 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers 2024-08-17T17:34:52.497 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:52.525 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers (28 ms) 2024-08-17T17:34:52.525 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVMismatch 2024-08-17T17:34:52.525 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:52.554 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVMismatch (29 ms) 2024-08-17T17:34:52.555 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVSuccess 2024-08-17T17:34:52.555 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:52.579 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVSuccess (25 ms) 2024-08-17T17:34:52.579 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestScatterGatherIO 2024-08-17T17:34:52.579 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:52.604 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestScatterGatherIO (25 ms) 2024-08-17T17:34:52.604 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestEmptyDiscard 2024-08-17T17:34:52.604 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:52.626 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.626 INFO:tasks.workunit.client.0.smithi130.stdout:aio discard: 0~1048576 = 0 2024-08-17T17:34:52.626 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.626 INFO:tasks.workunit.client.0.smithi130.stdout:aio discard: 0~4194304 = 0 2024-08-17T17:34:52.626 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.626 INFO:tasks.workunit.client.0.smithi130.stdout:aio discard: 3145728~1048576 = 0 2024-08-17T17:34:52.626 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestEmptyDiscard (22 ms) 2024-08-17T17:34:52.626 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestFUA 2024-08-17T17:34:52.626 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:52.725 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:52.725 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:52.726 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:52.726 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:52.726 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:52.726 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:52.726 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:52.726 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:52.726 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:52.726 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:52.726 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.726 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:52.726 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.726 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.726 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:52.727 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.727 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:52.727 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.727 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.727 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:52.727 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.727 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:52.727 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.727 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.727 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:52.727 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.727 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:52.727 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.728 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.728 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:52.728 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.728 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:52.728 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.728 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.728 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:52.728 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.728 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:52.728 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:52.728 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:52.728 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.728 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:52.728 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:52.729 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:52.729 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.729 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:52.729 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:52.729 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:52.729 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.729 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:52.729 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:52.729 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:52.729 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.729 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:52.729 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:52.729 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:52.729 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestFUA (99 ms) 2024-08-17T17:34:52.730 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestIOPP 2024-08-17T17:34:52.730 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:52.745 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:52.746 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:52.746 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:52.746 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:52.746 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:52.746 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.746 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:52.746 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.746 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.746 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:52.746 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.746 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:52.747 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.747 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.747 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:52.747 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.747 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:52.747 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.747 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.747 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:52.747 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.747 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:52.747 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.747 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.747 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:52.747 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.748 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:52.748 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.761 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.762 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:52.762 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:52.762 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:52.762 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:52.762 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:52.762 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:52.762 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:52.762 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:52.762 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:52.762 INFO:tasks.workunit.client.0.smithi130.stdout:start compare and write 2024-08-17T17:34:52.762 INFO:tasks.workunit.client.0.smithi130.stdout:compare and wrote: 512 2024-08-17T17:34:52.762 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.763 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:52.763 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.763 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.763 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:52.763 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.763 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:52.763 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.763 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.764 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:52.764 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.764 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:52.764 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.764 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.764 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:52.764 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.764 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:52.764 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.765 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.765 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:52.765 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.765 INFO:tasks.workunit.client.0.smithi130.stdout:started aio compare and write 2024-08-17T17:34:52.765 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.767 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:52.767 INFO:tasks.workunit.client.0.smithi130.stdout:finished aio compare and write 2024-08-17T17:34:52.767 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:52.767 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:52.767 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:52.767 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:52.767 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:52.767 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.767 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:52.767 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:52.767 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:52.767 INFO:tasks.workunit.client.0.smithi130.stdout:finished read 2024-08-17T17:34:52.768 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.768 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:52.768 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:52.768 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:52.768 INFO:tasks.workunit.client.0.smithi130.stdout:finished read 2024-08-17T17:34:52.768 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.768 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:52.768 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:52.769 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:52.769 INFO:tasks.workunit.client.0.smithi130.stdout:finished read 2024-08-17T17:34:52.769 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.769 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:52.769 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:52.769 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:52.769 INFO:tasks.workunit.client.0.smithi130.stdout:finished read 2024-08-17T17:34:52.769 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:52.769 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:52.769 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:52.769 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:52.769 INFO:tasks.workunit.client.0.smithi130.stdout:finished read 2024-08-17T17:34:52.769 INFO:tasks.workunit.client.0.smithi130.stdout:discard: 512~512 2024-08-17T17:34:52.770 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:52 smithi179 ceph-mon[93519]: osdmap e675: 8 total, 8 up, 8 in 2024-08-17T17:34:52.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:52 smithi130 ceph-mon[110068]: osdmap e675: 8 total, 8 up, 8 in 2024-08-17T17:34:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:52 smithi130 ceph-mon[105521]: osdmap e675: 8 total, 8 up, 8 in 2024-08-17T17:34:53.773 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:53.773 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:53.773 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:53.773 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:53.773 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:53.773 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 0 2024-08-17T17:34:53.773 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.773 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.773 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 0 2024-08-17T17:34:53.773 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.774 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.774 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 16384 2024-08-17T17:34:53.774 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.774 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.774 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 16384 2024-08-17T17:34:53.774 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.774 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.774 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:53.774 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:53.774 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 49152 2024-08-17T17:34:53.774 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.774 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.774 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 49152 2024-08-17T17:34:53.775 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.775 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.775 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 65536 2024-08-17T17:34:53.775 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.775 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.775 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 65536 2024-08-17T17:34:53.775 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.775 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.775 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:53.775 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:53.775 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 98304 2024-08-17T17:34:53.775 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.775 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.776 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 98304 2024-08-17T17:34:53.776 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.776 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.776 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 114688 2024-08-17T17:34:53.776 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.776 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.776 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 114688 2024-08-17T17:34:53.776 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.776 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.776 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:53.776 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:53.776 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 147456 2024-08-17T17:34:53.776 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.776 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.777 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 147456 2024-08-17T17:34:53.777 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.777 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.777 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 163840 2024-08-17T17:34:53.777 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.777 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.777 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 163840 2024-08-17T17:34:53.777 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.777 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.777 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:53.777 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:53.777 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 196608 2024-08-17T17:34:53.777 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.777 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.778 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 196608 2024-08-17T17:34:53.778 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.778 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.778 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 212992 2024-08-17T17:34:53.778 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.778 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.778 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 212992 2024-08-17T17:34:53.778 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.778 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.778 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:53.778 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:53.778 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.778 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.779 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:53.779 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:53.779 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:53.779 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.779 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.779 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.779 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.779 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:53.779 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:53.779 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:53.779 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.779 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.779 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.780 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.780 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:53.782 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:53.782 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:53.782 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.782 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.782 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.782 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.782 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:53.789 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:53.789 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:53.789 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.789 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.789 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.789 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.789 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:53.789 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.789 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.790 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:53.790 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.790 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.790 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:53.814 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:53.814 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:53.814 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.814 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.814 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.814 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.814 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:53.834 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:53.834 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:53.835 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.835 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.835 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.835 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.835 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:53.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:53 smithi179 ceph-mon[93519]: pgmap v683: 553 pgs: 8 creating+peering, 17 unknown, 528 active+clean; 33 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 525 KiB/s wr, 2.10k op/s 2024-08-17T17:34:53.861 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:53.861 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:53.861 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.861 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.861 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.861 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.861 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:53.890 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:53.890 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:53.890 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.890 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.890 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.890 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.890 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:53.890 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.890 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.890 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:53.891 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.891 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.891 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:53.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:53 smithi130 ceph-mon[105521]: pgmap v683: 553 pgs: 8 creating+peering, 17 unknown, 528 active+clean; 33 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 525 KiB/s wr, 2.10k op/s 2024-08-17T17:34:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:53 smithi130 ceph-mon[110068]: pgmap v683: 553 pgs: 8 creating+peering, 17 unknown, 528 active+clean; 33 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 525 KiB/s wr, 2.10k op/s 2024-08-17T17:34:53.938 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:53.938 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:53.938 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.938 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.938 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.938 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.938 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:53.981 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:53.981 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:53.981 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:53.981 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:53.982 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:53.982 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:53.982 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:54.046 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:54.046 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:54.046 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:54.046 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:54.046 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.046 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.047 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:54.090 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:54.090 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:54.090 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:54.090 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:54.090 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.090 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.090 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:54.090 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.090 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.090 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:54.090 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.091 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.091 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:54.159 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:54.159 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:54.159 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:54.159 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:54.159 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.159 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.159 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:54.218 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:54.218 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:54.218 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:54.218 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:54.218 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.218 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.218 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:54.294 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:54.294 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:54.294 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:54.294 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:54.294 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.294 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.294 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:54.360 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:54.360 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:54.360 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:54.360 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:54.361 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.361 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.361 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:54.361 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.361 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.361 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:54.361 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.361 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.361 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:54.453 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:54.453 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:54.453 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:54.454 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:54.454 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.454 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.454 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:54.538 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:54.539 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:54.539 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:54.539 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:54.539 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.539 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.539 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:54.638 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:54.638 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:54.638 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:54.638 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:54.638 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.638 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.638 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:54.728 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:54.728 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:54.728 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:54.728 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:54.728 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.729 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.729 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:54.729 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:54.729 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:54.729 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:54.729 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestIOPP (2003 ms) 2024-08-17T17:34:54.729 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP 2024-08-17T17:34:54.729 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:54.752 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP (24 ms) 2024-08-17T17:34:54.753 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP 2024-08-17T17:34:54.753 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:54.775 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP (23 ms) 2024-08-17T17:34:54.775 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP 2024-08-17T17:34:54.775 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:54.799 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP (24 ms) 2024-08-17T17:34:54.799 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP 2024-08-17T17:34:54.799 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:54.821 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP (22 ms) 2024-08-17T17:34:54.821 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchPP 2024-08-17T17:34:54.821 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:54.845 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchPP (24 ms) 2024-08-17T17:34:54.846 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchPP 2024-08-17T17:34:54.846 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:54.867 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:54.870 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchPP (25 ms) 2024-08-17T17:34:54.870 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP 2024-08-17T17:34:54.870 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:54.902 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP (32 ms) 2024-08-17T17:34:54.902 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP 2024-08-17T17:34:54.902 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:54.925 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:54.928 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP (26 ms) 2024-08-17T17:34:54.928 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessPP 2024-08-17T17:34:54.929 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:54.955 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessPP (27 ms) 2024-08-17T17:34:54.955 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessPP 2024-08-17T17:34:54.955 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:54.976 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:54.980 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessPP (25 ms) 2024-08-17T17:34:54.980 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP 2024-08-17T17:34:54.980 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:55.005 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP (25 ms) 2024-08-17T17:34:55.005 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP 2024-08-17T17:34:55.005 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:55.027 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:55.032 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP (27 ms) 2024-08-17T17:34:55.032 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP 2024-08-17T17:34:55.032 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:55.098 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP (66 ms) 2024-08-17T17:34:55.098 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP 2024-08-17T17:34:55.098 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:55.140 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:55.162 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP (64 ms) 2024-08-17T17:34:55.163 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLargePP 2024-08-17T17:34:55.163 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:55.226 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLargePP (64 ms) 2024-08-17T17:34:55.226 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLargePP 2024-08-17T17:34:55.227 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:55.279 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:55.299 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLargePP (73 ms) 2024-08-17T17:34:55.300 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP 2024-08-17T17:34:55.300 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:55.612 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP (312 ms) 2024-08-17T17:34:55.612 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP 2024-08-17T17:34:55.612 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:55.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:55 smithi179 ceph-mon[93519]: pgmap v684: 553 pgs: 553 active+clean; 75 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 8.3 MiB/s wr, 1.92k op/s 2024-08-17T17:34:55.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:55 smithi130 ceph-mon[105521]: pgmap v684: 553 pgs: 553 active+clean; 75 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 8.3 MiB/s wr, 1.92k op/s 2024-08-17T17:34:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:55 smithi130 ceph-mon[110068]: pgmap v684: 553 pgs: 553 active+clean; 75 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 8.3 MiB/s wr, 1.92k op/s 2024-08-17T17:34:55.925 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:55.948 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP (336 ms) 2024-08-17T17:34:55.948 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestIOPPWithIOHint 2024-08-17T17:34:55.948 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:55.974 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:55.975 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:55.975 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:55.975 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:55.975 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:34:55.975 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:55.975 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:55.975 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:55.975 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:55.975 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:55.976 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:55.976 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:55.976 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:55.976 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:55.976 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:55.976 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:55.976 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:55.976 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:55.977 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:55.977 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:55.977 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:55.977 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:55.977 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:55.980 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:55.980 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:55.980 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:55.980 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:34:55.980 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:55.981 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:55.981 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:34:55.981 INFO:tasks.workunit.client.0.smithi130.stdout:read: 512 2024-08-17T17:34:55.981 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:55.981 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:55.981 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:55.981 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:55.981 INFO:tasks.workunit.client.0.smithi130.stdout:finished read 2024-08-17T17:34:55.982 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:55.982 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:55.982 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:55.982 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:55.982 INFO:tasks.workunit.client.0.smithi130.stdout:finished read 2024-08-17T17:34:55.982 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:55.982 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:55.982 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:55.982 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:55.982 INFO:tasks.workunit.client.0.smithi130.stdout:finished read 2024-08-17T17:34:55.982 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:55.982 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:55.982 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:55.983 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:55.983 INFO:tasks.workunit.client.0.smithi130.stdout:finished read 2024-08-17T17:34:55.983 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:55.983 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:34:55.983 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:34:56.979 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:34:56.980 INFO:tasks.workunit.client.0.smithi130.stdout:finished read 2024-08-17T17:34:56.980 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 0 2024-08-17T17:34:56.980 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.980 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.980 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 0 2024-08-17T17:34:56.980 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.980 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.980 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 16384 2024-08-17T17:34:56.980 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.980 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.981 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 16384 2024-08-17T17:34:56.981 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.981 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.981 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:56.981 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:56.981 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 49152 2024-08-17T17:34:56.981 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.981 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.981 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 49152 2024-08-17T17:34:56.981 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.981 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.981 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 65536 2024-08-17T17:34:56.981 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.981 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.982 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 65536 2024-08-17T17:34:56.982 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.982 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.982 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:56.982 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:56.982 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 98304 2024-08-17T17:34:56.982 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.982 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.982 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 98304 2024-08-17T17:34:56.982 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.982 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.982 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 114688 2024-08-17T17:34:56.983 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.983 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.983 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 114688 2024-08-17T17:34:56.983 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.983 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.983 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:56.983 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:56.983 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 147456 2024-08-17T17:34:56.983 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.983 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.983 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 147456 2024-08-17T17:34:56.983 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.983 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.984 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 163840 2024-08-17T17:34:56.984 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.984 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.984 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 163840 2024-08-17T17:34:56.984 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.984 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.984 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:56.984 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:56.984 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 196608 2024-08-17T17:34:56.984 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.984 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.985 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 196608 2024-08-17T17:34:56.985 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.985 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.985 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 212992 2024-08-17T17:34:56.985 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.985 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.985 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 212992 2024-08-17T17:34:56.985 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.985 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.985 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:56.985 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:56.985 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:56.985 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:56.986 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:56.986 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:56.986 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:56.986 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.986 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.986 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:56.986 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:56.986 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:56.986 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:56.986 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:56.987 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.987 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.987 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:56.987 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:56.987 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:56.990 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:56.990 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:56.990 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:56.991 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:56.991 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:56.991 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:56.991 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:56.999 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:56.999 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:56.999 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.000 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.000 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.000 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.000 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:57.000 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.000 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.000 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:57.000 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.000 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.000 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.024 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.024 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.024 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.024 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.024 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.024 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.024 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.045 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.045 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.045 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.045 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.045 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.045 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.046 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.077 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.077 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.077 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.077 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.077 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.077 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.077 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.102 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.102 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.102 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.102 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.102 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.102 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.102 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:57.102 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.103 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.103 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:57.103 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.103 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.103 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.144 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.144 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.144 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.144 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.144 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.144 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.144 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.189 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.189 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.189 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.189 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.189 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.189 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.189 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.242 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.242 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.242 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.242 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.242 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.242 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.242 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.287 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.288 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.288 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.288 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.288 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.288 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.288 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:57.288 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.288 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.288 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:57.288 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.288 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.289 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.358 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.358 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.358 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.358 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.358 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.358 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.358 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.429 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.429 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.429 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.429 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.429 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.429 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.429 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.508 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.508 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.508 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.508 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.509 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.509 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.509 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.574 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.574 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.574 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.574 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.574 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.574 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.574 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:57.574 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.574 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.575 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:57.575 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.575 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.575 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.664 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.665 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.665 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.665 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.665 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.665 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.665 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.748 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.748 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.748 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.748 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.748 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.748 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.748 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.837 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.837 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.837 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.837 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.837 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.837 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.838 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:34:57.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:57 smithi179 ceph-mon[93519]: pgmap v685: 553 pgs: 553 active+clean; 75 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 6.7 MiB/s wr, 356 op/s 2024-08-17T17:34:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:57.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:57 smithi130 ceph-mon[105521]: pgmap v685: 553 pgs: 553 active+clean; 75 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 6.7 MiB/s wr, 356 op/s 2024-08-17T17:34:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:57 smithi130 ceph-mon[110068]: pgmap v685: 553 pgs: 553 active+clean; 75 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 6.7 MiB/s wr, 356 op/s 2024-08-17T17:34:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:57.935 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:34:57.935 INFO:tasks.workunit.client.0.smithi130.stdout:finished writesame 2024-08-17T17:34:57.935 INFO:tasks.workunit.client.0.smithi130.stdout:to verify the data 2024-08-17T17:34:57.935 INFO:tasks.workunit.client.0.smithi130.stdout:verified 2024-08-17T17:34:57.935 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.935 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.936 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:57.936 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:34:57.936 INFO:tasks.workunit.client.0.smithi130.stdout:started writesame 2024-08-17T17:34:57.936 INFO:tasks.workunit.client.0.smithi130.stdout:expected fail, finished writesame 2024-08-17T17:34:57.936 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestIOPPWithIOHint (1987 ms) 2024-08-17T17:34:57.936 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestIOToSnapshot 2024-08-17T17:34:57.936 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:34:59.603 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 0 2024-08-17T17:34:59.603 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 0 2024-08-17T17:34:59.603 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 1 2024-08-17T17:34:59.603 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:34:59.603 INFO:tasks.workunit.client.0.smithi130.stdout:snap: orig 2024-08-17T17:34:59.603 INFO:tasks.workunit.client.0.smithi130.stdout:found orig with size 2097152 2024-08-17T17:34:59.603 INFO:tasks.workunit.client.0.smithi130.stdout:read: 80 2024-08-17T17:34:59.603 INFO:tasks.workunit.client.0.smithi130.stdout:write test data! 2024-08-17T17:34:59.603 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 80 2024-08-17T17:34:59.603 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 2 2024-08-17T17:34:59.604 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 2 2024-08-17T17:34:59.604 INFO:tasks.workunit.client.0.smithi130.stdout:snap: orig 2024-08-17T17:34:59.604 INFO:tasks.workunit.client.0.smithi130.stdout:snap: written 2024-08-17T17:34:59.604 INFO:tasks.workunit.client.0.smithi130.stdout:found orig with size 2097152 2024-08-17T17:34:59.604 INFO:tasks.workunit.client.0.smithi130.stdout:found written with size 2097152 2024-08-17T17:34:59.604 INFO:tasks.workunit.client.0.smithi130.stdout:read: 80 2024-08-17T17:34:59.604 INFO:tasks.workunit.client.0.smithi130.stdout:read: 80 2024-08-17T17:34:59.604 INFO:tasks.workunit.client.0.smithi130.stdout:read: 80 2024-08-17T17:34:59.604 INFO:tasks.workunit.client.0.smithi130.stdout:write to snapshot returned -30 2024-08-17T17:34:59.604 INFO:tasks.workunit.client.0.smithi130.stdout:Read-only file system 2024-08-17T17:34:59.625 INFO:tasks.workunit.client.0.smithi130.stdout:read: 80 2024-08-17T17:34:59.625 INFO:tasks.workunit.client.0.smithi130.stdout:read: 80 2024-08-17T17:34:59.625 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 80 2024-08-17T17:34:59.625 INFO:tasks.workunit.client.0.smithi130.stdout:opening testimg@orig 2024-08-17T17:34:59.625 INFO:tasks.workunit.client.0.smithi130.stdout:read: 80 2024-08-17T17:34:59.626 INFO:tasks.workunit.client.0.smithi130.stdout:write to snapshot returned -30 2024-08-17T17:34:59.626 INFO:tasks.workunit.client.0.smithi130.stdout:Read-only file system 2024-08-17T17:34:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:59 smithi179 ceph-mon[93519]: pgmap v686: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 15 MiB/s wr, 2.41k op/s 2024-08-17T17:34:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:34:59 smithi179 ceph-mon[93519]: osdmap e676: 8 total, 8 up, 8 in 2024-08-17T17:34:59.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:59 smithi130 ceph-mon[110068]: pgmap v686: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 15 MiB/s wr, 2.41k op/s 2024-08-17T17:34:59.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:34:59 smithi130 ceph-mon[110068]: osdmap e676: 8 total, 8 up, 8 in 2024-08-17T17:34:59.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:59 smithi130 ceph-mon[105521]: pgmap v686: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 15 MiB/s wr, 2.41k op/s 2024-08-17T17:34:59.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:34:59 smithi130 ceph-mon[105521]: osdmap e676: 8 total, 8 up, 8 in 2024-08-17T17:34:59.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:34:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:34:59] "GET /metrics HTTP/1.1" 200 70196 "" "Prometheus/2.51.0" 2024-08-17T17:35:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:35:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:35:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:35:00.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:00 smithi130 ceph-mon[110068]: osdmap e677: 8 total, 8 up, 8 in 2024-08-17T17:35:00.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:00 smithi130 ceph-mon[110068]: osdmap e678: 8 total, 8 up, 8 in 2024-08-17T17:35:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:00 smithi130 ceph-mon[105521]: osdmap e677: 8 total, 8 up, 8 in 2024-08-17T17:35:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:00 smithi130 ceph-mon[105521]: osdmap e678: 8 total, 8 up, 8 in 2024-08-17T17:35:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:00 smithi179 ceph-mon[93519]: osdmap e677: 8 total, 8 up, 8 in 2024-08-17T17:35:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:00 smithi179 ceph-mon[93519]: osdmap e678: 8 total, 8 up, 8 in 2024-08-17T17:35:01.459 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 2 2024-08-17T17:35:01.459 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 2 2024-08-17T17:35:01.459 INFO:tasks.workunit.client.0.smithi130.stdout:snap: orig 2024-08-17T17:35:01.459 INFO:tasks.workunit.client.0.smithi130.stdout:snap: written 2024-08-17T17:35:01.459 INFO:tasks.workunit.client.0.smithi130.stdout:found orig with size 2097152 2024-08-17T17:35:01.459 INFO:tasks.workunit.client.0.smithi130.stdout:found written with size 2097152 2024-08-17T17:35:01.460 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 1 2024-08-17T17:35:01.460 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:35:01.460 INFO:tasks.workunit.client.0.smithi130.stdout:snap: orig 2024-08-17T17:35:01.460 INFO:tasks.workunit.client.0.smithi130.stdout:found orig with size 2097152 2024-08-17T17:35:01.460 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 0 2024-08-17T17:35:01.460 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 0 2024-08-17T17:35:01.460 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestIOToSnapshot (3524 ms) 2024-08-17T17:35:01.460 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestSnapshotDeletedIo 2024-08-17T17:35:01.460 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:01.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:01 smithi130 ceph-mon[105521]: pgmap v689: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 15 MiB/s wr, 2.43k op/s 2024-08-17T17:35:01.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:01 smithi130 ceph-mon[105521]: osdmap e679: 8 total, 8 up, 8 in 2024-08-17T17:35:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:01 smithi130 ceph-mon[110068]: pgmap v689: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 15 MiB/s wr, 2.43k op/s 2024-08-17T17:35:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:01 smithi130 ceph-mon[110068]: osdmap e679: 8 total, 8 up, 8 in 2024-08-17T17:35:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:01 smithi179 ceph-mon[93519]: pgmap v689: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 15 MiB/s wr, 2.43k op/s 2024-08-17T17:35:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:01 smithi179 ceph-mon[93519]: osdmap e679: 8 total, 8 up, 8 in 2024-08-17T17:35:03.467 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestSnapshotDeletedIo (2008 ms) 2024-08-17T17:35:03.467 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestClone 2024-08-17T17:35:03.467 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:03 smithi179 ceph-mon[93519]: pgmap v692: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 2.7 KiB/s wr, 28 op/s 2024-08-17T17:35:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:03 smithi179 ceph-mon[93519]: osdmap e680: 8 total, 8 up, 8 in 2024-08-17T17:35:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:03 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:03.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:03 smithi130 ceph-mon[105521]: pgmap v692: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 2.7 KiB/s wr, 28 op/s 2024-08-17T17:35:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:03 smithi130 ceph-mon[105521]: osdmap e680: 8 total, 8 up, 8 in 2024-08-17T17:35:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:03 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:03 smithi130 ceph-mon[110068]: pgmap v692: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 2.7 KiB/s wr, 28 op/s 2024-08-17T17:35:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:03 smithi130 ceph-mon[110068]: osdmap e680: 8 total, 8 up, 8 in 2024-08-17T17:35:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:03 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:04.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:04 smithi130 ceph-mon[105521]: osdmap e681: 8 total, 8 up, 8 in 2024-08-17T17:35:04.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:04 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:35:04.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:04 smithi130 ceph-mon[110068]: osdmap e681: 8 total, 8 up, 8 in 2024-08-17T17:35:04.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:04 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:35:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:04 smithi179 ceph-mon[93519]: osdmap e681: 8 total, 8 up, 8 in 2024-08-17T17:35:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:04 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:35:05.464 INFO:tasks.workunit.client.0.smithi130.stdout:made parent image "parent" 2024-08-17T17:35:05.464 INFO:tasks.workunit.client.0.smithi130.stdout:parent has no parent info 2024-08-17T17:35:05.464 INFO:tasks.workunit.client.0.smithi130.stdout:made snapshot "parent@parent_snap" 2024-08-17T17:35:05.464 INFO:tasks.workunit.client.0.smithi130.stdout:can't unprotect an unprotected snap 2024-08-17T17:35:05.464 INFO:tasks.workunit.client.0.smithi130.stdout:can't protect a protected snap 2024-08-17T17:35:05.464 INFO:tasks.workunit.client.0.smithi130.stdout:made and opened clone "child" 2024-08-17T17:35:05.464 INFO:tasks.workunit.client.0.smithi130.stdout:read: 8 2024-08-17T17:35:05.464 INFO:tasks.workunit.client.0.smithi130.stdout:read: 8 2024-08-17T17:35:05.465 INFO:tasks.workunit.client.0.smithi130.stdout:read: 8 2024-08-17T17:35:05.465 INFO:tasks.workunit.client.0.smithi130.stdout:sizes and overlaps are good between parent and child 2024-08-17T17:35:05.465 INFO:tasks.workunit.client.0.smithi130.stdout:child image successfully cloned all image-meta pairs 2024-08-17T17:35:05.465 INFO:tasks.workunit.client.0.smithi130.stdout:sized down clone, changed overlap 2024-08-17T17:35:05.465 INFO:tasks.workunit.client.0.smithi130.stdout:parent info: size 4194304 obj_size 4194304 parent_pool 18446744073709551615 2024-08-17T17:35:05.465 INFO:tasks.workunit.client.0.smithi130.stdout:sized up clone, changed size but not overlap or parent's size 2024-08-17T17:35:05.465 INFO:tasks.workunit.client.0.smithi130.stdout:can't remove parent while child still exists 2024-08-17T17:35:05.465 INFO:tasks.workunit.client.0.smithi130.stdout:can't remove parent while still protected 2024-08-17T17:35:05.465 INFO:tasks.workunit.client.0.smithi130.stdout:removed parent snap after unprotecting 2024-08-17T17:35:05.465 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestClone (1997 ms) 2024-08-17T17:35:05.465 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestClone2 2024-08-17T17:35:05.465 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:05.675 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:05 smithi179 ceph-mon[93519]: pgmap v695: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 181 op/s 2024-08-17T17:35:05.675 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:05 smithi179 ceph-mon[93519]: osdmap e682: 8 total, 8 up, 8 in 2024-08-17T17:35:05.675 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:05 smithi179 ceph-mon[93519]: osdmap e683: 8 total, 8 up, 8 in 2024-08-17T17:35:05.816 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:05 smithi130 ceph-mon[105521]: pgmap v695: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 181 op/s 2024-08-17T17:35:05.816 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:05 smithi130 ceph-mon[105521]: osdmap e682: 8 total, 8 up, 8 in 2024-08-17T17:35:05.816 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:05 smithi130 ceph-mon[105521]: osdmap e683: 8 total, 8 up, 8 in 2024-08-17T17:35:05.817 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:05 smithi130 ceph-mon[110068]: pgmap v695: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 181 op/s 2024-08-17T17:35:05.817 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:05 smithi130 ceph-mon[110068]: osdmap e682: 8 total, 8 up, 8 in 2024-08-17T17:35:05.817 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:05 smithi130 ceph-mon[110068]: osdmap e683: 8 total, 8 up, 8 in 2024-08-17T17:35:06.584 INFO:tasks.workunit.client.0.smithi130.stdout:made parent image "parent" 2024-08-17T17:35:06.584 INFO:tasks.workunit.client.0.smithi130.stdout:parent has no parent info 2024-08-17T17:35:06.584 INFO:tasks.workunit.client.0.smithi130.stdout:made snapshot "parent@parent_snap" 2024-08-17T17:35:06.584 INFO:tasks.workunit.client.0.smithi130.stdout:made and opened clone "child" 2024-08-17T17:35:06.585 INFO:tasks.workunit.client.0.smithi130.stdout:child image successfully cloned all image-meta pairs 2024-08-17T17:35:06.585 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestClone2 (1120 ms) 2024-08-17T17:35:06.585 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ListChildren 2024-08-17T17:35:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:07 smithi179 ceph-mon[93519]: pgmap v698: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 12 KiB/s wr, 178 op/s 2024-08-17T17:35:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:07 smithi179 ceph-mon[93519]: osdmap e684: 8 total, 8 up, 8 in 2024-08-17T17:35:07.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:07 smithi130 ceph-mon[105521]: pgmap v698: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 12 KiB/s wr, 178 op/s 2024-08-17T17:35:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:07 smithi130 ceph-mon[105521]: osdmap e684: 8 total, 8 up, 8 in 2024-08-17T17:35:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:07 smithi130 ceph-mon[110068]: pgmap v698: 553 pgs: 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 12 KiB/s wr, 178 op/s 2024-08-17T17:35:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:07 smithi130 ceph-mon[110068]: osdmap e684: 8 total, 8 up, 8 in 2024-08-17T17:35:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:08 smithi179 ceph-mon[93519]: osdmap e685: 8 total, 8 up, 8 in 2024-08-17T17:35:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:08 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2345862125' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:08 smithi130 ceph-mon[105521]: osdmap e685: 8 total, 8 up, 8 in 2024-08-17T17:35:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:08 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2345862125' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:08.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:08 smithi130 ceph-mon[110068]: osdmap e685: 8 total, 8 up, 8 in 2024-08-17T17:35:08.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:08 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2345862125' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:08.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:08.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[105521]: pgmap v701: 585 pgs: 19 creating+peering, 13 unknown, 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 380 KiB/s rd, 30 KiB/s wr, 464 op/s 2024-08-17T17:35:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2345862125' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[105521]: osdmap e686: 8 total, 8 up, 8 in 2024-08-17T17:35:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:35:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:35:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:35:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:09.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[105521]: osdmap e687: 8 total, 8 up, 8 in 2024-08-17T17:35:09.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3857164110' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:09.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[110068]: pgmap v701: 585 pgs: 19 creating+peering, 13 unknown, 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 380 KiB/s rd, 30 KiB/s wr, 464 op/s 2024-08-17T17:35:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2345862125' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[110068]: osdmap e686: 8 total, 8 up, 8 in 2024-08-17T17:35:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:35:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:35:09.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:35:09.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:09.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[110068]: osdmap e687: 8 total, 8 up, 8 in 2024-08-17T17:35:09.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3857164110' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:09.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:09 smithi179 ceph-mon[93519]: pgmap v701: 585 pgs: 19 creating+peering, 13 unknown, 553 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 380 KiB/s rd, 30 KiB/s wr, 464 op/s 2024-08-17T17:35:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:09 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2345862125' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:09 smithi179 ceph-mon[93519]: osdmap e686: 8 total, 8 up, 8 in 2024-08-17T17:35:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:09 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:35:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:09 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:09 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:09 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:35:09.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:09 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:35:09.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:09 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:09.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:09 smithi179 ceph-mon[93519]: osdmap e687: 8 total, 8 up, 8 in 2024-08-17T17:35:09.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:09 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3857164110' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:09.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:09 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:35:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:35:09] "GET /metrics HTTP/1.1" 200 70196 "" "Prometheus/2.51.0" 2024-08-17T17:35:10.447 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:35:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:35:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:35:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:11 smithi179 ceph-mon[93519]: pgmap v704: 617 pgs: 21 creating+peering, 33 unknown, 563 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 676 KiB/s rd, 48 KiB/s wr, 809 op/s 2024-08-17T17:35:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:11 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:11 smithi179 ceph-mon[93519]: osdmap e688: 8 total, 8 up, 8 in 2024-08-17T17:35:11.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:11 smithi130 ceph-mon[105521]: pgmap v704: 617 pgs: 21 creating+peering, 33 unknown, 563 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 676 KiB/s rd, 48 KiB/s wr, 809 op/s 2024-08-17T17:35:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:11 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:11 smithi130 ceph-mon[105521]: osdmap e688: 8 total, 8 up, 8 in 2024-08-17T17:35:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:11 smithi130 ceph-mon[110068]: pgmap v704: 617 pgs: 21 creating+peering, 33 unknown, 563 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 676 KiB/s rd, 48 KiB/s wr, 809 op/s 2024-08-17T17:35:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:11 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:11 smithi130 ceph-mon[110068]: osdmap e688: 8 total, 8 up, 8 in 2024-08-17T17:35:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:12 smithi179 ceph-mon[93519]: osdmap e689: 8 total, 8 up, 8 in 2024-08-17T17:35:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:12.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:12 smithi130 ceph-mon[105521]: osdmap e689: 8 total, 8 up, 8 in 2024-08-17T17:35:12.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:12 smithi130 ceph-mon[110068]: osdmap e689: 8 total, 8 up, 8 in 2024-08-17T17:35:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:13.457 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:35:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=sqlstore.transactions t=2024-08-17T17:35:13.132465044Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T17:35:13.457 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:35:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=cleanup t=2024-08-17T17:35:13.133708821Z level=info msg="Completed cleanup jobs" duration=12.573951ms 2024-08-17T17:35:13.723 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:13 smithi179 ceph-mon[93519]: pgmap v707: 617 pgs: 21 creating+peering, 28 unknown, 568 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 328 KiB/s rd, 20 KiB/s wr, 380 op/s 2024-08-17T17:35:13.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:13 smithi179 ceph-mon[93519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:35:13.724 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:13 smithi179 ceph-mon[93519]: osdmap e690: 8 total, 8 up, 8 in 2024-08-17T17:35:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:13 smithi130 ceph-mon[110068]: pgmap v707: 617 pgs: 21 creating+peering, 28 unknown, 568 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 328 KiB/s rd, 20 KiB/s wr, 380 op/s 2024-08-17T17:35:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:13 smithi130 ceph-mon[110068]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:35:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:13 smithi130 ceph-mon[110068]: osdmap e690: 8 total, 8 up, 8 in 2024-08-17T17:35:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:13 smithi130 ceph-mon[105521]: pgmap v707: 617 pgs: 21 creating+peering, 28 unknown, 568 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 328 KiB/s rd, 20 KiB/s wr, 380 op/s 2024-08-17T17:35:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:13 smithi130 ceph-mon[105521]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:35:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:13 smithi130 ceph-mon[105521]: osdmap e690: 8 total, 8 up, 8 in 2024-08-17T17:35:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:14 smithi179 ceph-mon[93519]: osdmap e691: 8 total, 8 up, 8 in 2024-08-17T17:35:14.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:14 smithi130 ceph-mon[105521]: osdmap e691: 8 total, 8 up, 8 in 2024-08-17T17:35:14.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:14 smithi130 ceph-mon[110068]: osdmap e691: 8 total, 8 up, 8 in 2024-08-17T17:35:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:15 smithi179 ceph-mon[93519]: pgmap v710: 617 pgs: 2 creating+peering, 615 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 3.7 KiB/s wr, 47 op/s 2024-08-17T17:35:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:15 smithi179 ceph-mon[93519]: osdmap e692: 8 total, 8 up, 8 in 2024-08-17T17:35:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:15 smithi179 ceph-mon[93519]: osdmap e693: 8 total, 8 up, 8 in 2024-08-17T17:35:15.906 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.906 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.906 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.906 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.906 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.906 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 1 2024-08-17T17:35:15.906 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:35:15.906 INFO:tasks.workunit.client.0.smithi130.stdout:child: image108 2024-08-17T17:35:15.907 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image108/d5132543ad0e 2024-08-17T17:35:15.907 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.907 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.907 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.907 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.907 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.907 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.908 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.908 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.908 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.908 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.908 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.909 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.909 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 2 2024-08-17T17:35:15.909 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 2 2024-08-17T17:35:15.909 INFO:tasks.workunit.client.0.smithi130.stdout:child: image109 2024-08-17T17:35:15.909 INFO:tasks.workunit.client.0.smithi130.stdout:child: image108 2024-08-17T17:35:15.909 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image108/d5132543ad0e 2024-08-17T17:35:15.909 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.909 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109/d513a4ef6e9c 2024-08-17T17:35:15.909 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.909 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.909 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.909 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.910 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.910 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.910 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.910 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.910 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.910 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.911 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.911 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.911 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.911 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-31/image110 2024-08-17T17:35:15.911 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.911 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.911 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image110 2024-08-17T17:35:15.911 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image110 2024-08-17T17:35:15.911 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.912 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 3 2024-08-17T17:35:15.912 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 3 2024-08-17T17:35:15.912 INFO:tasks.workunit.client.0.smithi130.stdout:child: image109 2024-08-17T17:35:15.912 INFO:tasks.workunit.client.0.smithi130.stdout:child: image108 2024-08-17T17:35:15.912 INFO:tasks.workunit.client.0.smithi130.stdout:child: image110 2024-08-17T17:35:15.912 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image108/d5132543ad0e 2024-08-17T17:35:15.912 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.912 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109/d513a4ef6e9c 2024-08-17T17:35:15.912 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.912 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image110/d5133154689d 2024-08-17T17:35:15.912 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.912 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.912 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.912 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.913 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.913 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.913 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.913 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.913 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.913 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.913 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.913 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.913 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 3 2024-08-17T17:35:15.913 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 3 2024-08-17T17:35:15.913 INFO:tasks.workunit.client.0.smithi130.stdout:child: image109 2024-08-17T17:35:15.913 INFO:tasks.workunit.client.0.smithi130.stdout:child: image108 2024-08-17T17:35:15.913 INFO:tasks.workunit.client.0.smithi130.stdout:child: image110 2024-08-17T17:35:15.913 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image108/d5132543ad0e 2024-08-17T17:35:15.914 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.914 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109/d513a4ef6e9c 2024-08-17T17:35:15.914 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.914 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image110/d5133154689d 2024-08-17T17:35:15.914 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.914 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.914 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.914 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.914 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.914 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.914 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.914 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.914 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.915 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.915 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.915 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.915 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.915 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-31/image111 2024-08-17T17:35:15.915 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.915 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.915 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image111 2024-08-17T17:35:15.915 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image111 2024-08-17T17:35:15.915 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.916 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 4 2024-08-17T17:35:15.916 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 4 2024-08-17T17:35:15.916 INFO:tasks.workunit.client.0.smithi130.stdout:child: image109 2024-08-17T17:35:15.916 INFO:tasks.workunit.client.0.smithi130.stdout:child: image108 2024-08-17T17:35:15.916 INFO:tasks.workunit.client.0.smithi130.stdout:child: image110 2024-08-17T17:35:15.916 INFO:tasks.workunit.client.0.smithi130.stdout:child: image111 2024-08-17T17:35:15.916 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image108/d5132543ad0e 2024-08-17T17:35:15.916 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.916 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109/d513a4ef6e9c 2024-08-17T17:35:15.916 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.916 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image110/d5133154689d 2024-08-17T17:35:15.916 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.916 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image111/d5135a2afb7e 2024-08-17T17:35:15.916 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.917 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.917 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.917 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.917 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.917 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.917 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.917 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.917 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.917 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.917 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.917 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.917 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.918 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-31/image110 2024-08-17T17:35:15.918 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.918 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:15.918 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image110 2024-08-17T17:35:15.918 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image110 2024-08-17T17:35:15.918 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.918 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:15.918 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-31/image111 2024-08-17T17:35:15.918 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:15.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:15 smithi130 ceph-mon[110068]: pgmap v710: 617 pgs: 2 creating+peering, 615 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 3.7 KiB/s wr, 47 op/s 2024-08-17T17:35:15.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:15 smithi130 ceph-mon[110068]: osdmap e692: 8 total, 8 up, 8 in 2024-08-17T17:35:15.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:15 smithi130 ceph-mon[110068]: osdmap e693: 8 total, 8 up, 8 in 2024-08-17T17:35:15.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:15 smithi130 ceph-mon[105521]: pgmap v710: 617 pgs: 2 creating+peering, 615 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 3.7 KiB/s wr, 47 op/s 2024-08-17T17:35:15.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:15 smithi130 ceph-mon[105521]: osdmap e692: 8 total, 8 up, 8 in 2024-08-17T17:35:15.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:15 smithi130 ceph-mon[105521]: osdmap e693: 8 total, 8 up, 8 in 2024-08-17T17:35:16.553 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image108 2024-08-17T17:35:16.553 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image110 2024-08-17T17:35:16.553 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image111 2024-08-17T17:35:16.553 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image111 2024-08-17T17:35:16.554 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.554 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 4 2024-08-17T17:35:16.554 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 4 2024-08-17T17:35:16.554 INFO:tasks.workunit.client.0.smithi130.stdout:child: image109 2024-08-17T17:35:16.554 INFO:tasks.workunit.client.0.smithi130.stdout:child: image108 2024-08-17T17:35:16.554 INFO:tasks.workunit.client.0.smithi130.stdout:child: image110 2024-08-17T17:35:16.554 INFO:tasks.workunit.client.0.smithi130.stdout:child: image111 2024-08-17T17:35:16.554 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image108/d5132543ad0e 2024-08-17T17:35:16.554 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.554 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109/d513a4ef6e9c 2024-08-17T17:35:16.554 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.554 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image110/d5133154689d 2024-08-17T17:35:16.554 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.554 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image111/d5135a2afb7e 2024-08-17T17:35:16.555 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.555 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.555 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:16.555 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:16.555 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:16.555 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.555 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.555 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-31/image110 2024-08-17T17:35:16.555 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:16.555 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image110 2024-08-17T17:35:16.555 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image110 2024-08-17T17:35:16.555 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-31/image111 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image110 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image111 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image111 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 3 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 3 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout:child: image109 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout:child: image110 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout:child: image111 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109/d513a4ef6e9c 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.556 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image110/d5133154689d 2024-08-17T17:35:16.557 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.557 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image111/d5135a2afb7e 2024-08-17T17:35:16.557 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.557 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.557 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:16.557 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:16.557 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:16.557 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.557 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.557 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-31/image111 2024-08-17T17:35:16.557 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:16.557 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-31/image111 2024-08-17T17:35:16.557 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image111 2024-08-17T17:35:16.558 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.558 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 2 2024-08-17T17:35:16.558 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 2 2024-08-17T17:35:16.558 INFO:tasks.workunit.client.0.smithi130.stdout:child: image109 2024-08-17T17:35:16.558 INFO:tasks.workunit.client.0.smithi130.stdout:child: image111 2024-08-17T17:35:16.558 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109/d513a4ef6e9c 2024-08-17T17:35:16.558 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.558 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-31/image111/d5135a2afb7e 2024-08-17T17:35:16.558 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.558 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.558 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:16.558 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:16.558 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109 2024-08-17T17:35:16.558 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.559 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 1 2024-08-17T17:35:16.559 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:35:16.559 INFO:tasks.workunit.client.0.smithi130.stdout:child: image109 2024-08-17T17:35:16.559 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-30/image109/d513a4ef6e9c 2024-08-17T17:35:16.559 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:16.559 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 0 2024-08-17T17:35:16.559 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 0 2024-08-17T17:35:16.559 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ListChildren (9969 ms) 2024-08-17T17:35:16.559 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ListChildrenTiered 2024-08-17T17:35:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:17 smithi179 ceph-mon[93519]: pgmap v713: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 1.5 KiB/s wr, 11 op/s 2024-08-17T17:35:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:17 smithi179 ceph-mon[93519]: osdmap e694: 8 total, 8 up, 8 in 2024-08-17T17:35:17.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:17 smithi130 ceph-mon[105521]: pgmap v713: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 1.5 KiB/s wr, 11 op/s 2024-08-17T17:35:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:17 smithi130 ceph-mon[105521]: osdmap e694: 8 total, 8 up, 8 in 2024-08-17T17:35:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:17 smithi130 ceph-mon[110068]: pgmap v713: 617 pgs: 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 1.5 KiB/s wr, 11 op/s 2024-08-17T17:35:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:17 smithi130 ceph-mon[110068]: osdmap e694: 8 total, 8 up, 8 in 2024-08-17T17:35:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:18 smithi179 ceph-mon[93519]: osdmap e695: 8 total, 8 up, 8 in 2024-08-17T17:35:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2541924935' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-32","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:18 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:18.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:18 smithi130 ceph-mon[105521]: osdmap e695: 8 total, 8 up, 8 in 2024-08-17T17:35:18.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2541924935' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-32","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:18.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:18 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:18.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:18 smithi130 ceph-mon[110068]: osdmap e695: 8 total, 8 up, 8 in 2024-08-17T17:35:18.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2541924935' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-32","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:18.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:18 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:19.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:19 smithi130 ceph-mon[105521]: pgmap v716: 649 pgs: 12 creating+peering, 20 unknown, 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 582 KiB/s rd, 13 KiB/s wr, 708 op/s 2024-08-17T17:35:19.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2541924935' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:19.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:19 smithi130 ceph-mon[105521]: osdmap e696: 8 total, 8 up, 8 in 2024-08-17T17:35:19.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:19 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:35:19.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:19 smithi130 ceph-mon[110068]: pgmap v716: 649 pgs: 12 creating+peering, 20 unknown, 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 582 KiB/s rd, 13 KiB/s wr, 708 op/s 2024-08-17T17:35:19.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2541924935' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:19.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:19 smithi130 ceph-mon[110068]: osdmap e696: 8 total, 8 up, 8 in 2024-08-17T17:35:19.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:19 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:35:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:19 smithi179 ceph-mon[93519]: pgmap v716: 649 pgs: 12 creating+peering, 20 unknown, 617 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 582 KiB/s rd, 13 KiB/s wr, 708 op/s 2024-08-17T17:35:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2541924935' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:19 smithi179 ceph-mon[93519]: osdmap e696: 8 total, 8 up, 8 in 2024-08-17T17:35:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:19 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:35:20.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:35:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:35:19] "GET /metrics HTTP/1.1" 200 74912 "" "Prometheus/2.51.0" 2024-08-17T17:35:20.519 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:35:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:35:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:35:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:20 smithi179 ceph-mon[93519]: osdmap e697: 8 total, 8 up, 8 in 2024-08-17T17:35:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1707655426' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:20 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:20 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:20 smithi179 ceph-mon[93519]: osdmap e698: 8 total, 8 up, 8 in 2024-08-17T17:35:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:20 smithi130 ceph-mon[110068]: osdmap e697: 8 total, 8 up, 8 in 2024-08-17T17:35:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1707655426' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:20 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:20 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:20 smithi130 ceph-mon[110068]: osdmap e698: 8 total, 8 up, 8 in 2024-08-17T17:35:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:20 smithi130 ceph-mon[105521]: osdmap e697: 8 total, 8 up, 8 in 2024-08-17T17:35:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1707655426' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:20 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:20.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:20 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:20.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:20 smithi130 ceph-mon[105521]: osdmap e698: 8 total, 8 up, 8 in 2024-08-17T17:35:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:21 smithi179 ceph-mon[93519]: pgmap v719: 681 pgs: 18 creating+peering, 31 unknown, 632 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 588 KiB/s rd, 15 KiB/s wr, 718 op/s 2024-08-17T17:35:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:21 smithi179 ceph-mon[93519]: osdmap e699: 8 total, 8 up, 8 in 2024-08-17T17:35:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:21 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2837514608' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:21 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:21.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:21 smithi130 ceph-mon[110068]: pgmap v719: 681 pgs: 18 creating+peering, 31 unknown, 632 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 588 KiB/s rd, 15 KiB/s wr, 718 op/s 2024-08-17T17:35:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:21 smithi130 ceph-mon[110068]: osdmap e699: 8 total, 8 up, 8 in 2024-08-17T17:35:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:21 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2837514608' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:21 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:21 smithi130 ceph-mon[105521]: pgmap v719: 681 pgs: 18 creating+peering, 31 unknown, 632 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 588 KiB/s rd, 15 KiB/s wr, 718 op/s 2024-08-17T17:35:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:21 smithi130 ceph-mon[105521]: osdmap e699: 8 total, 8 up, 8 in 2024-08-17T17:35:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:21 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2837514608' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:21 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:35:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:23 smithi179 ceph-mon[93519]: pgmap v722: 713 pgs: 19 creating+peering, 52 unknown, 642 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 1.5 KiB/s wr, 9 op/s 2024-08-17T17:35:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:23 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:23 smithi179 ceph-mon[93519]: osdmap e700: 8 total, 8 up, 8 in 2024-08-17T17:35:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:23 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34", "force_nonempty":""} : dispatch 2024-08-17T17:35:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:23 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34", "force_nonempty":""} : dispatch 2024-08-17T17:35:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:23 smithi130 ceph-mon[105521]: pgmap v722: 713 pgs: 19 creating+peering, 52 unknown, 642 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 1.5 KiB/s wr, 9 op/s 2024-08-17T17:35:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:23 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:23 smithi130 ceph-mon[105521]: osdmap e700: 8 total, 8 up, 8 in 2024-08-17T17:35:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:23 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34", "force_nonempty":""} : dispatch 2024-08-17T17:35:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:23 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34", "force_nonempty":""} : dispatch 2024-08-17T17:35:23.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:23 smithi130 ceph-mon[110068]: pgmap v722: 713 pgs: 19 creating+peering, 52 unknown, 642 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 1.5 KiB/s wr, 9 op/s 2024-08-17T17:35:23.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:23 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:35:23.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:23 smithi130 ceph-mon[110068]: osdmap e700: 8 total, 8 up, 8 in 2024-08-17T17:35:23.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:23 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34", "force_nonempty":""} : dispatch 2024-08-17T17:35:23.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:23 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34", "force_nonempty":""} : dispatch 2024-08-17T17:35:24.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:24 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34", "force_nonempty":""}]': finished 2024-08-17T17:35:24.731 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:24 smithi179 ceph-mon[93519]: osdmap e701: 8 total, 8 up, 8 in 2024-08-17T17:35:24.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:24 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi130-87148-34", "mode":"writeback"} : dispatch 2024-08-17T17:35:24.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:24 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi130-87148-34", "mode":"writeback"} : dispatch 2024-08-17T17:35:24.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:24 smithi179 ceph-mon[93519]: Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-08-17T17:35:24.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:24 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi130-87148-34", "mode":"writeback"}]': finished 2024-08-17T17:35:24.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:24 smithi179 ceph-mon[93519]: osdmap e702: 8 total, 8 up, 8 in 2024-08-17T17:35:24.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:24 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi130-87148-32", "overlaypool":"test-librbd-smithi130-87148-34"} : dispatch 2024-08-17T17:35:24.732 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:24 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi130-87148-32", "overlaypool":"test-librbd-smithi130-87148-34"} : dispatch 2024-08-17T17:35:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34", "force_nonempty":""}]': finished 2024-08-17T17:35:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[110068]: osdmap e701: 8 total, 8 up, 8 in 2024-08-17T17:35:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi130-87148-34", "mode":"writeback"} : dispatch 2024-08-17T17:35:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi130-87148-34", "mode":"writeback"} : dispatch 2024-08-17T17:35:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[110068]: Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-08-17T17:35:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi130-87148-34", "mode":"writeback"}]': finished 2024-08-17T17:35:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[110068]: osdmap e702: 8 total, 8 up, 8 in 2024-08-17T17:35:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi130-87148-32", "overlaypool":"test-librbd-smithi130-87148-34"} : dispatch 2024-08-17T17:35:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi130-87148-32", "overlaypool":"test-librbd-smithi130-87148-34"} : dispatch 2024-08-17T17:35:24.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34", "force_nonempty":""}]': finished 2024-08-17T17:35:24.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[105521]: osdmap e701: 8 total, 8 up, 8 in 2024-08-17T17:35:24.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi130-87148-34", "mode":"writeback"} : dispatch 2024-08-17T17:35:24.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi130-87148-34", "mode":"writeback"} : dispatch 2024-08-17T17:35:24.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[105521]: Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-08-17T17:35:24.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi130-87148-34", "mode":"writeback"}]': finished 2024-08-17T17:35:24.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[105521]: osdmap e702: 8 total, 8 up, 8 in 2024-08-17T17:35:24.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi130-87148-32", "overlaypool":"test-librbd-smithi130-87148-34"} : dispatch 2024-08-17T17:35:24.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:24 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi130-87148-32", "overlaypool":"test-librbd-smithi130-87148-34"} : dispatch 2024-08-17T17:35:25.448 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:25 smithi179 ceph-mon[93519]: pgmap v725: 713 pgs: 7 creating+peering, 6 unknown, 700 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:35:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:25 smithi179 ceph-mon[93519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:35:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:25 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi130-87148-32", "overlaypool":"test-librbd-smithi130-87148-34"}]': finished 2024-08-17T17:35:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:25 smithi179 ceph-mon[93519]: osdmap e703: 8 total, 8 up, 8 in 2024-08-17T17:35:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:25 smithi130 ceph-mon[110068]: pgmap v725: 713 pgs: 7 creating+peering, 6 unknown, 700 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:35:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:25 smithi130 ceph-mon[110068]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:35:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:25 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi130-87148-32", "overlaypool":"test-librbd-smithi130-87148-34"}]': finished 2024-08-17T17:35:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:25 smithi130 ceph-mon[110068]: osdmap e703: 8 total, 8 up, 8 in 2024-08-17T17:35:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:25 smithi130 ceph-mon[105521]: pgmap v725: 713 pgs: 7 creating+peering, 6 unknown, 700 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:35:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:25 smithi130 ceph-mon[105521]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:35:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:25 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi130-87148-32", "overlaypool":"test-librbd-smithi130-87148-34"}]': finished 2024-08-17T17:35:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:25 smithi130 ceph-mon[105521]: osdmap e703: 8 total, 8 up, 8 in 2024-08-17T17:35:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:27 smithi179 ceph-mon[93519]: pgmap v728: 713 pgs: 1 creating+peering, 712 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:35:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:27 smithi179 ceph-mon[93519]: osdmap e704: 8 total, 8 up, 8 in 2024-08-17T17:35:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:27.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:27 smithi130 ceph-mon[105521]: pgmap v728: 713 pgs: 1 creating+peering, 712 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:35:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:27 smithi130 ceph-mon[105521]: osdmap e704: 8 total, 8 up, 8 in 2024-08-17T17:35:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:27 smithi130 ceph-mon[110068]: pgmap v728: 713 pgs: 1 creating+peering, 712 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:35:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:27 smithi130 ceph-mon[110068]: osdmap e704: 8 total, 8 up, 8 in 2024-08-17T17:35:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:28 smithi179 ceph-mon[93519]: osdmap e705: 8 total, 8 up, 8 in 2024-08-17T17:35:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:28.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:28 smithi130 ceph-mon[110068]: osdmap e705: 8 total, 8 up, 8 in 2024-08-17T17:35:28.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:28 smithi130 ceph-mon[105521]: osdmap e705: 8 total, 8 up, 8 in 2024-08-17T17:35:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:29.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:29 smithi130 ceph-mon[105521]: pgmap v731: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 1.5 KiB/s wr, 11 op/s 2024-08-17T17:35:29.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:29 smithi130 ceph-mon[105521]: osdmap e706: 8 total, 8 up, 8 in 2024-08-17T17:35:29.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:29 smithi130 ceph-mon[110068]: pgmap v731: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 1.5 KiB/s wr, 11 op/s 2024-08-17T17:35:29.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:29 smithi130 ceph-mon[110068]: osdmap e706: 8 total, 8 up, 8 in 2024-08-17T17:35:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:29 smithi179 ceph-mon[93519]: pgmap v731: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 1.5 KiB/s wr, 11 op/s 2024-08-17T17:35:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:29 smithi179 ceph-mon[93519]: osdmap e706: 8 total, 8 up, 8 in 2024-08-17T17:35:30.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:35:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:35:29] "GET /metrics HTTP/1.1" 200 82165 "" "Prometheus/2.51.0" 2024-08-17T17:35:30.534 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:35:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:35:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:35:30.839 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:30 smithi130 ceph-mon[110068]: osdmap e707: 8 total, 8 up, 8 in 2024-08-17T17:35:30.839 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:30 smithi130 ceph-mon[105521]: osdmap e707: 8 total, 8 up, 8 in 2024-08-17T17:35:30.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:30 smithi179 ceph-mon[93519]: osdmap e707: 8 total, 8 up, 8 in 2024-08-17T17:35:31.022 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.022 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.022 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.022 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.022 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.023 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 1 2024-08-17T17:35:31.023 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:35:31.023 INFO:tasks.workunit.client.0.smithi130.stdout:child: image113 2024-08-17T17:35:31.023 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image113/d51316f69efa 2024-08-17T17:35:31.023 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.023 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.023 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.023 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.024 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.024 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.024 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.024 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.024 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.024 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.024 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.024 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.024 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 2 2024-08-17T17:35:31.024 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 2 2024-08-17T17:35:31.024 INFO:tasks.workunit.client.0.smithi130.stdout:child: image114 2024-08-17T17:35:31.024 INFO:tasks.workunit.client.0.smithi130.stdout:child: image113 2024-08-17T17:35:31.024 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image113/d51316f69efa 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114/d513e8224e2c 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.025 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.026 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-33/image115 2024-08-17T17:35:31.026 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.026 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.026 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image115 2024-08-17T17:35:31.026 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image115 2024-08-17T17:35:31.026 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.026 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 3 2024-08-17T17:35:31.026 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 3 2024-08-17T17:35:31.026 INFO:tasks.workunit.client.0.smithi130.stdout:child: image114 2024-08-17T17:35:31.026 INFO:tasks.workunit.client.0.smithi130.stdout:child: image113 2024-08-17T17:35:31.026 INFO:tasks.workunit.client.0.smithi130.stdout:child: image115 2024-08-17T17:35:31.026 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image113/d51316f69efa 2024-08-17T17:35:31.026 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.027 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114/d513e8224e2c 2024-08-17T17:35:31.027 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.027 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image115/d5131e1d46b9 2024-08-17T17:35:31.027 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.027 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.027 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.027 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.027 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.027 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.027 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.027 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.027 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.027 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.028 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.028 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.028 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 3 2024-08-17T17:35:31.028 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 3 2024-08-17T17:35:31.028 INFO:tasks.workunit.client.0.smithi130.stdout:child: image114 2024-08-17T17:35:31.028 INFO:tasks.workunit.client.0.smithi130.stdout:child: image113 2024-08-17T17:35:31.028 INFO:tasks.workunit.client.0.smithi130.stdout:child: image115 2024-08-17T17:35:31.028 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image113/d51316f69efa 2024-08-17T17:35:31.028 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.028 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114/d513e8224e2c 2024-08-17T17:35:31.028 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.028 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image115/d5131e1d46b9 2024-08-17T17:35:31.028 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.028 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.029 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.029 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.029 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.029 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.029 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.029 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.029 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.029 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.029 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.029 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.029 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.029 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-33/image116 2024-08-17T17:35:31.029 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.029 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.030 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image116 2024-08-17T17:35:31.030 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image116 2024-08-17T17:35:31.030 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.030 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 4 2024-08-17T17:35:31.030 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 4 2024-08-17T17:35:31.030 INFO:tasks.workunit.client.0.smithi130.stdout:child: image114 2024-08-17T17:35:31.030 INFO:tasks.workunit.client.0.smithi130.stdout:child: image113 2024-08-17T17:35:31.031 INFO:tasks.workunit.client.0.smithi130.stdout:child: image115 2024-08-17T17:35:31.031 INFO:tasks.workunit.client.0.smithi130.stdout:child: image116 2024-08-17T17:35:31.031 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image113/d51316f69efa 2024-08-17T17:35:31.031 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.031 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114/d513e8224e2c 2024-08-17T17:35:31.031 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.031 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image115/d5131e1d46b9 2024-08-17T17:35:31.031 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.031 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image116/d5138b9bb93d 2024-08-17T17:35:31.031 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.031 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.031 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.031 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.032 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.032 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.032 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.032 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.032 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.032 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.032 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.032 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.032 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.032 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-33/image115 2024-08-17T17:35:31.032 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.032 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:31.032 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image115 2024-08-17T17:35:31.032 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image115 2024-08-17T17:35:31.033 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.033 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:31.033 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-33/image116 2024-08-17T17:35:31.033 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:31 smithi179 ceph-mon[93519]: pgmap v734: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 1.5 KiB/s wr, 11 op/s 2024-08-17T17:35:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:31 smithi179 ceph-mon[93519]: osdmap e708: 8 total, 8 up, 8 in 2024-08-17T17:35:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:31 smithi130 ceph-mon[105521]: pgmap v734: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 1.5 KiB/s wr, 11 op/s 2024-08-17T17:35:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:31 smithi130 ceph-mon[105521]: osdmap e708: 8 total, 8 up, 8 in 2024-08-17T17:35:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:31 smithi130 ceph-mon[110068]: pgmap v734: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 1.5 KiB/s wr, 11 op/s 2024-08-17T17:35:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:31 smithi130 ceph-mon[110068]: osdmap e708: 8 total, 8 up, 8 in 2024-08-17T17:35:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:32 smithi179 ceph-mon[93519]: osdmap e709: 8 total, 8 up, 8 in 2024-08-17T17:35:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi130-87148-32"} : dispatch 2024-08-17T17:35:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:32 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi130-87148-32"} : dispatch 2024-08-17T17:35:32.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:32 smithi130 ceph-mon[110068]: osdmap e709: 8 total, 8 up, 8 in 2024-08-17T17:35:32.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi130-87148-32"} : dispatch 2024-08-17T17:35:32.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:32 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi130-87148-32"} : dispatch 2024-08-17T17:35:32.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:32 smithi130 ceph-mon[105521]: osdmap e709: 8 total, 8 up, 8 in 2024-08-17T17:35:32.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi130-87148-32"} : dispatch 2024-08-17T17:35:32.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:32 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi130-87148-32"} : dispatch 2024-08-17T17:35:33.547 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image113 2024-08-17T17:35:33.547 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image115 2024-08-17T17:35:33.547 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image116 2024-08-17T17:35:33.547 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image116 2024-08-17T17:35:33.547 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.547 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 4 2024-08-17T17:35:33.547 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 4 2024-08-17T17:35:33.548 INFO:tasks.workunit.client.0.smithi130.stdout:child: image114 2024-08-17T17:35:33.548 INFO:tasks.workunit.client.0.smithi130.stdout:child: image113 2024-08-17T17:35:33.548 INFO:tasks.workunit.client.0.smithi130.stdout:child: image115 2024-08-17T17:35:33.548 INFO:tasks.workunit.client.0.smithi130.stdout:child: image116 2024-08-17T17:35:33.548 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image113/d51316f69efa 2024-08-17T17:35:33.548 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.548 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114/d513e8224e2c 2024-08-17T17:35:33.548 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.548 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image115/d5131e1d46b9 2024-08-17T17:35:33.548 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.548 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image116/d5138b9bb93d 2024-08-17T17:35:33.548 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.548 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.548 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:33.549 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:33.549 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:33.549 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.549 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.549 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-33/image115 2024-08-17T17:35:33.549 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:33.549 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image115 2024-08-17T17:35:33.549 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image115 2024-08-17T17:35:33.549 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.549 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.549 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-33/image116 2024-08-17T17:35:33.549 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:33.550 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image115 2024-08-17T17:35:33.550 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image116 2024-08-17T17:35:33.550 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image116 2024-08-17T17:35:33.550 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.550 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 3 2024-08-17T17:35:33.550 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 3 2024-08-17T17:35:33.550 INFO:tasks.workunit.client.0.smithi130.stdout:child: image114 2024-08-17T17:35:33.550 INFO:tasks.workunit.client.0.smithi130.stdout:child: image115 2024-08-17T17:35:33.550 INFO:tasks.workunit.client.0.smithi130.stdout:child: image116 2024-08-17T17:35:33.550 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114/d513e8224e2c 2024-08-17T17:35:33.550 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.550 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image115/d5131e1d46b9 2024-08-17T17:35:33.550 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.551 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image116/d5138b9bb93d 2024-08-17T17:35:33.551 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.551 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.551 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:33.551 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:33.551 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:33.551 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.551 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.551 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-33/image116 2024-08-17T17:35:33.551 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:33.551 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-33/image116 2024-08-17T17:35:33.552 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image116 2024-08-17T17:35:33.552 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.552 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 2 2024-08-17T17:35:33.552 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 2 2024-08-17T17:35:33.552 INFO:tasks.workunit.client.0.smithi130.stdout:child: image114 2024-08-17T17:35:33.552 INFO:tasks.workunit.client.0.smithi130.stdout:child: image116 2024-08-17T17:35:33.552 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114/d513e8224e2c 2024-08-17T17:35:33.552 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.552 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-33/image116/d5138b9bb93d 2024-08-17T17:35:33.552 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.552 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.553 INFO:tasks.workunit.client.0.smithi130.stdout:trying to find test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:33.553 INFO:tasks.workunit.client.0.smithi130.stdout:checking test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:33.553 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114 2024-08-17T17:35:33.553 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.553 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 1 2024-08-17T17:35:33.553 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 1 2024-08-17T17:35:33.553 INFO:tasks.workunit.client.0.smithi130.stdout:child: image114 2024-08-17T17:35:33.553 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-32/image114/d513e8224e2c 2024-08-17T17:35:33.553 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:35:33.553 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 0 2024-08-17T17:35:33.553 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 0 2024-08-17T17:35:33.553 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ListChildrenTiered (16994 ms) 2024-08-17T17:35:33.553 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.LockingPP 2024-08-17T17:35:33.554 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:33.611 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.LockingPP (64 ms) 2024-08-17T17:35:33.611 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.FlushAio 2024-08-17T17:35:33.611 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:33.817 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.FlushAio (206 ms) 2024-08-17T17:35:33.817 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.FlushAioPP 2024-08-17T17:35:33.817 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:33 smithi179 ceph-mon[93519]: pgmap v737: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 1.5 KiB/s wr, 134 op/s 2024-08-17T17:35:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:33 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi130-87148-32"}]': finished 2024-08-17T17:35:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:33 smithi179 ceph-mon[93519]: osdmap e710: 8 total, 8 up, 8 in 2024-08-17T17:35:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:33 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34"} : dispatch 2024-08-17T17:35:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:33 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34"} : dispatch 2024-08-17T17:35:33.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:33 smithi130 ceph-mon[110068]: pgmap v737: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 1.5 KiB/s wr, 134 op/s 2024-08-17T17:35:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:33 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi130-87148-32"}]': finished 2024-08-17T17:35:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:33 smithi130 ceph-mon[110068]: osdmap e710: 8 total, 8 up, 8 in 2024-08-17T17:35:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:33 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34"} : dispatch 2024-08-17T17:35:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:33 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34"} : dispatch 2024-08-17T17:35:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:33 smithi130 ceph-mon[105521]: pgmap v737: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 1.5 KiB/s wr, 134 op/s 2024-08-17T17:35:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:33 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi130-87148-32"}]': finished 2024-08-17T17:35:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:33 smithi130 ceph-mon[105521]: osdmap e710: 8 total, 8 up, 8 in 2024-08-17T17:35:33.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:33 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34"} : dispatch 2024-08-17T17:35:33.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:33 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34"} : dispatch 2024-08-17T17:35:33.986 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.FlushAioPP (169 ms) 2024-08-17T17:35:33.987 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ZeroLengthWrite 2024-08-17T17:35:33.987 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:34.009 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ZeroLengthWrite (23 ms) 2024-08-17T17:35:34.009 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ZeroLengthDiscard 2024-08-17T17:35:34.009 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:34.038 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ZeroLengthDiscard (29 ms) 2024-08-17T17:35:34.038 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ZeroLengthRead 2024-08-17T17:35:34.038 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:34.053 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ZeroLengthRead (15 ms) 2024-08-17T17:35:34.053 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.LargeCacheRead 2024-08-17T17:35:34.053 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:34.095 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.LargeCacheRead (42 ms) 2024-08-17T17:35:34.095 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestPendingAio 2024-08-17T17:35:34.095 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:34.320 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestPendingAio (225 ms) 2024-08-17T17:35:34.320 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyup 2024-08-17T17:35:34.321 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:34.650 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:34 smithi179 ceph-mon[93519]: Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-08-17T17:35:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:34 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34"}]': finished 2024-08-17T17:35:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:34 smithi179 ceph-mon[93519]: osdmap e711: 8 total, 8 up, 8 in 2024-08-17T17:35:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:34 smithi130 ceph-mon[105521]: Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-08-17T17:35:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:34 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34"}]': finished 2024-08-17T17:35:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:34 smithi130 ceph-mon[105521]: osdmap e711: 8 total, 8 up, 8 in 2024-08-17T17:35:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:34 smithi130 ceph-mon[110068]: Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-08-17T17:35:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:34 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi130-87148-32", "tierpool":"test-librbd-smithi130-87148-34"}]': finished 2024-08-17T17:35:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:34 smithi130 ceph-mon[110068]: osdmap e711: 8 total, 8 up, 8 in 2024-08-17T17:35:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:35 smithi179 ceph-mon[93519]: pgmap v740: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 488 KiB/s rd, 12 KiB/s wr, 597 op/s 2024-08-17T17:35:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:35 smithi179 ceph-mon[93519]: osdmap e712: 8 total, 8 up, 8 in 2024-08-17T17:35:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:35 smithi179 ceph-mon[93519]: osdmap e713: 8 total, 8 up, 8 in 2024-08-17T17:35:35.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:35 smithi130 ceph-mon[105521]: pgmap v740: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 488 KiB/s rd, 12 KiB/s wr, 597 op/s 2024-08-17T17:35:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:35 smithi130 ceph-mon[105521]: osdmap e712: 8 total, 8 up, 8 in 2024-08-17T17:35:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:35 smithi130 ceph-mon[105521]: osdmap e713: 8 total, 8 up, 8 in 2024-08-17T17:35:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:35 smithi130 ceph-mon[110068]: pgmap v740: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 488 KiB/s rd, 12 KiB/s wr, 597 op/s 2024-08-17T17:35:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:35 smithi130 ceph-mon[110068]: osdmap e712: 8 total, 8 up, 8 in 2024-08-17T17:35:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:35 smithi130 ceph-mon[110068]: osdmap e713: 8 total, 8 up, 8 in 2024-08-17T17:35:36.492 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyup (2172 ms) 2024-08-17T17:35:36.492 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyupMismatch 2024-08-17T17:35:36.492 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:37.544 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:37 smithi179 ceph-mon[93519]: pgmap v743: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 490 KiB/s rd, 13 KiB/s wr, 599 op/s 2024-08-17T17:35:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:37 smithi179 ceph-mon[93519]: osdmap e714: 8 total, 8 up, 8 in 2024-08-17T17:35:37.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:37 smithi130 ceph-mon[105521]: pgmap v743: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 490 KiB/s rd, 13 KiB/s wr, 599 op/s 2024-08-17T17:35:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:37 smithi130 ceph-mon[105521]: osdmap e714: 8 total, 8 up, 8 in 2024-08-17T17:35:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:37 smithi130 ceph-mon[110068]: pgmap v743: 713 pgs: 713 active+clean; 131 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 490 KiB/s rd, 13 KiB/s wr, 599 op/s 2024-08-17T17:35:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:37 smithi130 ceph-mon[110068]: osdmap e714: 8 total, 8 up, 8 in 2024-08-17T17:35:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:38 smithi179 ceph-mon[93519]: osdmap e715: 8 total, 8 up, 8 in 2024-08-17T17:35:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:38 smithi130 ceph-mon[105521]: osdmap e715: 8 total, 8 up, 8 in 2024-08-17T17:35:38.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:38 smithi130 ceph-mon[110068]: osdmap e715: 8 total, 8 up, 8 in 2024-08-17T17:35:39.513 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyupMismatch (3021 ms) 2024-08-17T17:35:39.514 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.Flatten 2024-08-17T17:35:39.514 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:39.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:39 smithi130 ceph-mon[105521]: pgmap v746: 713 pgs: 713 active+clean; 135 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 882 KiB/s rd, 671 KiB/s wr, 618 op/s 2024-08-17T17:35:39.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:39 smithi130 ceph-mon[105521]: osdmap e716: 8 total, 8 up, 8 in 2024-08-17T17:35:39.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:39 smithi130 ceph-mon[110068]: pgmap v746: 713 pgs: 713 active+clean; 135 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 882 KiB/s rd, 671 KiB/s wr, 618 op/s 2024-08-17T17:35:39.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:39 smithi130 ceph-mon[110068]: osdmap e716: 8 total, 8 up, 8 in 2024-08-17T17:35:39.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:39 smithi179 ceph-mon[93519]: pgmap v746: 713 pgs: 713 active+clean; 135 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 882 KiB/s rd, 671 KiB/s wr, 618 op/s 2024-08-17T17:35:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:39 smithi179 ceph-mon[93519]: osdmap e716: 8 total, 8 up, 8 in 2024-08-17T17:35:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:35:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:35:39] "GET /metrics HTTP/1.1" 200 82165 "" "Prometheus/2.51.0" 2024-08-17T17:35:40.482 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:35:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:35:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:35:40.550 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.Flatten (1037 ms) 2024-08-17T17:35:40.550 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.Sparsify 2024-08-17T17:35:40.550 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:40.581 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 32768 2024-08-17T17:35:40.581 INFO:tasks.workunit.client.0.smithi130.stdout:read: 32768 2024-08-17T17:35:40.581 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.Sparsify (31 ms) 2024-08-17T17:35:40.581 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.SparsifyPP 2024-08-17T17:35:40.581 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:40.610 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.SparsifyPP (29 ms) 2024-08-17T17:35:40.610 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.SnapshotLimit 2024-08-17T17:35:40.610 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:40 smithi179 ceph-mon[93519]: osdmap e717: 8 total, 8 up, 8 in 2024-08-17T17:35:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:40 smithi179 ceph-mon[93519]: osdmap e718: 8 total, 8 up, 8 in 2024-08-17T17:35:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:40 smithi130 ceph-mon[105521]: osdmap e717: 8 total, 8 up, 8 in 2024-08-17T17:35:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:40 smithi130 ceph-mon[105521]: osdmap e718: 8 total, 8 up, 8 in 2024-08-17T17:35:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:40 smithi130 ceph-mon[110068]: osdmap e717: 8 total, 8 up, 8 in 2024-08-17T17:35:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:40 smithi130 ceph-mon[110068]: osdmap e718: 8 total, 8 up, 8 in 2024-08-17T17:35:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:41 smithi179 ceph-mon[93519]: pgmap v749: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 872 KiB/s rd, 740 KiB/s wr, 672 op/s 2024-08-17T17:35:41.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:41 smithi130 ceph-mon[105521]: pgmap v749: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 872 KiB/s rd, 740 KiB/s wr, 672 op/s 2024-08-17T17:35:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:41 smithi130 ceph-mon[110068]: pgmap v749: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 872 KiB/s rd, 740 KiB/s wr, 672 op/s 2024-08-17T17:35:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:42 smithi179 ceph-mon[93519]: osdmap e719: 8 total, 8 up, 8 in 2024-08-17T17:35:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:42 smithi130 ceph-mon[110068]: osdmap e719: 8 total, 8 up, 8 in 2024-08-17T17:35:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:42 smithi130 ceph-mon[105521]: osdmap e719: 8 total, 8 up, 8 in 2024-08-17T17:35:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:35:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:43 smithi179 ceph-mon[93519]: pgmap v752: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 162 KiB/s rd, 89 KiB/s wr, 244 op/s 2024-08-17T17:35:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:43 smithi179 ceph-mon[93519]: osdmap e720: 8 total, 8 up, 8 in 2024-08-17T17:35:43.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:43 smithi130 ceph-mon[105521]: pgmap v752: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 162 KiB/s rd, 89 KiB/s wr, 244 op/s 2024-08-17T17:35:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:43 smithi130 ceph-mon[105521]: osdmap e720: 8 total, 8 up, 8 in 2024-08-17T17:35:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:43 smithi130 ceph-mon[110068]: pgmap v752: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 162 KiB/s rd, 89 KiB/s wr, 244 op/s 2024-08-17T17:35:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:43 smithi130 ceph-mon[110068]: osdmap e720: 8 total, 8 up, 8 in 2024-08-17T17:35:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:44 smithi179 ceph-mon[93519]: osdmap e721: 8 total, 8 up, 8 in 2024-08-17T17:35:44.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:44 smithi130 ceph-mon[105521]: osdmap e721: 8 total, 8 up, 8 in 2024-08-17T17:35:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:44 smithi130 ceph-mon[110068]: osdmap e721: 8 total, 8 up, 8 in 2024-08-17T17:35:45.544 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.SnapshotLimit (4934 ms) 2024-08-17T17:35:45.544 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.SnapshotLimitPP 2024-08-17T17:35:45.544 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:45 smithi179 ceph-mon[93519]: pgmap v755: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 198 KiB/s rd, 35 KiB/s wr, 244 op/s 2024-08-17T17:35:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:45 smithi179 ceph-mon[93519]: osdmap e722: 8 total, 8 up, 8 in 2024-08-17T17:35:45.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:45 smithi130 ceph-mon[110068]: pgmap v755: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 198 KiB/s rd, 35 KiB/s wr, 244 op/s 2024-08-17T17:35:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:45 smithi130 ceph-mon[110068]: osdmap e722: 8 total, 8 up, 8 in 2024-08-17T17:35:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:45 smithi130 ceph-mon[105521]: pgmap v755: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 198 KiB/s rd, 35 KiB/s wr, 244 op/s 2024-08-17T17:35:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:45 smithi130 ceph-mon[105521]: osdmap e722: 8 total, 8 up, 8 in 2024-08-17T17:35:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:46 smithi179 ceph-mon[93519]: osdmap e723: 8 total, 8 up, 8 in 2024-08-17T17:35:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:46 smithi130 ceph-mon[110068]: osdmap e723: 8 total, 8 up, 8 in 2024-08-17T17:35:46.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:46 smithi130 ceph-mon[105521]: osdmap e723: 8 total, 8 up, 8 in 2024-08-17T17:35:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:47 smithi179 ceph-mon[93519]: pgmap v758: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 139 KiB/s rd, 17 KiB/s wr, 189 op/s 2024-08-17T17:35:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:47 smithi179 ceph-mon[93519]: osdmap e724: 8 total, 8 up, 8 in 2024-08-17T17:35:47.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:47 smithi130 ceph-mon[110068]: pgmap v758: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 139 KiB/s rd, 17 KiB/s wr, 189 op/s 2024-08-17T17:35:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:47 smithi130 ceph-mon[110068]: osdmap e724: 8 total, 8 up, 8 in 2024-08-17T17:35:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:47 smithi130 ceph-mon[105521]: pgmap v758: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 139 KiB/s rd, 17 KiB/s wr, 189 op/s 2024-08-17T17:35:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:47 smithi130 ceph-mon[105521]: osdmap e724: 8 total, 8 up, 8 in 2024-08-17T17:35:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:48 smithi179 ceph-mon[93519]: osdmap e725: 8 total, 8 up, 8 in 2024-08-17T17:35:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:48 smithi130 ceph-mon[110068]: osdmap e725: 8 total, 8 up, 8 in 2024-08-17T17:35:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:48 smithi130 ceph-mon[105521]: osdmap e725: 8 total, 8 up, 8 in 2024-08-17T17:35:49.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:49 smithi130 ceph-mon[110068]: pgmap v761: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 4.0 KiB/s wr, 54 op/s 2024-08-17T17:35:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:49 smithi130 ceph-mon[110068]: osdmap e726: 8 total, 8 up, 8 in 2024-08-17T17:35:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:49 smithi130 ceph-mon[105521]: pgmap v761: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 4.0 KiB/s wr, 54 op/s 2024-08-17T17:35:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:49 smithi130 ceph-mon[105521]: osdmap e726: 8 total, 8 up, 8 in 2024-08-17T17:35:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:49 smithi179 ceph-mon[93519]: pgmap v761: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 4.0 KiB/s wr, 54 op/s 2024-08-17T17:35:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:49 smithi179 ceph-mon[93519]: osdmap e726: 8 total, 8 up, 8 in 2024-08-17T17:35:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:35:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:35:49] "GET /metrics HTTP/1.1" 200 82260 "" "Prometheus/2.51.0" 2024-08-17T17:35:50.566 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:35:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:35:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:35:50.595 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.SnapshotLimitPP (5051 ms) 2024-08-17T17:35:50.595 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.RebuildObjectMapViaLockOwner 2024-08-17T17:35:50.595 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:50.652 INFO:tasks.workunit.client.0.smithi130.stdout:0.00% done 2024-08-17T17:35:50.652 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.RebuildObjectMapViaLockOwner (56 ms) 2024-08-17T17:35:50.652 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.RenameViaLockOwner 2024-08-17T17:35:50.652 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:50.716 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.RenameViaLockOwner (65 ms) 2024-08-17T17:35:50.716 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.SnapCreateViaLockOwner 2024-08-17T17:35:50.716 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:50 smithi179 ceph-mon[93519]: osdmap e727: 8 total, 8 up, 8 in 2024-08-17T17:35:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:50 smithi130 ceph-mon[105521]: osdmap e727: 8 total, 8 up, 8 in 2024-08-17T17:35:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:50 smithi130 ceph-mon[110068]: osdmap e727: 8 total, 8 up, 8 in 2024-08-17T17:35:51.604 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.SnapCreateViaLockOwner (888 ms) 2024-08-17T17:35:51.604 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.SnapRemoveViaLockOwner 2024-08-17T17:35:51.604 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:51 smithi179 ceph-mon[93519]: pgmap v764: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 4.2 KiB/s wr, 54 op/s 2024-08-17T17:35:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:51 smithi179 ceph-mon[93519]: osdmap e728: 8 total, 8 up, 8 in 2024-08-17T17:35:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:51 smithi130 ceph-mon[105521]: pgmap v764: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 4.2 KiB/s wr, 54 op/s 2024-08-17T17:35:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:51 smithi130 ceph-mon[105521]: osdmap e728: 8 total, 8 up, 8 in 2024-08-17T17:35:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:51 smithi130 ceph-mon[110068]: pgmap v764: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 4.2 KiB/s wr, 54 op/s 2024-08-17T17:35:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:51 smithi130 ceph-mon[110068]: osdmap e728: 8 total, 8 up, 8 in 2024-08-17T17:35:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:52 smithi179 ceph-mon[93519]: osdmap e729: 8 total, 8 up, 8 in 2024-08-17T17:35:52.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:52 smithi130 ceph-mon[105521]: osdmap e729: 8 total, 8 up, 8 in 2024-08-17T17:35:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:52 smithi130 ceph-mon[110068]: osdmap e729: 8 total, 8 up, 8 in 2024-08-17T17:35:53.636 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.SnapRemoveViaLockOwner (2032 ms) 2024-08-17T17:35:53.636 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.UpdateFeaturesViaLockOwner 2024-08-17T17:35:53.636 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:53.690 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.UpdateFeaturesViaLockOwner (54 ms) 2024-08-17T17:35:53.690 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.EnableJournalingViaLockOwner 2024-08-17T17:35:53.690 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:35:53.691 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.EnableJournalingViaLockOwner (0 ms) 2024-08-17T17:35:53.691 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.SnapRemove2 2024-08-17T17:35:53.691 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:53 smithi179 ceph-mon[93519]: pgmap v767: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 1.7 KiB/s wr, 45 op/s 2024-08-17T17:35:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:53 smithi179 ceph-mon[93519]: osdmap e730: 8 total, 8 up, 8 in 2024-08-17T17:35:53.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:53 smithi130 ceph-mon[105521]: pgmap v767: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 1.7 KiB/s wr, 45 op/s 2024-08-17T17:35:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:53 smithi130 ceph-mon[105521]: osdmap e730: 8 total, 8 up, 8 in 2024-08-17T17:35:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:53 smithi130 ceph-mon[110068]: pgmap v767: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 1.7 KiB/s wr, 45 op/s 2024-08-17T17:35:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:53 smithi130 ceph-mon[110068]: osdmap e730: 8 total, 8 up, 8 in 2024-08-17T17:35:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:54 smithi130 ceph-mon[105521]: osdmap e731: 8 total, 8 up, 8 in 2024-08-17T17:35:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:54 smithi130 ceph-mon[110068]: osdmap e731: 8 total, 8 up, 8 in 2024-08-17T17:35:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:54 smithi179 ceph-mon[93519]: osdmap e731: 8 total, 8 up, 8 in 2024-08-17T17:35:55.489 INFO:tasks.workunit.client.0.smithi130.stdout:100.00% done 2024-08-17T17:35:55.489 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.SnapRemove2 (1799 ms) 2024-08-17T17:35:55.489 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.SnapRenameViaLockOwner 2024-08-17T17:35:55.489 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:55.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:55 smithi130 ceph-mon[105521]: pgmap v770: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 102 KiB/s rd, 11 KiB/s wr, 143 op/s 2024-08-17T17:35:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:55 smithi130 ceph-mon[105521]: osdmap e732: 8 total, 8 up, 8 in 2024-08-17T17:35:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:55 smithi130 ceph-mon[105521]: osdmap e733: 8 total, 8 up, 8 in 2024-08-17T17:35:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:55 smithi130 ceph-mon[110068]: pgmap v770: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 102 KiB/s rd, 11 KiB/s wr, 143 op/s 2024-08-17T17:35:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:55 smithi130 ceph-mon[110068]: osdmap e732: 8 total, 8 up, 8 in 2024-08-17T17:35:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:55 smithi130 ceph-mon[110068]: osdmap e733: 8 total, 8 up, 8 in 2024-08-17T17:35:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:55 smithi179 ceph-mon[93519]: pgmap v770: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 102 KiB/s rd, 11 KiB/s wr, 143 op/s 2024-08-17T17:35:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:55 smithi179 ceph-mon[93519]: osdmap e732: 8 total, 8 up, 8 in 2024-08-17T17:35:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:55 smithi179 ceph-mon[93519]: osdmap e733: 8 total, 8 up, 8 in 2024-08-17T17:35:56.511 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.SnapRenameViaLockOwner (1022 ms) 2024-08-17T17:35:56.511 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.SnapProtectViaLockOwner 2024-08-17T17:35:56.511 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:57.513 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.SnapProtectViaLockOwner (1002 ms) 2024-08-17T17:35:57.513 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.SnapUnprotectViaLockOwner 2024-08-17T17:35:57.513 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:57 smithi179 ceph-mon[93519]: pgmap v773: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s rd, 9.2 KiB/s wr, 98 op/s 2024-08-17T17:35:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:57 smithi179 ceph-mon[93519]: osdmap e734: 8 total, 8 up, 8 in 2024-08-17T17:35:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:57.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:57 smithi130 ceph-mon[105521]: pgmap v773: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s rd, 9.2 KiB/s wr, 98 op/s 2024-08-17T17:35:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:57 smithi130 ceph-mon[105521]: osdmap e734: 8 total, 8 up, 8 in 2024-08-17T17:35:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:57 smithi130 ceph-mon[110068]: pgmap v773: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s rd, 9.2 KiB/s wr, 98 op/s 2024-08-17T17:35:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:57 smithi130 ceph-mon[110068]: osdmap e734: 8 total, 8 up, 8 in 2024-08-17T17:35:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:58.548 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.SnapUnprotectViaLockOwner (1035 ms) 2024-08-17T17:35:58.548 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.FlattenViaLockOwner 2024-08-17T17:35:58.548 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:58 smithi179 ceph-mon[93519]: osdmap e735: 8 total, 8 up, 8 in 2024-08-17T17:35:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:58 smithi130 ceph-mon[105521]: osdmap e735: 8 total, 8 up, 8 in 2024-08-17T17:35:58.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:58 smithi130 ceph-mon[110068]: osdmap e735: 8 total, 8 up, 8 in 2024-08-17T17:35:59.622 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.FlattenViaLockOwner (1074 ms) 2024-08-17T17:35:59.622 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ResizeViaLockOwner 2024-08-17T17:35:59.622 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:59.659 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ResizeViaLockOwner (37 ms) 2024-08-17T17:35:59.659 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.SparsifyViaLockOwner 2024-08-17T17:35:59.660 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:59.693 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.SparsifyViaLockOwner (34 ms) 2024-08-17T17:35:59.693 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ObjectMapConsistentSnap 2024-08-17T17:35:59.693 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:35:59.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:59 smithi130 ceph-mon[105521]: pgmap v776: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 11 KiB/s wr, 205 op/s 2024-08-17T17:35:59.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:35:59 smithi130 ceph-mon[105521]: osdmap e736: 8 total, 8 up, 8 in 2024-08-17T17:35:59.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:59 smithi130 ceph-mon[110068]: pgmap v776: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 11 KiB/s wr, 205 op/s 2024-08-17T17:35:59.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:35:59 smithi130 ceph-mon[110068]: osdmap e736: 8 total, 8 up, 8 in 2024-08-17T17:35:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:59 smithi179 ceph-mon[93519]: pgmap v776: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 11 KiB/s wr, 205 op/s 2024-08-17T17:35:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:35:59 smithi179 ceph-mon[93519]: osdmap e736: 8 total, 8 up, 8 in 2024-08-17T17:36:00.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:35:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:35:59] "GET /metrics HTTP/1.1" 200 82248 "" "Prometheus/2.51.0" 2024-08-17T17:36:00.507 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:36:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:36:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:36:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:00 smithi179 ceph-mon[93519]: osdmap e737: 8 total, 8 up, 8 in 2024-08-17T17:36:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:00 smithi130 ceph-mon[105521]: osdmap e737: 8 total, 8 up, 8 in 2024-08-17T17:36:00.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:00 smithi130 ceph-mon[110068]: osdmap e737: 8 total, 8 up, 8 in 2024-08-17T17:36:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:01 smithi179 ceph-mon[93519]: pgmap v779: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 214 KiB/s rd, 14 KiB/s wr, 283 op/s 2024-08-17T17:36:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:01 smithi179 ceph-mon[93519]: osdmap e738: 8 total, 8 up, 8 in 2024-08-17T17:36:01.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:01 smithi130 ceph-mon[105521]: pgmap v779: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 214 KiB/s rd, 14 KiB/s wr, 283 op/s 2024-08-17T17:36:01.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:01 smithi130 ceph-mon[105521]: osdmap e738: 8 total, 8 up, 8 in 2024-08-17T17:36:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:01 smithi130 ceph-mon[110068]: pgmap v779: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 214 KiB/s rd, 14 KiB/s wr, 283 op/s 2024-08-17T17:36:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:01 smithi130 ceph-mon[110068]: osdmap e738: 8 total, 8 up, 8 in 2024-08-17T17:36:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:02 smithi179 ceph-mon[93519]: osdmap e739: 8 total, 8 up, 8 in 2024-08-17T17:36:02.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:02 smithi130 ceph-mon[105521]: osdmap e739: 8 total, 8 up, 8 in 2024-08-17T17:36:02.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:02 smithi130 ceph-mon[110068]: osdmap e739: 8 total, 8 up, 8 in 2024-08-17T17:36:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:03 smithi179 ceph-mon[93519]: pgmap v782: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 4.2 KiB/s wr, 102 op/s 2024-08-17T17:36:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:03 smithi179 ceph-mon[93519]: osdmap e740: 8 total, 8 up, 8 in 2024-08-17T17:36:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:03 smithi130 ceph-mon[105521]: pgmap v782: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 4.2 KiB/s wr, 102 op/s 2024-08-17T17:36:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:03 smithi130 ceph-mon[105521]: osdmap e740: 8 total, 8 up, 8 in 2024-08-17T17:36:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:03 smithi130 ceph-mon[110068]: pgmap v782: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 4.2 KiB/s wr, 102 op/s 2024-08-17T17:36:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:03 smithi130 ceph-mon[110068]: osdmap e740: 8 total, 8 up, 8 in 2024-08-17T17:36:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:04 smithi179 ceph-mon[93519]: osdmap e741: 8 total, 8 up, 8 in 2024-08-17T17:36:04.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:04 smithi130 ceph-mon[105521]: osdmap e741: 8 total, 8 up, 8 in 2024-08-17T17:36:04.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:04 smithi130 ceph-mon[110068]: osdmap e741: 8 total, 8 up, 8 in 2024-08-17T17:36:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:05 smithi179 ceph-mon[93519]: pgmap v785: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 9.5 KiB/s wr, 195 op/s 2024-08-17T17:36:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:05 smithi179 ceph-mon[93519]: osdmap e742: 8 total, 8 up, 8 in 2024-08-17T17:36:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:05 smithi179 ceph-mon[93519]: osdmap e743: 8 total, 8 up, 8 in 2024-08-17T17:36:05.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:05 smithi130 ceph-mon[105521]: pgmap v785: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 9.5 KiB/s wr, 195 op/s 2024-08-17T17:36:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:05 smithi130 ceph-mon[105521]: osdmap e742: 8 total, 8 up, 8 in 2024-08-17T17:36:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:05 smithi130 ceph-mon[105521]: osdmap e743: 8 total, 8 up, 8 in 2024-08-17T17:36:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:05 smithi130 ceph-mon[110068]: pgmap v785: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 146 KiB/s rd, 9.5 KiB/s wr, 195 op/s 2024-08-17T17:36:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:05 smithi130 ceph-mon[110068]: osdmap e742: 8 total, 8 up, 8 in 2024-08-17T17:36:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:05 smithi130 ceph-mon[110068]: osdmap e743: 8 total, 8 up, 8 in 2024-08-17T17:36:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:07 smithi179 ceph-mon[93519]: pgmap v788: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 129 KiB/s rd, 8.2 KiB/s wr, 170 op/s 2024-08-17T17:36:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:07 smithi179 ceph-mon[93519]: osdmap e744: 8 total, 8 up, 8 in 2024-08-17T17:36:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:07 smithi130 ceph-mon[105521]: pgmap v788: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 129 KiB/s rd, 8.2 KiB/s wr, 170 op/s 2024-08-17T17:36:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:07 smithi130 ceph-mon[105521]: osdmap e744: 8 total, 8 up, 8 in 2024-08-17T17:36:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:07 smithi130 ceph-mon[110068]: pgmap v788: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 129 KiB/s rd, 8.2 KiB/s wr, 170 op/s 2024-08-17T17:36:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:07 smithi130 ceph-mon[110068]: osdmap e744: 8 total, 8 up, 8 in 2024-08-17T17:36:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:08 smithi179 ceph-mon[93519]: osdmap e745: 8 total, 8 up, 8 in 2024-08-17T17:36:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:08 smithi130 ceph-mon[105521]: osdmap e745: 8 total, 8 up, 8 in 2024-08-17T17:36:08.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:08 smithi130 ceph-mon[110068]: osdmap e745: 8 total, 8 up, 8 in 2024-08-17T17:36:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:09 smithi130 ceph-mon[105521]: pgmap v791: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 3.0 KiB/s wr, 89 op/s 2024-08-17T17:36:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:09 smithi130 ceph-mon[105521]: osdmap e746: 8 total, 8 up, 8 in 2024-08-17T17:36:09.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:09 smithi130 ceph-mon[110068]: pgmap v791: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 3.0 KiB/s wr, 89 op/s 2024-08-17T17:36:09.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:09 smithi130 ceph-mon[110068]: osdmap e746: 8 total, 8 up, 8 in 2024-08-17T17:36:09.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:09 smithi179 ceph-mon[93519]: pgmap v791: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 3.0 KiB/s wr, 89 op/s 2024-08-17T17:36:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:09 smithi179 ceph-mon[93519]: osdmap e746: 8 total, 8 up, 8 in 2024-08-17T17:36:10.085 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ObjectMapConsistentSnap (10392 ms) 2024-08-17T17:36:10.085 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.Metadata 2024-08-17T17:36:10.085 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:36:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:36:09] "GET /metrics HTTP/1.1" 200 82248 "" "Prometheus/2.51.0" 2024-08-17T17:36:10.478 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:36:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:36:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:36:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:10 smithi179 ceph-mon[93519]: osdmap e747: 8 total, 8 up, 8 in 2024-08-17T17:36:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:10 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:36:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:10 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:10 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:10.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:10 smithi130 ceph-mon[105521]: osdmap e747: 8 total, 8 up, 8 in 2024-08-17T17:36:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:10 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:36:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:10 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:10 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:10 smithi130 ceph-mon[110068]: osdmap e747: 8 total, 8 up, 8 in 2024-08-17T17:36:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:10 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:36:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:10 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:10 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:10.999 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.Metadata (914 ms) 2024-08-17T17:36:10.999 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.MetadataPP 2024-08-17T17:36:10.999 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:12.000 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.MetadataPP (1001 ms) 2024-08-17T17:36:12.001 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.UpdateFeatures 2024-08-17T17:36:12.001 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:12.094 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.UpdateFeatures (94 ms) 2024-08-17T17:36:12.094 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.FeaturesBitmaskString 2024-08-17T17:36:12.094 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.FeaturesBitmaskString (0 ms) 2024-08-17T17:36:12.094 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.RebuildObjectMap 2024-08-17T17:36:12.094 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:12.165 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:11 smithi179 ceph-mon[93519]: pgmap v794: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 3.7 KiB/s wr, 90 op/s 2024-08-17T17:36:12.165 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:11 smithi179 ceph-mon[93519]: osdmap e748: 8 total, 8 up, 8 in 2024-08-17T17:36:12.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:11 smithi130 ceph-mon[105521]: pgmap v794: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 3.7 KiB/s wr, 90 op/s 2024-08-17T17:36:12.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:11 smithi130 ceph-mon[105521]: osdmap e748: 8 total, 8 up, 8 in 2024-08-17T17:36:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:11 smithi130 ceph-mon[110068]: pgmap v794: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 3.7 KiB/s wr, 90 op/s 2024-08-17T17:36:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:11 smithi130 ceph-mon[110068]: osdmap e748: 8 total, 8 up, 8 in 2024-08-17T17:36:12.976 INFO:tasks.workunit.client.0.smithi130.stdout:0.00% done 2024-08-17T17:36:12.976 INFO:tasks.workunit.client.0.smithi130.stdout:25.00% done 2024-08-17T17:36:12.976 INFO:tasks.workunit.client.0.smithi130.stdout:50.00% done 2024-08-17T17:36:12.976 INFO:tasks.workunit.client.0.smithi130.stdout:75.00% done 2024-08-17T17:36:12.976 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.RebuildObjectMap (882 ms) 2024-08-17T17:36:12.977 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.RebuildNewObjectMap 2024-08-17T17:36:13.011 INFO:tasks.workunit.client.0.smithi130.stdout:0.00% done 2024-08-17T17:36:13.011 INFO:tasks.workunit.client.0.smithi130.stdout:25.00% done 2024-08-17T17:36:13.011 INFO:tasks.workunit.client.0.smithi130.stdout:50.00% done 2024-08-17T17:36:13.011 INFO:tasks.workunit.client.0.smithi130.stdout:75.00% done 2024-08-17T17:36:13.011 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.RebuildNewObjectMap (35 ms) 2024-08-17T17:36:13.011 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.CheckObjectMap 2024-08-17T17:36:13.011 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: osdmap e749: 8 total, 8 up, 8 in 2024-08-17T17:36:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]} : dispatch 2024-08-17T17:36:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]} : dispatch 2024-08-17T17:36:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]} : dispatch 2024-08-17T17:36:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]} : dispatch 2024-08-17T17:36:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]} : dispatch 2024-08-17T17:36:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]} : dispatch 2024-08-17T17:36:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]} : dispatch 2024-08-17T17:36:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]} : dispatch 2024-08-17T17:36:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]} : dispatch 2024-08-17T17:36:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]} : dispatch 2024-08-17T17:36:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]} : dispatch 2024-08-17T17:36:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]} : dispatch 2024-08-17T17:36:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.19", "id": [0, 3]} : dispatch 2024-08-17T17:36:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]} : dispatch 2024-08-17T17:36:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.0", "id": [7, 1]} : dispatch 2024-08-17T17:36:13.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]} : dispatch 2024-08-17T17:36:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]} : dispatch 2024-08-17T17:36:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.19", "id": [0, 3]} : dispatch 2024-08-17T17:36:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.0", "id": [7, 1]} : dispatch 2024-08-17T17:36:13.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]} : dispatch 2024-08-17T17:36:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: osdmap e749: 8 total, 8 up, 8 in 2024-08-17T17:36:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]} : dispatch 2024-08-17T17:36:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]} : dispatch 2024-08-17T17:36:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]} : dispatch 2024-08-17T17:36:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]} : dispatch 2024-08-17T17:36:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]} : dispatch 2024-08-17T17:36:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]} : dispatch 2024-08-17T17:36:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]} : dispatch 2024-08-17T17:36:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]} : dispatch 2024-08-17T17:36:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]} : dispatch 2024-08-17T17:36:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]} : dispatch 2024-08-17T17:36:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]} : dispatch 2024-08-17T17:36:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]} : dispatch 2024-08-17T17:36:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.19", "id": [0, 3]} : dispatch 2024-08-17T17:36:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]} : dispatch 2024-08-17T17:36:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.0", "id": [7, 1]} : dispatch 2024-08-17T17:36:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]} : dispatch 2024-08-17T17:36:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]} : dispatch 2024-08-17T17:36:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.19", "id": [0, 3]} : dispatch 2024-08-17T17:36:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.0", "id": [7, 1]} : dispatch 2024-08-17T17:36:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]} : dispatch 2024-08-17T17:36:13.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: osdmap e749: 8 total, 8 up, 8 in 2024-08-17T17:36:13.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:13.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]} : dispatch 2024-08-17T17:36:13.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]} : dispatch 2024-08-17T17:36:13.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]} : dispatch 2024-08-17T17:36:13.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]} : dispatch 2024-08-17T17:36:13.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]} : dispatch 2024-08-17T17:36:13.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]} : dispatch 2024-08-17T17:36:13.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]} : dispatch 2024-08-17T17:36:13.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]} : dispatch 2024-08-17T17:36:13.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]} : dispatch 2024-08-17T17:36:13.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]} : dispatch 2024-08-17T17:36:13.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]} : dispatch 2024-08-17T17:36:13.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]} : dispatch 2024-08-17T17:36:13.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.19", "id": [0, 3]} : dispatch 2024-08-17T17:36:13.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]} : dispatch 2024-08-17T17:36:13.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.0", "id": [7, 1]} : dispatch 2024-08-17T17:36:13.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]} : dispatch 2024-08-17T17:36:13.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]} : dispatch 2024-08-17T17:36:13.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.19", "id": [0, 3]} : dispatch 2024-08-17T17:36:13.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.0", "id": [7, 1]} : dispatch 2024-08-17T17:36:13.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]} : dispatch 2024-08-17T17:36:13.969 INFO:tasks.workunit.client.0.smithi130.stdout:0.00% done 2024-08-17T17:36:13.969 INFO:tasks.workunit.client.0.smithi130.stdout:25.00% done 2024-08-17T17:36:13.969 INFO:tasks.workunit.client.0.smithi130.stdout:50.00% done 2024-08-17T17:36:13.969 INFO:tasks.workunit.client.0.smithi130.stdout:75.00% done 2024-08-17T17:36:13.969 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.CheckObjectMap (958 ms) 2024-08-17T17:36:13.969 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.BlockingAIO 2024-08-17T17:36:13.969 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:13.997 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.BlockingAIO (28 ms) 2024-08-17T17:36:13.997 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ExclusiveLockTransition 2024-08-17T17:36:13.997 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:13 smithi179 ceph-mon[93519]: pgmap v797: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 324 KiB/s rd, 141 KiB/s wr, 532 op/s 2024-08-17T17:36:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]': finished 2024-08-17T17:36:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-08-17T17:36:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]}]': finished 2024-08-17T17:36:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]': finished 2024-08-17T17:36:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]}]': finished 2024-08-17T17:36:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]}]': finished 2024-08-17T17:36:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]}]': finished 2024-08-17T17:36:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.19", "id": [0, 3]}]': finished 2024-08-17T17:36:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.0", "id": [7, 1]}]': finished 2024-08-17T17:36:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]}]': finished 2024-08-17T17:36:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:13 smithi179 ceph-mon[93519]: osdmap e750: 8 total, 8 up, 8 in 2024-08-17T17:36:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:14.120 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ExclusiveLockTransition (123 ms) 2024-08-17T17:36:14.120 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ExclusiveLockReadTransition 2024-08-17T17:36:14.121 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:36:14.121 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ExclusiveLockReadTransition (0 ms) 2024-08-17T17:36:14.121 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.CacheMayCopyOnWrite 2024-08-17T17:36:14.121 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[105521]: pgmap v797: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 324 KiB/s rd, 141 KiB/s wr, 532 op/s 2024-08-17T17:36:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]': finished 2024-08-17T17:36:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-08-17T17:36:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]}]': finished 2024-08-17T17:36:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]': finished 2024-08-17T17:36:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]}]': finished 2024-08-17T17:36:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]}]': finished 2024-08-17T17:36:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]}]': finished 2024-08-17T17:36:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.19", "id": [0, 3]}]': finished 2024-08-17T17:36:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.0", "id": [7, 1]}]': finished 2024-08-17T17:36:14.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]}]': finished 2024-08-17T17:36:14.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[105521]: osdmap e750: 8 total, 8 up, 8 in 2024-08-17T17:36:14.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:14.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:14.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[110068]: pgmap v797: 713 pgs: 713 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 324 KiB/s rd, 141 KiB/s wr, 532 op/s 2024-08-17T17:36:14.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]': finished 2024-08-17T17:36:14.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-08-17T17:36:14.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.b", "id": [3, 4]}]': finished 2024-08-17T17:36:14.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.14", "id": [2, 5]}]': finished 2024-08-17T17:36:14.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]}]': finished 2024-08-17T17:36:14.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.13", "id": [4, 2]}]': finished 2024-08-17T17:36:14.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]}]': finished 2024-08-17T17:36:14.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.19", "id": [0, 3]}]': finished 2024-08-17T17:36:14.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.0", "id": [7, 1]}]': finished 2024-08-17T17:36:14.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]}]': finished 2024-08-17T17:36:14.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[110068]: osdmap e750: 8 total, 8 up, 8 in 2024-08-17T17:36:14.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:14.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:14.992 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.CacheMayCopyOnWrite (872 ms) 2024-08-17T17:36:14.992 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot 2024-08-17T17:36:14.992 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:15.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:14 smithi130 ceph-mon[105521]: osdmap e751: 8 total, 8 up, 8 in 2024-08-17T17:36:15.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:14 smithi130 ceph-mon[105521]: pgmap v800: 713 pgs: 8 peering, 705 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 646 KiB/s rd, 219 KiB/s wr, 1.02k op/s 2024-08-17T17:36:15.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:15.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:15.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:14 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:36:15.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:14 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:36:15.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:15.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:14 smithi130 ceph-mon[110068]: osdmap e751: 8 total, 8 up, 8 in 2024-08-17T17:36:15.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:14 smithi130 ceph-mon[110068]: pgmap v800: 713 pgs: 8 peering, 705 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 646 KiB/s rd, 219 KiB/s wr, 1.02k op/s 2024-08-17T17:36:15.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:15.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:15.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:14 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:36:15.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:14 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:36:15.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:15.331 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:14 smithi179 ceph-mon[93519]: osdmap e751: 8 total, 8 up, 8 in 2024-08-17T17:36:15.331 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:14 smithi179 ceph-mon[93519]: pgmap v800: 713 pgs: 8 peering, 705 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 646 KiB/s rd, 219 KiB/s wr, 1.02k op/s 2024-08-17T17:36:15.331 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:15.331 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:15.331 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:14 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:36:15.331 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:14 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:36:15.331 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:15.952 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot (960 ms) 2024-08-17T17:36:15.953 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestImageOptions 2024-08-17T17:36:16.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:15 smithi130 ceph-mon[105521]: osdmap e752: 8 total, 8 up, 8 in 2024-08-17T17:36:16.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:15 smithi130 ceph-mon[110068]: osdmap e752: 8 total, 8 up, 8 in 2024-08-17T17:36:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:15 smithi179 ceph-mon[93519]: osdmap e752: 8 total, 8 up, 8 in 2024-08-17T17:36:17.049 INFO:tasks.workunit.client.0.smithi130.stdout:0.00% done 2024-08-17T17:36:17.049 INFO:tasks.workunit.client.0.smithi130.stdout:100.00% done 2024-08-17T17:36:17.050 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestImageOptions (1097 ms) 2024-08-17T17:36:17.050 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestImageOptionsPP 2024-08-17T17:36:17.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:16 smithi179 ceph-mon[93519]: osdmap e753: 8 total, 8 up, 8 in 2024-08-17T17:36:17.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:16 smithi179 ceph-mon[93519]: pgmap v803: 713 pgs: 8 peering, 705 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 322 KiB/s rd, 79 KiB/s wr, 486 op/s 2024-08-17T17:36:17.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:16 smithi130 ceph-mon[105521]: osdmap e753: 8 total, 8 up, 8 in 2024-08-17T17:36:17.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:16 smithi130 ceph-mon[105521]: pgmap v803: 713 pgs: 8 peering, 705 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 322 KiB/s rd, 79 KiB/s wr, 486 op/s 2024-08-17T17:36:17.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:16 smithi130 ceph-mon[110068]: osdmap e753: 8 total, 8 up, 8 in 2024-08-17T17:36:17.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:16 smithi130 ceph-mon[110068]: pgmap v803: 713 pgs: 8 peering, 705 active+clean; 139 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 322 KiB/s rd, 79 KiB/s wr, 486 op/s 2024-08-17T17:36:18.031 INFO:tasks.workunit.client.0.smithi130.stdout:0.00% done 2024-08-17T17:36:18.031 INFO:tasks.workunit.client.0.smithi130.stdout:100.00% done 2024-08-17T17:36:18.031 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestImageOptionsPP (982 ms) 2024-08-17T17:36:18.031 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.EventSocketPipe 2024-08-17T17:36:18.031 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.EventSocketPipe (0 ms) 2024-08-17T17:36:18.031 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.EventSocketEventfd 2024-08-17T17:36:18.032 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.EventSocketEventfd (0 ms) 2024-08-17T17:36:18.032 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ImagePollIO 2024-08-17T17:36:18.032 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:18.072 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:36:18.072 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:36:18.072 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:36:18.072 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:36:18.072 INFO:tasks.workunit.client.0.smithi130.stdout:wrote: 512 2024-08-17T17:36:18.072 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:36:18.072 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:36:18.072 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:36:18.073 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:36:18.073 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:36:18.073 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:36:18.073 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:36:18.073 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:36:18.073 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:36:18.073 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:36:18.073 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:36:18.073 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:36:18.073 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:36:18.073 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:36:18.073 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:36:18.073 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:36:18.074 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:36:18.074 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:36:18.074 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:36:18.074 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:36:18.074 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:36:18.074 INFO:tasks.workunit.client.0.smithi130.stdout:started write 2024-08-17T17:36:18.074 INFO:tasks.workunit.client.0.smithi130.stdout:write completion cb called! 2024-08-17T17:36:18.074 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 0 2024-08-17T17:36:18.074 INFO:tasks.workunit.client.0.smithi130.stdout:finished write 2024-08-17T17:36:18.074 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:36:18.074 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:36:18.074 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:36:18.075 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:36:18.075 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:36:18.075 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:36:18.075 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:36:18.075 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:36:18.075 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:36:18.075 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:36:18.075 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:36:18.075 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:36:18.075 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:36:18.075 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:36:18.075 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:36:18.075 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:36:18.075 INFO:tasks.workunit.client.0.smithi130.stdout:created completion 2024-08-17T17:36:18.076 INFO:tasks.workunit.client.0.smithi130.stdout:started read 2024-08-17T17:36:18.076 INFO:tasks.workunit.client.0.smithi130.stdout:read completion cb called! 2024-08-17T17:36:18.076 INFO:tasks.workunit.client.0.smithi130.stdout:return value is: 512 2024-08-17T17:36:18.076 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ImagePollIO (41 ms) 2024-08-17T17:36:18.076 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.Mirror 2024-08-17T17:36:18.076 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:18.083 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:18.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:17 smithi179 ceph-mon[93519]: osdmap e754: 8 total, 8 up, 8 in 2024-08-17T17:36:18.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:17 smithi130 ceph-mon[105521]: osdmap e754: 8 total, 8 up, 8 in 2024-08-17T17:36:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:17 smithi130 ceph-mon[110068]: osdmap e754: 8 total, 8 up, 8 in 2024-08-17T17:36:19.085 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.Mirror (1013 ms) 2024-08-17T17:36:19.085 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.MirrorPeerAttributes 2024-08-17T17:36:19.099 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.MirrorPeerAttributes (14 ms) 2024-08-17T17:36:19.099 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.CreateWithMirrorEnabled 2024-08-17T17:36:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:18 smithi179 ceph-mon[93519]: osdmap e755: 8 total, 8 up, 8 in 2024-08-17T17:36:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:18 smithi179 ceph-mon[93519]: pgmap v806: 713 pgs: 713 active+clean; 144 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 373 KiB/s rd, 1.2 MiB/s wr, 485 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:36:19.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:18 smithi130 ceph-mon[105521]: osdmap e755: 8 total, 8 up, 8 in 2024-08-17T17:36:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:18 smithi130 ceph-mon[105521]: pgmap v806: 713 pgs: 713 active+clean; 144 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 373 KiB/s rd, 1.2 MiB/s wr, 485 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:36:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:18 smithi130 ceph-mon[110068]: osdmap e755: 8 total, 8 up, 8 in 2024-08-17T17:36:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:18 smithi130 ceph-mon[110068]: pgmap v806: 713 pgs: 713 active+clean; 144 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 373 KiB/s rd, 1.2 MiB/s wr, 485 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:36:20.169 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:36:19] "GET /metrics HTTP/1.1" 200 82249 "" "Prometheus/2.51.0" 2024-08-17T17:36:20.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[105521]: osdmap e756: 8 total, 8 up, 8 in 2024-08-17T17:36:20.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"} : dispatch 2024-08-17T17:36:20.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-08-17T17:36:20.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3201cc26-922f-49f0-84c0-8416bce2ecda"} : dispatch 2024-08-17T17:36:20.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3201cc26-922f-49f0-84c0-8416bce2ecda"}]': finished 2024-08-17T17:36:20.170 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/04be7d08-ebba-4610-8f57-c72408cab703"} : dispatch 2024-08-17T17:36:20.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/04be7d08-ebba-4610-8f57-c72408cab703"}]': finished 2024-08-17T17:36:20.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/91011cb8-e9b5-4bf6-ab6e-d2d505234b09"} : dispatch 2024-08-17T17:36:20.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/91011cb8-e9b5-4bf6-ab6e-d2d505234b09"}]': finished 2024-08-17T17:36:20.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/19/35ef1e44-dacf-4d97-969d-b092f3f2aaee"} : dispatch 2024-08-17T17:36:20.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' 2024-08-17T17:36:20.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/19/35ef1e44-dacf-4d97-969d-b092f3f2aaee"} : dispatch 2024-08-17T17:36:20.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/35ef1e44-dacf-4d97-969d-b092f3f2aaee"} : dispatch 2024-08-17T17:36:20.171 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/35ef1e44-dacf-4d97-969d-b092f3f2aaee"}]': finished 2024-08-17T17:36:20.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[110068]: osdmap e756: 8 total, 8 up, 8 in 2024-08-17T17:36:20.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"} : dispatch 2024-08-17T17:36:20.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-08-17T17:36:20.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3201cc26-922f-49f0-84c0-8416bce2ecda"} : dispatch 2024-08-17T17:36:20.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3201cc26-922f-49f0-84c0-8416bce2ecda"}]': finished 2024-08-17T17:36:20.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/04be7d08-ebba-4610-8f57-c72408cab703"} : dispatch 2024-08-17T17:36:20.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/04be7d08-ebba-4610-8f57-c72408cab703"}]': finished 2024-08-17T17:36:20.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/91011cb8-e9b5-4bf6-ab6e-d2d505234b09"} : dispatch 2024-08-17T17:36:20.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/91011cb8-e9b5-4bf6-ab6e-d2d505234b09"}]': finished 2024-08-17T17:36:20.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/19/35ef1e44-dacf-4d97-969d-b092f3f2aaee"} : dispatch 2024-08-17T17:36:20.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' 2024-08-17T17:36:20.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/19/35ef1e44-dacf-4d97-969d-b092f3f2aaee"} : dispatch 2024-08-17T17:36:20.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/35ef1e44-dacf-4d97-969d-b092f3f2aaee"} : dispatch 2024-08-17T17:36:20.172 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/35ef1e44-dacf-4d97-969d-b092f3f2aaee"}]': finished 2024-08-17T17:36:20.214 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:19 smithi179 ceph-mon[93519]: osdmap e756: 8 total, 8 up, 8 in 2024-08-17T17:36:20.214 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"} : dispatch 2024-08-17T17:36:20.214 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-08-17T17:36:20.214 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3201cc26-922f-49f0-84c0-8416bce2ecda"} : dispatch 2024-08-17T17:36:20.214 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/3201cc26-922f-49f0-84c0-8416bce2ecda"}]': finished 2024-08-17T17:36:20.214 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/04be7d08-ebba-4610-8f57-c72408cab703"} : dispatch 2024-08-17T17:36:20.214 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/04be7d08-ebba-4610-8f57-c72408cab703"}]': finished 2024-08-17T17:36:20.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/91011cb8-e9b5-4bf6-ab6e-d2d505234b09"} : dispatch 2024-08-17T17:36:20.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/91011cb8-e9b5-4bf6-ab6e-d2d505234b09"}]': finished 2024-08-17T17:36:20.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/19/35ef1e44-dacf-4d97-969d-b092f3f2aaee"} : dispatch 2024-08-17T17:36:20.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' 2024-08-17T17:36:20.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/19/35ef1e44-dacf-4d97-969d-b092f3f2aaee"} : dispatch 2024-08-17T17:36:20.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/35ef1e44-dacf-4d97-969d-b092f3f2aaee"} : dispatch 2024-08-17T17:36:20.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/245470950' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/35ef1e44-dacf-4d97-969d-b092f3f2aaee"}]': finished 2024-08-17T17:36:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:36:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:36:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:36:21.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:21 smithi130 ceph-mon[105521]: osdmap e757: 8 total, 8 up, 8 in 2024-08-17T17:36:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:21 smithi130 ceph-mon[105521]: pgmap v809: 713 pgs: 713 active+clean; 146 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 387 KiB/s rd, 1.7 MiB/s wr, 502 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:36:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:21 smithi130 ceph-mon[110068]: osdmap e757: 8 total, 8 up, 8 in 2024-08-17T17:36:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:21 smithi130 ceph-mon[110068]: pgmap v809: 713 pgs: 713 active+clean; 146 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 387 KiB/s rd, 1.7 MiB/s wr, 502 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:36:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:21 smithi179 ceph-mon[93519]: osdmap e757: 8 total, 8 up, 8 in 2024-08-17T17:36:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:21 smithi179 ceph-mon[93519]: pgmap v809: 713 pgs: 713 active+clean; 146 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 387 KiB/s rd, 1.7 MiB/s wr, 502 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:36:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:22 smithi130 ceph-mon[105521]: osdmap e758: 8 total, 8 up, 8 in 2024-08-17T17:36:22.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:22 smithi130 ceph-mon[110068]: osdmap e758: 8 total, 8 up, 8 in 2024-08-17T17:36:22.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:22 smithi179 ceph-mon[93519]: osdmap e758: 8 total, 8 up, 8 in 2024-08-17T17:36:23.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:23 smithi130 ceph-mon[105521]: osdmap e759: 8 total, 8 up, 8 in 2024-08-17T17:36:23.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:23 smithi130 ceph-mon[105521]: pgmap v812: 713 pgs: 713 active+clean; 147 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s rd, 906 KiB/s wr, 79 op/s 2024-08-17T17:36:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:23 smithi130 ceph-mon[110068]: osdmap e759: 8 total, 8 up, 8 in 2024-08-17T17:36:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:23 smithi130 ceph-mon[110068]: pgmap v812: 713 pgs: 713 active+clean; 147 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s rd, 906 KiB/s wr, 79 op/s 2024-08-17T17:36:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:23 smithi179 ceph-mon[93519]: osdmap e759: 8 total, 8 up, 8 in 2024-08-17T17:36:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:23 smithi179 ceph-mon[93519]: pgmap v812: 713 pgs: 713 active+clean; 147 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s rd, 906 KiB/s wr, 79 op/s 2024-08-17T17:36:24.167 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.CreateWithMirrorEnabled (5068 ms) 2024-08-17T17:36:24.168 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot 2024-08-17T17:36:24.168 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:24.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:24 smithi130 ceph-mon[105521]: osdmap e760: 8 total, 8 up, 8 in 2024-08-17T17:36:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:24 smithi130 ceph-mon[110068]: osdmap e760: 8 total, 8 up, 8 in 2024-08-17T17:36:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:24 smithi179 ceph-mon[93519]: osdmap e760: 8 total, 8 up, 8 in 2024-08-17T17:36:25.395 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:25 smithi179 ceph-mon[93519]: osdmap e761: 8 total, 8 up, 8 in 2024-08-17T17:36:25.395 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:25 smithi179 ceph-mon[93519]: pgmap v815: 713 pgs: 713 active+clean; 147 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 470 KiB/s wr, 314 op/s 2024-08-17T17:36:25.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:25 smithi130 ceph-mon[105521]: osdmap e761: 8 total, 8 up, 8 in 2024-08-17T17:36:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:25 smithi130 ceph-mon[105521]: pgmap v815: 713 pgs: 713 active+clean; 147 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 470 KiB/s wr, 314 op/s 2024-08-17T17:36:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:25 smithi130 ceph-mon[110068]: osdmap e761: 8 total, 8 up, 8 in 2024-08-17T17:36:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:25 smithi130 ceph-mon[110068]: pgmap v815: 713 pgs: 713 active+clean; 147 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 239 KiB/s rd, 470 KiB/s wr, 314 op/s 2024-08-17T17:36:26.174 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot (2007 ms) 2024-08-17T17:36:26.174 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ExclusiveLock 2024-08-17T17:36:26.174 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:26.224 INFO:tasks.workunit.client.0.smithi130.stdout:1: acquiring exclusive lock 2024-08-17T17:36:26.224 INFO:tasks.workunit.client.0.smithi130.stdout:2: acquiring exclusive lock 2024-08-17T17:36:26.227 INFO:tasks.workunit.client.0.smithi130.stdout:2: exclusive lock acquired 2024-08-17T17:36:26.271 INFO:tasks.workunit.client.0.smithi130.stdout:2: releasing exclusive lock 2024-08-17T17:36:26.273 INFO:tasks.workunit.client.0.smithi130.stdout:2: exclusive lock released 2024-08-17T17:36:26.274 INFO:tasks.workunit.client.0.smithi130.stdout:2: acquiring exclusive lock 2024-08-17T17:36:26.277 INFO:tasks.workunit.client.0.smithi130.stdout:1: exclusive lock acquired 2024-08-17T17:36:26.318 INFO:tasks.workunit.client.0.smithi130.stdout:1: releasing exclusive lock 2024-08-17T17:36:26.320 INFO:tasks.workunit.client.0.smithi130.stdout:1: exclusive lock released 2024-08-17T17:36:26.320 INFO:tasks.workunit.client.0.smithi130.stdout:1: acquiring exclusive lock 2024-08-17T17:36:26.324 INFO:tasks.workunit.client.0.smithi130.stdout:2: exclusive lock acquired 2024-08-17T17:36:26.342 INFO:tasks.workunit.client.0.smithi130.stdout:2: releasing exclusive lock 2024-08-17T17:36:26.345 INFO:tasks.workunit.client.0.smithi130.stdout:2: exclusive lock released 2024-08-17T17:36:26.345 INFO:tasks.workunit.client.0.smithi130.stdout:2: acquiring exclusive lock 2024-08-17T17:36:26.349 INFO:tasks.workunit.client.0.smithi130.stdout:2: exclusive lock acquired 2024-08-17T17:36:26.364 INFO:tasks.workunit.client.0.smithi130.stdout:2: releasing exclusive lock 2024-08-17T17:36:26.367 INFO:tasks.workunit.client.0.smithi130.stdout:2: exclusive lock released 2024-08-17T17:36:26.367 INFO:tasks.workunit.client.0.smithi130.stdout:2: acquiring exclusive lock 2024-08-17T17:36:26.370 INFO:tasks.workunit.client.0.smithi130.stdout:1: exclusive lock acquired 2024-08-17T17:36:26.380 INFO:tasks.workunit.client.0.smithi130.stdout:1: releasing exclusive lock 2024-08-17T17:36:26.382 INFO:tasks.workunit.client.0.smithi130.stdout:1: exclusive lock released 2024-08-17T17:36:26.383 INFO:tasks.workunit.client.0.smithi130.stdout:1: acquiring exclusive lock 2024-08-17T17:36:26.386 INFO:tasks.workunit.client.0.smithi130.stdout:1: exclusive lock acquired 2024-08-17T17:36:26.391 INFO:tasks.workunit.client.0.smithi130.stdout:1: releasing exclusive lock 2024-08-17T17:36:26.394 INFO:tasks.workunit.client.0.smithi130.stdout:1: exclusive lock released 2024-08-17T17:36:26.394 INFO:tasks.workunit.client.0.smithi130.stdout:1: acquiring exclusive lock 2024-08-17T17:36:26.397 INFO:tasks.workunit.client.0.smithi130.stdout:1: exclusive lock acquired 2024-08-17T17:36:26.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:26 smithi130 ceph-mon[105521]: osdmap e762: 8 total, 8 up, 8 in 2024-08-17T17:36:26.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:26 smithi130 ceph-mon[110068]: osdmap e762: 8 total, 8 up, 8 in 2024-08-17T17:36:26.447 INFO:tasks.workunit.client.0.smithi130.stdout:1: releasing exclusive lock 2024-08-17T17:36:26.450 INFO:tasks.workunit.client.0.smithi130.stdout:1: exclusive lock released 2024-08-17T17:36:26.450 INFO:tasks.workunit.client.0.smithi130.stdout:1: acquiring exclusive lock 2024-08-17T17:36:26.454 INFO:tasks.workunit.client.0.smithi130.stdout:2: exclusive lock acquired 2024-08-17T17:36:26.468 INFO:tasks.workunit.client.0.smithi130.stdout:2: releasing exclusive lock 2024-08-17T17:36:26.470 INFO:tasks.workunit.client.0.smithi130.stdout:2: exclusive lock released 2024-08-17T17:36:26.471 INFO:tasks.workunit.client.0.smithi130.stdout:2: acquiring exclusive lock 2024-08-17T17:36:26.473 INFO:tasks.workunit.client.0.smithi130.stdout:1: exclusive lock acquired 2024-08-17T17:36:26.479 INFO:tasks.workunit.client.0.smithi130.stdout:1: releasing exclusive lock 2024-08-17T17:36:26.482 INFO:tasks.workunit.client.0.smithi130.stdout:1: exclusive lock released 2024-08-17T17:36:26.485 INFO:tasks.workunit.client.0.smithi130.stdout:2: exclusive lock acquired 2024-08-17T17:36:26.507 INFO:tasks.workunit.client.0.smithi130.stdout:2: releasing exclusive lock 2024-08-17T17:36:26.509 INFO:tasks.workunit.client.0.smithi130.stdout:2: exclusive lock released 2024-08-17T17:36:26.522 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ExclusiveLock (349 ms) 2024-08-17T17:36:26.522 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.BreakLock 2024-08-17T17:36:26.538 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:26 smithi179 ceph-mon[93519]: osdmap e762: 8 total, 8 up, 8 in 2024-08-17T17:36:27.185 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.BreakLock (662 ms) 2024-08-17T17:36:27.185 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.DiscardAfterWrite 2024-08-17T17:36:27.185 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:27.209 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.DiscardAfterWrite (24 ms) 2024-08-17T17:36:27.209 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.DefaultFeatures 2024-08-17T17:36:27.209 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.DefaultFeatures (0 ms) 2024-08-17T17:36:27.209 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurge 2024-08-17T17:36:27.209 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:27.267 INFO:tasks.workunit.client.0.smithi130.stdout:0.00% done 2024-08-17T17:36:27.267 INFO:tasks.workunit.client.0.smithi130.stdout:1.56% done 2024-08-17T17:36:27.267 INFO:tasks.workunit.client.0.smithi130.stdout:3.12% done 2024-08-17T17:36:27.267 INFO:tasks.workunit.client.0.smithi130.stdout:4.69% done 2024-08-17T17:36:27.267 INFO:tasks.workunit.client.0.smithi130.stdout:6.25% done 2024-08-17T17:36:27.267 INFO:tasks.workunit.client.0.smithi130.stdout:7.81% done 2024-08-17T17:36:27.267 INFO:tasks.workunit.client.0.smithi130.stdout:9.38% done 2024-08-17T17:36:27.267 INFO:tasks.workunit.client.0.smithi130.stdout:10.94% done 2024-08-17T17:36:27.267 INFO:tasks.workunit.client.0.smithi130.stdout:12.50% done 2024-08-17T17:36:27.267 INFO:tasks.workunit.client.0.smithi130.stdout:14.06% done 2024-08-17T17:36:27.268 INFO:tasks.workunit.client.0.smithi130.stdout:15.62% done 2024-08-17T17:36:27.268 INFO:tasks.workunit.client.0.smithi130.stdout:17.19% done 2024-08-17T17:36:27.268 INFO:tasks.workunit.client.0.smithi130.stdout:18.75% done 2024-08-17T17:36:27.268 INFO:tasks.workunit.client.0.smithi130.stdout:20.31% done 2024-08-17T17:36:27.268 INFO:tasks.workunit.client.0.smithi130.stdout:21.88% done 2024-08-17T17:36:27.268 INFO:tasks.workunit.client.0.smithi130.stdout:23.44% done 2024-08-17T17:36:27.268 INFO:tasks.workunit.client.0.smithi130.stdout:25.00% done 2024-08-17T17:36:27.268 INFO:tasks.workunit.client.0.smithi130.stdout:26.56% done 2024-08-17T17:36:27.268 INFO:tasks.workunit.client.0.smithi130.stdout:28.12% done 2024-08-17T17:36:27.268 INFO:tasks.workunit.client.0.smithi130.stdout:29.69% done 2024-08-17T17:36:27.268 INFO:tasks.workunit.client.0.smithi130.stdout:31.25% done 2024-08-17T17:36:27.268 INFO:tasks.workunit.client.0.smithi130.stdout:32.81% done 2024-08-17T17:36:27.269 INFO:tasks.workunit.client.0.smithi130.stdout:34.38% done 2024-08-17T17:36:27.269 INFO:tasks.workunit.client.0.smithi130.stdout:35.94% done 2024-08-17T17:36:27.269 INFO:tasks.workunit.client.0.smithi130.stdout:37.50% done 2024-08-17T17:36:27.269 INFO:tasks.workunit.client.0.smithi130.stdout:39.06% done 2024-08-17T17:36:27.269 INFO:tasks.workunit.client.0.smithi130.stdout:40.62% done 2024-08-17T17:36:27.269 INFO:tasks.workunit.client.0.smithi130.stdout:42.19% done 2024-08-17T17:36:27.269 INFO:tasks.workunit.client.0.smithi130.stdout:43.75% done 2024-08-17T17:36:27.269 INFO:tasks.workunit.client.0.smithi130.stdout:45.31% done 2024-08-17T17:36:27.269 INFO:tasks.workunit.client.0.smithi130.stdout:46.88% done 2024-08-17T17:36:27.269 INFO:tasks.workunit.client.0.smithi130.stdout:48.44% done 2024-08-17T17:36:27.269 INFO:tasks.workunit.client.0.smithi130.stdout:50.00% done 2024-08-17T17:36:27.269 INFO:tasks.workunit.client.0.smithi130.stdout:51.56% done 2024-08-17T17:36:27.269 INFO:tasks.workunit.client.0.smithi130.stdout:53.12% done 2024-08-17T17:36:27.270 INFO:tasks.workunit.client.0.smithi130.stdout:54.69% done 2024-08-17T17:36:27.270 INFO:tasks.workunit.client.0.smithi130.stdout:56.25% done 2024-08-17T17:36:27.270 INFO:tasks.workunit.client.0.smithi130.stdout:57.81% done 2024-08-17T17:36:27.270 INFO:tasks.workunit.client.0.smithi130.stdout:59.38% done 2024-08-17T17:36:27.270 INFO:tasks.workunit.client.0.smithi130.stdout:60.94% done 2024-08-17T17:36:27.270 INFO:tasks.workunit.client.0.smithi130.stdout:62.50% done 2024-08-17T17:36:27.270 INFO:tasks.workunit.client.0.smithi130.stdout:64.06% done 2024-08-17T17:36:27.270 INFO:tasks.workunit.client.0.smithi130.stdout:65.62% done 2024-08-17T17:36:27.270 INFO:tasks.workunit.client.0.smithi130.stdout:67.19% done 2024-08-17T17:36:27.270 INFO:tasks.workunit.client.0.smithi130.stdout:68.75% done 2024-08-17T17:36:27.270 INFO:tasks.workunit.client.0.smithi130.stdout:70.31% done 2024-08-17T17:36:27.270 INFO:tasks.workunit.client.0.smithi130.stdout:71.88% done 2024-08-17T17:36:27.270 INFO:tasks.workunit.client.0.smithi130.stdout:73.44% done 2024-08-17T17:36:27.270 INFO:tasks.workunit.client.0.smithi130.stdout:75.00% done 2024-08-17T17:36:27.271 INFO:tasks.workunit.client.0.smithi130.stdout:76.56% done 2024-08-17T17:36:27.271 INFO:tasks.workunit.client.0.smithi130.stdout:78.12% done 2024-08-17T17:36:27.271 INFO:tasks.workunit.client.0.smithi130.stdout:79.69% done 2024-08-17T17:36:27.271 INFO:tasks.workunit.client.0.smithi130.stdout:81.25% done 2024-08-17T17:36:27.271 INFO:tasks.workunit.client.0.smithi130.stdout:82.81% done 2024-08-17T17:36:27.271 INFO:tasks.workunit.client.0.smithi130.stdout:84.38% done 2024-08-17T17:36:27.271 INFO:tasks.workunit.client.0.smithi130.stdout:85.94% done 2024-08-17T17:36:27.271 INFO:tasks.workunit.client.0.smithi130.stdout:87.50% done 2024-08-17T17:36:27.271 INFO:tasks.workunit.client.0.smithi130.stdout:89.06% done 2024-08-17T17:36:27.271 INFO:tasks.workunit.client.0.smithi130.stdout:90.62% done 2024-08-17T17:36:27.271 INFO:tasks.workunit.client.0.smithi130.stdout:92.19% done 2024-08-17T17:36:27.271 INFO:tasks.workunit.client.0.smithi130.stdout:93.75% done 2024-08-17T17:36:27.271 INFO:tasks.workunit.client.0.smithi130.stdout:95.31% done 2024-08-17T17:36:27.272 INFO:tasks.workunit.client.0.smithi130.stdout:96.88% done 2024-08-17T17:36:27.272 INFO:tasks.workunit.client.0.smithi130.stdout:98.44% done 2024-08-17T17:36:27.272 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurge (58 ms) 2024-08-17T17:36:27.272 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay 2024-08-17T17:36:27.272 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:27.321 INFO:tasks.workunit.client.0.smithi130.stdout:0.00% done 2024-08-17T17:36:27.321 INFO:tasks.workunit.client.0.smithi130.stdout:1.56% done 2024-08-17T17:36:27.321 INFO:tasks.workunit.client.0.smithi130.stdout:3.12% done 2024-08-17T17:36:27.321 INFO:tasks.workunit.client.0.smithi130.stdout:4.69% done 2024-08-17T17:36:27.321 INFO:tasks.workunit.client.0.smithi130.stdout:6.25% done 2024-08-17T17:36:27.322 INFO:tasks.workunit.client.0.smithi130.stdout:7.81% done 2024-08-17T17:36:27.322 INFO:tasks.workunit.client.0.smithi130.stdout:9.38% done 2024-08-17T17:36:27.322 INFO:tasks.workunit.client.0.smithi130.stdout:10.94% done 2024-08-17T17:36:27.322 INFO:tasks.workunit.client.0.smithi130.stdout:12.50% done 2024-08-17T17:36:27.322 INFO:tasks.workunit.client.0.smithi130.stdout:14.06% done 2024-08-17T17:36:27.322 INFO:tasks.workunit.client.0.smithi130.stdout:15.62% done 2024-08-17T17:36:27.322 INFO:tasks.workunit.client.0.smithi130.stdout:17.19% done 2024-08-17T17:36:27.322 INFO:tasks.workunit.client.0.smithi130.stdout:18.75% done 2024-08-17T17:36:27.322 INFO:tasks.workunit.client.0.smithi130.stdout:20.31% done 2024-08-17T17:36:27.322 INFO:tasks.workunit.client.0.smithi130.stdout:21.88% done 2024-08-17T17:36:27.322 INFO:tasks.workunit.client.0.smithi130.stdout:23.44% done 2024-08-17T17:36:27.322 INFO:tasks.workunit.client.0.smithi130.stdout:25.00% done 2024-08-17T17:36:27.323 INFO:tasks.workunit.client.0.smithi130.stdout:26.56% done 2024-08-17T17:36:27.323 INFO:tasks.workunit.client.0.smithi130.stdout:28.12% done 2024-08-17T17:36:27.323 INFO:tasks.workunit.client.0.smithi130.stdout:29.69% done 2024-08-17T17:36:27.323 INFO:tasks.workunit.client.0.smithi130.stdout:31.25% done 2024-08-17T17:36:27.323 INFO:tasks.workunit.client.0.smithi130.stdout:32.81% done 2024-08-17T17:36:27.323 INFO:tasks.workunit.client.0.smithi130.stdout:34.38% done 2024-08-17T17:36:27.323 INFO:tasks.workunit.client.0.smithi130.stdout:35.94% done 2024-08-17T17:36:27.323 INFO:tasks.workunit.client.0.smithi130.stdout:37.50% done 2024-08-17T17:36:27.323 INFO:tasks.workunit.client.0.smithi130.stdout:39.06% done 2024-08-17T17:36:27.323 INFO:tasks.workunit.client.0.smithi130.stdout:40.62% done 2024-08-17T17:36:27.323 INFO:tasks.workunit.client.0.smithi130.stdout:42.19% done 2024-08-17T17:36:27.323 INFO:tasks.workunit.client.0.smithi130.stdout:43.75% done 2024-08-17T17:36:27.323 INFO:tasks.workunit.client.0.smithi130.stdout:45.31% done 2024-08-17T17:36:27.323 INFO:tasks.workunit.client.0.smithi130.stdout:46.88% done 2024-08-17T17:36:27.324 INFO:tasks.workunit.client.0.smithi130.stdout:48.44% done 2024-08-17T17:36:27.324 INFO:tasks.workunit.client.0.smithi130.stdout:50.00% done 2024-08-17T17:36:27.324 INFO:tasks.workunit.client.0.smithi130.stdout:51.56% done 2024-08-17T17:36:27.324 INFO:tasks.workunit.client.0.smithi130.stdout:53.12% done 2024-08-17T17:36:27.324 INFO:tasks.workunit.client.0.smithi130.stdout:54.69% done 2024-08-17T17:36:27.324 INFO:tasks.workunit.client.0.smithi130.stdout:56.25% done 2024-08-17T17:36:27.324 INFO:tasks.workunit.client.0.smithi130.stdout:57.81% done 2024-08-17T17:36:27.324 INFO:tasks.workunit.client.0.smithi130.stdout:59.38% done 2024-08-17T17:36:27.324 INFO:tasks.workunit.client.0.smithi130.stdout:60.94% done 2024-08-17T17:36:27.324 INFO:tasks.workunit.client.0.smithi130.stdout:62.50% done 2024-08-17T17:36:27.324 INFO:tasks.workunit.client.0.smithi130.stdout:64.06% done 2024-08-17T17:36:27.324 INFO:tasks.workunit.client.0.smithi130.stdout:65.62% done 2024-08-17T17:36:27.324 INFO:tasks.workunit.client.0.smithi130.stdout:67.19% done 2024-08-17T17:36:27.324 INFO:tasks.workunit.client.0.smithi130.stdout:68.75% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:70.31% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:71.88% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:73.44% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:75.00% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:76.56% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:78.12% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:79.69% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:81.25% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:82.81% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:84.38% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:85.94% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:87.50% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:89.06% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:90.62% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:92.19% done 2024-08-17T17:36:27.325 INFO:tasks.workunit.client.0.smithi130.stdout:93.75% done 2024-08-17T17:36:27.326 INFO:tasks.workunit.client.0.smithi130.stdout:95.31% done 2024-08-17T17:36:27.326 INFO:tasks.workunit.client.0.smithi130.stdout:96.88% done 2024-08-17T17:36:27.326 INFO:tasks.workunit.client.0.smithi130.stdout:98.44% done 2024-08-17T17:36:27.326 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay (54 ms) 2024-08-17T17:36:27.326 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestTrashPurge 2024-08-17T17:36:27.326 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:27.329 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:27.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:27 smithi130 ceph-mon[105521]: osdmap e763: 8 total, 8 up, 8 in 2024-08-17T17:36:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:27 smithi130 ceph-mon[105521]: pgmap v818: 713 pgs: 713 active+clean; 147 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 193 KiB/s rd, 19 KiB/s wr, 253 op/s 2024-08-17T17:36:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:27 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/695423662"} : dispatch 2024-08-17T17:36:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:27 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/695423662"} : dispatch 2024-08-17T17:36:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:27 smithi130 ceph-mon[110068]: osdmap e763: 8 total, 8 up, 8 in 2024-08-17T17:36:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:27 smithi130 ceph-mon[110068]: pgmap v818: 713 pgs: 713 active+clean; 147 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 193 KiB/s rd, 19 KiB/s wr, 253 op/s 2024-08-17T17:36:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:27 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/695423662"} : dispatch 2024-08-17T17:36:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:27 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/695423662"} : dispatch 2024-08-17T17:36:27.443 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestTrashPurge (122 ms) 2024-08-17T17:36:27.443 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndRestore 2024-08-17T17:36:27.443 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:27.484 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestTrashMoveAndRestore (41 ms) 2024-08-17T17:36:27.484 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestListWatchers 2024-08-17T17:36:27.484 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:27.508 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestListWatchers (24 ms) 2024-08-17T17:36:27.508 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestSetSnapById 2024-08-17T17:36:27.508 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:27 smithi179 ceph-mon[93519]: osdmap e763: 8 total, 8 up, 8 in 2024-08-17T17:36:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:27 smithi179 ceph-mon[93519]: pgmap v818: 713 pgs: 713 active+clean; 147 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 193 KiB/s rd, 19 KiB/s wr, 253 op/s 2024-08-17T17:36:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:27 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2978656622' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/695423662"} : dispatch 2024-08-17T17:36:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:27 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/695423662"} : dispatch 2024-08-17T17:36:28.195 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestSetSnapById (686 ms) 2024-08-17T17:36:28.195 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.Namespaces 2024-08-17T17:36:28.216 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.Namespaces (22 ms) 2024-08-17T17:36:28.216 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.NamespacesPP 2024-08-17T17:36:28.247 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:28.251 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:28.324 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.NamespacesPP (108 ms) 2024-08-17T17:36:28.325 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.Migration 2024-08-17T17:36:28.325 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:28.325 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:28.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:28 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/695423662"}]': finished 2024-08-17T17:36:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:28 smithi130 ceph-mon[105521]: osdmap e764: 8 total, 8 up, 8 in 2024-08-17T17:36:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:28.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:28 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/695423662"}]': finished 2024-08-17T17:36:28.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:28 smithi130 ceph-mon[110068]: osdmap e764: 8 total, 8 up, 8 in 2024-08-17T17:36:28.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:28.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:28 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.130:0/695423662"}]': finished 2024-08-17T17:36:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:28 smithi179 ceph-mon[93519]: osdmap e764: 8 total, 8 up, 8 in 2024-08-17T17:36:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:28.695 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.Migration (371 ms) 2024-08-17T17:36:28.695 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.MigrationPP 2024-08-17T17:36:28.695 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:28.695 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:29.080 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.MigrationPP (385 ms) 2024-08-17T17:36:29.080 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestGetAccessTimestamp 2024-08-17T17:36:29.080 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:29.096 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestGetAccessTimestamp (16 ms) 2024-08-17T17:36:29.096 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.TestGetModifyTimestamp 2024-08-17T17:36:29.097 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:29.114 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.TestGetModifyTimestamp (18 ms) 2024-08-17T17:36:29.114 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ZeroOverlapFlatten 2024-08-17T17:36:29.114 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:29.251 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ZeroOverlapFlatten (137 ms) 2024-08-17T17:36:29.251 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.PoolMetadata 2024-08-17T17:36:29.279 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.PoolMetadata (28 ms) 2024-08-17T17:36:29.279 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.PoolMetadataPP 2024-08-17T17:36:29.295 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.PoolMetadataPP (16 ms) 2024-08-17T17:36:29.295 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.Config 2024-08-17T17:36:29.299 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:29.334 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.Config (39 ms) 2024-08-17T17:36:29.334 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ConfigPP 2024-08-17T17:36:29.338 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:29.373 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ConfigPP (39 ms) 2024-08-17T17:36:29.373 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.PoolStatsPP 2024-08-17T17:36:29.443 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:29 smithi130 ceph-mon[105521]: osdmap e765: 8 total, 8 up, 8 in 2024-08-17T17:36:29.443 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:29 smithi130 ceph-mon[105521]: pgmap v821: 713 pgs: 713 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 357 KiB/s rd, 84 KiB/s wr, 421 op/s 2024-08-17T17:36:29.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:29 smithi130 ceph-mon[110068]: osdmap e765: 8 total, 8 up, 8 in 2024-08-17T17:36:29.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:29 smithi130 ceph-mon[110068]: pgmap v821: 713 pgs: 713 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 357 KiB/s rd, 84 KiB/s wr, 421 op/s 2024-08-17T17:36:29.444 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:36:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:36:29] ENGINE Bus STOPPING 2024-08-17T17:36:29.444 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:36:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:36:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:36:29.444 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:36:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:36:29] ENGINE Bus STOPPED 2024-08-17T17:36:29.444 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:36:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:36:29] ENGINE Bus STARTING 2024-08-17T17:36:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:29 smithi179 ceph-mon[93519]: osdmap e765: 8 total, 8 up, 8 in 2024-08-17T17:36:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:29 smithi179 ceph-mon[93519]: pgmap v821: 713 pgs: 713 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 357 KiB/s rd, 84 KiB/s wr, 421 op/s 2024-08-17T17:36:29.783 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:36:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:36:29] ENGINE Serving on http://:::9283 2024-08-17T17:36:29.783 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:36:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:17:36:29] ENGINE Bus STARTED 2024-08-17T17:36:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:36:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:36:29] "GET /metrics HTTP/1.1" 200 82259 "" "Prometheus/2.51.0" 2024-08-17T17:36:30.470 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:30.576 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[105521]: osdmap e766: 8 total, 8 up, 8 in 2024-08-17T17:36:30.576 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' 2024-08-17T17:36:30.576 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:36:30.576 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:36:30.576 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[105521]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:36:30.576 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:30.576 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:30.576 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:36:30.576 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:36:30.577 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:30.577 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[110068]: osdmap e766: 8 total, 8 up, 8 in 2024-08-17T17:36:30.577 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' 2024-08-17T17:36:30.577 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:36:30.577 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:36:30.577 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[110068]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:36:30.577 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:30.577 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:30.577 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:36:30.577 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:36:30.578 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:30 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:36:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:36:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:36:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:30 smithi179 ceph-mon[93519]: osdmap e766: 8 total, 8 up, 8 in 2024-08-17T17:36:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:30 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' 2024-08-17T17:36:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:30 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:36:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:30 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-08-17T17:36:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:30 smithi179 ceph-mon[93519]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:36:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:30 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:30 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:30 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:36:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:30 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:36:30.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:30 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:31 smithi179 ceph-mon[93519]: pgmap v823: 713 pgs: 713 active+clean; 148 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 362 KiB/s rd, 151 KiB/s wr, 436 op/s 2024-08-17T17:36:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:31 smithi179 ceph-mon[93519]: osdmap e767: 8 total, 8 up, 8 in 2024-08-17T17:36:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/726604318' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/726604318' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:36:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:31 smithi179 ceph-mon[93519]: osdmap e768: 8 total, 8 up, 8 in 2024-08-17T17:36:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:31 smithi130 ceph-mon[105521]: pgmap v823: 713 pgs: 713 active+clean; 148 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 362 KiB/s rd, 151 KiB/s wr, 436 op/s 2024-08-17T17:36:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:31 smithi130 ceph-mon[105521]: osdmap e767: 8 total, 8 up, 8 in 2024-08-17T17:36:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/726604318' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/726604318' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:36:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:31 smithi130 ceph-mon[105521]: osdmap e768: 8 total, 8 up, 8 in 2024-08-17T17:36:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:31 smithi130 ceph-mon[110068]: pgmap v823: 713 pgs: 713 active+clean; 148 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 362 KiB/s rd, 151 KiB/s wr, 436 op/s 2024-08-17T17:36:31.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:31 smithi130 ceph-mon[110068]: osdmap e767: 8 total, 8 up, 8 in 2024-08-17T17:36:31.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/726604318' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:31.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/726604318' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:36:31.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:31 smithi130 ceph-mon[110068]: osdmap e768: 8 total, 8 up, 8 in 2024-08-17T17:36:32.500 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:32.509 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:32.517 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:32 smithi179 ceph-mon[93519]: osdmap e769: 8 total, 8 up, 8 in 2024-08-17T17:36:32.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:32 smithi130 ceph-mon[105521]: osdmap e769: 8 total, 8 up, 8 in 2024-08-17T17:36:32.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:32 smithi130 ceph-mon[110068]: osdmap e769: 8 total, 8 up, 8 in 2024-08-17T17:36:33.549 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.PoolStatsPP (4176 ms) 2024-08-17T17:36:33.549 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ImageSpec 2024-08-17T17:36:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:33 smithi179 ceph-mon[93519]: pgmap v827: 745 pgs: 29 unknown, 716 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 71 KiB/s wr, 98 op/s 2024-08-17T17:36:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:33 smithi179 ceph-mon[93519]: osdmap e770: 8 total, 8 up, 8 in 2024-08-17T17:36:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:33 smithi130 ceph-mon[105521]: pgmap v827: 745 pgs: 29 unknown, 716 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 71 KiB/s wr, 98 op/s 2024-08-17T17:36:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:33 smithi130 ceph-mon[105521]: osdmap e770: 8 total, 8 up, 8 in 2024-08-17T17:36:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:33 smithi130 ceph-mon[110068]: pgmap v827: 745 pgs: 29 unknown, 716 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 71 KiB/s wr, 98 op/s 2024-08-17T17:36:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:33 smithi130 ceph-mon[110068]: osdmap e770: 8 total, 8 up, 8 in 2024-08-17T17:36:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:34 smithi179 ceph-mon[93519]: osdmap e771: 8 total, 8 up, 8 in 2024-08-17T17:36:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:34 smithi130 ceph-mon[110068]: osdmap e771: 8 total, 8 up, 8 in 2024-08-17T17:36:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:34 smithi130 ceph-mon[105521]: osdmap e771: 8 total, 8 up, 8 in 2024-08-17T17:36:35.468 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:35 smithi179 ceph-mon[93519]: pgmap v830: 745 pgs: 4 unknown, 741 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 737 KiB/s rd, 46 KiB/s wr, 955 op/s 2024-08-17T17:36:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:35 smithi179 ceph-mon[93519]: osdmap e772: 8 total, 8 up, 8 in 2024-08-17T17:36:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:35 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2698610754' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:35 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2698610754' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:36:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:35 smithi179 ceph-mon[93519]: osdmap e773: 8 total, 8 up, 8 in 2024-08-17T17:36:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:35 smithi130 ceph-mon[110068]: pgmap v830: 745 pgs: 4 unknown, 741 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 737 KiB/s rd, 46 KiB/s wr, 955 op/s 2024-08-17T17:36:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:35 smithi130 ceph-mon[110068]: osdmap e772: 8 total, 8 up, 8 in 2024-08-17T17:36:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:35 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2698610754' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:35 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2698610754' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:36:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:35 smithi130 ceph-mon[110068]: osdmap e773: 8 total, 8 up, 8 in 2024-08-17T17:36:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:35 smithi130 ceph-mon[105521]: pgmap v830: 745 pgs: 4 unknown, 741 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 737 KiB/s rd, 46 KiB/s wr, 955 op/s 2024-08-17T17:36:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:35 smithi130 ceph-mon[105521]: osdmap e772: 8 total, 8 up, 8 in 2024-08-17T17:36:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:35 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2698610754' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:35 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2698610754' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:36:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:35 smithi130 ceph-mon[105521]: osdmap e773: 8 total, 8 up, 8 in 2024-08-17T17:36:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:37 smithi179 ceph-mon[93519]: pgmap v833: 777 pgs: 8 creating+peering, 24 unknown, 745 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 670 KiB/s rd, 42 KiB/s wr, 859 op/s 2024-08-17T17:36:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:37 smithi179 ceph-mon[93519]: osdmap e774: 8 total, 8 up, 8 in 2024-08-17T17:36:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:37 smithi130 ceph-mon[110068]: pgmap v833: 777 pgs: 8 creating+peering, 24 unknown, 745 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 670 KiB/s rd, 42 KiB/s wr, 859 op/s 2024-08-17T17:36:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:37 smithi130 ceph-mon[110068]: osdmap e774: 8 total, 8 up, 8 in 2024-08-17T17:36:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:37 smithi130 ceph-mon[105521]: pgmap v833: 777 pgs: 8 creating+peering, 24 unknown, 745 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 670 KiB/s rd, 42 KiB/s wr, 859 op/s 2024-08-17T17:36:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:37 smithi130 ceph-mon[105521]: osdmap e774: 8 total, 8 up, 8 in 2024-08-17T17:36:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:38 smithi179 ceph-mon[93519]: osdmap e775: 8 total, 8 up, 8 in 2024-08-17T17:36:38.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:38 smithi130 ceph-mon[105521]: osdmap e775: 8 total, 8 up, 8 in 2024-08-17T17:36:38.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:38 smithi130 ceph-mon[110068]: osdmap e775: 8 total, 8 up, 8 in 2024-08-17T17:36:39.635 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ImageSpec (6086 ms) 2024-08-17T17:36:39.635 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.SnapRemoveWithChildMissing 2024-08-17T17:36:39.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:39 smithi130 ceph-mon[105521]: pgmap v836: 777 pgs: 8 creating+peering, 2 unknown, 767 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 3.7 KiB/s wr, 31 op/s 2024-08-17T17:36:39.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:39 smithi130 ceph-mon[105521]: osdmap e776: 8 total, 8 up, 8 in 2024-08-17T17:36:39.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:39 smithi130 ceph-mon[110068]: pgmap v836: 777 pgs: 8 creating+peering, 2 unknown, 767 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 3.7 KiB/s wr, 31 op/s 2024-08-17T17:36:39.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:39 smithi130 ceph-mon[110068]: osdmap e776: 8 total, 8 up, 8 in 2024-08-17T17:36:39.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:39 smithi179 ceph-mon[93519]: pgmap v836: 777 pgs: 8 creating+peering, 2 unknown, 767 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 3.7 KiB/s wr, 31 op/s 2024-08-17T17:36:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:39 smithi179 ceph-mon[93519]: osdmap e776: 8 total, 8 up, 8 in 2024-08-17T17:36:40.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:36:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:36:39] "GET /metrics HTTP/1.1" 200 82259 "" "Prometheus/2.51.0" 2024-08-17T17:36:40.498 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:36:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:36:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:36:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:40 smithi179 ceph-mon[93519]: osdmap e777: 8 total, 8 up, 8 in 2024-08-17T17:36:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:40 smithi179 ceph-mon[93519]: osdmap e778: 8 total, 8 up, 8 in 2024-08-17T17:36:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:40 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1551519405' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-37","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:40 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-37","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:40 smithi130 ceph-mon[110068]: osdmap e777: 8 total, 8 up, 8 in 2024-08-17T17:36:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:40 smithi130 ceph-mon[110068]: osdmap e778: 8 total, 8 up, 8 in 2024-08-17T17:36:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:40 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1551519405' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-37","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:40 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-37","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:40 smithi130 ceph-mon[105521]: osdmap e777: 8 total, 8 up, 8 in 2024-08-17T17:36:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:40 smithi130 ceph-mon[105521]: osdmap e778: 8 total, 8 up, 8 in 2024-08-17T17:36:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:40 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1551519405' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-37","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:40.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:40 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-37","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:41.477 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:41 smithi179 ceph-mon[93519]: pgmap v839: 777 pgs: 777 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.0 KiB/s wr, 33 op/s 2024-08-17T17:36:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:41 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:36:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:41 smithi179 ceph-mon[93519]: osdmap e779: 8 total, 8 up, 8 in 2024-08-17T17:36:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:41 smithi130 ceph-mon[105521]: pgmap v839: 777 pgs: 777 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.0 KiB/s wr, 33 op/s 2024-08-17T17:36:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:41 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:36:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:41 smithi130 ceph-mon[105521]: osdmap e779: 8 total, 8 up, 8 in 2024-08-17T17:36:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:41 smithi130 ceph-mon[110068]: pgmap v839: 777 pgs: 777 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.0 KiB/s wr, 33 op/s 2024-08-17T17:36:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:41 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:36:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:41 smithi130 ceph-mon[110068]: osdmap e779: 8 total, 8 up, 8 in 2024-08-17T17:36:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:42 smithi179 ceph-mon[93519]: osdmap e780: 8 total, 8 up, 8 in 2024-08-17T17:36:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:42.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:42 smithi130 ceph-mon[105521]: osdmap e780: 8 total, 8 up, 8 in 2024-08-17T17:36:42.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:42.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:42 smithi130 ceph-mon[110068]: osdmap e780: 8 total, 8 up, 8 in 2024-08-17T17:36:43.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:43 smithi130 ceph-mon[105521]: pgmap v842: 809 pgs: 2 creating+activating, 29 unknown, 778 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s rd, 2.5 KiB/s wr, 83 op/s 2024-08-17T17:36:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:43 smithi130 ceph-mon[105521]: osdmap e781: 8 total, 8 up, 8 in 2024-08-17T17:36:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:43 smithi130 ceph-mon[110068]: pgmap v842: 809 pgs: 2 creating+activating, 29 unknown, 778 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s rd, 2.5 KiB/s wr, 83 op/s 2024-08-17T17:36:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:43 smithi130 ceph-mon[110068]: osdmap e781: 8 total, 8 up, 8 in 2024-08-17T17:36:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:43 smithi179 ceph-mon[93519]: pgmap v842: 809 pgs: 2 creating+activating, 29 unknown, 778 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 67 KiB/s rd, 2.5 KiB/s wr, 83 op/s 2024-08-17T17:36:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:43 smithi179 ceph-mon[93519]: osdmap e781: 8 total, 8 up, 8 in 2024-08-17T17:36:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:45 smithi179 ceph-mon[93519]: pgmap v845: 809 pgs: 2 creating+activating, 5 unknown, 802 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 126 KiB/s rd, 4.7 KiB/s wr, 155 op/s 2024-08-17T17:36:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:45 smithi179 ceph-mon[93519]: osdmap e782: 8 total, 8 up, 8 in 2024-08-17T17:36:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:45 smithi179 ceph-mon[93519]: osdmap e783: 8 total, 8 up, 8 in 2024-08-17T17:36:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:45 smithi130 ceph-mon[105521]: pgmap v845: 809 pgs: 2 creating+activating, 5 unknown, 802 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 126 KiB/s rd, 4.7 KiB/s wr, 155 op/s 2024-08-17T17:36:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:45 smithi130 ceph-mon[105521]: osdmap e782: 8 total, 8 up, 8 in 2024-08-17T17:36:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:45 smithi130 ceph-mon[105521]: osdmap e783: 8 total, 8 up, 8 in 2024-08-17T17:36:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:45 smithi130 ceph-mon[110068]: pgmap v845: 809 pgs: 2 creating+activating, 5 unknown, 802 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 126 KiB/s rd, 4.7 KiB/s wr, 155 op/s 2024-08-17T17:36:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:45 smithi130 ceph-mon[110068]: osdmap e782: 8 total, 8 up, 8 in 2024-08-17T17:36:45.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:45 smithi130 ceph-mon[110068]: osdmap e783: 8 total, 8 up, 8 in 2024-08-17T17:36:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:47 smithi179 ceph-mon[93519]: pgmap v848: 809 pgs: 2 creating+activating, 807 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s rd, 3.5 KiB/s wr, 84 op/s 2024-08-17T17:36:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:47 smithi179 ceph-mon[93519]: osdmap e784: 8 total, 8 up, 8 in 2024-08-17T17:36:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:47 smithi130 ceph-mon[105521]: pgmap v848: 809 pgs: 2 creating+activating, 807 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s rd, 3.5 KiB/s wr, 84 op/s 2024-08-17T17:36:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:47 smithi130 ceph-mon[105521]: osdmap e784: 8 total, 8 up, 8 in 2024-08-17T17:36:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:47 smithi130 ceph-mon[110068]: pgmap v848: 809 pgs: 2 creating+activating, 807 active+clean; 148 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 66 KiB/s rd, 3.5 KiB/s wr, 84 op/s 2024-08-17T17:36:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:47 smithi130 ceph-mon[110068]: osdmap e784: 8 total, 8 up, 8 in 2024-08-17T17:36:48.529 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 3 2024-08-17T17:36:48.529 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 3 2024-08-17T17:36:48.530 INFO:tasks.workunit.client.0.smithi130.stdout:child: image213 2024-08-17T17:36:48.530 INFO:tasks.workunit.client.0.smithi130.stdout:child: image215 2024-08-17T17:36:48.530 INFO:tasks.workunit.client.0.smithi130.stdout:child: image214 2024-08-17T17:36:48.530 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-15/image213/d513e0d278a4 2024-08-17T17:36:48.530 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:36:48.530 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-37/image214/d51359431631 2024-08-17T17:36:48.530 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:36:48.530 INFO:tasks.workunit.client.0.smithi130.stdout:found child test-librbd-smithi130-87148-15/image215/d513f5896970 2024-08-17T17:36:48.530 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:36:48.530 INFO:tasks.workunit.client.0.smithi130.stdout:num children is: 0 2024-08-17T17:36:48.530 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 0 2024-08-17T17:36:48.530 INFO:tasks.workunit.client.0.smithi130.stdout:num snaps is: 0 2024-08-17T17:36:48.530 INFO:tasks.workunit.client.0.smithi130.stdout:expected: 0 2024-08-17T17:36:48.531 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.SnapRemoveWithChildMissing (8894 ms) 2024-08-17T17:36:48.531 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.QuiesceWatch 2024-08-17T17:36:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:48 smithi179 ceph-mon[93519]: osdmap e785: 8 total, 8 up, 8 in 2024-08-17T17:36:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:48 smithi130 ceph-mon[105521]: osdmap e785: 8 total, 8 up, 8 in 2024-08-17T17:36:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:48 smithi130 ceph-mon[110068]: osdmap e785: 8 total, 8 up, 8 in 2024-08-17T17:36:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:49 smithi130 ceph-mon[105521]: pgmap v851: 777 pgs: 777 active+clean; 2.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:36:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:49 smithi130 ceph-mon[105521]: osdmap e786: 8 total, 8 up, 8 in 2024-08-17T17:36:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:49 smithi130 ceph-mon[110068]: pgmap v851: 777 pgs: 777 active+clean; 2.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:36:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:49 smithi130 ceph-mon[110068]: osdmap e786: 8 total, 8 up, 8 in 2024-08-17T17:36:49.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:49 smithi179 ceph-mon[93519]: pgmap v851: 777 pgs: 777 active+clean; 2.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:36:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:49 smithi179 ceph-mon[93519]: osdmap e786: 8 total, 8 up, 8 in 2024-08-17T17:36:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:36:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:36:49] "GET /metrics HTTP/1.1" 200 89428 "" "Prometheus/2.51.0" 2024-08-17T17:36:50.473 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:50.511 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:36:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:36:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:36:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:50 smithi179 ceph-mon[93519]: osdmap e787: 8 total, 8 up, 8 in 2024-08-17T17:36:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:50 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2731778917' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-38","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:50 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-38","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:50 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-38","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:36:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:50 smithi179 ceph-mon[93519]: osdmap e788: 8 total, 8 up, 8 in 2024-08-17T17:36:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:50 smithi130 ceph-mon[105521]: osdmap e787: 8 total, 8 up, 8 in 2024-08-17T17:36:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:50 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2731778917' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-38","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:50 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-38","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:50 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-38","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:36:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:50 smithi130 ceph-mon[105521]: osdmap e788: 8 total, 8 up, 8 in 2024-08-17T17:36:50.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:50 smithi130 ceph-mon[110068]: osdmap e787: 8 total, 8 up, 8 in 2024-08-17T17:36:50.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:50 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2731778917' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-38","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:50.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:50 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-38","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:36:50.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:50 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-38","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:36:50.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:50 smithi130 ceph-mon[110068]: osdmap e788: 8 total, 8 up, 8 in 2024-08-17T17:36:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:51 smithi179 ceph-mon[93519]: pgmap v854: 809 pgs: 1 creating+peering, 31 unknown, 777 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:36:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:51 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:36:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:51 smithi179 ceph-mon[93519]: osdmap e789: 8 total, 8 up, 8 in 2024-08-17T17:36:51.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:51 smithi130 ceph-mon[105521]: pgmap v854: 809 pgs: 1 creating+peering, 31 unknown, 777 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:36:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:51 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:36:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:51 smithi130 ceph-mon[105521]: osdmap e789: 8 total, 8 up, 8 in 2024-08-17T17:36:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:51 smithi130 ceph-mon[110068]: pgmap v854: 809 pgs: 1 creating+peering, 31 unknown, 777 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:36:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:51 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:36:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:51 smithi130 ceph-mon[110068]: osdmap e789: 8 total, 8 up, 8 in 2024-08-17T17:36:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:53 smithi179 ceph-mon[93519]: pgmap v857: 809 pgs: 1 creating+peering, 26 unknown, 782 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 2 op/s 2024-08-17T17:36:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:53 smithi179 ceph-mon[93519]: osdmap e790: 8 total, 8 up, 8 in 2024-08-17T17:36:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:53 smithi179 ceph-mon[93519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:36:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:53 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:53.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:53 smithi130 ceph-mon[105521]: pgmap v857: 809 pgs: 1 creating+peering, 26 unknown, 782 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 2 op/s 2024-08-17T17:36:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:53 smithi130 ceph-mon[105521]: osdmap e790: 8 total, 8 up, 8 in 2024-08-17T17:36:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:53 smithi130 ceph-mon[105521]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:36:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:53 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:53 smithi130 ceph-mon[110068]: pgmap v857: 809 pgs: 1 creating+peering, 26 unknown, 782 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 2 op/s 2024-08-17T17:36:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:53 smithi130 ceph-mon[110068]: osdmap e790: 8 total, 8 up, 8 in 2024-08-17T17:36:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:53 smithi130 ceph-mon[110068]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:36:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:53 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:36:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:54 smithi179 ceph-mon[93519]: osdmap e791: 8 total, 8 up, 8 in 2024-08-17T17:36:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:54 smithi130 ceph-mon[105521]: osdmap e791: 8 total, 8 up, 8 in 2024-08-17T17:36:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:54 smithi130 ceph-mon[110068]: osdmap e791: 8 total, 8 up, 8 in 2024-08-17T17:36:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:55 smithi179 ceph-mon[93519]: pgmap v860: 809 pgs: 1 creating+peering, 808 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 1.2 KiB/s wr, 12 op/s 2024-08-17T17:36:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:55 smithi179 ceph-mon[93519]: osdmap e792: 8 total, 8 up, 8 in 2024-08-17T17:36:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:55 smithi179 ceph-mon[93519]: osdmap e793: 8 total, 8 up, 8 in 2024-08-17T17:36:55.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:55 smithi130 ceph-mon[105521]: pgmap v860: 809 pgs: 1 creating+peering, 808 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 1.2 KiB/s wr, 12 op/s 2024-08-17T17:36:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:55 smithi130 ceph-mon[105521]: osdmap e792: 8 total, 8 up, 8 in 2024-08-17T17:36:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:55 smithi130 ceph-mon[105521]: osdmap e793: 8 total, 8 up, 8 in 2024-08-17T17:36:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:55 smithi130 ceph-mon[110068]: pgmap v860: 809 pgs: 1 creating+peering, 808 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 1.2 KiB/s wr, 12 op/s 2024-08-17T17:36:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:55 smithi130 ceph-mon[110068]: osdmap e792: 8 total, 8 up, 8 in 2024-08-17T17:36:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:55 smithi130 ceph-mon[110068]: osdmap e793: 8 total, 8 up, 8 in 2024-08-17T17:36:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:57 smithi179 ceph-mon[93519]: pgmap v863: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 2.5 KiB/s wr, 48 op/s 2024-08-17T17:36:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:57 smithi179 ceph-mon[93519]: osdmap e794: 8 total, 8 up, 8 in 2024-08-17T17:36:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:57 smithi130 ceph-mon[105521]: pgmap v863: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 2.5 KiB/s wr, 48 op/s 2024-08-17T17:36:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:57 smithi130 ceph-mon[105521]: osdmap e794: 8 total, 8 up, 8 in 2024-08-17T17:36:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:57 smithi130 ceph-mon[110068]: pgmap v863: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 2.5 KiB/s wr, 48 op/s 2024-08-17T17:36:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:57 smithi130 ceph-mon[110068]: osdmap e794: 8 total, 8 up, 8 in 2024-08-17T17:36:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:58.577 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.QuiesceWatch (10048 ms) 2024-08-17T17:36:58.578 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.QuiesceWatchPP 2024-08-17T17:36:58.578 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:36:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:58 smithi179 ceph-mon[93519]: osdmap e795: 8 total, 8 up, 8 in 2024-08-17T17:36:58.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:58 smithi130 ceph-mon[110068]: osdmap e795: 8 total, 8 up, 8 in 2024-08-17T17:36:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:58 smithi130 ceph-mon[105521]: osdmap e795: 8 total, 8 up, 8 in 2024-08-17T17:36:59.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:59 smithi130 ceph-mon[105521]: pgmap v866: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 2.2 KiB/s wr, 45 op/s 2024-08-17T17:36:59.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:36:59 smithi130 ceph-mon[105521]: osdmap e796: 8 total, 8 up, 8 in 2024-08-17T17:36:59.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:59 smithi130 ceph-mon[110068]: pgmap v866: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 2.2 KiB/s wr, 45 op/s 2024-08-17T17:36:59.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:36:59 smithi130 ceph-mon[110068]: osdmap e796: 8 total, 8 up, 8 in 2024-08-17T17:36:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:59 smithi179 ceph-mon[93519]: pgmap v866: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 2.2 KiB/s wr, 45 op/s 2024-08-17T17:36:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:36:59 smithi179 ceph-mon[93519]: osdmap e796: 8 total, 8 up, 8 in 2024-08-17T17:37:00.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:36:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:36:59] "GET /metrics HTTP/1.1" 200 89445 "" "Prometheus/2.51.0" 2024-08-17T17:37:00.504 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:37:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:37:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:37:00.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:00 smithi179 ceph-mon[93519]: osdmap e797: 8 total, 8 up, 8 in 2024-08-17T17:37:00.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:00 smithi179 ceph-mon[93519]: osdmap e798: 8 total, 8 up, 8 in 2024-08-17T17:37:00.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:00 smithi130 ceph-mon[110068]: osdmap e797: 8 total, 8 up, 8 in 2024-08-17T17:37:00.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:00 smithi130 ceph-mon[110068]: osdmap e798: 8 total, 8 up, 8 in 2024-08-17T17:37:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:00 smithi130 ceph-mon[105521]: osdmap e797: 8 total, 8 up, 8 in 2024-08-17T17:37:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:00 smithi130 ceph-mon[105521]: osdmap e798: 8 total, 8 up, 8 in 2024-08-17T17:37:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:01 smithi179 ceph-mon[93519]: pgmap v869: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 4.5 KiB/s wr, 69 op/s 2024-08-17T17:37:01.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:01 smithi130 ceph-mon[105521]: pgmap v869: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 4.5 KiB/s wr, 69 op/s 2024-08-17T17:37:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:01 smithi130 ceph-mon[110068]: pgmap v869: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 4.5 KiB/s wr, 69 op/s 2024-08-17T17:37:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:02 smithi179 ceph-mon[93519]: osdmap e799: 8 total, 8 up, 8 in 2024-08-17T17:37:02.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:02 smithi130 ceph-mon[105521]: osdmap e799: 8 total, 8 up, 8 in 2024-08-17T17:37:02.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:02 smithi130 ceph-mon[110068]: osdmap e799: 8 total, 8 up, 8 in 2024-08-17T17:37:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:03 smithi179 ceph-mon[93519]: pgmap v872: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 3.7 KiB/s wr, 63 op/s 2024-08-17T17:37:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:03 smithi179 ceph-mon[93519]: osdmap e800: 8 total, 8 up, 8 in 2024-08-17T17:37:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:03 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:03.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:03 smithi130 ceph-mon[105521]: pgmap v872: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 3.7 KiB/s wr, 63 op/s 2024-08-17T17:37:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:03 smithi130 ceph-mon[105521]: osdmap e800: 8 total, 8 up, 8 in 2024-08-17T17:37:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:03 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:03 smithi130 ceph-mon[110068]: pgmap v872: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 3.7 KiB/s wr, 63 op/s 2024-08-17T17:37:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:03 smithi130 ceph-mon[110068]: osdmap e800: 8 total, 8 up, 8 in 2024-08-17T17:37:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:03 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:04.629 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.QuiesceWatchPP (6052 ms) 2024-08-17T17:37:04.629 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.QuiesceWatchError 2024-08-17T17:37:04.629 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:37:04.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:04 smithi179 ceph-mon[93519]: osdmap e801: 8 total, 8 up, 8 in 2024-08-17T17:37:04.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:04 smithi130 ceph-mon[110068]: osdmap e801: 8 total, 8 up, 8 in 2024-08-17T17:37:04.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:04 smithi130 ceph-mon[105521]: osdmap e801: 8 total, 8 up, 8 in 2024-08-17T17:37:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:05 smithi179 ceph-mon[93519]: pgmap v875: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 5.7 KiB/s wr, 83 op/s 2024-08-17T17:37:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:05 smithi179 ceph-mon[93519]: osdmap e802: 8 total, 8 up, 8 in 2024-08-17T17:37:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:05 smithi179 ceph-mon[93519]: osdmap e803: 8 total, 8 up, 8 in 2024-08-17T17:37:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:05 smithi130 ceph-mon[105521]: pgmap v875: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 5.7 KiB/s wr, 83 op/s 2024-08-17T17:37:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:05 smithi130 ceph-mon[105521]: osdmap e802: 8 total, 8 up, 8 in 2024-08-17T17:37:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:05 smithi130 ceph-mon[105521]: osdmap e803: 8 total, 8 up, 8 in 2024-08-17T17:37:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:05 smithi130 ceph-mon[110068]: pgmap v875: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 5.7 KiB/s wr, 83 op/s 2024-08-17T17:37:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:05 smithi130 ceph-mon[110068]: osdmap e802: 8 total, 8 up, 8 in 2024-08-17T17:37:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:05 smithi130 ceph-mon[110068]: osdmap e803: 8 total, 8 up, 8 in 2024-08-17T17:37:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:07 smithi179 ceph-mon[93519]: pgmap v878: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 5.5 KiB/s wr, 81 op/s 2024-08-17T17:37:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:07 smithi179 ceph-mon[93519]: osdmap e804: 8 total, 8 up, 8 in 2024-08-17T17:37:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:07 smithi130 ceph-mon[105521]: pgmap v878: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 5.5 KiB/s wr, 81 op/s 2024-08-17T17:37:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:07 smithi130 ceph-mon[105521]: osdmap e804: 8 total, 8 up, 8 in 2024-08-17T17:37:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:07 smithi130 ceph-mon[110068]: pgmap v878: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 5.5 KiB/s wr, 81 op/s 2024-08-17T17:37:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:07 smithi130 ceph-mon[110068]: osdmap e804: 8 total, 8 up, 8 in 2024-08-17T17:37:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:08 smithi179 ceph-mon[93519]: osdmap e805: 8 total, 8 up, 8 in 2024-08-17T17:37:08.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:08 smithi130 ceph-mon[105521]: osdmap e805: 8 total, 8 up, 8 in 2024-08-17T17:37:08.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:08 smithi130 ceph-mon[110068]: osdmap e805: 8 total, 8 up, 8 in 2024-08-17T17:37:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:09 smithi130 ceph-mon[105521]: pgmap v881: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 78 KiB/s rd, 6.0 KiB/s wr, 106 op/s 2024-08-17T17:37:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:09 smithi130 ceph-mon[105521]: osdmap e806: 8 total, 8 up, 8 in 2024-08-17T17:37:09.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:09 smithi130 ceph-mon[110068]: pgmap v881: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 78 KiB/s rd, 6.0 KiB/s wr, 106 op/s 2024-08-17T17:37:09.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:09 smithi130 ceph-mon[110068]: osdmap e806: 8 total, 8 up, 8 in 2024-08-17T17:37:09.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:09 smithi179 ceph-mon[93519]: pgmap v881: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 78 KiB/s rd, 6.0 KiB/s wr, 106 op/s 2024-08-17T17:37:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:09 smithi179 ceph-mon[93519]: osdmap e806: 8 total, 8 up, 8 in 2024-08-17T17:37:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:37:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:37:09] "GET /metrics HTTP/1.1" 200 89445 "" "Prometheus/2.51.0" 2024-08-17T17:37:10.510 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:37:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:37:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:37:10.570 INFO:tasks.workunit.client.0.smithi130.stdout:10.00% done 2024-08-17T17:37:10.570 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.QuiesceWatchError (5941 ms) 2024-08-17T17:37:10.570 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.QuiesceWatchTimeout 2024-08-17T17:37:10.570 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:37:10.584 INFO:tasks.workunit.client.0.smithi130.stderr:test quiesce is not long enough to time out 2024-08-17T17:37:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:10 smithi179 ceph-mon[93519]: osdmap e807: 8 total, 8 up, 8 in 2024-08-17T17:37:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:10 smithi179 ceph-mon[93519]: osdmap e808: 8 total, 8 up, 8 in 2024-08-17T17:37:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:10 smithi130 ceph-mon[105521]: osdmap e807: 8 total, 8 up, 8 in 2024-08-17T17:37:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:10 smithi130 ceph-mon[105521]: osdmap e808: 8 total, 8 up, 8 in 2024-08-17T17:37:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:10 smithi130 ceph-mon[110068]: osdmap e807: 8 total, 8 up, 8 in 2024-08-17T17:37:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:10 smithi130 ceph-mon[110068]: osdmap e808: 8 total, 8 up, 8 in 2024-08-17T17:37:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:11 smithi179 ceph-mon[93519]: pgmap v884: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 6.0 KiB/s wr, 109 op/s 2024-08-17T17:37:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:11 smithi130 ceph-mon[105521]: pgmap v884: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 6.0 KiB/s wr, 109 op/s 2024-08-17T17:37:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:11 smithi130 ceph-mon[110068]: pgmap v884: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 6.0 KiB/s wr, 109 op/s 2024-08-17T17:37:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: pgmap v886: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 4.7 KiB/s wr, 94 op/s 2024-08-17T17:37:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.14", "id": [6, 7]} : dispatch 2024-08-17T17:37:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 2]} : dispatch 2024-08-17T17:37:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.14", "id": [6, 7]} : dispatch 2024-08-17T17:37:13.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]} : dispatch 2024-08-17T17:37:13.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.11", "id": [3, 1]} : dispatch 2024-08-17T17:37:13.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 2]} : dispatch 2024-08-17T17:37:13.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.15", "id": [5, 1]} : dispatch 2024-08-17T17:37:13.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]} : dispatch 2024-08-17T17:37:13.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1c", "id": [6, 7]} : dispatch 2024-08-17T17:37:13.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.11", "id": [3, 1]} : dispatch 2024-08-17T17:37:13.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1d", "id": [3, 7]} : dispatch 2024-08-17T17:37:13.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.15", "id": [5, 1]} : dispatch 2024-08-17T17:37:13.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1c", "id": [6, 7]} : dispatch 2024-08-17T17:37:13.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1d", "id": [3, 7]} : dispatch 2024-08-17T17:37:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: pgmap v886: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 4.7 KiB/s wr, 94 op/s 2024-08-17T17:37:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.14", "id": [6, 7]} : dispatch 2024-08-17T17:37:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 2]} : dispatch 2024-08-17T17:37:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.14", "id": [6, 7]} : dispatch 2024-08-17T17:37:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]} : dispatch 2024-08-17T17:37:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.11", "id": [3, 1]} : dispatch 2024-08-17T17:37:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 2]} : dispatch 2024-08-17T17:37:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.15", "id": [5, 1]} : dispatch 2024-08-17T17:37:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]} : dispatch 2024-08-17T17:37:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1c", "id": [6, 7]} : dispatch 2024-08-17T17:37:13.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.11", "id": [3, 1]} : dispatch 2024-08-17T17:37:13.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1d", "id": [3, 7]} : dispatch 2024-08-17T17:37:13.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.15", "id": [5, 1]} : dispatch 2024-08-17T17:37:13.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1c", "id": [6, 7]} : dispatch 2024-08-17T17:37:13.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1d", "id": [3, 7]} : dispatch 2024-08-17T17:37:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: pgmap v886: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 4.7 KiB/s wr, 94 op/s 2024-08-17T17:37:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.14", "id": [6, 7]} : dispatch 2024-08-17T17:37:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 2]} : dispatch 2024-08-17T17:37:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.14", "id": [6, 7]} : dispatch 2024-08-17T17:37:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]} : dispatch 2024-08-17T17:37:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.11", "id": [3, 1]} : dispatch 2024-08-17T17:37:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 2]} : dispatch 2024-08-17T17:37:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.15", "id": [5, 1]} : dispatch 2024-08-17T17:37:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]} : dispatch 2024-08-17T17:37:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1c", "id": [6, 7]} : dispatch 2024-08-17T17:37:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.11", "id": [3, 1]} : dispatch 2024-08-17T17:37:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1d", "id": [3, 7]} : dispatch 2024-08-17T17:37:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.15", "id": [5, 1]} : dispatch 2024-08-17T17:37:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1c", "id": [6, 7]} : dispatch 2024-08-17T17:37:13.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1d", "id": [3, 7]} : dispatch 2024-08-17T17:37:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.14", "id": [6, 7]}]': finished 2024-08-17T17:37:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 2]}]': finished 2024-08-17T17:37:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-08-17T17:37:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.11", "id": [3, 1]}]': finished 2024-08-17T17:37:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.15", "id": [5, 1]}]': finished 2024-08-17T17:37:14.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1c", "id": [6, 7]}]': finished 2024-08-17T17:37:14.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1d", "id": [3, 7]}]': finished 2024-08-17T17:37:14.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:14 smithi179 ceph-mon[93519]: osdmap e809: 8 total, 8 up, 8 in 2024-08-17T17:37:14.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.14", "id": [6, 7]}]': finished 2024-08-17T17:37:14.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 2]}]': finished 2024-08-17T17:37:14.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-08-17T17:37:14.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.11", "id": [3, 1]}]': finished 2024-08-17T17:37:14.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.15", "id": [5, 1]}]': finished 2024-08-17T17:37:14.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1c", "id": [6, 7]}]': finished 2024-08-17T17:37:14.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1d", "id": [3, 7]}]': finished 2024-08-17T17:37:14.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[105521]: osdmap e809: 8 total, 8 up, 8 in 2024-08-17T17:37:14.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.14", "id": [6, 7]}]': finished 2024-08-17T17:37:14.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.b", "id": [3, 2]}]': finished 2024-08-17T17:37:14.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.d", "id": [3, 1]}]': finished 2024-08-17T17:37:14.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.11", "id": [3, 1]}]': finished 2024-08-17T17:37:14.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.15", "id": [5, 1]}]': finished 2024-08-17T17:37:14.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1c", "id": [6, 7]}]': finished 2024-08-17T17:37:14.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1d", "id": [3, 7]}]': finished 2024-08-17T17:37:14.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:14 smithi130 ceph-mon[110068]: osdmap e809: 8 total, 8 up, 8 in 2024-08-17T17:37:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:15 smithi179 ceph-mon[93519]: pgmap v888: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 2.8 KiB/s wr, 53 op/s 2024-08-17T17:37:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:15 smithi179 ceph-mon[93519]: osdmap e810: 8 total, 8 up, 8 in 2024-08-17T17:37:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:15 smithi179 ceph-mon[93519]: osdmap e811: 8 total, 8 up, 8 in 2024-08-17T17:37:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:15 smithi130 ceph-mon[105521]: pgmap v888: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 2.8 KiB/s wr, 53 op/s 2024-08-17T17:37:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:15 smithi130 ceph-mon[105521]: osdmap e810: 8 total, 8 up, 8 in 2024-08-17T17:37:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:15 smithi130 ceph-mon[105521]: osdmap e811: 8 total, 8 up, 8 in 2024-08-17T17:37:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:15 smithi130 ceph-mon[110068]: pgmap v888: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 2.8 KiB/s wr, 53 op/s 2024-08-17T17:37:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:15 smithi130 ceph-mon[110068]: osdmap e810: 8 total, 8 up, 8 in 2024-08-17T17:37:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:15 smithi130 ceph-mon[110068]: osdmap e811: 8 total, 8 up, 8 in 2024-08-17T17:37:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:17 smithi179 ceph-mon[93519]: pgmap v891: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 2.8 KiB/s wr, 51 op/s 2024-08-17T17:37:17.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:17 smithi130 ceph-mon[110068]: pgmap v891: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 2.8 KiB/s wr, 51 op/s 2024-08-17T17:37:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:17 smithi130 ceph-mon[105521]: pgmap v891: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 2.8 KiB/s wr, 51 op/s 2024-08-17T17:37:19.570 INFO:tasks.workunit.client.0.smithi130.stderr:test quiesce is timed out 2024-08-17T17:37:19.573 INFO:tasks.workunit.client.0.smithi130.stderr:waiting for timed out ... 0 2024-08-17T17:37:19.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:19 smithi130 ceph-mon[105521]: pgmap v892: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 2.5 KiB/s wr, 45 op/s; 8 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:37:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:19 smithi130 ceph-mon[110068]: pgmap v892: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 2.5 KiB/s wr, 45 op/s; 8 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:37:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:19 smithi179 ceph-mon[93519]: pgmap v892: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 2.5 KiB/s wr, 45 op/s; 8 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:37:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:37:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:37:19] "GET /metrics HTTP/1.1" 200 89438 "" "Prometheus/2.51.0" 2024-08-17T17:37:20.562 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:37:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:37:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:37:20.573 INFO:tasks.workunit.client.0.smithi130.stderr:waiting for timed out ... 1 2024-08-17T17:37:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:20 smithi179 ceph-mon[93519]: osdmap e812: 8 total, 8 up, 8 in 2024-08-17T17:37:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:20 smithi179 ceph-mon[93519]: osdmap e813: 8 total, 8 up, 8 in 2024-08-17T17:37:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:20 smithi130 ceph-mon[105521]: osdmap e812: 8 total, 8 up, 8 in 2024-08-17T17:37:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:20 smithi130 ceph-mon[105521]: osdmap e813: 8 total, 8 up, 8 in 2024-08-17T17:37:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:20 smithi130 ceph-mon[110068]: osdmap e812: 8 total, 8 up, 8 in 2024-08-17T17:37:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:20 smithi130 ceph-mon[110068]: osdmap e813: 8 total, 8 up, 8 in 2024-08-17T17:37:21.573 INFO:tasks.workunit.client.0.smithi130.stderr:waiting for timed out ... 2 2024-08-17T17:37:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:21 smithi179 ceph-mon[93519]: pgmap v894: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:37:21.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:21 smithi130 ceph-mon[105521]: pgmap v894: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:37:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:21 smithi130 ceph-mon[110068]: pgmap v894: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:37:22.573 INFO:tasks.workunit.client.0.smithi130.stderr:waiting for timed out ... 3 2024-08-17T17:37:23.574 INFO:tasks.workunit.client.0.smithi130.stderr:waiting for timed out ... 4 2024-08-17T17:37:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:23 smithi179 ceph-mon[93519]: pgmap v896: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:37:23.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:23 smithi130 ceph-mon[105521]: pgmap v896: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:37:23.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:23 smithi130 ceph-mon[110068]: pgmap v896: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:37:24.574 INFO:tasks.workunit.client.0.smithi130.stderr:waiting for timed out ... 5 2024-08-17T17:37:25.574 INFO:tasks.workunit.client.0.smithi130.stderr:waiting for timed out ... 6 2024-08-17T17:37:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:25 smithi179 ceph-mon[93519]: pgmap v897: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:37:25.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:25 smithi130 ceph-mon[105521]: pgmap v897: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:37:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:25 smithi130 ceph-mon[110068]: pgmap v897: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-08-17T17:37:26.574 INFO:tasks.workunit.client.0.smithi130.stderr:waiting for timed out ... 7 2024-08-17T17:37:27.574 INFO:tasks.workunit.client.0.smithi130.stderr:waiting for timed out ... 8 2024-08-17T17:37:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:27 smithi179 ceph-mon[93519]: pgmap v898: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-08-17T17:37:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:27 smithi130 ceph-mon[105521]: pgmap v898: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-08-17T17:37:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:27 smithi130 ceph-mon[110068]: pgmap v898: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-08-17T17:37:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:28.574 INFO:tasks.workunit.client.0.smithi130.stderr:waiting for timed out ... 9 2024-08-17T17:37:29.574 INFO:tasks.workunit.client.0.smithi130.stderr:waiting for timed out ... 10 2024-08-17T17:37:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:29 smithi179 ceph-mon[93519]: pgmap v899: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 354 B/s wr, 4 op/s 2024-08-17T17:37:29.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:29 smithi130 ceph-mon[105521]: pgmap v899: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 354 B/s wr, 4 op/s 2024-08-17T17:37:29.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:29 smithi130 ceph-mon[110068]: pgmap v899: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 354 B/s wr, 4 op/s 2024-08-17T17:37:29.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:37:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:37:29] "GET /metrics HTTP/1.1" 200 89434 "" "Prometheus/2.51.0" 2024-08-17T17:37:30.497 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:37:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:37:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:37:30.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:30 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:37:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:30 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:37:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:30 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:37:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:31 smithi179 ceph-mon[93519]: pgmap v900: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 306 B/s wr, 3 op/s 2024-08-17T17:37:31.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:31 smithi130 ceph-mon[110068]: pgmap v900: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 306 B/s wr, 3 op/s 2024-08-17T17:37:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:31 smithi130 ceph-mon[105521]: pgmap v900: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.9 KiB/s rd, 306 B/s wr, 3 op/s 2024-08-17T17:37:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:33 smithi179 ceph-mon[93519]: pgmap v901: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.5 KiB/s rd, 261 B/s wr, 3 op/s 2024-08-17T17:37:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:33 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:33 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:33.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:33 smithi130 ceph-mon[105521]: pgmap v901: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.5 KiB/s rd, 261 B/s wr, 3 op/s 2024-08-17T17:37:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:33 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:33 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:33 smithi130 ceph-mon[110068]: pgmap v901: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.5 KiB/s rd, 261 B/s wr, 3 op/s 2024-08-17T17:37:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:33 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:33 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:35.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:35 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:35.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:35 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:35.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:35 smithi179 ceph-mon[93519]: pgmap v902: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-08-17T17:37:35.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:35 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:37:35.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:35 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:37:35.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:35 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:35 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:35 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:35 smithi130 ceph-mon[105521]: pgmap v902: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-08-17T17:37:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:35 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:37:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:35 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:37:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:35 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:35 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:35 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:35.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:35 smithi130 ceph-mon[110068]: pgmap v902: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-08-17T17:37:35.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:35 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:37:35.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:35 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:37:35.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:35 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:37:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:37 smithi179 ceph-mon[93519]: pgmap v903: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:37:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:37 smithi130 ceph-mon[105521]: pgmap v903: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:37:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:37 smithi130 ceph-mon[110068]: pgmap v903: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:37:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:39 smithi179 ceph-mon[93519]: pgmap v904: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:37:39.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:39 smithi130 ceph-mon[105521]: pgmap v904: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:37:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:39 smithi130 ceph-mon[110068]: pgmap v904: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:37:40.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:37:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:37:39] "GET /metrics HTTP/1.1" 200 89434 "" "Prometheus/2.51.0" 2024-08-17T17:37:40.574 INFO:tasks.workunit.client.0.smithi130.stderr:test retry succeeds 2024-08-17T17:37:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:37:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:37:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:37:41.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:41 smithi179 ceph-mon[93519]: pgmap v905: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:37:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:41 smithi130 ceph-mon[105521]: pgmap v905: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:37:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:41 smithi130 ceph-mon[110068]: pgmap v905: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:37:42.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:42 smithi179 ceph-mon[93519]: osdmap e814: 8 total, 8 up, 8 in 2024-08-17T17:37:42.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:42 smithi130 ceph-mon[105521]: osdmap e814: 8 total, 8 up, 8 in 2024-08-17T17:37:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:42 smithi130 ceph-mon[110068]: osdmap e814: 8 total, 8 up, 8 in 2024-08-17T17:37:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:43.381 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.QuiesceWatchTimeout (32811 ms) 2024-08-17T17:37:43.381 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.WriteZeroes 2024-08-17T17:37:43.382 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:37:43.443 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.WriteZeroes (62 ms) 2024-08-17T17:37:43.443 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.WriteZeroesThickProvision 2024-08-17T17:37:43.443 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:37:43.504 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.WriteZeroesThickProvision (61 ms) 2024-08-17T17:37:43.504 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestLibRBD.ConcurentOperations 2024-08-17T17:37:43.504 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:37:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:43 smithi179 ceph-mon[93519]: pgmap v907: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:37:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:43 smithi179 ceph-mon[93519]: osdmap e815: 8 total, 8 up, 8 in 2024-08-17T17:37:43.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:43 smithi130 ceph-mon[105521]: pgmap v907: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:37:43.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:43 smithi130 ceph-mon[105521]: osdmap e815: 8 total, 8 up, 8 in 2024-08-17T17:37:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:43 smithi130 ceph-mon[110068]: pgmap v907: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:37:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:43 smithi130 ceph-mon[110068]: osdmap e815: 8 total, 8 up, 8 in 2024-08-17T17:37:44.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:44 smithi179 ceph-mon[93519]: osdmap e816: 8 total, 8 up, 8 in 2024-08-17T17:37:44.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:44 smithi130 ceph-mon[105521]: osdmap e816: 8 total, 8 up, 8 in 2024-08-17T17:37:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:44 smithi130 ceph-mon[110068]: osdmap e816: 8 total, 8 up, 8 in 2024-08-17T17:37:45.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:45 smithi179 ceph-mon[93519]: pgmap v910: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 1.7 KiB/s wr, 24 op/s 2024-08-17T17:37:45.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:45 smithi179 ceph-mon[93519]: osdmap e817: 8 total, 8 up, 8 in 2024-08-17T17:37:45.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:45 smithi130 ceph-mon[105521]: pgmap v910: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 1.7 KiB/s wr, 24 op/s 2024-08-17T17:37:45.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:45 smithi130 ceph-mon[105521]: osdmap e817: 8 total, 8 up, 8 in 2024-08-17T17:37:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:45 smithi130 ceph-mon[110068]: pgmap v910: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 1.7 KiB/s wr, 24 op/s 2024-08-17T17:37:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:45 smithi130 ceph-mon[110068]: osdmap e817: 8 total, 8 up, 8 in 2024-08-17T17:37:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:46 smithi179 ceph-mon[93519]: osdmap e818: 8 total, 8 up, 8 in 2024-08-17T17:37:46.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:46 smithi130 ceph-mon[110068]: osdmap e818: 8 total, 8 up, 8 in 2024-08-17T17:37:46.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:46 smithi130 ceph-mon[105521]: osdmap e818: 8 total, 8 up, 8 in 2024-08-17T17:37:47.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:47 smithi179 ceph-mon[93519]: pgmap v913: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.7 KiB/s wr, 38 op/s 2024-08-17T17:37:47.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:47 smithi179 ceph-mon[93519]: osdmap e819: 8 total, 8 up, 8 in 2024-08-17T17:37:47.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:47 smithi130 ceph-mon[105521]: pgmap v913: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.7 KiB/s wr, 38 op/s 2024-08-17T17:37:47.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:47 smithi130 ceph-mon[105521]: osdmap e819: 8 total, 8 up, 8 in 2024-08-17T17:37:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:47 smithi130 ceph-mon[110068]: pgmap v913: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.7 KiB/s wr, 38 op/s 2024-08-17T17:37:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:47 smithi130 ceph-mon[110068]: osdmap e819: 8 total, 8 up, 8 in 2024-08-17T17:37:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:48 smithi179 ceph-mon[93519]: osdmap e820: 8 total, 8 up, 8 in 2024-08-17T17:37:48.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:48 smithi130 ceph-mon[105521]: osdmap e820: 8 total, 8 up, 8 in 2024-08-17T17:37:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:48 smithi130 ceph-mon[110068]: osdmap e820: 8 total, 8 up, 8 in 2024-08-17T17:37:49.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:49 smithi130 ceph-mon[105521]: pgmap v916: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 9.7 KiB/s wr, 151 op/s 2024-08-17T17:37:49.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:49 smithi130 ceph-mon[105521]: osdmap e821: 8 total, 8 up, 8 in 2024-08-17T17:37:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:49 smithi130 ceph-mon[110068]: pgmap v916: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 9.7 KiB/s wr, 151 op/s 2024-08-17T17:37:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:49 smithi130 ceph-mon[110068]: osdmap e821: 8 total, 8 up, 8 in 2024-08-17T17:37:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:49 smithi179 ceph-mon[93519]: pgmap v916: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 106 KiB/s rd, 9.7 KiB/s wr, 151 op/s 2024-08-17T17:37:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:49 smithi179 ceph-mon[93519]: osdmap e821: 8 total, 8 up, 8 in 2024-08-17T17:37:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:37:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:37:49] "GET /metrics HTTP/1.1" 200 89432 "" "Prometheus/2.51.0" 2024-08-17T17:37:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:50 smithi179 ceph-mon[93519]: osdmap e822: 8 total, 8 up, 8 in 2024-08-17T17:37:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:37:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:37:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:37:50.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:50 smithi130 ceph-mon[110068]: osdmap e822: 8 total, 8 up, 8 in 2024-08-17T17:37:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:50 smithi130 ceph-mon[105521]: osdmap e822: 8 total, 8 up, 8 in 2024-08-17T17:37:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:51 smithi130 ceph-mon[110068]: pgmap v919: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 9.7 KiB/s wr, 150 op/s 2024-08-17T17:37:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:51 smithi130 ceph-mon[110068]: osdmap e823: 8 total, 8 up, 8 in 2024-08-17T17:37:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:51 smithi130 ceph-mon[110068]: osdmap e824: 8 total, 8 up, 8 in 2024-08-17T17:37:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:51 smithi130 ceph-mon[105521]: pgmap v919: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 9.7 KiB/s wr, 150 op/s 2024-08-17T17:37:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:51 smithi130 ceph-mon[105521]: osdmap e823: 8 total, 8 up, 8 in 2024-08-17T17:37:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:51 smithi130 ceph-mon[105521]: osdmap e824: 8 total, 8 up, 8 in 2024-08-17T17:37:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:51 smithi179 ceph-mon[93519]: pgmap v919: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 9.7 KiB/s wr, 150 op/s 2024-08-17T17:37:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:51 smithi179 ceph-mon[93519]: osdmap e823: 8 total, 8 up, 8 in 2024-08-17T17:37:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:51 smithi179 ceph-mon[93519]: osdmap e824: 8 total, 8 up, 8 in 2024-08-17T17:37:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:52 smithi179 ceph-mon[93519]: osdmap e825: 8 total, 8 up, 8 in 2024-08-17T17:37:52.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:52 smithi130 ceph-mon[105521]: osdmap e825: 8 total, 8 up, 8 in 2024-08-17T17:37:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:52 smithi130 ceph-mon[110068]: osdmap e825: 8 total, 8 up, 8 in 2024-08-17T17:37:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:53 smithi179 ceph-mon[93519]: pgmap v923: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 264 B/s wr, 0 op/s 2024-08-17T17:37:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:53 smithi179 ceph-mon[93519]: osdmap e826: 8 total, 8 up, 8 in 2024-08-17T17:37:53.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:53 smithi130 ceph-mon[105521]: pgmap v923: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 264 B/s wr, 0 op/s 2024-08-17T17:37:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:53 smithi130 ceph-mon[105521]: osdmap e826: 8 total, 8 up, 8 in 2024-08-17T17:37:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:53 smithi130 ceph-mon[110068]: pgmap v923: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 264 B/s wr, 0 op/s 2024-08-17T17:37:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:53 smithi130 ceph-mon[110068]: osdmap e826: 8 total, 8 up, 8 in 2024-08-17T17:37:54.786 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:54 smithi179 ceph-mon[93519]: osdmap e827: 8 total, 8 up, 8 in 2024-08-17T17:37:54.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:54 smithi130 ceph-mon[105521]: osdmap e827: 8 total, 8 up, 8 in 2024-08-17T17:37:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:54 smithi130 ceph-mon[110068]: osdmap e827: 8 total, 8 up, 8 in 2024-08-17T17:37:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:55 smithi179 ceph-mon[93519]: pgmap v926: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 187 KiB/s rd, 5.2 KiB/s wr, 254 op/s 2024-08-17T17:37:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:55 smithi179 ceph-mon[93519]: osdmap e828: 8 total, 8 up, 8 in 2024-08-17T17:37:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:55 smithi130 ceph-mon[105521]: pgmap v926: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 187 KiB/s rd, 5.2 KiB/s wr, 254 op/s 2024-08-17T17:37:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:55 smithi130 ceph-mon[105521]: osdmap e828: 8 total, 8 up, 8 in 2024-08-17T17:37:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:55 smithi130 ceph-mon[110068]: pgmap v926: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 187 KiB/s rd, 5.2 KiB/s wr, 254 op/s 2024-08-17T17:37:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:55 smithi130 ceph-mon[110068]: osdmap e828: 8 total, 8 up, 8 in 2024-08-17T17:37:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:56 smithi179 ceph-mon[93519]: osdmap e829: 8 total, 8 up, 8 in 2024-08-17T17:37:56.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:56 smithi130 ceph-mon[105521]: osdmap e829: 8 total, 8 up, 8 in 2024-08-17T17:37:56.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:56 smithi130 ceph-mon[110068]: osdmap e829: 8 total, 8 up, 8 in 2024-08-17T17:37:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:57 smithi179 ceph-mon[93519]: pgmap v929: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 5.0 KiB/s wr, 244 op/s 2024-08-17T17:37:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:57 smithi179 ceph-mon[93519]: osdmap e830: 8 total, 8 up, 8 in 2024-08-17T17:37:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:57.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:57 smithi130 ceph-mon[105521]: pgmap v929: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 5.0 KiB/s wr, 244 op/s 2024-08-17T17:37:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:57 smithi130 ceph-mon[105521]: osdmap e830: 8 total, 8 up, 8 in 2024-08-17T17:37:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:57 smithi130 ceph-mon[110068]: pgmap v929: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 5.0 KiB/s wr, 244 op/s 2024-08-17T17:37:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:57 smithi130 ceph-mon[110068]: osdmap e830: 8 total, 8 up, 8 in 2024-08-17T17:37:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:58 smithi179 ceph-mon[93519]: osdmap e831: 8 total, 8 up, 8 in 2024-08-17T17:37:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:58 smithi130 ceph-mon[105521]: osdmap e831: 8 total, 8 up, 8 in 2024-08-17T17:37:58.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:58 smithi130 ceph-mon[110068]: osdmap e831: 8 total, 8 up, 8 in 2024-08-17T17:37:59.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:59 smithi130 ceph-mon[105521]: pgmap v932: 809 pgs: 1 active+clean+snaptrim_wait, 808 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 5.0 KiB/s wr, 23 op/s 2024-08-17T17:37:59.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:37:59 smithi130 ceph-mon[105521]: osdmap e832: 8 total, 8 up, 8 in 2024-08-17T17:37:59.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:59 smithi130 ceph-mon[110068]: pgmap v932: 809 pgs: 1 active+clean+snaptrim_wait, 808 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 5.0 KiB/s wr, 23 op/s 2024-08-17T17:37:59.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:37:59 smithi130 ceph-mon[110068]: osdmap e832: 8 total, 8 up, 8 in 2024-08-17T17:37:59.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:59 smithi179 ceph-mon[93519]: pgmap v932: 809 pgs: 1 active+clean+snaptrim_wait, 808 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 5.0 KiB/s wr, 23 op/s 2024-08-17T17:37:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:37:59 smithi179 ceph-mon[93519]: osdmap e832: 8 total, 8 up, 8 in 2024-08-17T17:38:00.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:37:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:37:59] "GET /metrics HTTP/1.1" 200 89438 "" "Prometheus/2.51.0" 2024-08-17T17:38:00.568 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:38:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:38:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:38:00.842 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:00 smithi130 ceph-mon[105521]: osdmap e833: 8 total, 8 up, 8 in 2024-08-17T17:38:00.842 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:00 smithi130 ceph-mon[105521]: osdmap e834: 8 total, 8 up, 8 in 2024-08-17T17:38:00.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:00 smithi130 ceph-mon[110068]: osdmap e833: 8 total, 8 up, 8 in 2024-08-17T17:38:00.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:00 smithi130 ceph-mon[110068]: osdmap e834: 8 total, 8 up, 8 in 2024-08-17T17:38:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:00 smithi179 ceph-mon[93519]: osdmap e833: 8 total, 8 up, 8 in 2024-08-17T17:38:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:00 smithi179 ceph-mon[93519]: osdmap e834: 8 total, 8 up, 8 in 2024-08-17T17:38:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:01 smithi179 ceph-mon[93519]: pgmap v935: 809 pgs: 1 active+clean+snaptrim_wait, 808 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 25 op/s 2024-08-17T17:38:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:01 smithi179 ceph-mon[93519]: osdmap e835: 8 total, 8 up, 8 in 2024-08-17T17:38:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:01 smithi130 ceph-mon[110068]: pgmap v935: 809 pgs: 1 active+clean+snaptrim_wait, 808 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 25 op/s 2024-08-17T17:38:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:01 smithi130 ceph-mon[110068]: osdmap e835: 8 total, 8 up, 8 in 2024-08-17T17:38:01.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:01 smithi130 ceph-mon[105521]: pgmap v935: 809 pgs: 1 active+clean+snaptrim_wait, 808 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 25 op/s 2024-08-17T17:38:01.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:01 smithi130 ceph-mon[105521]: osdmap e835: 8 total, 8 up, 8 in 2024-08-17T17:38:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:03 smithi179 ceph-mon[93519]: pgmap v938: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 0 B/s wr, 2 op/s 2024-08-17T17:38:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:03 smithi179 ceph-mon[93519]: osdmap e836: 8 total, 8 up, 8 in 2024-08-17T17:38:03.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:03 smithi130 ceph-mon[105521]: pgmap v938: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 0 B/s wr, 2 op/s 2024-08-17T17:38:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:03 smithi130 ceph-mon[105521]: osdmap e836: 8 total, 8 up, 8 in 2024-08-17T17:38:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:03 smithi130 ceph-mon[110068]: pgmap v938: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 0 B/s wr, 2 op/s 2024-08-17T17:38:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:03 smithi130 ceph-mon[110068]: osdmap e836: 8 total, 8 up, 8 in 2024-08-17T17:38:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:04 smithi179 ceph-mon[93519]: osdmap e837: 8 total, 8 up, 8 in 2024-08-17T17:38:04.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:04 smithi130 ceph-mon[105521]: osdmap e837: 8 total, 8 up, 8 in 2024-08-17T17:38:04.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:04 smithi130 ceph-mon[110068]: osdmap e837: 8 total, 8 up, 8 in 2024-08-17T17:38:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:05 smithi179 ceph-mon[93519]: pgmap v941: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.2 KiB/s wr, 45 op/s 2024-08-17T17:38:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:05 smithi179 ceph-mon[93519]: osdmap e838: 8 total, 8 up, 8 in 2024-08-17T17:38:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:05 smithi179 ceph-mon[93519]: osdmap e839: 8 total, 8 up, 8 in 2024-08-17T17:38:05.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:05 smithi130 ceph-mon[105521]: pgmap v941: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.2 KiB/s wr, 45 op/s 2024-08-17T17:38:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:05 smithi130 ceph-mon[105521]: osdmap e838: 8 total, 8 up, 8 in 2024-08-17T17:38:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:05 smithi130 ceph-mon[105521]: osdmap e839: 8 total, 8 up, 8 in 2024-08-17T17:38:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:05 smithi130 ceph-mon[110068]: pgmap v941: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.2 KiB/s wr, 45 op/s 2024-08-17T17:38:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:05 smithi130 ceph-mon[110068]: osdmap e838: 8 total, 8 up, 8 in 2024-08-17T17:38:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:05 smithi130 ceph-mon[110068]: osdmap e839: 8 total, 8 up, 8 in 2024-08-17T17:38:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:07 smithi179 ceph-mon[93519]: pgmap v944: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.2 KiB/s wr, 45 op/s 2024-08-17T17:38:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:07 smithi179 ceph-mon[93519]: osdmap e840: 8 total, 8 up, 8 in 2024-08-17T17:38:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:07 smithi130 ceph-mon[105521]: pgmap v944: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.2 KiB/s wr, 45 op/s 2024-08-17T17:38:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:07 smithi130 ceph-mon[105521]: osdmap e840: 8 total, 8 up, 8 in 2024-08-17T17:38:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:07 smithi130 ceph-mon[110068]: pgmap v944: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.2 KiB/s wr, 45 op/s 2024-08-17T17:38:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:07 smithi130 ceph-mon[110068]: osdmap e840: 8 total, 8 up, 8 in 2024-08-17T17:38:08.614 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestLibRBD.ConcurentOperations (25110 ms) 2024-08-17T17:38:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:08 smithi179 ceph-mon[93519]: osdmap e841: 8 total, 8 up, 8 in 2024-08-17T17:38:08.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:08 smithi130 ceph-mon[105521]: osdmap e841: 8 total, 8 up, 8 in 2024-08-17T17:38:08.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:08 smithi130 ceph-mon[110068]: osdmap e841: 8 total, 8 up, 8 in 2024-08-17T17:38:09.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:09 smithi130 ceph-mon[105521]: pgmap v947: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 5.0 KiB/s wr, 79 op/s 2024-08-17T17:38:09.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:09 smithi130 ceph-mon[105521]: osdmap e842: 8 total, 8 up, 8 in 2024-08-17T17:38:09.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:09 smithi130 ceph-mon[110068]: pgmap v947: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 5.0 KiB/s wr, 79 op/s 2024-08-17T17:38:09.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:09 smithi130 ceph-mon[110068]: osdmap e842: 8 total, 8 up, 8 in 2024-08-17T17:38:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:09 smithi179 ceph-mon[93519]: pgmap v947: 809 pgs: 809 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 5.0 KiB/s wr, 79 op/s 2024-08-17T17:38:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:09 smithi179 ceph-mon[93519]: osdmap e842: 8 total, 8 up, 8 in 2024-08-17T17:38:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:38:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:38:09] "GET /metrics HTTP/1.1" 200 89438 "" "Prometheus/2.51.0" 2024-08-17T17:38:10.543 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:38:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:38:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:38:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:10 smithi179 ceph-mon[93519]: osdmap e843: 8 total, 8 up, 8 in 2024-08-17T17:38:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:10 smithi179 ceph-mon[93519]: osdmap e844: 8 total, 8 up, 8 in 2024-08-17T17:38:10.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:10 smithi130 ceph-mon[105521]: osdmap e843: 8 total, 8 up, 8 in 2024-08-17T17:38:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:10 smithi130 ceph-mon[105521]: osdmap e844: 8 total, 8 up, 8 in 2024-08-17T17:38:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:10 smithi130 ceph-mon[110068]: osdmap e843: 8 total, 8 up, 8 in 2024-08-17T17:38:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:10 smithi130 ceph-mon[110068]: osdmap e844: 8 total, 8 up, 8 in 2024-08-17T17:38:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:11 smithi179 ceph-mon[93519]: pgmap v950: 777 pgs: 777 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 40 op/s 2024-08-17T17:38:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:11 smithi179 ceph-mon[93519]: osdmap e845: 8 total, 8 up, 8 in 2024-08-17T17:38:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:11 smithi130 ceph-mon[105521]: pgmap v950: 777 pgs: 777 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 40 op/s 2024-08-17T17:38:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:11 smithi130 ceph-mon[105521]: osdmap e845: 8 total, 8 up, 8 in 2024-08-17T17:38:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:11 smithi130 ceph-mon[110068]: pgmap v950: 777 pgs: 777 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 40 op/s 2024-08-17T17:38:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:11 smithi130 ceph-mon[110068]: osdmap e845: 8 total, 8 up, 8 in 2024-08-17T17:38:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:12 smithi179 ceph-mon[93519]: osdmap e846: 8 total, 8 up, 8 in 2024-08-17T17:38:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:12 smithi130 ceph-mon[105521]: osdmap e846: 8 total, 8 up, 8 in 2024-08-17T17:38:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:12 smithi130 ceph-mon[110068]: osdmap e846: 8 total, 8 up, 8 in 2024-08-17T17:38:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:13 smithi179 ceph-mon[93519]: pgmap v953: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:13 smithi179 ceph-mon[93519]: osdmap e847: 8 total, 8 up, 8 in 2024-08-17T17:38:13.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:13 smithi130 ceph-mon[105521]: pgmap v953: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:13.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:13 smithi130 ceph-mon[105521]: osdmap e847: 8 total, 8 up, 8 in 2024-08-17T17:38:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:13 smithi130 ceph-mon[110068]: pgmap v953: 713 pgs: 713 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:13 smithi130 ceph-mon[110068]: osdmap e847: 8 total, 8 up, 8 in 2024-08-17T17:38:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:15 smithi179 ceph-mon[93519]: pgmap v956: 649 pgs: 649 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:15 smithi179 ceph-mon[93519]: osdmap e848: 8 total, 8 up, 8 in 2024-08-17T17:38:15.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:15 smithi130 ceph-mon[105521]: pgmap v956: 649 pgs: 649 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:15 smithi130 ceph-mon[105521]: osdmap e848: 8 total, 8 up, 8 in 2024-08-17T17:38:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:15 smithi130 ceph-mon[110068]: pgmap v956: 649 pgs: 649 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:15 smithi130 ceph-mon[110068]: osdmap e848: 8 total, 8 up, 8 in 2024-08-17T17:38:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:16 smithi179 ceph-mon[93519]: osdmap e849: 8 total, 8 up, 8 in 2024-08-17T17:38:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:16 smithi179 ceph-mon[93519]: osdmap e850: 8 total, 8 up, 8 in 2024-08-17T17:38:16.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:16 smithi130 ceph-mon[105521]: osdmap e849: 8 total, 8 up, 8 in 2024-08-17T17:38:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:16 smithi130 ceph-mon[105521]: osdmap e850: 8 total, 8 up, 8 in 2024-08-17T17:38:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:16 smithi130 ceph-mon[110068]: osdmap e849: 8 total, 8 up, 8 in 2024-08-17T17:38:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:16 smithi130 ceph-mon[110068]: osdmap e850: 8 total, 8 up, 8 in 2024-08-17T17:38:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:17 smithi179 ceph-mon[93519]: pgmap v959: 585 pgs: 585 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:17 smithi179 ceph-mon[93519]: Health check update: 1 pool(s) full (POOL_FULL) 2024-08-17T17:38:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:17 smithi179 ceph-mon[93519]: osdmap e851: 8 total, 8 up, 8 in 2024-08-17T17:38:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:17 smithi130 ceph-mon[105521]: pgmap v959: 585 pgs: 585 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:17 smithi130 ceph-mon[105521]: Health check update: 1 pool(s) full (POOL_FULL) 2024-08-17T17:38:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:17 smithi130 ceph-mon[105521]: osdmap e851: 8 total, 8 up, 8 in 2024-08-17T17:38:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:17 smithi130 ceph-mon[110068]: pgmap v959: 585 pgs: 585 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:17 smithi130 ceph-mon[110068]: Health check update: 1 pool(s) full (POOL_FULL) 2024-08-17T17:38:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:17 smithi130 ceph-mon[110068]: osdmap e851: 8 total, 8 up, 8 in 2024-08-17T17:38:19.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:19 smithi130 ceph-mon[110068]: pgmap v962: 521 pgs: 521 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:19 smithi130 ceph-mon[110068]: osdmap e852: 8 total, 8 up, 8 in 2024-08-17T17:38:19.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:19 smithi130 ceph-mon[105521]: pgmap v962: 521 pgs: 521 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:19.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:19 smithi130 ceph-mon[105521]: osdmap e852: 8 total, 8 up, 8 in 2024-08-17T17:38:19.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:19 smithi179 ceph-mon[93519]: pgmap v962: 521 pgs: 521 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:19 smithi179 ceph-mon[93519]: osdmap e852: 8 total, 8 up, 8 in 2024-08-17T17:38:20.169 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:38:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:38:19] "GET /metrics HTTP/1.1" 200 82237 "" "Prometheus/2.51.0" 2024-08-17T17:38:20.521 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:38:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:38:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:38:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:20 smithi179 ceph-mon[93519]: osdmap e853: 8 total, 8 up, 8 in 2024-08-17T17:38:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:20 smithi179 ceph-mon[93519]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-08-17T17:38:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:20 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T17:38:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:20 smithi179 ceph-mon[93519]: osdmap e854: 8 total, 8 up, 8 in 2024-08-17T17:38:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:20 smithi130 ceph-mon[105521]: osdmap e853: 8 total, 8 up, 8 in 2024-08-17T17:38:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:20 smithi130 ceph-mon[105521]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-08-17T17:38:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:20 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T17:38:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:20 smithi130 ceph-mon[105521]: osdmap e854: 8 total, 8 up, 8 in 2024-08-17T17:38:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:20 smithi130 ceph-mon[110068]: osdmap e853: 8 total, 8 up, 8 in 2024-08-17T17:38:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:20 smithi130 ceph-mon[110068]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-08-17T17:38:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:20 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T17:38:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:20 smithi130 ceph-mon[110068]: osdmap e854: 8 total, 8 up, 8 in 2024-08-17T17:38:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:21 smithi179 ceph-mon[93519]: pgmap v965: 457 pgs: 457 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:21 smithi179 ceph-mon[93519]: osdmap e855: 8 total, 8 up, 8 in 2024-08-17T17:38:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:21 smithi130 ceph-mon[110068]: pgmap v965: 457 pgs: 457 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:21 smithi130 ceph-mon[110068]: osdmap e855: 8 total, 8 up, 8 in 2024-08-17T17:38:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:21 smithi130 ceph-mon[105521]: pgmap v965: 457 pgs: 457 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:21 smithi130 ceph-mon[105521]: osdmap e855: 8 total, 8 up, 8 in 2024-08-17T17:38:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:23 smithi179 ceph-mon[93519]: pgmap v968: 393 pgs: 393 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:23 smithi179 ceph-mon[93519]: osdmap e856: 8 total, 8 up, 8 in 2024-08-17T17:38:23.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:23 smithi130 ceph-mon[110068]: pgmap v968: 393 pgs: 393 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:23.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:23 smithi130 ceph-mon[110068]: osdmap e856: 8 total, 8 up, 8 in 2024-08-17T17:38:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:23 smithi130 ceph-mon[105521]: pgmap v968: 393 pgs: 393 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:23 smithi130 ceph-mon[105521]: osdmap e856: 8 total, 8 up, 8 in 2024-08-17T17:38:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:24 smithi179 ceph-mon[93519]: osdmap e857: 8 total, 8 up, 8 in 2024-08-17T17:38:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:24 smithi179 ceph-mon[93519]: osdmap e858: 8 total, 8 up, 8 in 2024-08-17T17:38:24.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:24 smithi130 ceph-mon[105521]: osdmap e857: 8 total, 8 up, 8 in 2024-08-17T17:38:24.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:24 smithi130 ceph-mon[105521]: osdmap e858: 8 total, 8 up, 8 in 2024-08-17T17:38:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:24 smithi130 ceph-mon[110068]: osdmap e857: 8 total, 8 up, 8 in 2024-08-17T17:38:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:24 smithi130 ceph-mon[110068]: osdmap e858: 8 total, 8 up, 8 in 2024-08-17T17:38:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:25 smithi179 ceph-mon[93519]: pgmap v971: 329 pgs: 329 active+clean; 587 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:25 smithi179 ceph-mon[93519]: osdmap e859: 8 total, 8 up, 8 in 2024-08-17T17:38:25.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:25 smithi130 ceph-mon[105521]: pgmap v971: 329 pgs: 329 active+clean; 587 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:25 smithi130 ceph-mon[105521]: osdmap e859: 8 total, 8 up, 8 in 2024-08-17T17:38:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:25 smithi130 ceph-mon[110068]: pgmap v971: 329 pgs: 329 active+clean; 587 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:25 smithi130 ceph-mon[110068]: osdmap e859: 8 total, 8 up, 8 in 2024-08-17T17:38:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:27 smithi179 ceph-mon[93519]: pgmap v974: 265 pgs: 265 active+clean; 587 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:27 smithi179 ceph-mon[93519]: osdmap e860: 8 total, 8 up, 8 in 2024-08-17T17:38:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:27 smithi130 ceph-mon[105521]: pgmap v974: 265 pgs: 265 active+clean; 587 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:27 smithi130 ceph-mon[105521]: osdmap e860: 8 total, 8 up, 8 in 2024-08-17T17:38:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:27 smithi130 ceph-mon[110068]: pgmap v974: 265 pgs: 265 active+clean; 587 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:27 smithi130 ceph-mon[110068]: osdmap e860: 8 total, 8 up, 8 in 2024-08-17T17:38:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:28 smithi179 ceph-mon[93519]: osdmap e861: 8 total, 8 up, 8 in 2024-08-17T17:38:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:28 smithi179 ceph-mon[93519]: osdmap e862: 8 total, 8 up, 8 in 2024-08-17T17:38:28.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:28 smithi130 ceph-mon[110068]: osdmap e861: 8 total, 8 up, 8 in 2024-08-17T17:38:28.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:28 smithi130 ceph-mon[110068]: osdmap e862: 8 total, 8 up, 8 in 2024-08-17T17:38:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:28 smithi130 ceph-mon[105521]: osdmap e861: 8 total, 8 up, 8 in 2024-08-17T17:38:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:28 smithi130 ceph-mon[105521]: osdmap e862: 8 total, 8 up, 8 in 2024-08-17T17:38:29.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:29 smithi130 ceph-mon[105521]: pgmap v977: 201 pgs: 201 active+clean; 587 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:29.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:29 smithi130 ceph-mon[105521]: osdmap e863: 8 total, 8 up, 8 in 2024-08-17T17:38:29.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:29 smithi130 ceph-mon[110068]: pgmap v977: 201 pgs: 201 active+clean; 587 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:29.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:29 smithi130 ceph-mon[110068]: osdmap e863: 8 total, 8 up, 8 in 2024-08-17T17:38:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:29 smithi179 ceph-mon[93519]: pgmap v977: 201 pgs: 201 active+clean; 587 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:29 smithi179 ceph-mon[93519]: osdmap e863: 8 total, 8 up, 8 in 2024-08-17T17:38:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:38:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:38:29] "GET /metrics HTTP/1.1" 200 46179 "" "Prometheus/2.51.0" 2024-08-17T17:38:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:38:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:38:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:38:31.524 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 161 tests from TestLibRBD (380798 ms total) 2024-08-17T17:38:31.524 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:38:31.524 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 19 tests from DiffIterateTest/0, where TypeParam = DiffIterateParams 2024-08-17T17:38:31.556 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterate 2024-08-17T17:38:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:31 smithi179 ceph-mon[93519]: pgmap v980: 137 pgs: 137 active+clean; 586 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:31 smithi179 ceph-mon[93519]: osdmap e864: 8 total, 8 up, 8 in 2024-08-17T17:38:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:31 smithi130 ceph-mon[105521]: pgmap v980: 137 pgs: 137 active+clean; 586 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:31 smithi130 ceph-mon[105521]: osdmap e864: 8 total, 8 up, 8 in 2024-08-17T17:38:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:31 smithi130 ceph-mon[110068]: pgmap v980: 137 pgs: 137 active+clean; 586 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:31 smithi130 ceph-mon[110068]: osdmap e864: 8 total, 8 up, 8 in 2024-08-17T17:38:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:32 smithi179 ceph-mon[93519]: osdmap e865: 8 total, 8 up, 8 in 2024-08-17T17:38:32.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:32 smithi130 ceph-mon[105521]: osdmap e865: 8 total, 8 up, 8 in 2024-08-17T17:38:32.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:32 smithi130 ceph-mon[110068]: osdmap e865: 8 total, 8 up, 8 in 2024-08-17T17:38:33.555 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:38:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:33 smithi179 ceph-mon[93519]: pgmap v983: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:33 smithi179 ceph-mon[93519]: osdmap e866: 8 total, 8 up, 8 in 2024-08-17T17:38:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:33 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3779485165' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-39","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:38:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:33 smithi130 ceph-mon[110068]: pgmap v983: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:33 smithi130 ceph-mon[110068]: osdmap e866: 8 total, 8 up, 8 in 2024-08-17T17:38:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:33 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3779485165' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-39","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:38:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:33 smithi130 ceph-mon[105521]: pgmap v983: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:33 smithi130 ceph-mon[105521]: osdmap e866: 8 total, 8 up, 8 in 2024-08-17T17:38:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:33 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3779485165' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-39","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:38:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:34 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3779485165' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-39","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:38:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:34 smithi179 ceph-mon[93519]: osdmap e867: 8 total, 8 up, 8 in 2024-08-17T17:38:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:34 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:38:34.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:34 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3779485165' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-39","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:38:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:34 smithi130 ceph-mon[105521]: osdmap e867: 8 total, 8 up, 8 in 2024-08-17T17:38:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:34 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:38:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:34 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3779485165' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-39","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:38:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:34 smithi130 ceph-mon[110068]: osdmap e867: 8 total, 8 up, 8 in 2024-08-17T17:38:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:34 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:38:35.586 INFO:tasks.workunit.client.0.smithi130.stdout: wrote [1058078~74008,3458280~89981,4053630~64566,4868485~76262,5988283~90383,12683721~40197,12732257~96659,13913658~93643,14545532~85286,15509055~54656] 2024-08-17T17:38:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:35 smithi179 ceph-mon[93519]: pgmap v986: 105 pgs: 24 creating+peering, 4 unknown, 77 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:35 smithi179 ceph-mon[93519]: osdmap e868: 8 total, 8 up, 8 in 2024-08-17T17:38:35.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:35 smithi130 ceph-mon[105521]: pgmap v986: 105 pgs: 24 creating+peering, 4 unknown, 77 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:35.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:35 smithi130 ceph-mon[105521]: osdmap e868: 8 total, 8 up, 8 in 2024-08-17T17:38:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:35 smithi130 ceph-mon[110068]: pgmap v986: 105 pgs: 24 creating+peering, 4 unknown, 77 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:35 smithi130 ceph-mon[110068]: osdmap e868: 8 total, 8 up, 8 in 2024-08-17T17:38:36.580 INFO:tasks.workunit.client.0.smithi130.stdout: wrote [316057~15052,4091999~32047,7359065~80958,7706204~4092,11234573~32602,11636787~59090,12196129~18517,12668316~82077,17450760~8489,17545508~82072] 2024-08-17T17:38:36.588 INFO:tasks.workunit.client.0.smithi130.stdout: diff was [316057~15052,4091999~32047,6078666~1631630,8388608~3826038,12668316~82077,16777216~850364] 2024-08-17T17:38:36.591 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterate (5035 ms) 2024-08-17T17:38:36.591 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministic 2024-08-17T17:38:36.592 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:38:36.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:36 smithi130 ceph-mon[110068]: osdmap e869: 8 total, 8 up, 8 in 2024-08-17T17:38:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:36 smithi130 ceph-mon[105521]: osdmap e869: 8 total, 8 up, 8 in 2024-08-17T17:38:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:36 smithi179 ceph-mon[93519]: osdmap e869: 8 total, 8 up, 8 in 2024-08-17T17:38:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:37 smithi179 ceph-mon[93519]: pgmap v989: 105 pgs: 24 creating+peering, 4 unknown, 77 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:37 smithi179 ceph-mon[93519]: osdmap e870: 8 total, 8 up, 8 in 2024-08-17T17:38:37.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:37 smithi130 ceph-mon[105521]: pgmap v989: 105 pgs: 24 creating+peering, 4 unknown, 77 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:37 smithi130 ceph-mon[105521]: osdmap e870: 8 total, 8 up, 8 in 2024-08-17T17:38:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:37 smithi130 ceph-mon[110068]: pgmap v989: 105 pgs: 24 creating+peering, 4 unknown, 77 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:38:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:37 smithi130 ceph-mon[110068]: osdmap e870: 8 total, 8 up, 8 in 2024-08-17T17:38:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:38 smithi179 ceph-mon[93519]: osdmap e871: 8 total, 8 up, 8 in 2024-08-17T17:38:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:38 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:38 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:38.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:38 smithi130 ceph-mon[105521]: osdmap e871: 8 total, 8 up, 8 in 2024-08-17T17:38:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:38 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:38 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:38.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:38 smithi130 ceph-mon[110068]: osdmap e871: 8 total, 8 up, 8 in 2024-08-17T17:38:38.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:38 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:38.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:38 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:39.614 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:38:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:39 smithi179 ceph-mon[93519]: pgmap v992: 105 pgs: 105 active+clean; 14 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 276 KiB/s wr, 36 op/s 2024-08-17T17:38:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:39 smithi179 ceph-mon[93519]: osdmap e872: 8 total, 8 up, 8 in 2024-08-17T17:38:39.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:39 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:39.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:39 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:39.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:39 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:38:39.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:39 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:38:39.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:39 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:39.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:38:39] "GET /metrics HTTP/1.1" 200 46179 "" "Prometheus/2.51.0" 2024-08-17T17:38:39.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-mon[110068]: pgmap v992: 105 pgs: 105 active+clean; 14 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 276 KiB/s wr, 36 op/s 2024-08-17T17:38:39.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-mon[110068]: osdmap e872: 8 total, 8 up, 8 in 2024-08-17T17:38:39.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:39.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:39.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:38:39.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:38:39.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:39.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-mon[105521]: pgmap v992: 105 pgs: 105 active+clean; 14 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 276 KiB/s wr, 36 op/s 2024-08-17T17:38:39.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-mon[105521]: osdmap e872: 8 total, 8 up, 8 in 2024-08-17T17:38:39.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:39.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:39.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:38:39.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:38:39.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:39 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:38:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:38:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:38:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:40 smithi130 ceph-mon[105521]: osdmap e873: 8 total, 8 up, 8 in 2024-08-17T17:38:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:40 smithi130 ceph-mon[110068]: osdmap e873: 8 total, 8 up, 8 in 2024-08-17T17:38:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:40 smithi179 ceph-mon[93519]: osdmap e873: 8 total, 8 up, 8 in 2024-08-17T17:38:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:41 smithi130 ceph-mon[110068]: pgmap v995: 105 pgs: 105 active+clean; 14 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 277 KiB/s wr, 52 op/s 2024-08-17T17:38:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:41 smithi130 ceph-mon[110068]: osdmap e874: 8 total, 8 up, 8 in 2024-08-17T17:38:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:41 smithi130 ceph-mon[105521]: pgmap v995: 105 pgs: 105 active+clean; 14 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 277 KiB/s wr, 52 op/s 2024-08-17T17:38:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:41 smithi130 ceph-mon[105521]: osdmap e874: 8 total, 8 up, 8 in 2024-08-17T17:38:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:41 smithi179 ceph-mon[93519]: pgmap v995: 105 pgs: 105 active+clean; 14 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 277 KiB/s wr, 52 op/s 2024-08-17T17:38:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:41 smithi179 ceph-mon[93519]: osdmap e874: 8 total, 8 up, 8 in 2024-08-17T17:38:42.641 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:38:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:42 smithi130 ceph-mon[105521]: osdmap e875: 8 total, 8 up, 8 in 2024-08-17T17:38:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:42 smithi130 ceph-mon[110068]: osdmap e875: 8 total, 8 up, 8 in 2024-08-17T17:38:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:42 smithi179 ceph-mon[93519]: osdmap e875: 8 total, 8 up, 8 in 2024-08-17T17:38:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:43 smithi130 ceph-mon[105521]: pgmap v998: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 25 KiB/s wr, 52 op/s 2024-08-17T17:38:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:43 smithi130 ceph-mon[105521]: osdmap e876: 8 total, 8 up, 8 in 2024-08-17T17:38:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:43 smithi130 ceph-mon[110068]: pgmap v998: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 25 KiB/s wr, 52 op/s 2024-08-17T17:38:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:43 smithi130 ceph-mon[110068]: osdmap e876: 8 total, 8 up, 8 in 2024-08-17T17:38:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:43.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:43 smithi179 ceph-mon[93519]: pgmap v998: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 25 KiB/s wr, 52 op/s 2024-08-17T17:38:43.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:43 smithi179 ceph-mon[93519]: osdmap e876: 8 total, 8 up, 8 in 2024-08-17T17:38:43.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:38:44.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:44 smithi130 ceph-mon[105521]: osdmap e877: 8 total, 8 up, 8 in 2024-08-17T17:38:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:44 smithi130 ceph-mon[110068]: osdmap e877: 8 total, 8 up, 8 in 2024-08-17T17:38:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:44 smithi179 ceph-mon[93519]: osdmap e877: 8 total, 8 up, 8 in 2024-08-17T17:38:45.526 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:38:45.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:45 smithi130 ceph-mon[105521]: pgmap v1001: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 32 KiB/s wr, 117 op/s 2024-08-17T17:38:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:45 smithi130 ceph-mon[105521]: osdmap e878: 8 total, 8 up, 8 in 2024-08-17T17:38:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:45 smithi130 ceph-mon[105521]: osdmap e879: 8 total, 8 up, 8 in 2024-08-17T17:38:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:45 smithi130 ceph-mon[110068]: pgmap v1001: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 32 KiB/s wr, 117 op/s 2024-08-17T17:38:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:45 smithi130 ceph-mon[110068]: osdmap e878: 8 total, 8 up, 8 in 2024-08-17T17:38:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:45 smithi130 ceph-mon[110068]: osdmap e879: 8 total, 8 up, 8 in 2024-08-17T17:38:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:45 smithi179 ceph-mon[93519]: pgmap v1001: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 32 KiB/s wr, 117 op/s 2024-08-17T17:38:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:45 smithi179 ceph-mon[93519]: osdmap e878: 8 total, 8 up, 8 in 2024-08-17T17:38:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:45 smithi179 ceph-mon[93519]: osdmap e879: 8 total, 8 up, 8 in 2024-08-17T17:38:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:47 smithi179 ceph-mon[93519]: pgmap v1004: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 48 KiB/s rd, 7.7 KiB/s wr, 79 op/s 2024-08-17T17:38:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:47 smithi179 ceph-mon[93519]: osdmap e880: 8 total, 8 up, 8 in 2024-08-17T17:38:47.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:47 smithi130 ceph-mon[105521]: pgmap v1004: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 48 KiB/s rd, 7.7 KiB/s wr, 79 op/s 2024-08-17T17:38:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:47 smithi130 ceph-mon[105521]: osdmap e880: 8 total, 8 up, 8 in 2024-08-17T17:38:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:47 smithi130 ceph-mon[110068]: pgmap v1004: 105 pgs: 105 active+clean; 17 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 48 KiB/s rd, 7.7 KiB/s wr, 79 op/s 2024-08-17T17:38:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:47 smithi130 ceph-mon[110068]: osdmap e880: 8 total, 8 up, 8 in 2024-08-17T17:38:48.543 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:38:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:48 smithi179 ceph-mon[93519]: osdmap e881: 8 total, 8 up, 8 in 2024-08-17T17:38:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:48 smithi130 ceph-mon[105521]: osdmap e881: 8 total, 8 up, 8 in 2024-08-17T17:38:48.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:48 smithi130 ceph-mon[110068]: osdmap e881: 8 total, 8 up, 8 in 2024-08-17T17:38:49.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:49 smithi130 ceph-mon[105521]: pgmap v1007: 105 pgs: 105 active+clean; 21 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 8.2 KiB/s wr, 85 op/s 2024-08-17T17:38:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:49 smithi130 ceph-mon[105521]: osdmap e882: 8 total, 8 up, 8 in 2024-08-17T17:38:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:49 smithi130 ceph-mon[110068]: pgmap v1007: 105 pgs: 105 active+clean; 21 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 8.2 KiB/s wr, 85 op/s 2024-08-17T17:38:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:49 smithi130 ceph-mon[110068]: osdmap e882: 8 total, 8 up, 8 in 2024-08-17T17:38:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:49 smithi179 ceph-mon[93519]: pgmap v1007: 105 pgs: 105 active+clean; 21 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 8.2 KiB/s wr, 85 op/s 2024-08-17T17:38:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:49 smithi179 ceph-mon[93519]: osdmap e882: 8 total, 8 up, 8 in 2024-08-17T17:38:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:38:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:38:49] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.51.0" 2024-08-17T17:38:50.524 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:38:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:38:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:38:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:50 smithi179 ceph-mon[93519]: osdmap e883: 8 total, 8 up, 8 in 2024-08-17T17:38:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:50 smithi130 ceph-mon[105521]: osdmap e883: 8 total, 8 up, 8 in 2024-08-17T17:38:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:50 smithi130 ceph-mon[110068]: osdmap e883: 8 total, 8 up, 8 in 2024-08-17T17:38:51.559 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministic (14968 ms) 2024-08-17T17:38:51.559 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministicPP 2024-08-17T17:38:51.559 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:38:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:51 smithi179 ceph-mon[93519]: pgmap v1010: 105 pgs: 105 active+clean; 21 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 8.5 KiB/s wr, 86 op/s 2024-08-17T17:38:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:51 smithi179 ceph-mon[93519]: osdmap e884: 8 total, 8 up, 8 in 2024-08-17T17:38:51.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:51 smithi130 ceph-mon[105521]: pgmap v1010: 105 pgs: 105 active+clean; 21 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 8.5 KiB/s wr, 86 op/s 2024-08-17T17:38:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:51 smithi130 ceph-mon[105521]: osdmap e884: 8 total, 8 up, 8 in 2024-08-17T17:38:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:51 smithi130 ceph-mon[110068]: pgmap v1010: 105 pgs: 105 active+clean; 21 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 8.5 KiB/s wr, 86 op/s 2024-08-17T17:38:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:51 smithi130 ceph-mon[110068]: osdmap e884: 8 total, 8 up, 8 in 2024-08-17T17:38:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:52 smithi179 ceph-mon[93519]: osdmap e885: 8 total, 8 up, 8 in 2024-08-17T17:38:52.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:52 smithi130 ceph-mon[105521]: osdmap e885: 8 total, 8 up, 8 in 2024-08-17T17:38:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:52 smithi130 ceph-mon[110068]: osdmap e885: 8 total, 8 up, 8 in 2024-08-17T17:38:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:53 smithi179 ceph-mon[93519]: pgmap v1013: 105 pgs: 105 active+clean; 25 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 2 op/s 2024-08-17T17:38:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:53 smithi179 ceph-mon[93519]: osdmap e886: 8 total, 8 up, 8 in 2024-08-17T17:38:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:53 smithi130 ceph-mon[105521]: pgmap v1013: 105 pgs: 105 active+clean; 25 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 2 op/s 2024-08-17T17:38:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:53 smithi130 ceph-mon[105521]: osdmap e886: 8 total, 8 up, 8 in 2024-08-17T17:38:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:53 smithi130 ceph-mon[110068]: pgmap v1013: 105 pgs: 105 active+clean; 25 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 2 op/s 2024-08-17T17:38:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:53 smithi130 ceph-mon[110068]: osdmap e886: 8 total, 8 up, 8 in 2024-08-17T17:38:54.579 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:38:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:54 smithi179 ceph-mon[93519]: osdmap e887: 8 total, 8 up, 8 in 2024-08-17T17:38:54.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:54 smithi130 ceph-mon[105521]: osdmap e887: 8 total, 8 up, 8 in 2024-08-17T17:38:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:54 smithi130 ceph-mon[110068]: osdmap e887: 8 total, 8 up, 8 in 2024-08-17T17:38:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:55 smithi179 ceph-mon[93519]: pgmap v1016: 105 pgs: 105 active+clean; 29 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 8.5 KiB/s wr, 106 op/s 2024-08-17T17:38:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:55 smithi179 ceph-mon[93519]: osdmap e888: 8 total, 8 up, 8 in 2024-08-17T17:38:55.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:55 smithi130 ceph-mon[105521]: pgmap v1016: 105 pgs: 105 active+clean; 29 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 8.5 KiB/s wr, 106 op/s 2024-08-17T17:38:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:55 smithi130 ceph-mon[105521]: osdmap e888: 8 total, 8 up, 8 in 2024-08-17T17:38:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:55 smithi130 ceph-mon[110068]: pgmap v1016: 105 pgs: 105 active+clean; 29 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 8.5 KiB/s wr, 106 op/s 2024-08-17T17:38:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:55 smithi130 ceph-mon[110068]: osdmap e888: 8 total, 8 up, 8 in 2024-08-17T17:38:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:56 smithi179 ceph-mon[93519]: osdmap e889: 8 total, 8 up, 8 in 2024-08-17T17:38:56.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:56 smithi130 ceph-mon[105521]: osdmap e889: 8 total, 8 up, 8 in 2024-08-17T17:38:56.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:56 smithi130 ceph-mon[110068]: osdmap e889: 8 total, 8 up, 8 in 2024-08-17T17:38:57.593 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:38:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:57 smithi179 ceph-mon[93519]: pgmap v1019: 105 pgs: 105 active+clean; 29 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 8.2 KiB/s wr, 103 op/s 2024-08-17T17:38:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:57 smithi179 ceph-mon[93519]: osdmap e890: 8 total, 8 up, 8 in 2024-08-17T17:38:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:57.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:57 smithi130 ceph-mon[105521]: pgmap v1019: 105 pgs: 105 active+clean; 29 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 8.2 KiB/s wr, 103 op/s 2024-08-17T17:38:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:57 smithi130 ceph-mon[105521]: osdmap e890: 8 total, 8 up, 8 in 2024-08-17T17:38:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:57 smithi130 ceph-mon[110068]: pgmap v1019: 105 pgs: 105 active+clean; 29 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 8.2 KiB/s wr, 103 op/s 2024-08-17T17:38:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:57 smithi130 ceph-mon[110068]: osdmap e890: 8 total, 8 up, 8 in 2024-08-17T17:38:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:58 smithi179 ceph-mon[93519]: osdmap e891: 8 total, 8 up, 8 in 2024-08-17T17:38:58.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:58 smithi130 ceph-mon[105521]: osdmap e891: 8 total, 8 up, 8 in 2024-08-17T17:38:58.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:58 smithi130 ceph-mon[110068]: osdmap e891: 8 total, 8 up, 8 in 2024-08-17T17:38:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:59 smithi179 ceph-mon[93519]: pgmap v1022: 105 pgs: 105 active+clean; 35 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 8.0 KiB/s wr, 85 op/s 2024-08-17T17:38:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:38:59 smithi179 ceph-mon[93519]: osdmap e892: 8 total, 8 up, 8 in 2024-08-17T17:38:59.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:59 smithi130 ceph-mon[105521]: pgmap v1022: 105 pgs: 105 active+clean; 35 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 8.0 KiB/s wr, 85 op/s 2024-08-17T17:38:59.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:38:59 smithi130 ceph-mon[105521]: osdmap e892: 8 total, 8 up, 8 in 2024-08-17T17:38:59.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:38:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:38:59] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.51.0" 2024-08-17T17:38:59.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:59 smithi130 ceph-mon[110068]: pgmap v1022: 105 pgs: 105 active+clean; 35 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 8.0 KiB/s wr, 85 op/s 2024-08-17T17:38:59.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:38:59 smithi130 ceph-mon[110068]: osdmap e892: 8 total, 8 up, 8 in 2024-08-17T17:39:00.586 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:39:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:39:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:39:00.614 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:39:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:00 smithi179 ceph-mon[93519]: osdmap e893: 8 total, 8 up, 8 in 2024-08-17T17:39:00.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:00 smithi130 ceph-mon[105521]: osdmap e893: 8 total, 8 up, 8 in 2024-08-17T17:39:00.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:00 smithi130 ceph-mon[110068]: osdmap e893: 8 total, 8 up, 8 in 2024-08-17T17:39:01.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:01 smithi130 ceph-mon[105521]: pgmap v1025: 105 pgs: 105 active+clean; 35 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 9.0 KiB/s wr, 98 op/s 2024-08-17T17:39:01.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:01 smithi130 ceph-mon[105521]: osdmap e894: 8 total, 8 up, 8 in 2024-08-17T17:39:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:01 smithi130 ceph-mon[110068]: pgmap v1025: 105 pgs: 105 active+clean; 35 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 9.0 KiB/s wr, 98 op/s 2024-08-17T17:39:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:01 smithi130 ceph-mon[110068]: osdmap e894: 8 total, 8 up, 8 in 2024-08-17T17:39:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:01 smithi179 ceph-mon[93519]: pgmap v1025: 105 pgs: 105 active+clean; 35 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 9.0 KiB/s wr, 98 op/s 2024-08-17T17:39:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:01 smithi179 ceph-mon[93519]: osdmap e894: 8 total, 8 up, 8 in 2024-08-17T17:39:02.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:02 smithi130 ceph-mon[105521]: osdmap e895: 8 total, 8 up, 8 in 2024-08-17T17:39:02.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:02 smithi130 ceph-mon[110068]: osdmap e895: 8 total, 8 up, 8 in 2024-08-17T17:39:03.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:02 smithi179 ceph-mon[93519]: osdmap e895: 8 total, 8 up, 8 in 2024-08-17T17:39:03.640 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:39:03.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:03 smithi130 ceph-mon[105521]: pgmap v1028: 105 pgs: 105 active+clean; 35 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 1.2 KiB/s wr, 14 op/s 2024-08-17T17:39:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:03 smithi130 ceph-mon[105521]: osdmap e896: 8 total, 8 up, 8 in 2024-08-17T17:39:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:03 smithi130 ceph-mon[110068]: pgmap v1028: 105 pgs: 105 active+clean; 35 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 1.2 KiB/s wr, 14 op/s 2024-08-17T17:39:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:03 smithi130 ceph-mon[110068]: osdmap e896: 8 total, 8 up, 8 in 2024-08-17T17:39:04.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:03 smithi179 ceph-mon[93519]: pgmap v1028: 105 pgs: 105 active+clean; 35 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 1.2 KiB/s wr, 14 op/s 2024-08-17T17:39:04.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:03 smithi179 ceph-mon[93519]: osdmap e896: 8 total, 8 up, 8 in 2024-08-17T17:39:04.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:04 smithi130 ceph-mon[105521]: osdmap e897: 8 total, 8 up, 8 in 2024-08-17T17:39:04.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:04 smithi130 ceph-mon[110068]: osdmap e897: 8 total, 8 up, 8 in 2024-08-17T17:39:05.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:04 smithi179 ceph-mon[93519]: osdmap e897: 8 total, 8 up, 8 in 2024-08-17T17:39:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:05 smithi130 ceph-mon[105521]: pgmap v1031: 105 pgs: 105 active+clean; 44 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 8.5 KiB/s wr, 84 op/s 2024-08-17T17:39:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:05 smithi130 ceph-mon[105521]: osdmap e898: 8 total, 8 up, 8 in 2024-08-17T17:39:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:05 smithi130 ceph-mon[110068]: pgmap v1031: 105 pgs: 105 active+clean; 44 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 8.5 KiB/s wr, 84 op/s 2024-08-17T17:39:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:05 smithi130 ceph-mon[110068]: osdmap e898: 8 total, 8 up, 8 in 2024-08-17T17:39:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:05 smithi179 ceph-mon[93519]: pgmap v1031: 105 pgs: 105 active+clean; 44 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 8.5 KiB/s wr, 84 op/s 2024-08-17T17:39:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:05 smithi179 ceph-mon[93519]: osdmap e898: 8 total, 8 up, 8 in 2024-08-17T17:39:06.670 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministicPP (15111 ms) 2024-08-17T17:39:06.670 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministicLUKS1 2024-08-17T17:39:06.670 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:39:06.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:06 smithi130 ceph-mon[105521]: osdmap e899: 8 total, 8 up, 8 in 2024-08-17T17:39:06.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:06 smithi130 ceph-mon[110068]: osdmap e899: 8 total, 8 up, 8 in 2024-08-17T17:39:07.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:06 smithi179 ceph-mon[93519]: osdmap e899: 8 total, 8 up, 8 in 2024-08-17T17:39:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:07 smithi130 ceph-mon[105521]: pgmap v1034: 105 pgs: 105 active+clean; 44 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 8.2 KiB/s wr, 83 op/s 2024-08-17T17:39:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:07 smithi130 ceph-mon[105521]: osdmap e900: 8 total, 8 up, 8 in 2024-08-17T17:39:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:07 smithi130 ceph-mon[110068]: pgmap v1034: 105 pgs: 105 active+clean; 44 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 8.2 KiB/s wr, 83 op/s 2024-08-17T17:39:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:07 smithi130 ceph-mon[110068]: osdmap e900: 8 total, 8 up, 8 in 2024-08-17T17:39:08.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:07 smithi179 ceph-mon[93519]: pgmap v1034: 105 pgs: 105 active+clean; 44 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 8.2 KiB/s wr, 83 op/s 2024-08-17T17:39:08.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:07 smithi179 ceph-mon[93519]: osdmap e900: 8 total, 8 up, 8 in 2024-08-17T17:39:09.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:09 smithi130 ceph-mon[105521]: pgmap v1036: 105 pgs: 105 active+clean; 55 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 6.0 KiB/s wr, 66 op/s 2024-08-17T17:39:09.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:09 smithi130 ceph-mon[110068]: pgmap v1036: 105 pgs: 105 active+clean; 55 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 6.0 KiB/s wr, 66 op/s 2024-08-17T17:39:09.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:39:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:39:09] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.51.0" 2024-08-17T17:39:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:09 smithi179 ceph-mon[93519]: pgmap v1036: 105 pgs: 105 active+clean; 55 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 6.0 KiB/s wr, 66 op/s 2024-08-17T17:39:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:39:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:39:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:39:11.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:11 smithi130 ceph-mon[105521]: pgmap v1037: 105 pgs: 105 active+clean; 55 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 5.0 KiB/s wr, 59 op/s 2024-08-17T17:39:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:11 smithi130 ceph-mon[105521]: osdmap e901: 8 total, 8 up, 8 in 2024-08-17T17:39:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:11 smithi130 ceph-mon[110068]: pgmap v1037: 105 pgs: 105 active+clean; 55 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 5.0 KiB/s wr, 59 op/s 2024-08-17T17:39:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:11 smithi130 ceph-mon[110068]: osdmap e901: 8 total, 8 up, 8 in 2024-08-17T17:39:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:11 smithi179 ceph-mon[93519]: pgmap v1037: 105 pgs: 105 active+clean; 55 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 5.0 KiB/s wr, 59 op/s 2024-08-17T17:39:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:11 smithi179 ceph-mon[93519]: osdmap e901: 8 total, 8 up, 8 in 2024-08-17T17:39:12.690 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:39:12.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:12 smithi130 ceph-mon[110068]: osdmap e902: 8 total, 8 up, 8 in 2024-08-17T17:39:12.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:12 smithi130 ceph-mon[105521]: osdmap e902: 8 total, 8 up, 8 in 2024-08-17T17:39:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:12 smithi179 ceph-mon[93519]: osdmap e902: 8 total, 8 up, 8 in 2024-08-17T17:39:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:13.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:13 smithi130 ceph-mon[110068]: pgmap v1040: 105 pgs: 105 active+clean; 55 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 5.0 KiB/s wr, 59 op/s 2024-08-17T17:39:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:13 smithi130 ceph-mon[110068]: osdmap e903: 8 total, 8 up, 8 in 2024-08-17T17:39:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:13 smithi130 ceph-mon[105521]: pgmap v1040: 105 pgs: 105 active+clean; 55 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 5.0 KiB/s wr, 59 op/s 2024-08-17T17:39:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:13 smithi130 ceph-mon[105521]: osdmap e903: 8 total, 8 up, 8 in 2024-08-17T17:39:13.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:13 smithi179 ceph-mon[93519]: pgmap v1040: 105 pgs: 105 active+clean; 55 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 5.0 KiB/s wr, 59 op/s 2024-08-17T17:39:13.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:13 smithi179 ceph-mon[93519]: osdmap e903: 8 total, 8 up, 8 in 2024-08-17T17:39:15.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:15 smithi130 ceph-mon[105521]: pgmap v1042: 105 pgs: 105 active+clean; 59 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 687 KiB/s wr, 47 op/s 2024-08-17T17:39:15.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:15 smithi130 ceph-mon[110068]: pgmap v1042: 105 pgs: 105 active+clean; 59 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 687 KiB/s wr, 47 op/s 2024-08-17T17:39:16.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:15 smithi179 ceph-mon[93519]: pgmap v1042: 105 pgs: 105 active+clean; 59 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 687 KiB/s wr, 47 op/s 2024-08-17T17:39:18.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:17 smithi179 ceph-mon[93519]: pgmap v1043: 105 pgs: 105 active+clean; 59 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 39 op/s 2024-08-17T17:39:18.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:17 smithi179 ceph-mon[93519]: osdmap e904: 8 total, 8 up, 8 in 2024-08-17T17:39:18.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:17 smithi130 ceph-mon[105521]: pgmap v1043: 105 pgs: 105 active+clean; 59 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 39 op/s 2024-08-17T17:39:18.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:17 smithi130 ceph-mon[105521]: osdmap e904: 8 total, 8 up, 8 in 2024-08-17T17:39:18.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:17 smithi130 ceph-mon[110068]: pgmap v1043: 105 pgs: 105 active+clean; 59 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 39 op/s 2024-08-17T17:39:18.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:17 smithi130 ceph-mon[110068]: osdmap e904: 8 total, 8 up, 8 in 2024-08-17T17:39:18.713 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:39:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:18 smithi179 ceph-mon[93519]: osdmap e905: 8 total, 8 up, 8 in 2024-08-17T17:39:19.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:18 smithi130 ceph-mon[105521]: osdmap e905: 8 total, 8 up, 8 in 2024-08-17T17:39:19.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:18 smithi130 ceph-mon[110068]: osdmap e905: 8 total, 8 up, 8 in 2024-08-17T17:39:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:19 smithi179 ceph-mon[93519]: pgmap v1046: 105 pgs: 105 active+clean; 64 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 1.3 MiB/s wr, 53 op/s 2024-08-17T17:39:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:19 smithi179 ceph-mon[93519]: osdmap e906: 8 total, 8 up, 8 in 2024-08-17T17:39:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:39:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:39:19] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.51.0" 2024-08-17T17:39:20.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:19 smithi130 ceph-mon[105521]: pgmap v1046: 105 pgs: 105 active+clean; 64 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 1.3 MiB/s wr, 53 op/s 2024-08-17T17:39:20.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:19 smithi130 ceph-mon[105521]: osdmap e906: 8 total, 8 up, 8 in 2024-08-17T17:39:20.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:19 smithi130 ceph-mon[110068]: pgmap v1046: 105 pgs: 105 active+clean; 64 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 1.3 MiB/s wr, 53 op/s 2024-08-17T17:39:20.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:19 smithi130 ceph-mon[110068]: osdmap e906: 8 total, 8 up, 8 in 2024-08-17T17:39:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:39:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:39:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:39:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:21 smithi179 ceph-mon[93519]: pgmap v1048: 105 pgs: 105 active+clean; 65 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 685 KiB/s wr, 20 op/s 2024-08-17T17:39:22.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:21 smithi130 ceph-mon[105521]: pgmap v1048: 105 pgs: 105 active+clean; 65 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 685 KiB/s wr, 20 op/s 2024-08-17T17:39:22.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:21 smithi130 ceph-mon[110068]: pgmap v1048: 105 pgs: 105 active+clean; 65 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 685 KiB/s wr, 20 op/s 2024-08-17T17:39:24.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:23 smithi179 ceph-mon[93519]: pgmap v1049: 105 pgs: 105 active+clean; 65 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 685 KiB/s wr, 20 op/s 2024-08-17T17:39:24.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:23 smithi179 ceph-mon[93519]: osdmap e907: 8 total, 8 up, 8 in 2024-08-17T17:39:24.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:23 smithi130 ceph-mon[105521]: pgmap v1049: 105 pgs: 105 active+clean; 65 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 685 KiB/s wr, 20 op/s 2024-08-17T17:39:24.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:23 smithi130 ceph-mon[105521]: osdmap e907: 8 total, 8 up, 8 in 2024-08-17T17:39:24.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:23 smithi130 ceph-mon[110068]: pgmap v1049: 105 pgs: 105 active+clean; 65 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 685 KiB/s wr, 20 op/s 2024-08-17T17:39:24.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:23 smithi130 ceph-mon[110068]: osdmap e907: 8 total, 8 up, 8 in 2024-08-17T17:39:24.746 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:39:25.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:24 smithi179 ceph-mon[93519]: osdmap e908: 8 total, 8 up, 8 in 2024-08-17T17:39:25.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:24 smithi130 ceph-mon[105521]: osdmap e908: 8 total, 8 up, 8 in 2024-08-17T17:39:25.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:24 smithi130 ceph-mon[110068]: osdmap e908: 8 total, 8 up, 8 in 2024-08-17T17:39:26.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:25 smithi179 ceph-mon[93519]: pgmap v1052: 105 pgs: 105 active+clean; 70 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 687 KiB/s wr, 37 op/s 2024-08-17T17:39:26.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:25 smithi179 ceph-mon[93519]: osdmap e909: 8 total, 8 up, 8 in 2024-08-17T17:39:26.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:25 smithi130 ceph-mon[105521]: pgmap v1052: 105 pgs: 105 active+clean; 70 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 687 KiB/s wr, 37 op/s 2024-08-17T17:39:26.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:25 smithi130 ceph-mon[105521]: osdmap e909: 8 total, 8 up, 8 in 2024-08-17T17:39:26.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:25 smithi130 ceph-mon[110068]: pgmap v1052: 105 pgs: 105 active+clean; 70 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 687 KiB/s wr, 37 op/s 2024-08-17T17:39:26.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:25 smithi130 ceph-mon[110068]: osdmap e909: 8 total, 8 up, 8 in 2024-08-17T17:39:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:27 smithi179 ceph-mon[93519]: pgmap v1054: 105 pgs: 105 active+clean; 70 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 685 KiB/s wr, 30 op/s 2024-08-17T17:39:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:28.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:27 smithi130 ceph-mon[105521]: pgmap v1054: 105 pgs: 105 active+clean; 70 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 685 KiB/s wr, 30 op/s 2024-08-17T17:39:28.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:28.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:27 smithi130 ceph-mon[110068]: pgmap v1054: 105 pgs: 105 active+clean; 70 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 685 KiB/s wr, 30 op/s 2024-08-17T17:39:28.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:29 smithi179 ceph-mon[93519]: pgmap v1055: 105 pgs: 105 active+clean; 71 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 688 KiB/s wr, 59 op/s 2024-08-17T17:39:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:29 smithi179 ceph-mon[93519]: osdmap e910: 8 total, 8 up, 8 in 2024-08-17T17:39:30.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:29 smithi130 ceph-mon[105521]: pgmap v1055: 105 pgs: 105 active+clean; 71 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 688 KiB/s wr, 59 op/s 2024-08-17T17:39:30.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:29 smithi130 ceph-mon[105521]: osdmap e910: 8 total, 8 up, 8 in 2024-08-17T17:39:30.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:39:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:39:29] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.51.0" 2024-08-17T17:39:30.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:29 smithi130 ceph-mon[110068]: pgmap v1055: 105 pgs: 105 active+clean; 71 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 688 KiB/s wr, 59 op/s 2024-08-17T17:39:30.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:29 smithi130 ceph-mon[110068]: osdmap e910: 8 total, 8 up, 8 in 2024-08-17T17:39:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:39:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:39:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:39:30.769 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:39:31.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:30 smithi179 ceph-mon[93519]: osdmap e911: 8 total, 8 up, 8 in 2024-08-17T17:39:31.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:30 smithi130 ceph-mon[105521]: osdmap e911: 8 total, 8 up, 8 in 2024-08-17T17:39:31.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:30 smithi130 ceph-mon[110068]: osdmap e911: 8 total, 8 up, 8 in 2024-08-17T17:39:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:31 smithi179 ceph-mon[93519]: pgmap v1058: 105 pgs: 105 active+clean; 71 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.8 KiB/s wr, 31 op/s 2024-08-17T17:39:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:31 smithi179 ceph-mon[93519]: osdmap e912: 8 total, 8 up, 8 in 2024-08-17T17:39:32.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:31 smithi130 ceph-mon[105521]: pgmap v1058: 105 pgs: 105 active+clean; 71 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.8 KiB/s wr, 31 op/s 2024-08-17T17:39:32.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:31 smithi130 ceph-mon[105521]: osdmap e912: 8 total, 8 up, 8 in 2024-08-17T17:39:32.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:31 smithi130 ceph-mon[110068]: pgmap v1058: 105 pgs: 105 active+clean; 71 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.8 KiB/s wr, 31 op/s 2024-08-17T17:39:32.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:31 smithi130 ceph-mon[110068]: osdmap e912: 8 total, 8 up, 8 in 2024-08-17T17:39:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:33 smithi179 ceph-mon[93519]: pgmap v1060: 105 pgs: 105 active+clean; 71 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 3.0 KiB/s wr, 32 op/s 2024-08-17T17:39:34.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:33 smithi130 ceph-mon[105521]: pgmap v1060: 105 pgs: 105 active+clean; 71 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 3.0 KiB/s wr, 32 op/s 2024-08-17T17:39:34.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:33 smithi130 ceph-mon[110068]: pgmap v1060: 105 pgs: 105 active+clean; 71 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 3.0 KiB/s wr, 32 op/s 2024-08-17T17:39:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:35 smithi179 ceph-mon[93519]: pgmap v1061: 105 pgs: 105 active+clean; 77 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 40 op/s 2024-08-17T17:39:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:35 smithi179 ceph-mon[93519]: osdmap e913: 8 total, 8 up, 8 in 2024-08-17T17:39:36.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:35 smithi130 ceph-mon[105521]: pgmap v1061: 105 pgs: 105 active+clean; 77 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 40 op/s 2024-08-17T17:39:36.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:35 smithi130 ceph-mon[105521]: osdmap e913: 8 total, 8 up, 8 in 2024-08-17T17:39:36.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:35 smithi130 ceph-mon[110068]: pgmap v1061: 105 pgs: 105 active+clean; 77 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 40 op/s 2024-08-17T17:39:36.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:35 smithi130 ceph-mon[110068]: osdmap e913: 8 total, 8 up, 8 in 2024-08-17T17:39:36.797 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministicLUKS1 (30127 ms) 2024-08-17T17:39:36.797 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministicLUKS1PP 2024-08-17T17:39:36.797 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:39:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:36 smithi179 ceph-mon[93519]: osdmap e914: 8 total, 8 up, 8 in 2024-08-17T17:39:37.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:36 smithi130 ceph-mon[105521]: osdmap e914: 8 total, 8 up, 8 in 2024-08-17T17:39:37.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:36 smithi130 ceph-mon[110068]: osdmap e914: 8 total, 8 up, 8 in 2024-08-17T17:39:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:37 smithi179 ceph-mon[93519]: pgmap v1064: 105 pgs: 105 active+clean; 77 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-08-17T17:39:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:37 smithi179 ceph-mon[93519]: osdmap e915: 8 total, 8 up, 8 in 2024-08-17T17:39:38.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:37 smithi130 ceph-mon[105521]: pgmap v1064: 105 pgs: 105 active+clean; 77 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-08-17T17:39:38.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:37 smithi130 ceph-mon[105521]: osdmap e915: 8 total, 8 up, 8 in 2024-08-17T17:39:38.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:37 smithi130 ceph-mon[110068]: pgmap v1064: 105 pgs: 105 active+clean; 77 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-08-17T17:39:38.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:37 smithi130 ceph-mon[110068]: osdmap e915: 8 total, 8 up, 8 in 2024-08-17T17:39:40.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:39 smithi130 ceph-mon[105521]: pgmap v1066: 105 pgs: 105 active+clean; 89 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 1.3 MiB/s wr, 74 op/s 2024-08-17T17:39:40.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:39 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:39:40.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:39 smithi130 ceph-mon[110068]: pgmap v1066: 105 pgs: 105 active+clean; 89 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 1.3 MiB/s wr, 74 op/s 2024-08-17T17:39:40.060 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:39 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:39:40.060 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:39:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:39:39] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.51.0" 2024-08-17T17:39:40.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:39 smithi179 ceph-mon[93519]: pgmap v1066: 105 pgs: 105 active+clean; 89 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 1.3 MiB/s wr, 74 op/s 2024-08-17T17:39:40.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:39 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:39:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:39:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:39:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:39:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:41 smithi179 ceph-mon[93519]: pgmap v1067: 105 pgs: 105 active+clean; 89 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-08-17T17:39:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:41 smithi179 ceph-mon[93519]: osdmap e916: 8 total, 8 up, 8 in 2024-08-17T17:39:42.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:41 smithi130 ceph-mon[105521]: pgmap v1067: 105 pgs: 105 active+clean; 89 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-08-17T17:39:42.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:41 smithi130 ceph-mon[105521]: osdmap e916: 8 total, 8 up, 8 in 2024-08-17T17:39:42.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:41 smithi130 ceph-mon[110068]: pgmap v1067: 105 pgs: 105 active+clean; 89 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-08-17T17:39:42.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:41 smithi130 ceph-mon[110068]: osdmap e916: 8 total, 8 up, 8 in 2024-08-17T17:39:42.807 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:39:42.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:42 smithi130 ceph-mon[105521]: osdmap e917: 8 total, 8 up, 8 in 2024-08-17T17:39:42.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:42 smithi130 ceph-mon[110068]: osdmap e917: 8 total, 8 up, 8 in 2024-08-17T17:39:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:42 smithi179 ceph-mon[93519]: osdmap e917: 8 total, 8 up, 8 in 2024-08-17T17:39:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:43.923 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:43 smithi179 ceph-mon[93519]: pgmap v1070: 105 pgs: 105 active+clean; 89 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 40 op/s 2024-08-17T17:39:43.923 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:43 smithi179 ceph-mon[93519]: osdmap e918: 8 total, 8 up, 8 in 2024-08-17T17:39:43.923 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:43.923 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:43.924 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:43.924 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:43 smithi130 ceph-mon[105521]: pgmap v1070: 105 pgs: 105 active+clean; 89 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 40 op/s 2024-08-17T17:39:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:43 smithi130 ceph-mon[105521]: osdmap e918: 8 total, 8 up, 8 in 2024-08-17T17:39:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:44.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:43 smithi130 ceph-mon[110068]: pgmap v1070: 105 pgs: 105 active+clean; 89 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 40 op/s 2024-08-17T17:39:44.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:43 smithi130 ceph-mon[110068]: osdmap e918: 8 total, 8 up, 8 in 2024-08-17T17:39:44.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:44.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:44.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:44.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:44 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:39:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:44 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:39:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:44 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:45.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:44 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:39:45.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:44 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:39:45.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:44 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:44 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:39:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:44 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:39:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:44 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:39:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:45 smithi179 ceph-mon[93519]: pgmap v1072: 105 pgs: 105 active+clean; 93 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 43 op/s 2024-08-17T17:39:46.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:45 smithi130 ceph-mon[105521]: pgmap v1072: 105 pgs: 105 active+clean; 93 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 43 op/s 2024-08-17T17:39:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:45 smithi130 ceph-mon[110068]: pgmap v1072: 105 pgs: 105 active+clean; 93 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 43 op/s 2024-08-17T17:39:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:47 smithi179 ceph-mon[93519]: pgmap v1073: 105 pgs: 105 active+clean; 93 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 42 op/s 2024-08-17T17:39:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:47 smithi179 ceph-mon[93519]: osdmap e919: 8 total, 8 up, 8 in 2024-08-17T17:39:48.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:47 smithi130 ceph-mon[105521]: pgmap v1073: 105 pgs: 105 active+clean; 93 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 42 op/s 2024-08-17T17:39:48.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:47 smithi130 ceph-mon[105521]: osdmap e919: 8 total, 8 up, 8 in 2024-08-17T17:39:48.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:47 smithi130 ceph-mon[110068]: pgmap v1073: 105 pgs: 105 active+clean; 93 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 42 op/s 2024-08-17T17:39:48.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:47 smithi130 ceph-mon[110068]: osdmap e919: 8 total, 8 up, 8 in 2024-08-17T17:39:48.840 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:39:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:48 smithi179 ceph-mon[93519]: osdmap e920: 8 total, 8 up, 8 in 2024-08-17T17:39:49.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:48 smithi130 ceph-mon[105521]: osdmap e920: 8 total, 8 up, 8 in 2024-08-17T17:39:49.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:48 smithi130 ceph-mon[110068]: osdmap e920: 8 total, 8 up, 8 in 2024-08-17T17:39:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:49 smithi179 ceph-mon[93519]: pgmap v1076: 105 pgs: 105 active+clean; 103 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 1.3 MiB/s wr, 60 op/s 2024-08-17T17:39:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:49 smithi179 ceph-mon[93519]: osdmap e921: 8 total, 8 up, 8 in 2024-08-17T17:39:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:39:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:39:49] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.51.0" 2024-08-17T17:39:50.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:49 smithi130 ceph-mon[105521]: pgmap v1076: 105 pgs: 105 active+clean; 103 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 1.3 MiB/s wr, 60 op/s 2024-08-17T17:39:50.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:49 smithi130 ceph-mon[105521]: osdmap e921: 8 total, 8 up, 8 in 2024-08-17T17:39:50.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:49 smithi130 ceph-mon[110068]: pgmap v1076: 105 pgs: 105 active+clean; 103 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 1.3 MiB/s wr, 60 op/s 2024-08-17T17:39:50.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:49 smithi130 ceph-mon[110068]: osdmap e921: 8 total, 8 up, 8 in 2024-08-17T17:39:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:39:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:39:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:39:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:51 smithi179 ceph-mon[93519]: pgmap v1078: 105 pgs: 105 active+clean; 103 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 685 KiB/s wr, 19 op/s 2024-08-17T17:39:52.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:51 smithi130 ceph-mon[105521]: pgmap v1078: 105 pgs: 105 active+clean; 103 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 685 KiB/s wr, 19 op/s 2024-08-17T17:39:52.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:51 smithi130 ceph-mon[110068]: pgmap v1078: 105 pgs: 105 active+clean; 103 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 685 KiB/s wr, 19 op/s 2024-08-17T17:39:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:53 smithi179 ceph-mon[93519]: pgmap v1079: 105 pgs: 105 active+clean; 103 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 686 KiB/s wr, 21 op/s 2024-08-17T17:39:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:53 smithi179 ceph-mon[93519]: osdmap e922: 8 total, 8 up, 8 in 2024-08-17T17:39:54.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:53 smithi130 ceph-mon[105521]: pgmap v1079: 105 pgs: 105 active+clean; 103 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 686 KiB/s wr, 21 op/s 2024-08-17T17:39:54.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:53 smithi130 ceph-mon[105521]: osdmap e922: 8 total, 8 up, 8 in 2024-08-17T17:39:54.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:53 smithi130 ceph-mon[110068]: pgmap v1079: 105 pgs: 105 active+clean; 103 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 686 KiB/s wr, 21 op/s 2024-08-17T17:39:54.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:53 smithi130 ceph-mon[110068]: osdmap e922: 8 total, 8 up, 8 in 2024-08-17T17:39:54.857 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:39:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:54 smithi179 ceph-mon[93519]: osdmap e923: 8 total, 8 up, 8 in 2024-08-17T17:39:55.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:54 smithi130 ceph-mon[105521]: osdmap e923: 8 total, 8 up, 8 in 2024-08-17T17:39:55.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:54 smithi130 ceph-mon[110068]: osdmap e923: 8 total, 8 up, 8 in 2024-08-17T17:39:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:55 smithi179 ceph-mon[93519]: pgmap v1082: 105 pgs: 105 active+clean; 106 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.8 KiB/s wr, 30 op/s 2024-08-17T17:39:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:55 smithi179 ceph-mon[93519]: osdmap e924: 8 total, 8 up, 8 in 2024-08-17T17:39:56.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:55 smithi130 ceph-mon[105521]: pgmap v1082: 105 pgs: 105 active+clean; 106 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.8 KiB/s wr, 30 op/s 2024-08-17T17:39:56.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:55 smithi130 ceph-mon[105521]: osdmap e924: 8 total, 8 up, 8 in 2024-08-17T17:39:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:55 smithi130 ceph-mon[110068]: pgmap v1082: 105 pgs: 105 active+clean; 106 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.8 KiB/s wr, 30 op/s 2024-08-17T17:39:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:55 smithi130 ceph-mon[110068]: osdmap e924: 8 total, 8 up, 8 in 2024-08-17T17:39:58.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:57 smithi179 ceph-mon[93519]: pgmap v1084: 105 pgs: 105 active+clean; 106 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.8 KiB/s wr, 30 op/s 2024-08-17T17:39:58.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:58.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:57 smithi130 ceph-mon[105521]: pgmap v1084: 105 pgs: 105 active+clean; 106 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.8 KiB/s wr, 30 op/s 2024-08-17T17:39:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:57 smithi130 ceph-mon[110068]: pgmap v1084: 105 pgs: 105 active+clean; 106 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.8 KiB/s wr, 30 op/s 2024-08-17T17:39:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:59 smithi130 ceph-mon[105521]: pgmap v1085: 105 pgs: 105 active+clean; 113 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 687 KiB/s wr, 54 op/s 2024-08-17T17:40:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:39:59 smithi130 ceph-mon[105521]: osdmap e925: 8 total, 8 up, 8 in 2024-08-17T17:40:00.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:39:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:39:59] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-08-17T17:40:00.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:59 smithi130 ceph-mon[110068]: pgmap v1085: 105 pgs: 105 active+clean; 113 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 687 KiB/s wr, 54 op/s 2024-08-17T17:40:00.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:39:59 smithi130 ceph-mon[110068]: osdmap e925: 8 total, 8 up, 8 in 2024-08-17T17:40:00.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:59 smithi179 ceph-mon[93519]: pgmap v1085: 105 pgs: 105 active+clean; 113 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 687 KiB/s wr, 54 op/s 2024-08-17T17:40:00.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:39:59 smithi179 ceph-mon[93519]: osdmap e925: 8 total, 8 up, 8 in 2024-08-17T17:40:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:40:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:40:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:40:00.891 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:40:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:00 smithi130 ceph-mon[105521]: osdmap e926: 8 total, 8 up, 8 in 2024-08-17T17:40:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:00 smithi130 ceph-mon[105521]: overall HEALTH_OK 2024-08-17T17:40:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:00 smithi130 ceph-mon[110068]: osdmap e926: 8 total, 8 up, 8 in 2024-08-17T17:40:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:00 smithi130 ceph-mon[110068]: overall HEALTH_OK 2024-08-17T17:40:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:00 smithi179 ceph-mon[93519]: osdmap e926: 8 total, 8 up, 8 in 2024-08-17T17:40:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:00 smithi179 ceph-mon[93519]: overall HEALTH_OK 2024-08-17T17:40:02.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:01 smithi130 ceph-mon[105521]: pgmap v1088: 105 pgs: 105 active+clean; 113 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 685 KiB/s wr, 30 op/s 2024-08-17T17:40:02.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:01 smithi130 ceph-mon[105521]: osdmap e927: 8 total, 8 up, 8 in 2024-08-17T17:40:02.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:01 smithi130 ceph-mon[110068]: pgmap v1088: 105 pgs: 105 active+clean; 113 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 685 KiB/s wr, 30 op/s 2024-08-17T17:40:02.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:01 smithi130 ceph-mon[110068]: osdmap e927: 8 total, 8 up, 8 in 2024-08-17T17:40:02.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:01 smithi179 ceph-mon[93519]: pgmap v1088: 105 pgs: 105 active+clean; 113 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 685 KiB/s wr, 30 op/s 2024-08-17T17:40:02.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:01 smithi179 ceph-mon[93519]: osdmap e927: 8 total, 8 up, 8 in 2024-08-17T17:40:04.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:03 smithi130 ceph-mon[105521]: pgmap v1090: 105 pgs: 105 active+clean; 117 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 1.3 MiB/s wr, 31 op/s 2024-08-17T17:40:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:03 smithi130 ceph-mon[110068]: pgmap v1090: 105 pgs: 105 active+clean; 117 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 1.3 MiB/s wr, 31 op/s 2024-08-17T17:40:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:03 smithi179 ceph-mon[93519]: pgmap v1090: 105 pgs: 105 active+clean; 117 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 1.3 MiB/s wr, 31 op/s 2024-08-17T17:40:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:05 smithi130 ceph-mon[105521]: pgmap v1091: 105 pgs: 105 active+clean; 123 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 687 KiB/s wr, 45 op/s 2024-08-17T17:40:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:05 smithi130 ceph-mon[105521]: osdmap e928: 8 total, 8 up, 8 in 2024-08-17T17:40:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:05 smithi130 ceph-mon[110068]: pgmap v1091: 105 pgs: 105 active+clean; 123 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 687 KiB/s wr, 45 op/s 2024-08-17T17:40:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:05 smithi130 ceph-mon[110068]: osdmap e928: 8 total, 8 up, 8 in 2024-08-17T17:40:06.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:05 smithi179 ceph-mon[93519]: pgmap v1091: 105 pgs: 105 active+clean; 123 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 687 KiB/s wr, 45 op/s 2024-08-17T17:40:06.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:05 smithi179 ceph-mon[93519]: osdmap e928: 8 total, 8 up, 8 in 2024-08-17T17:40:06.914 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministicLUKS1PP (30117 ms) 2024-08-17T17:40:06.914 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministicLUKS2 2024-08-17T17:40:06.914 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:40:07.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:06 smithi130 ceph-mon[105521]: osdmap e929: 8 total, 8 up, 8 in 2024-08-17T17:40:07.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:06 smithi130 ceph-mon[110068]: osdmap e929: 8 total, 8 up, 8 in 2024-08-17T17:40:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:06 smithi179 ceph-mon[93519]: osdmap e929: 8 total, 8 up, 8 in 2024-08-17T17:40:08.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:07 smithi130 ceph-mon[110068]: pgmap v1094: 105 pgs: 105 active+clean; 123 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 39 op/s 2024-08-17T17:40:08.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:07 smithi130 ceph-mon[110068]: osdmap e930: 8 total, 8 up, 8 in 2024-08-17T17:40:08.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:07 smithi130 ceph-mon[105521]: pgmap v1094: 105 pgs: 105 active+clean; 123 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 39 op/s 2024-08-17T17:40:08.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:07 smithi130 ceph-mon[105521]: osdmap e930: 8 total, 8 up, 8 in 2024-08-17T17:40:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:07 smithi179 ceph-mon[93519]: pgmap v1094: 105 pgs: 105 active+clean; 123 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 687 KiB/s wr, 39 op/s 2024-08-17T17:40:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:07 smithi179 ceph-mon[93519]: osdmap e930: 8 total, 8 up, 8 in 2024-08-17T17:40:10.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:09 smithi130 ceph-mon[105521]: pgmap v1096: 105 pgs: 105 active+clean; 135 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 689 KiB/s wr, 71 op/s 2024-08-17T17:40:10.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:40:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:40:09] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-08-17T17:40:10.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:09 smithi130 ceph-mon[110068]: pgmap v1096: 105 pgs: 105 active+clean; 135 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 689 KiB/s wr, 71 op/s 2024-08-17T17:40:10.212 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:09 smithi179 ceph-mon[93519]: pgmap v1096: 105 pgs: 105 active+clean; 135 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 689 KiB/s wr, 71 op/s 2024-08-17T17:40:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:40:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:40:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:40:12.147 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:11 smithi179 ceph-mon[93519]: pgmap v1097: 105 pgs: 105 active+clean; 135 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 686 KiB/s wr, 33 op/s 2024-08-17T17:40:12.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:11 smithi130 ceph-mon[105521]: pgmap v1097: 105 pgs: 105 active+clean; 135 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 686 KiB/s wr, 33 op/s 2024-08-17T17:40:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:11 smithi130 ceph-mon[110068]: pgmap v1097: 105 pgs: 105 active+clean; 135 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 686 KiB/s wr, 33 op/s 2024-08-17T17:40:12.593 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:40:12 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=sqlstore.transactions t=2024-08-17T17:40:12.146920292Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T17:40:13.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:13.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:14.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:13 smithi130 ceph-mon[110068]: pgmap v1098: 105 pgs: 105 active+clean; 135 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 558 KiB/s wr, 32 op/s 2024-08-17T17:40:14.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:13 smithi130 ceph-mon[110068]: osdmap e931: 8 total, 8 up, 8 in 2024-08-17T17:40:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:13 smithi130 ceph-mon[105521]: pgmap v1098: 105 pgs: 105 active+clean; 135 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 558 KiB/s wr, 32 op/s 2024-08-17T17:40:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:13 smithi130 ceph-mon[105521]: osdmap e931: 8 total, 8 up, 8 in 2024-08-17T17:40:14.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:13 smithi179 ceph-mon[93519]: pgmap v1098: 105 pgs: 105 active+clean; 135 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 558 KiB/s wr, 32 op/s 2024-08-17T17:40:14.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:13 smithi179 ceph-mon[93519]: osdmap e931: 8 total, 8 up, 8 in 2024-08-17T17:40:14.949 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:40:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:14 smithi179 ceph-mon[93519]: osdmap e932: 8 total, 8 up, 8 in 2024-08-17T17:40:15.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:14 smithi130 ceph-mon[105521]: osdmap e932: 8 total, 8 up, 8 in 2024-08-17T17:40:15.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:14 smithi130 ceph-mon[110068]: osdmap e932: 8 total, 8 up, 8 in 2024-08-17T17:40:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:15 smithi179 ceph-mon[93519]: pgmap v1101: 105 pgs: 105 active+clean; 151 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.2 MiB/s wr, 34 op/s 2024-08-17T17:40:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:15 smithi179 ceph-mon[93519]: osdmap e933: 8 total, 8 up, 8 in 2024-08-17T17:40:16.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:15 smithi130 ceph-mon[105521]: pgmap v1101: 105 pgs: 105 active+clean; 151 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.2 MiB/s wr, 34 op/s 2024-08-17T17:40:16.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:15 smithi130 ceph-mon[105521]: osdmap e933: 8 total, 8 up, 8 in 2024-08-17T17:40:16.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:15 smithi130 ceph-mon[110068]: pgmap v1101: 105 pgs: 105 active+clean; 151 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.2 MiB/s wr, 34 op/s 2024-08-17T17:40:16.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:15 smithi130 ceph-mon[110068]: osdmap e933: 8 total, 8 up, 8 in 2024-08-17T17:40:18.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:17 smithi179 ceph-mon[93519]: pgmap v1103: 105 pgs: 105 active+clean; 151 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 2.7 MiB/s wr, 16 op/s 2024-08-17T17:40:18.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:17 smithi130 ceph-mon[105521]: pgmap v1103: 105 pgs: 105 active+clean; 151 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 2.7 MiB/s wr, 16 op/s 2024-08-17T17:40:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:17 smithi130 ceph-mon[110068]: pgmap v1103: 105 pgs: 105 active+clean; 151 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 2.7 MiB/s wr, 16 op/s 2024-08-17T17:40:20.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:19 smithi130 ceph-mon[105521]: pgmap v1104: 105 pgs: 105 active+clean; 151 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:40:20.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:19 smithi130 ceph-mon[110068]: pgmap v1104: 105 pgs: 105 active+clean; 151 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:40:20.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:40:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:40:19] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-08-17T17:40:20.212 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:19 smithi179 ceph-mon[93519]: pgmap v1104: 105 pgs: 105 active+clean; 151 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:40:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:40:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:40:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:40:22.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:21 smithi179 ceph-mon[93519]: pgmap v1105: 105 pgs: 105 active+clean; 151 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.2 MiB/s wr, 36 op/s 2024-08-17T17:40:22.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:21 smithi179 ceph-mon[93519]: osdmap e934: 8 total, 8 up, 8 in 2024-08-17T17:40:22.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:21 smithi130 ceph-mon[110068]: pgmap v1105: 105 pgs: 105 active+clean; 151 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.2 MiB/s wr, 36 op/s 2024-08-17T17:40:22.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:21 smithi130 ceph-mon[110068]: osdmap e934: 8 total, 8 up, 8 in 2024-08-17T17:40:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:21 smithi130 ceph-mon[105521]: pgmap v1105: 105 pgs: 105 active+clean; 151 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.2 MiB/s wr, 36 op/s 2024-08-17T17:40:22.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:21 smithi130 ceph-mon[105521]: osdmap e934: 8 total, 8 up, 8 in 2024-08-17T17:40:22.987 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:40:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:22 smithi179 ceph-mon[93519]: osdmap e935: 8 total, 8 up, 8 in 2024-08-17T17:40:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:22 smithi179 ceph-mon[93519]: pgmap v1108: 105 pgs: 105 active+clean; 152 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 3.4 KiB/s wr, 29 op/s 2024-08-17T17:40:23.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:22 smithi130 ceph-mon[110068]: osdmap e935: 8 total, 8 up, 8 in 2024-08-17T17:40:23.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:22 smithi130 ceph-mon[110068]: pgmap v1108: 105 pgs: 105 active+clean; 152 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 3.4 KiB/s wr, 29 op/s 2024-08-17T17:40:23.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:22 smithi130 ceph-mon[105521]: osdmap e935: 8 total, 8 up, 8 in 2024-08-17T17:40:23.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:22 smithi130 ceph-mon[105521]: pgmap v1108: 105 pgs: 105 active+clean; 152 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 3.4 KiB/s wr, 29 op/s 2024-08-17T17:40:24.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:23 smithi179 ceph-mon[93519]: osdmap e936: 8 total, 8 up, 8 in 2024-08-17T17:40:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:23 smithi130 ceph-mon[105521]: osdmap e936: 8 total, 8 up, 8 in 2024-08-17T17:40:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:23 smithi130 ceph-mon[110068]: osdmap e936: 8 total, 8 up, 8 in 2024-08-17T17:40:25.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:24 smithi179 ceph-mon[93519]: pgmap v1110: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 40 op/s 2024-08-17T17:40:25.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:24 smithi130 ceph-mon[105521]: pgmap v1110: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 40 op/s 2024-08-17T17:40:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:24 smithi130 ceph-mon[110068]: pgmap v1110: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 40 op/s 2024-08-17T17:40:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:27 smithi179 ceph-mon[93519]: pgmap v1111: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 40 op/s 2024-08-17T17:40:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:27.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:27 smithi130 ceph-mon[110068]: pgmap v1111: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 40 op/s 2024-08-17T17:40:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:27 smithi130 ceph-mon[105521]: pgmap v1111: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 40 op/s 2024-08-17T17:40:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:29 smithi130 ceph-mon[110068]: pgmap v1112: 105 pgs: 105 active+clean; 169 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.2 MiB/s wr, 36 op/s 2024-08-17T17:40:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:29 smithi130 ceph-mon[105521]: pgmap v1112: 105 pgs: 105 active+clean; 169 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.2 MiB/s wr, 36 op/s 2024-08-17T17:40:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:29 smithi179 ceph-mon[93519]: pgmap v1112: 105 pgs: 105 active+clean; 169 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.2 MiB/s wr, 36 op/s 2024-08-17T17:40:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:40:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:40:29] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-08-17T17:40:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:30 smithi179 ceph-mon[93519]: osdmap e937: 8 total, 8 up, 8 in 2024-08-17T17:40:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:40:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:40:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:40:30.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:30 smithi130 ceph-mon[105521]: osdmap e937: 8 total, 8 up, 8 in 2024-08-17T17:40:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:30 smithi130 ceph-mon[110068]: osdmap e937: 8 total, 8 up, 8 in 2024-08-17T17:40:31.403 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:40:31.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:31 smithi130 ceph-mon[110068]: pgmap v1114: 105 pgs: 105 active+clean; 169 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-08-17T17:40:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:31 smithi130 ceph-mon[110068]: osdmap e938: 8 total, 8 up, 8 in 2024-08-17T17:40:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:31 smithi130 ceph-mon[105521]: pgmap v1114: 105 pgs: 105 active+clean; 169 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-08-17T17:40:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:31 smithi130 ceph-mon[105521]: osdmap e938: 8 total, 8 up, 8 in 2024-08-17T17:40:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:31 smithi179 ceph-mon[93519]: pgmap v1114: 105 pgs: 105 active+clean; 169 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-08-17T17:40:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:31 smithi179 ceph-mon[93519]: osdmap e938: 8 total, 8 up, 8 in 2024-08-17T17:40:32.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:32 smithi130 ceph-mon[105521]: osdmap e939: 8 total, 8 up, 8 in 2024-08-17T17:40:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:32 smithi130 ceph-mon[110068]: osdmap e939: 8 total, 8 up, 8 in 2024-08-17T17:40:32.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:32 smithi179 ceph-mon[93519]: osdmap e939: 8 total, 8 up, 8 in 2024-08-17T17:40:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:33 smithi130 ceph-mon[105521]: pgmap v1117: 105 pgs: 105 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 1.3 MiB/s wr, 8 op/s 2024-08-17T17:40:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:33 smithi130 ceph-mon[110068]: pgmap v1117: 105 pgs: 105 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 1.3 MiB/s wr, 8 op/s 2024-08-17T17:40:33.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:33 smithi179 ceph-mon[93519]: pgmap v1117: 105 pgs: 105 active+clean; 179 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 1.3 MiB/s wr, 8 op/s 2024-08-17T17:40:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:35 smithi130 ceph-mon[105521]: pgmap v1118: 105 pgs: 105 active+clean; 187 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:40:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:35 smithi130 ceph-mon[110068]: pgmap v1118: 105 pgs: 105 active+clean; 187 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:40:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:35 smithi179 ceph-mon[93519]: pgmap v1118: 105 pgs: 105 active+clean; 187 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:40:37.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:37 smithi130 ceph-mon[105521]: pgmap v1119: 105 pgs: 105 active+clean; 187 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.3 MiB/s wr, 38 op/s 2024-08-17T17:40:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:37 smithi130 ceph-mon[110068]: pgmap v1119: 105 pgs: 105 active+clean; 187 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.3 MiB/s wr, 38 op/s 2024-08-17T17:40:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:37 smithi179 ceph-mon[93519]: pgmap v1119: 105 pgs: 105 active+clean; 187 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.3 MiB/s wr, 38 op/s 2024-08-17T17:40:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:38 smithi130 ceph-mon[105521]: osdmap e940: 8 total, 8 up, 8 in 2024-08-17T17:40:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:38 smithi130 ceph-mon[110068]: osdmap e940: 8 total, 8 up, 8 in 2024-08-17T17:40:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:38 smithi179 ceph-mon[93519]: osdmap e940: 8 total, 8 up, 8 in 2024-08-17T17:40:39.439 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:40:39.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:39 smithi130 ceph-mon[110068]: pgmap v1121: 105 pgs: 105 active+clean; 199 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 3.5 MiB/s wr, 45 op/s 2024-08-17T17:40:39.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:39 smithi130 ceph-mon[110068]: osdmap e941: 8 total, 8 up, 8 in 2024-08-17T17:40:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:39 smithi130 ceph-mon[105521]: pgmap v1121: 105 pgs: 105 active+clean; 199 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 3.5 MiB/s wr, 45 op/s 2024-08-17T17:40:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:39 smithi130 ceph-mon[105521]: osdmap e941: 8 total, 8 up, 8 in 2024-08-17T17:40:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:39 smithi179 ceph-mon[93519]: pgmap v1121: 105 pgs: 105 active+clean; 199 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 3.5 MiB/s wr, 45 op/s 2024-08-17T17:40:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:39 smithi179 ceph-mon[93519]: osdmap e941: 8 total, 8 up, 8 in 2024-08-17T17:40:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:40:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:40:39] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-08-17T17:40:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:40 smithi179 ceph-mon[93519]: osdmap e942: 8 total, 8 up, 8 in 2024-08-17T17:40:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:40:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:40:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:40:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:40 smithi130 ceph-mon[105521]: osdmap e942: 8 total, 8 up, 8 in 2024-08-17T17:40:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:40 smithi130 ceph-mon[110068]: osdmap e942: 8 total, 8 up, 8 in 2024-08-17T17:40:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:41 smithi179 ceph-mon[93519]: pgmap v1124: 105 pgs: 105 active+clean; 200 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-08-17T17:40:41.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:41 smithi130 ceph-mon[105521]: pgmap v1124: 105 pgs: 105 active+clean; 200 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-08-17T17:40:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:41 smithi130 ceph-mon[110068]: pgmap v1124: 105 pgs: 105 active+clean; 200 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-08-17T17:40:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:42.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:43.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:43 smithi179 ceph-mon[93519]: pgmap v1125: 105 pgs: 105 active+clean; 204 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-08-17T17:40:43.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:43 smithi130 ceph-mon[105521]: pgmap v1125: 105 pgs: 105 active+clean; 204 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-08-17T17:40:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:43 smithi130 ceph-mon[110068]: pgmap v1125: 105 pgs: 105 active+clean; 204 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-08-17T17:40:44.793 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:44 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:40:44.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:44 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:40:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:44 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:40:45.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:45 smithi130 ceph-mon[105521]: pgmap v1126: 105 pgs: 105 active+clean; 205 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.3 MiB/s wr, 39 op/s 2024-08-17T17:40:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:45 smithi130 ceph-mon[110068]: pgmap v1126: 105 pgs: 105 active+clean; 205 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.3 MiB/s wr, 39 op/s 2024-08-17T17:40:45.704 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:45 smithi179 ceph-mon[93519]: pgmap v1126: 105 pgs: 105 active+clean; 205 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.3 MiB/s wr, 39 op/s 2024-08-17T17:40:46.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:46 smithi130 ceph-mon[105521]: osdmap e943: 8 total, 8 up, 8 in 2024-08-17T17:40:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:46 smithi130 ceph-mon[110068]: osdmap e943: 8 total, 8 up, 8 in 2024-08-17T17:40:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:46 smithi179 ceph-mon[93519]: osdmap e943: 8 total, 8 up, 8 in 2024-08-17T17:40:47.483 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministicLUKS2 (40569 ms) 2024-08-17T17:40:47.483 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministicLUKS2PP 2024-08-17T17:40:47.483 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:40:47.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:47 smithi130 ceph-mon[105521]: pgmap v1128: 105 pgs: 105 active+clean; 205 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 523 KiB/s wr, 23 op/s 2024-08-17T17:40:47.801 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:47 smithi130 ceph-mon[105521]: osdmap e944: 8 total, 8 up, 8 in 2024-08-17T17:40:47.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:47 smithi130 ceph-mon[110068]: pgmap v1128: 105 pgs: 105 active+clean; 205 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 523 KiB/s wr, 23 op/s 2024-08-17T17:40:47.801 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:47 smithi130 ceph-mon[110068]: osdmap e944: 8 total, 8 up, 8 in 2024-08-17T17:40:47.825 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:47 smithi179 ceph-mon[93519]: pgmap v1128: 105 pgs: 105 active+clean; 205 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 523 KiB/s wr, 23 op/s 2024-08-17T17:40:47.825 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:47 smithi179 ceph-mon[93519]: osdmap e944: 8 total, 8 up, 8 in 2024-08-17T17:40:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:48 smithi179 ceph-mon[93519]: osdmap e945: 8 total, 8 up, 8 in 2024-08-17T17:40:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:48 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:48 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:48 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:48 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:48 smithi130 ceph-mon[105521]: osdmap e945: 8 total, 8 up, 8 in 2024-08-17T17:40:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:48 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:48 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:48 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:48 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:48 smithi130 ceph-mon[110068]: osdmap e945: 8 total, 8 up, 8 in 2024-08-17T17:40:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:48 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:48 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:48 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:48 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:49 smithi130 ceph-mon[105521]: pgmap v1131: 105 pgs: 105 active+clean; 225 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 2.0 MiB/s wr, 61 op/s 2024-08-17T17:40:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:49 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:40:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:49 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:40:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:49 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:49 smithi130 ceph-mon[110068]: pgmap v1131: 105 pgs: 105 active+clean; 225 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 2.0 MiB/s wr, 61 op/s 2024-08-17T17:40:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:49 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:40:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:49 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:40:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:49 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:49.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:49 smithi179 ceph-mon[93519]: pgmap v1131: 105 pgs: 105 active+clean; 225 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 2.0 MiB/s wr, 61 op/s 2024-08-17T17:40:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:49 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:40:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:49 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:40:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:49 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:40:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:40:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:40:49] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-08-17T17:40:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:40:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:40:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:40:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:51 smithi179 ceph-mon[93519]: pgmap v1132: 105 pgs: 105 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:40:51.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:51 smithi130 ceph-mon[105521]: pgmap v1132: 105 pgs: 105 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:40:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:51 smithi130 ceph-mon[110068]: pgmap v1132: 105 pgs: 105 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:40:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:53 smithi179 ceph-mon[93519]: pgmap v1133: 105 pgs: 105 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.3 MiB/s wr, 39 op/s 2024-08-17T17:40:53.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:53 smithi130 ceph-mon[105521]: pgmap v1133: 105 pgs: 105 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.3 MiB/s wr, 39 op/s 2024-08-17T17:40:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:53 smithi130 ceph-mon[110068]: pgmap v1133: 105 pgs: 105 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.3 MiB/s wr, 39 op/s 2024-08-17T17:40:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:55 smithi179 ceph-mon[93519]: pgmap v1134: 105 pgs: 105 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-08-17T17:40:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:55 smithi179 ceph-mon[93519]: osdmap e946: 8 total, 8 up, 8 in 2024-08-17T17:40:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:55 smithi130 ceph-mon[105521]: pgmap v1134: 105 pgs: 105 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-08-17T17:40:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:55 smithi130 ceph-mon[105521]: osdmap e946: 8 total, 8 up, 8 in 2024-08-17T17:40:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:55 smithi130 ceph-mon[110068]: pgmap v1134: 105 pgs: 105 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-08-17T17:40:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:55 smithi130 ceph-mon[110068]: osdmap e946: 8 total, 8 up, 8 in 2024-08-17T17:40:56.520 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:40:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:56 smithi179 ceph-mon[93519]: osdmap e947: 8 total, 8 up, 8 in 2024-08-17T17:40:56.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:56 smithi130 ceph-mon[105521]: osdmap e947: 8 total, 8 up, 8 in 2024-08-17T17:40:56.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:56 smithi130 ceph-mon[110068]: osdmap e947: 8 total, 8 up, 8 in 2024-08-17T17:40:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:57 smithi179 ceph-mon[93519]: pgmap v1137: 105 pgs: 105 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 514 KiB/s wr, 9 op/s 2024-08-17T17:40:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:57 smithi179 ceph-mon[93519]: osdmap e948: 8 total, 8 up, 8 in 2024-08-17T17:40:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:57 smithi130 ceph-mon[105521]: pgmap v1137: 105 pgs: 105 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 514 KiB/s wr, 9 op/s 2024-08-17T17:40:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:57 smithi130 ceph-mon[105521]: osdmap e948: 8 total, 8 up, 8 in 2024-08-17T17:40:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:57 smithi130 ceph-mon[110068]: pgmap v1137: 105 pgs: 105 active+clean; 229 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 514 KiB/s wr, 9 op/s 2024-08-17T17:40:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:57 smithi130 ceph-mon[110068]: osdmap e948: 8 total, 8 up, 8 in 2024-08-17T17:40:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:59.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:40:59 smithi130 ceph-mon[105521]: pgmap v1139: 105 pgs: 105 active+clean; 241 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 32 op/s 2024-08-17T17:40:59.820 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:40:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:40:59] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-08-17T17:40:59.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:40:59 smithi130 ceph-mon[110068]: pgmap v1139: 105 pgs: 105 active+clean; 241 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 32 op/s 2024-08-17T17:40:59.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:40:59 smithi179 ceph-mon[93519]: pgmap v1139: 105 pgs: 105 active+clean; 241 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 32 op/s 2024-08-17T17:41:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:41:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:41:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:41:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:01 smithi179 ceph-mon[93519]: pgmap v1140: 105 pgs: 105 active+clean; 245 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:41:01.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:01 smithi130 ceph-mon[105521]: pgmap v1140: 105 pgs: 105 active+clean; 245 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:41:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:01 smithi130 ceph-mon[110068]: pgmap v1140: 105 pgs: 105 active+clean; 245 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:41:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:03 smithi179 ceph-mon[93519]: pgmap v1141: 105 pgs: 105 active+clean; 245 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.1 MiB/s wr, 34 op/s 2024-08-17T17:41:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:03 smithi179 ceph-mon[93519]: osdmap e949: 8 total, 8 up, 8 in 2024-08-17T17:41:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:03 smithi130 ceph-mon[105521]: pgmap v1141: 105 pgs: 105 active+clean; 245 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.1 MiB/s wr, 34 op/s 2024-08-17T17:41:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:03 smithi130 ceph-mon[105521]: osdmap e949: 8 total, 8 up, 8 in 2024-08-17T17:41:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:03 smithi130 ceph-mon[110068]: pgmap v1141: 105 pgs: 105 active+clean; 245 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.1 MiB/s wr, 34 op/s 2024-08-17T17:41:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:03 smithi130 ceph-mon[110068]: osdmap e949: 8 total, 8 up, 8 in 2024-08-17T17:41:04.556 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:41:04.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:04 smithi179 ceph-mon[93519]: osdmap e950: 8 total, 8 up, 8 in 2024-08-17T17:41:04.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:04 smithi130 ceph-mon[105521]: osdmap e950: 8 total, 8 up, 8 in 2024-08-17T17:41:04.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:04 smithi130 ceph-mon[110068]: osdmap e950: 8 total, 8 up, 8 in 2024-08-17T17:41:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:05 smithi179 ceph-mon[93519]: pgmap v1144: 105 pgs: 105 active+clean; 264 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-08-17T17:41:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:05 smithi179 ceph-mon[93519]: osdmap e951: 8 total, 8 up, 8 in 2024-08-17T17:41:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:05 smithi130 ceph-mon[110068]: pgmap v1144: 105 pgs: 105 active+clean; 264 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-08-17T17:41:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:05 smithi130 ceph-mon[110068]: osdmap e951: 8 total, 8 up, 8 in 2024-08-17T17:41:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:05 smithi130 ceph-mon[105521]: pgmap v1144: 105 pgs: 105 active+clean; 264 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-08-17T17:41:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:05 smithi130 ceph-mon[105521]: osdmap e951: 8 total, 8 up, 8 in 2024-08-17T17:41:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:07 smithi179 ceph-mon[93519]: pgmap v1146: 105 pgs: 105 active+clean; 264 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.7 MiB/s wr, 6 op/s 2024-08-17T17:41:07.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:07 smithi130 ceph-mon[105521]: pgmap v1146: 105 pgs: 105 active+clean; 264 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.7 MiB/s wr, 6 op/s 2024-08-17T17:41:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:07 smithi130 ceph-mon[110068]: pgmap v1146: 105 pgs: 105 active+clean; 264 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.7 MiB/s wr, 6 op/s 2024-08-17T17:41:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:09 smithi130 ceph-mon[105521]: pgmap v1147: 105 pgs: 105 active+clean; 267 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:41:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:09 smithi130 ceph-mon[110068]: pgmap v1147: 105 pgs: 105 active+clean; 267 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:41:09.820 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:41:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:41:09] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-08-17T17:41:09.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:09 smithi179 ceph-mon[93519]: pgmap v1147: 105 pgs: 105 active+clean; 267 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-08-17T17:41:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:41:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:41:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:41:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:11 smithi179 ceph-mon[93519]: pgmap v1148: 105 pgs: 105 active+clean; 267 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.1 MiB/s wr, 34 op/s 2024-08-17T17:41:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:11 smithi179 ceph-mon[93519]: osdmap e952: 8 total, 8 up, 8 in 2024-08-17T17:41:11.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:11 smithi130 ceph-mon[105521]: pgmap v1148: 105 pgs: 105 active+clean; 267 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.1 MiB/s wr, 34 op/s 2024-08-17T17:41:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:11 smithi130 ceph-mon[105521]: osdmap e952: 8 total, 8 up, 8 in 2024-08-17T17:41:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:11 smithi130 ceph-mon[110068]: pgmap v1148: 105 pgs: 105 active+clean; 267 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.1 MiB/s wr, 34 op/s 2024-08-17T17:41:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:11 smithi130 ceph-mon[110068]: osdmap e952: 8 total, 8 up, 8 in 2024-08-17T17:41:12.577 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:41:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:12 smithi179 ceph-mon[93519]: osdmap e953: 8 total, 8 up, 8 in 2024-08-17T17:41:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:12.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:12 smithi130 ceph-mon[105521]: osdmap e953: 8 total, 8 up, 8 in 2024-08-17T17:41:12.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:12 smithi130 ceph-mon[110068]: osdmap e953: 8 total, 8 up, 8 in 2024-08-17T17:41:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:13.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:13 smithi179 ceph-mon[93519]: pgmap v1151: 105 pgs: 105 active+clean; 267 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.7 KiB/s wr, 37 op/s 2024-08-17T17:41:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:13 smithi179 ceph-mon[93519]: osdmap e954: 8 total, 8 up, 8 in 2024-08-17T17:41:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:13 smithi130 ceph-mon[110068]: pgmap v1151: 105 pgs: 105 active+clean; 267 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.7 KiB/s wr, 37 op/s 2024-08-17T17:41:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:13 smithi130 ceph-mon[110068]: osdmap e954: 8 total, 8 up, 8 in 2024-08-17T17:41:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:13 smithi130 ceph-mon[105521]: pgmap v1151: 105 pgs: 105 active+clean; 267 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.7 KiB/s wr, 37 op/s 2024-08-17T17:41:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:13 smithi130 ceph-mon[105521]: osdmap e954: 8 total, 8 up, 8 in 2024-08-17T17:41:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:15 smithi179 ceph-mon[93519]: pgmap v1153: 105 pgs: 105 active+clean; 289 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-08-17T17:41:15.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:15 smithi130 ceph-mon[105521]: pgmap v1153: 105 pgs: 105 active+clean; 289 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-08-17T17:41:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:15 smithi130 ceph-mon[110068]: pgmap v1153: 105 pgs: 105 active+clean; 289 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-08-17T17:41:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:17 smithi179 ceph-mon[93519]: pgmap v1154: 105 pgs: 105 active+clean; 289 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-08-17T17:41:17.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:17 smithi130 ceph-mon[105521]: pgmap v1154: 105 pgs: 105 active+clean; 289 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-08-17T17:41:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:17 smithi130 ceph-mon[110068]: pgmap v1154: 105 pgs: 105 active+clean; 289 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-08-17T17:41:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:19 smithi179 ceph-mon[93519]: pgmap v1155: 105 pgs: 105 active+clean; 293 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.6 MiB/s wr, 35 op/s 2024-08-17T17:41:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:19 smithi179 ceph-mon[93519]: osdmap e955: 8 total, 8 up, 8 in 2024-08-17T17:41:19.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:19 smithi130 ceph-mon[105521]: pgmap v1155: 105 pgs: 105 active+clean; 293 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.6 MiB/s wr, 35 op/s 2024-08-17T17:41:19.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:19 smithi130 ceph-mon[105521]: osdmap e955: 8 total, 8 up, 8 in 2024-08-17T17:41:19.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:41:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:41:19] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-08-17T17:41:19.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:19 smithi130 ceph-mon[110068]: pgmap v1155: 105 pgs: 105 active+clean; 293 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.6 MiB/s wr, 35 op/s 2024-08-17T17:41:19.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:19 smithi130 ceph-mon[110068]: osdmap e955: 8 total, 8 up, 8 in 2024-08-17T17:41:20.577 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:41:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:41:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:41:20.605 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:41:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:20 smithi179 ceph-mon[93519]: osdmap e956: 8 total, 8 up, 8 in 2024-08-17T17:41:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:20 smithi130 ceph-mon[105521]: osdmap e956: 8 total, 8 up, 8 in 2024-08-17T17:41:20.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:20 smithi130 ceph-mon[110068]: osdmap e956: 8 total, 8 up, 8 in 2024-08-17T17:41:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:21 smithi179 ceph-mon[93519]: pgmap v1158: 105 pgs: 105 active+clean; 301 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 3.6 MiB/s wr, 33 op/s 2024-08-17T17:41:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:21 smithi179 ceph-mon[93519]: osdmap e957: 8 total, 8 up, 8 in 2024-08-17T17:41:21.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:21 smithi130 ceph-mon[105521]: pgmap v1158: 105 pgs: 105 active+clean; 301 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 3.6 MiB/s wr, 33 op/s 2024-08-17T17:41:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:21 smithi130 ceph-mon[105521]: osdmap e957: 8 total, 8 up, 8 in 2024-08-17T17:41:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:21 smithi130 ceph-mon[110068]: pgmap v1158: 105 pgs: 105 active+clean; 301 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 3.6 MiB/s wr, 33 op/s 2024-08-17T17:41:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:21 smithi130 ceph-mon[110068]: osdmap e957: 8 total, 8 up, 8 in 2024-08-17T17:41:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:23 smithi179 ceph-mon[93519]: pgmap v1160: 105 pgs: 105 active+clean; 305 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 2.7 MiB/s wr, 10 op/s 2024-08-17T17:41:23.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:23 smithi130 ceph-mon[110068]: pgmap v1160: 105 pgs: 105 active+clean; 305 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 2.7 MiB/s wr, 10 op/s 2024-08-17T17:41:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:23 smithi130 ceph-mon[105521]: pgmap v1160: 105 pgs: 105 active+clean; 305 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 2.7 MiB/s wr, 10 op/s 2024-08-17T17:41:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:25 smithi179 ceph-mon[93519]: pgmap v1161: 105 pgs: 105 active+clean; 311 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.0 MiB/s wr, 44 op/s 2024-08-17T17:41:25.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:25 smithi130 ceph-mon[105521]: pgmap v1161: 105 pgs: 105 active+clean; 311 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.0 MiB/s wr, 44 op/s 2024-08-17T17:41:25.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:25 smithi130 ceph-mon[110068]: pgmap v1161: 105 pgs: 105 active+clean; 311 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.0 MiB/s wr, 44 op/s 2024-08-17T17:41:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:27 smithi130 ceph-mon[105521]: pgmap v1162: 105 pgs: 105 active+clean; 311 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 1.6 MiB/s wr, 34 op/s 2024-08-17T17:41:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:27 smithi130 ceph-mon[105521]: osdmap e958: 8 total, 8 up, 8 in 2024-08-17T17:41:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:27 smithi130 ceph-mon[110068]: pgmap v1162: 105 pgs: 105 active+clean; 311 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 1.6 MiB/s wr, 34 op/s 2024-08-17T17:41:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:27 smithi130 ceph-mon[110068]: osdmap e958: 8 total, 8 up, 8 in 2024-08-17T17:41:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:27 smithi179 ceph-mon[93519]: pgmap v1162: 105 pgs: 105 active+clean; 311 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 1.6 MiB/s wr, 34 op/s 2024-08-17T17:41:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:27 smithi179 ceph-mon[93519]: osdmap e958: 8 total, 8 up, 8 in 2024-08-17T17:41:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:28.642 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministicLUKS2PP (41159 ms) 2024-08-17T17:41:28.643 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDiscard 2024-08-17T17:41:28.643 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:41:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:28 smithi130 ceph-mon[105521]: osdmap e959: 8 total, 8 up, 8 in 2024-08-17T17:41:28.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:28 smithi130 ceph-mon[110068]: osdmap e959: 8 total, 8 up, 8 in 2024-08-17T17:41:29.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:28 smithi179 ceph-mon[93519]: osdmap e959: 8 total, 8 up, 8 in 2024-08-17T17:41:29.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:29 smithi130 ceph-mon[105521]: pgmap v1165: 105 pgs: 105 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.6 MiB/s wr, 46 op/s 2024-08-17T17:41:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:29 smithi130 ceph-mon[105521]: osdmap e960: 8 total, 8 up, 8 in 2024-08-17T17:41:29.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:29 smithi130 ceph-mon[110068]: pgmap v1165: 105 pgs: 105 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.6 MiB/s wr, 46 op/s 2024-08-17T17:41:29.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:29 smithi130 ceph-mon[110068]: osdmap e960: 8 total, 8 up, 8 in 2024-08-17T17:41:29.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:41:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:41:29] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.51.0" 2024-08-17T17:41:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:29 smithi179 ceph-mon[93519]: pgmap v1165: 105 pgs: 105 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.6 MiB/s wr, 46 op/s 2024-08-17T17:41:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:29 smithi179 ceph-mon[93519]: osdmap e960: 8 total, 8 up, 8 in 2024-08-17T17:41:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:41:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:41:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:41:30.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:30 smithi130 ceph-mon[105521]: osdmap e961: 8 total, 8 up, 8 in 2024-08-17T17:41:30.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:30 smithi130 ceph-mon[110068]: osdmap e961: 8 total, 8 up, 8 in 2024-08-17T17:41:31.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:30 smithi179 ceph-mon[93519]: osdmap e961: 8 total, 8 up, 8 in 2024-08-17T17:41:31.655 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateDiscard (3013 ms) 2024-08-17T17:41:31.655 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateTruncate 2024-08-17T17:41:31.655 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:41:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:31 smithi130 ceph-mon[105521]: pgmap v1168: 105 pgs: 105 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 36 op/s 2024-08-17T17:41:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:31 smithi130 ceph-mon[105521]: osdmap e962: 8 total, 8 up, 8 in 2024-08-17T17:41:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:31 smithi130 ceph-mon[110068]: pgmap v1168: 105 pgs: 105 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 36 op/s 2024-08-17T17:41:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:31 smithi130 ceph-mon[110068]: osdmap e962: 8 total, 8 up, 8 in 2024-08-17T17:41:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:31 smithi179 ceph-mon[93519]: pgmap v1168: 105 pgs: 105 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 36 op/s 2024-08-17T17:41:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:31 smithi179 ceph-mon[93519]: osdmap e962: 8 total, 8 up, 8 in 2024-08-17T17:41:32.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:32 smithi130 ceph-mon[105521]: osdmap e963: 8 total, 8 up, 8 in 2024-08-17T17:41:32.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:32 smithi130 ceph-mon[110068]: osdmap e963: 8 total, 8 up, 8 in 2024-08-17T17:41:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:32 smithi179 ceph-mon[93519]: osdmap e963: 8 total, 8 up, 8 in 2024-08-17T17:41:33.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:33 smithi130 ceph-mon[105521]: pgmap v1171: 105 pgs: 105 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 1023 B/s wr, 0 op/s 2024-08-17T17:41:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:33 smithi130 ceph-mon[105521]: osdmap e964: 8 total, 8 up, 8 in 2024-08-17T17:41:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:33 smithi130 ceph-mon[110068]: pgmap v1171: 105 pgs: 105 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 1023 B/s wr, 0 op/s 2024-08-17T17:41:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:33 smithi130 ceph-mon[110068]: osdmap e964: 8 total, 8 up, 8 in 2024-08-17T17:41:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:33 smithi179 ceph-mon[93519]: pgmap v1171: 105 pgs: 105 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 1023 B/s wr, 0 op/s 2024-08-17T17:41:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:33 smithi179 ceph-mon[93519]: osdmap e964: 8 total, 8 up, 8 in 2024-08-17T17:41:34.689 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateTruncate (3034 ms) 2024-08-17T17:41:34.689 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateWriteAndTruncate 2024-08-17T17:41:34.689 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:41:34.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:34 smithi130 ceph-mon[105521]: osdmap e965: 8 total, 8 up, 8 in 2024-08-17T17:41:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:34 smithi130 ceph-mon[110068]: osdmap e965: 8 total, 8 up, 8 in 2024-08-17T17:41:35.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:34 smithi179 ceph-mon[93519]: osdmap e965: 8 total, 8 up, 8 in 2024-08-17T17:41:35.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:35 smithi130 ceph-mon[105521]: pgmap v1174: 105 pgs: 105 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 140 KiB/s wr, 117 op/s 2024-08-17T17:41:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:35 smithi130 ceph-mon[105521]: osdmap e966: 8 total, 8 up, 8 in 2024-08-17T17:41:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:35 smithi130 ceph-mon[110068]: pgmap v1174: 105 pgs: 105 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 140 KiB/s wr, 117 op/s 2024-08-17T17:41:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:35 smithi130 ceph-mon[110068]: osdmap e966: 8 total, 8 up, 8 in 2024-08-17T17:41:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:35 smithi179 ceph-mon[93519]: pgmap v1174: 105 pgs: 105 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 140 KiB/s wr, 117 op/s 2024-08-17T17:41:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:35 smithi179 ceph-mon[93519]: osdmap e966: 8 total, 8 up, 8 in 2024-08-17T17:41:36.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:36 smithi130 ceph-mon[105521]: osdmap e967: 8 total, 8 up, 8 in 2024-08-17T17:41:36.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:36 smithi130 ceph-mon[110068]: osdmap e967: 8 total, 8 up, 8 in 2024-08-17T17:41:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:36 smithi179 ceph-mon[93519]: osdmap e967: 8 total, 8 up, 8 in 2024-08-17T17:41:37.710 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateWriteAndTruncate (3021 ms) 2024-08-17T17:41:37.710 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStress 2024-08-17T17:41:37.710 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:41:37.801 INFO:tasks.workunit.client.0.smithi130.stdout: i=0 exists [26710920~4271223,43236907~6810075,60928690~8608441,81338580~5704653,143625453~4824448,154147428~3368679,168233823~6973570,348774268~4706953] wrote [26710920~4271223,43236907~6810075,60928690~8608441,81338580~5704653,143625453~4824448,154147428~3368679,168233823~6973570,348774268~4706953] 2024-08-17T17:41:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:37 smithi179 ceph-mon[93519]: pgmap v1177: 105 pgs: 105 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 140 KiB/s wr, 117 op/s 2024-08-17T17:41:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:37 smithi179 ceph-mon[93519]: osdmap e968: 8 total, 8 up, 8 in 2024-08-17T17:41:38.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:37 smithi130 ceph-mon[105521]: pgmap v1177: 105 pgs: 105 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 140 KiB/s wr, 117 op/s 2024-08-17T17:41:38.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:37 smithi130 ceph-mon[105521]: osdmap e968: 8 total, 8 up, 8 in 2024-08-17T17:41:38.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:37 smithi130 ceph-mon[110068]: pgmap v1177: 105 pgs: 105 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s rd, 140 KiB/s wr, 117 op/s 2024-08-17T17:41:38.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:37 smithi130 ceph-mon[110068]: osdmap e968: 8 total, 8 up, 8 in 2024-08-17T17:41:38.743 INFO:tasks.workunit.client.0.smithi130.stdout: i=1 exists [26710920~4271223,43236907~6810075,60928690~8608441,81338580~5704653,99353810~286411,143625453~4824448,154147428~3368679,165161949~10691865,184955152~8026724,206579328~7352908,253511811~9996474,292092374~772196,335173878~7912963,348774268~4706953] wrote [64100320~418905,99353810~286411,165161949~4961337,170345186~5508628,184955152~8026724,206579328~7352908,253511811~9996474,292092374~772196,335173878~7912963] 2024-08-17T17:41:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:38 smithi179 ceph-mon[93519]: osdmap e969: 8 total, 8 up, 8 in 2024-08-17T17:41:39.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:38 smithi130 ceph-mon[105521]: osdmap e969: 8 total, 8 up, 8 in 2024-08-17T17:41:39.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:38 smithi130 ceph-mon[110068]: osdmap e969: 8 total, 8 up, 8 in 2024-08-17T17:41:39.760 INFO:tasks.workunit.client.0.smithi130.stdout: i=2 exists [26710920~4271223,35941013~6104604,43236907~10304140,60928690~10841291,81338580~5704653,99353810~286411,114947089~7807371,143625453~4824448,154147428~3368679,165161949~10691865,184955152~8026724,206579328~7352908,242260764~5125880,253511811~9996474,292092374~772196,293482393~6349736,332861425~10225416,348774268~4706953,356921972~5383594] wrote [27668422~1989503,35941013~6104604,47355868~6185179,64681659~7088322,114947089~7807371,242260764~5125880,293482393~6349736,332861425~4368751,356921972~5383594] 2024-08-17T17:41:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:39 smithi179 ceph-mon[93519]: pgmap v1180: 105 pgs: 105 active+clean; 389 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 11 MiB/s wr, 109 op/s 2024-08-17T17:41:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:39 smithi179 ceph-mon[93519]: osdmap e970: 8 total, 8 up, 8 in 2024-08-17T17:41:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:41:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:41:39] "GET /metrics HTTP/1.1" 200 36578 "" "Prometheus/2.51.0" 2024-08-17T17:41:40.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:39 smithi130 ceph-mon[110068]: pgmap v1180: 105 pgs: 105 active+clean; 389 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 11 MiB/s wr, 109 op/s 2024-08-17T17:41:40.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:39 smithi130 ceph-mon[110068]: osdmap e970: 8 total, 8 up, 8 in 2024-08-17T17:41:40.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:39 smithi130 ceph-mon[105521]: pgmap v1180: 105 pgs: 105 active+clean; 389 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 11 MiB/s wr, 109 op/s 2024-08-17T17:41:40.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:39 smithi130 ceph-mon[105521]: osdmap e970: 8 total, 8 up, 8 in 2024-08-17T17:41:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:41:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:41:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:41:40.761 INFO:tasks.workunit.client.0.smithi130.stdout: i=3 exists [26710920~4271223,35941013~17600034,60928690~10841291,81338580~5704653,87479775~2420950,99353810~286411,114947089~7807371,143625453~4824448,154147428~5507727,165161949~10691865,184955152~8026724,206579328~7352908,242260764~5125880,253511811~9996474,292092374~772196,293482393~6349736,332861425~10225416,348774268~4706953,354460657~8143176,375999115~10637313] wrote [38674135~5744539,87479775~2420950,155052722~4602433,187244557~3088494,206785372~4793572,335970101~5471703,354460657~8143176,375999115~10637313] 2024-08-17T17:41:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:40 smithi179 ceph-mon[93519]: osdmap e971: 8 total, 8 up, 8 in 2024-08-17T17:41:41.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:40 smithi130 ceph-mon[105521]: osdmap e971: 8 total, 8 up, 8 in 2024-08-17T17:41:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:40 smithi130 ceph-mon[110068]: osdmap e971: 8 total, 8 up, 8 in 2024-08-17T17:41:41.768 INFO:tasks.workunit.client.0.smithi130.stdout: i=4 exists [26710920~4271223,35941013~17600034,60928690~10841291,81338580~5704653,87479775~2420950,99353810~286411,110470753~244936,114947089~7807371,143625453~4824448,153770951~5884204,165161949~10691865,177013685~1267197,184955152~8026724,206579328~9350837,220784358~5545290,228066662~7161224,242260764~5125880,253511811~9996474,292092374~772196,293482393~6349736,332861425~10225416,348774268~4706953,354460657~8143176,368552586~254797,372708801~13927627] wrote [64130590~509246,83855009~122571,110470753~244936,153770951~5448131,177013685~1267197,211815065~4115100,220784358~5545290,228066662~7161224,368552586~254797,372708801~6982505] 2024-08-17T17:41:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:41 smithi179 ceph-mon[93519]: pgmap v1183: 105 pgs: 105 active+clean; 392 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 12 MiB/s wr, 110 op/s 2024-08-17T17:41:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:41 smithi179 ceph-mon[93519]: osdmap e972: 8 total, 8 up, 8 in 2024-08-17T17:41:42.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:41 smithi130 ceph-mon[105521]: pgmap v1183: 105 pgs: 105 active+clean; 392 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 12 MiB/s wr, 110 op/s 2024-08-17T17:41:42.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:41 smithi130 ceph-mon[105521]: osdmap e972: 8 total, 8 up, 8 in 2024-08-17T17:41:42.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:41 smithi130 ceph-mon[110068]: pgmap v1183: 105 pgs: 105 active+clean; 392 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s rd, 12 MiB/s wr, 110 op/s 2024-08-17T17:41:42.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:41 smithi130 ceph-mon[110068]: osdmap e972: 8 total, 8 up, 8 in 2024-08-17T17:41:42.784 INFO:tasks.workunit.client.0.smithi130.stdout: i=5 exists [24616983~6365160,35941013~17600034,60928690~10841291,74007760~333146,81338580~5704653,87479775~2420950,99353810~286411,105261990~3433993,110470753~244936,114947089~7807371,143625453~4824448,151953600~842106,153770951~5884204,165161949~10691865,177013685~1267197,184955152~8026724,206579328~9350837,220784358~5545290,228066662~7161224,242260764~5125880,253511811~9996474,291073215~751556,292092374~772196,293482393~6349736,332649487~12754526,348774268~4706953,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540] wrote [24616983~5736360,74007760~333146,105261990~3433993,144750991~1159758,151953600~842106,242776912~2064207,291073215~751556,332649487~12754526,388486096~342540] 2024-08-17T17:41:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:42 smithi179 ceph-mon[93519]: osdmap e973: 8 total, 8 up, 8 in 2024-08-17T17:41:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:43.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:42 smithi130 ceph-mon[105521]: osdmap e973: 8 total, 8 up, 8 in 2024-08-17T17:41:43.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:42 smithi130 ceph-mon[110068]: osdmap e973: 8 total, 8 up, 8 in 2024-08-17T17:41:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:43.806 INFO:tasks.workunit.client.0.smithi130.stdout: i=6 exists [24616983~6365160,35941013~17600034,55859939~779155,60928690~10841291,74007760~333146,81338580~5704653,87479775~2420950,99353810~286411,105261990~3433993,110470753~244936,114947089~7807371,143625453~4824448,151953600~842106,153770951~5884204,165161949~10691865,177013685~1267197,184955152~15976866,206579328~9350837,220784358~5545290,228066662~7161224,239723295~7920536,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,332649487~20831734,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540] wrote [55859939~779155,189053332~11878686,229534798~3451158,239723295~7920536,269763691~4981986,342747177~8963796,355907560~4087411] 2024-08-17T17:41:43.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:43 smithi179 ceph-mon[93519]: pgmap v1186: 105 pgs: 105 active+clean; 423 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 99 KiB/s rd, 8.3 MiB/s wr, 163 op/s 2024-08-17T17:41:43.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:43 smithi179 ceph-mon[93519]: osdmap e974: 8 total, 8 up, 8 in 2024-08-17T17:41:44.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:43 smithi130 ceph-mon[105521]: pgmap v1186: 105 pgs: 105 active+clean; 423 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 99 KiB/s rd, 8.3 MiB/s wr, 163 op/s 2024-08-17T17:41:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:43 smithi130 ceph-mon[105521]: osdmap e974: 8 total, 8 up, 8 in 2024-08-17T17:41:44.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:43 smithi130 ceph-mon[110068]: pgmap v1186: 105 pgs: 105 active+clean; 423 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 99 KiB/s rd, 8.3 MiB/s wr, 163 op/s 2024-08-17T17:41:44.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:43 smithi130 ceph-mon[110068]: osdmap e974: 8 total, 8 up, 8 in 2024-08-17T17:41:44.851 INFO:tasks.workunit.client.0.smithi130.stdout: i=7 exists [24616983~6365160,35941013~17600034,55859939~2311241,60928690~10841291,74007760~333146,81338580~5704653,87479775~2420950,99353810~286411,105261990~3433993,110055987~12698473,143625453~4824448,151953600~842106,153770951~5884204,165161949~10691865,177013685~1267197,184955152~15976866,206579328~9350837,220784358~5545290,228066662~7161224,239723295~7920536,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,332649487~20831734,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540,395994076~4377232,407841230~7367153] wrote [36062054~6554620,56464863~1706317,82296588~1267250,110055987~6404653,333022625~6072123,373115509~11964022,395994076~4377232,407841230~7367153] 2024-08-17T17:41:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:44 smithi179 ceph-mon[93519]: osdmap e975: 8 total, 8 up, 8 in 2024-08-17T17:41:45.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:44 smithi130 ceph-mon[105521]: osdmap e975: 8 total, 8 up, 8 in 2024-08-17T17:41:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:44 smithi130 ceph-mon[110068]: osdmap e975: 8 total, 8 up, 8 in 2024-08-17T17:41:45.786 INFO:tasks.workunit.client.0.smithi130.stdout: i=8 exists [12311950~2412776,24616983~6365160,31503479~1175130,35941013~17600034,55859939~2311241,60928690~10841291,74007760~333146,81338580~5704653,87479775~2420950,90175397~5338809,99353810~286411,105261990~3433993,110055987~12698473,128194814~7129965,137226312~11223589,151953600~842106,153770951~5884204,165161949~10691865,177013685~3249038,184955152~15976866,206579328~9350837,220784358~5545290,228066662~7161224,239723295~7920536,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~4377232,407841230~7367153] wrote [12311950~2412776,31503479~1175130,90175397~5338809,128194814~7129965,137226312~7668218,177491177~2771546,322650436~784308,350982941~4053211,356919771~997495,385791049~1661831] 2024-08-17T17:41:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:45 smithi179 ceph-mon[93519]: pgmap v1189: 105 pgs: 105 active+clean; 621 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 144 KiB/s rd, 47 MiB/s wr, 242 op/s 2024-08-17T17:41:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:45 smithi179 ceph-mon[93519]: osdmap e976: 8 total, 8 up, 8 in 2024-08-17T17:41:46.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:45 smithi130 ceph-mon[105521]: pgmap v1189: 105 pgs: 105 active+clean; 621 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 144 KiB/s rd, 47 MiB/s wr, 242 op/s 2024-08-17T17:41:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:45 smithi130 ceph-mon[105521]: osdmap e976: 8 total, 8 up, 8 in 2024-08-17T17:41:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:45 smithi130 ceph-mon[110068]: pgmap v1189: 105 pgs: 105 active+clean; 621 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 144 KiB/s rd, 47 MiB/s wr, 242 op/s 2024-08-17T17:41:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:45 smithi130 ceph-mon[110068]: osdmap e976: 8 total, 8 up, 8 in 2024-08-17T17:41:46.853 INFO:tasks.workunit.client.0.smithi130.stdout: i=9 exists [12311950~2412776,24616983~6365160,31503479~1175130,35941013~17600034,55859939~2311241,60928690~10841291,74007760~333146,81338580~5704653,87479775~2420950,90175397~5338809,99353810~286411,105261990~3433993,110055987~12698473,127412771~21037130,151953600~842106,153770951~8826886,165161949~10691865,177013685~3249038,184955152~19344069,206579328~9350837,220784358~5545290,228066662~7161224,239723295~12307662,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] wrote [68742908~1813321,111220774~5205171,127412771~15004337,155757889~6839948,197702108~6597113,244002673~773270,247070758~4960199,312958845~2265984,400335542~7506643] 2024-08-17T17:41:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:46 smithi179 ceph-mon[93519]: osdmap e977: 8 total, 8 up, 8 in 2024-08-17T17:41:47.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:46 smithi130 ceph-mon[105521]: osdmap e977: 8 total, 8 up, 8 in 2024-08-17T17:41:47.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:46 smithi130 ceph-mon[110068]: osdmap e977: 8 total, 8 up, 8 in 2024-08-17T17:41:47.833 INFO:tasks.workunit.client.0.smithi130.stdout: i=10 exists [12311950~2412776,24616983~6365160,31503479~1175130,35941013~19477203,55859939~2311241,58199577~13570404,74007760~333146,75676394~14224331,90175397~5338809,99353810~286411,105261990~3433993,110055987~12698473,127412771~21883557,151953600~842106,153770951~8826886,165161949~10691865,177013685~3249038,184955152~19344069,206579328~9350837,220784358~5545290,228066662~7161224,239723295~12307662,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] wrote [52172688~3245528,58199577~3721268,75676394~6230067,85296870~4252304,141229321~8067007,155540399~1674274,166058561~2977181,240858522~5051700,272182852~2470550] 2024-08-17T17:41:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:47 smithi179 ceph-mon[93519]: pgmap v1192: 105 pgs: 105 active+clean; 621 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 45 KiB/s rd, 39 MiB/s wr, 79 op/s 2024-08-17T17:41:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:47 smithi179 ceph-mon[93519]: osdmap e978: 8 total, 8 up, 8 in 2024-08-17T17:41:48.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:47 smithi130 ceph-mon[105521]: pgmap v1192: 105 pgs: 105 active+clean; 621 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 45 KiB/s rd, 39 MiB/s wr, 79 op/s 2024-08-17T17:41:48.185 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:47 smithi130 ceph-mon[105521]: osdmap e978: 8 total, 8 up, 8 in 2024-08-17T17:41:48.185 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:47 smithi130 ceph-mon[110068]: pgmap v1192: 105 pgs: 105 active+clean; 621 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 45 KiB/s rd, 39 MiB/s wr, 79 op/s 2024-08-17T17:41:48.186 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:47 smithi130 ceph-mon[110068]: osdmap e978: 8 total, 8 up, 8 in 2024-08-17T17:41:48.845 INFO:tasks.workunit.client.0.smithi130.stdout: i=11 exists [12311950~6523919,24616983~6365160,31503479~1175130,33421453~21996763,55859939~15910042,74007760~333146,75676394~14224331,90175397~5338809,99353810~286411,105261990~3433993,110055987~12698473,127412771~21883557,151953600~842106,153770951~8826886,165161949~10691865,177013685~3249038,184955152~19344069,206579328~9350837,220784358~5850305,228066662~7161224,238191726~13839231,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] wrote [13040607~5795262,33421453~10210967,58014695~4138528,165571375~3062744,212123767~2607614,224516274~2118389,238191726~4859035,409521161~4155949] 2024-08-17T17:41:49.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:48 smithi179 ceph-mon[93519]: osdmap e979: 8 total, 8 up, 8 in 2024-08-17T17:41:49.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:48 smithi130 ceph-mon[105521]: osdmap e979: 8 total, 8 up, 8 in 2024-08-17T17:41:49.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:48 smithi130 ceph-mon[110068]: osdmap e979: 8 total, 8 up, 8 in 2024-08-17T17:41:49.787 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:49 smithi130 ceph-mon[110068]: pgmap v1195: 105 pgs: 105 active+clean; 819 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 185 KiB/s rd, 47 MiB/s wr, 298 op/s 2024-08-17T17:41:49.787 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:49 smithi130 ceph-mon[110068]: osdmap e980: 8 total, 8 up, 8 in 2024-08-17T17:41:49.787 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:49 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:41:49.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:49 smithi130 ceph-mon[105521]: pgmap v1195: 105 pgs: 105 active+clean; 819 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 185 KiB/s rd, 47 MiB/s wr, 298 op/s 2024-08-17T17:41:49.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:49 smithi130 ceph-mon[105521]: osdmap e980: 8 total, 8 up, 8 in 2024-08-17T17:41:49.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:49 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:41:49.819 INFO:tasks.workunit.client.0.smithi130.stdout: i=12 exists [12311950~6523919,24616983~6365160,31503479~1175130,33421453~21996763,55859939~15910042,74007760~333146,75676394~14224331,90175397~5338809,99353810~286411,101692190~7003793,110055987~12698473,127412771~21883557,151953600~842106,153615532~8982305,165161949~10691865,177013685~3249038,184955152~19344069,206579328~9350837,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,390263701~1637045,395994076~19214307] wrote [60052586~1275610,101692190~4549633,110253798~3900891,153615532~3715116,217652007~2893566,228885435~3355214,245682731~2511565,354087269~4254759,390263701~1637045,404079964~7145520] 2024-08-17T17:41:49.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:49 smithi179 ceph-mon[93519]: pgmap v1195: 105 pgs: 105 active+clean; 819 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 185 KiB/s rd, 47 MiB/s wr, 298 op/s 2024-08-17T17:41:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:49 smithi179 ceph-mon[93519]: osdmap e980: 8 total, 8 up, 8 in 2024-08-17T17:41:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:49 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:41:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:41:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:41:49] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-08-17T17:41:50.474 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:41:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:41:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:41:50.824 INFO:tasks.workunit.client.0.smithi130.stdout: i=13 exists [12311950~6523919,24616983~6365160,31503479~1175130,33421453~21996763,55859939~15910042,74007760~333146,75676394~14224331,90097398~5457279,99353810~9342173,110055987~12698473,127412771~21883557,151953600~842106,153615532~8982305,165161949~10691865,177013685~3765470,184955152~19344069,206579328~9350837,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,252773028~10735257,269763691~4981986,291073215~751556,292092374~772196,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,390263701~1637045,395994076~19214307] wrote [41118839~7378600,90097398~5457279,99526650~2264727,160518828~285849,167084727~8167846,178411768~2367387,252773028~8082862,294440898~6981222] 2024-08-17T17:41:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:50 smithi130 ceph-mon[105521]: osdmap e981: 8 total, 8 up, 8 in 2024-08-17T17:41:50.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:50 smithi130 ceph-mon[110068]: osdmap e981: 8 total, 8 up, 8 in 2024-08-17T17:41:51.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:50 smithi179 ceph-mon[93519]: osdmap e981: 8 total, 8 up, 8 in 2024-08-17T17:41:51.845 INFO:tasks.workunit.client.0.smithi130.stdout: i=14 exists [12311950~6523919,24616983~6365160,31503479~1175130,33421453~38348528,74007760~333146,75676394~14224331,90097398~32657062,127412771~21883557,151953600~842106,153615532~8982305,165161949~10691865,177013685~3765470,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~13839231,252773028~10735257,269763691~4981986,291073215~751556,292092374~772196,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~19425483,395994076~19214307] wrote [41388025~4298022,52330412~7185084,92352943~7343768,108486098~2678206,118553273~1355301,133306049~7681407,218372862~7495873,261307577~1507943,354913938~909027,384785269~7349015] 2024-08-17T17:41:52.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:51 smithi130 ceph-mon[105521]: pgmap v1198: 105 pgs: 105 active+clean; 843 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 185 KiB/s rd, 54 MiB/s wr, 301 op/s 2024-08-17T17:41:52.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:51 smithi130 ceph-mon[105521]: osdmap e982: 8 total, 8 up, 8 in 2024-08-17T17:41:52.019 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:51 smithi130 ceph-mon[110068]: pgmap v1198: 105 pgs: 105 active+clean; 843 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 185 KiB/s rd, 54 MiB/s wr, 301 op/s 2024-08-17T17:41:52.019 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:51 smithi130 ceph-mon[110068]: osdmap e982: 8 total, 8 up, 8 in 2024-08-17T17:41:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:51 smithi179 ceph-mon[93519]: pgmap v1198: 105 pgs: 105 active+clean; 843 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 185 KiB/s rd, 54 MiB/s wr, 301 op/s 2024-08-17T17:41:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:51 smithi179 ceph-mon[93519]: osdmap e982: 8 total, 8 up, 8 in 2024-08-17T17:41:52.835 INFO:tasks.workunit.client.0.smithi130.stdout: i=15 exists [6122489~2117206,9582259~1513694,12311950~18670193,31503479~1175130,33421453~38348528,74007760~333146,75676394~14224331,90097398~32657062,127412771~21883557,150192064~2603642,153615532~8982305,165161949~10691865,177013685~3765470,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~13839231,252773028~10735257,265186950~1824474,269763691~4981986,291073215~751556,292092374~772196,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,366925156~3770110,372708801~19425483,395994076~19214307] wrote [6122489~2117206,9582259~1513694,16874234~8111475,150192064~2291333,191934227~5441303,239310228~2700,265186950~1824474,348569567~2746233,366925156~3770110] 2024-08-17T17:41:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:52 smithi179 ceph-mon[93519]: osdmap e983: 8 total, 8 up, 8 in 2024-08-17T17:41:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:52 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:52 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:53.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:52 smithi130 ceph-mon[105521]: osdmap e983: 8 total, 8 up, 8 in 2024-08-17T17:41:53.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:52 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:53.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:52 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:52 smithi130 ceph-mon[110068]: osdmap e983: 8 total, 8 up, 8 in 2024-08-17T17:41:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:52 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:52 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:53.841 INFO:tasks.workunit.client.0.smithi130.stdout: i=16 exists [6122489~2117206,9582259~1513694,12311950~18670193,31503479~1175130,33421453~38348528,74007760~333146,75676394~14224331,90097398~32657062,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~13839231,252773028~10735257,265186950~1824474,269763691~8777870,291073215~751556,292092374~772196,293482393~7939727,312958845~2265984,322650436~784308,330570284~32033549,366925156~3770110,372708801~19425483,393360140~1439083,395994076~19214307] wrote [118030408~3183729,149924380~5105518,166825307~2678998,172403776~5654992,208255148~1681884,247539592~3515878,274541285~4000276,330570284~6844117,393360140~1439083,403188267~3969107] 2024-08-17T17:41:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:53 smithi179 ceph-mon[93519]: pgmap v1201: 105 pgs: 105 active+clean; 863 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 11 MiB/s wr, 193 op/s 2024-08-17T17:41:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:53 smithi179 ceph-mon[93519]: osdmap e984: 8 total, 8 up, 8 in 2024-08-17T17:41:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:53 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:53 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:53 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:41:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:53 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:41:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:53 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:54.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:53 smithi130 ceph-mon[110068]: pgmap v1201: 105 pgs: 105 active+clean; 863 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 11 MiB/s wr, 193 op/s 2024-08-17T17:41:54.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:53 smithi130 ceph-mon[110068]: osdmap e984: 8 total, 8 up, 8 in 2024-08-17T17:41:54.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:53 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:54.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:53 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:54.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:53 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:41:54.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:53 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:41:54.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:53 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:54.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:53 smithi130 ceph-mon[105521]: pgmap v1201: 105 pgs: 105 active+clean; 863 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 11 MiB/s wr, 193 op/s 2024-08-17T17:41:54.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:53 smithi130 ceph-mon[105521]: osdmap e984: 8 total, 8 up, 8 in 2024-08-17T17:41:54.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:53 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:54.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:53 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:54.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:53 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:41:54.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:53 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:41:54.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:53 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:41:54.917 INFO:tasks.workunit.client.0.smithi130.stdout: i=17 exists [6122489~2117206,8405456~22576687,31503479~1175130,33421453~38348528,72099906~2964160,75676394~14224331,90097398~32657062,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~13839231,252773028~10735257,265186950~1824474,269763691~8777870,291073215~751556,292092374~772196,293482393~7939727,312958845~6612414,322650436~784308,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] wrote [8405456~6091738,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,246931942~2452483,273290717~2471737,315124553~4446706,330928129~4964895,363159108~5625694] 2024-08-17T17:41:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:54 smithi179 ceph-mon[93519]: osdmap e985: 8 total, 8 up, 8 in 2024-08-17T17:41:55.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:54 smithi130 ceph-mon[105521]: osdmap e985: 8 total, 8 up, 8 in 2024-08-17T17:41:55.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:54 smithi130 ceph-mon[110068]: osdmap e985: 8 total, 8 up, 8 in 2024-08-17T17:41:55.890 INFO:tasks.workunit.client.0.smithi130.stdout: i=18 exists [6122489~2117206,8405456~24273153,33421453~38348528,72099906~2964160,75676394~14224331,90097398~32657062,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~7939727,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] wrote [30955299~1249356,152404635~3174744,210075422~3861992,248466245~7857272,266411827~7611252,286246187~4813712,302649343~6221358,310725355~1663245,315345966~5908629] 2024-08-17T17:41:56.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:55 smithi179 ceph-mon[93519]: pgmap v1204: 105 pgs: 105 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 227 KiB/s rd, 46 MiB/s wr, 358 op/s 2024-08-17T17:41:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:55 smithi179 ceph-mon[93519]: osdmap e986: 8 total, 8 up, 8 in 2024-08-17T17:41:56.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:55 smithi130 ceph-mon[105521]: pgmap v1204: 105 pgs: 105 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 227 KiB/s rd, 46 MiB/s wr, 358 op/s 2024-08-17T17:41:56.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:55 smithi130 ceph-mon[105521]: osdmap e986: 8 total, 8 up, 8 in 2024-08-17T17:41:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:55 smithi130 ceph-mon[110068]: pgmap v1204: 105 pgs: 105 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 227 KiB/s rd, 46 MiB/s wr, 358 op/s 2024-08-17T17:41:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:55 smithi130 ceph-mon[110068]: osdmap e986: 8 total, 8 up, 8 in 2024-08-17T17:41:56.923 INFO:tasks.workunit.client.0.smithi130.stdout: i=19 exists [1297677~31380932,33421453~38348528,72099906~2964160,75676394~47078066,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] wrote [1297677~6200185,7784300~5890796,30413398~2257211,81427530~881449,84600711~5606478,300835157~7348018,309044486~5067705,322065070~4478831,338106187~5516501] 2024-08-17T17:41:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:56 smithi179 ceph-mon[93519]: osdmap e987: 8 total, 8 up, 8 in 2024-08-17T17:41:57.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:56 smithi130 ceph-mon[105521]: osdmap e987: 8 total, 8 up, 8 in 2024-08-17T17:41:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:56 smithi130 ceph-mon[110068]: osdmap e987: 8 total, 8 up, 8 in 2024-08-17T17:41:57.821 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 1 diff [64100320~418905,99353810~286411,165161949~4961337,170345186~5508628,184955152~8026724,206579328~7352908,253511811~9996474,292092374~772196,335173878~7912963] 2024-08-17T17:41:57.821 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [64100320~418905,99353810~286411,165161949~4961337,170345186~5508628,184955152~8026724,206579328~7352908,253511811~9996474,292092374~772196,335173878~7912963] 2024-08-17T17:41:57.838 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [64100320~418905,96468992~3171229,163577856~6545430,170345186~5508628,184549376~8432500,205520896~8411340,251658240~11850045,289406976~3457594,331350016~11736825] 2024-08-17T17:41:57.838 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 2 diff [27668422~1989503,35941013~6104604,47355868~6185179,64100320~418905,64681659~7088322,99353810~286411,114947089~7807371,165161949~4961337,170345186~5508628,184955152~8026724,206579328~7352908,242260764~5125880,253511811~9996474,292092374~772196,293482393~6349736,332861425~10225416,356921972~5383594] 2024-08-17T17:41:57.838 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [27668422~1989503,35941013~6104604,47355868~6185179,64100320~418905,64681659~7088322,99353810~286411,114947089~7807371,165161949~4961337,170345186~5508628,184955152~8026724,206579328~7352908,242260764~5125880,253511811~9996474,292092374~772196,293482393~6349736,332861425~10225416,356921972~5383594] 2024-08-17T17:41:57.844 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [27668422~1989503,33554432~8491185,47355868~6185179,64100320~418905,64681659~7088322,96468992~3171229,113246208~9508252,163577856~6545430,170345186~5508628,184549376~8432500,205520896~8411340,239075328~8311316,251658240~11850045,289406976~10425153,331350016~11736825,356515840~5789726] 2024-08-17T17:41:57.844 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 3 diff [27668422~1989503,35941013~8477661,47355868~6185179,64100320~418905,64681659~7088322,87479775~2420950,99353810~286411,114947089~7807371,155052722~4602433,165161949~4961337,170345186~5508628,184955152~8026724,206579328~7352908,242260764~5125880,253511811~9996474,292092374~772196,293482393~6349736,332861425~10225416,354460657~8143176,375999115~10637313] 2024-08-17T17:41:57.844 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [27668422~1989503,35941013~8477661,47355868~6185179,64100320~418905,64681659~7088322,87479775~2420950,99353810~286411,114947089~7807371,155052722~4602433,165161949~4961337,170345186~5508628,184955152~8026724,206579328~7352908,242260764~5125880,253511811~9996474,292092374~772196,293482393~6349736,332861425~10225416,354460657~8143176,375999115~10637313] 2024-08-17T17:41:57.850 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [27668422~1989503,33554432~10864242,47355868~6185179,64100320~418905,64681659~7088322,87043233~2857492,96468992~3171229,113246208~9508252,155052722~4602433,163577856~6545430,170345186~5508628,184549376~8432500,205520896~8411340,239075328~8311316,251658240~11850045,289406976~10425153,331350016~11736825,353481221~9122612,373293056~13343372] 2024-08-17T17:41:57.850 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 4 diff [27668422~1989503,35941013~8477661,47355868~6185179,64100320~539516,64681659~7088322,83855009~122571,87479775~2420950,99353810~286411,110470753~244936,114947089~7807371,153770951~5884204,165161949~4961337,170345186~5508628,177013685~1267197,184955152~8026724,206579328~9350837,220784358~5545290,228066662~7161224,242260764~5125880,253511811~9996474,292092374~772196,293482393~6349736,332861425~10225416,354460657~8143176,368552586~254797,372708801~13927627] 2024-08-17T17:41:57.850 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [27668422~1989503,35941013~8477661,47355868~6185179,64100320~539516,64681659~7088322,83855009~122571,87479775~2420950,99353810~286411,110470753~244936,114947089~7807371,153770951~5884204,165161949~4961337,170345186~5508628,177013685~1267197,184955152~8026724,206579328~9350837,220784358~5545290,228066662~7161224,242260764~5125880,253511811~9996474,292092374~772196,293482393~6349736,332861425~10225416,354460657~8143176,368552586~254797,372708801~13927627] 2024-08-17T17:41:57.857 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [27668422~1989503,33554432~10864242,47355868~6185179,64100320~539516,64681659~7088322,83855009~122571,87043233~2857492,96468992~3171229,109051904~1663785,113246208~9508252,153770951~5884204,163577856~6545430,170345186~5508628,176160768~2120114,184549376~8432500,205520896~10409269,218103808~8225840,226492416~8735470,239075328~8311316,251658240~11850045,289406976~10425153,331350016~11736825,353481221~9122612,364904448~3902935,369098752~17537676] 2024-08-17T17:41:57.857 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 5 diff [24616983~5736360,35941013~8477661,47355868~6185179,64100320~539516,64681659~7088322,74007760~333146,83855009~122571,87479775~2420950,99353810~286411,105261990~3433993,110470753~244936,114947089~7807371,144750991~1159758,151953600~842106,153770951~5884204,165161949~4961337,170345186~5508628,177013685~1267197,184955152~8026724,206579328~9350837,220784358~5545290,228066662~7161224,242260764~5125880,253511811~9996474,291073215~751556,292092374~772196,293482393~6349736,332649487~12754526,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540] 2024-08-17T17:41:57.857 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,35941013~8477661,47355868~6185179,64100320~539516,64681659~7088322,74007760~333146,83855009~122571,87479775~2420950,99353810~286411,105261990~3433993,110470753~244936,114947089~7807371,144750991~1159758,151953600~842106,153770951~5884204,165161949~4961337,170345186~5508628,177013685~1267197,184955152~8026724,206579328~9350837,220784358~5545290,228066662~7161224,242260764~5125880,253511811~9996474,291073215~751556,292092374~772196,293482393~6349736,332649487~12754526,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540] 2024-08-17T17:41:57.863 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,33554432~10864242,47355868~6185179,64100320~539516,64681659~9659247,83855009~122571,87043233~2857492,96468992~3171229,104857600~3838383,109051904~1663785,113246208~9508252,144750991~1159758,151953600~842106,153770951~5884204,163577856~6545430,170345186~5508628,176160768~2120114,184549376~8432500,205520896~10409269,218103808~8225840,226492416~8735470,239075328~8311316,251658240~11850045,289406976~10425153,331350016~14053997,353481221~9122612,364904448~3902935,369098752~19729884] 2024-08-17T17:41:57.863 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 6 diff [24616983~5736360,35941013~8477661,47355868~6185179,55859939~779155,64100320~539516,64681659~7088322,74007760~333146,83855009~122571,87479775~2420950,99353810~286411,105261990~3433993,110470753~244936,114947089~7807371,144750991~1159758,151953600~842106,153770951~5884204,165161949~4961337,170345186~5508628,177013685~1267197,184955152~15976866,206579328~9350837,220784358~5545290,228066662~7161224,239723295~7920536,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,332649487~19061486,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540] 2024-08-17T17:41:57.863 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,35941013~8477661,47355868~6185179,55859939~779155,64100320~539516,64681659~7088322,74007760~333146,83855009~122571,87479775~2420950,99353810~286411,105261990~3433993,110470753~244936,114947089~7807371,144750991~1159758,151953600~842106,153770951~5884204,165161949~4961337,170345186~5508628,177013685~1267197,184955152~15976866,206579328~9350837,220784358~5545290,228066662~7161224,239723295~7920536,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,332649487~19061486,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540] 2024-08-17T17:41:57.868 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,33554432~10864242,47355868~6185179,54525952~2113142,64100320~539516,64681659~9659247,83855009~122571,87043233~2857492,96468992~3171229,104857600~3838383,109051904~1663785,113246208~9508252,144750991~1159758,151953600~842106,153770951~5884204,163577856~6545430,170345186~5508628,176160768~2120114,184549376~16382642,205520896~10409269,218103808~8225840,226492416~8735470,239075328~8568503,251658240~11850045,268435456~6310221,289406976~10425153,331350016~20360957,353481221~9122612,364904448~3902935,369098752~19729884] 2024-08-17T17:41:57.868 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 7 diff [24616983~5736360,35941013~8477661,47355868~6185179,55859939~2311241,64100320~539516,64681659~7088322,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,99353810~286411,105261990~3433993,110055987~12698473,144750991~1159758,151953600~842106,153770951~5884204,165161949~4961337,170345186~5508628,177013685~1267197,184955152~15976866,206579328~9350837,220784358~5545290,228066662~7161224,239723295~7920536,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,332649487~19061486,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:57.868 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,35941013~8477661,47355868~6185179,55859939~2311241,64100320~539516,64681659~7088322,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,99353810~286411,105261990~3433993,110055987~12698473,144750991~1159758,151953600~842106,153770951~5884204,165161949~4961337,170345186~5508628,177013685~1267197,184955152~15976866,206579328~9350837,220784358~5545290,228066662~7161224,239723295~7920536,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,332649487~19061486,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:57.874 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,33554432~10864242,47355868~6185179,54525952~3645228,64100320~539516,64681659~9659247,82296588~1267250,83855009~122571,87043233~2857492,96468992~3171229,104857600~3838383,109051904~13702556,144750991~1159758,151953600~842106,153770951~5884204,163577856~6545430,170345186~5508628,176160768~2120114,184549376~16382642,205520896~10409269,218103808~8225840,226492416~8735470,239075328~8568503,251658240~11850045,268435456~6310221,289406976~10425153,331350016~20360957,353481221~9122612,364904448~3902935,369098752~19729884,394264576~6106732,406847488~8360895] 2024-08-17T17:41:57.874 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 8 diff [12311950~2412776,24616983~5736360,31503479~1175130,35941013~8477661,47355868~6185179,55859939~2311241,64100320~539516,64681659~7088322,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,90175397~5338809,99353810~286411,105261990~3433993,110055987~12698473,128194814~7129965,137226312~8684437,151953600~842106,153770951~5884204,165161949~4961337,170345186~5508628,177013685~3249038,184955152~15976866,206579328~9350837,220784358~5545290,228066662~7161224,239723295~7920536,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:57.874 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,35941013~8477661,47355868~6185179,55859939~2311241,64100320~539516,64681659~7088322,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,90175397~5338809,99353810~286411,105261990~3433993,110055987~12698473,128194814~7129965,137226312~8684437,151953600~842106,153770951~5884204,165161949~4961337,170345186~5508628,177013685~3249038,184955152~15976866,206579328~9350837,220784358~5545290,228066662~7161224,239723295~7920536,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:57.880 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,33554432~10864242,47355868~6185179,54525952~3645228,64100320~539516,64681659~9659247,82296588~1267250,83855009~122571,87043233~8470973,96468992~3171229,104857600~3838383,109051904~13702556,125829120~20081629,151953600~842106,153770951~5884204,163577856~6545430,170345186~5508628,176160768~4101955,184549376~16382642,205520896~10409269,218103808~8225840,226492416~8735470,239075328~8568503,251658240~11850045,268435456~6310221,289406976~10425153,318767104~4667640,331350016~31253817,364904448~3902935,369098752~19729884,394264576~6106732,406847488~8360895] 2024-08-17T17:41:57.880 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 9 diff [12311950~2412776,24616983~5736360,31503479~1175130,35941013~8477661,47355868~6185179,55859939~2311241,64100320~539516,64681659~7088322,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,90175397~5338809,99353810~286411,105261990~3433993,110055987~12698473,127412771~18497978,151953600~842106,153770951~8826886,165161949~4961337,170345186~5508628,177013685~3249038,184955152~19344069,206579328~9350837,220784358~5545290,228066662~7161224,239723295~12307662,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:57.880 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,35941013~8477661,47355868~6185179,55859939~2311241,64100320~539516,64681659~7088322,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,90175397~5338809,99353810~286411,105261990~3433993,110055987~12698473,127412771~18497978,151953600~842106,153770951~8826886,165161949~4961337,170345186~5508628,177013685~3249038,184955152~19344069,206579328~9350837,220784358~5545290,228066662~7161224,239723295~12307662,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:57.886 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,33554432~10864242,47355868~6185179,54525952~3645228,64100320~539516,64681659~9659247,82296588~1267250,83855009~122571,87043233~8470973,96468992~3171229,104857600~3838383,109051904~13702556,125829120~20081629,151953600~842106,153770951~8826886,163577856~6545430,170345186~5508628,176160768~4101955,184549376~19749845,205520896~10409269,218103808~8225840,226492416~8735470,239075328~24432957,268435456~6310221,289406976~10425153,310378496~4846333,318767104~4667640,331350016~31253817,364904448~3902935,369098752~19729884,394264576~20943807] 2024-08-17T17:41:57.886 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 10 diff [12311950~2412776,24616983~5736360,31503479~1175130,35941013~8477661,47355868~8062348,55859939~2311241,58199577~3721268,64100320~539516,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,99353810~286411,105261990~3433993,110055987~12698473,127412771~21883557,151953600~842106,153770951~8826886,165161949~4961337,170345186~5508628,177013685~3249038,184955152~19344069,206579328~9350837,220784358~5545290,228066662~7161224,239723295~12307662,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:57.886 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,35941013~8477661,47355868~8062348,55859939~2311241,58199577~3721268,64100320~539516,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,99353810~286411,105261990~3433993,110055987~12698473,127412771~21883557,151953600~842106,153770951~8826886,165161949~4961337,170345186~5508628,177013685~3249038,184955152~19344069,206579328~9350837,220784358~5545290,228066662~7161224,239723295~12307662,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:57.892 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,33554432~10864242,47355868~14564977,64100320~539516,64681659~9659247,75497472~6408989,82296588~1267250,83855009~122571,85296870~10217336,96468992~3171229,104857600~3838383,109051904~13702556,125829120~23467208,151953600~842106,153770951~8826886,163577856~6545430,170345186~5508628,176160768~4101955,184549376~19749845,205520896~10409269,218103808~8225840,226492416~8735470,239075328~24432957,268435456~6310221,289406976~10425153,310378496~4846333,318767104~4667640,331350016~31253817,364904448~3902935,369098752~19729884,394264576~20943807] 2024-08-17T17:41:57.892 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 11 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10997221,47355868~8062348,55859939~6293284,64100320~539516,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,99353810~286411,105261990~3433993,110055987~12698473,127412771~21883557,151953600~842106,153770951~8826886,165161949~4961337,170345186~5508628,177013685~3249038,184955152~19344069,206579328~9350837,220784358~5850305,228066662~7161224,238191726~13839231,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:57.892 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10997221,47355868~8062348,55859939~6293284,64100320~539516,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,99353810~286411,105261990~3433993,110055987~12698473,127412771~21883557,151953600~842106,153770951~8826886,165161949~4961337,170345186~5508628,177013685~3249038,184955152~19344069,206579328~9350837,220784358~5850305,228066662~7161224,238191726~13839231,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:57.898 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~13436531,47355868~14797355,64100320~539516,64681659~9659247,75497472~6408989,82296588~1267250,83855009~122571,85296870~10217336,96468992~3171229,104857600~3838383,109051904~13702556,125829120~23467208,151953600~842106,153770951~8826886,163577856~6545430,170345186~5508628,176160768~4101955,184549376~19749845,205520896~10409269,218103808~45404477,268435456~6310221,289406976~10425153,310378496~4846333,318767104~4667640,331350016~31253817,364904448~3902935,369098752~19729884,394264576~20943807] 2024-08-17T17:41:57.898 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 12 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10997221,47355868~8062348,55859939~6293284,64100320~539516,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,99353810~286411,101692190~7003793,110055987~12698473,127412771~21883557,151953600~842106,153615532~8982305,165161949~4961337,170345186~5508628,177013685~3249038,184955152~19344069,206579328~9350837,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:57.898 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10997221,47355868~8062348,55859939~6293284,64100320~539516,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,99353810~286411,101692190~7003793,110055987~12698473,127412771~21883557,151953600~842106,153615532~8982305,165161949~4961337,170345186~5508628,177013685~3249038,184955152~19344069,206579328~9350837,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,253511811~9996474,269763691~4981986,291073215~751556,292092374~772196,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:57.904 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~13436531,47355868~14797355,64100320~539516,64681659~9659247,75497472~6408989,82296588~1267250,83855009~122571,85296870~10217336,96468992~3171229,100663296~8032687,109051904~13702556,125829120~23467208,151953600~842106,153615532~8982305,163577856~6545430,170345186~5508628,176160768~4101955,184549376~19749845,205520896~57987389,268435456~6310221,289406976~10425153,310378496~4846333,318767104~4667640,331350016~31253817,364904448~3902935,369098752~19729884,390070272~1830474,394264576~20943807] 2024-08-17T17:41:57.904 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 13 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~21996763,55859939~6293284,64100320~539516,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~5457279,99353810~9342173,110055987~12698473,127412771~21883557,151953600~842106,153615532~8982305,165161949~10691865,177013685~3765470,184955152~19344069,206579328~9350837,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,252773028~10735257,269763691~4981986,291073215~751556,292092374~772196,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:57.904 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~21996763,55859939~6293284,64100320~539516,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~5457279,99353810~9342173,110055987~12698473,127412771~21883557,151953600~842106,153615532~8982305,165161949~10691865,177013685~3765470,184955152~19344069,206579328~9350837,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,252773028~10735257,269763691~4981986,291073215~751556,292092374~772196,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:57.908 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~31171080,64100320~539516,64681659~9659247,75497472~6408989,82296588~1267250,83855009~122571,85296870~10257807,96468992~12226991,109051904~13702556,125829120~23467208,151953600~842106,153615532~8982305,163577856~12275958,176160768~4618387,184549376~19749845,205520896~57987389,268435456~6310221,289406976~12015144,310378496~4846333,318767104~4667640,331350016~31253817,364904448~3902935,369098752~19729884,390070272~1830474,394264576~20943807] 2024-08-17T17:41:57.909 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 14 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~28731770,64100320~539516,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,151953600~842106,153615532~8982305,165161949~10691865,177013685~3765470,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~13839231,252773028~10735257,269763691~4981986,291073215~751556,292092374~772196,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~19425483,395994076~19214307] 2024-08-17T17:41:57.909 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~28731770,64100320~539516,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,151953600~842106,153615532~8982305,165161949~10691865,177013685~3765470,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~13839231,252773028~10735257,269763691~4981986,291073215~751556,292092374~772196,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~19425483,395994076~19214307] 2024-08-17T17:41:57.914 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~31171080,64100320~539516,64681659~9659247,75497472~6408989,82296588~1267250,83855009~122571,85296870~37457590,125829120~23467208,151953600~842106,153615532~8982305,163577856~12275958,176160768~4618387,184549376~19749845,205520896~57987389,268435456~6310221,289406976~12015144,310378496~4846333,318767104~4667640,331350016~31253817,364904448~3902935,369098752~23035532,394264576~20943807] 2024-08-17T17:41:57.914 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 15 diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~28731770,64100320~539516,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,150192064~2603642,153615532~8982305,165161949~10691865,177013685~3765470,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~13839231,252773028~10735257,265186950~1824474,269763691~4981986,291073215~751556,292092374~772196,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,366925156~3770110,372708801~19425483,395994076~19214307] 2024-08-17T17:41:57.914 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~28731770,64100320~539516,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,150192064~2603642,153615532~8982305,165161949~10691865,177013685~3765470,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~13839231,252773028~10735257,265186950~1824474,269763691~4981986,291073215~751556,292092374~772196,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,366925156~3770110,372708801~19425483,395994076~19214307] 2024-08-17T17:41:57.920 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~31171080,64100320~539516,64681659~9659247,75497472~6408989,82296588~1267250,83855009~122571,85296870~37457590,125829120~26966586,153615532~8982305,163577856~12275958,176160768~4618387,184549376~19749845,205520896~57987389,264241152~2770272,268435456~6310221,289406976~12015144,310378496~4846333,318767104~4667640,331350016~31253817,364904448~27229836,394264576~20943807] 2024-08-17T17:41:57.920 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 16 diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~28731770,64100320~539516,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~13839231,252773028~10735257,265186950~1824474,269763691~8777870,291073215~751556,292092374~772196,293482393~7939727,312958845~2265984,322650436~784308,330570284~32033549,366925156~3770110,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:57.920 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~28731770,64100320~539516,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~13839231,252773028~10735257,265186950~1824474,269763691~8777870,291073215~751556,292092374~772196,293482393~7939727,312958845~2265984,322650436~784308,330570284~32033549,366925156~3770110,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:57.925 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~31171080,64100320~539516,64681659~9659247,75497472~6408989,82296588~1267250,83855009~122571,85296870~37457590,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~2770272,268435456~10106105,289406976~12015144,310378496~4846333,318767104~4667640,327155712~35448121,364904448~50303935] 2024-08-17T17:41:57.926 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 17 diff [6122489~2117206,8405456~21947887,31503479~1175130,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~13839231,252773028~10735257,265186950~1824474,269763691~8777870,291073215~751556,292092374~772196,293482393~7939727,312958845~6612414,322650436~784308,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:57.926 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~21947887,31503479~1175130,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~13839231,252773028~10735257,265186950~1824474,269763691~8777870,291073215~751556,292092374~772196,293482393~7939727,312958845~6612414,322650436~784308,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:57.931 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~31171080,64100320~539516,64681659~10382407,75497472~6408989,82296588~1267250,83855009~122571,85296870~37457590,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~2770272,268435456~10106105,289406976~12015144,310378496~13056248,327155712~88052671] 2024-08-17T17:41:57.931 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 18 diff [6122489~2117206,8405456~21947887,30955299~1723310,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~7939727,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:57.931 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~21947887,30955299~1723310,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~7939727,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:57.936 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30955299~31197924,64100320~539516,64681659~10382407,75497472~6408989,82296588~1267250,83855009~122571,85296870~37457590,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~16209448,301989888~6880813,310378496~13056248,327155712~88052671] 2024-08-17T17:41:57.937 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 19 diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:57.937 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:57.942 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:41:57.942 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 2 diff [27668422~1989503,35941013~6104604,47355868~6185179,64681659~7088322,114947089~7807371,242260764~5125880,293482393~6349736,332861425~4368751,356921972~5383594] 2024-08-17T17:41:57.942 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [27668422~1989503,35941013~6104604,47355868~6185179,64681659~7088322,114947089~7807371,242260764~5125880,293482393~6349736,332861425~4368751,356921972~5383594] 2024-08-17T17:41:57.947 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [27668422~1989503,33554432~8491185,47355868~6185179,64681659~7088322,113246208~9508252,239075328~8311316,292864570~6967559,332861425~4368751,356515840~5789726] 2024-08-17T17:41:57.947 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 3 diff [27668422~1989503,35941013~8477661,47355868~6185179,64681659~7088322,87479775~2420950,114947089~7807371,155052722~4602433,187244557~3088494,206785372~4793572,242260764~5125880,293482393~6349736,332861425~8580379,354460657~8143176,375999115~10637313] 2024-08-17T17:41:57.948 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [27668422~1989503,35941013~8477661,47355868~6185179,64681659~7088322,87479775~2420950,114947089~7807371,155052722~4602433,187244557~3088494,206785372~4793572,242260764~5125880,293482393~6349736,332861425~8580379,354460657~8143176,375999115~10637313] 2024-08-17T17:41:57.953 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [27668422~1989503,33554432~10864242,47355868~6185179,64681659~7088322,87043233~2857492,113246208~9508252,155052722~4602433,187244557~3088494,206785372~4793572,239075328~8311316,292864570~6967559,332861425~8580379,353481221~9122612,373293056~13343372] 2024-08-17T17:41:57.953 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 4 diff [27668422~1989503,35941013~8477661,47355868~6185179,64130590~509246,64681659~7088322,83855009~122571,87479775~2420950,110470753~244936,114947089~7807371,153770951~5884204,177013685~1267197,187244557~3088494,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,242260764~5125880,293482393~6349736,332861425~8580379,354460657~8143176,368552586~254797,372708801~13927627] 2024-08-17T17:41:57.953 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [27668422~1989503,35941013~8477661,47355868~6185179,64130590~509246,64681659~7088322,83855009~122571,87479775~2420950,110470753~244936,114947089~7807371,153770951~5884204,177013685~1267197,187244557~3088494,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,242260764~5125880,293482393~6349736,332861425~8580379,354460657~8143176,368552586~254797,372708801~13927627] 2024-08-17T17:41:57.958 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [27668422~1989503,33554432~10864242,47355868~6185179,64130590~509246,64681659~7088322,83855009~122571,87043233~2857492,109051904~1663785,113246208~9508252,153770951~5884204,176160768~2120114,187244557~3088494,206785372~4793572,211815065~4115100,218103808~8225840,226492416~8735470,239075328~8311316,292864570~6967559,332861425~8580379,353481221~9122612,364904448~3902935,369098752~17537676] 2024-08-17T17:41:57.959 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 5 diff [24616983~5736360,35941013~8477661,47355868~6185179,64130590~509246,64681659~7088322,74007760~333146,83855009~122571,87479775~2420950,105261990~3433993,110470753~244936,114947089~7807371,144750991~1159758,151953600~842106,153770951~5884204,177013685~1267197,187244557~3088494,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,242260764~5125880,291073215~751556,293482393~6349736,332649487~12754526,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540] 2024-08-17T17:41:57.959 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,35941013~8477661,47355868~6185179,64130590~509246,64681659~7088322,74007760~333146,83855009~122571,87479775~2420950,105261990~3433993,110470753~244936,114947089~7807371,144750991~1159758,151953600~842106,153770951~5884204,177013685~1267197,187244557~3088494,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,242260764~5125880,291073215~751556,293482393~6349736,332649487~12754526,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540] 2024-08-17T17:41:57.963 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,33554432~10864242,47355868~6185179,64130590~509246,64681659~9659247,83855009~122571,87043233~2857492,104857600~3838383,109051904~1663785,113246208~9508252,144750991~1159758,151953600~842106,153770951~5884204,176160768~2120114,187244557~3088494,206785372~4793572,211815065~4115100,218103808~8225840,226492416~8735470,239075328~8311316,291073215~751556,292864570~6967559,332649487~12754526,353481221~9122612,364904448~3902935,369098752~19729884] 2024-08-17T17:41:57.964 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 6 diff [24616983~5736360,35941013~8477661,47355868~6185179,55859939~779155,64130590~509246,64681659~7088322,74007760~333146,83855009~122571,87479775~2420950,105261990~3433993,110470753~244936,114947089~7807371,144750991~1159758,151953600~842106,153770951~5884204,177013685~1267197,187244557~13687461,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,293482393~6349736,332649487~19061486,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540] 2024-08-17T17:41:57.964 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,35941013~8477661,47355868~6185179,55859939~779155,64130590~509246,64681659~7088322,74007760~333146,83855009~122571,87479775~2420950,105261990~3433993,110470753~244936,114947089~7807371,144750991~1159758,151953600~842106,153770951~5884204,177013685~1267197,187244557~13687461,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,293482393~6349736,332649487~19061486,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540] 2024-08-17T17:41:57.969 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,33554432~10864242,47355868~6185179,54525952~2113142,64130590~509246,64681659~9659247,83855009~122571,87043233~2857492,104857600~3838383,109051904~1663785,113246208~9508252,144750991~1159758,151953600~842106,153770951~5884204,176160768~2120114,187244557~13687461,206785372~4793572,211815065~4115100,218103808~8225840,226492416~8735470,239075328~8568503,268435456~6310221,291073215~751556,292864570~6967559,332649487~19061486,353481221~9122612,364904448~3902935,369098752~19729884] 2024-08-17T17:41:57.969 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 7 diff [24616983~5736360,35941013~8477661,47355868~6185179,55859939~2311241,64130590~509246,64681659~7088322,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,105261990~3433993,110055987~12698473,144750991~1159758,151953600~842106,153770951~5884204,177013685~1267197,187244557~13687461,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,293482393~6349736,332649487~19061486,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:57.969 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,35941013~8477661,47355868~6185179,55859939~2311241,64130590~509246,64681659~7088322,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,105261990~3433993,110055987~12698473,144750991~1159758,151953600~842106,153770951~5884204,177013685~1267197,187244557~13687461,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,293482393~6349736,332649487~19061486,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:57.974 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,33554432~10864242,47355868~6185179,54525952~3645228,64130590~509246,64681659~9659247,82296588~1267250,83855009~122571,87043233~2857492,104857600~3838383,109051904~13702556,144750991~1159758,151953600~842106,153770951~5884204,176160768~2120114,187244557~13687461,206785372~4793572,211815065~4115100,218103808~8225840,226492416~8735470,239075328~8568503,268435456~6310221,291073215~751556,292864570~6967559,332649487~19061486,353481221~9122612,364904448~3902935,369098752~19729884,394264576~6106732,406847488~8360895] 2024-08-17T17:41:57.974 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 8 diff [12311950~2412776,24616983~5736360,31503479~1175130,35941013~8477661,47355868~6185179,55859939~2311241,64130590~509246,64681659~7088322,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,90175397~5338809,105261990~3433993,110055987~12698473,128194814~7129965,137226312~8684437,151953600~842106,153770951~5884204,177013685~3249038,187244557~13687461,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,293482393~6349736,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:57.974 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,35941013~8477661,47355868~6185179,55859939~2311241,64130590~509246,64681659~7088322,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,90175397~5338809,105261990~3433993,110055987~12698473,128194814~7129965,137226312~8684437,151953600~842106,153770951~5884204,177013685~3249038,187244557~13687461,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,293482393~6349736,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:57.979 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,33554432~10864242,47355868~6185179,54525952~3645228,64130590~509246,64681659~9659247,82296588~1267250,83855009~122571,87043233~8470973,104857600~3838383,109051904~13702556,125829120~20081629,151953600~842106,153770951~5884204,176160768~4101955,187244557~13687461,206785372~4793572,211815065~4115100,218103808~8225840,226492416~8735470,239075328~8568503,268435456~6310221,291073215~751556,292864570~6967559,318767104~4667640,332649487~29954346,364904448~3902935,369098752~19729884,394264576~6106732,406847488~8360895] 2024-08-17T17:41:57.980 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 9 diff [12311950~2412776,24616983~5736360,31503479~1175130,35941013~8477661,47355868~6185179,55859939~2311241,64130590~509246,64681659~7088322,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,90175397~5338809,105261990~3433993,110055987~12698473,127412771~18497978,151953600~842106,153770951~8826886,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~12307662,269763691~4981986,291073215~751556,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:57.980 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,35941013~8477661,47355868~6185179,55859939~2311241,64130590~509246,64681659~7088322,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,90175397~5338809,105261990~3433993,110055987~12698473,127412771~18497978,151953600~842106,153770951~8826886,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~12307662,269763691~4981986,291073215~751556,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:57.985 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,33554432~10864242,47355868~6185179,54525952~3645228,64130590~509246,64681659~9659247,82296588~1267250,83855009~122571,87043233~8470973,104857600~3838383,109051904~13702556,125829120~20081629,151953600~842106,153770951~8826886,176160768~4101955,187244557~17054664,206785372~4793572,211815065~4115100,218103808~8225840,226492416~8735470,239075328~12955629,268435456~6310221,291073215~751556,292864570~6967559,310378496~4846333,318767104~4667640,332649487~29954346,364904448~3902935,369098752~19729884,394264576~20943807] 2024-08-17T17:41:57.985 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 10 diff [12311950~2412776,24616983~5736360,31503479~1175130,35941013~8477661,47355868~8062348,55859939~2311241,58199577~3721268,64130590~509246,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,105261990~3433993,110055987~12698473,127412771~21883557,151953600~842106,153770951~8826886,166058561~2977181,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~12307662,269763691~4981986,291073215~751556,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:57.985 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,35941013~8477661,47355868~8062348,55859939~2311241,58199577~3721268,64130590~509246,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,105261990~3433993,110055987~12698473,127412771~21883557,151953600~842106,153770951~8826886,166058561~2977181,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~12307662,269763691~4981986,291073215~751556,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:57.990 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,33554432~10864242,47355868~14564977,64130590~509246,64681659~9659247,75497472~6408989,82296588~1267250,83855009~122571,85296870~10217336,104857600~3838383,109051904~13702556,125829120~23467208,151953600~842106,153770951~8826886,166058561~2977181,176160768~4101955,187244557~17054664,206785372~4793572,211815065~4115100,218103808~8225840,226492416~8735470,239075328~12955629,268435456~6310221,291073215~751556,292864570~6967559,310378496~4846333,318767104~4667640,332649487~29954346,364904448~3902935,369098752~19729884,394264576~20943807] 2024-08-17T17:41:57.991 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 11 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10997221,47355868~8062348,55859939~6293284,64130590~509246,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,105261990~3433993,110055987~12698473,127412771~21883557,151953600~842106,153770951~8826886,165571375~3464367,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,220784358~5850305,228066662~7161224,238191726~13839231,269763691~4981986,291073215~751556,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:57.991 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10997221,47355868~8062348,55859939~6293284,64130590~509246,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,105261990~3433993,110055987~12698473,127412771~21883557,151953600~842106,153770951~8826886,165571375~3464367,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,220784358~5850305,228066662~7161224,238191726~13839231,269763691~4981986,291073215~751556,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:57.996 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~13436531,47355868~14797355,64130590~509246,64681659~9659247,75497472~6408989,82296588~1267250,83855009~122571,85296870~10217336,104857600~3838383,109051904~13702556,125829120~23467208,151953600~842106,153770951~8826886,165571375~3464367,176160768~4101955,187244557~17054664,206785372~4793572,211815065~4115100,218103808~33927149,268435456~6310221,291073215~751556,292864570~6967559,310378496~4846333,318767104~4667640,332649487~29954346,364904448~3902935,369098752~19729884,394264576~20943807] 2024-08-17T17:41:57.996 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 12 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10997221,47355868~8062348,55859939~6293284,64130590~509246,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,101692190~7003793,110055987~12698473,127412771~21883557,151953600~842106,153615532~8982305,165571375~3464367,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,269763691~4981986,291073215~751556,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:57.996 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10997221,47355868~8062348,55859939~6293284,64130590~509246,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,101692190~7003793,110055987~12698473,127412771~21883557,151953600~842106,153615532~8982305,165571375~3464367,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,269763691~4981986,291073215~751556,293482393~6349736,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.001 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~13436531,47355868~14797355,64130590~509246,64681659~9659247,75497472~6408989,82296588~1267250,83855009~122571,85296870~10217336,100663296~8032687,109051904~13702556,125829120~23467208,151953600~842106,153615532~8982305,165571375~3464367,176160768~4101955,187244557~17054664,206785372~4793572,211815065~40215892,268435456~6310221,291073215~751556,292864570~6967559,310378496~4846333,318767104~4667640,332649487~29954346,364904448~3902935,369098752~19729884,390070272~1830474,394264576~20943807] 2024-08-17T17:41:58.001 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 13 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~21996763,55859939~6293284,64130590~509246,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~5457279,99526650~9169333,110055987~12698473,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177013685~3765470,187244557~17054664,206785372~4793572,211815065~4115100,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,252773028~8082862,269763691~4981986,291073215~751556,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.001 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~21996763,55859939~6293284,64130590~509246,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~5457279,99526650~9169333,110055987~12698473,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177013685~3765470,187244557~17054664,206785372~4793572,211815065~4115100,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,252773028~8082862,269763691~4981986,291073215~751556,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.006 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~31171080,64130590~509246,64681659~9659247,75497472~6408989,82296588~1267250,83855009~122571,85296870~10257807,99526650~9169333,109051904~13702556,125829120~23467208,151953600~842106,153615532~8982305,165571375~9681198,176160768~4618387,187244557~17054664,206785372~4793572,211815065~40215892,252773028~8082862,268435456~6310221,291073215~751556,292864570~8557550,310378496~4846333,318767104~4667640,332649487~29954346,364904448~3902935,369098752~19729884,390070272~1830474,394264576~20943807] 2024-08-17T17:41:58.006 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 14 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~28731770,64130590~509246,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177013685~3765470,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,269763691~4981986,291073215~751556,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~19425483,395994076~19214307] 2024-08-17T17:41:58.007 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~28731770,64130590~509246,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177013685~3765470,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,269763691~4981986,291073215~751556,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~19425483,395994076~19214307] 2024-08-17T17:41:58.012 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~31171080,64130590~509246,64681659~9659247,75497472~6408989,82296588~1267250,83855009~122571,85296870~37457590,125829120~23467208,151953600~842106,153615532~8982305,165571375~9681198,176160768~4618387,187244557~17054664,206785372~4793572,211815065~40215892,252773028~8082862,261307577~1507943,268435456~6310221,291073215~751556,292864570~8557550,310378496~4846333,318767104~4667640,332649487~29954346,364904448~3902935,369098752~23035532,394264576~20943807] 2024-08-17T17:41:58.012 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 15 diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~28731770,64130590~509246,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,150192064~2603642,153615532~8982305,165571375~9681198,177013685~3765470,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~4981986,291073215~751556,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,366925156~3770110,372708801~19425483,395994076~19214307] 2024-08-17T17:41:58.012 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~28731770,64130590~509246,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,150192064~2603642,153615532~8982305,165571375~9681198,177013685~3765470,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~4981986,291073215~751556,293482393~7939727,312958845~2265984,322650436~784308,332649487~29954346,366925156~3770110,372708801~19425483,395994076~19214307] 2024-08-17T17:41:58.017 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~31171080,64130590~509246,64681659~9659247,75497472~6408989,82296588~1267250,83855009~122571,85296870~37457590,125829120~26966586,153615532~8982305,165571375~9681198,176160768~4618387,187244557~17054664,206785372~4793572,211815065~40215892,252773028~8082862,261307577~1507943,264241152~2770272,268435456~6310221,291073215~751556,292864570~8557550,310378496~4846333,318767104~4667640,332649487~29954346,364904448~27229836,394264576~20943807] 2024-08-17T17:41:58.017 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 16 diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~28731770,64130590~509246,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,293482393~7939727,312958845~2265984,322650436~784308,330570284~32033549,366925156~3770110,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.017 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~28731770,64130590~509246,64681659~7088322,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,293482393~7939727,312958845~2265984,322650436~784308,330570284~32033549,366925156~3770110,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.022 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~31171080,64130590~509246,64681659~9659247,75497472~6408989,82296588~1267250,83855009~122571,85296870~37457590,125829120~36768717,165571375~15207780,187244557~17054664,206785372~4793572,211815065~40215892,252773028~8082862,261307577~1507943,264241152~2770272,268435456~10106105,291073215~751556,292864570~8557550,310378496~4846333,318767104~4667640,327155712~35448121,364904448~50303935] 2024-08-17T17:41:58.022 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 17 diff [6122489~2117206,8405456~21947887,31503479~1175130,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,293482393~7939727,312958845~6612414,322650436~784308,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.022 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~21947887,31503479~1175130,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,293482393~7939727,312958845~6612414,322650436~784308,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.028 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~31171080,64130590~509246,64681659~10382407,75497472~6408989,82296588~1267250,83855009~122571,85296870~37457590,125829120~36768717,165571375~15207780,187244557~17054664,206785372~4793572,211815065~40215892,252773028~8082862,261307577~1507943,264241152~2770272,268435456~10106105,291073215~751556,292864570~8557550,310378496~13056248,327155712~88052671] 2024-08-17T17:41:58.028 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 18 diff [6122489~2117206,8405456~21947887,30955299~1723310,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~7939727,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.028 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~21947887,30955299~1723310,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~32657062,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~7939727,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.033 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30955299~31197924,64130590~509246,64681659~10382407,75497472~6408989,82296588~1267250,83855009~122571,85296870~37457590,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~8557550,301989888~6880813,310378496~13056248,327155712~88052671] 2024-08-17T17:41:58.033 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 19 diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.033 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.038 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:41:58.038 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 3 diff [38674135~5744539,87479775~2420950,155052722~4602433,187244557~3088494,206785372~4793572,335970101~5471703,354460657~8143176,375999115~10637313] 2024-08-17T17:41:58.039 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [38674135~5744539,87479775~2420950,155052722~4602433,187244557~3088494,206785372~4793572,335970101~5471703,354460657~8143176,375999115~10637313] 2024-08-17T17:41:58.043 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [38674135~5744539,87043233~2857492,155052722~4602433,187244557~3088494,206785372~4793572,335970101~5471703,353481221~9122612,373293056~13343372] 2024-08-17T17:41:58.044 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 4 diff [38674135~5744539,64130590~509246,83855009~122571,87479775~2420950,110470753~244936,153770951~5884204,177013685~1267197,187244557~3088494,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,335970101~5471703,354460657~8143176,368552586~254797,372708801~13927627] 2024-08-17T17:41:58.044 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [38674135~5744539,64130590~509246,83855009~122571,87479775~2420950,110470753~244936,153770951~5884204,177013685~1267197,187244557~3088494,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,335970101~5471703,354460657~8143176,368552586~254797,372708801~13927627] 2024-08-17T17:41:58.048 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [38674135~5744539,64130590~509246,83855009~122571,87043233~2857492,109051904~1663785,153770951~5884204,176160768~2120114,187244557~3088494,206785372~4793572,211815065~4115100,218103808~8225840,226492416~8735470,335970101~5471703,353481221~9122612,364904448~3902935,369098752~17537676] 2024-08-17T17:41:58.049 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 5 diff [24616983~5736360,38674135~5744539,64130590~509246,74007760~333146,83855009~122571,87479775~2420950,105261990~3433993,110470753~244936,144750991~1159758,151953600~842106,153770951~5884204,177013685~1267197,187244557~3088494,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,242776912~2064207,291073215~751556,332649487~12754526,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540] 2024-08-17T17:41:58.049 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,38674135~5744539,64130590~509246,74007760~333146,83855009~122571,87479775~2420950,105261990~3433993,110470753~244936,144750991~1159758,151953600~842106,153770951~5884204,177013685~1267197,187244557~3088494,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,242776912~2064207,291073215~751556,332649487~12754526,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540] 2024-08-17T17:41:58.054 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,38674135~5744539,64130590~509246,71769981~2570925,83855009~122571,87043233~2857492,104857600~3838383,109051904~1663785,144750991~1159758,151953600~842106,153770951~5884204,176160768~2120114,187244557~3088494,206785372~4793572,211815065~4115100,218103808~8225840,226492416~8735470,242776912~2064207,291073215~751556,332649487~12754526,353481221~9122612,364904448~3902935,369098752~19729884] 2024-08-17T17:41:58.054 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 6 diff [24616983~5736360,38674135~5744539,55859939~779155,64130590~509246,74007760~333146,83855009~122571,87479775~2420950,105261990~3433993,110470753~244936,144750991~1159758,151953600~842106,153770951~5884204,177013685~1267197,187244557~13687461,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,332649487~19061486,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540] 2024-08-17T17:41:58.054 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,38674135~5744539,55859939~779155,64130590~509246,74007760~333146,83855009~122571,87479775~2420950,105261990~3433993,110470753~244936,144750991~1159758,151953600~842106,153770951~5884204,177013685~1267197,187244557~13687461,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,332649487~19061486,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540] 2024-08-17T17:41:58.059 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,38674135~5744539,54525952~2113142,64130590~509246,71769981~2570925,83855009~122571,87043233~2857492,104857600~3838383,109051904~1663785,144750991~1159758,151953600~842106,153770951~5884204,176160768~2120114,187244557~13687461,206785372~4793572,211815065~4115100,218103808~8225840,226492416~8735470,239723295~7920536,268435456~6310221,291073215~751556,332649487~19061486,353481221~9122612,364904448~3902935,369098752~19729884] 2024-08-17T17:41:58.059 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 7 diff [24616983~5736360,36062054~8356620,55859939~2311241,64130590~509246,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,105261990~3433993,110055987~6404653,144750991~1159758,151953600~842106,153770951~5884204,177013685~1267197,187244557~13687461,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,332649487~19061486,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.059 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,36062054~8356620,55859939~2311241,64130590~509246,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,105261990~3433993,110055987~6404653,144750991~1159758,151953600~842106,153770951~5884204,177013685~1267197,187244557~13687461,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,332649487~19061486,354460657~8143176,368552586~254797,372708801~13927627,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.064 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,36062054~8356620,54525952~3645228,64130590~509246,71769981~2570925,82296588~1267250,83855009~122571,87043233~2857492,104857600~3838383,109051904~7408736,144750991~1159758,151953600~842106,153770951~5884204,176160768~2120114,187244557~13687461,206785372~4793572,211815065~4115100,218103808~8225840,226492416~8735470,239723295~7920536,268435456~6310221,291073215~751556,332649487~19061486,353481221~9122612,364904448~3902935,369098752~19729884,394264576~6106732,406847488~8360895] 2024-08-17T17:41:58.065 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 8 diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~8356620,55859939~2311241,64130590~509246,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,90175397~5338809,105261990~3433993,110055987~6404653,128194814~7129965,137226312~8684437,151953600~842106,153770951~5884204,177013685~3249038,187244557~13687461,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.065 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~8356620,55859939~2311241,64130590~509246,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,90175397~5338809,105261990~3433993,110055987~6404653,128194814~7129965,137226312~8684437,151953600~842106,153770951~5884204,177013685~3249038,187244557~13687461,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.070 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,36062054~8356620,54525952~3645228,64130590~509246,71769981~2570925,82296588~1267250,83855009~122571,87043233~8470973,104857600~3838383,109051904~7408736,125829120~20081629,151953600~842106,153770951~5884204,176160768~4101955,187244557~13687461,206785372~4793572,211815065~4115100,218103808~8225840,226492416~8735470,239723295~7920536,268435456~6310221,291073215~751556,318767104~4667640,332649487~29954346,364904448~3902935,369098752~19729884,394264576~6106732,406847488~8360895] 2024-08-17T17:41:58.070 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 9 diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~8356620,55859939~2311241,64130590~509246,68742908~1813321,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,90175397~5338809,105261990~3433993,110055987~6404653,127412771~18497978,151953600~842106,153770951~8826886,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~12307662,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.070 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~8356620,55859939~2311241,64130590~509246,68742908~1813321,74007760~333146,82296588~1267250,83855009~122571,87479775~2420950,90175397~5338809,105261990~3433993,110055987~6404653,127412771~18497978,151953600~842106,153770951~8826886,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~12307662,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.075 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,36062054~8356620,54525952~3645228,64130590~509246,68742908~1813321,71769981~2570925,82296588~1267250,83855009~122571,87043233~8470973,104857600~3838383,109051904~7408736,125829120~20081629,151953600~842106,153770951~8826886,176160768~4101955,187244557~17054664,206785372~4793572,211815065~4115100,218103808~8225840,226492416~8735470,239723295~12307662,268435456~6310221,291073215~751556,310378496~4846333,318767104~4667640,332649487~29954346,364904448~3902935,369098752~19729884,394264576~20943807] 2024-08-17T17:41:58.075 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 10 diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~8356620,52172688~3245528,55859939~2311241,58199577~3721268,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,105261990~3433993,110055987~6404653,127412771~21883557,151953600~842106,153770951~8826886,166058561~2977181,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~12307662,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.075 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~8356620,52172688~3245528,55859939~2311241,58199577~3721268,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,105261990~3433993,110055987~6404653,127412771~21883557,151953600~842106,153770951~8826886,166058561~2977181,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,220784358~5545290,228066662~7161224,239723295~12307662,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.080 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,36062054~8356620,52172688~9748157,64130590~509246,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,83855009~122571,85296870~10217336,104857600~3838383,109051904~7408736,125829120~23467208,151953600~842106,153770951~8826886,166058561~2977181,176160768~4101955,187244557~17054664,206785372~4793572,211815065~4115100,218103808~8225840,226492416~8735470,239723295~12307662,268435456~6310221,291073215~751556,310378496~4846333,318767104~4667640,332649487~29954346,364904448~3902935,369098752~19729884,394264576~20943807] 2024-08-17T17:41:58.081 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 11 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10997221,52172688~3245528,55859939~6293284,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,105261990~3433993,110055987~6404653,127412771~21883557,151953600~842106,153770951~8826886,165571375~3464367,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,220784358~5850305,228066662~7161224,238191726~13839231,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.081 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10997221,52172688~3245528,55859939~6293284,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,105261990~3433993,110055987~6404653,127412771~21883557,151953600~842106,153770951~8826886,165571375~3464367,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,220784358~5850305,228066662~7161224,238191726~13839231,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.086 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~13436531,52172688~9980535,64130590~509246,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,83855009~122571,85296870~10217336,104857600~3838383,109051904~7408736,125829120~23467208,151953600~842106,153770951~8826886,165571375~3464367,176160768~4101955,187244557~17054664,206785372~4793572,211815065~4115100,218103808~33927149,268435456~6310221,291073215~751556,310378496~4846333,318767104~4667640,332649487~29954346,364904448~3902935,369098752~19729884,394264576~20943807] 2024-08-17T17:41:58.086 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 12 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10997221,52172688~3245528,55859939~6293284,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,101692190~7003793,110055987~6404653,127412771~21883557,151953600~842106,153615532~8982305,165571375~3464367,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.086 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10997221,52172688~3245528,55859939~6293284,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90175397~5338809,101692190~7003793,110055987~6404653,127412771~21883557,151953600~842106,153615532~8982305,165571375~3464367,177013685~3249038,187244557~17054664,206785372~4793572,211815065~4115100,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.091 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~13436531,52172688~9980535,64130590~509246,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,83855009~122571,85296870~10217336,100663296~8032687,109051904~7408736,125829120~23467208,151953600~842106,153615532~8982305,165571375~3464367,176160768~4101955,187244557~17054664,206785372~4793572,211815065~40215892,268435456~6310221,291073215~751556,310378496~4846333,318767104~4667640,332649487~29954346,364904448~3902935,369098752~19729884,390070272~1830474,394264576~20943807] 2024-08-17T17:41:58.091 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 13 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~15075986,52172688~3245528,55859939~6293284,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~5457279,99526650~9169333,110055987~6404653,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177013685~3765470,187244557~17054664,206785372~4793572,211815065~4115100,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,252773028~8082862,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.091 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~15075986,52172688~3245528,55859939~6293284,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~5457279,99526650~9169333,110055987~6404653,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177013685~3765470,187244557~17054664,206785372~4793572,211815065~4115100,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,252773028~8082862,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~14744079,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:57 smithi179 ceph-mon[93519]: pgmap v1207: 105 pgs: 105 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 110 KiB/s rd, 41 MiB/s wr, 167 op/s 2024-08-17T17:41:58.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:57 smithi179 ceph-mon[93519]: osdmap e988: 8 total, 8 up, 8 in 2024-08-17T17:41:58.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:58.096 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~17515296,52172688~9980535,64130590~509246,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,83855009~122571,85296870~10257807,99526650~9169333,109051904~7408736,125829120~23467208,151953600~842106,153615532~8982305,165571375~9681198,176160768~4618387,187244557~17054664,206785372~4793572,211815065~40215892,252773028~8082862,268435456~6310221,291073215~751556,294440898~6981222,310378496~4846333,318767104~4667640,332649487~29954346,364904448~3902935,369098752~19729884,390070272~1830474,394264576~20943807] 2024-08-17T17:41:58.096 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 14 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~15075986,52172688~9980535,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~26363242,118553273~1355301,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177013685~3765470,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~19425483,395994076~19214307] 2024-08-17T17:41:58.096 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~15075986,52172688~9980535,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~26363242,118553273~1355301,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177013685~3765470,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~29954346,368552586~254797,372708801~19425483,395994076~19214307] 2024-08-17T17:41:58.102 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~17515296,52172688~9980535,64130590~509246,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,83855009~122571,85296870~31163770,118553273~1355301,125829120~23467208,151953600~842106,153615532~8982305,165571375~9681198,176160768~4618387,187244557~17054664,206785372~4793572,211815065~40215892,252773028~8082862,261307577~1507943,268435456~6310221,291073215~751556,294440898~6981222,310378496~4846333,318767104~4667640,332649487~29954346,364904448~3902935,369098752~23035532,394264576~20943807] 2024-08-17T17:41:58.102 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 15 diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~15075986,52172688~9980535,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~26363242,118553273~1355301,127412771~21883557,150192064~2603642,153615532~8982305,165571375~9681198,177013685~3765470,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~29954346,366925156~3770110,372708801~19425483,395994076~19214307] 2024-08-17T17:41:58.102 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~15075986,52172688~9980535,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~26363242,118553273~1355301,127412771~21883557,150192064~2603642,153615532~8982305,165571375~9681198,177013685~3765470,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~29954346,366925156~3770110,372708801~19425483,395994076~19214307] 2024-08-17T17:41:58.107 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~17515296,52172688~9980535,64130590~509246,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,83855009~122571,85296870~31163770,118553273~1355301,125829120~26966586,153615532~8982305,165571375~9681198,176160768~4618387,187244557~17054664,206785372~4793572,211815065~40215892,252773028~8082862,261307577~1507943,264241152~2770272,268435456~6310221,291073215~751556,294440898~6981222,310378496~4846333,318767104~4667640,332649487~29954346,364904448~27229836,394264576~20943807] 2024-08-17T17:41:58.107 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 16 diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~15075986,52172688~9980535,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,330570284~32033549,366925156~3770110,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.107 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~15075986,52172688~9980535,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,330570284~32033549,366925156~3770110,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.112 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~17515296,52172688~9980535,64130590~509246,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,83855009~122571,85296870~31163770,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~4793572,211815065~40215892,252773028~8082862,261307577~1507943,264241152~2770272,268435456~10106105,291073215~751556,294440898~6981222,310378496~4846333,318767104~4667640,327155712~35448121,364904448~50303935] 2024-08-17T17:41:58.112 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 17 diff [6122489~2117206,8405456~21947887,31503479~1175130,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,294440898~6981222,312958845~6612414,322650436~784308,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.112 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~21947887,31503479~1175130,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~4793572,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,294440898~6981222,312958845~6612414,322650436~784308,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.118 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~19029698,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~6408989,82296588~1267250,83855009~122571,85296870~31163770,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~4793572,211815065~40215892,252773028~8082862,261307577~1507943,264241152~2770272,268435456~10106105,291073215~751556,294440898~6981222,310378496~13056248,327155712~88052671] 2024-08-17T17:41:58.118 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 18 diff [6122489~2117206,8405456~21947887,30955299~1723310,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~6981222,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.118 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~21947887,30955299~1723310,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4603855,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~6981222,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.123 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30955299~19056542,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~6408989,82296588~1267250,83855009~122571,85296870~31163770,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~6981222,301989888~6880813,310378496~13056248,327155712~88052671] 2024-08-17T17:41:58.123 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 19 diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.124 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.129 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:41:58.129 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 4 diff [64130590~509246,83855009~122571,110470753~244936,153770951~5448131,177013685~1267197,211815065~4115100,220784358~5545290,228066662~7161224,368552586~254797,372708801~6982505] 2024-08-17T17:41:58.129 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [64130590~509246,83855009~122571,110470753~244936,153770951~5448131,177013685~1267197,211815065~4115100,220784358~5545290,228066662~7161224,368552586~254797,372708801~6982505] 2024-08-17T17:41:58.134 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [64130590~509246,83855009~122571,109051904~1663785,153770951~5448131,176160768~2120114,211815065~4115100,218103808~8225840,226492416~8735470,364904448~3902935,369098752~10592554] 2024-08-17T17:41:58.134 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 5 diff [24616983~5736360,64130590~509246,74007760~333146,83855009~122571,105261990~3433993,110470753~244936,144750991~1159758,151953600~842106,153770951~5448131,177013685~1267197,211815065~4115100,220784358~5545290,228066662~7161224,242776912~2064207,291073215~751556,332649487~12754526,368552586~254797,372708801~6982505,388486096~342540] 2024-08-17T17:41:58.134 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,64130590~509246,74007760~333146,83855009~122571,105261990~3433993,110470753~244936,144750991~1159758,151953600~842106,153770951~5448131,177013685~1267197,211815065~4115100,220784358~5545290,228066662~7161224,242776912~2064207,291073215~751556,332649487~12754526,368552586~254797,372708801~6982505,388486096~342540] 2024-08-17T17:41:58.139 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,64130590~509246,71769981~2570925,83855009~122571,104857600~3838383,109051904~1663785,144750991~1159758,151953600~842106,153770951~5448131,176160768~2120114,211815065~4115100,218103808~8225840,226492416~8735470,242776912~2064207,291073215~751556,332649487~12754526,364904448~3902935,369098752~10592554,386636428~2192208] 2024-08-17T17:41:58.139 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 6 diff [24616983~5736360,55859939~779155,64130590~509246,74007760~333146,83855009~122571,105261990~3433993,110470753~244936,144750991~1159758,151953600~842106,153770951~5448131,177013685~1267197,189053332~11878686,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,332649487~19061486,355907560~4087411,368552586~254797,372708801~6982505,388486096~342540] 2024-08-17T17:41:58.139 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,55859939~779155,64130590~509246,74007760~333146,83855009~122571,105261990~3433993,110470753~244936,144750991~1159758,151953600~842106,153770951~5448131,177013685~1267197,189053332~11878686,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,332649487~19061486,355907560~4087411,368552586~254797,372708801~6982505,388486096~342540] 2024-08-17T17:41:58.144 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,54525952~2113142,64130590~509246,71769981~2570925,83855009~122571,104857600~3838383,109051904~1663785,144750991~1159758,151953600~842106,153770951~5448131,176160768~2120114,189053332~11878686,211815065~4115100,218103808~8225840,226492416~8735470,239723295~7920536,268435456~6310221,291073215~751556,332649487~19061486,355907560~4087411,364904448~3902935,369098752~10592554,386636428~2192208] 2024-08-17T17:41:58.144 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 7 diff [24616983~5736360,36062054~6554620,55859939~2311241,64130590~509246,74007760~333146,82296588~1267250,83855009~122571,105261990~3433993,110055987~6404653,144750991~1159758,151953600~842106,153770951~5448131,177013685~1267197,189053332~11878686,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,332649487~19061486,355907560~4087411,368552586~254797,372708801~12370730,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.144 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,36062054~6554620,55859939~2311241,64130590~509246,74007760~333146,82296588~1267250,83855009~122571,105261990~3433993,110055987~6404653,144750991~1159758,151953600~842106,153770951~5448131,177013685~1267197,189053332~11878686,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,332649487~19061486,355907560~4087411,368552586~254797,372708801~12370730,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.149 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,36062054~6554620,54525952~3645228,64130590~509246,71769981~2570925,82296588~1267250,83855009~122571,104857600~3838383,109051904~7408736,144750991~1159758,151953600~842106,153770951~5448131,176160768~2120114,189053332~11878686,211815065~4115100,218103808~8225840,226492416~8735470,239723295~7920536,268435456~6310221,291073215~751556,332649487~19061486,355907560~4087411,364904448~3902935,369098752~15980779,386636428~2192208,394264576~6106732,406847488~8360895] 2024-08-17T17:41:58.150 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 8 diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~6554620,55859939~2311241,64130590~509246,74007760~333146,82296588~1267250,83855009~122571,90175397~5338809,105261990~3433993,110055987~6404653,128194814~7129965,137226312~8684437,151953600~842106,153770951~5448131,177013685~3249038,189053332~11878686,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,322650436~784308,332649487~22386665,355907560~4087411,368552586~254797,372708801~12370730,385791049~1661831,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.150 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~6554620,55859939~2311241,64130590~509246,74007760~333146,82296588~1267250,83855009~122571,90175397~5338809,105261990~3433993,110055987~6404653,128194814~7129965,137226312~8684437,151953600~842106,153770951~5448131,177013685~3249038,189053332~11878686,211815065~4115100,220784358~5545290,228066662~7161224,239723295~7920536,269763691~4981986,291073215~751556,322650436~784308,332649487~22386665,355907560~4087411,368552586~254797,372708801~12370730,385791049~1661831,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.155 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,36062054~6554620,54525952~3645228,64130590~509246,71769981~2570925,82296588~1267250,83855009~122571,89900725~5613481,104857600~3838383,109051904~7408736,125829120~20081629,151953600~842106,153770951~5448131,176160768~4101955,189053332~11878686,211815065~4115100,218103808~8225840,226492416~8735470,239723295~7920536,268435456~6310221,291073215~751556,318767104~4667640,332649487~22386665,355907560~4087411,364904448~3902935,369098752~15980779,385791049~3037587,394264576~6106732,406847488~8360895] 2024-08-17T17:41:58.155 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 9 diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~6554620,55859939~2311241,64130590~509246,68742908~1813321,74007760~333146,82296588~1267250,83855009~122571,90175397~5338809,105261990~3433993,110055987~6404653,127412771~18497978,151953600~842106,153770951~8826886,177013685~3249038,189053332~15245889,211815065~4115100,220784358~5545290,228066662~7161224,239723295~12307662,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~22386665,355907560~4087411,368552586~254797,372708801~12370730,385791049~1661831,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.155 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~6554620,55859939~2311241,64130590~509246,68742908~1813321,74007760~333146,82296588~1267250,83855009~122571,90175397~5338809,105261990~3433993,110055987~6404653,127412771~18497978,151953600~842106,153770951~8826886,177013685~3249038,189053332~15245889,211815065~4115100,220784358~5545290,228066662~7161224,239723295~12307662,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~22386665,355907560~4087411,368552586~254797,372708801~12370730,385791049~1661831,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.160 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,36062054~6554620,54525952~3645228,64130590~509246,68742908~1813321,71769981~2570925,82296588~1267250,83855009~122571,89900725~5613481,104857600~3838383,109051904~7408736,125829120~20081629,151953600~842106,153770951~8826886,176160768~4101955,189053332~15245889,211815065~4115100,218103808~8225840,226492416~8735470,239723295~12307662,268435456~6310221,291073215~751556,310378496~4846333,318767104~4667640,332649487~22386665,355907560~4087411,364904448~3902935,369098752~15980779,385791049~3037587,394264576~20943807] 2024-08-17T17:41:58.160 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 10 diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~6554620,52172688~3245528,55859939~2311241,58199577~3721268,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90175397~5338809,105261990~3433993,110055987~6404653,127412771~21883557,151953600~842106,153770951~8826886,166058561~2977181,177013685~3249038,189053332~15245889,211815065~4115100,220784358~5545290,228066662~7161224,239723295~12307662,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~22386665,355907560~4087411,368552586~254797,372708801~12370730,385791049~1661831,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.160 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~6554620,52172688~3245528,55859939~2311241,58199577~3721268,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90175397~5338809,105261990~3433993,110055987~6404653,127412771~21883557,151953600~842106,153770951~8826886,166058561~2977181,177013685~3249038,189053332~15245889,211815065~4115100,220784358~5545290,228066662~7161224,239723295~12307662,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~22386665,355907560~4087411,368552586~254797,372708801~12370730,385791049~1661831,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.166 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,36062054~6554620,52172688~9748157,64130590~509246,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,83855009~122571,85296870~4252304,89900725~5613481,104857600~3838383,109051904~7408736,125829120~23467208,151953600~842106,153770951~8826886,166058561~2977181,176160768~4101955,189053332~15245889,211815065~4115100,218103808~8225840,226492416~8735470,239723295~12307662,268435456~6310221,291073215~751556,310378496~4846333,318767104~4667640,332649487~22386665,355907560~4087411,364904448~3902935,369098752~15980779,385791049~3037587,394264576~20943807] 2024-08-17T17:41:58.166 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 11 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10210967,52172688~3245528,55859939~6293284,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90175397~5338809,105261990~3433993,110055987~6404653,127412771~21883557,151953600~842106,153770951~8826886,165571375~3464367,177013685~3249038,189053332~15245889,211815065~4115100,220784358~5850305,228066662~7161224,238191726~13839231,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~22386665,355907560~4087411,368552586~254797,372708801~12370730,385791049~1661831,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.166 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10210967,52172688~3245528,55859939~6293284,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90175397~5338809,105261990~3433993,110055987~6404653,127412771~21883557,151953600~842106,153770951~8826886,165571375~3464367,177013685~3249038,189053332~15245889,211815065~4115100,220784358~5850305,228066662~7161224,238191726~13839231,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~22386665,355907560~4087411,368552586~254797,372708801~12370730,385791049~1661831,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:57 smithi130 ceph-mon[105521]: pgmap v1207: 105 pgs: 105 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 110 KiB/s rd, 41 MiB/s wr, 167 op/s 2024-08-17T17:41:58.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:57 smithi130 ceph-mon[105521]: osdmap e988: 8 total, 8 up, 8 in 2024-08-17T17:41:58.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:57 smithi130 ceph-mon[110068]: pgmap v1207: 105 pgs: 105 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 110 KiB/s rd, 41 MiB/s wr, 167 op/s 2024-08-17T17:41:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:57 smithi130 ceph-mon[110068]: osdmap e988: 8 total, 8 up, 8 in 2024-08-17T17:41:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:58.171 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~12650277,52172688~9980535,64130590~509246,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,83855009~122571,85296870~4252304,89900725~5613481,104857600~3838383,109051904~7408736,125829120~23467208,151953600~842106,153770951~8826886,165571375~3464367,176160768~4101955,189053332~15245889,211815065~4115100,218103808~33927149,268435456~6310221,291073215~751556,310378496~4846333,318767104~4667640,332649487~22386665,355907560~4087411,364904448~3902935,369098752~15980779,385791049~3037587,394264576~20943807] 2024-08-17T17:41:58.171 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 12 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10210967,52172688~3245528,55859939~6293284,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90175397~5338809,101692190~7003793,110055987~6404653,127412771~21883557,151953600~842106,153615532~8982305,165571375~3464367,177013685~3249038,189053332~15245889,211815065~4115100,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~27345484,368552586~254797,372708801~12370730,385791049~1661831,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.171 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10210967,52172688~3245528,55859939~6293284,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90175397~5338809,101692190~7003793,110055987~6404653,127412771~21883557,151953600~842106,153615532~8982305,165571375~3464367,177013685~3249038,189053332~15245889,211815065~4115100,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~27345484,368552586~254797,372708801~12370730,385791049~1661831,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.176 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~12650277,52172688~9980535,64130590~509246,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,83855009~122571,85296870~4252304,89900725~5613481,100663296~8032687,109051904~7408736,125829120~23467208,151953600~842106,153615532~8982305,165571375~3464367,176160768~4101955,189053332~15245889,211815065~40215892,268435456~6310221,291073215~751556,310378496~4846333,318767104~4667640,332649487~27345484,364904448~3902935,369098752~15980779,385791049~3037587,390070272~1830474,394264576~20943807] 2024-08-17T17:41:58.176 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 13 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~15075986,52172688~3245528,55859939~6293284,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90097398~5457279,99526650~9169333,110055987~6404653,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177013685~3765470,189053332~15245889,211815065~4115100,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,252773028~8082862,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~27345484,368552586~254797,372708801~12370730,385791049~1661831,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.177 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~15075986,52172688~3245528,55859939~6293284,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90097398~5457279,99526650~9169333,110055987~6404653,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177013685~3765470,189053332~15245889,211815065~4115100,217652007~2893566,220784358~5850305,228066662~7161224,238191726~13839231,252773028~8082862,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~27345484,368552586~254797,372708801~12370730,385791049~1661831,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.182 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~17515296,52172688~9980535,64130590~509246,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,83855009~122571,85296870~4252304,89900725~5653952,99526650~9169333,109051904~7408736,125829120~23467208,151953600~842106,153615532~8982305,165571375~9681198,176160768~4618387,189053332~15245889,211815065~40215892,252773028~8082862,268435456~6310221,291073215~751556,294440898~6981222,310378496~4846333,318767104~4667640,332649487~27345484,364904448~3902935,369098752~15980779,385791049~3037587,390070272~1830474,394264576~20943807] 2024-08-17T17:41:58.182 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 14 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~15075986,52172688~9980535,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90097398~26363242,118553273~1355301,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177013685~3765470,189053332~15245889,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~27345484,368552586~254797,372708801~19425483,395994076~19214307] 2024-08-17T17:41:58.182 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~15075986,52172688~9980535,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90097398~26363242,118553273~1355301,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177013685~3765470,189053332~15245889,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~27345484,368552586~254797,372708801~19425483,395994076~19214307] 2024-08-17T17:41:58.188 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~17515296,52172688~9980535,64130590~509246,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,83855009~122571,85296870~4252304,89900725~26559915,118553273~1355301,125829120~23467208,151953600~842106,153615532~8982305,165571375~9681198,176160768~4618387,189053332~15245889,211815065~40215892,252773028~8082862,261307577~1507943,268435456~6310221,291073215~751556,294440898~6981222,310378496~4846333,318767104~4667640,332649487~27345484,364904448~3902935,369098752~23035532,394264576~20943807] 2024-08-17T17:41:58.188 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 15 diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~15075986,52172688~9980535,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90097398~26363242,118553273~1355301,127412771~21883557,150192064~2603642,153615532~8982305,165571375~9681198,177013685~3765470,189053332~15245889,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~27345484,366925156~3770110,372708801~19425483,395994076~19214307] 2024-08-17T17:41:58.188 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~15075986,52172688~9980535,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90097398~26363242,118553273~1355301,127412771~21883557,150192064~2603642,153615532~8982305,165571375~9681198,177013685~3765470,189053332~15245889,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~27345484,366925156~3770110,372708801~19425483,395994076~19214307] 2024-08-17T17:41:58.193 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~17515296,52172688~9980535,64130590~509246,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,83855009~122571,85296870~4252304,89900725~26559915,118553273~1355301,125829120~26966586,153615532~8982305,165571375~9681198,176160768~4618387,189053332~15245889,211815065~40215892,252773028~8082862,261307577~1507943,264241152~2770272,268435456~6310221,291073215~751556,294440898~6981222,310378496~4846333,318767104~4667640,332649487~27345484,364904448~27229836,394264576~20943807] 2024-08-17T17:41:58.193 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 16 diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~15075986,52172688~9980535,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,330570284~29424687,366925156~3770110,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.193 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~15075986,52172688~9980535,64130590~509246,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,330570284~29424687,366925156~3770110,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.198 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~17515296,52172688~9980535,64130590~509246,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,83855009~122571,85296870~4252304,89900725~26559915,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,211815065~40215892,252773028~8082862,261307577~1507943,264241152~2770272,268435456~10106105,291073215~751556,294440898~6981222,310378496~4846333,318767104~4667640,327155712~32839259,364904448~50303935] 2024-08-17T17:41:58.198 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 17 diff [6122489~2117206,8405456~21947887,31503479~1175130,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,294440898~6981222,312958845~6612414,322650436~784308,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.198 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~21947887,31503479~1175130,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,211815065~4115100,217652007~8982656,228066662~7161224,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,294440898~6981222,312958845~6612414,322650436~784308,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.203 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~19029698,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~6408989,82296588~1267250,83855009~122571,85296870~4252304,89900725~26559915,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,211815065~40215892,252773028~8082862,261307577~1507943,264241152~2770272,268435456~10106105,291073215~751556,294440898~6981222,310378496~13056248,327155712~32839259,362603833~52604550] 2024-08-17T17:41:58.204 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 18 diff [6122489~2117206,8405456~21947887,30955299~1723310,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~6981222,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.204 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~21947887,30955299~1723310,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,83855009~122571,85296870~4252304,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~6981222,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.209 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30955299~19056542,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~6408989,82296588~1267250,83855009~122571,85296870~4252304,89900725~26559915,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~6981222,301989888~6880813,310378496~13056248,327155712~32839259,362603833~52604550] 2024-08-17T17:41:58.209 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 19 diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.209 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.214 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:41:58.214 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 5 diff [24616983~5736360,74007760~333146,105261990~3433993,144750991~1159758,151953600~842106,242776912~2064207,291073215~751556,332649487~12754526,388486096~342540] 2024-08-17T17:41:58.214 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,74007760~333146,105261990~3433993,144750991~1159758,151953600~842106,242776912~2064207,291073215~751556,332649487~12754526,388486096~342540] 2024-08-17T17:41:58.220 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,71769981~2570925,104857600~3838383,144750991~1159758,151953600~842106,242776912~2064207,291073215~751556,332649487~12754526,386636428~2192208] 2024-08-17T17:41:58.220 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 6 diff [24616983~5736360,55859939~779155,74007760~333146,105261990~3433993,144750991~1159758,151953600~842106,189053332~11878686,229534798~3451158,239723295~7920536,269763691~4981986,291073215~751556,332649487~19061486,355907560~4087411,388486096~342540] 2024-08-17T17:41:58.220 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,55859939~779155,74007760~333146,105261990~3433993,144750991~1159758,151953600~842106,189053332~11878686,229534798~3451158,239723295~7920536,269763691~4981986,291073215~751556,332649487~19061486,355907560~4087411,388486096~342540] 2024-08-17T17:41:58.225 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,54525952~2113142,71769981~2570925,104857600~3838383,144750991~1159758,151953600~842106,189053332~11878686,229534798~3451158,239723295~7920536,268435456~6310221,291073215~751556,332649487~19061486,355907560~4087411,386636428~2192208] 2024-08-17T17:41:58.225 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 7 diff [24616983~5736360,36062054~6554620,55859939~2311241,74007760~333146,82296588~1267250,105261990~3433993,110055987~6404653,144750991~1159758,151953600~842106,189053332~11878686,229534798~3451158,239723295~7920536,269763691~4981986,291073215~751556,332649487~19061486,355907560~4087411,373115509~11964022,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.225 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [24616983~5736360,36062054~6554620,55859939~2311241,74007760~333146,82296588~1267250,105261990~3433993,110055987~6404653,144750991~1159758,151953600~842106,189053332~11878686,229534798~3451158,239723295~7920536,269763691~4981986,291073215~751556,332649487~19061486,355907560~4087411,373115509~11964022,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.234 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [20971520~9381823,36062054~6554620,54525952~3645228,71769981~2570925,82296588~1267250,104857600~3838383,110055987~6404653,144750991~1159758,151953600~842106,189053332~11878686,229534798~3451158,239723295~7920536,268435456~6310221,291073215~751556,332649487~19061486,355907560~4087411,373115509~11964022,386636428~2192208,394264576~6106732,406847488~8360895] 2024-08-17T17:41:58.235 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 8 diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~6554620,55859939~2311241,74007760~333146,82296588~1267250,90175397~5338809,105261990~3433993,110055987~6404653,128194814~7129965,137226312~8684437,151953600~842106,177491177~2771546,189053332~11878686,229534798~3451158,239723295~7920536,269763691~4981986,291073215~751556,322650436~784308,332649487~22386665,355907560~4087411,373115509~11964022,385791049~1661831,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.235 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~6554620,55859939~2311241,74007760~333146,82296588~1267250,90175397~5338809,105261990~3433993,110055987~6404653,128194814~7129965,137226312~8684437,151953600~842106,177491177~2771546,189053332~11878686,229534798~3451158,239723295~7920536,269763691~4981986,291073215~751556,322650436~784308,332649487~22386665,355907560~4087411,373115509~11964022,385791049~1661831,388486096~342540,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.236 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,36062054~6554620,54525952~3645228,71769981~2570925,82296588~1267250,89900725~5613481,104857600~3838383,110055987~6404653,125829120~20081629,151953600~842106,177491177~2771546,189053332~11878686,229534798~3451158,239723295~7920536,268435456~6310221,291073215~751556,318767104~4667640,332649487~22386665,355907560~4087411,373115509~11964022,385791049~3037587,394264576~6106732,406847488~8360895] 2024-08-17T17:41:58.236 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 9 diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~6554620,55859939~2311241,68742908~1813321,74007760~333146,82296588~1267250,90175397~5338809,105261990~3433993,110055987~6404653,127412771~18497978,151953600~842106,155757889~6839948,177491177~2771546,189053332~15245889,229534798~3451158,239723295~12307662,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~22386665,355907560~4087411,373115509~11964022,385791049~1661831,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.236 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~6554620,55859939~2311241,68742908~1813321,74007760~333146,82296588~1267250,90175397~5338809,105261990~3433993,110055987~6404653,127412771~18497978,151953600~842106,155757889~6839948,177491177~2771546,189053332~15245889,229534798~3451158,239723295~12307662,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~22386665,355907560~4087411,373115509~11964022,385791049~1661831,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.241 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,36062054~6554620,54525952~3645228,68742908~1813321,71769981~2570925,82296588~1267250,89900725~5613481,104857600~3838383,110055987~6404653,125829120~20081629,151953600~842106,155757889~6839948,177491177~2771546,189053332~15245889,229534798~3451158,239723295~12307662,268435456~6310221,291073215~751556,310378496~4846333,318767104~4667640,332649487~22386665,355907560~4087411,373115509~11964022,385791049~3037587,394264576~20943807] 2024-08-17T17:41:58.241 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 10 diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~6554620,52172688~3245528,55859939~2311241,58199577~3721268,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,105261990~3433993,110055987~6404653,127412771~21883557,151953600~842106,155540399~7057438,166058561~2977181,177491177~2771546,189053332~15245889,229534798~3451158,239723295~12307662,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~22386665,355907560~4087411,373115509~11964022,385791049~1661831,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.241 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,24616983~5736360,31503479~1175130,36062054~6554620,52172688~3245528,55859939~2311241,58199577~3721268,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,105261990~3433993,110055987~6404653,127412771~21883557,151953600~842106,155540399~7057438,166058561~2977181,177491177~2771546,189053332~15245889,229534798~3451158,239723295~12307662,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~22386665,355907560~4087411,373115509~11964022,385791049~1661831,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.246 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,20971520~9381823,30982143~1696466,36062054~6554620,52172688~9748157,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,85296870~4252304,89900725~5613481,104857600~3838383,110055987~6404653,125829120~23467208,151953600~842106,155540399~7057438,166058561~2977181,177491177~2771546,189053332~15245889,229534798~3451158,239723295~12307662,268435456~6310221,291073215~751556,310378496~4846333,318767104~4667640,332649487~22386665,355907560~4087411,373115509~11964022,385791049~3037587,394264576~20943807] 2024-08-17T17:41:58.246 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 11 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10210967,52172688~3245528,55859939~6293284,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,105261990~3433993,110055987~6404653,127412771~21883557,151953600~842106,155540399~7057438,165571375~3464367,177491177~2771546,189053332~15245889,212123767~2607614,224516274~2118389,229534798~3451158,238191726~13839231,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~22386665,355907560~4087411,373115509~11964022,385791049~1661831,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.246 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10210967,52172688~3245528,55859939~6293284,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,105261990~3433993,110055987~6404653,127412771~21883557,151953600~842106,155540399~7057438,165571375~3464367,177491177~2771546,189053332~15245889,212123767~2607614,224516274~2118389,229534798~3451158,238191726~13839231,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~22386665,355907560~4087411,373115509~11964022,385791049~1661831,388486096~342540,395994076~19214307] 2024-08-17T17:41:58.251 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~12650277,52172688~9980535,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,85296870~4252304,89900725~5613481,104857600~3838383,110055987~6404653,125829120~23467208,151953600~842106,155540399~7057438,165571375~3464367,177491177~2771546,189053332~15245889,212123767~2607614,224516274~2118389,229534798~3451158,235227886~16803071,268435456~6310221,291073215~751556,310378496~4846333,318767104~4667640,332649487~22386665,355907560~4087411,373115509~11964022,385791049~3037587,394264576~20943807] 2024-08-17T17:41:58.251 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 12 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10210967,52172688~3245528,55859939~6293284,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,101692190~7003793,110055987~6404653,127412771~21883557,151953600~842106,153615532~8982305,165571375~3464367,177491177~2771546,189053332~15245889,212123767~2607614,217652007~2893566,224516274~2118389,228885435~4100521,238191726~13839231,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~27345484,373115509~11964022,385791049~1661831,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.251 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~10210967,52172688~3245528,55859939~6293284,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,101692190~7003793,110055987~6404653,127412771~21883557,151953600~842106,153615532~8982305,165571375~3464367,177491177~2771546,189053332~15245889,212123767~2607614,217652007~2893566,224516274~2118389,228885435~4100521,238191726~13839231,269763691~4981986,291073215~751556,312958845~2265984,322650436~784308,332649487~27345484,373115509~11964022,385791049~1661831,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.257 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~12650277,52172688~9980535,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,85296870~4252304,89900725~5613481,100663296~8032687,110055987~6404653,125829120~23467208,151953600~842106,153615532~8982305,165571375~3464367,177491177~2771546,189053332~15245889,212123767~2607614,215930165~4615408,224516274~2118389,228885435~4100521,235227886~16803071,268435456~6310221,291073215~751556,310378496~4846333,318767104~4667640,332649487~27345484,373115509~11964022,385791049~3037587,390070272~1830474,394264576~20943807] 2024-08-17T17:41:58.257 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 13 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~15075986,52172688~3245528,55859939~6293284,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,85296870~4252304,90097398~5457279,99526650~9169333,110055987~6404653,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,217652007~2893566,224516274~2118389,228885435~4100521,238191726~13839231,252773028~8082862,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~27345484,373115509~11964022,385791049~1661831,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.257 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~15075986,52172688~3245528,55859939~6293284,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,85296870~4252304,90097398~5457279,99526650~9169333,110055987~6404653,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,217652007~2893566,224516274~2118389,228885435~4100521,238191726~13839231,252773028~8082862,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~27345484,373115509~11964022,385791049~1661831,388486096~342540,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.262 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~17515296,52172688~9980535,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,85296870~4252304,89900725~5653952,99526650~9169333,110055987~6404653,125829120~23467208,151953600~842106,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,215930165~4615408,224516274~2118389,228885435~4100521,235227886~16803071,252773028~8082862,268435456~6310221,291073215~751556,294440898~6981222,310378496~4846333,318767104~4667640,332649487~27345484,373115509~11964022,385791049~3037587,390070272~1830474,394264576~20943807] 2024-08-17T17:41:58.262 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 14 diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,85296870~4252304,90097398~26363242,118553273~1355301,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~27345484,373115509~19018775,395994076~19214307] 2024-08-17T17:41:58.262 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,24616983~5736360,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,85296870~4252304,90097398~26363242,118553273~1355301,127412771~21883557,151953600~842106,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~27345484,373115509~19018775,395994076~19214307] 2024-08-17T17:41:58.268 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,20971520~9381823,30982143~17515296,52172688~9980535,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,85296870~4252304,89900725~26559915,118553273~1355301,125829120~23467208,151953600~842106,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,215930165~10704498,228885435~4100521,235227886~16803071,252773028~8082862,261307577~1507943,268435456~6310221,291073215~751556,294440898~6981222,310378496~4846333,318767104~4667640,332649487~27345484,373115509~19018775,394264576~20943807] 2024-08-17T17:41:58.268 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 15 diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,85296870~4252304,90097398~26363242,118553273~1355301,127412771~21883557,150192064~2603642,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~27345484,366925156~3770110,373115509~19018775,395994076~19214307] 2024-08-17T17:41:58.268 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,85296870~4252304,90097398~26363242,118553273~1355301,127412771~21883557,150192064~2603642,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~4981986,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,332649487~27345484,366925156~3770110,373115509~19018775,395994076~19214307] 2024-08-17T17:41:58.273 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~17515296,52172688~9980535,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,85296870~4252304,89900725~26559915,118553273~1355301,125829120~26966586,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,215930165~10704498,228885435~4100521,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,268435456~6310221,291073215~751556,294440898~6981222,310378496~4846333,318767104~4667640,332649487~27345484,366925156~3770110,373115509~19018775,394264576~20943807] 2024-08-17T17:41:58.273 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 16 diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,85296870~4252304,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,330570284~29424687,366925156~3770110,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.273 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~18041393,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,74007760~333146,75676394~6230067,82296588~1267250,85296870~4252304,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,294440898~6981222,312958845~2265984,322650436~784308,330570284~29424687,366925156~3770110,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.279 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~17515296,52172688~9980535,68742908~1813321,71769981~2570925,75497472~6408989,82296588~1267250,85296870~4252304,89900725~26559915,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,212123767~2607614,215930165~10704498,228885435~4100521,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,268435456~10106105,291073215~751556,294440898~6981222,310378496~4846333,318767104~4667640,327155712~32839259,366925156~3770110,373115509~42092874] 2024-08-17T17:41:58.279 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 17 diff [6122489~2117206,8405456~21947887,31503479~1175130,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,85296870~4252304,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,294440898~6981222,312958845~6612414,322650436~784308,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.279 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~21947887,31503479~1175130,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,85296870~4252304,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,291073215~751556,294440898~6981222,312958845~6612414,322650436~784308,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.285 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30982143~19029698,52172688~9980535,68742908~1813321,71769981~3294085,75497472~6408989,82296588~1267250,85296870~4252304,89900725~26559915,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,212123767~2607614,215930165~10704498,228885435~4100521,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,268435456~10106105,291073215~751556,294440898~6981222,310378496~13056248,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:41:58.285 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 18 diff [6122489~2117206,8405456~21947887,30955299~1723310,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,85296870~4252304,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~6981222,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.285 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~21947887,30955299~1723310,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,85296870~4252304,90097398~26363242,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~6981222,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.290 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~21964735,30955299~19056542,52172688~9980535,68742908~1813321,71769981~3294085,75497472~6408989,82296588~1267250,85296870~4252304,89900725~26559915,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~6981222,301989888~6880813,310378496~13056248,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:41:58.290 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 19 diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.290 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.295 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:41:58.295 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 6 diff [55859939~779155,189053332~11878686,229534798~3451158,239723295~7920536,269763691~4981986,342747177~8963796,355907560~4087411] 2024-08-17T17:41:58.296 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [55859939~779155,189053332~11878686,229534798~3451158,239723295~7920536,269763691~4981986,342747177~8963796,355907560~4087411] 2024-08-17T17:41:58.301 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [54525952~2113142,189053332~11878686,229534798~3451158,239723295~7920536,268435456~6310221,342747177~8963796,355907560~4087411] 2024-08-17T17:41:58.301 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 7 diff [36062054~6554620,55859939~2311241,82296588~1267250,110055987~6404653,189053332~11878686,229534798~3451158,239723295~7920536,269763691~4981986,333022625~6072123,342747177~8963796,355907560~4087411,373115509~11964022,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.301 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [36062054~6554620,55859939~2311241,82296588~1267250,110055987~6404653,189053332~11878686,229534798~3451158,239723295~7920536,269763691~4981986,333022625~6072123,342747177~8963796,355907560~4087411,373115509~11964022,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.306 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [36062054~6554620,54525952~3645228,82296588~1267250,110055987~6404653,189053332~11878686,229534798~3451158,239723295~7920536,268435456~6310221,333022625~6072123,342747177~8963796,355907560~4087411,373115509~11964022,394264576~6106732,406847488~8360895] 2024-08-17T17:41:58.306 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 8 diff [12311950~2412776,31503479~1175130,36062054~6554620,55859939~2311241,82296588~1267250,90175397~5338809,110055987~6404653,128194814~7129965,137226312~7668218,177491177~2771546,189053332~11878686,229534798~3451158,239723295~7920536,269763691~4981986,322650436~784308,333022625~6072123,342747177~12288975,355907560~4087411,373115509~11964022,385791049~1661831,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.306 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,31503479~1175130,36062054~6554620,55859939~2311241,82296588~1267250,90175397~5338809,110055987~6404653,128194814~7129965,137226312~7668218,177491177~2771546,189053332~11878686,229534798~3451158,239723295~7920536,269763691~4981986,322650436~784308,333022625~6072123,342747177~12288975,355907560~4087411,373115509~11964022,385791049~1661831,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.311 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,30982143~1696466,36062054~6554620,54525952~3645228,82296588~1267250,89900725~5613481,110055987~6404653,125829120~19065410,177491177~2771546,189053332~11878686,229534798~3451158,239723295~7920536,268435456~6310221,318767104~4667640,333022625~6072123,342747177~12288975,355907560~4087411,373115509~11964022,385791049~1661831,394264576~6106732,406847488~8360895] 2024-08-17T17:41:58.311 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 9 diff [12311950~2412776,31503479~1175130,36062054~6554620,55859939~2311241,68742908~1813321,82296588~1267250,90175397~5338809,110055987~6404653,127412771~17481759,155757889~6839948,177491177~2771546,189053332~15245889,229534798~3451158,239723295~12307662,269763691~4981986,312958845~2265984,322650436~784308,333022625~6072123,342747177~12288975,355907560~4087411,373115509~11964022,385791049~1661831,395994076~19214307] 2024-08-17T17:41:58.311 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,31503479~1175130,36062054~6554620,55859939~2311241,68742908~1813321,82296588~1267250,90175397~5338809,110055987~6404653,127412771~17481759,155757889~6839948,177491177~2771546,189053332~15245889,229534798~3451158,239723295~12307662,269763691~4981986,312958845~2265984,322650436~784308,333022625~6072123,342747177~12288975,355907560~4087411,373115509~11964022,385791049~1661831,395994076~19214307] 2024-08-17T17:41:58.316 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,30982143~1696466,36062054~6554620,54525952~3645228,68742908~1813321,82296588~1267250,89900725~5613481,110055987~6404653,125829120~19065410,155757889~6839948,177491177~2771546,189053332~15245889,229534798~3451158,239723295~12307662,268435456~6310221,310378496~4846333,318767104~4667640,333022625~6072123,342747177~12288975,355907560~4087411,373115509~11964022,385791049~1661831,394264576~20943807] 2024-08-17T17:41:58.316 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 10 diff [12311950~2412776,31503479~1175130,36062054~6554620,52172688~3245528,55859939~2311241,58199577~3721268,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,110055987~6404653,127412771~21883557,155540399~7057438,166058561~2977181,177491177~2771546,189053332~15245889,229534798~3451158,239723295~12307662,269763691~4981986,312958845~2265984,322650436~784308,333022625~6072123,342747177~12288975,355907560~4087411,373115509~11964022,385791049~1661831,395994076~19214307] 2024-08-17T17:41:58.317 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,31503479~1175130,36062054~6554620,52172688~3245528,55859939~2311241,58199577~3721268,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,110055987~6404653,127412771~21883557,155540399~7057438,166058561~2977181,177491177~2771546,189053332~15245889,229534798~3451158,239723295~12307662,269763691~4981986,312958845~2265984,322650436~784308,333022625~6072123,342747177~12288975,355907560~4087411,373115509~11964022,385791049~1661831,395994076~19214307] 2024-08-17T17:41:58.321 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,30982143~1696466,36062054~6554620,52172688~9748157,68742908~1813321,75497472~6408989,82296588~1267250,85296870~4252304,89900725~5613481,110055987~6404653,125829120~23467208,155540399~7057438,166058561~2977181,177491177~2771546,189053332~15245889,229534798~3451158,239723295~12307662,268435456~6310221,310378496~4846333,318767104~4667640,333022625~6072123,342747177~12288975,355907560~4087411,373115509~11964022,385791049~1661831,394264576~20943807] 2024-08-17T17:41:58.322 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 11 diff [12311950~6523919,31503479~1175130,33421453~10210967,52172688~3245528,55859939~6293284,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,110055987~6404653,127412771~21883557,155540399~7057438,165571375~3464367,177491177~2771546,189053332~15245889,212123767~2607614,224516274~2118389,229534798~3451158,238191726~13839231,269763691~4981986,312958845~2265984,322650436~784308,333022625~6072123,342747177~12288975,355907560~4087411,373115509~11964022,385791049~1661831,395994076~19214307] 2024-08-17T17:41:58.322 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,31503479~1175130,33421453~10210967,52172688~3245528,55859939~6293284,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,110055987~6404653,127412771~21883557,155540399~7057438,165571375~3464367,177491177~2771546,189053332~15245889,212123767~2607614,224516274~2118389,229534798~3451158,238191726~13839231,269763691~4981986,312958845~2265984,322650436~784308,333022625~6072123,342747177~12288975,355907560~4087411,373115509~11964022,385791049~1661831,395994076~19214307] 2024-08-17T17:41:58.327 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,30982143~12650277,52172688~9980535,68742908~1813321,75497472~6408989,82296588~1267250,85296870~4252304,89900725~5613481,110055987~6404653,125829120~23467208,155540399~7057438,165571375~3464367,177491177~2771546,189053332~15245889,212123767~2607614,224516274~2118389,229534798~3451158,235227886~16803071,268435456~6310221,310378496~4846333,318767104~4667640,333022625~6072123,342747177~12288975,355907560~4087411,373115509~11964022,385791049~1661831,394264576~20943807] 2024-08-17T17:41:58.327 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 12 diff [12311950~6523919,31503479~1175130,33421453~10210967,52172688~3245528,55859939~6293284,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,101692190~4549633,110055987~6404653,127412771~21883557,153615532~8982305,165571375~3464367,177491177~2771546,189053332~15245889,212123767~2607614,217652007~2893566,224516274~2118389,228885435~4100521,238191726~13839231,269763691~4981986,312958845~2265984,322650436~784308,333022625~6072123,342747177~17247794,373115509~11964022,385791049~1661831,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.327 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,31503479~1175130,33421453~10210967,52172688~3245528,55859939~6293284,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,101692190~4549633,110055987~6404653,127412771~21883557,153615532~8982305,165571375~3464367,177491177~2771546,189053332~15245889,212123767~2607614,217652007~2893566,224516274~2118389,228885435~4100521,238191726~13839231,269763691~4981986,312958845~2265984,322650436~784308,333022625~6072123,342747177~17247794,373115509~11964022,385791049~1661831,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.332 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,30982143~12650277,52172688~9980535,68742908~1813321,75497472~6408989,82296588~1267250,85296870~4252304,89900725~5613481,100663296~5578527,110055987~6404653,125829120~23467208,153615532~8982305,165571375~3464367,177491177~2771546,189053332~15245889,212123767~2607614,215930165~4615408,224516274~2118389,228885435~4100521,235227886~16803071,268435456~6310221,310378496~4846333,318767104~4667640,333022625~6072123,342747177~17247794,373115509~11964022,385791049~1661831,390070272~1830474,394264576~20943807] 2024-08-17T17:41:58.332 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 13 diff [12311950~6523919,31503479~1175130,33421453~15075986,52172688~3245528,55859939~6293284,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~5457279,99526650~6715173,110055987~6404653,127412771~21883557,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,217652007~2893566,224516274~2118389,228885435~4100521,238191726~13839231,252773028~8082862,269763691~4981986,294440898~6981222,312958845~2265984,322650436~784308,333022625~6072123,342747177~17247794,373115509~11964022,385791049~1661831,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.332 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,31503479~1175130,33421453~15075986,52172688~3245528,55859939~6293284,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~5457279,99526650~6715173,110055987~6404653,127412771~21883557,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,217652007~2893566,224516274~2118389,228885435~4100521,238191726~13839231,252773028~8082862,269763691~4981986,294440898~6981222,312958845~2265984,322650436~784308,333022625~6072123,342747177~17247794,373115509~11964022,385791049~1661831,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.337 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,30982143~17515296,52172688~9980535,68742908~1813321,75497472~6408989,82296588~1267250,85296870~4252304,89900725~5653952,99526650~6715173,110055987~6404653,125829120~23467208,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,215930165~4615408,224516274~2118389,228885435~4100521,235227886~16803071,252773028~8082862,268435456~6310221,294440898~6981222,310378496~4846333,318767104~4667640,333022625~6072123,342747177~17247794,373115509~11964022,385791049~1661831,390070272~1830474,394264576~20943807] 2024-08-17T17:41:58.338 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 14 diff [12311950~6523919,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118553273~1355301,127412771~21883557,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,269763691~4981986,294440898~6981222,312958845~2265984,322650436~784308,333022625~6072123,342747177~17247794,373115509~19018775,395994076~19214307] 2024-08-17T17:41:58.338 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118553273~1355301,127412771~21883557,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,269763691~4981986,294440898~6981222,312958845~2265984,322650436~784308,333022625~6072123,342747177~17247794,373115509~19018775,395994076~19214307] 2024-08-17T17:41:58.343 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,30982143~17515296,52172688~9980535,68742908~1813321,75497472~6408989,82296588~1267250,85296870~4252304,89900725~16341098,108486098~7974542,118553273~1355301,125829120~23467208,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,215930165~10704498,228885435~4100521,235227886~16803071,252773028~8082862,261307577~1507943,268435456~6310221,294440898~6981222,310378496~4846333,318767104~4667640,333022625~6072123,342747177~17247794,373115509~19018775,394264576~20943807] 2024-08-17T17:41:58.343 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 15 diff [6122489~2117206,9582259~1513694,12311950~12673759,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118553273~1355301,127412771~21883557,150192064~2291333,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~4981986,294440898~6981222,312958845~2265984,322650436~784308,333022625~6072123,342747177~17247794,366925156~3770110,373115509~19018775,395994076~19214307] 2024-08-17T17:41:58.343 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~12673759,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118553273~1355301,127412771~21883557,150192064~2291333,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~4981986,294440898~6981222,312958845~2265984,322650436~784308,333022625~6072123,342747177~17247794,366925156~3770110,373115509~19018775,395994076~19214307] 2024-08-17T17:41:58.348 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~16597101,30982143~17515296,52172688~9980535,68742908~1813321,75497472~6408989,82296588~1267250,85296870~4252304,89900725~16341098,108486098~7974542,118553273~1355301,125829120~26654277,153615532~8982305,165571375~9681198,177491177~3287978,189053332~15245889,212123767~2607614,215930165~10704498,228885435~4100521,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,268435456~6310221,294440898~6981222,310378496~4846333,318767104~4667640,333022625~6072123,342747177~17247794,366925156~3770110,373115509~19018775,394264576~20943807] 2024-08-17T17:41:58.348 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 16 diff [6122489~2117206,9582259~1513694,12311950~12673759,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,294440898~6981222,312958845~2265984,322650436~784308,330570284~8524464,342747177~17247794,366925156~3770110,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.348 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~12673759,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,294440898~6981222,312958845~2265984,322650436~784308,330570284~8524464,342747177~17247794,366925156~3770110,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.353 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~16597101,30982143~17515296,52172688~9980535,68742908~1813321,75497472~6408989,82296588~1267250,85296870~4252304,89900725~16341098,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,212123767~2607614,215930165~10704498,228885435~4100521,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,268435456~10106105,294440898~6981222,310378496~4846333,318767104~4667640,327155712~11939036,342747177~17247794,366925156~3770110,373115509~42092874] 2024-08-17T17:41:58.353 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 17 diff [6122489~2117206,8405456~16580253,31503479~1175130,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,294440898~6981222,312958845~6612414,322650436~784308,330570284~8524464,342747177~17247794,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.354 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~16580253,31503479~1175130,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,212123767~2607614,217652007~8982656,228885435~4100521,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,269763691~8777870,294440898~6981222,312958845~6612414,322650436~784308,330570284~8524464,342747177~17247794,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.359 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~16597101,30982143~19029698,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6408989,82296588~1267250,85296870~4252304,89900725~16341098,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,212123767~2607614,215930165~10704498,228885435~4100521,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,268435456~10106105,294440898~6981222,310378496~13056248,327155712~11939036,342747177~17247794,362603833~8091433,373115509~42092874] 2024-08-17T17:41:58.359 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 18 diff [6122489~2117206,8405456~16580253,30955299~1723310,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~8524464,342747177~17247794,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.359 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~16580253,30955299~1723310,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~8524464,342747177~17247794,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.364 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~16597101,30955299~19056542,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6408989,82296588~1267250,85296870~4252304,89900725~16341098,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~6981222,301989888~6880813,310378496~13056248,327155712~11939036,342747177~17247794,362603833~8091433,373115509~42092874] 2024-08-17T17:41:58.364 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 19 diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.364 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.369 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:41:58.369 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 7 diff [36062054~6554620,56464863~1706317,82296588~1267250,110055987~6404653,333022625~6072123,373115509~11964022,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.370 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [36062054~6554620,56464863~1706317,82296588~1267250,110055987~6404653,333022625~6072123,373115509~11964022,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.375 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [36062054~6554620,56464863~1706317,82296588~1267250,110055987~6404653,333022625~6072123,373115509~11964022,394264576~6106732,406847488~8360895] 2024-08-17T17:41:58.375 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 8 diff [12311950~2412776,31503479~1175130,36062054~6554620,56464863~1706317,82296588~1267250,90175397~5338809,110055987~6404653,128194814~7129965,137226312~7668218,177491177~2771546,322650436~784308,333022625~6072123,350982941~4053211,356919771~997495,373115509~11964022,385791049~1661831,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.375 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,31503479~1175130,36062054~6554620,56464863~1706317,82296588~1267250,90175397~5338809,110055987~6404653,128194814~7129965,137226312~7668218,177491177~2771546,322650436~784308,333022625~6072123,350982941~4053211,356919771~997495,373115509~11964022,385791049~1661831,395994076~4377232,407841230~7367153] 2024-08-17T17:41:58.380 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,30982143~1696466,36062054~6554620,56464863~1706317,82296588~1267250,89900725~5613481,110055987~6404653,125829120~19065410,177491177~2771546,318767104~4667640,333022625~6072123,350982941~4053211,356919771~997495,373115509~11964022,385791049~1661831,394264576~6106732,406847488~8360895] 2024-08-17T17:41:58.380 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 9 diff [12311950~2412776,31503479~1175130,36062054~6554620,56464863~1706317,68742908~1813321,82296588~1267250,90175397~5338809,110055987~6404653,127412771~17481759,155757889~6839948,177491177~2771546,197702108~6597113,244002673~773270,247070758~4960199,312958845~2265984,322650436~784308,333022625~6072123,350982941~4053211,356919771~997495,373115509~11964022,385791049~1661831,395994076~19214307] 2024-08-17T17:41:58.380 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,31503479~1175130,36062054~6554620,56464863~1706317,68742908~1813321,82296588~1267250,90175397~5338809,110055987~6404653,127412771~17481759,155757889~6839948,177491177~2771546,197702108~6597113,244002673~773270,247070758~4960199,312958845~2265984,322650436~784308,333022625~6072123,350982941~4053211,356919771~997495,373115509~11964022,385791049~1661831,395994076~19214307] 2024-08-17T17:41:58.385 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,30982143~1696466,36062054~6554620,56464863~1706317,68742908~1813321,82296588~1267250,89900725~5613481,110055987~6404653,125829120~19065410,155757889~6839948,177491177~2771546,197702108~6597113,244002673~773270,247070758~4960199,310378496~4846333,318767104~4667640,333022625~6072123,350982941~4053211,356919771~997495,373115509~11964022,385791049~1661831,394264576~20943807] 2024-08-17T17:41:58.385 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 10 diff [12311950~2412776,31503479~1175130,36062054~6554620,52172688~3245528,56464863~1706317,58199577~3721268,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,110055987~6404653,127412771~21883557,155540399~7057438,166058561~2977181,177491177~2771546,197702108~6597113,240858522~5051700,247070758~4960199,272182852~2470550,312958845~2265984,322650436~784308,333022625~6072123,350982941~4053211,356919771~997495,373115509~11964022,385791049~1661831,395994076~19214307] 2024-08-17T17:41:58.385 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,31503479~1175130,36062054~6554620,52172688~3245528,56464863~1706317,58199577~3721268,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,110055987~6404653,127412771~21883557,155540399~7057438,166058561~2977181,177491177~2771546,197702108~6597113,240858522~5051700,247070758~4960199,272182852~2470550,312958845~2265984,322650436~784308,333022625~6072123,350982941~4053211,356919771~997495,373115509~11964022,385791049~1661831,395994076~19214307] 2024-08-17T17:41:58.390 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,30982143~1696466,36062054~6554620,52172688~3245528,56464863~5455982,68742908~1813321,75497472~6408989,82296588~1267250,85296870~4252304,89900725~5613481,110055987~6404653,125829120~23467208,155540399~7057438,166058561~2977181,177491177~2771546,197702108~6597113,240858522~5051700,247070758~4960199,272182852~2470550,310378496~4846333,318767104~4667640,333022625~6072123,350982941~4053211,356919771~997495,373115509~11964022,385791049~1661831,394264576~20943807] 2024-08-17T17:41:58.390 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 11 diff [12311950~6523919,31503479~1175130,33421453~10210967,52172688~3245528,56464863~5688360,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,110055987~6404653,127412771~21883557,155540399~7057438,165571375~3464367,177491177~2771546,197702108~6597113,212123767~2607614,224516274~2118389,238191726~7718496,247070758~4960199,272182852~2470550,312958845~2265984,322650436~784308,333022625~6072123,350982941~4053211,356919771~997495,373115509~11964022,385791049~1661831,395994076~19214307] 2024-08-17T17:41:58.390 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,31503479~1175130,33421453~10210967,52172688~3245528,56464863~5688360,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,110055987~6404653,127412771~21883557,155540399~7057438,165571375~3464367,177491177~2771546,197702108~6597113,212123767~2607614,224516274~2118389,238191726~7718496,247070758~4960199,272182852~2470550,312958845~2265984,322650436~784308,333022625~6072123,350982941~4053211,356919771~997495,373115509~11964022,385791049~1661831,395994076~19214307] 2024-08-17T17:41:58.395 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,30982143~12650277,52172688~3245528,56464863~5688360,68742908~1813321,75497472~6408989,82296588~1267250,85296870~4252304,89900725~5613481,110055987~6404653,125829120~23467208,155540399~7057438,165571375~3464367,177491177~2771546,197702108~6597113,212123767~2607614,224516274~2118389,235227886~10682336,247070758~4960199,272182852~2470550,310378496~4846333,318767104~4667640,333022625~6072123,350982941~4053211,356919771~997495,373115509~11964022,385791049~1661831,394264576~20943807] 2024-08-17T17:41:58.396 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 12 diff [12311950~6523919,31503479~1175130,33421453~10210967,52172688~3245528,56464863~5688360,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,101692190~4549633,110055987~6404653,127412771~21883557,153615532~8982305,165571375~3464367,177491177~2771546,197702108~6597113,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~13839231,272182852~2470550,312958845~2265984,322650436~784308,333022625~6072123,350982941~7359087,373115509~11964022,385791049~1661831,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.396 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,31503479~1175130,33421453~10210967,52172688~3245528,56464863~5688360,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90175397~5338809,101692190~4549633,110055987~6404653,127412771~21883557,153615532~8982305,165571375~3464367,177491177~2771546,197702108~6597113,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~13839231,272182852~2470550,312958845~2265984,322650436~784308,333022625~6072123,350982941~7359087,373115509~11964022,385791049~1661831,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.401 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,30982143~12650277,52172688~3245528,56464863~5688360,68742908~1813321,75497472~6408989,82296588~1267250,85296870~4252304,89900725~5613481,100663296~5578527,110055987~6404653,125829120~23467208,153615532~8982305,165571375~3464367,177491177~2771546,197702108~6597113,212123767~2607614,215930165~4615408,224516274~2118389,228885435~3355214,235227886~16803071,272182852~2470550,310378496~4846333,318767104~4667640,333022625~6072123,350982941~7359087,373115509~11964022,385791049~1661831,390070272~1830474,394264576~20943807] 2024-08-17T17:41:58.401 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 13 diff [12311950~6523919,31503479~1175130,33421453~15075986,52172688~3245528,56464863~5688360,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~5457279,99526650~6715173,110055987~6404653,127412771~21883557,153615532~8982305,165571375~9681198,177491177~3287978,197702108~6597113,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~13839231,252773028~8082862,272182852~2470550,294440898~6981222,312958845~2265984,322650436~784308,333022625~6072123,350982941~7359087,373115509~11964022,385791049~1661831,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.401 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,31503479~1175130,33421453~15075986,52172688~3245528,56464863~5688360,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~5457279,99526650~6715173,110055987~6404653,127412771~21883557,153615532~8982305,165571375~9681198,177491177~3287978,197702108~6597113,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~13839231,252773028~8082862,272182852~2470550,294440898~6981222,312958845~2265984,322650436~784308,333022625~6072123,350982941~7359087,373115509~11964022,385791049~1661831,390263701~1637045,395994076~19214307] 2024-08-17T17:41:58.406 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,30982143~17515296,52172688~3245528,56464863~5688360,68742908~1813321,75497472~6408989,82296588~1267250,85296870~4252304,89900725~5653952,99526650~6715173,110055987~6404653,125829120~23467208,153615532~8982305,165571375~9681198,177491177~3287978,197702108~6597113,212123767~2607614,215930165~4615408,224516274~2118389,228885435~3355214,235227886~16803071,252773028~8082862,272182852~2470550,294440898~6981222,310378496~4846333,318767104~4667640,333022625~6072123,350982941~7359087,373115509~11964022,385791049~1661831,390070272~1830474,394264576~20943807] 2024-08-17T17:41:58.407 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 14 diff [12311950~6523919,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118553273~1355301,127412771~21883557,153615532~8982305,165571375~9681198,177491177~3287978,197702108~6597113,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,272182852~2470550,294440898~6981222,312958845~2265984,322650436~784308,333022625~6072123,350982941~7359087,373115509~19018775,395994076~19214307] 2024-08-17T17:41:58.407 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118553273~1355301,127412771~21883557,153615532~8982305,165571375~9681198,177491177~3287978,197702108~6597113,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,272182852~2470550,294440898~6981222,312958845~2265984,322650436~784308,333022625~6072123,350982941~7359087,373115509~19018775,395994076~19214307] 2024-08-17T17:41:58.412 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,30982143~17515296,52172688~9980535,68742908~1813321,75497472~6408989,82296588~1267250,85296870~4252304,89900725~16341098,108486098~7974542,118553273~1355301,125829120~23467208,153615532~8982305,165571375~9681198,177491177~3287978,197702108~6597113,212123767~2607614,215930165~10704498,228885435~3355214,235227886~16803071,252773028~8082862,261307577~1507943,272182852~2470550,294440898~6981222,310378496~4846333,318767104~4667640,333022625~6072123,350982941~7359087,373115509~19018775,394264576~20943807] 2024-08-17T17:41:58.412 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 15 diff [6122489~2117206,9582259~1513694,12311950~12673759,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118553273~1355301,127412771~21883557,150192064~2291333,153615532~8982305,165571375~9681198,177491177~3287978,191934227~5441303,197702108~6597113,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~2470550,294440898~6981222,312958845~2265984,322650436~784308,333022625~6072123,348569567~9772461,366925156~3770110,373115509~19018775,395994076~19214307] 2024-08-17T17:41:58.412 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~12673759,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118553273~1355301,127412771~21883557,150192064~2291333,153615532~8982305,165571375~9681198,177491177~3287978,191934227~5441303,197702108~6597113,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~2470550,294440898~6981222,312958845~2265984,322650436~784308,333022625~6072123,348569567~9772461,366925156~3770110,373115509~19018775,395994076~19214307] 2024-08-17T17:41:58.417 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~16597101,30982143~17515296,52172688~9980535,68742908~1813321,75497472~6408989,82296588~1267250,85296870~4252304,89900725~16341098,108486098~7974542,118553273~1355301,125829120~26654277,153615532~8982305,165571375~9681198,177491177~3287978,191934227~5441303,197702108~6597113,212123767~2607614,215930165~10704498,228885435~3355214,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,272182852~2470550,294440898~6981222,310378496~4846333,318767104~4667640,333022625~6072123,348569567~9772461,366925156~3770110,373115509~19018775,394264576~20943807] 2024-08-17T17:41:58.417 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 16 diff [6122489~2117206,9582259~1513694,12311950~12673759,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,312958845~2265984,322650436~784308,330570284~8524464,348569567~9772461,366925156~3770110,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.417 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~12673759,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,312958845~2265984,322650436~784308,330570284~8524464,348569567~9772461,366925156~3770110,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.423 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~16597101,30982143~17515296,52172688~9980535,68742908~1813321,75497472~6408989,82296588~1267250,85296870~4252304,89900725~16341098,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,215930165~10704498,228885435~3355214,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,272182852~6358709,294440898~6981222,310378496~4846333,318767104~4667640,327155712~11939036,348569567~9772461,366925156~3770110,373115509~42092874] 2024-08-17T17:41:58.423 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 17 diff [6122489~2117206,8405456~16580253,31503479~1175130,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,312958845~6612414,322650436~784308,330570284~8524464,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.423 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~16580253,31503479~1175130,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,312958845~6612414,322650436~784308,330570284~8524464,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.428 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~16597101,30982143~19029698,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6408989,82296588~1267250,85296870~4252304,89900725~16341098,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,215930165~10704498,228885435~3355214,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,272182852~6358709,294440898~6981222,310378496~13056248,327155712~11939036,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:41:58.428 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 18 diff [6122489~2117206,8405456~16580253,30955299~1723310,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~8524464,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.428 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~16580253,30955299~1723310,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,82296588~1267250,85296870~4252304,90097398~16144425,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~8524464,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.433 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~16597101,30955299~19056542,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6408989,82296588~1267250,85296870~4252304,89900725~16341098,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~6981222,301989888~6880813,310378496~13056248,327155712~11939036,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:41:58.433 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 19 diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.434 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:58.442 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~16466976,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:41:58.442 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 8 diff [12311950~2412776,31503479~1175130,90175397~5338809,128194814~7129965,137226312~7668218,177491177~2771546,322650436~784308,350982941~4053211,356919771~997495,385791049~1661831] 2024-08-17T17:41:58.442 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,31503479~1175130,90175397~5338809,128194814~7129965,137226312~7668218,177491177~2771546,322650436~784308,350982941~4053211,356919771~997495,385791049~1661831] 2024-08-17T17:41:58.444 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,30982143~1696466,89900725~5613481,125829120~19065410,177491177~2771546,318767104~4667640,350982941~4053211,356919771~997495,385791049~1661831] 2024-08-17T17:41:58.444 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 9 diff [12311950~2412776,31503479~1175130,68742908~1813321,90175397~5338809,111220774~5205171,127412771~17481759,155757889~6839948,177491177~2771546,197702108~6597113,244002673~773270,247070758~4960199,312958845~2265984,322650436~784308,350982941~4053211,356919771~997495,385791049~1661831,400335542~7506643] 2024-08-17T17:41:58.445 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,31503479~1175130,68742908~1813321,90175397~5338809,111220774~5205171,127412771~17481759,155757889~6839948,177491177~2771546,197702108~6597113,244002673~773270,247070758~4960199,312958845~2265984,322650436~784308,350982941~4053211,356919771~997495,385791049~1661831,400335542~7506643] 2024-08-17T17:41:58.450 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,30982143~1696466,68742908~1813321,89900725~5613481,111220774~5205171,125829120~19065410,155757889~6839948,177491177~2771546,197702108~6597113,244002673~773270,247070758~4960199,310378496~4846333,318767104~4667640,350982941~4053211,356919771~997495,385791049~1661831,400335542~7506643] 2024-08-17T17:41:58.450 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 10 diff [12311950~2412776,31503479~1175130,52172688~3245528,58199577~3721268,68742908~1813321,75676394~6230067,85296870~4252304,90175397~5338809,111220774~5205171,127412771~21883557,155540399~7057438,166058561~2977181,177491177~2771546,197702108~6597113,240858522~5051700,247070758~4960199,272182852~2470550,312958845~2265984,322650436~784308,350982941~4053211,356919771~997495,385791049~1661831,400335542~7506643] 2024-08-17T17:41:58.450 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~2412776,31503479~1175130,52172688~3245528,58199577~3721268,68742908~1813321,75676394~6230067,85296870~4252304,90175397~5338809,111220774~5205171,127412771~21883557,155540399~7057438,166058561~2977181,177491177~2771546,197702108~6597113,240858522~5051700,247070758~4960199,272182852~2470550,312958845~2265984,322650436~784308,350982941~4053211,356919771~997495,385791049~1661831,400335542~7506643] 2024-08-17T17:41:58.455 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~6336118,30982143~1696466,52172688~3245528,58171180~3749665,68742908~1813321,75497472~6408989,85296870~4252304,89900725~5613481,111220774~5205171,125829120~23467208,155540399~7057438,166058561~2977181,177491177~2771546,197702108~6597113,240858522~5051700,247070758~4960199,272182852~2470550,310378496~4846333,318767104~4667640,350982941~4053211,356919771~997495,385791049~1661831,400335542~7506643] 2024-08-17T17:41:58.455 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 11 diff [12311950~6523919,31503479~1175130,33421453~10210967,52172688~3245528,58014695~4138528,68742908~1813321,75676394~6230067,85296870~4252304,90175397~5338809,111220774~5205171,127412771~21883557,155540399~7057438,165571375~3464367,177491177~2771546,197702108~6597113,212123767~2607614,224516274~2118389,238191726~7718496,247070758~4960199,272182852~2470550,312958845~2265984,322650436~784308,350982941~4053211,356919771~997495,385791049~1661831,400335542~7506643,409521161~4155949] 2024-08-17T17:41:58.455 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,31503479~1175130,33421453~10210967,52172688~3245528,58014695~4138528,68742908~1813321,75676394~6230067,85296870~4252304,90175397~5338809,111220774~5205171,127412771~21883557,155540399~7057438,165571375~3464367,177491177~2771546,197702108~6597113,212123767~2607614,224516274~2118389,238191726~7718496,247070758~4960199,272182852~2470550,312958845~2265984,322650436~784308,350982941~4053211,356919771~997495,385791049~1661831,400335542~7506643,409521161~4155949] 2024-08-17T17:41:58.460 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,30982143~12650277,52172688~3245528,58014695~4138528,68742908~1813321,75497472~6408989,85296870~4252304,89900725~5613481,111220774~5205171,125829120~23467208,155540399~7057438,165571375~3464367,177491177~2771546,197702108~6597113,212123767~2607614,224516274~2118389,235227886~10682336,247070758~4960199,272182852~2470550,310378496~4846333,318767104~4667640,350982941~4053211,356919771~997495,385791049~1661831,400335542~7506643,409521161~4155949] 2024-08-17T17:41:58.460 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 12 diff [12311950~6523919,31503479~1175130,33421453~10210967,52172688~3245528,58014695~4138528,68742908~1813321,75676394~6230067,85296870~4252304,90175397~5338809,101692190~4549633,110253798~6172147,127412771~21883557,153615532~8982305,165571375~3464367,177491177~2771546,197702108~6597113,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~13839231,272182852~2470550,312958845~2265984,322650436~784308,350982941~7359087,385791049~1661831,390263701~1637045,400335542~13341568] 2024-08-17T17:41:58.460 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,31503479~1175130,33421453~10210967,52172688~3245528,58014695~4138528,68742908~1813321,75676394~6230067,85296870~4252304,90175397~5338809,101692190~4549633,110253798~6172147,127412771~21883557,153615532~8982305,165571375~3464367,177491177~2771546,197702108~6597113,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~13839231,272182852~2470550,312958845~2265984,322650436~784308,350982941~7359087,385791049~1661831,390263701~1637045,400335542~13341568] 2024-08-17T17:41:58.466 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,30982143~12650277,52172688~3245528,58014695~4138528,68742908~1813321,75497472~6408989,85296870~4252304,89900725~5613481,100663296~5578527,110253798~6172147,125829120~23467208,153615532~8982305,165571375~3464367,177491177~2771546,197702108~6597113,212123767~2607614,215930165~4615408,224516274~2118389,228885435~3355214,235227886~16803071,272182852~2470550,310378496~4846333,318767104~4667640,350982941~7359087,385791049~1661831,390070272~1830474,400335542~13341568] 2024-08-17T17:41:58.466 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 13 diff [12311950~6523919,31503479~1175130,33421453~15075986,52172688~3245528,58014695~4138528,68742908~1813321,75676394~6230067,85296870~4252304,90097398~5457279,99526650~6715173,110253798~6172147,127412771~21883557,153615532~8982305,165571375~9681198,177491177~3287978,197702108~6597113,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~13839231,252773028~8082862,272182852~2470550,294440898~6981222,312958845~2265984,322650436~784308,350982941~7359087,385791049~1661831,390263701~1637045,400335542~13341568] 2024-08-17T17:41:58.466 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,31503479~1175130,33421453~15075986,52172688~3245528,58014695~4138528,68742908~1813321,75676394~6230067,85296870~4252304,90097398~5457279,99526650~6715173,110253798~6172147,127412771~21883557,153615532~8982305,165571375~9681198,177491177~3287978,197702108~6597113,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~13839231,252773028~8082862,272182852~2470550,294440898~6981222,312958845~2265984,322650436~784308,350982941~7359087,385791049~1661831,390263701~1637045,400335542~13341568] 2024-08-17T17:41:58.471 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,30982143~17515296,52172688~3245528,58014695~4138528,68742908~1813321,75497472~6408989,85296870~4252304,89900725~5653952,99526650~6715173,110253798~6172147,125829120~23467208,153615532~8982305,165571375~9681198,177491177~3287978,197702108~6597113,212123767~2607614,215930165~4615408,224516274~2118389,228885435~3355214,235227886~16803071,252773028~8082862,272182852~2470550,294440898~6981222,310378496~4846333,318767104~4667640,350982941~7359087,385791049~1661831,390070272~1830474,400335542~13341568] 2024-08-17T17:41:58.471 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 14 diff [12311950~6523919,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118553273~1355301,127412771~21883557,153615532~8982305,165571375~9681198,177491177~3287978,197702108~6597113,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,272182852~2470550,294440898~6981222,312958845~2265984,322650436~784308,350982941~7359087,384785269~7349015,400335542~13341568] 2024-08-17T17:41:58.471 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12311950~6523919,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118553273~1355301,127412771~21883557,153615532~8982305,165571375~9681198,177491177~3287978,197702108~6597113,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,272182852~2470550,294440898~6981222,312958845~2265984,322650436~784308,350982941~7359087,384785269~7349015,400335542~13341568] 2024-08-17T17:41:58.476 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~10447261,30982143~17515296,52172688~9980535,68742908~1813321,75497472~6408989,85296870~4252304,89900725~16341098,108486098~7939847,118553273~1355301,125829120~23467208,153615532~8982305,165571375~9681198,177491177~3287978,197702108~6597113,212123767~2607614,215930165~10704498,228885435~3355214,235227886~16803071,252773028~8082862,261307577~1507943,272182852~2470550,294440898~6981222,310378496~4846333,318767104~4667640,350982941~7359087,384785269~7349015,400335542~13341568] 2024-08-17T17:41:58.476 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 15 diff [6122489~2117206,9582259~1513694,12311950~12673759,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118553273~1355301,127412771~21883557,150192064~2291333,153615532~8982305,165571375~9681198,177491177~3287978,191934227~5441303,197702108~6597113,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~2470550,294440898~6981222,312958845~2265984,322650436~784308,348569567~9772461,366925156~3770110,384785269~7349015,400335542~13341568] 2024-08-17T17:41:58.476 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~12673759,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118553273~1355301,127412771~21883557,150192064~2291333,153615532~8982305,165571375~9681198,177491177~3287978,191934227~5441303,197702108~6597113,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~2470550,294440898~6981222,312958845~2265984,322650436~784308,348569567~9772461,366925156~3770110,384785269~7349015,400335542~13341568] 2024-08-17T17:41:58.482 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~16597101,30982143~17515296,52172688~9980535,68742908~1813321,75497472~6408989,85296870~4252304,89900725~16341098,108486098~7939847,118553273~1355301,125829120~26654277,153615532~8982305,165571375~9681198,177491177~3287978,191934227~5441303,197702108~6597113,212123767~2607614,215930165~10704498,228885435~3355214,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,272182852~2470550,294440898~6981222,310378496~4846333,318767104~4667640,348569567~9772461,366925156~3770110,384785269~7349015,400335542~13341568] 2024-08-17T17:41:58.482 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 16 diff [6122489~2117206,9582259~1513694,12311950~12673759,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,312958845~2265984,322650436~784308,330570284~6844117,348569567~9772461,366925156~3770110,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.482 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,12311950~12673759,31503479~1175130,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,312958845~2265984,322650436~784308,330570284~6844117,348569567~9772461,366925156~3770110,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.487 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~16597101,30982143~17515296,52172688~9980535,68742908~1813321,75497472~6408989,85296870~4252304,89900725~16341098,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,215930165~10704498,228885435~3355214,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,272182852~6358709,294440898~6981222,310378496~4846333,318767104~4667640,327155712~10258689,348569567~9772461,366925156~3770110,384785269~10013954,400335542~13341568] 2024-08-17T17:41:58.487 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 17 diff [6122489~2117206,8405456~16580253,31503479~1175130,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,312958845~6612414,322650436~784308,330570284~6844117,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.487 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~16580253,31503479~1175130,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,312958845~6612414,322650436~784308,330570284~6844117,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.492 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~16597101,30982143~19029698,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6408989,85296870~4252304,89900725~16341098,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,215930165~10704498,228885435~3355214,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,272182852~6358709,294440898~6981222,310378496~13056248,327155712~10258689,348569567~9772461,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:41:58.492 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 18 diff [6122489~2117206,8405456~16580253,30955299~1723310,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~6844117,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.492 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~16580253,30955299~1723310,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,312958845~8295750,322650436~784308,330570284~6844117,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.497 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8388608~16597101,30955299~19056542,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6408989,85296870~4252304,89900725~16341098,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~6981222,301989888~6880813,310378496~13056248,327155712~10258689,348569567~9772461,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:41:58.497 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 19 diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.497 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.502 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~10258689,338106187~5516501,348569567~9772461,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:41:58.503 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 9 diff [68742908~1813321,111220774~5205171,127412771~15004337,155757889~6839948,197702108~6597113,244002673~773270,247070758~4960199,312958845~2265984,400335542~7506643] 2024-08-17T17:41:58.503 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [68742908~1813321,111220774~5205171,127412771~15004337,155757889~6839948,197702108~6597113,244002673~773270,247070758~4960199,312958845~2265984,400335542~7506643] 2024-08-17T17:41:58.508 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [68742908~1813321,111220774~5205171,127412771~15004337,155757889~6839948,197702108~6597113,244002673~773270,247070758~4960199,310378496~4846333,400335542~7506643] 2024-08-17T17:41:58.508 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 10 diff [52172688~3245528,58199577~3721268,68742908~1813321,75676394~6230067,85296870~4252304,111220774~5205171,127412771~21883557,155540399~7057438,166058561~2977181,197702108~6597113,240858522~5051700,247070758~4960199,272182852~2470550,312958845~2265984,400335542~7506643] 2024-08-17T17:41:58.508 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [52172688~3245528,58199577~3721268,68742908~1813321,75676394~6230067,85296870~4252304,111220774~5205171,127412771~21883557,155540399~7057438,166058561~2977181,197702108~6597113,240858522~5051700,247070758~4960199,272182852~2470550,312958845~2265984,400335542~7506643] 2024-08-17T17:41:58.513 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [52172688~3245528,58171180~3749665,68742908~1813321,75497472~6408989,85296870~4252304,111220774~5205171,127412771~21883557,155540399~7057438,166058561~2977181,197702108~6597113,240858522~5051700,247070758~4960199,272182852~2470550,310378496~4846333,400335542~7506643] 2024-08-17T17:41:58.513 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 11 diff [13040607~5795262,33421453~10210967,52172688~3245528,58014695~4138528,68742908~1813321,75676394~6230067,85296870~4252304,111220774~5205171,127412771~21883557,155540399~7057438,165571375~3464367,197702108~6597113,212123767~2607614,224516274~2118389,238191726~7718496,247070758~4960199,272182852~2470550,312958845~2265984,400335542~7506643,409521161~4155949] 2024-08-17T17:41:58.513 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [13040607~5795262,33421453~10210967,52172688~3245528,58014695~4138528,68742908~1813321,75676394~6230067,85296870~4252304,111220774~5205171,127412771~21883557,155540399~7057438,165571375~3464367,197702108~6597113,212123767~2607614,224516274~2118389,238191726~7718496,247070758~4960199,272182852~2470550,312958845~2265984,400335542~7506643,409521161~4155949] 2024-08-17T17:41:58.518 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [13040607~5795262,32678609~10953811,52172688~3245528,58014695~4138528,68742908~1813321,75497472~6408989,85296870~4252304,111220774~5205171,127412771~21883557,155540399~7057438,165571375~3464367,197702108~6597113,212123767~2607614,224516274~2118389,235227886~10682336,247070758~4960199,272182852~2470550,310378496~4846333,400335542~7506643,409521161~4155949] 2024-08-17T17:41:58.518 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 12 diff [13040607~5795262,33421453~10210967,52172688~3245528,58014695~4138528,68742908~1813321,75676394~6230067,85296870~4252304,101692190~4549633,110253798~6172147,127412771~21883557,153615532~8982305,165571375~3464367,197702108~6597113,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~13839231,272182852~2470550,312958845~2265984,354087269~4254759,390263701~1637045,400335542~13341568] 2024-08-17T17:41:58.518 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [13040607~5795262,33421453~10210967,52172688~3245528,58014695~4138528,68742908~1813321,75676394~6230067,85296870~4252304,101692190~4549633,110253798~6172147,127412771~21883557,153615532~8982305,165571375~3464367,197702108~6597113,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~13839231,272182852~2470550,312958845~2265984,354087269~4254759,390263701~1637045,400335542~13341568] 2024-08-17T17:41:58.523 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [13040607~5795262,32678609~10953811,52172688~3245528,58014695~4138528,68742908~1813321,75497472~6408989,85296870~4252304,100663296~5578527,110253798~6172147,127412771~21883557,153615532~8982305,165571375~3464367,197702108~6597113,212123767~2607614,215930165~4615408,224516274~2118389,228885435~3355214,235227886~16803071,272182852~2470550,310378496~4846333,354087269~4254759,390070272~1830474,400335542~13341568] 2024-08-17T17:41:58.523 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 13 diff [13040607~5795262,33421453~15075986,52172688~3245528,58014695~4138528,68742908~1813321,75676394~6230067,85296870~4252304,90097398~5457279,99526650~6715173,110253798~6172147,127412771~21883557,153615532~8982305,165571375~9681198,178411768~2367387,197702108~6597113,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~13839231,252773028~8082862,272182852~2470550,294440898~6981222,312958845~2265984,354087269~4254759,390263701~1637045,400335542~13341568] 2024-08-17T17:41:58.523 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [13040607~5795262,33421453~15075986,52172688~3245528,58014695~4138528,68742908~1813321,75676394~6230067,85296870~4252304,90097398~5457279,99526650~6715173,110253798~6172147,127412771~21883557,153615532~8982305,165571375~9681198,178411768~2367387,197702108~6597113,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~13839231,252773028~8082862,272182852~2470550,294440898~6981222,312958845~2265984,354087269~4254759,390263701~1637045,400335542~13341568] 2024-08-17T17:41:58.528 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [13040607~5795262,32678609~15818830,52172688~3245528,58014695~4138528,68742908~1813321,75497472~6408989,85296870~4252304,90097398~5457279,99526650~6715173,110253798~6172147,127412771~21883557,153615532~8982305,165571375~9681198,178411768~2367387,197702108~6597113,212123767~2607614,215930165~4615408,224516274~2118389,228885435~3355214,235227886~16803071,252773028~8082862,272182852~2470550,294440898~6981222,310378496~4846333,354087269~4254759,390070272~1830474,400335542~13341568] 2024-08-17T17:41:58.529 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 14 diff [13040607~5795262,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118553273~1355301,127412771~21883557,153615532~8982305,165571375~9681198,178411768~2367387,197702108~6597113,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,272182852~2470550,294440898~6981222,312958845~2265984,354087269~4254759,384785269~7349015,400335542~13341568] 2024-08-17T17:41:58.529 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [13040607~5795262,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118553273~1355301,127412771~21883557,153615532~8982305,165571375~9681198,178411768~2367387,197702108~6597113,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,272182852~2470550,294440898~6981222,312958845~2265984,354087269~4254759,384785269~7349015,400335542~13341568] 2024-08-17T17:41:58.534 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [13040607~5795262,32678609~15818830,52172688~9980535,68742908~1813321,75497472~6408989,85296870~4252304,90097398~16144425,108486098~7939847,118553273~1355301,127412771~21883557,153615532~8982305,165571375~9681198,178411768~2367387,197702108~6597113,212123767~2607614,215930165~10704498,228885435~3355214,235227886~16803071,252773028~8082862,261307577~1507943,272182852~2470550,294440898~6981222,310378496~4846333,354087269~4254759,384785269~7349015,400335542~13341568] 2024-08-17T17:41:58.534 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 15 diff [6122489~2117206,9582259~1513694,13040607~11945102,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118553273~1355301,127412771~21883557,150192064~2291333,153615532~8982305,165571375~9681198,178411768~2367387,191934227~5441303,197702108~6597113,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~2470550,294440898~6981222,312958845~2265984,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,400335542~13341568] 2024-08-17T17:41:58.534 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,13040607~11945102,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118553273~1355301,127412771~21883557,150192064~2291333,153615532~8982305,165571375~9681198,178411768~2367387,191934227~5441303,197702108~6597113,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~2470550,294440898~6981222,312958845~2265984,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,400335542~13341568] 2024-08-17T17:41:58.539 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,9582259~1513694,13040607~11945102,32678609~15818830,52172688~9980535,68742908~1813321,75497472~6408989,85296870~4252304,90097398~16144425,108486098~7939847,118553273~1355301,127412771~25070626,153615532~8982305,165571375~9681198,178411768~2367387,191934227~5441303,197702108~6597113,212123767~2607614,215930165~10704498,228885435~3355214,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,272182852~2470550,294440898~6981222,310378496~4846333,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,400335542~13341568] 2024-08-17T17:41:58.539 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 16 diff [6122489~2117206,9582259~1513694,13040607~11945102,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,312958845~2265984,330570284~6844117,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.539 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,13040607~11945102,33421453~15075986,52172688~9980535,68742908~1813321,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,312958845~2265984,330570284~6844117,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.545 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,9582259~1513694,13040607~11945102,32678609~15818830,52172688~9980535,68742908~1813321,75497472~6408989,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~35185066,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,215930165~10704498,228885435~3355214,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,272182852~6358709,294440898~6981222,310378496~4846333,327155712~10258689,348569567~2746233,354087269~4254759,366925156~3770110,384785269~10013954,400335542~13341568] 2024-08-17T17:41:58.545 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 17 diff [6122489~2117206,8405456~16580253,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,312958845~6612414,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.545 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~16580253,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~13839231,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,312958845~6612414,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.551 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8405456~16580253,32678609~17333232,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6408989,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~35185066,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,212123767~2607614,215930165~10704498,228885435~3355214,235227886~16803071,252773028~8082862,261307577~1507943,264241152~2770272,272182852~6358709,294440898~6981222,310378496~9192763,327155712~10258689,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:41:58.551 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 18 diff [6122489~2117206,8405456~16580253,30955299~1249356,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,312958845~8295750,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.551 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~16580253,30955299~1249356,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6230067,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,312958845~8295750,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.556 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8405456~16580253,30955299~1249356,32678609~17333232,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6408989,85296870~4252304,90097398~16144425,108486098~7939847,118030408~3183729,127412771~35185066,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~6981222,301989888~6880813,310378496~10876099,327155712~10258689,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:41:58.556 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 19 diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.556 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:58.562 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,127412771~35185066,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~26813697,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:41:58.562 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 10 diff [52172688~3245528,58199577~3721268,75676394~6230067,85296870~4252304,141229321~8067007,155540399~1674274,166058561~2977181,240858522~5051700,272182852~2470550] 2024-08-17T17:41:58.562 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [52172688~3245528,58199577~3721268,75676394~6230067,85296870~4252304,141229321~8067007,155540399~1674274,166058561~2977181,240858522~5051700,272182852~2470550] 2024-08-17T17:41:58.567 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [52172688~3245528,58171180~3749665,75497472~6408989,85296870~4252304,141229321~8067007,155540399~1674274,166058561~2977181,240858522~5051700,272182852~2470550] 2024-08-17T17:41:58.567 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 11 diff [13040607~5795262,33421453~10210967,52172688~3245528,58014695~4138528,75676394~6230067,85296870~4252304,141229321~8067007,155540399~1674274,165571375~3464367,212123767~2607614,224516274~2118389,238191726~7718496,272182852~2470550,409521161~4155949] 2024-08-17T17:41:58.567 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [13040607~5795262,33421453~10210967,52172688~3245528,58014695~4138528,75676394~6230067,85296870~4252304,141229321~8067007,155540399~1674274,165571375~3464367,212123767~2607614,224516274~2118389,238191726~7718496,272182852~2470550,409521161~4155949] 2024-08-17T17:41:58.572 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [13040607~5795262,32678609~10953811,52172688~3245528,58014695~4138528,75497472~6408989,85296870~4252304,141229321~8067007,155540399~1674274,165571375~3464367,212123767~2607614,224516274~2118389,235227886~10682336,272182852~2470550,409521161~4155949] 2024-08-17T17:41:58.572 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 12 diff [13040607~5795262,33421453~10210967,52172688~3245528,58014695~4138528,75676394~6230067,85296870~4252304,101692190~4549633,110253798~3900891,141229321~8067007,153615532~3715116,165571375~3464367,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~10002570,272182852~2470550,354087269~4254759,390263701~1637045,404079964~9597146] 2024-08-17T17:41:58.572 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [13040607~5795262,33421453~10210967,52172688~3245528,58014695~4138528,75676394~6230067,85296870~4252304,101692190~4549633,110253798~3900891,141229321~8067007,153615532~3715116,165571375~3464367,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~10002570,272182852~2470550,354087269~4254759,390263701~1637045,404079964~9597146] 2024-08-17T17:41:58.577 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [13040607~5795262,32678609~10953811,52172688~3245528,58014695~4138528,75497472~6408989,85296870~4252304,100663296~5578527,110253798~3900891,141229321~8067007,153615532~3715116,165571375~3464367,212123767~2607614,215930165~4615408,224516274~2118389,228885435~3355214,235227886~12966410,272182852~2470550,354087269~4254759,390070272~1830474,404079964~9597146] 2024-08-17T17:41:58.577 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 13 diff [13040607~5795262,33421453~15075986,52172688~3245528,58014695~4138528,75676394~6230067,85296870~4252304,90097398~5457279,99526650~6715173,110253798~3900891,141229321~8067007,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~10002570,252773028~8082862,272182852~2470550,294440898~6981222,354087269~4254759,390263701~1637045,404079964~9597146] 2024-08-17T17:41:58.577 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [13040607~5795262,33421453~15075986,52172688~3245528,58014695~4138528,75676394~6230067,85296870~4252304,90097398~5457279,99526650~6715173,110253798~3900891,141229321~8067007,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~10002570,252773028~8082862,272182852~2470550,294440898~6981222,354087269~4254759,390263701~1637045,404079964~9597146] 2024-08-17T17:41:58.583 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [13040607~5795262,32678609~15818830,52172688~3245528,58014695~4138528,75497472~6408989,85296870~4252304,90097398~5457279,99526650~6715173,110253798~3900891,141229321~8067007,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,212123767~2607614,215930165~4615408,224516274~2118389,228885435~3355214,235227886~12966410,252773028~8082862,272182852~2470550,294440898~6981222,354087269~4254759,390070272~1830474,404079964~9597146] 2024-08-17T17:41:58.583 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 14 diff [13040607~5795262,33421453~15075986,52172688~9980535,75676394~6230067,85296870~4252304,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,141229321~8067007,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,212123767~2607614,217652007~8982656,228885435~3355214,238191726~10002570,252773028~8082862,261307577~1507943,272182852~2470550,294440898~6981222,354087269~4254759,384785269~7349015,404079964~9597146] 2024-08-17T17:41:58.583 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [13040607~5795262,33421453~15075986,52172688~9980535,75676394~6230067,85296870~4252304,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,141229321~8067007,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,212123767~2607614,217652007~8982656,228885435~3355214,238191726~10002570,252773028~8082862,261307577~1507943,272182852~2470550,294440898~6981222,354087269~4254759,384785269~7349015,404079964~9597146] 2024-08-17T17:41:58.588 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [13040607~5795262,32678609~15818830,52172688~9980535,75497472~6408989,85296870~4252304,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,141229321~8067007,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,212123767~2607614,215930165~10704498,228885435~3355214,235227886~12966410,252773028~8082862,261307577~1507943,272182852~2470550,294440898~6981222,354087269~4254759,384785269~7349015,404079964~9597146] 2024-08-17T17:41:58.588 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 15 diff [6122489~2117206,9582259~1513694,13040607~11945102,33421453~15075986,52172688~9980535,75676394~6230067,85296870~4252304,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,141229321~8067007,150192064~2291333,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,191934227~5441303,212123767~2607614,217652007~8982656,228885435~3355214,238191726~10002570,252773028~8082862,261307577~1507943,265186950~1824474,272182852~2470550,294440898~6981222,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,404079964~9597146] 2024-08-17T17:41:58.588 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,13040607~11945102,33421453~15075986,52172688~9980535,75676394~6230067,85296870~4252304,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,141229321~8067007,150192064~2291333,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,191934227~5441303,212123767~2607614,217652007~8982656,228885435~3355214,238191726~10002570,252773028~8082862,261307577~1507943,265186950~1824474,272182852~2470550,294440898~6981222,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,404079964~9597146] 2024-08-17T17:41:58.593 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,9582259~1513694,13040607~11945102,32678609~15818830,52172688~9980535,75497472~6408989,85296870~4252304,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,141229321~11254076,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,191934227~5441303,212123767~2607614,215930165~10704498,228885435~3355214,235227886~12966410,252773028~8082862,261307577~1507943,264241152~2770272,272182852~2470550,294440898~6981222,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,404079964~9597146] 2024-08-17T17:41:58.593 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 16 diff [6122489~2117206,9582259~1513694,13040607~11945102,33421453~15075986,52172688~9980535,75676394~6230067,85296870~4252304,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~12863744,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,330570284~6844117,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.593 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,13040607~11945102,33421453~15075986,52172688~9980535,75676394~6230067,85296870~4252304,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~12863744,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,330570284~6844117,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.598 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,9582259~1513694,13040607~11945102,32678609~15818830,52172688~9980535,75497472~6408989,85296870~4252304,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,141229321~16101327,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,212123767~2607614,215930165~10704498,228885435~3355214,235227886~15827584,252773028~8082862,261307577~1507943,264241152~2770272,272182852~6358709,294440898~6981222,327155712~10258689,348569567~2746233,354087269~4254759,366925156~3770110,384785269~10013954,403188267~10488843] 2024-08-17T17:41:58.599 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 17 diff [6122489~2117206,8405456~16580253,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6230067,85296870~4252304,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~12863744,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,315124553~4446706,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.599 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~16580253,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6230067,85296870~4252304,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~12863744,252773028~8082862,261307577~1507943,265186950~1824474,272182852~6358709,294440898~6981222,315124553~4446706,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.604 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8405456~16580253,32678609~17333232,52172688~9980535,72099906~2964160,75497472~6408989,85296870~4252304,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,141229321~16101327,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,212123767~2607614,215930165~10704498,228885435~3355214,235227886~15827584,252773028~8082862,261307577~1507943,264241152~2770272,272182852~6358709,294440898~6981222,315124553~4446706,327155712~10258689,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:41:58.604 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 18 diff [6122489~2117206,8405456~16580253,30955299~1249356,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6230067,85296870~4252304,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,315124553~6130042,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.604 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~16580253,30955299~1249356,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6230067,85296870~4252304,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,315124553~6130042,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.609 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8405456~16580253,30955299~1249356,32678609~17333232,52172688~9980535,72099906~2964160,75497472~6408989,85296870~4252304,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,141229321~16101327,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~6981222,301989888~6880813,310725355~1663245,315124553~6130042,327155712~10258689,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:41:58.609 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 19 diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.609 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.615 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,72099906~2964160,75497472~6811507,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~16101327,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:41:58.615 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 11 diff [13040607~5795262,33421453~10210967,58014695~4138528,165571375~3062744,212123767~2607614,224516274~2118389,238191726~4859035,409521161~4155949] 2024-08-17T17:41:58.615 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [13040607~5795262,33421453~10210967,58014695~4138528,165571375~3062744,212123767~2607614,224516274~2118389,238191726~4859035,409521161~4155949] 2024-08-17T17:41:58.620 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [13040607~5795262,32678609~10953811,58014695~4138528,165571375~3062744,212123767~2607614,224516274~2118389,235227886~7822875,409521161~4155949] 2024-08-17T17:41:58.620 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 12 diff [13040607~5795262,33421453~10210967,58014695~4138528,101692190~4549633,110253798~3900891,153615532~3715116,165571375~3062744,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~4859035,245682731~2511565,354087269~4254759,390263701~1637045,404079964~9597146] 2024-08-17T17:41:58.620 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [13040607~5795262,33421453~10210967,58014695~4138528,101692190~4549633,110253798~3900891,153615532~3715116,165571375~3062744,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~4859035,245682731~2511565,354087269~4254759,390263701~1637045,404079964~9597146] 2024-08-17T17:41:58.625 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [13040607~5795262,32678609~10953811,58014695~4138528,100663296~5578527,110253798~3900891,153615532~3715116,165571375~3062744,212123767~2607614,215930165~4615408,224516274~2118389,228885435~3355214,235227886~7822875,245682731~2511565,354087269~4254759,390070272~1830474,404079964~9597146] 2024-08-17T17:41:58.625 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 13 diff [13040607~5795262,33421453~15075986,58014695~4138528,90097398~5457279,99526650~6715173,110253798~3900891,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~4859035,245682731~2511565,252773028~8082862,294440898~6981222,354087269~4254759,390263701~1637045,404079964~9597146] 2024-08-17T17:41:58.625 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [13040607~5795262,33421453~15075986,58014695~4138528,90097398~5457279,99526650~6715173,110253798~3900891,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,212123767~2607614,217652007~2893566,224516274~2118389,228885435~3355214,238191726~4859035,245682731~2511565,252773028~8082862,294440898~6981222,354087269~4254759,390263701~1637045,404079964~9597146] 2024-08-17T17:41:58.630 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [13040607~5795262,32678609~15818830,58014695~4138528,90097398~5457279,99526650~6715173,110253798~3900891,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,212123767~2607614,215930165~4615408,224516274~2118389,228885435~3355214,235227886~7822875,245682731~2511565,252773028~8082862,294440898~6981222,354087269~4254759,390070272~1830474,404079964~9597146] 2024-08-17T17:41:58.630 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 14 diff [13040607~5795262,33421453~15075986,52330412~9822811,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,212123767~2607614,217652007~8982656,228885435~3355214,238191726~4859035,245682731~2511565,252773028~8082862,261307577~1507943,294440898~6981222,354087269~4254759,384785269~7349015,404079964~9597146] 2024-08-17T17:41:58.630 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [13040607~5795262,33421453~15075986,52330412~9822811,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,212123767~2607614,217652007~8982656,228885435~3355214,238191726~4859035,245682731~2511565,252773028~8082862,261307577~1507943,294440898~6981222,354087269~4254759,384785269~7349015,404079964~9597146] 2024-08-17T17:41:58.636 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [13040607~5795262,32678609~15818830,52330412~9822811,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,212123767~2607614,215930165~10704498,228885435~3355214,235227886~7822875,245682731~2511565,252773028~8082862,261307577~1507943,294440898~6981222,354087269~4254759,384785269~7349015,404079964~9597146] 2024-08-17T17:41:58.636 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 15 diff [6122489~2117206,9582259~1513694,13040607~11945102,33421453~15075986,52330412~9822811,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,150192064~2291333,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,191934227~5441303,212123767~2607614,217652007~8982656,228885435~3355214,238191726~4859035,245682731~2511565,252773028~8082862,261307577~1507943,265186950~1824474,294440898~6981222,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,404079964~9597146] 2024-08-17T17:41:58.636 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,13040607~11945102,33421453~15075986,52330412~9822811,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,150192064~2291333,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,191934227~5441303,212123767~2607614,217652007~8982656,228885435~3355214,238191726~4859035,245682731~2511565,252773028~8082862,261307577~1507943,265186950~1824474,294440898~6981222,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,404079964~9597146] 2024-08-17T17:41:58.641 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,9582259~1513694,13040607~11945102,32678609~15818830,52330412~9822811,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,149296328~3187069,153615532~3715116,160518828~285849,165571375~9681198,178411768~2367387,191934227~5441303,212123767~2607614,215930165~10704498,228885435~3355214,235227886~7822875,245682731~2511565,252773028~8082862,261307577~1507943,264241152~2770272,294440898~6981222,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,404079964~9597146] 2024-08-17T17:41:58.641 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 16 diff [6122489~2117206,9582259~1513694,13040607~11945102,33421453~15075986,52330412~9822811,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~4859035,245682731~5372739,252773028~8082862,261307577~1507943,265186950~1824474,274541285~4000276,294440898~6981222,330570284~6844117,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.642 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,13040607~11945102,33421453~15075986,52330412~9822811,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~4859035,245682731~5372739,252773028~8082862,261307577~1507943,265186950~1824474,274541285~4000276,294440898~6981222,330570284~6844117,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.650 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,9582259~1513694,13040607~11945102,32678609~15818830,52330412~9822811,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,212123767~2607614,215930165~10704498,228885435~3355214,235227886~7822875,245682731~5372739,252773028~8082862,261307577~1507943,264241152~2770272,274541285~4000276,294440898~6981222,327155712~10258689,348569567~2746233,354087269~4254759,366925156~3770110,384785269~10013954,403188267~10488843] 2024-08-17T17:41:58.650 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 17 diff [6122489~2117206,8405456~16580253,33421453~16590388,52330412~9822811,72099906~2964160,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~4859035,245682731~5372739,252773028~8082862,261307577~1507943,265186950~1824474,273290717~5250844,294440898~6981222,315124553~4446706,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.650 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~16580253,33421453~16590388,52330412~9822811,72099906~2964160,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,212123767~2607614,217652007~8982656,228885435~3355214,238191726~4859035,245682731~5372739,252773028~8082862,261307577~1507943,265186950~1824474,273290717~5250844,294440898~6981222,315124553~4446706,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.653 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8405456~16580253,32678609~17333232,52330412~9822811,72099906~2964160,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,212123767~2607614,215930165~10704498,228885435~3355214,235227886~7822875,245682731~5372739,252773028~8082862,261307577~1507943,264241152~2770272,273290717~5250844,294440898~6981222,315124553~4446706,327155712~10258689,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:41:58.653 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 18 diff [6122489~2117206,8405456~16580253,30955299~1249356,33421453~16590388,52330412~9822811,72099906~2964160,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,315124553~6130042,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.653 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~16580253,30955299~1249356,33421453~16590388,52330412~9822811,72099906~2964160,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,315124553~6130042,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.658 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8405456~16580253,30955299~1249356,32678609~17333232,52330412~9822811,72099906~2964160,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~7822875,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~6981222,301989888~6880813,310725355~1663245,315124553~6130042,327155712~10258689,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:41:58.658 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 19 diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.658 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:58.663 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~7822875,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:41:58.663 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 12 diff [60052586~1275610,101692190~4549633,110253798~3900891,153615532~3715116,217652007~2893566,228885435~3355214,245682731~2511565,354087269~4254759,390263701~1637045,404079964~7145520] 2024-08-17T17:41:58.663 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [60052586~1275610,101692190~4549633,110253798~3900891,153615532~3715116,217652007~2893566,228885435~3355214,245682731~2511565,354087269~4254759,390263701~1637045,404079964~7145520] 2024-08-17T17:41:58.668 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [60052586~1275610,100663296~5578527,110253798~3900891,153615532~3715116,215930165~4615408,228885435~3355214,245682731~2511565,354087269~4254759,390070272~1830474,404079964~7145520] 2024-08-17T17:41:58.669 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 13 diff [41118839~7378600,60052586~1275610,90097398~5457279,99526650~6715173,110253798~3900891,153615532~3715116,160518828~285849,167084727~8167846,178411768~2367387,217652007~2893566,228885435~3355214,245682731~2511565,252773028~8082862,294440898~6981222,354087269~4254759,390263701~1637045,404079964~7145520] 2024-08-17T17:41:58.669 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [41118839~7378600,60052586~1275610,90097398~5457279,99526650~6715173,110253798~3900891,153615532~3715116,160518828~285849,167084727~8167846,178411768~2367387,217652007~2893566,228885435~3355214,245682731~2511565,252773028~8082862,294440898~6981222,354087269~4254759,390263701~1637045,404079964~7145520] 2024-08-17T17:41:58.674 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [41118839~7378600,60052586~1275610,90097398~5457279,99526650~6715173,110253798~3900891,153615532~3715116,160518828~285849,167084727~8167846,178411768~2367387,215930165~4615408,228885435~3355214,245682731~2511565,252773028~8082862,294440898~6981222,354087269~4254759,390070272~1830474,404079964~7145520] 2024-08-17T17:41:58.674 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 14 diff [41118839~7378600,52330412~7185084,60052586~1275610,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,153615532~3715116,160518828~285849,167084727~8167846,178411768~2367387,217652007~8216728,228885435~3355214,245682731~2511565,252773028~8082862,261307577~1507943,294440898~6981222,354087269~4254759,384785269~7349015,404079964~7145520] 2024-08-17T17:41:58.674 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [41118839~7378600,52330412~7185084,60052586~1275610,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,153615532~3715116,160518828~285849,167084727~8167846,178411768~2367387,217652007~8216728,228885435~3355214,245682731~2511565,252773028~8082862,261307577~1507943,294440898~6981222,354087269~4254759,384785269~7349015,404079964~7145520] 2024-08-17T17:41:58.679 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [41118839~7378600,52330412~7185084,60052586~1275610,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,153615532~3715116,160518828~285849,167084727~8167846,178411768~2367387,215930165~9938570,228885435~3355214,245682731~2511565,252773028~8082862,261307577~1507943,294440898~6981222,354087269~4254759,384785269~7349015,404079964~7145520] 2024-08-17T17:41:58.679 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 15 diff [6122489~2117206,9582259~1513694,16874234~8111475,41118839~7378600,52330412~7185084,60052586~1275610,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,150192064~2291333,153615532~3715116,160518828~285849,167084727~8167846,178411768~2367387,191934227~5441303,217652007~8216728,228885435~3355214,239310228~2700,245682731~2511565,252773028~8082862,261307577~1507943,265186950~1824474,294440898~6981222,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,404079964~7145520] 2024-08-17T17:41:58.679 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,16874234~8111475,41118839~7378600,52330412~7185084,60052586~1275610,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,150192064~2291333,153615532~3715116,160518828~285849,167084727~8167846,178411768~2367387,191934227~5441303,217652007~8216728,228885435~3355214,239310228~2700,245682731~2511565,252773028~8082862,261307577~1507943,265186950~1824474,294440898~6981222,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,404079964~7145520] 2024-08-17T17:41:58.684 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,9582259~1513694,16874234~8111475,41118839~7378600,52330412~7185084,60052586~1275610,90097398~16144425,108486098~5668591,118553273~1355301,133306049~7681407,149296328~3187069,153615532~3715116,160518828~285849,167084727~8167846,178411768~2367387,191934227~5441303,215930165~9938570,228885435~3355214,239310228~2700,245682731~2511565,252773028~8082862,261307577~1507943,264241152~2770272,294440898~6981222,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,404079964~7145520] 2024-08-17T17:41:58.684 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 16 diff [6122489~2117206,9582259~1513694,16874234~8111475,41118839~7378600,52330412~7185084,60052586~1275610,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,217652007~8216728,228885435~3355214,239310228~2700,245682731~5372739,252773028~8082862,261307577~1507943,265186950~1824474,274541285~4000276,294440898~6981222,330570284~6844117,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:41:58.685 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,16874234~8111475,41118839~7378600,52330412~7185084,60052586~1275610,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,217652007~8216728,228885435~3355214,239310228~2700,245682731~5372739,252773028~8082862,261307577~1507943,265186950~1824474,274541285~4000276,294440898~6981222,330570284~6844117,348569567~2746233,354087269~4254759,366925156~3770110,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:41:58.690 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,9582259~1513694,16874234~8111475,41118839~7378600,52330412~7185084,60052586~1275610,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,215930165~9938570,228885435~3355214,239310228~2700,245682731~5372739,252773028~8082862,261307577~1507943,264241152~2770272,274541285~4000276,294440898~6981222,327155712~10258689,348569567~2746233,354087269~4254759,366925156~3770110,384785269~10013954,403188267~8037217] 2024-08-17T17:41:58.690 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 17 diff [6122489~2117206,8405456~6091738,16874234~8111475,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,217652007~8216728,228885435~3355214,239310228~2700,245682731~5372739,252773028~8082862,261307577~1507943,265186950~1824474,273290717~5250844,294440898~6981222,315124553~4446706,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:41:58.690 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~6091738,16874234~8111475,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,217652007~8216728,228885435~3355214,239310228~2700,245682731~5372739,252773028~8082862,261307577~1507943,265186950~1824474,273290717~5250844,294440898~6981222,315124553~4446706,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:41:58.695 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8405456~6091738,16874234~8111475,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,215930165~9938570,228885435~3355214,239310228~2700,245682731~5372739,252773028~8082862,261307577~1507943,264241152~2770272,273290717~5250844,294440898~6981222,315124553~4446706,327155712~10258689,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~8037217] 2024-08-17T17:41:58.695 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 18 diff [6122489~2117206,8405456~6091738,16874234~8111475,30955299~1249356,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,315124553~6130042,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:41:58.695 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~6091738,16874234~8111475,30955299~1249356,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,315124553~6130042,330570284~6844117,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:41:58.700 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8405456~6091738,16874234~8111475,30955299~1249356,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,90097398~16144425,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,215930165~9938570,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~6981222,301989888~6880813,310725355~1663245,315124553~6130042,327155712~10258689,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~8037217] 2024-08-17T17:41:58.701 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 19 diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:41:58.701 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:41:58.706 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,215930165~9938570,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~8037217] 2024-08-17T17:41:58.706 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to 13 diff [41118839~7378600,90097398~5457279,99526650~2264727,160518828~285849,167084727~8167846,178411768~2367387,252773028~8082862,294440898~6981222] 2024-08-17T17:41:58.706 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [41118839~7378600,90097398~5457279,99526650~2264727,160518828~285849,167084727~8167846,178411768~2367387,252773028~8082862,294440898~6981222] 2024-08-17T17:41:58.711 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [41118839~7378600,90097398~5457279,99526650~2264727,160518828~285849,167084727~8167846,178411768~2367387,252773028~8082862,294440898~6981222] 2024-08-17T17:41:58.711 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to 14 diff [41118839~7378600,52330412~7185084,90097398~11693979,108486098~2678206,118553273~1355301,133306049~7681407,160518828~285849,167084727~8167846,178411768~2367387,218372862~7495873,252773028~8082862,261307577~1507943,294440898~6981222,354913938~909027,384785269~7349015] 2024-08-17T17:41:58.711 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [41118839~7378600,52330412~7185084,90097398~11693979,108486098~2678206,118553273~1355301,133306049~7681407,160518828~285849,167084727~8167846,178411768~2367387,218372862~7495873,252773028~8082862,261307577~1507943,294440898~6981222,354913938~909027,384785269~7349015] 2024-08-17T17:41:58.716 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [41118839~7378600,52330412~7185084,90097398~11693979,108486098~2678206,118553273~1355301,133306049~7681407,160518828~285849,167084727~8167846,178411768~2367387,218372862~7495873,252773028~8082862,261307577~1507943,294440898~6981222,354913938~909027,384785269~7349015] 2024-08-17T17:41:58.717 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to 15 diff [6122489~2117206,9582259~1513694,16874234~8111475,41118839~7378600,52330412~7185084,90097398~11693979,108486098~2678206,118553273~1355301,133306049~7681407,150192064~2291333,160518828~285849,167084727~8167846,178411768~2367387,191934227~5441303,218372862~7495873,239310228~2700,252773028~8082862,261307577~1507943,265186950~1824474,294440898~6981222,348569567~2746233,354913938~909027,366925156~3770110,384785269~7349015] 2024-08-17T17:41:58.717 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,16874234~8111475,41118839~7378600,52330412~7185084,90097398~11693979,108486098~2678206,118553273~1355301,133306049~7681407,150192064~2291333,160518828~285849,167084727~8167846,178411768~2367387,191934227~5441303,218372862~7495873,239310228~2700,252773028~8082862,261307577~1507943,265186950~1824474,294440898~6981222,348569567~2746233,354913938~909027,366925156~3770110,384785269~7349015] 2024-08-17T17:41:58.722 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,9582259~1513694,16874234~8111475,41118839~7378600,52330412~7185084,90097398~11693979,108486098~2678206,118553273~1355301,133306049~7681407,149296328~3187069,160518828~285849,167084727~8167846,178411768~2367387,191934227~5441303,218372862~7495873,239310228~2700,252773028~8082862,261307577~1507943,264241152~2770272,294440898~6981222,348569567~2746233,354913938~909027,366925156~3770110,384785269~7349015] 2024-08-17T17:41:58.722 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to 16 diff [6122489~2117206,9582259~1513694,16874234~8111475,41118839~7378600,52330412~7185084,90097398~11693979,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5105518,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,218372862~7495873,239310228~2700,247539592~3515878,252773028~8082862,261307577~1507943,265186950~1824474,274541285~4000276,294440898~6981222,330570284~6844117,348569567~2746233,354913938~909027,366925156~3770110,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.722 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,16874234~8111475,41118839~7378600,52330412~7185084,90097398~11693979,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5105518,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,218372862~7495873,239310228~2700,247539592~3515878,252773028~8082862,261307577~1507943,265186950~1824474,274541285~4000276,294440898~6981222,330570284~6844117,348569567~2746233,354913938~909027,366925156~3770110,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.727 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,9582259~1513694,16874234~8111475,41118839~7378600,52330412~7185084,90097398~11693979,108486098~2678206,118030408~3183729,133306049~7681407,149296328~5733570,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,218372862~7495873,239310228~2700,247539592~3515878,252773028~8082862,261307577~1507943,264241152~2770272,274541285~4000276,294440898~6981222,327155712~10258689,348569567~2746233,354913938~909027,366925156~3770110,384785269~10013954,403188267~3969107] 2024-08-17T17:41:58.727 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to 17 diff [6122489~2117206,8405456~6091738,16874234~8111475,41118839~8893002,52330412~7185084,72099906~2964160,90097398~11693979,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5105518,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,218372862~7495873,239310228~2700,246931942~4123528,252773028~8082862,261307577~1507943,265186950~1824474,273290717~5250844,294440898~6981222,315124553~4446706,330570284~6844117,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.727 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~6091738,16874234~8111475,41118839~8893002,52330412~7185084,72099906~2964160,90097398~11693979,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5105518,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,218372862~7495873,239310228~2700,246931942~4123528,252773028~8082862,261307577~1507943,265186950~1824474,273290717~5250844,294440898~6981222,315124553~4446706,330570284~6844117,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.732 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8405456~6091738,16874234~8111475,41118839~8893002,52330412~7185084,72099906~2964160,90097398~11693979,108486098~2678206,118030408~3183729,133306049~7681407,149296328~5733570,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,218372862~7495873,239310228~2700,246931942~4123528,252773028~8082862,261307577~1507943,264241152~2770272,273290717~5250844,294440898~6981222,315124553~4446706,327155712~10258689,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:41:58.732 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to 18 diff [6122489~2117206,8405456~6091738,16874234~8111475,30955299~1249356,41118839~8893002,52330412~7185084,72099906~2964160,90097398~11693979,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,315124553~6130042,330570284~6844117,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.732 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~6091738,16874234~8111475,30955299~1249356,41118839~8893002,52330412~7185084,72099906~2964160,90097398~11693979,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~6981222,302649343~6221358,310725355~1663245,315124553~6130042,330570284~6844117,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.738 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8405456~6091738,16874234~8111475,30955299~1249356,41118839~8893002,52330412~7185084,72099906~2964160,90097398~11693979,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,264241152~14300409,285212672~5847227,294440898~6981222,301989888~6880813,310725355~1663245,315124553~6130042,327155712~10258689,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:41:58.738 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to 19 diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.738 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.743 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:41:58.743 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to 14 diff [41388025~4298022,52330412~7185084,92352943~7343768,108486098~2678206,118553273~1355301,133306049~7681407,218372862~7495873,261307577~1507943,354913938~909027,384785269~7349015] 2024-08-17T17:41:58.743 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [41388025~4298022,52330412~7185084,92352943~7343768,108486098~2678206,118553273~1355301,133306049~7681407,218372862~7495873,261307577~1507943,354913938~909027,384785269~7349015] 2024-08-17T17:41:58.748 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [41388025~4298022,52330412~7185084,92352943~7343768,108486098~2678206,118553273~1355301,133306049~7681407,218372862~7495873,261307577~1507943,354913938~909027,384785269~7349015] 2024-08-17T17:41:58.748 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to 15 diff [6122489~2117206,9582259~1513694,16874234~8111475,41388025~4298022,52330412~7185084,92352943~7343768,108486098~2678206,118553273~1355301,133306049~7681407,150192064~2291333,191934227~5441303,218372862~7495873,239310228~2700,261307577~1507943,265186950~1824474,348569567~2746233,354913938~909027,366925156~3770110,384785269~7349015] 2024-08-17T17:41:58.748 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,16874234~8111475,41388025~4298022,52330412~7185084,92352943~7343768,108486098~2678206,118553273~1355301,133306049~7681407,150192064~2291333,191934227~5441303,218372862~7495873,239310228~2700,261307577~1507943,265186950~1824474,348569567~2746233,354913938~909027,366925156~3770110,384785269~7349015] 2024-08-17T17:41:58.753 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,9582259~1513694,16874234~8111475,41388025~4298022,52330412~7185084,92352943~7343768,108486098~2678206,118553273~1355301,133306049~7681407,149296328~3187069,191934227~5441303,218372862~7495873,239310228~2700,261307577~1507943,264241152~2770272,348569567~2746233,354913938~909027,366925156~3770110,384785269~7349015] 2024-08-17T17:41:58.753 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to 16 diff [6122489~2117206,9582259~1513694,16874234~8111475,41388025~4298022,52330412~7185084,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5105518,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,218372862~7495873,239310228~2700,247539592~3515878,261307577~1507943,265186950~1824474,274541285~4000276,330570284~6844117,348569567~2746233,354913938~909027,366925156~3770110,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.753 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,16874234~8111475,41388025~4298022,52330412~7185084,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5105518,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,218372862~7495873,239310228~2700,247539592~3515878,261307577~1507943,265186950~1824474,274541285~4000276,330570284~6844117,348569567~2746233,354913938~909027,366925156~3770110,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.758 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,9582259~1513694,16874234~8111475,41388025~4298022,52330412~7185084,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149296328~5733570,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,218372862~7495873,239310228~2700,247539592~3515878,261307577~1507943,264241152~2770272,274541285~4000276,327155712~10258689,348569567~2746233,354913938~909027,366925156~3770110,384785269~10013954,403188267~3969107] 2024-08-17T17:41:58.758 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to 17 diff [6122489~2117206,8405456~6091738,16874234~8111475,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5105518,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,218372862~7495873,239310228~2700,246931942~4123528,261307577~1507943,265186950~1824474,273290717~5250844,315124553~4446706,330570284~6844117,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.759 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~6091738,16874234~8111475,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5105518,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,218372862~7495873,239310228~2700,246931942~4123528,261307577~1507943,265186950~1824474,273290717~5250844,315124553~4446706,330570284~6844117,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.764 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8405456~6091738,16874234~8111475,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149296328~5733570,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,218372862~7495873,239310228~2700,246931942~4123528,261307577~1507943,264241152~2770272,273290717~5250844,315124553~4446706,327155712~10258689,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:41:58.764 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to 18 diff [6122489~2117206,8405456~6091738,16874234~8111475,30955299~1249356,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,265186950~13354611,286246187~4813712,302649343~6221358,310725355~1663245,315124553~6130042,330570284~6844117,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.764 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~6091738,16874234~8111475,30955299~1249356,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,265186950~13354611,286246187~4813712,302649343~6221358,310725355~1663245,315124553~6130042,330570284~6844117,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.769 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8405456~6091738,16874234~8111475,30955299~1249356,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,264241152~14300409,285212672~5847227,301989888~6880813,310725355~1663245,315124553~6130042,327155712~10258689,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:41:58.769 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to 19 diff [1297677~13199517,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.769 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.774 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,264241152~14300409,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:41:58.775 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to 15 diff [6122489~2117206,9582259~1513694,16874234~8111475,150192064~2291333,191934227~5441303,239310228~2700,265186950~1824474,348569567~2746233,366925156~3770110] 2024-08-17T17:41:58.775 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,16874234~8111475,150192064~2291333,191934227~5441303,239310228~2700,265186950~1824474,348569567~2746233,366925156~3770110] 2024-08-17T17:41:58.779 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,9582259~1513694,16874234~8111475,149296328~3187069,191934227~5441303,239310228~2700,264241152~2770272,348569567~2746233,366925156~3770110] 2024-08-17T17:41:58.780 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to 16 diff [6122489~2117206,9582259~1513694,16874234~8111475,118030408~3183729,149924380~5105518,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,239310228~2700,247539592~3515878,265186950~1824474,274541285~4000276,330570284~6844117,348569567~2746233,366925156~3770110,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.780 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,9582259~1513694,16874234~8111475,118030408~3183729,149924380~5105518,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,239310228~2700,247539592~3515878,265186950~1824474,274541285~4000276,330570284~6844117,348569567~2746233,366925156~3770110,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.785 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,9582259~1513694,16874234~8111475,118030408~3183729,149296328~5733570,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,239310228~2700,247539592~3515878,264241152~2770272,274541285~4000276,327155712~10258689,348569567~2746233,366925156~3770110,392134284~2664939,403188267~3969107] 2024-08-17T17:41:58.785 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to 17 diff [6122489~2117206,8405456~6091738,16874234~8111475,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,118030408~3183729,149924380~5105518,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,239310228~2700,246931942~4123528,265186950~1824474,273290717~5250844,315124553~4446706,330570284~6844117,348569567~2746233,363159108~7536158,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.785 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~6091738,16874234~8111475,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,118030408~3183729,149924380~5105518,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,239310228~2700,246931942~4123528,265186950~1824474,273290717~5250844,315124553~4446706,330570284~6844117,348569567~2746233,363159108~7536158,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.790 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8405456~6091738,16874234~8111475,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,118030408~3183729,149296328~5733570,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,239310228~2700,246931942~4123528,264241152~2770272,273290717~5250844,315124553~4446706,327155712~10258689,348569567~2746233,362603833~8091433,392134284~2664939,403188267~3969107] 2024-08-17T17:41:58.790 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to 18 diff [6122489~2117206,8405456~6091738,16874234~8111475,30955299~1249356,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,265186950~13354611,286246187~4813712,302649343~6221358,310725355~1663245,315124553~6130042,330570284~6844117,348569567~2746233,363159108~7536158,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.790 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [6122489~2117206,8405456~6091738,16874234~8111475,30955299~1249356,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,265186950~13354611,286246187~4813712,302649343~6221358,310725355~1663245,315124553~6130042,330570284~6844117,348569567~2746233,363159108~7536158,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.796 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~4045391,8405456~6091738,16874234~8111475,30955299~1249356,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,118030408~3183729,149296328~6283051,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,264241152~14300409,285212672~5847227,301989888~6880813,310725355~1663245,315124553~6130042,327155712~10258689,348569567~2746233,362603833~8091433,392134284~2664939,403188267~3969107] 2024-08-17T17:41:58.796 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to 19 diff [1297677~13199517,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,363159108~7536158,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.796 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,363159108~7536158,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.801 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149296328~6283051,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,264241152~14300409,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,362603833~8091433,392134284~2664939,403188267~3969107] 2024-08-17T17:41:58.801 INFO:tasks.workunit.client.0.smithi130.stdout:from 15 to 16 diff [118030408~3183729,149924380~5105518,166825307~2678998,172403776~5654992,208255148~1681884,247539592~3515878,274541285~4000276,330570284~6844117,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.801 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [118030408~3183729,149924380~5105518,166825307~2678998,172403776~5654992,208255148~1681884,247539592~3515878,274541285~4000276,330570284~6844117,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.806 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [118030408~3183729,149924380~5105518,166825307~2678998,172403776~5654992,208255148~1681884,247539592~3515878,274541285~4000276,327155712~10258689,392134284~2664939,403188267~3969107] 2024-08-17T17:41:58.806 INFO:tasks.workunit.client.0.smithi130.stdout:from 15 to 17 diff [8405456~6091738,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,118030408~3183729,149924380~5105518,166825307~2678998,172403776~5654992,208255148~1681884,246931942~4123528,273290717~5250844,315124553~4446706,330570284~6844117,363159108~5625694,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.806 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8405456~6091738,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,118030408~3183729,149924380~5105518,166825307~2678998,172403776~5654992,208255148~1681884,246931942~4123528,273290717~5250844,315124553~4446706,330570284~6844117,363159108~5625694,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.812 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8405456~6091738,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,118030408~3183729,149924380~5105518,166825307~2678998,172403776~5654992,208255148~1681884,246931942~4123528,273290717~5250844,315124553~4446706,327155712~10258689,362603833~6180969,392134284~2664939,403188267~3969107] 2024-08-17T17:41:58.813 INFO:tasks.workunit.client.0.smithi130.stdout:from 15 to 18 diff [8405456~6091738,30955299~1249356,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,286246187~4813712,302649343~6221358,310725355~1663245,315124553~6130042,330570284~6844117,363159108~5625694,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.813 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8405456~6091738,30955299~1249356,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,286246187~4813712,302649343~6221358,310725355~1663245,315124553~6130042,330570284~6844117,363159108~5625694,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.819 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8405456~6091738,30955299~1249356,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,285212672~5847227,301989888~6880813,310725355~1663245,315124553~6130042,327155712~10258689,362603833~6180969,392134284~2664939,403188267~3969107] 2024-08-17T17:41:58.819 INFO:tasks.workunit.client.0.smithi130.stdout:from 15 to 19 diff [1297677~6200185,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,363159108~5625694,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.819 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~6200185,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,363159108~5625694,393360140~1439083,403188267~3969107] 2024-08-17T17:41:58.824 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~7497862,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,362603833~6180969,392134284~2664939,403188267~3969107] 2024-08-17T17:41:58.824 INFO:tasks.workunit.client.0.smithi130.stdout:from 16 to 17 diff [8405456~6091738,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,246931942~2452483,273290717~2471737,315124553~4446706,330928129~4964895,363159108~5625694] 2024-08-17T17:41:58.824 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8405456~6091738,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,246931942~2452483,273290717~2471737,315124553~4446706,330928129~4964895,363159108~5625694] 2024-08-17T17:41:58.829 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8405456~6091738,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,246931942~2452483,273290717~2471737,315124553~4446706,330928129~4964895,362603833~6180969] 2024-08-17T17:41:58.829 INFO:tasks.workunit.client.0.smithi130.stdout:from 16 to 18 diff [8405456~6091738,30955299~1249356,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,152404635~3174744,210075422~3861992,246931942~9391575,266411827~9350627,286246187~4813712,302649343~6221358,310725355~1663245,315124553~6130042,330928129~4964895,363159108~5625694] 2024-08-17T17:41:58.829 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8405456~6091738,30955299~1249356,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,152404635~3174744,210075422~3861992,246931942~9391575,266411827~9350627,286246187~4813712,302649343~6221358,310725355~1663245,315124553~6130042,330928129~4964895,363159108~5625694] 2024-08-17T17:41:58.834 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8405456~6091738,30955299~1249356,46564393~3447448,52822277~4635371,72099906~2964160,95970646~579400,152404635~3174744,210075422~3861992,246931942~9391575,266411827~9350627,285212672~5847227,301989888~6880813,310725355~1663245,315124553~6130042,330928129~4964895,362603833~6180969] 2024-08-17T17:41:58.834 INFO:tasks.workunit.client.0.smithi130.stdout:from 16 to 19 diff [1297677~6200185,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,152404635~3174744,210075422~3861992,246931942~9391575,266411827~9350627,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330928129~4964895,338106187~5516501,363159108~5625694] 2024-08-17T17:41:58.834 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~6200185,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,152404635~3174744,210075422~3861992,246931942~9391575,266411827~9350627,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330928129~4964895,338106187~5516501,363159108~5625694] 2024-08-17T17:41:58.839 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~7497862,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,152404635~3174744,210075422~3861992,246931942~9391575,266411827~9350627,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,330928129~4964895,338106187~5516501,362603833~6180969] 2024-08-17T17:41:58.839 INFO:tasks.workunit.client.0.smithi130.stdout:from 17 to 18 diff [30955299~1249356,152404635~3174744,210075422~3861992,248466245~7857272,266411827~7611252,286246187~4813712,302649343~6221358,310725355~1663245,315345966~5908629] 2024-08-17T17:41:58.839 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [30955299~1249356,152404635~3174744,210075422~3861992,248466245~7857272,266411827~7611252,286246187~4813712,302649343~6221358,310725355~1663245,315345966~5908629] 2024-08-17T17:41:58.845 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [30955299~1249356,152404635~3174744,210075422~3861992,248466245~7857272,266411827~7611252,285212672~5847227,301989888~6880813,310725355~1663245,315345966~5908629] 2024-08-17T17:41:58.845 INFO:tasks.workunit.client.0.smithi130.stdout:from 17 to 19 diff [1297677~6200185,7784300~5890796,30413398~2257211,81427530~881449,84600711~5606478,152404635~3174744,210075422~3861992,248466245~7857272,266411827~7611252,286246187~4813712,300835157~8035544,309044486~5067705,315345966~5908629,322065070~4478831,338106187~5516501] 2024-08-17T17:41:58.845 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~6200185,7784300~5890796,30413398~2257211,81427530~881449,84600711~5606478,152404635~3174744,210075422~3861992,248466245~7857272,266411827~7611252,286246187~4813712,300835157~8035544,309044486~5067705,315345966~5908629,322065070~4478831,338106187~5516501] 2024-08-17T17:41:58.850 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~7497862,7784300~5890796,30413398~2257211,81427530~881449,84600711~5606478,152404635~3174744,210075422~3861992,248466245~7857272,266411827~7611252,285212672~5847227,300835157~13277034,315345966~5908629,322065070~4478831,338106187~5516501] 2024-08-17T17:41:58.850 INFO:tasks.workunit.client.0.smithi130.stdout:from 18 to 19 diff [1297677~6200185,7784300~5890796,30413398~2257211,81427530~881449,84600711~5606478,300835157~7348018,309044486~5067705,322065070~4478831,338106187~5516501] 2024-08-17T17:41:58.850 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~6200185,7784300~5890796,30413398~2257211,81427530~881449,84600711~5606478,300835157~7348018,309044486~5067705,322065070~4478831,338106187~5516501] 2024-08-17T17:41:58.855 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~7497862,7784300~5890796,30413398~2257211,81427530~881449,84600711~5606478,300835157~7348018,308870701~5241490,322065070~4478831,338106187~5516501] 2024-08-17T17:41:59.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:58 smithi130 ceph-mon[105521]: osdmap e989: 8 total, 8 up, 8 in 2024-08-17T17:41:59.068 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:58 smithi130 ceph-mon[110068]: osdmap e989: 8 total, 8 up, 8 in 2024-08-17T17:41:59.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:58 smithi179 ceph-mon[93519]: osdmap e989: 8 total, 8 up, 8 in 2024-08-17T17:41:59.828 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:59.829 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:59.848 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:41:59.848 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:59.848 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:59.853 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:41:59.853 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:59.854 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:59.859 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:41:59.859 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:59.859 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:41:59.864 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:41:59.864 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:59.864 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:59.870 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:41:59.870 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:59.870 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:59.876 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:41:59.876 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:59.876 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:41:59.881 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~16466976,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:41:59.881 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:59.881 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:59.887 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~10258689,338106187~5516501,348569567~9772461,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:41:59.887 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:59.887 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:41:59.892 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,127412771~35185066,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~26813697,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:41:59.892 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:59.892 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:59.898 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,72099906~2964160,75497472~6811507,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~16101327,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:41:59.898 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:59.898 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:41:59.903 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~7822875,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:41:59.903 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:41:59.904 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:41:59.908 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,215930165~9938570,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~8037217] 2024-08-17T17:41:59.909 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:59.909 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:59.915 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:41:59.915 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:59.915 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:41:59.919 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,264241152~14300409,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:41:59.919 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,363159108~7536158,393360140~1439083,403188267~3969107] 2024-08-17T17:41:59.919 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,363159108~7536158,393360140~1439083,403188267~3969107] 2024-08-17T17:41:59.925 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149296328~6283051,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,264241152~14300409,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,362603833~8091433,392134284~2664939,403188267~3969107] 2024-08-17T17:41:59.925 INFO:tasks.workunit.client.0.smithi130.stdout:from 15 to HEAD diff [1297677~6200185,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,363159108~5625694,393360140~1439083,403188267~3969107] 2024-08-17T17:41:59.925 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~6200185,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,363159108~5625694,393360140~1439083,403188267~3969107] 2024-08-17T17:41:59.931 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~7497862,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,362603833~6180969,392134284~2664939,403188267~3969107] 2024-08-17T17:41:59.931 INFO:tasks.workunit.client.0.smithi130.stdout:from 16 to HEAD diff [1297677~6200185,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,152404635~3174744,210075422~3861992,246931942~9391575,266411827~9350627,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330928129~4964895,338106187~5516501,363159108~5625694] 2024-08-17T17:41:59.931 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~6200185,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,152404635~3174744,210075422~3861992,246931942~9391575,266411827~9350627,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330928129~4964895,338106187~5516501,363159108~5625694] 2024-08-17T17:41:59.936 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~7497862,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,152404635~3174744,210075422~3861992,246931942~9391575,266411827~9350627,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,330928129~4964895,338106187~5516501,362603833~6180969] 2024-08-17T17:41:59.936 INFO:tasks.workunit.client.0.smithi130.stdout:from 17 to HEAD diff [1297677~6200185,7784300~5890796,30413398~2257211,81427530~881449,84600711~5606478,152404635~3174744,210075422~3861992,248466245~7857272,266411827~7611252,286246187~4813712,300835157~8035544,309044486~5067705,315345966~5908629,322065070~4478831,338106187~5516501] 2024-08-17T17:41:59.936 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~6200185,7784300~5890796,30413398~2257211,81427530~881449,84600711~5606478,152404635~3174744,210075422~3861992,248466245~7857272,266411827~7611252,286246187~4813712,300835157~8035544,309044486~5067705,315345966~5908629,322065070~4478831,338106187~5516501] 2024-08-17T17:41:59.941 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~7497862,7784300~5890796,30413398~2257211,81427530~881449,84600711~5606478,152404635~3174744,210075422~3861992,248466245~7857272,266411827~7611252,285212672~5847227,300835157~13277034,315345966~5908629,322065070~4478831,338106187~5516501] 2024-08-17T17:42:00.087 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:41:59 smithi130 ceph-mon[105521]: pgmap v1210: 105 pgs: 105 active+clean; 1.1 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 263 KiB/s rd, 37 MiB/s wr, 1.06k op/s 2024-08-17T17:42:00.087 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:41:59 smithi130 ceph-mon[110068]: pgmap v1210: 105 pgs: 105 active+clean; 1.1 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 263 KiB/s rd, 37 MiB/s wr, 1.06k op/s 2024-08-17T17:42:00.087 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:41:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:41:59] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-08-17T17:42:00.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:41:59 smithi179 ceph-mon[93519]: pgmap v1210: 105 pgs: 105 active+clean; 1.1 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 263 KiB/s rd, 37 MiB/s wr, 1.06k op/s 2024-08-17T17:42:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:42:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:42:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:42:00.844 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:00.844 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:00.854 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:00.854 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:00.854 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:00.859 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:00.860 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:00.860 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:00.865 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:00.865 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:00.865 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:00.871 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:42:00.871 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:00.871 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:00.876 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:00.876 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:00.876 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:00.882 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:00.882 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:00.882 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:00.887 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~16466976,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:42:00.887 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:00.887 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:00.893 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~10258689,338106187~5516501,348569567~9772461,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:00.893 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:00.893 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:00.898 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,127412771~35185066,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~26813697,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:00.898 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:00.898 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:00.904 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,72099906~2964160,75497472~6811507,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~16101327,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:00.904 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:00.904 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:00.909 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~7822875,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:00.909 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:42:00.910 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:42:00.915 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,215930165~9938570,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~8037217] 2024-08-17T17:42:00.915 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:00.915 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:00.920 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:42:00.920 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:00.921 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:00.926 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,264241152~14300409,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:42:00.926 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,363159108~7536158,393360140~1439083,403188267~3969107] 2024-08-17T17:42:00.926 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,363159108~7536158,393360140~1439083,403188267~3969107] 2024-08-17T17:42:00.931 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149296328~6283051,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,264241152~14300409,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,362603833~8091433,392134284~2664939,403188267~3969107] 2024-08-17T17:42:00.931 INFO:tasks.workunit.client.0.smithi130.stdout:from 15 to HEAD diff [1297677~6200185,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,363159108~5625694,393360140~1439083,403188267~3969107] 2024-08-17T17:42:00.932 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~6200185,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,363159108~5625694,393360140~1439083,403188267~3969107] 2024-08-17T17:42:00.937 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~7497862,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,362603833~6180969,392134284~2664939,403188267~3969107] 2024-08-17T17:42:00.937 INFO:tasks.workunit.client.0.smithi130.stdout:from 16 to HEAD diff [1297677~6200185,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,152404635~3174744,210075422~3861992,246931942~9391575,266411827~9350627,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330928129~4964895,338106187~5516501,363159108~5625694] 2024-08-17T17:42:00.937 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~6200185,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,152404635~3174744,210075422~3861992,246931942~9391575,266411827~9350627,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330928129~4964895,338106187~5516501,363159108~5625694] 2024-08-17T17:42:00.942 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~7497862,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,152404635~3174744,210075422~3861992,246931942~9391575,266411827~9350627,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,330928129~4964895,338106187~5516501,362603833~6180969] 2024-08-17T17:42:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:00 smithi179 ceph-mon[93519]: osdmap e990: 8 total, 8 up, 8 in 2024-08-17T17:42:01.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:00 smithi130 ceph-mon[105521]: osdmap e990: 8 total, 8 up, 8 in 2024-08-17T17:42:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:00 smithi130 ceph-mon[110068]: osdmap e990: 8 total, 8 up, 8 in 2024-08-17T17:42:01.849 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:01.849 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:01.862 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:01.862 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:01.862 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:01.867 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:01.867 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:01.867 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:01.873 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:01.873 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:01.873 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:01.878 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:42:01.878 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:01.878 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:01.883 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:01.883 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:01.883 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:01.888 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:01.888 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:01.888 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:01.893 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~16466976,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:42:01.893 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:01.893 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:01.898 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~10258689,338106187~5516501,348569567~9772461,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:01.899 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:01.899 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:01.904 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,127412771~35185066,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~26813697,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:01.904 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:01.904 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:01.909 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,72099906~2964160,75497472~6811507,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~16101327,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:01.909 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:01.909 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:01.914 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~7822875,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:01.914 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:42:01.914 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:42:01.920 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,215930165~9938570,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~8037217] 2024-08-17T17:42:01.920 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:01.920 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:01.925 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:42:01.925 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:01.925 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:01.931 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,264241152~14300409,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:42:01.931 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,363159108~7536158,393360140~1439083,403188267~3969107] 2024-08-17T17:42:01.931 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,363159108~7536158,393360140~1439083,403188267~3969107] 2024-08-17T17:42:01.936 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149296328~6283051,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,264241152~14300409,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,362603833~8091433,392134284~2664939,403188267~3969107] 2024-08-17T17:42:01.936 INFO:tasks.workunit.client.0.smithi130.stdout:from 15 to HEAD diff [1297677~6200185,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,363159108~5625694,393360140~1439083,403188267~3969107] 2024-08-17T17:42:01.936 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~6200185,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,363159108~5625694,393360140~1439083,403188267~3969107] 2024-08-17T17:42:01.941 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~7497862,7784300~6712894,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,208255148~1681884,210075422~3861992,246931942~9391575,266411827~12129734,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,362603833~6180969,392134284~2664939,403188267~3969107] 2024-08-17T17:42:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:01 smithi179 ceph-mon[93519]: pgmap v1212: 105 pgs: 105 active+clean; 1.2 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 238 KiB/s rd, 37 MiB/s wr, 1.31k op/s 2024-08-17T17:42:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:01 smithi179 ceph-mon[93519]: osdmap e991: 8 total, 8 up, 8 in 2024-08-17T17:42:02.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:01 smithi130 ceph-mon[105521]: pgmap v1212: 105 pgs: 105 active+clean; 1.2 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 238 KiB/s rd, 37 MiB/s wr, 1.31k op/s 2024-08-17T17:42:02.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:01 smithi130 ceph-mon[105521]: osdmap e991: 8 total, 8 up, 8 in 2024-08-17T17:42:02.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:01 smithi130 ceph-mon[110068]: pgmap v1212: 105 pgs: 105 active+clean; 1.2 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 238 KiB/s rd, 37 MiB/s wr, 1.31k op/s 2024-08-17T17:42:02.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:01 smithi130 ceph-mon[110068]: osdmap e991: 8 total, 8 up, 8 in 2024-08-17T17:42:02.861 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:02.861 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:02.875 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:02.875 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:02.875 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:02.880 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:02.880 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:02.881 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:02.886 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:02.886 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:02.887 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:02.892 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:42:02.892 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:02.892 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:02.898 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:02.898 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:02.898 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:02.903 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:02.903 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:02.903 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:02.908 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~16466976,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:42:02.908 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:02.908 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:02.914 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~10258689,338106187~5516501,348569567~9772461,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:02.914 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:02.914 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:02.919 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,127412771~35185066,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~26813697,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:02.919 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:02.920 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:02.924 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,72099906~2964160,75497472~6811507,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~16101327,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:02.924 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:02.925 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:02.930 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~7822875,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:02.930 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:42:02.930 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:42:02.935 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,215930165~9938570,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~8037217] 2024-08-17T17:42:02.935 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:02.935 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:02.940 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:42:02.941 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:02.941 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:02.946 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,264241152~14300409,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:42:02.946 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,363159108~7536158,393360140~1439083,403188267~3969107] 2024-08-17T17:42:02.946 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,363159108~7536158,393360140~1439083,403188267~3969107] 2024-08-17T17:42:02.951 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,46564393~3447448,52822277~4635371,72099906~2964160,81427530~881449,84600711~5606478,95970646~579400,118030408~3183729,149296328~6283051,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,239310228~2700,246931942~9391575,264241152~14300409,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,362603833~8091433,392134284~2664939,403188267~3969107] 2024-08-17T17:42:03.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:02 smithi179 ceph-mon[93519]: osdmap e992: 8 total, 8 up, 8 in 2024-08-17T17:42:03.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:02 smithi130 ceph-mon[105521]: osdmap e992: 8 total, 8 up, 8 in 2024-08-17T17:42:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:02 smithi130 ceph-mon[110068]: osdmap e992: 8 total, 8 up, 8 in 2024-08-17T17:42:03.858 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:03.858 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:03.875 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:03.876 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:03.876 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:03.882 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:03.882 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:03.882 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:03.888 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:03.888 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:03.888 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:03.893 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:42:03.893 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:03.894 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:03.899 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:03.899 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:03.899 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:03.904 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:03.904 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:03.905 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:03.910 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~16466976,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:42:03.910 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:03.910 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:03.915 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~10258689,338106187~5516501,348569567~9772461,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:03.915 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:03.916 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:03.921 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,127412771~35185066,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~26813697,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:03.921 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:03.921 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:03.927 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,72099906~2964160,75497472~6811507,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~16101327,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:03.927 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:03.927 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:03.932 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~7822875,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:03.932 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:42:03.932 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:42:03.937 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,215930165~9938570,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~8037217] 2024-08-17T17:42:03.937 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:03.937 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:03.942 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:42:03.942 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:03.943 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,265186950~13354611,286246187~4813712,300835157~8035544,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:03.947 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41388025~4298022,46564393~3447448,52330412~7185084,72099906~2964160,81427530~881449,84600711~5606478,92352943~7343768,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,166825307~2678998,172403776~5654992,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~9391575,261307577~1507943,264241152~14300409,285212672~5847227,300835157~13277034,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:42:04.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:03 smithi130 ceph-mon[105521]: pgmap v1215: 105 pgs: 105 active+clean; 1.2 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 165 KiB/s rd, 32 MiB/s wr, 1.95k op/s 2024-08-17T17:42:04.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:03 smithi130 ceph-mon[105521]: osdmap e993: 8 total, 8 up, 8 in 2024-08-17T17:42:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:03 smithi130 ceph-mon[110068]: pgmap v1215: 105 pgs: 105 active+clean; 1.2 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 165 KiB/s rd, 32 MiB/s wr, 1.95k op/s 2024-08-17T17:42:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:03 smithi130 ceph-mon[110068]: osdmap e993: 8 total, 8 up, 8 in 2024-08-17T17:42:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:03 smithi179 ceph-mon[93519]: pgmap v1215: 105 pgs: 105 active+clean; 1.2 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 165 KiB/s rd, 32 MiB/s wr, 1.95k op/s 2024-08-17T17:42:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:03 smithi179 ceph-mon[93519]: osdmap e993: 8 total, 8 up, 8 in 2024-08-17T17:42:04.872 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:04.872 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:04.889 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:04.889 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:04.889 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:04.894 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:04.894 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:04.894 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:04.900 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:04.900 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:04.900 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:04.905 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:42:04.905 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:04.905 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:04.910 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:04.911 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:04.911 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:04.916 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:04.916 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:04.916 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:04.921 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~16466976,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:42:04.922 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:04.922 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:04.927 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~10258689,338106187~5516501,348569567~9772461,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:04.927 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:04.927 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:04.932 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,127412771~35185066,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~26813697,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:04.933 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:04.933 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:04.938 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,72099906~2964160,75497472~6811507,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~16101327,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:04.938 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:04.938 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:04.943 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~7822875,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:04.943 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:42:04.943 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:42:04.949 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,215930165~9938570,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~8037217] 2024-08-17T17:42:04.949 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:04.949 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149924380~5654999,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354913938~909027,363159108~7536158,384785269~7349015,393360140~1439083,403188267~3969107] 2024-08-17T17:42:04.953 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,72099906~2964160,81427530~881449,84600711~17190666,108486098~2678206,118030408~3183729,133306049~7681407,149296328~6283051,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,218372862~7495873,239310228~2700,246931942~13923948,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354913938~909027,362603833~8091433,384785269~10013954,403188267~3969107] 2024-08-17T17:42:05.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:04 smithi130 ceph-mon[105521]: osdmap e994: 8 total, 8 up, 8 in 2024-08-17T17:42:05.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:04 smithi130 ceph-mon[110068]: osdmap e994: 8 total, 8 up, 8 in 2024-08-17T17:42:05.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:04 smithi179 ceph-mon[93519]: osdmap e994: 8 total, 8 up, 8 in 2024-08-17T17:42:05.551 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:05.551 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:05.574 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:05.574 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:05.574 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:05.580 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:05.580 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:05.580 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:05.585 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:05.585 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:05.585 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:05.591 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:42:05.591 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:05.591 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:05.596 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:05.597 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:05.597 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:05.602 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:05.602 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:05.602 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:05.607 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~16466976,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:42:05.607 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:05.607 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:05.612 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~10258689,338106187~5516501,348569567~9772461,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:05.613 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:05.613 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:05.618 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,127412771~35185066,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~26813697,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:05.618 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:05.618 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:05.623 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,72099906~2964160,75497472~6811507,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~16101327,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:05.623 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:05.623 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:05.628 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~7822875,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:05.628 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to HEAD diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:42:05.629 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~13199517,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,217652007~8216728,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~8037217] 2024-08-17T17:42:05.633 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~14497194,16874234~8111475,30413398~2257211,41118839~8893002,52330412~7185084,60052586~1275610,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,166825307~11233461,178411768~2367387,191934227~5441303,208255148~1681884,210075422~3861992,215930165~9938570,228885435~3355214,239310228~2700,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~8037217] 2024-08-17T17:42:06.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:05 smithi130 ceph-mon[105521]: pgmap v1218: 105 pgs: 105 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 176 KiB/s rd, 1.3 MiB/s wr, 5.37k op/s 2024-08-17T17:42:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:05 smithi130 ceph-mon[105521]: osdmap e995: 8 total, 8 up, 8 in 2024-08-17T17:42:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:05 smithi130 ceph-mon[105521]: osdmap e996: 8 total, 8 up, 8 in 2024-08-17T17:42:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:05 smithi130 ceph-mon[110068]: pgmap v1218: 105 pgs: 105 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 176 KiB/s rd, 1.3 MiB/s wr, 5.37k op/s 2024-08-17T17:42:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:05 smithi130 ceph-mon[110068]: osdmap e995: 8 total, 8 up, 8 in 2024-08-17T17:42:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:05 smithi130 ceph-mon[110068]: osdmap e996: 8 total, 8 up, 8 in 2024-08-17T17:42:06.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:05 smithi179 ceph-mon[93519]: pgmap v1218: 105 pgs: 105 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 176 KiB/s rd, 1.3 MiB/s wr, 5.37k op/s 2024-08-17T17:42:06.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:05 smithi179 ceph-mon[93519]: osdmap e995: 8 total, 8 up, 8 in 2024-08-17T17:42:06.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:05 smithi179 ceph-mon[93519]: osdmap e996: 8 total, 8 up, 8 in 2024-08-17T17:42:06.556 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:06.556 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:06.575 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:06.575 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:06.575 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:06.580 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:06.580 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:06.581 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:06.586 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:06.586 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:06.586 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:06.592 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:42:06.592 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:06.592 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:06.597 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:06.597 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:06.597 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:06.603 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:06.603 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:06.603 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:06.608 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~16466976,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:42:06.608 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:06.608 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:06.613 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~10258689,338106187~5516501,348569567~9772461,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:06.614 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:06.614 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:06.619 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,127412771~35185066,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~26813697,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:06.619 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:06.619 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:06.625 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,72099906~2964160,75497472~6811507,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~16101327,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:06.625 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:06.625 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~4859035,245682731~15173159,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:06.630 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52330412~9822811,72099906~2964160,81427530~881449,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,149296328~8034320,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~7822875,245682731~15173159,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:07.564 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:07.564 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:07.582 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:07.582 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:07.582 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:07.587 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:07.587 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:07.588 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:07.593 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:07.593 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:07.593 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:07.599 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:42:07.599 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:07.599 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:07.604 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:07.604 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:07.605 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:07.610 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:07.610 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:07.610 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:07.616 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~16466976,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:42:07.616 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:07.616 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:07.621 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~10258689,338106187~5516501,348569567~9772461,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:07.621 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:07.622 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:07.626 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,127412771~35185066,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~26813697,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:07.627 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:07.627 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,72099906~2964160,75676394~6632585,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~8067007,149924380~7406268,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~5067705,315124553~6130042,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,403188267~10488843] 2024-08-17T17:42:07.632 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,72099906~2964160,75497472~6811507,84600711~21641112,108486098~5668591,118030408~3183729,133306049~7681407,141229321~16101327,160518828~285849,165571375~12487393,178411768~2367387,191934227~5441303,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~19671293,315124553~6130042,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,403188267~10488843] 2024-08-17T17:42:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:07 smithi179 ceph-mon[93519]: pgmap v1221: 105 pgs: 105 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 143 KiB/s rd, 4.7 KiB/s wr, 4.47k op/s 2024-08-17T17:42:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:07 smithi179 ceph-mon[93519]: osdmap e997: 8 total, 8 up, 8 in 2024-08-17T17:42:07.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:07 smithi130 ceph-mon[105521]: pgmap v1221: 105 pgs: 105 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 143 KiB/s rd, 4.7 KiB/s wr, 4.47k op/s 2024-08-17T17:42:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:07 smithi130 ceph-mon[105521]: osdmap e997: 8 total, 8 up, 8 in 2024-08-17T17:42:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:07 smithi130 ceph-mon[110068]: pgmap v1221: 105 pgs: 105 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 143 KiB/s rd, 4.7 KiB/s wr, 4.47k op/s 2024-08-17T17:42:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:07 smithi130 ceph-mon[110068]: osdmap e997: 8 total, 8 up, 8 in 2024-08-17T17:42:08.579 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:08.579 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:08.594 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:08.594 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:08.594 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:08.599 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:08.599 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:08.599 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:08.604 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:08.605 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:08.605 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:08.610 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:42:08.692 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:08.692 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:08.693 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:08.693 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:08.694 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:08.694 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:08.694 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:08.694 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:08.694 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~16466976,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:42:08.694 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:08.694 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:08.694 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~10258689,338106187~5516501,348569567~9772461,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:08.694 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:08.694 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2257211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~2746233,354087269~4254759,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:08.694 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~2257211,32678609~17333232,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,127412771~35185066,165571375~12487393,178411768~2367387,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~26813697,322065070~4478831,327155712~10258689,338106187~5516501,348569567~2746233,354087269~4254759,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:08 smithi179 ceph-mon[93519]: osdmap e998: 8 total, 8 up, 8 in 2024-08-17T17:42:08.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:08 smithi130 ceph-mon[105521]: osdmap e998: 8 total, 8 up, 8 in 2024-08-17T17:42:08.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:08 smithi130 ceph-mon[110068]: osdmap e998: 8 total, 8 up, 8 in 2024-08-17T17:42:09.577 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:09.577 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:09.599 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:09.599 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:09.599 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:09.605 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:09.605 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:09.605 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:09.611 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:09.611 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:09.611 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:09.616 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:42:09.616 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:09.616 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:09.622 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:09.622 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:09.622 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:09.627 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:09.627 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:09.627 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:09.632 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~16466976,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:42:09.632 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:09.632 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~6632585,84600711~21641112,108486098~7939847,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~6844117,338106187~5516501,348569567~9772461,363159108~7536158,384785269~7349015,393360140~1439083,400335542~13341568] 2024-08-17T17:42:09.637 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~6811507,84600711~21641112,108486098~7939847,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~10258689,338106187~5516501,348569567~9772461,362603833~8091433,384785269~10013954,400335542~13341568] 2024-08-17T17:42:09.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:09 smithi130 ceph-mon[105521]: pgmap v1224: 105 pgs: 105 active+clean; 1.1 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 1.51k op/s 2024-08-17T17:42:09.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:09 smithi130 ceph-mon[105521]: osdmap e999: 8 total, 8 up, 8 in 2024-08-17T17:42:09.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:09 smithi130 ceph-mon[110068]: pgmap v1224: 105 pgs: 105 active+clean; 1.1 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 1.51k op/s 2024-08-17T17:42:09.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:09 smithi130 ceph-mon[110068]: osdmap e999: 8 total, 8 up, 8 in 2024-08-17T17:42:09.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:09 smithi179 ceph-mon[93519]: pgmap v1224: 105 pgs: 105 active+clean; 1.1 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 117 KiB/s rd, 6.2 KiB/s wr, 1.51k op/s 2024-08-17T17:42:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:09 smithi179 ceph-mon[93519]: osdmap e999: 8 total, 8 up, 8 in 2024-08-17T17:42:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:42:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:42:09] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.51.0" 2024-08-17T17:42:10.557 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:10.557 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:10.564 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:42:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:42:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:42:10.579 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:10.579 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:10.580 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:10.585 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:10.585 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:10.585 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:10.590 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:10.590 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:10.590 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:10.596 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:42:10.596 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:10.596 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:10.601 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:10.601 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:10.601 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:10.606 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:10.606 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:10.607 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,217652007~8982656,228885435~3355214,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~13052404,348569567~9772461,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:10.611 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,191934227~5441303,197702108~6597113,208255148~1681884,210075422~4655959,215930165~10704498,228885435~3355214,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~16466976,348569567~9772461,362603833~8091433,373115509~42092874] 2024-08-17T17:42:10.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:10 smithi179 ceph-mon[93519]: osdmap e1000: 8 total, 8 up, 8 in 2024-08-17T17:42:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:10 smithi179 ceph-mon[93519]: osdmap e1001: 8 total, 8 up, 8 in 2024-08-17T17:42:10.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:10 smithi130 ceph-mon[105521]: osdmap e1000: 8 total, 8 up, 8 in 2024-08-17T17:42:10.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:10 smithi130 ceph-mon[105521]: osdmap e1001: 8 total, 8 up, 8 in 2024-08-17T17:42:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:10 smithi130 ceph-mon[110068]: osdmap e1000: 8 total, 8 up, 8 in 2024-08-17T17:42:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:10 smithi130 ceph-mon[110068]: osdmap e1001: 8 total, 8 up, 8 in 2024-08-17T17:42:11.585 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:11.585 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:11.607 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:11.607 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:11.607 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:11.612 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:11.612 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:11.613 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:11.617 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:11.618 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:11.618 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:11.623 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:42:11.623 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:11.623 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:11.628 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:11.628 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:11.628 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~23688032,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~21641112,108486098~7974542,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:11.633 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~24985709,30413398~19598443,52172688~9980535,68742908~1813321,72099906~2964160,75497472~8066366,84600711~21641112,108486098~7974542,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:11.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:11 smithi130 ceph-mon[105521]: pgmap v1227: 105 pgs: 105 active+clean; 1.1 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 118 KiB/s rd, 6.2 KiB/s wr, 1.69k op/s 2024-08-17T17:42:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:11 smithi130 ceph-mon[110068]: pgmap v1227: 105 pgs: 105 active+clean; 1.1 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 118 KiB/s rd, 6.2 KiB/s wr, 1.69k op/s 2024-08-17T17:42:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:11 smithi179 ceph-mon[93519]: pgmap v1227: 105 pgs: 105 active+clean; 1.1 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 118 KiB/s rd, 6.2 KiB/s wr, 1.69k op/s 2024-08-17T17:42:12.594 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:12.594 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:12.618 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:12.619 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:12.619 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:12.624 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:12.624 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:12.624 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:12.630 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:12.630 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:12.630 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:12.635 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:42:12.635 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:12.635 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,68742908~1813321,72099906~2964160,75676394~7887444,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,217652007~8982656,228885435~4100521,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,373115509~19018775,393360140~1439083,395994076~19214307] 2024-08-17T17:42:12.640 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,68742908~1813321,71769981~3294085,75497472~8066366,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~4655959,215930165~10704498,228885435~4100521,235227886~25628004,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~8091433,373115509~42092874] 2024-08-17T17:42:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:12 smithi179 ceph-mon[93519]: osdmap e1002: 8 total, 8 up, 8 in 2024-08-17T17:42:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:12 smithi130 ceph-mon[105521]: osdmap e1002: 8 total, 8 up, 8 in 2024-08-17T17:42:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:12 smithi130 ceph-mon[110068]: osdmap e1002: 8 total, 8 up, 8 in 2024-08-17T17:42:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:13.603 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:13.604 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:13.621 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:13.621 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:13.621 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:13.627 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:13.627 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:13.627 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:13.633 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:13.633 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:13.633 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,189053332~15245889,208255148~1681884,210075422~5854743,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~29424687,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:13.638 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,189053332~15245889,208255148~1681884,210075422~50780468,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~32839259,362603833~52604550] 2024-08-17T17:42:13.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:13 smithi179 ceph-mon[93519]: pgmap v1230: 105 pgs: 105 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 1023 B/s wr, 347 op/s 2024-08-17T17:42:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:13 smithi179 ceph-mon[93519]: osdmap e1003: 8 total, 8 up, 8 in 2024-08-17T17:42:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:13 smithi130 ceph-mon[105521]: pgmap v1230: 105 pgs: 105 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 1023 B/s wr, 347 op/s 2024-08-17T17:42:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:13 smithi130 ceph-mon[105521]: osdmap e1003: 8 total, 8 up, 8 in 2024-08-17T17:42:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:13 smithi130 ceph-mon[110068]: pgmap v1230: 105 pgs: 105 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 3.0 KiB/s rd, 1023 B/s wr, 347 op/s 2024-08-17T17:42:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:13 smithi130 ceph-mon[110068]: osdmap e1003: 8 total, 8 up, 8 in 2024-08-17T17:42:14.622 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:14.622 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:14.636 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:14.636 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:14.637 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:14.642 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:14.642 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:14.642 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~16590388,52172688~9980535,64130590~509246,68742908~1813321,72099906~2964160,75676394~7887444,83855009~122571,84600711~31859929,118030408~3183729,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,294440898~14429803,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:14.647 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~19598443,52172688~9980535,64130590~509246,68742908~1813321,71769981~3294085,75497472~8066366,83855009~122571,84600711~31859929,118030408~3183729,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,294440898~32103003,327155712~88052671] 2024-08-17T17:42:14.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:14 smithi130 ceph-mon[105521]: osdmap e1004: 8 total, 8 up, 8 in 2024-08-17T17:42:14.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:14 smithi130 ceph-mon[110068]: osdmap e1004: 8 total, 8 up, 8 in 2024-08-17T17:42:15.009 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:14 smithi179 ceph-mon[93519]: osdmap e1004: 8 total, 8 up, 8 in 2024-08-17T17:42:15.553 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:15.553 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:15.569 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:15.569 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:15.569 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64130590~509246,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165571375~15207780,187244557~17054664,206785372~9144793,217652007~8982656,228066662~7161224,238191726~22664164,261307577~1507943,265186950~13354611,286246187~4813712,291073215~751556,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:15.574 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64130590~509246,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,165571375~15207780,187244557~17054664,206785372~54070518,261307577~1507943,264241152~14300409,285212672~5847227,291073215~751556,292864570~33679331,327155712~88052671] 2024-08-17T17:42:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:15 smithi130 ceph-mon[105521]: pgmap v1233: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 973 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 84 KiB/s rd, 6.0 KiB/s wr, 995 op/s 2024-08-17T17:42:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:15 smithi130 ceph-mon[105521]: osdmap e1005: 8 total, 8 up, 8 in 2024-08-17T17:42:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:15 smithi130 ceph-mon[105521]: osdmap e1006: 8 total, 8 up, 8 in 2024-08-17T17:42:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:15 smithi130 ceph-mon[110068]: pgmap v1233: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 973 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 84 KiB/s rd, 6.0 KiB/s wr, 995 op/s 2024-08-17T17:42:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:15 smithi130 ceph-mon[110068]: osdmap e1005: 8 total, 8 up, 8 in 2024-08-17T17:42:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:15 smithi130 ceph-mon[110068]: osdmap e1006: 8 total, 8 up, 8 in 2024-08-17T17:42:15.953 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:15 smithi179 ceph-mon[93519]: pgmap v1233: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 973 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 84 KiB/s rd, 6.0 KiB/s wr, 995 op/s 2024-08-17T17:42:15.953 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:15 smithi179 ceph-mon[93519]: osdmap e1005: 8 total, 8 up, 8 in 2024-08-17T17:42:15.953 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:15 smithi179 ceph-mon[93519]: osdmap e1006: 8 total, 8 up, 8 in 2024-08-17T17:42:16.549 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:16.549 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [1297677~29055666,30413398~2265211,33421453~28731770,64100320~539516,64681659~7088322,72099906~2964160,75676394~7887444,83855009~122571,84600711~38153749,127412771~21883557,149924380~12673457,165161949~15617206,184955152~19344069,206579328~9350837,217652007~8982656,228066662~7161224,238191726~25316559,265186950~13354611,286246187~4813712,291073215~751556,292092374~772196,293482393~15388308,309044486~12210109,322065070~4478831,330570284~32033549,363159108~7536158,372708801~19425483,393360140~1439083,395994076~19214307] 2024-08-17T17:42:16.565 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [0~30353343,30413398~31739825,64100320~539516,64681659~10382407,75497472~8066366,83855009~122571,84600711~38153749,125829120~36768717,163577856~17201299,184549376~19749845,205520896~57987389,264241152~14300409,285212672~41331229,327155712~88052671] 2024-08-17T17:42:17.572 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateStress (39862 ms) 2024-08-17T17:42:17.572 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParent 2024-08-17T17:42:17.572 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:17 smithi179 ceph-mon[93519]: pgmap v1236: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 973 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 81 KiB/s rd, 5.0 KiB/s wr, 826 op/s 2024-08-17T17:42:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:17 smithi179 ceph-mon[93519]: osdmap e1007: 8 total, 8 up, 8 in 2024-08-17T17:42:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:17 smithi130 ceph-mon[105521]: pgmap v1236: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 973 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 81 KiB/s rd, 5.0 KiB/s wr, 826 op/s 2024-08-17T17:42:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:17 smithi130 ceph-mon[105521]: osdmap e1007: 8 total, 8 up, 8 in 2024-08-17T17:42:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:17 smithi130 ceph-mon[110068]: pgmap v1236: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 973 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 81 KiB/s rd, 5.0 KiB/s wr, 826 op/s 2024-08-17T17:42:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:17 smithi130 ceph-mon[110068]: osdmap e1007: 8 total, 8 up, 8 in 2024-08-17T17:42:18.685 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateParent (1113 ms) 2024-08-17T17:42:18.685 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateIgnoreParent 2024-08-17T17:42:18.685 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:18 smithi179 ceph-mon[93519]: osdmap e1008: 8 total, 8 up, 8 in 2024-08-17T17:42:18.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:18 smithi130 ceph-mon[105521]: osdmap e1008: 8 total, 8 up, 8 in 2024-08-17T17:42:18.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:18 smithi130 ceph-mon[110068]: osdmap e1008: 8 total, 8 up, 8 in 2024-08-17T17:42:19.702 INFO:tasks.workunit.client.0.smithi130.stdout: wrote [856517~88915,1753179~4083,7417903~37321,10253921~73090,15384576~43960,16700695~75127,19032600~80335,19943378~75529,20045490~32615,20335891~15887] to clone 2024-08-17T17:42:19.702 INFO:tasks.workunit.client.0.smithi130.stdout: diff was [0~1757262,4194304~3260920,8388608~1938403,12582912~4192910,16777216~3574562] 2024-08-17T17:42:19.706 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateIgnoreParent (1021 ms) 2024-08-17T17:42:19.706 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateCallbackError 2024-08-17T17:42:19.706 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:19.725 INFO:tasks.workunit.client.0.smithi130.stdout: wrote [3275422~93757,4395914~38870,4498422~62370,6601874~88286,7460495~73720,8836164~86559,16678913~81145,16943321~50989,17689599~31827,19011578~95428] 2024-08-17T17:42:19.735 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateCallbackError (29 ms) 2024-08-17T17:42:19.735 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParentDiscard 2024-08-17T17:42:19.735 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:19.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:19 smithi130 ceph-mon[105521]: pgmap v1239: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 830 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 65 KiB/s rd, 3.0 MiB/s wr, 354 op/s 2024-08-17T17:42:19.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:19 smithi130 ceph-mon[105521]: osdmap e1009: 8 total, 8 up, 8 in 2024-08-17T17:42:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:19 smithi130 ceph-mon[110068]: pgmap v1239: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 830 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 65 KiB/s rd, 3.0 MiB/s wr, 354 op/s 2024-08-17T17:42:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:19 smithi130 ceph-mon[110068]: osdmap e1009: 8 total, 8 up, 8 in 2024-08-17T17:42:19.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:19 smithi179 ceph-mon[93519]: pgmap v1239: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 830 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 65 KiB/s rd, 3.0 MiB/s wr, 354 op/s 2024-08-17T17:42:19.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:19 smithi179 ceph-mon[93519]: osdmap e1009: 8 total, 8 up, 8 in 2024-08-17T17:42:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:42:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:42:19] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-08-17T17:42:20.569 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:42:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:42:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:42:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:20 smithi179 ceph-mon[93519]: osdmap e1010: 8 total, 8 up, 8 in 2024-08-17T17:42:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:20 smithi179 ceph-mon[93519]: osdmap e1011: 8 total, 8 up, 8 in 2024-08-17T17:42:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:20 smithi130 ceph-mon[105521]: osdmap e1010: 8 total, 8 up, 8 in 2024-08-17T17:42:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:20 smithi130 ceph-mon[105521]: osdmap e1011: 8 total, 8 up, 8 in 2024-08-17T17:42:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:20 smithi130 ceph-mon[110068]: osdmap e1010: 8 total, 8 up, 8 in 2024-08-17T17:42:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:20 smithi130 ceph-mon[110068]: osdmap e1011: 8 total, 8 up, 8 in 2024-08-17T17:42:21.672 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateParentDiscard (1937 ms) 2024-08-17T17:42:21.672 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnalignedSmall 2024-08-17T17:42:21.672 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:21.729 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnalignedSmall (57 ms) 2024-08-17T17:42:21.729 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnaligned 2024-08-17T17:42:21.729 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:21.824 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnaligned (94 ms) 2024-08-17T17:42:21.824 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateTryAcquireLock 2024-08-17T17:42:21.824 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:21.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:21 smithi179 ceph-mon[93519]: pgmap v1242: 105 pgs: 105 active+clean; 793 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 66 KiB/s rd, 3.0 MiB/s wr, 383 op/s 2024-08-17T17:42:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:21 smithi130 ceph-mon[105521]: pgmap v1242: 105 pgs: 105 active+clean; 793 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 66 KiB/s rd, 3.0 MiB/s wr, 383 op/s 2024-08-17T17:42:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:21 smithi130 ceph-mon[110068]: pgmap v1242: 105 pgs: 105 active+clean; 793 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 66 KiB/s rd, 3.0 MiB/s wr, 383 op/s 2024-08-17T17:42:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:22 smithi179 ceph-mon[93519]: osdmap e1012: 8 total, 8 up, 8 in 2024-08-17T17:42:22.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:22 smithi130 ceph-mon[105521]: osdmap e1012: 8 total, 8 up, 8 in 2024-08-17T17:42:22.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:22 smithi130 ceph-mon[110068]: osdmap e1012: 8 total, 8 up, 8 in 2024-08-17T17:42:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:23 smithi179 ceph-mon[93519]: pgmap v1245: 105 pgs: 105 active+clean; 792 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 34 op/s 2024-08-17T17:42:23.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:23 smithi130 ceph-mon[105521]: pgmap v1245: 105 pgs: 105 active+clean; 792 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 34 op/s 2024-08-17T17:42:23.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:23 smithi130 ceph-mon[110068]: pgmap v1245: 105 pgs: 105 active+clean; 792 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 1.2 KiB/s rd, 2.1 MiB/s wr, 34 op/s 2024-08-17T17:42:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:25 smithi179 ceph-mon[93519]: pgmap v1246: 105 pgs: 105 active+clean; 887 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 MiB/s rd, 9.8 MiB/s wr, 282 op/s 2024-08-17T17:42:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:25 smithi179 ceph-mon[93519]: osdmap e1013: 8 total, 8 up, 8 in 2024-08-17T17:42:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:25 smithi130 ceph-mon[105521]: pgmap v1246: 105 pgs: 105 active+clean; 887 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 MiB/s rd, 9.8 MiB/s wr, 282 op/s 2024-08-17T17:42:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:25 smithi130 ceph-mon[105521]: osdmap e1013: 8 total, 8 up, 8 in 2024-08-17T17:42:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:25 smithi130 ceph-mon[110068]: pgmap v1246: 105 pgs: 105 active+clean; 887 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 MiB/s rd, 9.8 MiB/s wr, 282 op/s 2024-08-17T17:42:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:25 smithi130 ceph-mon[110068]: osdmap e1013: 8 total, 8 up, 8 in 2024-08-17T17:42:26.859 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateTryAcquireLock (5036 ms) 2024-08-17T17:42:26.859 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStriping 2024-08-17T17:42:26.859 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:42:26.859 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/0.DiffIterateStriping (0 ms) 2024-08-17T17:42:27.594 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 19 tests from DiffIterateTest/0 (235303 ms total) 2024-08-17T17:42:27.594 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:42:27.594 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 19 tests from DiffIterateTest/1, where TypeParam = DiffIterateParams 2024-08-17T17:42:27.628 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterate 2024-08-17T17:42:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:27 smithi130 ceph-mon[105521]: pgmap v1248: 105 pgs: 105 active+clean; 887 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.4 MiB/s wr, 251 op/s 2024-08-17T17:42:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:27 smithi130 ceph-mon[110068]: pgmap v1248: 105 pgs: 105 active+clean; 887 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.4 MiB/s wr, 251 op/s 2024-08-17T17:42:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:27 smithi179 ceph-mon[93519]: pgmap v1248: 105 pgs: 105 active+clean; 887 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.4 MiB/s wr, 251 op/s 2024-08-17T17:42:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:28.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:28 smithi130 ceph-mon[105521]: osdmap e1014: 8 total, 8 up, 8 in 2024-08-17T17:42:28.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:28 smithi130 ceph-mon[110068]: osdmap e1014: 8 total, 8 up, 8 in 2024-08-17T17:42:29.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:28 smithi179 ceph-mon[93519]: osdmap e1014: 8 total, 8 up, 8 in 2024-08-17T17:42:29.620 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:29.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:42:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:42:29] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-08-17T17:42:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:29 smithi130 ceph-mon[105521]: pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:42:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:29 smithi130 ceph-mon[105521]: osdmap e1015: 8 total, 8 up, 8 in 2024-08-17T17:42:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1700197602' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-40","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:42:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:29 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-40","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:42:29.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:29 smithi130 ceph-mon[110068]: pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:42:29.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:29 smithi130 ceph-mon[110068]: osdmap e1015: 8 total, 8 up, 8 in 2024-08-17T17:42:29.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1700197602' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-40","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:42:29.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:29 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-40","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:42:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:29 smithi179 ceph-mon[93519]: pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:42:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:29 smithi179 ceph-mon[93519]: osdmap e1015: 8 total, 8 up, 8 in 2024-08-17T17:42:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1700197602' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-40","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:42:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:29 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-40","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:42:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:42:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:42:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:42:30.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:30 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-40","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:42:30.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:30 smithi130 ceph-mon[110068]: osdmap e1016: 8 total, 8 up, 8 in 2024-08-17T17:42:30.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:30 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-40","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:42:30.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:30 smithi130 ceph-mon[105521]: osdmap e1016: 8 total, 8 up, 8 in 2024-08-17T17:42:31.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:30 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi130-87148-40","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:42:31.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:30 smithi179 ceph-mon[93519]: osdmap e1016: 8 total, 8 up, 8 in 2024-08-17T17:42:31.663 INFO:tasks.workunit.client.0.smithi130.stdout: wrote [1140681~97241,1305128~70321,2925338~47862,3102041~30320,3747719~56977,3821151~86580,7619963~29502,12961911~65710,17256579~21953,20787120~55844] 2024-08-17T17:42:31.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:31 smithi130 ceph-mon[105521]: pgmap v1253: 105 pgs: 4 creating+peering, 24 unknown, 77 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:42:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:31 smithi130 ceph-mon[105521]: osdmap e1017: 8 total, 8 up, 8 in 2024-08-17T17:42:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:31 smithi130 ceph-mon[110068]: pgmap v1253: 105 pgs: 4 creating+peering, 24 unknown, 77 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:42:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:31 smithi130 ceph-mon[110068]: osdmap e1017: 8 total, 8 up, 8 in 2024-08-17T17:42:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:31 smithi179 ceph-mon[93519]: pgmap v1253: 105 pgs: 4 creating+peering, 24 unknown, 77 active+clean; 582 KiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:42:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:31 smithi179 ceph-mon[93519]: osdmap e1017: 8 total, 8 up, 8 in 2024-08-17T17:42:32.651 INFO:tasks.workunit.client.0.smithi130.stdout: wrote [0~20971520] 2024-08-17T17:42:32.662 INFO:tasks.workunit.client.0.smithi130.stdout: diff was [0~20971520] 2024-08-17T17:42:32.666 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterate (5038 ms) 2024-08-17T17:42:32.666 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministic 2024-08-17T17:42:32.666 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:32.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:32 smithi130 ceph-mon[105521]: osdmap e1018: 8 total, 8 up, 8 in 2024-08-17T17:42:32.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:32 smithi130 ceph-mon[110068]: osdmap e1018: 8 total, 8 up, 8 in 2024-08-17T17:42:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:32 smithi179 ceph-mon[93519]: osdmap e1018: 8 total, 8 up, 8 in 2024-08-17T17:42:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:33 smithi130 ceph-mon[105521]: pgmap v1256: 105 pgs: 4 creating+peering, 21 unknown, 80 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:42:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:33 smithi130 ceph-mon[105521]: osdmap e1019: 8 total, 8 up, 8 in 2024-08-17T17:42:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:33 smithi130 ceph-mon[110068]: pgmap v1256: 105 pgs: 4 creating+peering, 21 unknown, 80 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:42:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:33 smithi130 ceph-mon[110068]: osdmap e1019: 8 total, 8 up, 8 in 2024-08-17T17:42:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:33 smithi179 ceph-mon[93519]: pgmap v1256: 105 pgs: 4 creating+peering, 21 unknown, 80 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:42:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:33 smithi179 ceph-mon[93519]: osdmap e1019: 8 total, 8 up, 8 in 2024-08-17T17:42:34.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:34 smithi130 ceph-mon[105521]: osdmap e1020: 8 total, 8 up, 8 in 2024-08-17T17:42:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:34 smithi130 ceph-mon[110068]: osdmap e1020: 8 total, 8 up, 8 in 2024-08-17T17:42:35.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:34 smithi179 ceph-mon[93519]: osdmap e1020: 8 total, 8 up, 8 in 2024-08-17T17:42:35.689 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:35.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:35 smithi130 ceph-mon[105521]: pgmap v1259: 105 pgs: 105 active+clean; 13 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 286 KiB/s wr, 56 op/s 2024-08-17T17:42:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:35 smithi130 ceph-mon[105521]: osdmap e1021: 8 total, 8 up, 8 in 2024-08-17T17:42:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:35 smithi130 ceph-mon[110068]: pgmap v1259: 105 pgs: 105 active+clean; 13 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 286 KiB/s wr, 56 op/s 2024-08-17T17:42:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:35 smithi130 ceph-mon[110068]: osdmap e1021: 8 total, 8 up, 8 in 2024-08-17T17:42:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:35 smithi179 ceph-mon[93519]: pgmap v1259: 105 pgs: 105 active+clean; 13 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 286 KiB/s wr, 56 op/s 2024-08-17T17:42:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:35 smithi179 ceph-mon[93519]: osdmap e1021: 8 total, 8 up, 8 in 2024-08-17T17:42:36.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:36 smithi130 ceph-mon[105521]: osdmap e1022: 8 total, 8 up, 8 in 2024-08-17T17:42:36.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:36 smithi130 ceph-mon[110068]: osdmap e1022: 8 total, 8 up, 8 in 2024-08-17T17:42:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:36 smithi179 ceph-mon[93519]: osdmap e1022: 8 total, 8 up, 8 in 2024-08-17T17:42:37.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:37 smithi130 ceph-mon[105521]: pgmap v1262: 105 pgs: 105 active+clean; 13 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 286 KiB/s wr, 57 op/s 2024-08-17T17:42:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:37 smithi130 ceph-mon[105521]: osdmap e1023: 8 total, 8 up, 8 in 2024-08-17T17:42:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:37 smithi130 ceph-mon[110068]: pgmap v1262: 105 pgs: 105 active+clean; 13 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 286 KiB/s wr, 57 op/s 2024-08-17T17:42:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:37 smithi130 ceph-mon[110068]: osdmap e1023: 8 total, 8 up, 8 in 2024-08-17T17:42:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:37 smithi179 ceph-mon[93519]: pgmap v1262: 105 pgs: 105 active+clean; 13 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 286 KiB/s wr, 57 op/s 2024-08-17T17:42:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:37 smithi179 ceph-mon[93519]: osdmap e1023: 8 total, 8 up, 8 in 2024-08-17T17:42:38.702 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:38 smithi179 ceph-mon[93519]: osdmap e1024: 8 total, 8 up, 8 in 2024-08-17T17:42:39.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:38 smithi130 ceph-mon[105521]: osdmap e1024: 8 total, 8 up, 8 in 2024-08-17T17:42:39.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:38 smithi130 ceph-mon[110068]: osdmap e1024: 8 total, 8 up, 8 in 2024-08-17T17:42:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:39 smithi179 ceph-mon[93519]: pgmap v1265: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 19 KiB/s wr, 84 op/s 2024-08-17T17:42:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:39 smithi179 ceph-mon[93519]: osdmap e1025: 8 total, 8 up, 8 in 2024-08-17T17:42:40.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:39 smithi130 ceph-mon[105521]: pgmap v1265: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 19 KiB/s wr, 84 op/s 2024-08-17T17:42:40.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:39 smithi130 ceph-mon[105521]: osdmap e1025: 8 total, 8 up, 8 in 2024-08-17T17:42:40.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:39 smithi130 ceph-mon[110068]: pgmap v1265: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 19 KiB/s wr, 84 op/s 2024-08-17T17:42:40.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:39 smithi130 ceph-mon[110068]: osdmap e1025: 8 total, 8 up, 8 in 2024-08-17T17:42:40.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:42:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:42:39] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-08-17T17:42:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:42:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:42:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:42:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:40 smithi179 ceph-mon[93519]: osdmap e1026: 8 total, 8 up, 8 in 2024-08-17T17:42:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:40 smithi179 ceph-mon[93519]: osdmap e1027: 8 total, 8 up, 8 in 2024-08-17T17:42:41.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:40 smithi130 ceph-mon[105521]: osdmap e1026: 8 total, 8 up, 8 in 2024-08-17T17:42:41.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:40 smithi130 ceph-mon[105521]: osdmap e1027: 8 total, 8 up, 8 in 2024-08-17T17:42:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:40 smithi130 ceph-mon[110068]: osdmap e1026: 8 total, 8 up, 8 in 2024-08-17T17:42:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:40 smithi130 ceph-mon[110068]: osdmap e1027: 8 total, 8 up, 8 in 2024-08-17T17:42:41.579 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:41 smithi179 ceph-mon[93519]: pgmap v1268: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 19 KiB/s wr, 85 op/s 2024-08-17T17:42:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:41 smithi179 ceph-mon[93519]: osdmap e1028: 8 total, 8 up, 8 in 2024-08-17T17:42:42.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:41 smithi130 ceph-mon[105521]: pgmap v1268: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 19 KiB/s wr, 85 op/s 2024-08-17T17:42:42.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:41 smithi130 ceph-mon[105521]: osdmap e1028: 8 total, 8 up, 8 in 2024-08-17T17:42:42.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:41 smithi130 ceph-mon[110068]: pgmap v1268: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 19 KiB/s wr, 85 op/s 2024-08-17T17:42:42.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:41 smithi130 ceph-mon[110068]: osdmap e1028: 8 total, 8 up, 8 in 2024-08-17T17:42:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:42 smithi179 ceph-mon[93519]: osdmap e1029: 8 total, 8 up, 8 in 2024-08-17T17:42:43.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:43.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:42 smithi130 ceph-mon[105521]: osdmap e1029: 8 total, 8 up, 8 in 2024-08-17T17:42:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:42 smithi130 ceph-mon[110068]: osdmap e1029: 8 total, 8 up, 8 in 2024-08-17T17:42:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:43 smithi179 ceph-mon[93519]: pgmap v1271: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.5 KiB/s wr, 18 op/s 2024-08-17T17:42:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:43 smithi179 ceph-mon[93519]: osdmap e1030: 8 total, 8 up, 8 in 2024-08-17T17:42:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:43 smithi130 ceph-mon[105521]: pgmap v1271: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.5 KiB/s wr, 18 op/s 2024-08-17T17:42:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:43 smithi130 ceph-mon[105521]: osdmap e1030: 8 total, 8 up, 8 in 2024-08-17T17:42:44.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:43 smithi130 ceph-mon[110068]: pgmap v1271: 105 pgs: 105 active+clean; 16 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.5 KiB/s wr, 18 op/s 2024-08-17T17:42:44.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:44.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:43 smithi130 ceph-mon[110068]: osdmap e1030: 8 total, 8 up, 8 in 2024-08-17T17:42:44.581 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:45 smithi179 ceph-mon[93519]: pgmap v1274: 105 pgs: 105 active+clean; 19 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 8.7 KiB/s wr, 105 op/s 2024-08-17T17:42:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:45 smithi179 ceph-mon[93519]: osdmap e1031: 8 total, 8 up, 8 in 2024-08-17T17:42:45.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:45 smithi130 ceph-mon[105521]: pgmap v1274: 105 pgs: 105 active+clean; 19 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 8.7 KiB/s wr, 105 op/s 2024-08-17T17:42:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:45 smithi130 ceph-mon[105521]: osdmap e1031: 8 total, 8 up, 8 in 2024-08-17T17:42:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:45 smithi130 ceph-mon[110068]: pgmap v1274: 105 pgs: 105 active+clean; 19 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 71 KiB/s rd, 8.7 KiB/s wr, 105 op/s 2024-08-17T17:42:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:45 smithi130 ceph-mon[110068]: osdmap e1031: 8 total, 8 up, 8 in 2024-08-17T17:42:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:46 smithi179 ceph-mon[93519]: osdmap e1032: 8 total, 8 up, 8 in 2024-08-17T17:42:46.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:46 smithi130 ceph-mon[105521]: osdmap e1032: 8 total, 8 up, 8 in 2024-08-17T17:42:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:46 smithi130 ceph-mon[110068]: osdmap e1032: 8 total, 8 up, 8 in 2024-08-17T17:42:47.599 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministic (14933 ms) 2024-08-17T17:42:47.599 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministicPP 2024-08-17T17:42:47.599 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:47 smithi179 ceph-mon[93519]: pgmap v1277: 105 pgs: 105 active+clean; 19 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 7.0 KiB/s wr, 96 op/s 2024-08-17T17:42:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:47 smithi179 ceph-mon[93519]: osdmap e1033: 8 total, 8 up, 8 in 2024-08-17T17:42:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:47 smithi130 ceph-mon[105521]: pgmap v1277: 105 pgs: 105 active+clean; 19 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 7.0 KiB/s wr, 96 op/s 2024-08-17T17:42:47.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:47 smithi130 ceph-mon[105521]: osdmap e1033: 8 total, 8 up, 8 in 2024-08-17T17:42:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:47 smithi130 ceph-mon[110068]: pgmap v1277: 105 pgs: 105 active+clean; 19 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 7.0 KiB/s wr, 96 op/s 2024-08-17T17:42:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:47 smithi130 ceph-mon[110068]: osdmap e1033: 8 total, 8 up, 8 in 2024-08-17T17:42:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:48 smithi179 ceph-mon[93519]: osdmap e1034: 8 total, 8 up, 8 in 2024-08-17T17:42:48.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:48 smithi130 ceph-mon[105521]: osdmap e1034: 8 total, 8 up, 8 in 2024-08-17T17:42:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:48 smithi130 ceph-mon[110068]: osdmap e1034: 8 total, 8 up, 8 in 2024-08-17T17:42:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:49 smithi179 ceph-mon[93519]: pgmap v1280: 105 pgs: 105 active+clean; 20 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 5.7 KiB/s wr, 61 op/s 2024-08-17T17:42:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:49 smithi179 ceph-mon[93519]: osdmap e1035: 8 total, 8 up, 8 in 2024-08-17T17:42:49.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:49 smithi130 ceph-mon[105521]: pgmap v1280: 105 pgs: 105 active+clean; 20 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 5.7 KiB/s wr, 61 op/s 2024-08-17T17:42:49.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:49 smithi130 ceph-mon[105521]: osdmap e1035: 8 total, 8 up, 8 in 2024-08-17T17:42:49.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:42:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:42:49] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-08-17T17:42:49.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:49 smithi130 ceph-mon[110068]: pgmap v1280: 105 pgs: 105 active+clean; 20 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 5.7 KiB/s wr, 61 op/s 2024-08-17T17:42:49.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:49 smithi130 ceph-mon[110068]: osdmap e1035: 8 total, 8 up, 8 in 2024-08-17T17:42:50.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:42:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:42:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:42:50.626 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:50 smithi179 ceph-mon[93519]: osdmap e1036: 8 total, 8 up, 8 in 2024-08-17T17:42:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:50 smithi130 ceph-mon[105521]: osdmap e1036: 8 total, 8 up, 8 in 2024-08-17T17:42:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:50 smithi130 ceph-mon[110068]: osdmap e1036: 8 total, 8 up, 8 in 2024-08-17T17:42:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:51 smithi179 ceph-mon[93519]: pgmap v1283: 105 pgs: 105 active+clean; 24 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 8.5 KiB/s wr, 104 op/s 2024-08-17T17:42:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:51 smithi179 ceph-mon[93519]: osdmap e1037: 8 total, 8 up, 8 in 2024-08-17T17:42:51.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:51 smithi130 ceph-mon[105521]: pgmap v1283: 105 pgs: 105 active+clean; 24 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 8.5 KiB/s wr, 104 op/s 2024-08-17T17:42:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:51 smithi130 ceph-mon[105521]: osdmap e1037: 8 total, 8 up, 8 in 2024-08-17T17:42:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:51 smithi130 ceph-mon[110068]: pgmap v1283: 105 pgs: 105 active+clean; 24 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 8.5 KiB/s wr, 104 op/s 2024-08-17T17:42:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:51 smithi130 ceph-mon[110068]: osdmap e1037: 8 total, 8 up, 8 in 2024-08-17T17:42:52.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:52 smithi130 ceph-mon[105521]: osdmap e1038: 8 total, 8 up, 8 in 2024-08-17T17:42:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:52 smithi130 ceph-mon[110068]: osdmap e1038: 8 total, 8 up, 8 in 2024-08-17T17:42:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:52 smithi179 ceph-mon[93519]: osdmap e1038: 8 total, 8 up, 8 in 2024-08-17T17:42:53.637 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:53.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:53 smithi130 ceph-mon[105521]: pgmap v1286: 105 pgs: 105 active+clean; 28 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 4.2 KiB/s wr, 56 op/s 2024-08-17T17:42:53.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:53 smithi130 ceph-mon[105521]: osdmap e1039: 8 total, 8 up, 8 in 2024-08-17T17:42:53.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:53 smithi130 ceph-mon[110068]: pgmap v1286: 105 pgs: 105 active+clean; 28 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 4.2 KiB/s wr, 56 op/s 2024-08-17T17:42:53.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:53 smithi130 ceph-mon[110068]: osdmap e1039: 8 total, 8 up, 8 in 2024-08-17T17:42:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:53 smithi179 ceph-mon[93519]: pgmap v1286: 105 pgs: 105 active+clean; 28 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 40 KiB/s rd, 4.2 KiB/s wr, 56 op/s 2024-08-17T17:42:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:53 smithi179 ceph-mon[93519]: osdmap e1039: 8 total, 8 up, 8 in 2024-08-17T17:42:54.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:54 smithi130 ceph-mon[105521]: osdmap e1040: 8 total, 8 up, 8 in 2024-08-17T17:42:54.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:54 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:42:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:54 smithi130 ceph-mon[110068]: osdmap e1040: 8 total, 8 up, 8 in 2024-08-17T17:42:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:54 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:42:55.088 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:54 smithi179 ceph-mon[93519]: osdmap e1040: 8 total, 8 up, 8 in 2024-08-17T17:42:55.089 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:54 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:42:55.909 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:55 smithi130 ceph-mon[110068]: pgmap v1289: 105 pgs: 105 active+clean; 34 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 8.2 KiB/s wr, 78 op/s 2024-08-17T17:42:55.910 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:55 smithi130 ceph-mon[110068]: osdmap e1041: 8 total, 8 up, 8 in 2024-08-17T17:42:55.910 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:55 smithi130 ceph-mon[105521]: pgmap v1289: 105 pgs: 105 active+clean; 34 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 8.2 KiB/s wr, 78 op/s 2024-08-17T17:42:55.910 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:55 smithi130 ceph-mon[105521]: osdmap e1041: 8 total, 8 up, 8 in 2024-08-17T17:42:55.983 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:55 smithi179 ceph-mon[93519]: pgmap v1289: 105 pgs: 105 active+clean; 34 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 8.2 KiB/s wr, 78 op/s 2024-08-17T17:42:55.984 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:55 smithi179 ceph-mon[93519]: osdmap e1041: 8 total, 8 up, 8 in 2024-08-17T17:42:56.654 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:56.889 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:56 smithi130 ceph-mon[105521]: osdmap e1042: 8 total, 8 up, 8 in 2024-08-17T17:42:56.889 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:56 smithi130 ceph-mon[110068]: osdmap e1042: 8 total, 8 up, 8 in 2024-08-17T17:42:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:56 smithi179 ceph-mon[93519]: osdmap e1042: 8 total, 8 up, 8 in 2024-08-17T17:42:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:57 smithi179 ceph-mon[93519]: pgmap v1292: 105 pgs: 105 active+clean; 34 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 7.5 KiB/s wr, 76 op/s 2024-08-17T17:42:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:57 smithi179 ceph-mon[93519]: osdmap e1043: 8 total, 8 up, 8 in 2024-08-17T17:42:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:57 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:57 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:57.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:57 smithi130 ceph-mon[105521]: pgmap v1292: 105 pgs: 105 active+clean; 34 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 7.5 KiB/s wr, 76 op/s 2024-08-17T17:42:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:57 smithi130 ceph-mon[105521]: osdmap e1043: 8 total, 8 up, 8 in 2024-08-17T17:42:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:57 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:57 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:57 smithi130 ceph-mon[110068]: pgmap v1292: 105 pgs: 105 active+clean; 34 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 7.5 KiB/s wr, 76 op/s 2024-08-17T17:42:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:57 smithi130 ceph-mon[110068]: osdmap e1043: 8 total, 8 up, 8 in 2024-08-17T17:42:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:57 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:57 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:58 smithi130 ceph-mon[105521]: osdmap e1044: 8 total, 8 up, 8 in 2024-08-17T17:42:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:42:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:42:58.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:58 smithi130 ceph-mon[110068]: osdmap e1044: 8 total, 8 up, 8 in 2024-08-17T17:42:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:42:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:42:58.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:58 smithi179 ceph-mon[93519]: osdmap e1044: 8 total, 8 up, 8 in 2024-08-17T17:42:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:42:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:42:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:42:59.683 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:42:59.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:59 smithi130 ceph-mon[105521]: pgmap v1295: 105 pgs: 105 active+clean; 40 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 6.0 KiB/s wr, 54 op/s 2024-08-17T17:42:59.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:42:59 smithi130 ceph-mon[105521]: osdmap e1045: 8 total, 8 up, 8 in 2024-08-17T17:42:59.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:42:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:42:59] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.51.0" 2024-08-17T17:42:59.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:59 smithi130 ceph-mon[110068]: pgmap v1295: 105 pgs: 105 active+clean; 40 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 6.0 KiB/s wr, 54 op/s 2024-08-17T17:42:59.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:42:59 smithi130 ceph-mon[110068]: osdmap e1045: 8 total, 8 up, 8 in 2024-08-17T17:43:00.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:59 smithi179 ceph-mon[93519]: pgmap v1295: 105 pgs: 105 active+clean; 40 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 6.0 KiB/s wr, 54 op/s 2024-08-17T17:43:00.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:42:59 smithi179 ceph-mon[93519]: osdmap e1045: 8 total, 8 up, 8 in 2024-08-17T17:43:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:43:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:43:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:43:00.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:00 smithi130 ceph-mon[105521]: osdmap e1046: 8 total, 8 up, 8 in 2024-08-17T17:43:00.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:00 smithi130 ceph-mon[110068]: osdmap e1046: 8 total, 8 up, 8 in 2024-08-17T17:43:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:00 smithi179 ceph-mon[93519]: osdmap e1046: 8 total, 8 up, 8 in 2024-08-17T17:43:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:01 smithi130 ceph-mon[110068]: pgmap v1298: 105 pgs: 105 active+clean; 43 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 9.2 KiB/s wr, 110 op/s 2024-08-17T17:43:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:01 smithi130 ceph-mon[110068]: osdmap e1047: 8 total, 8 up, 8 in 2024-08-17T17:43:01.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:01 smithi130 ceph-mon[105521]: pgmap v1298: 105 pgs: 105 active+clean; 43 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 9.2 KiB/s wr, 110 op/s 2024-08-17T17:43:01.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:01 smithi130 ceph-mon[105521]: osdmap e1047: 8 total, 8 up, 8 in 2024-08-17T17:43:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:01 smithi179 ceph-mon[93519]: pgmap v1298: 105 pgs: 105 active+clean; 43 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 9.2 KiB/s wr, 110 op/s 2024-08-17T17:43:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:01 smithi179 ceph-mon[93519]: osdmap e1047: 8 total, 8 up, 8 in 2024-08-17T17:43:02.703 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministicPP (15104 ms) 2024-08-17T17:43:02.703 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministicLUKS1 2024-08-17T17:43:02.703 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:43:03.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:02 smithi179 ceph-mon[93519]: osdmap e1048: 8 total, 8 up, 8 in 2024-08-17T17:43:03.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:02 smithi130 ceph-mon[110068]: osdmap e1048: 8 total, 8 up, 8 in 2024-08-17T17:43:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:02 smithi130 ceph-mon[105521]: osdmap e1048: 8 total, 8 up, 8 in 2024-08-17T17:43:04.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:03 smithi179 ceph-mon[93519]: pgmap v1301: 105 pgs: 105 active+clean; 43 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 4.2 KiB/s wr, 63 op/s 2024-08-17T17:43:04.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:03 smithi179 ceph-mon[93519]: osdmap e1049: 8 total, 8 up, 8 in 2024-08-17T17:43:04.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:03 smithi130 ceph-mon[105521]: pgmap v1301: 105 pgs: 105 active+clean; 43 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 4.2 KiB/s wr, 63 op/s 2024-08-17T17:43:04.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:03 smithi130 ceph-mon[105521]: osdmap e1049: 8 total, 8 up, 8 in 2024-08-17T17:43:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:03 smithi130 ceph-mon[110068]: pgmap v1301: 105 pgs: 105 active+clean; 43 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 4.2 KiB/s wr, 63 op/s 2024-08-17T17:43:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:03 smithi130 ceph-mon[110068]: osdmap e1049: 8 total, 8 up, 8 in 2024-08-17T17:43:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:05 smithi179 ceph-mon[93519]: pgmap v1303: 105 pgs: 105 active+clean; 54 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 7.1 KiB/s wr, 86 op/s 2024-08-17T17:43:06.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:05 smithi130 ceph-mon[105521]: pgmap v1303: 105 pgs: 105 active+clean; 54 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 7.1 KiB/s wr, 86 op/s 2024-08-17T17:43:06.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:05 smithi130 ceph-mon[110068]: pgmap v1303: 105 pgs: 105 active+clean; 54 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 7.1 KiB/s wr, 86 op/s 2024-08-17T17:43:08.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:07 smithi179 ceph-mon[93519]: pgmap v1304: 105 pgs: 105 active+clean; 54 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 5.2 KiB/s wr, 64 op/s 2024-08-17T17:43:08.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:07 smithi179 ceph-mon[93519]: osdmap e1050: 8 total, 8 up, 8 in 2024-08-17T17:43:08.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:07 smithi130 ceph-mon[105521]: pgmap v1304: 105 pgs: 105 active+clean; 54 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 5.2 KiB/s wr, 64 op/s 2024-08-17T17:43:08.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:07 smithi130 ceph-mon[105521]: osdmap e1050: 8 total, 8 up, 8 in 2024-08-17T17:43:08.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:07 smithi130 ceph-mon[110068]: pgmap v1304: 105 pgs: 105 active+clean; 54 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 45 KiB/s rd, 5.2 KiB/s wr, 64 op/s 2024-08-17T17:43:08.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:07 smithi130 ceph-mon[110068]: osdmap e1050: 8 total, 8 up, 8 in 2024-08-17T17:43:08.724 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:43:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:08 smithi179 ceph-mon[93519]: osdmap e1051: 8 total, 8 up, 8 in 2024-08-17T17:43:09.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:08 smithi130 ceph-mon[105521]: osdmap e1051: 8 total, 8 up, 8 in 2024-08-17T17:43:09.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:08 smithi130 ceph-mon[110068]: osdmap e1051: 8 total, 8 up, 8 in 2024-08-17T17:43:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:09 smithi179 ceph-mon[93519]: pgmap v1307: 105 pgs: 105 active+clean; 58 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 690 KiB/s wr, 85 op/s 2024-08-17T17:43:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:09 smithi179 ceph-mon[93519]: osdmap e1052: 8 total, 8 up, 8 in 2024-08-17T17:43:10.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:09 smithi130 ceph-mon[105521]: pgmap v1307: 105 pgs: 105 active+clean; 58 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 690 KiB/s wr, 85 op/s 2024-08-17T17:43:10.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:09 smithi130 ceph-mon[105521]: osdmap e1052: 8 total, 8 up, 8 in 2024-08-17T17:43:10.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:09 smithi130 ceph-mon[110068]: pgmap v1307: 105 pgs: 105 active+clean; 58 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 690 KiB/s wr, 85 op/s 2024-08-17T17:43:10.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:09 smithi130 ceph-mon[110068]: osdmap e1052: 8 total, 8 up, 8 in 2024-08-17T17:43:10.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:43:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:43:09] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.51.0" 2024-08-17T17:43:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:43:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:43:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:43:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:11 smithi179 ceph-mon[93519]: pgmap v1309: 105 pgs: 105 active+clean; 58 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 686 KiB/s wr, 26 op/s 2024-08-17T17:43:12.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:11 smithi130 ceph-mon[105521]: pgmap v1309: 105 pgs: 105 active+clean; 58 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 686 KiB/s wr, 26 op/s 2024-08-17T17:43:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:11 smithi130 ceph-mon[110068]: pgmap v1309: 105 pgs: 105 active+clean; 58 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 686 KiB/s wr, 26 op/s 2024-08-17T17:43:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:13.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:13.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:13 smithi179 ceph-mon[93519]: pgmap v1310: 105 pgs: 105 active+clean; 58 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 686 KiB/s wr, 26 op/s 2024-08-17T17:43:13.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:13 smithi179 ceph-mon[93519]: osdmap e1053: 8 total, 8 up, 8 in 2024-08-17T17:43:14.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:13 smithi130 ceph-mon[110068]: pgmap v1310: 105 pgs: 105 active+clean; 58 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 686 KiB/s wr, 26 op/s 2024-08-17T17:43:14.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:13 smithi130 ceph-mon[110068]: osdmap e1053: 8 total, 8 up, 8 in 2024-08-17T17:43:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:13 smithi130 ceph-mon[105521]: pgmap v1310: 105 pgs: 105 active+clean; 58 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 686 KiB/s wr, 26 op/s 2024-08-17T17:43:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:13 smithi130 ceph-mon[105521]: osdmap e1053: 8 total, 8 up, 8 in 2024-08-17T17:43:14.744 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:43:15.017 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:14 smithi179 ceph-mon[93519]: osdmap e1054: 8 total, 8 up, 8 in 2024-08-17T17:43:15.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:14 smithi130 ceph-mon[105521]: osdmap e1054: 8 total, 8 up, 8 in 2024-08-17T17:43:15.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:14 smithi130 ceph-mon[110068]: osdmap e1054: 8 total, 8 up, 8 in 2024-08-17T17:43:16.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:15 smithi179 ceph-mon[93519]: pgmap v1313: 105 pgs: 105 active+clean; 62 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 685 KiB/s wr, 28 op/s 2024-08-17T17:43:16.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:15 smithi179 ceph-mon[93519]: osdmap e1055: 8 total, 8 up, 8 in 2024-08-17T17:43:16.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:15 smithi130 ceph-mon[105521]: pgmap v1313: 105 pgs: 105 active+clean; 62 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 685 KiB/s wr, 28 op/s 2024-08-17T17:43:16.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:15 smithi130 ceph-mon[105521]: osdmap e1055: 8 total, 8 up, 8 in 2024-08-17T17:43:16.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:15 smithi130 ceph-mon[110068]: pgmap v1313: 105 pgs: 105 active+clean; 62 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 685 KiB/s wr, 28 op/s 2024-08-17T17:43:16.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:15 smithi130 ceph-mon[110068]: osdmap e1055: 8 total, 8 up, 8 in 2024-08-17T17:43:18.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:17 smithi179 ceph-mon[93519]: pgmap v1315: 105 pgs: 105 active+clean; 62 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 35 op/s 2024-08-17T17:43:18.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:17 smithi130 ceph-mon[105521]: pgmap v1315: 105 pgs: 105 active+clean; 62 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 35 op/s 2024-08-17T17:43:18.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:17 smithi130 ceph-mon[110068]: pgmap v1315: 105 pgs: 105 active+clean; 62 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 35 op/s 2024-08-17T17:43:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:19 smithi179 ceph-mon[93519]: pgmap v1316: 105 pgs: 105 active+clean; 64 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 687 KiB/s wr, 59 op/s 2024-08-17T17:43:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:19 smithi179 ceph-mon[93519]: osdmap e1056: 8 total, 8 up, 8 in 2024-08-17T17:43:20.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:19 smithi130 ceph-mon[105521]: pgmap v1316: 105 pgs: 105 active+clean; 64 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 687 KiB/s wr, 59 op/s 2024-08-17T17:43:20.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:19 smithi130 ceph-mon[105521]: osdmap e1056: 8 total, 8 up, 8 in 2024-08-17T17:43:20.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:43:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:43:19] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.51.0" 2024-08-17T17:43:20.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:19 smithi130 ceph-mon[110068]: pgmap v1316: 105 pgs: 105 active+clean; 64 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 687 KiB/s wr, 59 op/s 2024-08-17T17:43:20.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:19 smithi130 ceph-mon[110068]: osdmap e1056: 8 total, 8 up, 8 in 2024-08-17T17:43:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:43:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:43:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:43:20.776 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:43:21.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:20 smithi179 ceph-mon[93519]: osdmap e1057: 8 total, 8 up, 8 in 2024-08-17T17:43:21.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:20 smithi130 ceph-mon[105521]: osdmap e1057: 8 total, 8 up, 8 in 2024-08-17T17:43:21.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:20 smithi130 ceph-mon[110068]: osdmap e1057: 8 total, 8 up, 8 in 2024-08-17T17:43:22.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:21 smithi179 ceph-mon[93519]: pgmap v1319: 105 pgs: 105 active+clean; 64 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.8 KiB/s wr, 37 op/s 2024-08-17T17:43:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:21 smithi179 ceph-mon[93519]: osdmap e1058: 8 total, 8 up, 8 in 2024-08-17T17:43:22.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:21 smithi130 ceph-mon[105521]: pgmap v1319: 105 pgs: 105 active+clean; 64 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.8 KiB/s wr, 37 op/s 2024-08-17T17:43:22.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:21 smithi130 ceph-mon[105521]: osdmap e1058: 8 total, 8 up, 8 in 2024-08-17T17:43:22.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:21 smithi130 ceph-mon[110068]: pgmap v1319: 105 pgs: 105 active+clean; 64 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.8 KiB/s wr, 37 op/s 2024-08-17T17:43:22.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:21 smithi130 ceph-mon[110068]: osdmap e1058: 8 total, 8 up, 8 in 2024-08-17T17:43:24.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:23 smithi179 ceph-mon[93519]: pgmap v1321: 105 pgs: 105 active+clean; 64 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.2 KiB/s wr, 28 op/s 2024-08-17T17:43:24.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:23 smithi130 ceph-mon[105521]: pgmap v1321: 105 pgs: 105 active+clean; 64 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.2 KiB/s wr, 28 op/s 2024-08-17T17:43:24.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:23 smithi130 ceph-mon[110068]: pgmap v1321: 105 pgs: 105 active+clean; 64 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.2 KiB/s wr, 28 op/s 2024-08-17T17:43:26.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:25 smithi179 ceph-mon[93519]: pgmap v1322: 105 pgs: 105 active+clean; 70 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 686 KiB/s wr, 42 op/s 2024-08-17T17:43:26.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:25 smithi179 ceph-mon[93519]: osdmap e1059: 8 total, 8 up, 8 in 2024-08-17T17:43:26.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:25 smithi130 ceph-mon[110068]: pgmap v1322: 105 pgs: 105 active+clean; 70 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 686 KiB/s wr, 42 op/s 2024-08-17T17:43:26.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:25 smithi130 ceph-mon[110068]: osdmap e1059: 8 total, 8 up, 8 in 2024-08-17T17:43:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:25 smithi130 ceph-mon[105521]: pgmap v1322: 105 pgs: 105 active+clean; 70 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 686 KiB/s wr, 42 op/s 2024-08-17T17:43:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:25 smithi130 ceph-mon[105521]: osdmap e1059: 8 total, 8 up, 8 in 2024-08-17T17:43:26.797 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:43:27.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:26 smithi179 ceph-mon[93519]: osdmap e1060: 8 total, 8 up, 8 in 2024-08-17T17:43:27.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:26 smithi130 ceph-mon[105521]: osdmap e1060: 8 total, 8 up, 8 in 2024-08-17T17:43:27.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:26 smithi130 ceph-mon[110068]: osdmap e1060: 8 total, 8 up, 8 in 2024-08-17T17:43:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:27 smithi179 ceph-mon[93519]: pgmap v1325: 105 pgs: 105 active+clean; 70 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 686 KiB/s wr, 39 op/s 2024-08-17T17:43:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:27 smithi179 ceph-mon[93519]: osdmap e1061: 8 total, 8 up, 8 in 2024-08-17T17:43:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:28.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:27 smithi130 ceph-mon[110068]: pgmap v1325: 105 pgs: 105 active+clean; 70 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 686 KiB/s wr, 39 op/s 2024-08-17T17:43:28.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:27 smithi130 ceph-mon[110068]: osdmap e1061: 8 total, 8 up, 8 in 2024-08-17T17:43:28.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:28.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:27 smithi130 ceph-mon[105521]: pgmap v1325: 105 pgs: 105 active+clean; 70 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 686 KiB/s wr, 39 op/s 2024-08-17T17:43:28.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:27 smithi130 ceph-mon[105521]: osdmap e1061: 8 total, 8 up, 8 in 2024-08-17T17:43:28.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:29 smithi179 ceph-mon[93519]: pgmap v1327: 105 pgs: 105 active+clean; 75 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 1.3 MiB/s wr, 71 op/s 2024-08-17T17:43:30.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:29 smithi130 ceph-mon[110068]: pgmap v1327: 105 pgs: 105 active+clean; 75 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 1.3 MiB/s wr, 71 op/s 2024-08-17T17:43:30.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:29 smithi130 ceph-mon[105521]: pgmap v1327: 105 pgs: 105 active+clean; 75 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 49 KiB/s rd, 1.3 MiB/s wr, 71 op/s 2024-08-17T17:43:30.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:43:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:43:29] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.51.0" 2024-08-17T17:43:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:43:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:43:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:43:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:31 smithi179 ceph-mon[93519]: pgmap v1328: 105 pgs: 105 active+clean; 76 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-08-17T17:43:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:31 smithi179 ceph-mon[93519]: osdmap e1062: 8 total, 8 up, 8 in 2024-08-17T17:43:32.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:31 smithi130 ceph-mon[105521]: pgmap v1328: 105 pgs: 105 active+clean; 76 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-08-17T17:43:32.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:31 smithi130 ceph-mon[105521]: osdmap e1062: 8 total, 8 up, 8 in 2024-08-17T17:43:32.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:31 smithi130 ceph-mon[110068]: pgmap v1328: 105 pgs: 105 active+clean; 76 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-08-17T17:43:32.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:31 smithi130 ceph-mon[110068]: osdmap e1062: 8 total, 8 up, 8 in 2024-08-17T17:43:32.817 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministicLUKS1 (30114 ms) 2024-08-17T17:43:32.817 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministicLUKS1PP 2024-08-17T17:43:32.817 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:43:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:32 smithi179 ceph-mon[93519]: osdmap e1063: 8 total, 8 up, 8 in 2024-08-17T17:43:33.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:32 smithi130 ceph-mon[105521]: osdmap e1063: 8 total, 8 up, 8 in 2024-08-17T17:43:33.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:32 smithi130 ceph-mon[110068]: osdmap e1063: 8 total, 8 up, 8 in 2024-08-17T17:43:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:33 smithi179 ceph-mon[93519]: pgmap v1331: 105 pgs: 105 active+clean; 76 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 687 KiB/s wr, 45 op/s 2024-08-17T17:43:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:33 smithi179 ceph-mon[93519]: osdmap e1064: 8 total, 8 up, 8 in 2024-08-17T17:43:34.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:33 smithi130 ceph-mon[105521]: pgmap v1331: 105 pgs: 105 active+clean; 76 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 687 KiB/s wr, 45 op/s 2024-08-17T17:43:34.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:33 smithi130 ceph-mon[105521]: osdmap e1064: 8 total, 8 up, 8 in 2024-08-17T17:43:34.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:33 smithi130 ceph-mon[110068]: pgmap v1331: 105 pgs: 105 active+clean; 76 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 687 KiB/s wr, 45 op/s 2024-08-17T17:43:34.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:33 smithi130 ceph-mon[110068]: osdmap e1064: 8 total, 8 up, 8 in 2024-08-17T17:43:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:35 smithi179 ceph-mon[93519]: pgmap v1333: 105 pgs: 105 active+clean; 84 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 686 KiB/s wr, 45 op/s 2024-08-17T17:43:36.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:35 smithi130 ceph-mon[105521]: pgmap v1333: 105 pgs: 105 active+clean; 84 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 686 KiB/s wr, 45 op/s 2024-08-17T17:43:36.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:35 smithi130 ceph-mon[110068]: pgmap v1333: 105 pgs: 105 active+clean; 84 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 686 KiB/s wr, 45 op/s 2024-08-17T17:43:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:37 smithi179 ceph-mon[93519]: pgmap v1334: 105 pgs: 105 active+clean; 88 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 686 KiB/s wr, 39 op/s 2024-08-17T17:43:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:37 smithi179 ceph-mon[93519]: osdmap e1065: 8 total, 8 up, 8 in 2024-08-17T17:43:38.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:37 smithi130 ceph-mon[105521]: pgmap v1334: 105 pgs: 105 active+clean; 88 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 686 KiB/s wr, 39 op/s 2024-08-17T17:43:38.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:37 smithi130 ceph-mon[105521]: osdmap e1065: 8 total, 8 up, 8 in 2024-08-17T17:43:38.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:37 smithi130 ceph-mon[110068]: pgmap v1334: 105 pgs: 105 active+clean; 88 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 686 KiB/s wr, 39 op/s 2024-08-17T17:43:38.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:37 smithi130 ceph-mon[110068]: osdmap e1065: 8 total, 8 up, 8 in 2024-08-17T17:43:38.836 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:43:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:38 smithi179 ceph-mon[93519]: osdmap e1066: 8 total, 8 up, 8 in 2024-08-17T17:43:39.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:38 smithi130 ceph-mon[105521]: osdmap e1066: 8 total, 8 up, 8 in 2024-08-17T17:43:39.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:38 smithi130 ceph-mon[110068]: osdmap e1066: 8 total, 8 up, 8 in 2024-08-17T17:43:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:39 smithi179 ceph-mon[93519]: pgmap v1337: 105 pgs: 105 active+clean; 92 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 1.3 MiB/s wr, 50 op/s 2024-08-17T17:43:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:39 smithi179 ceph-mon[93519]: osdmap e1067: 8 total, 8 up, 8 in 2024-08-17T17:43:40.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:39 smithi130 ceph-mon[110068]: pgmap v1337: 105 pgs: 105 active+clean; 92 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 1.3 MiB/s wr, 50 op/s 2024-08-17T17:43:40.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:39 smithi130 ceph-mon[110068]: osdmap e1067: 8 total, 8 up, 8 in 2024-08-17T17:43:40.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:39 smithi130 ceph-mon[105521]: pgmap v1337: 105 pgs: 105 active+clean; 92 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 1.3 MiB/s wr, 50 op/s 2024-08-17T17:43:40.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:39 smithi130 ceph-mon[105521]: osdmap e1067: 8 total, 8 up, 8 in 2024-08-17T17:43:40.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:43:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:43:39] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.51.0" 2024-08-17T17:43:40.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:43:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:43:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:43:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:41 smithi179 ceph-mon[93519]: pgmap v1339: 105 pgs: 105 active+clean; 92 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 686 KiB/s wr, 26 op/s 2024-08-17T17:43:42.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:41 smithi130 ceph-mon[105521]: pgmap v1339: 105 pgs: 105 active+clean; 92 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 686 KiB/s wr, 26 op/s 2024-08-17T17:43:42.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:41 smithi130 ceph-mon[110068]: pgmap v1339: 105 pgs: 105 active+clean; 92 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 686 KiB/s wr, 26 op/s 2024-08-17T17:43:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:43.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:43 smithi179 ceph-mon[93519]: pgmap v1340: 105 pgs: 105 active+clean; 92 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 686 KiB/s wr, 23 op/s 2024-08-17T17:43:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:43 smithi179 ceph-mon[93519]: osdmap e1068: 8 total, 8 up, 8 in 2024-08-17T17:43:44.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:43 smithi130 ceph-mon[105521]: pgmap v1340: 105 pgs: 105 active+clean; 92 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 686 KiB/s wr, 23 op/s 2024-08-17T17:43:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:43 smithi130 ceph-mon[105521]: osdmap e1068: 8 total, 8 up, 8 in 2024-08-17T17:43:44.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:43 smithi130 ceph-mon[110068]: pgmap v1340: 105 pgs: 105 active+clean; 92 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 686 KiB/s wr, 23 op/s 2024-08-17T17:43:44.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:43 smithi130 ceph-mon[110068]: osdmap e1068: 8 total, 8 up, 8 in 2024-08-17T17:43:44.871 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:43:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:44 smithi179 ceph-mon[93519]: osdmap e1069: 8 total, 8 up, 8 in 2024-08-17T17:43:45.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:44 smithi130 ceph-mon[105521]: osdmap e1069: 8 total, 8 up, 8 in 2024-08-17T17:43:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:44 smithi130 ceph-mon[110068]: osdmap e1069: 8 total, 8 up, 8 in 2024-08-17T17:43:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:45 smithi179 ceph-mon[93519]: pgmap v1343: 105 pgs: 105 active+clean; 95 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 4.0 KiB/s wr, 38 op/s 2024-08-17T17:43:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:45 smithi179 ceph-mon[93519]: osdmap e1070: 8 total, 8 up, 8 in 2024-08-17T17:43:46.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:45 smithi130 ceph-mon[105521]: pgmap v1343: 105 pgs: 105 active+clean; 95 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 4.0 KiB/s wr, 38 op/s 2024-08-17T17:43:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:45 smithi130 ceph-mon[105521]: osdmap e1070: 8 total, 8 up, 8 in 2024-08-17T17:43:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:45 smithi130 ceph-mon[110068]: pgmap v1343: 105 pgs: 105 active+clean; 95 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 4.0 KiB/s wr, 38 op/s 2024-08-17T17:43:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:45 smithi130 ceph-mon[110068]: osdmap e1070: 8 total, 8 up, 8 in 2024-08-17T17:43:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:47 smithi179 ceph-mon[93519]: pgmap v1345: 105 pgs: 105 active+clean; 99 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 685 KiB/s wr, 34 op/s 2024-08-17T17:43:48.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:47 smithi130 ceph-mon[105521]: pgmap v1345: 105 pgs: 105 active+clean; 99 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 685 KiB/s wr, 34 op/s 2024-08-17T17:43:48.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:47 smithi130 ceph-mon[110068]: pgmap v1345: 105 pgs: 105 active+clean; 99 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 685 KiB/s wr, 34 op/s 2024-08-17T17:43:50.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:49 smithi130 ceph-mon[105521]: pgmap v1346: 105 pgs: 105 active+clean; 102 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 688 KiB/s wr, 61 op/s 2024-08-17T17:43:50.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:49 smithi130 ceph-mon[105521]: osdmap e1071: 8 total, 8 up, 8 in 2024-08-17T17:43:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:43:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:43:49] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-08-17T17:43:50.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:49 smithi130 ceph-mon[110068]: pgmap v1346: 105 pgs: 105 active+clean; 102 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 688 KiB/s wr, 61 op/s 2024-08-17T17:43:50.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:49 smithi130 ceph-mon[110068]: osdmap e1071: 8 total, 8 up, 8 in 2024-08-17T17:43:50.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:49 smithi179 ceph-mon[93519]: pgmap v1346: 105 pgs: 105 active+clean; 102 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 688 KiB/s wr, 61 op/s 2024-08-17T17:43:50.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:49 smithi179 ceph-mon[93519]: osdmap e1071: 8 total, 8 up, 8 in 2024-08-17T17:43:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:43:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:43:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:43:50.897 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:43:51.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:50 smithi130 ceph-mon[105521]: osdmap e1072: 8 total, 8 up, 8 in 2024-08-17T17:43:51.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:50 smithi130 ceph-mon[110068]: osdmap e1072: 8 total, 8 up, 8 in 2024-08-17T17:43:51.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:50 smithi179 ceph-mon[93519]: osdmap e1072: 8 total, 8 up, 8 in 2024-08-17T17:43:52.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:51 smithi130 ceph-mon[105521]: pgmap v1349: 105 pgs: 105 active+clean; 106 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.3 MiB/s wr, 31 op/s 2024-08-17T17:43:52.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:51 smithi130 ceph-mon[105521]: osdmap e1073: 8 total, 8 up, 8 in 2024-08-17T17:43:52.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:51 smithi130 ceph-mon[110068]: pgmap v1349: 105 pgs: 105 active+clean; 106 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.3 MiB/s wr, 31 op/s 2024-08-17T17:43:52.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:51 smithi130 ceph-mon[110068]: osdmap e1073: 8 total, 8 up, 8 in 2024-08-17T17:43:52.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:51 smithi179 ceph-mon[93519]: pgmap v1349: 105 pgs: 105 active+clean; 106 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.3 MiB/s wr, 31 op/s 2024-08-17T17:43:52.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:51 smithi179 ceph-mon[93519]: osdmap e1073: 8 total, 8 up, 8 in 2024-08-17T17:43:54.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:53 smithi130 ceph-mon[105521]: pgmap v1351: 105 pgs: 105 active+clean; 106 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 685 KiB/s wr, 30 op/s 2024-08-17T17:43:54.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:53 smithi130 ceph-mon[110068]: pgmap v1351: 105 pgs: 105 active+clean; 106 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 685 KiB/s wr, 30 op/s 2024-08-17T17:43:54.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:53 smithi179 ceph-mon[93519]: pgmap v1351: 105 pgs: 105 active+clean; 106 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 685 KiB/s wr, 30 op/s 2024-08-17T17:43:56.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:55 smithi130 ceph-mon[105521]: pgmap v1352: 105 pgs: 105 active+clean; 112 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-08-17T17:43:56.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:55 smithi130 ceph-mon[105521]: osdmap e1074: 8 total, 8 up, 8 in 2024-08-17T17:43:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:55 smithi130 ceph-mon[110068]: pgmap v1352: 105 pgs: 105 active+clean; 112 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-08-17T17:43:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:55 smithi130 ceph-mon[110068]: osdmap e1074: 8 total, 8 up, 8 in 2024-08-17T17:43:56.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:55 smithi179 ceph-mon[93519]: pgmap v1352: 105 pgs: 105 active+clean; 112 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-08-17T17:43:56.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:55 smithi179 ceph-mon[93519]: osdmap e1074: 8 total, 8 up, 8 in 2024-08-17T17:43:56.920 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:43:57.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:56 smithi130 ceph-mon[105521]: osdmap e1075: 8 total, 8 up, 8 in 2024-08-17T17:43:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:56 smithi130 ceph-mon[110068]: osdmap e1075: 8 total, 8 up, 8 in 2024-08-17T17:43:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:56 smithi179 ceph-mon[93519]: osdmap e1075: 8 total, 8 up, 8 in 2024-08-17T17:43:58.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:57 smithi130 ceph-mon[105521]: pgmap v1355: 105 pgs: 105 active+clean; 115 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.5 KiB/s wr, 47 op/s 2024-08-17T17:43:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:57 smithi130 ceph-mon[105521]: osdmap e1076: 8 total, 8 up, 8 in 2024-08-17T17:43:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:57 smithi130 ceph-mon[110068]: pgmap v1355: 105 pgs: 105 active+clean; 115 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.5 KiB/s wr, 47 op/s 2024-08-17T17:43:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:57 smithi130 ceph-mon[110068]: osdmap e1076: 8 total, 8 up, 8 in 2024-08-17T17:43:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:57 smithi179 ceph-mon[93519]: pgmap v1355: 105 pgs: 105 active+clean; 115 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.5 KiB/s wr, 47 op/s 2024-08-17T17:43:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:57 smithi179 ceph-mon[93519]: osdmap e1076: 8 total, 8 up, 8 in 2024-08-17T17:43:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:59.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:43:59.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:43:59.206 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:44:00.083 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:43:59 smithi130 ceph-mon[105521]: pgmap v1357: 105 pgs: 105 active+clean; 122 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 689 KiB/s wr, 67 op/s 2024-08-17T17:44:00.083 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:43:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:43:59] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.51.0" 2024-08-17T17:44:00.084 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:43:59 smithi130 ceph-mon[110068]: pgmap v1357: 105 pgs: 105 active+clean; 122 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 689 KiB/s wr, 67 op/s 2024-08-17T17:44:00.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:43:59 smithi179 ceph-mon[93519]: pgmap v1357: 105 pgs: 105 active+clean; 122 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 689 KiB/s wr, 67 op/s 2024-08-17T17:44:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:44:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:44:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:44:02.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:01 smithi130 ceph-mon[105521]: pgmap v1358: 105 pgs: 105 active+clean; 122 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 687 KiB/s wr, 47 op/s 2024-08-17T17:44:02.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:01 smithi130 ceph-mon[105521]: osdmap e1077: 8 total, 8 up, 8 in 2024-08-17T17:44:02.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:01 smithi130 ceph-mon[110068]: pgmap v1358: 105 pgs: 105 active+clean; 122 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 687 KiB/s wr, 47 op/s 2024-08-17T17:44:02.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:01 smithi130 ceph-mon[110068]: osdmap e1077: 8 total, 8 up, 8 in 2024-08-17T17:44:02.240 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:01 smithi179 ceph-mon[93519]: pgmap v1358: 105 pgs: 105 active+clean; 122 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 687 KiB/s wr, 47 op/s 2024-08-17T17:44:02.240 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:01 smithi179 ceph-mon[93519]: osdmap e1077: 8 total, 8 up, 8 in 2024-08-17T17:44:02.937 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministicLUKS1PP (30120 ms) 2024-08-17T17:44:02.937 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministicLUKS2 2024-08-17T17:44:02.937 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:44:03.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:02 smithi130 ceph-mon[105521]: osdmap e1078: 8 total, 8 up, 8 in 2024-08-17T17:44:03.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:02 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:02 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:02 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:02 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:02 smithi130 ceph-mon[110068]: osdmap e1078: 8 total, 8 up, 8 in 2024-08-17T17:44:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:02 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:02 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:02 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:02 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:02 smithi179 ceph-mon[93519]: osdmap e1078: 8 total, 8 up, 8 in 2024-08-17T17:44:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:02 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:02 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:02 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:02 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:04.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:03 smithi130 ceph-mon[105521]: pgmap v1361: 105 pgs: 105 active+clean; 122 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 35 op/s 2024-08-17T17:44:04.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:03 smithi130 ceph-mon[105521]: osdmap e1079: 8 total, 8 up, 8 in 2024-08-17T17:44:04.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:03 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:44:04.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:03 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:44:04.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:03 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:03 smithi130 ceph-mon[110068]: pgmap v1361: 105 pgs: 105 active+clean; 122 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 35 op/s 2024-08-17T17:44:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:03 smithi130 ceph-mon[110068]: osdmap e1079: 8 total, 8 up, 8 in 2024-08-17T17:44:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:03 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:44:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:03 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:44:04.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:03 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:03 smithi179 ceph-mon[93519]: pgmap v1361: 105 pgs: 105 active+clean; 122 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 35 op/s 2024-08-17T17:44:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:03 smithi179 ceph-mon[93519]: osdmap e1079: 8 total, 8 up, 8 in 2024-08-17T17:44:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:03 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:44:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:03 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:44:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:03 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:44:06.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:05 smithi179 ceph-mon[93519]: pgmap v1363: 105 pgs: 105 active+clean; 134 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 687 KiB/s wr, 57 op/s 2024-08-17T17:44:06.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:05 smithi130 ceph-mon[105521]: pgmap v1363: 105 pgs: 105 active+clean; 134 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 687 KiB/s wr, 57 op/s 2024-08-17T17:44:06.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:05 smithi130 ceph-mon[110068]: pgmap v1363: 105 pgs: 105 active+clean; 134 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 687 KiB/s wr, 57 op/s 2024-08-17T17:44:08.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:07 smithi179 ceph-mon[93519]: pgmap v1364: 105 pgs: 105 active+clean; 134 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 686 KiB/s wr, 41 op/s 2024-08-17T17:44:08.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:07 smithi130 ceph-mon[105521]: pgmap v1364: 105 pgs: 105 active+clean; 134 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 686 KiB/s wr, 41 op/s 2024-08-17T17:44:08.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:07 smithi130 ceph-mon[110068]: pgmap v1364: 105 pgs: 105 active+clean; 134 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 686 KiB/s wr, 41 op/s 2024-08-17T17:44:10.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:09 smithi130 ceph-mon[110068]: pgmap v1365: 105 pgs: 105 active+clean; 134 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 555 KiB/s wr, 33 op/s 2024-08-17T17:44:10.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:09 smithi130 ceph-mon[110068]: osdmap e1080: 8 total, 8 up, 8 in 2024-08-17T17:44:10.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:09 smithi130 ceph-mon[105521]: pgmap v1365: 105 pgs: 105 active+clean; 134 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 555 KiB/s wr, 33 op/s 2024-08-17T17:44:10.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:09 smithi130 ceph-mon[105521]: osdmap e1080: 8 total, 8 up, 8 in 2024-08-17T17:44:10.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:44:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:44:09] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.51.0" 2024-08-17T17:44:10.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:09 smithi179 ceph-mon[93519]: pgmap v1365: 105 pgs: 105 active+clean; 134 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 555 KiB/s wr, 33 op/s 2024-08-17T17:44:10.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:09 smithi179 ceph-mon[93519]: osdmap e1080: 8 total, 8 up, 8 in 2024-08-17T17:44:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:44:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:44:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:44:10.974 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:44:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:10 smithi179 ceph-mon[93519]: osdmap e1081: 8 total, 8 up, 8 in 2024-08-17T17:44:11.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:10 smithi130 ceph-mon[105521]: osdmap e1081: 8 total, 8 up, 8 in 2024-08-17T17:44:11.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:10 smithi130 ceph-mon[110068]: osdmap e1081: 8 total, 8 up, 8 in 2024-08-17T17:44:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:11 smithi179 ceph-mon[93519]: pgmap v1368: 105 pgs: 105 active+clean; 138 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 1.1 MiB/s wr, 38 op/s 2024-08-17T17:44:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:11 smithi179 ceph-mon[93519]: osdmap e1082: 8 total, 8 up, 8 in 2024-08-17T17:44:12.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:11 smithi130 ceph-mon[105521]: pgmap v1368: 105 pgs: 105 active+clean; 138 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 1.1 MiB/s wr, 38 op/s 2024-08-17T17:44:12.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:11 smithi130 ceph-mon[105521]: osdmap e1082: 8 total, 8 up, 8 in 2024-08-17T17:44:12.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:11 smithi130 ceph-mon[110068]: pgmap v1368: 105 pgs: 105 active+clean; 138 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 1.1 MiB/s wr, 38 op/s 2024-08-17T17:44:12.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:11 smithi130 ceph-mon[110068]: osdmap e1082: 8 total, 8 up, 8 in 2024-08-17T17:44:13.244 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:13.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:14.256 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:13 smithi179 ceph-mon[93519]: pgmap v1370: 105 pgs: 105 active+clean; 142 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-08-17T17:44:14.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:13 smithi130 ceph-mon[105521]: pgmap v1370: 105 pgs: 105 active+clean; 142 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-08-17T17:44:14.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:13 smithi130 ceph-mon[110068]: pgmap v1370: 105 pgs: 105 active+clean; 142 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-08-17T17:44:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:15 smithi179 ceph-mon[93519]: pgmap v1371: 105 pgs: 105 active+clean; 150 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:44:16.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:15 smithi130 ceph-mon[105521]: pgmap v1371: 105 pgs: 105 active+clean; 150 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:44:16.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:15 smithi130 ceph-mon[110068]: pgmap v1371: 105 pgs: 105 active+clean; 150 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:44:18.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:17 smithi179 ceph-mon[93519]: pgmap v1372: 105 pgs: 105 active+clean; 150 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.2 MiB/s wr, 32 op/s 2024-08-17T17:44:18.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:17 smithi179 ceph-mon[93519]: osdmap e1083: 8 total, 8 up, 8 in 2024-08-17T17:44:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:17 smithi130 ceph-mon[105521]: pgmap v1372: 105 pgs: 105 active+clean; 150 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.2 MiB/s wr, 32 op/s 2024-08-17T17:44:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:17 smithi130 ceph-mon[105521]: osdmap e1083: 8 total, 8 up, 8 in 2024-08-17T17:44:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:17 smithi130 ceph-mon[110068]: pgmap v1372: 105 pgs: 105 active+clean; 150 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.2 MiB/s wr, 32 op/s 2024-08-17T17:44:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:17 smithi130 ceph-mon[110068]: osdmap e1083: 8 total, 8 up, 8 in 2024-08-17T17:44:19.015 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:44:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:18 smithi179 ceph-mon[93519]: osdmap e1084: 8 total, 8 up, 8 in 2024-08-17T17:44:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:18 smithi179 ceph-mon[93519]: pgmap v1375: 105 pgs: 105 active+clean; 159 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.7 MiB/s wr, 33 op/s 2024-08-17T17:44:19.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:18 smithi130 ceph-mon[105521]: osdmap e1084: 8 total, 8 up, 8 in 2024-08-17T17:44:19.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:18 smithi130 ceph-mon[105521]: pgmap v1375: 105 pgs: 105 active+clean; 159 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.7 MiB/s wr, 33 op/s 2024-08-17T17:44:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:18 smithi130 ceph-mon[110068]: osdmap e1084: 8 total, 8 up, 8 in 2024-08-17T17:44:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:18 smithi130 ceph-mon[110068]: pgmap v1375: 105 pgs: 105 active+clean; 159 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.7 MiB/s wr, 33 op/s 2024-08-17T17:44:20.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:19 smithi130 ceph-mon[105521]: osdmap e1085: 8 total, 8 up, 8 in 2024-08-17T17:44:20.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:19 smithi130 ceph-mon[110068]: osdmap e1085: 8 total, 8 up, 8 in 2024-08-17T17:44:20.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:44:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:44:19] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.51.0" 2024-08-17T17:44:20.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:19 smithi179 ceph-mon[93519]: osdmap e1085: 8 total, 8 up, 8 in 2024-08-17T17:44:20.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:44:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:44:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:44:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:20 smithi179 ceph-mon[93519]: pgmap v1377: 105 pgs: 105 active+clean; 167 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 2.7 MiB/s wr, 12 op/s 2024-08-17T17:44:21.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:20 smithi130 ceph-mon[105521]: pgmap v1377: 105 pgs: 105 active+clean; 167 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 2.7 MiB/s wr, 12 op/s 2024-08-17T17:44:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:20 smithi130 ceph-mon[110068]: pgmap v1377: 105 pgs: 105 active+clean; 167 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 2.7 MiB/s wr, 12 op/s 2024-08-17T17:44:23.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:23 smithi130 ceph-mon[105521]: pgmap v1378: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.7 MiB/s wr, 31 op/s 2024-08-17T17:44:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:23 smithi130 ceph-mon[110068]: pgmap v1378: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.7 MiB/s wr, 31 op/s 2024-08-17T17:44:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:23 smithi179 ceph-mon[93519]: pgmap v1378: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.7 MiB/s wr, 31 op/s 2024-08-17T17:44:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:25 smithi130 ceph-mon[105521]: pgmap v1379: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 2.2 MiB/s wr, 37 op/s 2024-08-17T17:44:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:25 smithi130 ceph-mon[105521]: osdmap e1086: 8 total, 8 up, 8 in 2024-08-17T17:44:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:25 smithi130 ceph-mon[110068]: pgmap v1379: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 2.2 MiB/s wr, 37 op/s 2024-08-17T17:44:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:25 smithi130 ceph-mon[110068]: osdmap e1086: 8 total, 8 up, 8 in 2024-08-17T17:44:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:25 smithi179 ceph-mon[93519]: pgmap v1379: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 2.2 MiB/s wr, 37 op/s 2024-08-17T17:44:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:25 smithi179 ceph-mon[93519]: osdmap e1086: 8 total, 8 up, 8 in 2024-08-17T17:44:26.434 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:44:26.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:26 smithi130 ceph-mon[105521]: osdmap e1087: 8 total, 8 up, 8 in 2024-08-17T17:44:26.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:26 smithi130 ceph-mon[110068]: osdmap e1087: 8 total, 8 up, 8 in 2024-08-17T17:44:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:26 smithi179 ceph-mon[93519]: osdmap e1087: 8 total, 8 up, 8 in 2024-08-17T17:44:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:27 smithi130 ceph-mon[105521]: pgmap v1382: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 560 KiB/s wr, 28 op/s 2024-08-17T17:44:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:27 smithi130 ceph-mon[105521]: osdmap e1088: 8 total, 8 up, 8 in 2024-08-17T17:44:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:27 smithi130 ceph-mon[110068]: pgmap v1382: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 560 KiB/s wr, 28 op/s 2024-08-17T17:44:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:27 smithi130 ceph-mon[110068]: osdmap e1088: 8 total, 8 up, 8 in 2024-08-17T17:44:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:27 smithi179 ceph-mon[93519]: pgmap v1382: 105 pgs: 105 active+clean; 168 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 560 KiB/s wr, 28 op/s 2024-08-17T17:44:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:27 smithi179 ceph-mon[93519]: osdmap e1088: 8 total, 8 up, 8 in 2024-08-17T17:44:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:29 smithi130 ceph-mon[105521]: pgmap v1384: 105 pgs: 105 active+clean; 185 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 2.7 MiB/s wr, 60 op/s 2024-08-17T17:44:29.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:29 smithi130 ceph-mon[110068]: pgmap v1384: 105 pgs: 105 active+clean; 185 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 2.7 MiB/s wr, 60 op/s 2024-08-17T17:44:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:29 smithi179 ceph-mon[93519]: pgmap v1384: 105 pgs: 105 active+clean; 185 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 2.7 MiB/s wr, 60 op/s 2024-08-17T17:44:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:44:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:44:29] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-17T17:44:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:44:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:44:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:44:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:31 smithi130 ceph-mon[105521]: pgmap v1385: 105 pgs: 105 active+clean; 186 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:44:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:31 smithi130 ceph-mon[110068]: pgmap v1385: 105 pgs: 105 active+clean; 186 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:44:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:31 smithi179 ceph-mon[93519]: pgmap v1385: 105 pgs: 105 active+clean; 186 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:44:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:33 smithi130 ceph-mon[105521]: pgmap v1386: 105 pgs: 105 active+clean; 186 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 34 op/s 2024-08-17T17:44:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:33 smithi130 ceph-mon[105521]: osdmap e1089: 8 total, 8 up, 8 in 2024-08-17T17:44:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:33 smithi130 ceph-mon[110068]: pgmap v1386: 105 pgs: 105 active+clean; 186 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 34 op/s 2024-08-17T17:44:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:33 smithi130 ceph-mon[110068]: osdmap e1089: 8 total, 8 up, 8 in 2024-08-17T17:44:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:33 smithi179 ceph-mon[93519]: pgmap v1386: 105 pgs: 105 active+clean; 186 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 34 op/s 2024-08-17T17:44:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:33 smithi179 ceph-mon[93519]: osdmap e1089: 8 total, 8 up, 8 in 2024-08-17T17:44:34.464 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:44:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:34 smithi179 ceph-mon[93519]: osdmap e1090: 8 total, 8 up, 8 in 2024-08-17T17:44:34.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:34 smithi130 ceph-mon[105521]: osdmap e1090: 8 total, 8 up, 8 in 2024-08-17T17:44:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:34 smithi130 ceph-mon[110068]: osdmap e1090: 8 total, 8 up, 8 in 2024-08-17T17:44:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:35 smithi179 ceph-mon[93519]: pgmap v1389: 105 pgs: 105 active+clean; 199 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.5 MiB/s wr, 40 op/s 2024-08-17T17:44:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:35 smithi179 ceph-mon[93519]: osdmap e1091: 8 total, 8 up, 8 in 2024-08-17T17:44:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:35 smithi130 ceph-mon[105521]: pgmap v1389: 105 pgs: 105 active+clean; 199 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.5 MiB/s wr, 40 op/s 2024-08-17T17:44:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:35 smithi130 ceph-mon[105521]: osdmap e1091: 8 total, 8 up, 8 in 2024-08-17T17:44:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:35 smithi130 ceph-mon[110068]: pgmap v1389: 105 pgs: 105 active+clean; 199 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.5 MiB/s wr, 40 op/s 2024-08-17T17:44:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:35 smithi130 ceph-mon[110068]: osdmap e1091: 8 total, 8 up, 8 in 2024-08-17T17:44:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:37 smithi179 ceph-mon[93519]: pgmap v1391: 105 pgs: 105 active+clean; 203 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-08-17T17:44:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:37 smithi130 ceph-mon[105521]: pgmap v1391: 105 pgs: 105 active+clean; 203 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-08-17T17:44:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:37 smithi130 ceph-mon[110068]: pgmap v1391: 105 pgs: 105 active+clean; 203 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-08-17T17:44:39.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:39 smithi130 ceph-mon[105521]: pgmap v1392: 105 pgs: 105 active+clean; 204 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-08-17T17:44:39.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:39 smithi130 ceph-mon[110068]: pgmap v1392: 105 pgs: 105 active+clean; 204 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-08-17T17:44:39.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:39 smithi179 ceph-mon[93519]: pgmap v1392: 105 pgs: 105 active+clean; 204 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-08-17T17:44:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:44:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:44:39] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-17T17:44:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:44:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:44:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:44:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:41 smithi179 ceph-mon[93519]: pgmap v1393: 105 pgs: 105 active+clean; 204 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 35 op/s 2024-08-17T17:44:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:41 smithi179 ceph-mon[93519]: osdmap e1092: 8 total, 8 up, 8 in 2024-08-17T17:44:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:41 smithi130 ceph-mon[105521]: pgmap v1393: 105 pgs: 105 active+clean; 204 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 35 op/s 2024-08-17T17:44:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:41 smithi130 ceph-mon[105521]: osdmap e1092: 8 total, 8 up, 8 in 2024-08-17T17:44:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:41 smithi130 ceph-mon[110068]: pgmap v1393: 105 pgs: 105 active+clean; 204 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 35 op/s 2024-08-17T17:44:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:41 smithi130 ceph-mon[110068]: osdmap e1092: 8 total, 8 up, 8 in 2024-08-17T17:44:42.496 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministicLUKS2 (39558 ms) 2024-08-17T17:44:42.496 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministicLUKS2PP 2024-08-17T17:44:42.496 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:44:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:42 smithi179 ceph-mon[93519]: osdmap e1093: 8 total, 8 up, 8 in 2024-08-17T17:44:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:42.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:42 smithi130 ceph-mon[110068]: osdmap e1093: 8 total, 8 up, 8 in 2024-08-17T17:44:42.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:42 smithi130 ceph-mon[105521]: osdmap e1093: 8 total, 8 up, 8 in 2024-08-17T17:44:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:43.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:43 smithi179 ceph-mon[93519]: pgmap v1396: 105 pgs: 105 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 1.5 MiB/s wr, 30 op/s 2024-08-17T17:44:43.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:43 smithi179 ceph-mon[93519]: osdmap e1094: 8 total, 8 up, 8 in 2024-08-17T17:44:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:43 smithi130 ceph-mon[105521]: pgmap v1396: 105 pgs: 105 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 1.5 MiB/s wr, 30 op/s 2024-08-17T17:44:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:43 smithi130 ceph-mon[105521]: osdmap e1094: 8 total, 8 up, 8 in 2024-08-17T17:44:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:43 smithi130 ceph-mon[110068]: pgmap v1396: 105 pgs: 105 active+clean; 212 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 1.5 MiB/s wr, 30 op/s 2024-08-17T17:44:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:43 smithi130 ceph-mon[110068]: osdmap e1094: 8 total, 8 up, 8 in 2024-08-17T17:44:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:45 smithi179 ceph-mon[93519]: pgmap v1398: 105 pgs: 105 active+clean; 228 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-08-17T17:44:45.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:45 smithi130 ceph-mon[105521]: pgmap v1398: 105 pgs: 105 active+clean; 228 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-08-17T17:44:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:45 smithi130 ceph-mon[110068]: pgmap v1398: 105 pgs: 105 active+clean; 228 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-08-17T17:44:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:47 smithi179 ceph-mon[93519]: pgmap v1399: 105 pgs: 105 active+clean; 228 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:44:47.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:47 smithi130 ceph-mon[105521]: pgmap v1399: 105 pgs: 105 active+clean; 228 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:44:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:47 smithi130 ceph-mon[110068]: pgmap v1399: 105 pgs: 105 active+clean; 228 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:44:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:49 smithi130 ceph-mon[105521]: pgmap v1400: 105 pgs: 105 active+clean; 240 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.6 MiB/s wr, 40 op/s 2024-08-17T17:44:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:49 smithi130 ceph-mon[105521]: osdmap e1095: 8 total, 8 up, 8 in 2024-08-17T17:44:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:49 smithi130 ceph-mon[110068]: pgmap v1400: 105 pgs: 105 active+clean; 240 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.6 MiB/s wr, 40 op/s 2024-08-17T17:44:49.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:49 smithi130 ceph-mon[110068]: osdmap e1095: 8 total, 8 up, 8 in 2024-08-17T17:44:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:49 smithi179 ceph-mon[93519]: pgmap v1400: 105 pgs: 105 active+clean; 240 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.6 MiB/s wr, 40 op/s 2024-08-17T17:44:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:49 smithi179 ceph-mon[93519]: osdmap e1095: 8 total, 8 up, 8 in 2024-08-17T17:44:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:44:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:44:49] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-08-17T17:44:50.488 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:44:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:44:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:44:50.518 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:44:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:50 smithi179 ceph-mon[93519]: osdmap e1096: 8 total, 8 up, 8 in 2024-08-17T17:44:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:50 smithi130 ceph-mon[105521]: osdmap e1096: 8 total, 8 up, 8 in 2024-08-17T17:44:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:50 smithi130 ceph-mon[110068]: osdmap e1096: 8 total, 8 up, 8 in 2024-08-17T17:44:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:51 smithi179 ceph-mon[93519]: pgmap v1403: 105 pgs: 105 active+clean; 244 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.1 MiB/s wr, 44 op/s 2024-08-17T17:44:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:51 smithi179 ceph-mon[93519]: osdmap e1097: 8 total, 8 up, 8 in 2024-08-17T17:44:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:51 smithi130 ceph-mon[105521]: pgmap v1403: 105 pgs: 105 active+clean; 244 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.1 MiB/s wr, 44 op/s 2024-08-17T17:44:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:51 smithi130 ceph-mon[105521]: osdmap e1097: 8 total, 8 up, 8 in 2024-08-17T17:44:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:51 smithi130 ceph-mon[110068]: pgmap v1403: 105 pgs: 105 active+clean; 244 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.1 MiB/s wr, 44 op/s 2024-08-17T17:44:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:51 smithi130 ceph-mon[110068]: osdmap e1097: 8 total, 8 up, 8 in 2024-08-17T17:44:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:53 smithi179 ceph-mon[93519]: pgmap v1405: 105 pgs: 105 active+clean; 244 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.7 MiB/s wr, 14 op/s 2024-08-17T17:44:53.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:53 smithi130 ceph-mon[105521]: pgmap v1405: 105 pgs: 105 active+clean; 244 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.7 MiB/s wr, 14 op/s 2024-08-17T17:44:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:53 smithi130 ceph-mon[110068]: pgmap v1405: 105 pgs: 105 active+clean; 244 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.7 MiB/s wr, 14 op/s 2024-08-17T17:44:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:55 smithi179 ceph-mon[93519]: pgmap v1406: 105 pgs: 105 active+clean; 244 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 687 KiB/s wr, 37 op/s 2024-08-17T17:44:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:55 smithi130 ceph-mon[110068]: pgmap v1406: 105 pgs: 105 active+clean; 244 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 687 KiB/s wr, 37 op/s 2024-08-17T17:44:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:55 smithi130 ceph-mon[105521]: pgmap v1406: 105 pgs: 105 active+clean; 244 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 687 KiB/s wr, 37 op/s 2024-08-17T17:44:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:57 smithi179 ceph-mon[93519]: pgmap v1407: 105 pgs: 105 active+clean; 244 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 524 KiB/s wr, 30 op/s 2024-08-17T17:44:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:57 smithi179 ceph-mon[93519]: osdmap e1098: 8 total, 8 up, 8 in 2024-08-17T17:44:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:57.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:57 smithi130 ceph-mon[105521]: pgmap v1407: 105 pgs: 105 active+clean; 244 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 524 KiB/s wr, 30 op/s 2024-08-17T17:44:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:57 smithi130 ceph-mon[105521]: osdmap e1098: 8 total, 8 up, 8 in 2024-08-17T17:44:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:57 smithi130 ceph-mon[110068]: pgmap v1407: 105 pgs: 105 active+clean; 244 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 524 KiB/s wr, 30 op/s 2024-08-17T17:44:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:57 smithi130 ceph-mon[110068]: osdmap e1098: 8 total, 8 up, 8 in 2024-08-17T17:44:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:44:58.545 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:44:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:58 smithi179 ceph-mon[93519]: osdmap e1099: 8 total, 8 up, 8 in 2024-08-17T17:44:58.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:58 smithi130 ceph-mon[105521]: osdmap e1099: 8 total, 8 up, 8 in 2024-08-17T17:44:58.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:58 smithi130 ceph-mon[110068]: osdmap e1099: 8 total, 8 up, 8 in 2024-08-17T17:44:59.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:59 smithi130 ceph-mon[110068]: pgmap v1410: 105 pgs: 105 active+clean; 258 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 1.0 MiB/s wr, 35 op/s 2024-08-17T17:44:59.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:44:59 smithi130 ceph-mon[110068]: osdmap e1100: 8 total, 8 up, 8 in 2024-08-17T17:44:59.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:59 smithi130 ceph-mon[105521]: pgmap v1410: 105 pgs: 105 active+clean; 258 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 1.0 MiB/s wr, 35 op/s 2024-08-17T17:44:59.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:44:59 smithi130 ceph-mon[105521]: osdmap e1100: 8 total, 8 up, 8 in 2024-08-17T17:44:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:59 smithi179 ceph-mon[93519]: pgmap v1410: 105 pgs: 105 active+clean; 258 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 1.0 MiB/s wr, 35 op/s 2024-08-17T17:44:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:44:59 smithi179 ceph-mon[93519]: osdmap e1100: 8 total, 8 up, 8 in 2024-08-17T17:45:00.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:44:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:44:59] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-08-17T17:45:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:45:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:45:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:45:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:01 smithi179 ceph-mon[93519]: pgmap v1412: 105 pgs: 105 active+clean; 266 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 2.7 MiB/s wr, 30 op/s 2024-08-17T17:45:01.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:01 smithi130 ceph-mon[105521]: pgmap v1412: 105 pgs: 105 active+clean; 266 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 2.7 MiB/s wr, 30 op/s 2024-08-17T17:45:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:01 smithi130 ceph-mon[110068]: pgmap v1412: 105 pgs: 105 active+clean; 266 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 2.7 MiB/s wr, 30 op/s 2024-08-17T17:45:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:03 smithi179 ceph-mon[93519]: pgmap v1413: 105 pgs: 105 active+clean; 266 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 2.7 MiB/s wr, 28 op/s 2024-08-17T17:45:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:03 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:45:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:03 smithi130 ceph-mon[105521]: pgmap v1413: 105 pgs: 105 active+clean; 266 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 2.7 MiB/s wr, 28 op/s 2024-08-17T17:45:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:03 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:45:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:03 smithi130 ceph-mon[110068]: pgmap v1413: 105 pgs: 105 active+clean; 266 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 18 KiB/s rd, 2.7 MiB/s wr, 28 op/s 2024-08-17T17:45:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:03 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:45:05.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:05 smithi130 ceph-mon[105521]: pgmap v1414: 105 pgs: 105 active+clean; 266 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 35 op/s 2024-08-17T17:45:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:05 smithi130 ceph-mon[105521]: osdmap e1101: 8 total, 8 up, 8 in 2024-08-17T17:45:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:05 smithi130 ceph-mon[110068]: pgmap v1414: 105 pgs: 105 active+clean; 266 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 35 op/s 2024-08-17T17:45:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:05 smithi130 ceph-mon[110068]: osdmap e1101: 8 total, 8 up, 8 in 2024-08-17T17:45:05.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:05 smithi179 ceph-mon[93519]: pgmap v1414: 105 pgs: 105 active+clean; 266 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 35 op/s 2024-08-17T17:45:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:05 smithi179 ceph-mon[93519]: osdmap e1101: 8 total, 8 up, 8 in 2024-08-17T17:45:06.559 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:45:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:06 smithi179 ceph-mon[93519]: osdmap e1102: 8 total, 8 up, 8 in 2024-08-17T17:45:06.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:06 smithi130 ceph-mon[105521]: osdmap e1102: 8 total, 8 up, 8 in 2024-08-17T17:45:06.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:06 smithi130 ceph-mon[110068]: osdmap e1102: 8 total, 8 up, 8 in 2024-08-17T17:45:07.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:07 smithi179 ceph-mon[93519]: pgmap v1417: 105 pgs: 105 active+clean; 266 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 1.0 MiB/s wr, 25 op/s 2024-08-17T17:45:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:07 smithi179 ceph-mon[93519]: osdmap e1103: 8 total, 8 up, 8 in 2024-08-17T17:45:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:07 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:07 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:07.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:07 smithi130 ceph-mon[105521]: pgmap v1417: 105 pgs: 105 active+clean; 266 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 1.0 MiB/s wr, 25 op/s 2024-08-17T17:45:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:07 smithi130 ceph-mon[105521]: osdmap e1103: 8 total, 8 up, 8 in 2024-08-17T17:45:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:07 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:07 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:07 smithi130 ceph-mon[110068]: pgmap v1417: 105 pgs: 105 active+clean; 266 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 1.0 MiB/s wr, 25 op/s 2024-08-17T17:45:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:07 smithi130 ceph-mon[110068]: osdmap e1103: 8 total, 8 up, 8 in 2024-08-17T17:45:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:07 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:07 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:45:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:45:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:08.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:08.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:08.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:45:08.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:45:08.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:45:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:45:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:45:09.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:09 smithi130 ceph-mon[105521]: pgmap v1419: 105 pgs: 105 active+clean; 284 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 2.0 MiB/s wr, 59 op/s 2024-08-17T17:45:09.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:09 smithi130 ceph-mon[110068]: pgmap v1419: 105 pgs: 105 active+clean; 284 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 2.0 MiB/s wr, 59 op/s 2024-08-17T17:45:09.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:45:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:45:09] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-08-17T17:45:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:09 smithi179 ceph-mon[93519]: pgmap v1419: 105 pgs: 105 active+clean; 284 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 2.0 MiB/s wr, 59 op/s 2024-08-17T17:45:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:45:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:45:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:45:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:11 smithi130 ceph-mon[105521]: pgmap v1420: 105 pgs: 105 active+clean; 288 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:45:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:11 smithi130 ceph-mon[110068]: pgmap v1420: 105 pgs: 105 active+clean; 288 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:45:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:11 smithi179 ceph-mon[93519]: pgmap v1420: 105 pgs: 105 active+clean; 288 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:45:12.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:13.593 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:45:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=cleanup t=2024-08-17T17:45:13.134854901Z level=info msg="Completed cleanup jobs" duration=12.997426ms 2024-08-17T17:45:13.865 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:13 smithi179 ceph-mon[93519]: pgmap v1421: 105 pgs: 105 active+clean; 288 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.1 MiB/s wr, 35 op/s 2024-08-17T17:45:13.865 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:13 smithi179 ceph-mon[93519]: osdmap e1104: 8 total, 8 up, 8 in 2024-08-17T17:45:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:13 smithi130 ceph-mon[105521]: pgmap v1421: 105 pgs: 105 active+clean; 288 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.1 MiB/s wr, 35 op/s 2024-08-17T17:45:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:13 smithi130 ceph-mon[105521]: osdmap e1104: 8 total, 8 up, 8 in 2024-08-17T17:45:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:13 smithi130 ceph-mon[110068]: pgmap v1421: 105 pgs: 105 active+clean; 288 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.1 MiB/s wr, 35 op/s 2024-08-17T17:45:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:13 smithi130 ceph-mon[110068]: osdmap e1104: 8 total, 8 up, 8 in 2024-08-17T17:45:14.677 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:45:14.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:14 smithi130 ceph-mon[105521]: osdmap e1105: 8 total, 8 up, 8 in 2024-08-17T17:45:14.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:14 smithi130 ceph-mon[110068]: osdmap e1105: 8 total, 8 up, 8 in 2024-08-17T17:45:15.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:14 smithi179 ceph-mon[93519]: osdmap e1105: 8 total, 8 up, 8 in 2024-08-17T17:45:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:15 smithi130 ceph-mon[105521]: pgmap v1424: 105 pgs: 105 active+clean; 300 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 3.6 MiB/s wr, 46 op/s 2024-08-17T17:45:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:15 smithi130 ceph-mon[105521]: osdmap e1106: 8 total, 8 up, 8 in 2024-08-17T17:45:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:15 smithi130 ceph-mon[110068]: pgmap v1424: 105 pgs: 105 active+clean; 300 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 3.6 MiB/s wr, 46 op/s 2024-08-17T17:45:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:15 smithi130 ceph-mon[110068]: osdmap e1106: 8 total, 8 up, 8 in 2024-08-17T17:45:16.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:15 smithi179 ceph-mon[93519]: pgmap v1424: 105 pgs: 105 active+clean; 300 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 3.6 MiB/s wr, 46 op/s 2024-08-17T17:45:16.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:15 smithi179 ceph-mon[93519]: osdmap e1106: 8 total, 8 up, 8 in 2024-08-17T17:45:17.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:17 smithi130 ceph-mon[105521]: pgmap v1426: 105 pgs: 105 active+clean; 307 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.7 MiB/s wr, 14 op/s 2024-08-17T17:45:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:17 smithi130 ceph-mon[110068]: pgmap v1426: 105 pgs: 105 active+clean; 307 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.7 MiB/s wr, 14 op/s 2024-08-17T17:45:18.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:17 smithi179 ceph-mon[93519]: pgmap v1426: 105 pgs: 105 active+clean; 307 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.7 MiB/s wr, 14 op/s 2024-08-17T17:45:19.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:19 smithi130 ceph-mon[105521]: pgmap v1427: 105 pgs: 105 active+clean; 310 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:45:19.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:19 smithi130 ceph-mon[110068]: pgmap v1427: 105 pgs: 105 active+clean; 310 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:45:19.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:45:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:45:19] "GET /metrics HTTP/1.1" 200 36574 "" "Prometheus/2.51.0" 2024-08-17T17:45:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:19 smithi179 ceph-mon[93519]: pgmap v1427: 105 pgs: 105 active+clean; 310 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 46 op/s 2024-08-17T17:45:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:45:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:45:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:45:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:21 smithi130 ceph-mon[105521]: pgmap v1428: 105 pgs: 105 active+clean; 314 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.6 MiB/s wr, 36 op/s 2024-08-17T17:45:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:21 smithi130 ceph-mon[105521]: osdmap e1107: 8 total, 8 up, 8 in 2024-08-17T17:45:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:21 smithi130 ceph-mon[110068]: pgmap v1428: 105 pgs: 105 active+clean; 314 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.6 MiB/s wr, 36 op/s 2024-08-17T17:45:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:21 smithi130 ceph-mon[110068]: osdmap e1107: 8 total, 8 up, 8 in 2024-08-17T17:45:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:21 smithi179 ceph-mon[93519]: pgmap v1428: 105 pgs: 105 active+clean; 314 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.6 MiB/s wr, 36 op/s 2024-08-17T17:45:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:21 smithi179 ceph-mon[93519]: osdmap e1107: 8 total, 8 up, 8 in 2024-08-17T17:45:22.705 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministicLUKS2PP (40209 ms) 2024-08-17T17:45:22.705 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDiscard 2024-08-17T17:45:22.705 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:45:23.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:22 smithi179 ceph-mon[93519]: osdmap e1108: 8 total, 8 up, 8 in 2024-08-17T17:45:23.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:22 smithi130 ceph-mon[105521]: osdmap e1108: 8 total, 8 up, 8 in 2024-08-17T17:45:23.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:22 smithi130 ceph-mon[110068]: osdmap e1108: 8 total, 8 up, 8 in 2024-08-17T17:45:24.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:23 smithi179 ceph-mon[93519]: pgmap v1431: 105 pgs: 105 active+clean; 318 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 1.6 MiB/s wr, 24 op/s 2024-08-17T17:45:24.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:23 smithi179 ceph-mon[93519]: osdmap e1109: 8 total, 8 up, 8 in 2024-08-17T17:45:24.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:23 smithi130 ceph-mon[105521]: pgmap v1431: 105 pgs: 105 active+clean; 318 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 1.6 MiB/s wr, 24 op/s 2024-08-17T17:45:24.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:23 smithi130 ceph-mon[105521]: osdmap e1109: 8 total, 8 up, 8 in 2024-08-17T17:45:24.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:23 smithi130 ceph-mon[110068]: pgmap v1431: 105 pgs: 105 active+clean; 318 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 1.6 MiB/s wr, 24 op/s 2024-08-17T17:45:24.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:23 smithi130 ceph-mon[110068]: osdmap e1109: 8 total, 8 up, 8 in 2024-08-17T17:45:25.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:24 smithi179 ceph-mon[93519]: osdmap e1110: 8 total, 8 up, 8 in 2024-08-17T17:45:25.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:24 smithi130 ceph-mon[105521]: osdmap e1110: 8 total, 8 up, 8 in 2024-08-17T17:45:25.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:24 smithi130 ceph-mon[110068]: osdmap e1110: 8 total, 8 up, 8 in 2024-08-17T17:45:25.725 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateDiscard (3020 ms) 2024-08-17T17:45:25.725 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateTruncate 2024-08-17T17:45:25.725 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:45:26.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:25 smithi179 ceph-mon[93519]: pgmap v1434: 105 pgs: 105 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 3.0 MiB/s wr, 80 op/s 2024-08-17T17:45:26.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:25 smithi179 ceph-mon[93519]: osdmap e1111: 8 total, 8 up, 8 in 2024-08-17T17:45:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:25 smithi130 ceph-mon[105521]: pgmap v1434: 105 pgs: 105 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 3.0 MiB/s wr, 80 op/s 2024-08-17T17:45:26.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:25 smithi130 ceph-mon[105521]: osdmap e1111: 8 total, 8 up, 8 in 2024-08-17T17:45:26.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:25 smithi130 ceph-mon[110068]: pgmap v1434: 105 pgs: 105 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 3.0 MiB/s wr, 80 op/s 2024-08-17T17:45:26.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:25 smithi130 ceph-mon[110068]: osdmap e1111: 8 total, 8 up, 8 in 2024-08-17T17:45:27.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:26 smithi179 ceph-mon[93519]: osdmap e1112: 8 total, 8 up, 8 in 2024-08-17T17:45:27.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:26 smithi130 ceph-mon[105521]: osdmap e1112: 8 total, 8 up, 8 in 2024-08-17T17:45:27.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:26 smithi130 ceph-mon[110068]: osdmap e1112: 8 total, 8 up, 8 in 2024-08-17T17:45:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:27 smithi179 ceph-mon[93519]: pgmap v1437: 105 pgs: 105 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 53 KiB/s rd, 2.0 MiB/s wr, 83 op/s 2024-08-17T17:45:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:27 smithi179 ceph-mon[93519]: osdmap e1113: 8 total, 8 up, 8 in 2024-08-17T17:45:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:28.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:27 smithi130 ceph-mon[105521]: pgmap v1437: 105 pgs: 105 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 53 KiB/s rd, 2.0 MiB/s wr, 83 op/s 2024-08-17T17:45:28.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:27 smithi130 ceph-mon[105521]: osdmap e1113: 8 total, 8 up, 8 in 2024-08-17T17:45:28.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:28.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:27 smithi130 ceph-mon[110068]: pgmap v1437: 105 pgs: 105 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 53 KiB/s rd, 2.0 MiB/s wr, 83 op/s 2024-08-17T17:45:28.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:27 smithi130 ceph-mon[110068]: osdmap e1113: 8 total, 8 up, 8 in 2024-08-17T17:45:28.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:28.767 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateTruncate (3042 ms) 2024-08-17T17:45:28.768 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateWriteAndTruncate 2024-08-17T17:45:28.768 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:45:29.068 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:28 smithi130 ceph-mon[105521]: osdmap e1114: 8 total, 8 up, 8 in 2024-08-17T17:45:29.068 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:28 smithi130 ceph-mon[110068]: osdmap e1114: 8 total, 8 up, 8 in 2024-08-17T17:45:29.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:28 smithi179 ceph-mon[93519]: osdmap e1114: 8 total, 8 up, 8 in 2024-08-17T17:45:30.089 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:29 smithi130 ceph-mon[105521]: pgmap v1440: 105 pgs: 105 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 4.5 KiB/s wr, 65 op/s 2024-08-17T17:45:30.089 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:29 smithi130 ceph-mon[105521]: osdmap e1115: 8 total, 8 up, 8 in 2024-08-17T17:45:30.090 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:45:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:45:29] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-08-17T17:45:30.090 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:29 smithi130 ceph-mon[110068]: pgmap v1440: 105 pgs: 105 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 4.5 KiB/s wr, 65 op/s 2024-08-17T17:45:30.090 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:29 smithi130 ceph-mon[110068]: osdmap e1115: 8 total, 8 up, 8 in 2024-08-17T17:45:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:29 smithi179 ceph-mon[93519]: pgmap v1440: 105 pgs: 105 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s rd, 4.5 KiB/s wr, 65 op/s 2024-08-17T17:45:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:29 smithi179 ceph-mon[93519]: osdmap e1115: 8 total, 8 up, 8 in 2024-08-17T17:45:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:45:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:45:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:45:31.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:30 smithi179 ceph-mon[93519]: osdmap e1116: 8 total, 8 up, 8 in 2024-08-17T17:45:31.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:30 smithi130 ceph-mon[105521]: osdmap e1116: 8 total, 8 up, 8 in 2024-08-17T17:45:31.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:30 smithi130 ceph-mon[110068]: osdmap e1116: 8 total, 8 up, 8 in 2024-08-17T17:45:31.774 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateWriteAndTruncate (3007 ms) 2024-08-17T17:45:31.774 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStress 2024-08-17T17:45:31.774 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:45:31.832 INFO:tasks.workunit.client.0.smithi130.stdout: i=0 exists [2083188~3680165,24581400~1314716,37666204~7665827,90744509~2968335,96274207~4184628,152322797~3440262,194901265~5440016,285636051~7602225,308030534~355067,340765516~1962307] wrote [2083188~3680165,24581400~1314716,37666204~7665827,90744509~2968335,96274207~4184628,152322797~3440262,194901265~5440016,285636051~7602225,308030534~355067,340765516~1962307] 2024-08-17T17:45:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:31 smithi179 ceph-mon[93519]: pgmap v1443: 105 pgs: 105 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 393 KiB/s wr, 107 op/s 2024-08-17T17:45:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:31 smithi179 ceph-mon[93519]: osdmap e1117: 8 total, 8 up, 8 in 2024-08-17T17:45:32.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:31 smithi130 ceph-mon[110068]: pgmap v1443: 105 pgs: 105 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 393 KiB/s wr, 107 op/s 2024-08-17T17:45:32.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:31 smithi130 ceph-mon[110068]: osdmap e1117: 8 total, 8 up, 8 in 2024-08-17T17:45:32.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:31 smithi130 ceph-mon[105521]: pgmap v1443: 105 pgs: 105 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 393 KiB/s wr, 107 op/s 2024-08-17T17:45:32.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:31 smithi130 ceph-mon[105521]: osdmap e1117: 8 total, 8 up, 8 in 2024-08-17T17:45:32.811 INFO:tasks.workunit.client.0.smithi130.stdout: i=1 exists [2083188~3680165,6377918~5737258,24581400~1314716,37666204~7665827,64140920~840509,83760579~9952265,96274207~4184628,111970624~4759817,125679365~6836804,141737503~4355181,152322797~3440262,187532292~2292075,194901265~5440016,285636051~7602225,308030534~355067,340765516~4839430,364248728~5289548] wrote [6377918~5737258,64140920~840509,83760579~7810634,111970624~4759817,125679365~6836804,141737503~4355181,187532292~2292075,289126636~1540826,342355156~3249790,364248728~5289548] 2024-08-17T17:45:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:32 smithi179 ceph-mon[93519]: osdmap e1118: 8 total, 8 up, 8 in 2024-08-17T17:45:33.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:32 smithi130 ceph-mon[105521]: osdmap e1118: 8 total, 8 up, 8 in 2024-08-17T17:45:33.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:32 smithi130 ceph-mon[110068]: osdmap e1118: 8 total, 8 up, 8 in 2024-08-17T17:45:33.826 INFO:tasks.workunit.client.0.smithi130.stdout: i=2 exists [2083188~3680165,6377918~5737258,24581400~1314716,37666204~7665827,64140920~840509,83760579~9952265,96274207~4184628,111970624~4759817,125679365~6836804,141737503~4355181,152322797~3440262,181009902~940898,183342564~3580888,187532292~2292075,194901265~5440016,212859013~22671,231395872~327897,253156921~7114424,285636051~7602225,308030534~355067,315204064~8410012,340765516~4839430,364248728~8255672] wrote [84071988~8058345,181009902~940898,183342564~3580888,212859013~22671,231395872~327897,253156921~7114424,315204064~8410012,364995449~7508951] 2024-08-17T17:45:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:33 smithi179 ceph-mon[93519]: pgmap v1446: 105 pgs: 105 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 389 KiB/s wr, 45 op/s 2024-08-17T17:45:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:33 smithi179 ceph-mon[93519]: osdmap e1119: 8 total, 8 up, 8 in 2024-08-17T17:45:34.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:33 smithi130 ceph-mon[105521]: pgmap v1446: 105 pgs: 105 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 389 KiB/s wr, 45 op/s 2024-08-17T17:45:34.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:33 smithi130 ceph-mon[105521]: osdmap e1119: 8 total, 8 up, 8 in 2024-08-17T17:45:34.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:33 smithi130 ceph-mon[110068]: pgmap v1446: 105 pgs: 105 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 389 KiB/s wr, 45 op/s 2024-08-17T17:45:34.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:33 smithi130 ceph-mon[110068]: osdmap e1119: 8 total, 8 up, 8 in 2024-08-17T17:45:34.861 INFO:tasks.workunit.client.0.smithi130.stdout: i=3 exists [2083188~3680165,6377918~5737258,24581400~1314716,37666204~7665827,64140920~840509,83760579~10566329,96274207~4184628,101984504~4273835,111970624~4759817,125679365~6836804,141737503~4355181,152322797~3440262,165267563~35954,181009902~12920691,194901265~5440016,212859013~22671,231039088~5949575,253156921~7114424,285636051~7602225,308030534~355067,315204064~8410012,340765516~4839430,353521984~5372799,364248728~8255672,375541081~7920853] wrote [88229119~6097789,101984504~4273835,113108222~155750,165267563~35954,181067445~12863148,231039088~5949575,254265982~4571380,353521984~5372799,375541081~7920853] 2024-08-17T17:45:35.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:34 smithi179 ceph-mon[93519]: osdmap e1120: 8 total, 8 up, 8 in 2024-08-17T17:45:35.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:34 smithi130 ceph-mon[105521]: osdmap e1120: 8 total, 8 up, 8 in 2024-08-17T17:45:35.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:34 smithi130 ceph-mon[110068]: osdmap e1120: 8 total, 8 up, 8 in 2024-08-17T17:45:35.817 INFO:tasks.workunit.client.0.smithi130.stdout: i=4 exists [2083188~3680165,6377918~5737258,17982662~2043600,20406330~5901884,37666204~7665827,47009112~5875010,64140920~840509,78422203~194672,83760579~11554319,96274207~4184628,101984504~4273835,110890003~5840438,125679365~6836804,141737503~4355181,152322797~3440262,165267563~35954,181009902~12920691,194901265~5440016,212859013~22671,231039088~5949575,238557622~5676701,253156921~10572590,285636051~7602225,308030534~355067,315204064~8410012,340765516~4839430,347763685~1349033,353521984~5372799,364248728~8255672,372649891~264708,375541081~7920853] wrote [17982662~2043600,20406330~5901884,47009112~5875010,78422203~194672,90214986~5099912,110890003~1980139,238557622~5676701,259506400~4223111,347763685~1349033,372649891~264708] 2024-08-17T17:45:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:35 smithi179 ceph-mon[93519]: pgmap v1449: 105 pgs: 105 active+clean; 438 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 17 MiB/s wr, 158 op/s 2024-08-17T17:45:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:35 smithi179 ceph-mon[93519]: osdmap e1121: 8 total, 8 up, 8 in 2024-08-17T17:45:36.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:35 smithi130 ceph-mon[105521]: pgmap v1449: 105 pgs: 105 active+clean; 438 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 17 MiB/s wr, 158 op/s 2024-08-17T17:45:36.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:35 smithi130 ceph-mon[105521]: osdmap e1121: 8 total, 8 up, 8 in 2024-08-17T17:45:36.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:35 smithi130 ceph-mon[110068]: pgmap v1449: 105 pgs: 105 active+clean; 438 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 17 MiB/s wr, 158 op/s 2024-08-17T17:45:36.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:35 smithi130 ceph-mon[110068]: osdmap e1121: 8 total, 8 up, 8 in 2024-08-17T17:45:36.824 INFO:tasks.workunit.client.0.smithi130.stdout: i=5 exists [2083188~3680165,6377918~5737258,13248743~447576,17982662~2043600,20406330~5901884,35151403~2183649,37666204~7665827,47009112~5875010,64140920~840509,69749754~5248397,76015500~4889267,83760579~11554319,96274207~4184628,101984504~4273835,110890003~5840438,125679365~6836804,141737503~4355181,152322797~3440262,165267563~35954,170974799~7132547,181009902~12920691,194901265~5440016,212857373~2310652,231039088~5949575,238557622~5676701,252085053~11644458,268301131~8129760,285636051~7602225,308030534~355067,315204064~8410012,340765516~4839430,347763685~1349033,353521984~5372799,364248728~8255672,372649891~264708,375541081~7920853,391383806~534171] wrote [13248743~447576,35151403~2183649,69749754~5248397,76015500~4889267,170974799~7132547,212857373~2310652,252085053~4334023,268301131~8129760,391383806~534171] 2024-08-17T17:45:37.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:36 smithi179 ceph-mon[93519]: osdmap e1122: 8 total, 8 up, 8 in 2024-08-17T17:45:37.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:36 smithi130 ceph-mon[105521]: osdmap e1122: 8 total, 8 up, 8 in 2024-08-17T17:45:37.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:36 smithi130 ceph-mon[110068]: osdmap e1122: 8 total, 8 up, 8 in 2024-08-17T17:45:37.840 INFO:tasks.workunit.client.0.smithi130.stdout: i=6 exists [2083188~3680165,6377918~5737258,13248743~447576,17019901~9288313,35151403~10180628,47009112~6398255,64140920~840509,69749754~5248397,76015500~4889267,81631944~13682954,96274207~4184628,101984504~4273835,110890003~5840438,123467982~9048187,141737503~4355181,152322797~3440262,165267563~35954,170974799~7132547,181009902~12920691,194901265~5440016,206187837~2876912,212857373~2310652,231039088~5949575,238557622~10507163,252085053~11644458,268301131~8129760,285636051~7602225,308030534~355067,315204064~8410012,340765516~4839430,347763685~1349033,353521984~5372799,364248728~8255672,372649891~264708,375271217~8190717,391383806~534171] wrote [17019901~8029786,36860424~3643790,49658437~3748930,81631944~6200204,123467982~7288018,183141903~4084621,206187837~2876912,243378184~5686601,375271217~4806185] 2024-08-17T17:45:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:37 smithi179 ceph-mon[93519]: pgmap v1452: 105 pgs: 105 active+clean; 460 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 133 KiB/s rd, 20 MiB/s wr, 219 op/s 2024-08-17T17:45:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:37 smithi179 ceph-mon[93519]: osdmap e1123: 8 total, 8 up, 8 in 2024-08-17T17:45:38.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:37 smithi130 ceph-mon[105521]: pgmap v1452: 105 pgs: 105 active+clean; 460 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 133 KiB/s rd, 20 MiB/s wr, 219 op/s 2024-08-17T17:45:38.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:37 smithi130 ceph-mon[105521]: osdmap e1123: 8 total, 8 up, 8 in 2024-08-17T17:45:38.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:37 smithi130 ceph-mon[110068]: pgmap v1452: 105 pgs: 105 active+clean; 460 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 133 KiB/s rd, 20 MiB/s wr, 219 op/s 2024-08-17T17:45:38.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:37 smithi130 ceph-mon[110068]: osdmap e1123: 8 total, 8 up, 8 in 2024-08-17T17:45:38.840 INFO:tasks.workunit.client.0.smithi130.stdout: i=7 exists [2083188~3680165,6377918~5737258,13248743~447576,17019901~9288313,35151403~10180628,47009112~6398255,64140920~840509,69749754~5248397,76015500~4889267,81631944~13682954,95741920~4716915,101984504~4273835,110890003~5840438,123467982~9048187,141737503~4355181,152322797~3440262,165267563~35954,170974799~7132547,181009902~12920691,194901265~5440016,206187837~2876912,212857373~2310652,231039088~5949575,238557622~10507163,252085053~11644458,268301131~8129760,285636051~7602225,308030534~355067,315204064~8410012,327014957~3775799,340765516~4839430,347763685~1349033,353521984~5372799,363635816~296614,364248728~8255672,372649891~264708,375271217~8190717,391383806~534171,400448910~6250157] wrote [95741920~4078537,154102663~1377987,186463939~4516023,195905960~2543796,257626798~4221732,327014957~3775799,363635816~296614,400448910~6250157] 2024-08-17T17:45:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:38 smithi179 ceph-mon[93519]: osdmap e1124: 8 total, 8 up, 8 in 2024-08-17T17:45:39.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:38 smithi130 ceph-mon[105521]: osdmap e1124: 8 total, 8 up, 8 in 2024-08-17T17:45:39.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:38 smithi130 ceph-mon[110068]: osdmap e1124: 8 total, 8 up, 8 in 2024-08-17T17:45:39.858 INFO:tasks.workunit.client.0.smithi130.stdout: i=8 exists [2083188~3680165,6377918~5737258,13248743~447576,17019901~9288313,35151403~10180628,47009112~6398255,64140920~840509,69749754~5248397,76015500~19299398,95741920~4716915,101984504~4273835,110890003~5840438,123467982~9048187,141737503~8166133,152322797~3440262,165267563~35954,170974799~7132547,181009902~12920691,194901265~5440016,206187837~2876912,212857373~2310652,231039088~18025697,252085053~11644458,268301131~8129760,285636051~11981354,308030534~355067,315204064~8410012,327014957~3775799,338138599~7466347,347763685~1349033,353521984~5372799,363635816~296614,364248728~8255672,372649891~264708,375271217~8825617,389442879~19588,391383806~534171,397099750~9599317] wrote [80343727~5058533,142848440~7055196,234241756~7269489,245969446~1861114,293018932~4598473,338138599~7295039,375492239~726683,378336091~5760743,389442879~19588,397099750~5355241] 2024-08-17T17:45:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:39 smithi179 ceph-mon[93519]: pgmap v1455: 105 pgs: 105 active+clean; 623 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 160 KiB/s rd, 38 MiB/s wr, 264 op/s 2024-08-17T17:45:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:39 smithi179 ceph-mon[93519]: osdmap e1125: 8 total, 8 up, 8 in 2024-08-17T17:45:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:45:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:45:39] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.51.0" 2024-08-17T17:45:40.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:39 smithi130 ceph-mon[110068]: pgmap v1455: 105 pgs: 105 active+clean; 623 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 160 KiB/s rd, 38 MiB/s wr, 264 op/s 2024-08-17T17:45:40.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:39 smithi130 ceph-mon[110068]: osdmap e1125: 8 total, 8 up, 8 in 2024-08-17T17:45:40.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:39 smithi130 ceph-mon[105521]: pgmap v1455: 105 pgs: 105 active+clean; 623 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 160 KiB/s rd, 38 MiB/s wr, 264 op/s 2024-08-17T17:45:40.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:39 smithi130 ceph-mon[105521]: osdmap e1125: 8 total, 8 up, 8 in 2024-08-17T17:45:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:45:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:45:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:45:40.858 INFO:tasks.workunit.client.0.smithi130.stdout: i=9 exists [2083188~3680165,6377918~5737258,13248743~447576,17019901~9288313,35151403~10180628,47009112~6398255,55125230~3175922,64140920~840509,69749754~5248397,76015500~19299398,95741920~10516419,110890003~5840438,120069466~1394051,123467982~9048187,140411071~9492565,152322797~3440262,164913761~488146,170974799~7132547,181009902~12920691,194901265~5440016,206187837~2876912,212857373~2310652,217138118~3217788,231039088~18025697,252085053~11644458,264037046~12393845,285636051~11981354,308030534~355067,315204064~8410012,327014957~3775799,338138599~7466347,347763685~1349033,353521984~5372799,363635816~296614,364248728~8255672,372649891~264708,375271217~8825617,389442879~19588,391383806~534171,397099750~9599317] wrote [55125230~3175922,98467697~7562190,120069466~1394051,140411071~5175182,164913761~488146,217138118~3217788,235774234~5765661,264037046~5078040,369629736~178164] 2024-08-17T17:45:41.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:40 smithi179 ceph-mon[93519]: osdmap e1126: 8 total, 8 up, 8 in 2024-08-17T17:45:41.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:40 smithi130 ceph-mon[105521]: osdmap e1126: 8 total, 8 up, 8 in 2024-08-17T17:45:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:40 smithi130 ceph-mon[110068]: osdmap e1126: 8 total, 8 up, 8 in 2024-08-17T17:45:41.867 INFO:tasks.workunit.client.0.smithi130.stdout: i=10 exists [2083188~10031988,13248743~447576,17019901~9288313,35151403~10180628,47009112~11292040,64139720~2505802,69749754~5248397,76015500~19299398,95741920~10516419,110890003~5840438,120069466~2990162,123467982~9048187,140411071~9492565,152322797~3440262,164913761~488146,170974799~7132547,181009902~12920691,194901265~5440016,206187837~2876912,212857373~2310652,217138118~3217788,231039088~18025697,252085053~11644458,264037046~12393845,285636051~11981354,308030534~355067,315204064~8410012,327014957~3775799,338138599~7466347,345802711~4415885,353521984~5372799,360313696~12190704,372649891~264708,375271217~8825617,389442879~19588,391383806~534171,397099750~9599317] wrote [5533399~4881877,22033504~1725158,50900403~6102470,64139720~2505802,76847202~4962089,86657349~7109408,121004185~2055443,345802711~4415885,360313696~9606336] 2024-08-17T17:45:42.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:41 smithi179 ceph-mon[93519]: pgmap v1458: 105 pgs: 105 active+clean; 679 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 178 KiB/s rd, 49 MiB/s wr, 292 op/s 2024-08-17T17:45:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:41 smithi179 ceph-mon[93519]: osdmap e1127: 8 total, 8 up, 8 in 2024-08-17T17:45:42.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:41 smithi130 ceph-mon[105521]: pgmap v1458: 105 pgs: 105 active+clean; 679 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 178 KiB/s rd, 49 MiB/s wr, 292 op/s 2024-08-17T17:45:42.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:41 smithi130 ceph-mon[105521]: osdmap e1127: 8 total, 8 up, 8 in 2024-08-17T17:45:42.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:41 smithi130 ceph-mon[110068]: pgmap v1458: 105 pgs: 105 active+clean; 679 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 178 KiB/s rd, 49 MiB/s wr, 292 op/s 2024-08-17T17:45:42.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:41 smithi130 ceph-mon[110068]: osdmap e1127: 8 total, 8 up, 8 in 2024-08-17T17:45:42.925 INFO:tasks.workunit.client.0.smithi130.stdout: i=11 exists [2083188~10031988,13248743~447576,17019901~9288313,35151403~10180628,47009112~11292040,64139720~2505802,69749754~5248397,76015500~19299398,95741920~10516419,110890003~7882948,120069466~2990162,123467982~9048187,140411071~9492565,152322797~3440262,164913761~488146,170974799~8140994,181009902~12920691,194901265~14412914,212857373~2310652,217138118~3217788,231039088~18025697,252085053~11644458,264037046~12393845,285636051~11981354,298155712~7002654,308030534~355067,315204064~8410012,327014957~3775799,334996414~10608532,345802711~4415885,353521984~5372799,360313696~12190704,372649891~264708,375271217~8825617,389442879~19588,391383806~534171,397099750~10463446] wrote [84432881~4031995,115691309~3081642,173081345~6034448,197563546~11750633,234518907~4694286,270304988~1427154,298155712~7002654,334996414~4522233,401604123~5959073] 2024-08-17T17:45:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:42 smithi179 ceph-mon[93519]: osdmap e1128: 8 total, 8 up, 8 in 2024-08-17T17:45:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:43.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:42 smithi130 ceph-mon[105521]: osdmap e1128: 8 total, 8 up, 8 in 2024-08-17T17:45:43.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:42 smithi130 ceph-mon[110068]: osdmap e1128: 8 total, 8 up, 8 in 2024-08-17T17:45:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:43.988 INFO:tasks.workunit.client.0.smithi130.stdout: i=12 exists [2083188~10031988,13248743~447576,17019901~9288313,35151403~10180628,47009112~11292040,64139720~2505802,69749754~5248397,76015500~19299398,95741920~12801676,110890003~7882948,120069466~2990162,123467982~9048187,140411071~9492565,152322797~3440262,164913761~488146,170974799~8140994,181009902~12920691,194901265~16451364,212363103~8135394,231039088~18025697,252085053~11644458,264037046~12393845,285636051~11981354,298155712~7002654,308030534~355067,315204064~8410012,327014957~3775799,334996414~10608532,345802711~4415885,353521984~5372799,360313696~12190704,372649891~264708,375271217~10283744,388453340~4487258,397099750~10463446] wrote [87894487~7007301,100809518~7734078,147533697~1340200,203427849~7924780,212363103~8135394,267288095~713251,379142641~6412320,388453340~4487258] 2024-08-17T17:45:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:43 smithi179 ceph-mon[93519]: pgmap v1461: 105 pgs: 105 active+clean; 707 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 62 KiB/s rd, 21 MiB/s wr, 93 op/s 2024-08-17T17:45:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:43 smithi179 ceph-mon[93519]: osdmap e1129: 8 total, 8 up, 8 in 2024-08-17T17:45:44.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:43 smithi130 ceph-mon[105521]: pgmap v1461: 105 pgs: 105 active+clean; 707 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 62 KiB/s rd, 21 MiB/s wr, 93 op/s 2024-08-17T17:45:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:43 smithi130 ceph-mon[105521]: osdmap e1129: 8 total, 8 up, 8 in 2024-08-17T17:45:44.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:43 smithi130 ceph-mon[110068]: pgmap v1461: 105 pgs: 105 active+clean; 707 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 62 KiB/s rd, 21 MiB/s wr, 93 op/s 2024-08-17T17:45:44.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:43 smithi130 ceph-mon[110068]: osdmap e1129: 8 total, 8 up, 8 in 2024-08-17T17:45:44.882 INFO:tasks.workunit.client.0.smithi130.stdout: i=13 exists [2083188~10031988,13248743~447576,17019901~9288313,35151403~10180628,47009112~11292040,64139720~2505802,68303526~154574,69749754~5248397,76015500~19299398,95741920~12801676,110890003~8644164,120069466~2990162,123467982~9048187,140411071~9492565,152322797~3440262,164913761~488146,170974799~8140994,181009902~12920691,194901265~16451364,212363103~8135394,231039088~20584764,252085053~11644458,264037046~12393845,285636051~11981354,298155712~7002654,308030534~355067,315204064~8410012,327014957~5940292,334996414~15222182,350762991~8131792,360313696~12190704,372649891~264708,375271217~10283744,388453340~4487258,397099750~10463446] wrote [17315832~5569764,68303526~154574,69975059~4340146,117947453~1586714,246421976~5201876,275311232~857137,328235721~4719528,339334387~6939182,350762991~4558471,361813028~25040] 2024-08-17T17:45:45.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:44 smithi179 ceph-mon[93519]: osdmap e1130: 8 total, 8 up, 8 in 2024-08-17T17:45:45.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:44 smithi130 ceph-mon[105521]: osdmap e1130: 8 total, 8 up, 8 in 2024-08-17T17:45:45.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:44 smithi130 ceph-mon[110068]: osdmap e1130: 8 total, 8 up, 8 in 2024-08-17T17:45:45.895 INFO:tasks.workunit.client.0.smithi130.stdout: i=14 exists [2083188~10031988,13248743~447576,17019901~9288313,28804386~3381031,35151403~10180628,47009112~11292040,64139720~2505802,68303526~154574,69749754~5248397,76015500~19299398,95741920~12801676,110890003~8644164,120069466~2990162,123467982~9048187,140411071~9492565,152322797~3440262,164913761~488146,170974799~8140994,181009902~12920691,194717316~16635313,212071326~175948,212363103~8135394,231039088~20584764,252085053~11644458,264037046~12393845,282456887~2915983,285636051~11981354,298155712~7002654,308030534~355067,315204064~8410012,327014957~23203639,350762991~8652789,360313696~12190704,372649891~264708,375271217~10283744,388453340~4487258,394273582~13410958] wrote [4790534~6897937,28804386~3381031,194717316~1955120,212071326~175948,282456887~2915983,327538796~12229392,356703786~2711994,394273582~3521823,407484479~200061] 2024-08-17T17:45:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:45 smithi179 ceph-mon[93519]: pgmap v1464: 105 pgs: 105 active+clean; 856 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 40 MiB/s wr, 208 op/s 2024-08-17T17:45:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:45 smithi179 ceph-mon[93519]: osdmap e1131: 8 total, 8 up, 8 in 2024-08-17T17:45:46.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:45 smithi130 ceph-mon[105521]: pgmap v1464: 105 pgs: 105 active+clean; 856 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 40 MiB/s wr, 208 op/s 2024-08-17T17:45:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:45 smithi130 ceph-mon[105521]: osdmap e1131: 8 total, 8 up, 8 in 2024-08-17T17:45:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:45 smithi130 ceph-mon[110068]: pgmap v1464: 105 pgs: 105 active+clean; 856 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 117 KiB/s rd, 40 MiB/s wr, 208 op/s 2024-08-17T17:45:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:45 smithi130 ceph-mon[110068]: osdmap e1131: 8 total, 8 up, 8 in 2024-08-17T17:45:46.943 INFO:tasks.workunit.client.0.smithi130.stdout: i=15 exists [2083188~11613131,17019901~9288313,28804386~3381031,35151403~10180628,47009112~11292040,64139720~2505802,68303526~154574,69749754~5248397,76015500~19299398,95741920~12801676,110890003~8644164,120069466~2990162,123467982~9048187,140411071~9492565,152322797~3440262,164913761~488146,170974799~8140994,181009902~12920691,194717316~16635313,212071326~175948,212363103~8135394,231039088~20584764,252085053~11644458,264037046~12393845,282456887~22701479,308030534~355067,315204064~8410012,327014957~32400823,360313696~12190704,372649891~264708,375271217~10283744,388453340~4487258,394273582~13410958] wrote [8606464~4963472,18776426~4996464,36389139~6651350,78997831~7620369,282881152~2895851,293789284~6663874,344050004~732019,346165020~5567209] 2024-08-17T17:45:47.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:46 smithi179 ceph-mon[93519]: osdmap e1132: 8 total, 8 up, 8 in 2024-08-17T17:45:47.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:46 smithi130 ceph-mon[105521]: osdmap e1132: 8 total, 8 up, 8 in 2024-08-17T17:45:47.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:46 smithi130 ceph-mon[110068]: osdmap e1132: 8 total, 8 up, 8 in 2024-08-17T17:45:47.909 INFO:tasks.workunit.client.0.smithi130.stdout: i=16 exists [2083188~11613131,17019901~9288313,28129093~6281616,35151403~10180628,47009112~11292040,58625054~8020468,68303526~154574,69749754~5248397,76015500~32528096,110890003~24351319,140411071~9492565,152322797~3440262,164913761~488146,169766981~9348812,181009902~12920691,194717316~16635313,212071326~175948,212363103~8135394,231039088~20584764,252085053~11644458,264037046~12393845,280489222~24669144,308030534~355067,315204064~8410012,327014957~32400823,360313696~12190704,372649891~264708,375271217~10283744,388453340~4487258,394273582~13410958] wrote [28129093~6281616,58625054~6906570,92901255~7357534,118389124~5869493,130934198~4307124,169766981~5349034,184315682~974913,187045628~3051162,280489222~7736721,302626023~1071126] 2024-08-17T17:45:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:47 smithi179 ceph-mon[93519]: pgmap v1467: 105 pgs: 105 active+clean; 900 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 212 KiB/s rd, 45 MiB/s wr, 337 op/s 2024-08-17T17:45:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:47 smithi179 ceph-mon[93519]: osdmap e1133: 8 total, 8 up, 8 in 2024-08-17T17:45:48.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:47 smithi130 ceph-mon[110068]: pgmap v1467: 105 pgs: 105 active+clean; 900 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 212 KiB/s rd, 45 MiB/s wr, 337 op/s 2024-08-17T17:45:48.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:47 smithi130 ceph-mon[110068]: osdmap e1133: 8 total, 8 up, 8 in 2024-08-17T17:45:48.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:47 smithi130 ceph-mon[105521]: pgmap v1467: 105 pgs: 105 active+clean; 900 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 212 KiB/s rd, 45 MiB/s wr, 337 op/s 2024-08-17T17:45:48.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:47 smithi130 ceph-mon[105521]: osdmap e1133: 8 total, 8 up, 8 in 2024-08-17T17:45:48.931 INFO:tasks.workunit.client.0.smithi130.stdout: i=17 exists [2083188~11613131,17019901~9288313,28129093~6599037,35151403~23149749,58625054~8020468,68303526~154574,69749754~5248397,76015500~32528096,110890003~24351319,140411071~9492565,152322797~3440262,163350297~3348378,169766981~9348812,181009902~12920691,194717316~25781181,231039088~20584764,252085053~11644458,264037046~15678576,280489222~24669144,308030534~355067,315204064~8410012,327014957~32400823,360313696~26575640,388453340~4487258,394273582~13410958] wrote [29028991~5699139,41888397~9729328,58939658~4294501,163350297~3348378,208374498~4586151,271957938~7757684,333139933~2066701,371026794~6748008,382793685~4095651] 2024-08-17T17:45:49.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:48 smithi130 ceph-mon[105521]: osdmap e1134: 8 total, 8 up, 8 in 2024-08-17T17:45:49.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:48 smithi130 ceph-mon[110068]: osdmap e1134: 8 total, 8 up, 8 in 2024-08-17T17:45:49.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:48 smithi179 ceph-mon[93519]: osdmap e1134: 8 total, 8 up, 8 in 2024-08-17T17:45:49.955 INFO:tasks.workunit.client.0.smithi130.stdout: i=18 exists [2083188~11613131,17019901~9288313,28129093~6736638,35151403~23149749,58625054~16373097,76015500~32528096,110890003~24351319,140411071~9492565,152322797~3440262,163350297~3348378,169766981~24163612,194717316~25781181,229647885~21975967,252085053~11644458,264037046~15678576,280489222~24669144,308030534~355067,315204064~8410012,327014957~59874379,388453340~5224445,394273582~13410958] wrote [20249391~5755304,28219998~6645733,51233535~969042,66233052~5049288,126268148~7977439,177035955~5935400,229647885~5634630,294130162~7251663,357893698~4868885,389563832~4113953] 2024-08-17T17:45:50.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:49 smithi130 ceph-mon[105521]: pgmap v1470: 105 pgs: 105 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 48 MiB/s wr, 325 op/s 2024-08-17T17:45:50.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:49 smithi130 ceph-mon[105521]: osdmap e1135: 8 total, 8 up, 8 in 2024-08-17T17:45:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:45:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:45:49] "GET /metrics HTTP/1.1" 200 36579 "" "Prometheus/2.51.0" 2024-08-17T17:45:50.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:49 smithi130 ceph-mon[110068]: pgmap v1470: 105 pgs: 105 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 48 MiB/s wr, 325 op/s 2024-08-17T17:45:50.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:49 smithi130 ceph-mon[110068]: osdmap e1135: 8 total, 8 up, 8 in 2024-08-17T17:45:50.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:49 smithi179 ceph-mon[93519]: pgmap v1470: 105 pgs: 105 active+clean; 1.0 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 205 KiB/s rd, 48 MiB/s wr, 325 op/s 2024-08-17T17:45:50.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:49 smithi179 ceph-mon[93519]: osdmap e1135: 8 total, 8 up, 8 in 2024-08-17T17:45:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:45:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:45:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:45:50.937 INFO:tasks.workunit.client.0.smithi130.stdout: i=19 exists [2083188~11613131,17019901~9288313,28129093~6736638,35151403~39846748,76015500~32528096,110890003~24351319,140411071~9492565,152322797~3440262,160008161~826776,163350297~3348378,169766981~24786017,194717316~25781181,229647885~21975967,252085053~11644458,264037046~15678576,280489222~24669144,308030534~355067,315204064~8410012,327014957~59874379,388453340~5224445,394233093~13451447] wrote [9844982~3837047,52463759~12645026,123710223~7737260,160008161~826776,188406139~6146859,257304647~1060146,333383457~5824313,380896749~5865408,394233093~3658585] 2024-08-17T17:45:51.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:50 smithi130 ceph-mon[105521]: osdmap e1136: 8 total, 8 up, 8 in 2024-08-17T17:45:51.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:50 smithi130 ceph-mon[110068]: osdmap e1136: 8 total, 8 up, 8 in 2024-08-17T17:45:51.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:50 smithi179 ceph-mon[93519]: osdmap e1136: 8 total, 8 up, 8 in 2024-08-17T17:45:51.883 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 1 diff [4194304~8388608,62914560~4194304,79691776~8388608,109051904~8388608,121634816~8388608,138412032~8388608,184549376~4194304,285212672~4194304,339738624~4194304,360710144~8388608] 2024-08-17T17:45:51.883 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,62914560~4194304,79691776~8388608,109051904~8388608,121634816~8388608,138412032~8388608,184549376~4194304,285212672~4194304,339738624~4194304,360710144~8388608] 2024-08-17T17:45:51.892 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,62914560~4194304,79691776~12582912,109051904~8388608,121634816~12582912,138412032~8388608,184549376~8388608,285212672~8388608,339738624~8388608,360710144~12582912] 2024-08-17T17:45:51.892 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 2 diff [4194304~8388608,62914560~4194304,79691776~8388608,109051904~8388608,121634816~8388608,138412032~8388608,180355072~8388608,209715200~4194304,230686720~4194304,251658240~8388608,285212672~4194304,314572800~12582912,339738624~4194304,360710144~8388608] 2024-08-17T17:45:51.892 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,62914560~4194304,79691776~8388608,109051904~8388608,121634816~8388608,138412032~8388608,180355072~8388608,209715200~4194304,230686720~4194304,251658240~8388608,285212672~4194304,314572800~12582912,339738624~4194304,360710144~8388608] 2024-08-17T17:45:51.895 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,62914560~4194304,79691776~12582912,109051904~8388608,121634816~12582912,138412032~8388608,180355072~12582912,209715200~4194304,230686720~4194304,251658240~12582912,285212672~8388608,314572800~12582912,339738624~8388608,360710144~12582912] 2024-08-17T17:45:51.895 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 3 diff [4194304~8388608,62914560~4194304,79691776~12582912,100663296~16777216,121634816~8388608,138412032~8388608,163577856~4194304,180355072~16777216,209715200~4194304,230686720~8388608,251658240~8388608,285212672~4194304,314572800~12582912,339738624~4194304,352321536~16777216,373293056~8388608] 2024-08-17T17:45:51.895 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,62914560~4194304,79691776~12582912,100663296~16777216,121634816~8388608,138412032~8388608,163577856~4194304,180355072~16777216,209715200~4194304,230686720~8388608,251658240~8388608,285212672~4194304,314572800~12582912,339738624~4194304,352321536~16777216,373293056~8388608] 2024-08-17T17:45:51.899 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,62914560~4194304,79691776~16777216,100663296~16777216,121634816~12582912,138412032~8388608,163577856~4194304,180355072~16777216,209715200~4194304,230686720~8388608,251658240~12582912,285212672~8388608,314572800~12582912,339738624~8388608,352321536~33554432] 2024-08-17T17:45:51.899 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 4 diff [4194304~8388608,16777216~8388608,46137344~8388608,62914560~4194304,75497472~16777216,100663296~16777216,121634816~8388608,138412032~8388608,163577856~4194304,180355072~16777216,209715200~4194304,230686720~12582912,251658240~12582912,285212672~4194304,314572800~12582912,339738624~8388608,352321536~29360128] 2024-08-17T17:45:51.899 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~8388608,46137344~8388608,62914560~4194304,75497472~16777216,100663296~16777216,121634816~8388608,138412032~8388608,163577856~4194304,180355072~16777216,209715200~4194304,230686720~12582912,251658240~12582912,285212672~4194304,314572800~12582912,339738624~8388608,352321536~29360128] 2024-08-17T17:45:51.903 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~12582912,46137344~8388608,62914560~4194304,75497472~20971520,100663296~16777216,121634816~12582912,138412032~8388608,163577856~4194304,180355072~16777216,209715200~4194304,230686720~16777216,251658240~12582912,285212672~8388608,314572800~12582912,339738624~46137344] 2024-08-17T17:45:51.903 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 5 diff [4194304~20971520,33554432~4194304,46137344~8388608,62914560~29360128,100663296~16777216,121634816~8388608,138412032~8388608,163577856~12582912,180355072~16777216,209715200~4194304,230686720~12582912,251658240~20971520,285212672~4194304,314572800~12582912,339738624~8388608,352321536~29360128,390070272~4194304] 2024-08-17T17:45:51.903 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~20971520,33554432~4194304,46137344~8388608,62914560~29360128,100663296~16777216,121634816~8388608,138412032~8388608,163577856~12582912,180355072~16777216,209715200~4194304,230686720~12582912,251658240~20971520,285212672~4194304,314572800~12582912,339738624~8388608,352321536~29360128,390070272~4194304] 2024-08-17T17:45:51.906 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~4194304,46137344~8388608,62914560~33554432,100663296~16777216,121634816~12582912,138412032~8388608,163577856~33554432,209715200~8388608,230686720~16777216,251658240~25165824,285212672~8388608,314572800~12582912,339738624~46137344,390070272~4194304] 2024-08-17T17:45:51.906 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 6 diff [4194304~25165824,33554432~8388608,46137344~8388608,62914560~33554432,100663296~16777216,121634816~12582912,138412032~8388608,163577856~12582912,180355072~16777216,205520896~8388608,230686720~16777216,251658240~20971520,285212672~4194304,314572800~12582912,339738624~8388608,352321536~29360128,390070272~4194304] 2024-08-17T17:45:51.906 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~8388608,62914560~33554432,100663296~16777216,121634816~12582912,138412032~8388608,163577856~12582912,180355072~16777216,205520896~8388608,230686720~16777216,251658240~20971520,285212672~4194304,314572800~12582912,339738624~8388608,352321536~29360128,390070272~4194304] 2024-08-17T17:45:51.910 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~8388608,62914560~33554432,100663296~16777216,121634816~12582912,138412032~8388608,163577856~33554432,205520896~12582912,230686720~46137344,285212672~8388608,314572800~12582912,339738624~46137344,390070272~4194304] 2024-08-17T17:45:51.910 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 7 diff [4194304~25165824,33554432~8388608,46137344~8388608,62914560~33554432,100663296~16777216,121634816~12582912,138412032~8388608,150994944~4194304,163577856~12582912,180355072~16777216,205520896~8388608,230686720~16777216,251658240~20971520,285212672~4194304,314572800~12582912,339738624~8388608,352321536~29360128,390070272~4194304,398458880~8388608] 2024-08-17T17:45:51.910 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~8388608,62914560~33554432,100663296~16777216,121634816~12582912,138412032~8388608,150994944~4194304,163577856~12582912,180355072~16777216,205520896~8388608,230686720~16777216,251658240~20971520,285212672~4194304,314572800~12582912,339738624~8388608,352321536~29360128,390070272~4194304,398458880~8388608] 2024-08-17T17:45:51.913 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~8388608,62914560~54525952,121634816~12582912,138412032~8388608,150994944~8388608,163577856~37748736,205520896~12582912,230686720~46137344,285212672~8388608,314572800~16777216,339738624~46137344,390070272~4194304,398458880~8388608] 2024-08-17T17:45:51.913 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 8 diff [4194304~25165824,33554432~8388608,46137344~8388608,62914560~33554432,100663296~16777216,121634816~12582912,138412032~8388608,150994944~4194304,163577856~12582912,180355072~16777216,205520896~8388608,230686720~41943040,285212672~12582912,314572800~12582912,335544320~12582912,352321536~54525952] 2024-08-17T17:45:51.913 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~8388608,62914560~33554432,100663296~16777216,121634816~12582912,138412032~8388608,150994944~4194304,163577856~12582912,180355072~16777216,205520896~8388608,230686720~41943040,285212672~12582912,314572800~12582912,335544320~12582912,352321536~54525952] 2024-08-17T17:45:51.917 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~8388608,62914560~54525952,121634816~12582912,138412032~20971520,163577856~37748736,205520896~12582912,230686720~46137344,285212672~12582912,314572800~16777216,335544320~71303168] 2024-08-17T17:45:51.918 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 9 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~25165824,138412032~16777216,163577856~12582912,180355072~16777216,205520896~12582912,230686720~41943040,285212672~12582912,314572800~12582912,335544320~12582912,352321536~54525952] 2024-08-17T17:45:51.918 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~25165824,138412032~16777216,163577856~12582912,180355072~16777216,205520896~12582912,230686720~41943040,285212672~12582912,314572800~12582912,335544320~12582912,352321536~54525952] 2024-08-17T17:45:51.922 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~37748736,205520896~16777216,230686720~46137344,285212672~12582912,314572800~16777216,335544320~71303168] 2024-08-17T17:45:51.922 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 10 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~25165824,138412032~16777216,163577856~12582912,180355072~16777216,205520896~12582912,230686720~41943040,285212672~12582912,314572800~12582912,335544320~71303168] 2024-08-17T17:45:51.922 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~25165824,138412032~16777216,163577856~12582912,180355072~16777216,205520896~12582912,230686720~41943040,285212672~12582912,314572800~12582912,335544320~71303168] 2024-08-17T17:45:51.928 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~37748736,205520896~16777216,230686720~46137344,285212672~12582912,314572800~16777216,335544320~71303168] 2024-08-17T17:45:51.928 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 11 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~25165824,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,285212672~20971520,314572800~12582912,331350016~75497472] 2024-08-17T17:45:51.928 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~25165824,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,285212672~20971520,314572800~12582912,331350016~75497472] 2024-08-17T17:45:51.935 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,285212672~20971520,314572800~96468992] 2024-08-17T17:45:51.935 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 12 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,285212672~20971520,314572800~12582912,331350016~75497472] 2024-08-17T17:45:51.935 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,285212672~20971520,314572800~12582912,331350016~75497472] 2024-08-17T17:45:51.942 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,285212672~20971520,314572800~96468992] 2024-08-17T17:45:51.943 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 13 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,285212672~20971520,314572800~92274688] 2024-08-17T17:45:51.943 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,285212672~20971520,314572800~92274688] 2024-08-17T17:45:51.949 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,285212672~20971520,314572800~96468992] 2024-08-17T17:45:51.950 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 14 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,281018368~25165824,314572800~96468992] 2024-08-17T17:45:51.950 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,281018368~25165824,314572800~96468992] 2024-08-17T17:45:51.958 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~37748736,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,281018368~25165824,314572800~96468992] 2024-08-17T17:45:51.958 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 15 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,281018368~20971520,314572800~96468992] 2024-08-17T17:45:51.958 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,281018368~20971520,314572800~96468992] 2024-08-17T17:45:51.966 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~54525952,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,281018368~25165824,314572800~96468992] 2024-08-17T17:45:51.966 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 16 diff [4194304~37748736,46137344~16777216,67108864~67108864,138412032~16777216,163577856~54525952,230686720~41943040,276824064~25165824,314572800~96468992] 2024-08-17T17:45:51.966 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~67108864,138412032~16777216,163577856~54525952,230686720~41943040,276824064~25165824,314572800~96468992] 2024-08-17T17:45:51.975 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~155189248,163577856~58720256,230686720~75497472,314572800~96468992] 2024-08-17T17:45:51.975 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 17 diff [4194304~58720256,67108864~67108864,138412032~16777216,159383552~4194304,167772160~54525952,230686720~71303168,314572800~96468992] 2024-08-17T17:45:51.975 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~58720256,67108864~67108864,138412032~16777216,159383552~4194304,167772160~54525952,230686720~71303168,314572800~96468992] 2024-08-17T17:45:51.983 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,230686720~75497472,314572800~96468992] 2024-08-17T17:45:51.983 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 18 diff [4194304~67108864,75497472~58720256,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~96468992] 2024-08-17T17:45:51.984 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~58720256,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~96468992] 2024-08-17T17:45:51.993 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:51.993 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to 19 diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:51.993 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:52.002 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:52.002 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 2 diff [83886080~8388608,180355072~4194304,209715200~4194304,230686720~4194304,251658240~8388608,314572800~12582912,364904448~8388608] 2024-08-17T17:45:52.003 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [83886080~8388608,180355072~4194304,209715200~4194304,230686720~4194304,251658240~8388608,314572800~12582912,364904448~8388608] 2024-08-17T17:45:52.004 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [83886080~8388608,180355072~8388608,209715200~4194304,230686720~4194304,251658240~12582912,314572800~12582912,364904448~8388608] 2024-08-17T17:45:52.004 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 3 diff [83886080~12582912,100663296~12582912,163577856~4194304,180355072~16777216,209715200~4194304,230686720~8388608,251658240~8388608,314572800~12582912,352321536~8388608,364904448~16777216] 2024-08-17T17:45:52.004 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [83886080~12582912,100663296~12582912,163577856~4194304,180355072~16777216,209715200~4194304,230686720~8388608,251658240~8388608,314572800~12582912,352321536~8388608,364904448~16777216] 2024-08-17T17:45:52.006 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [83886080~12582912,100663296~16777216,163577856~4194304,180355072~16777216,209715200~4194304,230686720~8388608,251658240~12582912,314572800~12582912,352321536~8388608,364904448~20971520] 2024-08-17T17:45:52.006 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 4 diff [16777216~8388608,46137344~8388608,75497472~4194304,83886080~12582912,100663296~12582912,163577856~4194304,180355072~16777216,209715200~4194304,230686720~12582912,251658240~12582912,314572800~12582912,343932928~4194304,352321536~8388608,364904448~16777216] 2024-08-17T17:45:52.007 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [16777216~8388608,46137344~8388608,75497472~4194304,83886080~12582912,100663296~12582912,163577856~4194304,180355072~16777216,209715200~4194304,230686720~12582912,251658240~12582912,314572800~12582912,343932928~4194304,352321536~8388608,364904448~16777216] 2024-08-17T17:45:52.008 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [16777216~12582912,46137344~8388608,75497472~4194304,83886080~12582912,100663296~16777216,163577856~4194304,180355072~16777216,209715200~4194304,230686720~16777216,251658240~12582912,314572800~12582912,343932928~16777216,364904448~20971520] 2024-08-17T17:45:52.009 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 5 diff [12582912~12582912,33554432~4194304,46137344~8388608,67108864~29360128,100663296~12582912,163577856~12582912,180355072~16777216,209715200~4194304,230686720~12582912,251658240~20971520,314572800~12582912,343932928~4194304,352321536~8388608,364904448~16777216,390070272~4194304] 2024-08-17T17:45:52.009 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~12582912,33554432~4194304,46137344~8388608,67108864~29360128,100663296~12582912,163577856~12582912,180355072~16777216,209715200~4194304,230686720~12582912,251658240~20971520,314572800~12582912,343932928~4194304,352321536~8388608,364904448~16777216,390070272~4194304] 2024-08-17T17:45:52.011 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~4194304,46137344~8388608,67108864~29360128,100663296~16777216,163577856~33554432,209715200~8388608,230686720~16777216,251658240~25165824,314572800~12582912,343932928~16777216,364904448~20971520,390070272~4194304] 2024-08-17T17:45:52.011 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 6 diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,100663296~12582912,121634816~8388608,163577856~12582912,180355072~16777216,205520896~8388608,230686720~16777216,251658240~20971520,314572800~12582912,343932928~4194304,352321536~8388608,364904448~16777216,390070272~4194304] 2024-08-17T17:45:52.011 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,100663296~12582912,121634816~8388608,163577856~12582912,180355072~16777216,205520896~8388608,230686720~16777216,251658240~20971520,314572800~12582912,343932928~4194304,352321536~8388608,364904448~16777216,390070272~4194304] 2024-08-17T17:45:52.014 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,100663296~16777216,121634816~12582912,163577856~33554432,205520896~12582912,230686720~46137344,314572800~12582912,343932928~16777216,364904448~20971520,390070272~4194304] 2024-08-17T17:45:52.014 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 7 diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,100663296~12582912,121634816~8388608,150994944~4194304,163577856~12582912,180355072~16777216,205520896~8388608,230686720~16777216,251658240~20971520,314572800~12582912,343932928~4194304,352321536~29360128,390070272~4194304,398458880~8388608] 2024-08-17T17:45:52.014 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,100663296~12582912,121634816~8388608,150994944~4194304,163577856~12582912,180355072~16777216,205520896~8388608,230686720~16777216,251658240~20971520,314572800~12582912,343932928~4194304,352321536~29360128,390070272~4194304,398458880~8388608] 2024-08-17T17:45:52.017 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~8388608,46137344~8388608,67108864~50331648,121634816~12582912,150994944~8388608,163577856~37748736,205520896~12582912,230686720~46137344,314572800~16777216,343932928~41943040,390070272~4194304,398458880~8388608] 2024-08-17T17:45:52.017 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 8 diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,100663296~12582912,121634816~8388608,142606336~12582912,163577856~12582912,180355072~16777216,205520896~8388608,230686720~41943040,289406976~8388608,314572800~12582912,335544320~12582912,352321536~54525952] 2024-08-17T17:45:52.017 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,100663296~12582912,121634816~8388608,142606336~12582912,163577856~12582912,180355072~16777216,205520896~8388608,230686720~41943040,289406976~8388608,314572800~12582912,335544320~12582912,352321536~54525952] 2024-08-17T17:45:52.021 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~8388608,46137344~8388608,67108864~50331648,121634816~12582912,142606336~16777216,163577856~37748736,205520896~12582912,230686720~46137344,289406976~8388608,314572800~16777216,335544320~71303168] 2024-08-17T17:45:52.021 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 9 diff [12582912~16777216,33554432~8388608,46137344~12582912,67108864~37748736,109051904~4194304,117440512~12582912,138412032~16777216,163577856~12582912,180355072~16777216,205520896~12582912,230686720~41943040,289406976~8388608,314572800~12582912,335544320~12582912,352321536~54525952] 2024-08-17T17:45:52.021 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~16777216,33554432~8388608,46137344~12582912,67108864~37748736,109051904~4194304,117440512~12582912,138412032~16777216,163577856~12582912,180355072~16777216,205520896~12582912,230686720~41943040,289406976~8388608,314572800~12582912,335544320~12582912,352321536~54525952] 2024-08-17T17:45:52.024 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~8388608,46137344~12582912,67108864~67108864,138412032~20971520,163577856~37748736,205520896~16777216,230686720~46137344,289406976~8388608,314572800~16777216,335544320~71303168] 2024-08-17T17:45:52.024 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 10 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~4194304,117440512~12582912,138412032~16777216,163577856~12582912,180355072~16777216,205520896~12582912,230686720~41943040,289406976~8388608,314572800~12582912,335544320~71303168] 2024-08-17T17:45:52.025 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~4194304,117440512~12582912,138412032~16777216,163577856~12582912,180355072~16777216,205520896~12582912,230686720~41943040,289406976~8388608,314572800~12582912,335544320~71303168] 2024-08-17T17:45:52.029 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~37748736,205520896~16777216,230686720~46137344,289406976~8388608,314572800~16777216,335544320~71303168] 2024-08-17T17:45:52.029 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 11 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~20971520,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,289406976~16777216,314572800~12582912,331350016~75497472] 2024-08-17T17:45:52.029 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~20971520,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,289406976~16777216,314572800~12582912,331350016~75497472] 2024-08-17T17:45:52.033 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,289406976~16777216,314572800~96468992] 2024-08-17T17:45:52.033 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 12 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,289406976~16777216,314572800~12582912,331350016~75497472] 2024-08-17T17:45:52.034 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,289406976~16777216,314572800~12582912,331350016~75497472] 2024-08-17T17:45:52.039 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,289406976~16777216,314572800~96468992] 2024-08-17T17:45:52.039 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 13 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,289406976~16777216,314572800~92274688] 2024-08-17T17:45:52.039 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,289406976~16777216,314572800~92274688] 2024-08-17T17:45:52.045 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,289406976~16777216,314572800~96468992] 2024-08-17T17:45:52.045 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 14 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,281018368~4194304,289406976~16777216,314572800~96468992] 2024-08-17T17:45:52.045 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,281018368~4194304,289406976~16777216,314572800~96468992] 2024-08-17T17:45:52.052 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~37748736,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,281018368~25165824,314572800~96468992] 2024-08-17T17:45:52.052 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 15 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,281018368~4194304,289406976~12582912,314572800~96468992] 2024-08-17T17:45:52.052 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,281018368~4194304,289406976~12582912,314572800~96468992] 2024-08-17T17:45:52.059 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~54525952,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,281018368~25165824,314572800~96468992] 2024-08-17T17:45:52.059 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 16 diff [4194304~37748736,46137344~16777216,67108864~88080384,163577856~54525952,230686720~41943040,276824064~8388608,289406976~12582912,314572800~96468992] 2024-08-17T17:45:52.059 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~88080384,163577856~54525952,230686720~41943040,276824064~8388608,289406976~12582912,314572800~96468992] 2024-08-17T17:45:52.067 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~155189248,163577856~58720256,230686720~75497472,314572800~96468992] 2024-08-17T17:45:52.067 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 17 diff [4194304~58720256,67108864~88080384,159383552~4194304,167772160~54525952,230686720~54525952,289406976~12582912,314572800~96468992] 2024-08-17T17:45:52.067 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~58720256,67108864~88080384,159383552~4194304,167772160~54525952,230686720~54525952,289406976~12582912,314572800~96468992] 2024-08-17T17:45:52.073 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,230686720~75497472,314572800~96468992] 2024-08-17T17:45:52.073 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 18 diff [4194304~67108864,75497472~58720256,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~96468992] 2024-08-17T17:45:52.073 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~58720256,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~96468992] 2024-08-17T17:45:52.079 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:52.079 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to 19 diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:52.080 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:52.088 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:52.088 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 3 diff [88080384~8388608,100663296~12582912,163577856~4194304,180355072~16777216,230686720~8388608,251658240~8388608,352321536~8388608,373293056~8388608] 2024-08-17T17:45:52.088 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [88080384~8388608,100663296~12582912,163577856~4194304,180355072~16777216,230686720~8388608,251658240~8388608,352321536~8388608,373293056~8388608] 2024-08-17T17:45:52.089 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [88080384~8388608,100663296~16777216,163577856~4194304,180355072~16777216,230686720~8388608,251658240~8388608,352321536~8388608,373293056~12582912] 2024-08-17T17:45:52.089 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 4 diff [16777216~8388608,46137344~8388608,75497472~4194304,88080384~8388608,100663296~12582912,163577856~4194304,180355072~16777216,230686720~12582912,251658240~12582912,343932928~4194304,352321536~8388608,369098752~12582912] 2024-08-17T17:45:52.089 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [16777216~8388608,46137344~8388608,75497472~4194304,88080384~8388608,100663296~12582912,163577856~4194304,180355072~16777216,230686720~12582912,251658240~12582912,343932928~4194304,352321536~8388608,369098752~12582912] 2024-08-17T17:45:52.091 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [16777216~12582912,46137344~8388608,75497472~4194304,88080384~8388608,100663296~16777216,163577856~4194304,180355072~16777216,230686720~16777216,251658240~12582912,343932928~16777216,369098752~16777216] 2024-08-17T17:45:52.091 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 5 diff [12582912~12582912,33554432~4194304,46137344~8388608,67108864~16777216,88080384~8388608,100663296~12582912,163577856~12582912,180355072~16777216,209715200~4194304,230686720~12582912,251658240~20971520,343932928~4194304,352321536~8388608,369098752~12582912,390070272~4194304] 2024-08-17T17:45:52.091 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~12582912,33554432~4194304,46137344~8388608,67108864~16777216,88080384~8388608,100663296~12582912,163577856~12582912,180355072~16777216,209715200~4194304,230686720~12582912,251658240~20971520,343932928~4194304,352321536~8388608,369098752~12582912,390070272~4194304] 2024-08-17T17:45:52.092 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~4194304,46137344~8388608,67108864~16777216,88080384~8388608,100663296~16777216,163577856~33554432,209715200~8388608,230686720~16777216,251658240~25165824,343932928~16777216,369098752~16777216,390070272~4194304] 2024-08-17T17:45:52.092 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 6 diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,100663296~12582912,121634816~8388608,163577856~12582912,180355072~16777216,205520896~8388608,230686720~16777216,251658240~20971520,343932928~4194304,352321536~8388608,369098752~12582912,390070272~4194304] 2024-08-17T17:45:52.093 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,100663296~12582912,121634816~8388608,163577856~12582912,180355072~16777216,205520896~8388608,230686720~16777216,251658240~20971520,343932928~4194304,352321536~8388608,369098752~12582912,390070272~4194304] 2024-08-17T17:45:52.095 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,100663296~16777216,121634816~12582912,163577856~33554432,205520896~12582912,230686720~46137344,343932928~16777216,369098752~16777216,390070272~4194304] 2024-08-17T17:45:52.095 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 7 diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,100663296~12582912,121634816~8388608,150994944~4194304,163577856~12582912,180355072~16777216,205520896~8388608,230686720~16777216,251658240~20971520,322961408~4194304,343932928~4194304,352321536~12582912,369098752~12582912,390070272~4194304,398458880~8388608] 2024-08-17T17:45:52.095 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,100663296~12582912,121634816~8388608,150994944~4194304,163577856~12582912,180355072~16777216,205520896~8388608,230686720~16777216,251658240~20971520,322961408~4194304,343932928~4194304,352321536~12582912,369098752~12582912,390070272~4194304,398458880~8388608] 2024-08-17T17:45:52.098 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~8388608,46137344~8388608,67108864~50331648,121634816~12582912,150994944~8388608,163577856~37748736,205520896~12582912,230686720~46137344,322961408~8388608,343932928~20971520,369098752~16777216,390070272~4194304,398458880~8388608] 2024-08-17T17:45:52.098 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 8 diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,100663296~12582912,121634816~8388608,142606336~12582912,163577856~12582912,180355072~16777216,205520896~8388608,230686720~41943040,289406976~8388608,322961408~4194304,335544320~12582912,352321536~12582912,369098752~37748736] 2024-08-17T17:45:52.098 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,100663296~12582912,121634816~8388608,142606336~12582912,163577856~12582912,180355072~16777216,205520896~8388608,230686720~41943040,289406976~8388608,322961408~4194304,335544320~12582912,352321536~12582912,369098752~37748736] 2024-08-17T17:45:52.100 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~8388608,46137344~8388608,67108864~50331648,121634816~12582912,142606336~16777216,163577856~37748736,205520896~12582912,230686720~46137344,289406976~8388608,322961408~8388608,335544320~29360128,369098752~37748736] 2024-08-17T17:45:52.101 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 9 diff [12582912~16777216,33554432~8388608,46137344~12582912,67108864~37748736,109051904~4194304,117440512~12582912,138412032~16777216,163577856~12582912,180355072~16777216,205520896~12582912,230686720~41943040,289406976~8388608,322961408~4194304,335544320~12582912,352321536~12582912,369098752~37748736] 2024-08-17T17:45:52.101 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~16777216,33554432~8388608,46137344~12582912,67108864~37748736,109051904~4194304,117440512~12582912,138412032~16777216,163577856~12582912,180355072~16777216,205520896~12582912,230686720~41943040,289406976~8388608,322961408~4194304,335544320~12582912,352321536~12582912,369098752~37748736] 2024-08-17T17:45:52.104 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~8388608,46137344~12582912,67108864~67108864,138412032~20971520,163577856~37748736,205520896~16777216,230686720~46137344,289406976~8388608,322961408~8388608,335544320~29360128,369098752~37748736] 2024-08-17T17:45:52.104 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 10 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~4194304,117440512~12582912,138412032~16777216,163577856~12582912,180355072~16777216,205520896~12582912,230686720~41943040,289406976~8388608,322961408~4194304,335544320~71303168] 2024-08-17T17:45:52.105 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~4194304,117440512~12582912,138412032~16777216,163577856~12582912,180355072~16777216,205520896~12582912,230686720~41943040,289406976~8388608,322961408~4194304,335544320~71303168] 2024-08-17T17:45:52.109 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~37748736,205520896~16777216,230686720~46137344,289406976~8388608,322961408~8388608,335544320~71303168] 2024-08-17T17:45:52.109 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 11 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~20971520,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,289406976~16777216,322961408~4194304,331350016~75497472] 2024-08-17T17:45:52.109 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~20971520,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,289406976~16777216,322961408~4194304,331350016~75497472] 2024-08-17T17:45:52.114 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,289406976~16777216,322961408~88080384] 2024-08-17T17:45:52.114 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 12 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,289406976~16777216,322961408~4194304,331350016~75497472] 2024-08-17T17:45:52.115 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,289406976~16777216,322961408~4194304,331350016~75497472] 2024-08-17T17:45:52.119 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,289406976~16777216,322961408~88080384] 2024-08-17T17:45:52.119 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 13 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,289406976~16777216,322961408~83886080] 2024-08-17T17:45:52.119 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,289406976~16777216,322961408~83886080] 2024-08-17T17:45:52.124 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,289406976~16777216,322961408~88080384] 2024-08-17T17:45:52.124 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 14 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,281018368~4194304,289406976~16777216,322961408~88080384] 2024-08-17T17:45:52.124 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,281018368~4194304,289406976~16777216,322961408~88080384] 2024-08-17T17:45:52.130 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~37748736,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,281018368~25165824,322961408~88080384] 2024-08-17T17:45:52.130 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 15 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,281018368~4194304,289406976~12582912,322961408~88080384] 2024-08-17T17:45:52.130 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~37748736,230686720~41943040,281018368~4194304,289406976~12582912,322961408~88080384] 2024-08-17T17:45:52.135 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~54525952,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,281018368~25165824,322961408~88080384] 2024-08-17T17:45:52.135 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 16 diff [4194304~37748736,46137344~16777216,67108864~88080384,163577856~54525952,230686720~41943040,276824064~8388608,289406976~12582912,322961408~88080384] 2024-08-17T17:45:52.135 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~88080384,163577856~54525952,230686720~41943040,276824064~8388608,289406976~12582912,322961408~88080384] 2024-08-17T17:45:52.141 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~155189248,163577856~58720256,230686720~75497472,322961408~88080384] 2024-08-17T17:45:52.141 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 17 diff [4194304~58720256,67108864~88080384,159383552~4194304,167772160~54525952,230686720~54525952,289406976~12582912,322961408~88080384] 2024-08-17T17:45:52.141 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~58720256,67108864~88080384,159383552~4194304,167772160~54525952,230686720~54525952,289406976~12582912,322961408~88080384] 2024-08-17T17:45:52.148 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,230686720~75497472,322961408~88080384] 2024-08-17T17:45:52.148 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 18 diff [4194304~67108864,75497472~58720256,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~88080384] 2024-08-17T17:45:52.148 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~58720256,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~88080384] 2024-08-17T17:45:52.155 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:52.155 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to 19 diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:52.155 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:52.162 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:52.162 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 4 diff [16777216~8388608,46137344~8388608,75497472~4194304,88080384~8388608,109051904~4194304,234881024~8388608,255852544~8388608,343932928~4194304,369098752~4194304] 2024-08-17T17:45:52.162 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [16777216~8388608,46137344~8388608,75497472~4194304,88080384~8388608,109051904~4194304,234881024~8388608,255852544~8388608,343932928~4194304,369098752~4194304] 2024-08-17T17:45:52.163 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [16777216~12582912,46137344~8388608,75497472~4194304,88080384~8388608,109051904~4194304,234881024~12582912,255852544~8388608,343932928~8388608,369098752~4194304] 2024-08-17T17:45:52.163 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 5 diff [12582912~12582912,33554432~4194304,46137344~8388608,67108864~16777216,88080384~8388608,109051904~4194304,167772160~8388608,209715200~4194304,234881024~8388608,251658240~20971520,343932928~4194304,369098752~4194304,390070272~4194304] 2024-08-17T17:45:52.163 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~12582912,33554432~4194304,46137344~8388608,67108864~16777216,88080384~8388608,109051904~4194304,167772160~8388608,209715200~4194304,234881024~8388608,251658240~20971520,343932928~4194304,369098752~4194304,390070272~4194304] 2024-08-17T17:45:52.165 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~4194304,46137344~8388608,67108864~16777216,88080384~8388608,109051904~4194304,167772160~12582912,209715200~8388608,234881024~12582912,251658240~25165824,343932928~8388608,369098752~4194304,390070272~4194304] 2024-08-17T17:45:52.165 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 6 diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,109051904~4194304,121634816~8388608,167772160~8388608,180355072~4194304,205520896~8388608,234881024~12582912,251658240~20971520,343932928~4194304,369098752~12582912,390070272~4194304] 2024-08-17T17:45:52.165 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,109051904~4194304,121634816~8388608,167772160~8388608,180355072~4194304,205520896~8388608,234881024~12582912,251658240~20971520,343932928~4194304,369098752~12582912,390070272~4194304] 2024-08-17T17:45:52.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:51 smithi130 ceph-mon[110068]: pgmap v1473: 105 pgs: 105 active+clean; 1.1 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 237 KiB/s rd, 48 MiB/s wr, 368 op/s 2024-08-17T17:45:52.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:51 smithi130 ceph-mon[110068]: osdmap e1137: 8 total, 8 up, 8 in 2024-08-17T17:45:52.167 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,109051904~4194304,121634816~12582912,167772160~20971520,205520896~12582912,234881024~41943040,343932928~8388608,369098752~12582912,390070272~4194304] 2024-08-17T17:45:52.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:51 smithi130 ceph-mon[105521]: pgmap v1473: 105 pgs: 105 active+clean; 1.1 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 237 KiB/s rd, 48 MiB/s wr, 368 op/s 2024-08-17T17:45:52.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:51 smithi130 ceph-mon[105521]: osdmap e1137: 8 total, 8 up, 8 in 2024-08-17T17:45:52.167 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 7 diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,109051904~4194304,121634816~8388608,150994944~4194304,167772160~8388608,180355072~8388608,192937984~4194304,205520896~8388608,234881024~12582912,251658240~20971520,322961408~4194304,343932928~4194304,360710144~4194304,369098752~12582912,390070272~4194304,398458880~8388608] 2024-08-17T17:45:52.168 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,109051904~4194304,121634816~8388608,150994944~4194304,167772160~8388608,180355072~8388608,192937984~4194304,205520896~8388608,234881024~12582912,251658240~20971520,322961408~4194304,343932928~4194304,360710144~4194304,369098752~12582912,390070272~4194304,398458880~8388608] 2024-08-17T17:45:52.168 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~8388608,46137344~8388608,67108864~33554432,109051904~4194304,121634816~12582912,150994944~8388608,167772160~33554432,205520896~12582912,234881024~41943040,322961408~8388608,343932928~8388608,360710144~4194304,369098752~12582912,390070272~4194304,398458880~8388608] 2024-08-17T17:45:52.168 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 8 diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,109051904~4194304,121634816~8388608,142606336~12582912,167772160~8388608,180355072~8388608,192937984~4194304,205520896~8388608,230686720~16777216,251658240~20971520,289406976~8388608,322961408~4194304,335544320~12582912,360710144~4194304,369098752~37748736] 2024-08-17T17:45:52.168 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~16777216,33554432~8388608,46137344~8388608,67108864~29360128,109051904~4194304,121634816~8388608,142606336~12582912,167772160~8388608,180355072~8388608,192937984~4194304,205520896~8388608,230686720~16777216,251658240~20971520,289406976~8388608,322961408~4194304,335544320~12582912,360710144~4194304,369098752~37748736] 2024-08-17T17:45:52.171 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~8388608,46137344~8388608,67108864~33554432,109051904~4194304,121634816~12582912,142606336~16777216,167772160~33554432,205520896~12582912,230686720~46137344,289406976~8388608,322961408~8388608,335544320~16777216,360710144~4194304,369098752~37748736] 2024-08-17T17:45:52.171 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 9 diff [12582912~16777216,33554432~8388608,46137344~12582912,67108864~37748736,109051904~4194304,117440512~12582912,138412032~16777216,163577856~12582912,180355072~8388608,192937984~4194304,205520896~12582912,230686720~16777216,251658240~20971520,289406976~8388608,322961408~4194304,335544320~12582912,360710144~4194304,369098752~37748736] 2024-08-17T17:45:52.171 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~16777216,33554432~8388608,46137344~12582912,67108864~37748736,109051904~4194304,117440512~12582912,138412032~16777216,163577856~12582912,180355072~8388608,192937984~4194304,205520896~12582912,230686720~16777216,251658240~20971520,289406976~8388608,322961408~4194304,335544320~12582912,360710144~4194304,369098752~37748736] 2024-08-17T17:45:52.173 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~16777216,33554432~8388608,46137344~12582912,67108864~46137344,117440512~16777216,138412032~20971520,163577856~37748736,205520896~16777216,230686720~46137344,289406976~8388608,322961408~8388608,335544320~16777216,360710144~4194304,369098752~37748736] 2024-08-17T17:45:52.173 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 10 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~4194304,117440512~12582912,138412032~16777216,163577856~12582912,180355072~8388608,192937984~4194304,205520896~12582912,230686720~16777216,251658240~20971520,289406976~8388608,322961408~4194304,335544320~16777216,356515840~50331648] 2024-08-17T17:45:52.173 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~4194304,117440512~12582912,138412032~16777216,163577856~12582912,180355072~8388608,192937984~4194304,205520896~12582912,230686720~16777216,251658240~20971520,289406976~8388608,322961408~4194304,335544320~16777216,356515840~50331648] 2024-08-17T17:45:52.177 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~50331648,117440512~16777216,138412032~20971520,163577856~37748736,205520896~16777216,230686720~46137344,289406976~8388608,322961408~8388608,335544320~16777216,356515840~50331648] 2024-08-17T17:45:52.177 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 11 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~20971520,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~16777216,251658240~20971520,289406976~16777216,322961408~4194304,331350016~20971520,356515840~50331648] 2024-08-17T17:45:52.177 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~41943040,109051904~20971520,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~16777216,251658240~20971520,289406976~16777216,322961408~4194304,331350016~20971520,356515840~50331648] 2024-08-17T17:45:52.182 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,289406976~16777216,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.182 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 12 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~16777216,251658240~20971520,289406976~16777216,322961408~4194304,331350016~20971520,356515840~50331648] 2024-08-17T17:45:52.182 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~16777216,251658240~20971520,289406976~16777216,322961408~4194304,331350016~20971520,356515840~50331648] 2024-08-17T17:45:52.187 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,289406976~16777216,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.187 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 13 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~41943040,289406976~16777216,322961408~83886080] 2024-08-17T17:45:52.187 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~41943040,289406976~16777216,322961408~83886080] 2024-08-17T17:45:52.191 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~25165824,33554432~8388608,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,289406976~16777216,322961408~88080384] 2024-08-17T17:45:52.191 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 14 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~41943040,281018368~4194304,289406976~16777216,322961408~88080384] 2024-08-17T17:45:52.191 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~41943040,281018368~4194304,289406976~16777216,322961408~88080384] 2024-08-17T17:45:52.196 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~37748736,46137344~12582912,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,281018368~25165824,322961408~88080384] 2024-08-17T17:45:52.196 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 15 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~41943040,281018368~4194304,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.196 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~67108864,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~41943040,281018368~4194304,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.201 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~54525952,62914560~71303168,138412032~20971520,163577856~58720256,230686720~46137344,281018368~25165824,322961408~88080384] 2024-08-17T17:45:52.201 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 16 diff [4194304~37748736,46137344~16777216,67108864~88080384,163577856~25165824,192937984~25165824,230686720~41943040,276824064~8388608,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.201 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~88080384,163577856~25165824,192937984~25165824,230686720~41943040,276824064~8388608,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.206 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~155189248,163577856~58720256,230686720~75497472,322961408~88080384] 2024-08-17T17:45:52.206 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 17 diff [4194304~58720256,67108864~88080384,159383552~4194304,167772160~20971520,192937984~29360128,230686720~54525952,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.206 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~58720256,67108864~88080384,159383552~4194304,167772160~20971520,192937984~29360128,230686720~54525952,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.211 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,230686720~75497472,322961408~88080384] 2024-08-17T17:45:52.211 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 18 diff [4194304~67108864,75497472~58720256,138412032~16777216,159383552~4194304,167772160~20971520,192937984~29360128,226492416~58720256,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.211 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~58720256,138412032~16777216,159383552~4194304,167772160~20971520,192937984~29360128,226492416~58720256,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.218 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:52.218 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to 19 diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.218 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.225 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:52.226 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 5 diff [12582912~4194304,33554432~4194304,67108864~16777216,167772160~8388608,209715200~4194304,251658240~8388608,264241152~8388608,390070272~4194304] 2024-08-17T17:45:52.226 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~4194304,33554432~4194304,67108864~16777216,167772160~8388608,209715200~4194304,251658240~8388608,264241152~8388608,390070272~4194304] 2024-08-17T17:45:52.226 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~4194304,33554432~4194304,67108864~16777216,167772160~12582912,209715200~8388608,251658240~8388608,264241152~12582912,390070272~4194304] 2024-08-17T17:45:52.226 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 6 diff [12582912~12582912,33554432~8388608,46137344~4194304,67108864~20971520,121634816~8388608,167772160~8388608,180355072~4194304,205520896~8388608,243269632~16777216,264241152~8388608,373293056~8388608,390070272~4194304] 2024-08-17T17:45:52.226 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~12582912,33554432~8388608,46137344~4194304,67108864~20971520,121634816~8388608,167772160~8388608,180355072~4194304,205520896~8388608,243269632~16777216,264241152~8388608,373293056~8388608,390070272~4194304] 2024-08-17T17:45:52.227 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~12582912,33554432~8388608,46137344~8388608,67108864~20971520,121634816~12582912,167772160~20971520,205520896~12582912,243269632~16777216,264241152~12582912,373293056~8388608,390070272~4194304] 2024-08-17T17:45:52.227 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 7 diff [12582912~12582912,33554432~8388608,46137344~4194304,67108864~20971520,92274688~4194304,121634816~8388608,150994944~4194304,167772160~8388608,180355072~8388608,192937984~4194304,205520896~8388608,243269632~29360128,322961408~4194304,360710144~4194304,373293056~8388608,390070272~4194304,398458880~8388608] 2024-08-17T17:45:52.227 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~12582912,33554432~8388608,46137344~4194304,67108864~20971520,92274688~4194304,121634816~8388608,150994944~4194304,167772160~8388608,180355072~8388608,192937984~4194304,205520896~8388608,243269632~29360128,322961408~4194304,360710144~4194304,373293056~8388608,390070272~4194304,398458880~8388608] 2024-08-17T17:45:52.228 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~12582912,33554432~8388608,46137344~8388608,67108864~20971520,92274688~8388608,121634816~12582912,150994944~8388608,167772160~33554432,205520896~12582912,243269632~33554432,322961408~8388608,360710144~4194304,373293056~8388608,390070272~4194304,398458880~8388608] 2024-08-17T17:45:52.228 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 8 diff [12582912~12582912,33554432~8388608,46137344~4194304,67108864~20971520,92274688~4194304,121634816~8388608,142606336~12582912,167772160~8388608,180355072~8388608,192937984~4194304,205520896~8388608,230686720~8388608,243269632~29360128,289406976~8388608,322961408~4194304,335544320~8388608,360710144~4194304,373293056~33554432] 2024-08-17T17:45:52.229 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~12582912,33554432~8388608,46137344~4194304,67108864~20971520,92274688~4194304,121634816~8388608,142606336~12582912,167772160~8388608,180355072~8388608,192937984~4194304,205520896~8388608,230686720~8388608,243269632~29360128,289406976~8388608,322961408~4194304,335544320~8388608,360710144~4194304,373293056~33554432] 2024-08-17T17:45:52.231 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~12582912,33554432~8388608,46137344~8388608,67108864~20971520,92274688~8388608,121634816~12582912,142606336~16777216,167772160~33554432,205520896~12582912,230686720~46137344,289406976~8388608,322961408~8388608,335544320~12582912,360710144~4194304,373293056~33554432] 2024-08-17T17:45:52.231 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 9 diff [12582912~12582912,33554432~8388608,46137344~4194304,54525952~4194304,67108864~20971520,92274688~12582912,117440512~12582912,138412032~16777216,163577856~12582912,180355072~8388608,192937984~4194304,205520896~12582912,230686720~8388608,243269632~29360128,289406976~8388608,322961408~4194304,335544320~8388608,360710144~4194304,369098752~37748736] 2024-08-17T17:45:52.231 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [12582912~12582912,33554432~8388608,46137344~4194304,54525952~4194304,67108864~20971520,92274688~12582912,117440512~12582912,138412032~16777216,163577856~12582912,180355072~8388608,192937984~4194304,205520896~12582912,230686720~8388608,243269632~29360128,289406976~8388608,322961408~4194304,335544320~8388608,360710144~4194304,369098752~37748736] 2024-08-17T17:45:52.233 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [12582912~12582912,33554432~8388608,46137344~12582912,67108864~20971520,92274688~16777216,117440512~16777216,138412032~20971520,163577856~37748736,205520896~16777216,230686720~46137344,289406976~8388608,322961408~8388608,335544320~12582912,360710144~4194304,369098752~37748736] 2024-08-17T17:45:52.234 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 10 diff [4194304~20971520,33554432~8388608,46137344~12582912,62914560~41943040,117440512~12582912,138412032~16777216,163577856~12582912,180355072~8388608,192937984~4194304,205520896~12582912,230686720~8388608,243269632~29360128,289406976~8388608,322961408~4194304,335544320~16777216,356515840~12582912,373293056~33554432] 2024-08-17T17:45:52.234 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~20971520,33554432~8388608,46137344~12582912,62914560~41943040,117440512~12582912,138412032~16777216,163577856~12582912,180355072~8388608,192937984~4194304,205520896~12582912,230686720~8388608,243269632~29360128,289406976~8388608,322961408~4194304,335544320~16777216,356515840~12582912,373293056~33554432] 2024-08-17T17:45:52.236 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~20971520,33554432~8388608,46137344~12582912,62914560~46137344,117440512~16777216,138412032~20971520,163577856~37748736,205520896~16777216,230686720~46137344,289406976~8388608,322961408~8388608,335544320~16777216,356515840~50331648] 2024-08-17T17:45:52.236 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 11 diff [4194304~20971520,33554432~8388608,46137344~12582912,62914560~41943040,113246208~16777216,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~8388608,243269632~29360128,289406976~16777216,322961408~4194304,331350016~20971520,356515840~12582912,373293056~33554432] 2024-08-17T17:45:52.236 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~20971520,33554432~8388608,46137344~12582912,62914560~41943040,113246208~16777216,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~8388608,243269632~29360128,289406976~16777216,322961408~4194304,331350016~20971520,356515840~12582912,373293056~33554432] 2024-08-17T17:45:52.240 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~20971520,33554432~8388608,46137344~12582912,62914560~46137344,113246208~20971520,138412032~20971520,163577856~58720256,230686720~46137344,289406976~16777216,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.240 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 12 diff [4194304~20971520,33554432~8388608,46137344~12582912,62914560~46137344,113246208~16777216,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~8388608,243269632~29360128,289406976~16777216,322961408~4194304,331350016~20971520,356515840~12582912,373293056~33554432] 2024-08-17T17:45:52.240 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~20971520,33554432~8388608,46137344~12582912,62914560~46137344,113246208~16777216,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~8388608,243269632~29360128,289406976~16777216,322961408~4194304,331350016~20971520,356515840~12582912,373293056~33554432] 2024-08-17T17:45:52.243 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~20971520,33554432~8388608,46137344~12582912,62914560~46137344,113246208~20971520,138412032~20971520,163577856~58720256,230686720~46137344,289406976~16777216,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.243 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 13 diff [4194304~20971520,33554432~8388608,46137344~12582912,62914560~46137344,113246208~16777216,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~8388608,243269632~29360128,289406976~16777216,322961408~46137344,373293056~33554432] 2024-08-17T17:45:52.244 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~20971520,33554432~8388608,46137344~12582912,62914560~46137344,113246208~16777216,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~8388608,243269632~29360128,289406976~16777216,322961408~46137344,373293056~33554432] 2024-08-17T17:45:52.247 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~20971520,33554432~8388608,46137344~12582912,62914560~46137344,113246208~20971520,138412032~20971520,163577856~58720256,230686720~46137344,289406976~16777216,322961408~88080384] 2024-08-17T17:45:52.247 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 14 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~46137344,113246208~16777216,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~8388608,243269632~29360128,281018368~4194304,289406976~16777216,322961408~46137344,373293056~37748736] 2024-08-17T17:45:52.247 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~46137344,113246208~16777216,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~8388608,243269632~29360128,281018368~4194304,289406976~16777216,322961408~46137344,373293056~37748736] 2024-08-17T17:45:52.251 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~37748736,46137344~12582912,62914560~46137344,113246208~20971520,138412032~20971520,163577856~58720256,230686720~46137344,281018368~25165824,322961408~88080384] 2024-08-17T17:45:52.252 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 15 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~46137344,113246208~16777216,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~8388608,243269632~29360128,281018368~4194304,289406976~12582912,322961408~29360128,356515840~12582912,373293056~37748736] 2024-08-17T17:45:52.252 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~46137344,113246208~16777216,138412032~16777216,163577856~12582912,180355072~8388608,192937984~25165824,230686720~8388608,243269632~29360128,281018368~4194304,289406976~12582912,322961408~29360128,356515840~12582912,373293056~37748736] 2024-08-17T17:45:52.256 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~54525952,62914560~46137344,113246208~20971520,138412032~20971520,163577856~58720256,230686720~46137344,281018368~25165824,322961408~88080384] 2024-08-17T17:45:52.256 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 16 diff [4194304~37748736,46137344~16777216,67108864~41943040,113246208~41943040,163577856~25165824,192937984~25165824,230686720~8388608,243269632~29360128,276824064~8388608,289406976~12582912,322961408~29360128,356515840~12582912,373293056~37748736] 2024-08-17T17:45:52.256 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~41943040,113246208~41943040,163577856~25165824,192937984~25165824,230686720~8388608,243269632~29360128,276824064~8388608,289406976~12582912,322961408~29360128,356515840~12582912,373293056~37748736] 2024-08-17T17:45:52.261 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~46137344,163577856~58720256,230686720~75497472,322961408~88080384] 2024-08-17T17:45:52.261 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 17 diff [4194304~58720256,67108864~41943040,113246208~41943040,159383552~4194304,167772160~20971520,192937984~29360128,230686720~8388608,243269632~41943040,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.261 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~58720256,67108864~41943040,113246208~41943040,159383552~4194304,167772160~20971520,192937984~29360128,230686720~8388608,243269632~41943040,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.266 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,230686720~75497472,322961408~88080384] 2024-08-17T17:45:52.266 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 18 diff [4194304~67108864,75497472~33554432,113246208~20971520,138412032~16777216,159383552~4194304,167772160~20971520,192937984~29360128,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.266 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~20971520,138412032~16777216,159383552~4194304,167772160~20971520,192937984~29360128,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.272 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:45:52.272 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to 19 diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.272 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.278 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:45:52.278 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 6 diff [16777216~8388608,33554432~4194304,46137344~4194304,79691776~8388608,121634816~8388608,180355072~4194304,205520896~4194304,243269632~8388608,373293056~8388608] 2024-08-17T17:45:52.278 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [16777216~8388608,33554432~4194304,46137344~4194304,79691776~8388608,121634816~8388608,180355072~4194304,205520896~4194304,243269632~8388608,373293056~8388608] 2024-08-17T17:45:52.279 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [16777216~8388608,33554432~8388608,46137344~8388608,79691776~8388608,121634816~12582912,180355072~8388608,205520896~4194304,243269632~8388608,373293056~8388608] 2024-08-17T17:45:52.279 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 7 diff [16777216~8388608,33554432~4194304,46137344~4194304,79691776~8388608,92274688~4194304,121634816~8388608,150994944~4194304,180355072~8388608,192937984~4194304,205520896~4194304,243269632~8388608,255852544~8388608,322961408~4194304,360710144~4194304,373293056~8388608,398458880~8388608] 2024-08-17T17:45:52.279 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [16777216~8388608,33554432~4194304,46137344~4194304,79691776~8388608,92274688~4194304,121634816~8388608,150994944~4194304,180355072~8388608,192937984~4194304,205520896~4194304,243269632~8388608,255852544~8388608,322961408~4194304,360710144~4194304,373293056~8388608,398458880~8388608] 2024-08-17T17:45:52.280 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [16777216~8388608,33554432~8388608,46137344~8388608,79691776~8388608,92274688~8388608,121634816~12582912,150994944~8388608,180355072~20971520,205520896~4194304,243269632~8388608,255852544~8388608,322961408~8388608,360710144~4194304,373293056~8388608,398458880~8388608] 2024-08-17T17:45:52.280 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 8 diff [16777216~8388608,33554432~4194304,46137344~4194304,79691776~8388608,92274688~4194304,121634816~8388608,142606336~12582912,180355072~8388608,192937984~4194304,205520896~4194304,230686720~8388608,243269632~8388608,255852544~8388608,289406976~8388608,322961408~4194304,335544320~8388608,360710144~4194304,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.280 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [16777216~8388608,33554432~4194304,46137344~4194304,79691776~8388608,92274688~4194304,121634816~8388608,142606336~12582912,180355072~8388608,192937984~4194304,205520896~4194304,230686720~8388608,243269632~8388608,255852544~8388608,289406976~8388608,322961408~4194304,335544320~8388608,360710144~4194304,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.282 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [16777216~8388608,33554432~8388608,46137344~8388608,79691776~8388608,92274688~8388608,121634816~12582912,142606336~16777216,180355072~20971520,205520896~4194304,230686720~20971520,255852544~8388608,289406976~8388608,322961408~8388608,335544320~12582912,360710144~4194304,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.282 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 9 diff [16777216~8388608,33554432~4194304,46137344~4194304,54525952~4194304,79691776~8388608,92274688~12582912,117440512~12582912,138412032~16777216,163577856~4194304,180355072~8388608,192937984~4194304,205520896~4194304,213909504~4194304,230686720~8388608,243269632~8388608,255852544~12582912,289406976~8388608,322961408~4194304,335544320~8388608,360710144~4194304,369098752~20971520,394264576~12582912] 2024-08-17T17:45:52.283 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [16777216~8388608,33554432~4194304,46137344~4194304,54525952~4194304,79691776~8388608,92274688~12582912,117440512~12582912,138412032~16777216,163577856~4194304,180355072~8388608,192937984~4194304,205520896~4194304,213909504~4194304,230686720~8388608,243269632~8388608,255852544~12582912,289406976~8388608,322961408~4194304,335544320~8388608,360710144~4194304,369098752~20971520,394264576~12582912] 2024-08-17T17:45:52.284 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [16777216~8388608,33554432~8388608,46137344~12582912,79691776~8388608,92274688~16777216,117440512~16777216,138412032~20971520,163577856~4194304,180355072~20971520,205520896~4194304,213909504~8388608,230686720~20971520,255852544~16777216,289406976~8388608,322961408~8388608,335544320~12582912,360710144~4194304,369098752~20971520,394264576~12582912] 2024-08-17T17:45:52.285 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 10 diff [4194304~8388608,16777216~8388608,33554432~4194304,46137344~12582912,62914560~4194304,75497472~29360128,117440512~12582912,138412032~16777216,163577856~4194304,180355072~8388608,192937984~4194304,205520896~4194304,213909504~4194304,230686720~8388608,243269632~8388608,255852544~12582912,289406976~8388608,322961408~4194304,335544320~16777216,356515840~12582912,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.285 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~8388608,33554432~4194304,46137344~12582912,62914560~4194304,75497472~29360128,117440512~12582912,138412032~16777216,163577856~4194304,180355072~8388608,192937984~4194304,205520896~4194304,213909504~4194304,230686720~8388608,243269632~8388608,255852544~12582912,289406976~8388608,322961408~4194304,335544320~16777216,356515840~12582912,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.287 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~8388608,33554432~8388608,46137344~12582912,62914560~4194304,75497472~33554432,117440512~16777216,138412032~20971520,163577856~4194304,180355072~20971520,205520896~4194304,213909504~8388608,230686720~20971520,255852544~16777216,289406976~8388608,322961408~8388608,335544320~16777216,356515840~33554432,394264576~12582912] 2024-08-17T17:45:52.287 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 11 diff [4194304~8388608,16777216~8388608,33554432~4194304,46137344~12582912,62914560~4194304,75497472~29360128,113246208~16777216,138412032~16777216,163577856~4194304,171966464~16777216,192937984~16777216,213909504~4194304,230686720~8388608,243269632~8388608,255852544~16777216,289406976~16777216,322961408~4194304,331350016~20971520,356515840~12582912,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.287 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~8388608,33554432~4194304,46137344~12582912,62914560~4194304,75497472~29360128,113246208~16777216,138412032~16777216,163577856~4194304,171966464~16777216,192937984~16777216,213909504~4194304,230686720~8388608,243269632~8388608,255852544~16777216,289406976~16777216,322961408~4194304,331350016~20971520,356515840~12582912,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.290 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~8388608,33554432~8388608,46137344~12582912,62914560~4194304,75497472~33554432,113246208~20971520,138412032~20971520,163577856~4194304,171966464~37748736,213909504~8388608,230686720~20971520,255852544~16777216,289406976~16777216,322961408~29360128,356515840~33554432,394264576~16777216] 2024-08-17T17:45:52.290 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 12 diff [4194304~8388608,16777216~8388608,33554432~4194304,46137344~12582912,62914560~4194304,75497472~33554432,113246208~16777216,138412032~16777216,163577856~4194304,171966464~16777216,192937984~25165824,230686720~8388608,243269632~8388608,255852544~16777216,289406976~16777216,322961408~4194304,331350016~20971520,356515840~12582912,373293056~33554432] 2024-08-17T17:45:52.290 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~8388608,33554432~4194304,46137344~12582912,62914560~4194304,75497472~33554432,113246208~16777216,138412032~16777216,163577856~4194304,171966464~16777216,192937984~25165824,230686720~8388608,243269632~8388608,255852544~16777216,289406976~16777216,322961408~4194304,331350016~20971520,356515840~12582912,373293056~33554432] 2024-08-17T17:45:52.293 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~8388608,33554432~8388608,46137344~12582912,62914560~4194304,75497472~33554432,113246208~20971520,138412032~20971520,163577856~4194304,171966464~50331648,230686720~20971520,255852544~16777216,289406976~16777216,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.293 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 13 diff [4194304~8388608,16777216~8388608,33554432~4194304,46137344~12582912,62914560~46137344,113246208~16777216,138412032~16777216,163577856~4194304,171966464~16777216,192937984~25165824,230686720~8388608,243269632~8388608,255852544~20971520,289406976~16777216,322961408~46137344,373293056~33554432] 2024-08-17T17:45:52.294 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~8388608,33554432~4194304,46137344~12582912,62914560~46137344,113246208~16777216,138412032~16777216,163577856~4194304,171966464~16777216,192937984~25165824,230686720~8388608,243269632~8388608,255852544~20971520,289406976~16777216,322961408~46137344,373293056~33554432] 2024-08-17T17:45:52.297 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~8388608,33554432~8388608,46137344~12582912,62914560~46137344,113246208~20971520,138412032~20971520,163577856~4194304,171966464~50331648,230686720~20971520,255852544~20971520,289406976~16777216,322961408~88080384] 2024-08-17T17:45:52.297 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 14 diff [4194304~8388608,16777216~12582912,33554432~4194304,46137344~12582912,62914560~46137344,113246208~16777216,138412032~16777216,163577856~4194304,171966464~16777216,192937984~25165824,230686720~8388608,243269632~8388608,255852544~20971520,281018368~4194304,289406976~16777216,322961408~46137344,373293056~37748736] 2024-08-17T17:45:52.297 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~12582912,33554432~4194304,46137344~12582912,62914560~46137344,113246208~16777216,138412032~16777216,163577856~4194304,171966464~16777216,192937984~25165824,230686720~8388608,243269632~8388608,255852544~20971520,281018368~4194304,289406976~16777216,322961408~46137344,373293056~37748736] 2024-08-17T17:45:52.301 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~25165824,46137344~12582912,62914560~46137344,113246208~20971520,138412032~20971520,163577856~4194304,171966464~50331648,230686720~20971520,255852544~20971520,281018368~25165824,322961408~88080384] 2024-08-17T17:45:52.301 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 15 diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~46137344,113246208~16777216,138412032~16777216,163577856~4194304,171966464~16777216,192937984~25165824,230686720~8388608,243269632~8388608,255852544~20971520,281018368~4194304,289406976~12582912,322961408~29360128,356515840~12582912,373293056~37748736] 2024-08-17T17:45:52.301 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,46137344~12582912,62914560~46137344,113246208~16777216,138412032~16777216,163577856~4194304,171966464~16777216,192937984~25165824,230686720~8388608,243269632~8388608,255852544~20971520,281018368~4194304,289406976~12582912,322961408~29360128,356515840~12582912,373293056~37748736] 2024-08-17T17:45:52.305 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~54525952,62914560~46137344,113246208~20971520,138412032~20971520,163577856~4194304,171966464~50331648,230686720~20971520,255852544~20971520,281018368~25165824,322961408~88080384] 2024-08-17T17:45:52.305 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 16 diff [4194304~37748736,46137344~16777216,67108864~41943040,113246208~41943040,163577856~25165824,192937984~25165824,230686720~8388608,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~12582912,373293056~37748736] 2024-08-17T17:45:52.305 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~37748736,46137344~16777216,67108864~41943040,113246208~41943040,163577856~25165824,192937984~25165824,230686720~8388608,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~12582912,373293056~37748736] 2024-08-17T17:45:52.310 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~46137344,163577856~58720256,230686720~20971520,255852544~50331648,322961408~88080384] 2024-08-17T17:45:52.310 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 17 diff [4194304~58720256,67108864~41943040,113246208~41943040,159383552~4194304,167772160~20971520,192937984~29360128,230686720~8388608,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.310 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~58720256,67108864~41943040,113246208~41943040,159383552~4194304,167772160~20971520,192937984~29360128,230686720~8388608,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.315 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,230686720~20971520,255852544~50331648,322961408~88080384] 2024-08-17T17:45:52.315 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 18 diff [4194304~67108864,75497472~33554432,113246208~20971520,138412032~16777216,159383552~4194304,167772160~20971520,192937984~29360128,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.315 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~20971520,138412032~16777216,159383552~4194304,167772160~20971520,192937984~29360128,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.319 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:52.319 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to 19 diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.320 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.326 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:52.326 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 7 diff [92274688~4194304,150994944~4194304,184549376~12582912,255852544~8388608,322961408~4194304,360710144~4194304,398458880~8388608] 2024-08-17T17:45:52.326 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [92274688~4194304,150994944~4194304,184549376~12582912,255852544~8388608,322961408~4194304,360710144~4194304,398458880~8388608] 2024-08-17T17:45:52.327 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [92274688~8388608,150994944~8388608,184549376~16777216,255852544~8388608,322961408~8388608,360710144~4194304,398458880~8388608] 2024-08-17T17:45:52.327 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 8 diff [79691776~8388608,92274688~4194304,142606336~12582912,184549376~12582912,230686720~8388608,243269632~4194304,255852544~8388608,289406976~8388608,322961408~4194304,335544320~8388608,360710144~4194304,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.327 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [79691776~8388608,92274688~4194304,142606336~12582912,184549376~12582912,230686720~8388608,243269632~4194304,255852544~8388608,289406976~8388608,322961408~4194304,335544320~8388608,360710144~4194304,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.328 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [79691776~8388608,92274688~8388608,142606336~16777216,184549376~16777216,230686720~20971520,255852544~8388608,289406976~8388608,322961408~8388608,335544320~12582912,360710144~4194304,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.328 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 9 diff [54525952~4194304,79691776~8388608,92274688~12582912,117440512~4194304,138412032~16777216,163577856~4194304,184549376~12582912,213909504~4194304,230686720~8388608,243269632~4194304,255852544~12582912,289406976~8388608,322961408~4194304,335544320~8388608,360710144~4194304,369098752~20971520,394264576~12582912] 2024-08-17T17:45:52.328 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [54525952~4194304,79691776~8388608,92274688~12582912,117440512~4194304,138412032~16777216,163577856~4194304,184549376~12582912,213909504~4194304,230686720~8388608,243269632~4194304,255852544~12582912,289406976~8388608,322961408~4194304,335544320~8388608,360710144~4194304,369098752~20971520,394264576~12582912] 2024-08-17T17:45:52.330 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [54525952~4194304,79691776~8388608,92274688~16777216,117440512~4194304,138412032~20971520,163577856~4194304,184549376~16777216,213909504~8388608,230686720~20971520,255852544~16777216,289406976~8388608,322961408~8388608,335544320~12582912,360710144~4194304,369098752~20971520,394264576~12582912] 2024-08-17T17:45:52.330 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 10 diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~29360128,117440512~4194304,138412032~16777216,163577856~4194304,184549376~12582912,213909504~4194304,230686720~8388608,243269632~4194304,255852544~12582912,289406976~8388608,322961408~4194304,335544320~16777216,356515840~12582912,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.330 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~29360128,117440512~4194304,138412032~16777216,163577856~4194304,184549376~12582912,213909504~4194304,230686720~8388608,243269632~4194304,255852544~12582912,289406976~8388608,322961408~4194304,335544320~16777216,356515840~12582912,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.333 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,117440512~8388608,138412032~20971520,163577856~4194304,184549376~16777216,213909504~8388608,230686720~20971520,255852544~16777216,289406976~8388608,322961408~8388608,335544320~16777216,356515840~33554432,394264576~12582912] 2024-08-17T17:45:52.333 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 11 diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~29360128,113246208~8388608,138412032~16777216,163577856~4194304,171966464~8388608,184549376~25165824,213909504~4194304,230686720~8388608,243269632~4194304,255852544~16777216,289406976~16777216,322961408~4194304,331350016~20971520,356515840~12582912,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.333 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~29360128,113246208~8388608,138412032~16777216,163577856~4194304,171966464~8388608,184549376~25165824,213909504~4194304,230686720~8388608,243269632~4194304,255852544~16777216,289406976~16777216,322961408~4194304,331350016~20971520,356515840~12582912,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.335 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,113246208~12582912,138412032~20971520,163577856~4194304,171966464~8388608,184549376~25165824,213909504~8388608,230686720~20971520,255852544~16777216,289406976~16777216,322961408~29360128,356515840~33554432,394264576~16777216] 2024-08-17T17:45:52.335 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 12 diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,113246208~8388608,138412032~16777216,163577856~4194304,171966464~8388608,184549376~33554432,230686720~8388608,243269632~4194304,255852544~16777216,289406976~16777216,322961408~4194304,331350016~20971520,356515840~12582912,373293056~33554432] 2024-08-17T17:45:52.335 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,113246208~8388608,138412032~16777216,163577856~4194304,171966464~8388608,184549376~33554432,230686720~8388608,243269632~4194304,255852544~16777216,289406976~16777216,322961408~4194304,331350016~20971520,356515840~12582912,373293056~33554432] 2024-08-17T17:45:52.338 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,113246208~12582912,138412032~20971520,163577856~4194304,171966464~8388608,184549376~37748736,230686720~20971520,255852544~16777216,289406976~16777216,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.338 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 13 diff [4194304~8388608,16777216~8388608,50331648~8388608,62914560~46137344,113246208~8388608,138412032~16777216,163577856~4194304,171966464~8388608,184549376~33554432,230686720~8388608,243269632~8388608,255852544~20971520,289406976~16777216,322961408~46137344,373293056~33554432] 2024-08-17T17:45:52.338 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~8388608,50331648~8388608,62914560~46137344,113246208~8388608,138412032~16777216,163577856~4194304,171966464~8388608,184549376~33554432,230686720~8388608,243269632~8388608,255852544~20971520,289406976~16777216,322961408~46137344,373293056~33554432] 2024-08-17T17:45:52.341 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~8388608,50331648~8388608,62914560~46137344,113246208~12582912,138412032~20971520,163577856~4194304,171966464~8388608,184549376~37748736,230686720~20971520,255852544~20971520,289406976~16777216,322961408~88080384] 2024-08-17T17:45:52.341 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 14 diff [4194304~8388608,16777216~12582912,50331648~8388608,62914560~46137344,113246208~8388608,138412032~16777216,163577856~4194304,171966464~8388608,184549376~33554432,230686720~8388608,243269632~8388608,255852544~20971520,281018368~4194304,289406976~16777216,322961408~46137344,373293056~37748736] 2024-08-17T17:45:52.341 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~12582912,50331648~8388608,62914560~46137344,113246208~8388608,138412032~16777216,163577856~4194304,171966464~8388608,184549376~33554432,230686720~8388608,243269632~8388608,255852544~20971520,281018368~4194304,289406976~16777216,322961408~46137344,373293056~37748736] 2024-08-17T17:45:52.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:51 smithi179 ceph-mon[93519]: pgmap v1473: 105 pgs: 105 active+clean; 1.1 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 237 KiB/s rd, 48 MiB/s wr, 368 op/s 2024-08-17T17:45:52.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:51 smithi179 ceph-mon[93519]: osdmap e1137: 8 total, 8 up, 8 in 2024-08-17T17:45:52.344 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~16777216,50331648~8388608,62914560~46137344,113246208~12582912,138412032~20971520,163577856~4194304,171966464~8388608,184549376~37748736,230686720~20971520,255852544~20971520,281018368~25165824,322961408~88080384] 2024-08-17T17:45:52.344 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 15 diff [4194304~25165824,33554432~8388608,50331648~8388608,62914560~46137344,113246208~8388608,138412032~16777216,163577856~4194304,171966464~8388608,184549376~33554432,230686720~8388608,243269632~8388608,255852544~20971520,281018368~4194304,289406976~12582912,322961408~29360128,356515840~12582912,373293056~37748736] 2024-08-17T17:45:52.344 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~8388608,62914560~46137344,113246208~8388608,138412032~16777216,163577856~4194304,171966464~8388608,184549376~33554432,230686720~8388608,243269632~8388608,255852544~20971520,281018368~4194304,289406976~12582912,322961408~29360128,356515840~12582912,373293056~37748736] 2024-08-17T17:45:52.348 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,50331648~8388608,62914560~46137344,113246208~12582912,138412032~20971520,163577856~4194304,171966464~8388608,184549376~37748736,230686720~20971520,255852544~20971520,281018368~25165824,322961408~88080384] 2024-08-17T17:45:52.348 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 16 diff [4194304~37748736,50331648~12582912,67108864~41943040,113246208~12582912,130023424~25165824,163577856~54525952,230686720~8388608,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~12582912,373293056~37748736] 2024-08-17T17:45:52.348 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~37748736,50331648~12582912,67108864~41943040,113246208~12582912,130023424~25165824,163577856~54525952,230686720~8388608,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~12582912,373293056~37748736] 2024-08-17T17:45:52.352 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,50331648~58720256,113246208~12582912,130023424~29360128,163577856~58720256,230686720~20971520,255852544~50331648,322961408~88080384] 2024-08-17T17:45:52.352 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 17 diff [4194304~58720256,67108864~41943040,113246208~12582912,130023424~25165824,159383552~4194304,167772160~54525952,230686720~8388608,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.352 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~58720256,67108864~41943040,113246208~12582912,130023424~25165824,159383552~4194304,167772160~54525952,230686720~8388608,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.357 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~12582912,130023424~92274688,230686720~20971520,255852544~50331648,322961408~88080384] 2024-08-17T17:45:52.357 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 18 diff [4194304~67108864,75497472~33554432,113246208~41943040,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.357 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~41943040,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~54525952] 2024-08-17T17:45:52.362 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:52.362 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to 19 diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.362 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.367 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:52.367 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 8 diff [79691776~8388608,142606336~8388608,230686720~8388608,243269632~4194304,289406976~8388608,335544320~8388608,373293056~16777216,394264576~8388608] 2024-08-17T17:45:52.368 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [79691776~8388608,142606336~8388608,230686720~8388608,243269632~4194304,289406976~8388608,335544320~8388608,373293056~16777216,394264576~8388608] 2024-08-17T17:45:52.368 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [79691776~8388608,142606336~8388608,230686720~20971520,289406976~8388608,335544320~12582912,373293056~16777216,394264576~8388608] 2024-08-17T17:45:52.368 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 9 diff [54525952~4194304,79691776~8388608,96468992~8388608,117440512~4194304,138412032~12582912,163577856~4194304,213909504~4194304,230686720~8388608,243269632~4194304,260046848~8388608,289406976~8388608,335544320~8388608,369098752~20971520,394264576~8388608] 2024-08-17T17:45:52.369 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [54525952~4194304,79691776~8388608,96468992~8388608,117440512~4194304,138412032~12582912,163577856~4194304,213909504~4194304,230686720~8388608,243269632~4194304,260046848~8388608,289406976~8388608,335544320~8388608,369098752~20971520,394264576~8388608] 2024-08-17T17:45:52.370 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [54525952~4194304,79691776~8388608,96468992~12582912,117440512~4194304,138412032~12582912,163577856~4194304,213909504~8388608,230686720~20971520,260046848~12582912,289406976~8388608,335544320~12582912,369098752~20971520,394264576~8388608] 2024-08-17T17:45:52.370 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 10 diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~16777216,96468992~8388608,117440512~4194304,138412032~12582912,163577856~4194304,213909504~4194304,230686720~8388608,243269632~4194304,260046848~8388608,289406976~8388608,335544320~16777216,356515840~12582912,373293056~16777216,394264576~8388608] 2024-08-17T17:45:52.370 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~16777216,96468992~8388608,117440512~4194304,138412032~12582912,163577856~4194304,213909504~4194304,230686720~8388608,243269632~4194304,260046848~8388608,289406976~8388608,335544320~16777216,356515840~12582912,373293056~16777216,394264576~8388608] 2024-08-17T17:45:52.372 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,117440512~8388608,138412032~12582912,163577856~4194304,213909504~8388608,230686720~20971520,260046848~12582912,289406976~8388608,335544320~16777216,356515840~33554432,394264576~8388608] 2024-08-17T17:45:52.372 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 11 diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~29360128,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,197132288~12582912,213909504~4194304,230686720~8388608,243269632~4194304,260046848~12582912,289406976~16777216,331350016~20971520,356515840~12582912,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.372 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~29360128,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,197132288~12582912,213909504~4194304,230686720~8388608,243269632~4194304,260046848~12582912,289406976~16777216,331350016~20971520,356515840~12582912,373293056~16777216,394264576~12582912] 2024-08-17T17:45:52.374 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,113246208~12582912,138412032~12582912,163577856~4194304,171966464~8388608,197132288~12582912,213909504~8388608,230686720~20971520,260046848~12582912,289406976~16777216,331350016~20971520,356515840~33554432,394264576~16777216] 2024-08-17T17:45:52.375 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 12 diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,197132288~20971520,230686720~8388608,243269632~4194304,260046848~12582912,289406976~16777216,331350016~20971520,356515840~12582912,373293056~33554432] 2024-08-17T17:45:52.375 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,197132288~20971520,230686720~8388608,243269632~4194304,260046848~12582912,289406976~16777216,331350016~20971520,356515840~12582912,373293056~33554432] 2024-08-17T17:45:52.377 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,113246208~12582912,138412032~12582912,163577856~4194304,171966464~8388608,197132288~25165824,230686720~20971520,260046848~12582912,289406976~16777216,331350016~20971520,356515840~54525952] 2024-08-17T17:45:52.377 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 13 diff [4194304~8388608,16777216~8388608,50331648~8388608,62914560~46137344,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,197132288~20971520,230686720~8388608,243269632~8388608,260046848~16777216,289406976~16777216,327155712~41943040,373293056~33554432] 2024-08-17T17:45:52.377 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~8388608,50331648~8388608,62914560~46137344,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,197132288~20971520,230686720~8388608,243269632~8388608,260046848~16777216,289406976~16777216,327155712~41943040,373293056~33554432] 2024-08-17T17:45:52.380 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~8388608,50331648~8388608,62914560~46137344,113246208~12582912,138412032~12582912,163577856~4194304,171966464~8388608,197132288~25165824,230686720~20971520,260046848~16777216,289406976~16777216,327155712~83886080] 2024-08-17T17:45:52.380 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 14 diff [4194304~8388608,16777216~12582912,50331648~8388608,62914560~46137344,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,260046848~16777216,281018368~4194304,289406976~16777216,327155712~41943040,373293056~37748736] 2024-08-17T17:45:52.380 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~12582912,50331648~8388608,62914560~46137344,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,260046848~16777216,281018368~4194304,289406976~16777216,327155712~41943040,373293056~37748736] 2024-08-17T17:45:52.383 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~16777216,50331648~8388608,62914560~46137344,113246208~12582912,138412032~12582912,163577856~4194304,171966464~8388608,192937984~29360128,230686720~20971520,260046848~16777216,281018368~25165824,327155712~83886080] 2024-08-17T17:45:52.383 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 15 diff [4194304~25165824,33554432~8388608,50331648~8388608,62914560~46137344,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,260046848~16777216,281018368~4194304,289406976~12582912,327155712~41943040,373293056~37748736] 2024-08-17T17:45:52.383 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~8388608,62914560~46137344,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,260046848~16777216,281018368~4194304,289406976~12582912,327155712~41943040,373293056~37748736] 2024-08-17T17:45:52.387 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,50331648~8388608,62914560~46137344,113246208~12582912,138412032~12582912,163577856~4194304,171966464~8388608,192937984~29360128,230686720~20971520,260046848~16777216,281018368~25165824,327155712~83886080] 2024-08-17T17:45:52.387 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 16 diff [4194304~37748736,50331648~12582912,67108864~41943040,113246208~12582912,130023424~20971520,163577856~25165824,192937984~25165824,230686720~8388608,243269632~8388608,260046848~25165824,289406976~12582912,327155712~41943040,373293056~37748736] 2024-08-17T17:45:52.387 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~37748736,50331648~12582912,67108864~41943040,113246208~12582912,130023424~20971520,163577856~25165824,192937984~25165824,230686720~8388608,243269632~8388608,260046848~25165824,289406976~12582912,327155712~41943040,373293056~37748736] 2024-08-17T17:45:52.391 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,50331648~58720256,113246208~12582912,130023424~20971520,163577856~58720256,230686720~20971520,260046848~46137344,327155712~83886080] 2024-08-17T17:45:52.391 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 17 diff [4194304~58720256,67108864~41943040,113246208~12582912,130023424~20971520,159383552~4194304,167772160~20971520,192937984~29360128,230686720~8388608,243269632~8388608,260046848~25165824,289406976~12582912,327155712~83886080] 2024-08-17T17:45:52.391 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~58720256,67108864~41943040,113246208~12582912,130023424~20971520,159383552~4194304,167772160~20971520,192937984~29360128,230686720~8388608,243269632~8388608,260046848~25165824,289406976~12582912,327155712~83886080] 2024-08-17T17:45:52.395 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~12582912,130023424~20971520,159383552~62914560,230686720~20971520,260046848~46137344,327155712~83886080] 2024-08-17T17:45:52.395 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 18 diff [4194304~67108864,75497472~33554432,113246208~37748736,159383552~4194304,167772160~20971520,192937984~29360128,226492416~12582912,243269632~8388608,260046848~25165824,289406976~12582912,327155712~83886080] 2024-08-17T17:45:52.396 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~37748736,159383552~4194304,167772160~20971520,192937984~29360128,226492416~12582912,243269632~8388608,260046848~25165824,289406976~12582912,327155712~83886080] 2024-08-17T17:45:52.400 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,260046848~46137344,327155712~83886080] 2024-08-17T17:45:52.400 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to 19 diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:52.400 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:52.405 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~50331648,327155712~83886080] 2024-08-17T17:45:52.405 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 9 diff [54525952~4194304,96468992~8388608,117440512~4194304,138412032~8388608,163577856~4194304,213909504~4194304,234881024~8388608,260046848~8388608,369098752~4194304] 2024-08-17T17:45:52.406 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [54525952~4194304,96468992~8388608,117440512~4194304,138412032~8388608,163577856~4194304,213909504~4194304,234881024~8388608,260046848~8388608,369098752~4194304] 2024-08-17T17:45:52.407 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [54525952~4194304,96468992~12582912,117440512~4194304,138412032~8388608,163577856~4194304,213909504~8388608,234881024~8388608,260046848~12582912,369098752~4194304] 2024-08-17T17:45:52.407 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 10 diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~16777216,96468992~8388608,117440512~4194304,138412032~8388608,163577856~4194304,213909504~4194304,234881024~8388608,260046848~8388608,343932928~8388608,356515840~12582912] 2024-08-17T17:45:52.407 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~16777216,96468992~8388608,117440512~4194304,138412032~8388608,163577856~4194304,213909504~4194304,234881024~8388608,260046848~8388608,343932928~8388608,356515840~12582912] 2024-08-17T17:45:52.408 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,117440512~8388608,138412032~8388608,163577856~4194304,213909504~8388608,234881024~8388608,260046848~12582912,343932928~8388608,356515840~16777216] 2024-08-17T17:45:52.409 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 11 diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~29360128,113246208~8388608,138412032~8388608,163577856~4194304,171966464~8388608,197132288~12582912,213909504~4194304,230686720~8388608,260046848~12582912,297795584~8388608,331350016~8388608,343932928~8388608,356515840~12582912,398458880~8388608] 2024-08-17T17:45:52.409 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~29360128,113246208~8388608,138412032~8388608,163577856~4194304,171966464~8388608,197132288~12582912,213909504~4194304,230686720~8388608,260046848~12582912,297795584~8388608,331350016~8388608,343932928~8388608,356515840~12582912,398458880~8388608] 2024-08-17T17:45:52.410 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,113246208~12582912,138412032~8388608,163577856~4194304,171966464~8388608,197132288~12582912,213909504~8388608,230686720~12582912,260046848~12582912,297795584~8388608,331350016~8388608,343932928~8388608,356515840~16777216,398458880~12582912] 2024-08-17T17:45:52.411 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 12 diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,197132288~20971520,230686720~8388608,260046848~12582912,297795584~8388608,331350016~8388608,343932928~8388608,356515840~12582912,377487360~16777216,398458880~8388608] 2024-08-17T17:45:52.411 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,197132288~20971520,230686720~8388608,260046848~12582912,297795584~8388608,331350016~8388608,343932928~8388608,356515840~12582912,377487360~16777216,398458880~8388608] 2024-08-17T17:45:52.413 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~33554432,113246208~12582912,138412032~12582912,163577856~4194304,171966464~8388608,197132288~25165824,230686720~12582912,260046848~12582912,297795584~8388608,331350016~8388608,343932928~8388608,356515840~16777216,377487360~16777216,398458880~12582912] 2024-08-17T17:45:52.413 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 13 diff [4194304~8388608,16777216~8388608,50331648~8388608,62914560~46137344,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,197132288~20971520,230686720~8388608,243269632~8388608,260046848~16777216,297795584~8388608,327155712~41943040,377487360~16777216,398458880~8388608] 2024-08-17T17:45:52.413 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~8388608,50331648~8388608,62914560~46137344,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,197132288~20971520,230686720~8388608,243269632~8388608,260046848~16777216,297795584~8388608,327155712~41943040,377487360~16777216,398458880~8388608] 2024-08-17T17:45:52.416 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~8388608,50331648~8388608,62914560~46137344,113246208~12582912,138412032~12582912,163577856~4194304,171966464~8388608,197132288~25165824,230686720~20971520,260046848~16777216,297795584~8388608,327155712~46137344,377487360~16777216,398458880~12582912] 2024-08-17T17:45:52.416 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 14 diff [4194304~8388608,16777216~12582912,50331648~8388608,62914560~46137344,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,260046848~16777216,281018368~4194304,297795584~8388608,327155712~41943040,377487360~29360128] 2024-08-17T17:45:52.416 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~12582912,50331648~8388608,62914560~46137344,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,260046848~16777216,281018368~4194304,297795584~8388608,327155712~41943040,377487360~29360128] 2024-08-17T17:45:52.420 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~16777216,50331648~8388608,62914560~46137344,113246208~12582912,138412032~12582912,163577856~4194304,171966464~8388608,192937984~29360128,230686720~20971520,260046848~16777216,281018368~8388608,297795584~8388608,327155712~46137344,377487360~33554432] 2024-08-17T17:45:52.420 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 15 diff [4194304~25165824,33554432~8388608,50331648~8388608,62914560~46137344,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,260046848~16777216,281018368~4194304,293601280~12582912,327155712~41943040,377487360~29360128] 2024-08-17T17:45:52.420 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~8388608,62914560~46137344,113246208~8388608,138412032~12582912,163577856~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,260046848~16777216,281018368~4194304,293601280~12582912,327155712~41943040,377487360~29360128] 2024-08-17T17:45:52.423 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,50331648~8388608,62914560~46137344,113246208~12582912,138412032~12582912,163577856~4194304,171966464~8388608,192937984~29360128,230686720~20971520,260046848~16777216,281018368~8388608,293601280~12582912,327155712~46137344,377487360~33554432] 2024-08-17T17:45:52.423 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 16 diff [4194304~37748736,50331648~12582912,67108864~41943040,113246208~12582912,130023424~20971520,163577856~25165824,192937984~25165824,230686720~8388608,243269632~8388608,260046848~25165824,293601280~12582912,327155712~41943040,377487360~29360128] 2024-08-17T17:45:52.423 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~37748736,50331648~12582912,67108864~41943040,113246208~12582912,130023424~20971520,163577856~25165824,192937984~25165824,230686720~8388608,243269632~8388608,260046848~25165824,293601280~12582912,327155712~41943040,377487360~29360128] 2024-08-17T17:45:52.426 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,50331648~58720256,113246208~12582912,130023424~20971520,163577856~58720256,230686720~20971520,260046848~29360128,293601280~12582912,327155712~46137344,377487360~33554432] 2024-08-17T17:45:52.426 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 17 diff [4194304~58720256,67108864~41943040,113246208~12582912,130023424~20971520,159383552~4194304,167772160~20971520,192937984~29360128,230686720~8388608,243269632~8388608,260046848~25165824,293601280~12582912,327155712~79691776] 2024-08-17T17:45:52.426 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~58720256,67108864~41943040,113246208~12582912,130023424~20971520,159383552~4194304,167772160~20971520,192937984~29360128,230686720~8388608,243269632~8388608,260046848~25165824,293601280~12582912,327155712~79691776] 2024-08-17T17:45:52.431 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~12582912,130023424~20971520,159383552~62914560,230686720~20971520,260046848~29360128,293601280~12582912,327155712~83886080] 2024-08-17T17:45:52.431 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 18 diff [4194304~67108864,75497472~33554432,113246208~37748736,159383552~4194304,167772160~20971520,192937984~29360128,226492416~12582912,243269632~8388608,260046848~25165824,293601280~12582912,327155712~79691776] 2024-08-17T17:45:52.431 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~37748736,159383552~4194304,167772160~20971520,192937984~29360128,226492416~12582912,243269632~8388608,260046848~25165824,293601280~12582912,327155712~79691776] 2024-08-17T17:45:52.436 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,260046848~29360128,293601280~12582912,327155712~83886080] 2024-08-17T17:45:52.436 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to 19 diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:52.436 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:52.441 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~33554432,293601280~12582912,327155712~83886080] 2024-08-17T17:45:52.441 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 10 diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~16777216,117440512~4194304,343932928~8388608,356515840~12582912] 2024-08-17T17:45:52.441 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~16777216,117440512~4194304,343932928~8388608,356515840~12582912] 2024-08-17T17:45:52.442 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~20971520,117440512~8388608,343932928~8388608,356515840~16777216] 2024-08-17T17:45:52.442 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 11 diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~20971520,113246208~8388608,171966464~8388608,197132288~12582912,230686720~8388608,268435456~4194304,297795584~8388608,331350016~8388608,343932928~8388608,356515840~12582912,398458880~8388608] 2024-08-17T17:45:52.442 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~20971520,113246208~8388608,171966464~8388608,197132288~12582912,230686720~8388608,268435456~4194304,297795584~8388608,331350016~8388608,343932928~8388608,356515840~12582912,398458880~8388608] 2024-08-17T17:45:52.443 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~20971520,113246208~12582912,171966464~8388608,197132288~12582912,230686720~12582912,268435456~4194304,297795584~8388608,331350016~8388608,343932928~8388608,356515840~16777216,398458880~12582912] 2024-08-17T17:45:52.444 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 12 diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~20971520,100663296~8388608,113246208~8388608,146800640~4194304,171966464~8388608,197132288~20971520,230686720~8388608,264241152~8388608,297795584~8388608,331350016~8388608,343932928~8388608,356515840~12582912,377487360~16777216,398458880~8388608] 2024-08-17T17:45:52.444 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~20971520,100663296~8388608,113246208~8388608,146800640~4194304,171966464~8388608,197132288~20971520,230686720~8388608,264241152~8388608,297795584~8388608,331350016~8388608,343932928~8388608,356515840~12582912,377487360~16777216,398458880~8388608] 2024-08-17T17:45:52.445 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,20971520~4194304,50331648~8388608,62914560~4194304,75497472~20971520,100663296~8388608,113246208~12582912,146800640~4194304,171966464~8388608,197132288~25165824,230686720~12582912,264241152~8388608,297795584~8388608,331350016~8388608,343932928~8388608,356515840~16777216,377487360~16777216,398458880~12582912] 2024-08-17T17:45:52.446 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 13 diff [4194304~8388608,16777216~8388608,50331648~8388608,62914560~33554432,100663296~8388608,113246208~8388608,146800640~4194304,171966464~8388608,197132288~20971520,230686720~8388608,243269632~8388608,264241152~12582912,297795584~8388608,327155712~41943040,377487360~16777216,398458880~8388608] 2024-08-17T17:45:52.446 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~8388608,50331648~8388608,62914560~33554432,100663296~8388608,113246208~8388608,146800640~4194304,171966464~8388608,197132288~20971520,230686720~8388608,243269632~8388608,264241152~12582912,297795584~8388608,327155712~41943040,377487360~16777216,398458880~8388608] 2024-08-17T17:45:52.448 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~8388608,50331648~8388608,62914560~33554432,100663296~8388608,113246208~12582912,146800640~4194304,171966464~8388608,197132288~25165824,230686720~20971520,264241152~12582912,297795584~8388608,327155712~46137344,377487360~16777216,398458880~12582912] 2024-08-17T17:45:52.448 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 14 diff [4194304~8388608,16777216~12582912,50331648~8388608,62914560~33554432,100663296~8388608,113246208~8388608,146800640~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,264241152~12582912,281018368~4194304,297795584~8388608,327155712~41943040,377487360~29360128] 2024-08-17T17:45:52.448 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~12582912,50331648~8388608,62914560~33554432,100663296~8388608,113246208~8388608,146800640~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,264241152~12582912,281018368~4194304,297795584~8388608,327155712~41943040,377487360~29360128] 2024-08-17T17:45:52.451 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~16777216,50331648~8388608,62914560~33554432,100663296~8388608,113246208~12582912,146800640~4194304,171966464~8388608,192937984~29360128,230686720~20971520,264241152~12582912,281018368~8388608,297795584~8388608,327155712~46137344,377487360~33554432] 2024-08-17T17:45:52.451 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 15 diff [4194304~25165824,33554432~8388608,50331648~8388608,62914560~33554432,100663296~8388608,113246208~8388608,146800640~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,264241152~12582912,281018368~4194304,293601280~12582912,327155712~41943040,377487360~29360128] 2024-08-17T17:45:52.451 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,50331648~8388608,62914560~33554432,100663296~8388608,113246208~8388608,146800640~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,264241152~12582912,281018368~4194304,293601280~12582912,327155712~41943040,377487360~29360128] 2024-08-17T17:45:52.453 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,50331648~8388608,62914560~33554432,100663296~8388608,113246208~12582912,146800640~4194304,171966464~8388608,192937984~29360128,230686720~20971520,264241152~12582912,281018368~8388608,293601280~12582912,327155712~46137344,377487360~33554432] 2024-08-17T17:45:52.454 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 16 diff [4194304~37748736,50331648~12582912,67108864~41943040,113246208~12582912,130023424~8388608,146800640~4194304,167772160~20971520,192937984~25165824,230686720~8388608,243269632~8388608,264241152~20971520,293601280~12582912,327155712~41943040,377487360~29360128] 2024-08-17T17:45:52.454 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~37748736,50331648~12582912,67108864~41943040,113246208~12582912,130023424~8388608,146800640~4194304,167772160~20971520,192937984~25165824,230686720~8388608,243269632~8388608,264241152~20971520,293601280~12582912,327155712~41943040,377487360~29360128] 2024-08-17T17:45:52.457 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,50331648~58720256,113246208~12582912,130023424~8388608,146800640~4194304,167772160~54525952,230686720~20971520,264241152~25165824,293601280~12582912,327155712~46137344,377487360~33554432] 2024-08-17T17:45:52.457 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 17 diff [4194304~58720256,67108864~41943040,113246208~12582912,130023424~8388608,146800640~4194304,159383552~4194304,167772160~20971520,192937984~29360128,230686720~8388608,243269632~8388608,264241152~20971520,293601280~12582912,327155712~79691776] 2024-08-17T17:45:52.457 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~58720256,67108864~41943040,113246208~12582912,130023424~8388608,146800640~4194304,159383552~4194304,167772160~20971520,192937984~29360128,230686720~8388608,243269632~8388608,264241152~20971520,293601280~12582912,327155712~79691776] 2024-08-17T17:45:52.460 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~12582912,130023424~8388608,146800640~4194304,159383552~62914560,230686720~20971520,264241152~25165824,293601280~12582912,327155712~83886080] 2024-08-17T17:45:52.460 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 18 diff [4194304~67108864,75497472~33554432,113246208~25165824,146800640~4194304,159383552~4194304,167772160~20971520,192937984~29360128,226492416~12582912,243269632~8388608,264241152~20971520,293601280~12582912,327155712~79691776] 2024-08-17T17:45:52.460 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~25165824,146800640~4194304,159383552~4194304,167772160~20971520,192937984~29360128,226492416~12582912,243269632~8388608,264241152~20971520,293601280~12582912,327155712~79691776] 2024-08-17T17:45:52.463 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,264241152~25165824,293601280~12582912,327155712~83886080] 2024-08-17T17:45:52.464 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to 19 diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:52.464 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:52.468 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~83886080] 2024-08-17T17:45:52.468 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 11 diff [83886080~4194304,113246208~4194304,171966464~8388608,197132288~12582912,230686720~8388608,268435456~4194304,297795584~8388608,331350016~8388608,398458880~8388608] 2024-08-17T17:45:52.468 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [83886080~4194304,113246208~4194304,171966464~8388608,197132288~12582912,230686720~8388608,268435456~4194304,297795584~8388608,331350016~8388608,398458880~8388608] 2024-08-17T17:45:52.469 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [83886080~8388608,113246208~8388608,171966464~8388608,197132288~12582912,230686720~12582912,268435456~4194304,297795584~8388608,331350016~8388608,398458880~12582912] 2024-08-17T17:45:52.469 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 12 diff [83886080~12582912,100663296~8388608,113246208~4194304,146800640~4194304,171966464~8388608,197132288~20971520,230686720~8388608,264241152~8388608,297795584~8388608,331350016~8388608,377487360~16777216,398458880~8388608] 2024-08-17T17:45:52.469 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [83886080~12582912,100663296~8388608,113246208~4194304,146800640~4194304,171966464~8388608,197132288~20971520,230686720~8388608,264241152~8388608,297795584~8388608,331350016~8388608,377487360~16777216,398458880~8388608] 2024-08-17T17:45:52.471 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [83886080~12582912,100663296~8388608,113246208~8388608,146800640~4194304,171966464~8388608,197132288~25165824,230686720~12582912,264241152~8388608,297795584~8388608,331350016~8388608,377487360~16777216,398458880~12582912] 2024-08-17T17:45:52.471 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 13 diff [16777216~8388608,67108864~8388608,83886080~12582912,100663296~8388608,113246208~4194304,146800640~4194304,171966464~8388608,197132288~20971520,230686720~8388608,243269632~8388608,264241152~12582912,297795584~8388608,327155712~16777216,348127232~8388608,360710144~4194304,377487360~16777216,398458880~8388608] 2024-08-17T17:45:52.471 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [16777216~8388608,67108864~8388608,83886080~12582912,100663296~8388608,113246208~4194304,146800640~4194304,171966464~8388608,197132288~20971520,230686720~8388608,243269632~8388608,264241152~12582912,297795584~8388608,327155712~16777216,348127232~8388608,360710144~4194304,377487360~16777216,398458880~8388608] 2024-08-17T17:45:52.473 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [16777216~8388608,67108864~8388608,83886080~12582912,100663296~8388608,113246208~8388608,146800640~4194304,171966464~8388608,197132288~25165824,230686720~20971520,264241152~12582912,297795584~8388608,327155712~29360128,360710144~4194304,377487360~16777216,398458880~12582912] 2024-08-17T17:45:52.473 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 14 diff [4194304~8388608,16777216~12582912,67108864~8388608,83886080~12582912,100663296~8388608,113246208~4194304,146800640~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,264241152~12582912,281018368~4194304,297795584~8388608,327155712~37748736,377487360~29360128] 2024-08-17T17:45:52.473 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~12582912,67108864~8388608,83886080~12582912,100663296~8388608,113246208~4194304,146800640~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,264241152~12582912,281018368~4194304,297795584~8388608,327155712~37748736,377487360~29360128] 2024-08-17T17:45:52.476 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~16777216,67108864~8388608,83886080~12582912,100663296~8388608,113246208~8388608,146800640~4194304,171966464~8388608,192937984~29360128,230686720~20971520,264241152~12582912,281018368~8388608,297795584~8388608,327155712~37748736,377487360~33554432] 2024-08-17T17:45:52.476 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 15 diff [4194304~25165824,33554432~8388608,67108864~25165824,100663296~8388608,113246208~4194304,146800640~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,264241152~12582912,281018368~4194304,293601280~12582912,327155712~37748736,377487360~29360128] 2024-08-17T17:45:52.476 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,67108864~25165824,100663296~8388608,113246208~4194304,146800640~4194304,171966464~8388608,192937984~25165824,230686720~8388608,243269632~8388608,264241152~12582912,281018368~4194304,293601280~12582912,327155712~37748736,377487360~29360128] 2024-08-17T17:45:52.478 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,67108864~29360128,100663296~8388608,113246208~8388608,146800640~4194304,171966464~8388608,192937984~29360128,230686720~20971520,264241152~12582912,281018368~8388608,293601280~12582912,327155712~37748736,377487360~33554432] 2024-08-17T17:45:52.478 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 16 diff [4194304~37748736,54525952~8388608,67108864~41943040,113246208~12582912,130023424~8388608,146800640~4194304,167772160~20971520,192937984~25165824,230686720~8388608,243269632~8388608,264241152~20971520,293601280~12582912,327155712~37748736,377487360~29360128] 2024-08-17T17:45:52.478 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~37748736,54525952~8388608,67108864~41943040,113246208~12582912,130023424~8388608,146800640~4194304,167772160~20971520,192937984~25165824,230686720~8388608,243269632~8388608,264241152~20971520,293601280~12582912,327155712~37748736,377487360~29360128] 2024-08-17T17:45:52.481 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,54525952~54525952,113246208~12582912,130023424~8388608,146800640~4194304,167772160~54525952,230686720~20971520,264241152~25165824,293601280~12582912,327155712~37748736,377487360~33554432] 2024-08-17T17:45:52.481 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 17 diff [4194304~46137344,54525952~8388608,67108864~41943040,113246208~12582912,130023424~8388608,146800640~4194304,159383552~4194304,167772160~20971520,192937984~29360128,230686720~8388608,243269632~8388608,264241152~20971520,293601280~12582912,327155712~37748736,369098752~37748736] 2024-08-17T17:45:52.481 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~46137344,54525952~8388608,67108864~41943040,113246208~12582912,130023424~8388608,146800640~4194304,159383552~4194304,167772160~20971520,192937984~29360128,230686720~8388608,243269632~8388608,264241152~20971520,293601280~12582912,327155712~37748736,369098752~37748736] 2024-08-17T17:45:52.485 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~12582912,130023424~8388608,146800640~4194304,159383552~62914560,230686720~20971520,264241152~25165824,293601280~12582912,327155712~37748736,369098752~41943040] 2024-08-17T17:45:52.485 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 18 diff [4194304~46137344,54525952~16777216,75497472~33554432,113246208~25165824,146800640~4194304,159383552~4194304,167772160~20971520,192937984~29360128,226492416~12582912,243269632~8388608,264241152~20971520,293601280~12582912,327155712~37748736,369098752~37748736] 2024-08-17T17:45:52.485 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~46137344,54525952~16777216,75497472~33554432,113246208~25165824,146800640~4194304,159383552~4194304,167772160~20971520,192937984~29360128,226492416~12582912,243269632~8388608,264241152~20971520,293601280~12582912,327155712~37748736,369098752~37748736] 2024-08-17T17:45:52.487 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,264241152~25165824,293601280~12582912,327155712~37748736,369098752~41943040] 2024-08-17T17:45:52.487 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to 19 diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:52.488 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:52.491 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~41943040] 2024-08-17T17:45:52.491 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 12 diff [83886080~8388608,100663296~8388608,146800640~4194304,201326592~16777216,264241152~4194304,377487360~16777216] 2024-08-17T17:45:52.491 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [83886080~8388608,100663296~8388608,146800640~4194304,201326592~16777216,264241152~4194304,377487360~16777216] 2024-08-17T17:45:52.492 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [83886080~12582912,100663296~8388608,146800640~4194304,201326592~20971520,264241152~4194304,377487360~16777216] 2024-08-17T17:45:52.492 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 13 diff [16777216~8388608,67108864~8388608,83886080~8388608,100663296~8388608,117440512~4194304,146800640~4194304,201326592~16777216,243269632~8388608,264241152~4194304,272629760~4194304,327155712~16777216,348127232~8388608,360710144~4194304,377487360~16777216] 2024-08-17T17:45:52.493 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [16777216~8388608,67108864~8388608,83886080~8388608,100663296~8388608,117440512~4194304,146800640~4194304,201326592~16777216,243269632~8388608,264241152~4194304,272629760~4194304,327155712~16777216,348127232~8388608,360710144~4194304,377487360~16777216] 2024-08-17T17:45:52.494 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [16777216~8388608,67108864~8388608,83886080~12582912,100663296~8388608,117440512~4194304,146800640~4194304,201326592~20971520,243269632~8388608,264241152~4194304,272629760~4194304,327155712~29360128,360710144~4194304,377487360~16777216] 2024-08-17T17:45:52.494 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 14 diff [4194304~8388608,16777216~12582912,67108864~8388608,83886080~8388608,100663296~8388608,117440512~4194304,146800640~4194304,192937984~4194304,201326592~16777216,243269632~8388608,264241152~4194304,272629760~4194304,281018368~4194304,327155712~37748736,377487360~20971520,406847488~4194304] 2024-08-17T17:45:52.494 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~12582912,67108864~8388608,83886080~8388608,100663296~8388608,117440512~4194304,146800640~4194304,192937984~4194304,201326592~16777216,243269632~8388608,264241152~4194304,272629760~4194304,281018368~4194304,327155712~37748736,377487360~20971520,406847488~4194304] 2024-08-17T17:45:52.496 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~16777216,67108864~8388608,83886080~12582912,100663296~8388608,117440512~4194304,146800640~4194304,192937984~4194304,201326592~20971520,243269632~8388608,264241152~4194304,272629760~4194304,281018368~8388608,327155712~37748736,377487360~20971520,406847488~4194304] 2024-08-17T17:45:52.496 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 15 diff [4194304~25165824,33554432~8388608,67108864~25165824,100663296~8388608,117440512~4194304,146800640~4194304,192937984~4194304,201326592~16777216,243269632~8388608,264241152~4194304,272629760~4194304,281018368~4194304,293601280~8388608,327155712~37748736,377487360~20971520,406847488~4194304] 2024-08-17T17:45:52.496 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,67108864~25165824,100663296~8388608,117440512~4194304,146800640~4194304,192937984~4194304,201326592~16777216,243269632~8388608,264241152~4194304,272629760~4194304,281018368~4194304,293601280~8388608,327155712~37748736,377487360~20971520,406847488~4194304] 2024-08-17T17:45:52.498 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,67108864~29360128,100663296~8388608,117440512~4194304,146800640~4194304,192937984~4194304,201326592~20971520,243269632~8388608,264241152~4194304,272629760~4194304,281018368~8388608,293601280~8388608,327155712~37748736,377487360~20971520,406847488~4194304] 2024-08-17T17:45:52.498 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 16 diff [4194304~37748736,54525952~8388608,67108864~29360128,100663296~8388608,117440512~8388608,130023424~8388608,146800640~4194304,167772160~8388608,180355072~8388608,192937984~4194304,201326592~16777216,243269632~8388608,264241152~4194304,272629760~12582912,293601280~12582912,327155712~37748736,377487360~20971520,406847488~4194304] 2024-08-17T17:45:52.498 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~37748736,54525952~8388608,67108864~29360128,100663296~8388608,117440512~8388608,130023424~8388608,146800640~4194304,167772160~8388608,180355072~8388608,192937984~4194304,201326592~16777216,243269632~8388608,264241152~4194304,272629760~12582912,293601280~12582912,327155712~37748736,377487360~20971520,406847488~4194304] 2024-08-17T17:45:52.500 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,54525952~54525952,117440512~8388608,130023424~8388608,146800640~4194304,167772160~8388608,180355072~16777216,201326592~20971520,243269632~8388608,264241152~4194304,272629760~16777216,293601280~12582912,327155712~37748736,377487360~20971520,406847488~4194304] 2024-08-17T17:45:52.501 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 17 diff [4194304~46137344,54525952~8388608,67108864~29360128,100663296~8388608,117440512~8388608,130023424~8388608,146800640~4194304,159383552~4194304,167772160~8388608,180355072~8388608,192937984~4194304,201326592~20971520,243269632~8388608,264241152~20971520,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:52.501 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~46137344,54525952~8388608,67108864~29360128,100663296~8388608,117440512~8388608,130023424~8388608,146800640~4194304,159383552~4194304,167772160~8388608,180355072~8388608,192937984~4194304,201326592~20971520,243269632~8388608,264241152~20971520,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:52.503 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,117440512~8388608,130023424~8388608,146800640~4194304,159383552~16777216,180355072~16777216,201326592~20971520,243269632~8388608,264241152~25165824,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:52.503 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 18 diff [4194304~46137344,54525952~16777216,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~20971520,192937984~4194304,201326592~20971520,226492416~8388608,243269632~8388608,264241152~20971520,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:52.504 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~46137344,54525952~16777216,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~20971520,192937984~4194304,201326592~20971520,226492416~8388608,243269632~8388608,264241152~20971520,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:52.507 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,117440512~20971520,146800640~4194304,159383552~37748736,201326592~20971520,226492416~12582912,243269632~8388608,264241152~25165824,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:52.507 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to 19 diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:52.507 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:52.510 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,117440512~20971520,146800640~4194304,159383552~37748736,201326592~20971520,226492416~12582912,243269632~8388608,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:52.511 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to 13 diff [16777216~8388608,67108864~8388608,117440512~4194304,243269632~8388608,272629760~4194304,327155712~16777216,348127232~8388608,360710144~4194304] 2024-08-17T17:45:52.511 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [16777216~8388608,67108864~8388608,117440512~4194304,243269632~8388608,272629760~4194304,327155712~16777216,348127232~8388608,360710144~4194304] 2024-08-17T17:45:52.511 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [16777216~8388608,67108864~8388608,117440512~4194304,243269632~8388608,272629760~4194304,327155712~29360128,360710144~4194304] 2024-08-17T17:45:52.511 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to 14 diff [4194304~8388608,16777216~12582912,67108864~8388608,117440512~4194304,192937984~4194304,209715200~4194304,243269632~8388608,272629760~4194304,281018368~4194304,327155712~37748736,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.512 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,16777216~12582912,67108864~8388608,117440512~4194304,192937984~4194304,209715200~4194304,243269632~8388608,272629760~4194304,281018368~4194304,327155712~37748736,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.513 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,16777216~16777216,67108864~8388608,117440512~4194304,192937984~4194304,209715200~4194304,243269632~8388608,272629760~4194304,281018368~8388608,327155712~37748736,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.513 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to 15 diff [4194304~25165824,33554432~8388608,67108864~16777216,117440512~4194304,192937984~4194304,209715200~4194304,243269632~8388608,272629760~4194304,281018368~4194304,293601280~8388608,327155712~37748736,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.513 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,67108864~16777216,117440512~4194304,192937984~4194304,209715200~4194304,243269632~8388608,272629760~4194304,281018368~4194304,293601280~8388608,327155712~37748736,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.514 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,67108864~20971520,117440512~4194304,192937984~4194304,209715200~4194304,243269632~8388608,272629760~4194304,281018368~8388608,293601280~8388608,327155712~37748736,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.514 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to 16 diff [4194304~37748736,54525952~8388608,67108864~16777216,92274688~8388608,117440512~8388608,130023424~8388608,167772160~8388608,180355072~8388608,192937984~4194304,209715200~4194304,243269632~8388608,272629760~12582912,293601280~12582912,327155712~37748736,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.514 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~37748736,54525952~8388608,67108864~16777216,92274688~8388608,117440512~8388608,130023424~8388608,167772160~8388608,180355072~8388608,192937984~4194304,209715200~4194304,243269632~8388608,272629760~12582912,293601280~12582912,327155712~37748736,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.516 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,54525952~33554432,92274688~8388608,117440512~8388608,130023424~8388608,167772160~8388608,180355072~16777216,209715200~4194304,243269632~8388608,272629760~16777216,293601280~12582912,327155712~37748736,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.517 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to 17 diff [4194304~46137344,54525952~8388608,67108864~16777216,92274688~8388608,117440512~8388608,130023424~8388608,159383552~4194304,167772160~8388608,180355072~8388608,192937984~4194304,205520896~8388608,243269632~8388608,268435456~16777216,293601280~12582912,327155712~37748736,369098752~8388608,381681664~4194304,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.517 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~46137344,54525952~8388608,67108864~16777216,92274688~8388608,117440512~8388608,130023424~8388608,159383552~4194304,167772160~8388608,180355072~8388608,192937984~4194304,205520896~8388608,243269632~8388608,268435456~16777216,293601280~12582912,327155712~37748736,369098752~8388608,381681664~4194304,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.519 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~83886080,92274688~8388608,117440512~8388608,130023424~8388608,159383552~16777216,180355072~16777216,205520896~8388608,243269632~8388608,268435456~20971520,293601280~12582912,327155712~37748736,369098752~20971520,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.519 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to 18 diff [4194304~46137344,54525952~16777216,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~20971520,192937984~4194304,205520896~8388608,226492416~8388608,243269632~8388608,268435456~16777216,293601280~12582912,327155712~37748736,369098752~8388608,381681664~8388608,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.519 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~46137344,54525952~16777216,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~20971520,192937984~4194304,205520896~8388608,226492416~8388608,243269632~8388608,268435456~16777216,293601280~12582912,327155712~37748736,369098752~8388608,381681664~8388608,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.522 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~83886080,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,243269632~8388608,268435456~20971520,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:52.522 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to 19 diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,243269632~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:52.522 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,243269632~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:52.525 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~83886080,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,243269632~8388608,255852544~4194304,268435456~20971520,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:52.525 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to 14 diff [4194304~8388608,25165824~4194304,192937984~4194304,209715200~4194304,281018368~4194304,327155712~12582912,356515840~4194304,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.525 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~8388608,25165824~4194304,192937984~4194304,209715200~4194304,281018368~4194304,327155712~12582912,356515840~4194304,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.526 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~8388608,25165824~8388608,192937984~4194304,209715200~4194304,281018368~8388608,327155712~16777216,356515840~4194304,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.526 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to 15 diff [4194304~25165824,33554432~8388608,75497472~8388608,192937984~4194304,209715200~4194304,281018368~4194304,293601280~8388608,327155712~12582912,343932928~8388608,356515840~4194304,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.526 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~25165824,33554432~8388608,75497472~8388608,192937984~4194304,209715200~4194304,281018368~4194304,293601280~8388608,327155712~12582912,343932928~8388608,356515840~4194304,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.528 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,75497472~12582912,192937984~4194304,209715200~4194304,281018368~8388608,293601280~8388608,327155712~25165824,356515840~4194304,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.528 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to 16 diff [4194304~37748736,54525952~8388608,75497472~8388608,92274688~8388608,117440512~8388608,130023424~8388608,167772160~8388608,180355072~8388608,192937984~4194304,209715200~4194304,276824064~8388608,293601280~12582912,327155712~12582912,343932928~8388608,356515840~4194304,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.528 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~37748736,54525952~8388608,75497472~8388608,92274688~8388608,117440512~8388608,130023424~8388608,167772160~8388608,180355072~8388608,192937984~4194304,209715200~4194304,276824064~8388608,293601280~12582912,327155712~12582912,343932928~8388608,356515840~4194304,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.529 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~41943040,54525952~12582912,75497472~12582912,92274688~8388608,117440512~8388608,130023424~8388608,167772160~8388608,180355072~16777216,209715200~4194304,276824064~12582912,293601280~12582912,327155712~25165824,356515840~4194304,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.530 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to 17 diff [4194304~46137344,54525952~8388608,75497472~8388608,92274688~8388608,117440512~8388608,130023424~8388608,159383552~4194304,167772160~8388608,180355072~8388608,192937984~4194304,205520896~8388608,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~4194304,369098752~8388608,381681664~4194304,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.530 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~46137344,54525952~8388608,75497472~8388608,92274688~8388608,117440512~8388608,130023424~8388608,159383552~4194304,167772160~8388608,180355072~8388608,192937984~4194304,205520896~8388608,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~4194304,369098752~8388608,381681664~4194304,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.532 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~62914560,75497472~12582912,92274688~8388608,117440512~8388608,130023424~8388608,159383552~16777216,180355072~16777216,205520896~8388608,268435456~20971520,293601280~12582912,327155712~25165824,356515840~4194304,369098752~20971520,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.532 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to 18 diff [4194304~46137344,54525952~16777216,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~20971520,192937984~4194304,205520896~8388608,226492416~8388608,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~8388608,369098752~8388608,381681664~8388608,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.532 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~46137344,54525952~16777216,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~20971520,192937984~4194304,205520896~8388608,226492416~8388608,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~8388608,369098752~8388608,381681664~8388608,394264576~4194304,406847488~4194304] 2024-08-17T17:45:52.534 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~67108864,75497472~12582912,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,268435456~20971520,293601280~12582912,327155712~25165824,356515840~8388608,369098752~29360128,406847488~4194304] 2024-08-17T17:45:52.534 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to 19 diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~8388608,369098752~25165824,406847488~4194304] 2024-08-17T17:45:52.534 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~8388608,369098752~25165824,406847488~4194304] 2024-08-17T17:45:52.537 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~67108864,75497472~12582912,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,327155712~25165824,356515840~8388608,369098752~29360128,406847488~4194304] 2024-08-17T17:45:52.538 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to 15 diff [8388608~16777216,33554432~8388608,75497472~8388608,281018368~4194304,293601280~8388608,343932928~8388608] 2024-08-17T17:45:52.538 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~16777216,33554432~8388608,75497472~8388608,281018368~4194304,293601280~8388608,343932928~8388608] 2024-08-17T17:45:52.538 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~16777216,33554432~12582912,75497472~12582912,281018368~8388608,293601280~8388608,343932928~8388608] 2024-08-17T17:45:52.539 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to 16 diff [8388608~33554432,54525952~8388608,75497472~8388608,92274688~8388608,117440512~8388608,130023424~8388608,167772160~8388608,180355072~8388608,276824064~8388608,293601280~12582912,343932928~8388608] 2024-08-17T17:45:52.539 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~33554432,54525952~8388608,75497472~8388608,92274688~8388608,117440512~8388608,130023424~8388608,167772160~8388608,180355072~8388608,276824064~8388608,293601280~12582912,343932928~8388608] 2024-08-17T17:45:52.540 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~37748736,54525952~12582912,75497472~12582912,92274688~8388608,117440512~8388608,130023424~8388608,167772160~8388608,180355072~12582912,276824064~12582912,293601280~12582912,343932928~8388608] 2024-08-17T17:45:52.540 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to 17 diff [8388608~41943040,54525952~8388608,75497472~8388608,92274688~8388608,117440512~8388608,130023424~8388608,159383552~4194304,167772160~8388608,180355072~8388608,205520896~8388608,268435456~16777216,293601280~12582912,331350016~4194304,343932928~8388608,369098752~8388608,381681664~4194304] 2024-08-17T17:45:52.540 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~41943040,54525952~8388608,75497472~8388608,92274688~8388608,117440512~8388608,130023424~8388608,159383552~4194304,167772160~8388608,180355072~8388608,205520896~8388608,268435456~16777216,293601280~12582912,331350016~4194304,343932928~8388608,369098752~8388608,381681664~4194304] 2024-08-17T17:45:52.542 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~58720256,75497472~12582912,92274688~8388608,117440512~8388608,130023424~8388608,159383552~16777216,180355072~12582912,205520896~8388608,268435456~20971520,293601280~12582912,331350016~4194304,343932928~8388608,369098752~20971520] 2024-08-17T17:45:52.542 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to 18 diff [8388608~41943040,54525952~16777216,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~20971520,205520896~8388608,226492416~8388608,268435456~16777216,293601280~12582912,331350016~4194304,343932928~8388608,356515840~8388608,369098752~8388608,381681664~8388608] 2024-08-17T17:45:52.542 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~41943040,54525952~16777216,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~20971520,205520896~8388608,226492416~8388608,268435456~16777216,293601280~12582912,331350016~4194304,343932928~8388608,356515840~8388608,369098752~8388608,381681664~8388608] 2024-08-17T17:45:52.544 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~62914560,75497472~12582912,92274688~8388608,117440512~20971520,159383552~33554432,205520896~8388608,226492416~12582912,268435456~20971520,293601280~12582912,331350016~4194304,343932928~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.544 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to 19 diff [8388608~62914560,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.544 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~62914560,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.547 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~62914560,75497472~12582912,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~29360128] 2024-08-17T17:45:52.548 INFO:tasks.workunit.client.0.smithi130.stdout:from 15 to 16 diff [25165824~8388608,54525952~8388608,92274688~8388608,117440512~8388608,130023424~8388608,167772160~8388608,180355072~8388608,276824064~8388608,301989888~4194304] 2024-08-17T17:45:52.548 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [25165824~8388608,54525952~8388608,92274688~8388608,117440512~8388608,130023424~8388608,167772160~8388608,180355072~8388608,276824064~8388608,301989888~4194304] 2024-08-17T17:45:52.548 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [25165824~12582912,54525952~12582912,92274688~8388608,117440512~8388608,130023424~8388608,167772160~8388608,180355072~12582912,276824064~12582912,301989888~4194304] 2024-08-17T17:45:52.549 INFO:tasks.workunit.client.0.smithi130.stdout:from 15 to 17 diff [25165824~8388608,37748736~12582912,54525952~8388608,92274688~8388608,117440512~8388608,130023424~8388608,159383552~4194304,167772160~8388608,180355072~8388608,205520896~8388608,268435456~16777216,301989888~4194304,331350016~4194304,369098752~8388608,381681664~4194304] 2024-08-17T17:45:52.549 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [25165824~8388608,37748736~12582912,54525952~8388608,92274688~8388608,117440512~8388608,130023424~8388608,159383552~4194304,167772160~8388608,180355072~8388608,205520896~8388608,268435456~16777216,301989888~4194304,331350016~4194304,369098752~8388608,381681664~4194304] 2024-08-17T17:45:52.550 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [25165824~41943040,92274688~8388608,117440512~8388608,130023424~8388608,159383552~16777216,180355072~12582912,205520896~8388608,268435456~20971520,301989888~4194304,331350016~4194304,369098752~20971520] 2024-08-17T17:45:52.550 INFO:tasks.workunit.client.0.smithi130.stdout:from 15 to 18 diff [16777216~16777216,37748736~12582912,54525952~16777216,92274688~8388608,117440512~20971520,159383552~4194304,167772160~20971520,205520896~8388608,226492416~8388608,268435456~16777216,293601280~12582912,331350016~4194304,356515840~8388608,369098752~8388608,381681664~8388608] 2024-08-17T17:45:52.550 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [16777216~16777216,37748736~12582912,54525952~16777216,92274688~8388608,117440512~20971520,159383552~4194304,167772160~20971520,205520896~8388608,226492416~8388608,268435456~16777216,293601280~12582912,331350016~4194304,356515840~8388608,369098752~8388608,381681664~8388608] 2024-08-17T17:45:52.552 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [16777216~54525952,92274688~8388608,117440512~20971520,159383552~33554432,205520896~8388608,226492416~12582912,268435456~20971520,293601280~12582912,331350016~4194304,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.552 INFO:tasks.workunit.client.0.smithi130.stdout:from 15 to 19 diff [8388608~4194304,16777216~16777216,37748736~33554432,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.552 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~4194304,16777216~16777216,37748736~33554432,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.554 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~62914560,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,331350016~8388608,356515840~8388608,369098752~29360128] 2024-08-17T17:45:52.554 INFO:tasks.workunit.client.0.smithi130.stdout:from 16 to 17 diff [25165824~8388608,37748736~12582912,58720256~8388608,159383552~4194304,205520896~8388608,268435456~8388608,331350016~4194304,369098752~8388608,381681664~4194304] 2024-08-17T17:45:52.554 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [25165824~8388608,37748736~12582912,58720256~8388608,159383552~4194304,205520896~8388608,268435456~8388608,331350016~4194304,369098752~8388608,381681664~4194304] 2024-08-17T17:45:52.555 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [25165824~29360128,58720256~8388608,159383552~8388608,205520896~8388608,268435456~12582912,331350016~4194304,369098752~20971520] 2024-08-17T17:45:52.555 INFO:tasks.workunit.client.0.smithi130.stdout:from 16 to 18 diff [16777216~16777216,37748736~12582912,58720256~12582912,125829120~8388608,159383552~4194304,176160768~8388608,205520896~8388608,226492416~8388608,268435456~8388608,293601280~8388608,331350016~4194304,356515840~8388608,369098752~8388608,381681664~8388608] 2024-08-17T17:45:52.555 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [16777216~16777216,37748736~12582912,58720256~12582912,125829120~8388608,159383552~4194304,176160768~8388608,205520896~8388608,226492416~8388608,268435456~8388608,293601280~8388608,331350016~4194304,356515840~8388608,369098752~8388608,381681664~8388608] 2024-08-17T17:45:52.557 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [16777216~37748736,58720256~12582912,125829120~12582912,159383552~8388608,176160768~8388608,205520896~8388608,226492416~12582912,268435456~12582912,293601280~8388608,331350016~4194304,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.557 INFO:tasks.workunit.client.0.smithi130.stdout:from 16 to 19 diff [8388608~4194304,16777216~16777216,37748736~33554432,121634816~12582912,159383552~4194304,176160768~16777216,205520896~8388608,226492416~8388608,255852544~4194304,268435456~8388608,293601280~8388608,331350016~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.557 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~4194304,16777216~16777216,37748736~33554432,121634816~12582912,159383552~4194304,176160768~16777216,205520896~8388608,226492416~8388608,255852544~4194304,268435456~8388608,293601280~8388608,331350016~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.559 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~62914560,121634816~16777216,159383552~8388608,176160768~20971520,205520896~8388608,226492416~12582912,255852544~4194304,268435456~12582912,293601280~8388608,331350016~8388608,356515840~8388608,369098752~29360128] 2024-08-17T17:45:52.559 INFO:tasks.workunit.client.0.smithi130.stdout:from 17 to 18 diff [16777216~16777216,50331648~4194304,62914560~8388608,125829120~8388608,176160768~8388608,226492416~8388608,293601280~8388608,356515840~8388608,385875968~4194304] 2024-08-17T17:45:52.559 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [16777216~16777216,50331648~4194304,62914560~8388608,125829120~8388608,176160768~8388608,226492416~8388608,293601280~8388608,356515840~8388608,385875968~4194304] 2024-08-17T17:45:52.560 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [16777216~20971520,50331648~4194304,62914560~8388608,125829120~12582912,176160768~8388608,226492416~12582912,293601280~8388608,356515840~8388608,385875968~8388608] 2024-08-17T17:45:52.560 INFO:tasks.workunit.client.0.smithi130.stdout:from 17 to 19 diff [8388608~4194304,16777216~16777216,50331648~20971520,121634816~12582912,159383552~4194304,176160768~16777216,226492416~8388608,255852544~4194304,293601280~8388608,331350016~8388608,356515840~8388608,377487360~16777216] 2024-08-17T17:45:52.560 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~4194304,16777216~16777216,50331648~20971520,121634816~12582912,159383552~4194304,176160768~16777216,226492416~8388608,255852544~4194304,293601280~8388608,331350016~8388608,356515840~8388608,377487360~16777216] 2024-08-17T17:45:52.562 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~29360128,50331648~20971520,121634816~16777216,159383552~4194304,176160768~20971520,226492416~12582912,255852544~4194304,293601280~8388608,331350016~8388608,356515840~8388608,377487360~20971520] 2024-08-17T17:45:52.563 INFO:tasks.workunit.client.0.smithi130.stdout:from 18 to 19 diff [8388608~4194304,50331648~16777216,121634816~8388608,159383552~4194304,184549376~8388608,255852544~4194304,331350016~8388608,377487360~8388608,390070272~4194304] 2024-08-17T17:45:52.563 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~4194304,50331648~16777216,121634816~8388608,159383552~4194304,184549376~8388608,255852544~4194304,331350016~8388608,377487360~8388608,390070272~4194304] 2024-08-17T17:45:52.564 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~8388608,50331648~16777216,121634816~12582912,159383552~4194304,184549376~12582912,255852544~4194304,331350016~8388608,377487360~20971520] 2024-08-17T17:45:52.879 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:52.879 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:52.899 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:52.899 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:52.899 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:52.906 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:52.906 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:52.906 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:52.913 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:52.913 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.913 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.918 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:52.918 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.918 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.924 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:45:52.924 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.924 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.929 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:52.929 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.929 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:52.934 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:52.934 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:52.934 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:52.940 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~50331648,327155712~83886080] 2024-08-17T17:45:52.940 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:52.940 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:52.943 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~33554432,293601280~12582912,327155712~83886080] 2024-08-17T17:45:52.943 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:52.943 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:52.947 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~83886080] 2024-08-17T17:45:52.948 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:52.948 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:52.952 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~41943040] 2024-08-17T17:45:52.952 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to HEAD diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:52.952 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:52.955 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,117440512~20971520,146800640~4194304,159383552~37748736,201326592~20971520,226492416~12582912,243269632~8388608,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:52.955 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to HEAD diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,243269632~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:52.955 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,243269632~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:52.958 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~83886080,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,243269632~8388608,255852544~4194304,268435456~20971520,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:52.958 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to HEAD diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~8388608,369098752~25165824,406847488~4194304] 2024-08-17T17:45:52.958 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~8388608,369098752~25165824,406847488~4194304] 2024-08-17T17:45:52.960 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~67108864,75497472~12582912,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,327155712~25165824,356515840~8388608,369098752~29360128,406847488~4194304] 2024-08-17T17:45:52.961 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to HEAD diff [8388608~62914560,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.961 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~62914560,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.963 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~62914560,75497472~12582912,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~29360128] 2024-08-17T17:45:52.963 INFO:tasks.workunit.client.0.smithi130.stdout:from 15 to HEAD diff [8388608~4194304,16777216~16777216,37748736~33554432,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.963 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~4194304,16777216~16777216,37748736~33554432,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.964 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~62914560,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,331350016~8388608,356515840~8388608,369098752~29360128] 2024-08-17T17:45:52.965 INFO:tasks.workunit.client.0.smithi130.stdout:from 16 to HEAD diff [8388608~4194304,16777216~16777216,37748736~33554432,121634816~12582912,159383552~4194304,176160768~16777216,205520896~8388608,226492416~8388608,255852544~4194304,268435456~8388608,293601280~8388608,331350016~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.965 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~4194304,16777216~16777216,37748736~33554432,121634816~12582912,159383552~4194304,176160768~16777216,205520896~8388608,226492416~8388608,255852544~4194304,268435456~8388608,293601280~8388608,331350016~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:52.966 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~62914560,121634816~16777216,159383552~8388608,176160768~20971520,205520896~8388608,226492416~12582912,255852544~4194304,268435456~12582912,293601280~8388608,331350016~8388608,356515840~8388608,369098752~29360128] 2024-08-17T17:45:52.966 INFO:tasks.workunit.client.0.smithi130.stdout:from 17 to HEAD diff [8388608~4194304,16777216~16777216,50331648~20971520,121634816~12582912,159383552~4194304,176160768~16777216,226492416~8388608,255852544~4194304,293601280~8388608,331350016~8388608,356515840~8388608,377487360~16777216] 2024-08-17T17:45:52.966 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~4194304,16777216~16777216,50331648~20971520,121634816~12582912,159383552~4194304,176160768~16777216,226492416~8388608,255852544~4194304,293601280~8388608,331350016~8388608,356515840~8388608,377487360~16777216] 2024-08-17T17:45:52.966 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~29360128,50331648~20971520,121634816~16777216,159383552~4194304,176160768~20971520,226492416~12582912,255852544~4194304,293601280~8388608,331350016~8388608,356515840~8388608,377487360~20971520] 2024-08-17T17:45:53.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:52 smithi130 ceph-mon[105521]: osdmap e1138: 8 total, 8 up, 8 in 2024-08-17T17:45:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:52 smithi130 ceph-mon[110068]: osdmap e1138: 8 total, 8 up, 8 in 2024-08-17T17:45:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:52 smithi179 ceph-mon[93519]: osdmap e1138: 8 total, 8 up, 8 in 2024-08-17T17:45:53.896 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:53.897 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:53.910 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:53.910 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:53.910 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:53.918 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:53.918 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:53.918 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:53.924 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:53.925 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:53.925 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:53.931 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:53.931 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:53.931 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:53.936 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:45:53.936 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:53.936 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:53.942 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:53.942 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:53.943 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:53.948 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:53.948 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:53.948 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:53.954 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~50331648,327155712~83886080] 2024-08-17T17:45:53.954 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:53.954 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:53.958 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~33554432,293601280~12582912,327155712~83886080] 2024-08-17T17:45:53.959 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:53.959 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:53.963 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~83886080] 2024-08-17T17:45:53.963 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:53.963 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:53.967 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~41943040] 2024-08-17T17:45:53.968 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to HEAD diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:53.968 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:53.970 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,117440512~20971520,146800640~4194304,159383552~37748736,201326592~20971520,226492416~12582912,243269632~8388608,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:53.970 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to HEAD diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,243269632~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:53.971 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,243269632~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:53.973 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~83886080,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,243269632~8388608,255852544~4194304,268435456~20971520,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:53.973 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to HEAD diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~8388608,369098752~25165824,406847488~4194304] 2024-08-17T17:45:53.973 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~8388608,369098752~25165824,406847488~4194304] 2024-08-17T17:45:53.976 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~67108864,75497472~12582912,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,327155712~25165824,356515840~8388608,369098752~29360128,406847488~4194304] 2024-08-17T17:45:53.976 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to HEAD diff [8388608~62914560,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:53.976 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~62914560,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:53.978 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~62914560,75497472~12582912,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~29360128] 2024-08-17T17:45:53.978 INFO:tasks.workunit.client.0.smithi130.stdout:from 15 to HEAD diff [8388608~4194304,16777216~16777216,37748736~33554432,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:53.978 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~4194304,16777216~16777216,37748736~33554432,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:53.979 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~62914560,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,331350016~8388608,356515840~8388608,369098752~29360128] 2024-08-17T17:45:53.979 INFO:tasks.workunit.client.0.smithi130.stdout:from 16 to HEAD diff [8388608~4194304,16777216~16777216,37748736~33554432,121634816~12582912,159383552~4194304,176160768~16777216,205520896~8388608,226492416~8388608,255852544~4194304,268435456~8388608,293601280~8388608,331350016~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:53.980 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~4194304,16777216~16777216,37748736~33554432,121634816~12582912,159383552~4194304,176160768~16777216,205520896~8388608,226492416~8388608,255852544~4194304,268435456~8388608,293601280~8388608,331350016~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:53.980 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~62914560,121634816~16777216,159383552~8388608,176160768~20971520,205520896~8388608,226492416~12582912,255852544~4194304,268435456~12582912,293601280~8388608,331350016~8388608,356515840~8388608,369098752~29360128] 2024-08-17T17:45:54.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:53 smithi130 ceph-mon[105521]: pgmap v1476: 105 pgs: 105 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 127 KiB/s rd, 18 MiB/s wr, 178 op/s 2024-08-17T17:45:54.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:53 smithi130 ceph-mon[105521]: osdmap e1139: 8 total, 8 up, 8 in 2024-08-17T17:45:54.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:53 smithi130 ceph-mon[110068]: pgmap v1476: 105 pgs: 105 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 127 KiB/s rd, 18 MiB/s wr, 178 op/s 2024-08-17T17:45:54.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:53 smithi130 ceph-mon[110068]: osdmap e1139: 8 total, 8 up, 8 in 2024-08-17T17:45:54.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:53 smithi179 ceph-mon[93519]: pgmap v1476: 105 pgs: 105 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 127 KiB/s rd, 18 MiB/s wr, 178 op/s 2024-08-17T17:45:54.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:53 smithi179 ceph-mon[93519]: osdmap e1139: 8 total, 8 up, 8 in 2024-08-17T17:45:54.905 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:54.905 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:54.923 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:54.923 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:54.923 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:54.931 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:54.931 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:54.931 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:54.938 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:54.938 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:54.938 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:54.944 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:54.944 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:54.945 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:54.951 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:45:54.951 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:54.951 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:54.957 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:54.957 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:54.957 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:54.962 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:54.962 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:54.963 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:54.968 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~50331648,327155712~83886080] 2024-08-17T17:45:54.968 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:54.968 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:54.973 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~33554432,293601280~12582912,327155712~83886080] 2024-08-17T17:45:54.973 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:54.973 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:54.976 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~83886080] 2024-08-17T17:45:54.976 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:54.976 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:54.979 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~41943040] 2024-08-17T17:45:54.980 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to HEAD diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:54.980 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:54.982 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,117440512~20971520,146800640~4194304,159383552~37748736,201326592~20971520,226492416~12582912,243269632~8388608,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:54.982 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to HEAD diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,243269632~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:54.982 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,243269632~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:54.985 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~83886080,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,243269632~8388608,255852544~4194304,268435456~20971520,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:54.985 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to HEAD diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~8388608,369098752~25165824,406847488~4194304] 2024-08-17T17:45:54.985 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~8388608,369098752~25165824,406847488~4194304] 2024-08-17T17:45:54.988 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~67108864,75497472~12582912,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,327155712~25165824,356515840~8388608,369098752~29360128,406847488~4194304] 2024-08-17T17:45:54.988 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to HEAD diff [8388608~62914560,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:54.988 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~62914560,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:54.989 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~62914560,75497472~12582912,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~29360128] 2024-08-17T17:45:54.989 INFO:tasks.workunit.client.0.smithi130.stdout:from 15 to HEAD diff [8388608~4194304,16777216~16777216,37748736~33554432,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:54.989 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~4194304,16777216~16777216,37748736~33554432,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:54.990 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~62914560,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,331350016~8388608,356515840~8388608,369098752~29360128] 2024-08-17T17:45:55.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:54 smithi130 ceph-mon[105521]: osdmap e1140: 8 total, 8 up, 8 in 2024-08-17T17:45:55.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:54 smithi130 ceph-mon[110068]: osdmap e1140: 8 total, 8 up, 8 in 2024-08-17T17:45:55.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:54 smithi179 ceph-mon[93519]: osdmap e1140: 8 total, 8 up, 8 in 2024-08-17T17:45:55.912 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:55.912 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:55.927 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:55.927 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:55.927 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:55.933 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:55.933 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:55.933 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:55.938 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:55.938 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:55.938 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:55.944 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:55.944 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:55.944 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:55.949 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:45:55.949 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:55.949 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:55.954 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:55.954 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:55.954 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:55.958 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:55.958 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:55.958 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:55.962 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~50331648,327155712~83886080] 2024-08-17T17:45:55.962 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:55.962 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:55.966 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~33554432,293601280~12582912,327155712~83886080] 2024-08-17T17:45:55.967 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:55.967 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:55.970 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~83886080] 2024-08-17T17:45:55.970 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:55.970 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:55.973 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~41943040] 2024-08-17T17:45:55.973 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to HEAD diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:55.973 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:55.976 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,117440512~20971520,146800640~4194304,159383552~37748736,201326592~20971520,226492416~12582912,243269632~8388608,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:55.976 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to HEAD diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,243269632~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:55.976 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,243269632~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:55.978 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~83886080,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,243269632~8388608,255852544~4194304,268435456~20971520,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:55.978 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to HEAD diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~8388608,369098752~25165824,406847488~4194304] 2024-08-17T17:45:55.978 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~8388608,369098752~25165824,406847488~4194304] 2024-08-17T17:45:55.980 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~67108864,75497472~12582912,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,327155712~25165824,356515840~8388608,369098752~29360128,406847488~4194304] 2024-08-17T17:45:55.980 INFO:tasks.workunit.client.0.smithi130.stdout:from 14 to HEAD diff [8388608~62914560,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:55.980 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [8388608~62914560,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~25165824,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~25165824] 2024-08-17T17:45:55.981 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [8388608~62914560,75497472~12582912,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,331350016~8388608,343932928~8388608,356515840~8388608,369098752~29360128] 2024-08-17T17:45:56.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:55 smithi130 ceph-mon[105521]: pgmap v1479: 105 pgs: 105 active+clean; 1.2 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 417 KiB/s rd, 33 MiB/s wr, 816 op/s 2024-08-17T17:45:56.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:55 smithi130 ceph-mon[105521]: osdmap e1141: 8 total, 8 up, 8 in 2024-08-17T17:45:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:55 smithi130 ceph-mon[110068]: pgmap v1479: 105 pgs: 105 active+clean; 1.2 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 417 KiB/s rd, 33 MiB/s wr, 816 op/s 2024-08-17T17:45:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:55 smithi130 ceph-mon[110068]: osdmap e1141: 8 total, 8 up, 8 in 2024-08-17T17:45:56.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:55 smithi179 ceph-mon[93519]: pgmap v1479: 105 pgs: 105 active+clean; 1.2 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 417 KiB/s rd, 33 MiB/s wr, 816 op/s 2024-08-17T17:45:56.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:55 smithi179 ceph-mon[93519]: osdmap e1141: 8 total, 8 up, 8 in 2024-08-17T17:45:56.922 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:56.922 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:56.938 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:56.939 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:56.939 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:56.945 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:56.945 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:56.945 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:56.950 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:56.950 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:56.951 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:56.956 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:56.956 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:56.956 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:56.961 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:45:56.961 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:56.961 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:56.965 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:56.965 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:56.965 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:56.968 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:56.968 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:56.968 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:56.971 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~50331648,327155712~83886080] 2024-08-17T17:45:56.971 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:56.971 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:56.974 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~33554432,293601280~12582912,327155712~83886080] 2024-08-17T17:45:56.975 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:56.975 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:56.977 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~83886080] 2024-08-17T17:45:56.977 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:56.977 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:56.979 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~41943040] 2024-08-17T17:45:56.979 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to HEAD diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:56.979 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:56.981 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,117440512~20971520,146800640~4194304,159383552~37748736,201326592~20971520,226492416~12582912,243269632~8388608,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:56.982 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to HEAD diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,243269632~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:56.982 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,243269632~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:56.983 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~83886080,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,243269632~8388608,255852544~4194304,268435456~20971520,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:56.983 INFO:tasks.workunit.client.0.smithi130.stdout:from 13 to HEAD diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~8388608,369098752~25165824,406847488~4194304] 2024-08-17T17:45:56.983 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~12582912,343932928~8388608,356515840~8388608,369098752~25165824,406847488~4194304] 2024-08-17T17:45:56.984 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~67108864,75497472~12582912,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,255852544~4194304,268435456~20971520,293601280~12582912,327155712~25165824,356515840~8388608,369098752~29360128,406847488~4194304] 2024-08-17T17:45:57.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:56 smithi130 ceph-mon[105521]: osdmap e1142: 8 total, 8 up, 8 in 2024-08-17T17:45:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:56 smithi130 ceph-mon[110068]: osdmap e1142: 8 total, 8 up, 8 in 2024-08-17T17:45:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:56 smithi179 ceph-mon[93519]: osdmap e1142: 8 total, 8 up, 8 in 2024-08-17T17:45:57.927 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:57.927 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:57.944 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:57.944 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:57.944 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:57.950 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:57.950 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:57.950 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:57.955 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:57.955 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:57.955 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:57.960 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:57.960 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:57.960 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:57.964 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:45:57.964 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:57.964 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:57.967 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:57.967 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:57.967 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:57.971 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:57.971 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:57.971 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:57.974 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~50331648,327155712~83886080] 2024-08-17T17:45:57.974 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:57.974 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:57.976 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~33554432,293601280~12582912,327155712~83886080] 2024-08-17T17:45:57.976 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:57.976 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:57.978 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~83886080] 2024-08-17T17:45:57.978 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:57.978 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:57.980 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~41943040] 2024-08-17T17:45:57.980 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to HEAD diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:57.980 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:57.981 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,117440512~20971520,146800640~4194304,159383552~37748736,201326592~20971520,226492416~12582912,243269632~8388608,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:57.982 INFO:tasks.workunit.client.0.smithi130.stdout:from 12 to HEAD diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,243269632~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:57.982 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~8388608,92274688~8388608,117440512~20971520,159383552~4194304,167772160~29360128,205520896~8388608,226492416~8388608,243269632~8388608,255852544~4194304,268435456~16777216,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:57.982 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~83886080,92274688~8388608,117440512~20971520,159383552~37748736,205520896~8388608,226492416~12582912,243269632~8388608,255852544~4194304,268435456~20971520,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:57 smithi130 ceph-mon[105521]: pgmap v1482: 105 pgs: 105 active+clean; 1.2 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 624 KiB/s rd, 29 MiB/s wr, 1.13k op/s 2024-08-17T17:45:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:57 smithi130 ceph-mon[105521]: osdmap e1143: 8 total, 8 up, 8 in 2024-08-17T17:45:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:57 smithi130 ceph-mon[110068]: pgmap v1482: 105 pgs: 105 active+clean; 1.2 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 624 KiB/s rd, 29 MiB/s wr, 1.13k op/s 2024-08-17T17:45:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:57 smithi130 ceph-mon[110068]: osdmap e1143: 8 total, 8 up, 8 in 2024-08-17T17:45:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:58.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:57 smithi179 ceph-mon[93519]: pgmap v1482: 105 pgs: 105 active+clean; 1.2 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 624 KiB/s rd, 29 MiB/s wr, 1.13k op/s 2024-08-17T17:45:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:57 smithi179 ceph-mon[93519]: osdmap e1143: 8 total, 8 up, 8 in 2024-08-17T17:45:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:45:58.931 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:58.931 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:58.952 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:58.952 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:58.952 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:58.957 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:58.957 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:58.957 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:58.962 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:58.962 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:58.962 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:58.966 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:58.966 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:58.966 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:58.970 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:45:58.970 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:58.970 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:58.973 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:58.973 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:58.973 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:58.976 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:58.976 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:58.976 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:58.979 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~50331648,327155712~83886080] 2024-08-17T17:45:58.979 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:58.979 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:58.982 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~33554432,293601280~12582912,327155712~83886080] 2024-08-17T17:45:58.982 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:58.982 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:58.983 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~83886080] 2024-08-17T17:45:58.983 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:58.984 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:58.985 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~41943040] 2024-08-17T17:45:58.985 INFO:tasks.workunit.client.0.smithi130.stdout:from 11 to HEAD diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:58.985 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~20971520,100663296~8388608,117440512~20971520,146800640~4194304,159383552~4194304,167772160~29360128,201326592~20971520,226492416~8388608,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,406847488~4194304] 2024-08-17T17:45:58.986 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,117440512~20971520,146800640~4194304,159383552~37748736,201326592~20971520,226492416~12582912,243269632~8388608,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~29360128,406847488~4194304] 2024-08-17T17:45:59.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:58 smithi179 ceph-mon[93519]: osdmap e1144: 8 total, 8 up, 8 in 2024-08-17T17:45:59.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:58 smithi130 ceph-mon[105521]: osdmap e1144: 8 total, 8 up, 8 in 2024-08-17T17:45:59.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:58 smithi130 ceph-mon[110068]: osdmap e1144: 8 total, 8 up, 8 in 2024-08-17T17:45:59.944 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:59.944 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:45:59.963 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:59.963 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:59.963 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:45:59.967 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:45:59.967 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:59.967 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:45:59.972 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:59.972 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:59.972 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:59.976 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:45:59.976 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:59.976 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:59.979 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:45:59.979 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:59.980 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:59.983 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:59.983 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:59.983 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:45:59.987 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:45:59.987 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:59.987 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:45:59.989 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~50331648,327155712~83886080] 2024-08-17T17:45:59.989 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:59.990 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:59.992 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~33554432,293601280~12582912,327155712~83886080] 2024-08-17T17:45:59.993 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:59.993 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:45:59.994 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~83886080] 2024-08-17T17:45:59.994 INFO:tasks.workunit.client.0.smithi130.stdout:from 10 to HEAD diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:59.994 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~67108864,75497472~33554432,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~37748736,369098752~25165824,398458880~8388608] 2024-08-17T17:45:59.995 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~37748736,369098752~41943040] 2024-08-17T17:46:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:59 smithi130 ceph-mon[105521]: pgmap v1485: 105 pgs: 105 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 470 KiB/s rd, 2.2 MiB/s wr, 846 op/s 2024-08-17T17:46:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:45:59 smithi130 ceph-mon[105521]: osdmap e1145: 8 total, 8 up, 8 in 2024-08-17T17:46:00.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:45:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:45:59] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-08-17T17:46:00.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:59 smithi130 ceph-mon[110068]: pgmap v1485: 105 pgs: 105 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 470 KiB/s rd, 2.2 MiB/s wr, 846 op/s 2024-08-17T17:46:00.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:45:59 smithi130 ceph-mon[110068]: osdmap e1145: 8 total, 8 up, 8 in 2024-08-17T17:46:00.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:59 smithi179 ceph-mon[93519]: pgmap v1485: 105 pgs: 105 active+clean; 1.1 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 470 KiB/s rd, 2.2 MiB/s wr, 846 op/s 2024-08-17T17:46:00.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:45:59 smithi179 ceph-mon[93519]: osdmap e1145: 8 total, 8 up, 8 in 2024-08-17T17:46:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:46:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:46:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:46:00.599 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:00.599 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:00.618 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:00.619 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:00.619 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:00.623 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:00.623 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:00.623 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:00.627 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:00.627 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:00.627 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:00.630 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:00.630 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:00.630 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:00.634 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:46:00.634 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:00.634 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:00.636 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:46:00.637 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:00.637 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:00.642 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:46:00.642 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:46:00.642 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:46:00.642 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~50331648,327155712~83886080] 2024-08-17T17:46:00.642 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:46:00.642 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:46:00.644 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~33554432,293601280~12582912,327155712~83886080] 2024-08-17T17:46:00.644 INFO:tasks.workunit.client.0.smithi130.stdout:from 9 to HEAD diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:46:00.644 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,146800640~4194304,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~4194304,264241152~20971520,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:46:00.645 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~25165824,146800640~4194304,159383552~62914560,226492416~25165824,255852544~4194304,264241152~25165824,293601280~12582912,327155712~83886080] 2024-08-17T17:46:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:00 smithi179 ceph-mon[93519]: osdmap e1146: 8 total, 8 up, 8 in 2024-08-17T17:46:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:00 smithi179 ceph-mon[93519]: osdmap e1147: 8 total, 8 up, 8 in 2024-08-17T17:46:01.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:00 smithi130 ceph-mon[105521]: osdmap e1146: 8 total, 8 up, 8 in 2024-08-17T17:46:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:00 smithi130 ceph-mon[105521]: osdmap e1147: 8 total, 8 up, 8 in 2024-08-17T17:46:01.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:00 smithi130 ceph-mon[110068]: osdmap e1146: 8 total, 8 up, 8 in 2024-08-17T17:46:01.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:00 smithi130 ceph-mon[110068]: osdmap e1147: 8 total, 8 up, 8 in 2024-08-17T17:46:01.598 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:01.598 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:01.615 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:01.616 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:01.616 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:01.619 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:01.619 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:01.619 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:01.623 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:01.623 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:01.623 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:01.626 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:01.626 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:01.626 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:01.629 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:46:01.629 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:01.629 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:01.631 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:46:01.631 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:01.631 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:01.633 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:46:01.633 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:46:01.633 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:46:01.635 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~50331648,327155712~83886080] 2024-08-17T17:46:01.635 INFO:tasks.workunit.client.0.smithi130.stdout:from 8 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:46:01.635 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,293601280~12582912,327155712~67108864,398458880~8388608] 2024-08-17T17:46:01.636 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~33554432,293601280~12582912,327155712~83886080] 2024-08-17T17:46:02.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:01 smithi179 ceph-mon[93519]: pgmap v1488: 105 pgs: 105 active+clean; 1.1 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 403 KiB/s rd, 6.2 KiB/s wr, 730 op/s 2024-08-17T17:46:02.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:01 smithi179 ceph-mon[93519]: osdmap e1148: 8 total, 8 up, 8 in 2024-08-17T17:46:02.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:01 smithi130 ceph-mon[105521]: pgmap v1488: 105 pgs: 105 active+clean; 1.1 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 403 KiB/s rd, 6.2 KiB/s wr, 730 op/s 2024-08-17T17:46:02.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:01 smithi130 ceph-mon[105521]: osdmap e1148: 8 total, 8 up, 8 in 2024-08-17T17:46:02.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:01 smithi130 ceph-mon[110068]: pgmap v1488: 105 pgs: 105 active+clean; 1.1 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 403 KiB/s rd, 6.2 KiB/s wr, 730 op/s 2024-08-17T17:46:02.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:01 smithi130 ceph-mon[110068]: osdmap e1148: 8 total, 8 up, 8 in 2024-08-17T17:46:02.598 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:02.598 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:02.619 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:02.619 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:02.619 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:02.623 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:02.623 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:02.623 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:02.626 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:02.627 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:02.627 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:02.629 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:02.629 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:02.629 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:02.632 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:46:02.632 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:02.632 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:02.634 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:46:02.634 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:02.634 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:02.636 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:46:02.636 INFO:tasks.workunit.client.0.smithi130.stdout:from 7 to HEAD diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:46:02.636 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~12582912,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,327155712~79691776] 2024-08-17T17:46:02.637 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~37748736,159383552~62914560,226492416~25165824,255852544~50331648,327155712~83886080] 2024-08-17T17:46:03.608 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:03.608 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:03.623 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:03.623 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:03.623 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:03.627 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:03.627 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:03.627 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:03.630 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:03.630 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:03.630 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:03.632 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:03.632 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:03.632 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:03.634 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:46:03.634 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:03.634 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:03.635 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:46:03.635 INFO:tasks.workunit.client.0.smithi130.stdout:from 6 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:03.635 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:03.636 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:46:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:03 smithi179 ceph-mon[93519]: pgmap v1491: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 1.1 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 156 KiB/s rd, 5.0 KiB/s wr, 236 op/s 2024-08-17T17:46:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:03 smithi179 ceph-mon[93519]: osdmap e1149: 8 total, 8 up, 8 in 2024-08-17T17:46:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:03 smithi130 ceph-mon[105521]: pgmap v1491: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 1.1 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 156 KiB/s rd, 5.0 KiB/s wr, 236 op/s 2024-08-17T17:46:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:03 smithi130 ceph-mon[105521]: osdmap e1149: 8 total, 8 up, 8 in 2024-08-17T17:46:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:03 smithi130 ceph-mon[110068]: pgmap v1491: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 1.1 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 156 KiB/s rd, 5.0 KiB/s wr, 236 op/s 2024-08-17T17:46:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:03 smithi130 ceph-mon[110068]: osdmap e1149: 8 total, 8 up, 8 in 2024-08-17T17:46:04.614 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:04.615 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:04.629 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:04.629 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:04.629 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:04.633 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:04.633 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:04.633 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:04.635 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:04.635 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:04.635 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:04.637 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:04.637 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:04.637 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:04.639 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:46:04.639 INFO:tasks.workunit.client.0.smithi130.stdout:from 5 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:04.639 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~8388608,255852544~29360128,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:04.640 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~25165824,255852544~50331648,322961408~88080384] 2024-08-17T17:46:04.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:04 smithi130 ceph-mon[105521]: osdmap e1150: 8 total, 8 up, 8 in 2024-08-17T17:46:04.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:04 smithi130 ceph-mon[110068]: osdmap e1150: 8 total, 8 up, 8 in 2024-08-17T17:46:05.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:04 smithi179 ceph-mon[93519]: osdmap e1150: 8 total, 8 up, 8 in 2024-08-17T17:46:05.606 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:05.606 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:05.617 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:05.617 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:05.617 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:05.619 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:05.620 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:05.620 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:05.622 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:05.622 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:05.622 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:05.623 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:05.623 INFO:tasks.workunit.client.0.smithi130.stdout:from 4 to HEAD diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:05.623 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~104857600,113246208~20971520,138412032~16777216,159383552~4194304,167772160~54525952,226492416~12582912,243269632~41943040,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:05.624 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~104857600,113246208~109051904,226492416~79691776,322961408~88080384] 2024-08-17T17:46:05.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:05 smithi130 ceph-mon[105521]: pgmap v1494: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 1.0 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 1.2 KiB/s wr, 169 op/s 2024-08-17T17:46:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:05 smithi130 ceph-mon[105521]: osdmap e1151: 8 total, 8 up, 8 in 2024-08-17T17:46:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:05 smithi130 ceph-mon[105521]: osdmap e1152: 8 total, 8 up, 8 in 2024-08-17T17:46:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:05 smithi130 ceph-mon[110068]: pgmap v1494: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 1.0 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 1.2 KiB/s wr, 169 op/s 2024-08-17T17:46:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:05 smithi130 ceph-mon[110068]: osdmap e1151: 8 total, 8 up, 8 in 2024-08-17T17:46:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:05 smithi130 ceph-mon[110068]: osdmap e1152: 8 total, 8 up, 8 in 2024-08-17T17:46:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:05 smithi179 ceph-mon[93519]: pgmap v1494: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 1.0 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 85 KiB/s rd, 1.2 KiB/s wr, 169 op/s 2024-08-17T17:46:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:05 smithi179 ceph-mon[93519]: osdmap e1151: 8 total, 8 up, 8 in 2024-08-17T17:46:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:05 smithi179 ceph-mon[93519]: osdmap e1152: 8 total, 8 up, 8 in 2024-08-17T17:46:06.624 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:06.624 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:06.635 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:06.636 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:06.636 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:06.638 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:06.639 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:06.639 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:06.640 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:06.640 INFO:tasks.workunit.client.0.smithi130.stdout:from 3 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:06.640 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~29360128,356515840~50331648] 2024-08-17T17:46:06.641 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:07.631 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:07.631 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:07.641 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:07.641 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:07.641 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:07.642 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:07.642 INFO:tasks.workunit.client.0.smithi130.stdout:from 2 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:07.642 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,322961408~83886080] 2024-08-17T17:46:07.644 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,322961408~88080384] 2024-08-17T17:46:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:07 smithi130 ceph-mon[105521]: pgmap v1497: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 1.0 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 KiB/s rd, 6.2 KiB/s wr, 287 op/s 2024-08-17T17:46:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:07 smithi130 ceph-mon[105521]: osdmap e1153: 8 total, 8 up, 8 in 2024-08-17T17:46:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:07 smithi130 ceph-mon[110068]: pgmap v1497: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 1.0 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 KiB/s rd, 6.2 KiB/s wr, 287 op/s 2024-08-17T17:46:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:07 smithi130 ceph-mon[110068]: osdmap e1153: 8 total, 8 up, 8 in 2024-08-17T17:46:08.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:07 smithi179 ceph-mon[93519]: pgmap v1497: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 1.0 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 170 KiB/s rd, 6.2 KiB/s wr, 287 op/s 2024-08-17T17:46:08.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:07 smithi179 ceph-mon[93519]: osdmap e1153: 8 total, 8 up, 8 in 2024-08-17T17:46:08.633 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:08.633 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:08.640 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:08.640 INFO:tasks.workunit.client.0.smithi130.stdout:from 1 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:08.641 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~58720256,289406976~12582912,314572800~92274688] 2024-08-17T17:46:08.642 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:08.865 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:08 smithi179 ceph-mon[93519]: osdmap e1154: 8 total, 8 up, 8 in 2024-08-17T17:46:08.865 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:46:08.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:08 smithi130 ceph-mon[105521]: osdmap e1154: 8 total, 8 up, 8 in 2024-08-17T17:46:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:46:08.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:08 smithi130 ceph-mon[110068]: osdmap e1154: 8 total, 8 up, 8 in 2024-08-17T17:46:08.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:46:09.634 INFO:tasks.workunit.client.0.smithi130.stdout:from 0 to HEAD diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:09.634 INFO:tasks.workunit.client.0.smithi130.stdout: limited diff [4194304~130023424,138412032~16777216,159383552~4194304,167772160~54525952,226492416~75497472,314572800~92274688] 2024-08-17T17:46:09.644 INFO:tasks.workunit.client.0.smithi130.stdout: actual was [4194304~218103808,226492416~79691776,314572800~96468992] 2024-08-17T17:46:09.912 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:09 smithi130 ceph-mon[105521]: pgmap v1500: 105 pgs: 105 active+clean; 918 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 132 KiB/s rd, 5.7 KiB/s wr, 208 op/s 2024-08-17T17:46:09.912 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:09 smithi130 ceph-mon[105521]: osdmap e1155: 8 total, 8 up, 8 in 2024-08-17T17:46:09.913 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:09 smithi130 ceph-mon[110068]: pgmap v1500: 105 pgs: 105 active+clean; 918 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 132 KiB/s rd, 5.7 KiB/s wr, 208 op/s 2024-08-17T17:46:09.913 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:09 smithi130 ceph-mon[110068]: osdmap e1155: 8 total, 8 up, 8 in 2024-08-17T17:46:09.913 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:46:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:46:09] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-08-17T17:46:10.019 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:09 smithi179 ceph-mon[93519]: pgmap v1500: 105 pgs: 105 active+clean; 918 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 132 KiB/s rd, 5.7 KiB/s wr, 208 op/s 2024-08-17T17:46:10.020 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:09 smithi179 ceph-mon[93519]: osdmap e1155: 8 total, 8 up, 8 in 2024-08-17T17:46:10.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:46:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:46:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:46:10.607 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateStress (38833 ms) 2024-08-17T17:46:10.607 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParent 2024-08-17T17:46:10.607 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:46:10.908 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:10 smithi179 ceph-mon[93519]: osdmap e1156: 8 total, 8 up, 8 in 2024-08-17T17:46:10.908 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:10 smithi179 ceph-mon[93519]: osdmap e1157: 8 total, 8 up, 8 in 2024-08-17T17:46:10.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:10 smithi130 ceph-mon[105521]: osdmap e1156: 8 total, 8 up, 8 in 2024-08-17T17:46:10.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:10 smithi130 ceph-mon[105521]: osdmap e1157: 8 total, 8 up, 8 in 2024-08-17T17:46:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:10 smithi130 ceph-mon[110068]: osdmap e1156: 8 total, 8 up, 8 in 2024-08-17T17:46:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:10 smithi130 ceph-mon[110068]: osdmap e1157: 8 total, 8 up, 8 in 2024-08-17T17:46:11.730 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateParent (1123 ms) 2024-08-17T17:46:11.730 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateIgnoreParent 2024-08-17T17:46:11.730 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:46:11.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:11 smithi179 ceph-mon[93519]: pgmap v1503: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 877 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 89 KiB/s rd, 5.7 KiB/s wr, 143 op/s 2024-08-17T17:46:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:11 smithi130 ceph-mon[105521]: pgmap v1503: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 877 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 89 KiB/s rd, 5.7 KiB/s wr, 143 op/s 2024-08-17T17:46:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:11 smithi130 ceph-mon[110068]: pgmap v1503: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 877 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 89 KiB/s rd, 5.7 KiB/s wr, 143 op/s 2024-08-17T17:46:12.750 INFO:tasks.workunit.client.0.smithi130.stdout: wrote [0~4194304,12582912~8388608] to clone 2024-08-17T17:46:12.750 INFO:tasks.workunit.client.0.smithi130.stdout: diff was [0~4194304,12582912~8388608] 2024-08-17T17:46:12.754 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateIgnoreParent (1024 ms) 2024-08-17T17:46:12.754 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateCallbackError 2024-08-17T17:46:12.754 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:46:12.781 INFO:tasks.workunit.client.0.smithi130.stdout: wrote [4432739~98836,12573219~62739,14193537~55086,15306328~32627,17255886~54042,18323304~79565,19087904~30108,20364370~46866] 2024-08-17T17:46:12.785 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateCallbackError (31 ms) 2024-08-17T17:46:12.786 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParentDiscard 2024-08-17T17:46:12.786 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:46:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:12 smithi130 ceph-mon[105521]: osdmap e1158: 8 total, 8 up, 8 in 2024-08-17T17:46:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:12.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:12 smithi130 ceph-mon[110068]: osdmap e1158: 8 total, 8 up, 8 in 2024-08-17T17:46:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:12 smithi179 ceph-mon[93519]: osdmap e1158: 8 total, 8 up, 8 in 2024-08-17T17:46:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:13 smithi130 ceph-mon[105521]: pgmap v1506: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 859 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 60 KiB/s rd, 5.0 KiB/s wr, 87 op/s 2024-08-17T17:46:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:13 smithi130 ceph-mon[105521]: osdmap e1159: 8 total, 8 up, 8 in 2024-08-17T17:46:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:46:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:46:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:13 smithi130 ceph-mon[110068]: pgmap v1506: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 859 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 60 KiB/s rd, 5.0 KiB/s wr, 87 op/s 2024-08-17T17:46:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:13 smithi130 ceph-mon[110068]: osdmap e1159: 8 total, 8 up, 8 in 2024-08-17T17:46:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:46:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:46:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:13 smithi179 ceph-mon[93519]: pgmap v1506: 105 pgs: 1 active+clean+snaptrim, 104 active+clean; 859 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 60 KiB/s rd, 5.0 KiB/s wr, 87 op/s 2024-08-17T17:46:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:13 smithi179 ceph-mon[93519]: osdmap e1159: 8 total, 8 up, 8 in 2024-08-17T17:46:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:46:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:46:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:46:14.803 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateParentDiscard (2018 ms) 2024-08-17T17:46:14.803 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnalignedSmall 2024-08-17T17:46:14.803 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:46:14.850 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnalignedSmall (47 ms) 2024-08-17T17:46:14.850 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnaligned 2024-08-17T17:46:14.850 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:46:14.920 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnaligned (70 ms) 2024-08-17T17:46:14.920 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateTryAcquireLock 2024-08-17T17:46:14.920 INFO:tasks.workunit.client.0.smithi130.stdout:using new format! 2024-08-17T17:46:15.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:14 smithi179 ceph-mon[93519]: osdmap e1160: 8 total, 8 up, 8 in 2024-08-17T17:46:15.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:14 smithi130 ceph-mon[105521]: osdmap e1160: 8 total, 8 up, 8 in 2024-08-17T17:46:15.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:14 smithi130 ceph-mon[110068]: osdmap e1160: 8 total, 8 up, 8 in 2024-08-17T17:46:16.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:15 smithi179 ceph-mon[93519]: pgmap v1509: 105 pgs: 105 active+clean; 807 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.2 MiB/s rd, 7.3 MiB/s wr, 231 op/s 2024-08-17T17:46:16.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:15 smithi179 ceph-mon[93519]: osdmap e1161: 8 total, 8 up, 8 in 2024-08-17T17:46:16.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:15 smithi179 ceph-mon[93519]: osdmap e1162: 8 total, 8 up, 8 in 2024-08-17T17:46:16.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:15 smithi130 ceph-mon[105521]: pgmap v1509: 105 pgs: 105 active+clean; 807 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.2 MiB/s rd, 7.3 MiB/s wr, 231 op/s 2024-08-17T17:46:16.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:15 smithi130 ceph-mon[105521]: osdmap e1161: 8 total, 8 up, 8 in 2024-08-17T17:46:16.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:15 smithi130 ceph-mon[105521]: osdmap e1162: 8 total, 8 up, 8 in 2024-08-17T17:46:16.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:15 smithi130 ceph-mon[110068]: pgmap v1509: 105 pgs: 105 active+clean; 807 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.2 MiB/s rd, 7.3 MiB/s wr, 231 op/s 2024-08-17T17:46:16.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:15 smithi130 ceph-mon[110068]: osdmap e1161: 8 total, 8 up, 8 in 2024-08-17T17:46:16.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:15 smithi130 ceph-mon[110068]: osdmap e1162: 8 total, 8 up, 8 in 2024-08-17T17:46:18.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:17 smithi179 ceph-mon[93519]: pgmap v1512: 105 pgs: 105 active+clean; 814 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 MiB/s rd, 9.8 MiB/s wr, 276 op/s 2024-08-17T17:46:18.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:17 smithi130 ceph-mon[105521]: pgmap v1512: 105 pgs: 105 active+clean; 814 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 MiB/s rd, 9.8 MiB/s wr, 276 op/s 2024-08-17T17:46:18.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:17 smithi130 ceph-mon[110068]: pgmap v1512: 105 pgs: 105 active+clean; 814 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 MiB/s rd, 9.8 MiB/s wr, 276 op/s 2024-08-17T17:46:19.962 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateTryAcquireLock (5042 ms) 2024-08-17T17:46:19.962 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStriping 2024-08-17T17:46:19.962 INFO:tasks.workunit.client.0.smithi130.stdout:SKIPPING 2024-08-17T17:46:19.963 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] DiffIterateTest/1.DiffIterateStriping (0 ms) 2024-08-17T17:46:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:19 smithi179 ceph-mon[93519]: pgmap v1513: 105 pgs: 105 active+clean; 845 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s rd, 11 MiB/s wr, 270 op/s 2024-08-17T17:46:20.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:19 smithi130 ceph-mon[105521]: pgmap v1513: 105 pgs: 105 active+clean; 845 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s rd, 11 MiB/s wr, 270 op/s 2024-08-17T17:46:20.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:19 smithi130 ceph-mon[110068]: pgmap v1513: 105 pgs: 105 active+clean; 845 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.0 MiB/s rd, 11 MiB/s wr, 270 op/s 2024-08-17T17:46:20.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:46:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:46:19] "GET /metrics HTTP/1.1" 200 36580 "" "Prometheus/2.51.0" 2024-08-17T17:46:20.592 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 19 tests from DiffIterateTest/1 (232334 ms total) 2024-08-17T17:46:20.592 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:46:20.592 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] Global test environment tear-down 2024-08-17T17:46:20.592 INFO:tasks.workunit.client.0.smithi130.stdout:[==========] 425 tests from 16 test suites ran. (1395804 ms total) 2024-08-17T17:46:20.593 INFO:tasks.workunit.client.0.smithi130.stdout:[ PASSED ] 425 tests. 2024-08-17T17:46:20.593 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:46:20.593 INFO:tasks.workunit.client.0.smithi130.stdout: YOU HAVE 2 DISABLED TESTS 2024-08-17T17:46:20.593 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T17:46:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:46:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:46:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:46:20.599 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-08-17T17:46:20.599 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-08-17T17:46:20.672 INFO:tasks.workunit:Stopping ['rbd/test_librbd.sh'] on client.0... 2024-08-17T17:46:20.672 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-08-17T17:46:20.933 DEBUG:teuthology.parallel:result is None 2024-08-17T17:46:20.934 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-08-17T17:46:20.958 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-08-17T17:46:20.959 DEBUG:teuthology.orchestra.run.smithi130:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-08-17T17:46:21.301 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-08-17T17:46:21.302 INFO:teuthology.task.sequential:In sequential, running task print... 2024-08-17T17:46:21.302 INFO:teuthology.task.print:**** done rbd/test_librbd.sh 7-workload 2024-08-17T17:46:21.302 DEBUG:teuthology.parallel:result is None 2024-08-17T17:46:21.303 INFO:teuthology.run_tasks:Running task print... 2024-08-17T17:46:21.313 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-08-17T17:46:21.313 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-08-17T17:46:21.322 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-17T17:46:21.322 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-17T17:46:21.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph mgr dump --format=json 2024-08-17T17:46:21.574 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:21.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:21 smithi130 ceph-mon[105521]: pgmap v1514: 105 pgs: 105 active+clean; 860 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 6.4 MiB/s wr, 112 op/s 2024-08-17T17:46:21.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:21 smithi130 ceph-mon[105521]: osdmap e1163: 8 total, 8 up, 8 in 2024-08-17T17:46:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:21 smithi130 ceph-mon[110068]: pgmap v1514: 105 pgs: 105 active+clean; 860 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 6.4 MiB/s wr, 112 op/s 2024-08-17T17:46:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:21 smithi130 ceph-mon[110068]: osdmap e1163: 8 total, 8 up, 8 in 2024-08-17T17:46:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:21 smithi179 ceph-mon[93519]: pgmap v1514: 105 pgs: 105 active+clean; 860 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s rd, 6.4 MiB/s wr, 112 op/s 2024-08-17T17:46:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:21 smithi179 ceph-mon[93519]: osdmap e1163: 8 total, 8 up, 8 in 2024-08-17T17:46:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:46:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:23 smithi179 ceph-mon[93519]: pgmap v1516: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:46:23.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:23 smithi130 ceph-mon[110068]: pgmap v1516: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:46:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:23 smithi130 ceph-mon[105521]: pgmap v1516: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:46:24.477 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":41,"flags":0,"active_gid":25982,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6800","nonce":4195061784},{"type":"v1","addr":"172.21.15.130:6801","nonce":4195061784}]},"active_addr":"172.21.15.130:6801/4195061784","active_change":"2024-08-17T17:26:11.940877+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":25516,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":255,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":4045433689}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":2829009340}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":2438552069}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.130:0","nonce":1411604073}]}]} 2024-08-17T17:46:24.481 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-17T17:46:24.481 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T17:46:24.482 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:46:24.739 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:24.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:24 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3660296892' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-17T17:46:24.861 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:24 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3660296892' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-17T17:46:24.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:24 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3660296892' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-17T17:46:25.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:25 smithi130 ceph-mon[105521]: pgmap v1517: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:46:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:25 smithi130 ceph-mon[110068]: pgmap v1517: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:46:26.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:25 smithi179 ceph-mon[93519]: pgmap v1517: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-08-17T17:46:26.604 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:46:26.604 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":1163,"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","created":"2024-08-17T17:15:09.488012+0000","modified":"2024-08-17T17:46:20.580694+0000","last_up_change":"2024-08-17T17:31:48.768268+0000","last_in_change":"2024-08-17T17:19:46.093156+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":44,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:18:10.192831+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:20:37.124673+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T17:21:51.081217+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T17:21:51.379206+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"30c31adb-2cda-466b-933e-a8e2cb714fab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":423,"up_from":438,"up_thru":1015,"down_at":424,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":3901615741},{"type":"v1","addr":"172.21.15.130:6803","nonce":3901615741}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":3901615741},{"type":"v1","addr":"172.21.15.130:6805","nonce":3901615741}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3901615741},{"type":"v1","addr":"172.21.15.130:6809","nonce":3901615741}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":3901615741},{"type":"v1","addr":"172.21.15.130:6807","nonce":3901615741}]},"public_addr":"172.21.15.130:6803/3901615741","cluster_addr":"172.21.15.130:6805/3901615741","heartbeat_back_addr":"172.21.15.130:6809/3901615741","heartbeat_front_addr":"172.21.15.130:6807/3901615741","state":["exists","up"]},{"osd":1,"uuid":"55049bb4-b976-42c6-8a5d-8b3e9ae20988","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":493,"up_from":506,"up_thru":1015,"down_at":494,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6811","nonce":790699819}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6813","nonce":790699819}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6817","nonce":790699819}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6815","nonce":790699819}]},"public_addr":"172.21.15.130:6811/790699819","cluster_addr":"172.21.15.130:6813/790699819","heartbeat_back_addr":"172.21.15.130:6817/790699819","heartbeat_front_addr":"172.21.15.130:6815/790699819","state":["exists","up"]},{"osd":2,"uuid":"3f40f088-27f7-4450-8f31-b449711365ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":526,"up_from":538,"up_thru":1015,"down_at":527,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":2944523904},{"type":"v1","addr":"172.21.15.130:6819","nonce":2944523904}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":2944523904},{"type":"v1","addr":"172.21.15.130:6821","nonce":2944523904}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2944523904},{"type":"v1","addr":"172.21.15.130:6825","nonce":2944523904}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":2944523904},{"type":"v1","addr":"172.21.15.130:6823","nonce":2944523904}]},"public_addr":"172.21.15.130:6819/2944523904","cluster_addr":"172.21.15.130:6821/2944523904","heartbeat_back_addr":"172.21.15.130:6825/2944523904","heartbeat_front_addr":"172.21.15.130:6823/2944523904","state":["exists","up"]},{"osd":3,"uuid":"7684c401-462a-4e35-bb9c-1be23dfe573b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":558,"up_from":571,"up_thru":1015,"down_at":559,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":3086744673},{"type":"v1","addr":"172.21.15.130:6827","nonce":3086744673}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":3086744673},{"type":"v1","addr":"172.21.15.130:6829","nonce":3086744673}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3086744673},{"type":"v1","addr":"172.21.15.130:6833","nonce":3086744673}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":3086744673},{"type":"v1","addr":"172.21.15.130:6831","nonce":3086744673}]},"public_addr":"172.21.15.130:6827/3086744673","cluster_addr":"172.21.15.130:6829/3086744673","heartbeat_back_addr":"172.21.15.130:6833/3086744673","heartbeat_front_addr":"172.21.15.130:6831/3086744673","state":["exists","up"]},{"osd":4,"uuid":"7a3a331f-533a-49de-910d-5989f2f84b65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":1015,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6800","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6801","nonce":2107797845}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6802","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6803","nonce":2107797845}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6806","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6807","nonce":2107797845}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6804","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6805","nonce":2107797845}]},"public_addr":"172.21.15.179:6801/2107797845","cluster_addr":"172.21.15.179:6803/2107797845","heartbeat_back_addr":"172.21.15.179:6807/2107797845","heartbeat_front_addr":"172.21.15.179:6805/2107797845","state":["exists","up"]},{"osd":5,"uuid":"d53d65ee-bbb2-45ec-b557-07cc62a45b05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":1015,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6808","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6809","nonce":2392026049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6810","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6811","nonce":2392026049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6814","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6815","nonce":2392026049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6812","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6813","nonce":2392026049}]},"public_addr":"172.21.15.179:6809/2392026049","cluster_addr":"172.21.15.179:6811/2392026049","heartbeat_back_addr":"172.21.15.179:6815/2392026049","heartbeat_front_addr":"172.21.15.179:6813/2392026049","state":["exists","up"]},{"osd":6,"uuid":"b84229ad-2c88-4064-a070-33b98714ef96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":1015,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6816","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6817","nonce":139895003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6818","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6819","nonce":139895003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6822","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6823","nonce":139895003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6820","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6821","nonce":139895003}]},"public_addr":"172.21.15.179:6817/139895003","cluster_addr":"172.21.15.179:6819/139895003","heartbeat_back_addr":"172.21.15.179:6823/139895003","heartbeat_front_addr":"172.21.15.179:6821/139895003","state":["exists","up"]},{"osd":7,"uuid":"bba7978f-8b19-40e6-86bd-692a0e282429","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":1015,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6824","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6825","nonce":2533639201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6826","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6827","nonce":2533639201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6830","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6831","nonce":2533639201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6828","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6829","nonce":2533639201}]},"public_addr":"172.21.15.179:6825/2533639201","cluster_addr":"172.21.15.179:6827/2533639201","heartbeat_back_addr":"172.21.15.179:6831/2533639201","heartbeat_front_addr":"172.21.15.179:6829/2533639201","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:19.220776+0000","dead_epoch":423},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:42.876683+0000","dead_epoch":493},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:07.028539+0000","dead_epoch":526},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:31.259509+0000","dead_epoch":558},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:52.172235+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:15.112109+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:37.781476+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:20:00.893216+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/485089907":"2024-08-17T18:31:20.602503+0000","172.21.15.130:6801/3641086954":"2024-08-18T17:26:11.940622+0000","172.21.15.130:6800/3641086954":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/2447795507":"2024-08-17T18:31:19.611103+0000","172.21.15.130:0/851371227":"2024-08-18T17:20:33.507974+0000","172.21.15.179:0/393594060":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/3071443649":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2578011752":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/1457627030":"2024-08-18T17:24:24.724005+0000","172.21.15.130:0/2870283877":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/3943801420":"2024-08-18T17:26:11.940622+0000","172.21.15.179:0/2395764912":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/1299294781":"2024-08-18T17:24:24.724005+0000","172.21.15.179:0/1781739324":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/1690071627":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/2991152682":"2024-08-18T17:26:11.940622+0000","172.21.15.130:6801/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2810193027":"2024-08-18T17:26:11.940622+0000","172.21.15.179:0/56636976":"2024-08-18T17:25:30.337581+0000","172.21.15.130:6800/824081166":"2024-08-18T17:24:24.724005+0000","172.21.15.130:0/2589122337":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/1247136467":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/4037898622":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/3453310625":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/3471387839":"2024-08-18T17:15:57.894717+0000","172.21.15.130:6801/824081166":"2024-08-18T17:24:24.724005+0000","172.21.15.130:0/368845317":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/695423662":"2024-08-17T18:36:26.582385+0000","172.21.15.130:0/2999329360":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6800/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.179:0/196590495":"2024-08-18T17:25:30.337581+0000","172.21.15.130:6800/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/854839742":"2024-08-18T17:24:24.724005+0000","172.21.15.179:6834/3911752105":"2024-08-18T17:25:30.337581+0000","172.21.15.130:6801/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.179:0/2399909254":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/1843620043":"2024-08-18T17:24:24.724005+0000","172.21.15.130:6800/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6801/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/88320168":"2024-08-18T17:15:34.935061+0000","172.21.15.179:6835/3911752105":"2024-08-18T17:25:30.337581+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":14,"snaps":[{"begin":8,"length":1}]},{"pool":16,"snaps":[{"begin":35,"length":3},{"begin":44,"length":4}]},{"pool":17,"snaps":[{"begin":17,"length":2},{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":44,"snaps":[{"begin":104,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:46:27.200 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T17:46:27.200 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:46:27.455 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:27.902 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:27 smithi130 ceph-mon[105521]: pgmap v1518: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:27.902 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:27 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3838822656' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:46:27.902 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:27.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:27 smithi130 ceph-mon[110068]: pgmap v1518: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:27.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:27 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3838822656' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:46:27.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:27 smithi179 ceph-mon[93519]: pgmap v1518: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:27 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3838822656' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:46:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:29.456 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:46:29.456 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":1163,"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","created":"2024-08-17T17:15:09.488012+0000","modified":"2024-08-17T17:46:20.580694+0000","last_up_change":"2024-08-17T17:31:48.768268+0000","last_in_change":"2024-08-17T17:19:46.093156+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":44,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:18:10.192831+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:20:37.124673+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T17:21:51.081217+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T17:21:51.379206+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"30c31adb-2cda-466b-933e-a8e2cb714fab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":423,"up_from":438,"up_thru":1015,"down_at":424,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":3901615741},{"type":"v1","addr":"172.21.15.130:6803","nonce":3901615741}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":3901615741},{"type":"v1","addr":"172.21.15.130:6805","nonce":3901615741}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":3901615741},{"type":"v1","addr":"172.21.15.130:6809","nonce":3901615741}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":3901615741},{"type":"v1","addr":"172.21.15.130:6807","nonce":3901615741}]},"public_addr":"172.21.15.130:6803/3901615741","cluster_addr":"172.21.15.130:6805/3901615741","heartbeat_back_addr":"172.21.15.130:6809/3901615741","heartbeat_front_addr":"172.21.15.130:6807/3901615741","state":["exists","up"]},{"osd":1,"uuid":"55049bb4-b976-42c6-8a5d-8b3e9ae20988","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":493,"up_from":506,"up_thru":1015,"down_at":494,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6811","nonce":790699819}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6813","nonce":790699819}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6817","nonce":790699819}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6815","nonce":790699819}]},"public_addr":"172.21.15.130:6811/790699819","cluster_addr":"172.21.15.130:6813/790699819","heartbeat_back_addr":"172.21.15.130:6817/790699819","heartbeat_front_addr":"172.21.15.130:6815/790699819","state":["exists","up"]},{"osd":2,"uuid":"3f40f088-27f7-4450-8f31-b449711365ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":526,"up_from":538,"up_thru":1015,"down_at":527,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":2944523904},{"type":"v1","addr":"172.21.15.130:6819","nonce":2944523904}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":2944523904},{"type":"v1","addr":"172.21.15.130:6821","nonce":2944523904}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":2944523904},{"type":"v1","addr":"172.21.15.130:6825","nonce":2944523904}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":2944523904},{"type":"v1","addr":"172.21.15.130:6823","nonce":2944523904}]},"public_addr":"172.21.15.130:6819/2944523904","cluster_addr":"172.21.15.130:6821/2944523904","heartbeat_back_addr":"172.21.15.130:6825/2944523904","heartbeat_front_addr":"172.21.15.130:6823/2944523904","state":["exists","up"]},{"osd":3,"uuid":"7684c401-462a-4e35-bb9c-1be23dfe573b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":558,"up_from":571,"up_thru":1015,"down_at":559,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":3086744673},{"type":"v1","addr":"172.21.15.130:6827","nonce":3086744673}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":3086744673},{"type":"v1","addr":"172.21.15.130:6829","nonce":3086744673}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3086744673},{"type":"v1","addr":"172.21.15.130:6833","nonce":3086744673}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":3086744673},{"type":"v1","addr":"172.21.15.130:6831","nonce":3086744673}]},"public_addr":"172.21.15.130:6827/3086744673","cluster_addr":"172.21.15.130:6829/3086744673","heartbeat_back_addr":"172.21.15.130:6833/3086744673","heartbeat_front_addr":"172.21.15.130:6831/3086744673","state":["exists","up"]},{"osd":4,"uuid":"7a3a331f-533a-49de-910d-5989f2f84b65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":1015,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6800","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6801","nonce":2107797845}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6802","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6803","nonce":2107797845}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6806","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6807","nonce":2107797845}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6804","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6805","nonce":2107797845}]},"public_addr":"172.21.15.179:6801/2107797845","cluster_addr":"172.21.15.179:6803/2107797845","heartbeat_back_addr":"172.21.15.179:6807/2107797845","heartbeat_front_addr":"172.21.15.179:6805/2107797845","state":["exists","up"]},{"osd":5,"uuid":"d53d65ee-bbb2-45ec-b557-07cc62a45b05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":1015,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6808","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6809","nonce":2392026049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6810","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6811","nonce":2392026049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6814","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6815","nonce":2392026049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6812","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6813","nonce":2392026049}]},"public_addr":"172.21.15.179:6809/2392026049","cluster_addr":"172.21.15.179:6811/2392026049","heartbeat_back_addr":"172.21.15.179:6815/2392026049","heartbeat_front_addr":"172.21.15.179:6813/2392026049","state":["exists","up"]},{"osd":6,"uuid":"b84229ad-2c88-4064-a070-33b98714ef96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":1015,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6816","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6817","nonce":139895003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6818","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6819","nonce":139895003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6822","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6823","nonce":139895003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6820","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6821","nonce":139895003}]},"public_addr":"172.21.15.179:6817/139895003","cluster_addr":"172.21.15.179:6819/139895003","heartbeat_back_addr":"172.21.15.179:6823/139895003","heartbeat_front_addr":"172.21.15.179:6821/139895003","state":["exists","up"]},{"osd":7,"uuid":"bba7978f-8b19-40e6-86bd-692a0e282429","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":1015,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6824","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6825","nonce":2533639201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6826","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6827","nonce":2533639201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6830","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6831","nonce":2533639201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6828","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6829","nonce":2533639201}]},"public_addr":"172.21.15.179:6825/2533639201","cluster_addr":"172.21.15.179:6827/2533639201","heartbeat_back_addr":"172.21.15.179:6831/2533639201","heartbeat_front_addr":"172.21.15.179:6829/2533639201","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:19.220776+0000","dead_epoch":423},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:42.876683+0000","dead_epoch":493},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:07.028539+0000","dead_epoch":526},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:31.259509+0000","dead_epoch":558},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:52.172235+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:15.112109+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:37.781476+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:20:00.893216+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/485089907":"2024-08-17T18:31:20.602503+0000","172.21.15.130:6801/3641086954":"2024-08-18T17:26:11.940622+0000","172.21.15.130:6800/3641086954":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/3453310625":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/2870283877":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/3943801420":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/2991152682":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/2447795507":"2024-08-17T18:31:19.611103+0000","172.21.15.130:0/851371227":"2024-08-18T17:20:33.507974+0000","172.21.15.130:6800/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/4037898622":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/1247136467":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2578011752":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/1457627030":"2024-08-18T17:24:24.724005+0000","172.21.15.130:0/1299294781":"2024-08-18T17:24:24.724005+0000","172.21.15.179:0/1781739324":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/1690071627":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/2810193027":"2024-08-18T17:26:11.940622+0000","172.21.15.130:6801/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.179:0/56636976":"2024-08-18T17:25:30.337581+0000","172.21.15.130:6800/824081166":"2024-08-18T17:24:24.724005+0000","172.21.15.130:6800/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/695423662":"2024-08-17T18:36:26.582385+0000","172.21.15.130:0/2999329360":"2024-08-18T17:15:34.935061+0000","172.21.15.179:0/196590495":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/2589122337":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/368845317":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/3071443649":"2024-08-18T17:15:57.894717+0000","172.21.15.179:0/2399909254":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/1843620043":"2024-08-18T17:24:24.724005+0000","172.21.15.130:6800/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6801/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/88320168":"2024-08-18T17:15:34.935061+0000","172.21.15.179:6835/3911752105":"2024-08-18T17:25:30.337581+0000","172.21.15.179:0/2395764912":"2024-08-18T17:25:30.337581+0000","172.21.15.179:6834/3911752105":"2024-08-18T17:25:30.337581+0000","172.21.15.130:6801/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/854839742":"2024-08-18T17:24:24.724005+0000","172.21.15.130:6801/824081166":"2024-08-18T17:24:24.724005+0000","172.21.15.130:0/3471387839":"2024-08-18T17:15:57.894717+0000","172.21.15.179:0/393594060":"2024-08-18T17:25:30.337581+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":14,"snaps":[{"begin":8,"length":1}]},{"pool":16,"snaps":[{"begin":35,"length":3},{"begin":44,"length":4}]},{"pool":17,"snaps":[{"begin":17,"length":2},{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[{"pool":44,"snaps":[{"begin":104,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:46:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:29 smithi130 ceph-mon[105521]: pgmap v1519: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:29.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3600470572' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:46:29.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:46:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:46:29] "GET /metrics HTTP/1.1" 200 34146 "" "Prometheus/2.51.0" 2024-08-17T17:46:29.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:29 smithi130 ceph-mon[110068]: pgmap v1519: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:29.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3600470572' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:46:30.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T17:46:30.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T17:46:30.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T17:46:30.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T17:46:30.081 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T17:46:30.081 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T17:46:30.082 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T17:46:30.082 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T17:46:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:29 smithi179 ceph-mon[93519]: pgmap v1519: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3600470572' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:46:30.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:46:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:46:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:46:30.668 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:30.668 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:30.669 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:30.672 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:30.676 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:30.676 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:30.679 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:30.683 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:31 smithi130 ceph-mon[105521]: pgmap v1520: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:31 smithi130 ceph-mon[110068]: pgmap v1520: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:31 smithi179 ceph-mon[93519]: pgmap v1520: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:33.807 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:33 smithi130 ceph-mon[105521]: pgmap v1521: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:33.807 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:33 smithi130 ceph-mon[110068]: pgmap v1521: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:34.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:33 smithi179 ceph-mon[93519]: pgmap v1521: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:35.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:35 smithi130 ceph-mon[105521]: pgmap v1522: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:35.886 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:35 smithi130 ceph-mon[110068]: pgmap v1522: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:35 smithi179 ceph-mon[93519]: pgmap v1522: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:37.889 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:37 smithi130 ceph-mon[105521]: pgmap v1523: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:37.889 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:37 smithi130 ceph-mon[110068]: pgmap v1523: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:37 smithi179 ceph-mon[93519]: pgmap v1523: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:39.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:39 smithi130 ceph-mon[105521]: pgmap v1524: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:39 smithi130 ceph-mon[110068]: pgmap v1524: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:39.887 INFO:teuthology.orchestra.run.smithi130.stdout:128849019217 2024-08-17T17:46:39.888 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T17:46:39.986 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:46:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:46:39] "GET /metrics HTTP/1.1" 200 34146 "" "Prometheus/2.51.0" 2024-08-17T17:46:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:39 smithi179 ceph-mon[93519]: pgmap v1524: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:46:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:46:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:46:41.876 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:41 smithi130 ceph-mon[105521]: pgmap v1525: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:41.877 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:41 smithi130 ceph-mon[110068]: pgmap v1525: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:41 smithi179 ceph-mon[93519]: pgmap v1525: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:42.606 INFO:teuthology.orchestra.run.smithi130.stdout:231928234309 2024-08-17T17:46:42.606 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T17:46:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:43.732 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:43 smithi130 ceph-mon[105521]: pgmap v1526: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:43.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:43 smithi130 ceph-mon[110068]: pgmap v1526: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:43.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:43 smithi179 ceph-mon[93519]: pgmap v1526: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:44.040 INFO:teuthology.orchestra.run.smithi130.stdout:163208757582 2024-08-17T17:46:44.041 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T17:46:44.433 INFO:teuthology.orchestra.run.smithi130.stdout:197568495945 2024-08-17T17:46:44.433 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T17:46:45.078 INFO:teuthology.orchestra.run.smithi130.stdout:2310692405435 2024-08-17T17:46:45.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T17:46:45.440 INFO:teuthology.orchestra.run.smithi130.stdout:2173253451969 2024-08-17T17:46:45.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T17:46:45.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:45 smithi130 ceph-mon[105521]: pgmap v1527: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:45 smithi130 ceph-mon[110068]: pgmap v1527: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:45 smithi179 ceph-mon[93519]: pgmap v1527: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:46.320 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:46.320 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:46.684 INFO:teuthology.orchestra.run.smithi130.stdout:1881195675856 2024-08-17T17:46:46.685 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T17:46:46.697 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:47.361 INFO:teuthology.orchestra.run.smithi130.stdout:2452426326197 2024-08-17T17:46:47.362 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T17:46:47.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:47 smithi130 ceph-mon[105521]: pgmap v1528: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:47.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:47 smithi130 ceph-mon[110068]: pgmap v1528: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:47 smithi179 ceph-mon[93519]: pgmap v1528: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:49.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:49 smithi130 ceph-mon[105521]: pgmap v1529: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:49.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:49 smithi130 ceph-mon[110068]: pgmap v1529: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:49.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:46:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:46:49] "GET /metrics HTTP/1.1" 200 34148 "" "Prometheus/2.51.0" 2024-08-17T17:46:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:49 smithi179 ceph-mon[93519]: pgmap v1529: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:46:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:46:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:46:50.759 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:50.759 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:50.956 INFO:teuthology.orchestra.run.smithi130.stdout:163208757584 2024-08-17T17:46:51.839 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:51 smithi130 ceph-mon[105521]: pgmap v1530: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:51.840 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:51 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1244699235' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:46:51.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:51 smithi130 ceph-mon[110068]: pgmap v1530: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:51.840 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:51 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1244699235' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:46:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:51 smithi179 ceph-mon[93519]: pgmap v1530: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:51 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1244699235' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:46:52.963 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757582 got 163208757584 for osd.5 2024-08-17T17:46:52.963 DEBUG:teuthology.parallel:result is None 2024-08-17T17:46:53.578 INFO:teuthology.orchestra.run.smithi130.stdout:231928234311 2024-08-17T17:46:53.626 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:53.626 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:53.796 INFO:teuthology.orchestra.run.smithi130.stdout:128849019219 2024-08-17T17:46:53.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:53 smithi130 ceph-mon[105521]: pgmap v1531: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:53.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:53 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2245511160' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:46:53.796 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:53 smithi130 ceph-mon[110068]: pgmap v1531: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:53.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:53 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2245511160' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:46:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:53 smithi179 ceph-mon[93519]: pgmap v1531: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:53 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2245511160' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:46:54.373 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234309 got 231928234311 for osd.7 2024-08-17T17:46:54.373 DEBUG:teuthology.parallel:result is None 2024-08-17T17:46:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1716205485' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:46:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1716205485' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:46:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1716205485' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:46:55.727 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:46:55.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:55 smithi130 ceph-mon[110068]: pgmap v1532: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:55 smithi130 ceph-mon[105521]: pgmap v1532: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:55 smithi179 ceph-mon[93519]: pgmap v1532: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:57.381 INFO:teuthology.orchestra.run.smithi130.stdout:197568495947 2024-08-17T17:46:57.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:57 smithi130 ceph-mon[105521]: pgmap v1533: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:57 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1776347830' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:46:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:57 smithi130 ceph-mon[110068]: pgmap v1533: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:57 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1776347830' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:46:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:57.956 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495945 got 197568495947 for osd.6 2024-08-17T17:46:57.956 DEBUG:teuthology.parallel:result is None 2024-08-17T17:46:58.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:57 smithi179 ceph-mon[93519]: pgmap v1533: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:58.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:57 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1776347830' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:46:58.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:46:59.119 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019217 got 128849019219 for osd.4 2024-08-17T17:46:59.119 DEBUG:teuthology.parallel:result is None 2024-08-17T17:46:59.764 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:46:59 smithi130 ceph-mon[105521]: pgmap v1534: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:46:59.765 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:46:59 smithi130 ceph-mon[110068]: pgmap v1534: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:00.034 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:46:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:46:59] "GET /metrics HTTP/1.1" 200 34151 "" "Prometheus/2.51.0" 2024-08-17T17:47:00.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:46:59 smithi179 ceph-mon[93519]: pgmap v1534: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:00.134 INFO:teuthology.orchestra.run.smithi130.stdout:2173253451973 2024-08-17T17:47:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:47:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:47:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:47:00.872 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:00 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2467434361' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:47:00.872 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:00 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2467434361' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:47:00.897 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2173253451969 got 2173253451973 for osd.1 2024-08-17T17:47:00.897 DEBUG:teuthology.parallel:result is None 2024-08-17T17:47:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:00 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2467434361' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:47:01.655 INFO:teuthology.orchestra.run.smithi130.stdout:2310692405439 2024-08-17T17:47:01.872 INFO:teuthology.orchestra.run.smithi130.stdout:1881195675860 2024-08-17T17:47:01.873 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:01 smithi130 ceph-mon[105521]: pgmap v1535: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:01.873 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:01 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/397440230' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:47:01.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:01 smithi130 ceph-mon[110068]: pgmap v1535: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:01.873 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:01 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/397440230' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:47:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:01 smithi179 ceph-mon[93519]: pgmap v1535: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:01 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/397440230' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:47:02.433 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1881195675856 got 1881195675860 for osd.0 2024-08-17T17:47:02.433 DEBUG:teuthology.parallel:result is None 2024-08-17T17:47:02.751 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:02 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3137296664' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:47:02.752 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:02 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3137296664' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:47:02.993 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2310692405435 got 2310692405439 for osd.2 2024-08-17T17:47:02.993 DEBUG:teuthology.parallel:result is None 2024-08-17T17:47:03.020 INFO:teuthology.orchestra.run.smithi130.stdout:2452426326200 2024-08-17T17:47:03.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:02 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3137296664' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:47:03.654 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2452426326197 got 2452426326200 for osd.3 2024-08-17T17:47:03.655 DEBUG:teuthology.parallel:result is None 2024-08-17T17:47:03.655 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T17:47:03.655 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:47:03.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:03 smithi130 ceph-mon[105521]: pgmap v1536: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:03 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2403262425' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:47:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:03 smithi130 ceph-mon[110068]: pgmap v1536: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:03 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2403262425' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:47:03.949 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:04.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:03 smithi179 ceph-mon[93519]: pgmap v1536: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:04.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:03 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2403262425' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:47:05.661 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:47:05.662 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:47:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:05 smithi130 ceph-mon[105521]: pgmap v1537: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:05 smithi130 ceph-mon[110068]: pgmap v1537: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:05 smithi179 ceph-mon[93519]: pgmap v1537: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:06.277 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1537,"stamp":"2024-08-17T17:47:04.376441+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1580024,"kb_used_data":153440,"kb_used_omap":223,"kb_used_meta":1426272,"kb_avail":748381192,"statfs":{"total":767960285184,"available":766342340608,"internally_reserved":0,"allocated":157122560,"data_stored":97033139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":228359,"internal_metadata":1460503545},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002875"},"pg_stats":[{"pgid":"3.1f","version":"71'4","reported_seq":1819,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918108+0000","last_change":"2024-08-17T17:31:15.551224+0000","last_active":"2024-08-17T17:46:20.918108+0000","last_peered":"2024-08-17T17:46:20.918108+0000","last_clean":"2024-08-17T17:46:20.918108+0000","last_became_active":"2024-08-17T17:31:15.550855+0000","last_became_peered":"2024-08-17T17:31:15.550855+0000","last_unstale":"2024-08-17T17:46:20.918108+0000","last_undegraded":"2024-08-17T17:46:20.918108+0000","last_fullsized":"2024-08-17T17:46:20.918108+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:05:07.081474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009009+0000","last_change":"2024-08-17T17:30:43.429215+0000","last_active":"2024-08-17T17:46:16.009009+0000","last_peered":"2024-08-17T17:46:16.009009+0000","last_clean":"2024-08-17T17:46:16.009009+0000","last_became_active":"2024-08-17T17:30:43.428891+0000","last_became_peered":"2024-08-17T17:30:43.428891+0000","last_unstale":"2024-08-17T17:46:16.009009+0000","last_undegraded":"2024-08-17T17:46:16.009009+0000","last_fullsized":"2024-08-17T17:46:16.009009+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:41:39.313325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1698,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911021+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:46:20.911021+0000","last_peered":"2024-08-17T17:46:20.911021+0000","last_clean":"2024-08-17T17:46:20.911021+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:46:20.911021+0000","last_undegraded":"2024-08-17T17:46:20.911021+0000","last_fullsized":"2024-08-17T17:46:20.911021+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1712,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911837+0000","last_change":"2024-08-17T17:31:49.809768+0000","last_active":"2024-08-17T17:46:20.911837+0000","last_peered":"2024-08-17T17:46:20.911837+0000","last_clean":"2024-08-17T17:46:20.911837+0000","last_became_active":"2024-08-17T17:31:49.808276+0000","last_became_peered":"2024-08-17T17:31:49.808276+0000","last_unstale":"2024-08-17T17:46:20.911837+0000","last_undegraded":"2024-08-17T17:46:20.911837+0000","last_fullsized":"2024-08-17T17:46:20.911837+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:51.600719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1103,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070283+0000","last_change":"2024-08-17T17:21:52.157731+0000","last_active":"2024-08-17T17:46:16.070283+0000","last_peered":"2024-08-17T17:46:16.070283+0000","last_clean":"2024-08-17T17:46:16.070283+0000","last_became_active":"2024-08-17T17:21:52.157453+0000","last_became_peered":"2024-08-17T17:21:52.157453+0000","last_unstale":"2024-08-17T17:46:16.070283+0000","last_undegraded":"2024-08-17T17:46:16.070283+0000","last_fullsized":"2024-08-17T17:46:16.070283+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:30:09.106088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009219+0000","last_change":"2024-08-17T17:31:49.806060+0000","last_active":"2024-08-17T17:46:16.009219+0000","last_peered":"2024-08-17T17:46:16.009219+0000","last_clean":"2024-08-17T17:46:16.009219+0000","last_became_active":"2024-08-17T17:31:49.805688+0000","last_became_peered":"2024-08-17T17:31:49.805688+0000","last_unstale":"2024-08-17T17:46:16.009219+0000","last_undegraded":"2024-08-17T17:46:16.009219+0000","last_fullsized":"2024-08-17T17:46:16.009219+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:42.831662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1120,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070144+0000","last_change":"2024-08-17T17:30:43.427884+0000","last_active":"2024-08-17T17:46:16.070144+0000","last_peered":"2024-08-17T17:46:16.070144+0000","last_clean":"2024-08-17T17:46:16.070144+0000","last_became_active":"2024-08-17T17:30:43.427497+0000","last_became_peered":"2024-08-17T17:30:43.427497+0000","last_unstale":"2024-08-17T17:46:16.070144+0000","last_undegraded":"2024-08-17T17:46:16.070144+0000","last_fullsized":"2024-08-17T17:46:16.070144+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:22.830092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009797+0000","last_change":"2024-08-17T17:31:49.799570+0000","last_active":"2024-08-17T17:46:16.009797+0000","last_peered":"2024-08-17T17:46:16.009797+0000","last_clean":"2024-08-17T17:46:16.009797+0000","last_became_active":"2024-08-17T17:31:49.799251+0000","last_became_peered":"2024-08-17T17:31:49.799251+0000","last_unstale":"2024-08-17T17:46:16.009797+0000","last_undegraded":"2024-08-17T17:46:16.009797+0000","last_fullsized":"2024-08-17T17:46:16.009797+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:58.973574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1804,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.920819+0000","last_change":"2024-08-17T17:29:31.897430+0000","last_active":"2024-08-17T17:46:20.920819+0000","last_peered":"2024-08-17T17:46:20.920819+0000","last_clean":"2024-08-17T17:46:20.920819+0000","last_became_active":"2024-08-17T17:29:31.896722+0000","last_became_peered":"2024-08-17T17:29:31.896722+0000","last_unstale":"2024-08-17T17:46:20.920819+0000","last_undegraded":"2024-08-17T17:46:20.920819+0000","last_fullsized":"2024-08-17T17:46:20.920819+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:41:41.942213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1740,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912786+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:46:20.912786+0000","last_peered":"2024-08-17T17:46:20.912786+0000","last_clean":"2024-08-17T17:46:20.912786+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:46:20.912786+0000","last_undegraded":"2024-08-17T17:46:20.912786+0000","last_fullsized":"2024-08-17T17:46:20.912786+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1135,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009572+0000","last_change":"2024-08-17T17:31:15.547327+0000","last_active":"2024-08-17T17:46:16.009572+0000","last_peered":"2024-08-17T17:46:16.009572+0000","last_clean":"2024-08-17T17:46:16.009572+0000","last_became_active":"2024-08-17T17:31:15.546959+0000","last_became_peered":"2024-08-17T17:31:15.546959+0000","last_unstale":"2024-08-17T17:46:16.009572+0000","last_undegraded":"2024-08-17T17:46:16.009572+0000","last_fullsized":"2024-08-17T17:46:16.009572+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:27:49.216367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910954+0000","last_change":"2024-08-17T17:31:49.802098+0000","last_active":"2024-08-17T17:46:20.910954+0000","last_peered":"2024-08-17T17:46:20.910954+0000","last_clean":"2024-08-17T17:46:20.910954+0000","last_became_active":"2024-08-17T17:31:49.801747+0000","last_became_peered":"2024-08-17T17:31:49.801747+0000","last_unstale":"2024-08-17T17:46:20.910954+0000","last_undegraded":"2024-08-17T17:46:20.910954+0000","last_fullsized":"2024-08-17T17:46:20.910954+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:00.212402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1759,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908264+0000","last_change":"2024-08-17T17:31:49.815142+0000","last_active":"2024-08-17T17:46:20.908264+0000","last_peered":"2024-08-17T17:46:20.908264+0000","last_clean":"2024-08-17T17:46:20.908264+0000","last_became_active":"2024-08-17T17:31:49.814873+0000","last_became_peered":"2024-08-17T17:31:49.814873+0000","last_unstale":"2024-08-17T17:46:20.908264+0000","last_undegraded":"2024-08-17T17:46:20.908264+0000","last_fullsized":"2024-08-17T17:46:20.908264+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:05.542100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1812,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.919007+0000","last_change":"2024-08-17T17:31:49.804738+0000","last_active":"2024-08-17T17:46:20.919007+0000","last_peered":"2024-08-17T17:46:20.919007+0000","last_clean":"2024-08-17T17:46:20.919007+0000","last_became_active":"2024-08-17T17:31:49.804476+0000","last_became_peered":"2024-08-17T17:31:49.804476+0000","last_unstale":"2024-08-17T17:46:20.919007+0000","last_undegraded":"2024-08-17T17:46:20.919007+0000","last_fullsized":"2024-08-17T17:46:20.919007+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:51.659079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1723,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909025+0000","last_change":"2024-08-17T17:31:49.808285+0000","last_active":"2024-08-17T17:46:20.909025+0000","last_peered":"2024-08-17T17:46:20.909025+0000","last_clean":"2024-08-17T17:46:20.909025+0000","last_became_active":"2024-08-17T17:31:49.805633+0000","last_became_peered":"2024-08-17T17:31:49.805633+0000","last_unstale":"2024-08-17T17:46:20.909025+0000","last_undegraded":"2024-08-17T17:46:20.909025+0000","last_fullsized":"2024-08-17T17:46:20.909025+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:38.815825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1113,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583821+0000","last_change":"2024-08-17T17:30:43.429169+0000","last_active":"2024-08-17T17:46:15.583821+0000","last_peered":"2024-08-17T17:46:15.583821+0000","last_clean":"2024-08-17T17:46:15.583821+0000","last_became_active":"2024-08-17T17:30:43.428850+0000","last_became_peered":"2024-08-17T17:30:43.428850+0000","last_unstale":"2024-08-17T17:46:15.583821+0000","last_undegraded":"2024-08-17T17:46:15.583821+0000","last_fullsized":"2024-08-17T17:46:15.583821+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:51:44.920103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908763+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:46:20.908763+0000","last_peered":"2024-08-17T17:46:20.908763+0000","last_clean":"2024-08-17T17:46:20.908763+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:46:20.908763+0000","last_undegraded":"2024-08-17T17:46:20.908763+0000","last_fullsized":"2024-08-17T17:46:20.908763+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918062+0000","last_change":"2024-08-17T17:31:49.805397+0000","last_active":"2024-08-17T17:46:20.918062+0000","last_peered":"2024-08-17T17:46:20.918062+0000","last_clean":"2024-08-17T17:46:20.918062+0000","last_became_active":"2024-08-17T17:31:49.804586+0000","last_became_peered":"2024-08-17T17:31:49.804586+0000","last_unstale":"2024-08-17T17:46:20.918062+0000","last_undegraded":"2024-08-17T17:46:20.918062+0000","last_fullsized":"2024-08-17T17:46:20.918062+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:58.697361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1661,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911739+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:46:20.911739+0000","last_peered":"2024-08-17T17:46:20.911739+0000","last_clean":"2024-08-17T17:46:20.911739+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:46:20.911739+0000","last_undegraded":"2024-08-17T17:46:20.911739+0000","last_fullsized":"2024-08-17T17:46:20.911739+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1118,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070708+0000","last_change":"2024-08-17T17:30:43.429232+0000","last_active":"2024-08-17T17:46:16.070708+0000","last_peered":"2024-08-17T17:46:16.070708+0000","last_clean":"2024-08-17T17:46:16.070708+0000","last_became_active":"2024-08-17T17:30:43.428944+0000","last_became_peered":"2024-08-17T17:30:43.428944+0000","last_unstale":"2024-08-17T17:46:16.070708+0000","last_undegraded":"2024-08-17T17:46:16.070708+0000","last_fullsized":"2024-08-17T17:46:16.070708+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:05.780222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798470+0000","last_change":"2024-08-17T17:31:49.801654+0000","last_active":"2024-08-17T17:46:15.798470+0000","last_peered":"2024-08-17T17:46:15.798470+0000","last_clean":"2024-08-17T17:46:15.798470+0000","last_became_active":"2024-08-17T17:31:49.801234+0000","last_became_peered":"2024-08-17T17:31:49.801234+0000","last_unstale":"2024-08-17T17:46:15.798470+0000","last_undegraded":"2024-08-17T17:46:15.798470+0000","last_fullsized":"2024-08-17T17:46:15.798470+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:21.590996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1760,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910427+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:46:20.910427+0000","last_peered":"2024-08-17T17:46:20.910427+0000","last_clean":"2024-08-17T17:46:20.910427+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:46:20.910427+0000","last_undegraded":"2024-08-17T17:46:20.910427+0000","last_fullsized":"2024-08-17T17:46:20.910427+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":1122,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009725+0000","last_change":"2024-08-17T17:21:52.156855+0000","last_active":"2024-08-17T17:46:16.009725+0000","last_peered":"2024-08-17T17:46:16.009725+0000","last_clean":"2024-08-17T17:46:16.009725+0000","last_became_active":"2024-08-17T17:21:52.156377+0000","last_became_peered":"2024-08-17T17:21:52.156377+0000","last_unstale":"2024-08-17T17:46:16.009725+0000","last_undegraded":"2024-08-17T17:46:16.009725+0000","last_fullsized":"2024-08-17T17:46:16.009725+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:29.248851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009846+0000","last_change":"2024-08-17T17:30:43.433235+0000","last_active":"2024-08-17T17:46:16.009846+0000","last_peered":"2024-08-17T17:46:16.009846+0000","last_clean":"2024-08-17T17:46:16.009846+0000","last_became_active":"2024-08-17T17:30:43.432889+0000","last_became_peered":"2024-08-17T17:30:43.432889+0000","last_unstale":"2024-08-17T17:46:16.009846+0000","last_undegraded":"2024-08-17T17:46:16.009846+0000","last_fullsized":"2024-08-17T17:46:16.009846+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:22.828100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1099,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009203+0000","last_change":"2024-08-17T17:21:53.161517+0000","last_active":"2024-08-17T17:46:16.009203+0000","last_peered":"2024-08-17T17:46:16.009203+0000","last_clean":"2024-08-17T17:46:16.009203+0000","last_became_active":"2024-08-17T17:21:53.161203+0000","last_became_peered":"2024-08-17T17:21:53.161203+0000","last_unstale":"2024-08-17T17:46:16.009203+0000","last_undegraded":"2024-08-17T17:46:16.009203+0000","last_fullsized":"2024-08-17T17:46:16.009203+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:46.059237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1126,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583834+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:46:15.583834+0000","last_peered":"2024-08-17T17:46:15.583834+0000","last_clean":"2024-08-17T17:46:15.583834+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:46:15.583834+0000","last_undegraded":"2024-08-17T17:46:15.583834+0000","last_fullsized":"2024-08-17T17:46:15.583834+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1816,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918808+0000","last_change":"2024-08-17T17:30:43.432449+0000","last_active":"2024-08-17T17:46:20.918808+0000","last_peered":"2024-08-17T17:46:20.918808+0000","last_clean":"2024-08-17T17:46:20.918808+0000","last_became_active":"2024-08-17T17:30:43.432085+0000","last_became_peered":"2024-08-17T17:30:43.432085+0000","last_unstale":"2024-08-17T17:46:20.918808+0000","last_undegraded":"2024-08-17T17:46:20.918808+0000","last_fullsized":"2024-08-17T17:46:20.918808+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:30:03.861379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1679,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908943+0000","last_change":"2024-08-17T17:31:49.799682+0000","last_active":"2024-08-17T17:46:20.908943+0000","last_peered":"2024-08-17T17:46:20.908943+0000","last_clean":"2024-08-17T17:46:20.908943+0000","last_became_active":"2024-08-17T17:31:49.799090+0000","last_became_peered":"2024-08-17T17:31:49.799090+0000","last_unstale":"2024-08-17T17:46:20.908943+0000","last_undegraded":"2024-08-17T17:46:20.908943+0000","last_fullsized":"2024-08-17T17:46:20.908943+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:15:53.576285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1758,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912213+0000","last_change":"2024-08-17T17:30:43.440126+0000","last_active":"2024-08-17T17:46:20.912213+0000","last_peered":"2024-08-17T17:46:20.912213+0000","last_clean":"2024-08-17T17:46:20.912213+0000","last_became_active":"2024-08-17T17:30:43.439694+0000","last_became_peered":"2024-08-17T17:30:43.439694+0000","last_unstale":"2024-08-17T17:46:20.912213+0000","last_undegraded":"2024-08-17T17:46:20.912213+0000","last_fullsized":"2024-08-17T17:46:20.912213+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:02.086846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1682,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912510+0000","last_change":"2024-08-17T17:31:49.807926+0000","last_active":"2024-08-17T17:46:20.912510+0000","last_peered":"2024-08-17T17:46:20.912510+0000","last_clean":"2024-08-17T17:46:20.912510+0000","last_became_active":"2024-08-17T17:31:49.802101+0000","last_became_peered":"2024-08-17T17:31:49.802101+0000","last_unstale":"2024-08-17T17:46:20.912510+0000","last_undegraded":"2024-08-17T17:46:20.912510+0000","last_fullsized":"2024-08-17T17:46:20.912510+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:30.530469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1801,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.920831+0000","last_change":"2024-08-17T17:29:31.898335+0000","last_active":"2024-08-17T17:46:20.920831+0000","last_peered":"2024-08-17T17:46:20.920831+0000","last_clean":"2024-08-17T17:46:20.920831+0000","last_became_active":"2024-08-17T17:29:31.897066+0000","last_became_peered":"2024-08-17T17:29:31.897066+0000","last_unstale":"2024-08-17T17:46:20.920831+0000","last_undegraded":"2024-08-17T17:46:20.920831+0000","last_fullsized":"2024-08-17T17:46:20.920831+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:17.743900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1126,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798623+0000","last_change":"2024-08-17T17:29:31.894594+0000","last_active":"2024-08-17T17:46:15.798623+0000","last_peered":"2024-08-17T17:46:15.798623+0000","last_clean":"2024-08-17T17:46:15.798623+0000","last_became_active":"2024-08-17T17:29:31.893900+0000","last_became_peered":"2024-08-17T17:29:31.893900+0000","last_unstale":"2024-08-17T17:46:15.798623+0000","last_undegraded":"2024-08-17T17:46:15.798623+0000","last_fullsized":"2024-08-17T17:46:15.798623+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:17:03.570954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1759,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909855+0000","last_change":"2024-08-17T17:31:15.540575+0000","last_active":"2024-08-17T17:46:20.909855+0000","last_peered":"2024-08-17T17:46:20.909855+0000","last_clean":"2024-08-17T17:46:20.909855+0000","last_became_active":"2024-08-17T17:31:15.540120+0000","last_became_peered":"2024-08-17T17:31:15.540120+0000","last_unstale":"2024-08-17T17:46:20.909855+0000","last_undegraded":"2024-08-17T17:46:20.909855+0000","last_fullsized":"2024-08-17T17:46:20.909855+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:13.718902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1728,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911458+0000","last_change":"2024-08-17T17:31:15.550627+0000","last_active":"2024-08-17T17:46:20.911458+0000","last_peered":"2024-08-17T17:46:20.911458+0000","last_clean":"2024-08-17T17:46:20.911458+0000","last_became_active":"2024-08-17T17:31:15.550283+0000","last_became_peered":"2024-08-17T17:31:15.550283+0000","last_unstale":"2024-08-17T17:46:20.911458+0000","last_undegraded":"2024-08-17T17:46:20.911458+0000","last_fullsized":"2024-08-17T17:46:20.911458+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:30.165773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1779,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909729+0000","last_change":"2024-08-17T17:30:43.435630+0000","last_active":"2024-08-17T17:46:20.909729+0000","last_peered":"2024-08-17T17:46:20.909729+0000","last_clean":"2024-08-17T17:46:20.909729+0000","last_became_active":"2024-08-17T17:30:43.435338+0000","last_became_peered":"2024-08-17T17:30:43.435338+0000","last_unstale":"2024-08-17T17:46:20.909729+0000","last_undegraded":"2024-08-17T17:46:20.909729+0000","last_fullsized":"2024-08-17T17:46:20.909729+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:44.675247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1135,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070456+0000","last_change":"2024-08-17T17:31:49.801416+0000","last_active":"2024-08-17T17:46:16.070456+0000","last_peered":"2024-08-17T17:46:16.070456+0000","last_clean":"2024-08-17T17:46:16.070456+0000","last_became_active":"2024-08-17T17:31:49.801160+0000","last_became_peered":"2024-08-17T17:31:49.801160+0000","last_unstale":"2024-08-17T17:46:16.070456+0000","last_undegraded":"2024-08-17T17:46:16.070456+0000","last_fullsized":"2024-08-17T17:46:16.070456+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:04.641376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1771,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909952+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:46:20.909952+0000","last_peered":"2024-08-17T17:46:20.909952+0000","last_clean":"2024-08-17T17:46:20.909952+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:46:20.909952+0000","last_undegraded":"2024-08-17T17:46:20.909952+0000","last_fullsized":"2024-08-17T17:46:20.909952+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009980+0000","last_change":"2024-08-17T17:29:31.897318+0000","last_active":"2024-08-17T17:46:16.009980+0000","last_peered":"2024-08-17T17:46:16.009980+0000","last_clean":"2024-08-17T17:46:16.009980+0000","last_became_active":"2024-08-17T17:29:31.896952+0000","last_became_peered":"2024-08-17T17:29:31.896952+0000","last_unstale":"2024-08-17T17:46:16.009980+0000","last_undegraded":"2024-08-17T17:46:16.009980+0000","last_fullsized":"2024-08-17T17:46:16.009980+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:07:09.908531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1132,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.069948+0000","last_change":"2024-08-17T17:30:43.432147+0000","last_active":"2024-08-17T17:46:16.069948+0000","last_peered":"2024-08-17T17:46:16.069948+0000","last_clean":"2024-08-17T17:46:16.069948+0000","last_became_active":"2024-08-17T17:30:43.431796+0000","last_became_peered":"2024-08-17T17:30:43.431796+0000","last_unstale":"2024-08-17T17:46:16.069948+0000","last_undegraded":"2024-08-17T17:46:16.069948+0000","last_fullsized":"2024-08-17T17:46:16.069948+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:43.435654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0008386,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1766,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912908+0000","last_change":"2024-08-17T17:30:43.435209+0000","last_active":"2024-08-17T17:46:20.912908+0000","last_peered":"2024-08-17T17:46:20.912908+0000","last_clean":"2024-08-17T17:46:20.912908+0000","last_became_active":"2024-08-17T17:30:43.434903+0000","last_became_peered":"2024-08-17T17:30:43.434903+0000","last_unstale":"2024-08-17T17:46:20.912908+0000","last_undegraded":"2024-08-17T17:46:20.912908+0000","last_fullsized":"2024-08-17T17:46:20.912908+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:27.551001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1760,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909154+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:46:20.909154+0000","last_peered":"2024-08-17T17:46:20.909154+0000","last_clean":"2024-08-17T17:46:20.909154+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:46:20.909154+0000","last_undegraded":"2024-08-17T17:46:20.909154+0000","last_fullsized":"2024-08-17T17:46:20.909154+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1769,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908437+0000","last_change":"2024-08-17T17:31:49.809141+0000","last_active":"2024-08-17T17:46:20.908437+0000","last_peered":"2024-08-17T17:46:20.908437+0000","last_clean":"2024-08-17T17:46:20.908437+0000","last_became_active":"2024-08-17T17:31:49.808855+0000","last_became_peered":"2024-08-17T17:31:49.808855+0000","last_unstale":"2024-08-17T17:46:20.908437+0000","last_undegraded":"2024-08-17T17:46:20.908437+0000","last_fullsized":"2024-08-17T17:46:20.908437+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:10.144929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1826,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.919131+0000","last_change":"2024-08-17T17:31:15.546567+0000","last_active":"2024-08-17T17:46:20.919131+0000","last_peered":"2024-08-17T17:46:20.919131+0000","last_clean":"2024-08-17T17:46:20.919131+0000","last_became_active":"2024-08-17T17:31:15.546117+0000","last_became_peered":"2024-08-17T17:31:15.546117+0000","last_unstale":"2024-08-17T17:46:20.919131+0000","last_undegraded":"2024-08-17T17:46:20.919131+0000","last_fullsized":"2024-08-17T17:46:20.919131+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:19.245231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1813,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.915571+0000","last_change":"2024-08-17T17:31:49.811704+0000","last_active":"2024-08-17T17:46:20.915571+0000","last_peered":"2024-08-17T17:46:20.915571+0000","last_clean":"2024-08-17T17:46:20.915571+0000","last_became_active":"2024-08-17T17:31:49.811385+0000","last_became_peered":"2024-08-17T17:31:49.811385+0000","last_unstale":"2024-08-17T17:46:20.915571+0000","last_undegraded":"2024-08-17T17:46:20.915571+0000","last_fullsized":"2024-08-17T17:46:20.915571+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:58.832774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1143,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798033+0000","last_change":"2024-08-17T17:30:43.432897+0000","last_active":"2024-08-17T17:46:15.798033+0000","last_peered":"2024-08-17T17:46:15.798033+0000","last_clean":"2024-08-17T17:46:15.798033+0000","last_became_active":"2024-08-17T17:30:43.432602+0000","last_became_peered":"2024-08-17T17:30:43.432602+0000","last_unstale":"2024-08-17T17:46:15.798033+0000","last_undegraded":"2024-08-17T17:46:15.798033+0000","last_fullsized":"2024-08-17T17:46:15.798033+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:51.083549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1096,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583544+0000","last_change":"2024-08-17T17:21:53.164672+0000","last_active":"2024-08-17T17:46:15.583544+0000","last_peered":"2024-08-17T17:46:15.583544+0000","last_clean":"2024-08-17T17:46:15.583544+0000","last_became_active":"2024-08-17T17:21:53.164381+0000","last_became_peered":"2024-08-17T17:21:53.164381+0000","last_unstale":"2024-08-17T17:46:15.583544+0000","last_undegraded":"2024-08-17T17:46:15.583544+0000","last_fullsized":"2024-08-17T17:46:15.583544+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:43:36.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1661,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909361+0000","last_change":"2024-08-17T17:31:49.807517+0000","last_active":"2024-08-17T17:46:20.909361+0000","last_peered":"2024-08-17T17:46:20.909361+0000","last_clean":"2024-08-17T17:46:20.909361+0000","last_became_active":"2024-08-17T17:31:49.802152+0000","last_became_peered":"2024-08-17T17:31:49.802152+0000","last_unstale":"2024-08-17T17:46:20.909361+0000","last_undegraded":"2024-08-17T17:46:20.909361+0000","last_fullsized":"2024-08-17T17:46:20.909361+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:02.934420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1131,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070064+0000","last_change":"2024-08-17T17:31:15.545691+0000","last_active":"2024-08-17T17:46:16.070064+0000","last_peered":"2024-08-17T17:46:16.070064+0000","last_clean":"2024-08-17T17:46:16.070064+0000","last_became_active":"2024-08-17T17:31:15.545335+0000","last_became_peered":"2024-08-17T17:31:15.545335+0000","last_unstale":"2024-08-17T17:46:16.070064+0000","last_undegraded":"2024-08-17T17:46:16.070064+0000","last_fullsized":"2024-08-17T17:46:16.070064+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:18.694039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085985900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1455,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798170+0000","last_change":"2024-08-17T17:29:31.894591+0000","last_active":"2024-08-17T17:46:15.798170+0000","last_peered":"2024-08-17T17:46:15.798170+0000","last_clean":"2024-08-17T17:46:15.798170+0000","last_became_active":"2024-08-17T17:29:31.894134+0000","last_became_peered":"2024-08-17T17:29:31.894134+0000","last_unstale":"2024-08-17T17:46:15.798170+0000","last_undegraded":"2024-08-17T17:46:15.798170+0000","last_fullsized":"2024-08-17T17:46:15.798170+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:12.586656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1760,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.913715+0000","last_change":"2024-08-17T17:31:49.810921+0000","last_active":"2024-08-17T17:46:20.913715+0000","last_peered":"2024-08-17T17:46:20.913715+0000","last_clean":"2024-08-17T17:46:20.913715+0000","last_became_active":"2024-08-17T17:31:49.810583+0000","last_became_peered":"2024-08-17T17:31:49.810583+0000","last_unstale":"2024-08-17T17:46:20.913715+0000","last_undegraded":"2024-08-17T17:46:20.913715+0000","last_fullsized":"2024-08-17T17:46:20.913715+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:44.272815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1707,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908901+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:46:20.908901+0000","last_peered":"2024-08-17T17:46:20.908901+0000","last_clean":"2024-08-17T17:46:20.908901+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:46:20.908901+0000","last_undegraded":"2024-08-17T17:46:20.908901+0000","last_fullsized":"2024-08-17T17:46:20.908901+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009910+0000","last_change":"2024-08-17T17:30:43.434276+0000","last_active":"2024-08-17T17:46:16.009910+0000","last_peered":"2024-08-17T17:46:16.009910+0000","last_clean":"2024-08-17T17:46:16.009910+0000","last_became_active":"2024-08-17T17:30:43.433844+0000","last_became_peered":"2024-08-17T17:30:43.433844+0000","last_unstale":"2024-08-17T17:46:16.009910+0000","last_undegraded":"2024-08-17T17:46:16.009910+0000","last_fullsized":"2024-08-17T17:46:16.009910+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:17:56.025107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009503+0000","last_change":"2024-08-17T17:31:15.546844+0000","last_active":"2024-08-17T17:46:16.009503+0000","last_peered":"2024-08-17T17:46:16.009503+0000","last_clean":"2024-08-17T17:46:16.009503+0000","last_became_active":"2024-08-17T17:31:15.546516+0000","last_became_peered":"2024-08-17T17:31:15.546516+0000","last_unstale":"2024-08-17T17:46:16.009503+0000","last_undegraded":"2024-08-17T17:46:16.009503+0000","last_fullsized":"2024-08-17T17:46:16.009503+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:10:18.148181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009616+0000","last_change":"2024-08-17T17:31:15.546671+0000","last_active":"2024-08-17T17:46:16.009616+0000","last_peered":"2024-08-17T17:46:16.009616+0000","last_clean":"2024-08-17T17:46:16.009616+0000","last_became_active":"2024-08-17T17:31:15.546355+0000","last_became_peered":"2024-08-17T17:31:15.546355+0000","last_unstale":"2024-08-17T17:46:16.009616+0000","last_undegraded":"2024-08-17T17:46:16.009616+0000","last_fullsized":"2024-08-17T17:46:16.009616+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:45.836713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1114,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583330+0000","last_change":"2024-08-17T17:30:43.433097+0000","last_active":"2024-08-17T17:46:15.583330+0000","last_peered":"2024-08-17T17:46:15.583330+0000","last_clean":"2024-08-17T17:46:15.583330+0000","last_became_active":"2024-08-17T17:30:43.432179+0000","last_became_peered":"2024-08-17T17:30:43.432179+0000","last_unstale":"2024-08-17T17:46:15.583330+0000","last_undegraded":"2024-08-17T17:46:15.583330+0000","last_fullsized":"2024-08-17T17:46:15.583330+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:44:40.966915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009644+0000","last_change":"2024-08-17T17:31:49.800286+0000","last_active":"2024-08-17T17:46:16.009644+0000","last_peered":"2024-08-17T17:46:16.009644+0000","last_clean":"2024-08-17T17:46:16.009644+0000","last_became_active":"2024-08-17T17:31:49.799834+0000","last_became_peered":"2024-08-17T17:31:49.799834+0000","last_unstale":"2024-08-17T17:46:16.009644+0000","last_undegraded":"2024-08-17T17:46:16.009644+0000","last_fullsized":"2024-08-17T17:46:16.009644+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:28.409397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1681,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910644+0000","last_change":"2024-08-17T17:31:49.801723+0000","last_active":"2024-08-17T17:46:20.910644+0000","last_peered":"2024-08-17T17:46:20.910644+0000","last_clean":"2024-08-17T17:46:20.910644+0000","last_became_active":"2024-08-17T17:31:49.801370+0000","last_became_peered":"2024-08-17T17:31:49.801370+0000","last_unstale":"2024-08-17T17:46:20.910644+0000","last_undegraded":"2024-08-17T17:46:20.910644+0000","last_fullsized":"2024-08-17T17:46:20.910644+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:28.686255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.920662+0000","last_change":"2024-08-17T17:30:43.431404+0000","last_active":"2024-08-17T17:46:20.920662+0000","last_peered":"2024-08-17T17:46:20.920662+0000","last_clean":"2024-08-17T17:46:20.920662+0000","last_became_active":"2024-08-17T17:30:43.431087+0000","last_became_peered":"2024-08-17T17:30:43.431087+0000","last_unstale":"2024-08-17T17:46:20.920662+0000","last_undegraded":"2024-08-17T17:46:20.920662+0000","last_fullsized":"2024-08-17T17:46:20.920662+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:15.830770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1118,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070176+0000","last_change":"2024-08-17T17:31:49.799082+0000","last_active":"2024-08-17T17:46:16.070176+0000","last_peered":"2024-08-17T17:46:16.070176+0000","last_clean":"2024-08-17T17:46:16.070176+0000","last_became_active":"2024-08-17T17:31:49.798764+0000","last_became_peered":"2024-08-17T17:31:49.798764+0000","last_unstale":"2024-08-17T17:46:16.070176+0000","last_undegraded":"2024-08-17T17:46:16.070176+0000","last_fullsized":"2024-08-17T17:46:16.070176+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:07.697148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1113,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583225+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:46:15.583225+0000","last_peered":"2024-08-17T17:46:15.583225+0000","last_clean":"2024-08-17T17:46:15.583225+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:46:15.583225+0000","last_undegraded":"2024-08-17T17:46:15.583225+0000","last_fullsized":"2024-08-17T17:46:15.583225+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1098,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798620+0000","last_change":"2024-08-17T17:21:52.151540+0000","last_active":"2024-08-17T17:46:15.798620+0000","last_peered":"2024-08-17T17:46:15.798620+0000","last_clean":"2024-08-17T17:46:15.798620+0000","last_became_active":"2024-08-17T17:21:52.151294+0000","last_became_peered":"2024-08-17T17:21:52.151294+0000","last_unstale":"2024-08-17T17:46:15.798620+0000","last_undegraded":"2024-08-17T17:46:15.798620+0000","last_fullsized":"2024-08-17T17:46:15.798620+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:03.533252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009427+0000","last_change":"2024-08-17T17:31:49.799847+0000","last_active":"2024-08-17T17:46:16.009427+0000","last_peered":"2024-08-17T17:46:16.009427+0000","last_clean":"2024-08-17T17:46:16.009427+0000","last_became_active":"2024-08-17T17:31:49.799523+0000","last_became_peered":"2024-08-17T17:31:49.799523+0000","last_unstale":"2024-08-17T17:46:16.009427+0000","last_undegraded":"2024-08-17T17:46:16.009427+0000","last_fullsized":"2024-08-17T17:46:16.009427+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:34.070172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798751+0000","last_change":"2024-08-17T17:30:43.437929+0000","last_active":"2024-08-17T17:46:15.798751+0000","last_peered":"2024-08-17T17:46:15.798751+0000","last_clean":"2024-08-17T17:46:15.798751+0000","last_became_active":"2024-08-17T17:30:43.437604+0000","last_became_peered":"2024-08-17T17:30:43.437604+0000","last_unstale":"2024-08-17T17:46:15.798751+0000","last_undegraded":"2024-08-17T17:46:15.798751+0000","last_fullsized":"2024-08-17T17:46:15.798751+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:55:21.953561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1100,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.069864+0000","last_change":"2024-08-17T17:21:53.161767+0000","last_active":"2024-08-17T17:46:16.069864+0000","last_peered":"2024-08-17T17:46:16.069864+0000","last_clean":"2024-08-17T17:46:16.069864+0000","last_became_active":"2024-08-17T17:21:53.161514+0000","last_became_peered":"2024-08-17T17:21:53.161514+0000","last_unstale":"2024-08-17T17:46:16.069864+0000","last_undegraded":"2024-08-17T17:46:16.069864+0000","last_fullsized":"2024-08-17T17:46:16.069864+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:59.837673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1115,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798770+0000","last_change":"2024-08-17T17:29:31.900876+0000","last_active":"2024-08-17T17:46:15.798770+0000","last_peered":"2024-08-17T17:46:15.798770+0000","last_clean":"2024-08-17T17:46:15.798770+0000","last_became_active":"2024-08-17T17:29:31.900496+0000","last_became_peered":"2024-08-17T17:29:31.900496+0000","last_unstale":"2024-08-17T17:46:15.798770+0000","last_undegraded":"2024-08-17T17:46:15.798770+0000","last_fullsized":"2024-08-17T17:46:15.798770+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:18.971519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1122,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583539+0000","last_change":"2024-08-17T17:29:31.891859+0000","last_active":"2024-08-17T17:46:15.583539+0000","last_peered":"2024-08-17T17:46:15.583539+0000","last_clean":"2024-08-17T17:46:15.583539+0000","last_became_active":"2024-08-17T17:29:31.891405+0000","last_became_peered":"2024-08-17T17:29:31.891405+0000","last_unstale":"2024-08-17T17:46:15.583539+0000","last_undegraded":"2024-08-17T17:46:15.583539+0000","last_fullsized":"2024-08-17T17:46:15.583539+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:48.087233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912321+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:46:20.912321+0000","last_peered":"2024-08-17T17:46:20.912321+0000","last_clean":"2024-08-17T17:46:20.912321+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:46:20.912321+0000","last_undegraded":"2024-08-17T17:46:20.912321+0000","last_fullsized":"2024-08-17T17:46:20.912321+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1097,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798097+0000","last_change":"2024-08-17T17:21:52.149957+0000","last_active":"2024-08-17T17:46:15.798097+0000","last_peered":"2024-08-17T17:46:15.798097+0000","last_clean":"2024-08-17T17:46:15.798097+0000","last_became_active":"2024-08-17T17:21:52.149673+0000","last_became_peered":"2024-08-17T17:21:52.149673+0000","last_unstale":"2024-08-17T17:46:15.798097+0000","last_undegraded":"2024-08-17T17:46:15.798097+0000","last_fullsized":"2024-08-17T17:46:15.798097+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:01:16.473279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918958+0000","last_change":"2024-08-17T17:31:49.805181+0000","last_active":"2024-08-17T17:46:20.918958+0000","last_peered":"2024-08-17T17:46:20.918958+0000","last_clean":"2024-08-17T17:46:20.918958+0000","last_became_active":"2024-08-17T17:31:49.804905+0000","last_became_peered":"2024-08-17T17:31:49.804905+0000","last_unstale":"2024-08-17T17:46:20.918958+0000","last_undegraded":"2024-08-17T17:46:20.918958+0000","last_fullsized":"2024-08-17T17:46:20.918958+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:47:16.879688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.916345+0000","last_change":"2024-08-17T17:31:49.812747+0000","last_active":"2024-08-17T17:46:20.916345+0000","last_peered":"2024-08-17T17:46:20.916345+0000","last_clean":"2024-08-17T17:46:20.916345+0000","last_became_active":"2024-08-17T17:31:49.812378+0000","last_became_peered":"2024-08-17T17:31:49.812378+0000","last_unstale":"2024-08-17T17:46:20.916345+0000","last_undegraded":"2024-08-17T17:46:20.916345+0000","last_fullsized":"2024-08-17T17:46:20.916345+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:35.124235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070375+0000","last_change":"2024-08-17T17:31:49.801380+0000","last_active":"2024-08-17T17:46:16.070375+0000","last_peered":"2024-08-17T17:46:16.070375+0000","last_clean":"2024-08-17T17:46:16.070375+0000","last_became_active":"2024-08-17T17:31:49.801103+0000","last_became_peered":"2024-08-17T17:31:49.801103+0000","last_unstale":"2024-08-17T17:46:16.070375+0000","last_undegraded":"2024-08-17T17:46:16.070375+0000","last_fullsized":"2024-08-17T17:46:16.070375+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:09:43.554983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798259+0000","last_change":"2024-08-17T17:31:15.543729+0000","last_active":"2024-08-17T17:46:15.798259+0000","last_peered":"2024-08-17T17:46:15.798259+0000","last_clean":"2024-08-17T17:46:15.798259+0000","last_became_active":"2024-08-17T17:31:15.543142+0000","last_became_peered":"2024-08-17T17:31:15.543142+0000","last_unstale":"2024-08-17T17:46:15.798259+0000","last_undegraded":"2024-08-17T17:46:15.798259+0000","last_fullsized":"2024-08-17T17:46:15.798259+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:00.612263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910980+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:46:20.910980+0000","last_peered":"2024-08-17T17:46:20.910980+0000","last_clean":"2024-08-17T17:46:20.910980+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:46:20.910980+0000","last_undegraded":"2024-08-17T17:46:20.910980+0000","last_fullsized":"2024-08-17T17:46:20.910980+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234314,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325232,"kb_used_data":19556,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93419920,"statfs":{"total":95995035648,"available":95661998080,"internally_reserved":0,"allocated":20025344,"data_stored":12500974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495949,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":329380,"kb_used_data":19544,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93415772,"statfs":{"total":95995035648,"available":95657750528,"internally_reserved":0,"allocated":20013056,"data_stored":12500532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757587,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316316,"kb_used_data":18960,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93428836,"statfs":{"total":95995035648,"available":95671128064,"internally_reserved":0,"allocated":19415040,"data_stored":11906313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326201,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":68888,"kb_used_data":18956,"kb_used_omap":57,"kb_used_meta":49862,"kb_avail":93676264,"statfs":{"total":95995035648,"available":95924494336,"internally_reserved":0,"allocated":19410944,"data_stored":11906029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":51059451},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405440,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":73108,"kb_used_data":18956,"kb_used_omap":61,"kb_used_meta":54082,"kb_avail":93672044,"statfs":{"total":95995035648,"available":95920173056,"internally_reserved":0,"allocated":19410944,"data_stored":11905882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":55380920},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253451974,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":73044,"kb_used_data":18956,"kb_used_omap":54,"kb_used_meta":54025,"kb_avail":93672108,"statfs":{"total":95995035648,"available":95920238592,"internally_reserved":0,"allocated":19410944,"data_stored":11905966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":55321998},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":438,"seq":1881195675860,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":65248,"kb_used_data":19540,"kb_used_omap":43,"kb_used_meta":45652,"kb_avail":93679904,"statfs":{"total":95995035648,"available":95928221696,"internally_reserved":0,"allocated":20008960,"data_stored":12496899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44456,"internal_metadata":46748248},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019222,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328808,"kb_used_data":18972,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93416344,"statfs":{"total":95995035648,"available":95658336256,"internally_reserved":0,"allocated":19427328,"data_stored":11910544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:47:06.280 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:47:06.535 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:06.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:06 smithi130 ceph-mon[105521]: from='client.35187 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:06.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:06 smithi130 ceph-mon[110068]: from='client.35187 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:07.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:06 smithi179 ceph-mon[93519]: from='client.35187 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:07.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:07 smithi130 ceph-mon[105521]: pgmap v1538: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:07 smithi130 ceph-mon[110068]: pgmap v1538: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:08.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:07 smithi179 ceph-mon[93519]: pgmap v1538: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:08.312 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:47:08.313 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:47:08.915 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1538,"stamp":"2024-08-17T17:47:06.376908+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1580024,"kb_used_data":153440,"kb_used_omap":223,"kb_used_meta":1426272,"kb_avail":748381192,"statfs":{"total":767960285184,"available":766342340608,"internally_reserved":0,"allocated":157122560,"data_stored":97033139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":228359,"internal_metadata":1460503545},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002994"},"pg_stats":[{"pgid":"3.1f","version":"71'4","reported_seq":1819,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918108+0000","last_change":"2024-08-17T17:31:15.551224+0000","last_active":"2024-08-17T17:46:20.918108+0000","last_peered":"2024-08-17T17:46:20.918108+0000","last_clean":"2024-08-17T17:46:20.918108+0000","last_became_active":"2024-08-17T17:31:15.550855+0000","last_became_peered":"2024-08-17T17:31:15.550855+0000","last_unstale":"2024-08-17T17:46:20.918108+0000","last_undegraded":"2024-08-17T17:46:20.918108+0000","last_fullsized":"2024-08-17T17:46:20.918108+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:05:07.081474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009009+0000","last_change":"2024-08-17T17:30:43.429215+0000","last_active":"2024-08-17T17:46:16.009009+0000","last_peered":"2024-08-17T17:46:16.009009+0000","last_clean":"2024-08-17T17:46:16.009009+0000","last_became_active":"2024-08-17T17:30:43.428891+0000","last_became_peered":"2024-08-17T17:30:43.428891+0000","last_unstale":"2024-08-17T17:46:16.009009+0000","last_undegraded":"2024-08-17T17:46:16.009009+0000","last_fullsized":"2024-08-17T17:46:16.009009+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:41:39.313325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1698,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911021+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:46:20.911021+0000","last_peered":"2024-08-17T17:46:20.911021+0000","last_clean":"2024-08-17T17:46:20.911021+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:46:20.911021+0000","last_undegraded":"2024-08-17T17:46:20.911021+0000","last_fullsized":"2024-08-17T17:46:20.911021+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1712,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911837+0000","last_change":"2024-08-17T17:31:49.809768+0000","last_active":"2024-08-17T17:46:20.911837+0000","last_peered":"2024-08-17T17:46:20.911837+0000","last_clean":"2024-08-17T17:46:20.911837+0000","last_became_active":"2024-08-17T17:31:49.808276+0000","last_became_peered":"2024-08-17T17:31:49.808276+0000","last_unstale":"2024-08-17T17:46:20.911837+0000","last_undegraded":"2024-08-17T17:46:20.911837+0000","last_fullsized":"2024-08-17T17:46:20.911837+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:51.600719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1103,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070283+0000","last_change":"2024-08-17T17:21:52.157731+0000","last_active":"2024-08-17T17:46:16.070283+0000","last_peered":"2024-08-17T17:46:16.070283+0000","last_clean":"2024-08-17T17:46:16.070283+0000","last_became_active":"2024-08-17T17:21:52.157453+0000","last_became_peered":"2024-08-17T17:21:52.157453+0000","last_unstale":"2024-08-17T17:46:16.070283+0000","last_undegraded":"2024-08-17T17:46:16.070283+0000","last_fullsized":"2024-08-17T17:46:16.070283+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:30:09.106088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009219+0000","last_change":"2024-08-17T17:31:49.806060+0000","last_active":"2024-08-17T17:46:16.009219+0000","last_peered":"2024-08-17T17:46:16.009219+0000","last_clean":"2024-08-17T17:46:16.009219+0000","last_became_active":"2024-08-17T17:31:49.805688+0000","last_became_peered":"2024-08-17T17:31:49.805688+0000","last_unstale":"2024-08-17T17:46:16.009219+0000","last_undegraded":"2024-08-17T17:46:16.009219+0000","last_fullsized":"2024-08-17T17:46:16.009219+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:42.831662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1120,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070144+0000","last_change":"2024-08-17T17:30:43.427884+0000","last_active":"2024-08-17T17:46:16.070144+0000","last_peered":"2024-08-17T17:46:16.070144+0000","last_clean":"2024-08-17T17:46:16.070144+0000","last_became_active":"2024-08-17T17:30:43.427497+0000","last_became_peered":"2024-08-17T17:30:43.427497+0000","last_unstale":"2024-08-17T17:46:16.070144+0000","last_undegraded":"2024-08-17T17:46:16.070144+0000","last_fullsized":"2024-08-17T17:46:16.070144+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:22.830092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009797+0000","last_change":"2024-08-17T17:31:49.799570+0000","last_active":"2024-08-17T17:46:16.009797+0000","last_peered":"2024-08-17T17:46:16.009797+0000","last_clean":"2024-08-17T17:46:16.009797+0000","last_became_active":"2024-08-17T17:31:49.799251+0000","last_became_peered":"2024-08-17T17:31:49.799251+0000","last_unstale":"2024-08-17T17:46:16.009797+0000","last_undegraded":"2024-08-17T17:46:16.009797+0000","last_fullsized":"2024-08-17T17:46:16.009797+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:58.973574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1804,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.920819+0000","last_change":"2024-08-17T17:29:31.897430+0000","last_active":"2024-08-17T17:46:20.920819+0000","last_peered":"2024-08-17T17:46:20.920819+0000","last_clean":"2024-08-17T17:46:20.920819+0000","last_became_active":"2024-08-17T17:29:31.896722+0000","last_became_peered":"2024-08-17T17:29:31.896722+0000","last_unstale":"2024-08-17T17:46:20.920819+0000","last_undegraded":"2024-08-17T17:46:20.920819+0000","last_fullsized":"2024-08-17T17:46:20.920819+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:41:41.942213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1740,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912786+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:46:20.912786+0000","last_peered":"2024-08-17T17:46:20.912786+0000","last_clean":"2024-08-17T17:46:20.912786+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:46:20.912786+0000","last_undegraded":"2024-08-17T17:46:20.912786+0000","last_fullsized":"2024-08-17T17:46:20.912786+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1135,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009572+0000","last_change":"2024-08-17T17:31:15.547327+0000","last_active":"2024-08-17T17:46:16.009572+0000","last_peered":"2024-08-17T17:46:16.009572+0000","last_clean":"2024-08-17T17:46:16.009572+0000","last_became_active":"2024-08-17T17:31:15.546959+0000","last_became_peered":"2024-08-17T17:31:15.546959+0000","last_unstale":"2024-08-17T17:46:16.009572+0000","last_undegraded":"2024-08-17T17:46:16.009572+0000","last_fullsized":"2024-08-17T17:46:16.009572+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:27:49.216367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910954+0000","last_change":"2024-08-17T17:31:49.802098+0000","last_active":"2024-08-17T17:46:20.910954+0000","last_peered":"2024-08-17T17:46:20.910954+0000","last_clean":"2024-08-17T17:46:20.910954+0000","last_became_active":"2024-08-17T17:31:49.801747+0000","last_became_peered":"2024-08-17T17:31:49.801747+0000","last_unstale":"2024-08-17T17:46:20.910954+0000","last_undegraded":"2024-08-17T17:46:20.910954+0000","last_fullsized":"2024-08-17T17:46:20.910954+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:00.212402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1759,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908264+0000","last_change":"2024-08-17T17:31:49.815142+0000","last_active":"2024-08-17T17:46:20.908264+0000","last_peered":"2024-08-17T17:46:20.908264+0000","last_clean":"2024-08-17T17:46:20.908264+0000","last_became_active":"2024-08-17T17:31:49.814873+0000","last_became_peered":"2024-08-17T17:31:49.814873+0000","last_unstale":"2024-08-17T17:46:20.908264+0000","last_undegraded":"2024-08-17T17:46:20.908264+0000","last_fullsized":"2024-08-17T17:46:20.908264+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:05.542100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1812,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.919007+0000","last_change":"2024-08-17T17:31:49.804738+0000","last_active":"2024-08-17T17:46:20.919007+0000","last_peered":"2024-08-17T17:46:20.919007+0000","last_clean":"2024-08-17T17:46:20.919007+0000","last_became_active":"2024-08-17T17:31:49.804476+0000","last_became_peered":"2024-08-17T17:31:49.804476+0000","last_unstale":"2024-08-17T17:46:20.919007+0000","last_undegraded":"2024-08-17T17:46:20.919007+0000","last_fullsized":"2024-08-17T17:46:20.919007+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:51.659079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1723,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909025+0000","last_change":"2024-08-17T17:31:49.808285+0000","last_active":"2024-08-17T17:46:20.909025+0000","last_peered":"2024-08-17T17:46:20.909025+0000","last_clean":"2024-08-17T17:46:20.909025+0000","last_became_active":"2024-08-17T17:31:49.805633+0000","last_became_peered":"2024-08-17T17:31:49.805633+0000","last_unstale":"2024-08-17T17:46:20.909025+0000","last_undegraded":"2024-08-17T17:46:20.909025+0000","last_fullsized":"2024-08-17T17:46:20.909025+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:38.815825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1113,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583821+0000","last_change":"2024-08-17T17:30:43.429169+0000","last_active":"2024-08-17T17:46:15.583821+0000","last_peered":"2024-08-17T17:46:15.583821+0000","last_clean":"2024-08-17T17:46:15.583821+0000","last_became_active":"2024-08-17T17:30:43.428850+0000","last_became_peered":"2024-08-17T17:30:43.428850+0000","last_unstale":"2024-08-17T17:46:15.583821+0000","last_undegraded":"2024-08-17T17:46:15.583821+0000","last_fullsized":"2024-08-17T17:46:15.583821+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:51:44.920103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908763+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:46:20.908763+0000","last_peered":"2024-08-17T17:46:20.908763+0000","last_clean":"2024-08-17T17:46:20.908763+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:46:20.908763+0000","last_undegraded":"2024-08-17T17:46:20.908763+0000","last_fullsized":"2024-08-17T17:46:20.908763+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918062+0000","last_change":"2024-08-17T17:31:49.805397+0000","last_active":"2024-08-17T17:46:20.918062+0000","last_peered":"2024-08-17T17:46:20.918062+0000","last_clean":"2024-08-17T17:46:20.918062+0000","last_became_active":"2024-08-17T17:31:49.804586+0000","last_became_peered":"2024-08-17T17:31:49.804586+0000","last_unstale":"2024-08-17T17:46:20.918062+0000","last_undegraded":"2024-08-17T17:46:20.918062+0000","last_fullsized":"2024-08-17T17:46:20.918062+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:58.697361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1661,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911739+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:46:20.911739+0000","last_peered":"2024-08-17T17:46:20.911739+0000","last_clean":"2024-08-17T17:46:20.911739+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:46:20.911739+0000","last_undegraded":"2024-08-17T17:46:20.911739+0000","last_fullsized":"2024-08-17T17:46:20.911739+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1118,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070708+0000","last_change":"2024-08-17T17:30:43.429232+0000","last_active":"2024-08-17T17:46:16.070708+0000","last_peered":"2024-08-17T17:46:16.070708+0000","last_clean":"2024-08-17T17:46:16.070708+0000","last_became_active":"2024-08-17T17:30:43.428944+0000","last_became_peered":"2024-08-17T17:30:43.428944+0000","last_unstale":"2024-08-17T17:46:16.070708+0000","last_undegraded":"2024-08-17T17:46:16.070708+0000","last_fullsized":"2024-08-17T17:46:16.070708+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:05.780222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798470+0000","last_change":"2024-08-17T17:31:49.801654+0000","last_active":"2024-08-17T17:46:15.798470+0000","last_peered":"2024-08-17T17:46:15.798470+0000","last_clean":"2024-08-17T17:46:15.798470+0000","last_became_active":"2024-08-17T17:31:49.801234+0000","last_became_peered":"2024-08-17T17:31:49.801234+0000","last_unstale":"2024-08-17T17:46:15.798470+0000","last_undegraded":"2024-08-17T17:46:15.798470+0000","last_fullsized":"2024-08-17T17:46:15.798470+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:21.590996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1760,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910427+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:46:20.910427+0000","last_peered":"2024-08-17T17:46:20.910427+0000","last_clean":"2024-08-17T17:46:20.910427+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:46:20.910427+0000","last_undegraded":"2024-08-17T17:46:20.910427+0000","last_fullsized":"2024-08-17T17:46:20.910427+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":1122,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009725+0000","last_change":"2024-08-17T17:21:52.156855+0000","last_active":"2024-08-17T17:46:16.009725+0000","last_peered":"2024-08-17T17:46:16.009725+0000","last_clean":"2024-08-17T17:46:16.009725+0000","last_became_active":"2024-08-17T17:21:52.156377+0000","last_became_peered":"2024-08-17T17:21:52.156377+0000","last_unstale":"2024-08-17T17:46:16.009725+0000","last_undegraded":"2024-08-17T17:46:16.009725+0000","last_fullsized":"2024-08-17T17:46:16.009725+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:29.248851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009846+0000","last_change":"2024-08-17T17:30:43.433235+0000","last_active":"2024-08-17T17:46:16.009846+0000","last_peered":"2024-08-17T17:46:16.009846+0000","last_clean":"2024-08-17T17:46:16.009846+0000","last_became_active":"2024-08-17T17:30:43.432889+0000","last_became_peered":"2024-08-17T17:30:43.432889+0000","last_unstale":"2024-08-17T17:46:16.009846+0000","last_undegraded":"2024-08-17T17:46:16.009846+0000","last_fullsized":"2024-08-17T17:46:16.009846+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:22.828100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1099,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009203+0000","last_change":"2024-08-17T17:21:53.161517+0000","last_active":"2024-08-17T17:46:16.009203+0000","last_peered":"2024-08-17T17:46:16.009203+0000","last_clean":"2024-08-17T17:46:16.009203+0000","last_became_active":"2024-08-17T17:21:53.161203+0000","last_became_peered":"2024-08-17T17:21:53.161203+0000","last_unstale":"2024-08-17T17:46:16.009203+0000","last_undegraded":"2024-08-17T17:46:16.009203+0000","last_fullsized":"2024-08-17T17:46:16.009203+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:46.059237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1126,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583834+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:46:15.583834+0000","last_peered":"2024-08-17T17:46:15.583834+0000","last_clean":"2024-08-17T17:46:15.583834+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:46:15.583834+0000","last_undegraded":"2024-08-17T17:46:15.583834+0000","last_fullsized":"2024-08-17T17:46:15.583834+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1816,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918808+0000","last_change":"2024-08-17T17:30:43.432449+0000","last_active":"2024-08-17T17:46:20.918808+0000","last_peered":"2024-08-17T17:46:20.918808+0000","last_clean":"2024-08-17T17:46:20.918808+0000","last_became_active":"2024-08-17T17:30:43.432085+0000","last_became_peered":"2024-08-17T17:30:43.432085+0000","last_unstale":"2024-08-17T17:46:20.918808+0000","last_undegraded":"2024-08-17T17:46:20.918808+0000","last_fullsized":"2024-08-17T17:46:20.918808+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:30:03.861379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1679,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908943+0000","last_change":"2024-08-17T17:31:49.799682+0000","last_active":"2024-08-17T17:46:20.908943+0000","last_peered":"2024-08-17T17:46:20.908943+0000","last_clean":"2024-08-17T17:46:20.908943+0000","last_became_active":"2024-08-17T17:31:49.799090+0000","last_became_peered":"2024-08-17T17:31:49.799090+0000","last_unstale":"2024-08-17T17:46:20.908943+0000","last_undegraded":"2024-08-17T17:46:20.908943+0000","last_fullsized":"2024-08-17T17:46:20.908943+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:15:53.576285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1758,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912213+0000","last_change":"2024-08-17T17:30:43.440126+0000","last_active":"2024-08-17T17:46:20.912213+0000","last_peered":"2024-08-17T17:46:20.912213+0000","last_clean":"2024-08-17T17:46:20.912213+0000","last_became_active":"2024-08-17T17:30:43.439694+0000","last_became_peered":"2024-08-17T17:30:43.439694+0000","last_unstale":"2024-08-17T17:46:20.912213+0000","last_undegraded":"2024-08-17T17:46:20.912213+0000","last_fullsized":"2024-08-17T17:46:20.912213+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:02.086846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1682,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912510+0000","last_change":"2024-08-17T17:31:49.807926+0000","last_active":"2024-08-17T17:46:20.912510+0000","last_peered":"2024-08-17T17:46:20.912510+0000","last_clean":"2024-08-17T17:46:20.912510+0000","last_became_active":"2024-08-17T17:31:49.802101+0000","last_became_peered":"2024-08-17T17:31:49.802101+0000","last_unstale":"2024-08-17T17:46:20.912510+0000","last_undegraded":"2024-08-17T17:46:20.912510+0000","last_fullsized":"2024-08-17T17:46:20.912510+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:30.530469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1801,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.920831+0000","last_change":"2024-08-17T17:29:31.898335+0000","last_active":"2024-08-17T17:46:20.920831+0000","last_peered":"2024-08-17T17:46:20.920831+0000","last_clean":"2024-08-17T17:46:20.920831+0000","last_became_active":"2024-08-17T17:29:31.897066+0000","last_became_peered":"2024-08-17T17:29:31.897066+0000","last_unstale":"2024-08-17T17:46:20.920831+0000","last_undegraded":"2024-08-17T17:46:20.920831+0000","last_fullsized":"2024-08-17T17:46:20.920831+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:17.743900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1126,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798623+0000","last_change":"2024-08-17T17:29:31.894594+0000","last_active":"2024-08-17T17:46:15.798623+0000","last_peered":"2024-08-17T17:46:15.798623+0000","last_clean":"2024-08-17T17:46:15.798623+0000","last_became_active":"2024-08-17T17:29:31.893900+0000","last_became_peered":"2024-08-17T17:29:31.893900+0000","last_unstale":"2024-08-17T17:46:15.798623+0000","last_undegraded":"2024-08-17T17:46:15.798623+0000","last_fullsized":"2024-08-17T17:46:15.798623+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:17:03.570954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1759,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909855+0000","last_change":"2024-08-17T17:31:15.540575+0000","last_active":"2024-08-17T17:46:20.909855+0000","last_peered":"2024-08-17T17:46:20.909855+0000","last_clean":"2024-08-17T17:46:20.909855+0000","last_became_active":"2024-08-17T17:31:15.540120+0000","last_became_peered":"2024-08-17T17:31:15.540120+0000","last_unstale":"2024-08-17T17:46:20.909855+0000","last_undegraded":"2024-08-17T17:46:20.909855+0000","last_fullsized":"2024-08-17T17:46:20.909855+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:13.718902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1728,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911458+0000","last_change":"2024-08-17T17:31:15.550627+0000","last_active":"2024-08-17T17:46:20.911458+0000","last_peered":"2024-08-17T17:46:20.911458+0000","last_clean":"2024-08-17T17:46:20.911458+0000","last_became_active":"2024-08-17T17:31:15.550283+0000","last_became_peered":"2024-08-17T17:31:15.550283+0000","last_unstale":"2024-08-17T17:46:20.911458+0000","last_undegraded":"2024-08-17T17:46:20.911458+0000","last_fullsized":"2024-08-17T17:46:20.911458+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:30.165773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1779,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909729+0000","last_change":"2024-08-17T17:30:43.435630+0000","last_active":"2024-08-17T17:46:20.909729+0000","last_peered":"2024-08-17T17:46:20.909729+0000","last_clean":"2024-08-17T17:46:20.909729+0000","last_became_active":"2024-08-17T17:30:43.435338+0000","last_became_peered":"2024-08-17T17:30:43.435338+0000","last_unstale":"2024-08-17T17:46:20.909729+0000","last_undegraded":"2024-08-17T17:46:20.909729+0000","last_fullsized":"2024-08-17T17:46:20.909729+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:44.675247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1135,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070456+0000","last_change":"2024-08-17T17:31:49.801416+0000","last_active":"2024-08-17T17:46:16.070456+0000","last_peered":"2024-08-17T17:46:16.070456+0000","last_clean":"2024-08-17T17:46:16.070456+0000","last_became_active":"2024-08-17T17:31:49.801160+0000","last_became_peered":"2024-08-17T17:31:49.801160+0000","last_unstale":"2024-08-17T17:46:16.070456+0000","last_undegraded":"2024-08-17T17:46:16.070456+0000","last_fullsized":"2024-08-17T17:46:16.070456+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:04.641376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1771,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909952+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:46:20.909952+0000","last_peered":"2024-08-17T17:46:20.909952+0000","last_clean":"2024-08-17T17:46:20.909952+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:46:20.909952+0000","last_undegraded":"2024-08-17T17:46:20.909952+0000","last_fullsized":"2024-08-17T17:46:20.909952+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009980+0000","last_change":"2024-08-17T17:29:31.897318+0000","last_active":"2024-08-17T17:46:16.009980+0000","last_peered":"2024-08-17T17:46:16.009980+0000","last_clean":"2024-08-17T17:46:16.009980+0000","last_became_active":"2024-08-17T17:29:31.896952+0000","last_became_peered":"2024-08-17T17:29:31.896952+0000","last_unstale":"2024-08-17T17:46:16.009980+0000","last_undegraded":"2024-08-17T17:46:16.009980+0000","last_fullsized":"2024-08-17T17:46:16.009980+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:07:09.908531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1132,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.069948+0000","last_change":"2024-08-17T17:30:43.432147+0000","last_active":"2024-08-17T17:46:16.069948+0000","last_peered":"2024-08-17T17:46:16.069948+0000","last_clean":"2024-08-17T17:46:16.069948+0000","last_became_active":"2024-08-17T17:30:43.431796+0000","last_became_peered":"2024-08-17T17:30:43.431796+0000","last_unstale":"2024-08-17T17:46:16.069948+0000","last_undegraded":"2024-08-17T17:46:16.069948+0000","last_fullsized":"2024-08-17T17:46:16.069948+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:43.435654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0008386,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1766,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912908+0000","last_change":"2024-08-17T17:30:43.435209+0000","last_active":"2024-08-17T17:46:20.912908+0000","last_peered":"2024-08-17T17:46:20.912908+0000","last_clean":"2024-08-17T17:46:20.912908+0000","last_became_active":"2024-08-17T17:30:43.434903+0000","last_became_peered":"2024-08-17T17:30:43.434903+0000","last_unstale":"2024-08-17T17:46:20.912908+0000","last_undegraded":"2024-08-17T17:46:20.912908+0000","last_fullsized":"2024-08-17T17:46:20.912908+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:27.551001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1760,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909154+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:46:20.909154+0000","last_peered":"2024-08-17T17:46:20.909154+0000","last_clean":"2024-08-17T17:46:20.909154+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:46:20.909154+0000","last_undegraded":"2024-08-17T17:46:20.909154+0000","last_fullsized":"2024-08-17T17:46:20.909154+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1769,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908437+0000","last_change":"2024-08-17T17:31:49.809141+0000","last_active":"2024-08-17T17:46:20.908437+0000","last_peered":"2024-08-17T17:46:20.908437+0000","last_clean":"2024-08-17T17:46:20.908437+0000","last_became_active":"2024-08-17T17:31:49.808855+0000","last_became_peered":"2024-08-17T17:31:49.808855+0000","last_unstale":"2024-08-17T17:46:20.908437+0000","last_undegraded":"2024-08-17T17:46:20.908437+0000","last_fullsized":"2024-08-17T17:46:20.908437+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:10.144929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1826,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.919131+0000","last_change":"2024-08-17T17:31:15.546567+0000","last_active":"2024-08-17T17:46:20.919131+0000","last_peered":"2024-08-17T17:46:20.919131+0000","last_clean":"2024-08-17T17:46:20.919131+0000","last_became_active":"2024-08-17T17:31:15.546117+0000","last_became_peered":"2024-08-17T17:31:15.546117+0000","last_unstale":"2024-08-17T17:46:20.919131+0000","last_undegraded":"2024-08-17T17:46:20.919131+0000","last_fullsized":"2024-08-17T17:46:20.919131+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:19.245231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1813,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.915571+0000","last_change":"2024-08-17T17:31:49.811704+0000","last_active":"2024-08-17T17:46:20.915571+0000","last_peered":"2024-08-17T17:46:20.915571+0000","last_clean":"2024-08-17T17:46:20.915571+0000","last_became_active":"2024-08-17T17:31:49.811385+0000","last_became_peered":"2024-08-17T17:31:49.811385+0000","last_unstale":"2024-08-17T17:46:20.915571+0000","last_undegraded":"2024-08-17T17:46:20.915571+0000","last_fullsized":"2024-08-17T17:46:20.915571+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:58.832774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1143,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798033+0000","last_change":"2024-08-17T17:30:43.432897+0000","last_active":"2024-08-17T17:46:15.798033+0000","last_peered":"2024-08-17T17:46:15.798033+0000","last_clean":"2024-08-17T17:46:15.798033+0000","last_became_active":"2024-08-17T17:30:43.432602+0000","last_became_peered":"2024-08-17T17:30:43.432602+0000","last_unstale":"2024-08-17T17:46:15.798033+0000","last_undegraded":"2024-08-17T17:46:15.798033+0000","last_fullsized":"2024-08-17T17:46:15.798033+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:51.083549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1096,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583544+0000","last_change":"2024-08-17T17:21:53.164672+0000","last_active":"2024-08-17T17:46:15.583544+0000","last_peered":"2024-08-17T17:46:15.583544+0000","last_clean":"2024-08-17T17:46:15.583544+0000","last_became_active":"2024-08-17T17:21:53.164381+0000","last_became_peered":"2024-08-17T17:21:53.164381+0000","last_unstale":"2024-08-17T17:46:15.583544+0000","last_undegraded":"2024-08-17T17:46:15.583544+0000","last_fullsized":"2024-08-17T17:46:15.583544+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:43:36.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1661,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909361+0000","last_change":"2024-08-17T17:31:49.807517+0000","last_active":"2024-08-17T17:46:20.909361+0000","last_peered":"2024-08-17T17:46:20.909361+0000","last_clean":"2024-08-17T17:46:20.909361+0000","last_became_active":"2024-08-17T17:31:49.802152+0000","last_became_peered":"2024-08-17T17:31:49.802152+0000","last_unstale":"2024-08-17T17:46:20.909361+0000","last_undegraded":"2024-08-17T17:46:20.909361+0000","last_fullsized":"2024-08-17T17:46:20.909361+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:02.934420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1131,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070064+0000","last_change":"2024-08-17T17:31:15.545691+0000","last_active":"2024-08-17T17:46:16.070064+0000","last_peered":"2024-08-17T17:46:16.070064+0000","last_clean":"2024-08-17T17:46:16.070064+0000","last_became_active":"2024-08-17T17:31:15.545335+0000","last_became_peered":"2024-08-17T17:31:15.545335+0000","last_unstale":"2024-08-17T17:46:16.070064+0000","last_undegraded":"2024-08-17T17:46:16.070064+0000","last_fullsized":"2024-08-17T17:46:16.070064+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:18.694039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085985900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1455,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798170+0000","last_change":"2024-08-17T17:29:31.894591+0000","last_active":"2024-08-17T17:46:15.798170+0000","last_peered":"2024-08-17T17:46:15.798170+0000","last_clean":"2024-08-17T17:46:15.798170+0000","last_became_active":"2024-08-17T17:29:31.894134+0000","last_became_peered":"2024-08-17T17:29:31.894134+0000","last_unstale":"2024-08-17T17:46:15.798170+0000","last_undegraded":"2024-08-17T17:46:15.798170+0000","last_fullsized":"2024-08-17T17:46:15.798170+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:12.586656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1760,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.913715+0000","last_change":"2024-08-17T17:31:49.810921+0000","last_active":"2024-08-17T17:46:20.913715+0000","last_peered":"2024-08-17T17:46:20.913715+0000","last_clean":"2024-08-17T17:46:20.913715+0000","last_became_active":"2024-08-17T17:31:49.810583+0000","last_became_peered":"2024-08-17T17:31:49.810583+0000","last_unstale":"2024-08-17T17:46:20.913715+0000","last_undegraded":"2024-08-17T17:46:20.913715+0000","last_fullsized":"2024-08-17T17:46:20.913715+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:44.272815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1707,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908901+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:46:20.908901+0000","last_peered":"2024-08-17T17:46:20.908901+0000","last_clean":"2024-08-17T17:46:20.908901+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:46:20.908901+0000","last_undegraded":"2024-08-17T17:46:20.908901+0000","last_fullsized":"2024-08-17T17:46:20.908901+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009910+0000","last_change":"2024-08-17T17:30:43.434276+0000","last_active":"2024-08-17T17:46:16.009910+0000","last_peered":"2024-08-17T17:46:16.009910+0000","last_clean":"2024-08-17T17:46:16.009910+0000","last_became_active":"2024-08-17T17:30:43.433844+0000","last_became_peered":"2024-08-17T17:30:43.433844+0000","last_unstale":"2024-08-17T17:46:16.009910+0000","last_undegraded":"2024-08-17T17:46:16.009910+0000","last_fullsized":"2024-08-17T17:46:16.009910+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:17:56.025107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009503+0000","last_change":"2024-08-17T17:31:15.546844+0000","last_active":"2024-08-17T17:46:16.009503+0000","last_peered":"2024-08-17T17:46:16.009503+0000","last_clean":"2024-08-17T17:46:16.009503+0000","last_became_active":"2024-08-17T17:31:15.546516+0000","last_became_peered":"2024-08-17T17:31:15.546516+0000","last_unstale":"2024-08-17T17:46:16.009503+0000","last_undegraded":"2024-08-17T17:46:16.009503+0000","last_fullsized":"2024-08-17T17:46:16.009503+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:10:18.148181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009616+0000","last_change":"2024-08-17T17:31:15.546671+0000","last_active":"2024-08-17T17:46:16.009616+0000","last_peered":"2024-08-17T17:46:16.009616+0000","last_clean":"2024-08-17T17:46:16.009616+0000","last_became_active":"2024-08-17T17:31:15.546355+0000","last_became_peered":"2024-08-17T17:31:15.546355+0000","last_unstale":"2024-08-17T17:46:16.009616+0000","last_undegraded":"2024-08-17T17:46:16.009616+0000","last_fullsized":"2024-08-17T17:46:16.009616+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:45.836713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1114,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583330+0000","last_change":"2024-08-17T17:30:43.433097+0000","last_active":"2024-08-17T17:46:15.583330+0000","last_peered":"2024-08-17T17:46:15.583330+0000","last_clean":"2024-08-17T17:46:15.583330+0000","last_became_active":"2024-08-17T17:30:43.432179+0000","last_became_peered":"2024-08-17T17:30:43.432179+0000","last_unstale":"2024-08-17T17:46:15.583330+0000","last_undegraded":"2024-08-17T17:46:15.583330+0000","last_fullsized":"2024-08-17T17:46:15.583330+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:44:40.966915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009644+0000","last_change":"2024-08-17T17:31:49.800286+0000","last_active":"2024-08-17T17:46:16.009644+0000","last_peered":"2024-08-17T17:46:16.009644+0000","last_clean":"2024-08-17T17:46:16.009644+0000","last_became_active":"2024-08-17T17:31:49.799834+0000","last_became_peered":"2024-08-17T17:31:49.799834+0000","last_unstale":"2024-08-17T17:46:16.009644+0000","last_undegraded":"2024-08-17T17:46:16.009644+0000","last_fullsized":"2024-08-17T17:46:16.009644+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:28.409397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1681,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910644+0000","last_change":"2024-08-17T17:31:49.801723+0000","last_active":"2024-08-17T17:46:20.910644+0000","last_peered":"2024-08-17T17:46:20.910644+0000","last_clean":"2024-08-17T17:46:20.910644+0000","last_became_active":"2024-08-17T17:31:49.801370+0000","last_became_peered":"2024-08-17T17:31:49.801370+0000","last_unstale":"2024-08-17T17:46:20.910644+0000","last_undegraded":"2024-08-17T17:46:20.910644+0000","last_fullsized":"2024-08-17T17:46:20.910644+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:28.686255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.920662+0000","last_change":"2024-08-17T17:30:43.431404+0000","last_active":"2024-08-17T17:46:20.920662+0000","last_peered":"2024-08-17T17:46:20.920662+0000","last_clean":"2024-08-17T17:46:20.920662+0000","last_became_active":"2024-08-17T17:30:43.431087+0000","last_became_peered":"2024-08-17T17:30:43.431087+0000","last_unstale":"2024-08-17T17:46:20.920662+0000","last_undegraded":"2024-08-17T17:46:20.920662+0000","last_fullsized":"2024-08-17T17:46:20.920662+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:15.830770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1118,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070176+0000","last_change":"2024-08-17T17:31:49.799082+0000","last_active":"2024-08-17T17:46:16.070176+0000","last_peered":"2024-08-17T17:46:16.070176+0000","last_clean":"2024-08-17T17:46:16.070176+0000","last_became_active":"2024-08-17T17:31:49.798764+0000","last_became_peered":"2024-08-17T17:31:49.798764+0000","last_unstale":"2024-08-17T17:46:16.070176+0000","last_undegraded":"2024-08-17T17:46:16.070176+0000","last_fullsized":"2024-08-17T17:46:16.070176+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:07.697148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1113,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583225+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:46:15.583225+0000","last_peered":"2024-08-17T17:46:15.583225+0000","last_clean":"2024-08-17T17:46:15.583225+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:46:15.583225+0000","last_undegraded":"2024-08-17T17:46:15.583225+0000","last_fullsized":"2024-08-17T17:46:15.583225+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1098,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798620+0000","last_change":"2024-08-17T17:21:52.151540+0000","last_active":"2024-08-17T17:46:15.798620+0000","last_peered":"2024-08-17T17:46:15.798620+0000","last_clean":"2024-08-17T17:46:15.798620+0000","last_became_active":"2024-08-17T17:21:52.151294+0000","last_became_peered":"2024-08-17T17:21:52.151294+0000","last_unstale":"2024-08-17T17:46:15.798620+0000","last_undegraded":"2024-08-17T17:46:15.798620+0000","last_fullsized":"2024-08-17T17:46:15.798620+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:03.533252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009427+0000","last_change":"2024-08-17T17:31:49.799847+0000","last_active":"2024-08-17T17:46:16.009427+0000","last_peered":"2024-08-17T17:46:16.009427+0000","last_clean":"2024-08-17T17:46:16.009427+0000","last_became_active":"2024-08-17T17:31:49.799523+0000","last_became_peered":"2024-08-17T17:31:49.799523+0000","last_unstale":"2024-08-17T17:46:16.009427+0000","last_undegraded":"2024-08-17T17:46:16.009427+0000","last_fullsized":"2024-08-17T17:46:16.009427+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:34.070172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798751+0000","last_change":"2024-08-17T17:30:43.437929+0000","last_active":"2024-08-17T17:46:15.798751+0000","last_peered":"2024-08-17T17:46:15.798751+0000","last_clean":"2024-08-17T17:46:15.798751+0000","last_became_active":"2024-08-17T17:30:43.437604+0000","last_became_peered":"2024-08-17T17:30:43.437604+0000","last_unstale":"2024-08-17T17:46:15.798751+0000","last_undegraded":"2024-08-17T17:46:15.798751+0000","last_fullsized":"2024-08-17T17:46:15.798751+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:55:21.953561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1100,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.069864+0000","last_change":"2024-08-17T17:21:53.161767+0000","last_active":"2024-08-17T17:46:16.069864+0000","last_peered":"2024-08-17T17:46:16.069864+0000","last_clean":"2024-08-17T17:46:16.069864+0000","last_became_active":"2024-08-17T17:21:53.161514+0000","last_became_peered":"2024-08-17T17:21:53.161514+0000","last_unstale":"2024-08-17T17:46:16.069864+0000","last_undegraded":"2024-08-17T17:46:16.069864+0000","last_fullsized":"2024-08-17T17:46:16.069864+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:59.837673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1115,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798770+0000","last_change":"2024-08-17T17:29:31.900876+0000","last_active":"2024-08-17T17:46:15.798770+0000","last_peered":"2024-08-17T17:46:15.798770+0000","last_clean":"2024-08-17T17:46:15.798770+0000","last_became_active":"2024-08-17T17:29:31.900496+0000","last_became_peered":"2024-08-17T17:29:31.900496+0000","last_unstale":"2024-08-17T17:46:15.798770+0000","last_undegraded":"2024-08-17T17:46:15.798770+0000","last_fullsized":"2024-08-17T17:46:15.798770+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:18.971519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1122,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583539+0000","last_change":"2024-08-17T17:29:31.891859+0000","last_active":"2024-08-17T17:46:15.583539+0000","last_peered":"2024-08-17T17:46:15.583539+0000","last_clean":"2024-08-17T17:46:15.583539+0000","last_became_active":"2024-08-17T17:29:31.891405+0000","last_became_peered":"2024-08-17T17:29:31.891405+0000","last_unstale":"2024-08-17T17:46:15.583539+0000","last_undegraded":"2024-08-17T17:46:15.583539+0000","last_fullsized":"2024-08-17T17:46:15.583539+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:48.087233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912321+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:46:20.912321+0000","last_peered":"2024-08-17T17:46:20.912321+0000","last_clean":"2024-08-17T17:46:20.912321+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:46:20.912321+0000","last_undegraded":"2024-08-17T17:46:20.912321+0000","last_fullsized":"2024-08-17T17:46:20.912321+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1097,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798097+0000","last_change":"2024-08-17T17:21:52.149957+0000","last_active":"2024-08-17T17:46:15.798097+0000","last_peered":"2024-08-17T17:46:15.798097+0000","last_clean":"2024-08-17T17:46:15.798097+0000","last_became_active":"2024-08-17T17:21:52.149673+0000","last_became_peered":"2024-08-17T17:21:52.149673+0000","last_unstale":"2024-08-17T17:46:15.798097+0000","last_undegraded":"2024-08-17T17:46:15.798097+0000","last_fullsized":"2024-08-17T17:46:15.798097+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:01:16.473279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918958+0000","last_change":"2024-08-17T17:31:49.805181+0000","last_active":"2024-08-17T17:46:20.918958+0000","last_peered":"2024-08-17T17:46:20.918958+0000","last_clean":"2024-08-17T17:46:20.918958+0000","last_became_active":"2024-08-17T17:31:49.804905+0000","last_became_peered":"2024-08-17T17:31:49.804905+0000","last_unstale":"2024-08-17T17:46:20.918958+0000","last_undegraded":"2024-08-17T17:46:20.918958+0000","last_fullsized":"2024-08-17T17:46:20.918958+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:47:16.879688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.916345+0000","last_change":"2024-08-17T17:31:49.812747+0000","last_active":"2024-08-17T17:46:20.916345+0000","last_peered":"2024-08-17T17:46:20.916345+0000","last_clean":"2024-08-17T17:46:20.916345+0000","last_became_active":"2024-08-17T17:31:49.812378+0000","last_became_peered":"2024-08-17T17:31:49.812378+0000","last_unstale":"2024-08-17T17:46:20.916345+0000","last_undegraded":"2024-08-17T17:46:20.916345+0000","last_fullsized":"2024-08-17T17:46:20.916345+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:35.124235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070375+0000","last_change":"2024-08-17T17:31:49.801380+0000","last_active":"2024-08-17T17:46:16.070375+0000","last_peered":"2024-08-17T17:46:16.070375+0000","last_clean":"2024-08-17T17:46:16.070375+0000","last_became_active":"2024-08-17T17:31:49.801103+0000","last_became_peered":"2024-08-17T17:31:49.801103+0000","last_unstale":"2024-08-17T17:46:16.070375+0000","last_undegraded":"2024-08-17T17:46:16.070375+0000","last_fullsized":"2024-08-17T17:46:16.070375+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:09:43.554983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798259+0000","last_change":"2024-08-17T17:31:15.543729+0000","last_active":"2024-08-17T17:46:15.798259+0000","last_peered":"2024-08-17T17:46:15.798259+0000","last_clean":"2024-08-17T17:46:15.798259+0000","last_became_active":"2024-08-17T17:31:15.543142+0000","last_became_peered":"2024-08-17T17:31:15.543142+0000","last_unstale":"2024-08-17T17:46:15.798259+0000","last_undegraded":"2024-08-17T17:46:15.798259+0000","last_fullsized":"2024-08-17T17:46:15.798259+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:00.612263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910980+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:46:20.910980+0000","last_peered":"2024-08-17T17:46:20.910980+0000","last_clean":"2024-08-17T17:46:20.910980+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:46:20.910980+0000","last_undegraded":"2024-08-17T17:46:20.910980+0000","last_fullsized":"2024-08-17T17:46:20.910980+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234314,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325232,"kb_used_data":19556,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93419920,"statfs":{"total":95995035648,"available":95661998080,"internally_reserved":0,"allocated":20025344,"data_stored":12500974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495949,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":329380,"kb_used_data":19544,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93415772,"statfs":{"total":95995035648,"available":95657750528,"internally_reserved":0,"allocated":20013056,"data_stored":12500532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757587,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316316,"kb_used_data":18960,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93428836,"statfs":{"total":95995035648,"available":95671128064,"internally_reserved":0,"allocated":19415040,"data_stored":11906313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326201,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":68888,"kb_used_data":18956,"kb_used_omap":57,"kb_used_meta":49862,"kb_avail":93676264,"statfs":{"total":95995035648,"available":95924494336,"internally_reserved":0,"allocated":19410944,"data_stored":11906029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":51059451},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405440,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":73108,"kb_used_data":18956,"kb_used_omap":61,"kb_used_meta":54082,"kb_avail":93672044,"statfs":{"total":95995035648,"available":95920173056,"internally_reserved":0,"allocated":19410944,"data_stored":11905882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":55380920},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253451974,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":73044,"kb_used_data":18956,"kb_used_omap":54,"kb_used_meta":54025,"kb_avail":93672108,"statfs":{"total":95995035648,"available":95920238592,"internally_reserved":0,"allocated":19410944,"data_stored":11905966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":55321998},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":438,"seq":1881195675861,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":65248,"kb_used_data":19540,"kb_used_omap":43,"kb_used_meta":45652,"kb_avail":93679904,"statfs":{"total":95995035648,"available":95928221696,"internally_reserved":0,"allocated":20008960,"data_stored":12496899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44456,"internal_metadata":46748248},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019222,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328808,"kb_used_data":18972,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93416344,"statfs":{"total":95995035648,"available":95658336256,"internally_reserved":0,"allocated":19427328,"data_stored":11910544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:47:08.919 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T17:47:08.919 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-17T17:47:08.919 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-17T17:47:08.919 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph health --format=json 2024-08-17T17:47:09.175 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:09 smithi130 ceph-mon[105521]: from='client.45077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:09.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:09 smithi130 ceph-mon[105521]: pgmap v1539: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:09.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:09 smithi130 ceph-mon[110068]: from='client.45077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:09.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:09 smithi130 ceph-mon[110068]: pgmap v1539: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:09 smithi179 ceph-mon[93519]: from='client.45077 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:09 smithi179 ceph-mon[93519]: pgmap v1539: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:47:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:47:09] "GET /metrics HTTP/1.1" 200 34151 "" "Prometheus/2.51.0" 2024-08-17T17:47:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:47:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:47:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:47:10.985 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:47:10.986 INFO:teuthology.orchestra.run.smithi130.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-17T17:47:11.560 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-17T17:47:11.561 INFO:teuthology.run_tasks:Running task print... 2024-08-17T17:47:11.571 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-08-17T17:47:11.571 INFO:teuthology.run_tasks:Running task parallel... 2024-08-17T17:47:11.579 INFO:teuthology.task.parallel:starting parallel... 2024-08-17T17:47:11.579 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-17T17:47:11.580 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-08-17T17:47:11.582 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-08-17T17:47:11.582 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'PG_AVAILABILITY'], 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '91f88e4d132ac08dc53355fa3d9717a53236a3a9'}} 2024-08-17T17:47:11.582 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-08-17T17:47:11.582 INFO:tasks.thrashosds:Beginning thrashosds... 2024-08-17T17:47:11.582 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T17:47:11.583 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:47:11.838 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:11.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:11 smithi130 ceph-mon[105521]: pgmap v1540: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:11 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2875332159' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-17T17:47:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:11 smithi130 ceph-mon[110068]: pgmap v1540: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:11 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2875332159' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-17T17:47:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:11 smithi179 ceph-mon[93519]: pgmap v1540: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:11 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2875332159' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-17T17:47:12.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:12.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:13.626 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:47:13.627 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:47:13.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:13 smithi130 ceph-mon[105521]: pgmap v1541: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:47:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:13 smithi130 ceph-mon[110068]: pgmap v1541: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:47:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:13.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:13.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:13 smithi179 ceph-mon[93519]: pgmap v1541: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:13.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:47:13.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:13.994 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:13.995 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:13.995 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:14.224 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1541,"stamp":"2024-08-17T17:47:12.378176+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1580024,"kb_used_data":153440,"kb_used_omap":223,"kb_used_meta":1426272,"kb_avail":748381192,"statfs":{"total":767960285184,"available":766342340608,"internally_reserved":0,"allocated":157122560,"data_stored":97033139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":228359,"internal_metadata":1460503545},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002504"},"pg_stats":[{"pgid":"3.1f","version":"71'4","reported_seq":1819,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918108+0000","last_change":"2024-08-17T17:31:15.551224+0000","last_active":"2024-08-17T17:46:20.918108+0000","last_peered":"2024-08-17T17:46:20.918108+0000","last_clean":"2024-08-17T17:46:20.918108+0000","last_became_active":"2024-08-17T17:31:15.550855+0000","last_became_peered":"2024-08-17T17:31:15.550855+0000","last_unstale":"2024-08-17T17:46:20.918108+0000","last_undegraded":"2024-08-17T17:46:20.918108+0000","last_fullsized":"2024-08-17T17:46:20.918108+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:05:07.081474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009009+0000","last_change":"2024-08-17T17:30:43.429215+0000","last_active":"2024-08-17T17:46:16.009009+0000","last_peered":"2024-08-17T17:46:16.009009+0000","last_clean":"2024-08-17T17:46:16.009009+0000","last_became_active":"2024-08-17T17:30:43.428891+0000","last_became_peered":"2024-08-17T17:30:43.428891+0000","last_unstale":"2024-08-17T17:46:16.009009+0000","last_undegraded":"2024-08-17T17:46:16.009009+0000","last_fullsized":"2024-08-17T17:46:16.009009+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:41:39.313325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1698,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911021+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:46:20.911021+0000","last_peered":"2024-08-17T17:46:20.911021+0000","last_clean":"2024-08-17T17:46:20.911021+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:46:20.911021+0000","last_undegraded":"2024-08-17T17:46:20.911021+0000","last_fullsized":"2024-08-17T17:46:20.911021+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1712,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911837+0000","last_change":"2024-08-17T17:31:49.809768+0000","last_active":"2024-08-17T17:46:20.911837+0000","last_peered":"2024-08-17T17:46:20.911837+0000","last_clean":"2024-08-17T17:46:20.911837+0000","last_became_active":"2024-08-17T17:31:49.808276+0000","last_became_peered":"2024-08-17T17:31:49.808276+0000","last_unstale":"2024-08-17T17:46:20.911837+0000","last_undegraded":"2024-08-17T17:46:20.911837+0000","last_fullsized":"2024-08-17T17:46:20.911837+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:51.600719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1103,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070283+0000","last_change":"2024-08-17T17:21:52.157731+0000","last_active":"2024-08-17T17:46:16.070283+0000","last_peered":"2024-08-17T17:46:16.070283+0000","last_clean":"2024-08-17T17:46:16.070283+0000","last_became_active":"2024-08-17T17:21:52.157453+0000","last_became_peered":"2024-08-17T17:21:52.157453+0000","last_unstale":"2024-08-17T17:46:16.070283+0000","last_undegraded":"2024-08-17T17:46:16.070283+0000","last_fullsized":"2024-08-17T17:46:16.070283+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:30:09.106088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009219+0000","last_change":"2024-08-17T17:31:49.806060+0000","last_active":"2024-08-17T17:46:16.009219+0000","last_peered":"2024-08-17T17:46:16.009219+0000","last_clean":"2024-08-17T17:46:16.009219+0000","last_became_active":"2024-08-17T17:31:49.805688+0000","last_became_peered":"2024-08-17T17:31:49.805688+0000","last_unstale":"2024-08-17T17:46:16.009219+0000","last_undegraded":"2024-08-17T17:46:16.009219+0000","last_fullsized":"2024-08-17T17:46:16.009219+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:42.831662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1120,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070144+0000","last_change":"2024-08-17T17:30:43.427884+0000","last_active":"2024-08-17T17:46:16.070144+0000","last_peered":"2024-08-17T17:46:16.070144+0000","last_clean":"2024-08-17T17:46:16.070144+0000","last_became_active":"2024-08-17T17:30:43.427497+0000","last_became_peered":"2024-08-17T17:30:43.427497+0000","last_unstale":"2024-08-17T17:46:16.070144+0000","last_undegraded":"2024-08-17T17:46:16.070144+0000","last_fullsized":"2024-08-17T17:46:16.070144+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:22.830092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009797+0000","last_change":"2024-08-17T17:31:49.799570+0000","last_active":"2024-08-17T17:46:16.009797+0000","last_peered":"2024-08-17T17:46:16.009797+0000","last_clean":"2024-08-17T17:46:16.009797+0000","last_became_active":"2024-08-17T17:31:49.799251+0000","last_became_peered":"2024-08-17T17:31:49.799251+0000","last_unstale":"2024-08-17T17:46:16.009797+0000","last_undegraded":"2024-08-17T17:46:16.009797+0000","last_fullsized":"2024-08-17T17:46:16.009797+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:58.973574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1804,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.920819+0000","last_change":"2024-08-17T17:29:31.897430+0000","last_active":"2024-08-17T17:46:20.920819+0000","last_peered":"2024-08-17T17:46:20.920819+0000","last_clean":"2024-08-17T17:46:20.920819+0000","last_became_active":"2024-08-17T17:29:31.896722+0000","last_became_peered":"2024-08-17T17:29:31.896722+0000","last_unstale":"2024-08-17T17:46:20.920819+0000","last_undegraded":"2024-08-17T17:46:20.920819+0000","last_fullsized":"2024-08-17T17:46:20.920819+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:41:41.942213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1740,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912786+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:46:20.912786+0000","last_peered":"2024-08-17T17:46:20.912786+0000","last_clean":"2024-08-17T17:46:20.912786+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:46:20.912786+0000","last_undegraded":"2024-08-17T17:46:20.912786+0000","last_fullsized":"2024-08-17T17:46:20.912786+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1135,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009572+0000","last_change":"2024-08-17T17:31:15.547327+0000","last_active":"2024-08-17T17:46:16.009572+0000","last_peered":"2024-08-17T17:46:16.009572+0000","last_clean":"2024-08-17T17:46:16.009572+0000","last_became_active":"2024-08-17T17:31:15.546959+0000","last_became_peered":"2024-08-17T17:31:15.546959+0000","last_unstale":"2024-08-17T17:46:16.009572+0000","last_undegraded":"2024-08-17T17:46:16.009572+0000","last_fullsized":"2024-08-17T17:46:16.009572+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:27:49.216367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910954+0000","last_change":"2024-08-17T17:31:49.802098+0000","last_active":"2024-08-17T17:46:20.910954+0000","last_peered":"2024-08-17T17:46:20.910954+0000","last_clean":"2024-08-17T17:46:20.910954+0000","last_became_active":"2024-08-17T17:31:49.801747+0000","last_became_peered":"2024-08-17T17:31:49.801747+0000","last_unstale":"2024-08-17T17:46:20.910954+0000","last_undegraded":"2024-08-17T17:46:20.910954+0000","last_fullsized":"2024-08-17T17:46:20.910954+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:00.212402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1759,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908264+0000","last_change":"2024-08-17T17:31:49.815142+0000","last_active":"2024-08-17T17:46:20.908264+0000","last_peered":"2024-08-17T17:46:20.908264+0000","last_clean":"2024-08-17T17:46:20.908264+0000","last_became_active":"2024-08-17T17:31:49.814873+0000","last_became_peered":"2024-08-17T17:31:49.814873+0000","last_unstale":"2024-08-17T17:46:20.908264+0000","last_undegraded":"2024-08-17T17:46:20.908264+0000","last_fullsized":"2024-08-17T17:46:20.908264+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:05.542100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1812,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.919007+0000","last_change":"2024-08-17T17:31:49.804738+0000","last_active":"2024-08-17T17:46:20.919007+0000","last_peered":"2024-08-17T17:46:20.919007+0000","last_clean":"2024-08-17T17:46:20.919007+0000","last_became_active":"2024-08-17T17:31:49.804476+0000","last_became_peered":"2024-08-17T17:31:49.804476+0000","last_unstale":"2024-08-17T17:46:20.919007+0000","last_undegraded":"2024-08-17T17:46:20.919007+0000","last_fullsized":"2024-08-17T17:46:20.919007+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:51.659079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1723,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909025+0000","last_change":"2024-08-17T17:31:49.808285+0000","last_active":"2024-08-17T17:46:20.909025+0000","last_peered":"2024-08-17T17:46:20.909025+0000","last_clean":"2024-08-17T17:46:20.909025+0000","last_became_active":"2024-08-17T17:31:49.805633+0000","last_became_peered":"2024-08-17T17:31:49.805633+0000","last_unstale":"2024-08-17T17:46:20.909025+0000","last_undegraded":"2024-08-17T17:46:20.909025+0000","last_fullsized":"2024-08-17T17:46:20.909025+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:38.815825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1113,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583821+0000","last_change":"2024-08-17T17:30:43.429169+0000","last_active":"2024-08-17T17:46:15.583821+0000","last_peered":"2024-08-17T17:46:15.583821+0000","last_clean":"2024-08-17T17:46:15.583821+0000","last_became_active":"2024-08-17T17:30:43.428850+0000","last_became_peered":"2024-08-17T17:30:43.428850+0000","last_unstale":"2024-08-17T17:46:15.583821+0000","last_undegraded":"2024-08-17T17:46:15.583821+0000","last_fullsized":"2024-08-17T17:46:15.583821+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:51:44.920103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908763+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:46:20.908763+0000","last_peered":"2024-08-17T17:46:20.908763+0000","last_clean":"2024-08-17T17:46:20.908763+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:46:20.908763+0000","last_undegraded":"2024-08-17T17:46:20.908763+0000","last_fullsized":"2024-08-17T17:46:20.908763+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918062+0000","last_change":"2024-08-17T17:31:49.805397+0000","last_active":"2024-08-17T17:46:20.918062+0000","last_peered":"2024-08-17T17:46:20.918062+0000","last_clean":"2024-08-17T17:46:20.918062+0000","last_became_active":"2024-08-17T17:31:49.804586+0000","last_became_peered":"2024-08-17T17:31:49.804586+0000","last_unstale":"2024-08-17T17:46:20.918062+0000","last_undegraded":"2024-08-17T17:46:20.918062+0000","last_fullsized":"2024-08-17T17:46:20.918062+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:58.697361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1661,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911739+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:46:20.911739+0000","last_peered":"2024-08-17T17:46:20.911739+0000","last_clean":"2024-08-17T17:46:20.911739+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:46:20.911739+0000","last_undegraded":"2024-08-17T17:46:20.911739+0000","last_fullsized":"2024-08-17T17:46:20.911739+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1118,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070708+0000","last_change":"2024-08-17T17:30:43.429232+0000","last_active":"2024-08-17T17:46:16.070708+0000","last_peered":"2024-08-17T17:46:16.070708+0000","last_clean":"2024-08-17T17:46:16.070708+0000","last_became_active":"2024-08-17T17:30:43.428944+0000","last_became_peered":"2024-08-17T17:30:43.428944+0000","last_unstale":"2024-08-17T17:46:16.070708+0000","last_undegraded":"2024-08-17T17:46:16.070708+0000","last_fullsized":"2024-08-17T17:46:16.070708+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:05.780222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798470+0000","last_change":"2024-08-17T17:31:49.801654+0000","last_active":"2024-08-17T17:46:15.798470+0000","last_peered":"2024-08-17T17:46:15.798470+0000","last_clean":"2024-08-17T17:46:15.798470+0000","last_became_active":"2024-08-17T17:31:49.801234+0000","last_became_peered":"2024-08-17T17:31:49.801234+0000","last_unstale":"2024-08-17T17:46:15.798470+0000","last_undegraded":"2024-08-17T17:46:15.798470+0000","last_fullsized":"2024-08-17T17:46:15.798470+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:21.590996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1760,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910427+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:46:20.910427+0000","last_peered":"2024-08-17T17:46:20.910427+0000","last_clean":"2024-08-17T17:46:20.910427+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:46:20.910427+0000","last_undegraded":"2024-08-17T17:46:20.910427+0000","last_fullsized":"2024-08-17T17:46:20.910427+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":1122,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009725+0000","last_change":"2024-08-17T17:21:52.156855+0000","last_active":"2024-08-17T17:46:16.009725+0000","last_peered":"2024-08-17T17:46:16.009725+0000","last_clean":"2024-08-17T17:46:16.009725+0000","last_became_active":"2024-08-17T17:21:52.156377+0000","last_became_peered":"2024-08-17T17:21:52.156377+0000","last_unstale":"2024-08-17T17:46:16.009725+0000","last_undegraded":"2024-08-17T17:46:16.009725+0000","last_fullsized":"2024-08-17T17:46:16.009725+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:29.248851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009846+0000","last_change":"2024-08-17T17:30:43.433235+0000","last_active":"2024-08-17T17:46:16.009846+0000","last_peered":"2024-08-17T17:46:16.009846+0000","last_clean":"2024-08-17T17:46:16.009846+0000","last_became_active":"2024-08-17T17:30:43.432889+0000","last_became_peered":"2024-08-17T17:30:43.432889+0000","last_unstale":"2024-08-17T17:46:16.009846+0000","last_undegraded":"2024-08-17T17:46:16.009846+0000","last_fullsized":"2024-08-17T17:46:16.009846+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:22.828100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1099,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009203+0000","last_change":"2024-08-17T17:21:53.161517+0000","last_active":"2024-08-17T17:46:16.009203+0000","last_peered":"2024-08-17T17:46:16.009203+0000","last_clean":"2024-08-17T17:46:16.009203+0000","last_became_active":"2024-08-17T17:21:53.161203+0000","last_became_peered":"2024-08-17T17:21:53.161203+0000","last_unstale":"2024-08-17T17:46:16.009203+0000","last_undegraded":"2024-08-17T17:46:16.009203+0000","last_fullsized":"2024-08-17T17:46:16.009203+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:46.059237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1126,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583834+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:46:15.583834+0000","last_peered":"2024-08-17T17:46:15.583834+0000","last_clean":"2024-08-17T17:46:15.583834+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:46:15.583834+0000","last_undegraded":"2024-08-17T17:46:15.583834+0000","last_fullsized":"2024-08-17T17:46:15.583834+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1816,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918808+0000","last_change":"2024-08-17T17:30:43.432449+0000","last_active":"2024-08-17T17:46:20.918808+0000","last_peered":"2024-08-17T17:46:20.918808+0000","last_clean":"2024-08-17T17:46:20.918808+0000","last_became_active":"2024-08-17T17:30:43.432085+0000","last_became_peered":"2024-08-17T17:30:43.432085+0000","last_unstale":"2024-08-17T17:46:20.918808+0000","last_undegraded":"2024-08-17T17:46:20.918808+0000","last_fullsized":"2024-08-17T17:46:20.918808+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:30:03.861379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1679,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908943+0000","last_change":"2024-08-17T17:31:49.799682+0000","last_active":"2024-08-17T17:46:20.908943+0000","last_peered":"2024-08-17T17:46:20.908943+0000","last_clean":"2024-08-17T17:46:20.908943+0000","last_became_active":"2024-08-17T17:31:49.799090+0000","last_became_peered":"2024-08-17T17:31:49.799090+0000","last_unstale":"2024-08-17T17:46:20.908943+0000","last_undegraded":"2024-08-17T17:46:20.908943+0000","last_fullsized":"2024-08-17T17:46:20.908943+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:15:53.576285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1758,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912213+0000","last_change":"2024-08-17T17:30:43.440126+0000","last_active":"2024-08-17T17:46:20.912213+0000","last_peered":"2024-08-17T17:46:20.912213+0000","last_clean":"2024-08-17T17:46:20.912213+0000","last_became_active":"2024-08-17T17:30:43.439694+0000","last_became_peered":"2024-08-17T17:30:43.439694+0000","last_unstale":"2024-08-17T17:46:20.912213+0000","last_undegraded":"2024-08-17T17:46:20.912213+0000","last_fullsized":"2024-08-17T17:46:20.912213+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:02.086846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1682,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912510+0000","last_change":"2024-08-17T17:31:49.807926+0000","last_active":"2024-08-17T17:46:20.912510+0000","last_peered":"2024-08-17T17:46:20.912510+0000","last_clean":"2024-08-17T17:46:20.912510+0000","last_became_active":"2024-08-17T17:31:49.802101+0000","last_became_peered":"2024-08-17T17:31:49.802101+0000","last_unstale":"2024-08-17T17:46:20.912510+0000","last_undegraded":"2024-08-17T17:46:20.912510+0000","last_fullsized":"2024-08-17T17:46:20.912510+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:30.530469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1801,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.920831+0000","last_change":"2024-08-17T17:29:31.898335+0000","last_active":"2024-08-17T17:46:20.920831+0000","last_peered":"2024-08-17T17:46:20.920831+0000","last_clean":"2024-08-17T17:46:20.920831+0000","last_became_active":"2024-08-17T17:29:31.897066+0000","last_became_peered":"2024-08-17T17:29:31.897066+0000","last_unstale":"2024-08-17T17:46:20.920831+0000","last_undegraded":"2024-08-17T17:46:20.920831+0000","last_fullsized":"2024-08-17T17:46:20.920831+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:17.743900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1126,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798623+0000","last_change":"2024-08-17T17:29:31.894594+0000","last_active":"2024-08-17T17:46:15.798623+0000","last_peered":"2024-08-17T17:46:15.798623+0000","last_clean":"2024-08-17T17:46:15.798623+0000","last_became_active":"2024-08-17T17:29:31.893900+0000","last_became_peered":"2024-08-17T17:29:31.893900+0000","last_unstale":"2024-08-17T17:46:15.798623+0000","last_undegraded":"2024-08-17T17:46:15.798623+0000","last_fullsized":"2024-08-17T17:46:15.798623+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:17:03.570954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1759,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909855+0000","last_change":"2024-08-17T17:31:15.540575+0000","last_active":"2024-08-17T17:46:20.909855+0000","last_peered":"2024-08-17T17:46:20.909855+0000","last_clean":"2024-08-17T17:46:20.909855+0000","last_became_active":"2024-08-17T17:31:15.540120+0000","last_became_peered":"2024-08-17T17:31:15.540120+0000","last_unstale":"2024-08-17T17:46:20.909855+0000","last_undegraded":"2024-08-17T17:46:20.909855+0000","last_fullsized":"2024-08-17T17:46:20.909855+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:13.718902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1728,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911458+0000","last_change":"2024-08-17T17:31:15.550627+0000","last_active":"2024-08-17T17:46:20.911458+0000","last_peered":"2024-08-17T17:46:20.911458+0000","last_clean":"2024-08-17T17:46:20.911458+0000","last_became_active":"2024-08-17T17:31:15.550283+0000","last_became_peered":"2024-08-17T17:31:15.550283+0000","last_unstale":"2024-08-17T17:46:20.911458+0000","last_undegraded":"2024-08-17T17:46:20.911458+0000","last_fullsized":"2024-08-17T17:46:20.911458+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:30.165773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1779,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909729+0000","last_change":"2024-08-17T17:30:43.435630+0000","last_active":"2024-08-17T17:46:20.909729+0000","last_peered":"2024-08-17T17:46:20.909729+0000","last_clean":"2024-08-17T17:46:20.909729+0000","last_became_active":"2024-08-17T17:30:43.435338+0000","last_became_peered":"2024-08-17T17:30:43.435338+0000","last_unstale":"2024-08-17T17:46:20.909729+0000","last_undegraded":"2024-08-17T17:46:20.909729+0000","last_fullsized":"2024-08-17T17:46:20.909729+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:44.675247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1135,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070456+0000","last_change":"2024-08-17T17:31:49.801416+0000","last_active":"2024-08-17T17:46:16.070456+0000","last_peered":"2024-08-17T17:46:16.070456+0000","last_clean":"2024-08-17T17:46:16.070456+0000","last_became_active":"2024-08-17T17:31:49.801160+0000","last_became_peered":"2024-08-17T17:31:49.801160+0000","last_unstale":"2024-08-17T17:46:16.070456+0000","last_undegraded":"2024-08-17T17:46:16.070456+0000","last_fullsized":"2024-08-17T17:46:16.070456+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:04.641376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1771,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909952+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:46:20.909952+0000","last_peered":"2024-08-17T17:46:20.909952+0000","last_clean":"2024-08-17T17:46:20.909952+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:46:20.909952+0000","last_undegraded":"2024-08-17T17:46:20.909952+0000","last_fullsized":"2024-08-17T17:46:20.909952+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009980+0000","last_change":"2024-08-17T17:29:31.897318+0000","last_active":"2024-08-17T17:46:16.009980+0000","last_peered":"2024-08-17T17:46:16.009980+0000","last_clean":"2024-08-17T17:46:16.009980+0000","last_became_active":"2024-08-17T17:29:31.896952+0000","last_became_peered":"2024-08-17T17:29:31.896952+0000","last_unstale":"2024-08-17T17:46:16.009980+0000","last_undegraded":"2024-08-17T17:46:16.009980+0000","last_fullsized":"2024-08-17T17:46:16.009980+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:07:09.908531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1132,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.069948+0000","last_change":"2024-08-17T17:30:43.432147+0000","last_active":"2024-08-17T17:46:16.069948+0000","last_peered":"2024-08-17T17:46:16.069948+0000","last_clean":"2024-08-17T17:46:16.069948+0000","last_became_active":"2024-08-17T17:30:43.431796+0000","last_became_peered":"2024-08-17T17:30:43.431796+0000","last_unstale":"2024-08-17T17:46:16.069948+0000","last_undegraded":"2024-08-17T17:46:16.069948+0000","last_fullsized":"2024-08-17T17:46:16.069948+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:43.435654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0008386,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1766,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912908+0000","last_change":"2024-08-17T17:30:43.435209+0000","last_active":"2024-08-17T17:46:20.912908+0000","last_peered":"2024-08-17T17:46:20.912908+0000","last_clean":"2024-08-17T17:46:20.912908+0000","last_became_active":"2024-08-17T17:30:43.434903+0000","last_became_peered":"2024-08-17T17:30:43.434903+0000","last_unstale":"2024-08-17T17:46:20.912908+0000","last_undegraded":"2024-08-17T17:46:20.912908+0000","last_fullsized":"2024-08-17T17:46:20.912908+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:27.551001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1760,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909154+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:46:20.909154+0000","last_peered":"2024-08-17T17:46:20.909154+0000","last_clean":"2024-08-17T17:46:20.909154+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:46:20.909154+0000","last_undegraded":"2024-08-17T17:46:20.909154+0000","last_fullsized":"2024-08-17T17:46:20.909154+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1769,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908437+0000","last_change":"2024-08-17T17:31:49.809141+0000","last_active":"2024-08-17T17:46:20.908437+0000","last_peered":"2024-08-17T17:46:20.908437+0000","last_clean":"2024-08-17T17:46:20.908437+0000","last_became_active":"2024-08-17T17:31:49.808855+0000","last_became_peered":"2024-08-17T17:31:49.808855+0000","last_unstale":"2024-08-17T17:46:20.908437+0000","last_undegraded":"2024-08-17T17:46:20.908437+0000","last_fullsized":"2024-08-17T17:46:20.908437+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:10.144929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1826,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.919131+0000","last_change":"2024-08-17T17:31:15.546567+0000","last_active":"2024-08-17T17:46:20.919131+0000","last_peered":"2024-08-17T17:46:20.919131+0000","last_clean":"2024-08-17T17:46:20.919131+0000","last_became_active":"2024-08-17T17:31:15.546117+0000","last_became_peered":"2024-08-17T17:31:15.546117+0000","last_unstale":"2024-08-17T17:46:20.919131+0000","last_undegraded":"2024-08-17T17:46:20.919131+0000","last_fullsized":"2024-08-17T17:46:20.919131+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:19.245231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1813,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.915571+0000","last_change":"2024-08-17T17:31:49.811704+0000","last_active":"2024-08-17T17:46:20.915571+0000","last_peered":"2024-08-17T17:46:20.915571+0000","last_clean":"2024-08-17T17:46:20.915571+0000","last_became_active":"2024-08-17T17:31:49.811385+0000","last_became_peered":"2024-08-17T17:31:49.811385+0000","last_unstale":"2024-08-17T17:46:20.915571+0000","last_undegraded":"2024-08-17T17:46:20.915571+0000","last_fullsized":"2024-08-17T17:46:20.915571+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:58.832774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1143,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798033+0000","last_change":"2024-08-17T17:30:43.432897+0000","last_active":"2024-08-17T17:46:15.798033+0000","last_peered":"2024-08-17T17:46:15.798033+0000","last_clean":"2024-08-17T17:46:15.798033+0000","last_became_active":"2024-08-17T17:30:43.432602+0000","last_became_peered":"2024-08-17T17:30:43.432602+0000","last_unstale":"2024-08-17T17:46:15.798033+0000","last_undegraded":"2024-08-17T17:46:15.798033+0000","last_fullsized":"2024-08-17T17:46:15.798033+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:51.083549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1096,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583544+0000","last_change":"2024-08-17T17:21:53.164672+0000","last_active":"2024-08-17T17:46:15.583544+0000","last_peered":"2024-08-17T17:46:15.583544+0000","last_clean":"2024-08-17T17:46:15.583544+0000","last_became_active":"2024-08-17T17:21:53.164381+0000","last_became_peered":"2024-08-17T17:21:53.164381+0000","last_unstale":"2024-08-17T17:46:15.583544+0000","last_undegraded":"2024-08-17T17:46:15.583544+0000","last_fullsized":"2024-08-17T17:46:15.583544+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:43:36.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1661,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909361+0000","last_change":"2024-08-17T17:31:49.807517+0000","last_active":"2024-08-17T17:46:20.909361+0000","last_peered":"2024-08-17T17:46:20.909361+0000","last_clean":"2024-08-17T17:46:20.909361+0000","last_became_active":"2024-08-17T17:31:49.802152+0000","last_became_peered":"2024-08-17T17:31:49.802152+0000","last_unstale":"2024-08-17T17:46:20.909361+0000","last_undegraded":"2024-08-17T17:46:20.909361+0000","last_fullsized":"2024-08-17T17:46:20.909361+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:02.934420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1131,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070064+0000","last_change":"2024-08-17T17:31:15.545691+0000","last_active":"2024-08-17T17:46:16.070064+0000","last_peered":"2024-08-17T17:46:16.070064+0000","last_clean":"2024-08-17T17:46:16.070064+0000","last_became_active":"2024-08-17T17:31:15.545335+0000","last_became_peered":"2024-08-17T17:31:15.545335+0000","last_unstale":"2024-08-17T17:46:16.070064+0000","last_undegraded":"2024-08-17T17:46:16.070064+0000","last_fullsized":"2024-08-17T17:46:16.070064+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:18.694039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085985900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1455,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798170+0000","last_change":"2024-08-17T17:29:31.894591+0000","last_active":"2024-08-17T17:46:15.798170+0000","last_peered":"2024-08-17T17:46:15.798170+0000","last_clean":"2024-08-17T17:46:15.798170+0000","last_became_active":"2024-08-17T17:29:31.894134+0000","last_became_peered":"2024-08-17T17:29:31.894134+0000","last_unstale":"2024-08-17T17:46:15.798170+0000","last_undegraded":"2024-08-17T17:46:15.798170+0000","last_fullsized":"2024-08-17T17:46:15.798170+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:12.586656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1760,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.913715+0000","last_change":"2024-08-17T17:31:49.810921+0000","last_active":"2024-08-17T17:46:20.913715+0000","last_peered":"2024-08-17T17:46:20.913715+0000","last_clean":"2024-08-17T17:46:20.913715+0000","last_became_active":"2024-08-17T17:31:49.810583+0000","last_became_peered":"2024-08-17T17:31:49.810583+0000","last_unstale":"2024-08-17T17:46:20.913715+0000","last_undegraded":"2024-08-17T17:46:20.913715+0000","last_fullsized":"2024-08-17T17:46:20.913715+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:44.272815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1707,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908901+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:46:20.908901+0000","last_peered":"2024-08-17T17:46:20.908901+0000","last_clean":"2024-08-17T17:46:20.908901+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:46:20.908901+0000","last_undegraded":"2024-08-17T17:46:20.908901+0000","last_fullsized":"2024-08-17T17:46:20.908901+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009910+0000","last_change":"2024-08-17T17:30:43.434276+0000","last_active":"2024-08-17T17:46:16.009910+0000","last_peered":"2024-08-17T17:46:16.009910+0000","last_clean":"2024-08-17T17:46:16.009910+0000","last_became_active":"2024-08-17T17:30:43.433844+0000","last_became_peered":"2024-08-17T17:30:43.433844+0000","last_unstale":"2024-08-17T17:46:16.009910+0000","last_undegraded":"2024-08-17T17:46:16.009910+0000","last_fullsized":"2024-08-17T17:46:16.009910+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:17:56.025107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009503+0000","last_change":"2024-08-17T17:31:15.546844+0000","last_active":"2024-08-17T17:46:16.009503+0000","last_peered":"2024-08-17T17:46:16.009503+0000","last_clean":"2024-08-17T17:46:16.009503+0000","last_became_active":"2024-08-17T17:31:15.546516+0000","last_became_peered":"2024-08-17T17:31:15.546516+0000","last_unstale":"2024-08-17T17:46:16.009503+0000","last_undegraded":"2024-08-17T17:46:16.009503+0000","last_fullsized":"2024-08-17T17:46:16.009503+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:10:18.148181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009616+0000","last_change":"2024-08-17T17:31:15.546671+0000","last_active":"2024-08-17T17:46:16.009616+0000","last_peered":"2024-08-17T17:46:16.009616+0000","last_clean":"2024-08-17T17:46:16.009616+0000","last_became_active":"2024-08-17T17:31:15.546355+0000","last_became_peered":"2024-08-17T17:31:15.546355+0000","last_unstale":"2024-08-17T17:46:16.009616+0000","last_undegraded":"2024-08-17T17:46:16.009616+0000","last_fullsized":"2024-08-17T17:46:16.009616+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:45.836713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1114,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583330+0000","last_change":"2024-08-17T17:30:43.433097+0000","last_active":"2024-08-17T17:46:15.583330+0000","last_peered":"2024-08-17T17:46:15.583330+0000","last_clean":"2024-08-17T17:46:15.583330+0000","last_became_active":"2024-08-17T17:30:43.432179+0000","last_became_peered":"2024-08-17T17:30:43.432179+0000","last_unstale":"2024-08-17T17:46:15.583330+0000","last_undegraded":"2024-08-17T17:46:15.583330+0000","last_fullsized":"2024-08-17T17:46:15.583330+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:44:40.966915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009644+0000","last_change":"2024-08-17T17:31:49.800286+0000","last_active":"2024-08-17T17:46:16.009644+0000","last_peered":"2024-08-17T17:46:16.009644+0000","last_clean":"2024-08-17T17:46:16.009644+0000","last_became_active":"2024-08-17T17:31:49.799834+0000","last_became_peered":"2024-08-17T17:31:49.799834+0000","last_unstale":"2024-08-17T17:46:16.009644+0000","last_undegraded":"2024-08-17T17:46:16.009644+0000","last_fullsized":"2024-08-17T17:46:16.009644+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:28.409397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1681,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910644+0000","last_change":"2024-08-17T17:31:49.801723+0000","last_active":"2024-08-17T17:46:20.910644+0000","last_peered":"2024-08-17T17:46:20.910644+0000","last_clean":"2024-08-17T17:46:20.910644+0000","last_became_active":"2024-08-17T17:31:49.801370+0000","last_became_peered":"2024-08-17T17:31:49.801370+0000","last_unstale":"2024-08-17T17:46:20.910644+0000","last_undegraded":"2024-08-17T17:46:20.910644+0000","last_fullsized":"2024-08-17T17:46:20.910644+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:28.686255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.920662+0000","last_change":"2024-08-17T17:30:43.431404+0000","last_active":"2024-08-17T17:46:20.920662+0000","last_peered":"2024-08-17T17:46:20.920662+0000","last_clean":"2024-08-17T17:46:20.920662+0000","last_became_active":"2024-08-17T17:30:43.431087+0000","last_became_peered":"2024-08-17T17:30:43.431087+0000","last_unstale":"2024-08-17T17:46:20.920662+0000","last_undegraded":"2024-08-17T17:46:20.920662+0000","last_fullsized":"2024-08-17T17:46:20.920662+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:15.830770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1118,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070176+0000","last_change":"2024-08-17T17:31:49.799082+0000","last_active":"2024-08-17T17:46:16.070176+0000","last_peered":"2024-08-17T17:46:16.070176+0000","last_clean":"2024-08-17T17:46:16.070176+0000","last_became_active":"2024-08-17T17:31:49.798764+0000","last_became_peered":"2024-08-17T17:31:49.798764+0000","last_unstale":"2024-08-17T17:46:16.070176+0000","last_undegraded":"2024-08-17T17:46:16.070176+0000","last_fullsized":"2024-08-17T17:46:16.070176+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:07.697148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1113,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583225+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:46:15.583225+0000","last_peered":"2024-08-17T17:46:15.583225+0000","last_clean":"2024-08-17T17:46:15.583225+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:46:15.583225+0000","last_undegraded":"2024-08-17T17:46:15.583225+0000","last_fullsized":"2024-08-17T17:46:15.583225+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1098,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798620+0000","last_change":"2024-08-17T17:21:52.151540+0000","last_active":"2024-08-17T17:46:15.798620+0000","last_peered":"2024-08-17T17:46:15.798620+0000","last_clean":"2024-08-17T17:46:15.798620+0000","last_became_active":"2024-08-17T17:21:52.151294+0000","last_became_peered":"2024-08-17T17:21:52.151294+0000","last_unstale":"2024-08-17T17:46:15.798620+0000","last_undegraded":"2024-08-17T17:46:15.798620+0000","last_fullsized":"2024-08-17T17:46:15.798620+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:03.533252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009427+0000","last_change":"2024-08-17T17:31:49.799847+0000","last_active":"2024-08-17T17:46:16.009427+0000","last_peered":"2024-08-17T17:46:16.009427+0000","last_clean":"2024-08-17T17:46:16.009427+0000","last_became_active":"2024-08-17T17:31:49.799523+0000","last_became_peered":"2024-08-17T17:31:49.799523+0000","last_unstale":"2024-08-17T17:46:16.009427+0000","last_undegraded":"2024-08-17T17:46:16.009427+0000","last_fullsized":"2024-08-17T17:46:16.009427+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:34.070172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798751+0000","last_change":"2024-08-17T17:30:43.437929+0000","last_active":"2024-08-17T17:46:15.798751+0000","last_peered":"2024-08-17T17:46:15.798751+0000","last_clean":"2024-08-17T17:46:15.798751+0000","last_became_active":"2024-08-17T17:30:43.437604+0000","last_became_peered":"2024-08-17T17:30:43.437604+0000","last_unstale":"2024-08-17T17:46:15.798751+0000","last_undegraded":"2024-08-17T17:46:15.798751+0000","last_fullsized":"2024-08-17T17:46:15.798751+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:55:21.953561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1100,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.069864+0000","last_change":"2024-08-17T17:21:53.161767+0000","last_active":"2024-08-17T17:46:16.069864+0000","last_peered":"2024-08-17T17:46:16.069864+0000","last_clean":"2024-08-17T17:46:16.069864+0000","last_became_active":"2024-08-17T17:21:53.161514+0000","last_became_peered":"2024-08-17T17:21:53.161514+0000","last_unstale":"2024-08-17T17:46:16.069864+0000","last_undegraded":"2024-08-17T17:46:16.069864+0000","last_fullsized":"2024-08-17T17:46:16.069864+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:59.837673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1115,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798770+0000","last_change":"2024-08-17T17:29:31.900876+0000","last_active":"2024-08-17T17:46:15.798770+0000","last_peered":"2024-08-17T17:46:15.798770+0000","last_clean":"2024-08-17T17:46:15.798770+0000","last_became_active":"2024-08-17T17:29:31.900496+0000","last_became_peered":"2024-08-17T17:29:31.900496+0000","last_unstale":"2024-08-17T17:46:15.798770+0000","last_undegraded":"2024-08-17T17:46:15.798770+0000","last_fullsized":"2024-08-17T17:46:15.798770+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:18.971519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1122,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583539+0000","last_change":"2024-08-17T17:29:31.891859+0000","last_active":"2024-08-17T17:46:15.583539+0000","last_peered":"2024-08-17T17:46:15.583539+0000","last_clean":"2024-08-17T17:46:15.583539+0000","last_became_active":"2024-08-17T17:29:31.891405+0000","last_became_peered":"2024-08-17T17:29:31.891405+0000","last_unstale":"2024-08-17T17:46:15.583539+0000","last_undegraded":"2024-08-17T17:46:15.583539+0000","last_fullsized":"2024-08-17T17:46:15.583539+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:48.087233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912321+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:46:20.912321+0000","last_peered":"2024-08-17T17:46:20.912321+0000","last_clean":"2024-08-17T17:46:20.912321+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:46:20.912321+0000","last_undegraded":"2024-08-17T17:46:20.912321+0000","last_fullsized":"2024-08-17T17:46:20.912321+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1097,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798097+0000","last_change":"2024-08-17T17:21:52.149957+0000","last_active":"2024-08-17T17:46:15.798097+0000","last_peered":"2024-08-17T17:46:15.798097+0000","last_clean":"2024-08-17T17:46:15.798097+0000","last_became_active":"2024-08-17T17:21:52.149673+0000","last_became_peered":"2024-08-17T17:21:52.149673+0000","last_unstale":"2024-08-17T17:46:15.798097+0000","last_undegraded":"2024-08-17T17:46:15.798097+0000","last_fullsized":"2024-08-17T17:46:15.798097+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:01:16.473279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918958+0000","last_change":"2024-08-17T17:31:49.805181+0000","last_active":"2024-08-17T17:46:20.918958+0000","last_peered":"2024-08-17T17:46:20.918958+0000","last_clean":"2024-08-17T17:46:20.918958+0000","last_became_active":"2024-08-17T17:31:49.804905+0000","last_became_peered":"2024-08-17T17:31:49.804905+0000","last_unstale":"2024-08-17T17:46:20.918958+0000","last_undegraded":"2024-08-17T17:46:20.918958+0000","last_fullsized":"2024-08-17T17:46:20.918958+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:47:16.879688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.916345+0000","last_change":"2024-08-17T17:31:49.812747+0000","last_active":"2024-08-17T17:46:20.916345+0000","last_peered":"2024-08-17T17:46:20.916345+0000","last_clean":"2024-08-17T17:46:20.916345+0000","last_became_active":"2024-08-17T17:31:49.812378+0000","last_became_peered":"2024-08-17T17:31:49.812378+0000","last_unstale":"2024-08-17T17:46:20.916345+0000","last_undegraded":"2024-08-17T17:46:20.916345+0000","last_fullsized":"2024-08-17T17:46:20.916345+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:35.124235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070375+0000","last_change":"2024-08-17T17:31:49.801380+0000","last_active":"2024-08-17T17:46:16.070375+0000","last_peered":"2024-08-17T17:46:16.070375+0000","last_clean":"2024-08-17T17:46:16.070375+0000","last_became_active":"2024-08-17T17:31:49.801103+0000","last_became_peered":"2024-08-17T17:31:49.801103+0000","last_unstale":"2024-08-17T17:46:16.070375+0000","last_undegraded":"2024-08-17T17:46:16.070375+0000","last_fullsized":"2024-08-17T17:46:16.070375+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:09:43.554983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798259+0000","last_change":"2024-08-17T17:31:15.543729+0000","last_active":"2024-08-17T17:46:15.798259+0000","last_peered":"2024-08-17T17:46:15.798259+0000","last_clean":"2024-08-17T17:46:15.798259+0000","last_became_active":"2024-08-17T17:31:15.543142+0000","last_became_peered":"2024-08-17T17:31:15.543142+0000","last_unstale":"2024-08-17T17:46:15.798259+0000","last_undegraded":"2024-08-17T17:46:15.798259+0000","last_fullsized":"2024-08-17T17:46:15.798259+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:00.612263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910980+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:46:20.910980+0000","last_peered":"2024-08-17T17:46:20.910980+0000","last_clean":"2024-08-17T17:46:20.910980+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:46:20.910980+0000","last_undegraded":"2024-08-17T17:46:20.910980+0000","last_fullsized":"2024-08-17T17:46:20.910980+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234315,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325232,"kb_used_data":19556,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93419920,"statfs":{"total":95995035648,"available":95661998080,"internally_reserved":0,"allocated":20025344,"data_stored":12500974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495950,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":329380,"kb_used_data":19544,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93415772,"statfs":{"total":95995035648,"available":95657750528,"internally_reserved":0,"allocated":20013056,"data_stored":12500532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757588,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316316,"kb_used_data":18960,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93428836,"statfs":{"total":95995035648,"available":95671128064,"internally_reserved":0,"allocated":19415040,"data_stored":11906313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326202,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":68888,"kb_used_data":18956,"kb_used_omap":57,"kb_used_meta":49862,"kb_avail":93676264,"statfs":{"total":95995035648,"available":95924494336,"internally_reserved":0,"allocated":19410944,"data_stored":11906029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":51059451},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405441,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":73108,"kb_used_data":18956,"kb_used_omap":61,"kb_used_meta":54082,"kb_avail":93672044,"statfs":{"total":95995035648,"available":95920173056,"internally_reserved":0,"allocated":19410944,"data_stored":11905882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":55380920},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253451976,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":73044,"kb_used_data":18956,"kb_used_omap":54,"kb_used_meta":54025,"kb_avail":93672108,"statfs":{"total":95995035648,"available":95920238592,"internally_reserved":0,"allocated":19410944,"data_stored":11905966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":55321998},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":438,"seq":1881195675862,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":65248,"kb_used_data":19540,"kb_used_omap":43,"kb_used_meta":45652,"kb_avail":93679904,"statfs":{"total":95995035648,"available":95928221696,"internally_reserved":0,"allocated":20008960,"data_stored":12496899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44456,"internal_metadata":46748248},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019223,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328808,"kb_used_data":18972,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93416344,"statfs":{"total":95995035648,"available":95658336256,"internally_reserved":0,"allocated":19427328,"data_stored":11910544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:47:14.228 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:47:14.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:14 smithi130 ceph-mon[105521]: from='client.45089 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:14.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:14 smithi130 ceph-mon[110068]: from='client.45089 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:14.995 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:14 smithi179 ceph-mon[93519]: from='client.45089 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:15.051 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:15.698 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:15 smithi130 ceph-mon[105521]: pgmap v1542: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:15.699 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:15 smithi130 ceph-mon[110068]: pgmap v1542: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:16.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:15 smithi179 ceph-mon[93519]: pgmap v1542: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:16.856 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:47:16.857 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:47:17.609 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1543,"stamp":"2024-08-17T17:47:16.378978+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1580024,"kb_used_data":153440,"kb_used_omap":223,"kb_used_meta":1426272,"kb_avail":748381192,"statfs":{"total":767960285184,"available":766342340608,"internally_reserved":0,"allocated":157122560,"data_stored":97033139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":228359,"internal_metadata":1460503545},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002536"},"pg_stats":[{"pgid":"3.1f","version":"71'4","reported_seq":1819,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918108+0000","last_change":"2024-08-17T17:31:15.551224+0000","last_active":"2024-08-17T17:46:20.918108+0000","last_peered":"2024-08-17T17:46:20.918108+0000","last_clean":"2024-08-17T17:46:20.918108+0000","last_became_active":"2024-08-17T17:31:15.550855+0000","last_became_peered":"2024-08-17T17:31:15.550855+0000","last_unstale":"2024-08-17T17:46:20.918108+0000","last_undegraded":"2024-08-17T17:46:20.918108+0000","last_fullsized":"2024-08-17T17:46:20.918108+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:05:07.081474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009009+0000","last_change":"2024-08-17T17:30:43.429215+0000","last_active":"2024-08-17T17:46:16.009009+0000","last_peered":"2024-08-17T17:46:16.009009+0000","last_clean":"2024-08-17T17:46:16.009009+0000","last_became_active":"2024-08-17T17:30:43.428891+0000","last_became_peered":"2024-08-17T17:30:43.428891+0000","last_unstale":"2024-08-17T17:46:16.009009+0000","last_undegraded":"2024-08-17T17:46:16.009009+0000","last_fullsized":"2024-08-17T17:46:16.009009+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:41:39.313325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1698,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911021+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:46:20.911021+0000","last_peered":"2024-08-17T17:46:20.911021+0000","last_clean":"2024-08-17T17:46:20.911021+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:46:20.911021+0000","last_undegraded":"2024-08-17T17:46:20.911021+0000","last_fullsized":"2024-08-17T17:46:20.911021+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1712,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911837+0000","last_change":"2024-08-17T17:31:49.809768+0000","last_active":"2024-08-17T17:46:20.911837+0000","last_peered":"2024-08-17T17:46:20.911837+0000","last_clean":"2024-08-17T17:46:20.911837+0000","last_became_active":"2024-08-17T17:31:49.808276+0000","last_became_peered":"2024-08-17T17:31:49.808276+0000","last_unstale":"2024-08-17T17:46:20.911837+0000","last_undegraded":"2024-08-17T17:46:20.911837+0000","last_fullsized":"2024-08-17T17:46:20.911837+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:51.600719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":1103,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070283+0000","last_change":"2024-08-17T17:21:52.157731+0000","last_active":"2024-08-17T17:46:16.070283+0000","last_peered":"2024-08-17T17:46:16.070283+0000","last_clean":"2024-08-17T17:46:16.070283+0000","last_became_active":"2024-08-17T17:21:52.157453+0000","last_became_peered":"2024-08-17T17:21:52.157453+0000","last_unstale":"2024-08-17T17:46:16.070283+0000","last_undegraded":"2024-08-17T17:46:16.070283+0000","last_fullsized":"2024-08-17T17:46:16.070283+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:30:09.106088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009219+0000","last_change":"2024-08-17T17:31:49.806060+0000","last_active":"2024-08-17T17:46:16.009219+0000","last_peered":"2024-08-17T17:46:16.009219+0000","last_clean":"2024-08-17T17:46:16.009219+0000","last_became_active":"2024-08-17T17:31:49.805688+0000","last_became_peered":"2024-08-17T17:31:49.805688+0000","last_unstale":"2024-08-17T17:46:16.009219+0000","last_undegraded":"2024-08-17T17:46:16.009219+0000","last_fullsized":"2024-08-17T17:46:16.009219+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:42.831662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":1120,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070144+0000","last_change":"2024-08-17T17:30:43.427884+0000","last_active":"2024-08-17T17:46:16.070144+0000","last_peered":"2024-08-17T17:46:16.070144+0000","last_clean":"2024-08-17T17:46:16.070144+0000","last_became_active":"2024-08-17T17:30:43.427497+0000","last_became_peered":"2024-08-17T17:30:43.427497+0000","last_unstale":"2024-08-17T17:46:16.070144+0000","last_undegraded":"2024-08-17T17:46:16.070144+0000","last_fullsized":"2024-08-17T17:46:16.070144+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:22.830092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009797+0000","last_change":"2024-08-17T17:31:49.799570+0000","last_active":"2024-08-17T17:46:16.009797+0000","last_peered":"2024-08-17T17:46:16.009797+0000","last_clean":"2024-08-17T17:46:16.009797+0000","last_became_active":"2024-08-17T17:31:49.799251+0000","last_became_peered":"2024-08-17T17:31:49.799251+0000","last_unstale":"2024-08-17T17:46:16.009797+0000","last_undegraded":"2024-08-17T17:46:16.009797+0000","last_fullsized":"2024-08-17T17:46:16.009797+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:58.973574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1804,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.920819+0000","last_change":"2024-08-17T17:29:31.897430+0000","last_active":"2024-08-17T17:46:20.920819+0000","last_peered":"2024-08-17T17:46:20.920819+0000","last_clean":"2024-08-17T17:46:20.920819+0000","last_became_active":"2024-08-17T17:29:31.896722+0000","last_became_peered":"2024-08-17T17:29:31.896722+0000","last_unstale":"2024-08-17T17:46:20.920819+0000","last_undegraded":"2024-08-17T17:46:20.920819+0000","last_fullsized":"2024-08-17T17:46:20.920819+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:41:41.942213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1740,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912786+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:46:20.912786+0000","last_peered":"2024-08-17T17:46:20.912786+0000","last_clean":"2024-08-17T17:46:20.912786+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:46:20.912786+0000","last_undegraded":"2024-08-17T17:46:20.912786+0000","last_fullsized":"2024-08-17T17:46:20.912786+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":1135,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009572+0000","last_change":"2024-08-17T17:31:15.547327+0000","last_active":"2024-08-17T17:46:16.009572+0000","last_peered":"2024-08-17T17:46:16.009572+0000","last_clean":"2024-08-17T17:46:16.009572+0000","last_became_active":"2024-08-17T17:31:15.546959+0000","last_became_peered":"2024-08-17T17:31:15.546959+0000","last_unstale":"2024-08-17T17:46:16.009572+0000","last_undegraded":"2024-08-17T17:46:16.009572+0000","last_fullsized":"2024-08-17T17:46:16.009572+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:27:49.216367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910954+0000","last_change":"2024-08-17T17:31:49.802098+0000","last_active":"2024-08-17T17:46:20.910954+0000","last_peered":"2024-08-17T17:46:20.910954+0000","last_clean":"2024-08-17T17:46:20.910954+0000","last_became_active":"2024-08-17T17:31:49.801747+0000","last_became_peered":"2024-08-17T17:31:49.801747+0000","last_unstale":"2024-08-17T17:46:20.910954+0000","last_undegraded":"2024-08-17T17:46:20.910954+0000","last_fullsized":"2024-08-17T17:46:20.910954+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:00.212402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1759,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908264+0000","last_change":"2024-08-17T17:31:49.815142+0000","last_active":"2024-08-17T17:46:20.908264+0000","last_peered":"2024-08-17T17:46:20.908264+0000","last_clean":"2024-08-17T17:46:20.908264+0000","last_became_active":"2024-08-17T17:31:49.814873+0000","last_became_peered":"2024-08-17T17:31:49.814873+0000","last_unstale":"2024-08-17T17:46:20.908264+0000","last_undegraded":"2024-08-17T17:46:20.908264+0000","last_fullsized":"2024-08-17T17:46:20.908264+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:05.542100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1812,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.919007+0000","last_change":"2024-08-17T17:31:49.804738+0000","last_active":"2024-08-17T17:46:20.919007+0000","last_peered":"2024-08-17T17:46:20.919007+0000","last_clean":"2024-08-17T17:46:20.919007+0000","last_became_active":"2024-08-17T17:31:49.804476+0000","last_became_peered":"2024-08-17T17:31:49.804476+0000","last_unstale":"2024-08-17T17:46:20.919007+0000","last_undegraded":"2024-08-17T17:46:20.919007+0000","last_fullsized":"2024-08-17T17:46:20.919007+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:13:51.659079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1723,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909025+0000","last_change":"2024-08-17T17:31:49.808285+0000","last_active":"2024-08-17T17:46:20.909025+0000","last_peered":"2024-08-17T17:46:20.909025+0000","last_clean":"2024-08-17T17:46:20.909025+0000","last_became_active":"2024-08-17T17:31:49.805633+0000","last_became_peered":"2024-08-17T17:31:49.805633+0000","last_unstale":"2024-08-17T17:46:20.909025+0000","last_undegraded":"2024-08-17T17:46:20.909025+0000","last_fullsized":"2024-08-17T17:46:20.909025+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:38.815825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1113,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583821+0000","last_change":"2024-08-17T17:30:43.429169+0000","last_active":"2024-08-17T17:46:15.583821+0000","last_peered":"2024-08-17T17:46:15.583821+0000","last_clean":"2024-08-17T17:46:15.583821+0000","last_became_active":"2024-08-17T17:30:43.428850+0000","last_became_peered":"2024-08-17T17:30:43.428850+0000","last_unstale":"2024-08-17T17:46:15.583821+0000","last_undegraded":"2024-08-17T17:46:15.583821+0000","last_fullsized":"2024-08-17T17:46:15.583821+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:51:44.920103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908763+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:46:20.908763+0000","last_peered":"2024-08-17T17:46:20.908763+0000","last_clean":"2024-08-17T17:46:20.908763+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:46:20.908763+0000","last_undegraded":"2024-08-17T17:46:20.908763+0000","last_fullsized":"2024-08-17T17:46:20.908763+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918062+0000","last_change":"2024-08-17T17:31:49.805397+0000","last_active":"2024-08-17T17:46:20.918062+0000","last_peered":"2024-08-17T17:46:20.918062+0000","last_clean":"2024-08-17T17:46:20.918062+0000","last_became_active":"2024-08-17T17:31:49.804586+0000","last_became_peered":"2024-08-17T17:31:49.804586+0000","last_unstale":"2024-08-17T17:46:20.918062+0000","last_undegraded":"2024-08-17T17:46:20.918062+0000","last_fullsized":"2024-08-17T17:46:20.918062+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:58.697361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1661,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911739+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:46:20.911739+0000","last_peered":"2024-08-17T17:46:20.911739+0000","last_clean":"2024-08-17T17:46:20.911739+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:46:20.911739+0000","last_undegraded":"2024-08-17T17:46:20.911739+0000","last_fullsized":"2024-08-17T17:46:20.911739+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1118,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070708+0000","last_change":"2024-08-17T17:30:43.429232+0000","last_active":"2024-08-17T17:46:16.070708+0000","last_peered":"2024-08-17T17:46:16.070708+0000","last_clean":"2024-08-17T17:46:16.070708+0000","last_became_active":"2024-08-17T17:30:43.428944+0000","last_became_peered":"2024-08-17T17:30:43.428944+0000","last_unstale":"2024-08-17T17:46:16.070708+0000","last_undegraded":"2024-08-17T17:46:16.070708+0000","last_fullsized":"2024-08-17T17:46:16.070708+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:05.780222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798470+0000","last_change":"2024-08-17T17:31:49.801654+0000","last_active":"2024-08-17T17:46:15.798470+0000","last_peered":"2024-08-17T17:46:15.798470+0000","last_clean":"2024-08-17T17:46:15.798470+0000","last_became_active":"2024-08-17T17:31:49.801234+0000","last_became_peered":"2024-08-17T17:31:49.801234+0000","last_unstale":"2024-08-17T17:46:15.798470+0000","last_undegraded":"2024-08-17T17:46:15.798470+0000","last_fullsized":"2024-08-17T17:46:15.798470+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:21.590996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1760,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910427+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:46:20.910427+0000","last_peered":"2024-08-17T17:46:20.910427+0000","last_clean":"2024-08-17T17:46:20.910427+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:46:20.910427+0000","last_undegraded":"2024-08-17T17:46:20.910427+0000","last_fullsized":"2024-08-17T17:46:20.910427+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":1122,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009725+0000","last_change":"2024-08-17T17:21:52.156855+0000","last_active":"2024-08-17T17:46:16.009725+0000","last_peered":"2024-08-17T17:46:16.009725+0000","last_clean":"2024-08-17T17:46:16.009725+0000","last_became_active":"2024-08-17T17:21:52.156377+0000","last_became_peered":"2024-08-17T17:21:52.156377+0000","last_unstale":"2024-08-17T17:46:16.009725+0000","last_undegraded":"2024-08-17T17:46:16.009725+0000","last_fullsized":"2024-08-17T17:46:16.009725+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:29.248851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009846+0000","last_change":"2024-08-17T17:30:43.433235+0000","last_active":"2024-08-17T17:46:16.009846+0000","last_peered":"2024-08-17T17:46:16.009846+0000","last_clean":"2024-08-17T17:46:16.009846+0000","last_became_active":"2024-08-17T17:30:43.432889+0000","last_became_peered":"2024-08-17T17:30:43.432889+0000","last_unstale":"2024-08-17T17:46:16.009846+0000","last_undegraded":"2024-08-17T17:46:16.009846+0000","last_fullsized":"2024-08-17T17:46:16.009846+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:22.828100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1099,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009203+0000","last_change":"2024-08-17T17:21:53.161517+0000","last_active":"2024-08-17T17:46:16.009203+0000","last_peered":"2024-08-17T17:46:16.009203+0000","last_clean":"2024-08-17T17:46:16.009203+0000","last_became_active":"2024-08-17T17:21:53.161203+0000","last_became_peered":"2024-08-17T17:21:53.161203+0000","last_unstale":"2024-08-17T17:46:16.009203+0000","last_undegraded":"2024-08-17T17:46:16.009203+0000","last_fullsized":"2024-08-17T17:46:16.009203+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:46.059237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1126,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583834+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:46:15.583834+0000","last_peered":"2024-08-17T17:46:15.583834+0000","last_clean":"2024-08-17T17:46:15.583834+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:46:15.583834+0000","last_undegraded":"2024-08-17T17:46:15.583834+0000","last_fullsized":"2024-08-17T17:46:15.583834+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1816,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918808+0000","last_change":"2024-08-17T17:30:43.432449+0000","last_active":"2024-08-17T17:46:20.918808+0000","last_peered":"2024-08-17T17:46:20.918808+0000","last_clean":"2024-08-17T17:46:20.918808+0000","last_became_active":"2024-08-17T17:30:43.432085+0000","last_became_peered":"2024-08-17T17:30:43.432085+0000","last_unstale":"2024-08-17T17:46:20.918808+0000","last_undegraded":"2024-08-17T17:46:20.918808+0000","last_fullsized":"2024-08-17T17:46:20.918808+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:30:03.861379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1679,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908943+0000","last_change":"2024-08-17T17:31:49.799682+0000","last_active":"2024-08-17T17:46:20.908943+0000","last_peered":"2024-08-17T17:46:20.908943+0000","last_clean":"2024-08-17T17:46:20.908943+0000","last_became_active":"2024-08-17T17:31:49.799090+0000","last_became_peered":"2024-08-17T17:31:49.799090+0000","last_unstale":"2024-08-17T17:46:20.908943+0000","last_undegraded":"2024-08-17T17:46:20.908943+0000","last_fullsized":"2024-08-17T17:46:20.908943+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:15:53.576285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1758,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912213+0000","last_change":"2024-08-17T17:30:43.440126+0000","last_active":"2024-08-17T17:46:20.912213+0000","last_peered":"2024-08-17T17:46:20.912213+0000","last_clean":"2024-08-17T17:46:20.912213+0000","last_became_active":"2024-08-17T17:30:43.439694+0000","last_became_peered":"2024-08-17T17:30:43.439694+0000","last_unstale":"2024-08-17T17:46:20.912213+0000","last_undegraded":"2024-08-17T17:46:20.912213+0000","last_fullsized":"2024-08-17T17:46:20.912213+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:02.086846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1682,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912510+0000","last_change":"2024-08-17T17:31:49.807926+0000","last_active":"2024-08-17T17:46:20.912510+0000","last_peered":"2024-08-17T17:46:20.912510+0000","last_clean":"2024-08-17T17:46:20.912510+0000","last_became_active":"2024-08-17T17:31:49.802101+0000","last_became_peered":"2024-08-17T17:31:49.802101+0000","last_unstale":"2024-08-17T17:46:20.912510+0000","last_undegraded":"2024-08-17T17:46:20.912510+0000","last_fullsized":"2024-08-17T17:46:20.912510+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:30.530469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1801,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.920831+0000","last_change":"2024-08-17T17:29:31.898335+0000","last_active":"2024-08-17T17:46:20.920831+0000","last_peered":"2024-08-17T17:46:20.920831+0000","last_clean":"2024-08-17T17:46:20.920831+0000","last_became_active":"2024-08-17T17:29:31.897066+0000","last_became_peered":"2024-08-17T17:29:31.897066+0000","last_unstale":"2024-08-17T17:46:20.920831+0000","last_undegraded":"2024-08-17T17:46:20.920831+0000","last_fullsized":"2024-08-17T17:46:20.920831+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:17.743900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1126,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798623+0000","last_change":"2024-08-17T17:29:31.894594+0000","last_active":"2024-08-17T17:46:15.798623+0000","last_peered":"2024-08-17T17:46:15.798623+0000","last_clean":"2024-08-17T17:46:15.798623+0000","last_became_active":"2024-08-17T17:29:31.893900+0000","last_became_peered":"2024-08-17T17:29:31.893900+0000","last_unstale":"2024-08-17T17:46:15.798623+0000","last_undegraded":"2024-08-17T17:46:15.798623+0000","last_fullsized":"2024-08-17T17:46:15.798623+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:17:03.570954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1759,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909855+0000","last_change":"2024-08-17T17:31:15.540575+0000","last_active":"2024-08-17T17:46:20.909855+0000","last_peered":"2024-08-17T17:46:20.909855+0000","last_clean":"2024-08-17T17:46:20.909855+0000","last_became_active":"2024-08-17T17:31:15.540120+0000","last_became_peered":"2024-08-17T17:31:15.540120+0000","last_unstale":"2024-08-17T17:46:20.909855+0000","last_undegraded":"2024-08-17T17:46:20.909855+0000","last_fullsized":"2024-08-17T17:46:20.909855+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:13.718902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1728,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.911458+0000","last_change":"2024-08-17T17:31:15.550627+0000","last_active":"2024-08-17T17:46:20.911458+0000","last_peered":"2024-08-17T17:46:20.911458+0000","last_clean":"2024-08-17T17:46:20.911458+0000","last_became_active":"2024-08-17T17:31:15.550283+0000","last_became_peered":"2024-08-17T17:31:15.550283+0000","last_unstale":"2024-08-17T17:46:20.911458+0000","last_undegraded":"2024-08-17T17:46:20.911458+0000","last_fullsized":"2024-08-17T17:46:20.911458+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:30.165773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1779,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909729+0000","last_change":"2024-08-17T17:30:43.435630+0000","last_active":"2024-08-17T17:46:20.909729+0000","last_peered":"2024-08-17T17:46:20.909729+0000","last_clean":"2024-08-17T17:46:20.909729+0000","last_became_active":"2024-08-17T17:30:43.435338+0000","last_became_peered":"2024-08-17T17:30:43.435338+0000","last_unstale":"2024-08-17T17:46:20.909729+0000","last_undegraded":"2024-08-17T17:46:20.909729+0000","last_fullsized":"2024-08-17T17:46:20.909729+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:42:44.675247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1135,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070456+0000","last_change":"2024-08-17T17:31:49.801416+0000","last_active":"2024-08-17T17:46:16.070456+0000","last_peered":"2024-08-17T17:46:16.070456+0000","last_clean":"2024-08-17T17:46:16.070456+0000","last_became_active":"2024-08-17T17:31:49.801160+0000","last_became_peered":"2024-08-17T17:31:49.801160+0000","last_unstale":"2024-08-17T17:46:16.070456+0000","last_undegraded":"2024-08-17T17:46:16.070456+0000","last_fullsized":"2024-08-17T17:46:16.070456+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:04.641376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1771,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909952+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:46:20.909952+0000","last_peered":"2024-08-17T17:46:20.909952+0000","last_clean":"2024-08-17T17:46:20.909952+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:46:20.909952+0000","last_undegraded":"2024-08-17T17:46:20.909952+0000","last_fullsized":"2024-08-17T17:46:20.909952+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009980+0000","last_change":"2024-08-17T17:29:31.897318+0000","last_active":"2024-08-17T17:46:16.009980+0000","last_peered":"2024-08-17T17:46:16.009980+0000","last_clean":"2024-08-17T17:46:16.009980+0000","last_became_active":"2024-08-17T17:29:31.896952+0000","last_became_peered":"2024-08-17T17:29:31.896952+0000","last_unstale":"2024-08-17T17:46:16.009980+0000","last_undegraded":"2024-08-17T17:46:16.009980+0000","last_fullsized":"2024-08-17T17:46:16.009980+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:07:09.908531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":1132,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.069948+0000","last_change":"2024-08-17T17:30:43.432147+0000","last_active":"2024-08-17T17:46:16.069948+0000","last_peered":"2024-08-17T17:46:16.069948+0000","last_clean":"2024-08-17T17:46:16.069948+0000","last_became_active":"2024-08-17T17:30:43.431796+0000","last_became_peered":"2024-08-17T17:30:43.431796+0000","last_unstale":"2024-08-17T17:46:16.069948+0000","last_undegraded":"2024-08-17T17:46:16.069948+0000","last_fullsized":"2024-08-17T17:46:16.069948+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:43.435654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0008386,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1766,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912908+0000","last_change":"2024-08-17T17:30:43.435209+0000","last_active":"2024-08-17T17:46:20.912908+0000","last_peered":"2024-08-17T17:46:20.912908+0000","last_clean":"2024-08-17T17:46:20.912908+0000","last_became_active":"2024-08-17T17:30:43.434903+0000","last_became_peered":"2024-08-17T17:30:43.434903+0000","last_unstale":"2024-08-17T17:46:20.912908+0000","last_undegraded":"2024-08-17T17:46:20.912908+0000","last_fullsized":"2024-08-17T17:46:20.912908+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:27.551001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1760,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909154+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:46:20.909154+0000","last_peered":"2024-08-17T17:46:20.909154+0000","last_clean":"2024-08-17T17:46:20.909154+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:46:20.909154+0000","last_undegraded":"2024-08-17T17:46:20.909154+0000","last_fullsized":"2024-08-17T17:46:20.909154+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1769,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908437+0000","last_change":"2024-08-17T17:31:49.809141+0000","last_active":"2024-08-17T17:46:20.908437+0000","last_peered":"2024-08-17T17:46:20.908437+0000","last_clean":"2024-08-17T17:46:20.908437+0000","last_became_active":"2024-08-17T17:31:49.808855+0000","last_became_peered":"2024-08-17T17:31:49.808855+0000","last_unstale":"2024-08-17T17:46:20.908437+0000","last_undegraded":"2024-08-17T17:46:20.908437+0000","last_fullsized":"2024-08-17T17:46:20.908437+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:10.144929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1826,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.919131+0000","last_change":"2024-08-17T17:31:15.546567+0000","last_active":"2024-08-17T17:46:20.919131+0000","last_peered":"2024-08-17T17:46:20.919131+0000","last_clean":"2024-08-17T17:46:20.919131+0000","last_became_active":"2024-08-17T17:31:15.546117+0000","last_became_peered":"2024-08-17T17:31:15.546117+0000","last_unstale":"2024-08-17T17:46:20.919131+0000","last_undegraded":"2024-08-17T17:46:20.919131+0000","last_fullsized":"2024-08-17T17:46:20.919131+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:19.245231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1813,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.915571+0000","last_change":"2024-08-17T17:31:49.811704+0000","last_active":"2024-08-17T17:46:20.915571+0000","last_peered":"2024-08-17T17:46:20.915571+0000","last_clean":"2024-08-17T17:46:20.915571+0000","last_became_active":"2024-08-17T17:31:49.811385+0000","last_became_peered":"2024-08-17T17:31:49.811385+0000","last_unstale":"2024-08-17T17:46:20.915571+0000","last_undegraded":"2024-08-17T17:46:20.915571+0000","last_fullsized":"2024-08-17T17:46:20.915571+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:58.832774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1143,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798033+0000","last_change":"2024-08-17T17:30:43.432897+0000","last_active":"2024-08-17T17:46:15.798033+0000","last_peered":"2024-08-17T17:46:15.798033+0000","last_clean":"2024-08-17T17:46:15.798033+0000","last_became_active":"2024-08-17T17:30:43.432602+0000","last_became_peered":"2024-08-17T17:30:43.432602+0000","last_unstale":"2024-08-17T17:46:15.798033+0000","last_undegraded":"2024-08-17T17:46:15.798033+0000","last_fullsized":"2024-08-17T17:46:15.798033+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:51.083549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1096,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583544+0000","last_change":"2024-08-17T17:21:53.164672+0000","last_active":"2024-08-17T17:46:15.583544+0000","last_peered":"2024-08-17T17:46:15.583544+0000","last_clean":"2024-08-17T17:46:15.583544+0000","last_became_active":"2024-08-17T17:21:53.164381+0000","last_became_peered":"2024-08-17T17:21:53.164381+0000","last_unstale":"2024-08-17T17:46:15.583544+0000","last_undegraded":"2024-08-17T17:46:15.583544+0000","last_fullsized":"2024-08-17T17:46:15.583544+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:43:36.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1661,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.909361+0000","last_change":"2024-08-17T17:31:49.807517+0000","last_active":"2024-08-17T17:46:20.909361+0000","last_peered":"2024-08-17T17:46:20.909361+0000","last_clean":"2024-08-17T17:46:20.909361+0000","last_became_active":"2024-08-17T17:31:49.802152+0000","last_became_peered":"2024-08-17T17:31:49.802152+0000","last_unstale":"2024-08-17T17:46:20.909361+0000","last_undegraded":"2024-08-17T17:46:20.909361+0000","last_fullsized":"2024-08-17T17:46:20.909361+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:02.934420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":1131,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070064+0000","last_change":"2024-08-17T17:31:15.545691+0000","last_active":"2024-08-17T17:46:16.070064+0000","last_peered":"2024-08-17T17:46:16.070064+0000","last_clean":"2024-08-17T17:46:16.070064+0000","last_became_active":"2024-08-17T17:31:15.545335+0000","last_became_peered":"2024-08-17T17:31:15.545335+0000","last_unstale":"2024-08-17T17:46:16.070064+0000","last_undegraded":"2024-08-17T17:46:16.070064+0000","last_fullsized":"2024-08-17T17:46:16.070064+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:18.694039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085985900000000004,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1455,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798170+0000","last_change":"2024-08-17T17:29:31.894591+0000","last_active":"2024-08-17T17:46:15.798170+0000","last_peered":"2024-08-17T17:46:15.798170+0000","last_clean":"2024-08-17T17:46:15.798170+0000","last_became_active":"2024-08-17T17:29:31.894134+0000","last_became_peered":"2024-08-17T17:29:31.894134+0000","last_unstale":"2024-08-17T17:46:15.798170+0000","last_undegraded":"2024-08-17T17:46:15.798170+0000","last_fullsized":"2024-08-17T17:46:15.798170+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:12.586656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1760,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.913715+0000","last_change":"2024-08-17T17:31:49.810921+0000","last_active":"2024-08-17T17:46:20.913715+0000","last_peered":"2024-08-17T17:46:20.913715+0000","last_clean":"2024-08-17T17:46:20.913715+0000","last_became_active":"2024-08-17T17:31:49.810583+0000","last_became_peered":"2024-08-17T17:31:49.810583+0000","last_unstale":"2024-08-17T17:46:20.913715+0000","last_undegraded":"2024-08-17T17:46:20.913715+0000","last_fullsized":"2024-08-17T17:46:20.913715+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:41:44.272815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1707,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.908901+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:46:20.908901+0000","last_peered":"2024-08-17T17:46:20.908901+0000","last_clean":"2024-08-17T17:46:20.908901+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:46:20.908901+0000","last_undegraded":"2024-08-17T17:46:20.908901+0000","last_fullsized":"2024-08-17T17:46:20.908901+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009910+0000","last_change":"2024-08-17T17:30:43.434276+0000","last_active":"2024-08-17T17:46:16.009910+0000","last_peered":"2024-08-17T17:46:16.009910+0000","last_clean":"2024-08-17T17:46:16.009910+0000","last_became_active":"2024-08-17T17:30:43.433844+0000","last_became_peered":"2024-08-17T17:30:43.433844+0000","last_unstale":"2024-08-17T17:46:16.009910+0000","last_undegraded":"2024-08-17T17:46:16.009910+0000","last_fullsized":"2024-08-17T17:46:16.009910+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:17:56.025107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009503+0000","last_change":"2024-08-17T17:31:15.546844+0000","last_active":"2024-08-17T17:46:16.009503+0000","last_peered":"2024-08-17T17:46:16.009503+0000","last_clean":"2024-08-17T17:46:16.009503+0000","last_became_active":"2024-08-17T17:31:15.546516+0000","last_became_peered":"2024-08-17T17:31:15.546516+0000","last_unstale":"2024-08-17T17:46:16.009503+0000","last_undegraded":"2024-08-17T17:46:16.009503+0000","last_fullsized":"2024-08-17T17:46:16.009503+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:10:18.148181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009616+0000","last_change":"2024-08-17T17:31:15.546671+0000","last_active":"2024-08-17T17:46:16.009616+0000","last_peered":"2024-08-17T17:46:16.009616+0000","last_clean":"2024-08-17T17:46:16.009616+0000","last_became_active":"2024-08-17T17:31:15.546355+0000","last_became_peered":"2024-08-17T17:31:15.546355+0000","last_unstale":"2024-08-17T17:46:16.009616+0000","last_undegraded":"2024-08-17T17:46:16.009616+0000","last_fullsized":"2024-08-17T17:46:16.009616+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:45.836713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1114,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583330+0000","last_change":"2024-08-17T17:30:43.433097+0000","last_active":"2024-08-17T17:46:15.583330+0000","last_peered":"2024-08-17T17:46:15.583330+0000","last_clean":"2024-08-17T17:46:15.583330+0000","last_became_active":"2024-08-17T17:30:43.432179+0000","last_became_peered":"2024-08-17T17:30:43.432179+0000","last_unstale":"2024-08-17T17:46:15.583330+0000","last_undegraded":"2024-08-17T17:46:15.583330+0000","last_fullsized":"2024-08-17T17:46:15.583330+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:44:40.966915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009644+0000","last_change":"2024-08-17T17:31:49.800286+0000","last_active":"2024-08-17T17:46:16.009644+0000","last_peered":"2024-08-17T17:46:16.009644+0000","last_clean":"2024-08-17T17:46:16.009644+0000","last_became_active":"2024-08-17T17:31:49.799834+0000","last_became_peered":"2024-08-17T17:31:49.799834+0000","last_unstale":"2024-08-17T17:46:16.009644+0000","last_undegraded":"2024-08-17T17:46:16.009644+0000","last_fullsized":"2024-08-17T17:46:16.009644+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:28.409397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1681,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910644+0000","last_change":"2024-08-17T17:31:49.801723+0000","last_active":"2024-08-17T17:46:20.910644+0000","last_peered":"2024-08-17T17:46:20.910644+0000","last_clean":"2024-08-17T17:46:20.910644+0000","last_became_active":"2024-08-17T17:31:49.801370+0000","last_became_peered":"2024-08-17T17:31:49.801370+0000","last_unstale":"2024-08-17T17:46:20.910644+0000","last_undegraded":"2024-08-17T17:46:20.910644+0000","last_fullsized":"2024-08-17T17:46:20.910644+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:28.686255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.920662+0000","last_change":"2024-08-17T17:30:43.431404+0000","last_active":"2024-08-17T17:46:20.920662+0000","last_peered":"2024-08-17T17:46:20.920662+0000","last_clean":"2024-08-17T17:46:20.920662+0000","last_became_active":"2024-08-17T17:30:43.431087+0000","last_became_peered":"2024-08-17T17:30:43.431087+0000","last_unstale":"2024-08-17T17:46:20.920662+0000","last_undegraded":"2024-08-17T17:46:20.920662+0000","last_fullsized":"2024-08-17T17:46:20.920662+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:15.830770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1118,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070176+0000","last_change":"2024-08-17T17:31:49.799082+0000","last_active":"2024-08-17T17:46:16.070176+0000","last_peered":"2024-08-17T17:46:16.070176+0000","last_clean":"2024-08-17T17:46:16.070176+0000","last_became_active":"2024-08-17T17:31:49.798764+0000","last_became_peered":"2024-08-17T17:31:49.798764+0000","last_unstale":"2024-08-17T17:46:16.070176+0000","last_undegraded":"2024-08-17T17:46:16.070176+0000","last_fullsized":"2024-08-17T17:46:16.070176+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:09:07.697148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1113,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583225+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:46:15.583225+0000","last_peered":"2024-08-17T17:46:15.583225+0000","last_clean":"2024-08-17T17:46:15.583225+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:46:15.583225+0000","last_undegraded":"2024-08-17T17:46:15.583225+0000","last_fullsized":"2024-08-17T17:46:15.583225+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1098,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798620+0000","last_change":"2024-08-17T17:21:52.151540+0000","last_active":"2024-08-17T17:46:15.798620+0000","last_peered":"2024-08-17T17:46:15.798620+0000","last_clean":"2024-08-17T17:46:15.798620+0000","last_became_active":"2024-08-17T17:21:52.151294+0000","last_became_peered":"2024-08-17T17:21:52.151294+0000","last_unstale":"2024-08-17T17:46:15.798620+0000","last_undegraded":"2024-08-17T17:46:15.798620+0000","last_fullsized":"2024-08-17T17:46:15.798620+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:03.533252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1133,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.009427+0000","last_change":"2024-08-17T17:31:49.799847+0000","last_active":"2024-08-17T17:46:16.009427+0000","last_peered":"2024-08-17T17:46:16.009427+0000","last_clean":"2024-08-17T17:46:16.009427+0000","last_became_active":"2024-08-17T17:31:49.799523+0000","last_became_peered":"2024-08-17T17:31:49.799523+0000","last_unstale":"2024-08-17T17:46:16.009427+0000","last_undegraded":"2024-08-17T17:46:16.009427+0000","last_fullsized":"2024-08-17T17:46:16.009427+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:34.070172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798751+0000","last_change":"2024-08-17T17:30:43.437929+0000","last_active":"2024-08-17T17:46:15.798751+0000","last_peered":"2024-08-17T17:46:15.798751+0000","last_clean":"2024-08-17T17:46:15.798751+0000","last_became_active":"2024-08-17T17:30:43.437604+0000","last_became_peered":"2024-08-17T17:30:43.437604+0000","last_unstale":"2024-08-17T17:46:15.798751+0000","last_undegraded":"2024-08-17T17:46:15.798751+0000","last_fullsized":"2024-08-17T17:46:15.798751+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:55:21.953561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1100,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.069864+0000","last_change":"2024-08-17T17:21:53.161767+0000","last_active":"2024-08-17T17:46:16.069864+0000","last_peered":"2024-08-17T17:46:16.069864+0000","last_clean":"2024-08-17T17:46:16.069864+0000","last_became_active":"2024-08-17T17:21:53.161514+0000","last_became_peered":"2024-08-17T17:21:53.161514+0000","last_unstale":"2024-08-17T17:46:16.069864+0000","last_undegraded":"2024-08-17T17:46:16.069864+0000","last_fullsized":"2024-08-17T17:46:16.069864+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:59.837673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1115,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798770+0000","last_change":"2024-08-17T17:29:31.900876+0000","last_active":"2024-08-17T17:46:15.798770+0000","last_peered":"2024-08-17T17:46:15.798770+0000","last_clean":"2024-08-17T17:46:15.798770+0000","last_became_active":"2024-08-17T17:29:31.900496+0000","last_became_peered":"2024-08-17T17:29:31.900496+0000","last_unstale":"2024-08-17T17:46:15.798770+0000","last_undegraded":"2024-08-17T17:46:15.798770+0000","last_fullsized":"2024-08-17T17:46:15.798770+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:18.971519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1122,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.583539+0000","last_change":"2024-08-17T17:29:31.891859+0000","last_active":"2024-08-17T17:46:15.583539+0000","last_peered":"2024-08-17T17:46:15.583539+0000","last_clean":"2024-08-17T17:46:15.583539+0000","last_became_active":"2024-08-17T17:29:31.891405+0000","last_became_peered":"2024-08-17T17:29:31.891405+0000","last_unstale":"2024-08-17T17:46:15.583539+0000","last_undegraded":"2024-08-17T17:46:15.583539+0000","last_fullsized":"2024-08-17T17:46:15.583539+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:48.087233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.912321+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:46:20.912321+0000","last_peered":"2024-08-17T17:46:20.912321+0000","last_clean":"2024-08-17T17:46:20.912321+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:46:20.912321+0000","last_undegraded":"2024-08-17T17:46:20.912321+0000","last_fullsized":"2024-08-17T17:46:20.912321+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1097,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798097+0000","last_change":"2024-08-17T17:21:52.149957+0000","last_active":"2024-08-17T17:46:15.798097+0000","last_peered":"2024-08-17T17:46:15.798097+0000","last_clean":"2024-08-17T17:46:15.798097+0000","last_became_active":"2024-08-17T17:21:52.149673+0000","last_became_peered":"2024-08-17T17:21:52.149673+0000","last_unstale":"2024-08-17T17:46:15.798097+0000","last_undegraded":"2024-08-17T17:46:15.798097+0000","last_fullsized":"2024-08-17T17:46:15.798097+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:01:16.473279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.918958+0000","last_change":"2024-08-17T17:31:49.805181+0000","last_active":"2024-08-17T17:46:20.918958+0000","last_peered":"2024-08-17T17:46:20.918958+0000","last_clean":"2024-08-17T17:46:20.918958+0000","last_became_active":"2024-08-17T17:31:49.804905+0000","last_became_peered":"2024-08-17T17:31:49.804905+0000","last_unstale":"2024-08-17T17:46:20.918958+0000","last_undegraded":"2024-08-17T17:46:20.918958+0000","last_fullsized":"2024-08-17T17:46:20.918958+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:47:16.879688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1811,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.916345+0000","last_change":"2024-08-17T17:31:49.812747+0000","last_active":"2024-08-17T17:46:20.916345+0000","last_peered":"2024-08-17T17:46:20.916345+0000","last_clean":"2024-08-17T17:46:20.916345+0000","last_became_active":"2024-08-17T17:31:49.812378+0000","last_became_peered":"2024-08-17T17:31:49.812378+0000","last_unstale":"2024-08-17T17:46:20.916345+0000","last_undegraded":"2024-08-17T17:46:20.916345+0000","last_fullsized":"2024-08-17T17:46:20.916345+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:35.124235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1117,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:16.070375+0000","last_change":"2024-08-17T17:31:49.801380+0000","last_active":"2024-08-17T17:46:16.070375+0000","last_peered":"2024-08-17T17:46:16.070375+0000","last_clean":"2024-08-17T17:46:16.070375+0000","last_became_active":"2024-08-17T17:31:49.801103+0000","last_became_peered":"2024-08-17T17:31:49.801103+0000","last_unstale":"2024-08-17T17:46:16.070375+0000","last_undegraded":"2024-08-17T17:46:16.070375+0000","last_fullsized":"2024-08-17T17:46:16.070375+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:09:43.554983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1116,"reported_epoch":1162,"state":"active+clean","last_fresh":"2024-08-17T17:46:15.798259+0000","last_change":"2024-08-17T17:31:15.543729+0000","last_active":"2024-08-17T17:46:15.798259+0000","last_peered":"2024-08-17T17:46:15.798259+0000","last_clean":"2024-08-17T17:46:15.798259+0000","last_became_active":"2024-08-17T17:31:15.543142+0000","last_became_peered":"2024-08-17T17:31:15.543142+0000","last_unstale":"2024-08-17T17:46:15.798259+0000","last_undegraded":"2024-08-17T17:46:15.798259+0000","last_fullsized":"2024-08-17T17:46:15.798259+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:00.612263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1705,"reported_epoch":1163,"state":"active+clean","last_fresh":"2024-08-17T17:46:20.910980+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:46:20.910980+0000","last_peered":"2024-08-17T17:46:20.910980+0000","last_clean":"2024-08-17T17:46:20.910980+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:46:20.910980+0000","last_undegraded":"2024-08-17T17:46:20.910980+0000","last_fullsized":"2024-08-17T17:46:20.910980+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234316,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325232,"kb_used_data":19556,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93419920,"statfs":{"total":95995035648,"available":95661998080,"internally_reserved":0,"allocated":20025344,"data_stored":12500974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495951,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":329380,"kb_used_data":19544,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93415772,"statfs":{"total":95995035648,"available":95657750528,"internally_reserved":0,"allocated":20013056,"data_stored":12500532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757589,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316316,"kb_used_data":18960,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93428836,"statfs":{"total":95995035648,"available":95671128064,"internally_reserved":0,"allocated":19415040,"data_stored":11906313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326203,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":68888,"kb_used_data":18956,"kb_used_omap":57,"kb_used_meta":49862,"kb_avail":93676264,"statfs":{"total":95995035648,"available":95924494336,"internally_reserved":0,"allocated":19410944,"data_stored":11906029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":51059451},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405442,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":73108,"kb_used_data":18956,"kb_used_omap":61,"kb_used_meta":54082,"kb_avail":93672044,"statfs":{"total":95995035648,"available":95920173056,"internally_reserved":0,"allocated":19410944,"data_stored":11905882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":55380920},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253451976,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":73044,"kb_used_data":18956,"kb_used_omap":54,"kb_used_meta":54025,"kb_avail":93672108,"statfs":{"total":95995035648,"available":95920238592,"internally_reserved":0,"allocated":19410944,"data_stored":11905966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":55321998},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":438,"seq":1881195675863,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":65248,"kb_used_data":19540,"kb_used_omap":43,"kb_used_meta":45652,"kb_avail":93679904,"statfs":{"total":95995035648,"available":95928221696,"internally_reserved":0,"allocated":20008960,"data_stored":12496899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44456,"internal_metadata":46748248},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019224,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328808,"kb_used_data":18972,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93416344,"statfs":{"total":95995035648,"available":95658336256,"internally_reserved":0,"allocated":19427328,"data_stored":11910544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:47:17.613 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T17:47:17.613 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd dump 2024-08-17T17:47:17.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:17 smithi130 ceph-mon[105521]: pgmap v1543: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:17 smithi130 ceph-mon[105521]: from='client.35211 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:17 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:17 smithi130 ceph-mon[110068]: pgmap v1543: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:17 smithi130 ceph-mon[110068]: from='client.35211 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:17 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:18.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:17 smithi179 ceph-mon[93519]: pgmap v1543: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:18.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:17 smithi179 ceph-mon[93519]: from='client.35211 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:47:18.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:17 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:18.882 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:19.005 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:18 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:19.006 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:18 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:18 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:19 smithi179 ceph-mon[93519]: pgmap v1544: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:20.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:19 smithi130 ceph-mon[105521]: pgmap v1544: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:20.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:19 smithi130 ceph-mon[110068]: pgmap v1544: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:20.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:47:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:47:19] "GET /metrics HTTP/1.1" 200 34151 "" "Prometheus/2.51.0" 2024-08-17T17:47:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:47:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:47:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:47:21.137 INFO:teuthology.orchestra.run.smithi130.stdout:epoch 1163 2024-08-17T17:47:21.137 INFO:teuthology.orchestra.run.smithi130.stdout:fsid 25127030-5cbc-11ef-bccf-c7b262605968 2024-08-17T17:47:21.137 INFO:teuthology.orchestra.run.smithi130.stdout:created 2024-08-17T17:15:09.488012+0000 2024-08-17T17:47:21.138 INFO:teuthology.orchestra.run.smithi130.stdout:modified 2024-08-17T17:46:20.580694+0000 2024-08-17T17:47:21.138 INFO:teuthology.orchestra.run.smithi130.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-08-17T17:47:21.138 INFO:teuthology.orchestra.run.smithi130.stdout:crush_version 22 2024-08-17T17:47:21.138 INFO:teuthology.orchestra.run.smithi130.stdout:full_ratio 0.95 2024-08-17T17:47:21.138 INFO:teuthology.orchestra.run.smithi130.stdout:backfillfull_ratio 0.9 2024-08-17T17:47:21.138 INFO:teuthology.orchestra.run.smithi130.stdout:nearfull_ratio 0.85 2024-08-17T17:47:21.138 INFO:teuthology.orchestra.run.smithi130.stdout:require_min_compat_client luminous 2024-08-17T17:47:21.138 INFO:teuthology.orchestra.run.smithi130.stdout:min_compat_client luminous 2024-08-17T17:47:21.138 INFO:teuthology.orchestra.run.smithi130.stdout:require_osd_release quincy 2024-08-17T17:47:21.138 INFO:teuthology.orchestra.run.smithi130.stdout:stretch_mode_enabled false 2024-08-17T17:47:21.138 INFO:teuthology.orchestra.run.smithi130.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-08-17T17:47:21.138 INFO:teuthology.orchestra.run.smithi130.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 65 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2024-08-17T17:47:21.138 INFO:teuthology.orchestra.run.smithi130.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 70 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2024-08-17T17:47:21.139 INFO:teuthology.orchestra.run.smithi130.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 71 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-08-17T17:47:21.139 INFO:teuthology.orchestra.run.smithi130.stdout:max_osd 8 2024-08-17T17:47:21.139 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0 up in weight 1 up_from 438 up_thru 1015 down_at 424 last_clean_interval [8,423) [v2:172.21.15.130:6802/3901615741,v1:172.21.15.130:6803/3901615741] [v2:172.21.15.130:6804/3901615741,v1:172.21.15.130:6805/3901615741] exists,up 30c31adb-2cda-466b-933e-a8e2cb714fab 2024-08-17T17:47:21.139 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1 up in weight 1 up_from 506 up_thru 1015 down_at 494 last_clean_interval [13,493) [v2:172.21.15.130:6810/790699819,v1:172.21.15.130:6811/790699819] [v2:172.21.15.130:6812/790699819,v1:172.21.15.130:6813/790699819] exists,up 55049bb4-b976-42c6-8a5d-8b3e9ae20988 2024-08-17T17:47:21.139 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2 up in weight 1 up_from 538 up_thru 1015 down_at 527 last_clean_interval [18,526) [v2:172.21.15.130:6818/2944523904,v1:172.21.15.130:6819/2944523904] [v2:172.21.15.130:6820/2944523904,v1:172.21.15.130:6821/2944523904] exists,up 3f40f088-27f7-4450-8f31-b449711365ab 2024-08-17T17:47:21.139 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3 up in weight 1 up_from 571 up_thru 1015 down_at 559 last_clean_interval [25,558) [v2:172.21.15.130:6826/3086744673,v1:172.21.15.130:6827/3086744673] [v2:172.21.15.130:6828/3086744673,v1:172.21.15.130:6829/3086744673] exists,up 7684c401-462a-4e35-bb9c-1be23dfe573b 2024-08-17T17:47:21.139 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4 up in weight 1 up_from 30 up_thru 1015 down_at 0 last_clean_interval [0,0) [v2:172.21.15.179:6800/2107797845,v1:172.21.15.179:6801/2107797845] [v2:172.21.15.179:6802/2107797845,v1:172.21.15.179:6803/2107797845] exists,up 7a3a331f-533a-49de-910d-5989f2f84b65 2024-08-17T17:47:21.139 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5 up in weight 1 up_from 38 up_thru 1015 down_at 0 last_clean_interval [0,0) [v2:172.21.15.179:6808/2392026049,v1:172.21.15.179:6809/2392026049] [v2:172.21.15.179:6810/2392026049,v1:172.21.15.179:6811/2392026049] exists,up d53d65ee-bbb2-45ec-b557-07cc62a45b05 2024-08-17T17:47:21.139 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6 up in weight 1 up_from 46 up_thru 1015 down_at 0 last_clean_interval [0,0) [v2:172.21.15.179:6816/139895003,v1:172.21.15.179:6817/139895003] [v2:172.21.15.179:6818/139895003,v1:172.21.15.179:6819/139895003] exists,up b84229ad-2c88-4064-a070-33b98714ef96 2024-08-17T17:47:21.139 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7 up in weight 1 up_from 54 up_thru 1015 down_at 0 last_clean_interval [0,0) [v2:172.21.15.179:6824/2533639201,v1:172.21.15.179:6825/2533639201] [v2:172.21.15.179:6826/2533639201,v1:172.21.15.179:6827/2533639201] exists,up bba7978f-8b19-40e6-86bd-692a0e282429 2024-08-17T17:47:21.139 INFO:teuthology.orchestra.run.smithi130.stdout:pg_upmap_items 4.2 [1,2] 2024-08-17T17:47:21.139 INFO:teuthology.orchestra.run.smithi130.stdout:pg_upmap_items 4.6 [1,5] 2024-08-17T17:47:21.139 INFO:teuthology.orchestra.run.smithi130.stdout:pg_upmap_items 4.10 [1,2] 2024-08-17T17:47:21.140 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/485089907 expires 2024-08-17T18:31:20.602503+0000 2024-08-17T17:47:21.140 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:6801/3641086954 expires 2024-08-18T17:26:11.940622+0000 2024-08-17T17:47:21.140 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:6800/3641086954 expires 2024-08-18T17:26:11.940622+0000 2024-08-17T17:47:21.140 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/3453310625 expires 2024-08-18T17:26:11.940622+0000 2024-08-17T17:47:21.140 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/2870283877 expires 2024-08-18T17:26:11.940622+0000 2024-08-17T17:47:21.140 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/3943801420 expires 2024-08-18T17:26:11.940622+0000 2024-08-17T17:47:21.140 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/2991152682 expires 2024-08-18T17:26:11.940622+0000 2024-08-17T17:47:21.140 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/2447795507 expires 2024-08-17T18:31:19.611103+0000 2024-08-17T17:47:21.140 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/851371227 expires 2024-08-18T17:20:33.507974+0000 2024-08-17T17:47:21.140 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:6800/368286319 expires 2024-08-18T17:15:57.894717+0000 2024-08-17T17:47:21.140 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/4037898622 expires 2024-08-18T17:26:11.940622+0000 2024-08-17T17:47:21.140 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/1247136467 expires 2024-08-18T17:15:57.894717+0000 2024-08-17T17:47:21.140 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/2578011752 expires 2024-08-18T17:20:33.507974+0000 2024-08-17T17:47:21.140 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/1457627030 expires 2024-08-18T17:24:24.724005+0000 2024-08-17T17:47:21.141 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/1299294781 expires 2024-08-18T17:24:24.724005+0000 2024-08-17T17:47:21.141 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.179:0/1781739324 expires 2024-08-18T17:25:30.337581+0000 2024-08-17T17:47:21.141 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/1690071627 expires 2024-08-18T17:15:34.935061+0000 2024-08-17T17:47:21.141 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/2810193027 expires 2024-08-18T17:26:11.940622+0000 2024-08-17T17:47:21.141 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:6801/368286319 expires 2024-08-18T17:15:57.894717+0000 2024-08-17T17:47:21.141 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.179:0/56636976 expires 2024-08-18T17:25:30.337581+0000 2024-08-17T17:47:21.141 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:6800/824081166 expires 2024-08-18T17:24:24.724005+0000 2024-08-17T17:47:21.141 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:6800/3883563384 expires 2024-08-18T17:20:33.507974+0000 2024-08-17T17:47:21.141 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/695423662 expires 2024-08-17T18:36:26.582385+0000 2024-08-17T17:47:21.141 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/2999329360 expires 2024-08-18T17:15:34.935061+0000 2024-08-17T17:47:21.141 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.179:0/196590495 expires 2024-08-18T17:25:30.337581+0000 2024-08-17T17:47:21.141 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/2589122337 expires 2024-08-18T17:20:33.507974+0000 2024-08-17T17:47:21.142 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/368845317 expires 2024-08-18T17:20:33.507974+0000 2024-08-17T17:47:21.142 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/3071443649 expires 2024-08-18T17:15:57.894717+0000 2024-08-17T17:47:21.142 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.179:0/2399909254 expires 2024-08-18T17:25:30.337581+0000 2024-08-17T17:47:21.142 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/1843620043 expires 2024-08-18T17:24:24.724005+0000 2024-08-17T17:47:21.142 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:6800/1239370657 expires 2024-08-18T17:15:34.935061+0000 2024-08-17T17:47:21.142 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:6801/1239370657 expires 2024-08-18T17:15:34.935061+0000 2024-08-17T17:47:21.142 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/88320168 expires 2024-08-18T17:15:34.935061+0000 2024-08-17T17:47:21.142 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.179:6835/3911752105 expires 2024-08-18T17:25:30.337581+0000 2024-08-17T17:47:21.142 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.179:0/2395764912 expires 2024-08-18T17:25:30.337581+0000 2024-08-17T17:47:21.142 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.179:6834/3911752105 expires 2024-08-18T17:25:30.337581+0000 2024-08-17T17:47:21.142 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:6801/3883563384 expires 2024-08-18T17:20:33.507974+0000 2024-08-17T17:47:21.142 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/854839742 expires 2024-08-18T17:24:24.724005+0000 2024-08-17T17:47:21.143 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:6801/824081166 expires 2024-08-18T17:24:24.724005+0000 2024-08-17T17:47:21.143 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.130:0/3471387839 expires 2024-08-18T17:15:57.894717+0000 2024-08-17T17:47:21.143 INFO:teuthology.orchestra.run.smithi130.stdout:blocklist 172.21.15.179:0/393594060 expires 2024-08-18T17:25:30.337581+0000 2024-08-17T17:47:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:21 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:21 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:21 smithi179 ceph-mon[93519]: pgmap v1545: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:21 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:47:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:21 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:47:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:21 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:21 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:21 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:21 smithi130 ceph-mon[105521]: pgmap v1545: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:21 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:47:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:21 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:47:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:21 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:21 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:21 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:21 smithi130 ceph-mon[110068]: pgmap v1545: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:21.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:21 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:47:21.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:21 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:47:21.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:21 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:21.808 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 438 up_thru 1015 down_at 424 last_clean_interval [8,423) [v2:172.21.15.130:6802/3901615741,v1:172.21.15.130:6803/3901615741] [v2:172.21.15.130:6804/3901615741,v1:172.21.15.130:6805/3901615741] exists,up 30c31adb-2cda-466b-933e-a8e2cb714fab', 'osd.1 up in weight 1 up_from 506 up_thru 1015 down_at 494 last_clean_interval [13,493) [v2:172.21.15.130:6810/790699819,v1:172.21.15.130:6811/790699819] [v2:172.21.15.130:6812/790699819,v1:172.21.15.130:6813/790699819] exists,up 55049bb4-b976-42c6-8a5d-8b3e9ae20988', 'osd.2 up in weight 1 up_from 538 up_thru 1015 down_at 527 last_clean_interval [18,526) [v2:172.21.15.130:6818/2944523904,v1:172.21.15.130:6819/2944523904] [v2:172.21.15.130:6820/2944523904,v1:172.21.15.130:6821/2944523904] exists,up 3f40f088-27f7-4450-8f31-b449711365ab', 'osd.3 up in weight 1 up_from 571 up_thru 1015 down_at 559 last_clean_interval [25,558) [v2:172.21.15.130:6826/3086744673,v1:172.21.15.130:6827/3086744673] [v2:172.21.15.130:6828/3086744673,v1:172.21.15.130:6829/3086744673] exists,up 7684c401-462a-4e35-bb9c-1be23dfe573b', 'osd.4 up in weight 1 up_from 30 up_thru 1015 down_at 0 last_clean_interval [0,0) [v2:172.21.15.179:6800/2107797845,v1:172.21.15.179:6801/2107797845] [v2:172.21.15.179:6802/2107797845,v1:172.21.15.179:6803/2107797845] exists,up 7a3a331f-533a-49de-910d-5989f2f84b65', 'osd.5 up in weight 1 up_from 38 up_thru 1015 down_at 0 last_clean_interval [0,0) [v2:172.21.15.179:6808/2392026049,v1:172.21.15.179:6809/2392026049] [v2:172.21.15.179:6810/2392026049,v1:172.21.15.179:6811/2392026049] exists,up d53d65ee-bbb2-45ec-b557-07cc62a45b05', 'osd.6 up in weight 1 up_from 46 up_thru 1015 down_at 0 last_clean_interval [0,0) [v2:172.21.15.179:6816/139895003,v1:172.21.15.179:6817/139895003] [v2:172.21.15.179:6818/139895003,v1:172.21.15.179:6819/139895003] exists,up b84229ad-2c88-4064-a070-33b98714ef96', 'osd.7 up in weight 1 up_from 54 up_thru 1015 down_at 0 last_clean_interval [0,0) [v2:172.21.15.179:6824/2533639201,v1:172.21.15.179:6825/2533639201] [v2:172.21.15.179:6826/2533639201,v1:172.21.15.179:6827/2533639201] exists,up bba7978f-8b19-40e6-86bd-692a0e282429'] 2024-08-17T17:47:21.809 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon mon.a config show 2024-08-17T17:47:22.062 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:22.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:22 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1665645724' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-17T17:47:22.395 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:22 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1665645724' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-17T17:47:22.396 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:22 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1665645724' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-17T17:47:23.158 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:23 smithi130 ceph-mon[110068]: pgmap v1546: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:23.158 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:23 smithi130 ceph-mon[105521]: pgmap v1546: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:23 smithi179 ceph-mon[93519]: pgmap v1546: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:23.644 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:47:23.644 INFO:teuthology.orchestra.run.smithi130.stdout: "name": "mon.a", 2024-08-17T17:47:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "cluster": "ceph", 2024-08-17T17:47:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-08-17T17:47:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "admin_socket_mode": "", 2024-08-17T17:47:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-08-17T17:47:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "auth_client_required": "cephx, none", 2024-08-17T17:47:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "auth_cluster_required": "cephx", 2024-08-17T17:47:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "auth_debug": "true", 2024-08-17T17:47:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-08-17T17:47:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-08-17T17:47:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "auth_service_required": "cephx", 2024-08-17T17:47:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "auth_service_ticket_ttl": "240.000000", 2024-08-17T17:47:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "auth_supported": "", 2024-08-17T17:47:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_aio": "true", 2024-08-17T17:47:23.645 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_aio_max_queue_depth": "1024", 2024-08-17T17:47:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_aio_poll_ms": "250", 2024-08-17T17:47:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_aio_reap_max": "16", 2024-08-17T17:47:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_async_discard_threads": "0", 2024-08-17T17:47:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_block_size": "4096", 2024-08-17T17:47:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_debug_aio": "false", 2024-08-17T17:47:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-08-17T17:47:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-08-17T17:47:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_debug_inflight_ios": "false", 2024-08-17T17:47:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_enable_discard": "false", 2024-08-17T17:47:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_flock_retry": "3", 2024-08-17T17:47:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_flock_retry_interval": "0.100000", 2024-08-17T17:47:23.646 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_inject_crash": "0", 2024-08-17T17:47:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_inject_crash_flush_delay": "2", 2024-08-17T17:47:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_ioring": "false", 2024-08-17T17:47:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_ioring_hipri": "false", 2024-08-17T17:47:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_ioring_sqthread_poll": "false", 2024-08-17T17:47:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-08-17T17:47:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_read_buffer_alignment": "4096", 2024-08-17T17:47:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-08-17T17:47:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_stalled_read_warn_lifetime": "86400", 2024-08-17T17:47:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_stalled_read_warn_threshold": "1", 2024-08-17T17:47:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "bdev_type": "", 2024-08-17T17:47:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_alloc_size": "1048576", 2024-08-17T17:47:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_allocator": "hybrid", 2024-08-17T17:47:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_buffered_io": "true", 2024-08-17T17:47:23.647 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_check_for_zeros": "false", 2024-08-17T17:47:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_check_volume_selector_often": "false", 2024-08-17T17:47:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-08-17T17:47:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_compact_log_sync": "false", 2024-08-17T17:47:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-08-17T17:47:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-08-17T17:47:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_log_compact_min_size": "16777216", 2024-08-17T17:47:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_log_replay_check_allocations": "true", 2024-08-17T17:47:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_max_log_runway": "4194304", 2024-08-17T17:47:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_max_prefetch": "1048576", 2024-08-17T17:47:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_min_flush_size": "524288", 2024-08-17T17:47:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_min_log_runway": "1048576", 2024-08-17T17:47:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_replay_recovery": "false", 2024-08-17T17:47:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-08-17T17:47:23.648 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_shared_alloc_size": "65536", 2024-08-17T17:47:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "bluefs_sync_write": "false", 2024-08-17T17:47:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-08-17T17:47:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-08-17T17:47:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-08-17T17:47:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_allocation_from_file": "true", 2024-08-17T17:47:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_allocator": "hybrid", 2024-08-17T17:47:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_async_db_compaction": "true", 2024-08-17T17:47:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-08-17T17:47:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-08-17T17:47:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-08-17T17:47:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-08-17T17:47:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_bdev_label_multi": "true", 2024-08-17T17:47:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_bdev_label_multi_upgrade": "false", 2024-08-17T17:47:23.649 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_bdev_label_require_all": "true", 2024-08-17T17:47:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-08-17T17:47:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-08-17T17:47:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_blobid_prealloc": "10240", 2024-08-17T17:47:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_block_create": "true", 2024-08-17T17:47:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_block_db_create": "false", 2024-08-17T17:47:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_block_db_path": "", 2024-08-17T17:47:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_block_db_size": "0", 2024-08-17T17:47:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_block_path": "", 2024-08-17T17:47:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_block_preallocate_file": "false", 2024-08-17T17:47:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_block_size": "107374182400", 2024-08-17T17:47:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_block_wal_create": "false", 2024-08-17T17:47:23.650 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_block_wal_path": "", 2024-08-17T17:47:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_block_wal_size": "100663296", 2024-08-17T17:47:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_bluefs": "true", 2024-08-17T17:47:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-08-17T17:47:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_bluefs_env_mirror": "false", 2024-08-17T17:47:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-08-17T17:47:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_btree2_alloc_weight_factor": "2.000000", 2024-08-17T17:47:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-08-17T17:47:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-08-17T17:47:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-08-17T17:47:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-08-17T17:47:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-08-17T17:47:23.651 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_autotune": "true", 2024-08-17T17:47:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-08-17T17:47:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-08-17T17:47:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-08-17T17:47:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-08-17T17:47:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_size": "0", 2024-08-17T17:47:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-08-17T17:47:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-08-17T17:47:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-08-17T17:47:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-08-17T17:47:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cache_type": "2q", 2024-08-17T17:47:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-08-17T17:47:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_clone_cow": "true", 2024-08-17T17:47:23.652 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_compression_algorithm": "snappy", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_compression_max_blob_size": "0", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_compression_min_blob_size": "0", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_compression_mode": "none", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_csum_type": "crc32c", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_enforce_settings": "default", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_freelist": "false", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_fsck_abort": "false", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-08-17T17:47:23.653 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-08-17T17:47:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_inject_read_err": "false", 2024-08-17T17:47:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_legacy_omap": "false", 2024-08-17T17:47:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-08-17T17:47:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-08-17T17:47:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-08-17T17:47:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-08-17T17:47:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-08-17T17:47:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-08-17T17:47:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-08-17T17:47:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_small_allocations": "0", 2024-08-17T17:47:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-08-17T17:47:23.654 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_default_buffered_read": "true", 2024-08-17T17:47:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_default_buffered_write": "false", 2024-08-17T17:47:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_deferred_batch_ops": "0", 2024-08-17T17:47:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-08-17T17:47:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-08-17T17:47:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_elastic_shared_blobs": "true", 2024-08-17T17:47:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-08-17T17:47:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-08-17T17:47:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-08-17T17:47:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-08-17T17:47:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-08-17T17:47:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fail_eio": "false", 2024-08-17T17:47:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-08-17T17:47:23.655 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-08-17T17:47:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-08-17T17:47:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-08-17T17:47:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_on_mkfs": "true", 2024-08-17T17:47:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-08-17T17:47:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_on_mount": "false", 2024-08-17T17:47:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-08-17T17:47:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_on_umount": "false", 2024-08-17T17:47:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-08-17T17:47:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-08-17T17:47:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-08-17T17:47:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-08-17T17:47:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-08-17T17:47:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-08-17T17:47:23.656 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-08-17T17:47:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-08-17T17:47:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_ignore_data_csum": "false", 2024-08-17T17:47:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-08-17T17:47:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_kvbackend": "rocksdb", 2024-08-17T17:47:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-08-17T17:47:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-08-17T17:47:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_log_op_age": "5.000000", 2024-08-17T17:47:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_max_alloc_size": "0", 2024-08-17T17:47:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_max_blob_size": "0", 2024-08-17T17:47:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-08-17T17:47:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-08-17T17:47:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_max_defer_interval": "3.000000", 2024-08-17T17:47:23.657 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_max_deferred_txc": "32", 2024-08-17T17:47:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_min_alloc_size": "0", 2024-08-17T17:47:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-08-17T17:47:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-08-17T17:47:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_nid_prealloc": "1024", 2024-08-17T17:47:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_prefer_deferred_size": "0", 2024-08-17T17:47:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-08-17T17:47:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-08-17T17:47:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_qfsck_on_mount": "true", 2024-08-17T17:47:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_retry_disk_reads": "3", 2024-08-17T17:47:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_rocksdb_cf": "true", 2024-08-17T17:47:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-08-17T17:47:23.658 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-08-17T17:47:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_rocksdb_options_annex": "", 2024-08-17T17:47:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_slow_ops_warn_lifetime": "86400", 2024-08-17T17:47:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_slow_ops_warn_threshold": "1", 2024-08-17T17:47:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_spdk_coremask": "0x1", 2024-08-17T17:47:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_spdk_io_sleep": "5", 2024-08-17T17:47:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_spdk_max_io_completion": "0", 2024-08-17T17:47:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_spdk_mem": "512", 2024-08-17T17:47:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_sync_submit_transaction": "false", 2024-08-17T17:47:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_throttle_bytes": "67108864", 2024-08-17T17:47:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_throttle_cost_per_io": "0", 2024-08-17T17:47:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-08-17T17:47:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-08-17T17:47:23.659 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-08-17T17:47:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-08-17T17:47:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_tracing": "false", 2024-08-17T17:47:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-08-17T17:47:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-08-17T17:47:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_volume_selection_reserved": "0", 2024-08-17T17:47:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-08-17T17:47:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-08-17T17:47:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-08-17T17:47:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-08-17T17:47:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-08-17T17:47:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-08-17T17:47:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_write_v2": "false", 2024-08-17T17:47:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_write_v2_random": "false", 2024-08-17T17:47:23.660 INFO:teuthology.orchestra.run.smithi130.stdout: "bluestore_zero_block_detection": "false", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephfs_mirror_action_update_interval": "2", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephfs_mirror_mount_timeout": "10", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-08-17T17:47:23.661 INFO:teuthology.orchestra.run.smithi130.stdout: "cephx_cluster_require_signatures": "false", 2024-08-17T17:47:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "cephx_cluster_require_version": "2", 2024-08-17T17:47:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "cephx_require_signatures": "false", 2024-08-17T17:47:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "cephx_require_version": "2", 2024-08-17T17:47:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "cephx_service_require_signatures": "false", 2024-08-17T17:47:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "cephx_service_require_version": "2", 2024-08-17T17:47:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "cephx_sign_messages": "true", 2024-08-17T17:47:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "chdir": "", 2024-08-17T17:47:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "client_acl_type": "", 2024-08-17T17:47:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "client_asio_thread_count": "2", 2024-08-17T17:47:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "client_cache_mid": "0.750000", 2024-08-17T17:47:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "client_cache_size": "16384", 2024-08-17T17:47:23.662 INFO:teuthology.orchestra.run.smithi130.stdout: "client_caps_release_delay": "5", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_check_pool_perm": "true", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_collect_and_send_global_metrics": "false", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_debug_force_sync_read": "false", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_debug_getattr_caps": "false", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_debug_inject_features": "", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_debug_inject_tick_delay": "0", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_die_on_failed_remount": "false", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_dirsize_rbytes": "true", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_force_lazyio": "false", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_fs": "", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_inject_fixed_oldest_tid": "false", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_inject_release_failure": "false", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_max_inline_size": "4096", 2024-08-17T17:47:23.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_max_retries_on_remount_failure": "5", 2024-08-17T17:47:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_mds_namespace": "", 2024-08-17T17:47:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_metadata": "", 2024-08-17T17:47:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_mount_gid": "-1", 2024-08-17T17:47:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_mount_timeout": "300", 2024-08-17T17:47:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_mount_uid": "-1", 2024-08-17T17:47:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_mountpoint": "/", 2024-08-17T17:47:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_notify_timeout": "10", 2024-08-17T17:47:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_oc": "true", 2024-08-17T17:47:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_oc_max_dirty": "104857600", 2024-08-17T17:47:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_oc_max_dirty_age": "5.000000", 2024-08-17T17:47:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_oc_max_objects": "1000", 2024-08-17T17:47:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_oc_size": "209715200", 2024-08-17T17:47:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_oc_target_dirty": "8388608", 2024-08-17T17:47:23.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_permissions": "true", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_quota": "true", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_quota_df": "true", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_readahead_max_bytes": "0", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_readahead_max_periods": "4", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_readahead_min": "131072", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_reconnect_stale": "false", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_shutdown_timeout": "30", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_snapdir": ".snap", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_tick_interval": "1", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_trace": "", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_try_dentry_invalidate": "false", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_use_faked_inos": "false", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_use_random_mds": "false", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "clog_to_graylog": "false", 2024-08-17T17:47:23.665 INFO:teuthology.orchestra.run.smithi130.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-08-17T17:47:23.666 INFO:teuthology.orchestra.run.smithi130.stdout: "clog_to_graylog_port": "12201", 2024-08-17T17:47:23.666 INFO:teuthology.orchestra.run.smithi130.stdout: "clog_to_monitors": "default=true", 2024-08-17T17:47:23.666 INFO:teuthology.orchestra.run.smithi130.stdout: "clog_to_syslog": "false", 2024-08-17T17:47:23.666 INFO:teuthology.orchestra.run.smithi130.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-08-17T17:47:23.666 INFO:teuthology.orchestra.run.smithi130.stdout: "clog_to_syslog_level": "info", 2024-08-17T17:47:23.666 INFO:teuthology.orchestra.run.smithi130.stdout: "cluster_addr": "-", 2024-08-17T17:47:23.666 INFO:teuthology.orchestra.run.smithi130.stdout: "cluster_network": "", 2024-08-17T17:47:23.666 INFO:teuthology.orchestra.run.smithi130.stdout: "cluster_network_interface": "", 2024-08-17T17:47:23.666 INFO:teuthology.orchestra.run.smithi130.stdout: "compressor_zlib_isal": "false", 2024-08-17T17:47:23.666 INFO:teuthology.orchestra.run.smithi130.stdout: "compressor_zlib_level": "5", 2024-08-17T17:47:23.666 INFO:teuthology.orchestra.run.smithi130.stdout: "compressor_zlib_winsize": "-15", 2024-08-17T17:47:23.666 INFO:teuthology.orchestra.run.smithi130.stdout: "compressor_zstd_level": "1", 2024-08-17T17:47:23.666 INFO:teuthology.orchestra.run.smithi130.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:47:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-08-17T17:47:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "crimson_alien_op_num_threads": "6", 2024-08-17T17:47:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "crimson_alien_thread_cpu_cores": "", 2024-08-17T17:47:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "crimson_osd_obc_lru_size": "512", 2024-08-17T17:47:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-08-17T17:47:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "crimson_osd_stat_interval": "0", 2024-08-17T17:47:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "crimson_seastar_cpu_cores": "", 2024-08-17T17:47:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "crimson_seastar_num_threads": "0", 2024-08-17T17:47:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "crush_location": "", 2024-08-17T17:47:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "crush_location_hook": "", 2024-08-17T17:47:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "crush_location_hook_timeout": "10", 2024-08-17T17:47:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "daemonize": "false", 2024-08-17T17:47:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "daos_pool": "tank", 2024-08-17T17:47:23.667 INFO:teuthology.orchestra.run.smithi130.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-08-17T17:47:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-08-17T17:47:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "dbstore_db_name_prefix": "dbstore", 2024-08-17T17:47:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_alienstore": "0/5", 2024-08-17T17:47:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_allow_any_pool_priority": "false", 2024-08-17T17:47:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_asok": "1/5", 2024-08-17T17:47:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_asok_assert_abort": "false", 2024-08-17T17:47:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_asserts_on_shutdown": "true", 2024-08-17T17:47:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_auth": "1/5", 2024-08-17T17:47:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_bdev": "1/3", 2024-08-17T17:47:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_bluefs": "1/5", 2024-08-17T17:47:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_bluestore": "1/5", 2024-08-17T17:47:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_buffer": "0/1", 2024-08-17T17:47:23.668 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_ceph_dedup": "0/5", 2024-08-17T17:47:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_ceph_exporter": "1/5", 2024-08-17T17:47:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_cephfs_mirror": "0/5", 2024-08-17T17:47:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_cephsqlite": "0/5", 2024-08-17T17:47:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_client": "0/5", 2024-08-17T17:47:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_compressor": "1/5", 2024-08-17T17:47:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_context": "0/1", 2024-08-17T17:47:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_crimson_interrupt": "0/5", 2024-08-17T17:47:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_crush": "1/1", 2024-08-17T17:47:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_crypto": "1/5", 2024-08-17T17:47:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_cyanstore": "0/5", 2024-08-17T17:47:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_deliberately_leak_memory": "false", 2024-08-17T17:47:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_disable_randomized_ping": "false", 2024-08-17T17:47:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_dpdk": "1/5", 2024-08-17T17:47:23.669 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_eventtrace": "1/5", 2024-08-17T17:47:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_filer": "0/1", 2024-08-17T17:47:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_filestore": "1/3", 2024-08-17T17:47:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_finisher": "1/1", 2024-08-17T17:47:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_fuse": "1/5", 2024-08-17T17:47:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_heartbeat_testing_span": "0", 2024-08-17T17:47:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_heartbeatmap": "1/5", 2024-08-17T17:47:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_immutable_obj_cache": "0/5", 2024-08-17T17:47:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_javaclient": "1/5", 2024-08-17T17:47:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_journal": "1/3", 2024-08-17T17:47:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_journaler": "0/5", 2024-08-17T17:47:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_kstore": "1/5", 2024-08-17T17:47:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_lockdep": "0/1", 2024-08-17T17:47:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_mclock": "1/5", 2024-08-17T17:47:23.670 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_mds": "1/5", 2024-08-17T17:47:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_mds_balancer": "1/5", 2024-08-17T17:47:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_mds_locker": "1/5", 2024-08-17T17:47:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_mds_log": "1/5", 2024-08-17T17:47:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_mds_log_expire": "1/5", 2024-08-17T17:47:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_mds_migrator": "1/5", 2024-08-17T17:47:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_mds_quiesce": "3/5", 2024-08-17T17:47:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_memstore": "1/5", 2024-08-17T17:47:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_mgr": "2/5", 2024-08-17T17:47:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_mgrc": "1/5", 2024-08-17T17:47:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_mon": "20/20", 2024-08-17T17:47:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_monc": "0/10", 2024-08-17T17:47:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_ms": "1/1", 2024-08-17T17:47:23.671 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_none": "0/5", 2024-08-17T17:47:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_objclass": "0/5", 2024-08-17T17:47:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_objectcacher": "0/5", 2024-08-17T17:47:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_objecter": "0/1", 2024-08-17T17:47:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_optracker": "0/5", 2024-08-17T17:47:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_osd": "1/5", 2024-08-17T17:47:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_paxos": "20/20", 2024-08-17T17:47:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_perfcounter": "1/5", 2024-08-17T17:47:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_prioritycache": "1/5", 2024-08-17T17:47:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_rados": "0/5", 2024-08-17T17:47:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_rbd": "0/5", 2024-08-17T17:47:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_rbd_mirror": "0/5", 2024-08-17T17:47:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_rbd_pwl": "0/5", 2024-08-17T17:47:23.672 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_rbd_replay": "0/5", 2024-08-17T17:47:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_refs": "0/0", 2024-08-17T17:47:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_reserver": "1/1", 2024-08-17T17:47:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_rgw": "1/5", 2024-08-17T17:47:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_rgw_access": "1/5", 2024-08-17T17:47:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_rgw_datacache": "1/5", 2024-08-17T17:47:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_rgw_dbstore": "1/5", 2024-08-17T17:47:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_rgw_flight": "1/5", 2024-08-17T17:47:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_rgw_lifecycle": "1/5", 2024-08-17T17:47:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_rgw_notification": "1/5", 2024-08-17T17:47:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_rgw_sync": "1/5", 2024-08-17T17:47:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_rocksdb": "4/5", 2024-08-17T17:47:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_seastore": "0/5", 2024-08-17T17:47:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_seastore_backref": "0/5", 2024-08-17T17:47:23.673 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_seastore_cache": "0/5", 2024-08-17T17:47:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_seastore_cleaner": "0/5", 2024-08-17T17:47:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_seastore_device": "0/5", 2024-08-17T17:47:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_seastore_epm": "0/5", 2024-08-17T17:47:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-08-17T17:47:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_seastore_journal": "0/5", 2024-08-17T17:47:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_seastore_lba": "0/5", 2024-08-17T17:47:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_seastore_odata": "0/5", 2024-08-17T17:47:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_seastore_omap": "0/5", 2024-08-17T17:47:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_seastore_onode": "0/5", 2024-08-17T17:47:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_seastore_t": "0/5", 2024-08-17T17:47:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_seastore_tm": "0/5", 2024-08-17T17:47:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_striper": "0/1", 2024-08-17T17:47:23.674 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_test": "0/5", 2024-08-17T17:47:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_throttle": "1/1", 2024-08-17T17:47:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_timer": "0/1", 2024-08-17T17:47:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_tp": "0/5", 2024-08-17T17:47:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "debug_trace": "1/5", 2024-08-17T17:47:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-08-17T17:47:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "device_failure_prediction_mode": "none", 2024-08-17T17:47:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-08-17T17:47:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-08-17T17:47:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "err_to_graylog": "false", 2024-08-17T17:47:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "err_to_journald": "false", 2024-08-17T17:47:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "err_to_stderr": "true", 2024-08-17T17:47:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "err_to_syslog": "false", 2024-08-17T17:47:23.675 INFO:teuthology.orchestra.run.smithi130.stdout: "event_tracing": "false", 2024-08-17T17:47:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "exporter_addr": "0.0.0.0", 2024-08-17T17:47:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "exporter_cert_file": "", 2024-08-17T17:47:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "exporter_http_port": "9926", 2024-08-17T17:47:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "exporter_key_file": "", 2024-08-17T17:47:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "exporter_prio_limit": "5", 2024-08-17T17:47:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-08-17T17:47:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "exporter_sort_metrics": "true", 2024-08-17T17:47:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "exporter_stats_period": "5", 2024-08-17T17:47:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "fake_statfs_for_testing": "0", 2024-08-17T17:47:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "fatal_signal_handlers": "true", 2024-08-17T17:47:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "filer_max_purge_ops": "10", 2024-08-17T17:47:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "filer_max_truncate_ops": "128", 2024-08-17T17:47:23.676 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_apply_finisher_threads": "1", 2024-08-17T17:47:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_blackhole": "false", 2024-08-17T17:47:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_btrfs_clone_range": "true", 2024-08-17T17:47:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_btrfs_snap": "true", 2024-08-17T17:47:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_caller_concurrency": "10", 2024-08-17T17:47:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_collect_device_partition_information": "true", 2024-08-17T17:47:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_commit_timeout": "600.000000", 2024-08-17T17:47:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_debug_inject_read_err": "false", 2024-08-17T17:47:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_debug_omap_check": "false", 2024-08-17T17:47:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_debug_verify_split": "false", 2024-08-17T17:47:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_dump_file": "", 2024-08-17T17:47:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-08-17T17:47:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-08-17T17:47:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_fadvise": "true", 2024-08-17T17:47:23.677 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_fail_eio": "true", 2024-08-17T17:47:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_fd_cache_shards": "16", 2024-08-17T17:47:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_fd_cache_size": "128", 2024-08-17T17:47:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_fiemap": "false", 2024-08-17T17:47:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_fiemap_threshold": "4096", 2024-08-17T17:47:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-08-17T17:47:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_index_retry_probability": "0.000000", 2024-08-17T17:47:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_inject_stall": "0", 2024-08-17T17:47:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_journal_parallel": "false", 2024-08-17T17:47:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_journal_trailing": "false", 2024-08-17T17:47:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_journal_writeahead": "false", 2024-08-17T17:47:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_kill_at": "0", 2024-08-17T17:47:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-08-17T17:47:23.678 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_max_inline_xattr_size": "0", 2024-08-17T17:47:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-08-17T17:47:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-08-17T17:47:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-08-17T17:47:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_max_inline_xattrs": "0", 2024-08-17T17:47:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-08-17T17:47:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_max_inline_xattrs_other": "2", 2024-08-17T17:47:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-08-17T17:47:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_max_sync_interval": "5.000000", 2024-08-17T17:47:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_max_xattr_value_size": "0", 2024-08-17T17:47:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-08-17T17:47:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-08-17T17:47:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-08-17T17:47:23.679 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_merge_threshold": "-10", 2024-08-17T17:47:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_min_sync_interval": "0.010000", 2024-08-17T17:47:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_odsync_write": "false", 2024-08-17T17:47:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_omap_backend": "rocksdb", 2024-08-17T17:47:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_omap_backend_path": "", 2024-08-17T17:47:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_omap_header_cache_size": "1024", 2024-08-17T17:47:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_ondisk_finisher_threads": "1", 2024-08-17T17:47:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-08-17T17:47:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_op_thread_timeout": "60", 2024-08-17T17:47:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_op_threads": "2", 2024-08-17T17:47:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_punch_hole": "false", 2024-08-17T17:47:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-08-17T17:47:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-08-17T17:47:23.680 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-08-17T17:47:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-08-17T17:47:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-08-17T17:47:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_queue_max_bytes": "104857600", 2024-08-17T17:47:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-08-17T17:47:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-08-17T17:47:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-08-17T17:47:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_queue_max_ops": "50", 2024-08-17T17:47:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-08-17T17:47:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_seek_data_hole": "false", 2024-08-17T17:47:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_sloppy_crc": "false", 2024-08-17T17:47:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-08-17T17:47:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_splice": "false", 2024-08-17T17:47:23.681 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_split_multiple": "2", 2024-08-17T17:47:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_split_rand_factor": "20", 2024-08-17T17:47:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_update_to": "1000", 2024-08-17T17:47:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-08-17T17:47:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-08-17T17:47:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-08-17T17:47:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-08-17T17:47:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-08-17T17:47:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-08-17T17:47:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_wbthrottle_enable": "true", 2024-08-17T17:47:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-08-17T17:47:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-08-17T17:47:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-08-17T17:47:23.682 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-08-17T17:47:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-08-17T17:47:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-08-17T17:47:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_xfs_extsize": "false", 2024-08-17T17:47:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "filestore_zfs_snap": "false", 2024-08-17T17:47:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "fio_dir": "/tmp/fio", 2024-08-17T17:47:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "fsid": "25127030-5cbc-11ef-bccf-c7b262605968", 2024-08-17T17:47:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_allow_other": "true", 2024-08-17T17:47:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_atomic_o_trunc": "true", 2024-08-17T17:47:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_big_writes": "true", 2024-08-17T17:47:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_debug": "false", 2024-08-17T17:47:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_default_permissions": "false", 2024-08-17T17:47:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_disable_pagecache": "false", 2024-08-17T17:47:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_max_write": "0", 2024-08-17T17:47:23.683 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_multithreaded": "true", 2024-08-17T17:47:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_require_active_mds": "true", 2024-08-17T17:47:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_set_user_groups": "true", 2024-08-17T17:47:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_splice_move": "true", 2024-08-17T17:47:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_splice_read": "true", 2024-08-17T17:47:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_splice_write": "true", 2024-08-17T17:47:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_syncfs_on_mksnap": "true", 2024-08-17T17:47:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "fuse_use_invalidate_cb": "true", 2024-08-17T17:47:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-08-17T17:47:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "gss_target_name": "ceph", 2024-08-17T17:47:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "heartbeat_file": "", 2024-08-17T17:47:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "heartbeat_inject_failure": "0", 2024-08-17T17:47:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "heartbeat_interval": "5", 2024-08-17T17:47:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "host": "smithi130", 2024-08-17T17:47:23.684 INFO:teuthology.orchestra.run.smithi130.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-08-17T17:47:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-08-17T17:47:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "immutable_object_cache_max_size": "1073741824", 2024-08-17T17:47:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-08-17T17:47:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-08-17T17:47:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-08-17T17:47:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-08-17T17:47:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-08-17T17:47:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-08-17T17:47:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-08-17T17:47:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-08-17T17:47:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-08-17T17:47:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "immutable_object_cache_watermark": "0.900000", 2024-08-17T17:47:23.685 INFO:teuthology.orchestra.run.smithi130.stdout: "inject_early_sigterm": "false", 2024-08-17T17:47:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "jaeger_agent_port": "6799", 2024-08-17T17:47:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "jaeger_tracing_enable": "false", 2024-08-17T17:47:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_aio": "true", 2024-08-17T17:47:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_align_min_size": "65536", 2024-08-17T17:47:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_block_align": "true", 2024-08-17T17:47:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_block_size": "4096", 2024-08-17T17:47:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_dio": "true", 2024-08-17T17:47:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_discard": "false", 2024-08-17T17:47:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_force_aio": "false", 2024-08-17T17:47:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_ignore_corruption": "false", 2024-08-17T17:47:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_max_write_bytes": "10485760", 2024-08-17T17:47:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_max_write_entries": "100", 2024-08-17T17:47:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_replay_from": "0", 2024-08-17T17:47:23.686 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_throttle_high_multiple": "0.000000", 2024-08-17T17:47:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-08-17T17:47:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-08-17T17:47:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_throttle_max_multiple": "0.000000", 2024-08-17T17:47:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_write_header_frequency": "0", 2024-08-17T17:47:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "journal_zero_on_create": "false", 2024-08-17T17:47:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "journaler_prefetch_periods": "10", 2024-08-17T17:47:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "journaler_prezero_periods": "5", 2024-08-17T17:47:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "journaler_write_head_interval": "15", 2024-08-17T17:47:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "key": "", 2024-08-17T17:47:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "keyfile": "", 2024-08-17T17:47:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-08-17T17:47:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "kstore_backend": "rocksdb", 2024-08-17T17:47:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "kstore_default_stripe_size": "65536", 2024-08-17T17:47:23.687 INFO:teuthology.orchestra.run.smithi130.stdout: "kstore_fsck_on_mount": "false", 2024-08-17T17:47:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "kstore_fsck_on_mount_deep": "true", 2024-08-17T17:47:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "kstore_max_bytes": "67108864", 2024-08-17T17:47:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "kstore_max_ops": "512", 2024-08-17T17:47:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "kstore_nid_prealloc": "1024", 2024-08-17T17:47:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "kstore_onode_map_size": "1024", 2024-08-17T17:47:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-08-17T17:47:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "kstore_sync_submit_transaction": "false", 2024-08-17T17:47:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "kstore_sync_transaction": "false", 2024-08-17T17:47:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "librados_thread_count": "2", 2024-08-17T17:47:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "lockdep": "false", 2024-08-17T17:47:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "lockdep_force_backtrace": "false", 2024-08-17T17:47:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "log_coarse_timestamps": "true", 2024-08-17T17:47:23.688 INFO:teuthology.orchestra.run.smithi130.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-08-17T17:47:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "log_flush_on_exit": "false", 2024-08-17T17:47:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "log_graylog_host": "127.0.0.1", 2024-08-17T17:47:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "log_graylog_port": "12201", 2024-08-17T17:47:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "log_max_new": "1000", 2024-08-17T17:47:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "log_max_recent": "10000", 2024-08-17T17:47:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "log_stderr_prefix": "", 2024-08-17T17:47:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "log_stop_at_utilization": "0.970000", 2024-08-17T17:47:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "log_to_file": "true", 2024-08-17T17:47:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "log_to_graylog": "false", 2024-08-17T17:47:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "log_to_journald": "false", 2024-08-17T17:47:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "log_to_stderr": "false", 2024-08-17T17:47:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "log_to_syslog": "false", 2024-08-17T17:47:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "max_rotating_auth_attempts": "10", 2024-08-17T17:47:23.689 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-08-17T17:47:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_action_on_write_error": "1", 2024-08-17T17:47:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_alternate_name_max": "8192", 2024-08-17T17:47:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_asio_thread_count": "2", 2024-08-17T17:47:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_export_pin": "true", 2024-08-17T17:47:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_fragment_dirs": "true", 2024-08-17T17:47:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-08-17T17:47:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_fragment_interval": "5", 2024-08-17T17:47:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_fragment_size_max": "100000", 2024-08-17T17:47:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_idle_threshold": "0.000000", 2024-08-17T17:47:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_interval": "10", 2024-08-17T17:47:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_max": "-1", 2024-08-17T17:47:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_max_until": "-1", 2024-08-17T17:47:23.690 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_merge_size": "50", 2024-08-17T17:47:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_midchunk": "0.300000", 2024-08-17T17:47:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_min_rebalance": "0.100000", 2024-08-17T17:47:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_min_start": "0.200000", 2024-08-17T17:47:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_minchunk": "0.001000", 2024-08-17T17:47:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_mode": "0", 2024-08-17T17:47:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_need_max": "1.200000", 2024-08-17T17:47:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_need_min": "0.800000", 2024-08-17T17:47:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_overload_epochs": "2", 2024-08-17T17:47:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-08-17T17:47:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_sample_interval": "3.000000", 2024-08-17T17:47:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_split_bits": "3", 2024-08-17T17:47:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_split_rd": "25000.000000", 2024-08-17T17:47:23.691 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_split_size": "10000", 2024-08-17T17:47:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_split_wr": "10000.000000", 2024-08-17T17:47:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_target_decay": "10.000000", 2024-08-17T17:47:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-08-17T17:47:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_beacon_grace": "15.000000", 2024-08-17T17:47:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_beacon_interval": "4.000000", 2024-08-17T17:47:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_beacon_mon_down_grace": "60", 2024-08-17T17:47:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache_memory_limit": "4294967296", 2024-08-17T17:47:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache_mid": "0.700000", 2024-08-17T17:47:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-08-17T17:47:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache_quiesce_delay": "0", 2024-08-17T17:47:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache_quiesce_sleep": "200", 2024-08-17T17:47:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache_quiesce_splitauth": "true", 2024-08-17T17:47:23.692 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache_quiesce_threshold": "524288", 2024-08-17T17:47:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache_release_free_interval": "10", 2024-08-17T17:47:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache_reservation": "0.050000", 2024-08-17T17:47:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-08-17T17:47:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache_trim_interval": "1", 2024-08-17T17:47:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cache_trim_threshold": "262144", 2024-08-17T17:47:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-08-17T17:47:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-08-17T17:47:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_client_delegate_inos_pct": "50", 2024-08-17T17:47:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_client_prealloc_inos": "1000", 2024-08-17T17:47:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-08-17T17:47:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_connect_bootstrapping": "false", 2024-08-17T17:47:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_damage_table_max_entries": "10000", 2024-08-17T17:47:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-08-17T17:47:23.693 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_debug_auth_pins": "false", 2024-08-17T17:47:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_debug_frag": "false", 2024-08-17T17:47:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_debug_scatterstat": "false", 2024-08-17T17:47:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_debug_subtrees": "false", 2024-08-17T17:47:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_decay_halflife": "5.000000", 2024-08-17T17:47:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_default_dir_hash": "2", 2024-08-17T17:47:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_defer_session_stale": "true", 2024-08-17T17:47:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_deny_all_reconnect": "false", 2024-08-17T17:47:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_dir_keys_per_op": "16384", 2024-08-17T17:47:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_dir_max_commit_size": "10", 2024-08-17T17:47:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_dir_max_entries": "0", 2024-08-17T17:47:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_dir_prefetch": "true", 2024-08-17T17:47:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_dirstat_min_interval": "1.000000", 2024-08-17T17:47:23.694 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_dump_cache_after_rejoin": "false", 2024-08-17T17:47:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_dump_cache_on_map": "false", 2024-08-17T17:47:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_dump_cache_threshold_file": "0", 2024-08-17T17:47:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-08-17T17:47:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_early_reply": "true", 2024-08-17T17:47:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_enable_op_tracker": "true", 2024-08-17T17:47:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_enforce_unique_name": "true", 2024-08-17T17:47:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_export_ephemeral_distributed": "true", 2024-08-17T17:47:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-08-17T17:47:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_export_ephemeral_random": "true", 2024-08-17T17:47:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-08-17T17:47:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-08-17T17:47:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_forward_all_requests_to_auth": "false", 2024-08-17T17:47:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-08-17T17:47:23.695 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-08-17T17:47:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-08-17T17:47:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-08-17T17:47:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_health_cache_threshold": "1.500000", 2024-08-17T17:47:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_health_summarize_threshold": "10", 2024-08-17T17:47:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_heartbeat_grace": "15.000000", 2024-08-17T17:47:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_heartbeat_reset_grace": "1000", 2024-08-17T17:47:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_inject_health_dummy": "false", 2024-08-17T17:47:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-08-17T17:47:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_inject_migrator_session_race": "false", 2024-08-17T17:47:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-08-17T17:47:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-08-17T17:47:23.696 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-08-17T17:47:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_join_fs": "", 2024-08-17T17:47:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_journal_format": "1", 2024-08-17T17:47:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_kill_after_journal_logs_flushed": "false", 2024-08-17T17:47:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_kill_create_at": "0", 2024-08-17T17:47:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_kill_dirfrag_at": "0", 2024-08-17T17:47:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_kill_export_at": "0", 2024-08-17T17:47:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_kill_import_at": "0", 2024-08-17T17:47:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_kill_journal_at": "0", 2024-08-17T17:47:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_kill_journal_expire_at": "0", 2024-08-17T17:47:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_kill_journal_replay_at": "0", 2024-08-17T17:47:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_kill_link_at": "0", 2024-08-17T17:47:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_kill_mdstable_at": "0", 2024-08-17T17:47:23.697 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_kill_openc_at": "0", 2024-08-17T17:47:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_kill_rename_at": "0", 2024-08-17T17:47:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_kill_shutdown_at": "0", 2024-08-17T17:47:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log_event_large_threshold": "524288", 2024-08-17T17:47:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log_events_per_segment": "1024", 2024-08-17T17:47:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log_major_segment_event_ratio": "12", 2024-08-17T17:47:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log_max_events": "-1", 2024-08-17T17:47:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log_max_segments": "128", 2024-08-17T17:47:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log_pause": "false", 2024-08-17T17:47:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log_segment_size": "0", 2024-08-17T17:47:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log_skip_corrupt_events": "false", 2024-08-17T17:47:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log_skip_unbounded_events": "false", 2024-08-17T17:47:23.698 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-08-17T17:47:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log_trim_threshold": "128", 2024-08-17T17:47:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-08-17T17:47:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_log_warn_factor": "2.000000", 2024-08-17T17:47:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_max_caps_per_client": "1048576", 2024-08-17T17:47:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_max_completed_flushes": "100000", 2024-08-17T17:47:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_max_completed_requests": "100000", 2024-08-17T17:47:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_max_export_size": "20971520", 2024-08-17T17:47:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_max_file_recover": "32", 2024-08-17T17:47:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_max_purge_files": "64", 2024-08-17T17:47:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_max_purge_ops": "8192", 2024-08-17T17:47:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-08-17T17:47:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-08-17T17:47:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_max_snaps_per_dir": "100", 2024-08-17T17:47:23.699 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_metrics_update_interval": "2", 2024-08-17T17:47:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_min_caps_per_client": "100", 2024-08-17T17:47:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_min_caps_working_set": "10000", 2024-08-17T17:47:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-08-17T17:47:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_numa_node": "-1", 2024-08-17T17:47:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-08-17T17:47:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_op_complaint_time": "30.000000", 2024-08-17T17:47:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_op_history_duration": "600", 2024-08-17T17:47:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_op_history_size": "20", 2024-08-17T17:47:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_op_history_slow_op_size": "20", 2024-08-17T17:47:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_op_history_slow_op_threshold": "10", 2024-08-17T17:47:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_op_log_threshold": "5", 2024-08-17T17:47:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_ping_grace": "15", 2024-08-17T17:47:23.700 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_ping_interval": "5", 2024-08-17T17:47:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-08-17T17:47:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-08-17T17:47:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_recall_max_caps": "30000", 2024-08-17T17:47:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-08-17T17:47:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_recall_max_decay_threshold": "131072", 2024-08-17T17:47:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-08-17T17:47:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_recall_warning_threshold": "262144", 2024-08-17T17:47:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_reconnect_timeout": "45.000000", 2024-08-17T17:47:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_replay_interval": "1.000000", 2024-08-17T17:47:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-08-17T17:47:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-08-17T17:47:23.701 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_root_ino_gid": "0", 2024-08-17T17:47:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_root_ino_uid": "0", 2024-08-17T17:47:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-08-17T17:47:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_server_dispatch_client_request_delay": "0", 2024-08-17T17:47:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_server_dispatch_killpoint_random": "0.000000", 2024-08-17T17:47:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_session_blocklist_on_evict": "true", 2024-08-17T17:47:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_session_blocklist_on_timeout": "true", 2024-08-17T17:47:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-08-17T17:47:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-08-17T17:47:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-08-17T17:47:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-08-17T17:47:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-08-17T17:47:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_session_metadata_threshold": "16777216", 2024-08-17T17:47:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-08-17T17:47:23.702 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_shutdown_check": "0", 2024-08-17T17:47:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_skip_ino": "0", 2024-08-17T17:47:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_sleep_rank_change": "0.000000", 2024-08-17T17:47:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_snap_max_uid": "4294967294", 2024-08-17T17:47:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_snap_min_uid": "0", 2024-08-17T17:47:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_snap_rstat": "false", 2024-08-17T17:47:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_standby_replay_damaged": "false", 2024-08-17T17:47:23.703 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_symlink_recovery": "true", 2024-08-17T17:47:23.704 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_task_status_update_interval": "2.000000", 2024-08-17T17:47:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_thrash_exports": "0", 2024-08-17T17:47:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_thrash_fragments": "0", 2024-08-17T17:47:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_tick_interval": "5.000000", 2024-08-17T17:47:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_valgrind_exit": "false", 2024-08-17T17:47:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_verify_backtrace": "1", 2024-08-17T17:47:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_verify_scatter": "false", 2024-08-17T17:47:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_wipe_ino_prealloc": "false", 2024-08-17T17:47:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "mds_wipe_sessions": "false", 2024-08-17T17:47:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "mempool_debug": "false", 2024-08-17T17:47:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "memstore_debug_omit_block_device_write": "false", 2024-08-17T17:47:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "memstore_device_bytes": "1073741824", 2024-08-17T17:47:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "memstore_page_set": "false", 2024-08-17T17:47:23.705 INFO:teuthology.orchestra.run.smithi130.stdout: "memstore_page_size": "65536", 2024-08-17T17:47:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_client_bytes": "134217728", 2024-08-17T17:47:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_client_messages": "512", 2024-08-17T17:47:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-08-17T17:47:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_connect_retry_interval": "1.000000", 2024-08-17T17:47:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-08-17T17:47:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-08-17T17:47:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_disabled_modules": "", 2024-08-17T17:47:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_enable_op_tracker": "true", 2024-08-17T17:47:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-08-17T17:47:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_max_pg_creating": "1024", 2024-08-17T17:47:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_max_pg_num_change": "128", 2024-08-17T17:47:23.706 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_mds_bytes": "134217728", 2024-08-17T17:47:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_mds_messages": "128", 2024-08-17T17:47:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-08-17T17:47:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_mon_bytes": "134217728", 2024-08-17T17:47:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_mon_messages": "128", 2024-08-17T17:47:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_num_op_tracker_shard": "32", 2024-08-17T17:47:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_op_complaint_time": "30.000000", 2024-08-17T17:47:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_op_history_duration": "600", 2024-08-17T17:47:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_op_history_size": "20", 2024-08-17T17:47:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_op_history_slow_op_size": "20", 2024-08-17T17:47:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_op_history_slow_op_threshold": "10.000000", 2024-08-17T17:47:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_op_log_threshold": "5", 2024-08-17T17:47:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_osd_bytes": "536870912", 2024-08-17T17:47:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_osd_messages": "8192", 2024-08-17T17:47:23.707 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_pool": "true", 2024-08-17T17:47:23.708 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_service_beacon_grace": "60.000000", 2024-08-17T17:47:23.708 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_standby_modules": "true", 2024-08-17T17:47:23.708 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_stats_period": "5", 2024-08-17T17:47:23.708 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_stats_threshold": "5", 2024-08-17T17:47:23.708 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_tick_period": "2", 2024-08-17T17:47:23.708 INFO:teuthology.orchestra.run.smithi130.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-08-17T17:47:23.708 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_accept_timeout_factor": "2.000000", 2024-08-17T17:47:23.708 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_allow_pool_delete": "true", 2024-08-17T17:47:23.708 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_allow_pool_size_one": "false", 2024-08-17T17:47:23.708 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_auth_validate_all_caps": "true", 2024-08-17T17:47:23.708 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-08-17T17:47:23.708 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-08-17T17:47:23.708 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_client_bytes": "104857600", 2024-08-17T17:47:23.708 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_client_directed_command_retry": "2", 2024-08-17T17:47:23.709 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_client_hunt_interval": "3.000000", 2024-08-17T17:47:23.709 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-08-17T17:47:23.709 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-08-17T17:47:23.709 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-08-17T17:47:23.709 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_client_hunt_parallel": "3", 2024-08-17T17:47:23.709 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_client_log_interval": "1.000000", 2024-08-17T17:47:23.709 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-08-17T17:47:23.709 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_client_ping_interval": "10.000000", 2024-08-17T17:47:23.709 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_client_ping_timeout": "30.000000", 2024-08-17T17:47:23.709 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_client_target_rank": "-1", 2024-08-17T17:47:23.709 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_clock_drift_allowed": "1.000000", 2024-08-17T17:47:23.709 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-08-17T17:47:23.709 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-08-17T17:47:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_cluster_log_level": "debug", 2024-08-17T17:47:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_cluster_log_to_file": "true", 2024-08-17T17:47:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_cluster_log_to_graylog": "false", 2024-08-17T17:47:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-08-17T17:47:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-08-17T17:47:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_cluster_log_to_journald": "true", 2024-08-17T17:47:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_cluster_log_to_stderr": "false", 2024-08-17T17:47:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-08-17T17:47:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-08-17T17:47:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_compact_on_bootstrap": "false", 2024-08-17T17:47:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_compact_on_start": "false", 2024-08-17T17:47:23.710 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_compact_on_trim": "true", 2024-08-17T17:47:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_con_tracker_persist_interval": "10", 2024-08-17T17:47:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_con_tracker_score_halflife": "43200", 2024-08-17T17:47:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_config_key_max_entry_size": "65536", 2024-08-17T17:47:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_cpu_threads": "4", 2024-08-17T17:47:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_crush_min_required_version": "hammer", 2024-08-17T17:47:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_daemon_bytes": "419430400", 2024-08-17T17:47:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-08-17T17:47:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_data_avail_crit": "5", 2024-08-17T17:47:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_data_avail_warn": "5", 2024-08-17T17:47:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_data_size_warn": "16106127360", 2024-08-17T17:47:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_debug_block_osdmap_trim": "false", 2024-08-17T17:47:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-08-17T17:47:23.711 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_debug_dump_json": "false", 2024-08-17T17:47:23.712 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-08-17T17:47:23.712 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_debug_dump_transactions": "false", 2024-08-17T17:47:23.712 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_debug_extra_checks": "false", 2024-08-17T17:47:23.712 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-08-17T17:47:23.712 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-08-17T17:47:23.712 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_debug_no_require_reef": "false", 2024-08-17T17:47:23.712 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_debug_no_require_squid": "false", 2024-08-17T17:47:23.712 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-08-17T17:47:23.712 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_delta_reset_interval": "10.000000", 2024-08-17T17:47:23.712 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_dns_srv_name": "ceph-mon", 2024-08-17T17:47:23.712 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_down_mkfs_grace": "60", 2024-08-17T17:47:23.712 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_down_uptime_grace": "60", 2024-08-17T17:47:23.713 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_election_default_strategy": "1", 2024-08-17T17:47:23.713 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_election_timeout": "5.000000", 2024-08-17T17:47:23.713 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-08-17T17:47:23.713 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_elector_ping_divisor": "2", 2024-08-17T17:47:23.713 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_elector_ping_timeout": "2.000000", 2024-08-17T17:47:23.713 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_enable_op_tracker": "true", 2024-08-17T17:47:23.713 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_fake_pool_delete": "false", 2024-08-17T17:47:23.713 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_force_quorum_join": "false", 2024-08-17T17:47:23.713 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_fsmap_prune_threshold": "300", 2024-08-17T17:47:23.713 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_globalid_prealloc": "10000", 2024-08-17T17:47:23.713 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_health_detail_to_clog": "true", 2024-08-17T17:47:23.713 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_health_log_update_period": "5", 2024-08-17T17:47:23.713 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_health_max_detail": "50", 2024-08-17T17:47:23.714 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_health_to_clog": "true", 2024-08-17T17:47:23.714 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_health_to_clog_interval": "600", 2024-08-17T17:47:23.714 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-08-17T17:47:23.714 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_host": "[v2:172.21.15.130:3300/0,v1:172.21.15.130:6789/0] [v2:172.21.15.179:3300/0,v1:172.21.15.179:6789/0] [v2:172.21.15.130:3301/0,v1:172.21.15.130:6790/0]", 2024-08-17T17:47:23.714 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_host_override": "", 2024-08-17T17:47:23.714 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_initial_members": "", 2024-08-17T17:47:23.714 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-08-17T17:47:23.714 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-08-17T17:47:23.714 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-08-17T17:47:23.714 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-08-17T17:47:23.714 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_keyvaluedb": "rocksdb", 2024-08-17T17:47:23.714 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_lease": "5.000000", 2024-08-17T17:47:23.715 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-08-17T17:47:23.715 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-08-17T17:47:23.715 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_log_full_interval": "50", 2024-08-17T17:47:23.715 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_log_max": "10000", 2024-08-17T17:47:23.715 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_log_max_summary": "50", 2024-08-17T17:47:23.715 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_max_log_entries_per_event": "4096", 2024-08-17T17:47:23.715 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_max_log_epochs": "500", 2024-08-17T17:47:23.715 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_max_mdsmap_epochs": "500", 2024-08-17T17:47:23.715 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_max_mgrmap_epochs": "500", 2024-08-17T17:47:23.715 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_max_nvmeof_epochs": "500", 2024-08-17T17:47:23.715 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_max_osd": "10000", 2024-08-17T17:47:23.715 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_max_pg_per_osd": "10000", 2024-08-17T17:47:23.715 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_max_pool_pg_num": "65536", 2024-08-17T17:47:23.716 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-08-17T17:47:23.716 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-08-17T17:47:23.716 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_mds_force_trim_to": "0", 2024-08-17T17:47:23.716 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_mds_skip_sanity": "true", 2024-08-17T17:47:23.716 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_memory_autotune": "true", 2024-08-17T17:47:23.716 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_memory_target": "2147483648", 2024-08-17T17:47:23.716 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_mgr_beacon_grace": "30", 2024-08-17T17:47:23.716 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-08-17T17:47:23.716 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_mgr_digest_period": "5", 2024-08-17T17:47:23.716 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_mgr_inactive_grace": "60", 2024-08-17T17:47:23.716 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_mgr_mkfs_grace": "240", 2024-08-17T17:47:23.716 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-08-17T17:47:23.716 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_min_osdmap_epochs": "500", 2024-08-17T17:47:23.717 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_nvmeofgw_beacon_grace": "10", 2024-08-17T17:47:23.717 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_nvmeofgw_set_group_id_retry": "1000", 2024-08-17T17:47:23.717 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_op_complaint_time": "30", 2024-08-17T17:47:23.717 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_op_history_duration": "600", 2024-08-17T17:47:23.717 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_op_history_size": "20", 2024-08-17T17:47:23.717 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_op_history_slow_op_size": "20", 2024-08-17T17:47:23.717 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_op_history_slow_op_threshold": "10", 2024-08-17T17:47:23.717 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_op_log_threshold": "5", 2024-08-17T17:47:23.717 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-08-17T17:47:23.717 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-08-17T17:47:23.717 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-08-17T17:47:23.717 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_auto_mark_in": "false", 2024-08-17T17:47:23.717 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_auto_mark_new_in": "true", 2024-08-17T17:47:23.717 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-08-17T17:47:23.718 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-08-17T17:47:23.718 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_cache_size": "500", 2024-08-17T17:47:23.718 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_cache_size_min": "134217728", 2024-08-17T17:47:23.718 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_crush_smoke_test": "true", 2024-08-17T17:47:23.718 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_destroyed_out_interval": "600", 2024-08-17T17:47:23.718 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_down_out_interval": "600", 2024-08-17T17:47:23.718 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-08-17T17:47:23.718 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-08-17T17:47:23.718 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_force_trim_to": "0", 2024-08-17T17:47:23.718 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_full_ratio": "0.950000", 2024-08-17T17:47:23.718 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-08-17T17:47:23.718 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_laggy_halflife": "3600", 2024-08-17T17:47:23.719 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_laggy_max_interval": "300", 2024-08-17T17:47:23.719 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_laggy_weight": "0.300000", 2024-08-17T17:47:23.719 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-08-17T17:47:23.719 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_max_initial_pgs": "1024", 2024-08-17T17:47:23.719 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_min_down_reporters": "2", 2024-08-17T17:47:23.719 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-08-17T17:47:23.719 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-08-17T17:47:23.719 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-08-17T17:47:23.719 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_prime_pg_temp": "true", 2024-08-17T17:47:23.719 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-08-17T17:47:23.719 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-08-17T17:47:23.719 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_report_timeout": "900", 2024-08-17T17:47:23.719 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-08-17T17:47:23.719 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-08-17T17:47:23.720 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_warn_num_repaired": "10", 2024-08-17T17:47:23.720 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osd_warn_op_age": "32.000000", 2024-08-17T17:47:23.720 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-08-17T17:47:23.720 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osdmap_full_prune_interval": "10", 2024-08-17T17:47:23.720 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osdmap_full_prune_min": "10000", 2024-08-17T17:47:23.720 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-08-17T17:47:23.720 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-08-17T17:47:23.720 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_pg_stuck_threshold": "60", 2024-08-17T17:47:23.720 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-08-17T17:47:23.720 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_pg_warn_min_objects": "10000", 2024-08-17T17:47:23.720 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_pg_warn_min_per_osd": "0", 2024-08-17T17:47:23.720 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-08-17T17:47:23.720 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_pool_quota_crit_threshold": "0", 2024-08-17T17:47:23.721 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_pool_quota_warn_threshold": "0", 2024-08-17T17:47:23.721 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_probe_timeout": "2.000000", 2024-08-17T17:47:23.721 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_reweight_max_change": "0.050000", 2024-08-17T17:47:23.721 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_reweight_max_osds": "4", 2024-08-17T17:47:23.721 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-08-17T17:47:23.721 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-08-17T17:47:23.721 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-08-17T17:47:23.721 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-08-17T17:47:23.721 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-08-17T17:47:23.721 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_scrub_interval": "86400", 2024-08-17T17:47:23.721 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_scrub_max_keys": "100", 2024-08-17T17:47:23.721 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_scrub_timeout": "300", 2024-08-17T17:47:23.721 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_session_timeout": "300", 2024-08-17T17:47:23.722 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_smart_report_timeout": "5", 2024-08-17T17:47:23.722 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_stat_smooth_intervals": "6", 2024-08-17T17:47:23.722 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-08-17T17:47:23.722 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_stretch_pool_min_size": "2", 2024-08-17T17:47:23.722 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_stretch_pool_size": "4", 2024-08-17T17:47:23.722 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-08-17T17:47:23.722 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_subscribe_interval": "86400.000000", 2024-08-17T17:47:23.722 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_sync_debug": "false", 2024-08-17T17:47:23.722 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_sync_max_payload_keys": "2000", 2024-08-17T17:47:23.722 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_sync_max_payload_size": "1048576", 2024-08-17T17:47:23.722 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_sync_provider_kill_at": "0", 2024-08-17T17:47:23.722 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_sync_requester_kill_at": "0", 2024-08-17T17:47:23.722 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_sync_timeout": "60.000000", 2024-08-17T17:47:23.723 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_target_pg_per_osd": "100", 2024-08-17T17:47:23.723 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_tick_interval": "5", 2024-08-17T17:47:23.723 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_timecheck_interval": "300.000000", 2024-08-17T17:47:23.723 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-08-17T17:47:23.723 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_use_min_delay_socket": "false", 2024-08-17T17:47:23.723 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_older_version_delay": "604800", 2024-08-17T17:47:23.723 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-08-17T17:47:23.723 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-08-17T17:47:23.723 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-08-17T17:47:23.723 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_filestore_osds": "true", 2024-08-17T17:47:23.723 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-08-17T17:47:23.723 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-08-17T17:47:23.724 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-08-17T17:47:23.724 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_misplaced": "false", 2024-08-17T17:47:23.724 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-08-17T17:47:23.724 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_older_version": "true", 2024-08-17T17:47:23.724 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-08-17T17:47:23.724 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_pool_no_app": "true", 2024-08-17T17:47:23.724 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-08-17T17:47:23.724 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-08-17T17:47:23.724 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-08-17T17:47:23.724 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-08-17T17:47:23.724 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_on_too_few_osds": "false", 2024-08-17T17:47:23.724 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-08-17T17:47:23.724 INFO:teuthology.orchestra.run.smithi130.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-08-17T17:47:23.725 INFO:teuthology.orchestra.run.smithi130.stdout: "monmap": "", 2024-08-17T17:47:23.725 INFO:teuthology.orchestra.run.smithi130.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-08-17T17:47:23.725 INFO:teuthology.orchestra.run.smithi130.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-08-17T17:47:23.725 INFO:teuthology.orchestra.run.smithi130.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-08-17T17:47:23.725 INFO:teuthology.orchestra.run.smithi130.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-08-17T17:47:23.725 INFO:teuthology.orchestra.run.smithi130.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-08-17T17:47:23.725 INFO:teuthology.orchestra.run.smithi130.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-08-17T17:47:23.725 INFO:teuthology.orchestra.run.smithi130.stdout: "motr_tracing_enabled": "false", 2024-08-17T17:47:23.725 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_op_threads": "3", 2024-08-17T17:47:23.725 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_buffer_size": "131072", 2024-08-17T17:47:23.725 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_cm": "false", 2024-08-17T17:47:23.725 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_device_name": "", 2024-08-17T17:47:23.725 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_dscp": "96", 2024-08-17T17:47:23.726 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-08-17T17:47:23.726 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_gid_idx": "0", 2024-08-17T17:47:23.726 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_local_gid": "", 2024-08-17T17:47:23.726 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_polling_us": "1000", 2024-08-17T17:47:23.726 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_port_num": "1", 2024-08-17T17:47:23.726 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-08-17T17:47:23.726 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-08-17T17:47:23.726 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_roce_ver": "1", 2024-08-17T17:47:23.726 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_send_buffers": "1024", 2024-08-17T17:47:23.726 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_sl": "3", 2024-08-17T17:47:23.726 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_support_srq": "true", 2024-08-17T17:47:23.726 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_rdma_type": "ib", 2024-08-17T17:47:23.726 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_async_reap_threshold": "5", 2024-08-17T17:47:23.727 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_bind_before_connect": "false", 2024-08-17T17:47:23.727 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_bind_ipv4": "true", 2024-08-17T17:47:23.727 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_bind_ipv6": "false", 2024-08-17T17:47:23.727 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_bind_msgr1": "true", 2024-08-17T17:47:23.727 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_bind_msgr2": "true", 2024-08-17T17:47:23.727 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_bind_port_max": "7568", 2024-08-17T17:47:23.727 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_bind_port_min": "6800", 2024-08-17T17:47:23.727 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_bind_prefer_ipv4": "false", 2024-08-17T17:47:23.727 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_bind_retry_count": "3", 2024-08-17T17:47:23.727 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_bind_retry_delay": "5", 2024-08-17T17:47:23.727 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_blackhole_client": "false", 2024-08-17T17:47:23.727 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_blackhole_mds": "false", 2024-08-17T17:47:23.727 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_blackhole_mgr": "false", 2024-08-17T17:47:23.728 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_blackhole_mon": "false", 2024-08-17T17:47:23.728 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_blackhole_osd": "false", 2024-08-17T17:47:23.728 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_client_mode": "crc secure", 2024-08-17T17:47:23.728 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_client_throttle_retry_time_interval": "5000", 2024-08-17T17:47:23.728 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_cluster_mode": "crc secure", 2024-08-17T17:47:23.728 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_cluster_type": "", 2024-08-17T17:47:23.728 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_compress_secure": "false", 2024-08-17T17:47:23.728 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_connection_idle_timeout": "900", 2024-08-17T17:47:23.728 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_connection_ready_timeout": "10", 2024-08-17T17:47:23.728 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_crc_data": "true", 2024-08-17T17:47:23.728 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_crc_header": "true", 2024-08-17T17:47:23.728 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_die_on_bad_msg": "false", 2024-08-17T17:47:23.728 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_die_on_bug": "true", 2024-08-17T17:47:23.729 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_die_on_old_message": "true", 2024-08-17T17:47:23.729 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_die_on_skipped_message": "false", 2024-08-17T17:47:23.729 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_die_on_unhandled_msg": "false", 2024-08-17T17:47:23.729 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-08-17T17:47:23.729 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_coremask": "0xF", 2024-08-17T17:47:23.729 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-08-17T17:47:23.729 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_devs_allowlist": "", 2024-08-17T17:47:23.729 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_enable_tso": "true", 2024-08-17T17:47:23.729 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-08-17T17:47:23.729 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-08-17T17:47:23.729 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_hugepages": "", 2024-08-17T17:47:23.729 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_hw_flow_control": "true", 2024-08-17T17:47:23.729 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-08-17T17:47:23.729 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_lro": "true", 2024-08-17T17:47:23.730 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_memory_channel": "4", 2024-08-17T17:47:23.730 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-08-17T17:47:23.730 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_pmd": "", 2024-08-17T17:47:23.730 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_port_id": "0", 2024-08-17T17:47:23.730 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-08-17T17:47:23.730 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dump_corrupt_message_level": "1", 2024-08-17T17:47:23.730 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_dump_on_send": "false", 2024-08-17T17:47:23.730 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_initial_backoff": "0.200000", 2024-08-17T17:47:23.730 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_inject_delay_max": "1.000000", 2024-08-17T17:47:23.730 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_inject_delay_probability": "0.000000", 2024-08-17T17:47:23.730 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_inject_delay_type": "", 2024-08-17T17:47:23.730 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_inject_internal_delays": "0.000000", 2024-08-17T17:47:23.731 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_inject_network_congestion": "0", 2024-08-17T17:47:23.731 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_inject_socket_failures": "0", 2024-08-17T17:47:23.731 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_learn_addr_from_peer": "true", 2024-08-17T17:47:23.731 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_max_accept_failures": "4", 2024-08-17T17:47:23.731 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_max_backoff": "15.000000", 2024-08-17T17:47:23.731 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_mon_client_mode": "secure crc", 2024-08-17T17:47:23.731 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_mon_cluster_mode": "secure crc", 2024-08-17T17:47:23.731 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_mon_service_mode": "secure crc", 2024-08-17T17:47:23.731 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_osd_compress_min_size": "1024", 2024-08-17T17:47:23.731 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_osd_compress_mode": "none", 2024-08-17T17:47:23.731 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_osd_compression_algorithm": "snappy", 2024-08-17T17:47:23.731 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-08-17T17:47:23.731 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_pq_min_cost": "65536", 2024-08-17T17:47:23.732 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_public_type": "", 2024-08-17T17:47:23.732 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_service_mode": "crc secure", 2024-08-17T17:47:23.732 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_tcp_listen_backlog": "512", 2024-08-17T17:47:23.732 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_tcp_nodelay": "true", 2024-08-17T17:47:23.732 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-08-17T17:47:23.732 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_tcp_rcvbuf": "0", 2024-08-17T17:47:23.732 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_time_events_min_wait_interval": "1000", 2024-08-17T17:47:23.732 INFO:teuthology.orchestra.run.smithi130.stdout: "ms_type": "async+posix", 2024-08-17T17:47:23.732 INFO:teuthology.orchestra.run.smithi130.stdout: "no_config_file": "false", 2024-08-17T17:47:23.732 INFO:teuthology.orchestra.run.smithi130.stdout: "nvmeof_mon_client_disconnect_panic": "100", 2024-08-17T17:47:23.732 INFO:teuthology.orchestra.run.smithi130.stdout: "nvmeof_mon_client_tick_period": "2", 2024-08-17T17:47:23.732 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter_completion_locks_per_session": "32", 2024-08-17T17:47:23.732 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter_debug_inject_relock_delay": "false", 2024-08-17T17:47:23.733 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter_inflight_op_bytes": "104857600", 2024-08-17T17:47:23.733 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter_inflight_ops": "1024", 2024-08-17T17:47:23.733 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter_inject_no_watch_ping": "false", 2024-08-17T17:47:23.733 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-08-17T17:47:23.733 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter_tick_interval": "5.000000", 2024-08-17T17:47:23.733 INFO:teuthology.orchestra.run.smithi130.stdout: "objecter_timeout": "10.000000", 2024-08-17T17:47:23.733 INFO:teuthology.orchestra.run.smithi130.stdout: "objectstore_blackhole": "false", 2024-08-17T17:47:23.733 INFO:teuthology.orchestra.run.smithi130.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-08-17T17:47:23.733 INFO:teuthology.orchestra.run.smithi130.stdout: "openssl_engine_opts": "", 2024-08-17T17:47:23.733 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_agent_delay_time": "5.000000", 2024-08-17T17:47:23.733 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_agent_hist_halflife": "1000", 2024-08-17T17:47:23.733 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_agent_max_low_ops": "2", 2024-08-17T17:47:23.733 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_agent_max_ops": "4", 2024-08-17T17:47:23.734 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-08-17T17:47:23.734 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_agent_quantize_effort": "0.100000", 2024-08-17T17:47:23.734 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_agent_slop": "0.020000", 2024-08-17T17:47:23.734 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-08-17T17:47:23.734 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_allow_recovery_below_min_size": "true", 2024-08-17T17:47:23.734 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_asio_thread_count": "2", 2024-08-17T17:47:23.734 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_async_recovery_min_cost": "100", 2024-08-17T17:47:23.734 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_auto_mark_unfound_lost": "false", 2024-08-17T17:47:23.734 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_backfill_retry_interval": "30.000000", 2024-08-17T17:47:23.734 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_backfill_scan_max": "512", 2024-08-17T17:47:23.734 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_backfill_scan_min": "64", 2024-08-17T17:47:23.734 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_backoff_on_degraded": "false", 2024-08-17T17:47:23.734 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_backoff_on_peering": "false", 2024-08-17T17:47:23.734 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_backoff_on_unfound": "true", 2024-08-17T17:47:23.735 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_beacon_report_interval": "300", 2024-08-17T17:47:23.735 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_bench_duration": "30", 2024-08-17T17:47:23.735 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-08-17T17:47:23.735 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_bench_max_block_size": "67108864", 2024-08-17T17:47:23.735 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_bench_small_size_max_iops": "100", 2024-08-17T17:47:23.735 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_blkin_trace_all": "false", 2024-08-17T17:47:23.735 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_blocked_scrub_grace_period": "120", 2024-08-17T17:47:23.735 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-08-17T17:47:23.735 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-08-17T17:47:23.735 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-08-17T17:47:23.735 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_check_for_log_corruption": "false", 2024-08-17T17:47:23.735 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-08-17T17:47:23.735 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-08-17T17:47:23.736 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-08-17T17:47:23.736 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-08-17T17:47:23.736 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_class_update_on_start": "true", 2024-08-17T17:47:23.736 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_client_message_cap": "256", 2024-08-17T17:47:23.736 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_client_message_size_cap": "524288000", 2024-08-17T17:47:23.736 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_client_op_priority": "63", 2024-08-17T17:47:23.736 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_client_watch_timeout": "30", 2024-08-17T17:47:23.736 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_command_max_records": "256", 2024-08-17T17:47:23.736 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_compact_on_start": "false", 2024-08-17T17:47:23.736 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-08-17T17:47:23.736 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_crush_chooseleaf_type": "1", 2024-08-17T17:47:23.736 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_crush_initial_weight": "-1.000000", 2024-08-17T17:47:23.736 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_crush_update_on_start": "true", 2024-08-17T17:47:23.737 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_crush_update_weight_set": "true", 2024-08-17T17:47:23.737 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-08-17T17:47:23.737 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-08-17T17:47:23.737 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-08-17T17:47:23.737 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_drop_ping_duration": "0", 2024-08-17T17:47:23.737 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-08-17T17:47:23.737 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_feed_pullee": "-1", 2024-08-17T17:47:23.737 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-08-17T17:47:23.737 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-08-17T17:47:23.737 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-08-17T17:47:23.737 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_misdirected_ops": "false", 2024-08-17T17:47:23.737 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_no_acting_change": "false", 2024-08-17T17:47:23.737 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_no_purge_strays": "false", 2024-08-17T17:47:23.738 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_op_order": "false", 2024-08-17T17:47:23.738 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_pg_log_writeout": "false", 2024-08-17T17:47:23.738 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_pretend_recovery_active": "false", 2024-08-17T17:47:23.738 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-08-17T17:47:23.738 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-08-17T17:47:23.738 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_shutdown": "false", 2024-08-17T17:47:23.738 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-08-17T17:47:23.738 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-08-17T17:47:23.738 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_trim_objects": "false", 2024-08-17T17:47:23.738 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_verify_cached_snaps": "false", 2024-08-17T17:47:23.738 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_verify_missing_on_start": "false", 2024-08-17T17:47:23.738 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_verify_snaps": "false", 2024-08-17T17:47:23.738 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-08-17T17:47:23.739 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-08-17T17:47:23.739 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_deep_scrub_keys": "1024", 2024-08-17T17:47:23.739 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-08-17T17:47:23.739 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-08-17T17:47:23.739 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-08-17T17:47:23.739 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_deep_scrub_stride": "524288", 2024-08-17T17:47:23.739 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-08-17T17:47:23.739 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_default_data_pool_replay_window": "45", 2024-08-17T17:47:23.739 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_default_notify_timeout": "30", 2024-08-17T17:47:23.739 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_delete_sleep": "0.000000", 2024-08-17T17:47:23.739 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-08-17T17:47:23.739 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-08-17T17:47:23.739 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-08-17T17:47:23.740 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_discard_disconnected_ops": "true", 2024-08-17T17:47:23.740 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_ec_partial_reads": "true", 2024-08-17T17:47:23.740 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_enable_op_tracker": "true", 2024-08-17T17:47:23.740 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-08-17T17:47:23.740 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_extblkdev_plugins": "vdo", 2024-08-17T17:47:23.740 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-08-17T17:47:23.740 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-08-17T17:47:23.740 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_fast_info": "true", 2024-08-17T17:47:23.740 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_fast_shutdown": "true", 2024-08-17T17:47:23.740 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-08-17T17:47:23.740 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_fast_shutdown_timeout": "15", 2024-08-17T17:47:23.740 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-08-17T17:47:23.740 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-08-17T17:47:23.740 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-08-17T17:47:23.741 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_function_tracing": "false", 2024-08-17T17:47:23.741 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_heartbeat_grace": "20", 2024-08-17T17:47:23.741 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_heartbeat_interval": "6", 2024-08-17T17:47:23.741 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-08-17T17:47:23.741 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_heartbeat_min_peers": "10", 2024-08-17T17:47:23.741 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_heartbeat_min_size": "2000", 2024-08-17T17:47:23.741 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_heartbeat_stale": "600", 2024-08-17T17:47:23.741 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-08-17T17:47:23.741 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_hit_set_max_size": "100000", 2024-08-17T17:47:23.741 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_hit_set_min_size": "1000", 2024-08-17T17:47:23.741 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-08-17T17:47:23.741 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-08-17T17:47:23.741 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-08-17T17:47:23.741 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-08-17T17:47:23.742 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-08-17T17:47:23.742 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_journal_flush_on_shutdown": "true", 2024-08-17T17:47:23.742 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_journal_size": "5120", 2024-08-17T17:47:23.742 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_kill_backfill_at": "0", 2024-08-17T17:47:23.742 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_loop_before_reset_tphandle": "64", 2024-08-17T17:47:23.742 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_map_cache_size": "50", 2024-08-17T17:47:23.742 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_map_dedup": "true", 2024-08-17T17:47:23.742 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_map_message_max": "40", 2024-08-17T17:47:23.742 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_map_message_max_bytes": "10485760", 2024-08-17T17:47:23.742 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_map_share_max_epochs": "40", 2024-08-17T17:47:23.742 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_attr_name_len": "100", 2024-08-17T17:47:23.742 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_attr_size": "0", 2024-08-17T17:47:23.743 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_backfills": "1", 2024-08-17T17:47:23.743 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_markdown_count": "5", 2024-08-17T17:47:23.743 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_markdown_period": "600", 2024-08-17T17:47:23.743 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_object_name_len": "2048", 2024-08-17T17:47:23.743 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_object_namespace_len": "256", 2024-08-17T17:47:23.743 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_object_size": "134217728", 2024-08-17T17:47:23.743 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-08-17T17:47:23.743 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_omap_entries_per_request": "1024", 2024-08-17T17:47:23.743 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_pg_blocked_by": "16", 2024-08-17T17:47:23.743 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_pg_log_entries": "10000", 2024-08-17T17:47:23.745 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-08-17T17:47:23.745 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_pgls": "1024", 2024-08-17T17:47:23.745 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_push_cost": "8388608", 2024-08-17T17:47:23.745 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_push_objects": "10", 2024-08-17T17:47:23.745 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_scrubs": "3", 2024-08-17T17:47:23.745 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-08-17T17:47:23.745 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_trimming_pgs": "2", 2024-08-17T17:47:23.746 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_write_op_reply_len": "64", 2024-08-17T17:47:23.746 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_max_write_size": "90", 2024-08-17T17:47:23.746 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-08-17T17:47:23.746 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-08-17T17:47:23.746 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-08-17T17:47:23.746 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-08-17T17:47:23.746 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-08-17T17:47:23.746 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-08-17T17:47:23.746 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-08-17T17:47:23.746 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_override_recovery_settings": "false", 2024-08-17T17:47:23.746 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_profile": "balanced", 2024-08-17T17:47:23.746 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-08-17T17:47:23.746 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-08-17T17:47:23.747 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-08-17T17:47:23.747 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-08-17T17:47:23.747 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-08-17T17:47:23.747 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-08-17T17:47:23.747 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-08-17T17:47:23.747 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-08-17T17:47:23.747 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-08-17T17:47:23.747 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-08-17T17:47:23.747 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mclock_skip_benchmark": "false", 2024-08-17T17:47:23.747 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_memory_base": "805306368", 2024-08-17T17:47:23.747 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_memory_cache_min": "134217728", 2024-08-17T17:47:23.747 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-08-17T17:47:23.748 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-08-17T17:47:23.748 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_memory_target": "4294967296", 2024-08-17T17:47:23.748 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_memory_target_autotune": "false", 2024-08-17T17:47:23.748 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-08-17T17:47:23.748 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_min_pg_log_entries": "250", 2024-08-17T17:47:23.748 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_min_recovery_priority": "0", 2024-08-17T17:47:23.748 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mon_heartbeat_interval": "30", 2024-08-17T17:47:23.748 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-08-17T17:47:23.748 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mon_report_interval": "5", 2024-08-17T17:47:23.748 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mon_report_max_in_flight": "2", 2024-08-17T17:47:23.748 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-08-17T17:47:23.748 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_num_cache_shards": "32", 2024-08-17T17:47:23.748 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_num_op_tracker_shard": "32", 2024-08-17T17:47:23.749 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_numa_auto_affinity": "true", 2024-08-17T17:47:23.749 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_numa_node": "-1", 2024-08-17T17:47:23.749 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_numa_prefer_iface": "true", 2024-08-17T17:47:23.749 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-08-17T17:47:23.749 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_objecter_finishers": "1", 2024-08-17T17:47:23.749 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_objectstore": "bluestore", 2024-08-17T17:47:23.749 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_objectstore_fuse": "false", 2024-08-17T17:47:23.749 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_objectstore_ideal_list_max": "64", 2024-08-17T17:47:23.749 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_objectstore_tracing": "false", 2024-08-17T17:47:23.749 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_complaint_time": "30.000000", 2024-08-17T17:47:23.749 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_history_duration": "600", 2024-08-17T17:47:23.749 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_history_size": "20", 2024-08-17T17:47:23.749 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_history_slow_op_size": "20", 2024-08-17T17:47:23.749 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-08-17T17:47:23.750 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_log_threshold": "5", 2024-08-17T17:47:23.750 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_num_shards": "0", 2024-08-17T17:47:23.750 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_num_shards_hdd": "5", 2024-08-17T17:47:23.750 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_num_shards_ssd": "8", 2024-08-17T17:47:23.750 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_num_threads_per_shard": "0", 2024-08-17T17:47:23.750 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-08-17T17:47:23.750 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-08-17T17:47:23.750 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-08-17T17:47:23.750 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_pq_min_cost": "65536", 2024-08-17T17:47:23.750 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_queue": "mclock_scheduler", 2024-08-17T17:47:23.750 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_queue_cut_off": "high", 2024-08-17T17:47:23.750 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_thread_suicide_timeout": "150", 2024-08-17T17:47:23.750 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_op_thread_timeout": "15", 2024-08-17T17:47:23.750 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_open_classes_on_start": "true", 2024-08-17T17:47:23.751 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_os_flags": "0", 2024-08-17T17:47:23.751 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_peering_op_priority": "255", 2024-08-17T17:47:23.751 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pg_delete_cost": "1048576", 2024-08-17T17:47:23.751 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pg_delete_priority": "5", 2024-08-17T17:47:23.751 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-08-17T17:47:23.751 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-08-17T17:47:23.751 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pg_log_dups_tracked": "3000", 2024-08-17T17:47:23.751 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pg_log_trim_max": "10000", 2024-08-17T17:47:23.751 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pg_log_trim_min": "100", 2024-08-17T17:47:23.751 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-08-17T17:47:23.751 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pg_object_context_cache_count": "64", 2024-08-17T17:47:23.751 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-08-17T17:47:23.751 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-08-17T17:47:23.752 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-08-17T17:47:23.752 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-08-17T17:47:23.752 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-08-17T17:47:23.752 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-08-17T17:47:23.752 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-08-17T17:47:23.752 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-08-17T17:47:23.752 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_crimson": "false", 2024-08-17T17:47:23.752 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_crush_rule": "-1", 2024-08-17T17:47:23.752 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_ec_fast_read": "false", 2024-08-17T17:47:23.752 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-08-17T17:47:23.752 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_flag_bulk": "false", 2024-08-17T17:47:23.752 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-08-17T17:47:23.752 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_flag_nodelete": "false", 2024-08-17T17:47:23.752 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-08-17T17:47:23.753 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-08-17T17:47:23.753 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_flags": "0", 2024-08-17T17:47:23.753 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-08-17T17:47:23.753 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_min_size": "0", 2024-08-17T17:47:23.753 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-08-17T17:47:23.753 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_pg_num": "32", 2024-08-17T17:47:23.753 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_pgp_num": "0", 2024-08-17T17:47:23.753 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-08-17T17:47:23.753 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_read_ratio": "70", 2024-08-17T17:47:23.753 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_size": "3", 2024-08-17T17:47:23.753 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_default_type": "replicated", 2024-08-17T17:47:23.753 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-08-17T17:47:23.753 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_pool_use_gmt_hitset": "true", 2024-08-17T17:47:23.754 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_push_per_object_cost": "1000", 2024-08-17T17:47:23.754 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_read_ec_check_for_errors": "false", 2024-08-17T17:47:23.754 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recover_clone_overlap": "true", 2024-08-17T17:47:23.754 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recover_clone_overlap_limit": "10", 2024-08-17T17:47:23.754 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_cost": "20971520", 2024-08-17T17:47:23.754 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_delay_start": "0.000000", 2024-08-17T17:47:23.754 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_max_active": "0", 2024-08-17T17:47:23.754 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_max_active_hdd": "3", 2024-08-17T17:47:23.754 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_max_active_ssd": "10", 2024-08-17T17:47:23.754 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_max_chunk": "8388608", 2024-08-17T17:47:23.754 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-08-17T17:47:23.754 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_max_single_start": "1", 2024-08-17T17:47:23.754 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_op_priority": "3", 2024-08-17T17:47:23.755 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_op_warn_multiple": "16", 2024-08-17T17:47:23.755 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_priority": "5", 2024-08-17T17:47:23.755 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_retry_interval": "30.000000", 2024-08-17T17:47:23.755 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_sleep": "0.000000", 2024-08-17T17:47:23.755 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-08-17T17:47:23.755 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-08-17T17:47:23.755 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-08-17T17:47:23.755 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_repair_during_recovery": "false", 2024-08-17T17:47:23.755 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_requested_scrub_priority": "120", 2024-08-17T17:47:23.755 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-08-17T17:47:23.755 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_rollback_to_cluster_snap": "", 2024-08-17T17:47:23.755 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_auto_repair": "false", 2024-08-17T17:47:23.756 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-08-17T17:47:23.756 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-08-17T17:47:23.756 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_begin_hour": "0", 2024-08-17T17:47:23.756 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_begin_week_day": "0", 2024-08-17T17:47:23.756 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_chunk_max": "25", 2024-08-17T17:47:23.756 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_chunk_min": "5", 2024-08-17T17:47:23.756 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_cost": "52428800", 2024-08-17T17:47:23.756 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-08-17T17:47:23.756 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_during_recovery": "false", 2024-08-17T17:47:23.756 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_end_hour": "0", 2024-08-17T17:47:23.756 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_end_week_day": "0", 2024-08-17T17:47:23.756 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_event_cost": "4096", 2024-08-17T17:47:23.756 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-08-17T17:47:23.757 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-08-17T17:47:23.757 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_invalid_stats": "true", 2024-08-17T17:47:23.757 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_load_threshold": "0.500000", 2024-08-17T17:47:23.757 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_max_interval": "604800.000000", 2024-08-17T17:47:23.757 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_max_preemptions": "5", 2024-08-17T17:47:23.757 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_min_interval": "86400.000000", 2024-08-17T17:47:23.757 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_priority": "5", 2024-08-17T17:47:23.757 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_scrub_sleep": "0.000000", 2024-08-17T17:47:23.757 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-08-17T17:47:23.757 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-08-17T17:47:23.757 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_shutdown_pgref_assert": "false", 2024-08-17T17:47:23.757 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-08-17T17:47:23.757 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_skip_data_digest": "false", 2024-08-17T17:47:23.757 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_smart_report_timeout": "5", 2024-08-17T17:47:23.758 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_snap_trim_cost": "1048576", 2024-08-17T17:47:23.758 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_snap_trim_priority": "5", 2024-08-17T17:47:23.758 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_snap_trim_sleep": "0.000000", 2024-08-17T17:47:23.758 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-08-17T17:47:23.758 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-08-17T17:47:23.758 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-08-17T17:47:23.758 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-08-17T17:47:23.758 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_stats_update_period_scrubbing": "15", 2024-08-17T17:47:23.758 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-08-17T17:47:23.758 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_target_transaction_size": "30", 2024-08-17T17:47:23.758 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_tier_default_cache_mode": "writeback", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_tracing": "false", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_use_stale_snap": "false", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "osdc_blkin_trace_all": "false", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "paxos_kill_at": "0", 2024-08-17T17:47:23.759 INFO:teuthology.orchestra.run.smithi130.stdout: "paxos_max_join_drift": "10", 2024-08-17T17:47:23.760 INFO:teuthology.orchestra.run.smithi130.stdout: "paxos_min": "500", 2024-08-17T17:47:23.760 INFO:teuthology.orchestra.run.smithi130.stdout: "paxos_min_wait": "0.050000", 2024-08-17T17:47:23.760 INFO:teuthology.orchestra.run.smithi130.stdout: "paxos_propose_interval": "1.000000", 2024-08-17T17:47:23.760 INFO:teuthology.orchestra.run.smithi130.stdout: "paxos_service_trim_max": "500", 2024-08-17T17:47:23.760 INFO:teuthology.orchestra.run.smithi130.stdout: "paxos_service_trim_max_multiplier": "20", 2024-08-17T17:47:23.760 INFO:teuthology.orchestra.run.smithi130.stdout: "paxos_service_trim_min": "250", 2024-08-17T17:47:23.760 INFO:teuthology.orchestra.run.smithi130.stdout: "paxos_stash_full_interval": "25", 2024-08-17T17:47:23.760 INFO:teuthology.orchestra.run.smithi130.stdout: "paxos_trim_max": "500", 2024-08-17T17:47:23.760 INFO:teuthology.orchestra.run.smithi130.stdout: "paxos_trim_min": "250", 2024-08-17T17:47:23.760 INFO:teuthology.orchestra.run.smithi130.stdout: "perf": "true", 2024-08-17T17:47:23.760 INFO:teuthology.orchestra.run.smithi130.stdout: "pid_file": "", 2024-08-17T17:47:23.760 INFO:teuthology.orchestra.run.smithi130.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-08-17T17:47:23.761 INFO:teuthology.orchestra.run.smithi130.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-08-17T17:47:23.761 INFO:teuthology.orchestra.run.smithi130.stdout: "public_addr": "-", 2024-08-17T17:47:23.761 INFO:teuthology.orchestra.run.smithi130.stdout: "public_addrv": "", 2024-08-17T17:47:23.761 INFO:teuthology.orchestra.run.smithi130.stdout: "public_bind_addr": "-", 2024-08-17T17:47:23.761 INFO:teuthology.orchestra.run.smithi130.stdout: "public_network": "172.21.0.0/20", 2024-08-17T17:47:23.761 INFO:teuthology.orchestra.run.smithi130.stdout: "public_network_interface": "", 2024-08-17T17:47:23.761 INFO:teuthology.orchestra.run.smithi130.stdout: "qat_compressor_busy_polling": "false", 2024-08-17T17:47:23.761 INFO:teuthology.orchestra.run.smithi130.stdout: "qat_compressor_enabled": "false", 2024-08-17T17:47:23.761 INFO:teuthology.orchestra.run.smithi130.stdout: "qat_compressor_session_max_number": "256", 2024-08-17T17:47:23.761 INFO:teuthology.orchestra.run.smithi130.stdout: "rados_mon_op_timeout": "0", 2024-08-17T17:47:23.761 INFO:teuthology.orchestra.run.smithi130.stdout: "rados_osd_op_timeout": "0", 2024-08-17T17:47:23.761 INFO:teuthology.orchestra.run.smithi130.stdout: "rados_tracing": "false", 2024-08-17T17:47:23.761 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_atime_update_interval": "60", 2024-08-17T17:47:23.761 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-08-17T17:47:23.762 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_balance_parent_reads": "false", 2024-08-17T17:47:23.762 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_balance_snap_reads": "false", 2024-08-17T17:47:23.762 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_blkin_trace_all": "false", 2024-08-17T17:47:23.762 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_blocklist_expire_seconds": "0", 2024-08-17T17:47:23.762 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_blocklist_on_break_lock": "true", 2024-08-17T17:47:23.762 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_cache": "true", 2024-08-17T17:47:23.762 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_cache_block_writes_upfront": "false", 2024-08-17T17:47:23.762 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_cache_max_dirty": "25165824", 2024-08-17T17:47:23.762 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-08-17T17:47:23.762 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_cache_max_dirty_object": "0", 2024-08-17T17:47:23.762 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_cache_policy": "writearound", 2024-08-17T17:47:23.762 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_cache_size": "33554432", 2024-08-17T17:47:23.763 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_cache_target_dirty": "16777216", 2024-08-17T17:47:23.763 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-08-17T17:47:23.763 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_clone_copy_on_read": "false", 2024-08-17T17:47:23.763 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_compression_hint": "none", 2024-08-17T17:47:23.763 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_concurrent_management_ops": "10", 2024-08-17T17:47:23.763 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_config_pool_override_update_timestamp": "1723916189", 2024-08-17T17:47:23.763 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_default_clone_format": "auto", 2024-08-17T17:47:23.763 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_default_data_pool": "", 2024-08-17T17:47:23.763 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_default_features": "61", 2024-08-17T17:47:23.763 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_default_format": "2", 2024-08-17T17:47:23.763 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_default_map_options": "", 2024-08-17T17:47:23.763 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_default_order": "22", 2024-08-17T17:47:23.763 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_default_pool": "rbd", 2024-08-17T17:47:23.764 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-08-17T17:47:23.764 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_default_stripe_count": "0", 2024-08-17T17:47:23.764 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_default_stripe_unit": "0", 2024-08-17T17:47:23.764 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_disable_zero_copy_writes": "true", 2024-08-17T17:47:23.764 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_discard_granularity_bytes": "65536", 2024-08-17T17:47:23.764 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-08-17T17:47:23.764 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_enable_alloc_hint": "true", 2024-08-17T17:47:23.764 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-08-17T17:47:23.764 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_io_scheduler": "simple", 2024-08-17T17:47:23.764 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-08-17T17:47:23.764 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_journal_commit_age": "5.000000", 2024-08-17T17:47:23.764 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-08-17T17:47:23.764 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-08-17T17:47:23.765 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-08-17T17:47:23.765 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-08-17T17:47:23.765 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_journal_object_flush_interval": "0", 2024-08-17T17:47:23.765 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-08-17T17:47:23.765 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-08-17T17:47:23.765 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_journal_order": "24", 2024-08-17T17:47:23.765 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_journal_pool": "", 2024-08-17T17:47:23.765 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_journal_splay_width": "4", 2024-08-17T17:47:23.765 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_localize_parent_reads": "false", 2024-08-17T17:47:23.765 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_localize_snap_reads": "false", 2024-08-17T17:47:23.765 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-08-17T17:47:23.765 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-08-17T17:47:23.765 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-08-17T17:47:23.766 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-08-17T17:47:23.766 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-08-17T17:47:23.766 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-08-17T17:47:23.766 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_image_policy_type": "simple", 2024-08-17T17:47:23.766 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-08-17T17:47:23.766 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-08-17T17:47:23.766 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-08-17T17:47:23.766 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-08-17T17:47:23.766 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-08-17T17:47:23.766 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-08-17T17:47:23.766 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-08-17T17:47:23.766 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_memory_autotune": "true", 2024-08-17T17:47:23.766 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_memory_base": "805306368", 2024-08-17T17:47:23.767 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-08-17T17:47:23.767 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-08-17T17:47:23.767 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-08-17T17:47:23.767 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-08-17T17:47:23.767 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_memory_target": "4294967296", 2024-08-17T17:47:23.767 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-08-17T17:47:23.767 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-08-17T17:47:23.767 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-08-17T17:47:23.767 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirroring_delete_delay": "0", 2024-08-17T17:47:23.767 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-08-17T17:47:23.767 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirroring_replay_delay": "0", 2024-08-17T17:47:23.767 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-08-17T17:47:23.767 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-08-17T17:47:23.768 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_move_to_trash_on_remove": "false", 2024-08-17T17:47:23.768 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-08-17T17:47:23.768 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_mtime_update_interval": "60", 2024-08-17T17:47:23.768 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_non_blocking_aio": "true", 2024-08-17T17:47:23.768 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_op_thread_timeout": "60", 2024-08-17T17:47:23.768 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_op_threads": "1", 2024-08-17T17:47:23.768 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_parent_cache_enabled": "false", 2024-08-17T17:47:23.768 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_persistent_cache_mode": "disabled", 2024-08-17T17:47:23.768 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_persistent_cache_path": "/tmp", 2024-08-17T17:47:23.768 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_persistent_cache_size": "1073741824", 2024-08-17T17:47:23.768 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_plugins": "", 2024-08-17T17:47:23.768 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_bps_burst": "0", 2024-08-17T17:47:23.768 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-08-17T17:47:23.769 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_bps_limit": "0", 2024-08-17T17:47:23.769 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_exclude_ops": "0", 2024-08-17T17:47:23.769 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_iops_burst": "0", 2024-08-17T17:47:23.769 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-08-17T17:47:23.769 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_iops_limit": "0", 2024-08-17T17:47:23.769 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_read_bps_burst": "0", 2024-08-17T17:47:23.769 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-08-17T17:47:23.769 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_read_bps_limit": "0", 2024-08-17T17:47:23.769 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_read_iops_burst": "0", 2024-08-17T17:47:23.769 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-08-17T17:47:23.769 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_read_iops_limit": "0", 2024-08-17T17:47:23.769 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_schedule_tick_min": "50", 2024-08-17T17:47:23.769 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_write_bps_burst": "0", 2024-08-17T17:47:23.770 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-08-17T17:47:23.770 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_write_bps_limit": "0", 2024-08-17T17:47:23.770 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_write_iops_burst": "0", 2024-08-17T17:47:23.770 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-08-17T17:47:23.770 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_qos_write_iops_limit": "0", 2024-08-17T17:47:23.770 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_quiesce_notification_attempts": "10", 2024-08-17T17:47:23.770 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_read_from_replica_policy": "default", 2024-08-17T17:47:23.770 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-08-17T17:47:23.770 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_readahead_max_bytes": "524288", 2024-08-17T17:47:23.770 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_readahead_trigger_requests": "10", 2024-08-17T17:47:23.770 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_request_timed_out_seconds": "30", 2024-08-17T17:47:23.770 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_skip_partial_discard": "true", 2024-08-17T17:47:23.770 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-08-17T17:47:23.771 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_tracing": "false", 2024-08-17T17:47:23.771 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_validate_names": "true", 2024-08-17T17:47:23.771 INFO:teuthology.orchestra.run.smithi130.stdout: "rbd_validate_pool": "true", 2024-08-17T17:47:23.771 INFO:teuthology.orchestra.run.smithi130.stdout: "restapi_base_url": "", 2024-08-17T17:47:23.771 INFO:teuthology.orchestra.run.smithi130.stdout: "restapi_log_level": "", 2024-08-17T17:47:23.771 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_account_default_quota_max_objects": "-1", 2024-08-17T17:47:23.771 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_account_default_quota_max_size": "-1", 2024-08-17T17:47:23.771 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_acl_grants_max_num": "100", 2024-08-17T17:47:23.771 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_admin_entry": "admin", 2024-08-17T17:47:23.771 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-08-17T17:47:23.771 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_asio_assert_yielding": "false", 2024-08-17T17:47:23.771 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_backend_store": "rados", 2024-08-17T17:47:23.771 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_barbican_url": "", 2024-08-17T17:47:23.772 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_beast_enable_async": "true", 2024-08-17T17:47:23.772 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_bucket_counters_cache": "false", 2024-08-17T17:47:23.772 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-08-17T17:47:23.772 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-08-17T17:47:23.772 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-08-17T17:47:23.772 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_bucket_index_max_aio": "128", 2024-08-17T17:47:23.772 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-08-17T17:47:23.772 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-08-17T17:47:23.772 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_bucket_quota_ttl": "600", 2024-08-17T17:47:23.772 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-08-17T17:47:23.772 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_cache_enabled": "true", 2024-08-17T17:47:23.772 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_cache_expiry_interval": "900", 2024-08-17T17:47:23.772 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_cache_lru_size": "25000", 2024-08-17T17:47:23.773 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_config_store": "rados", 2024-08-17T17:47:23.773 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_content_length_compat": "false", 2024-08-17T17:47:23.773 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_copy_obj_progress": "true", 2024-08-17T17:47:23.773 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-08-17T17:47:23.773 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_cors_rules_max_num": "100", 2024-08-17T17:47:23.773 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_cross_domain_policy": "", 2024-08-17T17:47:23.773 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_default_encryption_key": "", 2024-08-17T17:47:23.773 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_kmip_addr": "", 2024-08-17T17:47:23.773 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_kmip_ca_path": "", 2024-08-17T17:47:23.773 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_kmip_client_cert": "", 2024-08-17T17:47:23.773 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_kmip_client_key": "", 2024-08-17T17:47:23.773 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-08-17T17:47:23.773 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_kmip_password": "", 2024-08-17T17:47:23.774 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-08-17T17:47:23.774 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_kmip_username": "", 2024-08-17T17:47:23.774 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_require_ssl": "true", 2024-08-17T17:47:23.774 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-08-17T17:47:23.774 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-08-17T17:47:23.774 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-08-17T17:47:23.774 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-08-17T17:47:23.774 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-08-17T17:47:23.774 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-08-17T17:47:23.774 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-08-17T17:47:23.774 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-08-17T17:47:23.774 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-08-17T17:47:23.775 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-08-17T17:47:23.775 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-08-17T17:47:23.775 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-08-17T17:47:23.775 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-08-17T17:47:23.775 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-08-17T17:47:23.775 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_suppress_logs": "true", 2024-08-17T17:47:23.775 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_vault_addr": "", 2024-08-17T17:47:23.775 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_vault_auth": "token", 2024-08-17T17:47:23.775 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_vault_namespace": "", 2024-08-17T17:47:23.775 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_vault_prefix": "", 2024-08-17T17:47:23.775 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-08-17T17:47:23.775 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-08-17T17:47:23.776 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-08-17T17:47:23.776 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-08-17T17:47:23.776 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_vault_token_file": "", 2024-08-17T17:47:23.776 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-08-17T17:47:23.776 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_curl_buffersize": "524288", 2024-08-17T17:47:23.776 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_curl_low_speed_limit": "1024", 2024-08-17T17:47:23.776 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_curl_low_speed_time": "300", 2024-08-17T17:47:23.776 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_curl_tcp_keepalive": "0", 2024-08-17T17:47:23.776 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-08-17T17:47:23.776 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-08-17T17:47:23.776 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-08-17T17:47:23.776 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-08-17T17:47:23.776 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-08-17T17:47:23.776 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d3n_l1_fadvise": "4", 2024-08-17T17:47:23.777 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-08-17T17:47:23.777 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-08-17T17:47:23.777 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-08-17T17:47:23.777 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d4n_address": "127.0.0.1:6379", 2024-08-17T17:47:23.777 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d4n_l1_datacache_address": "127.0.0.1:6379", 2024-08-17T17:47:23.777 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-08-17T17:47:23.777 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-08-17T17:47:23.777 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-08-17T17:47:23.777 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d4n_l1_fadvise": "4", 2024-08-17T17:47:23.777 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-08-17T17:47:23.777 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-08-17T17:47:23.777 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-08-17T17:47:23.778 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_data_log_changes_size": "1000", 2024-08-17T17:47:23.778 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_data_log_num_shards": "128", 2024-08-17T17:47:23.778 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-08-17T17:47:23.778 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_data_log_window": "30", 2024-08-17T17:47:23.778 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_data_notify_interval_msec": "0", 2024-08-17T17:47:23.778 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_data_sync_poll_interval": "20", 2024-08-17T17:47:23.778 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_data_sync_spawn_window": "20", 2024-08-17T17:47:23.778 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-08-17T17:47:23.778 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-08-17T17:47:23.778 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-08-17T17:47:23.778 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_default_data_log_backing": "fifo", 2024-08-17T17:47:23.778 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-08-17T17:47:23.779 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_default_region_info_oid": "default.region", 2024-08-17T17:47:23.779 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-08-17T17:47:23.779 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-08-17T17:47:23.779 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_defer_to_bucket_acls": "", 2024-08-17T17:47:23.779 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-08-17T17:47:23.779 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_disable_s3select": "false", 2024-08-17T17:47:23.779 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-08-17T17:47:23.779 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-08-17T17:47:23.779 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-08-17T17:47:23.779 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-08-17T17:47:23.779 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-08-17T17:47:23.779 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-08-17T17:47:23.779 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-08-17T17:47:23.780 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dmclock_data_res": "500.000000", 2024-08-17T17:47:23.780 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-08-17T17:47:23.780 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-08-17T17:47:23.780 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-08-17T17:47:23.780 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-08-17T17:47:23.780 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dns_name": "", 2024-08-17T17:47:23.780 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dns_s3website_name": "", 2024-08-17T17:47:23.780 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dynamic_resharding": "true", 2024-08-17T17:47:23.780 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dynamic_resharding_may_reduce": "true", 2024-08-17T17:47:23.780 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_dynamic_resharding_reduction_wait": "120", 2024-08-17T17:47:23.780 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-08-17T17:47:23.780 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_enable_gc_threads": "true", 2024-08-17T17:47:23.780 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_enable_lc_threads": "true", 2024-08-17T17:47:23.781 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_enable_mdsearch": "true", 2024-08-17T17:47:23.781 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_enable_ops_log": "false", 2024-08-17T17:47:23.781 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_enable_quota_threads": "true", 2024-08-17T17:47:23.781 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_enable_static_website": "false", 2024-08-17T17:47:23.781 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_enable_usage_log": "false", 2024-08-17T17:47:23.781 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_enforce_swift_acls": "true", 2024-08-17T17:47:23.781 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_exit_timeout_secs": "120", 2024-08-17T17:47:23.781 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_expose_bucket": "false", 2024-08-17T17:47:23.781 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_extended_http_attrs": "", 2024-08-17T17:47:23.781 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_filter": "none", 2024-08-17T17:47:23.781 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-08-17T17:47:23.781 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_frontends": "beast port=7480", 2024-08-17T17:47:23.781 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_gc_max_concurrent_io": "10", 2024-08-17T17:47:23.782 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_gc_max_deferred": "50", 2024-08-17T17:47:23.782 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-08-17T17:47:23.782 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_gc_max_objs": "32", 2024-08-17T17:47:23.782 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_gc_max_queue_size": "134213632", 2024-08-17T17:47:23.782 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_gc_max_trim_chunk": "16", 2024-08-17T17:47:23.782 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_gc_obj_min_wait": "7200", 2024-08-17T17:47:23.782 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_gc_processor_max_time": "3600", 2024-08-17T17:47:23.782 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_gc_processor_period": "3600", 2024-08-17T17:47:23.782 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-08-17T17:47:23.782 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_get_obj_window_size": "16777216", 2024-08-17T17:47:23.782 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_healthcheck_disabling_path": "", 2024-08-17T17:47:23.782 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_ignore_get_invalid_range": "false", 2024-08-17T17:47:23.782 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_init_timeout": "300", 2024-08-17T17:47:23.783 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-08-17T17:47:23.783 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-08-17T17:47:23.783 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_kafka_connection_idle": "300", 2024-08-17T17:47:23.783 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_kafka_message_timeout": "5000", 2024-08-17T17:47:23.783 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_kafka_sleep_timeout": "10", 2024-08-17T17:47:23.783 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-08-17T17:47:23.783 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-08-17T17:47:23.783 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-08-17T17:47:23.783 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_admin_domain": "", 2024-08-17T17:47:23.783 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_admin_password": "", 2024-08-17T17:47:23.783 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_admin_password_path": "", 2024-08-17T17:47:23.783 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_admin_project": "", 2024-08-17T17:47:23.783 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_admin_tenant": "", 2024-08-17T17:47:23.783 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_admin_token": "", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_admin_token_path": "", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_admin_user": "", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_api_version": "2", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_barbican_domain": "", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_barbican_password": "", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_barbican_project": "", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_barbican_tenant": "", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_barbican_user": "", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_implicit_tenants": "false", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_service_token_enabled": "false", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_token_cache_size": "10000", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_url": "", 2024-08-17T17:47:23.784 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_keystone_verify_ssl": "true", 2024-08-17T17:47:23.785 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_lc_debug_interval": "-1", 2024-08-17T17:47:23.785 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_lc_lock_max_time": "90", 2024-08-17T17:47:23.785 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_lc_max_objs": "32", 2024-08-17T17:47:23.785 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_lc_max_rules": "1000", 2024-08-17T17:47:23.785 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_lc_max_worker": "3", 2024-08-17T17:47:23.785 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_lc_max_wp_worker": "3", 2024-08-17T17:47:23.785 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_lc_thread_delay": "0", 2024-08-17T17:47:23.785 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-08-17T17:47:23.785 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_ldap_dnattr": "uid", 2024-08-17T17:47:23.785 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-08-17T17:47:23.785 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_ldap_searchfilter": "", 2024-08-17T17:47:23.785 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-08-17T17:47:23.785 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_ldap_uri": "", 2024-08-17T17:47:23.786 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_lfuda_sync_frequency": "60", 2024-08-17T17:47:23.786 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-08-17T17:47:23.786 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-08-17T17:47:23.786 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-08-17T17:47:23.786 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_log_http_headers": "", 2024-08-17T17:47:23.786 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_log_nonexistent_bucket": "false", 2024-08-17T17:47:23.786 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-08-17T17:47:23.786 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_log_object_name_utc": "false", 2024-08-17T17:47:23.786 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-08-17T17:47:23.786 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-08-17T17:47:23.786 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_max_attr_name_len": "0", 2024-08-17T17:47:23.786 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_max_attr_size": "0", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_max_attrs_num_in_req": "0", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_max_chunk_size": "4194304", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_max_concurrent_requests": "1024", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_max_dynamic_shards": "1999", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_max_listing_results": "1000", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_max_notify_retries": "10", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_max_objs_per_shard": "100000", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_max_put_param_size": "1048576", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_max_put_size": "5368709120", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_max_slo_entries": "1000", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_md_log_max_shards": "64", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_md_notify_interval_msec": "200", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_meta_sync_poll_interval": "20", 2024-08-17T17:47:23.787 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_meta_sync_spawn_window": "20", 2024-08-17T17:47:23.788 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-08-17T17:47:23.788 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_mp_lock_max_time": "600", 2024-08-17T17:47:23.788 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-08-17T17:47:23.788 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_multipart_min_part_size": "5242880", 2024-08-17T17:47:23.788 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-08-17T17:47:23.788 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-08-17T17:47:23.788 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_nfs_fhcache_size": "2017", 2024-08-17T17:47:23.788 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-08-17T17:47:23.788 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-08-17T17:47:23.788 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_nfs_lru_lanes": "5", 2024-08-17T17:47:23.788 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_nfs_max_gc": "300", 2024-08-17T17:47:23.788 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-08-17T17:47:23.788 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_nfs_run_gc_threads": "false", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_nfs_run_lc_threads": "false", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_nfs_run_quota_threads": "false", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_nfs_run_sync_thread": "false", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_num_async_rados_threads": "32", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_num_control_oids": "8", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_numa_node": "-1", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_obj_stripe_size": "4194304", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_objexp_chunk_size": "100", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_objexp_gc_interval": "600", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_objexp_hints_num_shards": "127", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-08-17T17:47:23.789 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-08-17T17:47:23.790 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_op_thread_timeout": "600", 2024-08-17T17:47:23.790 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_op_tracing": "false", 2024-08-17T17:47:23.790 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_opa_token": "", 2024-08-17T17:47:23.790 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_opa_url": "", 2024-08-17T17:47:23.790 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_opa_verify_ssl": "true", 2024-08-17T17:47:23.790 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-08-17T17:47:23.790 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-08-17T17:47:23.790 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_ops_log_rados": "false", 2024-08-17T17:47:23.790 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_ops_log_socket_path": "", 2024-08-17T17:47:23.790 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-08-17T17:47:23.790 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-08-17T17:47:23.790 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-08-17T17:47:23.790 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_period_push_interval": "2.000000", 2024-08-17T17:47:23.790 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_period_push_interval_max": "30.000000", 2024-08-17T17:47:23.791 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_period_root_pool": ".rgw.root", 2024-08-17T17:47:23.791 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-08-17T17:47:23.791 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-08-17T17:47:23.791 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_posix_cache_lanes": "3", 2024-08-17T17:47:23.791 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-08-17T17:47:23.791 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_posix_cache_max_buckets": "100", 2024-08-17T17:47:23.791 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_posix_cache_partitions": "3", 2024-08-17T17:47:23.791 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-08-17T17:47:23.791 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_print_continue": "true", 2024-08-17T17:47:23.791 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_print_prohibited_content_length": "false", 2024-08-17T17:47:23.791 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-08-17T17:47:23.791 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-08-17T17:47:23.791 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-08-17T17:47:23.792 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-08-17T17:47:23.792 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_rados_tracing": "false", 2024-08-17T17:47:23.792 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_realm": "", 2024-08-17T17:47:23.792 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_realm_id": "", 2024-08-17T17:47:23.792 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-08-17T17:47:23.792 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_region": "", 2024-08-17T17:47:23.792 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_region_root_pool": ".rgw.root", 2024-08-17T17:47:23.792 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_relaxed_region_enforcement": "false", 2024-08-17T17:47:23.792 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-08-17T17:47:23.792 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-08-17T17:47:23.792 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_request_uri": "", 2024-08-17T17:47:23.792 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_reshard_batch_size": "64", 2024-08-17T17:47:23.792 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-08-17T17:47:23.793 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_reshard_debug_interval": "-1", 2024-08-17T17:47:23.793 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_reshard_max_aio": "128", 2024-08-17T17:47:23.793 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_reshard_num_logs": "16", 2024-08-17T17:47:23.793 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_reshard_thread_interval": "600", 2024-08-17T17:47:23.793 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_resolve_cname": "false", 2024-08-17T17:47:23.793 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_rest_getusage_op_compat": "false", 2024-08-17T17:47:23.793 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_run_sync_thread": "true", 2024-08-17T17:47:23.793 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-08-17T17:47:23.793 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-08-17T17:47:23.793 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_s3_auth_use_keystone": "false", 2024-08-17T17:47:23.793 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_s3_auth_use_ldap": "false", 2024-08-17T17:47:23.793 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_s3_auth_use_rados": "true", 2024-08-17T17:47:23.793 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_s3_auth_use_sts": "false", 2024-08-17T17:47:23.794 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-08-17T17:47:23.794 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_s3_success_create_obj_status": "0", 2024-08-17T17:47:23.794 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-08-17T17:47:23.794 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_scheduler_type": "throttler", 2024-08-17T17:47:23.794 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_script_uri": "", 2024-08-17T17:47:23.794 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_service_provider_name": "", 2024-08-17T17:47:23.794 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-08-17T17:47:23.794 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sts_client_id": "", 2024-08-17T17:47:23.794 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sts_client_secret": "", 2024-08-17T17:47:23.794 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sts_entry": "sts", 2024-08-17T17:47:23.794 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sts_key": "sts", 2024-08-17T17:47:23.794 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sts_max_session_duration": "43200", 2024-08-17T17:47:23.794 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sts_min_session_duration": "900", 2024-08-17T17:47:23.794 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sts_token_introspection_url": "", 2024-08-17T17:47:23.795 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_swift_account_in_url": "false", 2024-08-17T17:47:23.795 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_swift_auth_entry": "auth", 2024-08-17T17:47:23.795 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_swift_auth_url": "", 2024-08-17T17:47:23.795 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_swift_custom_header": "", 2024-08-17T17:47:23.795 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_swift_enforce_content_length": "false", 2024-08-17T17:47:23.795 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_swift_need_stats": "true", 2024-08-17T17:47:23.795 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_swift_tenant_name": "", 2024-08-17T17:47:23.795 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_swift_token_expiration": "86400", 2024-08-17T17:47:23.795 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_swift_url": "", 2024-08-17T17:47:23.795 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_swift_url_prefix": "swift", 2024-08-17T17:47:23.795 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_swift_versioning_enabled": "false", 2024-08-17T17:47:23.795 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-08-17T17:47:23.795 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-08-17T17:47:23.795 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sync_lease_period": "120", 2024-08-17T17:47:23.796 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-08-17T17:47:23.796 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sync_log_trim_interval": "1200", 2024-08-17T17:47:23.796 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-08-17T17:47:23.796 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-08-17T17:47:23.796 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-08-17T17:47:23.796 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sync_obj_etag_verify": "false", 2024-08-17T17:47:23.796 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sync_trace_history_size": "4096", 2024-08-17T17:47:23.796 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-08-17T17:47:23.796 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-08-17T17:47:23.796 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_thread_pool_size": "512", 2024-08-17T17:47:23.796 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_topic_persistency_max_retries": "0", 2024-08-17T17:47:23.796 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-08-17T17:47:23.796 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-08-17T17:47:23.797 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_topic_require_publish_policy": "false", 2024-08-17T17:47:23.797 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_torrent_comment": "", 2024-08-17T17:47:23.797 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_torrent_createby": "", 2024-08-17T17:47:23.797 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_torrent_encoding": "", 2024-08-17T17:47:23.797 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_torrent_flag": "false", 2024-08-17T17:47:23.797 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_torrent_max_size": "5368709120", 2024-08-17T17:47:23.797 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_torrent_origin": "", 2024-08-17T17:47:23.797 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_torrent_sha_unit": "524288", 2024-08-17T17:47:23.797 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_torrent_tracker": "", 2024-08-17T17:47:23.797 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_trust_forwarded_https": "false", 2024-08-17T17:47:23.797 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-08-17T17:47:23.797 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_usage_log_tick_interval": "30", 2024-08-17T17:47:23.797 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_usage_max_shards": "32", 2024-08-17T17:47:23.797 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_usage_max_user_shards": "1", 2024-08-17T17:47:23.798 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_use_opa_authz": "false", 2024-08-17T17:47:23.798 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_user_counters_cache": "false", 2024-08-17T17:47:23.798 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_user_counters_cache_size": "10000", 2024-08-17T17:47:23.798 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-08-17T17:47:23.798 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_user_default_quota_max_size": "-1", 2024-08-17T17:47:23.798 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_user_max_buckets": "1000", 2024-08-17T17:47:23.798 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_user_policies_max_num": "100", 2024-08-17T17:47:23.798 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-08-17T17:47:23.798 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-08-17T17:47:23.798 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_user_quota_sync_interval": "86400", 2024-08-17T17:47:23.798 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-08-17T17:47:23.798 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_user_unique_email": "true", 2024-08-17T17:47:23.798 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_verify_ssl": "true", 2024-08-17T17:47:23.799 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_website_routing_rules_max_num": "50", 2024-08-17T17:47:23.799 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_zone": "", 2024-08-17T17:47:23.799 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_zone_id": "", 2024-08-17T17:47:23.799 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-08-17T17:47:23.799 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_zonegroup": "", 2024-08-17T17:47:23.799 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_zonegroup_id": "", 2024-08-17T17:47:23.799 INFO:teuthology.orchestra.run.smithi130.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-08-17T17:47:23.799 INFO:teuthology.orchestra.run.smithi130.stdout: "rgwlc_auto_session_clear": "true", 2024-08-17T17:47:23.799 INFO:teuthology.orchestra.run.smithi130.stdout: "rgwlc_skip_bucket_step": "false", 2024-08-17T17:47:23.799 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_block_size": "4096", 2024-08-17T17:47:23.799 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-08-17T17:47:23.799 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-08-17T17:47:23.799 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-08-17T17:47:23.799 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-08-17T17:47:23.800 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_cache_shard_bits": "4", 2024-08-17T17:47:23.800 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_cache_size": "536870912", 2024-08-17T17:47:23.800 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_cache_type": "binned_lru", 2024-08-17T17:47:23.800 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-08-17T17:47:23.800 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-08-17T17:47:23.800 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-08-17T17:47:23.800 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_collect_compaction_stats": "false", 2024-08-17T17:47:23.800 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_collect_extended_stats": "false", 2024-08-17T17:47:23.800 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_collect_memory_stats": "false", 2024-08-17T17:47:23.800 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-08-17T17:47:23.800 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_index_type": "binary_search", 2024-08-17T17:47:23.800 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_log_to_ceph_log": "true", 2024-08-17T17:47:23.800 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_metadata_block_size": "4096", 2024-08-17T17:47:23.801 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_partition_filters": "false", 2024-08-17T17:47:23.801 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_perf": "false", 2024-08-17T17:47:23.801 INFO:teuthology.orchestra.run.smithi130.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-08-17T17:47:23.801 INFO:teuthology.orchestra.run.smithi130.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-08-17T17:47:23.801 INFO:teuthology.orchestra.run.smithi130.stdout: "rotating_keys_renewal_timeout": "10", 2024-08-17T17:47:23.801 INFO:teuthology.orchestra.run.smithi130.stdout: "run_dir": "/var/run/ceph", 2024-08-17T17:47:23.801 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_block_create": "true", 2024-08-17T17:47:23.801 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_cache_lru_size": "67108864", 2024-08-17T17:47:23.801 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_cbjournal_size": "5368709120", 2024-08-17T17:47:23.801 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_data_delta_based_overwrite": "0", 2024-08-17T17:47:23.801 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_default_max_object_size": "16777216", 2024-08-17T17:47:23.801 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-08-17T17:47:23.801 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_device_size": "53687091200", 2024-08-17T17:47:23.802 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_disable_end_to_end_data_protection": "true", 2024-08-17T17:47:23.802 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_full_integrity_check": "false", 2024-08-17T17:47:23.802 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_journal_batch_capacity": "16", 2024-08-17T17:47:23.802 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-08-17T17:47:23.802 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-08-17T17:47:23.802 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_journal_iodepth_limit": "5", 2024-08-17T17:47:23.802 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_main_device_type": "SSD", 2024-08-17T17:47:23.802 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_max_concurrent_transactions": "8", 2024-08-17T17:47:23.802 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_max_data_allocation_size": "32768", 2024-08-17T17:47:23.802 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-08-17T17:47:23.802 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-08-17T17:47:23.802 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-08-17T17:47:23.802 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-08-17T17:47:23.803 INFO:teuthology.orchestra.run.smithi130.stdout: "seastore_segment_size": "67108864", 2024-08-17T17:47:23.803 INFO:teuthology.orchestra.run.smithi130.stdout: "set_keepcaps": "false", 2024-08-17T17:47:23.803 INFO:teuthology.orchestra.run.smithi130.stdout: "setgroup": "ceph", 2024-08-17T17:47:23.803 INFO:teuthology.orchestra.run.smithi130.stdout: "setuser": "ceph", 2024-08-17T17:47:23.803 INFO:teuthology.orchestra.run.smithi130.stdout: "setuser_match_path": "", 2024-08-17T17:47:23.803 INFO:teuthology.orchestra.run.smithi130.stdout: "target_max_misplaced_ratio": "0.050000", 2024-08-17T17:47:23.803 INFO:teuthology.orchestra.run.smithi130.stdout: "thp": "false", 2024-08-17T17:47:23.803 INFO:teuthology.orchestra.run.smithi130.stdout: "threadpool_default_timeout": "60", 2024-08-17T17:47:23.803 INFO:teuthology.orchestra.run.smithi130.stdout: "threadpool_empty_queue_max_wait": "2", 2024-08-17T17:47:23.803 INFO:teuthology.orchestra.run.smithi130.stdout: "throttler_perf_counter": "true", 2024-08-17T17:47:23.803 INFO:teuthology.orchestra.run.smithi130.stdout: "tmp_dir": "/tmp", 2024-08-17T17:47:23.803 INFO:teuthology.orchestra.run.smithi130.stdout: "tmp_file_template": "/tmp/ceph-mon.a.XXXXXX" 2024-08-17T17:47:23.803 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:47:24.661 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-08-17T17:47:24.915 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:25 smithi130 ceph-mon[105521]: pgmap v1547: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:25 smithi130 ceph-mon[110068]: pgmap v1547: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:25 smithi179 ceph-mon[93519]: pgmap v1547: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:26.614 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a: {} 2024-08-17T17:47:26.614 INFO:teuthology.orchestra.run.smithi130.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:47:26.620 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b: {} 2024-08-17T17:47:26.621 INFO:teuthology.orchestra.run.smithi130.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:47:26.629 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c: {} 2024-08-17T17:47:26.629 INFO:teuthology.orchestra.run.smithi130.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:47:27.275 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-08-17T17:47:27.275 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-08-17T17:47:27.276 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:47:27.278 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:47:27.278 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-08-17T17:47:27.278 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_0 16 2024-08-17T17:47:27.279 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-08-17T17:47:27.279 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:47:27.279 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:47:27.279 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:47:27.279 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-08-17T17:47:27.280 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.0 2024-08-17T17:47:27.280 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-08-17T17:47:27.280 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:27.280 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-08-17T17:47:27.281 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:47:27.281 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-08-17T17:47:27.281 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:47:27.282 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-08-17T17:47:27.282 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:47:27.309 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:27.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:27.411 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:27.437 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:27.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:27.538 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:27.563 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:27.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:27.665 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:27 smithi130 ceph-mon[105521]: pgmap v1548: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:27 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2932203669' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:47:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:27 smithi130 ceph-mon[110068]: pgmap v1548: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:27 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2932203669' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:47:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:27.667 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:47:27 smithi130 systemd[1]: Stopping Ceph osd.0 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:47:27.690 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:27.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:27.692 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:27.729 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:27.792 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:27.815 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:27.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:27 smithi179 ceph-mon[93519]: pgmap v1548: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:27 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2932203669' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:47:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:27.917 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:27.942 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:27.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:27.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:27.984 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:47:27 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[118233]: 2024-08-17T17:47:27.692+0000 7f6248232640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:47:27.984 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:47:27 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[118233]: 2024-08-17T17:47:27.692+0000 7f6248232640 -1 osd.0 1163 *** Got signal Terminated *** 2024-08-17T17:47:27.984 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:47:27 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[118233]: 2024-08-17T17:47:27.692+0000 7f6248232640 -1 osd.0 1163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:47:28.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:28.065 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:28.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:28.167 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:28.192 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:28.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:28.294 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:28.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.320 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:28.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:28.421 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:28.447 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:28.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:28.550 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:28.571 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:28.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:28.576 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:28 smithi130 ceph-mon[105521]: osd.0 marked itself down and dead 2024-08-17T17:47:28.577 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:28 smithi130 ceph-mon[110068]: osd.0 marked itself down and dead 2024-08-17T17:47:28.672 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:28.698 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:28.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:28.800 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:28.825 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:28.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:28.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:28 smithi179 ceph-mon[93519]: osd.0 marked itself down and dead 2024-08-17T17:47:28.927 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:28.953 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:28.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:28.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:29.054 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:29.079 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:29.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:29.156 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:47:28 smithi130 podman[169377]: 2024-08-17 17:47:28.840290449 +0000 UTC m=+1.278266963 container died 26704f9d719b0d5a8ee12bc7e5da5b61e42a1935ff27ad52ef4dcff342137b17 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-08-17T17:47:29.181 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:29.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.203 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:29.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:29.270 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:29.304 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:29.327 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:29.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:29.416 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:47:29 smithi130 podman[169377]: 2024-08-17 17:47:29.25435353 +0000 UTC m=+1.692330043 container cleanup 26704f9d719b0d5a8ee12bc7e5da5b61e42a1935ff27ad52ef4dcff342137b17 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True) 2024-08-17T17:47:29.416 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:47:29 smithi130 bash[169377]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0 2024-08-17T17:47:29.429 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:29.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.455 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:29.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:29.556 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:29.582 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:29.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:29.683 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:29.706 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:29.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:29.767 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:29 smithi130 ceph-mon[105521]: pgmap v1549: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:29.767 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:29 smithi130 ceph-mon[105521]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:47:29.767 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:29 smithi130 ceph-mon[105521]: osdmap e1164: 8 total, 7 up, 8 in 2024-08-17T17:47:29.767 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:29 smithi130 ceph-mon[110068]: pgmap v1549: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:29.767 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:29 smithi130 ceph-mon[110068]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:47:29.767 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:29 smithi130 ceph-mon[110068]: osdmap e1164: 8 total, 7 up, 8 in 2024-08-17T17:47:29.807 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:29.828 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:29.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:29.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:29 smithi179 ceph-mon[93519]: pgmap v1549: 73 pgs: 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:29 smithi179 ceph-mon[93519]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:47:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:29 smithi179 ceph-mon[93519]: osdmap e1164: 8 total, 7 up, 8 in 2024-08-17T17:47:29.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:29.955 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:29.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:29.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:30.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:30.078 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:30.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:47:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:47:29] "GET /metrics HTTP/1.1" 200 34144 "" "Prometheus/2.51.0" 2024-08-17T17:47:30.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:47:29 smithi130 podman[169624]: 2024-08-17 17:47:29.766526868 +0000 UTC m=+1.005243985 container remove 26704f9d719b0d5a8ee12bc7e5da5b61e42a1935ff27ad52ef4dcff342137b17 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, ceph=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:47:30.180 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:30.206 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:30.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:30.308 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:30.329 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:30.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:30.431 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:30.458 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:30.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:30.560 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:30.581 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:30.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:30 smithi179 ceph-mon[93519]: osdmap e1165: 8 total, 7 up, 8 in 2024-08-17T17:47:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:47:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:47:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:47:30.659 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:30.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:30 smithi130 ceph-mon[105521]: osdmap e1165: 8 total, 7 up, 8 in 2024-08-17T17:47:30.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:30 smithi130 ceph-mon[110068]: osdmap e1165: 8 total, 7 up, 8 in 2024-08-17T17:47:30.683 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:30.709 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:30.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:30.810 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:30.832 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:30.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:30.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:30.954 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:30.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:30.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:31.056 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:31.077 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:31.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:31.179 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:31.201 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:31.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:31.302 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:31.323 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:31.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:31.425 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:31.451 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:31.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:31.553 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:31.579 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:31.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:31.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:31 smithi130 ceph-mon[105521]: pgmap v1552: 73 pgs: 1 peering, 11 stale+active+clean, 61 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:31 smithi130 ceph-mon[105521]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:47:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:31 smithi130 ceph-mon[110068]: pgmap v1552: 73 pgs: 1 peering, 11 stale+active+clean, 61 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:31 smithi130 ceph-mon[110068]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:47:31.681 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:31.706 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:31.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:31.808 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:31.831 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:31.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:31.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:31 smithi179 ceph-mon[93519]: pgmap v1552: 73 pgs: 1 peering, 11 stale+active+clean, 61 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-08-17T17:47:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:31 smithi179 ceph-mon[93519]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:47:31.932 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:31.953 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:31.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:31.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:32.055 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:32.076 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:32.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:32.177 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:32.203 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:32.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:32.304 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:32.329 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:32.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:32.431 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:32.456 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:32.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:32.558 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:32.581 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:32.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:32.682 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:32.708 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:32.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:32.774 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:32 smithi130 ceph-mon[105521]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:47:32.774 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:32 smithi130 ceph-mon[110068]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:47:32.810 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:32.832 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:32.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:32.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:32 smithi179 ceph-mon[93519]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:47:32.934 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:32.959 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:32.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:32.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:33.061 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:33.086 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:33.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:33.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:33.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.213 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:33.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:33.314 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:33.338 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:33.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:33.412 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:47:33.412 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-08-17T17:47:33.412 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:47:33.428 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:47:33.428 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:47:33.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:33.444 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:47:33.444 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:47:33.460 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:47:33.460 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:47:33.462 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:33.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:33.476 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:47:33.476 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:47:33.491 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:47:33.492 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:47:33.508 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:47:33.508 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:47:33.516 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:47:33.564 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:33.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:33.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:33 smithi130 ceph-mon[105521]: pgmap v1553: 73 pgs: 3 active+undersized, 1 peering, 9 stale+active+clean, 1 active+undersized+degraded, 59 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-08-17T17:47:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:33 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/199038983' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:47:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:33 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:47:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:33 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2518879424' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:33 smithi130 ceph-mon[110068]: pgmap v1553: 73 pgs: 3 active+undersized, 1 peering, 9 stale+active+clean, 1 active+undersized+degraded, 59 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-08-17T17:47:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:33 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/199038983' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:47:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:33 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:47:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:33 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2518879424' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:33.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:33.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:33.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:33.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:33.836 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:33.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:33 smithi179 ceph-mon[93519]: pgmap v1553: 73 pgs: 3 active+undersized, 1 peering, 9 stale+active+clean, 1 active+undersized+degraded, 59 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1/78 objects degraded (1.282%) 2024-08-17T17:47:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:33 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/199038983' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:47:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:33 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:47:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:33 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2518879424' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:33.937 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:33.959 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:33.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:33.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:34.061 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:34.085 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:34.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:34.187 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:34.210 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:34.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:34.312 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:34.334 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:34.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:34.436 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:34.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:34.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:34.563 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:34.585 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:34.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:34.687 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:34.708 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:34.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:34.810 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:34.831 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:34.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:34 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:34 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:47:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:34 smithi179 ceph-mon[93519]: osdmap e1166: 8 total, 7 up, 8 in 2024-08-17T17:47:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:34 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:34 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:47:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:34 smithi130 ceph-mon[110068]: osdmap e1166: 8 total, 7 up, 8 in 2024-08-17T17:47:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:34 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:34 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:47:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:34 smithi130 ceph-mon[105521]: osdmap e1166: 8 total, 7 up, 8 in 2024-08-17T17:47:34.932 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:34.953 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:34.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:34.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:35.056 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:35.081 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:35.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:35.183 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:35.209 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:35.310 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:35.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:35.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:35.436 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:35.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.459 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:35.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:35.473 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:47:35.474 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:47:35.541 INFO:teuthology.orchestra.run.smithi130.stderr:pool 'unique_pool_0' created 2024-08-17T17:47:35.561 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:35.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:35.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:35.687 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:35.708 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:35.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:35.712 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:35 smithi130 ceph-mon[105521]: pgmap v1555: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:35.712 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:35 smithi130 ceph-mon[105521]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:47:35.713 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:35 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3424225999' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-17T17:47:35.713 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:35 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-17T17:47:35.713 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:35 smithi130 ceph-mon[110068]: pgmap v1555: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:35.713 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:35 smithi130 ceph-mon[110068]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:47:35.713 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:35 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3424225999' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-17T17:47:35.713 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:35 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-17T17:47:35.726 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:47:35.727 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:47:35.810 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:35.835 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:35.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:35.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:35 smithi179 ceph-mon[93519]: pgmap v1555: 73 pgs: 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:35 smithi179 ceph-mon[93519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-08-17T17:47:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:35 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3424225999' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-17T17:47:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:35 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-08-17T17:47:35.936 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:35.958 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:35.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:35.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:36.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:36.082 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:36.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:36.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:36.205 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:36.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:36.307 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:36.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.333 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:36.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:36.402 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:47:36.435 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:36.460 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:36.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:36.562 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:36.586 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:36.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:36 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-17T17:47:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:36 smithi130 ceph-mon[105521]: osdmap e1167: 8 total, 7 up, 8 in 2024-08-17T17:47:36.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:36 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-17T17:47:36.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:36 smithi130 ceph-mon[110068]: osdmap e1167: 8 total, 7 up, 8 in 2024-08-17T17:47:36.687 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:36.709 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:36.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:36.811 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:36.836 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:36.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:36.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:36 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-17T17:47:36.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:36 smithi179 ceph-mon[93519]: osdmap e1167: 8 total, 7 up, 8 in 2024-08-17T17:47:36.938 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:36.964 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:36.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:36.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:36.969 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:47:36 smithi130 podman[170618]: 2024-08-17 17:47:36.61628453 +0000 UTC m=+1.551712883 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:47:36.969 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:47:36 smithi130 podman[170618]: 2024-08-17 17:47:36.821816048 +0000 UTC m=+1.757244386 container create e4ffb795bfae6b58887ffb52a36fda283e6a672d8789de0876db6e44f5fc72c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True) 2024-08-17T17:47:37.030 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:47:37.066 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:37.091 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:37.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:37.193 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:37.215 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:37.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:37.316 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:37.338 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:37.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:37.423 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-08-17T17:47:37.440 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:37.466 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:37.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:37.474 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:47:37.568 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:37.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.591 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:37.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:37 smithi130 ceph-mon[105521]: pgmap v1557: 89 pgs: 16 unknown, 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:37 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:47:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:37 smithi130 ceph-mon[105521]: osdmap e1168: 8 total, 7 up, 8 in 2024-08-17T17:47:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:37 smithi130 ceph-mon[110068]: pgmap v1557: 89 pgs: 16 unknown, 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:37 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:47:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:37 smithi130 ceph-mon[110068]: osdmap e1168: 8 total, 7 up, 8 in 2024-08-17T17:47:37.667 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:47:37 smithi130 podman[170618]: 2024-08-17 17:47:37.527334255 +0000 UTC m=+2.462762618 container init e4ffb795bfae6b58887ffb52a36fda283e6a672d8789de0876db6e44f5fc72c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:47:37.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:47:37 smithi130 podman[170618]: 2024-08-17 17:47:37.533712463 +0000 UTC m=+2.469140807 container start e4ffb795bfae6b58887ffb52a36fda283e6a672d8789de0876db6e44f5fc72c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, ceph=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-08-17T17:47:37.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:47:37 smithi130 podman[170618]: 2024-08-17 17:47:37.635504738 +0000 UTC m=+2.570933075 container attach e4ffb795bfae6b58887ffb52a36fda283e6a672d8789de0876db6e44f5fc72c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, ceph=True, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:47:37.692 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:37.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.719 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:37.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:37.745 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:37.820 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:37.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:37 smithi179 ceph-mon[93519]: pgmap v1557: 89 pgs: 16 unknown, 19 active+undersized, 7 active+undersized+degraded, 47 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:37 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:47:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:37 smithi179 ceph-mon[93519]: osdmap e1168: 8 total, 7 up, 8 in 2024-08-17T17:47:37.845 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:37.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:37.947 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:37.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:37.971 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:37.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:38.072 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:38.090 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:47:37 smithi130 podman[170618]: 2024-08-17 17:47:37.779014398 +0000 UTC m=+2.714442736 container died e4ffb795bfae6b58887ffb52a36fda283e6a672d8789de0876db6e44f5fc72c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:47:38.097 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:38.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:38.199 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:38.223 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:38.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:38.325 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:38.349 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:38.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:38.370 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.0.service' 2024-08-17T17:47:38.414 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:38.414 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:38.415 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:38.420 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:47:38.420 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-08-17T17:47:38.451 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:38.475 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:38.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:38.577 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:38.599 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:38.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:38.614 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:38 smithi130 ceph-mon[105521]: osdmap e1169: 8 total, 7 up, 8 in 2024-08-17T17:47:38.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:38 smithi130 ceph-mon[110068]: osdmap e1169: 8 total, 7 up, 8 in 2024-08-17T17:47:38.701 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:38.727 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:38.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:38.829 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:38.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:38 smithi179 ceph-mon[93519]: osdmap e1169: 8 total, 7 up, 8 in 2024-08-17T17:47:38.855 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:38.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:38.957 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:38.983 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:38.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:38.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:39.085 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:39.110 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:39.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:39.211 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:39.237 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:39.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:39.338 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:39.363 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:39.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:39.465 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:39.490 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:39.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:39.592 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:39.617 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:39.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:39.719 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:39.744 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:39.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:39.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:39 smithi130 ceph-mon[105521]: pgmap v1560: 89 pgs: 2 unknown, 24 active+undersized, 7 active+undersized+degraded, 56 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:39.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:39 smithi130 ceph-mon[110068]: pgmap v1560: 89 pgs: 2 unknown, 24 active+undersized, 7 active+undersized+degraded, 56 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:39.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:39 smithi179 ceph-mon[93519]: pgmap v1560: 89 pgs: 2 unknown, 24 active+undersized, 7 active+undersized+degraded, 56 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:39.846 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:39.870 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:39.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:39.972 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:39.997 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:39.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:39.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:40.099 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:40.124 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:40.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:40.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:47:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:47:39] "GET /metrics HTTP/1.1" 200 34144 "" "Prometheus/2.51.0" 2024-08-17T17:47:40.226 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:40.251 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:40.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:40.353 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:40.376 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:40.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:40.478 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:40.501 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:40.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:47:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:47:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:47:40.603 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:40.626 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:40.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:40.728 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:40.754 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:40.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:40.855 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:40.879 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:40.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:40.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:40.981 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:41.006 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:41.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:41.108 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:41.133 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:41.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:41.234 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:41.260 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:41.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:41.362 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:41.385 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:41.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:41.487 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:41.510 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:41.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:41.612 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:41.638 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:41.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:41.739 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:41.764 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:41.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:41.831 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:41 smithi130 ceph-mon[105521]: pgmap v1561: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:41.832 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:41 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 12/78 objects degraded (15.385%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:47:41.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:41 smithi130 ceph-mon[110068]: pgmap v1561: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:41.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:41 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 12/78 objects degraded (15.385%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:47:41.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:41 smithi179 ceph-mon[93519]: pgmap v1561: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:41.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:41 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 12/78 objects degraded (15.385%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:47:41.866 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:41.888 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:41.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:41.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:41.989 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:42.011 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:42.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:42.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:42.138 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:42.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:42.240 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:42.261 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:42.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:42.363 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:42.389 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:42.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:42.490 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:42.512 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:42.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:42.614 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:42.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.635 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:42.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:42.736 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:42.757 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:42.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:42.859 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:42.884 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:42.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:42.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:42.894 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:47:42.894 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-08-17T17:47:42.894 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:47:42.901 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:47:42.902 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:47:42.917 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:47:42.918 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:47:42.934 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:47:42.934 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:47:42.949 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:47:42.949 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:47:42.965 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:47:42.965 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:47:42.981 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:47:42.982 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:47:42.986 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:43.010 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:43.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:43.112 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:43.134 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:43.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:43.235 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:43.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.261 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:43.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:43.362 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:43.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.384 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:43.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:43.421 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:47:43.421 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:47:43.422 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-08-17T17:47:43.422 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:47:43.422 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:47:43.422 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-08-17T17:47:43.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:43.507 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:43.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:43.540 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:43.540 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:43.540 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:43 smithi130 ceph-mon[105521]: pgmap v1562: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:43.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:43 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1794133156' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:43.541 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:43.541 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:43.541 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:43 smithi130 ceph-mon[110068]: pgmap v1562: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:43.541 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:43 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1794133156' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:43 smithi179 ceph-mon[93519]: pgmap v1562: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:43 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1794133156' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:43.610 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:43.637 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:43.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:43.739 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:43.761 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:43.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:43.863 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:43.885 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:43.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:43.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:43.987 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:44.009 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:44.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:44.111 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:44.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.133 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:44.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:44.234 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:44.256 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:44.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:44.358 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:44.380 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:44.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:44.481 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:44.502 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:44.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:44.604 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:44.626 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:44.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:44.728 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:44.749 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:44.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:44.850 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:44.873 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:44.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:44.891 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:47:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:47:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T17:47:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T17:47:44.891 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:47:44.975 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:44.997 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:44.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:44.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:45.098 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:45.119 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:45.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:45.221 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:45.243 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:45.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:45.345 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:45.365 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:45.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:45.444 INFO:teuthology.orchestra.run.smithi130.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-08-17T17:47:45.445 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T17:47:45.467 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:45.488 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:45.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:45.589 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:45.610 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:45.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:45.637 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:45 smithi130 ceph-mon[105521]: pgmap v1563: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:45.637 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:45 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3816869739' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:45.637 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:45 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3834633024' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:47:45.638 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:45 smithi130 ceph-mon[110068]: pgmap v1563: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:45.638 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:45 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3816869739' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:45.638 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:45 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3834633024' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:47:45.660 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:47:45.660 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:47:45.712 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:45.733 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:45.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:45.834 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:45.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:45 smithi179 ceph-mon[93519]: pgmap v1563: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:45.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:45 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3816869739' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:47:45.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:45 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3834633024' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:47:45.856 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:45.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:45.957 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:45.979 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:45.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:45.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:46.081 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:46.103 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:46.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:46.204 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:46.226 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:46.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:46.328 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:46.350 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:46.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:46.451 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:46.473 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:46.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:46.517 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:46 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:46.517 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:46 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3816869739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:47:46.517 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:46 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3834633024' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:47:46.518 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:46 smithi130 ceph-mon[105521]: osdmap e1170: 8 total, 7 up, 8 in 2024-08-17T17:47:46.518 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:46 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:46.518 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:46 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3816869739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:47:46.518 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:46 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3834633024' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:47:46.518 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:46 smithi130 ceph-mon[110068]: osdmap e1170: 8 total, 7 up, 8 in 2024-08-17T17:47:46.541 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:47:46.549 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:46.575 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:46.597 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:46.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:46.699 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:46.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:46.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:46.823 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:46.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:46 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:47:46.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:46 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3816869739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:47:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:46 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3834633024' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:47:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:46 smithi179 ceph-mon[93519]: osdmap e1170: 8 total, 7 up, 8 in 2024-08-17T17:47:46.846 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:46.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:46.948 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:46.969 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:46.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:46.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:47.071 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:47.095 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:47.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:47.197 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:47.220 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:47.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:47.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:47.344 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:47.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:47.446 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:47.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:47.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:47.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:47.595 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:47.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:47.633 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:47 smithi130 ceph-mon[105521]: pgmap v1565: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:47.633 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:47 smithi130 ceph-mon[105521]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:47:47.634 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:47 smithi130 ceph-mon[110068]: pgmap v1565: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:47.634 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:47 smithi130 ceph-mon[110068]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:47:47.661 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:47:47.697 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:47.719 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:47.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:47.820 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:47.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:47 smithi179 ceph-mon[93519]: pgmap v1565: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:47 smithi179 ceph-mon[93519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:47:47.846 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:47.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:47.947 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:47.972 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:47.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:47.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:48.074 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:48.090 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:48.099 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:48.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:48.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:48.223 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:48.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:48.324 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:48.350 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:48.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:48.451 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:48.476 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:48.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:48.578 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:48.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:48.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:48.658 DEBUG:teuthology.orchestra.run.smithi130:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-08-17T17:47:48.661 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:47:48.661 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:47:48.704 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:48.706 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:hints = 1 2024-08-17T17:47:48.706 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:47:48.706 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Object prefix: benchmark_data_smithi130_172551 2024-08-17T17:47:48.707 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:47:48.707 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:47:48.743 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:48.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:48.791 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:48.845 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:48.876 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:48.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:48.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:48.978 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:49.010 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:49.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:49.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:49.146 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:49.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:49.231 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:47:49.248 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:49.315 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:49.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:49.417 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:49.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.458 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:49.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:49.559 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:49.590 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:49.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:49 smithi130 ceph-mon[105521]: pgmap v1566: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:49 smithi130 ceph-mon[110068]: pgmap v1566: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:49.691 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:49.706 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 1 16 4328 4312 269.484 269.5 0.00246189 0.00370125 2024-08-17T17:47:49.720 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:49.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:49.822 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:49.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:49 smithi179 ceph-mon[93519]: pgmap v1566: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:49.859 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:49.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:49.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:49.961 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:50.000 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:50.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:50.101 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:50.111 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:47:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:47:49] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.51.0" 2024-08-17T17:47:50.133 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:50.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:50.235 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:50.269 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:50.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:50.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:50.404 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:50.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:50.506 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:50.544 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:50.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:50.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:47:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:47:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:47:50.646 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:50.678 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:50.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:50.706 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 2 16 8337 8321 260.012 250.562 0.00359954 0.00383711 2024-08-17T17:47:50.780 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:50.811 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:50.913 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:50.947 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:50.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:50.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:51.049 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:51.050 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:47:51.050 INFO:teuthology.orchestra.run.smithi130.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:47:51.050 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:47:51.086 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:51.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:51.188 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:51.212 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:51.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:51.314 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:51.347 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:51.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:51.449 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:51.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.481 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:51.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:51.582 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:51.617 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:51.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:51.706 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 3 16 12293 12277 255.75 247.25 0.00233407 0.00390468 2024-08-17T17:47:51.725 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:51.748 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:51 smithi130 ceph-mon[110068]: pgmap v1567: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:51.749 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:51 smithi130 ceph-mon[105521]: pgmap v1567: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:51.761 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:51.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:51.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:51 smithi179 ceph-mon[93519]: pgmap v1567: 89 pgs: 25 active+undersized, 7 active+undersized+degraded, 57 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12/78 objects degraded (15.385%) 2024-08-17T17:47:51.862 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:51.892 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:51.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:51.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:51.994 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:52.029 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:52.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:52.131 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:52.167 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:52.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:52.268 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:52.305 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:52.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:52.407 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:52.441 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:52.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:52.543 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:52.577 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:52.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:52.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:52 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 783/4683 objects degraded (16.720%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:47:52.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:52 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 783/4683 objects degraded (16.720%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:47:52.684 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:52.706 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 4 16 15928 15912 248.605 227.188 0.00438913 0.00401752 2024-08-17T17:47:52.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:52.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:52.817 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:52.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:52 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 783/4683 objects degraded (16.720%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:47:52.849 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:52.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:52.951 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:52.991 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:52.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:53.092 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:53.123 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:53.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:53.129 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:47:53.129 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T17:47:53.129 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T17:47:53.129 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T17:47:53.129 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.129 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.44921.0:1919 43.f 43:f6eac2f1:::rbd_data.af79f435bb66.0000000000000001:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e936)", 2024-08-17T17:47:53.129 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:40:28.313616+0000", 2024-08-17T17:47:53.129 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 444.81220984300001, 2024-08-17T17:47:53.130 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.043409425000000001, 2024-08-17T17:47:53.130 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.130 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.130 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.130 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.130 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.44921", 2024-08-17T17:47:53.130 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2187674054", 2024-08-17T17:47:53.130 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 1919 2024-08-17T17:47:53.130 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.130 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.130 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.130 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.130 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:28.313616+0000", 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:28.313616+0000", 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:28.313618+0000", 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.629e-06 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.131 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:28.316374+0000", 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027560979999999998 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:28.316407+0000", 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3118999999999999e-05 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:28.316417+0000", 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0655999999999999e-05 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:28.316498+0000", 2024-08-17T17:47:53.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0269e-05 2024-08-17T17:47:53.133 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.133 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.133 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.133 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:28.316633+0000", 2024-08-17T17:47:53.133 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013489200000000001 2024-08-17T17:47:53.133 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.133 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.133 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:47:53.133 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:28.316749+0000", 2024-08-17T17:47:53.133 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000115992 2024-08-17T17:47:53.133 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.133 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.133 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.133 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:28.333997+0000", 2024-08-17T17:47:53.134 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017248348 2024-08-17T17:47:53.134 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.134 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.134 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.134 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:28.354803+0000", 2024-08-17T17:47:53.134 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020805872 2024-08-17T17:47:53.134 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.134 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.134 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.134 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:28.356904+0000", 2024-08-17T17:47:53.134 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021010909999999998 2024-08-17T17:47:53.134 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.134 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:28.356970+0000", 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5813000000000002e-05 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:28.357025+0000", 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5646000000000002e-05 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.44921.0:2084 43.13 43:cb6b4db3:::rbd_data.af797f41ec27.0000000000000002:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e939)", 2024-08-17T17:47:53.135 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:40:37.152623+0000", 2024-08-17T17:47:53.136 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 435.97320278299998, 2024-08-17T17:47:53.136 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.043402745, 2024-08-17T17:47:53.136 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.136 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.136 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.136 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.136 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.44921", 2024-08-17T17:47:53.136 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2187674054", 2024-08-17T17:47:53.136 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 2084 2024-08-17T17:47:53.136 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.136 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.136 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.136 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.136 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:37.152623+0000", 2024-08-17T17:47:53.137 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.137 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.137 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.137 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.137 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:37.152623+0000", 2024-08-17T17:47:53.137 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.137 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.137 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.137 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.137 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:37.152627+0000", 2024-08-17T17:47:53.137 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6839999999999998e-06 2024-08-17T17:47:53.137 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.137 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.137 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:37.155760+0000", 2024-08-17T17:47:53.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0031332819999999998 2024-08-17T17:47:53.138 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.138 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.138 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:37.155786+0000", 2024-08-17T17:47:53.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.584e-05 2024-08-17T17:47:53.138 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.138 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.138 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:37.155804+0000", 2024-08-17T17:47:53.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.791e-05 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:37.155877+0000", 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3672999999999997e-05 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:37.156002+0000", 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012508800000000001 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:37.156124+0000", 2024-08-17T17:47:53.139 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012154699999999999 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:37.166359+0000", 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010234767000000001 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:37.166899+0000", 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00054047199999999996 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:37.195952+0000", 2024-08-17T17:47:53.140 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.029052360999999999 2024-08-17T17:47:53.141 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.141 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.141 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.141 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:37.195993+0000", 2024-08-17T17:47:53.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1131999999999999e-05 2024-08-17T17:47:53.141 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.141 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.141 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.141 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:40:37.196026+0000", 2024-08-17T17:47:53.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2988999999999999e-05 2024-08-17T17:47:53.141 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.141 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.141 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.142 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.142 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.35043.0:3114 43.f 43:f72abb4c:::rbd_data.88e3ea1ec06e.0000000000000050:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 67=[67] ondisk+write+known_if_redirected+supports_pool_eio e970)", 2024-08-17T17:47:53.142 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:41:38.743296+0000", 2024-08-17T17:47:53.142 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 374.38253003800003, 2024-08-17T17:47:53.142 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.044787759000000003, 2024-08-17T17:47:53.142 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.142 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.142 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.143 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.143 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.35043", 2024-08-17T17:47:53.143 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/606020930", 2024-08-17T17:47:53.143 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 3114 2024-08-17T17:47:53.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.143 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.143 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:38.743296+0000", 2024-08-17T17:47:53.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:38.743296+0000", 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:38.743299+0000", 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9380000000000001e-06 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:38.746083+0000", 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027840339999999999 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.144 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:38.746115+0000", 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.18e-05 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:38.746127+0000", 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2768999999999999e-05 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:38.746275+0000", 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014817699999999999 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.145 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.146 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:38.746514+0000", 2024-08-17T17:47:53.146 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00023885399999999999 2024-08-17T17:47:53.146 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.146 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.146 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:47:53.146 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:38.746737+0000", 2024-08-17T17:47:53.146 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000222518 2024-08-17T17:47:53.146 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.146 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.146 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.146 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:38.770088+0000", 2024-08-17T17:47:53.146 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023351 2024-08-17T17:47:53.146 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:38.777093+0000", 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0070051250000000001 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:38.787993+0000", 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010899714 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:38.788045+0000", 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2259999999999998e-05 2024-08-17T17:47:53.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:38.788084+0000", 2024-08-17T17:47:53.148 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8569999999999998e-05 2024-08-17T17:47:53.148 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.148 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.148 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.148 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.35043.0:3380 43.5 43:a1305e89:::rbd_data.88e3ea1ec06e.000000000000002e:head [write 2998~4191306] snapc 6c=[6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e975)", 2024-08-17T17:47:53.148 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:41:43.753315+0000", 2024-08-17T17:47:53.148 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 369.37251092600002, 2024-08-17T17:47:53.148 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.044596437000000003, 2024-08-17T17:47:53.148 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.35043", 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/606020930", 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 3380 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:43.753315+0000", 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.149 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.150 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:43.753315+0000", 2024-08-17T17:47:53.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.150 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.150 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.150 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.150 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:43.753318+0000", 2024-08-17T17:47:53.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.208e-06 2024-08-17T17:47:53.150 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.150 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.150 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.150 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:43.756936+0000", 2024-08-17T17:47:53.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0036182290000000001 2024-08-17T17:47:53.150 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:43.756971+0000", 2024-08-17T17:47:53.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4474000000000002e-05 2024-08-17T17:47:53.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:43.756992+0000", 2024-08-17T17:47:53.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1328999999999999e-05 2024-08-17T17:47:53.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:43.758220+0000", 2024-08-17T17:47:53.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0012281760000000001 2024-08-17T17:47:53.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.152 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:43.758389+0000", 2024-08-17T17:47:53.152 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016829 2024-08-17T17:47:53.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:47:53.152 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:43.758713+0000", 2024-08-17T17:47:53.152 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00032463499999999998 2024-08-17T17:47:53.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.152 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:43.775359+0000", 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0166456 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:43.797728+0000", 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022369457999999998 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:43.797782+0000", 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4042999999999997e-05 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.153 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:43.797857+0000", 2024-08-17T17:47:53.154 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4846999999999997e-05 2024-08-17T17:47:53.154 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.154 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.154 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.154 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:43.797911+0000", 2024-08-17T17:47:53.154 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4147999999999997e-05 2024-08-17T17:47:53.154 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.154 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.154 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.154 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.154 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.154 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.35043.0:3429 43.10 e976/866 43:099ea058:::rbd_data.88e3ea1ec06e.000000000000005a:head v 976'47, mlcod=973'44)", 2024-08-17T17:47:53.154 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:41:44.758324+0000", 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 368.36750172699999, 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.043112824000000001, 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.758324+0000", 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.758324+0000", 2024-08-17T17:47:53.155 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.156 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.156 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.156 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.156 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.758325+0000", 2024-08-17T17:47:53.156 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7800000000000001e-07 2024-08-17T17:47:53.156 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.156 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.156 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.156 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.767291+0000", 2024-08-17T17:47:53.156 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0089662579999999995 2024-08-17T17:47:53.156 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.156 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.156 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.156 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.767314+0000", 2024-08-17T17:47:53.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2402999999999999e-05 2024-08-17T17:47:53.157 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.157 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.767328+0000", 2024-08-17T17:47:53.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4657999999999999e-05 2024-08-17T17:47:53.157 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.157 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.767430+0000", 2024-08-17T17:47:53.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000102309 2024-08-17T17:47:53.157 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.158 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.767454+0000", 2024-08-17T17:47:53.158 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.332e-05 2024-08-17T17:47:53.158 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.158 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.158 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.158 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.801389+0000", 2024-08-17T17:47:53.158 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.033935143000000001 2024-08-17T17:47:53.158 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.158 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.158 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.158 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.801437+0000", 2024-08-17T17:47:53.158 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7954999999999997e-05 2024-08-17T17:47:53.159 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.159 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.159 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.159 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.159 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.159 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.35043.0:3436 43.12 43:4cac7c54:::rbd_data.88e3ea1ec06e.0000000000000008:head [write 2507622~1686682] snapc 6d=[6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e976)", 2024-08-17T17:47:53.159 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:41:44.760378+0000", 2024-08-17T17:47:53.159 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 368.365447948, 2024-08-17T17:47:53.159 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.043740846999999999, 2024-08-17T17:47:53.159 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.159 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.159 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.159 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.159 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.35043", 2024-08-17T17:47:53.160 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/606020930", 2024-08-17T17:47:53.160 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 3436 2024-08-17T17:47:53.160 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.160 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.160 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.160 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.160 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.760378+0000", 2024-08-17T17:47:53.160 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.160 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.160 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.160 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.160 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.760378+0000", 2024-08-17T17:47:53.160 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.160 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.760381+0000", 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8789999999999999e-06 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.762028+0000", 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0016474129999999999 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.762086+0000", 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7544000000000003e-05 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.162 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.162 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.762107+0000", 2024-08-17T17:47:53.162 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1742e-05 2024-08-17T17:47:53.162 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.162 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.162 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.162 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.762272+0000", 2024-08-17T17:47:53.162 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016437 2024-08-17T17:47:53.162 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.162 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.162 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.162 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.762414+0000", 2024-08-17T17:47:53.162 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014237599999999999 2024-08-17T17:47:53.163 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.163 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.163 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:47:53.163 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.762666+0000", 2024-08-17T17:47:53.163 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00025196600000000001 2024-08-17T17:47:53.163 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.163 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.163 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.163 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.795411+0000", 2024-08-17T17:47:53.163 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.032744693999999998 2024-08-17T17:47:53.163 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.163 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.163 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.163 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.803483+0000", 2024-08-17T17:47:53.164 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0080722639999999991 2024-08-17T17:47:53.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.164 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.164 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.164 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.803950+0000", 2024-08-17T17:47:53.164 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00046703700000000001 2024-08-17T17:47:53.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.164 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.164 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.164 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.804049+0000", 2024-08-17T17:47:53.164 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8648999999999997e-05 2024-08-17T17:47:53.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.164 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.804119+0000", 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9913000000000006e-05 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.35043.0:3432 43.1d e976/866 43:ba3897ef:::rbd_data.88e3ea1ec06e.0000000000000061:head v 976'40, mlcod=970'38)", 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:41:44.760848+0000", 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 368.36497805099998, 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.040548936000000001, 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.165 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.166 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.760848+0000", 2024-08-17T17:47:53.166 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.166 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.760848+0000", 2024-08-17T17:47:53.166 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.166 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.760850+0000", 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.351e-06 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.766894+0000", 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0060439459999999997 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.766932+0000", 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7772999999999999e-05 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.167 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.766988+0000", 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5983999999999998e-05 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.767147+0000", 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015953000000000001 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.767177+0000", 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0020000000000001e-05 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.801350+0000", 2024-08-17T17:47:53.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.034172352000000003 2024-08-17T17:47:53.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:44.801397+0000", 2024-08-17T17:47:53.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6980000000000001e-05 2024-08-17T17:47:53.169 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.169 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.169 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.169 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.35043.0:3614 43.5 43:a2fcbaf2:::rbd_data.88e3ea1ec06e.000000000000000c:head [write 1841040~2353264] snapc 70=[70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e979)", 2024-08-17T17:47:53.169 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:41:47.799282+0000", 2024-08-17T17:47:53.169 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 365.32654344299999, 2024-08-17T17:47:53.170 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042764048999999998, 2024-08-17T17:47:53.170 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.170 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.170 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.170 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.170 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.35043", 2024-08-17T17:47:53.170 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/606020930", 2024-08-17T17:47:53.170 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 3614 2024-08-17T17:47:53.170 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.799282+0000", 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.799282+0000", 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.799285+0000", 2024-08-17T17:47:53.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9529999999999999e-06 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.802868+0000", 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0035828800000000001 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.802910+0000", 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1427000000000002e-05 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.802927+0000", 2024-08-17T17:47:53.172 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7638e-05 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.803052+0000", 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000124788 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.803196+0000", 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014411900000000001 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.803424+0000", 2024-08-17T17:47:53.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00022827399999999999 2024-08-17T17:47:53.174 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.834599+0000", 2024-08-17T17:47:53.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.031174535999999999 2024-08-17T17:47:53.174 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.834654+0000", 2024-08-17T17:47:53.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5488000000000001e-05 2024-08-17T17:47:53.174 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.841922+0000", 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072677949999999996 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.841978+0000", 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6161e-05 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.842046+0000", 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7990000000000005e-05 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.175 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.35043.0:3610 43.10 e979/866 43:0dffe567:::rbd_data.88e3ea1ec06e.000000000000000e:head v 979'61, mlcod=978'57)", 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:41:47.801243+0000", 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 365.32458233099999, 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.039570528000000001, 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.801243+0000", 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.801243+0000", 2024-08-17T17:47:53.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.801246+0000", 2024-08-17T17:47:53.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0180000000000002e-06 2024-08-17T17:47:53.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.804571+0000", 2024-08-17T17:47:53.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003324596 2024-08-17T17:47:53.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.178 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.804590+0000", 2024-08-17T17:47:53.178 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8559999999999998e-05 2024-08-17T17:47:53.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.178 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.804599+0000", 2024-08-17T17:47:53.178 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9060000000000008e-06 2024-08-17T17:47:53.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.178 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.804673+0000", 2024-08-17T17:47:53.178 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4627000000000002e-05 2024-08-17T17:47:53.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.804696+0000", 2024-08-17T17:47:53.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2722999999999999e-05 2024-08-17T17:47:53.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.840758+0000", 2024-08-17T17:47:53.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.036061890999999999 2024-08-17T17:47:53.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.840814+0000", 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6206999999999999e-05 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.35043.0:3612 43.1d e979/866 43:bdd86ec0:::rbd_data.88e3ea1ec06e.0000000000000028:head v 979'44, mlcod=976'40)", 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:41:47.801608+0000", 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 365.32421796900002, 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.040504633999999998, 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.180 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.801608+0000", 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.801608+0000", 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.801610+0000", 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9599999999999999e-06 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.181 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.803437+0000", 2024-08-17T17:47:53.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0018276869999999999 2024-08-17T17:47:53.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.803451+0000", 2024-08-17T17:47:53.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3468e-05 2024-08-17T17:47:53.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.803460+0000", 2024-08-17T17:47:53.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4040000000000001e-06 2024-08-17T17:47:53.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.803558+0000", 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7208999999999997e-05 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.803586+0000", 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8833999999999999e-05 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.842058+0000", 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.038471205000000001 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:47.842112+0000", 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4867000000000002e-05 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.35043.0:3978 43.1 e986/866 43:810a446c:::rbd_data.88e3ea1ec06e.000000000000004b:head v 986'49, mlcod=978'45)", 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:41:54.874810+0000", 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 358.25101566400002, 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042232862000000003, 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.184 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:54.874810+0000", 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:54.874810+0000", 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:54.874810+0000", 2024-08-17T17:47:53.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3000000000000002e-07 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:54.900828+0000", 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.026017332000000001 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:54.900840+0000", 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.219e-05 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:54.900846+0000", 2024-08-17T17:47:53.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2759999999999997e-06 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:54.900902+0000", 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5259000000000001e-05 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:54.900923+0000", 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1922000000000001e-05 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:54.917014+0000", 2024-08-17T17:47:53.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016090436 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:41:54.917043+0000", 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9116999999999999e-05 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.35043.0:40752 43.13 43:ce894a28:::rbd_data.88e3a6881d3b.0000000000000000:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1008)", 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:42:17.600265+0000", 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 335.52556054899998, 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.040032986999999999, 2024-08-17T17:47:53.188 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.189 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.189 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.189 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.189 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.35043", 2024-08-17T17:47:53.189 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/606020930", 2024-08-17T17:47:53.189 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 40752 2024-08-17T17:47:53.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.189 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.600265+0000", 2024-08-17T17:47:53.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.190 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.190 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.600265+0000", 2024-08-17T17:47:53.190 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.190 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.190 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.190 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.190 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.600267+0000", 2024-08-17T17:47:53.190 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5770000000000001e-06 2024-08-17T17:47:53.190 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.190 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.190 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.190 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.603074+0000", 2024-08-17T17:47:53.190 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028076360000000001 2024-08-17T17:47:53.191 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.191 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.191 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.191 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.603113+0000", 2024-08-17T17:47:53.191 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8566000000000003e-05 2024-08-17T17:47:53.191 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.191 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.191 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.191 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.603128+0000", 2024-08-17T17:47:53.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.505e-05 2024-08-17T17:47:53.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.192 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.603242+0000", 2024-08-17T17:47:53.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000113791 2024-08-17T17:47:53.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.192 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.603386+0000", 2024-08-17T17:47:53.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000144086 2024-08-17T17:47:53.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:47:53.193 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.603500+0000", 2024-08-17T17:47:53.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000114539 2024-08-17T17:47:53.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.193 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.193 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.625779+0000", 2024-08-17T17:47:53.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022278183 2024-08-17T17:47:53.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.193 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.193 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.626015+0000", 2024-08-17T17:47:53.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00023583 2024-08-17T17:47:53.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.640220+0000", 2024-08-17T17:47:53.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014205906000000001 2024-08-17T17:47:53.194 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.640264+0000", 2024-08-17T17:47:53.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3245999999999999e-05 2024-08-17T17:47:53.194 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.195 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.640298+0000", 2024-08-17T17:47:53.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4576999999999998e-05 2024-08-17T17:47:53.195 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.195 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.195 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.195 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.35043.0:40755 43.5 43:a13b024c:::rbd_data.88e3a6881d3b.0000000000000003:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1008)", 2024-08-17T17:47:53.195 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:42:17.603187+0000", 2024-08-17T17:47:53.195 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 335.522638992, 2024-08-17T17:47:53.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.040505521000000003, 2024-08-17T17:47:53.195 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.195 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.195 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.35043", 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/606020930", 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 40755 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.603187+0000", 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.603187+0000", 2024-08-17T17:47:53.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.603188+0000", 2024-08-17T17:47:53.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.328e-06 2024-08-17T17:47:53.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.198 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.607742+0000", 2024-08-17T17:47:53.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0045540119999999996 2024-08-17T17:47:53.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.198 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.198 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.198 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.607813+0000", 2024-08-17T17:47:53.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0494999999999995e-05 2024-08-17T17:47:53.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.198 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.198 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.198 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.607838+0000", 2024-08-17T17:47:53.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5092999999999998e-05 2024-08-17T17:47:53.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.607942+0000", 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000104479 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.608335+0000", 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000393037 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.608574+0000", 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00023896099999999999 2024-08-17T17:47:53.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.629163+0000", 2024-08-17T17:47:53.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020588839000000001 2024-08-17T17:47:53.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.642238+0000", 2024-08-17T17:47:53.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013075148 2024-08-17T17:47:53.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.643624+0000", 2024-08-17T17:47:53.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0013859110000000001 2024-08-17T17:47:53.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.201 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.201 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.643663+0000", 2024-08-17T17:47:53.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8463000000000001e-05 2024-08-17T17:47:53.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.201 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.201 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:42:17.643692+0000", 2024-08-17T17:47:53.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9754999999999999e-05 2024-08-17T17:47:53.201 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.201 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.201 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.202 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.202 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.35055.0:1100 44.2 44:468c3f17:::rbd_data.88ef47b08a78.0000000000000000:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1079)", 2024-08-17T17:47:53.202 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:44:08.216096+0000", 2024-08-17T17:47:53.202 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 224.90973001399999, 2024-08-17T17:47:53.202 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.040624346999999998, 2024-08-17T17:47:53.202 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.202 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.202 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.202 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.202 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.35055", 2024-08-17T17:47:53.202 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2711769310", 2024-08-17T17:47:53.202 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 1100 2024-08-17T17:47:53.202 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:44:08.216096+0000", 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:44:08.216096+0000", 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:44:08.216100+0000", 2024-08-17T17:47:53.203 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6e-06 2024-08-17T17:47:53.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:44:08.218723+0000", 2024-08-17T17:47:53.204 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026229700000000001 2024-08-17T17:47:53.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:44:08.218759+0000", 2024-08-17T17:47:53.204 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5952999999999999e-05 2024-08-17T17:47:53.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:44:08.218775+0000", 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5316e-05 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:44:08.218879+0000", 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000104083 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:44:08.219044+0000", 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016493500000000001 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:44:08.219206+0000", 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016268399999999999 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:44:08.235745+0000", 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016539046000000002 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:44:08.241567+0000", 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0058221159999999996 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.206 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.207 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:44:08.256611+0000", 2024-08-17T17:47:53.207 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.01504313 2024-08-17T17:47:53.207 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.207 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.207 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.207 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:44:08.256671+0000", 2024-08-17T17:47:53.207 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0627000000000001e-05 2024-08-17T17:47:53.207 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.207 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.207 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.207 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:44:08.256720+0000", 2024-08-17T17:47:53.207 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8887000000000001e-05 2024-08-17T17:47:53.207 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.207 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.44945.0:1647 44.1f 44:fff7e504:::rbd_data.af913911a12.0000000000000002:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1106)", 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:45:19.968622+0000", 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 153.15720390199999, 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.040743443999999997, 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.44945", 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/735940947", 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 1647 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.208 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:19.968622+0000", 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:19.968622+0000", 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:19.968626+0000", 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5920000000000002e-06 2024-08-17T17:47:53.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.210 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.210 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:19.972785+0000", 2024-08-17T17:47:53.210 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0041583039999999998 2024-08-17T17:47:53.210 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.210 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.210 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:19.972820+0000", 2024-08-17T17:47:53.210 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5058999999999998e-05 2024-08-17T17:47:53.210 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.210 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.210 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:19.972832+0000", 2024-08-17T17:47:53.210 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2104e-05 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:19.972918+0000", 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6297000000000004e-05 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:19.973079+0000", 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000160688 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:19.973243+0000", 2024-08-17T17:47:53.211 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016455800000000001 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:19.986690+0000", 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013446911000000001 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:19.988800+0000", 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021095250000000001 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:20.009253+0000", 2024-08-17T17:47:53.212 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020453356999999998 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:20.009316+0000", 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.224e-05 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:20.009365+0000", 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9808999999999997e-05 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.213 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.214 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.44945.0:2588 44.2 44:4227f19d:::rbd_data.af91c8ed8b9b.0000000000000039:head [write 0~2435917] snapc 6e=[6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1126)", 2024-08-17T17:47:53.214 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:45:39.808103+0000", 2024-08-17T17:47:53.214 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 133.317723129, 2024-08-17T17:47:53.214 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.048259846000000002, 2024-08-17T17:47:53.214 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.214 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.214 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.214 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.214 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.44945", 2024-08-17T17:47:53.214 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/735940947", 2024-08-17T17:47:53.214 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 2588 2024-08-17T17:47:53.214 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.214 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.214 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.215 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.215 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.808103+0000", 2024-08-17T17:47:53.215 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.215 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.215 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.215 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.215 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.808103+0000", 2024-08-17T17:47:53.215 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.215 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.215 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.808107+0000", 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3610000000000003e-06 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.810161+0000", 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0020536439999999999 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.810195+0000", 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4208999999999997e-05 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.216 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.217 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.217 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.810204+0000", 2024-08-17T17:47:53.217 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9809999999999992e-06 2024-08-17T17:47:53.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.217 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.217 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.217 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.810269+0000", 2024-08-17T17:47:53.217 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4979000000000002e-05 2024-08-17T17:47:53.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.217 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.217 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.217 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.810386+0000", 2024-08-17T17:47:53.217 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000117179 2024-08-17T17:47:53.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.218 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:47:53.218 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.810572+0000", 2024-08-17T17:47:53.218 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018590300000000001 2024-08-17T17:47:53.218 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.218 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.218 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.821060+0000", 2024-08-17T17:47:53.218 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010487896999999999 2024-08-17T17:47:53.218 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.219 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.219 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.821099+0000", 2024-08-17T17:47:53.219 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8689999999999997e-05 2024-08-17T17:47:53.219 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.219 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.219 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.219 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.856256+0000", 2024-08-17T17:47:53.219 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.035157257999999997 2024-08-17T17:47:53.219 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.219 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.219 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.219 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.856294+0000", 2024-08-17T17:47:53.219 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8665000000000003e-05 2024-08-17T17:47:53.220 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.220 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.220 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.220 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.856363+0000", 2024-08-17T17:47:53.220 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8079999999999999e-05 2024-08-17T17:47:53.220 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.220 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.220 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.220 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.220 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.220 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.44945.0:2590 44.2 44:46ff3400:::rbd_data.af91c8ed8b9b.000000000000005b:head [write 0~2415170] snapc 6e=[6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1126)", 2024-08-17T17:47:53.220 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:45:39.811742+0000", 2024-08-17T17:47:53.220 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 133.31408361999999, 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.044764342999999998, 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.44945", 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/735940947", 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 2590 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.811742+0000", 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.221 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.811742+0000", 2024-08-17T17:47:53.222 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.811744+0000", 2024-08-17T17:47:53.222 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2000000000000001e-06 2024-08-17T17:47:53.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.814321+0000", 2024-08-17T17:47:53.222 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025761780000000001 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.814353+0000", 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2319e-05 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.814368+0000", 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5192e-05 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.816722+0000", 2024-08-17T17:47:53.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002353913 2024-08-17T17:47:53.224 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.224 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.224 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.224 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.816860+0000", 2024-08-17T17:47:53.224 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000138193 2024-08-17T17:47:53.224 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.224 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.224 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:47:53.224 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.817095+0000", 2024-08-17T17:47:53.224 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:53.225 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000234625 2024-08-17T17:47:53.225 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.225 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.225 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.225 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.827823+0000", 2024-08-17T17:47:53.226 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010728609 2024-08-17T17:47:53.226 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.226 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.226 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.226 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.828153+0000", 2024-08-17T17:47:53.226 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00032924199999999999 2024-08-17T17:47:53.226 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.226 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.226 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.226 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.856410+0000", 2024-08-17T17:47:53.226 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.02825766 2024-08-17T17:47:53.226 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.226 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.226 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.856469+0000", 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9083000000000003e-05 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:39.856507+0000", 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7129000000000003e-05 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.44945.0:2745 44.e 44:76f3a971:::rbd_data.af91c8ed8b9b.000000000000002a:head [write 0~2955025] snapc 71=[71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1129)", 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:45:42.827825+0000", 2024-08-17T17:47:53.227 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 130.29800074600001, 2024-08-17T17:47:53.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.04529859, 2024-08-17T17:47:53.228 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.228 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.228 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.228 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.228 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.44945", 2024-08-17T17:47:53.228 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/735940947", 2024-08-17T17:47:53.228 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 2745 2024-08-17T17:47:53.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.228 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.228 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.228 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.228 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.827825+0000", 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.827825+0000", 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.827829+0000", 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9190000000000001e-06 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.830623+0000", 2024-08-17T17:47:53.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027942510000000002 2024-08-17T17:47:53.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.230 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.230 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.830654+0000", 2024-08-17T17:47:53.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0539e-05 2024-08-17T17:47:53.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.230 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.230 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.830668+0000", 2024-08-17T17:47:53.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4683e-05 2024-08-17T17:47:53.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.830807+0000", 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000138367 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op must wait for map", 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.830813+0000", 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9200000000000001e-06 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.835702+0000", 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0048891300000000002 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.232 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.232 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.835894+0000", 2024-08-17T17:47:53.232 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000192287 2024-08-17T17:47:53.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.232 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:47:53.232 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.836364+0000", 2024-08-17T17:47:53.232 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000469457 2024-08-17T17:47:53.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.232 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.232 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.852498+0000", 2024-08-17T17:47:53.232 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016134406 2024-08-17T17:47:53.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.233 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.233 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.872900+0000", 2024-08-17T17:47:53.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020402053999999999 2024-08-17T17:47:53.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.233 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.233 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.872998+0000", 2024-08-17T17:47:53.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8424999999999994e-05 2024-08-17T17:47:53.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.233 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.233 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.873069+0000", 2024-08-17T17:47:53.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0936e-05 2024-08-17T17:47:53.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:42.873124+0000", 2024-08-17T17:47:53.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4215999999999997e-05 2024-08-17T17:47:53.234 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.234 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.234 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.234 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.44945.0:2806 44.2 44:46ff3400:::rbd_data.af91c8ed8b9b.000000000000005b:head [write 0~3873297] snapc 72=[72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-08-17T17:47:53.234 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:45:43.831742+0000", 2024-08-17T17:47:53.234 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 129.294083607, 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14714302000000001, 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.44945", 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/735940947", 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 2806 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:43.831742+0000", 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:43.831742+0000", 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:43.831745+0000", 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5790000000000002e-06 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:43.835206+0000", 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0034613119999999998 2024-08-17T17:47:53.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:43.835231+0000", 2024-08-17T17:47:53.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5103e-05 2024-08-17T17:47:53.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:43.835242+0000", 2024-08-17T17:47:53.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0556e-05 2024-08-17T17:47:53.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:43.835322+0000", 2024-08-17T17:47:53.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0479e-05 2024-08-17T17:47:53.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:43.835438+0000", 2024-08-17T17:47:53.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011595300000000001 2024-08-17T17:47:53.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:47:53.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:43.835644+0000", 2024-08-17T17:47:53.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020622199999999999 2024-08-17T17:47:53.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:43.856617+0000", 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020972477 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:43.868297+0000", 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011680612999999999 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:43.978776+0000", 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.11047860499999999 2024-08-17T17:47:53.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.240 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.240 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.240 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:43.978837+0000", 2024-08-17T17:47:53.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1267999999999997e-05 2024-08-17T17:47:53.240 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.240 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.240 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.240 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:43.978885+0000", 2024-08-17T17:47:53.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7852999999999997e-05 2024-08-17T17:47:53.240 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.240 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.240 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.240 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.240 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.241 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.44945.0:2867 44.e 44:7757f7bc:::rbd_data.af91c8ed8b9b.000000000000003b:head [write 0~4159916] snapc 73=[73,72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1131)", 2024-08-17T17:47:53.241 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:45:44.854401+0000", 2024-08-17T17:47:53.241 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 128.271424588, 2024-08-17T17:47:53.241 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.044591960999999999, 2024-08-17T17:47:53.241 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:47:53.241 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:47:53.241 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:47:53.241 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:47:53.241 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.44945", 2024-08-17T17:47:53.241 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/735940947", 2024-08-17T17:47:53.241 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 2867 2024-08-17T17:47:53.241 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.241 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:47:53.241 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:47:53.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.854401+0000", 2024-08-17T17:47:53.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.242 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.242 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:47:53.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.854401+0000", 2024-08-17T17:47:53.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:47:53.242 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.242 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:47:53.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.854404+0000", 2024-08-17T17:47:53.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0489999999999999e-06 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.858324+0000", 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039193910000000004 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.858340+0000", 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6073e-05 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.858352+0000", 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2021999999999999e-05 2024-08-17T17:47:53.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.858410+0000", 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8418000000000002e-05 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op must wait for map", 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.858415+0000", 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6670000000000004e-06 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.865584+0000", 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0071687119999999998 2024-08-17T17:47:53.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:47:53.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.865679+0000", 2024-08-17T17:47:53.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5636000000000004e-05 2024-08-17T17:47:53.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:47:53.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.865987+0000", 2024-08-17T17:47:53.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00030768100000000002 2024-08-17T17:47:53.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:47:53.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.873690+0000", 2024-08-17T17:47:53.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0077033420000000002 2024-08-17T17:47:53.246 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.246 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.246 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.898769+0000", 2024-08-17T17:47:53.246 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025078343999999999 2024-08-17T17:47:53.246 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.246 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:47:53.246 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.898881+0000", 2024-08-17T17:47:53.246 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000112342 2024-08-17T17:47:53.246 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.246 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:47:53.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.898920+0000", 2024-08-17T17:47:53.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9251000000000001e-05 2024-08-17T17:47:53.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:47:53.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:47:53.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:47:53.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:45:44.898993+0000", 2024-08-17T17:47:53.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3033000000000004e-05 2024-08-17T17:47:53.247 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.247 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.247 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.247 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:47:53.247 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:47:53.247 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:47:53.279 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:53.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:53.381 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:53.414 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:53.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:53.516 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:53.549 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:53.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:53.651 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:53.694 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:53.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:53.706 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 5 16 19237 19221 240.243 206.812 0.0031522 0.00415776 2024-08-17T17:47:53.796 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:53.829 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:53.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:53.838 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:53 smithi130 ceph-mon[110068]: pgmap v1568: 89 pgs: 24 active+undersized, 8 active+undersized+degraded, 57 active+clean; 97 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s wr, 153 op/s; 783/4683 objects degraded (16.720%) 2024-08-17T17:47:53.838 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:53 smithi130 ceph-mon[105521]: pgmap v1568: 89 pgs: 24 active+undersized, 8 active+undersized+degraded, 57 active+clean; 97 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s wr, 153 op/s; 783/4683 objects degraded (16.720%) 2024-08-17T17:47:53.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:53 smithi179 ceph-mon[93519]: pgmap v1568: 89 pgs: 24 active+undersized, 8 active+undersized+degraded, 57 active+clean; 97 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s wr, 153 op/s; 783/4683 objects degraded (16.720%) 2024-08-17T17:47:53.931 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:53.962 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:53.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:53.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:53.998 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-08-17T17:47:54.009 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:47:54.022 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:47:54.022 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:47:54.056 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:47:54.056 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:47:54.064 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:54.090 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:47:54.090 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:47:54.094 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:54.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:54.121 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:47:54.121 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:47:54.153 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:47:54.153 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:47:54.186 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:47:54.186 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:47:54.195 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:54.215 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:47:54.216 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:47:54.229 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:54.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:54.330 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:54.367 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:54.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:54.470 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:54.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.514 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:54.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:54.518 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:47:54.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2197282813' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:54.616 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:54.646 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:54.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:54.707 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 6 16 22309 22293 232.199 192 0.00379843 0.00430111 2024-08-17T17:47:54.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2197282813' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:54.732 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2197282813' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:47:54.748 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:54.771 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:54.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:54.874 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:54.912 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:54.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:54.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:55.014 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:55.051 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:55.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:55.152 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:55.187 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:55.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:55.288 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:55.319 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:55.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:55.421 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:55.453 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:55.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:55.555 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:55.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.592 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:55.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:55.693 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:55.707 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 7 16 24846 24830 221.678 158.562 0.00594479 0.00450584 2024-08-17T17:47:55.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.725 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:55.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:55.826 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:55.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:55 smithi179 ceph-mon[93519]: pgmap v1569: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.0 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s; 6283/50268 objects degraded (12.499%) 2024-08-17T17:47:55.861 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:55.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:55.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:55.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:55 smithi130 ceph-mon[105521]: pgmap v1569: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.0 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s; 6283/50268 objects degraded (12.499%) 2024-08-17T17:47:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:55 smithi130 ceph-mon[110068]: pgmap v1569: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.0 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s; 6283/50268 objects degraded (12.499%) 2024-08-17T17:47:55.963 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:55.999 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:56.101 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:56.136 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:56.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:56.239 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:56.272 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:56.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:56.373 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:56.399 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:56.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:56.501 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:56.537 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:56.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:56.640 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:56.660 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:56.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:56.707 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 8 16 27310 27294 213.216 154 0.00319633 0.00468532 2024-08-17T17:47:56.762 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:47:56.796 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:56.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:47:56.898 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:47:56.919 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:56.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:56.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:47:57.021 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:57.042 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:57.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:57.111 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:47:57.111 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:47:57.111 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:47:57.111 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd out 5 2024-08-17T17:47:57.145 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:57.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.194 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:57.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:57.295 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:57.316 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:57.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:57.418 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:57.454 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:57.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:57.556 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:57.591 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:57.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:57.693 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:57.707 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 9 16 29367 29351 203.809 128.562 0.00501783 0.00490275 2024-08-17T17:47:57.729 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:57.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:57.831 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:57 smithi179 ceph-mon[93519]: pgmap v1570: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.0 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s; 6283/50268 objects degraded (12.499%) 2024-08-17T17:47:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:57 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:57.864 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:57.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:57.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:57 smithi130 ceph-mon[105521]: pgmap v1570: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.0 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s; 6283/50268 objects degraded (12.499%) 2024-08-17T17:47:57.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:57 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:57 smithi130 ceph-mon[110068]: pgmap v1570: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.0 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s; 6283/50268 objects degraded (12.499%) 2024-08-17T17:47:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:57 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:47:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:47:57.965 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:57.990 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:57.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:57.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:58.091 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:58.128 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:58.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:58.230 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:47:58.260 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:58.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:47:58.362 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:58.383 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:58.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:58.484 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:58.505 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:58.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:58.607 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:58.627 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:58.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:58.707 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 10 16 31206 31190 194.921 114.938 0.00371055 0.00512432 2024-08-17T17:47:58.729 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:58.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.765 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:58.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:58 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 10539/85980 objects degraded (12.258%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:47:58.865 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:47:58.891 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:58.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:58.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:47:58.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:58 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 10539/85980 objects degraded (12.258%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:47:58.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:58 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 10539/85980 objects degraded (12.258%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:47:58.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:59.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.031 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:59.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:59.132 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:59.170 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:59.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:59.272 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:59.292 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:59.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:59.394 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:47:59.432 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:59.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:47:59.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:59.554 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:59.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:47:59.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:59.689 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:59.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:59.707 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 11 16 33487 33471 190.16 142.562 0.00305679 0.00525446 2024-08-17T17:47:59.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:47:59.817 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:47:59.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:47:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:47:59 smithi179 ceph-mon[93519]: pgmap v1571: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.7 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s; 10539/85980 objects degraded (12.258%) 2024-08-17T17:47:59.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:47:59 smithi130 ceph-mon[105521]: pgmap v1571: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.7 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s; 10539/85980 objects degraded (12.258%) 2024-08-17T17:47:59.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:47:59 smithi130 ceph-mon[110068]: pgmap v1571: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.7 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s; 10539/85980 objects degraded (12.258%) 2024-08-17T17:47:59.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:47:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:47:59] "GET /metrics HTTP/1.1" 200 36622 "" "Prometheus/2.51.0" 2024-08-17T17:47:59.920 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:47:59.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:47:59.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:47:59.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:00.059 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:00.084 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:00.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:00.186 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:00.220 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:00.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:00.322 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:00.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:00.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:00.457 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:00.491 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:00.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:48:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:48:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:48:00.593 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:00.626 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:00.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:00.676 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:48:00.676 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:48:00.707 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 12 16 35539 35523 184.999 128.25 0.00500287 0.00540104 2024-08-17T17:48:00.728 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:00.759 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:00.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:00.861 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:00.891 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:00.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:00.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:00.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:01.032 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:01.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:01.134 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:01.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.172 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:01.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:01.273 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:01.307 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:01.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:01.409 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:01.441 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:01.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:01.542 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:01.577 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:01.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:01.679 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:01.704 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:01.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:01.707 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 13 16 37880 37864 182.02 146.312 0.0119299 0.00548673 2024-08-17T17:48:01.722 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:01.806 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:01.840 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:01.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:01.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:01 smithi179 ceph-mon[93519]: pgmap v1572: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.8 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s; 11280/90234 objects degraded (12.501%) 2024-08-17T17:48:01.871 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:01 smithi130 ceph-mon[105521]: pgmap v1572: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.8 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s; 11280/90234 objects degraded (12.501%) 2024-08-17T17:48:01.871 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:01 smithi130 ceph-mon[110068]: pgmap v1572: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.8 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s; 11280/90234 objects degraded (12.501%) 2024-08-17T17:48:01.943 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:01.980 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:01.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:01.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:02.003 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:02.082 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:02.114 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:02.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:02.216 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:02.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:02.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:02.352 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:02.390 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:02.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:02.492 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:02.523 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:02.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:02.625 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:02.655 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:02.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:02.677 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:48:02.707 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 14 16 40185 40169 179.308 144.062 0.00574484 0.00557188 2024-08-17T17:48:02.757 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:02.786 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:02.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:02.888 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:02.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:02.929 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:02.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:03.030 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:03.064 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:03.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:03.165 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:03.200 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:03.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:03.302 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:03.344 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:03.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:03.446 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:03.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.491 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:03.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:03.592 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:03.603 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:03 smithi130 ceph-mon[105521]: pgmap v1573: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.9 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 163 MiB/s wr, 2.61k op/s; 11910/93957 objects degraded (12.676%) 2024-08-17T17:48:03.603 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:03 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1281837326' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:03.603 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:03 smithi130 ceph-mon[110068]: pgmap v1573: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.9 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 163 MiB/s wr, 2.61k op/s; 11910/93957 objects degraded (12.676%) 2024-08-17T17:48:03.603 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:03 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1281837326' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:03.603 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T17:48:03.624 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:03.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:03.708 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 15 16 42438 42422 176.741 140.812 0.00948895 0.00565337 2024-08-17T17:48:03.725 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:03.758 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:03.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:03 smithi179 ceph-mon[93519]: pgmap v1573: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 1.9 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 163 MiB/s wr, 2.61k op/s; 11910/93957 objects degraded (12.676%) 2024-08-17T17:48:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:03 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1281837326' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:48:03.860 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:03.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:03.895 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:03.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:03.996 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:04.031 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:04.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:04.133 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:04.164 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:04.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:04.266 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:04.298 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:04.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:04.400 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:04.434 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:04.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:04.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:04.568 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:04.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:04.656 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:04 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:04.656 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:04 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1281837326' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:48:04.656 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:04 smithi130 ceph-mon[105521]: osdmap e1171: 8 total, 7 up, 8 in 2024-08-17T17:48:04.657 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:04 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:04.657 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:04 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1281837326' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:48:04.657 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:04 smithi130 ceph-mon[110068]: osdmap e1171: 8 total, 7 up, 8 in 2024-08-17T17:48:04.669 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:04.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.702 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:04.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:04.708 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 16 16 44793 44777 174.893 147.188 0.00859412 0.00571318 2024-08-17T17:48:04.803 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:04.837 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:04.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:04.939 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:04.971 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:04.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:04.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:05.072 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:05.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:04 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:05.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:04 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1281837326' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:48:05.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:04 smithi179 ceph-mon[93519]: osdmap e1171: 8 total, 7 up, 8 in 2024-08-17T17:48:05.130 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:05.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:05.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:05.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.275 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:05.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:05.375 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:05.405 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:05.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:05.507 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:05.537 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:05.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:05.639 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:05.675 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:05.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:05.708 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 17 16 46985 46969 172.663 137 0.00585216 0.00578748 2024-08-17T17:48:05.777 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:05.808 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:05.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:05.910 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:05.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:05 smithi130 ceph-mon[105521]: pgmap v1575: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 2.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 15235/122274 objects degraded (12.460%) 2024-08-17T17:48:05.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:05 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15235/122274 objects degraded (12.460%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:05 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T17:48:05.614+0000 7fa856f19640 -1 mon.a@0(leader).osd e1171 definitely_dead 0 2024-08-17T17:48:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:05 smithi130 ceph-mon[110068]: pgmap v1575: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 2.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 15235/122274 objects degraded (12.460%) 2024-08-17T17:48:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:05 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15235/122274 objects degraded (12.460%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:05.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:05.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:05.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:06.045 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:06.078 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:06.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:06.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:05 smithi179 ceph-mon[93519]: pgmap v1575: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 2.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 15235/122274 objects degraded (12.460%) 2024-08-17T17:48:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:05 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15235/122274 objects degraded (12.460%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:06.180 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:06.217 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:06.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:06.241 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:06.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:06.358 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:06.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:06.460 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:06.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.493 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:06.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:06.594 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:06.613 INFO:teuthology.orchestra.run.smithi130.stderr:marked out osd.5. 2024-08-17T17:48:06.641 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:06.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:06.708 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 18 16 48926 48910 169.809 121.312 0.00232872 0.00586033 2024-08-17T17:48:06.730 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:06 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2275486717' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-17T17:48:06.730 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:06 smithi130 ceph-mon[105521]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-17T17:48:06.730 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:06 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2275486717' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-17T17:48:06.730 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:06 smithi130 ceph-mon[110068]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-17T17:48:06.743 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:06.764 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:06.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:06.865 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:06.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:06.887 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:06.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:06.988 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:07.009 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:07.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:07.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:06 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2275486717' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-17T17:48:07.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:06 smithi179 ceph-mon[93519]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-17T17:48:07.111 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:07.132 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:07.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:07.234 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:07.255 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:07.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:07.357 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:07.377 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:07.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:07.479 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:07.501 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:07.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:07.603 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:07.634 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:07.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:07.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:07 smithi130 ceph-mon[110068]: pgmap v1576: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 2.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 15235/122274 objects degraded (12.460%) 2024-08-17T17:48:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:07 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2275486717' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-17T17:48:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:07 smithi130 ceph-mon[110068]: osdmap e1172: 8 total, 7 up, 7 in 2024-08-17T17:48:07.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:07 smithi130 ceph-mon[105521]: pgmap v1576: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 2.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 15235/122274 objects degraded (12.460%) 2024-08-17T17:48:07.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:07 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2275486717' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-17T17:48:07.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:07 smithi130 ceph-mon[105521]: osdmap e1172: 8 total, 7 up, 7 in 2024-08-17T17:48:07.708 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 19 16 48926 48910 160.872 0 - 0.00586033 2024-08-17T17:48:07.736 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:07.758 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:07.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:07.859 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:07.880 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:07.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:07.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:07.982 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:08.003 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:08.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:08.024 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:48:08.024 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:48:08.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:07 smithi179 ceph-mon[93519]: pgmap v1576: 89 pgs: 19 active+undersized, 13 active+undersized+degraded, 57 active+clean; 2.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 15235/122274 objects degraded (12.460%) 2024-08-17T17:48:08.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:07 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2275486717' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-17T17:48:08.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:07 smithi179 ceph-mon[93519]: osdmap e1172: 8 total, 7 up, 7 in 2024-08-17T17:48:08.105 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:08.126 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:08.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:08.228 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:08.248 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:48:08.250 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:08.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:08.352 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:08.374 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:08.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:08.476 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:08.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.498 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:08.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:08.599 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:08.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.626 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:08.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:08.708 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T17:48:08.708807+0000 min lat: 0.000907997 max lat: 0.27836 avg lat: 0.00586033 2024-08-17T17:48:08.709 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:48:08.709 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 20 16 48926 48910 152.827 0 - 0.00586033 2024-08-17T17:48:08.727 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:08.761 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:08.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:08.863 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:08.901 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:08 smithi130 ceph-mon[105521]: osdmap e1173: 8 total, 7 up, 7 in 2024-08-17T17:48:08.901 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:08 smithi130 ceph-mon[105521]: osdmap e1174: 8 total, 7 up, 7 in 2024-08-17T17:48:08.901 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:08 smithi130 ceph-mon[110068]: osdmap e1173: 8 total, 7 up, 7 in 2024-08-17T17:48:08.901 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:08 smithi130 ceph-mon[110068]: osdmap e1174: 8 total, 7 up, 7 in 2024-08-17T17:48:08.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:08.913 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:08.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:09.014 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:09.046 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:09.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:09.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:08 smithi179 ceph-mon[93519]: osdmap e1173: 8 total, 7 up, 7 in 2024-08-17T17:48:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:08 smithi179 ceph-mon[93519]: osdmap e1174: 8 total, 7 up, 7 in 2024-08-17T17:48:09.148 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:09.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.194 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:09.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:09.295 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:09.329 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:09.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:09.430 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:09.436 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:48:09.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:09.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:09.541 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:48:09.541 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:48:09.541 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:48:09.541 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:48:09.541 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:48:09.541 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:48:09.541 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:48:09.542 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-08-17T17:48:09.542 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:48:09.542 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:48:09.542 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:48:09.542 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:48:09.542 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:48:09.542 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:48:09.542 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:48:09.563 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:09.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.594 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:09.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:09.695 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:09.709 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 21 16 50173 50157 149.26 25.9792 0.0039909 0.00669335 2024-08-17T17:48:09.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.749 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:09.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:09.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:09 smithi130 ceph-mon[105521]: pgmap v1579: 89 pgs: 9 remapped+peering, 5 peering, 16 active+undersized, 9 active+undersized+degraded, 50 active+clean; 2.9 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 175 MiB/s wr, 2.79k op/s; 8750/144279 objects degraded (6.065%); 11 B/s, 0 objects/s recovering 2024-08-17T17:48:09.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:09 smithi130 ceph-mon[105521]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-17T17:48:09.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:09 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1338024910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:09 smithi130 ceph-mon[110068]: pgmap v1579: 89 pgs: 9 remapped+peering, 5 peering, 16 active+undersized, 9 active+undersized+degraded, 50 active+clean; 2.9 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 175 MiB/s wr, 2.79k op/s; 8750/144279 objects degraded (6.065%); 11 B/s, 0 objects/s recovering 2024-08-17T17:48:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:09 smithi130 ceph-mon[110068]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-17T17:48:09.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:09 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1338024910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:09.851 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:09.881 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:09.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:09.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:09.983 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:10.021 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:10.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:10.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:09 smithi179 ceph-mon[93519]: pgmap v1579: 89 pgs: 9 remapped+peering, 5 peering, 16 active+undersized, 9 active+undersized+degraded, 50 active+clean; 2.9 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 175 MiB/s wr, 2.79k op/s; 8750/144279 objects degraded (6.065%); 11 B/s, 0 objects/s recovering 2024-08-17T17:48:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:09 smithi179 ceph-mon[93519]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-17T17:48:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:09 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1338024910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:10.123 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:10.155 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:10.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:10.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:48:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:48:09] "GET /metrics HTTP/1.1" 200 36622 "" "Prometheus/2.51.0" 2024-08-17T17:48:10.253 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:10.257 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:10.262 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:48:10.263 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:48:10.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:10.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:10.353 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:10.419 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:10.452 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:10.554 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:10.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:10.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:10.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:48:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:48:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:48:10.688 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:10.709 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 22 16 51544 51528 146.37 85.6875 0.00967312 0.00682659 2024-08-17T17:48:10.724 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:10.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:10.730 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:10 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 8750/144279 objects degraded (6.065%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:10.730 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:10 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 8750/144279 objects degraded (6.065%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:10.825 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:10.852 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:10.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:10.954 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:10.992 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:10.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:10.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:10 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 8750/144279 objects degraded (6.065%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:11.094 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:11.130 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:11.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:11.232 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:11.263 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:11.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:11.365 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:11.397 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:11.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:11.499 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:11.536 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:11.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:11.638 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:11.668 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:11.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:11.709 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 23 16 52888 52872 143.658 84 0.0205797 0.00695574 2024-08-17T17:48:11.770 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:11.794 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:11.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:11.896 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:11.913 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:11 smithi130 ceph-mon[105521]: pgmap v1581: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+clean+remapped, 9 remapped+peering, 5 peering, 16 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.0 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 12226/146823 objects degraded (8.327%); 2/146823 objects misplaced (0.001%); 106 B/s, 0 objects/s recovering 2024-08-17T17:48:11.913 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:11 smithi130 ceph-mon[110068]: pgmap v1581: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+clean+remapped, 9 remapped+peering, 5 peering, 16 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.0 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 12226/146823 objects degraded (8.327%); 2/146823 objects misplaced (0.001%); 106 B/s, 0 objects/s recovering 2024-08-17T17:48:11.931 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:11.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:11.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:12.032 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:12.065 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:12.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:12.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:11 smithi179 ceph-mon[93519]: pgmap v1581: 89 pgs: 1 activating+undersized+degraded+remapped, 2 active+clean+remapped, 9 remapped+peering, 5 peering, 16 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.0 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 12226/146823 objects degraded (8.327%); 2/146823 objects misplaced (0.001%); 106 B/s, 0 objects/s recovering 2024-08-17T17:48:12.167 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:12.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.204 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:12.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:12.263 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:48:12.305 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:12.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.359 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:12.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:12.460 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:12.500 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:12.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:12.602 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:12.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.646 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:12.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:12.709 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 24 16 54176 54160 141.026 80.5 0.00603713 0.0070852 2024-08-17T17:48:12.748 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:12.779 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:12.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:12.784 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:48:12.784 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:48:12.784 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2024-08-17T17:48:12.784 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:48:12.784 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:48:12.784 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 config set filestore_inject_stall 3 2024-08-17T17:48:12.881 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:12.916 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:12.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:12.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:13.018 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:13.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.065 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:13.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:13.166 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:13.206 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:13.239 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:48:13.308 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:13.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.348 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:13.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:13.449 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:13.481 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:13.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:13.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:13.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:13.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:13 smithi130 ceph-mon[105521]: pgmap v1582: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 6 active+clean+remapped, 9 remapped+peering, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.0 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 18405/148191 objects degraded (12.420%); 3328/148191 objects misplaced (2.246%); 106 B/s, 0 objects/s recovering 2024-08-17T17:48:13.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:13.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:13.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:13 smithi130 ceph-mon[110068]: pgmap v1582: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 6 active+clean+remapped, 9 remapped+peering, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.0 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 18405/148191 objects degraded (12.420%); 3328/148191 objects misplaced (2.246%); 106 B/s, 0 objects/s recovering 2024-08-17T17:48:13.562 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:13.562 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:13.562 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:13 smithi179 ceph-mon[93519]: pgmap v1582: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 6 active+clean+remapped, 9 remapped+peering, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.0 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 18405/148191 objects degraded (12.420%); 3328/148191 objects misplaced (2.246%); 106 B/s, 0 objects/s recovering 2024-08-17T17:48:13.583 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:13.613 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:13.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:13.710 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 25 16 55441 55425 138.545 79.0625 0.00711804 0.00721082 2024-08-17T17:48:13.715 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:13.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.766 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:13.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:13.867 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:13.909 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:13.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:13.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:14.011 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:14.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.049 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:14.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:14.150 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:14.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.190 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:14.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:14.292 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:14.320 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:14.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:14.408 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:48:14.408 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:48:14.408 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T17:48:14.408 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T17:48:14.408 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:48:14.422 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:14.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.471 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:14.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:14.572 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:14 smithi179 ceph-mon[93519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:48:14.610 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:14.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:14 smithi130 ceph-mon[105521]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:48:14.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:14 smithi130 ceph-mon[110068]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-08-17T17:48:14.709 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 26 16 56656 56640 136.137 75.9375 0.01653 0.00733936 2024-08-17T17:48:14.712 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:14.749 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:14.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:14.788 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:14.850 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:14.883 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:14.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:14.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:14.985 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:15.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.025 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:15.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:15.126 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:15.171 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:15.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:15.228 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:48:15.272 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:15.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.308 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:15.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:15.409 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:15.446 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:15.532 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T17:48:15.548 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:15.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.577 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:15.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:15.678 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:15.710 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 27 16 57873 57857 133.911 76.0625 0.00919652 0.00746253 2024-08-17T17:48:15.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:15.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:15.817 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:15.838 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:15.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:15.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:15 smithi179 ceph-mon[93519]: pgmap v1583: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s; 26002/163659 objects degraded (15.888%); 15694/163659 objects misplaced (9.589%); 4.0 MiB/s, 65 objects/s recovering 2024-08-17T17:48:15.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:15 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/67890004' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:15 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:15 smithi130 ceph-mon[105521]: pgmap v1583: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s; 26002/163659 objects degraded (15.888%); 15694/163659 objects misplaced (9.589%); 4.0 MiB/s, 65 objects/s recovering 2024-08-17T17:48:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:15 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/67890004' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:15 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:15 smithi130 ceph-mon[110068]: pgmap v1583: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 111 MiB/s wr, 1.77k op/s; 26002/163659 objects degraded (15.888%); 15694/163659 objects misplaced (9.589%); 4.0 MiB/s, 65 objects/s recovering 2024-08-17T17:48:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:15 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/67890004' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:15.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:15 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:15.940 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:15.979 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:15.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:15.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:16.081 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:16.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.122 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:16.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:16.223 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:16.260 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:16.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:16.361 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:16.392 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:16.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:16.493 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:16.530 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:16.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:16 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:48:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:16 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:16 smithi179 ceph-mon[93519]: osdmap e1175: 8 total, 7 up, 7 in 2024-08-17T17:48:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 26002/163659 objects degraded (15.888%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:16.632 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:16.665 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:16.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:16.710 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 28 16 59124 59108 131.921 78.1875 0.0152612 0.00757487 2024-08-17T17:48:16.749 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:48:16.749 INFO:teuthology.orchestra.run.smithi179.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:48:16.749 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:48:16.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:16.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.807 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:16.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:16.908 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:16.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:16 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:48:16.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:16 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:16.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:16 smithi130 ceph-mon[105521]: osdmap e1175: 8 total, 7 up, 7 in 2024-08-17T17:48:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 26002/163659 objects degraded (15.888%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:16 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:48:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:16 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:16 smithi130 ceph-mon[110068]: osdmap e1175: 8 total, 7 up, 7 in 2024-08-17T17:48:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 26002/163659 objects degraded (15.888%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:16.933 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:16.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:16.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:17.034 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:17.068 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:17.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:17.170 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:17.203 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:17.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:17.305 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:17.337 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:17.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:17.438 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:17.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.478 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:17.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:17.580 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:17.618 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:17.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:17.710 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 29 16 60417 60401 130.159 80.8125 0.00651386 0.00767771 2024-08-17T17:48:17.720 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:17.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.757 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:17.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:17.761 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:17 smithi130 ceph-mon[105521]: pgmap v1585: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 50 MiB/s wr, 807 op/s; 26002/163659 objects degraded (15.888%); 15694/163659 objects misplaced (9.589%); 3.9 MiB/s, 63 objects/s recovering 2024-08-17T17:48:17.761 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:17 smithi130 ceph-mon[110068]: pgmap v1585: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 50 MiB/s wr, 807 op/s; 26002/163659 objects degraded (15.888%); 15694/163659 objects misplaced (9.589%); 3.9 MiB/s, 63 objects/s recovering 2024-08-17T17:48:17.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:17 smithi179 ceph-mon[93519]: pgmap v1585: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 50 MiB/s wr, 807 op/s; 26002/163659 objects degraded (15.888%); 15694/163659 objects misplaced (9.589%); 3.9 MiB/s, 63 objects/s recovering 2024-08-17T17:48:17.859 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:17.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:17.895 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:17.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:17.996 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:18.027 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:18.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:18.129 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:18.158 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:18.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:18.260 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:18.291 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:18.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.316 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:18.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:18.417 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:18.452 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:18.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:18.554 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:18.592 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:18.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:18.694 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:18.712 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 30 16 61681 61665 128.446 79 0.00707635 0.00777934 2024-08-17T17:48:18.736 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:18.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:18.838 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:18.878 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:18.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:18.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:18.980 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:19.010 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:19.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:19.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:19.141 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:19.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:19.243 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:19.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.282 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:19.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:19.383 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:19.412 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:19.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:19.514 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:19.554 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:19.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:19.603 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-08-17T17:48:19.638 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:48:19.638 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:48:19.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:19.671 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:48:19.671 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:48:19.683 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:19.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:19.703 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:48:19.703 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:48:19.703 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:19 smithi130 ceph-mon[105521]: pgmap v1586: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.7 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 27192/180129 objects degraded (15.096%); 16536/180129 objects misplaced (9.180%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:48:19.704 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:19 smithi130 ceph-mon[110068]: pgmap v1586: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.7 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 27192/180129 objects degraded (15.096%); 16536/180129 objects misplaced (9.180%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:48:19.712 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 31 16 62956 62940 126.873 79.6875 0.013606 0.00787597 2024-08-17T17:48:19.738 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:48:19.738 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:48:19.767 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:48:19.767 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:48:19.784 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:19.801 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:48:19.801 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:48:19.818 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:19.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:19.831 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:48:19.837 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:48:19.838 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:48:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:19 smithi179 ceph-mon[93519]: pgmap v1586: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.7 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 27192/180129 objects degraded (15.096%); 16536/180129 objects misplaced (9.180%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:48:19.920 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:19.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:19.948 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:19.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:20.049 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:20.089 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:20.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:20.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:48:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:48:19] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-08-17T17:48:20.191 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:20.216 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:20.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:20.318 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:20.359 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:20.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:20.461 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:20.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.492 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:20.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:20.530 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:48:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:48:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:48:20.593 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:20.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.618 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:20.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:20.712 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 32 16 63971 63955 124.89 63.4375 0.0283603 0.0080011 2024-08-17T17:48:20.719 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:20.754 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:20.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4120697827' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:20.856 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:20.885 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:20.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:20.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4120697827' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4120697827' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:20.987 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:21.031 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:21.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:21.133 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:21.175 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:21.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:21.277 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:21.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.325 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:21.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:21.426 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:21.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.463 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:21.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:21.564 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:21.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:21.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:21.703 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:21.712 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 33 16 65061 65045 123.17 68.125 0.0179774 0.00811353 2024-08-17T17:48:21.742 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:21.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:21 smithi179 ceph-mon[93519]: pgmap v1587: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 27565/182475 objects degraded (15.106%); 16343/182475 objects misplaced (8.956%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:48:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 27565/182475 objects degraded (15.106%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:21 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:48:21.844 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:21.875 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:21.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:21.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:21.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:21 smithi130 ceph-mon[105521]: pgmap v1587: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 27565/182475 objects degraded (15.106%); 16343/182475 objects misplaced (8.956%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:48:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 27565/182475 objects degraded (15.106%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:21 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:48:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:21 smithi130 ceph-mon[110068]: pgmap v1587: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 27565/182475 objects degraded (15.106%); 16343/182475 objects misplaced (8.956%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:48:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 27565/182475 objects degraded (15.106%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:21 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:48:21.977 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:22.011 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:22.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:22.112 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:22.148 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:22.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:22.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:22.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.290 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:22.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:22.391 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:22.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.466 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:22.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:22.497 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:48:22.497 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:48:22.497 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2024-08-17T17:48:22.497 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2024-08-17T17:48:22.497 DEBUG:teuthology.orchestra.run.smithi130:osd.0> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.0.service 2024-08-17T17:48:22.500 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl start ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.0 2024-08-17T17:48:22.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:22.603 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:22.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:22.705 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:22.712 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 34 16 66280 66264 121.788 76.1875 0.0165872 0.0082057 2024-08-17T17:48:22.735 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:22.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:22.837 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:22.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:22.880 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:22.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:22.916 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:22 smithi130 systemd[1]: Starting Ceph osd.0 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T17:48:22.981 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:23.014 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:23.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:23.116 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:23.151 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:23.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:23.253 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:23.290 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:23.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:23.392 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:23.414 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:23.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:23.516 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:23.548 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:23.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:23.650 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:23.688 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:23.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:23.712 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 35 16 67402 67386 120.312 70.125 0.0168313 0.00830693 2024-08-17T17:48:23.790 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:23.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:23 smithi130 ceph-mon[105521]: pgmap v1588: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.8 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 27565/184551 objects degraded (14.936%); 16680/184551 objects misplaced (9.038%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:48:23.811 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:23 smithi130 ceph-mon[110068]: pgmap v1588: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.8 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 27565/184551 objects degraded (14.936%); 16680/184551 objects misplaced (9.038%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:48:23.812 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:48:23.812 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:48:23.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.839 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:23.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:23.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:23 smithi179 ceph-mon[93519]: pgmap v1588: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 3.8 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 27565/184551 objects degraded (14.936%); 16680/184551 objects misplaced (9.038%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:48:23.940 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:23.966 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:23.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:23.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:24.068 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:24.095 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:24.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:24.197 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:24.229 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:24.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:24.331 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:24.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.393 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:24.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:24.403 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:24 smithi130 podman[176570]: 2024-08-17 17:48:24.329933026 +0000 UTC m=+0.276717625 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:48:24.494 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:24.528 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:24.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:24.630 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:24.665 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:24.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:24.712 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 36 16 68471 68455 118.825 66.8125 0.0125323 0.00841021 2024-08-17T17:48:24.766 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:24.799 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:24.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:24.805 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:24 smithi130 podman[176570]: 2024-08-17 17:48:24.46766587 +0000 UTC m=+0.414450463 container create a28f6a213d5b00a607402d11b3bcfe1231739960488eb6dc5f393c65854a2d1a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:48:24.901 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:24.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:24.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:24.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:24.997 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:48:24.997 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:48:25.029 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:25.061 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:25.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:25.070 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:24 smithi130 podman[176570]: 2024-08-17 17:48:24.804855967 +0000 UTC m=+0.751640559 container init a28f6a213d5b00a607402d11b3bcfe1231739960488eb6dc5f393c65854a2d1a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-08-17T17:48:25.071 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:24 smithi130 podman[176570]: 2024-08-17 17:48:24.815744557 +0000 UTC m=+0.762529167 container start a28f6a213d5b00a607402d11b3bcfe1231739960488eb6dc5f393c65854a2d1a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-08-17T17:48:25.071 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:24 smithi130 podman[176570]: 2024-08-17 17:48:24.888196786 +0000 UTC m=+0.834981372 container attach a28f6a213d5b00a607402d11b3bcfe1231739960488eb6dc5f393c65854a2d1a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-08-17T17:48:25.071 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[176652]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:48:25.071 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 bash[176570]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:48:25.163 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:25.207 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:25.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:25.227 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:25.309 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:25.358 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:25.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:25.375 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[176652]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:48:25.375 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 bash[176570]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:48:25.375 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[176652]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:48:25.375 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[176652]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:48:25.375 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 bash[176570]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:48:25.376 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 bash[176570]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:48:25.376 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[176652]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:48:25.376 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 bash[176570]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:48:25.376 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[176652]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:48:25.376 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 bash[176570]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:48:25.376 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[176652]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-17T17:48:25.376 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 bash[176570]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-17T17:48:25.460 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:25.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.511 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:25.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:25.611 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:25.644 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:25.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:25 smithi130 ceph-mon[105521]: pgmap v1589: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 29071/199938 objects degraded (14.540%); 16774/199938 objects misplaced (8.390%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:48:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:25 smithi130 ceph-mon[110068]: pgmap v1589: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 29071/199938 objects degraded (14.540%); 16774/199938 objects misplaced (8.390%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:48:25.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[176652]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:48:25.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 bash[176570]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:48:25.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[176652]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:48:25.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 bash[176570]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:48:25.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[176652]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-17T17:48:25.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 bash[176570]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-17T17:48:25.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[176652]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:48:25.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 bash[176570]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:48:25.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate[176652]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-17T17:48:25.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 bash[176570]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-17T17:48:25.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:25 smithi130 podman[176570]: 2024-08-17 17:48:25.571107225 +0000 UTC m=+1.517891823 container died a28f6a213d5b00a607402d11b3bcfe1231739960488eb6dc5f393c65854a2d1a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, ceph=True) 2024-08-17T17:48:25.713 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 37 16 69571 69555 117.47 68.75 0.0272879 0.00850643 2024-08-17T17:48:25.745 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:25.774 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:25.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:25.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:48:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:25 smithi179 ceph-mon[93519]: pgmap v1589: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 29071/199938 objects degraded (14.540%); 16774/199938 objects misplaced (8.390%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:48:25.876 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:25.905 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:25.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:25.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:26.007 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:26.046 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:26.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:26.148 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:26.185 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:26.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:26.288 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:26.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.337 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:26.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:26.438 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:26.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:26.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:26.574 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:26.614 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:26.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:26.623 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 29071/199938 objects degraded (14.540%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:26.623 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:26 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:26.623 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:26 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:26.627 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 29071/199938 objects degraded (14.540%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:26.627 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:26 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:26.627 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:26 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:26.669 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:48:26.713 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 38 16 70695 70679 116.227 70.25 0.0072524 0.00859558 2024-08-17T17:48:26.716 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:26.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.746 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:26.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 29071/199938 objects degraded (14.540%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:26 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:26 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:26.846 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:26.891 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:26.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:26.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:26.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:27.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.042 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:27.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:27.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:27.182 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:27.284 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:27.315 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:27.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:27.417 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:27.443 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:27.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:27.544 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:27.573 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:27.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:27 smithi130 ceph-mon[110068]: pgmap v1590: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 29071/199938 objects degraded (14.540%); 16774/199938 objects misplaced (8.390%); 6.5 MiB/s, 103 objects/s recovering 2024-08-17T17:48:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:27 smithi130 ceph-mon[105521]: pgmap v1590: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 29071/199938 objects degraded (14.540%); 16774/199938 objects misplaced (8.390%); 6.5 MiB/s, 103 objects/s recovering 2024-08-17T17:48:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:27.668 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:27 smithi130 podman[176859]: 2024-08-17 17:48:27.384872144 +0000 UTC m=+1.808496279 container remove a28f6a213d5b00a607402d11b3bcfe1231739960488eb6dc5f393c65854a2d1a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:48:27.675 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:27.711 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:27.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:27.713 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 39 16 71828 71812 115.063 70.8125 0.00507166 0.00868533 2024-08-17T17:48:27.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:27 smithi179 ceph-mon[93519]: pgmap v1590: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 29071/199938 objects degraded (14.540%); 16774/199938 objects misplaced (8.390%); 6.5 MiB/s, 103 objects/s recovering 2024-08-17T17:48:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:27.856 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:27.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:27.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:27.958 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:28.000 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:28.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:28.102 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:28.146 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:28.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:28.249 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:28.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.283 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:28.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:28.384 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:28.425 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:28.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:28.526 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:28.555 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:28.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:28.591 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:28.609 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:28.657 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:28.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:28.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:28.714 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T17:48:28.714400+0000 min lat: 0.000907997 max lat: 2.20072 avg lat: 0.00876979 2024-08-17T17:48:28.714 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:48:28.714 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 40 16 72954 72938 113.944 70.375 0.0285594 0.00876979 2024-08-17T17:48:28.787 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:28.816 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:28.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:28.918 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:28.948 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:28.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:28.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:29.050 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:29.087 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:29.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:29.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:28 smithi130 podman[177300]: 2024-08-17 17:48:28.843729825 +0000 UTC m=+0.178518226 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:48:29.189 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:29.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.230 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:29.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:29.331 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:29.376 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:29.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:29.478 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:29.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.506 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:29.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:29.607 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:29.642 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:29.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:29.714 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 41 16 74038 74022 112.817 67.75 0.0228308 0.00885783 2024-08-17T17:48:29.744 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:29.784 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:29.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:29.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:29 smithi130 ceph-mon[105521]: pgmap v1591: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s; 30096/214437 objects degraded (14.035%); 17412/214437 objects misplaced (8.120%); 8.0 MiB/s, 128 objects/s recovering 2024-08-17T17:48:29.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:29 smithi130 ceph-mon[110068]: pgmap v1591: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s; 30096/214437 objects degraded (14.035%); 17412/214437 objects misplaced (8.120%); 8.0 MiB/s, 128 objects/s recovering 2024-08-17T17:48:29.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:29 smithi179 ceph-mon[93519]: pgmap v1591: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s; 30096/214437 objects degraded (14.035%); 17412/214437 objects misplaced (8.120%); 8.0 MiB/s, 128 objects/s recovering 2024-08-17T17:48:29.886 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:29.914 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:29.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:29.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:30.016 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:30.045 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:30.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:30.147 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:48:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:48:29] "GET /metrics HTTP/1.1" 200 36723 "" "Prometheus/2.51.0" 2024-08-17T17:48:30.184 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:30.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:30.286 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:30.315 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:30.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:30.417 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:30.450 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:30.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:30.552 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:30.588 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:30.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:30.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:48:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:48:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:48:30.690 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:30.714 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 42 16 75007 74991 111.573 60.5625 0.00381264 0.00895419 2024-08-17T17:48:30.716 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:30.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:30.818 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:30.847 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:30.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:30.949 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:30.971 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:30.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:30.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:31.073 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:31.102 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:31.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:31.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:31.234 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:31.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:31.338 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:31.369 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:31.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:31.471 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:31.516 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:31.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:31.618 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:31.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.658 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:31.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:31.715 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 43 16 75977 75961 110.385 60.625 0.0117248 0.00905348 2024-08-17T17:48:31.754 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:31 smithi130 ceph-mon[105521]: pgmap v1592: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 30466/216621 objects degraded (14.064%); 17210/216621 objects misplaced (7.945%); 7.4 MiB/s, 118 objects/s recovering 2024-08-17T17:48:31.754 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 30466/216621 objects degraded (14.064%), 13 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:48:31.754 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:31 smithi130 ceph-mon[110068]: pgmap v1592: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 30466/216621 objects degraded (14.064%); 17210/216621 objects misplaced (7.945%); 7.4 MiB/s, 118 objects/s recovering 2024-08-17T17:48:31.754 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 30466/216621 objects degraded (14.064%), 13 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:48:31.754 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:31 smithi130 podman[177300]: 2024-08-17 17:48:31.490251173 +0000 UTC m=+2.825039543 container create 691ff0a9846456e8efbe04563345e30cbcceeb592b170c12c38b6b79112a3637 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-08-17T17:48:31.759 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:31.785 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:31.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:31.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:31 smithi179 ceph-mon[93519]: pgmap v1592: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 30466/216621 objects degraded (14.064%); 17210/216621 objects misplaced (7.945%); 7.4 MiB/s, 118 objects/s recovering 2024-08-17T17:48:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 30466/216621 objects degraded (14.064%), 13 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:48:31.887 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:31.917 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:31.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:31.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:32.019 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:32.049 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:32.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:32.151 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:32.166 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:31 smithi130 podman[177300]: 2024-08-17 17:48:31.862195931 +0000 UTC m=+3.196984310 container init 691ff0a9846456e8efbe04563345e30cbcceeb592b170c12c38b6b79112a3637 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-08-17T17:48:32.167 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:31 smithi130 podman[177300]: 2024-08-17 17:48:31.873458144 +0000 UTC m=+3.208246521 container start 691ff0a9846456e8efbe04563345e30cbcceeb592b170c12c38b6b79112a3637 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-08-17T17:48:32.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.182 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:32.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:32.283 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:32.314 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:32.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:32.416 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:32.459 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:32.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:32.561 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:32.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:32.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:32.690 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:32.715 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 44 16 77131 77115 109.515 72.125 0.0154377 0.00912559 2024-08-17T17:48:32.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.722 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:32.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:32.823 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:32.853 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:32.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:32.954 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:32.983 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:32.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:32.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:33.017 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:48:33.085 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:33.165 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:33.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:33.230 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:33 smithi130 bash[177300]: 691ff0a9846456e8efbe04563345e30cbcceeb592b170c12c38b6b79112a3637 2024-08-17T17:48:33.231 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:33 smithi130 systemd[1]: Started Ceph osd.0 for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T17:48:33.267 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:33.295 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:33.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:33.397 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:33.435 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:33.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:33.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:33.565 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:33.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:33.567 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:33 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[177501]: 2024-08-17T17:48:33.266+0000 7fbbac67c640 -1 Falling back to public interface 2024-08-17T17:48:33.582 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:48:33.666 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:33.705 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:33.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:33.715 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 45 16 78114 78098 108.447 61.4375 0.0169947 0.00921547 2024-08-17T17:48:33.807 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:33.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.843 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:33.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:33 smithi179 ceph-mon[93519]: pgmap v1593: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s; 30466/218541 objects degraded (13.941%); 17512/218541 objects misplaced (8.013%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:48:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:33 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2862320600' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:33 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:33.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:33 smithi130 ceph-mon[105521]: pgmap v1593: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s; 30466/218541 objects degraded (13.941%); 17512/218541 objects misplaced (8.013%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:48:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:33 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2862320600' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:33 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:33 smithi130 ceph-mon[110068]: pgmap v1593: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 63 MiB/s wr, 1.00k op/s; 30466/218541 objects degraded (13.941%); 17512/218541 objects misplaced (8.013%); 6.4 MiB/s, 102 objects/s recovering 2024-08-17T17:48:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:33 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2862320600' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:33 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:48:33.943 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:33.984 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:33.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:33.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:34.085 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:34.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.117 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:34.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:34.218 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:34.250 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:34.352 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:34.394 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:34.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:34.496 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:34.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.527 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:34.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:34.627 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:34.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.653 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:34.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:34.715 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 46 16 79140 79124 107.483 64.125 0.0770601 0.0092983 2024-08-17T17:48:34.754 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:34.797 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:34.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:34 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:34 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:48:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:34 smithi179 ceph-mon[93519]: osdmap e1176: 8 total, 7 up, 7 in 2024-08-17T17:48:34.898 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:34 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:34.898 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:34 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:48:34.898 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:34 smithi130 ceph-mon[105521]: osdmap e1176: 8 total, 7 up, 7 in 2024-08-17T17:48:34.898 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:34 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:34.899 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:34 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:48:34.899 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:34 smithi130 ceph-mon[110068]: osdmap e1176: 8 total, 7 up, 7 in 2024-08-17T17:48:34.899 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:34.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:34.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:34.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:35.029 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:35.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.061 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:35.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:35.162 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:35.190 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:35.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:35.291 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:35.320 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:35.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:35.422 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:35.465 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:35.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:35.567 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:35.600 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:35.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:35.702 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:35.716 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 47 16 80279 80263 106.711 71.1875 0.0180013 0.00936548 2024-08-17T17:48:35.740 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:35.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:35.805 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:35 smithi130 ceph-mon[105521]: pgmap v1595: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s; 31795/232290 objects degraded (13.688%); 17391/232290 objects misplaced (7.487%); 8.7 MiB/s, 139 objects/s recovering 2024-08-17T17:48:35.805 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:35 smithi130 ceph-mon[110068]: pgmap v1595: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s; 31795/232290 objects degraded (13.688%); 17391/232290 objects misplaced (7.487%); 8.7 MiB/s, 139 objects/s recovering 2024-08-17T17:48:35.842 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:35 smithi179 ceph-mon[93519]: pgmap v1595: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s; 31795/232290 objects degraded (13.688%); 17391/232290 objects misplaced (7.487%); 8.7 MiB/s, 139 objects/s recovering 2024-08-17T17:48:35.870 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:35.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:35.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:35.972 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:36.015 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:36.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:36.117 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:36.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.151 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:36.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:36.252 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:36.285 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:36.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:36.387 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:36.451 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:36.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:36.552 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:36.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:36.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:36.638 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 31795/232290 objects degraded (13.688%), 13 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:48:36.639 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 31795/232290 objects degraded (13.688%), 13 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:48:36.687 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:36.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:36.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:36.717 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 48 16 81326 81310 105.849 65.4375 0.0112742 0.00944223 2024-08-17T17:48:36.815 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:36.842 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:36.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:36.945 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:36.973 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:36.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:36.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:37.075 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:37.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 31795/232290 objects degraded (13.688%), 13 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:48:37.113 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:37.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:37.195 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[177501]: 2024-08-17T17:48:37.056+0000 7fbbac67c640 -1 osd.0 1163 log_to_monitors true 2024-08-17T17:48:37.195 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-08-17T17:48:37.214 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:37.220 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:48:37.235 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:48:37.235 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:48:37.248 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:37.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:37.268 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:48:37.268 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:48:37.311 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:48:37.311 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:48:37.339 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:48:37.339 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:48:37.350 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:37.372 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:48:37.372 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:48:37.379 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:37.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:37.403 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:48:37.404 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:48:37.435 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:48:37.481 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:37.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.549 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:37.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:37.650 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:37.680 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:37.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:37.719 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 49 16 82285 82269 104.908 59.9375 0.0064075 0.00952607 2024-08-17T17:48:37.742 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:48:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T17:48:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.45158.0:82293 45.b e1177/1173 45:dac44ee3:::benchmark_data_smithi130_172551_object82292:head v 1177'5179, mlcod=0'0)", 2024-08-17T17:48:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:37.733245+0000", 2024-08-17T17:48:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0078292540000000008, 2024-08-17T17:48:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0078549480000000005, 2024-08-17T17:48:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:48:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:48:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:48:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:48:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:48:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.733245+0000", 2024-08-17T17:48:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:48:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.733245+0000", 2024-08-17T17:48:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:48:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.733245+0000", 2024-08-17T17:48:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7599999999999997e-07 2024-08-17T17:48:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:48:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.733268+0000", 2024-08-17T17:48:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2503000000000001e-05 2024-08-17T17:48:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:48:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.733276+0000", 2024-08-17T17:48:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.695e-06 2024-08-17T17:48:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.733281+0000", 2024-08-17T17:48:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3040000000000004e-06 2024-08-17T17:48:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:48:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.733379+0000", 2024-08-17T17:48:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8523999999999994e-05 2024-08-17T17:48:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:48:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.733404+0000", 2024-08-17T17:48:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.429e-05 2024-08-17T17:48:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:48:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740999+0000", 2024-08-17T17:48:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0075950719999999996 2024-08-17T17:48:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:48:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:82273 45.3 45:ca10e3ce:::benchmark_data_smithi130_172551_object82272:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:48:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:37.708357+0000", 2024-08-17T17:48:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.032717698000000003, 2024-08-17T17:48:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.032811970000000003, 2024-08-17T17:48:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:48:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:48:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:48:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:48:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:48:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:48:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 82273 2024-08-17T17:48:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:48:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:48:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.708357+0000", 2024-08-17T17:48:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:48:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.708357+0000", 2024-08-17T17:48:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:48:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.708358+0000", 2024-08-17T17:48:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.418e-06 2024-08-17T17:48:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:48:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.708384+0000", 2024-08-17T17:48:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5689000000000001e-05 2024-08-17T17:48:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:48:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.708393+0000", 2024-08-17T17:48:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6190000000000006e-06 2024-08-17T17:48:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.708401+0000", 2024-08-17T17:48:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3289999999999996e-06 2024-08-17T17:48:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:48:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.722494+0000", 2024-08-17T17:48:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014093006 2024-08-17T17:48:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:48:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.722830+0000", 2024-08-17T17:48:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00033663399999999999 2024-08-17T17:48:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:48:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.723218+0000", 2024-08-17T17:48:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00038737800000000002 2024-08-17T17:48:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.732486+0000", 2024-08-17T17:48:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0092684220000000001 2024-08-17T17:48:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:48:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740661+0000", 2024-08-17T17:48:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0081750139999999995 2024-08-17T17:48:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:48:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:82297 45.d 45:b3c0e3c7:::benchmark_data_smithi130_172551_object82296:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:48:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:37.736955+0000", 2024-08-17T17:48:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0041188580000000004, 2024-08-17T17:48:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0042922389999999998, 2024-08-17T17:48:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:48:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:48:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:48:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:48:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:48:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:48:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 82297 2024-08-17T17:48:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:48:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:48:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.736955+0000", 2024-08-17T17:48:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:48:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.736955+0000", 2024-08-17T17:48:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:48:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.736957+0000", 2024-08-17T17:48:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.15e-06 2024-08-17T17:48:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:48:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.736980+0000", 2024-08-17T17:48:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3326999999999999e-05 2024-08-17T17:48:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:48:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.736989+0000", 2024-08-17T17:48:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4089999999999992e-06 2024-08-17T17:48:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.736996+0000", 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4330000000000001e-06 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.737745+0000", 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00074916499999999999 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.737868+0000", 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000123249 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 6", 2024-08-17T17:48:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.737990+0000", 2024-08-17T17:48:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000122187 2024-08-17T17:48:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:48:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.45158.0:82294 45.f e1177/1167 45:f7be7276:::benchmark_data_smithi130_172551_object82293:head v 1177'5128, mlcod=1177'5126)", 2024-08-17T17:48:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:37.738618+0000", 2024-08-17T17:48:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0024560509999999999, 2024-08-17T17:48:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026938769999999999, 2024-08-17T17:48:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:48:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:48:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:48:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:48:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:48:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.738618+0000", 2024-08-17T17:48:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:48:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.738618+0000", 2024-08-17T17:48:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:48:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.738619+0000", 2024-08-17T17:48:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4600000000000003e-07 2024-08-17T17:48:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:48:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.738643+0000", 2024-08-17T17:48:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4417999999999999e-05 2024-08-17T17:48:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:48:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.738655+0000", 2024-08-17T17:48:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1392999999999999e-05 2024-08-17T17:48:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.738662+0000", 2024-08-17T17:48:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3089999999999998e-06 2024-08-17T17:48:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:48:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.738733+0000", 2024-08-17T17:48:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0878999999999997e-05 2024-08-17T17:48:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:48:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.738752+0000", 2024-08-17T17:48:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9355e-05 2024-08-17T17:48:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:48:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:82300 45.0 45:08f9121e:::benchmark_data_smithi130_172551_object82299:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:48:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:37.740807+0000", 2024-08-17T17:48:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.00026712600000000002, 2024-08-17T17:48:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00055730999999999999, 2024-08-17T17:48:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:48:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:48:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:48:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:48:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:48:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:48:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 82300 2024-08-17T17:48:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:48:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:48:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740807+0000", 2024-08-17T17:48:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:48:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740807+0000", 2024-08-17T17:48:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:48:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740808+0000", 2024-08-17T17:48:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.28e-06 2024-08-17T17:48:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740833+0000", 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4692e-05 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740860+0000", 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6917999999999999e-05 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740872+0000", 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2264e-05 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:48:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.741007+0000", 2024-08-17T17:48:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000134306 2024-08-17T17:48:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:48:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.741200+0000", 2024-08-17T17:48:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00019319899999999999 2024-08-17T17:48:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:48:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.741361+0000", 2024-08-17T17:48:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016108499999999999 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:82302 45.5 45:abc7be09:::benchmark_data_smithi130_172551_object82301:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:37.740908+0000", 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.000166236, 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00051960999999999999, 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:48:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:48:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:48:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 82302 2024-08-17T17:48:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:48:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:48:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740908+0000", 2024-08-17T17:48:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:48:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740908+0000", 2024-08-17T17:48:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740909+0000", 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4000000000000002e-07 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740934+0000", 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5867000000000002e-05 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740946+0000", 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1688999999999999e-05 2024-08-17T17:48:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740952+0000", 2024-08-17T17:48:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3539999999999996e-06 2024-08-17T17:48:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:48:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.741197+0000", 2024-08-17T17:48:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000244078 2024-08-17T17:48:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:48:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.741321+0000", 2024-08-17T17:48:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000124535 2024-08-17T17:48:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:48:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:82289 45.3 45:ce2467b6:::benchmark_data_smithi130_172551_object82288:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:48:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:37.726070+0000", 2024-08-17T17:48:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.015003805, 2024-08-17T17:48:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015421093, 2024-08-17T17:48:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:48:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:48:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:48:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:48:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:48:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:48:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 82289 2024-08-17T17:48:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:48:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:48:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.726070+0000", 2024-08-17T17:48:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:48:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.726070+0000", 2024-08-17T17:48:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:48:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.726071+0000", 2024-08-17T17:48:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3099999999999998e-07 2024-08-17T17:48:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:48:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.726090+0000", 2024-08-17T17:48:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9507000000000001e-05 2024-08-17T17:48:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:48:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.726198+0000", 2024-08-17T17:48:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000107978 2024-08-17T17:48:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.726205+0000", 2024-08-17T17:48:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5209999999999999e-06 2024-08-17T17:48:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:48:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.729897+0000", 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003692145 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.730016+0000", 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011933 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.730152+0000", 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013585 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:48:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.738699+0000", 2024-08-17T17:48:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0085466880000000002 2024-08-17T17:48:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:48:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.740667+0000", 2024-08-17T17:48:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001968015 2024-08-17T17:48:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:48:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:82291 45.5 45:a6880f7c:::benchmark_data_smithi130_172551_object82290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:48:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:37.731689+0000", 2024-08-17T17:48:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0093848590000000006, 2024-08-17T17:48:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0098842200000000009, 2024-08-17T17:48:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:48:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:48:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:48:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:48:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:48:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:48:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 82291 2024-08-17T17:48:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:48:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:48:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.731689+0000", 2024-08-17T17:48:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:48:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.731689+0000", 2024-08-17T17:48:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:48:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.731691+0000", 2024-08-17T17:48:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2100000000000001e-06 2024-08-17T17:48:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:48:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.731715+0000", 2024-08-17T17:48:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4049999999999998e-05 2024-08-17T17:48:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:48:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.731725+0000", 2024-08-17T17:48:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9960000000000005e-06 2024-08-17T17:48:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:48:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.731731+0000", 2024-08-17T17:48:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.494e-06 2024-08-17T17:48:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:48:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.731835+0000", 2024-08-17T17:48:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010338199999999999 2024-08-17T17:48:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:48:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.731959+0000", 2024-08-17T17:48:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000124325 2024-08-17T17:48:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2", 2024-08-17T17:48:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.732090+0000", 2024-08-17T17:48:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000131273 2024-08-17T17:48:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:48:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:37.736887+0000", 2024-08-17T17:48:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0047968220000000001 2024-08-17T17:48:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:48:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T17:48:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 8 2024-08-17T17:48:37.780 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:48:37.781 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:37 smithi130 ceph-mon[105521]: pgmap v1596: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s; 31795/232290 objects degraded (13.688%); 17391/232290 objects misplaced (7.487%); 8.7 MiB/s, 139 objects/s recovering 2024-08-17T17:48:37.781 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:37 smithi130 ceph-mon[105521]: from='osd.0 [v2:172.21.15.130:6802/2575115774,v1:172.21.15.130:6803/2575115774]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:48:37.781 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:37 smithi130 ceph-mon[105521]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:48:37.781 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:37 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1916447772' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:37.781 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:37 smithi130 ceph-mon[110068]: pgmap v1596: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s; 31795/232290 objects degraded (13.688%); 17391/232290 objects misplaced (7.487%); 8.7 MiB/s, 139 objects/s recovering 2024-08-17T17:48:37.781 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:37 smithi130 ceph-mon[110068]: from='osd.0 [v2:172.21.15.130:6802/2575115774,v1:172.21.15.130:6803/2575115774]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:48:37.781 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:37 smithi130 ceph-mon[110068]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:48:37.781 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:37 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1916447772' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:37.782 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 17:48:37 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[177501]: 2024-08-17T17:48:37.657+0000 7fbba4426640 -1 osd.0 1163 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:48:37.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:37.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.826 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:37.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:37.927 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:37.960 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:37.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:37.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:38.062 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:38.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:37 smithi179 ceph-mon[93519]: pgmap v1596: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 4.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 67 MiB/s wr, 1.08k op/s; 31795/232290 objects degraded (13.688%); 17391/232290 objects misplaced (7.487%); 8.7 MiB/s, 139 objects/s recovering 2024-08-17T17:48:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:37 smithi179 ceph-mon[93519]: from='osd.0 [v2:172.21.15.130:6802/2575115774,v1:172.21.15.130:6803/2575115774]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:48:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:37 smithi179 ceph-mon[93519]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:48:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:37 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1916447772' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:38.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:38.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:38.198 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:38.226 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:38.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:38.328 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:38.350 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:38.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:38.452 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:38.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.492 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:38.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:38.593 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:38.624 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:38.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:38.719 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 50 16 83271 83255 104.043 61.625 0.00738105 0.00959756 2024-08-17T17:48:38.726 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:38.748 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:38.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:38.850 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:38.871 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:38.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:38.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:38 smithi130 ceph-mon[105521]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:48:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:38 smithi130 ceph-mon[105521]: from='osd.0 [v2:172.21.15.130:6802/2575115774,v1:172.21.15.130:6803/2575115774]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:48:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:38 smithi130 ceph-mon[105521]: osdmap e1177: 8 total, 7 up, 7 in 2024-08-17T17:48:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:38 smithi130 ceph-mon[105521]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:48:38.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:38 smithi130 ceph-mon[110068]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:48:38.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:38 smithi130 ceph-mon[110068]: from='osd.0 [v2:172.21.15.130:6802/2575115774,v1:172.21.15.130:6803/2575115774]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:48:38.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:38 smithi130 ceph-mon[110068]: osdmap e1177: 8 total, 7 up, 7 in 2024-08-17T17:48:38.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:38 smithi130 ceph-mon[110068]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:48:38.973 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:38.995 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:38.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:38.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:38 smithi179 ceph-mon[93519]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:48:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:38 smithi179 ceph-mon[93519]: from='osd.0 [v2:172.21.15.130:6802/2575115774,v1:172.21.15.130:6803/2575115774]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:48:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:38 smithi179 ceph-mon[93519]: osdmap e1177: 8 total, 7 up, 7 in 2024-08-17T17:48:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:38 smithi179 ceph-mon[93519]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T17:48:39.097 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:39.122 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:39.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:39.223 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:39.248 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:39.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:39.350 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:39.375 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:39.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:39.477 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:39.500 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:39.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:39.601 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:39.627 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:39.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:39.659 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:39 smithi130 ceph-mon[105521]: pgmap v1598: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 32807/246117 objects degraded (13.330%); 17682/246117 objects misplaced (7.184%); 12 MiB/s, 185 objects/s recovering 2024-08-17T17:48:39.659 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:39 smithi130 ceph-mon[105521]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:48:39.659 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:39 smithi130 ceph-mon[105521]: osd.0 [v2:172.21.15.130:6802/2575115774,v1:172.21.15.130:6803/2575115774] boot 2024-08-17T17:48:39.659 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:39 smithi130 ceph-mon[105521]: osdmap e1178: 8 total, 8 up, 7 in 2024-08-17T17:48:39.659 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:39 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:48:39.719 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 51 16 83274 83258 102.007 0.1875 0.00371359 0.00959734 2024-08-17T17:48:39.729 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:39.754 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:39.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:39.856 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:39.881 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:39.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:39.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:39.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:48:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:48:39] "GET /metrics HTTP/1.1" 200 36723 "" "Prometheus/2.51.0" 2024-08-17T17:48:39.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:39 smithi130 ceph-mon[110068]: pgmap v1598: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 32807/246117 objects degraded (13.330%); 17682/246117 objects misplaced (7.184%); 12 MiB/s, 185 objects/s recovering 2024-08-17T17:48:39.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:39 smithi130 ceph-mon[110068]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:48:39.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:39 smithi130 ceph-mon[110068]: osd.0 [v2:172.21.15.130:6802/2575115774,v1:172.21.15.130:6803/2575115774] boot 2024-08-17T17:48:39.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:39 smithi130 ceph-mon[110068]: osdmap e1178: 8 total, 8 up, 7 in 2024-08-17T17:48:39.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:39 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:48:39.984 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:40.009 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:40.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:39 smithi179 ceph-mon[93519]: pgmap v1598: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized, 8 active+undersized+degraded, 48 active+clean; 5.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 32807/246117 objects degraded (13.330%); 17682/246117 objects misplaced (7.184%); 12 MiB/s, 185 objects/s recovering 2024-08-17T17:48:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:39 smithi179 ceph-mon[93519]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:48:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:39 smithi179 ceph-mon[93519]: osd.0 [v2:172.21.15.130:6802/2575115774,v1:172.21.15.130:6803/2575115774] boot 2024-08-17T17:48:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:39 smithi179 ceph-mon[93519]: osdmap e1178: 8 total, 8 up, 7 in 2024-08-17T17:48:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:39 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:48:40.111 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:40.136 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:40.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:40.237 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:40.260 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:40.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:40.361 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:40.384 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:40.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:40.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:40.513 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:40.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:40.525 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:48:40.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:48:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:48:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:48:40.615 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:40.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.650 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:40.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:40.719 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 52 16 83274 83258 100.045 0 - 0.00959734 2024-08-17T17:48:40.751 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:40.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.799 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:40.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:40.900 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:40.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:40 smithi130 ceph-mon[105521]: osdmap e1179: 8 total, 8 up, 7 in 2024-08-17T17:48:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:40 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 32807/246117 objects degraded (13.330%), 13 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:48:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:40 smithi130 ceph-mon[105521]: osdmap e1180: 8 total, 8 up, 7 in 2024-08-17T17:48:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:40 smithi130 ceph-mon[110068]: osdmap e1179: 8 total, 8 up, 7 in 2024-08-17T17:48:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:40 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 32807/246117 objects degraded (13.330%), 13 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:48:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:40 smithi130 ceph-mon[110068]: osdmap e1180: 8 total, 8 up, 7 in 2024-08-17T17:48:40.921 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:40.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:40.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:41.023 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:41.044 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:41.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:40 smithi179 ceph-mon[93519]: osdmap e1179: 8 total, 8 up, 7 in 2024-08-17T17:48:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:40 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 32807/246117 objects degraded (13.330%), 13 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:48:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:40 smithi179 ceph-mon[93519]: osdmap e1180: 8 total, 8 up, 7 in 2024-08-17T17:48:41.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:41.170 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:41.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:41.272 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:41.296 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:41.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:41.328 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:41.361 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:48:41.361 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:48:41.399 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:41.444 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:41.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:41.547 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:41.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.627 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:41.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:41.721 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 53 16 83639 83623 98.585 11.4062 0.0359331 0.0101374 2024-08-17T17:48:41.728 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:41.784 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:41.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:41.883 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:41 smithi130 ceph-mon[110068]: pgmap v1601: 89 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 8 active+undersized+degraded, 48 active+clean; 5.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 51 MiB/s wr, 820 op/s; 32807/247065 objects degraded (13.279%); 17448/247065 objects misplaced (7.062%); 9.8 MiB/s, 157 objects/s recovering 2024-08-17T17:48:41.884 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:41 smithi130 ceph-mon[105521]: pgmap v1601: 89 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 8 active+undersized+degraded, 48 active+clean; 5.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 51 MiB/s wr, 820 op/s; 32807/247065 objects degraded (13.279%); 17448/247065 objects misplaced (7.062%); 9.8 MiB/s, 157 objects/s recovering 2024-08-17T17:48:41.886 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:41.907 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:41.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:41.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:41.983 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:42.009 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:42.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.063 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:42.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:42.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:41 smithi179 ceph-mon[93519]: pgmap v1601: 89 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 10 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 8 active+undersized+degraded, 48 active+clean; 5.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 51 MiB/s wr, 820 op/s; 32807/247065 objects degraded (13.279%); 17448/247065 objects misplaced (7.062%); 9.8 MiB/s, 157 objects/s recovering 2024-08-17T17:48:42.164 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:42.209 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:42.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:42.311 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:42.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:42.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:42.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:42.460 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:42.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:42.562 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:42.587 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:42.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:42.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:42.712 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:42.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:42.721 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 54 16 84267 84251 97.4863 39.25 0.00609578 0.0102503 2024-08-17T17:48:42.769 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:42.814 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:42.875 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:42.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:42.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:42.977 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:43.000 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:43.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:43.102 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:43.127 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:43.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:43.229 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:43.251 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:43.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:43.353 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:43.362 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:48:43.375 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:43.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:43.477 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:43.501 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:43.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:43.528 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:43.528 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:43.528 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:43 smithi130 ceph-mon[105521]: pgmap v1603: 89 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 8 active+undersized+degraded, 51 active+clean; 5.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 59 MiB/s wr, 944 op/s; 34912/247911 objects degraded (14.082%); 15181/247911 objects misplaced (6.124%); 16 MiB/s, 255 objects/s recovering 2024-08-17T17:48:43.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:43.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:43.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:43 smithi130 ceph-mon[110068]: pgmap v1603: 89 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 8 active+undersized+degraded, 51 active+clean; 5.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 59 MiB/s wr, 944 op/s; 34912/247911 objects degraded (14.082%); 15181/247911 objects misplaced (6.124%); 16 MiB/s, 255 objects/s recovering 2024-08-17T17:48:43.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:43.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:43 smithi179 ceph-mon[93519]: pgmap v1603: 89 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+clean+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 8 active+undersized+degraded, 51 active+clean; 5.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 59 MiB/s wr, 944 op/s; 34912/247911 objects degraded (14.082%); 15181/247911 objects misplaced (6.124%); 16 MiB/s, 255 objects/s recovering 2024-08-17T17:48:43.603 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:43.627 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:43.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:43.723 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 55 16 84873 84857 96.4001 37.875 0.0378201 0.0103667 2024-08-17T17:48:43.729 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:43.753 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:43.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:43.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:43.877 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:43.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:43.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:43.979 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:44.006 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:44.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:44.108 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:44.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.158 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:44.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:44.259 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:44.308 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:44.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:44.410 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:44.457 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:44.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:44.559 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:44.587 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:44.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:44.689 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:44.723 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 56 16 85454 85438 95.3273 36.3125 0.0236496 0.0104837 2024-08-17T17:48:44.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.737 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:44.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:44.838 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:44.863 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:44.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:44.965 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:44.989 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:44.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:44.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:45.090 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:45.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:45.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:45.215 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:45.237 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:45.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:45.339 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:45.392 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:45.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:45.493 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:45.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.556 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:45.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:45.657 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:45.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:45 smithi130 ceph-mon[105521]: pgmap v1604: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 26 MiB/s wr, 412 op/s; 45851/253536 objects degraded (18.085%); 4048/253536 objects misplaced (1.597%); 12 MiB/s, 191 objects/s recovering 2024-08-17T17:48:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:45 smithi130 ceph-mon[110068]: pgmap v1604: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 26 MiB/s wr, 412 op/s; 45851/253536 objects degraded (18.085%); 4048/253536 objects misplaced (1.597%); 12 MiB/s, 191 objects/s recovering 2024-08-17T17:48:45.678 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:45.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:45.722 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 57 16 86080 86064 94.3414 39.125 0.0170955 0.0105935 2024-08-17T17:48:45.780 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:45.805 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:45.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:45.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:45 smithi179 ceph-mon[93519]: pgmap v1604: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 26 MiB/s wr, 412 op/s; 45851/253536 objects degraded (18.085%); 4048/253536 objects misplaced (1.597%); 12 MiB/s, 191 objects/s recovering 2024-08-17T17:48:45.907 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:45.932 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:45.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:45.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:46.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:46.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.092 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:46.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:46.193 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:46.221 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:46.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:46.323 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:46.347 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:46.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:46.449 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:46.493 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:46.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:46.595 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:46.631 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:46.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:46.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 45851/253536 objects degraded (18.085%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 45851/253536 objects degraded (18.085%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:46.723 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 58 16 86718 86702 93.4024 39.875 0.0672219 0.0106989 2024-08-17T17:48:46.733 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:46.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.784 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:46.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:46.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 45851/253536 objects degraded (18.085%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:46.885 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:46.906 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:46.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:46.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:47.008 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:47.030 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:47.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:47.132 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:47.154 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:47.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:47.256 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:47.277 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:47.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:47.379 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:47.404 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:47.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:47.435 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:47.506 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:47.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.608 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:47.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:47.709 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:47.723 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 59 16 87265 87249 92.3979 34.1875 0.0160269 0.0108136 2024-08-17T17:48:47.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.761 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:47.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:47.862 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:47.867 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:48:47.867 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:48:47.867 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T17:48:47.867 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T17:48:47.867 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:48:47.885 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:47.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:47.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:47.987 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:48.010 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:48.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:47 smithi179 ceph-mon[93519]: pgmap v1605: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 20 MiB/s wr, 318 op/s; 45851/253536 objects degraded (18.085%); 4048/253536 objects misplaced (1.597%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:48:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:47 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:47 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:47 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:48:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:47 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:48:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:47 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:48.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:48.136 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:48.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:48.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:47 smithi130 ceph-mon[110068]: pgmap v1605: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 20 MiB/s wr, 318 op/s; 45851/253536 objects degraded (18.085%); 4048/253536 objects misplaced (1.597%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:48:48.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:47 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:48.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:47 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:48.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:47 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:48:48.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:47 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:48:48.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:47 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:48.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:47 smithi130 ceph-mon[105521]: pgmap v1605: 89 pgs: 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 20 MiB/s wr, 318 op/s; 45851/253536 objects degraded (18.085%); 4048/253536 objects misplaced (1.597%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:48:48.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:47 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:48.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:47 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:48.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:47 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:48:48.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:47 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:48:48.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:47 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:48.238 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:48.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.276 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:48.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:48.377 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:48.400 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:48.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:48.501 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:48.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.557 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:48.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:48.658 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:48.680 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:48.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:48.723 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T17:48:48.723908+0000 min lat: 0.000907997 max lat: 2.49826 avg lat: 0.0109218 2024-08-17T17:48:48.724 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:48:48.724 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 60 16 87881 87865 91.4997 38.5 0.00928823 0.0109218 2024-08-17T17:48:48.782 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:48.804 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:48.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:48.906 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:48.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:48.949 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:48.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:49.008 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:48 smithi130 ceph-mon[105521]: pgmap v1606: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 20 MiB/s wr, 313 op/s; 45851/254583 objects degraded (18.010%); 3902/254583 objects misplaced (1.533%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:48:49.008 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:48 smithi130 ceph-mon[105521]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-17T17:48:49.008 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:48 smithi130 ceph-mon[110068]: pgmap v1606: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 20 MiB/s wr, 313 op/s; 45851/254583 objects degraded (18.010%); 3902/254583 objects misplaced (1.533%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:48:49.008 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:48 smithi130 ceph-mon[110068]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-17T17:48:49.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:49.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.076 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:49.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:49.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:48 smithi179 ceph-mon[93519]: pgmap v1606: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 20 MiB/s wr, 313 op/s; 45851/254583 objects degraded (18.010%); 3902/254583 objects misplaced (1.533%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:48:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:48 smithi179 ceph-mon[93519]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-08-17T17:48:49.114 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:48:49.114 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T17:48:49.114 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.114 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.45158.0:87907 45.0 e1180/1179 45:0343d2dc:::benchmark_data_smithi130_172551_object87906:head v 1180'5384, mlcod=0'0)", 2024-08-17T17:48:49.114 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:48.804802+0000", 2024-08-17T17:48:49.114 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0099325960000000001, 2024-08-17T17:48:49.114 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0099645319999999999, 2024-08-17T17:48:49.115 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:48:49.115 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:48:49.115 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:48:49.115 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:48:49.115 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.115 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:48:49.115 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.804802+0000", 2024-08-17T17:48:49.115 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:49.115 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.115 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.115 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:48:49.115 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.804802+0000", 2024-08-17T17:48:49.115 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:49.115 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.116 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.116 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:48:49.116 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.804804+0000", 2024-08-17T17:48:49.116 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.282e-06 2024-08-17T17:48:49.116 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.116 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.116 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:48:49.116 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.804837+0000", 2024-08-17T17:48:49.116 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2827000000000003e-05 2024-08-17T17:48:49.116 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.116 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.116 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:48:49.116 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.804856+0000", 2024-08-17T17:48:49.116 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9709000000000001e-05 2024-08-17T17:48:49.117 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.117 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.117 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:48:49.117 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.804865+0000", 2024-08-17T17:48:49.117 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0999999999999993e-06 2024-08-17T17:48:49.117 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.117 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.117 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:48:49.117 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.805072+0000", 2024-08-17T17:48:49.117 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000206753 2024-08-17T17:48:49.117 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.117 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.117 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:48:49.117 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.805145+0000", 2024-08-17T17:48:49.118 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2775999999999996e-05 2024-08-17T17:48:49.118 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:49.118 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:48:49.118 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:49.118 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.118 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.118 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.45158.0:87917 45.5 e1180/1179 45:aaa1155f:::benchmark_data_smithi130_172551_object87916:head v 1180'5441, mlcod=0'0)", 2024-08-17T17:48:49.118 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:48.812692+0000", 2024-08-17T17:48:49.118 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0020427459999999998, 2024-08-17T17:48:49.118 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021333709999999998, 2024-08-17T17:48:49.118 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:48:49.118 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:48:49.118 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.812692+0000", 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.812692+0000", 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.812694+0000", 2024-08-17T17:48:49.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5460000000000001e-06 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.812723+0000", 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8728e-05 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.812740+0000", 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7825e-05 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.812749+0000", 2024-08-17T17:48:49.120 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4540000000000007e-06 2024-08-17T17:48:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:48:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.812996+0000", 2024-08-17T17:48:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00024711900000000002 2024-08-17T17:48:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:48:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.813059+0000", 2024-08-17T17:48:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2837999999999997e-05 2024-08-17T17:48:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:48:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:49.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.122 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.122 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.45158.0:87918 45.e e1180/1179 45:7c7f59de:::benchmark_data_smithi130_172551_object87917:head v 1180'5472, mlcod=0'0)", 2024-08-17T17:48:49.122 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:48.813693+0000", 2024-08-17T17:48:49.122 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.001041644, 2024-08-17T17:48:49.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001175, 2024-08-17T17:48:49.122 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:48:49.122 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:48:49.122 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:48:49.122 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:48:49.122 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:48:49.122 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.813693+0000", 2024-08-17T17:48:49.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:49.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:48:49.123 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.813693+0000", 2024-08-17T17:48:49.123 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:48:49.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:48:49.123 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.813696+0000", 2024-08-17T17:48:49.123 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6970000000000002e-06 2024-08-17T17:48:49.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.813744+0000", 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8276999999999999e-05 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.813762+0000", 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7139999999999999e-05 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.813770+0000", 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.952e-06 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.124 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:48:49.125 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.813863+0000", 2024-08-17T17:48:49.125 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2245999999999997e-05 2024-08-17T17:48:49.125 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:48:49.125 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:48:49.125 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:48:49.125 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:48.813893+0000", 2024-08-17T17:48:49.125 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0611999999999998e-05 2024-08-17T17:48:49.125 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:49.125 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:48:49.125 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:49.125 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:48:49.125 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T17:48:49.125 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 3 2024-08-17T17:48:49.125 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:48:49.177 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:49.201 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:49.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:49.303 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:49.345 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:49.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:49.447 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:49.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:49.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:49.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:49.609 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:49.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:49.711 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:49.723 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 61 16 88464 88448 90.5972 36.4375 0.00540621 0.0110309 2024-08-17T17:48:49.733 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:49.835 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:49.836 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T17:48:49.856 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:49.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:49.958 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:49.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:49.999 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:49.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:49 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1470602859' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:50.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:49 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:50.099 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:50.122 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:50.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:50.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:49 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1470602859' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:50.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:49 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:48:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:48:49] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.51.0" 2024-08-17T17:48:50.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:49 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1470602859' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:50.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:49 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:48:50.224 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:50.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.272 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:50.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:50.373 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:50.395 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:50.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:50.497 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:50.540 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:50.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:48:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:48:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:48:50.642 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:50.665 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:50.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:50.724 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 62 16 89036 89020 89.7126 35.75 0.0453354 0.01114 2024-08-17T17:48:50.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:50.811 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:50.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:50.954 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:50.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:50.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:51.058 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:51.081 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:51.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:51.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:50 smithi179 ceph-mon[93519]: pgmap v1607: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 559 op/s; 45851/262548 objects degraded (17.464%); 3694/262548 objects misplaced (1.407%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:48:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:50 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:50 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:50 smithi179 ceph-mon[93519]: osdmap e1181: 8 total, 8 up, 7 in 2024-08-17T17:48:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:50 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 45851/262548 objects degraded (17.464%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:51.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:50 smithi130 ceph-mon[105521]: pgmap v1607: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 559 op/s; 45851/262548 objects degraded (17.464%); 3694/262548 objects misplaced (1.407%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:48:51.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:50 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:51.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:50 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:51.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:50 smithi130 ceph-mon[105521]: osdmap e1181: 8 total, 8 up, 7 in 2024-08-17T17:48:51.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:50 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 45851/262548 objects degraded (17.464%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:51.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:50 smithi130 ceph-mon[110068]: pgmap v1607: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 35 MiB/s wr, 559 op/s; 45851/262548 objects degraded (17.464%); 3694/262548 objects misplaced (1.407%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:48:51.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:50 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:48:51.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:50 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:48:51.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:50 smithi130 ceph-mon[110068]: osdmap e1181: 8 total, 8 up, 7 in 2024-08-17T17:48:51.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:50 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 45851/262548 objects degraded (17.464%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:51.183 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:51.204 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:51.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:51.306 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:51.328 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:51.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:51.430 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:51.457 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:51.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:51.559 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:51.584 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:51.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:51.685 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:51.724 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 63 16 89671 89655 88.9179 39.6875 0.0425594 0.0112392 2024-08-17T17:48:51.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.725 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:51.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:51.826 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:51.879 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:51.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:51.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:51.960 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:48:51.981 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:52.061 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:52.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:52.163 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:52.188 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:52.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:52.290 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:52.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.347 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:52.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:52.448 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:52.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:52.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:52.581 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:52.622 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:52.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:52.724 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:52.725 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 64 16 90271 90255 88.1147 37.5 0.0279745 0.011343 2024-08-17T17:48:52.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.761 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:52.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:52.862 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:52.906 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:52.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:52.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:53.008 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:53.044 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:53.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:52 smithi179 ceph-mon[93519]: pgmap v1609: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 33 MiB/s wr, 529 op/s; 45851/262548 objects degraded (17.464%); 3694/262548 objects misplaced (1.407%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T17:48:53.152 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:53.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:52 smithi130 ceph-mon[105521]: pgmap v1609: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 33 MiB/s wr, 529 op/s; 45851/262548 objects degraded (17.464%); 3694/262548 objects misplaced (1.407%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T17:48:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:52 smithi130 ceph-mon[110068]: pgmap v1609: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.3 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 33 MiB/s wr, 529 op/s; 45851/262548 objects degraded (17.464%); 3694/262548 objects misplaced (1.407%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T17:48:53.191 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:53.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:53.293 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:53.333 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:53.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:53.435 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:53.458 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:53.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:53.560 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:53.582 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:53.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:53.683 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:53.708 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:53.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:53.724 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 65 16 90858 90842 87.3236 36.6875 0.0287366 0.0114448 2024-08-17T17:48:53.810 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:53.833 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:53.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:53.934 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:53.977 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:53.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:53.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:54.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:54.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.125 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:54.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:54.226 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:54.251 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:54.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:54.352 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:54.395 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:54.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:54.496 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:54.520 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:54.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:54.622 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:54.659 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:54.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:54.724 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 66 16 91417 91401 86.53 34.9375 0.030133 0.0115499 2024-08-17T17:48:54.761 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:54.785 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:54.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:54.887 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:54.935 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:54.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:54.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:55.037 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:55.062 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:55.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:55.107 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:54 smithi179 ceph-mon[93519]: pgmap v1610: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 37 MiB/s wr, 590 op/s; 45851/269874 objects degraded (16.990%); 3553/269874 objects misplaced (1.317%); 3.4 MiB/s, 53 objects/s recovering 2024-08-17T17:48:55.164 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:55.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:54 smithi130 ceph-mon[110068]: pgmap v1610: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 37 MiB/s wr, 590 op/s; 45851/269874 objects degraded (16.990%); 3553/269874 objects misplaced (1.317%); 3.4 MiB/s, 53 objects/s recovering 2024-08-17T17:48:55.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:54 smithi130 ceph-mon[105521]: pgmap v1610: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 37 MiB/s wr, 590 op/s; 45851/269874 objects degraded (16.990%); 3553/269874 objects misplaced (1.317%); 3.4 MiB/s, 53 objects/s recovering 2024-08-17T17:48:55.201 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:55.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:55.303 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:55.327 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:55.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:55.429 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:55.477 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:55.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:55.578 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:55.603 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:55.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:55.704 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:55.725 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 67 16 92046 92030 85.8253 39.3125 0.0101845 0.011644 2024-08-17T17:48:55.727 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:55.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:55.829 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:55.879 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:55.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:55.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:55.980 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:56.005 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:56.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:56.107 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:56.141 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:56.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:56.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:55 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 45851/269874 objects degraded (16.990%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:55 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 45851/269874 objects degraded (16.990%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:56.243 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:56.296 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:56.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:56.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:55 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 45851/269874 objects degraded (16.990%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:48:56.398 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:48:56.422 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:56.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:48:56.523 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:56.546 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:56.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:56.648 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:56.670 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:56.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:56.715 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:56.725 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 68 16 92645 92629 85.113 37.4375 0.103845 0.0117401 2024-08-17T17:48:56.772 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:56.795 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:56.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:56.897 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:56.947 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:56.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:56.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:57.048 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:57.096 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:57.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:57.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:56 smithi130 ceph-mon[105521]: pgmap v1611: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 45851/271518 objects degraded (16.887%); 3329/271518 objects misplaced (1.226%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:48:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:56 smithi130 ceph-mon[110068]: pgmap v1611: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 45851/271518 objects degraded (16.887%); 3329/271518 objects misplaced (1.226%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:48:57.198 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:57.239 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:57.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:57.340 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:56 smithi179 ceph-mon[93519]: pgmap v1611: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 45851/271518 objects degraded (16.887%); 3329/271518 objects misplaced (1.226%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:48:57.364 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:57.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:57.445 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T17:48:57.446 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:48:57.446 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-08-17T17:48:57.446 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:48:57.446 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:48:57.446 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-08-17T17:48:57.465 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:57.503 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:57.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:57.605 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:57.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.649 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:57.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:57.725 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 69 16 93243 93227 84.4212 37.375 0.016828 0.011839 2024-08-17T17:48:57.750 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:57.765 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:48:57.775 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:57.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:57.876 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:57.899 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:57.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:57.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:58.001 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:58.053 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:58.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:58.129 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:48:58.130 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:48:58.156 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:58.175 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:48:58.175 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:48:58.177 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:58.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:58.209 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:48:58.210 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:48:58.254 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:48:58.255 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:48:58.279 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:58.289 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:48:58.289 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:48:58.321 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:48:58.321 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:48:58.323 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:58.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:58.352 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:48:58.353 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:48:58.382 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:48:58.382 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:48:58.383 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:58.383 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:58.383 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:58 smithi130 ceph-mon[105521]: pgmap v1612: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 600 op/s; 45851/272619 objects degraded (16.819%); 3180/272619 objects misplaced (1.166%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:48:58.383 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:48:58 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3740296886' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:58.383 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:58.383 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:58.383 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:58 smithi130 ceph-mon[110068]: pgmap v1612: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 600 op/s; 45851/272619 objects degraded (16.819%); 3180/272619 objects misplaced (1.166%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:48:58.383 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:48:58 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3740296886' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:58.425 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:58.474 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:58.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:58.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:48:58.576 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:58.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:48:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:48:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:58 smithi179 ceph-mon[93519]: pgmap v1612: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 38 MiB/s wr, 600 op/s; 45851/272619 objects degraded (16.819%); 3180/272619 objects misplaced (1.166%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:48:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:48:58 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3740296886' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:48:58.599 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:58.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:58.700 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:58.723 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:58.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:58.725 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 70 16 93809 93793 83.7206 35.375 0.0271153 0.0119376 2024-08-17T17:48:58.824 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:48:58.869 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:58.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:58.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:48:58.971 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:48:59.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.022 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:59.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:48:59.123 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:48:59.165 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:59.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:48:59.267 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:59.291 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:59.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:59.393 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:48:59.450 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:48:59.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:48:59.552 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:48:59.575 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:59.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:48:59.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:48:59.704 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:59.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:48:59.726 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 71 16 94428 94412 83.0864 38.6875 0.021997 0.0120282 2024-08-17T17:48:59.745 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:48:59.745 INFO:teuthology.orchestra.run.smithi179.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:48:59.745 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:48:59.806 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:59.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:59.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:48:59.933 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:48:59.942 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:48:59.955 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:48:59.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:48:59.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:00.057 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:00.082 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:00.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:00.089 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:48:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:48:59] "GET /metrics HTTP/1.1" 200 36796 "" "Prometheus/2.51.0" 2024-08-17T17:49:00.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:00.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.236 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:00.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:00.337 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:00.342 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:49:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:49:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:49:00.360 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:00.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:00.461 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:00.483 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:00.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:00.584 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:00.607 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:00.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:00.709 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:00.726 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 72 16 94996 94980 82.4255 35.5 0.0398266 0.0121246 2024-08-17T17:49:00.733 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:00.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:00.834 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:00 smithi179 ceph-mon[93519]: pgmap v1613: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 45851/280467 objects degraded (16.348%); 2956/280467 objects misplaced (1.054%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:49:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:00 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 45851/280467 objects degraded (16.348%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:00.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:00.883 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:00.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:00.902 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:00 smithi130 ceph-mon[105521]: pgmap v1613: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 45851/280467 objects degraded (16.348%); 2956/280467 objects misplaced (1.054%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:49:00.902 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:00 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 45851/280467 objects degraded (16.348%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:00.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:00 smithi130 ceph-mon[110068]: pgmap v1613: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 37 MiB/s wr, 596 op/s; 45851/280467 objects degraded (16.348%); 2956/280467 objects misplaced (1.054%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:49:00.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:00 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 45851/280467 objects degraded (16.348%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:00.984 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:01.009 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:01.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:01.110 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:01.135 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:01.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:01.236 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:01.246 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:01.246 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:01.258 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:01.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:01.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:49:01.360 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:01.381 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:01.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:01.483 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:01.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.506 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:01.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:01.607 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:01.630 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:01.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:01.727 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 73 16 95510 95494 81.736 32.125 0.0386452 0.0122276 2024-08-17T17:49:01.732 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:01.754 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:01.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:01.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:01.898 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:01.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:01.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:02.000 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:02.055 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:02.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:02.152 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:49:02.157 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:02.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:02.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:02.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:02.364 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:02.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:02.460 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:02.466 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:02.514 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:02.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:02.616 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:02.642 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:02.726 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 74 16 96070 96054 81.1045 35 0.0240713 0.0123236 2024-08-17T17:49:02.744 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:02.770 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:02.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:02.823 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:02 smithi130 ceph-mon[110068]: pgmap v1614: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 32 MiB/s wr, 506 op/s; 45851/280467 objects degraded (16.348%); 2956/280467 objects misplaced (1.054%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T17:49:02.824 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:02 smithi130 ceph-mon[105521]: pgmap v1614: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 32 MiB/s wr, 506 op/s; 45851/280467 objects degraded (16.348%); 2956/280467 objects misplaced (1.054%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T17:49:02.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:02 smithi179 ceph-mon[93519]: pgmap v1614: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 32 MiB/s wr, 506 op/s; 45851/280467 objects degraded (16.348%); 2956/280467 objects misplaced (1.054%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T17:49:02.872 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:02.893 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:02.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:02.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:02.995 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:03.020 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:03.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:03.122 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:03.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.172 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:03.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:03.273 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:03.294 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:03.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:03.396 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:03.450 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:03.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:03.535 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:03.552 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:03.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.611 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:03.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:03.712 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:03.726 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 75 16 96621 96605 80.4824 34.4375 0.0256183 0.0124182 2024-08-17T17:49:03.737 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:03.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:03.839 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:03.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:03.902 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:03.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:04.003 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:04.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.043 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:04.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:04.144 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:04.169 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:04.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:04.272 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:04.316 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:04.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:04.373 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:04.373 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:04.417 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:04.440 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:04.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:04.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:04.591 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:04.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:04.692 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:04.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:04.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:04.719 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T17:49:04.726 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 76 16 97210 97194 79.9078 36.8125 0.0519304 0.0125068 2024-08-17T17:49:04.816 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:04.843 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:04.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:04.904 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:04 smithi130 ceph-mon[105521]: pgmap v1615: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 688 op/s; 45851/287334 objects degraded (15.957%); 2824/287334 objects misplaced (0.983%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:49:04.904 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:04 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1864473639' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:04.904 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:04 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:04.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:04 smithi130 ceph-mon[110068]: pgmap v1615: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 688 op/s; 45851/287334 objects degraded (15.957%); 2824/287334 objects misplaced (0.983%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:49:04.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:04 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1864473639' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:04.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:04 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:04.945 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:04.967 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:04.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:04.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:05.069 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:05.092 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:05.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:04 smithi179 ceph-mon[93519]: pgmap v1615: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 43 MiB/s wr, 688 op/s; 45851/287334 objects degraded (15.957%); 2824/287334 objects misplaced (0.983%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:49:05.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:04 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1864473639' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:05.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:04 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:05.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:05.131 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:49:05.194 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:05.239 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:05.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:05.341 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:05.371 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:05.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:05.473 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:05.497 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:05.499 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:05.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:05.572 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T17:49:05.572 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:49:05.572 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-08-17T17:49:05.572 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:49:05.572 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:49:05.572 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-08-17T17:49:05.601 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:05.668 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:05.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:05.727 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 77 16 97792 97776 79.3425 36.375 0.0843171 0.0125943 2024-08-17T17:49:05.770 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:05.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.820 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:05.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:05.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:05 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:05.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:05 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:05.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:05 smithi130 ceph-mon[105521]: osdmap e1182: 8 total, 8 up, 7 in 2024-08-17T17:49:05.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:05 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 45851/287334 objects degraded (15.957%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:05.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:05 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:05.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:05 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:05.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:05 smithi130 ceph-mon[110068]: osdmap e1182: 8 total, 8 up, 7 in 2024-08-17T17:49:05.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:05 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 45851/287334 objects degraded (15.957%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:05.921 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:05.944 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:05.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:05.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:06.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:06.084 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:06.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:06.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:05 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:05 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:05 smithi179 ceph-mon[93519]: osdmap e1182: 8 total, 8 up, 7 in 2024-08-17T17:49:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:05 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 45851/287334 objects degraded (15.957%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:49:06.186 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:06.208 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:06.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:06.310 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:06.332 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:06.358 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:49:06.358 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:49:06.380 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:06.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:06.404 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:49:06.404 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:49:06.445 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:49:06.445 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:49:06.482 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:06.488 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:49:06.488 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:49:06.515 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:49:06.515 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:49:06.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.534 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:06.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:06.543 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:49:06.543 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:49:06.576 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:49:06.576 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:49:06.609 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:49:06.609 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:49:06.635 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:06.658 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:06.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:06.727 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 78 16 98330 98314 78.7564 33.625 0.0657443 0.0126883 2024-08-17T17:49:06.759 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:06.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.807 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:06.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:06.908 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:06 smithi130 ceph-mon[105521]: pgmap v1617: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 36 MiB/s wr, 582 op/s; 45851/288996 objects degraded (15.866%); 2600/288996 objects misplaced (0.900%); 4.6 MiB/s, 72 objects/s recovering 2024-08-17T17:49:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:06 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2966117953' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:06.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:06 smithi130 ceph-mon[110068]: pgmap v1617: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 36 MiB/s wr, 582 op/s; 45851/288996 objects degraded (15.866%); 2600/288996 objects misplaced (0.900%); 4.6 MiB/s, 72 objects/s recovering 2024-08-17T17:49:06.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:06 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2966117953' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:06.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:06.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:06.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:06.984 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:06 smithi179 ceph-mon[93519]: pgmap v1617: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 36 MiB/s wr, 582 op/s; 45851/288996 objects degraded (15.866%); 2600/288996 objects misplaced (0.900%); 4.6 MiB/s, 72 objects/s recovering 2024-08-17T17:49:06.984 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:06 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2966117953' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:07.052 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:07.076 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:07.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:07.178 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:07.218 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:07.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:07.320 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:07.346 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:07.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:07.448 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:07.470 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:07.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:07.572 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:07.596 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:07.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:07.667 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:49:07.667 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:49:07.668 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T17:49:07.668 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T17:49:07.668 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:49:07.668 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:49:07.698 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:07.721 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:07.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:07.727 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 79 16 98952 98936 78.2512 38.875 0.0149989 0.0127721 2024-08-17T17:49:07.823 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:07.847 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:07.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:07.949 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:07.970 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:07.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:07.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:08.072 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:08.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.130 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:08.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:08.231 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:08.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.292 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:08.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:08.393 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:08.418 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:08.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:08.520 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:08.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.572 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:08.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:08.662 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:49:08.673 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:08.727 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T17:49:08.727915+0000 min lat: 0.000907997 max lat: 2.49826 avg lat: 0.0128632 2024-08-17T17:49:08.728 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:49:08.728 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 80 16 99506 99490 77.7059 34.625 0.0292172 0.0128632 2024-08-17T17:49:08.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.731 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:08.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:08.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:08 smithi130 ceph-mon[105521]: pgmap v1618: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 36 MiB/s wr, 582 op/s; 45851/290091 objects degraded (15.806%); 2457/290091 objects misplaced (0.847%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:49:08.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:08 smithi130 ceph-mon[110068]: pgmap v1618: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 36 MiB/s wr, 582 op/s; 45851/290091 objects degraded (15.806%); 2457/290091 objects misplaced (0.847%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:49:08.832 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:08.859 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:08.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:08.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:08.923 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:08.961 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:08.972 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:49:08.972 INFO:teuthology.orchestra.run.smithi130.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:49:08.973 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:49:08.977 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:09.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.030 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:09.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:08 smithi179 ceph-mon[93519]: pgmap v1618: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 36 MiB/s wr, 582 op/s; 45851/290091 objects degraded (15.806%); 2457/290091 objects misplaced (0.847%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T17:49:09.131 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:09.189 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:09.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:09.291 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:09.332 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:09.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:09.434 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:09.479 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:09.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:09.581 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:09.620 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:09.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:09.722 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:09.729 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 81 16 100073 100057 77.1834 35.4375 0.0127759 0.0129479 2024-08-17T17:49:09.767 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:09.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:09.845 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:49:09.869 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:09.894 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:09.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:09.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:09.996 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:10.054 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:10.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:10.156 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:10.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:49:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:49:09] "GET /metrics HTTP/1.1" 200 36796 "" "Prometheus/2.51.0" 2024-08-17T17:49:10.180 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:10.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:10.282 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:10.306 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:10.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:10.408 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:10.463 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:10.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:10.470 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:49:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:49:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:49:10.566 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:10.593 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:10.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:10.695 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:10.719 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:10.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:10.728 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 82 16 100641 100625 76.6751 35.5 0.0242479 0.0130345 2024-08-17T17:49:10.821 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:10.847 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:10.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:10.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:10.921 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:49:10.921 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T17:49:10.921 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T17:49:10.921 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T17:49:10.922 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.922 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:62437 45.1 45:86d1cc6a:::benchmark_data_smithi130_172551_object62436:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1175)", 2024-08-17T17:49:10.922 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:19.293774+0000", 2024-08-17T17:49:10.922 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 51.609507925999999, 2024-08-17T17:49:10.922 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.061076134999999997, 2024-08-17T17:49:10.922 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:10.922 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:10.922 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:10.923 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:10.923 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:10.923 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 62437 2024-08-17T17:49:10.923 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.923 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:10.923 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.923 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:10.923 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.293774+0000", 2024-08-17T17:49:10.923 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.923 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.923 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.923 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:10.923 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.293774+0000", 2024-08-17T17:49:10.924 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.924 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.924 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.924 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:10.924 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.293773+0000", 2024-08-17T17:49:10.924 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:10.924 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.924 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.924 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:10.924 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.293803+0000", 2024-08-17T17:49:10.924 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9963000000000001e-05 2024-08-17T17:49:10.924 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.924 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.925 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:10.925 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.293818+0000", 2024-08-17T17:49:10.925 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4817000000000001e-05 2024-08-17T17:49:10.925 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.925 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.925 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:10.925 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.293824+0000", 2024-08-17T17:49:10.925 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9519999999999999e-06 2024-08-17T17:49:10.925 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.925 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.925 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.925 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297092+0000", 2024-08-17T17:49:10.925 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0032682980000000002 2024-08-17T17:49:10.926 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.926 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.926 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:10.926 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297241+0000", 2024-08-17T17:49:10.926 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000149209 2024-08-17T17:49:10.926 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.926 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.926 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:49:10.926 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297446+0000", 2024-08-17T17:49:10.926 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000204623 2024-08-17T17:49:10.926 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.926 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.927 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.927 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.311374+0000", 2024-08-17T17:49:10.927 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013928464999999999 2024-08-17T17:49:10.927 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.927 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.927 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:10.927 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.313504+0000", 2024-08-17T17:49:10.927 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0021295340000000002 2024-08-17T17:49:10.927 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.927 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.927 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.927 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.353265+0000", 2024-08-17T17:49:10.927 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.039761813999999999 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.353315+0000", 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9768999999999997e-05 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.354850+0000", 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001535088 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.928 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.929 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83241 45.1 45:82a6944f:::benchmark_data_smithi130_172551_object83240:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:49:10.929 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.647075+0000", 2024-08-17T17:49:10.929 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 32.256207443999998, 2024-08-17T17:49:10.929 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4851373099999998, 2024-08-17T17:49:10.929 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:10.929 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:10.929 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:10.929 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:10.929 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:10.929 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83241 2024-08-17T17:49:10.929 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.929 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:10.929 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.930 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:10.930 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647075+0000", 2024-08-17T17:49:10.930 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.930 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.930 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.930 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:10.930 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647075+0000", 2024-08-17T17:49:10.930 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.930 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.930 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.930 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:10.930 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647073+0000", 2024-08-17T17:49:10.930 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:49:10.931 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.931 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.931 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:10.931 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647109+0000", 2024-08-17T17:49:10.931 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6040000000000001e-05 2024-08-17T17:49:10.931 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.931 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.931 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:10.931 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647128+0000", 2024-08-17T17:49:10.931 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.9045000000000001e-05 2024-08-17T17:49:10.931 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.931 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.932 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:10.932 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647136+0000", 2024-08-17T17:49:10.932 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9589999999999995e-06 2024-08-17T17:49:10.932 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.932 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.932 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.932 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647230+0000", 2024-08-17T17:49:10.932 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4471999999999999e-05 2024-08-17T17:49:10.932 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.932 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.932 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:49:10.932 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647244+0000", 2024-08-17T17:49:10.932 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4146e-05 2024-08-17T17:49:10.932 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.933 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.933 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.933 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636452+0000", 2024-08-17T17:49:10.933 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98920747099999995 2024-08-17T17:49:10.933 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.933 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.933 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:49:10.933 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636467+0000", 2024-08-17T17:49:10.933 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5031999999999999e-05 2024-08-17T17:49:10.933 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.933 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.933 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.933 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.102636+0000", 2024-08-17T17:49:10.934 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4661687780000001 2024-08-17T17:49:10.934 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.934 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.934 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:10.934 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.102811+0000", 2024-08-17T17:49:10.934 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017539199999999999 2024-08-17T17:49:10.934 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.934 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.934 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:10.934 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.102997+0000", 2024-08-17T17:49:10.934 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000185706 2024-08-17T17:49:10.934 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.109961+0000", 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0069645180000000003 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.109972+0000", 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.093e-05 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.129457+0000", 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.019484379999999999 2024-08-17T17:49:10.935 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.129554+0000", 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7465000000000003e-05 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.132212+0000", 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002657867 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.936 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83244 45.1 45:81aa3e3d:::benchmark_data_smithi130_172551_object83243:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:49:10.937 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.658809+0000", 2024-08-17T17:49:10.937 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 32.244473208999999, 2024-08-17T17:49:10.937 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4758367149999998, 2024-08-17T17:49:10.937 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:10.937 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:10.937 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:10.937 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:10.937 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:10.937 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83244 2024-08-17T17:49:10.937 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.937 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:10.937 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.937 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:10.938 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658809+0000", 2024-08-17T17:49:10.938 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.938 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.938 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.938 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:10.938 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658809+0000", 2024-08-17T17:49:10.938 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.938 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.938 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.938 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:10.938 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658807+0000", 2024-08-17T17:49:10.938 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:49:10.938 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.939 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.939 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:10.939 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658846+0000", 2024-08-17T17:49:10.939 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8951e-05 2024-08-17T17:49:10.939 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.939 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.939 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:10.939 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658857+0000", 2024-08-17T17:49:10.939 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0569e-05 2024-08-17T17:49:10.939 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.939 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.939 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:10.939 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658861+0000", 2024-08-17T17:49:10.939 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0130000000000004e-06 2024-08-17T17:49:10.940 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.940 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.940 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.940 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658909+0000", 2024-08-17T17:49:10.940 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7958999999999999e-05 2024-08-17T17:49:10.940 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.940 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.940 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:49:10.940 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658922+0000", 2024-08-17T17:49:10.940 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3519e-05 2024-08-17T17:49:10.940 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.940 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.940 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.940 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636515+0000", 2024-08-17T17:49:10.941 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97759250399999997 2024-08-17T17:49:10.941 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.941 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.941 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:49:10.941 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636526+0000", 2024-08-17T17:49:10.941 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1593000000000001e-05 2024-08-17T17:49:10.941 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.941 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.941 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.941 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.107864+0000", 2024-08-17T17:49:10.941 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4713374130000001 2024-08-17T17:49:10.941 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.941 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.942 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:10.942 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.107963+0000", 2024-08-17T17:49:10.942 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9687999999999998e-05 2024-08-17T17:49:10.942 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.942 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.942 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:10.942 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.108070+0000", 2024-08-17T17:49:10.942 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010689200000000001 2024-08-17T17:49:10.942 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.942 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.942 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.943 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112059+0000", 2024-08-17T17:49:10.943 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0039884810000000003 2024-08-17T17:49:10.943 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.943 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.943 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:10.943 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112120+0000", 2024-08-17T17:49:10.943 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1134000000000001e-05 2024-08-17T17:49:10.943 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.943 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.943 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.943 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.132267+0000", 2024-08-17T17:49:10.943 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.020146641999999999 2024-08-17T17:49:10.944 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.944 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.944 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:10.944 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.132303+0000", 2024-08-17T17:49:10.944 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6529000000000002e-05 2024-08-17T17:49:10.944 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.944 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.944 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:10.944 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.134646+0000", 2024-08-17T17:49:10.944 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0023425199999999998 2024-08-17T17:49:10.944 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.944 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:10.945 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.945 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.945 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.945 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83246 45.1 45:8e7e3f88:::benchmark_data_smithi130_172551_object83245:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:49:10.945 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.659302+0000", 2024-08-17T17:49:10.945 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 32.24398016, 2024-08-17T17:49:10.945 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.500952952, 2024-08-17T17:49:10.945 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:10.945 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:10.945 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:10.945 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:10.946 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:10.946 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83246 2024-08-17T17:49:10.946 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.946 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:10.946 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.946 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:10.946 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659302+0000", 2024-08-17T17:49:10.946 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.946 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.946 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.946 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:10.946 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659302+0000", 2024-08-17T17:49:10.947 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.947 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.947 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.947 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:10.947 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659301+0000", 2024-08-17T17:49:10.947 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:49:10.947 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.947 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.947 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:10.947 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659364+0000", 2024-08-17T17:49:10.947 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2366000000000006e-05 2024-08-17T17:49:10.947 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.947 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.948 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:10.948 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659374+0000", 2024-08-17T17:49:10.948 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0380999999999999e-05 2024-08-17T17:49:10.948 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.948 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.948 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:10.948 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659379+0000", 2024-08-17T17:49:10.948 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5950000000000001e-06 2024-08-17T17:49:10.948 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.948 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.948 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.948 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659426+0000", 2024-08-17T17:49:10.949 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7775000000000003e-05 2024-08-17T17:49:10.949 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.949 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.949 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:49:10.949 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659438+0000", 2024-08-17T17:49:10.949 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1267000000000001e-05 2024-08-17T17:49:10.949 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.949 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.949 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.949 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636564+0000", 2024-08-17T17:49:10.949 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97712642100000002 2024-08-17T17:49:10.949 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.949 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.950 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:49:10.950 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636570+0000", 2024-08-17T17:49:10.950 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9120000000000003e-06 2024-08-17T17:49:10.950 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.950 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.950 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.950 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.109992+0000", 2024-08-17T17:49:10.950 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.473421493 2024-08-17T17:49:10.950 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.950 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.950 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:10.950 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.110115+0000", 2024-08-17T17:49:10.950 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000122934 2024-08-17T17:49:10.950 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.951 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.951 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:10.951 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.110209+0000", 2024-08-17T17:49:10.951 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4676e-05 2024-08-17T17:49:10.951 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.951 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.951 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.951 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112110+0000", 2024-08-17T17:49:10.951 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019009809999999999 2024-08-17T17:49:10.951 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.951 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.951 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:10.951 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.116034+0000", 2024-08-17T17:49:10.952 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0039233669999999997 2024-08-17T17:49:10.952 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.952 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.952 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.952 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.157325+0000", 2024-08-17T17:49:10.952 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.041291094 2024-08-17T17:49:10.952 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.952 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.952 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:10.952 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.157364+0000", 2024-08-17T17:49:10.952 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9023999999999998e-05 2024-08-17T17:49:10.952 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.952 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.953 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:10.953 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.160255+0000", 2024-08-17T17:49:10.953 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028912460000000001 2024-08-17T17:49:10.953 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.953 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:10.953 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.953 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.953 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.953 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83252 45.1 45:87ce6d24:::benchmark_data_smithi130_172551_object83251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:49:10.953 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.670101+0000", 2024-08-17T17:49:10.953 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 32.233181109999997, 2024-08-17T17:49:10.953 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.493777863, 2024-08-17T17:49:10.953 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:10.954 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:10.954 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:10.954 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:10.954 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:10.954 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83252 2024-08-17T17:49:10.954 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.954 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:10.954 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.954 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:10.954 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670101+0000", 2024-08-17T17:49:10.954 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.954 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.954 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670101+0000", 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670100+0000", 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670158+0000", 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8179e-05 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.955 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670170+0000", 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2e-05 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670175+0000", 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3290000000000001e-06 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670225+0000", 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0126000000000001e-05 2024-08-17T17:49:10.956 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.957 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.957 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:49:10.957 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670237+0000", 2024-08-17T17:49:10.957 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1832e-05 2024-08-17T17:49:10.957 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.957 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.957 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.957 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636594+0000", 2024-08-17T17:49:10.957 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.96635708200000003 2024-08-17T17:49:10.957 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.957 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.957 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:49:10.957 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636603+0000", 2024-08-17T17:49:10.958 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7139999999999999e-06 2024-08-17T17:49:10.958 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.958 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.958 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.958 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112146+0000", 2024-08-17T17:49:10.958 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.475543396 2024-08-17T17:49:10.958 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.958 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.958 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:10.958 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112248+0000", 2024-08-17T17:49:10.958 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0001016 2024-08-17T17:49:10.958 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.958 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.958 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:10.959 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112350+0000", 2024-08-17T17:49:10.959 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000102067 2024-08-17T17:49:10.959 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.959 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.959 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.959 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.116088+0000", 2024-08-17T17:49:10.959 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037383759999999999 2024-08-17T17:49:10.959 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.959 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.959 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:10.959 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.117798+0000", 2024-08-17T17:49:10.959 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0017097239999999999 2024-08-17T17:49:10.959 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.959 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.960 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.960 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.160341+0000", 2024-08-17T17:49:10.960 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.042542888000000001 2024-08-17T17:49:10.960 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.960 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.960 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:10.960 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.160400+0000", 2024-08-17T17:49:10.960 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9277000000000002e-05 2024-08-17T17:49:10.960 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.960 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.960 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:10.960 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.163879+0000", 2024-08-17T17:49:10.960 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0034785689999999999 2024-08-17T17:49:10.961 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.961 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:10.961 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.961 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.961 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.961 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:84124 45.1 45:81e39771:::benchmark_data_smithi130_172551_object84123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1180)", 2024-08-17T17:49:10.961 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:42.500377+0000", 2024-08-17T17:49:10.961 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 28.402905393000001, 2024-08-17T17:49:10.961 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.065064997999999999, 2024-08-17T17:49:10.961 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:10.961 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:10.961 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:10.961 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:10.962 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:10.962 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 84124 2024-08-17T17:49:10.962 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.962 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:10.962 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.962 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:10.962 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.500377+0000", 2024-08-17T17:49:10.962 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.962 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.962 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.962 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:10.962 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.500377+0000", 2024-08-17T17:49:10.962 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.963 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.963 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.963 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:10.963 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.500376+0000", 2024-08-17T17:49:10.963 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:49:10.963 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.963 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.963 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:10.963 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.500428+0000", 2024-08-17T17:49:10.963 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1697999999999999e-05 2024-08-17T17:49:10.963 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.963 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.964 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:10.964 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.500433+0000", 2024-08-17T17:49:10.964 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0340000000000004e-06 2024-08-17T17:49:10.964 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.964 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.964 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:10.964 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.500436+0000", 2024-08-17T17:49:10.964 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.7980000000000001e-06 2024-08-17T17:49:10.964 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.964 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.964 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.964 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.502998+0000", 2024-08-17T17:49:10.964 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025626720000000002 2024-08-17T17:49:10.964 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.965 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.965 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:10.965 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.503102+0000", 2024-08-17T17:49:10.965 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000103798 2024-08-17T17:49:10.965 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.965 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.965 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:10.965 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.503207+0000", 2024-08-17T17:49:10.965 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010469200000000001 2024-08-17T17:49:10.965 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.965 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.965 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.965 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.505371+0000", 2024-08-17T17:49:10.965 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0021643999999999999 2024-08-17T17:49:10.966 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.966 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.966 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:10.966 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.505434+0000", 2024-08-17T17:49:10.966 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3047999999999997e-05 2024-08-17T17:49:10.966 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.966 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.966 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.966 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.561855+0000", 2024-08-17T17:49:10.966 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.056421048000000001 2024-08-17T17:49:10.966 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.966 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.966 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:10.967 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.561937+0000", 2024-08-17T17:49:10.967 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.1668000000000005e-05 2024-08-17T17:49:10.967 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.967 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.967 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:10.967 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.565442+0000", 2024-08-17T17:49:10.967 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0035046729999999998 2024-08-17T17:49:10.967 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.967 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:10.967 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.967 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.967 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.967 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:84130 45.1 45:88b8f3de:::benchmark_data_smithi130_172551_object84129:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1180)", 2024-08-17T17:49:10.967 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:42.506115+0000", 2024-08-17T17:49:10.968 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 28.397166963, 2024-08-17T17:49:10.968 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.062221560000000002, 2024-08-17T17:49:10.968 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:10.968 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:10.968 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:10.968 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:10.968 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:10.968 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 84130 2024-08-17T17:49:10.968 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.968 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:10.968 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.968 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:10.968 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.506115+0000", 2024-08-17T17:49:10.969 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.969 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.969 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.969 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:10.969 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.506115+0000", 2024-08-17T17:49:10.969 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.969 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.969 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.969 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:10.969 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.506114+0000", 2024-08-17T17:49:10.969 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:49:10.969 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.969 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.970 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:10.970 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.506211+0000", 2024-08-17T17:49:10.970 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6966999999999995e-05 2024-08-17T17:49:10.970 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.970 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.970 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:10.970 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.506222+0000", 2024-08-17T17:49:10.970 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0885e-05 2024-08-17T17:49:10.970 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.970 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.970 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:10.970 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.506226+0000", 2024-08-17T17:49:10.970 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1339999999999997e-06 2024-08-17T17:49:10.971 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.971 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.971 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.971 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.506270+0000", 2024-08-17T17:49:10.971 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3665000000000003e-05 2024-08-17T17:49:10.971 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.971 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.971 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:10.971 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.506408+0000", 2024-08-17T17:49:10.971 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000138446 2024-08-17T17:49:10.971 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.971 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.971 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:10.971 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.506572+0000", 2024-08-17T17:49:10.972 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016404200000000001 2024-08-17T17:49:10.972 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.972 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.972 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:10.972 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.509227+0000", 2024-08-17T17:49:10.972 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002655129 2024-08-17T17:49:10.972 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.972 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.972 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.972 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.512212+0000", 2024-08-17T17:49:10.972 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002985157 2024-08-17T17:49:10.972 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.972 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.972 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.973 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.565514+0000", 2024-08-17T17:49:10.973 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.053301343000000001 2024-08-17T17:49:10.973 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.973 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.973 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:10.973 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.565580+0000", 2024-08-17T17:49:10.973 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6514999999999996e-05 2024-08-17T17:49:10.973 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.973 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.973 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:10.973 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:42.568337+0000", 2024-08-17T17:49:10.973 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027564009999999999 2024-08-17T17:49:10.973 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.974 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:10.974 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.974 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.974 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.974 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:88238 45.1 45:8de64967:::benchmark_data_smithi130_172551_object88237:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1180)", 2024-08-17T17:49:10.974 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:49.366656+0000", 2024-08-17T17:49:10.974 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 21.536626621, 2024-08-17T17:49:10.974 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.079152609999999998, 2024-08-17T17:49:10.974 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:10.974 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:10.974 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:10.974 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:10.974 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:10.975 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 88238 2024-08-17T17:49:10.975 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.975 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:10.975 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.975 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:10.975 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366656+0000", 2024-08-17T17:49:10.975 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.975 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.975 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.975 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:10.975 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366656+0000", 2024-08-17T17:49:10.975 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.975 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.976 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.976 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:10.976 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366655+0000", 2024-08-17T17:49:10.976 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:49:10.976 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.976 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.976 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:10.976 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366709+0000", 2024-08-17T17:49:10.976 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.4617999999999998e-05 2024-08-17T17:49:10.976 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.976 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.976 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:10.976 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366720+0000", 2024-08-17T17:49:10.976 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.009e-05 2024-08-17T17:49:10.977 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.977 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.977 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:10.977 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366723+0000", 2024-08-17T17:49:10.977 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8800000000000001e-06 2024-08-17T17:49:10.977 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.977 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.977 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.977 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.380121+0000", 2024-08-17T17:49:10.977 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013397166 2024-08-17T17:49:10.977 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.977 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.977 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:10.978 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.380213+0000", 2024-08-17T17:49:10.978 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2111e-05 2024-08-17T17:49:10.978 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.978 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.978 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:10.978 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.380314+0000", 2024-08-17T17:49:10.978 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000101746 2024-08-17T17:49:10.978 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.978 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.978 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.978 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.383932+0000", 2024-08-17T17:49:10.978 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00361722 2024-08-17T17:49:10.979 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.979 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.979 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:10.979 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:10.980 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:10.981 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.383975+0000", 2024-08-17T17:49:10.981 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3120000000000001e-05 2024-08-17T17:49:10.981 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.981 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.982 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.982 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.443093+0000", 2024-08-17T17:49:10.982 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.059118638000000001 2024-08-17T17:49:10.982 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.982 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.982 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:10.982 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.443138+0000", 2024-08-17T17:49:10.982 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4335999999999997e-05 2024-08-17T17:49:10.982 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.982 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.982 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:10.982 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.445808+0000", 2024-08-17T17:49:10.982 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002670389 2024-08-17T17:49:10.983 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.983 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:10.983 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.983 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.983 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.983 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:88246 45.1 45:8c252ee5:::benchmark_data_smithi130_172551_object88245:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1180)", 2024-08-17T17:49:10.983 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:49.370303+0000", 2024-08-17T17:49:10.983 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 21.532978934999999, 2024-08-17T17:49:10.983 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.078072420000000003, 2024-08-17T17:49:10.983 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:10.983 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:10.983 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:10.983 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:10.983 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:10.984 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 88246 2024-08-17T17:49:10.984 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.984 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:10.984 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.984 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:10.984 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370303+0000", 2024-08-17T17:49:10.984 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.984 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.984 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.984 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:10.984 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370303+0000", 2024-08-17T17:49:10.984 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.984 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.985 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.985 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:10.985 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370302+0000", 2024-08-17T17:49:10.985 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:10.985 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.985 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.985 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:10.985 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370372+0000", 2024-08-17T17:49:10.985 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0137999999999998e-05 2024-08-17T17:49:10.985 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.985 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.985 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:10.985 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370386+0000", 2024-08-17T17:49:10.986 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4062e-05 2024-08-17T17:49:10.986 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.986 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.986 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:10.986 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370392+0000", 2024-08-17T17:49:10.986 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8690000000000001e-06 2024-08-17T17:49:10.986 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.986 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.986 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.986 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.381694+0000", 2024-08-17T17:49:10.986 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.011302395999999999 2024-08-17T17:49:10.986 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.986 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.987 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:10.987 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.381800+0000", 2024-08-17T17:49:10.987 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105775 2024-08-17T17:49:10.987 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.987 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.987 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:10.987 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.381899+0000", 2024-08-17T17:49:10.987 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.8529999999999993e-05 2024-08-17T17:49:10.987 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.987 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.987 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.987 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.383968+0000", 2024-08-17T17:49:10.987 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020689520000000002 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.384010+0000", 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2639999999999998e-05 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.445861+0000", 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.061850694999999997 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.445900+0000", 2024-08-17T17:49:10.988 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8890999999999997e-05 2024-08-17T17:49:10.989 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.989 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.989 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:10.989 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.448376+0000", 2024-08-17T17:49:10.989 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024758029999999999 2024-08-17T17:49:10.989 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.989 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:10.989 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.989 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.989 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.989 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:88852 45.1 45:8bcd85bb:::benchmark_data_smithi130_172551_object88851:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:49:10.989 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:50.423711+0000", 2024-08-17T17:49:10.989 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 20.479571498999999, 2024-08-17T17:49:10.989 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.067301394, 2024-08-17T17:49:10.990 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:10.990 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:10.990 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:10.990 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:10.990 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:10.990 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 88852 2024-08-17T17:49:10.990 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.990 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:10.990 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.990 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:10.990 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423711+0000", 2024-08-17T17:49:10.990 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.990 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.990 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.991 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:10.991 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423711+0000", 2024-08-17T17:49:10.991 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.991 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.991 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.991 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:10.991 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423703+0000", 2024-08-17T17:49:10.991 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999928 2024-08-17T17:49:10.991 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.991 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.991 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:10.991 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423743+0000", 2024-08-17T17:49:10.991 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9957999999999999e-05 2024-08-17T17:49:10.991 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.992 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.992 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:10.992 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423758+0000", 2024-08-17T17:49:10.992 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4294000000000001e-05 2024-08-17T17:49:10.992 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.992 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.992 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:10.992 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423764+0000", 2024-08-17T17:49:10.992 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2419999999999997e-06 2024-08-17T17:49:10.992 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.992 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.992 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.992 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423809+0000", 2024-08-17T17:49:10.992 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5754000000000003e-05 2024-08-17T17:49:10.993 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.993 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.993 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:10.993 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423915+0000", 2024-08-17T17:49:10.993 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105951 2024-08-17T17:49:10.993 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.993 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.993 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:10.993 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.424027+0000", 2024-08-17T17:49:10.993 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000111777 2024-08-17T17:49:10.993 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.993 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.993 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:10.994 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.430304+0000", 2024-08-17T17:49:10.994 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0062763059999999997 2024-08-17T17:49:10.994 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.994 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.994 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.994 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.430329+0000", 2024-08-17T17:49:10.994 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5187000000000001e-05 2024-08-17T17:49:10.994 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.994 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.994 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:10.994 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.488439+0000", 2024-08-17T17:49:10.994 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.058110436000000001 2024-08-17T17:49:10.994 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.994 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.995 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:10.995 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.488489+0000", 2024-08-17T17:49:10.995 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0201000000000003e-05 2024-08-17T17:49:10.995 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.995 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.995 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:10.995 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.491012+0000", 2024-08-17T17:49:10.995 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025226620000000002 2024-08-17T17:49:10.995 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.995 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:10.995 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:10.995 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.995 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:93092 45.1 45:8cdd61ff:::benchmark_data_smithi130_172551_object93091:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:57.470550+0000", 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 13.432731633, 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.063797922000000007, 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 93092 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.470550+0000", 2024-08-17T17:49:10.996 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.470550+0000", 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.470549+0000", 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.470601+0000", 2024-08-17T17:49:10.997 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1428999999999998e-05 2024-08-17T17:49:10.998 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.998 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.998 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:10.998 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.470611+0000", 2024-08-17T17:49:10.998 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0107e-05 2024-08-17T17:49:10.998 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.998 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.998 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:10.998 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.470616+0000", 2024-08-17T17:49:10.998 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7039999999999999e-06 2024-08-17T17:49:10.998 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.998 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.998 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:10.999 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.473070+0000", 2024-08-17T17:49:10.999 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002453757 2024-08-17T17:49:10.999 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.999 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.999 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:10.999 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.473243+0000", 2024-08-17T17:49:10.999 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017314000000000001 2024-08-17T17:49:10.999 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.999 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:10.999 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:10.999 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.473425+0000", 2024-08-17T17:49:10.999 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000181886 2024-08-17T17:49:10.999 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:10.999 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.000 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.000 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.480014+0000", 2024-08-17T17:49:11.000 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0065883959999999998 2024-08-17T17:49:11.000 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.000 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.000 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:11.000 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.483378+0000", 2024-08-17T17:49:11.000 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0033649029999999998 2024-08-17T17:49:11.000 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.000 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.000 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.000 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.531658+0000", 2024-08-17T17:49:11.000 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.048279387 2024-08-17T17:49:11.000 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.531697+0000", 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9515999999999999e-05 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.534348+0000", 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0026510549999999998 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.001 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:93095 45.1 45:85a693d9:::benchmark_data_smithi130_172551_object93094:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:49:11.002 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:57.476222+0000", 2024-08-17T17:49:11.002 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 13.427060051, 2024-08-17T17:49:11.002 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.061604867000000001, 2024-08-17T17:49:11.002 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:11.002 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:11.002 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:11.002 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:11.002 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:11.002 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 93095 2024-08-17T17:49:11.002 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.002 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:11.002 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.003 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:11.003 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.476222+0000", 2024-08-17T17:49:11.003 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.003 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.003 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.003 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:11.003 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.476222+0000", 2024-08-17T17:49:11.003 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.003 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.003 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.003 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:11.003 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.476220+0000", 2024-08-17T17:49:11.003 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.476281+0000", 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0767999999999998e-05 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.476295+0000", 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3378e-05 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.476300+0000", 2024-08-17T17:49:11.004 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6269999999999997e-06 2024-08-17T17:49:11.005 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.005 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.005 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:11.005 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.480050+0000", 2024-08-17T17:49:11.005 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037495829999999999 2024-08-17T17:49:11.005 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.005 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.005 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:11.005 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.480211+0000", 2024-08-17T17:49:11.005 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016159599999999999 2024-08-17T17:49:11.005 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.005 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.005 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:11.005 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.480381+0000", 2024-08-17T17:49:11.006 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016930499999999999 2024-08-17T17:49:11.006 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.006 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.006 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.006 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.488693+0000", 2024-08-17T17:49:11.006 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0083123759999999998 2024-08-17T17:49:11.006 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.006 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.006 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:11.006 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.488707+0000", 2024-08-17T17:49:11.006 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3484e-05 2024-08-17T17:49:11.006 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.007 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.007 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.007 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.534424+0000", 2024-08-17T17:49:11.007 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.045717239 2024-08-17T17:49:11.007 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.007 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.007 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:11.007 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.534493+0000", 2024-08-17T17:49:11.007 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.9610000000000006e-05 2024-08-17T17:49:11.007 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.007 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.007 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:11.007 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.537827+0000", 2024-08-17T17:49:11.007 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003333487 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:93097 45.1 45:84b07cc9:::benchmark_data_smithi130_172551_object93096:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:57.476311+0000", 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 13.426970989000001, 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.063975298999999999, 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:11.008 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 93097 2024-08-17T17:49:11.009 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.009 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:11.009 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.009 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:11.009 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.476311+0000", 2024-08-17T17:49:11.009 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.009 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.009 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.009 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:11.009 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.476311+0000", 2024-08-17T17:49:11.009 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.009 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.009 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.009 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:11.010 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.476311+0000", 2024-08-17T17:49:11.010 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:49:11.010 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.010 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.010 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:11.010 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.476339+0000", 2024-08-17T17:49:11.010 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8833999999999999e-05 2024-08-17T17:49:11.010 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.010 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.010 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:11.010 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.476347+0000", 2024-08-17T17:49:11.010 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7430000000000002e-06 2024-08-17T17:49:11.011 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.011 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.011 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:11.011 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.476351+0000", 2024-08-17T17:49:11.011 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.941e-06 2024-08-17T17:49:11.011 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.011 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.011 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:11.011 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.483407+0000", 2024-08-17T17:49:11.011 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0070562180000000004 2024-08-17T17:49:11.011 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.011 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.011 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:11.011 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.483554+0000", 2024-08-17T17:49:11.012 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014654499999999999 2024-08-17T17:49:11.012 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.012 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.012 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:11.012 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.483735+0000", 2024-08-17T17:49:11.012 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00018135199999999999 2024-08-17T17:49:11.012 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.012 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.012 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.012 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.488744+0000", 2024-08-17T17:49:11.012 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0050090899999999999 2024-08-17T17:49:11.012 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.012 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.012 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:11.013 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.488819+0000", 2024-08-17T17:49:11.013 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.4501999999999999e-05 2024-08-17T17:49:11.013 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.013 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.013 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.013 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.537897+0000", 2024-08-17T17:49:11.013 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.049077717 2024-08-17T17:49:11.013 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.013 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.013 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:11.013 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.537961+0000", 2024-08-17T17:49:11.013 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4974999999999994e-05 2024-08-17T17:49:11.014 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.014 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.014 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:11.014 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.540286+0000", 2024-08-17T17:49:11.014 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002324951 2024-08-17T17:49:11.014 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.014 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:11.014 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.014 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.014 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.014 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:93101 45.1 45:8aba74a4:::benchmark_data_smithi130_172551_object93100:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:49:11.014 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:57.477753+0000", 2024-08-17T17:49:11.014 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 13.42552914, 2024-08-17T17:49:11.015 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.066044696, 2024-08-17T17:49:11.015 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:11.015 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:11.015 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:11.015 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:11.015 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:11.015 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 93101 2024-08-17T17:49:11.015 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.015 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:11.015 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.015 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:11.015 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.477753+0000", 2024-08-17T17:49:11.015 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.477753+0000", 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.477752+0000", 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.477813+0000", 2024-08-17T17:49:11.016 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1459999999999998e-05 2024-08-17T17:49:11.017 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.017 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.017 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:11.017 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.477826+0000", 2024-08-17T17:49:11.017 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2670000000000001e-05 2024-08-17T17:49:11.017 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.017 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.017 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:11.017 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.477833+0000", 2024-08-17T17:49:11.017 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.1949999999999997e-06 2024-08-17T17:49:11.017 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.017 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.017 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:11.018 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.486323+0000", 2024-08-17T17:49:11.018 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0084897389999999996 2024-08-17T17:49:11.018 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.018 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.018 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:11.018 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.486457+0000", 2024-08-17T17:49:11.018 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000134387 2024-08-17T17:49:11.018 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.018 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.018 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:11.018 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.486576+0000", 2024-08-17T17:49:11.018 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000119034 2024-08-17T17:49:11.018 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.018 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.019 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.019 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.488809+0000", 2024-08-17T17:49:11.019 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0022323920000000001 2024-08-17T17:49:11.019 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.019 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.019 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:11.019 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.488827+0000", 2024-08-17T17:49:11.019 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.8814999999999999e-05 2024-08-17T17:49:11.019 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.019 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.019 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.019 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.540362+0000", 2024-08-17T17:49:11.019 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.051534785999999999 2024-08-17T17:49:11.020 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.020 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.020 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:11.020 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.540434+0000", 2024-08-17T17:49:11.020 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.2237999999999995e-05 2024-08-17T17:49:11.020 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.020 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.020 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:11.020 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.543798+0000", 2024-08-17T17:49:11.020 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0033632750000000002 2024-08-17T17:49:11.020 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.020 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:11.020 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.020 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.021 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.021 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:93103 45.1 45:89ecacb5:::benchmark_data_smithi130_172551_object93102:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:49:11.021 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:57.486105+0000", 2024-08-17T17:49:11.021 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 13.417177068000001, 2024-08-17T17:49:11.021 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.060837211000000002, 2024-08-17T17:49:11.021 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:11.021 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:11.021 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:11.021 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:11.021 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:11.021 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 93103 2024-08-17T17:49:11.021 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.021 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:11.021 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.022 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:11.022 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.486105+0000", 2024-08-17T17:49:11.022 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.022 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.022 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.022 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:11.022 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.486105+0000", 2024-08-17T17:49:11.022 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.022 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.022 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.022 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:11.022 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.486104+0000", 2024-08-17T17:49:11.022 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:11.023 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.023 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.023 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:11.023 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.486145+0000", 2024-08-17T17:49:11.023 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0821000000000001e-05 2024-08-17T17:49:11.023 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.023 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.023 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:11.023 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.486158+0000", 2024-08-17T17:49:11.023 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3023000000000001e-05 2024-08-17T17:49:11.023 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.023 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.023 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:11.024 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.486163+0000", 2024-08-17T17:49:11.024 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8720000000000001e-06 2024-08-17T17:49:11.024 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.024 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.024 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:11.024 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.488843+0000", 2024-08-17T17:49:11.024 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0026802940000000002 2024-08-17T17:49:11.024 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.024 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.024 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:11.024 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.488941+0000", 2024-08-17T17:49:11.024 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7818000000000004e-05 2024-08-17T17:49:11.024 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.024 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.489045+0000", 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000104702 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.495121+0000", 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0060752510000000003 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.495176+0000", 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5884999999999998e-05 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.025 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.026 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.026 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.543862+0000", 2024-08-17T17:49:11.026 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.048685910999999998 2024-08-17T17:49:11.026 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.026 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.026 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:11.026 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.543908+0000", 2024-08-17T17:49:11.026 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5123999999999997e-05 2024-08-17T17:49:11.026 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.026 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.026 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:11.026 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.546942+0000", 2024-08-17T17:49:11.026 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0030347479999999999 2024-08-17T17:49:11.027 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.027 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:11.027 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.027 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.027 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.027 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:93104 45.1 45:8e5a074a:::benchmark_data_smithi130_172551_object93103:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:49:11.027 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:57.486959+0000", 2024-08-17T17:49:11.027 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 13.416322996, 2024-08-17T17:49:11.027 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.064041281000000005, 2024-08-17T17:49:11.027 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:11.027 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:11.027 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:11.027 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:11.027 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:11.028 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 93104 2024-08-17T17:49:11.028 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.028 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:11.028 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.028 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:11.028 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.486959+0000", 2024-08-17T17:49:11.028 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.028 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.028 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.028 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:11.028 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.486959+0000", 2024-08-17T17:49:11.028 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.028 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.028 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.029 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:11.029 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.486958+0000", 2024-08-17T17:49:11.029 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:49:11.029 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.029 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.029 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:11.029 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.487017+0000", 2024-08-17T17:49:11.029 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8591000000000002e-05 2024-08-17T17:49:11.029 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.029 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.029 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:11.029 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.487029+0000", 2024-08-17T17:49:11.029 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1780000000000001e-05 2024-08-17T17:49:11.029 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.487035+0000", 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1279999999999996e-06 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.491786+0000", 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0047514130000000003 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.491980+0000", 2024-08-17T17:49:11.030 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019342200000000001 2024-08-17T17:49:11.031 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.031 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.031 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:11.031 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.492183+0000", 2024-08-17T17:49:11.031 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000203477 2024-08-17T17:49:11.031 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.031 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.031 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.031 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.495167+0000", 2024-08-17T17:49:11.031 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0029844160000000002 2024-08-17T17:49:11.031 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.031 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.031 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:11.032 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.495195+0000", 2024-08-17T17:49:11.032 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.7800000000000001e-05 2024-08-17T17:49:11.032 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.032 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.032 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.032 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.547004+0000", 2024-08-17T17:49:11.032 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.051808373999999997 2024-08-17T17:49:11.032 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.032 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.032 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:11.032 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.547134+0000", 2024-08-17T17:49:11.032 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012993400000000001 2024-08-17T17:49:11.032 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.032 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:57.551000+0000", 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0038668079999999998 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:99564 45.1 45:877a8ac3:::benchmark_data_smithi130_172551_object99563:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1182)", 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:08.821429+0000", 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 2.0818535740000002, 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.069042615000000002, 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:11.033 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:11.034 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:11.034 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:11.034 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 99564 2024-08-17T17:49:11.034 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.034 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:11.034 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.034 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:11.034 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821429+0000", 2024-08-17T17:49:11.034 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.034 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.034 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.034 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:11.034 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821429+0000", 2024-08-17T17:49:11.035 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.035 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.035 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.035 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:11.035 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821427+0000", 2024-08-17T17:49:11.035 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:49:11.035 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.035 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.035 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:11.035 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821522+0000", 2024-08-17T17:49:11.035 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5486e-05 2024-08-17T17:49:11.035 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.035 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.035 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:11.036 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821543+0000", 2024-08-17T17:49:11.036 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.1069000000000001e-05 2024-08-17T17:49:11.036 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.036 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.036 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:11.036 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821552+0000", 2024-08-17T17:49:11.036 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.4780000000000008e-06 2024-08-17T17:49:11.036 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.036 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.036 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:11.036 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821602+0000", 2024-08-17T17:49:11.036 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0368999999999998e-05 2024-08-17T17:49:11.036 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.036 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821791+0000", 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00018891099999999999 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821980+0000", 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00018913400000000001 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827561+0000", 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0055811910000000001 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.037 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.038 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.038 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830393+0000", 2024-08-17T17:49:11.038 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028316940000000001 2024-08-17T17:49:11.038 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.038 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.038 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.038 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.887201+0000", 2024-08-17T17:49:11.038 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.056807923000000003 2024-08-17T17:49:11.038 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.038 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.038 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:11.038 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.887243+0000", 2024-08-17T17:49:11.038 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.163e-05 2024-08-17T17:49:11.039 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.039 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.039 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:11.039 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.890471+0000", 2024-08-17T17:49:11.039 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003228608 2024-08-17T17:49:11.039 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.039 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:11.039 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.039 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.039 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.039 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:99567 45.1 45:86b126dd:::benchmark_data_smithi130_172551_object99566:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1182)", 2024-08-17T17:49:11.039 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:08.821921+0000", 2024-08-17T17:49:11.039 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 2.081360863, 2024-08-17T17:49:11.039 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.071874980000000005, 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 99567 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821921+0000", 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.040 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:11.041 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821921+0000", 2024-08-17T17:49:11.041 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.041 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.041 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.041 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:11.041 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821920+0000", 2024-08-17T17:49:11.041 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:49:11.041 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.041 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.041 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:11.041 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821993+0000", 2024-08-17T17:49:11.041 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.2989000000000002e-05 2024-08-17T17:49:11.041 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.041 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.042 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:11.042 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.822006+0000", 2024-08-17T17:49:11.042 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2729e-05 2024-08-17T17:49:11.042 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.042 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.042 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:11.042 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.822012+0000", 2024-08-17T17:49:11.042 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.003e-06 2024-08-17T17:49:11.042 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.042 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.042 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:11.042 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.824701+0000", 2024-08-17T17:49:11.042 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0026888110000000002 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.824892+0000", 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019161699999999999 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.825092+0000", 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019969200000000001 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830339+0000", 2024-08-17T17:49:11.043 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00524714 2024-08-17T17:49:11.044 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.044 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.044 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.044 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830563+0000", 2024-08-17T17:49:11.044 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00022378 2024-08-17T17:49:11.044 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.044 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.044 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.044 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.890526+0000", 2024-08-17T17:49:11.044 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.059962847 2024-08-17T17:49:11.044 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.044 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.044 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:11.045 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.890569+0000", 2024-08-17T17:49:11.045 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2808000000000001e-05 2024-08-17T17:49:11.045 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.045 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.045 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:11.045 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.893796+0000", 2024-08-17T17:49:11.045 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0032274970000000002 2024-08-17T17:49:11.045 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.045 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:11.045 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.045 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.045 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.045 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:99571 45.1 45:89cfe7f4:::benchmark_data_smithi130_172551_object99570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1182)", 2024-08-17T17:49:11.046 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:08.827641+0000", 2024-08-17T17:49:11.046 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 2.075640752, 2024-08-17T17:49:11.046 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.069045208999999996, 2024-08-17T17:49:11.046 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:11.046 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:11.046 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:11.046 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:11.046 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:11.046 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 99571 2024-08-17T17:49:11.046 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.046 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:11.046 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.046 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:11.046 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827641+0000", 2024-08-17T17:49:11.047 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.047 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.047 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.047 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:11.047 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827641+0000", 2024-08-17T17:49:11.047 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.047 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.047 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.047 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:11.047 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827640+0000", 2024-08-17T17:49:11.047 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:11.047 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.047 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827677+0000", 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7431000000000001e-05 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827689+0000", 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1399999999999999e-05 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827694+0000", 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0180000000000001e-06 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.048 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.049 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:11.049 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827736+0000", 2024-08-17T17:49:11.049 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1878000000000001e-05 2024-08-17T17:49:11.049 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.049 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.049 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:11.049 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827845+0000", 2024-08-17T17:49:11.049 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000109201 2024-08-17T17:49:11.049 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.049 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.049 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:11.049 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827950+0000", 2024-08-17T17:49:11.049 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105064 2024-08-17T17:49:11.049 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.050 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.050 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:11.050 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830518+0000", 2024-08-17T17:49:11.050 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025678049999999998 2024-08-17T17:49:11.050 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.050 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.050 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.050 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830749+0000", 2024-08-17T17:49:11.050 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00023170999999999999 2024-08-17T17:49:11.050 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.050 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.050 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.050 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.893864+0000", 2024-08-17T17:49:11.051 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.063114049000000005 2024-08-17T17:49:11.051 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.051 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.051 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:11.051 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.893911+0000", 2024-08-17T17:49:11.051 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7811e-05 2024-08-17T17:49:11.051 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.051 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.051 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:11.051 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.896687+0000", 2024-08-17T17:49:11.051 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002775226 2024-08-17T17:49:11.051 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.051 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:11.051 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.052 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.052 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.052 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:99576 45.1 45:8f20544f:::benchmark_data_smithi130_172551_object99575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1182)", 2024-08-17T17:49:11.052 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:08.835336+0000", 2024-08-17T17:49:11.052 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 2.0679456780000001, 2024-08-17T17:49:11.052 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.064230339999999997, 2024-08-17T17:49:11.052 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:11.052 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:11.052 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:11.052 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:11.052 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:11.052 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 99576 2024-08-17T17:49:11.052 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.052 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:11.053 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.053 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:11.053 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.835336+0000", 2024-08-17T17:49:11.053 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.053 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.053 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.053 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:11.053 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.835336+0000", 2024-08-17T17:49:11.053 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:11.053 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.053 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.053 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:11.053 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.835335+0000", 2024-08-17T17:49:11.054 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:49:11.054 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.054 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.054 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:11.054 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.835431+0000", 2024-08-17T17:49:11.054 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5878000000000006e-05 2024-08-17T17:49:11.054 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.054 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.054 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:11.054 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.835448+0000", 2024-08-17T17:49:11.054 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.7691000000000001e-05 2024-08-17T17:49:11.054 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.054 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.054 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:11.055 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.835455+0000", 2024-08-17T17:49:11.055 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2700000000000001e-06 2024-08-17T17:49:11.055 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.055 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.055 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:11.055 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.835507+0000", 2024-08-17T17:49:11.055 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2147000000000001e-05 2024-08-17T17:49:11.055 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.055 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.055 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:11.055 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.835700+0000", 2024-08-17T17:49:11.055 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019319300000000001 2024-08-17T17:49:11.055 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.055 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.056 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:49:11.056 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.835910+0000", 2024-08-17T17:49:11.056 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00020976499999999999 2024-08-17T17:49:11.056 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.056 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.056 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:11.056 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.838931+0000", 2024-08-17T17:49:11.056 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003021433 2024-08-17T17:49:11.056 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.056 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.056 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.056 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.840518+0000", 2024-08-17T17:49:11.056 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0015868049999999999 2024-08-17T17:49:11.056 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.057 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.057 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:11.057 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.896740+0000", 2024-08-17T17:49:11.057 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.056222363999999997 2024-08-17T17:49:11.057 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.057 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.057 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:11.057 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.896793+0000", 2024-08-17T17:49:11.057 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.304e-05 2024-08-17T17:49:11.057 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:11.057 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:11.057 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:11.057 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.899567+0000", 2024-08-17T17:49:11.057 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027734719999999999 2024-08-17T17:49:11.058 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.058 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:11.058 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.058 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:11.058 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:11.058 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:49:11.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:10 smithi130 ceph-mon[105521]: pgmap v1619: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 36 MiB/s wr, 570 op/s; 45851/297600 objects degraded (15.407%); 2194/297600 objects misplaced (0.737%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:49:11.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:10 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 45851/297600 objects degraded (15.407%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:11.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:10 smithi130 ceph-mon[110068]: pgmap v1619: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 36 MiB/s wr, 570 op/s; 45851/297600 objects degraded (15.407%); 2194/297600 objects misplaced (0.737%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:49:11.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:10 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 45851/297600 objects degraded (15.407%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:10 smithi179 ceph-mon[93519]: pgmap v1619: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 36 MiB/s wr, 570 op/s; 45851/297600 objects degraded (15.407%); 2194/297600 objects misplaced (0.737%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:49:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:10 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 45851/297600 objects degraded (15.407%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:11.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.111 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:11.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:11.212 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:11.236 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:11.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:11.338 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:11.387 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:11.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:11.489 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:11.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.547 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:11.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:11.648 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:11.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.673 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:11.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:11.729 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 83 16 101204 101188 76.1753 35.1875 0.0137186 0.0131198 2024-08-17T17:49:11.774 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:11.795 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:11.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:11.897 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:11.933 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:11.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:11.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:12.027 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:49:12.035 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:12.093 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:12.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:12.195 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:12.218 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:12.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:12.313 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:12.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:12.346 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:12.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:12.448 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:12.469 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:12.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:12.570 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:12.593 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:12.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:12.695 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:12.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.719 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:12.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:12.728 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 84 16 101430 101414 75.4367 14.125 0.00732088 0.0131351 2024-08-17T17:49:12.747 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T17:49:12.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:12.861 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:12.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:12.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:12 smithi130 ceph-mon[105521]: pgmap v1620: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 36 MiB/s wr, 570 op/s; 45851/297600 objects degraded (15.407%); 2194/297600 objects misplaced (0.737%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:49:12.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:12 smithi130 ceph-mon[105521]: osdmap e1183: 8 total, 8 up, 7 in 2024-08-17T17:49:12.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:12 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2969410603' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:12.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:12.882 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:12 smithi130 ceph-mon[110068]: pgmap v1620: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 36 MiB/s wr, 570 op/s; 45851/297600 objects degraded (15.407%); 2194/297600 objects misplaced (0.737%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:49:12.883 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:12 smithi130 ceph-mon[110068]: osdmap e1183: 8 total, 8 up, 7 in 2024-08-17T17:49:12.883 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:12 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2969410603' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:12.883 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:12.964 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:12.977 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:12 smithi179 ceph-mon[93519]: pgmap v1620: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 36 MiB/s wr, 570 op/s; 45851/297600 objects degraded (15.407%); 2194/297600 objects misplaced (0.737%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:49:12.977 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:12 smithi179 ceph-mon[93519]: osdmap e1183: 8 total, 8 up, 7 in 2024-08-17T17:49:12.977 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:12 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2969410603' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:12.977 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:12.990 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:12.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:12.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:13.092 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:13.135 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:13.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:13.237 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:13.260 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:13.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:13.362 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:13.408 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:13.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:13.510 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:13.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.560 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:13.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:13.662 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:13.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.707 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:13.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:13.729 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 85 16 101950 101934 74.9316 32.5 0.0305177 0.0133387 2024-08-17T17:49:13.809 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:13.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.856 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:13.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:13.864 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:49:13.864 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:49:13.864 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:49:13.864 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:49:13.864 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:49:13.864 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:49:13.864 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:49:13.864 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:49:13.865 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:49:13.865 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:49:13.865 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:49:13.865 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:49:13.865 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:49:13.865 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:49:13.866 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:49:13.866 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:49:13.957 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:13.981 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:13.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:13.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:14.082 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:13 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:49:14.082 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:13 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2969410603' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:14.082 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:13 smithi179 ceph-mon[93519]: osdmap e1184: 8 total, 8 up, 7 in 2024-08-17T17:49:14.082 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:13 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2271745967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:14.083 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:14.124 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:14.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:14.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:13 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:49:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:13 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2969410603' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:13 smithi130 ceph-mon[105521]: osdmap e1184: 8 total, 8 up, 7 in 2024-08-17T17:49:14.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:13 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2271745967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:14.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:13 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:49:14.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:13 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2969410603' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:14.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:13 smithi130 ceph-mon[110068]: osdmap e1184: 8 total, 8 up, 7 in 2024-08-17T17:49:14.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:13 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2271745967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:14.226 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:14.282 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:14.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:14.384 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:14.411 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:14.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:14.513 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:14.539 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:14.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:14.641 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:14.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.667 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:14.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:14.713 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:14.713 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:14.729 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 86 16 102587 102571 74.5233 39.8125 0.00780134 0.0134113 2024-08-17T17:49:14.768 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:14.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:14.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:14.912 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:14.942 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:14 smithi179 ceph-mon[93519]: pgmap v1623: 89 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38 MiB/s wr, 613 op/s; 42823/303726 objects degraded (14.099%); 4950/303726 objects misplaced (1.630%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:49:14.953 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:14.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:14.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:15.055 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:15.079 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:15.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:15.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:14 smithi130 ceph-mon[105521]: pgmap v1623: 89 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38 MiB/s wr, 613 op/s; 42823/303726 objects degraded (14.099%); 4950/303726 objects misplaced (1.630%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:49:15.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:14 smithi130 ceph-mon[110068]: pgmap v1623: 89 pgs: 1 active+recovery_wait+degraded, 3 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 38 MiB/s wr, 613 op/s; 42823/303726 objects degraded (14.099%); 4950/303726 objects misplaced (1.630%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:49:15.181 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:15.205 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:15.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:15.307 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:15.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.332 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:15.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:15.370 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T17:49:15.370 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:49:15.370 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:49:15.370 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 1.000000 2024-08-17T17:49:15.370 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 6 1 2024-08-17T17:49:15.433 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:15.459 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:49:15.459 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:15.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:15.561 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:15.625 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:15.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:15.727 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:15.734 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 87 16 103185 103169 74.0963 37.375 0.0174268 0.0134882 2024-08-17T17:49:15.744 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:49:15.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.789 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:15.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:15.841 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:15.881 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:15.890 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:15.955 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:15.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:15.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:16.055 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:15 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 42823/303726 objects degraded (14.099%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:16.055 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:15 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 42823/303726 objects degraded (14.099%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:16.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:16.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:15 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 42823/303726 objects degraded (14.099%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:16.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.125 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:16.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:16.226 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:16.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:16.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:16.352 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:16.399 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:16.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:16.501 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:16.525 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:16.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:16.555 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:49:16.626 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:16.648 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:16.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:16.729 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 88 16 103715 103699 73.6308 33.125 0.0165603 0.0135729 2024-08-17T17:49:16.749 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:16.772 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:16.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:16.874 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:16.949 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:16.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:16.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:17.051 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:17.076 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:16 smithi130 ceph-mon[105521]: pgmap v1624: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39 MiB/s wr, 624 op/s; 42823/305073 objects degraded (14.037%); 4833/305073 objects misplaced (1.584%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:49:17.079 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:16 smithi130 ceph-mon[110068]: pgmap v1624: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39 MiB/s wr, 624 op/s; 42823/305073 objects degraded (14.037%); 4833/305073 objects misplaced (1.584%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:49:17.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:16 smithi179 ceph-mon[93519]: pgmap v1624: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 39 MiB/s wr, 624 op/s; 42823/305073 objects degraded (14.037%); 4833/305073 objects misplaced (1.584%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:49:17.106 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:17.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:17.207 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:17.242 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:17.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:17.344 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:17.371 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:17.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:17.411 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:17.473 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:17.494 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:17.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:17.596 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:17.622 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:17.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:17.724 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:17.729 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 89 16 104259 104243 73.1855 34 0.019372 0.0136567 2024-08-17T17:49:17.748 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:17.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:17.850 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:17.873 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:17.873 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:17.875 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:17.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:17.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:17.977 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:18.000 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:18.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:18.101 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:18.124 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:18.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:18.226 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:18.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:18.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:18.351 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:18.375 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:18.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:18.477 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:18.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.528 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:18.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:18.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:49:18.629 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:18.673 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:18.729 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Total time run: 90.0167 2024-08-17T17:49:18.729 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Total writes made: 104804 2024-08-17T17:49:18.729 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Write size: 65536 2024-08-17T17:49:18.729 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Object size: 65536 2024-08-17T17:49:18.729 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Bandwidth (MB/sec): 72.767 2024-08-17T17:49:18.729 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev Bandwidth: 57.3403 2024-08-17T17:49:18.730 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max bandwidth (MB/sec): 269.5 2024-08-17T17:49:18.730 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min bandwidth (MB/sec): 0 2024-08-17T17:49:18.730 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Average IOPS: 1164 2024-08-17T17:49:18.730 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev IOPS: 917.458 2024-08-17T17:49:18.730 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max IOPS: 4312 2024-08-17T17:49:18.730 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min IOPS: 0 2024-08-17T17:49:18.730 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Average Latency(s): 0.0137366 2024-08-17T17:49:18.730 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev Latency(s): 0.0433564 2024-08-17T17:49:18.730 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max latency(s): 2.49826 2024-08-17T17:49:18.730 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min latency(s): 0.000907997 2024-08-17T17:49:18.741 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:49:18.775 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:18.824 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:18.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:18.864 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:18.881 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:18.926 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:18.950 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:18.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:18.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:19.052 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:19.090 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:19.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:19.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:18 smithi179 ceph-mon[93519]: pgmap v1625: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 22 MiB/s wr, 352 op/s; 42823/306072 objects degraded (13.991%); 4609/306072 objects misplaced (1.506%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:49:19.106 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:18 smithi130 ceph-mon[105521]: pgmap v1625: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 22 MiB/s wr, 352 op/s; 42823/306072 objects degraded (13.991%); 4609/306072 objects misplaced (1.506%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:49:19.106 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:18 smithi130 ceph-mon[110068]: pgmap v1625: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 22 MiB/s wr, 352 op/s; 42823/306072 objects degraded (13.991%); 4609/306072 objects misplaced (1.506%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T17:49:19.193 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:19.229 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:19.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:19.331 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:19.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:19.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:19.457 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:19.493 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:19.594 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:19.617 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:19.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:19.719 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:19.740 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:19.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:19.841 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:19.864 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:19.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:19.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:49:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:49:19] "GET /metrics HTTP/1.1" 200 36796 "" "Prometheus/2.51.0" 2024-08-17T17:49:19.966 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:19.989 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:19.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:19.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:20.091 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:20.136 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:20.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:20.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:20.238 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:20.261 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:20.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:20.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:20.363 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:20.385 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:20.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:20.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:20.487 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:20.500 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:49:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:49:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:49:20.537 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:20.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:20.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:20.639 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:20.663 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:20.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:20.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:20.765 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:20.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:20.794 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:20.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:20.812 INFO:teuthology.orchestra.run.smithi130.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-08-17T17:49:20.835 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:20.835 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:20.896 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:20.919 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:20.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:20.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:21.020 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:21.044 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:21.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:21.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:21.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:20 smithi179 ceph-mon[93519]: pgmap v1626: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 662 op/s; 42823/313383 objects degraded (13.665%); 4464/313383 objects misplaced (1.424%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T17:49:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/777530899' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:49:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:20 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:49:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:20 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 42823/313383 objects degraded (13.665%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:21.145 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:21.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:20 smithi130 ceph-mon[105521]: pgmap v1626: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 662 op/s; 42823/313383 objects degraded (13.665%); 4464/313383 objects misplaced (1.424%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T17:49:21.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/777530899' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:49:21.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:20 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:49:21.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:20 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 42823/313383 objects degraded (13.665%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:21.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:20 smithi130 ceph-mon[110068]: pgmap v1626: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 662 op/s; 42823/313383 objects degraded (13.665%); 4464/313383 objects misplaced (1.424%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T17:49:21.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/777530899' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:49:21.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:20 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:49:21.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:20 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 42823/313383 objects degraded (13.665%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:21.198 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:21.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:21.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:21.300 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:21.338 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:21.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:21.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:21.440 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:21.463 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:21.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:21.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:21.537 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:49:21.565 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:21.609 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:21.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:21.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:21.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:21.748 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:21.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:21.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:21.840 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:21.850 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:21.877 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:21.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:21.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:21 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:49:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:21 smithi130 ceph-mon[105521]: osdmap e1185: 8 total, 8 up, 7 in 2024-08-17T17:49:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:21 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:49:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:21 smithi130 ceph-mon[110068]: osdmap e1185: 8 total, 8 up, 7 in 2024-08-17T17:49:21.978 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:22.003 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:22.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:22.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:21 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:49:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:21 smithi179 ceph-mon[93519]: osdmap e1185: 8 total, 8 up, 7 in 2024-08-17T17:49:22.105 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:22.125 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:22.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:22.227 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:22.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.263 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:22.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:22.364 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:22.388 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:22.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:22.489 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:22.541 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:22.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:22.643 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:22.667 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:22.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:22.769 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:22.792 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:22.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:22.893 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:22.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:22.916 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:22.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:23.017 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:23.071 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:23.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:23.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:22 smithi179 ceph-mon[93519]: pgmap v1628: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 556 op/s; 42823/313383 objects degraded (13.665%); 4464/313383 objects misplaced (1.424%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:49:23.137 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:22 smithi130 ceph-mon[110068]: pgmap v1628: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 556 op/s; 42823/313383 objects degraded (13.665%); 4464/313383 objects misplaced (1.424%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:49:23.138 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:22 smithi130 ceph-mon[105521]: pgmap v1628: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 35 MiB/s wr, 556 op/s; 42823/313383 objects degraded (13.665%); 4464/313383 objects misplaced (1.424%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:49:23.173 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:23.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:23.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:23.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:23.228 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:49:23.228 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:49:23.270 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:49:23.270 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:49:23.305 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:49:23.305 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:49:23.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:23.338 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:49:23.338 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:49:23.354 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:23.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:23.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:23.366 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:49:23.366 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:49:23.394 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:49:23.394 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:49:23.423 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:49:23.423 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:49:23.452 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:49:23.452 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:49:23.456 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:23.502 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:23.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:23.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:23.604 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:23.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:23.628 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:23.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:23.729 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:23.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:23.753 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:23.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:23.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:23.891 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:23.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:23.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:23.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:24.026 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:24.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:24.033 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:23 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2924124965' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:24.038 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:49:24.038 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T17:49:24.038 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.039 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:107270 45.a 45:5d3bcdd3:::benchmark_data_smithi130_172551_object2463:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:49:24.039 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:24.024205+0000", 2024-08-17T17:49:24.039 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.0086981420000000007, 2024-08-17T17:49:24.039 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0087227859999999997, 2024-08-17T17:49:24.039 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:24.039 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:49:24.039 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:24.039 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:24.039 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:24.039 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 107270 2024-08-17T17:49:24.039 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.039 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:24.039 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.040 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:24.040 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.024205+0000", 2024-08-17T17:49:24.040 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:24.040 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.040 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.040 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:24.040 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.024205+0000", 2024-08-17T17:49:24.040 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:24.040 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.040 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.040 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:24.040 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.024204+0000", 2024-08-17T17:49:24.040 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:49:24.040 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.041 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.041 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:24.041 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.024209+0000", 2024-08-17T17:49:24.041 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.4e-06 2024-08-17T17:49:24.041 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.041 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.041 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:24.041 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.024218+0000", 2024-08-17T17:49:24.041 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7649999999999999e-06 2024-08-17T17:49:24.041 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.041 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.041 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:24.041 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.024222+0000", 2024-08-17T17:49:24.041 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0749999999999999e-06 2024-08-17T17:49:24.042 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.042 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.042 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:24.042 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.024257+0000", 2024-08-17T17:49:24.042 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4858999999999999e-05 2024-08-17T17:49:24.042 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.042 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.042 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:24.042 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.024328+0000", 2024-08-17T17:49:24.042 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0844000000000001e-05 2024-08-17T17:49:24.042 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.042 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.043 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:49:24.043 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.024391+0000", 2024-08-17T17:49:24.043 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2749999999999994e-05 2024-08-17T17:49:24.043 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.043 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.043 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:24.043 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.026935+0000", 2024-08-17T17:49:24.043 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002544166 2024-08-17T17:49:24.043 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.043 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.043 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:24.043 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.029224+0000", 2024-08-17T17:49:24.043 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0022890979999999998 2024-08-17T17:49:24.043 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:24.044 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:24.044 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:24.044 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.044 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.044 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:107273 45.9 45:9f404ff3:::benchmark_data_smithi130_172551_object2466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:49:24.044 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:24.030213+0000", 2024-08-17T17:49:24.044 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.0026896569999999998, 2024-08-17T17:49:24.044 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027846949999999998, 2024-08-17T17:49:24.044 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:24.044 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:49:24.044 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:24.044 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:24.044 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:24.044 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 107273 2024-08-17T17:49:24.045 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.045 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:24.045 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.045 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:24.045 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030213+0000", 2024-08-17T17:49:24.045 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:24.045 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.045 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.045 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:24.045 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030213+0000", 2024-08-17T17:49:24.045 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:24.045 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.045 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.046 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:24.046 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030213+0000", 2024-08-17T17:49:24.046 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:49:24.046 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.046 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.046 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:24.046 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030216+0000", 2024-08-17T17:49:24.046 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.5489999999999998e-06 2024-08-17T17:49:24.046 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.046 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.046 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:24.046 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030224+0000", 2024-08-17T17:49:24.046 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.5120000000000002e-06 2024-08-17T17:49:24.047 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.047 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.047 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:24.047 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030227+0000", 2024-08-17T17:49:24.047 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2279999999999998e-06 2024-08-17T17:49:24.047 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.047 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.047 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:24.047 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030267+0000", 2024-08-17T17:49:24.047 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9657999999999999e-05 2024-08-17T17:49:24.047 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.047 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.047 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030379+0000", 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00011218900000000001 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030471+0000", 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.1976000000000002e-05 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.45158.0:107271 45.1 e1185/1179 45:8d267227:::benchmark_data_smithi130_172551_object2464:head v 1185'6830, mlcod=0'0)", 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:24.030567+0000", 2024-08-17T17:49:24.048 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.0023361810000000001, 2024-08-17T17:49:24.049 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024708289999999999, 2024-08-17T17:49:24.049 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:24.049 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "started", 2024-08-17T17:49:24.049 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:24.049 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.049 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:24.049 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030567+0000", 2024-08-17T17:49:24.049 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:24.049 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.049 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.049 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:24.049 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030567+0000", 2024-08-17T17:49:24.049 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:24.049 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.050 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.050 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:24.050 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030566+0000", 2024-08-17T17:49:24.050 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:49:24.050 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.050 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.050 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:24.050 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030573+0000", 2024-08-17T17:49:24.050 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4339999999999997e-06 2024-08-17T17:49:24.050 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.050 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.050 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:24.050 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030584+0000", 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0774999999999999e-05 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030589+0000", 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3190000000000002e-06 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030696+0000", 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010732 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:24.030724+0000", 2024-08-17T17:49:24.051 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.7532e-05 2024-08-17T17:49:24.052 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:24.052 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:24.052 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:24.052 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:24.052 INFO:teuthology.orchestra.run.smithi179.stdout: ], 2024-08-17T17:49:24.052 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 3 2024-08-17T17:49:24.052 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:49:24.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:24.161 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:24.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:24.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:23 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2924124965' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:24.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:23 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2924124965' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:24.263 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:24.307 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:24.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:24.409 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:24.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:24.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:24.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:24.559 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:24.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:24.661 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:24.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.708 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:24.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:24.809 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:24.851 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:24.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:24.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:24 smithi130 ceph-mon[105521]: pgmap v1629: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 22 MiB/s wr, 521 op/s; 42352/309609 objects degraded (13.679%); 4063/309609 objects misplaced (1.312%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:49:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:24 smithi130 ceph-mon[110068]: pgmap v1629: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 22 MiB/s wr, 521 op/s; 42352/309609 objects degraded (13.679%); 4063/309609 objects misplaced (1.312%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:49:24.953 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:24.974 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:24.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:24.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:25.076 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:25.078 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:24 smithi179 ceph-mon[93519]: pgmap v1629: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 22 MiB/s wr, 521 op/s; 42352/309609 objects degraded (13.679%); 4063/309609 objects misplaced (1.312%); 7.0 MiB/s, 112 objects/s recovering 2024-08-17T17:49:25.105 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:49:25.122 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:25.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:25.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:25.224 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:25.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:25.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:25.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:25.352 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:25.389 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:25.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:25.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:25.427 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:25.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:25.517 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:25.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:25.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:25.619 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:25.643 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:25.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:25.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:25.745 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:25.774 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:25.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:25.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:25.876 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:25.911 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:25.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:25.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:26.013 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:26.037 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:26.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:26.139 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:26.163 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:26.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:26.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:25 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 42352/309609 objects degraded (13.679%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:26.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:25 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 42352/309609 objects degraded (13.679%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:26.265 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:26.309 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:26.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:26.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:25 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 42352/309609 objects degraded (13.679%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:26.410 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:26.433 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:26.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:26.535 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:26.559 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:26.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:26.661 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:26.668 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T17:49:26.668 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:49:26.668 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-08-17T17:49:26.668 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:49:26.668 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:49:26.669 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-08-17T17:49:26.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.683 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:26.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:26.783 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:26.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.820 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:26.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:26.920 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:26.969 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:26.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:26.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:27.032 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:26 smithi179 ceph-mon[93519]: pgmap v1630: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 20 MiB/s wr, 520 op/s; 42352/308283 objects degraded (13.738%); 4063/308283 objects misplaced (1.318%); 6.3 MiB/s, 100 objects/s recovering 2024-08-17T17:49:27.070 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:27.123 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:27.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:27.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:27.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:26 smithi130 ceph-mon[105521]: pgmap v1630: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 20 MiB/s wr, 520 op/s; 42352/308283 objects degraded (13.738%); 4063/308283 objects misplaced (1.318%); 6.3 MiB/s, 100 objects/s recovering 2024-08-17T17:49:27.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:26 smithi130 ceph-mon[110068]: pgmap v1630: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 20 MiB/s wr, 520 op/s; 42352/308283 objects degraded (13.738%); 4063/308283 objects misplaced (1.318%); 6.3 MiB/s, 100 objects/s recovering 2024-08-17T17:49:27.225 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:27.253 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:27.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:27.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:27.303 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:49:27.303 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:49:27.303 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T17:49:27.303 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T17:49:27.303 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:49:27.355 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:27.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:27.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:27.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:27.507 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:27.529 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:27.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:27.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:27.631 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:27.654 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:27.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:27.756 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:27.799 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:27.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:27.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:27.900 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:27.921 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:27.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:27.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:28.023 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:28.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.074 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:28.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:28.174 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:28.215 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:28.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:28.317 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:28.340 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:28.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:28.442 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:28.491 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:28.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:28.592 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:49:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:28.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:28 smithi179 ceph-mon[93519]: pgmap v1631: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 18 MiB/s wr, 515 op/s; 42064/307434 objects degraded (13.682%); 3855/307434 objects misplaced (1.254%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T17:49:28.615 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:28.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:28.623 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:49:28.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:49:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:28 smithi130 ceph-mon[105521]: pgmap v1631: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 18 MiB/s wr, 515 op/s; 42064/307434 objects degraded (13.682%); 3855/307434 objects misplaced (1.254%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T17:49:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:49:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:28 smithi130 ceph-mon[110068]: pgmap v1631: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 18 MiB/s wr, 515 op/s; 42064/307434 objects degraded (13.682%); 3855/307434 objects misplaced (1.254%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T17:49:28.718 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:28.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.763 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:28.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:28.863 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:28.889 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:28.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:28.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:28.964 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:28.992 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:29.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:29.031 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:29.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:29.132 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:29.164 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:29.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:29.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:29.267 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:29.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:29.310 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:29.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:29.411 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:29.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:29.453 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:29.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:29.554 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:29.602 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:29.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:29.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:29.704 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:29.749 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:29.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:29.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:29.851 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:29.879 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:29.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:29.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:29.884 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:49:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:49:29] "GET /metrics HTTP/1.1" 200 36804 "" "Prometheus/2.51.0" 2024-08-17T17:49:29.981 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:30.018 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:30.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:30.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:30.120 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:30.147 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:30.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:30.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:30.248 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:30.281 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:30.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:30.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:30.383 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:30.406 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:30.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:30.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:30.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:30.548 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:30.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:30.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:30.567 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:49:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:49:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:49:30.649 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:30.672 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:30.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:30.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:30.774 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:30.796 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:30.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:30.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:30.827 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:30 smithi179 ceph-mon[93519]: pgmap v1632: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 2.2 MiB/s wr, 464 op/s; 41793/301539 objects degraded (13.860%); 3681/301539 objects misplaced (1.221%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:49:30.827 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:30 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 41793/301539 objects degraded (13.860%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:30.828 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:49:30.828 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T17:49:30.828 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T17:49:30.828 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T17:49:30.828 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.828 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.44945.0:2808 44.d 44:b7f70492:::rbd_data.af91c8ed8b9b.000000000000005d:head [write 0~2870326] snapc 72=[72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-08-17T17:49:30.828 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:45:43.833685+0000", 2024-08-17T17:49:30.828 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 226.99176354700001, 2024-08-17T17:49:30.828 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.14144047200000001, 2024-08-17T17:49:30.829 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.829 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.829 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.829 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.44945", 2024-08-17T17:49:30.829 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/735940947", 2024-08-17T17:49:30.829 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 2808 2024-08-17T17:49:30.829 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.829 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.829 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.829 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.829 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.833685+0000", 2024-08-17T17:49:30.830 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.830 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.830 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.830 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.830 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.833685+0000", 2024-08-17T17:49:30.830 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.830 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.830 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.830 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.830 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.833679+0000", 2024-08-17T17:49:30.830 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999943 2024-08-17T17:49:30.830 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.831 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.831 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.831 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.838832+0000", 2024-08-17T17:49:30.831 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0051528889999999999 2024-08-17T17:49:30.831 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.831 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.831 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.831 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.838862+0000", 2024-08-17T17:49:30.831 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9703999999999999e-05 2024-08-17T17:49:30.831 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.831 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.831 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.831 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.838867+0000", 2024-08-17T17:49:30.831 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.31e-06 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.838905+0000", 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7942999999999999e-05 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.838985+0000", 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9950000000000005e-05 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.839104+0000", 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00011874 2024-08-17T17:49:30.832 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.833 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.833 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.833 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.843852+0000", 2024-08-17T17:49:30.833 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0047476289999999997 2024-08-17T17:49:30.833 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.833 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.833 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.833 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.868377+0000", 2024-08-17T17:49:30.833 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.024524962000000001 2024-08-17T17:49:30.833 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.834 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.834 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.834 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.975041+0000", 2024-08-17T17:49:30.834 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.106664704 2024-08-17T17:49:30.834 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.834 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.834 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.834 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.975080+0000", 2024-08-17T17:49:30.834 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9104999999999999e-05 2024-08-17T17:49:30.834 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.834 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.834 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.835 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.975126+0000", 2024-08-17T17:49:30.835 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5176000000000003e-05 2024-08-17T17:49:30.835 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.835 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.835 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.835 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.835 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.835 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:59331 45.7 45:e4b8b26e:::benchmark_data_smithi130_172551_object59330:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1175)", 2024-08-17T17:49:30.835 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:16.888202+0000", 2024-08-17T17:49:30.835 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 73.937247157000002, 2024-08-17T17:49:30.835 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.064058130000000005, 2024-08-17T17:49:30.835 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.836 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.836 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.836 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.836 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.836 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 59331 2024-08-17T17:49:30.836 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.836 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.836 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.836 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.836 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:16.888202+0000", 2024-08-17T17:49:30.836 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.836 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.836 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.836 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.837 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:16.888202+0000", 2024-08-17T17:49:30.837 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.837 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.837 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.837 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.837 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:16.888200+0000", 2024-08-17T17:49:30.837 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:30.837 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.837 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.837 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.837 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:16.888233+0000", 2024-08-17T17:49:30.837 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3253999999999997e-05 2024-08-17T17:49:30.837 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.838 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.838 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.838 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:16.888242+0000", 2024-08-17T17:49:30.838 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.1109999999999997e-06 2024-08-17T17:49:30.838 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.838 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.838 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.838 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:16.888247+0000", 2024-08-17T17:49:30.838 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4730000000000002e-06 2024-08-17T17:49:30.838 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.838 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.838 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.838 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:16.889483+0000", 2024-08-17T17:49:30.839 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0012357049999999999 2024-08-17T17:49:30.839 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.839 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.839 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.839 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:16.889667+0000", 2024-08-17T17:49:30.839 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00018454699999999999 2024-08-17T17:49:30.839 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.839 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.839 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:49:30.839 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:16.889791+0000", 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000124375 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:16.891696+0000", 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019046289999999999 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:16.892029+0000", 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000332555 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.840 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:16.952162+0000", 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.060133661999999997 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:16.952216+0000", 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.4069e-05 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:16.952260+0000", 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3296e-05 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.841 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.842 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:82940 45.c 45:37c40ad6:::benchmark_data_smithi130_172551_object82939:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:49:30.842 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.327053+0000", 2024-08-17T17:49:30.842 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 52.498395928000001, 2024-08-17T17:49:30.842 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.080169211000000004, 2024-08-17T17:49:30.842 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.842 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.842 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.842 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.843 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.843 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 82940 2024-08-17T17:49:30.843 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.843 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.843 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.843 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.843 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327053+0000", 2024-08-17T17:49:30.843 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.843 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327053+0000", 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327051+0000", 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327120+0000", 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.8663000000000003e-05 2024-08-17T17:49:30.844 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.845 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.845 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.845 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327135+0000", 2024-08-17T17:49:30.845 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5429000000000001e-05 2024-08-17T17:49:30.845 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.845 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.845 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.845 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327142+0000", 2024-08-17T17:49:30.845 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4280000000000001e-06 2024-08-17T17:49:30.845 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.845 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.845 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.845 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327185+0000", 2024-08-17T17:49:30.845 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3405999999999997e-05 2024-08-17T17:49:30.846 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.846 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.846 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.846 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327319+0000", 2024-08-17T17:49:30.846 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013401299999999999 2024-08-17T17:49:30.846 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.846 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.846 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:49:30.846 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327462+0000", 2024-08-17T17:49:30.846 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014319699999999999 2024-08-17T17:49:30.846 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.846 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.846 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.846 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.332836+0000", 2024-08-17T17:49:30.847 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0053734919999999997 2024-08-17T17:49:30.847 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.847 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.847 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.847 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.333565+0000", 2024-08-17T17:49:30.847 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00072900500000000002 2024-08-17T17:49:30.847 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.847 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.847 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.847 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.407138+0000", 2024-08-17T17:49:30.847 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.073573349999999996 2024-08-17T17:49:30.847 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.847 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.847 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.848 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.407185+0000", 2024-08-17T17:49:30.848 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7311999999999997e-05 2024-08-17T17:49:30.848 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.848 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.848 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.848 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.407222+0000", 2024-08-17T17:49:30.848 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6621000000000001e-05 2024-08-17T17:49:30.848 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.848 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.848 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.848 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.848 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.848 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:82944 45.c 45:3e028341:::benchmark_data_smithi130_172551_object82943:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:49:30.848 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.329686+0000", 2024-08-17T17:49:30.849 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 52.495762339000002, 2024-08-17T17:49:30.849 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.091734019, 2024-08-17T17:49:30.849 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.849 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.849 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.849 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.849 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.849 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 82944 2024-08-17T17:49:30.849 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.849 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.849 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.849 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.849 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.329686+0000", 2024-08-17T17:49:30.849 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.329686+0000", 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.329686+0000", 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.329753+0000", 2024-08-17T17:49:30.850 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7683999999999998e-05 2024-08-17T17:49:30.851 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.851 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.851 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.851 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.329764+0000", 2024-08-17T17:49:30.851 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0334e-05 2024-08-17T17:49:30.851 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.851 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.851 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.851 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.329769+0000", 2024-08-17T17:49:30.851 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8099999999999997e-06 2024-08-17T17:49:30.851 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.851 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.329961+0000", 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019194700000000001 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.330073+0000", 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000112793 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.330203+0000", 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000129212 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.852 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.853 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.853 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.332999+0000", 2024-08-17T17:49:30.853 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027968870000000001 2024-08-17T17:49:30.853 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.853 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.853 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.853 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.333696+0000", 2024-08-17T17:49:30.853 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00069637200000000001 2024-08-17T17:49:30.853 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.853 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.853 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.853 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.421298+0000", 2024-08-17T17:49:30.853 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.087602625000000003 2024-08-17T17:49:30.853 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.854 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.854 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.854 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.421379+0000", 2024-08-17T17:49:30.854 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.1094999999999995e-05 2024-08-17T17:49:30.854 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.855 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.855 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.855 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.421420+0000", 2024-08-17T17:49:30.855 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0859999999999998e-05 2024-08-17T17:49:30.855 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.855 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.856 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.856 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.856 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.856 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:90850 45.a 45:52267e40:::benchmark_data_smithi130_172551_object90849:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:49:30.856 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:53.711337+0000", 2024-08-17T17:49:30.856 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 37.114111454000003, 2024-08-17T17:49:30.856 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.078178569000000003, 2024-08-17T17:49:30.856 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.856 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.856 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.856 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.856 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.856 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 90850 2024-08-17T17:49:30.856 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.857 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.857 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.857 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.857 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711337+0000", 2024-08-17T17:49:30.857 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.858 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.858 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.858 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.858 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711337+0000", 2024-08-17T17:49:30.858 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.858 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.858 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.858 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.858 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711336+0000", 2024-08-17T17:49:30.858 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:49:30.858 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.859 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.859 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.859 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711402+0000", 2024-08-17T17:49:30.859 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6050000000000006e-05 2024-08-17T17:49:30.859 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.859 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.859 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.859 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711411+0000", 2024-08-17T17:49:30.859 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.5560000000000008e-06 2024-08-17T17:49:30.859 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.859 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.859 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.859 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711415+0000", 2024-08-17T17:49:30.860 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1999999999999996e-06 2024-08-17T17:49:30.860 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.860 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.860 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.860 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711456+0000", 2024-08-17T17:49:30.860 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1356000000000002e-05 2024-08-17T17:49:30.860 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.860 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.860 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.860 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711588+0000", 2024-08-17T17:49:30.860 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013130800000000001 2024-08-17T17:49:30.860 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.860 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.860 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:49:30.861 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711732+0000", 2024-08-17T17:49:30.861 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000144392 2024-08-17T17:49:30.861 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.861 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.861 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.861 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.714345+0000", 2024-08-17T17:49:30.861 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002613074 2024-08-17T17:49:30.861 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.861 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.861 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.861 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.787326+0000", 2024-08-17T17:49:30.861 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.072981089999999998 2024-08-17T17:49:30.862 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.862 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.862 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.862 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.789423+0000", 2024-08-17T17:49:30.862 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020972270000000001 2024-08-17T17:49:30.862 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.862 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.862 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.862 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.789474+0000", 2024-08-17T17:49:30.862 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0813000000000003e-05 2024-08-17T17:49:30.862 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.862 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.789516+0000", 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1517999999999998e-05 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:90856 45.9 45:9cde683e:::benchmark_data_smithi130_172551_object90855:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:53.717331+0000", 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 37.108117518, 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.073021671999999996, 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.863 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 90856 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717331+0000", 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717331+0000", 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.864 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717330+0000", 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717387+0000", 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7080999999999998e-05 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717401+0000", 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3961000000000001e-05 2024-08-17T17:49:30.865 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717407+0000", 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0630000000000002e-06 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717455+0000", 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7966000000000001e-05 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717595+0000", 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014063300000000001 2024-08-17T17:49:30.866 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717749+0000", 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000154222 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.720647+0000", 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028974209999999999 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.720929+0000", 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00028230200000000001 2024-08-17T17:49:30.867 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.868 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.868 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.868 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.790271+0000", 2024-08-17T17:49:30.868 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.069341497000000002 2024-08-17T17:49:30.868 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.868 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.868 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.868 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.790320+0000", 2024-08-17T17:49:30.868 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9051999999999997e-05 2024-08-17T17:49:30.868 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.868 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.868 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.868 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.790353+0000", 2024-08-17T17:49:30.868 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3086000000000001e-05 2024-08-17T17:49:30.869 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.869 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.869 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.869 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.869 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.869 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:90857 45.a 45:574290ca:::benchmark_data_smithi130_172551_object90856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:49:30.869 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:53.717418+0000", 2024-08-17T17:49:30.869 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 37.108030931000002, 2024-08-17T17:49:30.869 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.072220353000000001, 2024-08-17T17:49:30.869 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.869 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.869 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 90857 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717418+0000", 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717418+0000", 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.870 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717417+0000", 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717454+0000", 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6261999999999999e-05 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717466+0000", 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1992e-05 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.871 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.872 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.872 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717469+0000", 2024-08-17T17:49:30.872 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.839e-06 2024-08-17T17:49:30.872 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.872 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.872 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.872 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717504+0000", 2024-08-17T17:49:30.872 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.481e-05 2024-08-17T17:49:30.872 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.872 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.872 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.872 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717625+0000", 2024-08-17T17:49:30.872 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000121107 2024-08-17T17:49:30.872 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717757+0000", 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013128700000000001 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.720559+0000", 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028020409999999999 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.789228+0000", 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.068669189000000005 2024-08-17T17:49:30.873 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.789555+0000", 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00032744700000000002 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.789602+0000", 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6687999999999998e-05 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.789638+0000", 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6140000000000003e-05 2024-08-17T17:49:30.874 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.875 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.875 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.875 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.875 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.875 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:91365 45.c 45:381dbc23:::benchmark_data_smithi130_172551_object91364:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:49:30.875 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:54.621472+0000", 2024-08-17T17:49:30.875 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 36.203976697999998, 2024-08-17T17:49:30.875 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.067773287000000002, 2024-08-17T17:49:30.875 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.875 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.875 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.875 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.875 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.876 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 91365 2024-08-17T17:49:30.876 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.876 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.876 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.876 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.876 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621472+0000", 2024-08-17T17:49:30.876 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.876 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.876 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.876 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.876 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621472+0000", 2024-08-17T17:49:30.876 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.877 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.877 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.877 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.877 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621471+0000", 2024-08-17T17:49:30.877 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:49:30.877 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.877 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.877 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.877 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621555+0000", 2024-08-17T17:49:30.877 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3290999999999999e-05 2024-08-17T17:49:30.877 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.877 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.877 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621565+0000", 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0910000000000001e-05 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621570+0000", 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2209999999999999e-06 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621630+0000", 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0263999999999998e-05 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.878 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621709+0000", 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.8641999999999997e-05 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621798+0000", 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.9588999999999999e-05 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.627027+0000", 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0052288689999999997 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.879 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.880 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.629739+0000", 2024-08-17T17:49:30.880 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027122449999999998 2024-08-17T17:49:30.880 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.880 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.880 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.880 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689175+0000", 2024-08-17T17:49:30.880 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.059435756999999999 2024-08-17T17:49:30.880 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.880 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.880 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.880 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689218+0000", 2024-08-17T17:49:30.880 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3129e-05 2024-08-17T17:49:30.880 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.880 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.881 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.881 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689245+0000", 2024-08-17T17:49:30.881 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.711e-05 2024-08-17T17:49:30.881 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.881 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.881 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.882 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.882 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.882 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:91366 45.c 45:31c17a2b:::benchmark_data_smithi130_172551_object91365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:49:30.882 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:54.622876+0000", 2024-08-17T17:49:30.882 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 36.202572609999997, 2024-08-17T17:49:30.882 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.066455942000000004, 2024-08-17T17:49:30.882 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.883 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.883 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.883 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.883 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.883 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 91366 2024-08-17T17:49:30.883 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.883 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.883 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.883 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.883 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.622876+0000", 2024-08-17T17:49:30.883 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.883 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.883 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.883 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.622876+0000", 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.622875+0000", 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.622924+0000", 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9209999999999998e-05 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.884 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.622939+0000", 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4564e-05 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.622954+0000", 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5536000000000002e-05 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.623961+0000", 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0010069879999999999 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.885 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624089+0000", 2024-08-17T17:49:30.886 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000127747 2024-08-17T17:49:30.886 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.886 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.886 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:49:30.886 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624337+0000", 2024-08-17T17:49:30.886 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00024801899999999998 2024-08-17T17:49:30.886 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.886 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.886 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.886 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.629686+0000", 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0053493459999999996 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.629849+0000", 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016321900000000001 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689281+0000", 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.059432064 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.887 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689311+0000", 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9754e-05 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689332+0000", 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0818999999999998e-05 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:91369 45.c 45:3fbff2c1:::benchmark_data_smithi130_172551_object91368:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:54.624118+0000", 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 36.201330740000003, 2024-08-17T17:49:30.888 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.065287511000000006, 2024-08-17T17:49:30.889 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.889 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.889 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.889 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.889 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.889 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 91369 2024-08-17T17:49:30.889 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.889 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.889 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.889 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.889 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624118+0000", 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624118+0000", 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624117+0000", 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.890 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624160+0000", 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2967999999999999e-05 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624167+0000", 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.8990000000000004e-06 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624172+0000", 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8319999999999996e-06 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.891 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.627046+0000", 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028742910000000002 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.627158+0000", 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00011133600000000001 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.627294+0000", 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000136415 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.629898+0000", 2024-08-17T17:49:30.892 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0026042819999999999 2024-08-17T17:49:30.893 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.893 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.893 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.893 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.630090+0000", 2024-08-17T17:49:30.893 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019181999999999999 2024-08-17T17:49:30.893 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.893 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.893 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.893 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689364+0000", 2024-08-17T17:49:30.893 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.059274134999999999 2024-08-17T17:49:30.893 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.893 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689388+0000", 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3839999999999999e-05 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689405+0000", 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.7360999999999999e-05 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:108187 45.a 45:56008bcf:::benchmark_data_smithi130_172551_object3380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:49:30.894 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:25.915418+0000", 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 4.9100303690000002, 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.104042786, 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 108187 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915418+0000", 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.895 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.896 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.896 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.896 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915418+0000", 2024-08-17T17:49:30.896 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.896 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.896 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.896 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.896 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915417+0000", 2024-08-17T17:49:30.896 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:49:30.896 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.896 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.896 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.897 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915425+0000", 2024-08-17T17:49:30.897 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7860000000000002e-06 2024-08-17T17:49:30.897 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.897 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.897 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.897 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915441+0000", 2024-08-17T17:49:30.897 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5480000000000001e-05 2024-08-17T17:49:30.897 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.897 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.897 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.897 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915447+0000", 2024-08-17T17:49:30.898 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6139999999999997e-06 2024-08-17T17:49:30.898 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.898 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.898 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.898 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915496+0000", 2024-08-17T17:49:30.898 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8374999999999997e-05 2024-08-17T17:49:30.898 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.898 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.898 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.898 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915635+0000", 2024-08-17T17:49:30.898 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000138758 2024-08-17T17:49:30.898 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.898 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.899 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:49:30.899 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915761+0000", 2024-08-17T17:49:30.899 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000126573 2024-08-17T17:49:30.899 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.899 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.899 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.899 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.918644+0000", 2024-08-17T17:49:30.899 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028828930000000001 2024-08-17T17:49:30.899 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.899 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.899 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.899 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.017278+0000", 2024-08-17T17:49:30.900 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.098633437000000004 2024-08-17T17:49:30.900 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.900 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.900 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.900 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.019367+0000", 2024-08-17T17:49:30.900 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020891909999999998 2024-08-17T17:49:30.900 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:30.901 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.901 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.901 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.901 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.019417+0000", 2024-08-17T17:49:30.901 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9982999999999999e-05 2024-08-17T17:49:30.901 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.901 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.902 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.902 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.019461+0000", 2024-08-17T17:49:30.902 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4385999999999998e-05 2024-08-17T17:49:30.902 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.902 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.902 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.902 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.902 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.902 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:108356 45.a 45:5e4e7849:::benchmark_data_smithi130_172551_object3549:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:49:30.902 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:26.371218+0000", 2024-08-17T17:49:30.902 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 4.4542308119999996, 2024-08-17T17:49:30.902 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.077600958999999997, 2024-08-17T17:49:30.902 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.903 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.903 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.903 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.903 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.903 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 108356 2024-08-17T17:49:30.903 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.903 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.903 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.903 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.903 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371218+0000", 2024-08-17T17:49:30.903 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.903 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.903 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.903 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.904 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371218+0000", 2024-08-17T17:49:30.904 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.904 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.904 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.904 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.904 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371217+0000", 2024-08-17T17:49:30.904 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:30.904 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.904 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.904 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.904 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371222+0000", 2024-08-17T17:49:30.904 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2700000000000004e-06 2024-08-17T17:49:30.904 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.905 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.905 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.905 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371233+0000", 2024-08-17T17:49:30.905 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1413999999999999e-05 2024-08-17T17:49:30.905 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.905 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.905 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.905 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371238+0000", 2024-08-17T17:49:30.905 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4100000000000001e-06 2024-08-17T17:49:30.905 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.905 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.905 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.905 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371281+0000", 2024-08-17T17:49:30.905 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3653999999999999e-05 2024-08-17T17:49:30.906 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.906 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.906 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.906 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371366+0000", 2024-08-17T17:49:30.906 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.4938999999999994e-05 2024-08-17T17:49:30.906 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.906 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.906 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:49:30.906 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371441+0000", 2024-08-17T17:49:30.906 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.4386000000000002e-05 2024-08-17T17:49:30.906 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.906 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.906 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.907 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.374160+0000", 2024-08-17T17:49:30.907 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027197570000000002 2024-08-17T17:49:30.907 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.907 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.907 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.907 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.375982+0000", 2024-08-17T17:49:30.907 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001821671 2024-08-17T17:49:30.907 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.907 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.907 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.907 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.448743+0000", 2024-08-17T17:49:30.907 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.072761274000000001 2024-08-17T17:49:30.907 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.907 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.908 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.908 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.448774+0000", 2024-08-17T17:49:30.908 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1004000000000003e-05 2024-08-17T17:49:30.908 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.908 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.908 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.908 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.448819+0000", 2024-08-17T17:49:30.908 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4406000000000001e-05 2024-08-17T17:49:30.908 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.908 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.909 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.909 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.909 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.909 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109023 45.a 45:5d1a62c7:::benchmark_data_smithi130_172551_object4216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:49:30.909 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:27.877934+0000", 2024-08-17T17:49:30.909 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 2.9475151560000001, 2024-08-17T17:49:30.909 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.119736067, 2024-08-17T17:49:30.909 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.909 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.909 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.909 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.909 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.909 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109023 2024-08-17T17:49:30.909 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.910 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.910 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.910 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.910 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.877934+0000", 2024-08-17T17:49:30.910 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.910 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.910 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.910 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.910 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.877934+0000", 2024-08-17T17:49:30.910 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.910 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.910 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.910 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.910 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.877933+0000", 2024-08-17T17:49:30.911 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:49:30.911 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.911 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.911 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.911 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.877938+0000", 2024-08-17T17:49:30.911 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.755e-06 2024-08-17T17:49:30.911 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.911 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.911 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.911 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.877951+0000", 2024-08-17T17:49:30.911 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2995e-05 2024-08-17T17:49:30.911 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.911 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.912 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.912 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.877957+0000", 2024-08-17T17:49:30.912 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7690000000000002e-06 2024-08-17T17:49:30.912 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.912 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.912 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.912 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.878003+0000", 2024-08-17T17:49:30.912 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5886000000000001e-05 2024-08-17T17:49:30.912 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.912 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.912 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.912 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.878117+0000", 2024-08-17T17:49:30.912 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000114449 2024-08-17T17:49:30.912 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.878221+0000", 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010336399999999999 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.880965+0000", 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027437289999999999 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.886359+0000", 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0053943530000000002 2024-08-17T17:49:30.913 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.997513+0000", 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.111154077 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.997617+0000", 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000103948 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.997670+0000", 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2692999999999998e-05 2024-08-17T17:49:30.914 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109151 45.a 45:566f02b3:::benchmark_data_smithi130_172551_object4344:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:28.221634+0000", 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 2.6038146310000001, 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.079254931000000001, 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109151 2024-08-17T17:49:30.915 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.916 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.916 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.916 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.916 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221634+0000", 2024-08-17T17:49:30.916 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.916 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.916 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.916 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.916 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221634+0000", 2024-08-17T17:49:30.916 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.916 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.916 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221633+0000", 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221637+0000", 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8539999999999999e-06 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221644+0000", 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.401e-06 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.917 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.918 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.918 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221648+0000", 2024-08-17T17:49:30.918 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3050000000000001e-06 2024-08-17T17:49:30.918 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.918 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.918 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.918 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221680+0000", 2024-08-17T17:49:30.918 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2404999999999999e-05 2024-08-17T17:49:30.918 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.918 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.918 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.918 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221753+0000", 2024-08-17T17:49:30.918 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3312000000000006e-05 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221844+0000", 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0953999999999997e-05 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.224607+0000", 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002762376 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.237911+0000", 2024-08-17T17:49:30.919 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013304744 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.300805+0000", 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.062893874000000002 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.300867+0000", 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1592000000000003e-05 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.300889+0000", 2024-08-17T17:49:30.920 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.2019e-05 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109163 45.a 45:54edb7bc:::benchmark_data_smithi130_172551_object4356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:28.242813+0000", 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 2.5826355620000001, 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.079628663000000002, 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.921 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109163 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242813+0000", 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242813+0000", 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.922 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242812+0000", 2024-08-17T17:49:30.923 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:49:30.923 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.923 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.923 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.923 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242816+0000", 2024-08-17T17:49:30.923 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3449999999999999e-06 2024-08-17T17:49:30.923 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.923 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.923 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.923 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242827+0000", 2024-08-17T17:49:30.923 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0137e-05 2024-08-17T17:49:30.923 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.923 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242832+0000", 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5439999999999998e-06 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242873+0000", 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0948999999999997e-05 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242992+0000", 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000119027 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.924 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.925 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:49:30.925 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.243110+0000", 2024-08-17T17:49:30.925 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000118292 2024-08-17T17:49:30.925 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.925 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.925 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.925 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.246498+0000", 2024-08-17T17:49:30.925 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0033872749999999999 2024-08-17T17:49:30.925 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.925 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.925 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.926 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.247473+0000", 2024-08-17T17:49:30.926 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00097564699999999995 2024-08-17T17:49:30.926 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.926 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.926 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.926 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.322376+0000", 2024-08-17T17:49:30.926 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.074903091000000005 2024-08-17T17:49:30.926 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.926 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.926 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.926 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.322409+0000", 2024-08-17T17:49:30.926 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2419000000000003e-05 2024-08-17T17:49:30.926 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.927 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.927 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.927 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.322442+0000", 2024-08-17T17:49:30.927 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2947000000000002e-05 2024-08-17T17:49:30.927 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.927 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.927 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.927 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.927 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.927 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109168 45.a 45:51a73ff8:::benchmark_data_smithi130_172551_object4361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:49:30.927 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:28.249348+0000", 2024-08-17T17:49:30.927 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 2.5761004330000001, 2024-08-17T17:49:30.928 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.073180470999999997, 2024-08-17T17:49:30.928 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.928 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.928 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.928 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.928 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.928 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109168 2024-08-17T17:49:30.928 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.928 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.928 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.929 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.929 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249348+0000", 2024-08-17T17:49:30.929 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.929 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.929 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.929 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.929 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249348+0000", 2024-08-17T17:49:30.929 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.929 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.929 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.929 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.929 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249347+0000", 2024-08-17T17:49:30.929 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:49:30.929 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.930 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.930 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.930 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249352+0000", 2024-08-17T17:49:30.930 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9239999999999998e-06 2024-08-17T17:49:30.930 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.930 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.930 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.930 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249362+0000", 2024-08-17T17:49:30.930 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9110000000000005e-06 2024-08-17T17:49:30.930 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249366+0000", 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2629999999999997e-06 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249400+0000", 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3858e-05 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249476+0000", 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.5542000000000003e-05 2024-08-17T17:49:30.931 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.932 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.932 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:49:30.932 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249543+0000", 2024-08-17T17:49:30.932 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7414000000000002e-05 2024-08-17T17:49:30.932 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.932 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.932 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.932 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.252152+0000", 2024-08-17T17:49:30.932 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002609077 2024-08-17T17:49:30.932 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.932 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.932 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.932 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.252895+0000", 2024-08-17T17:49:30.932 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00074267699999999996 2024-08-17T17:49:30.933 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.933 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.933 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.933 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.322474+0000", 2024-08-17T17:49:30.933 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.069579616999999996 2024-08-17T17:49:30.933 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.933 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.933 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.933 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.322505+0000", 2024-08-17T17:49:30.933 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0766000000000003e-05 2024-08-17T17:49:30.933 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.933 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.933 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.933 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.322529+0000", 2024-08-17T17:49:30.934 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3533e-05 2024-08-17T17:49:30.934 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.934 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.934 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.934 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.934 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.934 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109173 45.a 45:5728d6f3:::benchmark_data_smithi130_172551_object4366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:49:30.934 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:28.266456+0000", 2024-08-17T17:49:30.934 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 2.5589925660000001, 2024-08-17T17:49:30.934 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.062958629000000002, 2024-08-17T17:49:30.934 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.935 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.935 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.935 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.935 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.935 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109173 2024-08-17T17:49:30.935 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.935 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.935 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.935 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.935 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.266456+0000", 2024-08-17T17:49:30.935 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.935 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.935 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.936 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.936 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.266456+0000", 2024-08-17T17:49:30.936 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.936 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.936 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.936 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.936 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.266455+0000", 2024-08-17T17:49:30.936 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:49:30.936 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.936 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.936 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.936 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.266463+0000", 2024-08-17T17:49:30.936 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0970000000000006e-06 2024-08-17T17:49:30.936 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.937 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.937 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.937 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.266472+0000", 2024-08-17T17:49:30.937 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2940000000000006e-06 2024-08-17T17:49:30.937 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.937 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.937 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.937 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.266475+0000", 2024-08-17T17:49:30.937 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.703e-06 2024-08-17T17:49:30.937 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.937 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.937 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.937 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.266505+0000", 2024-08-17T17:49:30.938 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9773000000000001e-05 2024-08-17T17:49:30.938 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.938 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.938 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.938 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.266586+0000", 2024-08-17T17:49:30.938 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.1451000000000003e-05 2024-08-17T17:49:30.938 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.938 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.938 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:49:30.938 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.266655+0000", 2024-08-17T17:49:30.938 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.8441000000000004e-05 2024-08-17T17:49:30.938 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.938 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.938 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.939 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.269275+0000", 2024-08-17T17:49:30.939 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002620402 2024-08-17T17:49:30.939 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.939 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.939 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.939 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.269299+0000", 2024-08-17T17:49:30.939 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3801000000000001e-05 2024-08-17T17:49:30.939 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.939 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.939 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.329344+0000", 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.060044638999999997 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.329380+0000", 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6007000000000003e-05 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.329415+0000", 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4941999999999999e-05 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.940 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109535 45.a 45:50e21fca:::benchmark_data_smithi130_172551_object4728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:29.154039+0000", 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 1.6714098050000001, 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.12176530300000001, 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109535 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.941 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154039+0000", 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154039+0000", 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154038+0000", 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.942 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.943 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.943 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154043+0000", 2024-08-17T17:49:30.943 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0560000000000004e-06 2024-08-17T17:49:30.943 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.943 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.943 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.943 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154054+0000", 2024-08-17T17:49:30.943 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.057e-05 2024-08-17T17:49:30.943 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.943 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.943 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.943 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154058+0000", 2024-08-17T17:49:30.944 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.702e-06 2024-08-17T17:49:30.944 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.944 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.944 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.944 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154097+0000", 2024-08-17T17:49:30.944 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8847999999999998e-05 2024-08-17T17:49:30.944 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.944 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.944 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.944 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154205+0000", 2024-08-17T17:49:30.944 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000107408 2024-08-17T17:49:30.944 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.944 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.944 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:49:30.945 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154309+0000", 2024-08-17T17:49:30.945 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010401 2024-08-17T17:49:30.945 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.945 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.945 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.945 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.157600+0000", 2024-08-17T17:49:30.945 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003291661 2024-08-17T17:49:30.945 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.945 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.945 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.945 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.158088+0000", 2024-08-17T17:49:30.945 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00048783099999999999 2024-08-17T17:49:30.946 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.946 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.946 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.946 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.275734+0000", 2024-08-17T17:49:30.946 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.117646024 2024-08-17T17:49:30.946 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.946 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.946 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.946 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.275771+0000", 2024-08-17T17:49:30.946 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6519000000000001e-05 2024-08-17T17:49:30.946 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.947 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.947 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.947 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.275804+0000", 2024-08-17T17:49:30.947 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3569999999999999e-05 2024-08-17T17:49:30.947 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.947 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.947 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.947 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.947 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.947 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109546 45.a 45:5a3ddbc0:::benchmark_data_smithi130_172551_object4739:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:49:30.947 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:29.173181+0000", 2024-08-17T17:49:30.947 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 1.652267986, 2024-08-17T17:49:30.948 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.109487797, 2024-08-17T17:49:30.948 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.948 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.948 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.948 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.948 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.948 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109546 2024-08-17T17:49:30.948 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.948 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.948 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.948 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.948 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173181+0000", 2024-08-17T17:49:30.949 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.949 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.949 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.949 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.949 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173181+0000", 2024-08-17T17:49:30.949 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.949 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.949 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.949 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.949 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173180+0000", 2024-08-17T17:49:30.949 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:49:30.949 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.949 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.950 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.950 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173186+0000", 2024-08-17T17:49:30.950 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2029999999999997e-06 2024-08-17T17:49:30.950 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.950 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.950 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.950 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173197+0000", 2024-08-17T17:49:30.950 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1167e-05 2024-08-17T17:49:30.950 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.950 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.950 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173203+0000", 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.241e-06 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173232+0000", 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8569999999999999e-05 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173308+0000", 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.6097e-05 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.951 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:49:30.952 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173373+0000", 2024-08-17T17:49:30.952 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4941e-05 2024-08-17T17:49:30.952 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.952 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.952 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.952 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.175940+0000", 2024-08-17T17:49:30.952 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025670039999999999 2024-08-17T17:49:30.952 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.952 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.952 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.952 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.182032+0000", 2024-08-17T17:49:30.953 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0060918719999999999 2024-08-17T17:49:30.953 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.953 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.953 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.953 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.282582+0000", 2024-08-17T17:49:30.953 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.10055022700000001 2024-08-17T17:49:30.953 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.953 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.953 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.953 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.282629+0000", 2024-08-17T17:49:30.953 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6981999999999998e-05 2024-08-17T17:49:30.953 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.954 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.954 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.954 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.282668+0000", 2024-08-17T17:49:30.954 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9573999999999998e-05 2024-08-17T17:49:30.954 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.954 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.954 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.954 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.954 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.954 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109547 45.a 45:5fae21b8:::benchmark_data_smithi130_172551_object4740:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:49:30.954 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:29.176203+0000", 2024-08-17T17:49:30.954 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 1.649245281, 2024-08-17T17:49:30.954 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.106643664, 2024-08-17T17:49:30.954 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:30.955 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:49:30.955 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:30.955 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:30.955 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:30.955 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109547 2024-08-17T17:49:30.955 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.955 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:30.955 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.955 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:30.955 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176203+0000", 2024-08-17T17:49:30.955 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176203+0000", 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176203+0000", 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176210+0000", 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.1670000000000002e-06 2024-08-17T17:49:30.956 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.957 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.957 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:30.957 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176220+0000", 2024-08-17T17:49:30.957 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0344e-05 2024-08-17T17:49:30.957 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.957 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.957 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:30.957 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176226+0000", 2024-08-17T17:49:30.957 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9499999999999998e-06 2024-08-17T17:49:30.957 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.957 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.957 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:30.957 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176253+0000", 2024-08-17T17:49:30.957 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.6786000000000001e-05 2024-08-17T17:49:30.958 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.958 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.958 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:30.958 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176323+0000", 2024-08-17T17:49:30.958 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0520999999999998e-05 2024-08-17T17:49:30.958 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.958 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.958 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:49:30.958 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176387+0000", 2024-08-17T17:49:30.958 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3226999999999996e-05 2024-08-17T17:49:30.958 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.958 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.958 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:30.958 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.178044+0000", 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0016577759999999999 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.182089+0000", 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0040443559999999998 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.282793+0000", 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.10070444200000001 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:49:30.959 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.282825+0000", 2024-08-17T17:49:30.960 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1643000000000002e-05 2024-08-17T17:49:30.960 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:30.960 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:30.960 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:49:30.960 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.282847+0000", 2024-08-17T17:49:30.960 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.213e-05 2024-08-17T17:49:30.960 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.960 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.960 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.960 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:30.960 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:30.960 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:49:30.961 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:30 smithi130 ceph-mon[105521]: pgmap v1632: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 2.2 MiB/s wr, 464 op/s; 41793/301539 objects degraded (13.860%); 3681/301539 objects misplaced (1.221%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:49:30.961 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:30 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 41793/301539 objects degraded (13.860%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:30.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:30 smithi130 ceph-mon[110068]: pgmap v1632: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 102 B/s rd, 2.2 MiB/s wr, 464 op/s; 41793/301539 objects degraded (13.860%); 3681/301539 objects misplaced (1.221%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:49:30.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:30 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 41793/301539 objects degraded (13.860%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:31.009 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:31.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:31.110 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:31.171 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:31.172 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:31.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:31.273 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:31.296 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:31.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:31.398 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:31.421 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:31.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:31.523 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:31.547 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:31.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:31.648 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:31.689 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:31.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:31.719 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:49:31.766 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2996957960' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:49:31.767 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:31 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:49:31.767 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2996957960' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:49:31.767 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:31 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:49:31.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:31.816 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:31.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:31.883 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:49:31.917 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:31.949 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:31.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:31.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:32.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:32.075 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:32.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:32.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2996957960' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:49:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:31 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:49:32.177 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:32.187 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:32.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.225 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:32.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:32.326 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:32.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.350 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:32.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:32.452 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:32.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.495 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:32.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:32.596 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:32.620 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:32.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:32.722 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:32.725 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:49:32.769 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:32.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:32.871 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:32.895 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:32.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:32.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:32.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:32 smithi130 ceph-mon[105521]: pgmap v1633: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 94 B/s rd, 2.0 MiB/s wr, 428 op/s; 41793/301539 objects degraded (13.860%); 3681/301539 objects misplaced (1.221%); 7.5 MiB/s, 119 objects/s recovering 2024-08-17T17:49:32.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:32 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:32.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:32 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:49:32.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:32 smithi130 ceph-mon[105521]: osdmap e1186: 8 total, 8 up, 7 in 2024-08-17T17:49:32.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:32 smithi130 ceph-mon[110068]: pgmap v1633: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 94 B/s rd, 2.0 MiB/s wr, 428 op/s; 41793/301539 objects degraded (13.860%); 3681/301539 objects misplaced (1.221%); 7.5 MiB/s, 119 objects/s recovering 2024-08-17T17:49:32.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:32 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:32.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:32 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:49:32.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:32 smithi130 ceph-mon[110068]: osdmap e1186: 8 total, 8 up, 7 in 2024-08-17T17:49:32.997 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:33.020 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:33.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:33.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:32 smithi179 ceph-mon[93519]: pgmap v1633: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 94 B/s rd, 2.0 MiB/s wr, 428 op/s; 41793/301539 objects degraded (13.860%); 3681/301539 objects misplaced (1.221%); 7.5 MiB/s, 119 objects/s recovering 2024-08-17T17:49:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:32 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:32 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:49:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:32 smithi179 ceph-mon[93519]: osdmap e1186: 8 total, 8 up, 7 in 2024-08-17T17:49:33.122 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:33.199 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:33.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:33.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:33.304 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:33.366 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:33.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:33.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:33.468 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:33.510 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:33.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:33.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:33.525 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:33.612 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:33.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:33.663 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:33.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:33.764 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:33.799 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:33.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:33.859 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:49:33.859 INFO:teuthology.orchestra.run.smithi130.stdout: "success": "" 2024-08-17T17:49:33.860 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:49:33.900 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:33.922 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:33.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:33.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:34.023 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:34.045 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:34.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:34.147 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:34.169 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:34.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:34.271 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:34.309 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:34.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:34.411 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:34.432 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:34.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:34.535 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:34.575 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:34.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:34.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:34.678 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:49:34.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.706 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:34.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:34.719 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:49:34.720 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T17:49:34.720 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.720 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:111961 45.e 45:7f832eef:::benchmark_data_smithi130_172551_object7154:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:49:34.720 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:34.696471+0000", 2024-08-17T17:49:34.720 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.022187205000000002, 2024-08-17T17:49:34.720 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.022220054999999999, 2024-08-17T17:49:34.720 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:34.720 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:49:34.720 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:34.720 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:34.720 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:34.720 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 111961 2024-08-17T17:49:34.721 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.721 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:34.721 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.721 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:34.721 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696471+0000", 2024-08-17T17:49:34.721 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:34.721 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.721 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.721 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:34.721 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696471+0000", 2024-08-17T17:49:34.721 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:34.721 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.721 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.722 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:34.722 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696469+0000", 2024-08-17T17:49:34.722 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:49:34.722 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.722 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.722 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:34.722 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696477+0000", 2024-08-17T17:49:34.722 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.8909999999999995e-06 2024-08-17T17:49:34.722 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.722 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.722 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:34.722 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696494+0000", 2024-08-17T17:49:34.723 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.7598e-05 2024-08-17T17:49:34.723 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.723 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.723 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:34.723 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696502+0000", 2024-08-17T17:49:34.723 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3679999999999996e-06 2024-08-17T17:49:34.723 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.723 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.723 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:34.723 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696553+0000", 2024-08-17T17:49:34.723 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1257000000000001e-05 2024-08-17T17:49:34.723 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.723 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.723 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:34.724 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696719+0000", 2024-08-17T17:49:34.724 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016579999999999999 2024-08-17T17:49:34.724 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.724 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.724 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:49:34.724 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696881+0000", 2024-08-17T17:49:34.724 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000162032 2024-08-17T17:49:34.724 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.724 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.724 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:34.724 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.704517+0000", 2024-08-17T17:49:34.724 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0076363769999999997 2024-08-17T17:49:34.724 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.725 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.725 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:34.725 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.715300+0000", 2024-08-17T17:49:34.725 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.010782449 2024-08-17T17:49:34.725 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:34.725 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:34.725 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:34.725 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.725 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.725 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:111962 45.e 45:72363baa:::benchmark_data_smithi130_172551_object7155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:49:34.725 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:34.696568+0000", 2024-08-17T17:49:34.725 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.022089419999999999, 2024-08-17T17:49:34.725 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.022174520999999999, 2024-08-17T17:49:34.726 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:34.726 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:49:34.726 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:34.726 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:34.726 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:34.726 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 111962 2024-08-17T17:49:34.726 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.726 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:34.726 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.726 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:34.726 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696568+0000", 2024-08-17T17:49:34.726 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:34.726 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.727 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.727 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:34.727 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696568+0000", 2024-08-17T17:49:34.727 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:34.727 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.727 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.727 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:34.727 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696567+0000", 2024-08-17T17:49:34.727 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:49:34.727 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.727 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.727 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:34.727 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696577+0000", 2024-08-17T17:49:34.728 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5580000000000006e-06 2024-08-17T17:49:34.728 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.728 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.728 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:34.728 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696589+0000", 2024-08-17T17:49:34.728 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2169e-05 2024-08-17T17:49:34.728 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.728 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.728 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:34.728 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696595+0000", 2024-08-17T17:49:34.728 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1720000000000004e-06 2024-08-17T17:49:34.728 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.728 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.728 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:34.729 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.700919+0000", 2024-08-17T17:49:34.729 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0043241140000000004 2024-08-17T17:49:34.729 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.729 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.729 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:34.729 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.701110+0000", 2024-08-17T17:49:34.729 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000191139 2024-08-17T17:49:34.729 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.729 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.729 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:49:34.729 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.701281+0000", 2024-08-17T17:49:34.729 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017102999999999999 2024-08-17T17:49:34.729 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.730 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.730 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:49:34.730 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.704703+0000", 2024-08-17T17:49:34.730 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003422043 2024-08-17T17:49:34.730 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.730 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.730 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:49:34.730 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.715329+0000", 2024-08-17T17:49:34.730 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.010625921999999999 2024-08-17T17:49:34.730 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:34.730 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:34.730 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:34.730 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.731 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.731 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:111969 45.e 45:7a8e6614:::benchmark_data_smithi130_172551_object7162:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:49:34.731 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:34.718263+0000", 2024-08-17T17:49:34.731 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.000394497, 2024-08-17T17:49:34.731 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00052961000000000002, 2024-08-17T17:49:34.731 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:49:34.731 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:49:34.731 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:49:34.731 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:49:34.731 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:49:34.731 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 111969 2024-08-17T17:49:34.731 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.731 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:49:34.731 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.732 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:49:34.732 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.718263+0000", 2024-08-17T17:49:34.732 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:34.732 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.732 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.732 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:49:34.732 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.718263+0000", 2024-08-17T17:49:34.732 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:49:34.732 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.732 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.732 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:49:34.732 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.718262+0000", 2024-08-17T17:49:34.732 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:49:34.732 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.733 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.733 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:49:34.733 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.718268+0000", 2024-08-17T17:49:34.733 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0059999999999998e-06 2024-08-17T17:49:34.733 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.733 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.733 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:49:34.733 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.718280+0000", 2024-08-17T17:49:34.733 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1725e-05 2024-08-17T17:49:34.733 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.733 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.734 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:49:34.734 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.718286+0000", 2024-08-17T17:49:34.734 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9989999999999997e-06 2024-08-17T17:49:34.734 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.734 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.734 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:49:34.734 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.718322+0000", 2024-08-17T17:49:34.734 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6038999999999998e-05 2024-08-17T17:49:34.734 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.734 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.734 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:49:34.734 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.718437+0000", 2024-08-17T17:49:34.735 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000115739 2024-08-17T17:49:34.735 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:49:34.735 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:49:34.735 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:49:34.735 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.718558+0000", 2024-08-17T17:49:34.735 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000120067 2024-08-17T17:49:34.735 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:34.735 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:49:34.735 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:34.735 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:49:34.735 INFO:teuthology.orchestra.run.smithi179.stdout: ], 2024-08-17T17:49:34.735 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 3 2024-08-17T17:49:34.735 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:49:34.807 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:34.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:34.870 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:34.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:34.971 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:34.993 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:34 smithi130 ceph-mon[110068]: pgmap v1635: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 41258/296217 objects degraded (13.928%); 3307/296217 objects misplaced (1.116%); 8.1 MiB/s, 130 objects/s recovering 2024-08-17T17:49:34.993 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:34 smithi130 ceph-mon[105521]: pgmap v1635: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 41258/296217 objects degraded (13.928%); 3307/296217 objects misplaced (1.116%); 8.1 MiB/s, 130 objects/s recovering 2024-08-17T17:49:35.022 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:35.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:35.046 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:34 smithi179 ceph-mon[93519]: pgmap v1635: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 446 op/s; 41258/296217 objects degraded (13.928%); 3307/296217 objects misplaced (1.116%); 8.1 MiB/s, 130 objects/s recovering 2024-08-17T17:49:35.124 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:35.148 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:35.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:35.250 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:35.273 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:35.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:35.375 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:35.397 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:35.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:35.499 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:35.524 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:35.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:35.539 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:49:35.626 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:35.652 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:35.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:35.754 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:35.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.801 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:35.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:35 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 41258/296217 objects degraded (13.928%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:35.881 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:35.902 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:35.927 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:35.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:35.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:36.029 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:36.054 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:36.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:36.093 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:35 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 41258/296217 objects degraded (13.928%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:36.094 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:35 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 41258/296217 objects degraded (13.928%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:36.156 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:36.199 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:36.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:36.301 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:36.340 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:36.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:36.442 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:36.465 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:36.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:36.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:36.589 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:36.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:36.691 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:36.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.737 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:36.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:36.838 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:36.844 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:36.860 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:36.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:36.962 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:36.986 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:36.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:36.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:36.989 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:36 smithi130 ceph-mon[105521]: pgmap v1636: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 41258/295029 objects degraded (13.984%); 3307/295029 objects misplaced (1.121%); 8.1 MiB/s, 130 objects/s recovering 2024-08-17T17:49:36.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:36 smithi130 ceph-mon[110068]: pgmap v1636: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 41258/295029 objects degraded (13.984%); 3307/295029 objects misplaced (1.121%); 8.1 MiB/s, 130 objects/s recovering 2024-08-17T17:49:37.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:37.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:36 smithi179 ceph-mon[93519]: pgmap v1636: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 41258/295029 objects degraded (13.984%); 3307/295029 objects misplaced (1.121%); 8.1 MiB/s, 130 objects/s recovering 2024-08-17T17:49:37.143 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:37.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:37.245 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:37.270 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:37.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:37.372 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:37.394 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:37.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:37.496 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:37.545 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:37.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:37.647 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:37.672 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:37.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:37.774 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:37.800 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:37.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:37.811 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:49:37.811 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:49:37.811 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T17:49:37.812 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T17:49:37.812 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:49:37.902 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:37.926 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:37.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:37.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:38.027 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:38.049 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:38.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:38.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:38.151 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:38.175 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:38.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:38.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:38.277 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:38.333 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:38.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:38.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:38.435 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:38.457 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:38.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:38.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:38.554 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:49:38.560 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:38.618 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:38.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:38.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:38.720 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:38.742 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:49:38.742 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:49:38.743 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:49:38.743 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:49:38.743 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:49:38.743 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:49:38.743 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:49:38.743 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:49:38.743 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:49:38.743 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:49:38.744 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:49:38.744 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:49:38.744 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:49:38.744 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:49:38.744 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:49:38.744 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:49:38.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:38.746 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:38.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:38.842 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:49:38.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:38 smithi179 ceph-mon[93519]: pgmap v1637: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 442 op/s; 40989/294141 objects degraded (13.935%); 3079/294141 objects misplaced (1.047%); 8.2 MiB/s, 131 objects/s recovering 2024-08-17T17:49:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:38 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3198499849' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:38.847 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:38.893 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:38.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:38.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:38.995 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:39.019 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:39.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:39.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:38 smithi130 ceph-mon[105521]: pgmap v1637: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 442 op/s; 40989/294141 objects degraded (13.935%); 3079/294141 objects misplaced (1.047%); 8.2 MiB/s, 131 objects/s recovering 2024-08-17T17:49:39.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:38 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3198499849' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:39.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:38 smithi130 ceph-mon[110068]: pgmap v1637: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 6.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 442 op/s; 40989/294141 objects degraded (13.935%); 3079/294141 objects misplaced (1.047%); 8.2 MiB/s, 131 objects/s recovering 2024-08-17T17:49:39.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:38 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3198499849' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:39.121 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:39.145 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:39.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:39.247 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:39.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.270 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:39.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:39.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:39.394 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:39.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:39.496 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:39.536 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:39.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:39.637 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:39.675 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:39.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:39.767 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 6, 7] out_osds: [5] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T17:49:39.767 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:49:39.767 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 2, 3, 4, 6, 7] 2024-08-17T17:49:39.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd out 4 2024-08-17T17:49:39.777 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:39.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.799 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:39.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:39.900 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:39.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:49:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:49:39] "GET /metrics HTTP/1.1" 200 36804 "" "Prometheus/2.51.0" 2024-08-17T17:49:39.923 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:39.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:39.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:40.024 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:40.098 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:40.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:40.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:40.203 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:40.262 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:40.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:40.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:40.267 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:40.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:49:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:49:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:49:40.364 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:40.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:40.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:40.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:40.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:40.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:40.540 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:40.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:40.641 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:40.687 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:40.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:40.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:40.752 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T17:49:40.789 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:40.840 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:40.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:40.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:40.869 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:40 smithi130 ceph-mon[105521]: pgmap v1638: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 451 op/s; 40679/288003 objects degraded (14.125%); 2858/288003 objects misplaced (0.992%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:49:40.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:40 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/620666059' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:40.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:40 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:40.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:40 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 40679/288003 objects degraded (14.125%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:40.870 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:40 smithi130 ceph-mon[110068]: pgmap v1638: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 451 op/s; 40679/288003 objects degraded (14.125%); 2858/288003 objects misplaced (0.992%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:49:40.870 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:40 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/620666059' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:40.870 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:40 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:40.870 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:40 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 40679/288003 objects degraded (14.125%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:40.943 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:40.954 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:40 smithi179 ceph-mon[93519]: pgmap v1638: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 451 op/s; 40679/288003 objects degraded (14.125%); 2858/288003 objects misplaced (0.992%); 8.6 MiB/s, 137 objects/s recovering 2024-08-17T17:49:40.955 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:40 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/620666059' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:40.955 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:40 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:49:40.955 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:40 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 40679/288003 objects degraded (14.125%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:40.969 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:40.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:40.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:40.972 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:40.973 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:41.071 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:41.094 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:41.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:41.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:41.196 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:41.239 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:41.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:41.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:41.341 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:41.381 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:41.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:41.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:41.483 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:41.533 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:41.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:41.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:41.635 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:41.660 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:41.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:41.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:41.707 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:49:41.762 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:41.785 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:41.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:41.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:41.887 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:41.896 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:41 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:41.897 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:41 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:41.897 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:41 smithi130 ceph-mon[105521]: osdmap e1187: 8 total, 8 up, 7 in 2024-08-17T17:49:41.897 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:41 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:41.897 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:41 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:41.897 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:41 smithi130 ceph-mon[110068]: osdmap e1187: 8 total, 8 up, 7 in 2024-08-17T17:49:41.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:41.952 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:41.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:42.009 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:49:42.053 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:42.077 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:42.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:42.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:41 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:41 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:49:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:41 smithi179 ceph-mon[93519]: osdmap e1187: 8 total, 8 up, 7 in 2024-08-17T17:49:42.179 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:42.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.201 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:42.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:42.302 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:42.367 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:42.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:42.384 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:42.468 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:42.490 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:42.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:42.592 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:42.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.644 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:42.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:42.745 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:42.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.798 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:42.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:42.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:42 smithi130 ceph-mon[110068]: pgmap v1640: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 40679/288003 objects degraded (14.125%); 2858/288003 objects misplaced (0.992%); 8.7 MiB/s, 138 objects/s recovering 2024-08-17T17:49:42.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:49:42.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:42.815 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:42 smithi130 ceph-mon[105521]: pgmap v1640: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 40679/288003 objects degraded (14.125%); 2858/288003 objects misplaced (0.992%); 8.7 MiB/s, 138 objects/s recovering 2024-08-17T17:49:42.816 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:49:42.816 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:42.899 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:42.945 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:42.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:42.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:43.047 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:43.070 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:43.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:43.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:42 smithi179 ceph-mon[93519]: pgmap v1640: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 40679/288003 objects degraded (14.125%); 2858/288003 objects misplaced (0.992%); 8.7 MiB/s, 138 objects/s recovering 2024-08-17T17:49:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:49:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:43.173 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:43.196 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:43.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:43.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:43.298 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:43.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:43.351 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:43.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:43.451 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:43.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:43.476 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:43.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:43.507 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:43.578 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:43.601 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:43.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:43.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:43.703 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:43.759 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:43.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:43.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:43.861 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:43.883 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:43.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:43.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:43.984 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:44.007 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:44.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:44.108 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:44.131 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:44.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:44.233 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:44.284 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:44.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:44.334 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:49:44.386 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:44.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.409 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:44.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:44.510 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:44.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.567 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:44.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:44.668 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:44.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:44 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T17:49:44.452+0000 7fa856f19640 -1 mon.a@0(leader).osd e1187 definitely_dead 0 2024-08-17T17:49:44.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:44.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:44.793 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:44.800 INFO:teuthology.orchestra.run.smithi130.stderr:marked out osd.4. 2024-08-17T17:49:44.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.818 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:44.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:44.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:44.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:44.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:44.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:45.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:45.072 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:45.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:45.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:44 smithi179 ceph-mon[93519]: pgmap v1641: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 40096/282759 objects degraded (14.180%); 2364/282759 objects misplaced (0.836%); 9.5 MiB/s, 152 objects/s recovering 2024-08-17T17:49:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:44 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1506075925' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-08-17T17:49:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:44 smithi179 ceph-mon[93519]: Client client.admin marked osd.4 out, while it was still marked up 2024-08-17T17:49:45.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:44 smithi130 ceph-mon[105521]: pgmap v1641: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 40096/282759 objects degraded (14.180%); 2364/282759 objects misplaced (0.836%); 9.5 MiB/s, 152 objects/s recovering 2024-08-17T17:49:45.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:44 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1506075925' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-08-17T17:49:45.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:44 smithi130 ceph-mon[105521]: Client client.admin marked osd.4 out, while it was still marked up 2024-08-17T17:49:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:44 smithi130 ceph-mon[110068]: pgmap v1641: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.8 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 40096/282759 objects degraded (14.180%); 2364/282759 objects misplaced (0.836%); 9.5 MiB/s, 152 objects/s recovering 2024-08-17T17:49:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:44 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1506075925' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-08-17T17:49:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:44 smithi130 ceph-mon[110068]: Client client.admin marked osd.4 out, while it was still marked up 2024-08-17T17:49:45.175 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:45.198 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:45.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:45.300 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:45.322 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:45.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:45.424 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:45.449 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:45.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:45.551 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:45.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.574 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:45.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:45.675 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:45.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.706 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:45.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:45.807 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:45.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.834 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:45.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:45.935 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:45.962 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:45.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:45.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:46.063 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:46.084 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:46.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:45 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1506075925' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-08-17T17:49:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:45 smithi179 ceph-mon[93519]: osdmap e1188: 8 total, 8 up, 6 in 2024-08-17T17:49:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:45 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 40096/282759 objects degraded (14.180%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:45 smithi179 ceph-mon[93519]: osdmap e1189: 8 total, 8 up, 6 in 2024-08-17T17:49:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:45 smithi179 ceph-mon[93519]: 45.b starting backfill to osd.2 from (0'0,0'0] MAX to 1187'7308 2024-08-17T17:49:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:45 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1506075925' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-08-17T17:49:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:45 smithi130 ceph-mon[105521]: osdmap e1188: 8 total, 8 up, 6 in 2024-08-17T17:49:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:45 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 40096/282759 objects degraded (14.180%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:45 smithi130 ceph-mon[105521]: osdmap e1189: 8 total, 8 up, 6 in 2024-08-17T17:49:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:45 smithi130 ceph-mon[105521]: 45.b starting backfill to osd.2 from (0'0,0'0] MAX to 1187'7308 2024-08-17T17:49:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:45 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1506075925' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-08-17T17:49:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:45 smithi130 ceph-mon[110068]: osdmap e1188: 8 total, 8 up, 6 in 2024-08-17T17:49:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:45 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 40096/282759 objects degraded (14.180%), 8 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:46.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:45 smithi130 ceph-mon[110068]: osdmap e1189: 8 total, 8 up, 6 in 2024-08-17T17:49:46.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:45 smithi130 ceph-mon[110068]: 45.b starting backfill to osd.2 from (0'0,0'0] MAX to 1187'7308 2024-08-17T17:49:46.186 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:46.207 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:46.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:46.309 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:46.332 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:46.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:46.434 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:46.458 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:46.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:46.560 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:46.582 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:46.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:46.684 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:46.693 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:46.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.730 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:46.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:46.831 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:46.856 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:46.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:46.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:46 smithi130 ceph-mon[105521]: pgmap v1644: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 365 op/s; 40096/281424 objects degraded (14.248%); 2364/281424 objects misplaced (0.840%); 8.2 MiB/s, 130 objects/s recovering 2024-08-17T17:49:46.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:46 smithi130 ceph-mon[105521]: osdmap e1190: 8 total, 8 up, 6 in 2024-08-17T17:49:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:46 smithi130 ceph-mon[110068]: pgmap v1644: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 365 op/s; 40096/281424 objects degraded (14.248%); 2364/281424 objects misplaced (0.840%); 8.2 MiB/s, 130 objects/s recovering 2024-08-17T17:49:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:46 smithi130 ceph-mon[110068]: osdmap e1190: 8 total, 8 up, 6 in 2024-08-17T17:49:46.959 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:46.981 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:46.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:46.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:47.082 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:47.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:46 smithi179 ceph-mon[93519]: pgmap v1644: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 365 op/s; 40096/281424 objects degraded (14.248%); 2364/281424 objects misplaced (0.840%); 8.2 MiB/s, 130 objects/s recovering 2024-08-17T17:49:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:46 smithi179 ceph-mon[93519]: osdmap e1190: 8 total, 8 up, 6 in 2024-08-17T17:49:47.103 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:47.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:47.205 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:47.227 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:47.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:47.329 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:47.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.351 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:47.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:47.452 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:47.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.475 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:47.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:47.576 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:47.597 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:47.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:47.699 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:47.725 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:47.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:47.790 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:47.790 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:47.826 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:47.847 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:47.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:47.949 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:47.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:47.971 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:47.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:48.072 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:48.094 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:48.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:48.196 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:48.221 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:48.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:48.323 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:48.344 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:48.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:48.446 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:48.469 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:48.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:48.571 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:48.597 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:48.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:48.698 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:48.721 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:48.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:48.822 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:48.843 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:48.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:48.944 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:48.966 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:48.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:48.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:49.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:48 smithi130 ceph-mon[105521]: pgmap v1646: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 401 op/s; 40370/280782 objects degraded (14.378%); 7506/280782 objects misplaced (2.673%); 11 MiB/s, 172 objects/s recovering 2024-08-17T17:49:49.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:48 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:49:49.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:48 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1463309563' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:49.018 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:48 smithi130 ceph-mon[110068]: pgmap v1646: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 401 op/s; 40370/280782 objects degraded (14.378%); 7506/280782 objects misplaced (2.673%); 11 MiB/s, 172 objects/s recovering 2024-08-17T17:49:49.018 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:48 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:49:49.018 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:48 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1463309563' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:49.053 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:49:49.067 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:49.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:48 smithi179 ceph-mon[93519]: pgmap v1646: 89 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 401 op/s; 40370/280782 objects degraded (14.378%); 7506/280782 objects misplaced (2.673%); 11 MiB/s, 172 objects/s recovering 2024-08-17T17:49:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:48 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:49:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:48 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1463309563' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:49:49.121 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:49.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:49.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:49.223 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:49.244 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:49.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:49.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:49.346 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:49.370 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:49.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:49.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:49.472 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:49.493 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:49.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:49.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:49.595 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:49.632 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:49.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:49.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:49.733 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:49.758 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:49.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:49.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:49.860 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:49.865 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:49:49.893 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:49.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:49.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:49.996 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:50.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.019 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:50.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:50.120 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:50.141 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:50.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:49:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:49:49] "GET /metrics HTTP/1.1" 200 36800 "" "Prometheus/2.51.0" 2024-08-17T17:49:50.243 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:50.264 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:50.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:50.366 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:50.394 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:50.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:50.495 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:50.517 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:50.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:49:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:49:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:49:50.618 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:50.643 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:50.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:50.745 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:50.767 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:50.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:50.869 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:50.891 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:50.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:50.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:51.018 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:51.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:50 smithi179 ceph-mon[93519]: pgmap v1647: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 244 op/s; 43661/278361 objects degraded (15.685%); 10025/278361 objects misplaced (3.601%); 8.6 MiB/s, 1 keys/s, 139 objects/s recovering 2024-08-17T17:49:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:50 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/390164490' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:50 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:50 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 43661/278361 objects degraded (15.685%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:51.120 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:51.141 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:51.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:51.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:50 smithi130 ceph-mon[105521]: pgmap v1647: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 244 op/s; 43661/278361 objects degraded (15.685%); 10025/278361 objects misplaced (3.601%); 8.6 MiB/s, 1 keys/s, 139 objects/s recovering 2024-08-17T17:49:51.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:50 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/390164490' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:51.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:50 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:51.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:50 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 43661/278361 objects degraded (15.685%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:51.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:50 smithi130 ceph-mon[110068]: pgmap v1647: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 244 op/s; 43661/278361 objects degraded (15.685%); 10025/278361 objects misplaced (3.601%); 8.6 MiB/s, 1 keys/s, 139 objects/s recovering 2024-08-17T17:49:51.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:50 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/390164490' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:51.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:50 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:49:51.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:50 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 43661/278361 objects degraded (15.685%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:51.243 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:51.266 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:51.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:51.368 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:51.392 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:51.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:51.494 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:51.517 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:51.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:51.527 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T17:49:51.619 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:51.644 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:51.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:51.692 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7] out_osds: [5, 4] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T17:49:51.692 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:49:51.746 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:51.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.770 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:51.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:51.871 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:51.901 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:51.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:51.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:52.003 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:52.024 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:52.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:51 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:51 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:51 smithi179 ceph-mon[93519]: osdmap e1191: 8 total, 8 up, 6 in 2024-08-17T17:49:52.126 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:52.148 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:52.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:52.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:51 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:52.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:51 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:52.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:51 smithi130 ceph-mon[105521]: osdmap e1191: 8 total, 8 up, 6 in 2024-08-17T17:49:52.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:51 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:49:52.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:51 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:49:52.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:51 smithi130 ceph-mon[110068]: osdmap e1191: 8 total, 8 up, 6 in 2024-08-17T17:49:52.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:52.272 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:52.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:52.374 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:52.396 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:52.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:52.498 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:52.520 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:52.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:52.621 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:52.645 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:52.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:52.747 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:52.772 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:52.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:52.874 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:52.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:52.897 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:52.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:52.998 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:53.024 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:53.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:53.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:52 smithi179 ceph-mon[93519]: pgmap v1649: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 43661/278361 objects degraded (15.685%); 10025/278361 objects misplaced (3.601%); 8.6 MiB/s, 1 keys/s, 138 objects/s recovering 2024-08-17T17:49:53.125 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:53.146 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:53.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:53.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:52 smithi130 ceph-mon[105521]: pgmap v1649: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 43661/278361 objects degraded (15.685%); 10025/278361 objects misplaced (3.601%); 8.6 MiB/s, 1 keys/s, 138 objects/s recovering 2024-08-17T17:49:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:52 smithi130 ceph-mon[110068]: pgmap v1649: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 43661/278361 objects degraded (15.685%); 10025/278361 objects misplaced (3.601%); 8.6 MiB/s, 1 keys/s, 138 objects/s recovering 2024-08-17T17:49:53.248 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:53.269 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:53.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:53.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:53.392 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:53.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:53.494 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:53.517 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:53.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:53.619 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:53.645 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:53.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:53.747 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:53.770 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:53.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:53.872 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:53.897 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:53.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:53.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:53.999 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:54.023 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:54.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:54.125 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:54.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.150 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:54.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:54.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:54.274 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:54.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:54.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:53 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:49:54.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:53 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:49:54.376 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:54.401 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:54.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:54.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:53 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:49:54.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:53 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:49:54.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:53 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:49:54.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:53 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:49:54.503 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:54.527 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:54.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:54.716 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:54.741 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:54.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:54.843 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:54.865 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:54.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:54.967 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:54.991 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:54.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:54.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:55.093 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:55.116 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:55.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:55.218 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:55.242 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:55.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:54 smithi179 ceph-mon[93519]: pgmap v1650: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 43371/278262 objects degraded (15.586%); 9741/278262 objects misplaced (3.501%); 8.6 MiB/s, 1 keys/s, 138 objects/s recovering 2024-08-17T17:49:55.344 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:55.371 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:55.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:55.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:54 smithi130 ceph-mon[105521]: pgmap v1650: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 43371/278262 objects degraded (15.586%); 9741/278262 objects misplaced (3.501%); 8.6 MiB/s, 1 keys/s, 138 objects/s recovering 2024-08-17T17:49:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:54 smithi130 ceph-mon[110068]: pgmap v1650: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 43371/278262 objects degraded (15.586%); 9741/278262 objects misplaced (3.501%); 8.6 MiB/s, 1 keys/s, 138 objects/s recovering 2024-08-17T17:49:55.473 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:55.498 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:55.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:55.600 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:55.625 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:55.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:55.726 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:55.750 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:55.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:55.852 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:55.878 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:55.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:55.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:55.980 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:56.006 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:56.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:56.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:56.108 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:56.134 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:56.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:56.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:56.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 43371/278262 objects degraded (15.586%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 43371/278262 objects degraded (15.586%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:56.235 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:56.262 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:56.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:56.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:56.331 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 43371/278262 objects degraded (15.586%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:49:56.364 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:56.388 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:56.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:56.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:56.490 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:56.513 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:56.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:56.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:56.615 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:56.640 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:56.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:56.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:56.741 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:56.766 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:56.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:56.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:56.868 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:49:56.893 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:56.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:56.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:49:56.995 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:57.020 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:57.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:57.121 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:57.145 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:57.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:57.246 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:57.270 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:57.271 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:57 smithi130 ceph-mon[105521]: pgmap v1651: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 42996/278262 objects degraded (15.452%); 9741/278262 objects misplaced (3.501%); 10 MiB/s, 1 keys/s, 165 objects/s recovering 2024-08-17T17:49:57.271 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:57 smithi130 ceph-mon[110068]: pgmap v1651: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 42996/278262 objects degraded (15.452%); 9741/278262 objects misplaced (3.501%); 10 MiB/s, 1 keys/s, 165 objects/s recovering 2024-08-17T17:49:57.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:57.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:57 smithi179 ceph-mon[93519]: pgmap v1651: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 42996/278262 objects degraded (15.452%); 9741/278262 objects misplaced (3.501%); 10 MiB/s, 1 keys/s, 165 objects/s recovering 2024-08-17T17:49:57.373 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:57.396 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:57.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:57.498 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:57.523 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:57.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:57.625 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:57.652 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:57.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:57.753 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:57.777 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:57.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:57.879 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:57.904 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:57.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:57.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:58.006 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:58.033 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:58.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:58.135 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:58.163 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:58.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:58.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:58.265 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:58.291 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:58.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:58.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:49:58.393 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:58.419 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:58.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:58.521 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:58.544 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:58.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:49:58.645 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:58.671 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:58.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:58.772 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:49:58.797 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:58.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:49:58.899 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:49:58.924 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:58.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:58.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:49:59.025 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:59.051 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:59.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:59.153 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:49:59.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:49:59 smithi130 ceph-mon[105521]: pgmap v1652: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 83 op/s; 42996/278262 objects degraded (15.452%); 9741/278262 objects misplaced (3.501%); 7.6 MiB/s, 0 keys/s, 123 objects/s recovering 2024-08-17T17:49:59.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:49:59 smithi130 ceph-mon[110068]: pgmap v1652: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 83 op/s; 42996/278262 objects degraded (15.452%); 9741/278262 objects misplaced (3.501%); 7.6 MiB/s, 0 keys/s, 123 objects/s recovering 2024-08-17T17:49:59.177 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:59.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:49:59.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:59.303 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:59.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:59.309 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:49:59.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:49:59 smithi179 ceph-mon[93519]: pgmap v1652: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 83 op/s; 42996/278262 objects degraded (15.452%); 9741/278262 objects misplaced (3.501%); 7.6 MiB/s, 0 keys/s, 123 objects/s recovering 2024-08-17T17:49:59.405 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:49:59.433 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:59.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:49:59.535 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:49:59.560 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:49:59.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:49:59.661 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:59.691 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:59.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:59.792 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:49:59.833 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:59.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:49:59.910 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:49:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:49:59] "GET /metrics HTTP/1.1" 200 36809 "" "Prometheus/2.51.0" 2024-08-17T17:49:59.930 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:49:59.930 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:49:59.936 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:49:59.984 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:49:59.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:49:59.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:00.086 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:00.111 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:00.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); nodeep-scrub flag(s) set; Degraded data redundancy: 42996/278262 objects degraded (15.452%), 10 pgs degraded, 8 pgs undersized 2024-08-17T17:50:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-17T17:50:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: daemon osd.0 on smithi130 is in unknown state 2024-08-17T17:50:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-08-17T17:50:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: [WRN] PG_DEGRADED: Degraded data redundancy: 42996/278262 objects degraded (15.452%), 10 pgs degraded, 8 pgs undersized 2024-08-17T17:50:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: pg 45.0 is stuck undersized for 77s, current state active+recovering+undersized+degraded+remapped, last acting [3,2] 2024-08-17T17:50:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: pg 45.1 is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,6] 2024-08-17T17:50:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: pg 45.5 is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,3] 2024-08-17T17:50:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: pg 45.6 is stuck undersized for 109s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,1] 2024-08-17T17:50:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: pg 45.7 is active+recovery_wait+undersized+degraded+remapped, acting [6,1] 2024-08-17T17:50:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: pg 45.8 is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,7] 2024-08-17T17:50:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: pg 45.b is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2024-08-17T17:50:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: pg 45.d is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2024-08-17T17:50:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: pg 45.e is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,3] 2024-08-17T17:50:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[105521]: pg 45.f is stuck undersized for 77s, current state active+recovering+undersized+degraded+remapped, last acting [2,3] 2024-08-17T17:50:00.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); nodeep-scrub flag(s) set; Degraded data redundancy: 42996/278262 objects degraded (15.452%), 10 pgs degraded, 8 pgs undersized 2024-08-17T17:50:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-17T17:50:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: daemon osd.0 on smithi130 is in unknown state 2024-08-17T17:50:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-08-17T17:50:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: [WRN] PG_DEGRADED: Degraded data redundancy: 42996/278262 objects degraded (15.452%), 10 pgs degraded, 8 pgs undersized 2024-08-17T17:50:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: pg 45.0 is stuck undersized for 77s, current state active+recovering+undersized+degraded+remapped, last acting [3,2] 2024-08-17T17:50:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: pg 45.1 is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,6] 2024-08-17T17:50:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: pg 45.5 is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,3] 2024-08-17T17:50:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: pg 45.6 is stuck undersized for 109s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,1] 2024-08-17T17:50:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: pg 45.7 is active+recovery_wait+undersized+degraded+remapped, acting [6,1] 2024-08-17T17:50:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: pg 45.8 is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,7] 2024-08-17T17:50:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: pg 45.b is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2024-08-17T17:50:00.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: pg 45.d is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2024-08-17T17:50:00.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: pg 45.e is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,3] 2024-08-17T17:50:00.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:00 smithi130 ceph-mon[110068]: pg 45.f is stuck undersized for 77s, current state active+recovering+undersized+degraded+remapped, last acting [2,3] 2024-08-17T17:50:00.213 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:00.239 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:00.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:00.340 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:00.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); nodeep-scrub flag(s) set; Degraded data redundancy: 42996/278262 objects degraded (15.452%), 10 pgs degraded, 8 pgs undersized 2024-08-17T17:50:00.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-08-17T17:50:00.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: daemon osd.0 on smithi130 is in unknown state 2024-08-17T17:50:00.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-08-17T17:50:00.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: [WRN] PG_DEGRADED: Degraded data redundancy: 42996/278262 objects degraded (15.452%), 10 pgs degraded, 8 pgs undersized 2024-08-17T17:50:00.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: pg 45.0 is stuck undersized for 77s, current state active+recovering+undersized+degraded+remapped, last acting [3,2] 2024-08-17T17:50:00.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: pg 45.1 is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,6] 2024-08-17T17:50:00.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: pg 45.5 is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,3] 2024-08-17T17:50:00.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: pg 45.6 is stuck undersized for 109s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,1] 2024-08-17T17:50:00.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: pg 45.7 is active+recovery_wait+undersized+degraded+remapped, acting [6,1] 2024-08-17T17:50:00.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: pg 45.8 is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,7] 2024-08-17T17:50:00.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: pg 45.b is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2024-08-17T17:50:00.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: pg 45.d is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,6] 2024-08-17T17:50:00.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: pg 45.e is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,3] 2024-08-17T17:50:00.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-mon[93519]: pg 45.f is stuck undersized for 77s, current state active+recovering+undersized+degraded+remapped, last acting [2,3] 2024-08-17T17:50:00.345 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:50:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:50:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:50:00.366 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:00.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:00.468 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:00.493 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:00.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:00.594 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:00.620 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:00.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:00.722 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:00.750 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:00.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:00.852 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:00.877 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:00.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:00.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:00.979 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:01.004 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:01.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:01.106 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:01.132 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:01.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:01 smithi130 ceph-mon[105521]: pgmap v1653: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 42285/278262 objects degraded (15.196%); 9448/278262 objects misplaced (3.395%); 8.5 MiB/s, 136 objects/s recovering 2024-08-17T17:50:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 42285/278262 objects degraded (15.196%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:01 smithi130 ceph-mon[110068]: pgmap v1653: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 42285/278262 objects degraded (15.196%); 9448/278262 objects misplaced (3.395%); 8.5 MiB/s, 136 objects/s recovering 2024-08-17T17:50:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 42285/278262 objects degraded (15.196%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:01.233 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:01.260 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:01.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:01 smithi179 ceph-mon[93519]: pgmap v1653: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 42285/278262 objects degraded (15.196%); 9448/278262 objects misplaced (3.395%); 8.5 MiB/s, 136 objects/s recovering 2024-08-17T17:50:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 42285/278262 objects degraded (15.196%), 10 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:01.362 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:01.388 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:01.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:01.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:01.515 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:01.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:01.617 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:01.642 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:01.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:01.744 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:01.769 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:01.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:01.871 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:01.896 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:01.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:01.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:01.998 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:02.022 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:02.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:02.124 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:02.148 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:02.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:02.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:02.277 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:02.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:02.379 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:02.404 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:02.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:02.506 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:02.531 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:02.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:02.632 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:02.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:02.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:02.758 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:02.783 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:02.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:02.885 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:02.908 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:02.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:02.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:03.010 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:03.036 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:03.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:03.138 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:03.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.164 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:03.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:03.265 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:03.290 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:03.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:03 smithi179 ceph-mon[93519]: pgmap v1654: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 42285/278262 objects degraded (15.196%); 9448/278262 objects misplaced (3.395%); 7.9 MiB/s, 125 objects/s recovering 2024-08-17T17:50:03.392 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:03.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:03 smithi130 ceph-mon[105521]: pgmap v1654: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 42285/278262 objects degraded (15.196%); 9448/278262 objects misplaced (3.395%); 7.9 MiB/s, 125 objects/s recovering 2024-08-17T17:50:03.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:03 smithi130 ceph-mon[110068]: pgmap v1654: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 3 op/s; 42285/278262 objects degraded (15.196%); 9448/278262 objects misplaced (3.395%); 7.9 MiB/s, 125 objects/s recovering 2024-08-17T17:50:03.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:03.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:03.519 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:03.544 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:03.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:03.601 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:50:03.601 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:50:03.628 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:50:03.628 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:50:03.646 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:03.654 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:50:03.654 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:50:03.672 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:03.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:03.674 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:50:03.674 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:50:03.697 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:50:03.698 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:50:03.719 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:50:03.719 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:50:03.739 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:50:03.739 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:50:03.762 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:50:03.762 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:50:03.773 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:03.797 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:03.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:03.899 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:03.923 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:03.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:03.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:04.025 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:04.051 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:04.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:04.152 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:04.179 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:04.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:04.281 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:04.306 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:04.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:04.408 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:04.428 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:50:04.433 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:04.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:04.536 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:04.562 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:04.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:04.664 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:04.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:04.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:04.792 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:04.816 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:04.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:04.918 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:04.944 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:04.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:04.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:05.045 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:05.071 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:05.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:05.173 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:05.198 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:05.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:05.300 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:05.325 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:05.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:05.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:05 smithi179 ceph-mon[93519]: pgmap v1655: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 48667/278097 objects degraded (17.500%); 9148/278097 objects misplaced (3.289%); 8.7 MiB/s, 138 objects/s recovering 2024-08-17T17:50:05.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:05 smithi130 ceph-mon[105521]: pgmap v1655: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 48667/278097 objects degraded (17.500%); 9148/278097 objects misplaced (3.289%); 8.7 MiB/s, 138 objects/s recovering 2024-08-17T17:50:05.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:05 smithi130 ceph-mon[110068]: pgmap v1655: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 7 op/s; 48667/278097 objects degraded (17.500%); 9148/278097 objects misplaced (3.289%); 8.7 MiB/s, 138 objects/s recovering 2024-08-17T17:50:05.427 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:05.452 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:05.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:05.554 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:05.580 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:05.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:05.682 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:05.709 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:05.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:05.811 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:05.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:05.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:05.937 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:05.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:05.964 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:05.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:06.065 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:06.091 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:06.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:06.192 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:06.218 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:06.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:06.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 48667/278097 objects degraded (17.500%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:06.344 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:06.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:06.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 48667/278097 objects degraded (17.500%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:06.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 48667/278097 objects degraded (17.500%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:06.445 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:06.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:06.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:06.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:06.602 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:06.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:06.704 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:06.729 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:06.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:06.831 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:06.857 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:06.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:06.959 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:06.983 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:06.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:06.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:07.085 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:07.112 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:07.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:07.213 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:07.332 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:07.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:07.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:07 smithi179 ceph-mon[93519]: pgmap v1656: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 48238/278097 objects degraded (17.346%); 9148/278097 objects misplaced (3.289%); 9.5 MiB/s, 151 objects/s recovering 2024-08-17T17:50:07.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:07 smithi130 ceph-mon[105521]: pgmap v1656: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 48238/278097 objects degraded (17.346%); 9148/278097 objects misplaced (3.289%); 9.5 MiB/s, 151 objects/s recovering 2024-08-17T17:50:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:07 smithi130 ceph-mon[110068]: pgmap v1656: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 48238/278097 objects degraded (17.346%); 9148/278097 objects misplaced (3.289%); 9.5 MiB/s, 151 objects/s recovering 2024-08-17T17:50:07.434 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:07.459 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:07.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:07.561 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:07.587 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:07.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:07.689 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:07.713 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:07.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:07.815 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:07.840 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:07.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:07.942 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:07.968 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:07.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:07.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:08.069 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:08.093 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:08.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:08.195 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:08.219 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:08.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:08.321 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:08.346 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:08.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:08.448 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:08.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:08.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:08.574 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:08.600 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:08.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:08.701 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:08.726 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:08.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:08.828 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:08.852 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:08.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:08.954 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:08.978 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:08.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:08.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:09.079 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:09.105 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:09.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:09.163 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:09.165 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:09.206 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:09.236 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:09.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:09.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:09 smithi130 ceph-mon[105521]: pgmap v1657: 89 pgs: 1 active+recovery_wait, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 48238/278097 objects degraded (17.346%); 9148/278097 objects misplaced (3.289%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T17:50:09.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:09 smithi130 ceph-mon[110068]: pgmap v1657: 89 pgs: 1 active+recovery_wait, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 48238/278097 objects degraded (17.346%); 9148/278097 objects misplaced (3.289%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T17:50:09.338 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:09.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:09 smithi179 ceph-mon[93519]: pgmap v1657: 89 pgs: 1 active+recovery_wait, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 48238/278097 objects degraded (17.346%); 9148/278097 objects misplaced (3.289%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T17:50:09.364 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:09.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:09.456 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:09.465 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:09.489 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:09.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:09.591 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:09.616 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:09.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:09.718 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:09.749 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:09.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:09.851 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:09.877 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:09.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:09.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:09.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:50:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:50:09] "GET /metrics HTTP/1.1" 200 36809 "" "Prometheus/2.51.0" 2024-08-17T17:50:09.979 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:10.004 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:10.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:10.106 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:10.131 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:10.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:10.233 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:10.259 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:10.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:10.361 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:10.387 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:10.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:10.489 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:10.514 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:10.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:50:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:50:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:50:10.616 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:10.643 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:10.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:10.745 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:10.770 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:10.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:10.872 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:10.897 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:10.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:10.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:10.999 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:11.025 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:11.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:11.040 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:11 smithi130 ceph-mon[105521]: pgmap v1658: 89 pgs: 1 active+recovery_wait, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 47413/278097 objects degraded (17.049%); 8812/278097 objects misplaced (3.169%); 12 MiB/s, 188 objects/s recovering 2024-08-17T17:50:11.040 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 47413/278097 objects degraded (17.049%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:11.127 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:11.140 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:50:11.151 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:11.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:11.252 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:11.278 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:11.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:11.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:11 smithi179 ceph-mon[93519]: pgmap v1658: 89 pgs: 1 active+recovery_wait, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 47413/278097 objects degraded (17.049%); 8812/278097 objects misplaced (3.169%); 12 MiB/s, 188 objects/s recovering 2024-08-17T17:50:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 47413/278097 objects degraded (17.049%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:11.380 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:11.403 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:11.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:11 smithi130 ceph-mon[110068]: pgmap v1658: 89 pgs: 1 active+recovery_wait, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 47413/278097 objects degraded (17.049%); 8812/278097 objects misplaced (3.169%); 12 MiB/s, 188 objects/s recovering 2024-08-17T17:50:11.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 47413/278097 objects degraded (17.049%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:11.505 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:11.529 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:11.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:11.630 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:11.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:11.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:11.758 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:11.783 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:11.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:11.885 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:11.911 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:11.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:11.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:12.012 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:12.041 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:12.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:12.143 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:12.166 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:12.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:12.268 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:12.296 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:12.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:12.398 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:12.421 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:12.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:12.523 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:12.544 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:12.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:12.646 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:12.668 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:12.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:12.770 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:12.792 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:12.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:12.893 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:12.915 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:12.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:12.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:13.017 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:13.050 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:13.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:13.152 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:13 smithi130 ceph-mon[105521]: pgmap v1659: 89 pgs: 1 active+undersized+degraded+remapped+laggy, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 47413/278097 objects degraded (17.049%); 8812/278097 objects misplaced (3.169%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:50:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:13 smithi130 ceph-mon[110068]: pgmap v1659: 89 pgs: 1 active+undersized+degraded+remapped+laggy, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 47413/278097 objects degraded (17.049%); 8812/278097 objects misplaced (3.169%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:50:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:13.178 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:13.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:13.280 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:13.309 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:13.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:13 smithi179 ceph-mon[93519]: pgmap v1659: 89 pgs: 1 active+undersized+degraded+remapped+laggy, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 47413/278097 objects degraded (17.049%); 8812/278097 objects misplaced (3.169%); 8.1 MiB/s, 129 objects/s recovering 2024-08-17T17:50:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:13.411 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:13.437 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:13.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:13.540 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:13.563 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:13.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:13.665 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:13.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:13.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:13.795 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:13.817 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:13.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:13.919 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:13.946 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:13.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:13.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:14.048 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:14.049 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:50:14.083 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:14.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:14.099 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:14 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2997243967' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:14.099 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:14 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:14.099 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:14 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2997243967' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:14.099 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:14 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:14.119 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:50:14.119 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T17:50:14.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T17:50:14.119 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T17:50:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:83230 45.f 45.bf51175f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:50:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:38.650088+0000", 2024-08-17T17:50:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 95.412384238000001, 2024-08-17T17:50:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0079560940000001, 2024-08-17T17:50:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T17:50:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:50:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:50:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:50:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 83230 2024-08-17T17:50:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.650088+0000", 2024-08-17T17:50:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.650088+0000", 2024-08-17T17:50:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.650090+0000", 2024-08-17T17:50:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6810000000000001e-06 2024-08-17T17:50:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.650140+0000", 2024-08-17T17:50:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9613000000000001e-05 2024-08-17T17:50:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.650149+0000", 2024-08-17T17:50:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0890000000000006e-06 2024-08-17T17:50:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.650156+0000", 2024-08-17T17:50:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4880000000000001e-06 2024-08-17T17:50:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.677517+0000", 2024-08-17T17:50:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.027360993 2024-08-17T17:50:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:50:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.677600+0000", 2024-08-17T17:50:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2998999999999996e-05 2024-08-17T17:50:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.658034+0000", 2024-08-17T17:50:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98043390699999999 2024-08-17T17:50:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.658045+0000", 2024-08-17T17:50:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0324e-05 2024-08-17T17:50:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:83250 45.f 45.ee8905ff (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:50:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:38.668025+0000", 2024-08-17T17:50:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 95.394447546999999, 2024-08-17T17:50:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.99005247200000002, 2024-08-17T17:50:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T17:50:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:50:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:50:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:50:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 83250 2024-08-17T17:50:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.668025+0000", 2024-08-17T17:50:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.668025+0000", 2024-08-17T17:50:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.668027+0000", 2024-08-17T17:50:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.511e-06 2024-08-17T17:50:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.668077+0000", 2024-08-17T17:50:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0092000000000001e-05 2024-08-17T17:50:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.668087+0000", 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0317e-05 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.668095+0000", 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9370000000000004e-06 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.677625+0000", 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0095303720000000005 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:50:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.677630+0000", 2024-08-17T17:50:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9150000000000004e-06 2024-08-17T17:50:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.658070+0000", 2024-08-17T17:50:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98043984799999995 2024-08-17T17:50:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.658078+0000", 2024-08-17T17:50:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4800000000000004e-06 2024-08-17T17:50:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:83267 45.f 45.30d26fbf (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:50:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:38.698006+0000", 2024-08-17T17:50:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 95.364466250999996, 2024-08-17T17:50:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.96010115299999998, 2024-08-17T17:50:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T17:50:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:50:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:50:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:50:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 83267 2024-08-17T17:50:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.698006+0000", 2024-08-17T17:50:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.698006+0000", 2024-08-17T17:50:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.698009+0000", 2024-08-17T17:50:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2539999999999999e-06 2024-08-17T17:50:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.698065+0000", 2024-08-17T17:50:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6135999999999999e-05 2024-08-17T17:50:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.698088+0000", 2024-08-17T17:50:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3587e-05 2024-08-17T17:50:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.698098+0000", 2024-08-17T17:50:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7000000000000003e-06 2024-08-17T17:50:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.698176+0000", 2024-08-17T17:50:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8300000000000006e-05 2024-08-17T17:50:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:50:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.698186+0000", 2024-08-17T17:50:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3829999999999998e-06 2024-08-17T17:50:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.658099+0000", 2024-08-17T17:50:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.95991286799999997 2024-08-17T17:50:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.658108+0000", 2024-08-17T17:50:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9250000000000001e-06 2024-08-17T17:50:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "MOSDPGPush(45.1 1191/1189 [PushOp(45:84f44d4d:::benchmark_data_smithi130_172551_object11808:head, version: 1170'731, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(45:84f44d4d:::benchmark_data_smithi130_172551_object11808:head@1170'731, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:50:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:10.455277+0000", 2024-08-17T17:50:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 3.6071957609999998, 2024-08-17T17:50:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.194915319, 2024-08-17T17:50:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:50:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.455277+0000", 2024-08-17T17:50:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.455277+0000", 2024-08-17T17:50:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.455278+0000", 2024-08-17T17:50:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8699999999999995e-07 2024-08-17T17:50:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.455342+0000", 2024-08-17T17:50:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4744000000000002e-05 2024-08-17T17:50:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.455359+0000", 2024-08-17T17:50:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6339999999999999e-05 2024-08-17T17:50:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.455376+0000", 2024-08-17T17:50:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6980999999999999e-05 2024-08-17T17:50:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.576005+0000", 2024-08-17T17:50:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.120629809 2024-08-17T17:50:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:50:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.576839+0000", 2024-08-17T17:50:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000833766 2024-08-17T17:50:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.650192+0000", 2024-08-17T17:50:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.073353091999999995 2024-08-17T17:50:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "MOSDPGPushReply(45.1 1191/1189 [PushReplyOp(45:8e8b2239:::benchmark_data_smithi130_172551_object11770:head)])", 2024-08-17T17:50:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:10.790605+0000", 2024-08-17T17:50:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 3.2718675250000002, 2024-08-17T17:50:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.181738923, 2024-08-17T17:50:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T17:50:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.790605+0000", 2024-08-17T17:50:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.790605+0000", 2024-08-17T17:50:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.790606+0000", 2024-08-17T17:50:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9400000000000004e-07 2024-08-17T17:50:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.790611+0000", 2024-08-17T17:50:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1000000000000003e-06 2024-08-17T17:50:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.790621+0000", 2024-08-17T17:50:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0346000000000001e-05 2024-08-17T17:50:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.790627+0000", 2024-08-17T17:50:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9950000000000002e-06 2024-08-17T17:50:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.933648+0000", 2024-08-17T17:50:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14302089100000001 2024-08-17T17:50:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:10.972344+0000", 2024-08-17T17:50:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.038695796999999997 2024-08-17T17:50:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "MOSDPGPush(45.1 1191/1189 [PushOp(45:8ff6836c:::benchmark_data_smithi130_172551_object12058:head, version: 1170'739, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(45:8ff6836c:::benchmark_data_smithi130_172551_object12058:head@1170'739, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:50:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:11.189460+0000", 2024-08-17T17:50:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 2.8730125929999999, 2024-08-17T17:50:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14134516599999999, 2024-08-17T17:50:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:50:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.189460+0000", 2024-08-17T17:50:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.189460+0000", 2024-08-17T17:50:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.189461+0000", 2024-08-17T17:50:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4900000000000005e-07 2024-08-17T17:50:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.189522+0000", 2024-08-17T17:50:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1163999999999999e-05 2024-08-17T17:50:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.189538+0000", 2024-08-17T17:50:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5483999999999999e-05 2024-08-17T17:50:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.189550+0000", 2024-08-17T17:50:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2215000000000001e-05 2024-08-17T17:50:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.264190+0000", 2024-08-17T17:50:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.074640247000000007 2024-08-17T17:50:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:50:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.266115+0000", 2024-08-17T17:50:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001925169 2024-08-17T17:50:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.330805+0000", 2024-08-17T17:50:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.064690038000000005 2024-08-17T17:50:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "MOSDPGPush(45.1 1191/1189 [PushOp(45:80cd2ee9:::benchmark_data_smithi130_172551_object12137:head, version: 1170'742, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(45:80cd2ee9:::benchmark_data_smithi130_172551_object12137:head@1170'742, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:50:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:11.409936+0000", 2024-08-17T17:50:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 2.6525370740000001, 2024-08-17T17:50:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16064526800000001, 2024-08-17T17:50:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:50:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.409936+0000", 2024-08-17T17:50:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.409936+0000", 2024-08-17T17:50:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.409937+0000", 2024-08-17T17:50:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5899999999999995e-07 2024-08-17T17:50:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.410005+0000", 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8374999999999995e-05 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.410026+0000", 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0818999999999998e-05 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.410041+0000", 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5413999999999998e-05 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.511539+0000", 2024-08-17T17:50:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.101498139 2024-08-17T17:50:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:50:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.512973+0000", 2024-08-17T17:50:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0014341239999999999 2024-08-17T17:50:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.570581+0000", 2024-08-17T17:50:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.057607538 2024-08-17T17:50:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "MOSDPGPush(45.1 1191/1189 [PushOp(45:8799f42e:::benchmark_data_smithi130_172551_object12173:head, version: 1170'744, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(45:8799f42e:::benchmark_data_smithi130_172551_object12173:head@1170'744, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:50:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:11.602131+0000", 2024-08-17T17:50:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 2.460342078, 2024-08-17T17:50:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.217960244, 2024-08-17T17:50:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:50:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.602131+0000", 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.602131+0000", 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.602131+0000", 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.37e-07 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.602201+0000", 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9369000000000006e-05 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.602217+0000", 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6376999999999999e-05 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.602227+0000", 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7999999999999993e-06 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.780976+0000", 2024-08-17T17:50:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.178749451 2024-08-17T17:50:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:50:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.781860+0000", 2024-08-17T17:50:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00088400800000000004 2024-08-17T17:50:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.820091+0000", 2024-08-17T17:50:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.038230701999999998 2024-08-17T17:50:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "MOSDPGPushReply(45.1 1191/1189 [PushReplyOp(45:81a2a0f3:::benchmark_data_smithi130_172551_object11843:head)])", 2024-08-17T17:50:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:11.904784+0000", 2024-08-17T17:50:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 2.1576883609999999, 2024-08-17T17:50:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.17998902899999999, 2024-08-17T17:50:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T17:50:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.904784+0000", 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.904784+0000", 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.904785+0000", 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7399999999999991e-07 2024-08-17T17:50:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.904792+0000", 2024-08-17T17:50:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6900000000000003e-06 2024-08-17T17:50:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.904815+0000", 2024-08-17T17:50:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3289999999999999e-05 2024-08-17T17:50:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.904824+0000", 2024-08-17T17:50:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5750000000000001e-06 2024-08-17T17:50:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.042070+0000", 2024-08-17T17:50:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.13724587099999999 2024-08-17T17:50:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.084773+0000", 2024-08-17T17:50:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042703629 2024-08-17T17:50:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "MOSDPGPushReply(45.1 1191/1189 [PushReplyOp(45:84f44d4d:::benchmark_data_smithi130_172551_object11808:head)])", 2024-08-17T17:50:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:11.914370+0000", 2024-08-17T17:50:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 2.1481023970000002, 2024-08-17T17:50:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26862054299999999, 2024-08-17T17:50:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T17:50:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.914370+0000", 2024-08-17T17:50:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.914370+0000", 2024-08-17T17:50:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.914371+0000", 2024-08-17T17:50:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.119e-06 2024-08-17T17:50:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.914379+0000", 2024-08-17T17:50:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0399999999999993e-06 2024-08-17T17:50:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.914409+0000", 2024-08-17T17:50:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9490999999999999e-05 2024-08-17T17:50:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.914420+0000", 2024-08-17T17:50:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0617e-05 2024-08-17T17:50:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.130286+0000", 2024-08-17T17:50:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.21586593200000001 2024-08-17T17:50:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.182991+0000", 2024-08-17T17:50:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.052705344000000001 2024-08-17T17:50:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop_reply(client.45158.0:116601 45.1 e1191/1189 ondisk, result = 0)", 2024-08-17T17:50:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:12.032131+0000", 2024-08-17T17:50:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 2.0303414690000001, 2024-08-17T17:50:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.23322725, 2024-08-17T17:50:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:50:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.032131+0000", 2024-08-17T17:50:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.032131+0000", 2024-08-17T17:50:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.032132+0000", 2024-08-17T17:50:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4300000000000001e-07 2024-08-17T17:50:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.032138+0000", 2024-08-17T17:50:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6010000000000002e-06 2024-08-17T17:50:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.032152+0000", 2024-08-17T17:50:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4068e-05 2024-08-17T17:50:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.032160+0000", 2024-08-17T17:50:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8979999999999996e-06 2024-08-17T17:50:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.230171+0000", 2024-08-17T17:50:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.198010872 2024-08-17T17:50:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:50:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.231672+0000", 2024-08-17T17:50:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001501177 2024-08-17T17:50:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.265359+0000", 2024-08-17T17:50:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.033686690999999998 2024-08-17T17:50:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop_reply(client.45158.0:116650 45.1 e1191/1189 ondisk, result = 0)", 2024-08-17T17:50:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:12.112621+0000", 2024-08-17T17:50:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 1.9498520830000001, 2024-08-17T17:50:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.15981368700000001, 2024-08-17T17:50:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:50:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.112621+0000", 2024-08-17T17:50:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.112621+0000", 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.112622+0000", 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0299999999999997e-07 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.112627+0000", 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9749999999999998e-06 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.112638+0000", 2024-08-17T17:50:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1888999999999999e-05 2024-08-17T17:50:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.112646+0000", 2024-08-17T17:50:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7530000000000001e-06 2024-08-17T17:50:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.266210+0000", 2024-08-17T17:50:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.15356365799999999 2024-08-17T17:50:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:50:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.267218+0000", 2024-08-17T17:50:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0010084289999999999 2024-08-17T17:50:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.272434+0000", 2024-08-17T17:50:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0052160799999999997 2024-08-17T17:50:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop_reply(client.45158.0:116650 45.1 e1191/1189 ondisk, result = 0)", 2024-08-17T17:50:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:12.115724+0000", 2024-08-17T17:50:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 1.946748368, 2024-08-17T17:50:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.20348612999999999, 2024-08-17T17:50:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:50:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.115724+0000", 2024-08-17T17:50:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.115724+0000", 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.115725+0000", 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4499999999999996e-07 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.115730+0000", 2024-08-17T17:50:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1789999999999999e-06 2024-08-17T17:50:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.115743+0000", 2024-08-17T17:50:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2636000000000001e-05 2024-08-17T17:50:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.115750+0000", 2024-08-17T17:50:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3239999999999997e-06 2024-08-17T17:50:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.274235+0000", 2024-08-17T17:50:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.15848426700000001 2024-08-17T17:50:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:50:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.276033+0000", 2024-08-17T17:50:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0017987910000000001 2024-08-17T17:50:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.319210+0000", 2024-08-17T17:50:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.043177187999999998 2024-08-17T17:50:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop_reply(client.45158.0:116615 45.1 e1191/1189 ondisk, result = 0)", 2024-08-17T17:50:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:12.222108+0000", 2024-08-17T17:50:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 1.840365172, 2024-08-17T17:50:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.148941515, 2024-08-17T17:50:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:50:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.222108+0000", 2024-08-17T17:50:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.222108+0000", 2024-08-17T17:50:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.222109+0000", 2024-08-17T17:50:14.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.251e-06 2024-08-17T17:50:14.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.222117+0000", 2024-08-17T17:50:14.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7349999999999996e-06 2024-08-17T17:50:14.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.222130+0000", 2024-08-17T17:50:14.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3436e-05 2024-08-17T17:50:14.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.222139+0000", 2024-08-17T17:50:14.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6090000000000001e-06 2024-08-17T17:50:14.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.326591+0000", 2024-08-17T17:50:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.104452085 2024-08-17T17:50:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:50:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.328277+0000", 2024-08-17T17:50:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0016865649999999999 2024-08-17T17:50:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.371049+0000", 2024-08-17T17:50:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042771834000000002 2024-08-17T17:50:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "MOSDPGPush(45.1 1191/1189 [PushOp(45:86707d23:::benchmark_data_smithi130_172551_object12220:head, version: 1170'746, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(45:86707d23:::benchmark_data_smithi130_172551_object12220:head@1170'746, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:50:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:12.412525+0000", 2024-08-17T17:50:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 1.649948086, 2024-08-17T17:50:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.195332856, 2024-08-17T17:50:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:50:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.412525+0000", 2024-08-17T17:50:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.412525+0000", 2024-08-17T17:50:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.412526+0000", 2024-08-17T17:50:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.435e-06 2024-08-17T17:50:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.412586+0000", 2024-08-17T17:50:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9602000000000003e-05 2024-08-17T17:50:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.412608+0000", 2024-08-17T17:50:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2033000000000001e-05 2024-08-17T17:50:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.412627+0000", 2024-08-17T17:50:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9315000000000001e-05 2024-08-17T17:50:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.573781+0000", 2024-08-17T17:50:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16115368799999999 2024-08-17T17:50:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:50:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.574755+0000", 2024-08-17T17:50:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00097406900000000004 2024-08-17T17:50:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.607858+0000", 2024-08-17T17:50:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.033102713999999998 2024-08-17T17:50:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "MOSDPGPush(45.1 1191/1189 [PushOp(45:84049d28:::benchmark_data_smithi130_172551_object12256:head, version: 1170'748, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(45:84049d28:::benchmark_data_smithi130_172551_object12256:head@1170'748, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:50:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:12.708908+0000", 2024-08-17T17:50:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 1.353564483, 2024-08-17T17:50:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.18127293999999999, 2024-08-17T17:50:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:50:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.708908+0000", 2024-08-17T17:50:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.708908+0000", 2024-08-17T17:50:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.708909+0000", 2024-08-17T17:50:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1209999999999999e-06 2024-08-17T17:50:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.709013+0000", 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010374699999999999 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.709029+0000", 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5763000000000001e-05 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.709042+0000", 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2853999999999999e-05 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.854548+0000", 2024-08-17T17:50:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14550655800000001 2024-08-17T17:50:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:50:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.855839+0000", 2024-08-17T17:50:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001290395 2024-08-17T17:50:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.890181+0000", 2024-08-17T17:50:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.034342501999999997 2024-08-17T17:50:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "MOSDPGPushReply(45.1 1191/1189 [PushReplyOp(45:8bf2fc93:::benchmark_data_smithi130_172551_object11976:head)])", 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:13.327545+0000", 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.73492801200000002, 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.20010570599999999, 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.327545+0000", 2024-08-17T17:50:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.327545+0000", 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.327546+0000", 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2999999999999999e-07 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.327551+0000", 2024-08-17T17:50:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5210000000000002e-06 2024-08-17T17:50:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.327565+0000", 2024-08-17T17:50:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3881e-05 2024-08-17T17:50:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.327572+0000", 2024-08-17T17:50:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4089999999999997e-06 2024-08-17T17:50:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.481968+0000", 2024-08-17T17:50:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.154395279 2024-08-17T17:50:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.527650+0000", 2024-08-17T17:50:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.045682686 2024-08-17T17:50:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop_reply(client.45158.0:116750 45.1 e1191/1189 ondisk, result = 0)", 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:13.468888+0000", 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.59358440300000004, 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.146657814, 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.468888+0000", 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.468888+0000", 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.468889+0000", 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.093e-06 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.468902+0000", 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2310000000000001e-05 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.468914+0000", 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2221e-05 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.468922+0000", 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3480000000000004e-06 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.575393+0000", 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.106470327 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:50:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.576421+0000", 2024-08-17T17:50:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0010286360000000001 2024-08-17T17:50:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.615546+0000", 2024-08-17T17:50:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.039124879000000001 2024-08-17T17:50:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "MOSDPGPush(45.1 1191/1189 [PushOp(45:8d9e8ca7:::benchmark_data_smithi130_172551_object12289:head, version: 1170'752, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(45:8d9e8ca7:::benchmark_data_smithi130_172551_object12289:head@1170'752, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:50:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:13.701078+0000", 2024-08-17T17:50:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.36139455999999998, 2024-08-17T17:50:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.233112545, 2024-08-17T17:50:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:50:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.701078+0000", 2024-08-17T17:50:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.701078+0000", 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.701079+0000", 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8800000000000001e-07 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.701147+0000", 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8272e-05 2024-08-17T17:50:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.701166+0000", 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8397e-05 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.701181+0000", 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5387999999999999e-05 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.894512+0000", 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.19333073100000001 2024-08-17T17:50:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:50:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.895362+0000", 2024-08-17T17:50:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00085060500000000005 2024-08-17T17:50:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.934191+0000", 2024-08-17T17:50:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.038828264000000001 2024-08-17T17:50:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop_reply(client.45158.0:116785 45.1 e1191/1189 ondisk, result = 0)", 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:13.784783+0000", 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.277690031, 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.21343172399999999, 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.784783+0000", 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:50:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.784783+0000", 2024-08-17T17:50:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:50:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:50:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.784784+0000", 2024-08-17T17:50:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2100000000000001e-06 2024-08-17T17:50:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:50:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.784792+0000", 2024-08-17T17:50:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9710000000000004e-06 2024-08-17T17:50:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.784822+0000", 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0148999999999999e-05 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.784831+0000", 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7020000000000007e-06 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.953075+0000", 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16824396 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:50:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.954263+0000", 2024-08-17T17:50:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0011884930000000001 2024-08-17T17:50:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:50:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:50:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:50:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.998214+0000", 2024-08-17T17:50:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.043951239000000003 2024-08-17T17:50:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:50:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:50:14.210 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:50:14.211 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:14.232 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:14.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:14.334 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:14.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:14 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2997243967' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:14.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:14 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:50:14.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:14.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:14.457 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:14.483 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:14.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:14.585 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:14.612 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:14.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:14.714 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:14.739 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:14.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:14.841 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:14.863 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:14.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:14.965 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:14.986 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:14.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:14.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:15.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:15.112 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:15.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:15.214 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:15.235 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:15.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:15.337 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:15.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:15 smithi179 ceph-mon[93519]: pgmap v1660: 89 pgs: 1 active+recovery_wait, 1 active+undersized+degraded+remapped+laggy, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 12 op/s; 47062/277800 objects degraded (16.941%); 8464/277800 objects misplaced (3.047%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:50:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:15 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:50:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:15 smithi179 ceph-mon[93519]: osdmap e1192: 8 total, 8 up, 6 in 2024-08-17T17:50:15.368 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:15.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:15.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:15 smithi130 ceph-mon[105521]: pgmap v1660: 89 pgs: 1 active+recovery_wait, 1 active+undersized+degraded+remapped+laggy, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 12 op/s; 47062/277800 objects degraded (16.941%); 8464/277800 objects misplaced (3.047%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:50:15.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:15 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:50:15.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:15 smithi130 ceph-mon[105521]: osdmap e1192: 8 total, 8 up, 6 in 2024-08-17T17:50:15.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:15 smithi130 ceph-mon[110068]: pgmap v1660: 89 pgs: 1 active+recovery_wait, 1 active+undersized+degraded+remapped+laggy, 2 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 12 op/s; 47062/277800 objects degraded (16.941%); 8464/277800 objects misplaced (3.047%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:50:15.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:15 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:50:15.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:15 smithi130 ceph-mon[110068]: osdmap e1192: 8 total, 8 up, 6 in 2024-08-17T17:50:15.470 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:15.497 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:15.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:15.599 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:15.623 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:15.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:15.726 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:15.748 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:15.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:15.850 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:15.871 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:15.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:15.973 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:15.994 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:15.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:15.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:16.096 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:16.117 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:16.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:16.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:16.219 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:16.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:16.241 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:16.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:16.343 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 47062/277800 objects degraded (16.941%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:16.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:16 smithi179 ceph-mon[93519]: pgmap v1662: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 14 op/s; 46523/277665 objects degraded (16.755%); 8464/277665 objects misplaced (3.048%); 11 MiB/s, 168 objects/s recovering 2024-08-17T17:50:16.364 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:16.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:16.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:16.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 47062/277800 objects degraded (16.941%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:16.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:16 smithi130 ceph-mon[105521]: pgmap v1662: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 14 op/s; 46523/277665 objects degraded (16.755%); 8464/277665 objects misplaced (3.048%); 11 MiB/s, 168 objects/s recovering 2024-08-17T17:50:16.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 47062/277800 objects degraded (16.941%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:16.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:16 smithi130 ceph-mon[110068]: pgmap v1662: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 14 op/s; 46523/277665 objects degraded (16.755%); 8464/277665 objects misplaced (3.048%); 11 MiB/s, 168 objects/s recovering 2024-08-17T17:50:16.466 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:16.487 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:16.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:16.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:16.589 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:16.610 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:16.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:16.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:16.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:16.742 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:16.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:16.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:16.844 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:16.878 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:16.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:16.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:16.980 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:17.003 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:17.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:17.104 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:17.129 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:17.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:17.231 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:17.252 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:17.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:17.354 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:17.375 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:17.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:17.476 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:17.497 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:17.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:17.599 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:17.620 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:17.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:17.722 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:17.743 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:17.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:17.845 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:17.866 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:17.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:17.967 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:17.988 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:17.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:17.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:18.090 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:18.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:17 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3168387221' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:18.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:17 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:18.117 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:18.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:18.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:17 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3168387221' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:18.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:17 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:18.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:17 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3168387221' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:18.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:17 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:50:18.219 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:18.244 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:18.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:18.346 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:18.375 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:18.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:18.469 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T17:50:18.477 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:18.498 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:18.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:18.600 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:18.625 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:18.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:18.727 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:18.748 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:18.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:18.850 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:18.870 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:18.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:18.972 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:18.994 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:18.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:18.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:18 smithi179 ceph-mon[93519]: pgmap v1663: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 18 op/s; 46491/277551 objects degraded (16.750%); 8443/277551 objects misplaced (3.042%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:50:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:18 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:50:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:18 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:50:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:18 smithi179 ceph-mon[93519]: osdmap e1193: 8 total, 8 up, 6 in 2024-08-17T17:50:19.095 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:19.117 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:19.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:19.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:18 smithi130 ceph-mon[105521]: pgmap v1663: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 18 op/s; 46491/277551 objects degraded (16.750%); 8443/277551 objects misplaced (3.042%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:50:19.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:18 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:50:19.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:18 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:50:19.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:18 smithi130 ceph-mon[105521]: osdmap e1193: 8 total, 8 up, 6 in 2024-08-17T17:50:19.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:18 smithi130 ceph-mon[110068]: pgmap v1663: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 18 op/s; 46491/277551 objects degraded (16.750%); 8443/277551 objects misplaced (3.042%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:50:19.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:18 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:50:19.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:18 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:50:19.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:18 smithi130 ceph-mon[110068]: osdmap e1193: 8 total, 8 up, 6 in 2024-08-17T17:50:19.219 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:19.243 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:19.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:19.345 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:19.367 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:19.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:19.468 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:19.492 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:19.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:19.594 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:19.619 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:19.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:19.721 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:19.742 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:19.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:19.844 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:19.872 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:19.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:19.973 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:19.994 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:19.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:19.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:20.096 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:20.120 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:20.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:20.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:50:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:50:19] "GET /metrics HTTP/1.1" 200 36799 "" "Prometheus/2.51.0" 2024-08-17T17:50:20.222 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:20.243 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:20.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:20.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:20.345 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:20.366 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:20.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:20.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:20.468 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:20.489 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:20.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:20.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:20.591 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:50:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:50:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:50:20.611 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:20.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:20.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:20.713 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:20.740 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:20.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:20.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:20.842 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:20.874 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:20.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:20.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:20.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:21.016 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:21.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:20 smithi179 ceph-mon[93519]: pgmap v1665: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 57 op/s; 45600/276726 objects degraded (16.478%); 8082/276726 objects misplaced (2.921%); 14 MiB/s, 220 objects/s recovering 2024-08-17T17:50:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:20 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 45600/276726 objects degraded (16.478%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:21.118 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:21.140 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:21.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:21.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:20 smithi130 ceph-mon[105521]: pgmap v1665: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 57 op/s; 45600/276726 objects degraded (16.478%); 8082/276726 objects misplaced (2.921%); 14 MiB/s, 220 objects/s recovering 2024-08-17T17:50:21.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:20 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 45600/276726 objects degraded (16.478%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:21.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:20 smithi130 ceph-mon[110068]: pgmap v1665: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 57 op/s; 45600/276726 objects degraded (16.478%); 8082/276726 objects misplaced (2.921%); 14 MiB/s, 220 objects/s recovering 2024-08-17T17:50:21.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:20 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 45600/276726 objects degraded (16.478%), 10 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:21.242 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:21.263 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:21.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:21.365 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:21.387 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:21.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:21.489 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:21.515 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:21.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:21.617 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:21.638 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:21.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:21.740 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:21.761 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:21.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:21.863 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:21.884 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:21.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:21.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:21.986 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:22.008 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:22.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:22.109 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:22.131 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:22.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:22.233 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:22.254 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:22.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:22.356 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:22.382 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:22.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:22.485 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:22.510 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:22.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:22.612 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:22.634 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:22.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:22.736 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:22.763 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:22.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:22.865 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:22.887 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:22.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:22.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:22.989 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:23.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.011 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:23.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:23.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:22 smithi179 ceph-mon[93519]: pgmap v1666: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 48 op/s; 45553/276645 objects degraded (16.466%); 8082/276645 objects misplaced (2.921%); 11 MiB/s, 180 objects/s recovering 2024-08-17T17:50:23.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:23.134 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:23.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:23.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:22 smithi130 ceph-mon[105521]: pgmap v1666: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 48 op/s; 45553/276645 objects degraded (16.466%); 8082/276645 objects misplaced (2.921%); 11 MiB/s, 180 objects/s recovering 2024-08-17T17:50:23.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:22 smithi130 ceph-mon[110068]: pgmap v1666: 89 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 48 op/s; 45553/276645 objects degraded (16.466%); 8082/276645 objects misplaced (2.921%); 11 MiB/s, 180 objects/s recovering 2024-08-17T17:50:23.236 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:23.257 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:23.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:23.359 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:23.380 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:23.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:23.482 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:23.503 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:23.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:23.605 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:23.632 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:23.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:23.733 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:23.766 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:23.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:23.867 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:23.890 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:23.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:23.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:23.992 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:24.013 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:24.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:24.076 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:24.115 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:24.117 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:50:24.185 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:24.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:24.275 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:50:24.286 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:24.308 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:24.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:24.409 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:24.431 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:24.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:24.533 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:24.554 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:24.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:24.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:24.677 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:24.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:24.779 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:24.800 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:24.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:24.902 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:24.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:24 smithi130 ceph-mon[105521]: pgmap v1667: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 69 op/s; 45321/275700 objects degraded (16.439%); 7570/275700 objects misplaced (2.746%); 9.4 MiB/s, 150 objects/s recovering 2024-08-17T17:50:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:24 smithi130 ceph-mon[110068]: pgmap v1667: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 69 op/s; 45321/275700 objects degraded (16.439%); 7570/275700 objects misplaced (2.746%); 9.4 MiB/s, 150 objects/s recovering 2024-08-17T17:50:24.930 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:24.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:24.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:25.033 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:25.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.055 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:25.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:25.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:24 smithi179 ceph-mon[93519]: pgmap v1667: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 69 op/s; 45321/275700 objects degraded (16.439%); 7570/275700 objects misplaced (2.746%); 9.4 MiB/s, 150 objects/s recovering 2024-08-17T17:50:25.156 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:25.183 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:25.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:25.285 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:25.332 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:25.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:25.434 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:25.455 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:25.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:25.557 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:25.578 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:25.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:25.680 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:25.705 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:25.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:25.807 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:25.831 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:25.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:25.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:25 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 45321/275700 objects degraded (16.439%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:25.933 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:25.954 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:25.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:25.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:26.056 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:26.086 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:26.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:26.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:25 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 45321/275700 objects degraded (16.439%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:26.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:25 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 45321/275700 objects degraded (16.439%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:26.188 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:26.212 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:26.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:26.314 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:26.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:26.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:26.437 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:26.463 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:26.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:26.565 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:26.587 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:26.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:26.689 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:26.713 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:26.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:26.815 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:26.836 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:26.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:26.938 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:26.961 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:26.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:26.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:26.995 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:50:27.029 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:26 smithi130 ceph-mon[105521]: pgmap v1668: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 44675/275403 objects degraded (16.222%); 7570/275403 objects misplaced (2.749%); 13 MiB/s, 204 objects/s recovering 2024-08-17T17:50:27.029 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:26 smithi130 ceph-mon[110068]: pgmap v1668: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 44675/275403 objects degraded (16.222%); 7570/275403 objects misplaced (2.749%); 13 MiB/s, 204 objects/s recovering 2024-08-17T17:50:27.064 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:27.093 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:27.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:26 smithi179 ceph-mon[93519]: pgmap v1668: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 44675/275403 objects degraded (16.222%); 7570/275403 objects misplaced (2.749%); 13 MiB/s, 204 objects/s recovering 2024-08-17T17:50:27.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:27.195 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:27.216 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:27.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:27.318 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:27.340 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:27.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:27.442 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:27.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:27.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:27.564 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:27.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:27.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:27.687 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:27.714 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:27.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:27.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:27.841 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:27.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:27.942 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:27.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:27.972 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:27.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:28.073 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:28.094 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:28.195 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:28.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:28.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:28.320 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:28.343 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:28.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:28.445 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:28.465 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:28.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:28.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:28.590 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:28.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:28.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:28.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:28.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:28 smithi179 ceph-mon[93519]: pgmap v1669: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 76 op/s; 44613/275247 objects degraded (16.208%); 7549/275247 objects misplaced (2.743%); 13 MiB/s, 206 objects/s recovering 2024-08-17T17:50:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:28 smithi130 ceph-mon[105521]: pgmap v1669: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 76 op/s; 44613/275247 objects degraded (16.208%); 7549/275247 objects misplaced (2.743%); 13 MiB/s, 206 objects/s recovering 2024-08-17T17:50:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:28 smithi130 ceph-mon[110068]: pgmap v1669: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 76 op/s; 44613/275247 objects degraded (16.208%); 7549/275247 objects misplaced (2.743%); 13 MiB/s, 206 objects/s recovering 2024-08-17T17:50:28.692 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:28.715 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:28.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:28.817 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:28.843 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:28.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:28.945 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:28.970 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:28.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:28.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:29.072 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:29.094 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:29.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:29.196 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:29.222 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:29.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:29.324 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:29.349 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:29.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:29.451 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:29.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:29.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:29.574 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:29.598 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:29.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:29.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:29.730 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:29.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:29.832 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:29.853 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:29.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:29.954 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:29.975 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:29.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:29.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:30.077 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:30.097 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:30.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:50:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:50:29] "GET /metrics HTTP/1.1" 200 36805 "" "Prometheus/2.51.0" 2024-08-17T17:50:30.199 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:30.224 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:30.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:30.326 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:30.357 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:30.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:30.459 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:30.488 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:30.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:30.590 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:50:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:50:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:50:30.611 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:30.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:30.713 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:30.734 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:30.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:30.836 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:30.856 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:30.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:30.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:30 smithi130 ceph-mon[105521]: pgmap v1670: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 43887/274119 objects degraded (16.010%); 6941/274119 objects misplaced (2.532%); 18 MiB/s, 285 objects/s recovering 2024-08-17T17:50:30.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:30 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 43887/274119 objects degraded (16.010%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:30.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:30 smithi130 ceph-mon[110068]: pgmap v1670: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 43887/274119 objects degraded (16.010%); 6941/274119 objects misplaced (2.532%); 18 MiB/s, 285 objects/s recovering 2024-08-17T17:50:30.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:30 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 43887/274119 objects degraded (16.010%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:30.958 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:30.985 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:30.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:30.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:31.086 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:31.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:30 smithi179 ceph-mon[93519]: pgmap v1670: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 43887/274119 objects degraded (16.010%); 6941/274119 objects misplaced (2.532%); 18 MiB/s, 285 objects/s recovering 2024-08-17T17:50:31.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:30 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 43887/274119 objects degraded (16.010%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:31.108 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:31.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:31.210 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:31.231 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:31.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:31.333 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:31.354 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:31.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:31.455 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:31.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.477 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:31.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:31.578 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:31.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.600 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:31.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:31.701 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:31.723 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:31.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:31.825 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:31.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.847 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:31.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:31.948 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:31.973 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:31.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:31.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:32.074 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:32.099 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:32.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:32.201 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:32.223 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:32.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:32.324 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:32.352 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:32.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:32.454 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:32.476 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:32.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:32.577 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:32.599 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:32.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:32.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:32.735 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:32.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:32.837 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:32.858 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:32.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:32.959 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:32.985 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:32.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:32.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:33.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:32 smithi130 ceph-mon[105521]: pgmap v1671: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 43839/274041 objects degraded (15.997%); 6941/274041 objects misplaced (2.533%); 13 MiB/s, 210 objects/s recovering 2024-08-17T17:50:33.065 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:32 smithi130 ceph-mon[110068]: pgmap v1671: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 43839/274041 objects degraded (15.997%); 6941/274041 objects misplaced (2.533%); 13 MiB/s, 210 objects/s recovering 2024-08-17T17:50:33.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:33.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:32 smithi179 ceph-mon[93519]: pgmap v1671: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 43839/274041 objects degraded (15.997%); 6941/274041 objects misplaced (2.533%); 13 MiB/s, 210 objects/s recovering 2024-08-17T17:50:33.111 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:33.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:33.213 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:33.240 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:33.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:33.342 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:33.368 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:33.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:33.470 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:33.492 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:33.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:33.594 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:33.616 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:33.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:33.717 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:33.750 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:33.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:33.851 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:33.873 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:33.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:33.974 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:33.996 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:33.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:33.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:34.097 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:34.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.120 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:34.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:34.221 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:34.242 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:34.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:34.344 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:34.370 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:34.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:34.472 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:34.496 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:34.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:34.597 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:34.619 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:34.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:34.721 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:34.744 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:34.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:34.846 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:34.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.868 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:34.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:34.969 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:34.987 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:34 smithi130 ceph-mon[105521]: pgmap v1672: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 43799/273606 objects degraded (16.008%); 6918/273606 objects misplaced (2.528%); 13 MiB/s, 210 objects/s recovering 2024-08-17T17:50:34.987 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:34 smithi130 ceph-mon[105521]: osdmap e1194: 8 total, 8 up, 6 in 2024-08-17T17:50:34.987 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:34 smithi130 ceph-mon[110068]: pgmap v1672: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 43799/273606 objects degraded (16.008%); 6918/273606 objects misplaced (2.528%); 13 MiB/s, 210 objects/s recovering 2024-08-17T17:50:34.987 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:34 smithi130 ceph-mon[110068]: osdmap e1194: 8 total, 8 up, 6 in 2024-08-17T17:50:34.990 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:34.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:34.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:35.092 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:35.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:34 smithi179 ceph-mon[93519]: pgmap v1672: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 43799/273606 objects degraded (16.008%); 6918/273606 objects misplaced (2.528%); 13 MiB/s, 210 objects/s recovering 2024-08-17T17:50:35.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:34 smithi179 ceph-mon[93519]: osdmap e1194: 8 total, 8 up, 6 in 2024-08-17T17:50:35.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:35.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:35.120 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:50:35.120 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:50:35.128 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:35.129 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:35.130 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:35.143 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:50:35.144 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:50:35.175 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:50:35.175 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:50:35.203 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:50:35.204 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:50:35.214 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:35.219 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:50:35.220 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:50:35.235 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:35.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:35.239 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:50:35.240 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:50:35.257 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:50:35.258 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:50:35.276 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:50:35.276 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:50:35.336 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:35.357 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:35.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:35.459 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:35.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:35.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:35.582 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:35.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.604 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:35.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:35.705 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:35.734 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:35.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:35.835 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:35.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.859 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:35.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:35.879 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:35 smithi130 ceph-mon[105521]: osdmap e1195: 8 total, 8 up, 6 in 2024-08-17T17:50:35.879 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:35 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/221405942' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:35.879 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:35 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 43799/273606 objects degraded (16.008%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:35.879 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:35 smithi130 ceph-mon[105521]: osdmap e1196: 8 total, 8 up, 6 in 2024-08-17T17:50:35.879 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:35 smithi130 ceph-mon[110068]: osdmap e1195: 8 total, 8 up, 6 in 2024-08-17T17:50:35.879 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:35 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/221405942' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:35.879 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:35 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 43799/273606 objects degraded (16.008%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:35.879 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:35 smithi130 ceph-mon[110068]: osdmap e1196: 8 total, 8 up, 6 in 2024-08-17T17:50:35.960 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:35.988 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:35.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:35.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:36.090 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:35 smithi179 ceph-mon[93519]: osdmap e1195: 8 total, 8 up, 6 in 2024-08-17T17:50:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:35 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/221405942' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:35 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 43799/273606 objects degraded (16.008%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:35 smithi179 ceph-mon[93519]: osdmap e1196: 8 total, 8 up, 6 in 2024-08-17T17:50:36.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.112 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:36.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:36.213 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:36.235 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:36.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:36.337 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:36.358 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:36.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:36.459 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:36.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:36.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:36.582 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:36.606 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:36.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:36.707 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:36.729 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:36.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:36.831 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:36.854 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:36.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:36.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:36 smithi130 ceph-mon[105521]: pgmap v1676: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 45 op/s; 43790/273303 objects degraded (16.023%); 6102/273303 objects misplaced (2.233%); 9.1 MiB/s, 144 objects/s recovering 2024-08-17T17:50:36.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:36 smithi130 ceph-mon[105521]: osdmap e1197: 8 total, 8 up, 6 in 2024-08-17T17:50:36.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:36 smithi130 ceph-mon[110068]: pgmap v1676: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 45 op/s; 43790/273303 objects degraded (16.023%); 6102/273303 objects misplaced (2.233%); 9.1 MiB/s, 144 objects/s recovering 2024-08-17T17:50:36.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:36 smithi130 ceph-mon[110068]: osdmap e1197: 8 total, 8 up, 6 in 2024-08-17T17:50:36.955 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:36.984 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:36.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:37.085 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:37.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:36 smithi179 ceph-mon[93519]: pgmap v1676: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 7 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 45 op/s; 43790/273303 objects degraded (16.023%); 6102/273303 objects misplaced (2.233%); 9.1 MiB/s, 144 objects/s recovering 2024-08-17T17:50:37.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:36 smithi179 ceph-mon[93519]: osdmap e1197: 8 total, 8 up, 6 in 2024-08-17T17:50:37.111 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:37.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:37.213 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:37.236 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:37.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:37.337 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:37.359 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:37.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:37.460 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:37.481 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:37.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:37.583 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:37.610 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:37.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:37.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:37.737 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:37.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:37.839 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:37.860 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:37.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:37.962 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:37.985 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:37.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:37.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:38.086 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:38.111 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:38.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:38.212 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:38.236 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:38.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:38.338 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:38.360 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:38.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:38.461 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:38.489 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:38.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:38.590 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:38.615 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:38.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:38.717 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:38.742 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:38.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:38.844 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:38.872 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:38.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:38.974 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:38.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:38.996 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:38.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:39.083 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:38 smithi130 ceph-mon[105521]: pgmap v1678: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 44 op/s; 42769/273072 objects degraded (15.662%); 6085/273072 objects misplaced (2.228%); 16 MiB/s, 450 objects/s recovering 2024-08-17T17:50:39.083 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:38 smithi130 ceph-mon[110068]: pgmap v1678: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 44 op/s; 42769/273072 objects degraded (15.662%); 6085/273072 objects misplaced (2.228%); 16 MiB/s, 450 objects/s recovering 2024-08-17T17:50:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:38 smithi179 ceph-mon[93519]: pgmap v1678: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 44 op/s; 42769/273072 objects degraded (15.662%); 6085/273072 objects misplaced (2.228%); 16 MiB/s, 450 objects/s recovering 2024-08-17T17:50:39.097 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:39.117 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:39.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:39.219 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:39.240 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:39.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:39.342 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:39.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.363 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:39.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:39.464 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:39.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.486 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:39.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:39.587 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:39.608 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:39.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:39.709 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:39.732 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:39.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:39.834 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:39.856 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:39.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:39.958 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:39.980 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:39.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:39.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:40.082 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:40.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.106 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:40.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:50:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:50:39] "GET /metrics HTTP/1.1" 200 36805 "" "Prometheus/2.51.0" 2024-08-17T17:50:40.207 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:40.228 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:40.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:40.330 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:40.351 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:40.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:40.453 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:40.477 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:40.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:40.578 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:40.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:50:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:50:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:50:40.608 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:40.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:40.710 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:40.731 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:40.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:40.752 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:50:40.832 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:40.853 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:40.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:40.955 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:40.976 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:40.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:40.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:41.078 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:41.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:40 smithi179 ceph-mon[93519]: pgmap v1679: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 42456/272118 objects degraded (15.602%); 5652/272118 objects misplaced (2.077%); 10 MiB/s, 290 objects/s recovering 2024-08-17T17:50:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:40 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1073815560' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:50:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:40 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:50:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:40 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 42456/272118 objects degraded (15.602%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:41.103 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:41.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:40 smithi130 ceph-mon[110068]: pgmap v1679: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 42456/272118 objects degraded (15.602%); 5652/272118 objects misplaced (2.077%); 10 MiB/s, 290 objects/s recovering 2024-08-17T17:50:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:40 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1073815560' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:50:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:40 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:50:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:40 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 42456/272118 objects degraded (15.602%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:41.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:40 smithi130 ceph-mon[105521]: pgmap v1679: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 42456/272118 objects degraded (15.602%); 5652/272118 objects misplaced (2.077%); 10 MiB/s, 290 objects/s recovering 2024-08-17T17:50:41.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:40 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1073815560' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:50:41.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:40 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:50:41.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:40 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 42456/272118 objects degraded (15.602%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:41.204 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:41.231 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:41.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:41.333 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:41.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:41.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:41.457 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:41.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.479 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:41.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:41.536 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:50:41.580 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:41.609 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:41.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:41.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:41.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.748 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:41.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:41.821 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:41 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:50:41.821 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:41 smithi130 ceph-mon[105521]: osdmap e1198: 8 total, 8 up, 6 in 2024-08-17T17:50:41.822 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:41 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:41.822 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:41 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:41.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:41 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:50:41.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:41 smithi130 ceph-mon[110068]: osdmap e1198: 8 total, 8 up, 6 in 2024-08-17T17:50:41.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:41 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:41.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:41 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:41.850 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:41.852 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:50:41.852 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:50:41.871 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:41.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:41.973 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:41.996 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:41.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:41.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:41 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:50:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:41 smithi179 ceph-mon[93519]: osdmap e1198: 8 total, 8 up, 6 in 2024-08-17T17:50:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:41 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:41 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:42.098 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:42.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:42.121 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:42.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:42.222 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:42.248 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:42.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:42.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:42.283 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:50:42.353 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:42.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:42.375 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:42.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:42.476 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:42.500 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:42.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:42.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:42.602 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:42.623 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:42.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:42.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:42.725 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:42.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:42.746 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:42.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:42.847 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:42.873 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:42.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:42.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:42.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:42 smithi130 ceph-mon[105521]: pgmap v1681: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 76 op/s; 42395/271923 objects degraded (15.591%); 5652/271923 objects misplaced (2.079%); 10 MiB/s, 291 objects/s recovering 2024-08-17T17:50:42.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:42 smithi130 ceph-mon[110068]: pgmap v1681: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 76 op/s; 42395/271923 objects degraded (15.591%); 5652/271923 objects misplaced (2.079%); 10 MiB/s, 291 objects/s recovering 2024-08-17T17:50:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:42.974 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:43.005 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:43.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:43.025 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:50:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:42 smithi179 ceph-mon[93519]: pgmap v1681: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 76 op/s; 42395/271923 objects degraded (15.591%); 5652/271923 objects misplaced (2.079%); 10 MiB/s, 291 objects/s recovering 2024-08-17T17:50:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:43.107 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:43.142 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:43.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:43.244 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:43.266 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:43.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:43.368 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:43.407 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:43.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:43.409 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:43.412 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:43.436 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:43.509 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:43.534 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:43.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:43.636 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:43.661 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:43.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:43.763 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:43.784 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:43.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:43.886 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:43.907 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:43.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:43.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:44.002 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:50:44.008 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:44.020 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:50:44.020 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:50:44.020 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:43 smithi130 ceph-mon[105521]: pgmap v1682: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 68 op/s; 42351/271791 objects degraded (15.582%); 5630/271791 objects misplaced (2.071%); 8.8 MiB/s, 242 objects/s recovering 2024-08-17T17:50:44.021 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:44.021 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:43 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2467040353' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:44.021 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:43 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:44.021 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:50:44.021 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:50:44.021 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:43 smithi130 ceph-mon[110068]: pgmap v1682: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 68 op/s; 42351/271791 objects degraded (15.582%); 5630/271791 objects misplaced (2.071%); 8.8 MiB/s, 242 objects/s recovering 2024-08-17T17:50:44.021 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:44.021 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:43 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2467040353' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:44.021 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:43 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:44.035 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:44.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:50:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:50:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:43 smithi179 ceph-mon[93519]: pgmap v1682: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 68 op/s; 42351/271791 objects degraded (15.582%); 5630/271791 objects misplaced (2.071%); 8.8 MiB/s, 242 objects/s recovering 2024-08-17T17:50:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:43 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2467040353' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:43 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:50:44.137 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:44.159 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:44.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:44.261 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:44.282 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:44.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:44.384 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:44.406 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:44.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:44.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:44.533 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:44.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:44.635 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:44.658 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:44.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:44.759 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:44.785 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:44.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:44.887 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:44.910 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:44.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:44.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:45.011 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:45.026 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:44 smithi179 ceph-mon[93519]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-08-17T17:50:45.026 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:44 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:45.026 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:50:45.026 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:44 smithi179 ceph-mon[93519]: osdmap e1199: 8 total, 8 up, 6 in 2024-08-17T17:50:45.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.034 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:45.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:45.134 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:45.156 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:45.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:45.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:44 smithi130 ceph-mon[105521]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-08-17T17:50:45.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:44 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:45.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:50:45.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:44 smithi130 ceph-mon[105521]: osdmap e1199: 8 total, 8 up, 6 in 2024-08-17T17:50:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:44 smithi130 ceph-mon[110068]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-08-17T17:50:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:44 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:50:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:50:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:44 smithi130 ceph-mon[110068]: osdmap e1199: 8 total, 8 up, 6 in 2024-08-17T17:50:45.257 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:45.282 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:45.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:45.384 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:45.406 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:45.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:45.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:45.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.538 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:45.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:45.639 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:45.661 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:45.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:45.764 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:45.786 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:45.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:45.888 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:45.909 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:45.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:45.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:46.011 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:46.035 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:46.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:45 smithi179 ceph-mon[93519]: pgmap v1684: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 42017/271026 objects degraded (15.503%); 5630/271026 objects misplaced (2.077%); 9.7 MiB/s, 154 objects/s recovering 2024-08-17T17:50:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:45 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 42351/271791 objects degraded (15.582%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:46.137 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:46.158 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:46.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:46.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:45 smithi130 ceph-mon[105521]: pgmap v1684: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 42017/271026 objects degraded (15.503%); 5630/271026 objects misplaced (2.077%); 9.7 MiB/s, 154 objects/s recovering 2024-08-17T17:50:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:45 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 42351/271791 objects degraded (15.582%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:45 smithi130 ceph-mon[110068]: pgmap v1684: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 42017/271026 objects degraded (15.503%); 5630/271026 objects misplaced (2.077%); 9.7 MiB/s, 154 objects/s recovering 2024-08-17T17:50:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:45 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 42351/271791 objects degraded (15.582%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:46.260 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:46.281 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:46.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:46.383 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:46.407 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:46.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:46.509 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:46.530 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:46.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:46.632 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:46.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:46.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:46.758 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:46.778 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:46.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:46.880 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:46.906 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:46.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:46.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:47.008 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:47.030 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:47.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:47.132 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:47.153 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:47.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:47.255 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:47.276 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:47.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:47.378 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:47.401 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:47.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:47.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:47 smithi130 ceph-mon[105521]: pgmap v1685: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 41887/270657 objects degraded (15.476%); 5600/270657 objects misplaced (2.069%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T17:50:47.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:47 smithi130 ceph-mon[110068]: pgmap v1685: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 41887/270657 objects degraded (15.476%); 5600/270657 objects misplaced (2.069%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T17:50:47.503 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:47.527 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:47.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:47.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:47 smithi179 ceph-mon[93519]: pgmap v1685: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 66 op/s; 41887/270657 objects degraded (15.476%); 5600/270657 objects misplaced (2.069%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T17:50:47.628 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:47.651 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:47.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:47.753 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:47.778 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:47.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:47.880 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:47.901 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:47.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:47.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:48.003 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:48.026 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:48.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:48.128 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:48.155 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:48.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:48.256 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:48.279 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:48.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:48.381 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:48.406 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:48.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:48.508 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:48.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.529 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:48.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:48.630 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:48.653 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:48.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:48.754 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:48.776 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:48.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:48.877 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:48.901 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:48.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:48.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:49.003 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:49.024 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:49.126 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:49.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.147 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:49.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:49.248 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:49.273 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:49.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:49.375 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:49.397 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:49.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:49.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:49 smithi130 ceph-mon[105521]: pgmap v1686: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 41526/269619 objects degraded (15.402%); 5600/269619 objects misplaced (2.077%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T17:50:49.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:49 smithi130 ceph-mon[110068]: pgmap v1686: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 41526/269619 objects degraded (15.402%); 5600/269619 objects misplaced (2.077%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T17:50:49.500 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:49.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.523 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:49.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:49.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:49 smithi179 ceph-mon[93519]: pgmap v1686: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 41526/269619 objects degraded (15.402%); 5600/269619 objects misplaced (2.077%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T17:50:49.624 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:49.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.647 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:49.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:49.749 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:49.770 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:49.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:49.872 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:49.898 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:49.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:49.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:50.000 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:50.021 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:50.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:50.123 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:50.146 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:50.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:50:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:50:49] "GET /metrics HTTP/1.1" 200 36799 "" "Prometheus/2.51.0" 2024-08-17T17:50:50.248 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:50.270 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:50.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:50.372 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:50.393 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:50.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:50.494 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:50.518 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:50.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:50:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:50:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:50:50.619 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:50.641 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:50.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:50.743 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:50.767 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:50.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:50.869 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:50.892 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:50.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:50.993 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:51.015 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:51.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:50 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 41526/269619 objects degraded (15.402%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:51.117 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:51.138 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:50 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 41526/269619 objects degraded (15.402%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:51.138 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:50 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 41526/269619 objects degraded (15.402%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:51.139 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:51.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:51.241 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:51.262 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:51.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:51.364 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:51.385 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:51.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:51.487 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:51.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.508 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:51.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:51.610 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:51.633 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:51.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:51.708 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:50:51.737 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:51.758 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:51.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:51.860 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:51.881 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:51.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:51.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:51.983 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:52.003 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:51 smithi130 ceph-mon[105521]: pgmap v1687: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 89 op/s; 41526/269427 objects degraded (15.413%); 5600/269427 objects misplaced (2.078%); 5.2 MiB/s, 83 objects/s recovering 2024-08-17T17:50:52.003 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:51 smithi130 ceph-mon[110068]: pgmap v1687: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 89 op/s; 41526/269427 objects degraded (15.413%); 5600/269427 objects misplaced (2.078%); 5.2 MiB/s, 83 objects/s recovering 2024-08-17T17:50:52.004 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:52.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:51 smithi179 ceph-mon[93519]: pgmap v1687: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 89 op/s; 41526/269427 objects degraded (15.413%); 5600/269427 objects misplaced (2.078%); 5.2 MiB/s, 83 objects/s recovering 2024-08-17T17:50:52.106 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:52.127 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:52.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:52.229 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:52.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:52.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:52.352 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:52.374 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:52.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:52.476 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:52.501 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:52.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:52.602 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:52.627 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:52.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:52.729 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:52.750 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:50:52.750 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:50:52.750 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:50:52.750 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:50:52.750 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:50:52.750 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:50:52.751 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:50:52.751 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:50:52.751 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:50:52.751 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:50:52.751 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:50:52.751 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:50:52.751 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:50:52.752 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:50:52.752 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:50:52.752 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:50:52.752 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:52.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:52.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:52.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:52.877 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:52.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:52.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:52 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1003923986' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:52.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:52 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1003923986' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:52.978 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:53.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.001 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:53.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:52 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1003923986' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:50:53.102 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:53.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.125 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:53.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:53.226 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:53.249 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:53.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:53.350 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:53.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.374 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:53.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:53.475 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:53.501 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:53.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:53.603 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:53.629 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:53.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:53.730 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:53.753 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:53.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:53.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:53.880 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:53.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:53.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:53.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:53 smithi130 ceph-mon[105521]: pgmap v1688: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 41404/269058 objects degraded (15.389%); 5570/269058 objects misplaced (2.070%); 5.1 MiB/s, 82 objects/s recovering 2024-08-17T17:50:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:53 smithi130 ceph-mon[110068]: pgmap v1688: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 41404/269058 objects degraded (15.389%); 5570/269058 objects misplaced (2.070%); 5.1 MiB/s, 82 objects/s recovering 2024-08-17T17:50:53.981 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:54.006 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:54.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:53 smithi179 ceph-mon[93519]: pgmap v1688: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 41404/269058 objects degraded (15.389%); 5570/269058 objects misplaced (2.070%); 5.1 MiB/s, 82 objects/s recovering 2024-08-17T17:50:54.108 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:54.129 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:54.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:54.231 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:54.256 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:54.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:54.288 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:50:54.288 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:50:54.288 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T17:50:54.288 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T17:50:54.288 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:50:54.358 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:54.383 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:54.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:54.484 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:54.508 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:54.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:54.609 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:54.630 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:54.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:54.732 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:54.755 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:54.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:54.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:54.874 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:54.883 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:54.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:54.985 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:55.009 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:55.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:55.110 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:55.133 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:55.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:55.235 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:55.256 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:55.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:55.344 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:50:55.358 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:55.370 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:55 smithi130 ceph-mon[105521]: pgmap v1689: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 41012/268167 objects degraded (15.293%); 5570/268167 objects misplaced (2.077%); 5.0 MiB/s, 80 objects/s recovering 2024-08-17T17:50:55.370 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:55 smithi130 ceph-mon[110068]: pgmap v1689: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 41012/268167 objects degraded (15.293%); 5570/268167 objects misplaced (2.077%); 5.0 MiB/s, 80 objects/s recovering 2024-08-17T17:50:55.397 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:55.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:55.499 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:55.520 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:55.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:55.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:55 smithi179 ceph-mon[93519]: pgmap v1689: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 90 op/s; 41012/268167 objects degraded (15.293%); 5570/268167 objects misplaced (2.077%); 5.0 MiB/s, 80 objects/s recovering 2024-08-17T17:50:55.622 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:55.643 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:55.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:55.746 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:55.766 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:55.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:55.868 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:55.889 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:55.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:55.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:55.991 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:56.055 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:56.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:56.157 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:56.182 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:56.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:56.285 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:56.311 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:56.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:56.412 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:56.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 41012/268167 objects degraded (15.293%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:56.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 41012/268167 objects degraded (15.293%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:56.433 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:56.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:56.535 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:56.558 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:56.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:56.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 41012/268167 objects degraded (15.293%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:50:56.660 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:56.682 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:56.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:56.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:56.807 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:56.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:56.909 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:56.930 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:56.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:56.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:57.032 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:57.053 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:57.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:57.155 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:57.182 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:57.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:57.283 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:57.307 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:57.408 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:57.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:57 smithi130 ceph-mon[105521]: pgmap v1690: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 40887/267870 objects degraded (15.264%); 5546/267870 objects misplaced (2.070%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:50:57.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:57 smithi130 ceph-mon[110068]: pgmap v1690: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 40887/267870 objects degraded (15.264%); 5546/267870 objects misplaced (2.070%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:50:57.431 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:57.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:57.533 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:57.555 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:57.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:57.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:57 smithi179 ceph-mon[93519]: pgmap v1690: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 87 op/s; 40887/267870 objects degraded (15.264%); 5546/267870 objects misplaced (2.070%); 4.9 MiB/s, 78 objects/s recovering 2024-08-17T17:50:57.656 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:57.670 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:50:57.682 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:57.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:57.784 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:57.805 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:57.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:57.907 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:57.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:57.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:57.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:58.030 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:50:58.055 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:58.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:50:58.156 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:58.178 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:58.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:58.281 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:50:58.303 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:58.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:50:58.405 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:58.428 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:58.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:58.530 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:58.541 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:58.541 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:58.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:58.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:58.555 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:58.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:50:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:50:58.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:50:58.677 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:58.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:50:58.779 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:58.803 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:58.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:58.905 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:58.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:58.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:58.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:59.028 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:59.056 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:59.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:59.157 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:50:59.178 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:59.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:50:59.280 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:59.301 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:59.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:59.306 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:50:59.324 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:50:59.403 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:50:59.424 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:50:59.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:50:59.526 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:50:59.553 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:59.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:50:59.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:59 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2012122522' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:59 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:50:59 smithi179 ceph-mon[93519]: pgmap v1691: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 40470/267108 objects degraded (15.151%); 5546/267108 objects misplaced (2.076%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T17:50:59.645 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:59 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2012122522' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:59.645 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:59 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:59.645 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:50:59 smithi130 ceph-mon[105521]: pgmap v1691: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 40470/267108 objects degraded (15.151%); 5546/267108 objects misplaced (2.076%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T17:50:59.646 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:59 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2012122522' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:59.646 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:59 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:50:59.646 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:50:59 smithi130 ceph-mon[110068]: pgmap v1691: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 40470/267108 objects degraded (15.151%); 5546/267108 objects misplaced (2.076%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T17:50:59.655 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:59.680 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:59.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:50:59.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:50:59.806 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:59.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:50:59.908 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:50:59.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:50:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:50:59] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-08-17T17:50:59.934 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:50:59.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:50:59.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:00.036 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:00.060 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:00.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:00.162 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:00.165 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:51:00.186 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:00.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:00.289 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:00.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:00.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:00.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:00 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:51:00.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:00 smithi130 ceph-mon[105521]: osdmap e1200: 8 total, 8 up, 6 in 2024-08-17T17:51:00.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:00 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:51:00.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:00 smithi130 ceph-mon[110068]: osdmap e1200: 8 total, 8 up, 6 in 2024-08-17T17:51:00.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:00.457 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:00.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:00.559 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:00.581 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:00.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:00.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:00 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:51:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:00 smithi179 ceph-mon[93519]: osdmap e1200: 8 total, 8 up, 6 in 2024-08-17T17:51:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:51:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:51:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:51:00.682 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:00.714 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:00.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:00.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:00.842 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:00.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:00.908 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:00.944 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:00.974 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:00.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:00.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:01.076 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:01.101 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:01.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:01.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:01.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:01.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:01.319 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T17:51:01.326 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:01.353 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:01.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:01.454 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:01.486 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:01.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:01.588 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:01.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 40470/267108 objects degraded (15.151%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:01.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:01 smithi130 ceph-mon[105521]: pgmap v1693: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 40470/266943 objects degraded (15.161%); 5546/266943 objects misplaced (2.078%); 5.8 MiB/s, 93 objects/s recovering 2024-08-17T17:51:01.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:01 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2625235030' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:01.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:01 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:01.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 40470/267108 objects degraded (15.151%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:01.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:01 smithi130 ceph-mon[110068]: pgmap v1693: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 40470/266943 objects degraded (15.161%); 5546/266943 objects misplaced (2.078%); 5.8 MiB/s, 93 objects/s recovering 2024-08-17T17:51:01.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:01 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2625235030' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:01.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:01 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:01.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 40470/267108 objects degraded (15.151%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:01.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:01 smithi179 ceph-mon[93519]: pgmap v1693: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 40470/266943 objects degraded (15.161%); 5546/266943 objects misplaced (2.078%); 5.8 MiB/s, 93 objects/s recovering 2024-08-17T17:51:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:01 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2625235030' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:01 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:01.610 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:01.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:01.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:01.738 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:01.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:01.840 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:01.862 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:01.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:01.963 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:01.984 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:01.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:01.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:02.085 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:02.111 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:02.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:02.213 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:02.234 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:02.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:02.336 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:02.357 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:02.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:02.459 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:02.483 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:02.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:02.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:02 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:02.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:02 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:51:02.547 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:02 smithi130 ceph-mon[105521]: osdmap e1201: 8 total, 8 up, 6 in 2024-08-17T17:51:02.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:02 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:02.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:02 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:51:02.548 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:02 smithi130 ceph-mon[110068]: osdmap e1201: 8 total, 8 up, 6 in 2024-08-17T17:51:02.585 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:02 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:02 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:51:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:02 smithi179 ceph-mon[93519]: osdmap e1201: 8 total, 8 up, 6 in 2024-08-17T17:51:02.609 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:02.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:02.698 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:02.710 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:02.731 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:02.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:02.833 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:02.854 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:02.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:02.955 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:02.976 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:02.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:02.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:03.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:03.099 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:03.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:03.200 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:03.221 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:03.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:03.323 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:03.336 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:03 smithi130 ceph-mon[110068]: pgmap v1695: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 68 op/s; 40332/266514 objects degraded (15.133%); 5512/266514 objects misplaced (2.068%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:51:03.337 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:03 smithi130 ceph-mon[105521]: pgmap v1695: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 68 op/s; 40332/266514 objects degraded (15.133%); 5512/266514 objects misplaced (2.068%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:51:03.344 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:03.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:03.446 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:03.474 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:03.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:03.576 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:03.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:03 smithi179 ceph-mon[93519]: pgmap v1695: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 68 op/s; 40332/266514 objects degraded (15.133%); 5512/266514 objects misplaced (2.068%); 4.3 MiB/s, 68 objects/s recovering 2024-08-17T17:51:03.597 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:03.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:03.699 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:03.720 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:03.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:03.822 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:03.848 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:03.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:03.950 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:03.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:03.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:03.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:04.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:04.099 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:04.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:04.201 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:04.222 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:51:04.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:04.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:04.325 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:04.346 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:04.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:04.448 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:04.475 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:04.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:04.576 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:04.599 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:04.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:04.701 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:04.721 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:04.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:04.823 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:04.846 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:04.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:04.948 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:04.971 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:04.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:04.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:05.073 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:05.095 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:05.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:05.196 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:05.213 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:51:05.214 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:51:05.217 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:05.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:05.233 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:51:05.233 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:51:05.250 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:51:05.251 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:51:05.251 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:05 smithi130 ceph-mon[105521]: pgmap v1696: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 110 op/s; 39907/265206 objects degraded (15.048%); 5512/265206 objects misplaced (2.078%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T17:51:05.251 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:05 smithi130 ceph-mon[110068]: pgmap v1696: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 110 op/s; 39907/265206 objects degraded (15.048%); 5512/265206 objects misplaced (2.078%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T17:51:05.269 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:51:05.269 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:51:05.285 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:51:05.285 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:51:05.302 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:51:05.302 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:51:05.319 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:05.322 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:51:05.323 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:51:05.339 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:05.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:05.348 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:51:05.348 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:51:05.441 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:05.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:05.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:05.565 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:05.586 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:05.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:05.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:05 smithi179 ceph-mon[93519]: pgmap v1696: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 110 op/s; 39907/265206 objects degraded (15.048%); 5512/265206 objects misplaced (2.078%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T17:51:05.688 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:05.713 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:05.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:05.815 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:05.837 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:05.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:05.939 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:05.962 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:05.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:05.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:06.063 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:06.085 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:06.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:06.186 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:06.207 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:06.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:06.309 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:06.333 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:06.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:06.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:06 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3622598937' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:06.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 39907/265206 objects degraded (15.048%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:06.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:06 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3622598937' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:06.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 39907/265206 objects degraded (15.048%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:06.435 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:06.460 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:06.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:06.562 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:06.583 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:06.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:06.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:06 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3622598937' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:06.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 39907/265206 objects degraded (15.048%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:06.684 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:06.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.706 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:06.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:06.807 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:06.830 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:06.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:06.869 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:51:06.869 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:51:06.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:06.956 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:06.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:06.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:07.058 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:07.080 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:07.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:07.109 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:07 smithi130 ceph-mon[105521]: pgmap v1697: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 39785/264849 objects degraded (15.022%); 5483/264849 objects misplaced (2.070%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:51:07.110 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:07 smithi130 ceph-mon[110068]: pgmap v1697: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 39785/264849 objects degraded (15.022%); 5483/264849 objects misplaced (2.070%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:51:07.181 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:07.203 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:07.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:07.304 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:07.326 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:07.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:07.428 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:07.449 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:07.551 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:07.579 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:07.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:07.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:07 smithi179 ceph-mon[93519]: pgmap v1697: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 39785/264849 objects degraded (15.022%); 5483/264849 objects misplaced (2.070%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T17:51:07.681 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:07.703 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:07.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:07.804 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:07.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:07.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:07.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:07.952 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:07.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:07.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:08.053 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:08.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:51:08.078 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:08.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:08.116 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:51:08.133 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:51:08.180 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:08.201 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:08.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:08.303 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:08.341 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:08.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:08.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:08 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2764455036' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:51:08.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:08 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:51:08.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:08 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2764455036' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:51:08.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:08 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:51:08.443 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:08.465 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:08.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:08.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:08.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:08.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:08.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:08 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2764455036' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:51:08.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:08 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:51:08.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:08.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:08.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:08.812 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:08.815 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:08.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.843 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:08.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:08.846 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:51:08.944 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:08.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:08.965 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:08.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:09.066 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:09.087 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:09.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:09.190 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:09.212 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:09.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:09.314 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:09.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:09.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:09.344 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:09 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:51:09.344 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:09 smithi130 ceph-mon[105521]: osdmap e1202: 8 total, 8 up, 6 in 2024-08-17T17:51:09.344 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:09 smithi130 ceph-mon[105521]: pgmap v1699: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 39271/263961 objects degraded (14.878%); 5483/263961 objects misplaced (2.077%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:51:09.344 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:09 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:51:09.344 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:09 smithi130 ceph-mon[110068]: osdmap e1202: 8 total, 8 up, 6 in 2024-08-17T17:51:09.345 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:09 smithi130 ceph-mon[110068]: pgmap v1699: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 39271/263961 objects degraded (14.878%); 5483/263961 objects misplaced (2.077%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:51:09.436 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:09.461 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:09.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:09.553 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:09.564 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:09.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:09 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:51:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:09 smithi179 ceph-mon[93519]: osdmap e1202: 8 total, 8 up, 6 in 2024-08-17T17:51:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:09 smithi179 ceph-mon[93519]: pgmap v1699: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 39271/263961 objects degraded (14.878%); 5483/263961 objects misplaced (2.077%); 8.3 MiB/s, 133 objects/s recovering 2024-08-17T17:51:09.602 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:09.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:09.612 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:09.704 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:09.725 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:09.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:09.826 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:09.849 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:09.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:09.950 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:09.967 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:51:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:51:09] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-08-17T17:51:09.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:09.972 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:09.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:10.073 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:10.094 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:10.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:10.196 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:10.220 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:10.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:10.322 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:10.343 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:10.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:10.445 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:10.466 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:10.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:10.567 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:10.591 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:10.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:51:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:51:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:51:10.692 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:10.713 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:10.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:10.815 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:10.836 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:10.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:10.903 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:10.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:10 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 39271/263961 objects degraded (14.878%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:10 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 39271/263961 objects degraded (14.878%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:10.938 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:10.960 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:10.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:10.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:11.061 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:11.083 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:11.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:11.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:10 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 39271/263961 objects degraded (14.878%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:11.185 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:11.212 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:11.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:11.313 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:11.339 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:11.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:11.441 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:11.465 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:11.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:11.566 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:11.589 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:11.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:11.691 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:11.724 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:11.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:11.826 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:11.847 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:11.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:11 smithi130 ceph-mon[105521]: pgmap v1700: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 39271/263823 objects degraded (14.885%); 5483/263823 objects misplaced (2.078%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T17:51:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:11 smithi130 ceph-mon[110068]: pgmap v1700: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 39271/263823 objects degraded (14.885%); 5483/263823 objects misplaced (2.078%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T17:51:11.948 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:11.969 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:11.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:11.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:12.071 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:11 smithi179 ceph-mon[93519]: pgmap v1700: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 39271/263823 objects degraded (14.885%); 5483/263823 objects misplaced (2.078%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T17:51:12.100 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:12.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:12.202 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:12.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:12.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:12.325 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:12.347 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:12.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:12.448 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:12.473 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:12.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:12.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:12.596 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:12.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:12.698 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:12.719 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:12.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:12.731 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T17:51:12.821 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:12.842 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:12.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:12.944 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:12.966 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:12.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:12.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:13.068 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:13.091 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:13.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:12 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2088023852' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:51:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:12 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:51:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:13.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:12 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2088023852' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:51:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:12 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:51:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:12 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2088023852' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:51:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:12 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:51:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:13.193 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:13.217 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:13.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:13.319 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:13.341 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:13.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:13.443 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:13.468 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:13.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:13.570 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:13.600 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:13.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:13.701 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:13.722 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:13.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:13.824 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:13.850 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:13.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:13.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:13 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:13 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:51:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:13 smithi130 ceph-mon[105521]: osdmap e1203: 8 total, 8 up, 6 in 2024-08-17T17:51:13.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:13 smithi130 ceph-mon[105521]: pgmap v1702: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 70 op/s; 39182/263517 objects degraded (14.869%); 5447/263517 objects misplaced (2.067%); 4.8 MiB/s, 77 objects/s recovering 2024-08-17T17:51:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:13 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:13 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:51:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:13 smithi130 ceph-mon[110068]: osdmap e1203: 8 total, 8 up, 6 in 2024-08-17T17:51:13.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:13 smithi130 ceph-mon[110068]: pgmap v1702: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 70 op/s; 39182/263517 objects degraded (14.869%); 5447/263517 objects misplaced (2.067%); 4.8 MiB/s, 77 objects/s recovering 2024-08-17T17:51:13.952 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:13.973 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:13.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:13.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:14.075 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:14.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:13 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:14.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:13 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:51:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:13 smithi179 ceph-mon[93519]: osdmap e1203: 8 total, 8 up, 6 in 2024-08-17T17:51:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:13 smithi179 ceph-mon[93519]: pgmap v1702: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 70 op/s; 39182/263517 objects degraded (14.869%); 5447/263517 objects misplaced (2.067%); 4.8 MiB/s, 77 objects/s recovering 2024-08-17T17:51:14.095 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:14.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:14.197 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:14.218 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:14.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:14.320 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:14.341 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:14.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:14.442 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:14.463 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:14.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:14.565 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:14.586 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:14.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:14.688 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:14.709 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:14.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:14.810 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:14.832 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:14.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:14.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:14.960 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:14.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:14.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:15.062 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:15.088 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:15.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:15.190 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:15.211 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:15.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:15.307 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:51:15.307 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:51:15.314 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:15.330 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:51:15.331 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:51:15.334 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:15.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:15.351 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:51:15.352 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:51:15.352 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:15 smithi130 ceph-mon[105521]: pgmap v1703: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 96 op/s; 38586/262530 objects degraded (14.698%); 5447/262530 objects misplaced (2.075%); 9.0 MiB/s, 144 objects/s recovering 2024-08-17T17:51:15.352 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:15 smithi130 ceph-mon[110068]: pgmap v1703: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 96 op/s; 38586/262530 objects degraded (14.698%); 5447/262530 objects misplaced (2.075%); 9.0 MiB/s, 144 objects/s recovering 2024-08-17T17:51:15.377 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:51:15.377 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:51:15.396 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:51:15.396 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:51:15.417 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:51:15.417 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:51:15.436 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:15.436 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:51:15.437 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:51:15.458 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:15.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:15.468 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:51:15.469 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:51:15.560 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:15.581 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:15.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:15.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:15 smithi179 ceph-mon[93519]: pgmap v1703: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 96 op/s; 38586/262530 objects degraded (14.698%); 5447/262530 objects misplaced (2.075%); 9.0 MiB/s, 144 objects/s recovering 2024-08-17T17:51:15.683 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:15.704 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:15.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:15.805 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:15.829 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:15.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:15.931 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:15.952 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:15.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:15.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:15.995 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:51:16.054 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:16.078 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:16.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:16.180 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:16.203 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:16.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:16.305 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:16.329 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:16.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:16.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:16 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/498688648' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:16.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 38586/262530 objects degraded (14.698%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:16.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:16 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/498688648' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:16.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 38586/262530 objects degraded (14.698%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:16.431 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:16.454 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:16.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:16.556 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:16.578 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:16.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:16.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:16 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/498688648' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:16.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 38586/262530 objects degraded (14.698%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:16.680 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:16.714 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:16.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:16.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:16.839 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:16.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:16.941 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:16.962 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:16.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:16.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:17.063 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:17.088 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:17.190 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:17.190 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:17 smithi130 ceph-mon[105521]: pgmap v1704: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 38464/262194 objects degraded (14.670%); 5414/262194 objects misplaced (2.065%); 8.5 MiB/s, 136 objects/s recovering 2024-08-17T17:51:17.191 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:17 smithi130 ceph-mon[110068]: pgmap v1704: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 38464/262194 objects degraded (14.670%); 5414/262194 objects misplaced (2.065%); 8.5 MiB/s, 136 objects/s recovering 2024-08-17T17:51:17.215 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:17.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:17.317 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:17.338 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:17.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:17.417 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:17.440 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:17.440 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:51:17.440 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:51:17.440 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 0 2024-08-17T17:51:17.440 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:51:17.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:17.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:17.562 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:17.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.584 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:17.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:17.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:17 smithi179 ceph-mon[93519]: pgmap v1704: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 38464/262194 objects degraded (14.670%); 5414/262194 objects misplaced (2.065%); 8.5 MiB/s, 136 objects/s recovering 2024-08-17T17:51:17.685 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:17.710 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:17.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:17.815 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:17.837 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:17.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:17.938 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:17.959 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:17.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:17.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:18.061 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:18.082 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:18.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:18.113 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:51:18.126 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:51:18.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:18.206 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:18.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:18.308 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:18.330 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:18.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:18.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2499474430' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:51:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2499474430' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:51:18.432 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:18.453 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:18.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:18.555 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:18.579 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:18.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2499474430' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:51:18.681 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:18.704 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:18.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:18.806 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:18.833 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:18.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:18.851 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:51:18.935 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:18.958 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:18.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:18.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:19.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:19.089 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:19.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:19.191 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:19.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.214 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:19.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:19.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:19.338 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:19.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:19.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2499474430' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:51:19.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:19 smithi130 ceph-mon[105521]: osdmap e1204: 8 total, 8 up, 6 in 2024-08-17T17:51:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:19 smithi130 ceph-mon[105521]: pgmap v1706: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 37730/261390 objects degraded (14.434%); 5414/261390 objects misplaced (2.071%); 11 MiB/s, 179 objects/s recovering 2024-08-17T17:51:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2499474430' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:51:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:19 smithi130 ceph-mon[110068]: osdmap e1204: 8 total, 8 up, 6 in 2024-08-17T17:51:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:19 smithi130 ceph-mon[110068]: pgmap v1706: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 37730/261390 objects degraded (14.434%); 5414/261390 objects misplaced (2.071%); 11 MiB/s, 179 objects/s recovering 2024-08-17T17:51:19.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:19.467 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:19.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:19.568 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:19.590 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:19.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2499474430' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:51:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:19 smithi179 ceph-mon[93519]: osdmap e1204: 8 total, 8 up, 6 in 2024-08-17T17:51:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:19 smithi179 ceph-mon[93519]: pgmap v1706: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 37730/261390 objects degraded (14.434%); 5414/261390 objects misplaced (2.071%); 11 MiB/s, 179 objects/s recovering 2024-08-17T17:51:19.691 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:19.712 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:19.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:19.814 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:19.837 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:19.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:19.938 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:19.964 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:19.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:19.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:20.066 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:20.087 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:20.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:20.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:51:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:51:19] "GET /metrics HTTP/1.1" 200 36815 "" "Prometheus/2.51.0" 2024-08-17T17:51:20.189 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:20.215 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:20.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:20.317 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:20.343 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:20.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:20.445 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:20.468 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:20.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:20.570 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:20.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:51:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:51:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:51:20.597 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:20.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:20.698 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:20.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.721 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:20.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:20.822 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:20.843 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:20.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:20.945 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:20.968 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:20.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:20.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:21.070 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:21.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:20 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 37730/261390 objects degraded (14.434%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:21.095 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:21.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:21.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:20 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 37730/261390 objects degraded (14.434%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:21.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:20 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 37730/261390 objects degraded (14.434%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:21.196 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:21.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:21.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:21.320 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:21.340 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:21.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:21.442 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:21.465 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:21.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:21.566 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:21.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:21.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:21.690 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:21.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:21.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:21.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:21.836 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:21.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:21.937 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:21.960 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:21.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:21.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:21.983 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:21 smithi130 ceph-mon[105521]: pgmap v1707: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 37730/261225 objects degraded (14.443%); 5414/261225 objects misplaced (2.073%); 10 MiB/s, 167 objects/s recovering 2024-08-17T17:51:21.984 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:21 smithi130 ceph-mon[110068]: pgmap v1707: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 37730/261225 objects degraded (14.443%); 5414/261225 objects misplaced (2.073%); 10 MiB/s, 167 objects/s recovering 2024-08-17T17:51:22.062 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:22.087 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:22.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:22.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:21 smithi179 ceph-mon[93519]: pgmap v1707: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 37730/261225 objects degraded (14.443%); 5414/261225 objects misplaced (2.073%); 10 MiB/s, 167 objects/s recovering 2024-08-17T17:51:22.189 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:22.211 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:22.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:22.313 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:22.337 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:22.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:22.439 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:22.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:22.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:22.563 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:22.589 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:22.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:22.691 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:22.715 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:22.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:22.817 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:22.838 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:22.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:22.940 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:22.965 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:22.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:22.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:23.067 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:23.090 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:23.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:23.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:23 smithi130 ceph-mon[105521]: pgmap v1708: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 37606/260850 objects degraded (14.417%); 5380/260850 objects misplaced (2.062%); 9.1 MiB/s, 145 objects/s recovering 2024-08-17T17:51:23.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:23 smithi130 ceph-mon[110068]: pgmap v1708: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 37606/260850 objects degraded (14.417%); 5380/260850 objects misplaced (2.062%); 9.1 MiB/s, 145 objects/s recovering 2024-08-17T17:51:23.192 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:23.217 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:23.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:23.319 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:23.346 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:23.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:23.448 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:23.470 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:23.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:23.572 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:23 smithi179 ceph-mon[93519]: pgmap v1708: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 37606/260850 objects degraded (14.417%); 5380/260850 objects misplaced (2.062%); 9.1 MiB/s, 145 objects/s recovering 2024-08-17T17:51:23.594 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:23.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:23.695 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:23.719 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:23.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:23.820 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:23.842 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:23.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:23.943 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:23.972 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:23.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:23.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:24.074 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:24.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:24.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:24.198 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:24.220 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:24.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:24.322 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:24.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.344 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:24.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:24.445 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:24.470 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:24.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:24.572 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:24.593 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:24.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:24.695 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:24.718 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:24.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:24.820 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:24.842 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:24.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:24.944 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:24.965 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:24.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:24.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:25.067 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:25.088 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:25.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:25.190 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:25.213 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:25.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:25.315 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:25.338 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:25.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:25.440 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:25.464 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:25.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:25.566 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:25.591 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:25.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:25.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:25 smithi179 ceph-mon[93519]: osdmap e1205: 8 total, 8 up, 6 in 2024-08-17T17:51:25.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:25 smithi179 ceph-mon[93519]: pgmap v1710: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 36468/259809 objects degraded (14.036%); 5380/259809 objects misplaced (2.071%); 9.1 MiB/s, 241 objects/s recovering 2024-08-17T17:51:25.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:25 smithi130 ceph-mon[105521]: osdmap e1205: 8 total, 8 up, 6 in 2024-08-17T17:51:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:25 smithi130 ceph-mon[105521]: pgmap v1710: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 36468/259809 objects degraded (14.036%); 5380/259809 objects misplaced (2.071%); 9.1 MiB/s, 241 objects/s recovering 2024-08-17T17:51:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:25 smithi130 ceph-mon[110068]: osdmap e1205: 8 total, 8 up, 6 in 2024-08-17T17:51:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:25 smithi130 ceph-mon[110068]: pgmap v1710: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 36468/259809 objects degraded (14.036%); 5380/259809 objects misplaced (2.071%); 9.1 MiB/s, 241 objects/s recovering 2024-08-17T17:51:25.692 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:25.714 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:25.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:25.816 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:25.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.838 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:25.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:25.939 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:25.960 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:25.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:25.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:26.061 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:26.084 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:26.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:26.186 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:26.208 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:26.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:26.309 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:26.331 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:26.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:26.433 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:26.456 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:26.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:26.558 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:26.579 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:26.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:26.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:26 smithi179 ceph-mon[93519]: osdmap e1206: 8 total, 8 up, 6 in 2024-08-17T17:51:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 36468/259809 objects degraded (14.036%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:26.624 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:26 smithi130 ceph-mon[105521]: osdmap e1206: 8 total, 8 up, 6 in 2024-08-17T17:51:26.624 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 36468/259809 objects degraded (14.036%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:26.625 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:26 smithi130 ceph-mon[110068]: osdmap e1206: 8 total, 8 up, 6 in 2024-08-17T17:51:26.625 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 36468/259809 objects degraded (14.036%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:26.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:51:26.681 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:26.704 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:26.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:26.841 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:26.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.865 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:26.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:26.966 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:26.989 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:26.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:26.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:27.091 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:27.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:27.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:27.214 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:27.236 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:27.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:27.321 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:27 smithi130 ceph-mon[105521]: pgmap v1712: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 83 op/s; 25176/259389 objects degraded (9.706%); 15039/259389 objects misplaced (5.798%); 3.9 MiB/s, 158 objects/s recovering 2024-08-17T17:51:27.321 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:27 smithi130 ceph-mon[110068]: pgmap v1712: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 83 op/s; 25176/259389 objects degraded (9.706%); 15039/259389 objects misplaced (5.798%); 3.9 MiB/s, 158 objects/s recovering 2024-08-17T17:51:27.338 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:27.364 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:27.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:27.408 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:27.466 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:27.488 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:27.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:27.589 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:27 smithi179 ceph-mon[93519]: pgmap v1712: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 83 op/s; 25176/259389 objects degraded (9.706%); 15039/259389 objects misplaced (5.798%); 3.9 MiB/s, 158 objects/s recovering 2024-08-17T17:51:27.617 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:27.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:27.718 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:27.740 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:27.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:27.841 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:27.871 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:27.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:27.973 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:27.994 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:27.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:27.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:28.096 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:28.118 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:28.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:28.134 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:28.134 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:28.220 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:28.240 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:28.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:28.343 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:28.364 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:28.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:28.466 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:28.487 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:28.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:28.589 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:28.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:28.615 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:28.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:28.717 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:28.739 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:28.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:28.841 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:28.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.863 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:28.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:28.964 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:28.986 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:28.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:28.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:29.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:29.111 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:29.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:29.195 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T17:51:29.212 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2263538805' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:29.212 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:29 smithi130 ceph-mon[105521]: pgmap v1713: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 21018/258504 objects degraded (8.131%); 18758/258504 objects misplaced (7.256%); 6.4 MiB/s, 213 objects/s recovering 2024-08-17T17:51:29.212 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2263538805' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:29.212 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:29 smithi130 ceph-mon[110068]: pgmap v1713: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 21018/258504 objects degraded (8.131%); 18758/258504 objects misplaced (7.256%); 6.4 MiB/s, 213 objects/s recovering 2024-08-17T17:51:29.213 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:29.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.235 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:29.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:29.336 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:29.358 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:29.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:29.460 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:29.484 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:29.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:29.586 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:29.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2263538805' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:29 smithi179 ceph-mon[93519]: pgmap v1713: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 21018/258504 objects degraded (8.131%); 18758/258504 objects misplaced (7.256%); 6.4 MiB/s, 213 objects/s recovering 2024-08-17T17:51:29.611 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:29.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:29.712 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:29.734 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:29.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:29.836 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:29.857 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:29.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:29.959 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:29.981 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:29.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:29.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:30.083 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:30.104 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:30.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:51:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:51:29] "GET /metrics HTTP/1.1" 200 36816 "" "Prometheus/2.51.0" 2024-08-17T17:51:30.206 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:30.230 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:30.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:30.332 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:30.353 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:30.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:30.455 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:30.478 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:30.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:30.580 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:30.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:30 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:51:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:30 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2263538805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:51:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:30 smithi179 ceph-mon[93519]: osdmap e1207: 8 total, 8 up, 6 in 2024-08-17T17:51:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:51:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:51:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:51:30.601 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:30.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:30.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:30 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:51:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:30 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2263538805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:51:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:30 smithi130 ceph-mon[110068]: osdmap e1207: 8 total, 8 up, 6 in 2024-08-17T17:51:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:30 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:51:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:30 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2263538805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:51:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:30 smithi130 ceph-mon[105521]: osdmap e1207: 8 total, 8 up, 6 in 2024-08-17T17:51:30.702 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:30.724 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:30.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:30.825 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:30.849 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:30.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:30.950 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:30.975 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:30.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:30.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:31.077 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:31.098 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:31.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:31.200 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:31.221 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:31.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:31.323 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:31.346 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:31.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:31.448 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:31.470 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:31.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:31.538 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 21018/258504 objects degraded (8.131%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:31.538 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:31 smithi130 ceph-mon[105521]: pgmap v1715: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 71 op/s; 21018/258342 objects degraded (8.136%); 18758/258342 objects misplaced (7.261%); 3.6 MiB/s, 74 objects/s recovering 2024-08-17T17:51:31.538 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 21018/258504 objects degraded (8.131%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:31.538 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:31 smithi130 ceph-mon[110068]: pgmap v1715: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 71 op/s; 21018/258342 objects degraded (8.136%); 18758/258342 objects misplaced (7.261%); 3.6 MiB/s, 74 objects/s recovering 2024-08-17T17:51:31.572 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 21018/258504 objects degraded (8.131%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:31 smithi179 ceph-mon[93519]: pgmap v1715: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 71 op/s; 21018/258342 objects degraded (8.136%); 18758/258342 objects misplaced (7.261%); 3.6 MiB/s, 74 objects/s recovering 2024-08-17T17:51:31.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.594 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:31.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:31.694 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:31.716 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:31.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:31.818 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:31.846 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:31.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:31.948 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:31.969 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:31.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:31.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:32.070 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:32.093 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:32.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:32.195 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:32.217 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:32.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:32.318 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:32.340 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:32.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:32.442 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:32.464 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:32.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:32.566 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:32.591 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:32.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:32.693 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:32.720 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:32.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:32.822 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:32.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.845 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:32.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:32.946 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:32.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:32.968 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:32.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:33.070 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:33.091 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:33.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:33.192 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:33.216 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:33.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:33.266 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:51:33.267 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:51:33.267 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T17:51:33.267 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T17:51:33.267 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:51:33.278 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:33 smithi130 ceph-mon[105521]: pgmap v1716: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 20976/258027 objects degraded (8.129%); 18666/258027 objects misplaced (7.234%); 3.5 MiB/s, 70 objects/s recovering 2024-08-17T17:51:33.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:33 smithi130 ceph-mon[110068]: pgmap v1716: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 20976/258027 objects degraded (8.129%); 18666/258027 objects misplaced (7.234%); 3.5 MiB/s, 70 objects/s recovering 2024-08-17T17:51:33.318 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:33.343 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:33.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:33.444 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:33.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.468 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:33.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:33.569 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:33.590 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:33.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:33.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:33 smithi179 ceph-mon[93519]: pgmap v1716: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 20976/258027 objects degraded (8.129%); 18666/258027 objects misplaced (7.234%); 3.5 MiB/s, 70 objects/s recovering 2024-08-17T17:51:33.692 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:33.715 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:33.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:33.798 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:51:33.798 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:51:33.817 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:33.824 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:51:33.824 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:51:33.839 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:33.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:33.851 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:51:33.852 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:51:33.879 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:51:33.879 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:51:33.903 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:51:33.903 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:51:33.930 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:51:33.931 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:51:33.940 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:33.949 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:51:33.949 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:51:33.964 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:33.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:33.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:33.971 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:51:33.972 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:51:34.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:34.089 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:34.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:34.116 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:51:34.190 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:34.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:34.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:34.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:34.337 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:34.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:34.393 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:34 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3683972697' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:51:34.393 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:34 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:51:34.393 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:34 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1502460949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:34.394 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:34 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3683972697' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:51:34.394 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:34 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:51:34.394 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:34 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1502460949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:34.438 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:34.446 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:51:34.462 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:34.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:34.564 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:34.588 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:34.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:34.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:34 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3683972697' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:51:34.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:34 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:51:34.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:34 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1502460949' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:34.690 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:34.717 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:34.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:34.818 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:34.841 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:34.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:34.943 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:34.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:34.968 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:34.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:35.069 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:35.090 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:35.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:35.192 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:35.213 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:35.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:35.279 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:51:35.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:35.346 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:35 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:51:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:35 smithi130 ceph-mon[105521]: osdmap e1208: 8 total, 8 up, 6 in 2024-08-17T17:51:35.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:35 smithi130 ceph-mon[105521]: pgmap v1718: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 20976/256887 objects degraded (8.165%); 18238/256887 objects misplaced (7.100%); 6.3 MiB/s, 115 objects/s recovering 2024-08-17T17:51:35.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:35 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:51:35.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:35 smithi130 ceph-mon[110068]: osdmap e1208: 8 total, 8 up, 6 in 2024-08-17T17:51:35.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:35 smithi130 ceph-mon[110068]: pgmap v1718: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 20976/256887 objects degraded (8.165%); 18238/256887 objects misplaced (7.100%); 6.3 MiB/s, 115 objects/s recovering 2024-08-17T17:51:35.448 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:35.474 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:35.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:35.546 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:35.575 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:35.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:35 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:51:35.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:35 smithi179 ceph-mon[93519]: osdmap e1208: 8 total, 8 up, 6 in 2024-08-17T17:51:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:35 smithi179 ceph-mon[93519]: pgmap v1718: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 20976/256887 objects degraded (8.165%); 18238/256887 objects misplaced (7.100%); 6.3 MiB/s, 115 objects/s recovering 2024-08-17T17:51:35.601 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:35.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:35.603 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:35.704 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:35.730 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:35.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:35.832 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:35.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.854 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:35.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:35.955 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:35.982 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:35.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:35.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:36.070 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:51:36.070 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:51:36.084 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:36.138 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:36.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:36.240 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:36.264 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:36.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:36.302 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 20976/256887 objects degraded (8.165%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:36.302 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 20976/256887 objects degraded (8.165%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:36.366 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:36.388 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:36.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:36.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:36.511 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:36.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 20976/256887 objects degraded (8.165%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:36.613 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:36.634 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:36.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:36.736 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:36.758 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:36.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:36.835 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:51:36.861 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:36.903 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:36.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:36.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:37.005 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:37.030 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:37.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:37.132 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:37.153 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:37.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:37.255 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:37.277 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:37.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:37.378 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:37.399 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:37.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:37.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:37 smithi130 ceph-mon[105521]: pgmap v1719: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 20930/256566 objects degraded (8.158%); 18156/256566 objects misplaced (7.077%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:51:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:37 smithi130 ceph-mon[110068]: pgmap v1719: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 20930/256566 objects degraded (8.158%); 18156/256566 objects misplaced (7.077%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:51:37.419 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:37.473 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:37.501 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:37.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.524 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:37.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:37.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:37 smithi179 ceph-mon[93519]: pgmap v1719: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 20930/256566 objects degraded (8.158%); 18156/256566 objects misplaced (7.077%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:51:37.625 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:37.647 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:37.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:37.748 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:37.771 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:37.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:37.872 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:37.897 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:37.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:37.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:37.999 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:38.022 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:38.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:38.123 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:38.145 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:38.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:38.247 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:38.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.269 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:38.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:38.370 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:38.393 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:38.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:38.495 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:38.518 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:38.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:38.620 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:38.645 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:38.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:38.747 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:38.768 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:38.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:38.870 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:38.891 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:38.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:38.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:38.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:39.018 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:39.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:39.120 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:39.147 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:39.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:39.236 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:39 smithi130 ceph-mon[105521]: pgmap v1720: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 20930/255483 objects degraded (8.192%); 17700/255483 objects misplaced (6.928%); 6.3 MiB/s, 101 objects/s recovering 2024-08-17T17:51:39.236 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:39 smithi130 ceph-mon[110068]: pgmap v1720: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 20930/255483 objects degraded (8.192%); 17700/255483 objects misplaced (6.928%); 6.3 MiB/s, 101 objects/s recovering 2024-08-17T17:51:39.248 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:39.269 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:39.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:39.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:39.393 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:39.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:39.495 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:39.517 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:39.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:39 smithi179 ceph-mon[93519]: pgmap v1720: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 20930/255483 objects degraded (8.192%); 17700/255483 objects misplaced (6.928%); 6.3 MiB/s, 101 objects/s recovering 2024-08-17T17:51:39.619 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:39.641 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:39.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:39.743 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:39.764 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:39.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:39.866 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:39.888 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:39.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:39.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:39.990 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:40.011 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:40.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:40.113 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:40.134 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:40.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:51:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:51:39] "GET /metrics HTTP/1.1" 200 36816 "" "Prometheus/2.51.0" 2024-08-17T17:51:40.236 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:40.260 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:40.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:40.362 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:40.388 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:40.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:40.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:40.511 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:40.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:40.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:51:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:51:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:51:40.613 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:40.634 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:40.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:40.736 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:40.757 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:40.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:40.858 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:40.880 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:40.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:40.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:40.982 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:41.002 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:40 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 20930/255483 objects degraded (8.192%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:41.002 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:40 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 20930/255483 objects degraded (8.192%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:41.010 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:41.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:40 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 20930/255483 objects degraded (8.192%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:41.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:41.135 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:41.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:41.237 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:41.259 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:41.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:41.262 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:51:41.262 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:51:41.361 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:41.382 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:41.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:41.483 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:41.504 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:41.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:41.606 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:41.634 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:41.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:41.724 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:51:41.735 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:41.764 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:41.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:41.866 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:41.892 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:41.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:41.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:41 smithi130 ceph-mon[105521]: pgmap v1721: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 20930/255342 objects degraded (8.197%); 17700/255342 objects misplaced (6.932%); 6.2 MiB/s, 99 objects/s recovering 2024-08-17T17:51:41.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:41 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2930445368' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:51:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:41 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:51:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:41 smithi130 ceph-mon[110068]: pgmap v1721: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 20930/255342 objects degraded (8.197%); 17700/255342 objects misplaced (6.932%); 6.2 MiB/s, 99 objects/s recovering 2024-08-17T17:51:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:41 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2930445368' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:51:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:41 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:51:41.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:42.016 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:42.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:41 smithi179 ceph-mon[93519]: pgmap v1721: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 20930/255342 objects degraded (8.197%); 17700/255342 objects misplaced (6.932%); 6.2 MiB/s, 99 objects/s recovering 2024-08-17T17:51:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:41 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2930445368' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:51:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:41 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:51:42.117 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:42.139 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:42.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:42.240 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:42.264 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:42.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:42.365 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:42.387 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:42.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:42.489 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:42.511 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:42.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:42.593 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:51:42.593 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:51:42.593 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:51:42.593 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:51:42.593 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:51:42.594 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:51:42.594 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:51:42.594 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:51:42.594 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:51:42.594 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:51:42.594 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:51:42.594 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:51:42.594 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:51:42.595 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:51:42.595 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:51:42.595 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:51:42.613 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:42.637 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:42.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:42.739 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:42.764 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:42.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:42.866 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:42.891 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:42.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:42.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:42.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:43.021 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:43.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:42 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:42 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:51:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:42 smithi179 ceph-mon[93519]: osdmap e1209: 8 total, 8 up, 6 in 2024-08-17T17:51:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:42 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1755409630' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:43.110 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:42 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:43.110 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:42 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:51:43.110 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:42 smithi130 ceph-mon[105521]: osdmap e1209: 8 total, 8 up, 6 in 2024-08-17T17:51:43.110 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:42 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1755409630' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:43.110 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:43.110 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:42 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:51:43.110 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:42 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:51:43.110 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:42 smithi130 ceph-mon[110068]: osdmap e1209: 8 total, 8 up, 6 in 2024-08-17T17:51:43.110 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:42 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1755409630' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:43.110 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:43.123 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:43.144 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:43.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:43.245 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:43.271 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:43.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:43.373 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:43.402 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:43.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:43.504 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:43.530 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:43.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:43.556 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:51:43.631 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:43.654 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:43.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:43.756 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:43.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.791 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:43.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:43.891 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:43.915 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:43.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:43.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:43 smithi130 ceph-mon[105521]: pgmap v1723: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 71 op/s; 20886/254979 objects degraded (8.191%); 17602/254979 objects misplaced (6.903%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:51:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:51:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:43 smithi130 ceph-mon[110068]: pgmap v1723: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 71 op/s; 20886/254979 objects degraded (8.191%); 17602/254979 objects misplaced (6.903%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:51:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:51:44.017 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:44.045 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:44.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:44.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:43 smithi179 ceph-mon[93519]: pgmap v1723: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 71 op/s; 20886/254979 objects degraded (8.191%); 17602/254979 objects misplaced (6.903%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:51:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:51:44.147 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:44.169 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:44.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:44.271 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:44.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.295 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:44.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:44.396 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:44.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:44.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:44.519 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:44.543 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:44.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:44.645 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:44.666 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:44.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:44.768 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:44.790 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:44.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:44.892 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:44.914 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:44.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:44.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:45.016 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:45.038 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:45.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:45.140 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:45.161 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:45.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:45.263 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:45.285 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:45.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:45.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:45 smithi179 ceph-mon[93519]: pgmap v1724: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 110 op/s; 20886/253566 objects degraded (8.237%); 17097/253566 objects misplaced (6.743%); 6.7 MiB/s, 106 objects/s recovering 2024-08-17T17:51:45.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:45 smithi130 ceph-mon[105521]: pgmap v1724: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 110 op/s; 20886/253566 objects degraded (8.237%); 17097/253566 objects misplaced (6.743%); 6.7 MiB/s, 106 objects/s recovering 2024-08-17T17:51:45.385 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:45 smithi130 ceph-mon[110068]: pgmap v1724: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 110 op/s; 20886/253566 objects degraded (8.237%); 17097/253566 objects misplaced (6.743%); 6.7 MiB/s, 106 objects/s recovering 2024-08-17T17:51:45.387 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:45.411 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:45.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:45.513 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:45.537 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:45.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:45.639 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:45.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.661 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:45.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:45.661 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:45.762 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:45.784 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:45.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:45.864 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:51:45.865 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:51:45.886 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:45.908 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:45.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:45.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:46.010 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:46.039 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:46.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:46.063 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:51:46.141 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:46.167 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:46.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:46.269 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:46.297 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:46.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:46.312 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 20886/253566 objects degraded (8.237%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:46.312 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:46 smithi130 ceph-mon[110068]: from='client.35487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:51:46.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 20886/253566 objects degraded (8.237%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:46.313 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:46 smithi130 ceph-mon[105521]: from='client.35487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:51:46.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 20886/253566 objects degraded (8.237%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:46.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:46 smithi179 ceph-mon[93519]: from='client.35487 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:51:46.399 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:46.422 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:46.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:46.523 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:46.548 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:46.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:46.649 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:46.672 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:46.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:46.773 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1724,"stamp":"2024-08-17T17:51:45.011005+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5538060261,"num_objects":84522,"num_object_clones":0,"num_object_copies":253566,"num_objects_missing_on_primary":4794,"num_objects_missing":4794,"num_objects_degraded":20886,"num_objects_misplaced":17097,"num_objects_unfound":0,"num_objects_dirty":84522,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":125392,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29102,"num_bytes_recovered":1799691277,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105995,"ondisk_log_size":105995,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":15430060,"kb_used_data":14100660,"kb_used_omap":295,"kb_used_meta":1329048,"kb_avail":547040852,"statfs":{"total":575970213888,"available":560169832448,"internally_reserved":0,"allocated":14439075840,"data_stored":14410246874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1360945615},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-72548352,"num_objects":-1107,"num_object_clones":0,"num_object_copies":-3321,"num_objects_missing_on_primary":-66,"num_objects_missing":-66,"num_objects_degraded":-90,"num_objects_misplaced":-1141,"num_objects_unfound":0,"num_objects_dirty":-1107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1107,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1070,"num_bytes_recovered":70123520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002765"},"pg_stats":[{"pgid":"45.f","version":"1209'7823","reported_seq":18319,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.908425+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:51:41.908425+0000","last_peered":"2024-08-17T17:51:41.908425+0000","last_clean":"2024-08-17T17:51:41.908425+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:51:41.908425+0000","last_undegraded":"2024-08-17T17:51:41.908425+0000","last_fullsized":"2024-08-17T17:51:41.908425+0000","mapping_epoch":1194,"log_start":"1170'931","ondisk_log_start":"1170'931","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6892,"log_dups_size":0,"ondisk_log_size":6892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350027776,"num_objects":5341,"num_object_clones":0,"num_object_copies":16023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7823,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1209'7773","reported_seq":11383,"reported_epoch":1209,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:51:43.160188+0000","last_change":"2024-08-17T17:51:24.127184+0000","last_active":"2024-08-17T17:51:43.160188+0000","last_peered":"2024-08-17T17:51:43.160188+0000","last_clean":"2024-08-17T17:48:38.634240+0000","last_became_active":"2024-08-17T17:48:41.062541+0000","last_became_peered":"2024-08-17T17:48:41.062541+0000","last_unstale":"2024-08-17T17:51:43.160188+0000","last_undegraded":"2024-08-17T17:48:40.667079+0000","last_fullsized":"2024-08-17T17:48:40.640181+0000","mapping_epoch":1179,"log_start":"1170'732","ondisk_log_start":"1170'732","created":1167,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7041,"log_dups_size":0,"ondisk_log_size":7041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:16.003483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342163456,"num_objects":5221,"num_object_clones":0,"num_object_copies":15663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1084,"num_objects_misplaced":2330,"num_objects_unfound":0,"num_objects_dirty":5221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7773,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1768,"num_bytes_recovered":115867648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"3,5,7","objects":2330},{"shards":"3,7","objects":1807}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1209'8015","reported_seq":18818,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:43.483116+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:51:43.483116+0000","last_peered":"2024-08-17T17:51:43.483116+0000","last_clean":"2024-08-17T17:51:43.483116+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:51:43.483116+0000","last_undegraded":"2024-08-17T17:51:43.483116+0000","last_fullsized":"2024-08-17T17:51:43.483116+0000","mapping_epoch":1205,"log_start":"1170'2068","ondisk_log_start":"1170'2068","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5947,"log_dups_size":0,"ondisk_log_size":5947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351404056,"num_objects":5363,"num_object_clones":0,"num_object_copies":16089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5363,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8015,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1209'7785","reported_seq":7835,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:43.028044+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:51:43.028044+0000","last_peered":"2024-08-17T17:51:43.028044+0000","last_clean":"2024-08-17T17:51:43.028044+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:51:43.028044+0000","last_undegraded":"2024-08-17T17:51:43.028044+0000","last_fullsized":"2024-08-17T17:51:43.028044+0000","mapping_epoch":1167,"log_start":"1170'1089","ondisk_log_start":"1170'1089","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6696,"log_dups_size":0,"ondisk_log_size":6696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341770240,"num_objects":5215,"num_object_clones":0,"num_object_copies":15645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7785,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1208'7851","reported_seq":13060,"reported_epoch":1209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:51:41.851535+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:51:41.851535+0000","last_peered":"2024-08-17T17:51:41.851535+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:51:41.851535+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1170'1926","ondisk_log_start":"1170'1926","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5925,"log_dups_size":0,"ondisk_log_size":5925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344915968,"num_objects":5263,"num_object_clones":0,"num_object_copies":15789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":5263,"num_objects_unfound":0,"num_objects_dirty":5263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7851,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":4745},{"shards":"1,4,5","objects":518}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1209'7856","reported_seq":7906,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:42.851742+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:51:42.851742+0000","last_peered":"2024-08-17T17:51:42.851742+0000","last_clean":"2024-08-17T17:51:42.851742+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:51:42.851742+0000","last_undegraded":"2024-08-17T17:51:42.851742+0000","last_fullsized":"2024-08-17T17:51:42.851742+0000","mapping_epoch":1167,"log_start":"1170'1177","ondisk_log_start":"1170'1177","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6679,"log_dups_size":0,"ondisk_log_size":6679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343277568,"num_objects":5238,"num_object_clones":0,"num_object_copies":15714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7856,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1209'7873","reported_seq":7923,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:42.953343+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:51:42.953343+0000","last_peered":"2024-08-17T17:51:42.953343+0000","last_clean":"2024-08-17T17:51:42.953343+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:51:42.953343+0000","last_undegraded":"2024-08-17T17:51:42.953343+0000","last_fullsized":"2024-08-17T17:51:42.953343+0000","mapping_epoch":1167,"log_start":"1170'1169","ondisk_log_start":"1170'1169","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6704,"log_dups_size":0,"ondisk_log_size":6704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350814208,"num_objects":5353,"num_object_clones":0,"num_object_copies":16059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7873,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1209'7753","reported_seq":11629,"reported_epoch":1209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:51:41.859614+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:51:41.859614+0000","last_peered":"2024-08-17T17:51:41.859614+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:51:41.859614+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1170'1775","ondisk_log_start":"1170'1775","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5978,"log_dups_size":0,"ondisk_log_size":5978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343212032,"num_objects":5237,"num_object_clones":0,"num_object_copies":15711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5767,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7753,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1258,"num_bytes_recovered":82444288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1850},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1209'7775","reported_seq":7835,"reported_epoch":1209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:51:42.945403+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:51:42.945403+0000","last_peered":"2024-08-17T17:51:42.945403+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:51:42.945403+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'1077","ondisk_log_start":"1170'1077","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6698,"log_dups_size":0,"ondisk_log_size":6698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347406336,"num_objects":5301,"num_object_clones":0,"num_object_copies":15903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7775,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":5301},{"shards":"1,6","objects":-532}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1209'7805","reported_seq":10847,"reported_epoch":1209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:51:43.633238+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:51:43.633238+0000","last_peered":"2024-08-17T17:51:43.633238+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:51:43.633238+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1170'1906","ondisk_log_start":"1170'1906","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5899,"log_dups_size":0,"ondisk_log_size":5899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351862784,"num_objects":5369,"num_object_clones":0,"num_object_copies":16107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7805,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":3808},{"shards":"1,3,5","objects":1561}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1209'7838","reported_seq":7936,"reported_epoch":1209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:51:44.064175+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:51:44.064175+0000","last_peered":"2024-08-17T17:51:44.064175+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:51:44.064175+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7838,"log_dups_size":0,"ondisk_log_size":7838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343146496,"num_objects":5236,"num_object_clones":0,"num_object_copies":15708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7838,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":83}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1209'7816","reported_seq":13974,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:43.843437+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:51:43.843437+0000","last_peered":"2024-08-17T17:51:43.843437+0000","last_clean":"2024-08-17T17:51:43.843437+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:51:43.843437+0000","last_undegraded":"2024-08-17T17:51:43.843437+0000","last_fullsized":"2024-08-17T17:51:43.843437+0000","mapping_epoch":1183,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7816,"log_dups_size":0,"ondisk_log_size":7816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342753280,"num_objects":5230,"num_object_clones":0,"num_object_copies":15690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7816,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1209'7811","reported_seq":7907,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:43.270704+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:51:43.270704+0000","last_peered":"2024-08-17T17:51:43.270704+0000","last_clean":"2024-08-17T17:51:43.270704+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:51:43.270704+0000","last_undegraded":"2024-08-17T17:51:43.270704+0000","last_fullsized":"2024-08-17T17:51:43.270704+0000","mapping_epoch":1167,"log_start":"1170'1891","ondisk_log_start":"1170'1891","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5920,"log_dups_size":0,"ondisk_log_size":5920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350552064,"num_objects":5349,"num_object_clones":0,"num_object_copies":16047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7811,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1209'7731","reported_seq":7781,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:43.024630+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:51:43.024630+0000","last_peered":"2024-08-17T17:51:43.024630+0000","last_clean":"2024-08-17T17:51:43.024630+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:51:43.024630+0000","last_undegraded":"2024-08-17T17:51:43.024630+0000","last_fullsized":"2024-08-17T17:51:43.024630+0000","mapping_epoch":1167,"log_start":"1170'725","ondisk_log_start":"1170'725","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7006,"log_dups_size":0,"ondisk_log_size":7006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340983808,"num_objects":5203,"num_object_clones":0,"num_object_copies":15609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7731,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1208'7927","reported_seq":15260,"reported_epoch":1209,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:51:41.840060+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:51:41.840060+0000","last_peered":"2024-08-17T17:51:41.840060+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:51:41.840060+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'1091","ondisk_log_start":"1170'1091","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6836,"log_dups_size":0,"ondisk_log_size":6836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356319232,"num_objects":5437,"num_object_clones":0,"num_object_copies":16311,"num_objects_missing_on_primary":4794,"num_objects_missing":4794,"num_objects_degraded":265,"num_objects_misplaced":9504,"num_objects_unfound":0,"num_objects_dirty":5437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7927,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3351,"num_bytes_recovered":219611136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":14},{"shards":"0,4,6","objects":1356},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":3802}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1209'7682","reported_seq":20965,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:43.020916+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:51:43.020916+0000","last_peered":"2024-08-17T17:51:43.020916+0000","last_clean":"2024-08-17T17:51:43.020916+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:51:43.020916+0000","last_undegraded":"2024-08-17T17:51:43.020916+0000","last_fullsized":"2024-08-17T17:51:43.020916+0000","mapping_epoch":1196,"log_start":"1170'1769","ondisk_log_start":"1170'1769","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5913,"log_dups_size":0,"ondisk_log_size":5913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336855040,"num_objects":5140,"num_object_clones":0,"num_object_copies":15420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7682,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1873,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790518+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:51:41.790518+0000","last_peered":"2024-08-17T17:51:41.790518+0000","last_clean":"2024-08-17T17:51:41.790518+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:51:41.790518+0000","last_undegraded":"2024-08-17T17:51:41.790518+0000","last_fullsized":"2024-08-17T17:51:41.790518+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":486,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.112688+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:51:34.112688+0000","last_peered":"2024-08-17T17:51:34.112688+0000","last_clean":"2024-08-17T17:51:34.112688+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:51:34.112688+0000","last_undegraded":"2024-08-17T17:51:34.112688+0000","last_fullsized":"2024-08-17T17:51:34.112688+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1792,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.877108+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:51:41.877108+0000","last_peered":"2024-08-17T17:51:41.877108+0000","last_clean":"2024-08-17T17:51:41.877108+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:51:41.877108+0000","last_undegraded":"2024-08-17T17:51:41.877108+0000","last_fullsized":"2024-08-17T17:51:41.877108+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1817,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876244+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:51:41.876244+0000","last_peered":"2024-08-17T17:51:41.876244+0000","last_clean":"2024-08-17T17:51:41.876244+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:51:41.876244+0000","last_undegraded":"2024-08-17T17:51:41.876244+0000","last_fullsized":"2024-08-17T17:51:41.876244+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":48,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113239+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:51:34.113239+0000","last_peered":"2024-08-17T17:51:34.113239+0000","last_clean":"2024-08-17T17:51:34.113239+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:51:34.113239+0000","last_undegraded":"2024-08-17T17:51:34.113239+0000","last_fullsized":"2024-08-17T17:51:34.113239+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1210,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876420+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:51:41.876420+0000","last_peered":"2024-08-17T17:51:41.876420+0000","last_clean":"2024-08-17T17:51:41.876420+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:51:41.876420+0000","last_undegraded":"2024-08-17T17:51:41.876420+0000","last_fullsized":"2024-08-17T17:51:41.876420+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":68,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850634+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:51:41.850634+0000","last_peered":"2024-08-17T17:51:41.850634+0000","last_clean":"2024-08-17T17:51:41.850634+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:51:41.850634+0000","last_undegraded":"2024-08-17T17:51:41.850634+0000","last_fullsized":"2024-08-17T17:51:41.850634+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":588,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876718+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:51:41.876718+0000","last_peered":"2024-08-17T17:51:41.876718+0000","last_clean":"2024-08-17T17:51:41.876718+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:51:41.876718+0000","last_undegraded":"2024-08-17T17:51:41.876718+0000","last_fullsized":"2024-08-17T17:51:41.876718+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1832,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790565+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:51:41.790565+0000","last_peered":"2024-08-17T17:51:41.790565+0000","last_clean":"2024-08-17T17:51:41.790565+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:51:41.790565+0000","last_undegraded":"2024-08-17T17:51:41.790565+0000","last_fullsized":"2024-08-17T17:51:41.790565+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1834,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.865155+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:51:41.865155+0000","last_peered":"2024-08-17T17:51:41.865155+0000","last_clean":"2024-08-17T17:51:41.865155+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:51:41.865155+0000","last_undegraded":"2024-08-17T17:51:41.865155+0000","last_fullsized":"2024-08-17T17:51:41.865155+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":53,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.851213+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:51:41.851213+0000","last_peered":"2024-08-17T17:51:41.851213+0000","last_clean":"2024-08-17T17:51:41.851213+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:51:41.851213+0000","last_undegraded":"2024-08-17T17:51:41.851213+0000","last_fullsized":"2024-08-17T17:51:41.851213+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1805,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.877738+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:51:41.877738+0000","last_peered":"2024-08-17T17:51:41.877738+0000","last_clean":"2024-08-17T17:51:41.877738+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:51:41.877738+0000","last_undegraded":"2024-08-17T17:51:41.877738+0000","last_fullsized":"2024-08-17T17:51:41.877738+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1860,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850309+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:51:41.850309+0000","last_peered":"2024-08-17T17:51:41.850309+0000","last_clean":"2024-08-17T17:51:41.850309+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:51:41.850309+0000","last_undegraded":"2024-08-17T17:51:41.850309+0000","last_fullsized":"2024-08-17T17:51:41.850309+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1854,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790711+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:51:41.790711+0000","last_peered":"2024-08-17T17:51:41.790711+0000","last_clean":"2024-08-17T17:51:41.790711+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:51:41.790711+0000","last_undegraded":"2024-08-17T17:51:41.790711+0000","last_fullsized":"2024-08-17T17:51:41.790711+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1828,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876543+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:51:41.876543+0000","last_peered":"2024-08-17T17:51:41.876543+0000","last_clean":"2024-08-17T17:51:41.876543+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:51:41.876543+0000","last_undegraded":"2024-08-17T17:51:41.876543+0000","last_fullsized":"2024-08-17T17:51:41.876543+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1183,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113334+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:51:34.113334+0000","last_peered":"2024-08-17T17:51:34.113334+0000","last_clean":"2024-08-17T17:51:34.113334+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:51:34.113334+0000","last_undegraded":"2024-08-17T17:51:34.113334+0000","last_fullsized":"2024-08-17T17:51:34.113334+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1799,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876247+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:51:41.876247+0000","last_peered":"2024-08-17T17:51:41.876247+0000","last_clean":"2024-08-17T17:51:41.876247+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:51:41.876247+0000","last_undegraded":"2024-08-17T17:51:41.876247+0000","last_fullsized":"2024-08-17T17:51:41.876247+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1849,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790622+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:51:41.790622+0000","last_peered":"2024-08-17T17:51:41.790622+0000","last_clean":"2024-08-17T17:51:41.790622+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:51:41.790622+0000","last_undegraded":"2024-08-17T17:51:41.790622+0000","last_fullsized":"2024-08-17T17:51:41.790622+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1755,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876240+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:51:41.876240+0000","last_peered":"2024-08-17T17:51:41.876240+0000","last_clean":"2024-08-17T17:51:41.876240+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:51:41.876240+0000","last_undegraded":"2024-08-17T17:51:41.876240+0000","last_fullsized":"2024-08-17T17:51:41.876240+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":502,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.125078+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:51:34.125078+0000","last_peered":"2024-08-17T17:51:34.125078+0000","last_clean":"2024-08-17T17:51:34.125078+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:51:34.125078+0000","last_undegraded":"2024-08-17T17:51:34.125078+0000","last_fullsized":"2024-08-17T17:51:34.125078+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1179,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124870+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:51:34.124870+0000","last_peered":"2024-08-17T17:51:34.124870+0000","last_clean":"2024-08-17T17:51:34.124870+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:51:34.124870+0000","last_undegraded":"2024-08-17T17:51:34.124870+0000","last_fullsized":"2024-08-17T17:51:34.124870+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1854,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.851335+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:51:41.851335+0000","last_peered":"2024-08-17T17:51:41.851335+0000","last_clean":"2024-08-17T17:51:41.851335+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:51:41.851335+0000","last_undegraded":"2024-08-17T17:51:41.851335+0000","last_fullsized":"2024-08-17T17:51:41.851335+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":340,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124979+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:51:34.124979+0000","last_peered":"2024-08-17T17:51:34.124979+0000","last_clean":"2024-08-17T17:51:34.124979+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:51:34.124979+0000","last_undegraded":"2024-08-17T17:51:34.124979+0000","last_fullsized":"2024-08-17T17:51:34.124979+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":486,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113079+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:51:34.113079+0000","last_peered":"2024-08-17T17:51:34.113079+0000","last_clean":"2024-08-17T17:51:34.113079+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:51:34.113079+0000","last_undegraded":"2024-08-17T17:51:34.113079+0000","last_fullsized":"2024-08-17T17:51:34.113079+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1164,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113040+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:51:34.113040+0000","last_peered":"2024-08-17T17:51:34.113040+0000","last_clean":"2024-08-17T17:51:34.113040+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:51:34.113040+0000","last_undegraded":"2024-08-17T17:51:34.113040+0000","last_fullsized":"2024-08-17T17:51:34.113040+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1172,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113406+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:51:34.113406+0000","last_peered":"2024-08-17T17:51:34.113406+0000","last_clean":"2024-08-17T17:51:34.113406+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:51:34.113406+0000","last_undegraded":"2024-08-17T17:51:34.113406+0000","last_fullsized":"2024-08-17T17:51:34.113406+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1901,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790481+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:51:41.790481+0000","last_peered":"2024-08-17T17:51:41.790481+0000","last_clean":"2024-08-17T17:51:41.790481+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:51:41.790481+0000","last_undegraded":"2024-08-17T17:51:41.790481+0000","last_fullsized":"2024-08-17T17:51:41.790481+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1784,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876478+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:51:41.876478+0000","last_peered":"2024-08-17T17:51:41.876478+0000","last_clean":"2024-08-17T17:51:41.876478+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:51:41.876478+0000","last_undegraded":"2024-08-17T17:51:41.876478+0000","last_fullsized":"2024-08-17T17:51:41.876478+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1859,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850724+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:51:41.850724+0000","last_peered":"2024-08-17T17:51:41.850724+0000","last_clean":"2024-08-17T17:51:41.850724+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:51:41.850724+0000","last_undegraded":"2024-08-17T17:51:41.850724+0000","last_fullsized":"2024-08-17T17:51:41.850724+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1787,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.877243+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:51:41.877243+0000","last_peered":"2024-08-17T17:51:41.877243+0000","last_clean":"2024-08-17T17:51:41.877243+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:51:41.877243+0000","last_undegraded":"2024-08-17T17:51:41.877243+0000","last_fullsized":"2024-08-17T17:51:41.877243+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1819,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790437+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:51:41.790437+0000","last_peered":"2024-08-17T17:51:41.790437+0000","last_clean":"2024-08-17T17:51:41.790437+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:51:41.790437+0000","last_undegraded":"2024-08-17T17:51:41.790437+0000","last_fullsized":"2024-08-17T17:51:41.790437+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1199,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.125017+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:51:34.125017+0000","last_peered":"2024-08-17T17:51:34.125017+0000","last_clean":"2024-08-17T17:51:34.125017+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:51:34.125017+0000","last_undegraded":"2024-08-17T17:51:34.125017+0000","last_fullsized":"2024-08-17T17:51:34.125017+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1866,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850824+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:51:41.850824+0000","last_peered":"2024-08-17T17:51:41.850824+0000","last_clean":"2024-08-17T17:51:41.850824+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:51:41.850824+0000","last_undegraded":"2024-08-17T17:51:41.850824+0000","last_fullsized":"2024-08-17T17:51:41.850824+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":52,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.851153+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:51:41.851153+0000","last_peered":"2024-08-17T17:51:41.851153+0000","last_clean":"2024-08-17T17:51:41.851153+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:51:41.851153+0000","last_undegraded":"2024-08-17T17:51:41.851153+0000","last_fullsized":"2024-08-17T17:51:41.851153+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1885,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850731+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:51:41.850731+0000","last_peered":"2024-08-17T17:51:41.850731+0000","last_clean":"2024-08-17T17:51:41.850731+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:51:41.850731+0000","last_undegraded":"2024-08-17T17:51:41.850731+0000","last_fullsized":"2024-08-17T17:51:41.850731+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":621,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876994+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:51:41.876994+0000","last_peered":"2024-08-17T17:51:41.876994+0000","last_clean":"2024-08-17T17:51:41.876994+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:51:41.876994+0000","last_undegraded":"2024-08-17T17:51:41.876994+0000","last_fullsized":"2024-08-17T17:51:41.876994+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1866,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850901+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:51:41.850901+0000","last_peered":"2024-08-17T17:51:41.850901+0000","last_clean":"2024-08-17T17:51:41.850901+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:51:41.850901+0000","last_undegraded":"2024-08-17T17:51:41.850901+0000","last_fullsized":"2024-08-17T17:51:41.850901+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1194,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790203+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:51:41.790203+0000","last_peered":"2024-08-17T17:51:41.790203+0000","last_clean":"2024-08-17T17:51:41.790203+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:51:41.790203+0000","last_undegraded":"2024-08-17T17:51:41.790203+0000","last_fullsized":"2024-08-17T17:51:41.790203+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":112,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.851288+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:51:41.851288+0000","last_peered":"2024-08-17T17:51:41.851288+0000","last_clean":"2024-08-17T17:51:41.851288+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:51:41.851288+0000","last_undegraded":"2024-08-17T17:51:41.851288+0000","last_fullsized":"2024-08-17T17:51:41.851288+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1876,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850877+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:51:41.850877+0000","last_peered":"2024-08-17T17:51:41.850877+0000","last_clean":"2024-08-17T17:51:41.850877+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:51:41.850877+0000","last_undegraded":"2024-08-17T17:51:41.850877+0000","last_fullsized":"2024-08-17T17:51:41.850877+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1854,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850170+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:51:41.850170+0000","last_peered":"2024-08-17T17:51:41.850170+0000","last_clean":"2024-08-17T17:51:41.850170+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:51:41.850170+0000","last_undegraded":"2024-08-17T17:51:41.850170+0000","last_fullsized":"2024-08-17T17:51:41.850170+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1874,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.864874+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:51:41.864874+0000","last_peered":"2024-08-17T17:51:41.864874+0000","last_clean":"2024-08-17T17:51:41.864874+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:51:41.864874+0000","last_undegraded":"2024-08-17T17:51:41.864874+0000","last_fullsized":"2024-08-17T17:51:41.864874+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1880,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790758+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:51:41.790758+0000","last_peered":"2024-08-17T17:51:41.790758+0000","last_clean":"2024-08-17T17:51:41.790758+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:51:41.790758+0000","last_undegraded":"2024-08-17T17:51:41.790758+0000","last_fullsized":"2024-08-17T17:51:41.790758+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1860,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.833186+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:51:41.833186+0000","last_peered":"2024-08-17T17:51:41.833186+0000","last_clean":"2024-08-17T17:51:41.833186+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:51:41.833186+0000","last_undegraded":"2024-08-17T17:51:41.833186+0000","last_fullsized":"2024-08-17T17:51:41.833186+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1206,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124907+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:51:34.124907+0000","last_peered":"2024-08-17T17:51:34.124907+0000","last_clean":"2024-08-17T17:51:34.124907+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:51:34.124907+0000","last_undegraded":"2024-08-17T17:51:34.124907+0000","last_fullsized":"2024-08-17T17:51:34.124907+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1152,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113230+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:51:34.113230+0000","last_peered":"2024-08-17T17:51:34.113230+0000","last_clean":"2024-08-17T17:51:34.113230+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:51:34.113230+0000","last_undegraded":"2024-08-17T17:51:34.113230+0000","last_fullsized":"2024-08-17T17:51:34.113230+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1769,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876358+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:51:41.876358+0000","last_peered":"2024-08-17T17:51:41.876358+0000","last_clean":"2024-08-17T17:51:41.876358+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:51:41.876358+0000","last_undegraded":"2024-08-17T17:51:41.876358+0000","last_fullsized":"2024-08-17T17:51:41.876358+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":100,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.865829+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:51:41.865829+0000","last_peered":"2024-08-17T17:51:41.865829+0000","last_clean":"2024-08-17T17:51:41.865829+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:51:41.865829+0000","last_undegraded":"2024-08-17T17:51:41.865829+0000","last_fullsized":"2024-08-17T17:51:41.865829+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1518,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124676+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:51:34.124676+0000","last_peered":"2024-08-17T17:51:34.124676+0000","last_clean":"2024-08-17T17:51:34.124676+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:51:34.124676+0000","last_undegraded":"2024-08-17T17:51:34.124676+0000","last_fullsized":"2024-08-17T17:51:34.124676+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1861,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.851009+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:51:41.851009+0000","last_peered":"2024-08-17T17:51:41.851009+0000","last_clean":"2024-08-17T17:51:41.851009+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:51:41.851009+0000","last_undegraded":"2024-08-17T17:51:41.851009+0000","last_fullsized":"2024-08-17T17:51:41.851009+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1801,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876355+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:51:41.876355+0000","last_peered":"2024-08-17T17:51:41.876355+0000","last_clean":"2024-08-17T17:51:41.876355+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:51:41.876355+0000","last_undegraded":"2024-08-17T17:51:41.876355+0000","last_fullsized":"2024-08-17T17:51:41.876355+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":486,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113200+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:51:34.113200+0000","last_peered":"2024-08-17T17:51:34.113200+0000","last_clean":"2024-08-17T17:51:34.113200+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:51:34.113200+0000","last_undegraded":"2024-08-17T17:51:34.113200+0000","last_fullsized":"2024-08-17T17:51:34.113200+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":539,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.864674+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:51:41.864674+0000","last_peered":"2024-08-17T17:51:41.864674+0000","last_clean":"2024-08-17T17:51:41.864674+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:51:41.864674+0000","last_undegraded":"2024-08-17T17:51:41.864674+0000","last_fullsized":"2024-08-17T17:51:41.864674+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":555,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.864847+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:51:41.864847+0000","last_peered":"2024-08-17T17:51:41.864847+0000","last_clean":"2024-08-17T17:51:41.864847+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:51:41.864847+0000","last_undegraded":"2024-08-17T17:51:41.864847+0000","last_fullsized":"2024-08-17T17:51:41.864847+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1170,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113060+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:51:34.113060+0000","last_peered":"2024-08-17T17:51:34.113060+0000","last_clean":"2024-08-17T17:51:34.113060+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:51:34.113060+0000","last_undegraded":"2024-08-17T17:51:34.113060+0000","last_fullsized":"2024-08-17T17:51:34.113060+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":571,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876780+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:51:41.876780+0000","last_peered":"2024-08-17T17:51:41.876780+0000","last_clean":"2024-08-17T17:51:41.876780+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:51:41.876780+0000","last_undegraded":"2024-08-17T17:51:41.876780+0000","last_fullsized":"2024-08-17T17:51:41.876780+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1797,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876808+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:51:41.876808+0000","last_peered":"2024-08-17T17:51:41.876808+0000","last_clean":"2024-08-17T17:51:41.876808+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:51:41.876808+0000","last_undegraded":"2024-08-17T17:51:41.876808+0000","last_fullsized":"2024-08-17T17:51:41.876808+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1893,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790525+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:51:41.790525+0000","last_peered":"2024-08-17T17:51:41.790525+0000","last_clean":"2024-08-17T17:51:41.790525+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:51:41.790525+0000","last_undegraded":"2024-08-17T17:51:41.790525+0000","last_fullsized":"2024-08-17T17:51:41.790525+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":604,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876694+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:51:41.876694+0000","last_peered":"2024-08-17T17:51:41.876694+0000","last_clean":"2024-08-17T17:51:41.876694+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:51:41.876694+0000","last_undegraded":"2024-08-17T17:51:41.876694+0000","last_fullsized":"2024-08-17T17:51:41.876694+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1159,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.112931+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:51:34.112931+0000","last_peered":"2024-08-17T17:51:34.112931+0000","last_clean":"2024-08-17T17:51:34.112931+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:51:34.112931+0000","last_undegraded":"2024-08-17T17:51:34.112931+0000","last_fullsized":"2024-08-17T17:51:34.112931+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1158,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124959+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:51:34.124959+0000","last_peered":"2024-08-17T17:51:34.124959+0000","last_clean":"2024-08-17T17:51:34.124959+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:51:34.124959+0000","last_undegraded":"2024-08-17T17:51:34.124959+0000","last_fullsized":"2024-08-17T17:51:34.124959+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":588,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850420+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:51:41.850420+0000","last_peered":"2024-08-17T17:51:41.850420+0000","last_clean":"2024-08-17T17:51:41.850420+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:51:41.850420+0000","last_undegraded":"2024-08-17T17:51:41.850420+0000","last_fullsized":"2024-08-17T17:51:41.850420+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1176,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.125254+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:51:34.125254+0000","last_peered":"2024-08-17T17:51:34.125254+0000","last_clean":"2024-08-17T17:51:34.125254+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:51:34.125254+0000","last_undegraded":"2024-08-17T17:51:34.125254+0000","last_fullsized":"2024-08-17T17:51:34.125254+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":60,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124592+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:51:34.124592+0000","last_peered":"2024-08-17T17:51:34.124592+0000","last_clean":"2024-08-17T17:51:34.124592+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:51:34.124592+0000","last_undegraded":"2024-08-17T17:51:34.124592+0000","last_fullsized":"2024-08-17T17:51:34.124592+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1192,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.125129+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:51:34.125129+0000","last_peered":"2024-08-17T17:51:34.125129+0000","last_clean":"2024-08-17T17:51:34.125129+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:51:34.125129+0000","last_undegraded":"2024-08-17T17:51:34.125129+0000","last_fullsized":"2024-08-17T17:51:34.125129+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1209,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.112820+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:51:34.112820+0000","last_peered":"2024-08-17T17:51:34.112820+0000","last_clean":"2024-08-17T17:51:34.112820+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:51:34.112820+0000","last_undegraded":"2024-08-17T17:51:34.112820+0000","last_fullsized":"2024-08-17T17:51:34.112820+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1799,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.877190+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:51:41.877190+0000","last_peered":"2024-08-17T17:51:41.877190+0000","last_clean":"2024-08-17T17:51:41.877190+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:51:41.877190+0000","last_undegraded":"2024-08-17T17:51:41.877190+0000","last_fullsized":"2024-08-17T17:51:41.877190+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1153,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.125058+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:51:34.125058+0000","last_peered":"2024-08-17T17:51:34.125058+0000","last_clean":"2024-08-17T17:51:34.125058+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:51:34.125058+0000","last_undegraded":"2024-08-17T17:51:34.125058+0000","last_fullsized":"2024-08-17T17:51:34.125058+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1852,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790584+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:51:41.790584+0000","last_peered":"2024-08-17T17:51:41.790584+0000","last_clean":"2024-08-17T17:51:41.790584+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:51:41.790584+0000","last_undegraded":"2024-08-17T17:51:41.790584+0000","last_fullsized":"2024-08-17T17:51:41.790584+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1853,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790291+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:51:41.790291+0000","last_peered":"2024-08-17T17:51:41.790291+0000","last_clean":"2024-08-17T17:51:41.790291+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:51:41.790291+0000","last_undegraded":"2024-08-17T17:51:41.790291+0000","last_fullsized":"2024-08-17T17:51:41.790291+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":566,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124801+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:51:34.124801+0000","last_peered":"2024-08-17T17:51:34.124801+0000","last_clean":"2024-08-17T17:51:34.124801+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:51:34.124801+0000","last_undegraded":"2024-08-17T17:51:34.124801+0000","last_fullsized":"2024-08-17T17:51:34.124801+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1178,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124974+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:51:34.124974+0000","last_peered":"2024-08-17T17:51:34.124974+0000","last_clean":"2024-08-17T17:51:34.124974+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:51:34.124974+0000","last_undegraded":"2024-08-17T17:51:34.124974+0000","last_fullsized":"2024-08-17T17:51:34.124974+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1799,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876810+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:51:41.876810+0000","last_peered":"2024-08-17T17:51:41.876810+0000","last_clean":"2024-08-17T17:51:41.876810+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:51:41.876810+0000","last_undegraded":"2024-08-17T17:51:41.876810+0000","last_fullsized":"2024-08-17T17:51:41.876810+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":5537464344,"num_objects":84496,"num_object_clones":0,"num_object_copies":253488,"num_objects_missing_on_primary":4794,"num_objects_missing":4794,"num_objects_degraded":20886,"num_objects_misplaced":17097,"num_objects_unfound":0,"num_objects_dirty":84496,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":125114,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29073,"num_bytes_recovered":1797914624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17099206656,"data_stored":17099194440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105788,"ondisk_log_size":105788,"up":48,"acting":41,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234370,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2564484,"kb_used_data":2254648,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91180668,"statfs":{"total":95995035648,"available":93369004032,"internally_reserved":0,"allocated":2308759552,"data_stored":2303969129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496005,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2926408,"kb_used_data":2616572,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":90818744,"statfs":{"total":95995035648,"available":92998393856,"internally_reserved":0,"allocated":2679369728,"data_stored":2674575710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757643,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326257,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3214444,"kb_used_data":3020196,"kb_used_omap":57,"kb_used_meta":194182,"kb_avail":90530708,"statfs":{"total":95995035648,"available":92703444992,"internally_reserved":0,"allocated":3092680704,"data_stored":3087905461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":198843131},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405496,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2893300,"kb_used_data":2703148,"kb_used_omap":61,"kb_used_meta":190082,"kb_avail":90851852,"statfs":{"total":95995035648,"available":93032296448,"internally_reserved":0,"allocated":2768023552,"data_stored":2763245281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":194644920},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452030,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2208472,"kb_used_data":2042000,"kb_used_omap":54,"kb_used_meta":166409,"kb_avail":91536680,"statfs":{"total":95995035648,"available":93733560320,"internally_reserved":0,"allocated":2091008000,"data_stored":2086244125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":170403214},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474722,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1622952,"kb_used_data":1464096,"kb_used_omap":119,"kb_used_meta":158728,"kb_avail":92122200,"statfs":{"total":95995035648,"available":94333132800,"internally_reserved":0,"allocated":1499234304,"data_stored":1494307168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":162538206},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019278,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1489113088,"data_stored":1489109016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2077622272,"data_stored":2077622272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2751463424,"data_stored":2751463424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3077246976,"data_stored":3077242904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1116798976,"data_stored":1116798976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1630404608,"data_stored":1630404608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2664566784,"data_stored":2664562712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2291990528,"data_stored":2291990528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:51:46.778 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:51:46.778 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:46.801 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:46.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:46.903 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:46.924 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:46.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:46.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:47.026 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:47.048 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:47.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:47.149 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:51:47.150 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:47.172 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:47.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:47.274 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:47.299 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:47.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:47.361 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:47 smithi130 ceph-mon[105521]: pgmap v1725: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 20786/253068 objects degraded (8.214%); 16990/253068 objects misplaced (6.714%); 6.9 MiB/s, 109 objects/s recovering 2024-08-17T17:51:47.362 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:47 smithi130 ceph-mon[110068]: pgmap v1725: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 20786/253068 objects degraded (8.214%); 16990/253068 objects misplaced (6.714%); 6.9 MiB/s, 109 objects/s recovering 2024-08-17T17:51:47.401 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:47.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.423 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:47.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:47.524 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:47.549 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:47 smithi179 ceph-mon[93519]: pgmap v1725: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 20786/253068 objects degraded (8.214%); 16990/253068 objects misplaced (6.714%); 6.9 MiB/s, 109 objects/s recovering 2024-08-17T17:51:47.551 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:47.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:47.653 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:47.675 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:47.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:47.777 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:47.802 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:47.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:47.904 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:47.928 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:47.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:47.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:48.002 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:48.030 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:48.052 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:48.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:48.153 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:48.178 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:48.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:48.280 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:48.303 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:48.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:48.405 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:48.431 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:48.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:48.533 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:48.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.555 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:48.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:48.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:48.677 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:48.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:48.778 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:48.802 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:48.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:48.904 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:48.925 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:48.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:48.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:49.027 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:49.049 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:49.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:49.151 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:49.175 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:49.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:49.277 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:49.298 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:49.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:49.341 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:49 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:51:49.341 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:49 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:51:49.341 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:49 smithi130 ceph-mon[105521]: pgmap v1726: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 20786/252072 objects degraded (8.246%); 16480/252072 objects misplaced (6.538%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:51:49.341 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:49 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:51:49.341 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:49 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:51:49.342 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:49 smithi130 ceph-mon[110068]: pgmap v1726: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 20786/252072 objects degraded (8.246%); 16480/252072 objects misplaced (6.538%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:51:49.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:49 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:51:49.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:49 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:51:49.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:49 smithi179 ceph-mon[93519]: pgmap v1726: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 20786/252072 objects degraded (8.246%); 16480/252072 objects misplaced (6.538%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:51:49.400 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:49.422 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:49.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:49.523 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:49.544 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:49.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:49.646 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:49.671 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:49.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:49.773 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:49.795 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:49.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:49.897 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:49.926 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:49.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:49.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:50.028 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:50.055 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:50.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:50.157 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:51:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:51:49] "GET /metrics HTTP/1.1" 200 36813 "" "Prometheus/2.51.0" 2024-08-17T17:51:50.180 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:50.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:50.281 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:50.303 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:50.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:50.405 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:50.427 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:50.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:50.529 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:50.551 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:50.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:50.577 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:51:50.577 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:51:50.577 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 0 2024-08-17T17:51:50.577 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:51:50.584 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:50.584 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:51:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:51:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:51:50.653 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:50.676 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:50.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:50.778 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:50.805 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:50.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:50.895 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:50 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 20786/252072 objects degraded (8.246%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:50.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:50 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 20786/252072 objects degraded (8.246%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:50.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:50.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:50.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:50.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:51.029 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:51.056 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:51.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:51.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:50 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 20786/252072 objects degraded (8.246%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:51.158 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:51.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.181 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:51.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:51.282 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:51.309 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:51.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:51.411 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:51.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.433 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:51.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:51.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:51.561 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:51.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:51.663 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:51.690 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:51.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:51.792 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:51.815 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:51.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:51.818 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:51:51.917 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:51.937 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:51.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:51.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:52.018 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:51 smithi130 ceph-mon[105521]: pgmap v1727: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 20786/251898 objects degraded (8.252%); 16480/251898 objects misplaced (6.542%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:51:52.018 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:51 smithi130 ceph-mon[110068]: pgmap v1727: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 20786/251898 objects degraded (8.252%); 16480/251898 objects misplaced (6.542%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:51:52.039 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:52.061 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:52.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:51 smithi179 ceph-mon[93519]: pgmap v1727: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 20786/251898 objects degraded (8.252%); 16480/251898 objects misplaced (6.542%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:51:52.162 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:52.184 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:52.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:52.286 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:52.307 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:52.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:52.409 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:52.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.431 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:52.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:52.532 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:52.553 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:52.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:52.655 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:52.677 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:52.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:52.779 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:52.802 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:52.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:52.904 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:52.926 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:52.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:53.027 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:53.055 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:53.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:53.157 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:53.179 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:53.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:53.281 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:53.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:53 smithi130 ceph-mon[110068]: pgmap v1728: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 20702/251481 objects degraded (8.232%); 16381/251481 objects misplaced (6.514%); 6.8 MiB/s, 108 objects/s recovering 2024-08-17T17:51:53.287 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:53 smithi130 ceph-mon[105521]: pgmap v1728: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 20702/251481 objects degraded (8.232%); 16381/251481 objects misplaced (6.514%); 6.8 MiB/s, 108 objects/s recovering 2024-08-17T17:51:53.307 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:53.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:53.408 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:53.430 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:53.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:53.532 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:53.553 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:53.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:53.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:53 smithi179 ceph-mon[93519]: pgmap v1728: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 20702/251481 objects degraded (8.232%); 16381/251481 objects misplaced (6.514%); 6.8 MiB/s, 108 objects/s recovering 2024-08-17T17:51:53.654 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:53.676 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:53.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:53.778 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:53.804 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:53.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:53.906 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:53.930 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:53.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:53.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:54.032 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:54.053 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:54.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:54.154 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:54.177 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:54.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:54.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:54.301 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:54.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:54.402 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:54.424 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:54.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:54.526 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:54.550 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:54.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:54.652 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:54.677 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:54.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:54.779 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:54.801 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:54.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:54.903 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:54.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:54.925 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:54.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:54.969 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4100627036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:54.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4100627036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:55.027 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:55.048 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:55.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:55.149 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:55.151 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:51:55.151 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:51:55.151 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:51:55.151 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:51:55.151 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:51:55.151 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:51:55.152 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:51:55.152 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:51:55.152 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:51:55.152 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:51:55.152 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:51:55.152 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:51:55.152 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:51:55.152 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:51:55.153 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:51:55.153 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:51:55.159 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:51:55.160 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:51:55.178 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:55.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:55.280 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:55.301 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:55.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:55.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4100627036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:51:55.403 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:55.425 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:55.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:55.527 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:55.548 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:55.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:55.650 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:55.674 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:55.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:55.776 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:55.800 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:55.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:55.901 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:55.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:55.924 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:55.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:56.025 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:56.047 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:56.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:56.148 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:56.162 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:55 smithi179 ceph-mon[93519]: pgmap v1729: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 20702/250191 objects degraded (8.274%); 15740/250191 objects misplaced (6.291%); 9.4 MiB/s, 150 objects/s recovering 2024-08-17T17:51:56.162 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:55 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 20702/251481 objects degraded (8.232%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:56.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:55 smithi130 ceph-mon[105521]: pgmap v1729: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 20702/250191 objects degraded (8.274%); 15740/250191 objects misplaced (6.291%); 9.4 MiB/s, 150 objects/s recovering 2024-08-17T17:51:56.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:55 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 20702/251481 objects degraded (8.232%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:55 smithi130 ceph-mon[110068]: pgmap v1729: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 20702/250191 objects degraded (8.274%); 15740/250191 objects misplaced (6.291%); 9.4 MiB/s, 150 objects/s recovering 2024-08-17T17:51:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:55 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 20702/251481 objects degraded (8.232%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:51:56.170 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:56.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:56.268 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1729,"stamp":"2024-08-17T17:51:55.013825+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5464332261,"num_objects":83397,"num_object_clones":0,"num_object_copies":250191,"num_objects_missing_on_primary":4730,"num_objects_missing":4730,"num_objects_degraded":20702,"num_objects_misplaced":15740,"num_objects_unfound":0,"num_objects_dirty":83397,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":126517,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30409,"num_bytes_recovered":1885346829,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106320,"ondisk_log_size":106320,"up":267,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":15329940,"kb_used_data":13983908,"kb_used_omap":295,"kb_used_meta":1345688,"kb_avail":547140972,"statfs":{"total":575970213888,"available":560272355328,"internally_reserved":0,"allocated":14319521792,"data_stored":14290788744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1377984975},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,0,0,1,1,1,2,4,6,3],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-104595456,"num_objects":-1596,"num_object_clones":0,"num_object_copies":-4788,"num_objects_missing_on_primary":-64,"num_objects_missing":-64,"num_objects_degraded":-184,"num_objects_misplaced":-1862,"num_objects_unfound":0,"num_objects_dirty":-1596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1596,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1812,"num_bytes_recovered":118751232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003472"},"pg_stats":[{"pgid":"45.f","version":"1209'7895","reported_seq":18391,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:51.739136+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:51:51.739136+0000","last_peered":"2024-08-17T17:51:51.739136+0000","last_clean":"2024-08-17T17:51:51.739136+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:51:51.739136+0000","last_undegraded":"2024-08-17T17:51:51.739136+0000","last_fullsized":"2024-08-17T17:51:51.739136+0000","mapping_epoch":1194,"log_start":"1170'1031","ondisk_log_start":"1170'1031","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6864,"log_dups_size":0,"ondisk_log_size":6864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345309184,"num_objects":5269,"num_object_clones":0,"num_object_copies":15807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7895,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1209'7839","reported_seq":13751,"reported_epoch":1209,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:51:53.167471+0000","last_change":"2024-08-17T17:51:24.127184+0000","last_active":"2024-08-17T17:51:53.167471+0000","last_peered":"2024-08-17T17:51:53.167471+0000","last_clean":"2024-08-17T17:48:38.634240+0000","last_became_active":"2024-08-17T17:48:41.062541+0000","last_became_peered":"2024-08-17T17:48:41.062541+0000","last_unstale":"2024-08-17T17:51:53.167471+0000","last_undegraded":"2024-08-17T17:48:40.667079+0000","last_fullsized":"2024-08-17T17:48:40.640181+0000","mapping_epoch":1179,"log_start":"1170'832","ondisk_log_start":"1170'832","created":1167,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7007,"log_dups_size":0,"ondisk_log_size":7007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:16.003483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337838080,"num_objects":5155,"num_object_clones":0,"num_object_copies":15465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1084,"num_objects_misplaced":1179,"num_objects_unfound":0,"num_objects_dirty":5155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7839,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2919,"num_bytes_recovered":191299584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"3,5,7","objects":1179},{"shards":"3,7","objects":2892}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1209'8099","reported_seq":18902,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:53.426143+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:51:53.426143+0000","last_peered":"2024-08-17T17:51:53.426143+0000","last_clean":"2024-08-17T17:51:53.426143+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:51:53.426143+0000","last_undegraded":"2024-08-17T17:51:53.426143+0000","last_fullsized":"2024-08-17T17:51:53.426143+0000","mapping_epoch":1205,"log_start":"1170'2168","ondisk_log_start":"1170'2168","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5931,"log_dups_size":0,"ondisk_log_size":5931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345899032,"num_objects":5279,"num_object_clones":0,"num_object_copies":15837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5279,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8099,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1209'7859","reported_seq":7909,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:53.033212+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:51:53.033212+0000","last_peered":"2024-08-17T17:51:53.033212+0000","last_clean":"2024-08-17T17:51:53.033212+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:51:53.033212+0000","last_undegraded":"2024-08-17T17:51:53.033212+0000","last_fullsized":"2024-08-17T17:51:53.033212+0000","mapping_epoch":1167,"log_start":"1170'1189","ondisk_log_start":"1170'1189","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6670,"log_dups_size":0,"ondisk_log_size":6670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336920576,"num_objects":5141,"num_object_clones":0,"num_object_copies":15423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7859,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1209'7929","reported_seq":13138,"reported_epoch":1209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:51:51.742600+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:51:51.742600+0000","last_peered":"2024-08-17T17:51:51.742600+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:51:51.742600+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1170'2026","ondisk_log_start":"1170'2026","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5903,"log_dups_size":0,"ondisk_log_size":5903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339804160,"num_objects":5185,"num_object_clones":0,"num_object_copies":15555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":5185,"num_objects_unfound":0,"num_objects_dirty":5185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7929,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":4667},{"shards":"1,4,5","objects":518}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1209'7927","reported_seq":7977,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:52.455895+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:51:52.455895+0000","last_peered":"2024-08-17T17:51:52.455895+0000","last_clean":"2024-08-17T17:51:52.455895+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:51:52.455895+0000","last_undegraded":"2024-08-17T17:51:52.455895+0000","last_fullsized":"2024-08-17T17:51:52.455895+0000","mapping_epoch":1167,"log_start":"1170'1177","ondisk_log_start":"1170'1177","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6750,"log_dups_size":0,"ondisk_log_size":6750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338624512,"num_objects":5167,"num_object_clones":0,"num_object_copies":15501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7927,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1209'7949","reported_seq":7999,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:53.128019+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:51:53.128019+0000","last_peered":"2024-08-17T17:51:53.128019+0000","last_clean":"2024-08-17T17:51:53.128019+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:51:53.128019+0000","last_undegraded":"2024-08-17T17:51:53.128019+0000","last_fullsized":"2024-08-17T17:51:53.128019+0000","mapping_epoch":1167,"log_start":"1170'1269","ondisk_log_start":"1170'1269","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6680,"log_dups_size":0,"ondisk_log_size":6680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345833472,"num_objects":5277,"num_object_clones":0,"num_object_copies":15831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7949,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1209'7845","reported_seq":11997,"reported_epoch":1209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:51:51.708021+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:51:51.708021+0000","last_peered":"2024-08-17T17:51:51.708021+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:51:51.708021+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1170'1875","ondisk_log_start":"1170'1875","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5970,"log_dups_size":0,"ondisk_log_size":5970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337182720,"num_objects":5145,"num_object_clones":0,"num_object_copies":15435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5583,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7845,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1350,"num_bytes_recovered":88473600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1758},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1209'7834","reported_seq":7894,"reported_epoch":1209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:51:53.051119+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:51:53.051119+0000","last_peered":"2024-08-17T17:51:53.051119+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:51:53.051119+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'1077","ondisk_log_start":"1170'1077","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6757,"log_dups_size":0,"ondisk_log_size":6757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343539712,"num_objects":5242,"num_object_clones":0,"num_object_copies":15726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7834,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":5242},{"shards":"1,6","objects":-591}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1209'7865","reported_seq":10907,"reported_epoch":1209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:51:53.664469+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:51:53.664469+0000","last_peered":"2024-08-17T17:51:53.664469+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:51:53.664469+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1170'1906","ondisk_log_start":"1170'1906","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5959,"log_dups_size":0,"ondisk_log_size":5959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347930624,"num_objects":5309,"num_object_clones":0,"num_object_copies":15927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7865,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":3748},{"shards":"1,3,5","objects":1561}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1209'7910","reported_seq":8008,"reported_epoch":1209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:51:53.404665+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:51:53.404665+0000","last_peered":"2024-08-17T17:51:53.404665+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:51:53.404665+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7910,"log_dups_size":0,"ondisk_log_size":7910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338427904,"num_objects":5164,"num_object_clones":0,"num_object_copies":15492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7910,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":11}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1209'7871","reported_seq":14030,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:53.669580+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:51:53.669580+0000","last_peered":"2024-08-17T17:51:53.669580+0000","last_clean":"2024-08-17T17:51:53.669580+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:51:53.669580+0000","last_undegraded":"2024-08-17T17:51:53.669580+0000","last_fullsized":"2024-08-17T17:51:53.669580+0000","mapping_epoch":1183,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7871,"log_dups_size":0,"ondisk_log_size":7871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339148800,"num_objects":5175,"num_object_clones":0,"num_object_copies":15525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7871,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1209'7884","reported_seq":7981,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:53.391620+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:51:53.391620+0000","last_peered":"2024-08-17T17:51:53.391620+0000","last_clean":"2024-08-17T17:51:53.391620+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:51:53.391620+0000","last_undegraded":"2024-08-17T17:51:53.391620+0000","last_fullsized":"2024-08-17T17:51:53.391620+0000","mapping_epoch":1167,"log_start":"1170'1991","ondisk_log_start":"1170'1991","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5893,"log_dups_size":0,"ondisk_log_size":5893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345767936,"num_objects":5276,"num_object_clones":0,"num_object_copies":15828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7884,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1209'7792","reported_seq":7842,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:52.887769+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:51:52.887769+0000","last_peered":"2024-08-17T17:51:52.887769+0000","last_clean":"2024-08-17T17:51:52.887769+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:51:52.887769+0000","last_undegraded":"2024-08-17T17:51:52.887769+0000","last_fullsized":"2024-08-17T17:51:52.887769+0000","mapping_epoch":1167,"log_start":"1170'725","ondisk_log_start":"1170'725","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7067,"log_dups_size":0,"ondisk_log_size":7067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336986112,"num_objects":5142,"num_object_clones":0,"num_object_copies":15426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7792,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1209'7990","reported_seq":15515,"reported_epoch":1209,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:51:52.018052+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:51:52.018052+0000","last_peered":"2024-08-17T17:51:52.018052+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:51:52.018052+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'1091","ondisk_log_start":"1170'1091","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6899,"log_dups_size":0,"ondisk_log_size":6899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352190464,"num_objects":5374,"num_object_clones":0,"num_object_copies":16122,"num_objects_missing_on_primary":4730,"num_objects_missing":4730,"num_objects_degraded":265,"num_objects_misplaced":9376,"num_objects_unfound":0,"num_objects_dirty":5374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7990,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3415,"num_bytes_recovered":223805440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":14},{"shards":"0,4,6","objects":1356},{"shards":"0,6","objects":1},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":3738}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1209'7751","reported_seq":21034,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:53.291690+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:51:53.291690+0000","last_peered":"2024-08-17T17:51:53.291690+0000","last_clean":"2024-08-17T17:51:53.291690+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:51:53.291690+0000","last_undegraded":"2024-08-17T17:51:53.291690+0000","last_fullsized":"2024-08-17T17:51:53.291690+0000","mapping_epoch":1196,"log_start":"1170'1769","ondisk_log_start":"1170'1769","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5982,"log_dups_size":0,"ondisk_log_size":5982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332333056,"num_objects":5071,"num_object_clones":0,"num_object_copies":15213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7751,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1873,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790518+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:51:41.790518+0000","last_peered":"2024-08-17T17:51:41.790518+0000","last_clean":"2024-08-17T17:51:41.790518+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:51:41.790518+0000","last_undegraded":"2024-08-17T17:51:41.790518+0000","last_fullsized":"2024-08-17T17:51:41.790518+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":486,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.112688+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:51:34.112688+0000","last_peered":"2024-08-17T17:51:34.112688+0000","last_clean":"2024-08-17T17:51:34.112688+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:51:34.112688+0000","last_undegraded":"2024-08-17T17:51:34.112688+0000","last_fullsized":"2024-08-17T17:51:34.112688+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1793,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.877163+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:51:41.877163+0000","last_peered":"2024-08-17T17:51:41.877163+0000","last_clean":"2024-08-17T17:51:41.877163+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:51:41.877163+0000","last_undegraded":"2024-08-17T17:51:41.877163+0000","last_fullsized":"2024-08-17T17:51:41.877163+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1817,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876244+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:51:41.876244+0000","last_peered":"2024-08-17T17:51:41.876244+0000","last_clean":"2024-08-17T17:51:41.876244+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:51:41.876244+0000","last_undegraded":"2024-08-17T17:51:41.876244+0000","last_fullsized":"2024-08-17T17:51:41.876244+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":48,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113239+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:51:34.113239+0000","last_peered":"2024-08-17T17:51:34.113239+0000","last_clean":"2024-08-17T17:51:34.113239+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:51:34.113239+0000","last_undegraded":"2024-08-17T17:51:34.113239+0000","last_fullsized":"2024-08-17T17:51:34.113239+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1210,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876420+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:51:41.876420+0000","last_peered":"2024-08-17T17:51:41.876420+0000","last_clean":"2024-08-17T17:51:41.876420+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:51:41.876420+0000","last_undegraded":"2024-08-17T17:51:41.876420+0000","last_fullsized":"2024-08-17T17:51:41.876420+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":68,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850634+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:51:41.850634+0000","last_peered":"2024-08-17T17:51:41.850634+0000","last_clean":"2024-08-17T17:51:41.850634+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:51:41.850634+0000","last_undegraded":"2024-08-17T17:51:41.850634+0000","last_fullsized":"2024-08-17T17:51:41.850634+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":588,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876718+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:51:41.876718+0000","last_peered":"2024-08-17T17:51:41.876718+0000","last_clean":"2024-08-17T17:51:41.876718+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:51:41.876718+0000","last_undegraded":"2024-08-17T17:51:41.876718+0000","last_fullsized":"2024-08-17T17:51:41.876718+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1832,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790565+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:51:41.790565+0000","last_peered":"2024-08-17T17:51:41.790565+0000","last_clean":"2024-08-17T17:51:41.790565+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:51:41.790565+0000","last_undegraded":"2024-08-17T17:51:41.790565+0000","last_fullsized":"2024-08-17T17:51:41.790565+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1834,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.865155+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:51:41.865155+0000","last_peered":"2024-08-17T17:51:41.865155+0000","last_clean":"2024-08-17T17:51:41.865155+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:51:41.865155+0000","last_undegraded":"2024-08-17T17:51:41.865155+0000","last_fullsized":"2024-08-17T17:51:41.865155+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":53,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.851213+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:51:41.851213+0000","last_peered":"2024-08-17T17:51:41.851213+0000","last_clean":"2024-08-17T17:51:41.851213+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:51:41.851213+0000","last_undegraded":"2024-08-17T17:51:41.851213+0000","last_fullsized":"2024-08-17T17:51:41.851213+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1806,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.877762+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:51:41.877762+0000","last_peered":"2024-08-17T17:51:41.877762+0000","last_clean":"2024-08-17T17:51:41.877762+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:51:41.877762+0000","last_undegraded":"2024-08-17T17:51:41.877762+0000","last_fullsized":"2024-08-17T17:51:41.877762+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1860,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850309+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:51:41.850309+0000","last_peered":"2024-08-17T17:51:41.850309+0000","last_clean":"2024-08-17T17:51:41.850309+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:51:41.850309+0000","last_undegraded":"2024-08-17T17:51:41.850309+0000","last_fullsized":"2024-08-17T17:51:41.850309+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1854,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790711+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:51:41.790711+0000","last_peered":"2024-08-17T17:51:41.790711+0000","last_clean":"2024-08-17T17:51:41.790711+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:51:41.790711+0000","last_undegraded":"2024-08-17T17:51:41.790711+0000","last_fullsized":"2024-08-17T17:51:41.790711+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1828,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876543+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:51:41.876543+0000","last_peered":"2024-08-17T17:51:41.876543+0000","last_clean":"2024-08-17T17:51:41.876543+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:51:41.876543+0000","last_undegraded":"2024-08-17T17:51:41.876543+0000","last_fullsized":"2024-08-17T17:51:41.876543+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1183,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113334+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:51:34.113334+0000","last_peered":"2024-08-17T17:51:34.113334+0000","last_clean":"2024-08-17T17:51:34.113334+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:51:34.113334+0000","last_undegraded":"2024-08-17T17:51:34.113334+0000","last_fullsized":"2024-08-17T17:51:34.113334+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1799,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876247+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:51:41.876247+0000","last_peered":"2024-08-17T17:51:41.876247+0000","last_clean":"2024-08-17T17:51:41.876247+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:51:41.876247+0000","last_undegraded":"2024-08-17T17:51:41.876247+0000","last_fullsized":"2024-08-17T17:51:41.876247+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1849,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790622+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:51:41.790622+0000","last_peered":"2024-08-17T17:51:41.790622+0000","last_clean":"2024-08-17T17:51:41.790622+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:51:41.790622+0000","last_undegraded":"2024-08-17T17:51:41.790622+0000","last_fullsized":"2024-08-17T17:51:41.790622+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1755,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876240+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:51:41.876240+0000","last_peered":"2024-08-17T17:51:41.876240+0000","last_clean":"2024-08-17T17:51:41.876240+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:51:41.876240+0000","last_undegraded":"2024-08-17T17:51:41.876240+0000","last_fullsized":"2024-08-17T17:51:41.876240+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":502,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.125078+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:51:34.125078+0000","last_peered":"2024-08-17T17:51:34.125078+0000","last_clean":"2024-08-17T17:51:34.125078+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:51:34.125078+0000","last_undegraded":"2024-08-17T17:51:34.125078+0000","last_fullsized":"2024-08-17T17:51:34.125078+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1179,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124870+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:51:34.124870+0000","last_peered":"2024-08-17T17:51:34.124870+0000","last_clean":"2024-08-17T17:51:34.124870+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:51:34.124870+0000","last_undegraded":"2024-08-17T17:51:34.124870+0000","last_fullsized":"2024-08-17T17:51:34.124870+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1854,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.851335+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:51:41.851335+0000","last_peered":"2024-08-17T17:51:41.851335+0000","last_clean":"2024-08-17T17:51:41.851335+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:51:41.851335+0000","last_undegraded":"2024-08-17T17:51:41.851335+0000","last_fullsized":"2024-08-17T17:51:41.851335+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":340,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124979+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:51:34.124979+0000","last_peered":"2024-08-17T17:51:34.124979+0000","last_clean":"2024-08-17T17:51:34.124979+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:51:34.124979+0000","last_undegraded":"2024-08-17T17:51:34.124979+0000","last_fullsized":"2024-08-17T17:51:34.124979+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":486,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113079+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:51:34.113079+0000","last_peered":"2024-08-17T17:51:34.113079+0000","last_clean":"2024-08-17T17:51:34.113079+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:51:34.113079+0000","last_undegraded":"2024-08-17T17:51:34.113079+0000","last_fullsized":"2024-08-17T17:51:34.113079+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1164,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113040+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:51:34.113040+0000","last_peered":"2024-08-17T17:51:34.113040+0000","last_clean":"2024-08-17T17:51:34.113040+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:51:34.113040+0000","last_undegraded":"2024-08-17T17:51:34.113040+0000","last_fullsized":"2024-08-17T17:51:34.113040+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1172,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113406+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:51:34.113406+0000","last_peered":"2024-08-17T17:51:34.113406+0000","last_clean":"2024-08-17T17:51:34.113406+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:51:34.113406+0000","last_undegraded":"2024-08-17T17:51:34.113406+0000","last_fullsized":"2024-08-17T17:51:34.113406+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1901,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790481+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:51:41.790481+0000","last_peered":"2024-08-17T17:51:41.790481+0000","last_clean":"2024-08-17T17:51:41.790481+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:51:41.790481+0000","last_undegraded":"2024-08-17T17:51:41.790481+0000","last_fullsized":"2024-08-17T17:51:41.790481+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1785,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876513+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:51:41.876513+0000","last_peered":"2024-08-17T17:51:41.876513+0000","last_clean":"2024-08-17T17:51:41.876513+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:51:41.876513+0000","last_undegraded":"2024-08-17T17:51:41.876513+0000","last_fullsized":"2024-08-17T17:51:41.876513+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1859,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850724+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:51:41.850724+0000","last_peered":"2024-08-17T17:51:41.850724+0000","last_clean":"2024-08-17T17:51:41.850724+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:51:41.850724+0000","last_undegraded":"2024-08-17T17:51:41.850724+0000","last_fullsized":"2024-08-17T17:51:41.850724+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1787,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.877243+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:51:41.877243+0000","last_peered":"2024-08-17T17:51:41.877243+0000","last_clean":"2024-08-17T17:51:41.877243+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:51:41.877243+0000","last_undegraded":"2024-08-17T17:51:41.877243+0000","last_fullsized":"2024-08-17T17:51:41.877243+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1819,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790437+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:51:41.790437+0000","last_peered":"2024-08-17T17:51:41.790437+0000","last_clean":"2024-08-17T17:51:41.790437+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:51:41.790437+0000","last_undegraded":"2024-08-17T17:51:41.790437+0000","last_fullsized":"2024-08-17T17:51:41.790437+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1199,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.125017+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:51:34.125017+0000","last_peered":"2024-08-17T17:51:34.125017+0000","last_clean":"2024-08-17T17:51:34.125017+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:51:34.125017+0000","last_undegraded":"2024-08-17T17:51:34.125017+0000","last_fullsized":"2024-08-17T17:51:34.125017+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1866,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850824+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:51:41.850824+0000","last_peered":"2024-08-17T17:51:41.850824+0000","last_clean":"2024-08-17T17:51:41.850824+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:51:41.850824+0000","last_undegraded":"2024-08-17T17:51:41.850824+0000","last_fullsized":"2024-08-17T17:51:41.850824+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":53,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.851192+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:51:41.851192+0000","last_peered":"2024-08-17T17:51:41.851192+0000","last_clean":"2024-08-17T17:51:41.851192+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:51:41.851192+0000","last_undegraded":"2024-08-17T17:51:41.851192+0000","last_fullsized":"2024-08-17T17:51:41.851192+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1885,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850731+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:51:41.850731+0000","last_peered":"2024-08-17T17:51:41.850731+0000","last_clean":"2024-08-17T17:51:41.850731+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:51:41.850731+0000","last_undegraded":"2024-08-17T17:51:41.850731+0000","last_fullsized":"2024-08-17T17:51:41.850731+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":621,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876994+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:51:41.876994+0000","last_peered":"2024-08-17T17:51:41.876994+0000","last_clean":"2024-08-17T17:51:41.876994+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:51:41.876994+0000","last_undegraded":"2024-08-17T17:51:41.876994+0000","last_fullsized":"2024-08-17T17:51:41.876994+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1866,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850901+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:51:41.850901+0000","last_peered":"2024-08-17T17:51:41.850901+0000","last_clean":"2024-08-17T17:51:41.850901+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:51:41.850901+0000","last_undegraded":"2024-08-17T17:51:41.850901+0000","last_fullsized":"2024-08-17T17:51:41.850901+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1194,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790203+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:51:41.790203+0000","last_peered":"2024-08-17T17:51:41.790203+0000","last_clean":"2024-08-17T17:51:41.790203+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:51:41.790203+0000","last_undegraded":"2024-08-17T17:51:41.790203+0000","last_fullsized":"2024-08-17T17:51:41.790203+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":112,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.851288+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:51:41.851288+0000","last_peered":"2024-08-17T17:51:41.851288+0000","last_clean":"2024-08-17T17:51:41.851288+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:51:41.851288+0000","last_undegraded":"2024-08-17T17:51:41.851288+0000","last_fullsized":"2024-08-17T17:51:41.851288+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1876,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850877+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:51:41.850877+0000","last_peered":"2024-08-17T17:51:41.850877+0000","last_clean":"2024-08-17T17:51:41.850877+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:51:41.850877+0000","last_undegraded":"2024-08-17T17:51:41.850877+0000","last_fullsized":"2024-08-17T17:51:41.850877+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1854,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850170+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:51:41.850170+0000","last_peered":"2024-08-17T17:51:41.850170+0000","last_clean":"2024-08-17T17:51:41.850170+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:51:41.850170+0000","last_undegraded":"2024-08-17T17:51:41.850170+0000","last_fullsized":"2024-08-17T17:51:41.850170+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1874,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.864874+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:51:41.864874+0000","last_peered":"2024-08-17T17:51:41.864874+0000","last_clean":"2024-08-17T17:51:41.864874+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:51:41.864874+0000","last_undegraded":"2024-08-17T17:51:41.864874+0000","last_fullsized":"2024-08-17T17:51:41.864874+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1880,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790758+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:51:41.790758+0000","last_peered":"2024-08-17T17:51:41.790758+0000","last_clean":"2024-08-17T17:51:41.790758+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:51:41.790758+0000","last_undegraded":"2024-08-17T17:51:41.790758+0000","last_fullsized":"2024-08-17T17:51:41.790758+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1860,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.833186+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:51:41.833186+0000","last_peered":"2024-08-17T17:51:41.833186+0000","last_clean":"2024-08-17T17:51:41.833186+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:51:41.833186+0000","last_undegraded":"2024-08-17T17:51:41.833186+0000","last_fullsized":"2024-08-17T17:51:41.833186+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1206,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124907+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:51:34.124907+0000","last_peered":"2024-08-17T17:51:34.124907+0000","last_clean":"2024-08-17T17:51:34.124907+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:51:34.124907+0000","last_undegraded":"2024-08-17T17:51:34.124907+0000","last_fullsized":"2024-08-17T17:51:34.124907+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1152,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113230+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:51:34.113230+0000","last_peered":"2024-08-17T17:51:34.113230+0000","last_clean":"2024-08-17T17:51:34.113230+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:51:34.113230+0000","last_undegraded":"2024-08-17T17:51:34.113230+0000","last_fullsized":"2024-08-17T17:51:34.113230+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1769,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876358+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:51:41.876358+0000","last_peered":"2024-08-17T17:51:41.876358+0000","last_clean":"2024-08-17T17:51:41.876358+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:51:41.876358+0000","last_undegraded":"2024-08-17T17:51:41.876358+0000","last_fullsized":"2024-08-17T17:51:41.876358+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":100,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.865829+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:51:41.865829+0000","last_peered":"2024-08-17T17:51:41.865829+0000","last_clean":"2024-08-17T17:51:41.865829+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:51:41.865829+0000","last_undegraded":"2024-08-17T17:51:41.865829+0000","last_fullsized":"2024-08-17T17:51:41.865829+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1518,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124676+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:51:34.124676+0000","last_peered":"2024-08-17T17:51:34.124676+0000","last_clean":"2024-08-17T17:51:34.124676+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:51:34.124676+0000","last_undegraded":"2024-08-17T17:51:34.124676+0000","last_fullsized":"2024-08-17T17:51:34.124676+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1861,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.851009+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:51:41.851009+0000","last_peered":"2024-08-17T17:51:41.851009+0000","last_clean":"2024-08-17T17:51:41.851009+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:51:41.851009+0000","last_undegraded":"2024-08-17T17:51:41.851009+0000","last_fullsized":"2024-08-17T17:51:41.851009+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1801,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876355+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:51:41.876355+0000","last_peered":"2024-08-17T17:51:41.876355+0000","last_clean":"2024-08-17T17:51:41.876355+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:51:41.876355+0000","last_undegraded":"2024-08-17T17:51:41.876355+0000","last_fullsized":"2024-08-17T17:51:41.876355+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":486,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113200+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:51:34.113200+0000","last_peered":"2024-08-17T17:51:34.113200+0000","last_clean":"2024-08-17T17:51:34.113200+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:51:34.113200+0000","last_undegraded":"2024-08-17T17:51:34.113200+0000","last_fullsized":"2024-08-17T17:51:34.113200+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":539,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.864674+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:51:41.864674+0000","last_peered":"2024-08-17T17:51:41.864674+0000","last_clean":"2024-08-17T17:51:41.864674+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:51:41.864674+0000","last_undegraded":"2024-08-17T17:51:41.864674+0000","last_fullsized":"2024-08-17T17:51:41.864674+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":555,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.864847+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:51:41.864847+0000","last_peered":"2024-08-17T17:51:41.864847+0000","last_clean":"2024-08-17T17:51:41.864847+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:51:41.864847+0000","last_undegraded":"2024-08-17T17:51:41.864847+0000","last_fullsized":"2024-08-17T17:51:41.864847+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1170,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.113060+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:51:34.113060+0000","last_peered":"2024-08-17T17:51:34.113060+0000","last_clean":"2024-08-17T17:51:34.113060+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:51:34.113060+0000","last_undegraded":"2024-08-17T17:51:34.113060+0000","last_fullsized":"2024-08-17T17:51:34.113060+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":571,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876780+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:51:41.876780+0000","last_peered":"2024-08-17T17:51:41.876780+0000","last_clean":"2024-08-17T17:51:41.876780+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:51:41.876780+0000","last_undegraded":"2024-08-17T17:51:41.876780+0000","last_fullsized":"2024-08-17T17:51:41.876780+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1797,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876808+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:51:41.876808+0000","last_peered":"2024-08-17T17:51:41.876808+0000","last_clean":"2024-08-17T17:51:41.876808+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:51:41.876808+0000","last_undegraded":"2024-08-17T17:51:41.876808+0000","last_fullsized":"2024-08-17T17:51:41.876808+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1893,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790525+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:51:41.790525+0000","last_peered":"2024-08-17T17:51:41.790525+0000","last_clean":"2024-08-17T17:51:41.790525+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:51:41.790525+0000","last_undegraded":"2024-08-17T17:51:41.790525+0000","last_fullsized":"2024-08-17T17:51:41.790525+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":604,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876694+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:51:41.876694+0000","last_peered":"2024-08-17T17:51:41.876694+0000","last_clean":"2024-08-17T17:51:41.876694+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:51:41.876694+0000","last_undegraded":"2024-08-17T17:51:41.876694+0000","last_fullsized":"2024-08-17T17:51:41.876694+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1159,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.112931+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:51:34.112931+0000","last_peered":"2024-08-17T17:51:34.112931+0000","last_clean":"2024-08-17T17:51:34.112931+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:51:34.112931+0000","last_undegraded":"2024-08-17T17:51:34.112931+0000","last_fullsized":"2024-08-17T17:51:34.112931+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1158,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124959+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:51:34.124959+0000","last_peered":"2024-08-17T17:51:34.124959+0000","last_clean":"2024-08-17T17:51:34.124959+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:51:34.124959+0000","last_undegraded":"2024-08-17T17:51:34.124959+0000","last_fullsized":"2024-08-17T17:51:34.124959+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":588,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.850420+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:51:41.850420+0000","last_peered":"2024-08-17T17:51:41.850420+0000","last_clean":"2024-08-17T17:51:41.850420+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:51:41.850420+0000","last_undegraded":"2024-08-17T17:51:41.850420+0000","last_fullsized":"2024-08-17T17:51:41.850420+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1176,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.125254+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:51:34.125254+0000","last_peered":"2024-08-17T17:51:34.125254+0000","last_clean":"2024-08-17T17:51:34.125254+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:51:34.125254+0000","last_undegraded":"2024-08-17T17:51:34.125254+0000","last_fullsized":"2024-08-17T17:51:34.125254+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":60,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124592+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:51:34.124592+0000","last_peered":"2024-08-17T17:51:34.124592+0000","last_clean":"2024-08-17T17:51:34.124592+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:51:34.124592+0000","last_undegraded":"2024-08-17T17:51:34.124592+0000","last_fullsized":"2024-08-17T17:51:34.124592+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1192,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.125129+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:51:34.125129+0000","last_peered":"2024-08-17T17:51:34.125129+0000","last_clean":"2024-08-17T17:51:34.125129+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:51:34.125129+0000","last_undegraded":"2024-08-17T17:51:34.125129+0000","last_fullsized":"2024-08-17T17:51:34.125129+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1209,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.112820+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:51:34.112820+0000","last_peered":"2024-08-17T17:51:34.112820+0000","last_clean":"2024-08-17T17:51:34.112820+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:51:34.112820+0000","last_undegraded":"2024-08-17T17:51:34.112820+0000","last_fullsized":"2024-08-17T17:51:34.112820+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1799,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.877190+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:51:41.877190+0000","last_peered":"2024-08-17T17:51:41.877190+0000","last_clean":"2024-08-17T17:51:41.877190+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:51:41.877190+0000","last_undegraded":"2024-08-17T17:51:41.877190+0000","last_fullsized":"2024-08-17T17:51:41.877190+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1153,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.125058+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:51:34.125058+0000","last_peered":"2024-08-17T17:51:34.125058+0000","last_clean":"2024-08-17T17:51:34.125058+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:51:34.125058+0000","last_undegraded":"2024-08-17T17:51:34.125058+0000","last_fullsized":"2024-08-17T17:51:34.125058+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1852,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790584+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:51:41.790584+0000","last_peered":"2024-08-17T17:51:41.790584+0000","last_clean":"2024-08-17T17:51:41.790584+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:51:41.790584+0000","last_undegraded":"2024-08-17T17:51:41.790584+0000","last_fullsized":"2024-08-17T17:51:41.790584+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1853,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.790291+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:51:41.790291+0000","last_peered":"2024-08-17T17:51:41.790291+0000","last_clean":"2024-08-17T17:51:41.790291+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:51:41.790291+0000","last_undegraded":"2024-08-17T17:51:41.790291+0000","last_fullsized":"2024-08-17T17:51:41.790291+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":566,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124801+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:51:34.124801+0000","last_peered":"2024-08-17T17:51:34.124801+0000","last_clean":"2024-08-17T17:51:34.124801+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:51:34.124801+0000","last_undegraded":"2024-08-17T17:51:34.124801+0000","last_fullsized":"2024-08-17T17:51:34.124801+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1178,"reported_epoch":1208,"state":"active+clean","last_fresh":"2024-08-17T17:51:34.124974+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:51:34.124974+0000","last_peered":"2024-08-17T17:51:34.124974+0000","last_clean":"2024-08-17T17:51:34.124974+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:51:34.124974+0000","last_undegraded":"2024-08-17T17:51:34.124974+0000","last_fullsized":"2024-08-17T17:51:34.124974+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1799,"reported_epoch":1209,"state":"active+clean","last_fresh":"2024-08-17T17:51:41.876810+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:51:41.876810+0000","last_peered":"2024-08-17T17:51:41.876810+0000","last_clean":"2024-08-17T17:51:41.876810+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:51:41.876810+0000","last_undegraded":"2024-08-17T17:51:41.876810+0000","last_fullsized":"2024-08-17T17:51:41.876810+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":5463736344,"num_objects":83371,"num_object_clones":0,"num_object_copies":250113,"num_objects_missing_on_primary":4730,"num_objects_missing":4730,"num_objects_degraded":20702,"num_objects_misplaced":15740,"num_objects_unfound":0,"num_objects_dirty":83371,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":126239,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30380,"num_bytes_recovered":1883570176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16962564096,"data_stored":16962551880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106113,"ondisk_log_size":106113,"up":48,"acting":41,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234372,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2537600,"kb_used_data":2227768,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91207552,"statfs":{"total":95995035648,"available":93396533248,"internally_reserved":0,"allocated":2281234432,"data_stored":2276444009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496007,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2891076,"kb_used_data":2581244,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":90854076,"statfs":{"total":95995035648,"available":93034573824,"internally_reserved":0,"allocated":2643193856,"data_stored":2638399838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757645,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326259,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3176556,"kb_used_data":2978148,"kb_used_omap":57,"kb_used_meta":198342,"kb_avail":90568596,"statfs":{"total":95995035648,"available":92742242304,"internally_reserved":0,"allocated":3049623552,"data_stored":3044848309,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":203102971},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405498,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2861428,"kb_used_data":2667116,"kb_used_omap":61,"kb_used_meta":194242,"kb_avail":90883724,"statfs":{"total":95995035648,"available":93064933376,"internally_reserved":0,"allocated":2731126784,"data_stored":2726348513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":198904760},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452032,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2184744,"kb_used_data":2014112,"kb_used_omap":54,"kb_used_meta":170569,"kb_avail":91560408,"statfs":{"total":95995035648,"available":93757857792,"internally_reserved":0,"allocated":2062450688,"data_stored":2057679406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":174663054},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474724,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1678536,"kb_used_data":1515520,"kb_used_omap":119,"kb_used_meta":162888,"kb_avail":92066616,"statfs":{"total":95995035648,"available":94276214784,"internally_reserved":0,"allocated":1551892480,"data_stored":1547068669,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":166798046},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,1,1,1,2,4,6,3],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019280,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1543835648,"data_stored":1543831576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2049179648,"data_stored":2049179648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2717712384,"data_stored":2717712384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3035762688,"data_stored":3035758616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1111556096,"data_stored":1111556096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1610743808,"data_stored":1610743808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2628390912,"data_stored":2628386840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2265382912,"data_stored":2265382912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:51:56.272 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:51:56.272 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:56.294 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:56.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:56.396 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:56.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:56.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:56.519 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:56.545 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:56.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:56.647 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:56.670 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:56.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:56.772 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:56.799 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:56.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:56.901 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:56.923 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:56.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:56.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:57.025 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:57.052 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:57.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:57.154 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:57.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:56 smithi130 ceph-mon[105521]: from='client.35499 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:51:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:56 smithi130 ceph-mon[110068]: from='client.35499 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:51:57.180 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:57.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:57.281 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:57.306 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:57.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:57.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:56 smithi179 ceph-mon[93519]: from='client.35499 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:51:57.407 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:57.508 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:57.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:57.610 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:51:57.632 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:57.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:51:57.734 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:57.755 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:57.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:57.857 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:57.879 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:57.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:57.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:57.981 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:58.002 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:58.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:58.104 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:58.126 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:58.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:51:58.159 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:57 smithi130 ceph-mon[105521]: pgmap v1730: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 20630/249981 objects degraded (8.253%); 15706/249981 objects misplaced (6.283%); 7.0 MiB/s, 111 objects/s recovering 2024-08-17T17:51:58.159 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:58.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:57 smithi130 ceph-mon[110068]: pgmap v1730: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 20630/249981 objects degraded (8.253%); 15706/249981 objects misplaced (6.283%); 7.0 MiB/s, 111 objects/s recovering 2024-08-17T17:51:58.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:58.227 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:51:58.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:58.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:51:58.306 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:57 smithi179 ceph-mon[93519]: pgmap v1730: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 20630/249981 objects degraded (8.253%); 15706/249981 objects misplaced (6.283%); 7.0 MiB/s, 111 objects/s recovering 2024-08-17T17:51:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:51:58.352 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:58.359 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:51:58.382 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:58.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:58.483 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:58.507 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:58.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:58.609 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:58.638 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:58.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:58.740 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:58.765 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:58.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:58.867 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:58.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:58.896 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:58.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:58.898 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T17:51:58.975 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:58 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3117596672' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:58.975 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:58 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:58 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3117596672' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:58.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:58 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:58.998 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:59.024 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:59.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:59.126 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:51:59.151 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:59.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:51:59.252 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:59.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:59.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:58 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3117596672' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:59.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:58 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:51:59.376 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:51:59.398 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:59.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:51:59.499 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:51:59.520 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:59.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:51:59.622 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:51:59.645 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:59.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:51:59.716 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:51:59.747 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:51:59.780 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:51:59.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:51:59.882 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:51:59.904 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:51:59.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:51:59.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:00.006 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:00.017 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:51:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:51:59] "GET /metrics HTTP/1.1" 200 36816 "" "Prometheus/2.51.0" 2024-08-17T17:52:00.017 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:59 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:00.017 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:59 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:00.017 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:59 smithi130 ceph-mon[105521]: osdmap e1210: 8 total, 8 up, 6 in 2024-08-17T17:52:00.017 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:51:59 smithi130 ceph-mon[105521]: pgmap v1732: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 20630/248817 objects degraded (8.291%); 14846/248817 objects misplaced (5.967%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:52:00.018 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:59 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:00.018 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:59 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:00.018 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:59 smithi130 ceph-mon[110068]: osdmap e1210: 8 total, 8 up, 6 in 2024-08-17T17:52:00.018 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:51:59 smithi130 ceph-mon[110068]: pgmap v1732: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 20630/248817 objects degraded (8.291%); 14846/248817 objects misplaced (5.967%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:52:00.035 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:00.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:00.137 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:00.158 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:00.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:00.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:59 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:00.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:59 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:00.214 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:59 smithi179 ceph-mon[93519]: osdmap e1210: 8 total, 8 up, 6 in 2024-08-17T17:52:00.214 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:51:59 smithi179 ceph-mon[93519]: pgmap v1732: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 20630/248817 objects degraded (8.291%); 14846/248817 objects misplaced (5.967%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:52:00.260 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:00.282 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:00.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:00.383 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:00.408 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:00.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:00.510 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:00.531 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:00.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:00.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:52:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:52:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:52:00.633 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:00.654 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:00.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:00.756 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:00.777 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:00.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:00.879 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:00.903 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:00.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:00.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:01.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:01.026 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:01.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:01.128 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:01.149 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:01.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:01.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:00 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 20630/248817 objects degraded (8.291%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:00 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 20630/248817 objects degraded (8.291%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:01.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:01.280 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:01.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:01.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:00 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 20630/248817 objects degraded (8.291%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:01.381 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:01.407 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:01.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:01.509 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:01.533 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:01.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:01.635 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:01.659 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:01.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:01.761 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:01.788 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:01.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:01.889 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:01.914 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:01.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:01.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:02.016 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:02.039 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:02.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:02.141 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:02.163 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:02.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:02.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:01 smithi130 ceph-mon[105521]: pgmap v1733: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 20630/248628 objects degraded (8.298%); 14846/248628 objects misplaced (5.971%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:52:02.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:01 smithi130 ceph-mon[110068]: pgmap v1733: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 20630/248628 objects degraded (8.298%); 14846/248628 objects misplaced (5.971%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:52:02.265 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:02.296 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:02.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:02.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:01 smithi179 ceph-mon[93519]: pgmap v1733: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 20630/248628 objects degraded (8.298%); 14846/248628 objects misplaced (5.971%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:52:02.398 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:02.430 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:02.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:02.532 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:02.553 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:02.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:02.655 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:02.681 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:02.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:02.783 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:02.804 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:02.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:02.906 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:02.929 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:02.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:02.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:03.030 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:03.051 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:03.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:03.153 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:03.177 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:03.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:03.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:03.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.302 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:03.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:03.352 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:52:03.352 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:52:03.404 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:03.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:03 smithi130 ceph-mon[105521]: pgmap v1734: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 20556/248262 objects degraded (8.280%); 14743/248262 objects misplaced (5.938%); 9.8 MiB/s, 157 objects/s recovering 2024-08-17T17:52:03.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:03 smithi130 ceph-mon[110068]: pgmap v1734: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 20556/248262 objects degraded (8.280%); 14743/248262 objects misplaced (5.938%); 9.8 MiB/s, 157 objects/s recovering 2024-08-17T17:52:03.425 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:03.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:03.527 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:03.548 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:03.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:03.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:03 smithi179 ceph-mon[93519]: pgmap v1734: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 20556/248262 objects degraded (8.280%); 14743/248262 objects misplaced (5.938%); 9.8 MiB/s, 157 objects/s recovering 2024-08-17T17:52:03.650 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:03.673 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:03.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:03.698 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:52:03.776 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:03.799 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:03.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:03.901 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:03.922 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:03.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:03.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:04.023 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:04.048 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:04.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:04.110 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:52:04.150 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:04.172 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:04.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:04.273 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:04.282 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:04.310 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:04.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:04.412 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:04.433 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:04.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:04.535 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:04.562 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:04.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:04.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:04.687 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:04.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:04.789 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:04.810 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:04.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:04.912 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:04.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:04.934 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:04.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:05.035 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:05.058 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:05.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:05.160 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:05.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.182 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:05.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:05.283 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:05.304 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:05.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:05.327 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:05 smithi130 ceph-mon[105521]: osdmap e1211: 8 total, 8 up, 6 in 2024-08-17T17:52:05.328 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:05 smithi130 ceph-mon[105521]: pgmap v1736: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 19803/247197 objects degraded (8.011%); 14360/247197 objects misplaced (5.809%); 10 MiB/s, 254 objects/s recovering 2024-08-17T17:52:05.328 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:05 smithi130 ceph-mon[110068]: osdmap e1211: 8 total, 8 up, 6 in 2024-08-17T17:52:05.328 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:05 smithi130 ceph-mon[110068]: pgmap v1736: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 19803/247197 objects degraded (8.011%); 14360/247197 objects misplaced (5.809%); 10 MiB/s, 254 objects/s recovering 2024-08-17T17:52:05.406 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:05.428 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:05.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:05.529 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:05.550 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:05.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:05 smithi179 ceph-mon[93519]: osdmap e1211: 8 total, 8 up, 6 in 2024-08-17T17:52:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:05 smithi179 ceph-mon[93519]: pgmap v1736: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 19803/247197 objects degraded (8.011%); 14360/247197 objects misplaced (5.809%); 10 MiB/s, 254 objects/s recovering 2024-08-17T17:52:05.652 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:05.674 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:05.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:05.776 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:05.798 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:05.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:05.900 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:05.924 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:05.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:05.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:05.974 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:06.026 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:06.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.050 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:06.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:06.152 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:06.178 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:06.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:06.280 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:06.311 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:06.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:06.312 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:52:06.312 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 19803/247197 objects degraded (8.011%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1447,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1212,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":82},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+degraded+remapped","count":2}],"num_pgs":89,"num_pools":5,"num_objects":82399,"data_bytes":5398927333,"bytes_used":15581220864,"bytes_avail":560388993024,"bytes_total":575970213888,"degraded_objects":19803,"degraded_total":247197,"degraded_ratio":0.080110195512081456,"misplaced_objects":14360,"misplaced_total":247197,"misplaced_ratio":0.058091319878477495,"recovering_objects_per_sec":254,"recovering_bytes_per_sec":10499755,"recovering_keys_per_sec":0,"num_objects_recovered":2035,"num_bytes_recovered":84017152,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":115},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":25,"modified":"2024-08-17T17:49:39.646037+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (4m)\n [=========================...] (remaining: 23s)","progress":0.92134833335876465,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:52:06.356 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:06 smithi130 ceph-mon[105521]: osdmap e1212: 8 total, 8 up, 6 in 2024-08-17T17:52:06.356 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 19803/247197 objects degraded (8.011%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:06.356 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:06 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:06.356 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:06 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:06.356 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:06 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:52:06.356 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:06 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:52:06.356 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:06 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:06.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:06 smithi130 ceph-mon[110068]: osdmap e1212: 8 total, 8 up, 6 in 2024-08-17T17:52:06.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 19803/247197 objects degraded (8.011%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:06.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:06 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:06.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:06 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:06.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:06 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:52:06.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:06 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:52:06.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:06 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:06.413 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:06.434 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:06.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:06.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:06.560 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:06.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:06 smithi179 ceph-mon[93519]: osdmap e1212: 8 total, 8 up, 6 in 2024-08-17T17:52:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 19803/247197 objects degraded (8.011%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:06 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:06 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:06 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:52:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:06 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:52:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:06 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:06.662 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:06.684 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:06.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:06.785 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:06.808 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:06.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:06.910 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:06.931 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:06.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:06.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:07.033 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:07.058 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:07.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:07.160 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:07.186 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:07.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:07.288 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:07.310 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:07.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:07.412 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:07.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:07 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2317347357' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:52:07.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:07 smithi130 ceph-mon[105521]: pgmap v1738: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 19755/247038 objects degraded (7.997%); 14331/247038 objects misplaced (5.801%); 3.7 MiB/s, 153 objects/s recovering 2024-08-17T17:52:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:07 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2317347357' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:52:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:07 smithi130 ceph-mon[110068]: pgmap v1738: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 19755/247038 objects degraded (7.997%); 14331/247038 objects misplaced (5.801%); 3.7 MiB/s, 153 objects/s recovering 2024-08-17T17:52:07.434 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:07.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:07.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:07.560 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:07.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:07.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:07 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2317347357' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:52:07.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:07 smithi179 ceph-mon[93519]: pgmap v1738: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 74 op/s; 19755/247038 objects degraded (7.997%); 14331/247038 objects misplaced (5.801%); 3.7 MiB/s, 153 objects/s recovering 2024-08-17T17:52:07.662 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:07.685 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:07.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:07.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:07.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:07.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:07.912 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:07.938 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:07.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:07.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:08.040 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:08.061 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:08.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:08.163 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:08.189 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:08.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:08.291 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:08.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.313 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:08.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:08.414 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:08.440 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:08.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:08.542 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:08.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.564 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:08.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:08.630 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:52:08.631 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:52:08.636 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:08.665 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:08.697 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:08.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:08.799 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:08.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:08.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:08.923 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:08.951 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:08.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:08.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:09.053 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:09.077 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:09.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:09.179 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:09.188 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:09 smithi130 ceph-mon[110068]: pgmap v1739: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 19424/245844 objects degraded (7.901%); 14266/245844 objects misplaced (5.803%); 4.0 MiB/s, 198 objects/s recovering 2024-08-17T17:52:09.188 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:09 smithi130 ceph-mon[105521]: pgmap v1739: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 19424/245844 objects degraded (7.901%); 14266/245844 objects misplaced (5.803%); 4.0 MiB/s, 198 objects/s recovering 2024-08-17T17:52:09.202 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:09.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:09.304 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:09.325 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:09.427 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:09.450 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:09.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:09.510 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:52:09.510 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:52:09.545 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:52:09.545 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:52:09.552 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:09.569 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:52:09.569 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:52:09.573 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:09.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:09.587 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:52:09.587 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:52:09.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:09 smithi179 ceph-mon[93519]: pgmap v1739: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 19424/245844 objects degraded (7.901%); 14266/245844 objects misplaced (5.803%); 4.0 MiB/s, 198 objects/s recovering 2024-08-17T17:52:09.603 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:52:09.604 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:52:09.631 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:52:09.631 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:52:09.651 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:52:09.652 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:52:09.671 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:52:09.671 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:52:09.675 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:09.700 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:09.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:09.802 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:09.824 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:09.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:09.925 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:09.947 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:09.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:09.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:10.049 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:10.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.072 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:10.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:10.113 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:52:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:52:09] "GET /metrics HTTP/1.1" 200 36816 "" "Prometheus/2.51.0" 2024-08-17T17:52:10.113 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T17:52:10.172 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:10.194 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:10.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:10.296 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:10.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:10.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:10.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:10 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4144802480' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:10.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:10 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3890625235' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:10.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:10 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:10.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:10 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4144802480' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:10.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:10 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3890625235' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:10.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:10 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:10.420 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:10.442 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:10.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:10.544 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:10.566 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:10.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:10 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4144802480' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:10 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3890625235' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:10 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:52:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:52:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:52:10.668 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:10.690 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:10.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:10.792 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:10.815 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:10.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:10.918 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:10.944 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:10.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:10.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:11.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:11.068 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:11.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:11.170 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:11.194 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:11.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:11.243 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:11.296 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:11.317 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:11.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:11.393 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:11 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:11.393 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:11 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:52:11.393 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:11 smithi130 ceph-mon[105521]: osdmap e1213: 8 total, 8 up, 6 in 2024-08-17T17:52:11.393 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 19424/245844 objects degraded (7.901%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:11.393 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:11 smithi130 ceph-mon[105521]: pgmap v1741: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 19424/245667 objects degraded (7.907%); 14266/245667 objects misplaced (5.807%); 532 KiB/s, 57 objects/s recovering 2024-08-17T17:52:11.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:11 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:11.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:11 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:52:11.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:11 smithi130 ceph-mon[110068]: osdmap e1213: 8 total, 8 up, 6 in 2024-08-17T17:52:11.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 19424/245844 objects degraded (7.901%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:11.394 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:11 smithi130 ceph-mon[110068]: pgmap v1741: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 19424/245667 objects degraded (7.907%); 14266/245667 objects misplaced (5.807%); 532 KiB/s, 57 objects/s recovering 2024-08-17T17:52:11.419 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:11.440 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:11.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:11.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:11.566 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:11.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:11 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:11 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:52:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:11 smithi179 ceph-mon[93519]: osdmap e1213: 8 total, 8 up, 6 in 2024-08-17T17:52:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 19424/245844 objects degraded (7.901%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:11 smithi179 ceph-mon[93519]: pgmap v1741: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 75 op/s; 19424/245667 objects degraded (7.907%); 14266/245667 objects misplaced (5.807%); 532 KiB/s, 57 objects/s recovering 2024-08-17T17:52:11.668 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:11.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:11.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:11.796 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:11.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:11.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:11.922 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:11.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:11.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:11.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:12.045 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:12.072 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:12.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:12.175 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:12.197 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:12.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:12.299 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:12.321 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:12.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:12.423 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:12.444 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:12.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:12.546 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:12.568 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:12.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:12.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:12.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:12.669 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:12.695 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:12.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:12.797 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:12.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.820 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:12.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:12.826 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:52:12.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:12.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:12.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:12.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:12.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:13.058 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:13.079 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:13.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:13.181 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:13.204 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:13.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:13.306 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:13.327 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:13.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:13.428 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:13.450 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:13.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:13.552 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:13.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.580 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:13.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:13.664 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:13 smithi130 ceph-mon[105521]: pgmap v1742: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 19354/245256 objects degraded (7.891%); 14158/245256 objects misplaced (5.773%); 1016 KiB/s, 57 objects/s recovering 2024-08-17T17:52:13.664 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:13 smithi130 ceph-mon[110068]: pgmap v1742: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 19354/245256 objects degraded (7.891%); 14158/245256 objects misplaced (5.773%); 1016 KiB/s, 57 objects/s recovering 2024-08-17T17:52:13.681 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:13.702 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:13.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:13.804 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:13.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:13.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:13 smithi179 ceph-mon[93519]: pgmap v1742: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 80 op/s; 19354/245256 objects degraded (7.891%); 14158/245256 objects misplaced (5.773%); 1016 KiB/s, 57 objects/s recovering 2024-08-17T17:52:13.857 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:52:13.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:13.951 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:13.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:13.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:14.053 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:14.074 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:14.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:14.117 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:52:14.117 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T17:52:14.117 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T17:52:14.118 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T17:52:14.118 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.118 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:83237 45.0 45:024aba9a:::benchmark_data_smithi130_172551_object83236:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:52:14.118 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:38.636045+0000", 2024-08-17T17:52:14.118 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 215.47795588899999, 2024-08-17T17:52:14.118 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4221936899999998, 2024-08-17T17:52:14.118 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.118 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.118 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T17:52:14.119 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.119 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.119 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.119 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 83237 2024-08-17T17:52:14.119 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.119 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.119 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.119 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.119 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.636045+0000", 2024-08-17T17:52:14.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.119 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.636045+0000", 2024-08-17T17:52:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.636046+0000", 2024-08-17T17:52:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2309999999999999e-06 2024-08-17T17:52:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.636069+0000", 2024-08-17T17:52:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2107999999999999e-05 2024-08-17T17:52:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.636082+0000", 2024-08-17T17:52:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.322e-05 2024-08-17T17:52:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.121 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.636089+0000", 2024-08-17T17:52:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1010000000000003e-06 2024-08-17T17:52:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.636154+0000", 2024-08-17T17:52:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5227000000000004e-05 2024-08-17T17:52:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.636301+0000", 2024-08-17T17:52:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014697700000000001 2024-08-17T17:52:14.122 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,5", 2024-08-17T17:52:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.636461+0000", 2024-08-17T17:52:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000159827 2024-08-17T17:52:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.645771+0000", 2024-08-17T17:52:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0093104940000000008 2024-08-17T17:52:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.659868+0000", 2024-08-17T17:52:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014096876 2024-08-17T17:52:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.659881+0000", 2024-08-17T17:52:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2921999999999999e-05 2024-08-17T17:52:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.640362+0000", 2024-08-17T17:52:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98048062899999999 2024-08-17T17:52:14.124 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.640375+0000", 2024-08-17T17:52:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3532e-05 2024-08-17T17:52:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.058121+0000", 2024-08-17T17:52:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4177454380000001 2024-08-17T17:52:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.125 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.058239+0000", 2024-08-17T17:52:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000118108 2024-08-17T17:52:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:83247 45.d 45:b01d2b16:::benchmark_data_smithi130_172551_object83246:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:52:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:38.661129+0000", 2024-08-17T17:52:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 215.45287252200001, 2024-08-17T17:52:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.478937975, 2024-08-17T17:52:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.126 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 83247 2024-08-17T17:52:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.661129+0000", 2024-08-17T17:52:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.127 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.661129+0000", 2024-08-17T17:52:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.661130+0000", 2024-08-17T17:52:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3710000000000001e-06 2024-08-17T17:52:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.128 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.661156+0000", 2024-08-17T17:52:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6358000000000001e-05 2024-08-17T17:52:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.661169+0000", 2024-08-17T17:52:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2315e-05 2024-08-17T17:52:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.129 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.661177+0000", 2024-08-17T17:52:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0560000000000005e-06 2024-08-17T17:52:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.661642+0000", 2024-08-17T17:52:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00046559100000000001 2024-08-17T17:52:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.130 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.661653+0000", 2024-08-17T17:52:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0623e-05 2024-08-17T17:52:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.643472+0000", 2024-08-17T17:52:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98181879699999997 2024-08-17T17:52:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.131 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.643482+0000", 2024-08-17T17:52:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0018e-05 2024-08-17T17:52:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.117634+0000", 2024-08-17T17:52:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4741520459999999 2024-08-17T17:52:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.117773+0000", 2024-08-17T17:52:14.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000139228 2024-08-17T17:52:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:52:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.117953+0000", 2024-08-17T17:52:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000179846 2024-08-17T17:52:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.128883+0000", 2024-08-17T17:52:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010930637 2024-08-17T17:52:14.133 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.128898+0000", 2024-08-17T17:52:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4972e-05 2024-08-17T17:52:14.134 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.136099+0000", 2024-08-17T17:52:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072003170000000003 2024-08-17T17:52:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.136370+0000", 2024-08-17T17:52:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00027156800000000003 2024-08-17T17:52:14.135 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.140067+0000", 2024-08-17T17:52:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0036962319999999998 2024-08-17T17:52:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:83258 45.d 45:b7be69cf:::benchmark_data_smithi130_172551_object83257:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:52:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:38.683681+0000", 2024-08-17T17:52:14.136 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 215.43032046900001, 2024-08-17T17:52:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.467982847, 2024-08-17T17:52:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 83258 2024-08-17T17:52:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.137 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.683681+0000", 2024-08-17T17:52:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.683681+0000", 2024-08-17T17:52:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.138 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.683682+0000", 2024-08-17T17:52:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.722e-06 2024-08-17T17:52:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.683706+0000", 2024-08-17T17:52:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4006999999999999e-05 2024-08-17T17:52:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.139 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.683720+0000", 2024-08-17T17:52:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4134000000000001e-05 2024-08-17T17:52:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.683728+0000", 2024-08-17T17:52:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.25e-06 2024-08-17T17:52:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.683810+0000", 2024-08-17T17:52:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2373999999999994e-05 2024-08-17T17:52:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.140 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.683824+0000", 2024-08-17T17:52:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3463e-05 2024-08-17T17:52:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.643503+0000", 2024-08-17T17:52:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.95967992899999999 2024-08-17T17:52:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.643511+0000", 2024-08-17T17:52:14.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5320000000000001e-06 2024-08-17T17:52:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.128924+0000", 2024-08-17T17:52:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4854133780000001 2024-08-17T17:52:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.129096+0000", 2024-08-17T17:52:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017131000000000001 2024-08-17T17:52:14.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:52:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.129309+0000", 2024-08-17T17:52:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000213177 2024-08-17T17:52:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.143714+0000", 2024-08-17T17:52:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014404884 2024-08-17T17:52:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.143726+0000", 2024-08-17T17:52:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2249000000000001e-05 2024-08-17T17:52:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.147659+0000", 2024-08-17T17:52:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039328599999999998 2024-08-17T17:52:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.144 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.147813+0000", 2024-08-17T17:52:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015405200000000001 2024-08-17T17:52:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.151663+0000", 2024-08-17T17:52:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0038505259999999999 2024-08-17T17:52:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.145 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:83264 45.d 45:bde6869b:::benchmark_data_smithi130_172551_object83263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:52:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:38.693805+0000", 2024-08-17T17:52:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 215.42019627600001, 2024-08-17T17:52:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.471907217, 2024-08-17T17:52:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.146 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 83264 2024-08-17T17:52:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.693805+0000", 2024-08-17T17:52:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.693805+0000", 2024-08-17T17:52:14.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.693806+0000", 2024-08-17T17:52:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.638e-06 2024-08-17T17:52:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.693831+0000", 2024-08-17T17:52:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4363e-05 2024-08-17T17:52:14.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.693845+0000", 2024-08-17T17:52:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4168000000000001e-05 2024-08-17T17:52:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.693852+0000", 2024-08-17T17:52:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5059999999999998e-06 2024-08-17T17:52:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.693924+0000", 2024-08-17T17:52:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1962e-05 2024-08-17T17:52:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.693937+0000", 2024-08-17T17:52:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2927e-05 2024-08-17T17:52:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.150 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.643531+0000", 2024-08-17T17:52:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94959395899999999 2024-08-17T17:52:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.643539+0000", 2024-08-17T17:52:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3780000000000003e-06 2024-08-17T17:52:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.132610+0000", 2024-08-17T17:52:14.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4890710739999999 2024-08-17T17:52:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.132739+0000", 2024-08-17T17:52:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000129047 2024-08-17T17:52:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:52:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.132894+0000", 2024-08-17T17:52:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015534300000000001 2024-08-17T17:52:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.143731+0000", 2024-08-17T17:52:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010836722999999999 2024-08-17T17:52:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.143762+0000", 2024-08-17T17:52:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0845e-05 2024-08-17T17:52:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.157252+0000", 2024-08-17T17:52:14.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013490181 2024-08-17T17:52:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.157528+0000", 2024-08-17T17:52:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00027577500000000002 2024-08-17T17:52:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.165712+0000", 2024-08-17T17:52:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0081843279999999994 2024-08-17T17:52:14.154 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:83274 45.0 45:0c74311e:::benchmark_data_smithi130_172551_object83273:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:52:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:38.727132+0000", 2024-08-17T17:52:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 215.386868685, 2024-08-17T17:52:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3657750229999999, 2024-08-17T17:52:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.155 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 83274 2024-08-17T17:52:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.727132+0000", 2024-08-17T17:52:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.156 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.727132+0000", 2024-08-17T17:52:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.727133+0000", 2024-08-17T17:52:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0550000000000001e-06 2024-08-17T17:52:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.727154+0000", 2024-08-17T17:52:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0962999999999999e-05 2024-08-17T17:52:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.727167+0000", 2024-08-17T17:52:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.269e-05 2024-08-17T17:52:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.158 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.727173+0000", 2024-08-17T17:52:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1600000000000003e-06 2024-08-17T17:52:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.727246+0000", 2024-08-17T17:52:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2434000000000005e-05 2024-08-17T17:52:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.727258+0000", 2024-08-17T17:52:14.159 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2381999999999999e-05 2024-08-17T17:52:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.642451+0000", 2024-08-17T17:52:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.91519264499999997 2024-08-17T17:52:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.642460+0000", 2024-08-17T17:52:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4420000000000004e-06 2024-08-17T17:52:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.160 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.062571+0000", 2024-08-17T17:52:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.420110918 2024-08-17T17:52:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.062780+0000", 2024-08-17T17:52:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020887600000000001 2024-08-17T17:52:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:52:14.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.062987+0000", 2024-08-17T17:52:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020701699999999999 2024-08-17T17:52:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.065926+0000", 2024-08-17T17:52:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029391719999999999 2024-08-17T17:52:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.068725+0000", 2024-08-17T17:52:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027984329999999999 2024-08-17T17:52:14.162 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.088377+0000", 2024-08-17T17:52:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019652539 2024-08-17T17:52:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.088535+0000", 2024-08-17T17:52:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015805600000000001 2024-08-17T17:52:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.163 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.092907+0000", 2024-08-17T17:52:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0043722409999999998 2024-08-17T17:52:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108413 45.0 45:09fc7132:::benchmark_data_smithi130_172551_object3606:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:52:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:26.524118+0000", 2024-08-17T17:52:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 167.589883474, 2024-08-17T17:52:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.317561502, 2024-08-17T17:52:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.164 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108413 2024-08-17T17:52:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.524118+0000", 2024-08-17T17:52:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.165 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.524118+0000", 2024-08-17T17:52:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.524119+0000", 2024-08-17T17:52:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7570000000000001e-06 2024-08-17T17:52:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.524126+0000", 2024-08-17T17:52:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8809999999999998e-06 2024-08-17T17:52:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.524139+0000", 2024-08-17T17:52:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.252e-05 2024-08-17T17:52:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.524147+0000", 2024-08-17T17:52:14.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4260000000000003e-06 2024-08-17T17:52:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.697702+0000", 2024-08-17T17:52:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.17355443100000001 2024-08-17T17:52:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:52:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.699245+0000", 2024-08-17T17:52:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0015431329999999999 2024-08-17T17:52:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.745402+0000", 2024-08-17T17:52:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.046157255000000001 2024-08-17T17:52:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.745490+0000", 2024-08-17T17:52:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7861000000000005e-05 2024-08-17T17:52:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:52:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.745607+0000", 2024-08-17T17:52:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000117614 2024-08-17T17:52:14.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.793097+0000", 2024-08-17T17:52:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.047489189000000001 2024-08-17T17:52:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.837029+0000", 2024-08-17T17:52:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.043932589000000001 2024-08-17T17:52:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.170 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.837129+0000", 2024-08-17T17:52:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9783000000000003e-05 2024-08-17T17:52:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.837197+0000", 2024-08-17T17:52:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7507000000000003e-05 2024-08-17T17:52:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.841679+0000", 2024-08-17T17:52:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0044825560000000004 2024-08-17T17:52:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108418 45.0 45:0b767fca:::benchmark_data_smithi130_172551_object3611:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:52:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:26.533633+0000", 2024-08-17T17:52:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 167.58036777199999, 2024-08-17T17:52:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.31202963, 2024-08-17T17:52:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.172 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108418 2024-08-17T17:52:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.533633+0000", 2024-08-17T17:52:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.533633+0000", 2024-08-17T17:52:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.533635+0000", 2024-08-17T17:52:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.655e-06 2024-08-17T17:52:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.533641+0000", 2024-08-17T17:52:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7490000000000004e-06 2024-08-17T17:52:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.533657+0000", 2024-08-17T17:52:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6475000000000001e-05 2024-08-17T17:52:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.533668+0000", 2024-08-17T17:52:14.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0705e-05 2024-08-17T17:52:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.699295+0000", 2024-08-17T17:52:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.165626936 2024-08-17T17:52:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:52:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.700492+0000", 2024-08-17T17:52:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001196903 2024-08-17T17:52:14.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.757371+0000", 2024-08-17T17:52:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.056879486999999999 2024-08-17T17:52:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.757469+0000", 2024-08-17T17:52:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8193e-05 2024-08-17T17:52:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:52:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.757605+0000", 2024-08-17T17:52:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013586499999999999 2024-08-17T17:52:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.793106+0000", 2024-08-17T17:52:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.035500907999999998 2024-08-17T17:52:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.837083+0000", 2024-08-17T17:52:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.043977117000000003 2024-08-17T17:52:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.841787+0000", 2024-08-17T17:52:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0047041160000000004 2024-08-17T17:52:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.841852+0000", 2024-08-17T17:52:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4290999999999998e-05 2024-08-17T17:52:14.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.845663+0000", 2024-08-17T17:52:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0038112300000000001 2024-08-17T17:52:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108431 45.0 45:0e82cb6d:::benchmark_data_smithi130_172551_object3624:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:52:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:26.557141+0000", 2024-08-17T17:52:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 167.556860296, 2024-08-17T17:52:14.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.29216014499999998, 2024-08-17T17:52:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108431 2024-08-17T17:52:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.557141+0000", 2024-08-17T17:52:14.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.557141+0000", 2024-08-17T17:52:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.557142+0000", 2024-08-17T17:52:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.624e-06 2024-08-17T17:52:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.557150+0000", 2024-08-17T17:52:14.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6820000000000003e-06 2024-08-17T17:52:14.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.557163+0000", 2024-08-17T17:52:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3321e-05 2024-08-17T17:52:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.557173+0000", 2024-08-17T17:52:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1710000000000008e-06 2024-08-17T17:52:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.700537+0000", 2024-08-17T17:52:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14336425999999999 2024-08-17T17:52:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:52:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.703817+0000", 2024-08-17T17:52:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00328021 2024-08-17T17:52:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.772051+0000", 2024-08-17T17:52:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.068233550000000004 2024-08-17T17:52:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.772157+0000", 2024-08-17T17:52:14.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000106643 2024-08-17T17:52:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:52:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.772300+0000", 2024-08-17T17:52:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014247500000000001 2024-08-17T17:52:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.796714+0000", 2024-08-17T17:52:14.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.024414048000000001 2024-08-17T17:52:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.841738+0000", 2024-08-17T17:52:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.045024649 2024-08-17T17:52:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.845720+0000", 2024-08-17T17:52:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003981849 2024-08-17T17:52:14.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.845783+0000", 2024-08-17T17:52:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2738999999999997e-05 2024-08-17T17:52:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.849301+0000", 2024-08-17T17:52:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003517924 2024-08-17T17:52:14.190 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:110550 45.0 45:0d013c34:::benchmark_data_smithi130_172551_object5743:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:52:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:31.580582+0000", 2024-08-17T17:52:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 162.533418977, 2024-08-17T17:52:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.29428469600000001, 2024-08-17T17:52:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.191 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 110550 2024-08-17T17:52:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.580582+0000", 2024-08-17T17:52:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.192 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.580582+0000", 2024-08-17T17:52:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.580584+0000", 2024-08-17T17:52:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.669e-06 2024-08-17T17:52:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.580590+0000", 2024-08-17T17:52:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1190000000000002e-06 2024-08-17T17:52:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.580602+0000", 2024-08-17T17:52:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1929999999999999e-05 2024-08-17T17:52:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.580609+0000", 2024-08-17T17:52:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9519999999999997e-06 2024-08-17T17:52:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.589964+0000", 2024-08-17T17:52:14.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0093548780000000005 2024-08-17T17:52:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:52:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.590612+0000", 2024-08-17T17:52:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00064793499999999998 2024-08-17T17:52:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.815758+0000", 2024-08-17T17:52:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22514637200000001 2024-08-17T17:52:14.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.815885+0000", 2024-08-17T17:52:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000127186 2024-08-17T17:52:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:52:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.816017+0000", 2024-08-17T17:52:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013159900000000001 2024-08-17T17:52:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.831696+0000", 2024-08-17T17:52:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015679761 2024-08-17T17:52:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.831748+0000", 2024-08-17T17:52:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1882999999999998e-05 2024-08-17T17:52:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.869763+0000", 2024-08-17T17:52:14.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.038014394 2024-08-17T17:52:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.870234+0000", 2024-08-17T17:52:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00047148699999999998 2024-08-17T17:52:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.874867+0000", 2024-08-17T17:52:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046325309999999996 2024-08-17T17:52:14.198 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:110573 45.0 45:0bf98e72:::benchmark_data_smithi130_172551_object5766:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:52:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:31.631653+0000", 2024-08-17T17:52:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 162.48234761200001, 2024-08-17T17:52:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.32332876199999999, 2024-08-17T17:52:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.199 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 110573 2024-08-17T17:52:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.631653+0000", 2024-08-17T17:52:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.631653+0000", 2024-08-17T17:52:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.631655+0000", 2024-08-17T17:52:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5600000000000001e-06 2024-08-17T17:52:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.631660+0000", 2024-08-17T17:52:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0849999999999996e-06 2024-08-17T17:52:14.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.631676+0000", 2024-08-17T17:52:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5478e-05 2024-08-17T17:52:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.631683+0000", 2024-08-17T17:52:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1210000000000001e-06 2024-08-17T17:52:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.202 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.819618+0000", 2024-08-17T17:52:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.18793518400000001 2024-08-17T17:52:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:52:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.823985+0000", 2024-08-17T17:52:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0043673330000000002 2024-08-17T17:52:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.882909+0000", 2024-08-17T17:52:14.203 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.058924218 2024-08-17T17:52:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.882996+0000", 2024-08-17T17:52:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6561e-05 2024-08-17T17:52:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:52:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.883084+0000", 2024-08-17T17:52:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7651000000000005e-05 2024-08-17T17:52:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.920650+0000", 2024-08-17T17:52:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.037566135 2024-08-17T17:52:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.947225+0000", 2024-08-17T17:52:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.026575686000000001 2024-08-17T17:52:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.947276+0000", 2024-08-17T17:52:14.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0358999999999997e-05 2024-08-17T17:52:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.947426+0000", 2024-08-17T17:52:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015055400000000001 2024-08-17T17:52:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:31.954982+0000", 2024-08-17T17:52:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0075558370000000001 2024-08-17T17:52:14.206 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:111545 45.0 45:06dac3c9:::benchmark_data_smithi130_172551_object6738:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:52:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:33.783364+0000", 2024-08-17T17:52:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 160.33063710600001, 2024-08-17T17:52:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.31075924100000002, 2024-08-17T17:52:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.207 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 111545 2024-08-17T17:52:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.783364+0000", 2024-08-17T17:52:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.208 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.783364+0000", 2024-08-17T17:52:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.783366+0000", 2024-08-17T17:52:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.604e-06 2024-08-17T17:52:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.783371+0000", 2024-08-17T17:52:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5559999999999998e-06 2024-08-17T17:52:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.783383+0000", 2024-08-17T17:52:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.172e-05 2024-08-17T17:52:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.211 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.783391+0000", 2024-08-17T17:52:14.211 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8779999999999998e-06 2024-08-17T17:52:14.211 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.211 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.211 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.211 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.937483+0000", 2024-08-17T17:52:14.211 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.154091861 2024-08-17T17:52:14.211 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.211 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.211 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:52:14.211 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.938344+0000", 2024-08-17T17:52:14.212 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00086137300000000002 2024-08-17T17:52:14.212 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.212 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.212 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.212 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.979600+0000", 2024-08-17T17:52:14.212 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.041256412999999999 2024-08-17T17:52:14.212 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.212 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.212 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.212 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.979821+0000", 2024-08-17T17:52:14.212 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00022081800000000001 2024-08-17T17:52:14.212 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.213 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.213 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:52:14.213 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.980181+0000", 2024-08-17T17:52:14.213 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00035984599999999998 2024-08-17T17:52:14.213 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.213 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.213 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.213 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.008960+0000", 2024-08-17T17:52:14.213 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028778556 2024-08-17T17:52:14.213 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.213 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.213 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.213 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.078579+0000", 2024-08-17T17:52:14.214 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.069619173000000006 2024-08-17T17:52:14.214 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.214 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.214 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.214 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.090264+0000", 2024-08-17T17:52:14.214 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011684831999999999 2024-08-17T17:52:14.214 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.214 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.214 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.214 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.090594+0000", 2024-08-17T17:52:14.215 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000330768 2024-08-17T17:52:14.215 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.215 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.215 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.215 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.094123+0000", 2024-08-17T17:52:14.215 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0035288429999999998 2024-08-17T17:52:14.215 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.215 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.215 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.215 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.215 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.215 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:111621 45.0 45:0ed4ac4c:::benchmark_data_smithi130_172551_object6814:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:52:14.216 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:33.985569+0000", 2024-08-17T17:52:14.216 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 160.12843159100001, 2024-08-17T17:52:14.216 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.28945718599999998, 2024-08-17T17:52:14.216 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.216 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.216 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.216 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.216 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.216 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.216 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 111621 2024-08-17T17:52:14.216 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.216 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.216 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.216 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.217 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.985569+0000", 2024-08-17T17:52:14.217 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.217 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.217 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.217 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.985569+0000", 2024-08-17T17:52:14.217 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.217 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.217 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.217 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.985571+0000", 2024-08-17T17:52:14.217 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6670000000000001e-06 2024-08-17T17:52:14.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.218 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.218 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.985578+0000", 2024-08-17T17:52:14.218 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4099999999999996e-06 2024-08-17T17:52:14.218 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.218 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.218 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.985626+0000", 2024-08-17T17:52:14.218 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8384999999999998e-05 2024-08-17T17:52:14.218 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.219 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.219 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:33.985635+0000", 2024-08-17T17:52:14.219 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4360000000000008e-06 2024-08-17T17:52:14.219 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.219 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.219 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.219 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.121609+0000", 2024-08-17T17:52:14.219 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.135973857 2024-08-17T17:52:14.219 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.219 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.219 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:52:14.219 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.122435+0000", 2024-08-17T17:52:14.220 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00082575599999999997 2024-08-17T17:52:14.220 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.220 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.220 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.220 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.166020+0000", 2024-08-17T17:52:14.220 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.043585327 2024-08-17T17:52:14.220 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.220 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.220 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.220 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.166139+0000", 2024-08-17T17:52:14.221 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000118589 2024-08-17T17:52:14.221 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.221 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.221 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:52:14.221 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.166310+0000", 2024-08-17T17:52:14.221 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017074899999999999 2024-08-17T17:52:14.221 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.221 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.221 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.221 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.229771+0000", 2024-08-17T17:52:14.221 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.063461135000000002 2024-08-17T17:52:14.221 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.271052+0000", 2024-08-17T17:52:14.222 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.041280772 2024-08-17T17:52:14.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.271169+0000", 2024-08-17T17:52:14.222 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000117433 2024-08-17T17:52:14.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.271381+0000", 2024-08-17T17:52:14.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00021222999999999999 2024-08-17T17:52:14.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.275027+0000", 2024-08-17T17:52:14.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0036454399999999998 2024-08-17T17:52:14.223 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.223 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.223 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.224 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.224 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.224 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:111704 45.0 45:0a1efdea:::benchmark_data_smithi130_172551_object6897:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:52:14.224 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:34.155590+0000", 2024-08-17T17:52:14.224 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 159.95841101600001, 2024-08-17T17:52:14.224 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.30355789300000002, 2024-08-17T17:52:14.224 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.224 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.224 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.224 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.224 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.224 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.224 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 111704 2024-08-17T17:52:14.225 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.225 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.225 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.225 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.225 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.155590+0000", 2024-08-17T17:52:14.225 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.225 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.225 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.225 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.225 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.155590+0000", 2024-08-17T17:52:14.225 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.225 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.225 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.226 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.226 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.155592+0000", 2024-08-17T17:52:14.226 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9489999999999999e-06 2024-08-17T17:52:14.226 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.226 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.226 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.226 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.155599+0000", 2024-08-17T17:52:14.226 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5889999999999999e-06 2024-08-17T17:52:14.226 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.226 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.226 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.226 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.155611+0000", 2024-08-17T17:52:14.227 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2231e-05 2024-08-17T17:52:14.227 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.227 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.227 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.155618+0000", 2024-08-17T17:52:14.227 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8510000000000001e-06 2024-08-17T17:52:14.227 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.227 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.227 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.303096+0000", 2024-08-17T17:52:14.227 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14747803500000001 2024-08-17T17:52:14.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.228 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.228 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:52:14.228 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.304231+0000", 2024-08-17T17:52:14.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001135626 2024-08-17T17:52:14.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.228 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.228 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.228 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.362334+0000", 2024-08-17T17:52:14.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.058102839000000003 2024-08-17T17:52:14.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.362429+0000", 2024-08-17T17:52:14.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4962000000000003e-05 2024-08-17T17:52:14.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:52:14.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.362509+0000", 2024-08-17T17:52:14.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9914999999999996e-05 2024-08-17T17:52:14.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.378068+0000", 2024-08-17T17:52:14.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015559373 2024-08-17T17:52:14.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.230 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.230 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.455438+0000", 2024-08-17T17:52:14.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.077369678999999997 2024-08-17T17:52:14.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.230 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.230 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.455526+0000", 2024-08-17T17:52:14.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8200000000000003e-05 2024-08-17T17:52:14.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.455607+0000", 2024-08-17T17:52:14.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0563999999999996e-05 2024-08-17T17:52:14.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.459148+0000", 2024-08-17T17:52:14.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0035410799999999998 2024-08-17T17:52:14.231 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.232 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.232 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.232 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:111711 45.0 45:0e493dcf:::benchmark_data_smithi130_172551_object6904:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:52:14.232 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:34.171456+0000", 2024-08-17T17:52:14.232 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 159.94254479, 2024-08-17T17:52:14.232 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.29106030399999999, 2024-08-17T17:52:14.232 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.232 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.232 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.233 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.233 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.233 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.233 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 111711 2024-08-17T17:52:14.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.233 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.233 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.233 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.171456+0000", 2024-08-17T17:52:14.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.171456+0000", 2024-08-17T17:52:14.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.171459+0000", 2024-08-17T17:52:14.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4140000000000001e-06 2024-08-17T17:52:14.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.235 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.235 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.171466+0000", 2024-08-17T17:52:14.235 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2409999999999998e-06 2024-08-17T17:52:14.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.235 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.235 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.235 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.171482+0000", 2024-08-17T17:52:14.235 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5943e-05 2024-08-17T17:52:14.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.235 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.235 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.171490+0000", 2024-08-17T17:52:14.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5560000000000008e-06 2024-08-17T17:52:14.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.304275+0000", 2024-08-17T17:52:14.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.13278488099999999 2024-08-17T17:52:14.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:52:14.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.307096+0000", 2024-08-17T17:52:14.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028208690000000002 2024-08-17T17:52:14.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.373341+0000", 2024-08-17T17:52:14.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.066245233000000001 2024-08-17T17:52:14.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.373448+0000", 2024-08-17T17:52:14.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010676800000000001 2024-08-17T17:52:14.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:52:14.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.373614+0000", 2024-08-17T17:52:14.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016540599999999999 2024-08-17T17:52:14.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.412743+0000", 2024-08-17T17:52:14.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.039129882999999997 2024-08-17T17:52:14.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.455483+0000", 2024-08-17T17:52:14.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.04273979 2024-08-17T17:52:14.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.459202+0000", 2024-08-17T17:52:14.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0037182449999999998 2024-08-17T17:52:14.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.240 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.459253+0000", 2024-08-17T17:52:14.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1706000000000003e-05 2024-08-17T17:52:14.240 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.240 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.240 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.240 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.462517+0000", 2024-08-17T17:52:14.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032633689999999999 2024-08-17T17:52:14.240 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.240 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.240 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.240 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.241 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.241 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:111715 45.0 45:0714473e:::benchmark_data_smithi130_172551_object6908:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:52:14.241 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:34.178977+0000", 2024-08-17T17:52:14.241 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 159.93502381799999, 2024-08-17T17:52:14.241 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.291867724, 2024-08-17T17:52:14.241 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.241 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.241 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.241 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.241 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.241 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.242 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 111715 2024-08-17T17:52:14.242 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.242 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.242 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.178977+0000", 2024-08-17T17:52:14.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.242 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.242 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.178977+0000", 2024-08-17T17:52:14.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.178978+0000", 2024-08-17T17:52:14.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2900000000000002e-07 2024-08-17T17:52:14.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.178982+0000", 2024-08-17T17:52:14.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4220000000000002e-06 2024-08-17T17:52:14.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.178995+0000", 2024-08-17T17:52:14.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2791e-05 2024-08-17T17:52:14.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.179002+0000", 2024-08-17T17:52:14.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0990000000000001e-06 2024-08-17T17:52:14.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.309585+0000", 2024-08-17T17:52:14.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.13058278800000001 2024-08-17T17:52:14.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:52:14.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.311575+0000", 2024-08-17T17:52:14.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0019897360000000002 2024-08-17T17:52:14.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.246 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.246 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.400410+0000", 2024-08-17T17:52:14.246 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.088834725000000003 2024-08-17T17:52:14.246 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.246 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.246 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.400605+0000", 2024-08-17T17:52:14.246 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000195515 2024-08-17T17:52:14.246 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:52:14.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.401066+0000", 2024-08-17T17:52:14.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00046045500000000002 2024-08-17T17:52:14.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.416526+0000", 2024-08-17T17:52:14.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.01546 2024-08-17T17:52:14.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.248 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.467215+0000", 2024-08-17T17:52:14.248 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.050689873000000003 2024-08-17T17:52:14.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.248 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.248 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.467259+0000", 2024-08-17T17:52:14.248 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3022000000000002e-05 2024-08-17T17:52:14.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.248 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.248 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.467315+0000", 2024-08-17T17:52:14.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6678000000000002e-05 2024-08-17T17:52:14.249 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.249 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.249 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.249 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:34.470845+0000", 2024-08-17T17:52:14.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003529791 2024-08-17T17:52:14.249 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.249 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.249 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.249 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.249 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.249 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:123156 45.d 45:b1bbc672:::benchmark_data_smithi130_172551_object18349:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1204)", 2024-08-17T17:52:14.250 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:51:24.170478+0000", 2024-08-17T17:52:14.250 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 49.943523120000002, 2024-08-17T17:52:14.250 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0378718060000001, 2024-08-17T17:52:14.250 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.250 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.250 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T17:52:14.250 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.250 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.250 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.250 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 123156 2024-08-17T17:52:14.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.250 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.251 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.170478+0000", 2024-08-17T17:52:14.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.251 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.170478+0000", 2024-08-17T17:52:14.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.170480+0000", 2024-08-17T17:52:14.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9369999999999998e-06 2024-08-17T17:52:14.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.170487+0000", 2024-08-17T17:52:14.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3969999999999997e-06 2024-08-17T17:52:14.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.170501+0000", 2024-08-17T17:52:14.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3906e-05 2024-08-17T17:52:14.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.170509+0000", 2024-08-17T17:52:14.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5979999999999999e-06 2024-08-17T17:52:14.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.194407+0000", 2024-08-17T17:52:14.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023897966 2024-08-17T17:52:14.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.194556+0000", 2024-08-17T17:52:14.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014946600000000001 2024-08-17T17:52:14.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:52:14.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.194699+0000", 2024-08-17T17:52:14.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000142614 2024-08-17T17:52:14.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.200425+0000", 2024-08-17T17:52:14.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0057257460000000003 2024-08-17T17:52:14.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.200435+0000", 2024-08-17T17:52:14.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0519999999999999e-05 2024-08-17T17:52:14.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.208269+0000", 2024-08-17T17:52:14.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.007834267 2024-08-17T17:52:14.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.208350+0000", 2024-08-17T17:52:14.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0389000000000006e-05 2024-08-17T17:52:14.256 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.257 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.257 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.257 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:123159 45.d 45:b2552ba5:::benchmark_data_smithi130_172551_object18352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1205)", 2024-08-17T17:52:14.257 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:51:24.616417+0000", 2024-08-17T17:52:14.257 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 49.497584275999998, 2024-08-17T17:52:14.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.59829702900000004, 2024-08-17T17:52:14.257 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.257 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.257 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.258 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.258 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.258 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.258 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 123159 2024-08-17T17:52:14.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.258 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616417+0000", 2024-08-17T17:52:14.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.259 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.259 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616417+0000", 2024-08-17T17:52:14.259 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.259 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.259 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616418+0000", 2024-08-17T17:52:14.259 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.561e-06 2024-08-17T17:52:14.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616425+0000", 2024-08-17T17:52:14.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5950000000000004e-06 2024-08-17T17:52:14.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616439+0000", 2024-08-17T17:52:14.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3927000000000001e-05 2024-08-17T17:52:14.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616446+0000", 2024-08-17T17:52:14.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6810000000000001e-06 2024-08-17T17:52:14.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616515+0000", 2024-08-17T17:52:14.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9352999999999998e-05 2024-08-17T17:52:14.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616527+0000", 2024-08-17T17:52:14.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2452e-05 2024-08-17T17:52:14.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.208825+0000", 2024-08-17T17:52:14.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.59229791399999998 2024-08-17T17:52:14.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.208935+0000", 2024-08-17T17:52:14.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010940100000000001 2024-08-17T17:52:14.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:52:14.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.209032+0000", 2024-08-17T17:52:14.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7205000000000002e-05 2024-08-17T17:52:14.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214543+0000", 2024-08-17T17:52:14.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005510923 2024-08-17T17:52:14.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.264 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214577+0000", 2024-08-17T17:52:14.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4337e-05 2024-08-17T17:52:14.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.264 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214620+0000", 2024-08-17T17:52:14.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3101e-05 2024-08-17T17:52:14.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214670+0000", 2024-08-17T17:52:14.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9857000000000001e-05 2024-08-17T17:52:14.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214714+0000", 2024-08-17T17:52:14.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3721999999999999e-05 2024-08-17T17:52:14.265 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.265 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.265 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.266 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:123165 45.d 45:b3c02af0:::benchmark_data_smithi130_172551_object18358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1205)", 2024-08-17T17:52:14.266 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:51:24.775611+0000", 2024-08-17T17:52:14.266 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 49.338389601000003, 2024-08-17T17:52:14.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.442593493, 2024-08-17T17:52:14.266 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.266 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.266 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.266 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.266 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.266 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.267 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 123165 2024-08-17T17:52:14.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.267 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775611+0000", 2024-08-17T17:52:14.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775611+0000", 2024-08-17T17:52:14.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.268 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775613+0000", 2024-08-17T17:52:14.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.536e-06 2024-08-17T17:52:14.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.268 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775619+0000", 2024-08-17T17:52:14.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9959999999999999e-06 2024-08-17T17:52:14.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775633+0000", 2024-08-17T17:52:14.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4225e-05 2024-08-17T17:52:14.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775641+0000", 2024-08-17T17:52:14.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.391e-06 2024-08-17T17:52:14.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775698+0000", 2024-08-17T17:52:14.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7506000000000001e-05 2024-08-17T17:52:14.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775707+0000", 2024-08-17T17:52:14.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.687e-06 2024-08-17T17:52:14.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.212049+0000", 2024-08-17T17:52:14.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.436342444 2024-08-17T17:52:14.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.212137+0000", 2024-08-17T17:52:14.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8209999999999997e-05 2024-08-17T17:52:14.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:52:14.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.212229+0000", 2024-08-17T17:52:14.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1496999999999995e-05 2024-08-17T17:52:14.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214764+0000", 2024-08-17T17:52:14.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025347389999999998 2024-08-17T17:52:14.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218082+0000", 2024-08-17T17:52:14.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0033186040000000002 2024-08-17T17:52:14.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218097+0000", 2024-08-17T17:52:14.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4748e-05 2024-08-17T17:52:14.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218154+0000", 2024-08-17T17:52:14.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7399999999999999e-05 2024-08-17T17:52:14.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218205+0000", 2024-08-17T17:52:14.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0510000000000003e-05 2024-08-17T17:52:14.273 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.273 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.273 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.274 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:123166 45.d 45:b8803e29:::benchmark_data_smithi130_172551_object18359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1205)", 2024-08-17T17:52:14.274 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:51:24.792323+0000", 2024-08-17T17:52:14.274 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 49.321677739000002, 2024-08-17T17:52:14.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.42608478500000002, 2024-08-17T17:52:14.274 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.274 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.274 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.274 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.274 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.275 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.275 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 123166 2024-08-17T17:52:14.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.275 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.275 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.275 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792323+0000", 2024-08-17T17:52:14.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.275 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.275 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792323+0000", 2024-08-17T17:52:14.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792325+0000", 2024-08-17T17:52:14.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.903e-06 2024-08-17T17:52:14.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792331+0000", 2024-08-17T17:52:14.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0929999999999999e-06 2024-08-17T17:52:14.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792345+0000", 2024-08-17T17:52:14.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3763e-05 2024-08-17T17:52:14.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792352+0000", 2024-08-17T17:52:14.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3939999999999998e-06 2024-08-17T17:52:14.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792425+0000", 2024-08-17T17:52:14.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2426000000000001e-05 2024-08-17T17:52:14.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792437+0000", 2024-08-17T17:52:14.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1905e-05 2024-08-17T17:52:14.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214813+0000", 2024-08-17T17:52:14.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.42237641500000001 2024-08-17T17:52:14.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214895+0000", 2024-08-17T17:52:14.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1496999999999996e-05 2024-08-17T17:52:14.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.280 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:52:14.280 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.215021+0000", 2024-08-17T17:52:14.280 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012648100000000001 2024-08-17T17:52:14.280 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.280 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.280 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.280 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218257+0000", 2024-08-17T17:52:14.280 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032360790000000002 2024-08-17T17:52:14.280 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.280 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.280 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218313+0000", 2024-08-17T17:52:14.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5809e-05 2024-08-17T17:52:14.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218327+0000", 2024-08-17T17:52:14.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3713999999999999e-05 2024-08-17T17:52:14.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218371+0000", 2024-08-17T17:52:14.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3788000000000002e-05 2024-08-17T17:52:14.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.282 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.282 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218408+0000", 2024-08-17T17:52:14.282 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7518000000000002e-05 2024-08-17T17:52:14.282 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.282 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.282 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.283 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:123176 45.d 45:b4b9fb89:::benchmark_data_smithi130_172551_object18369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1205)", 2024-08-17T17:52:14.283 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:51:24.872841+0000", 2024-08-17T17:52:14.283 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 49.241160346000001, 2024-08-17T17:52:14.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35194215600000001, 2024-08-17T17:52:14.283 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:52:14.283 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:52:14.283 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:14.283 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:52:14.283 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:52:14.283 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:14.283 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 123176 2024-08-17T17:52:14.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.283 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:52:14.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:52:14.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872841+0000", 2024-08-17T17:52:14.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:52:14.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872841+0000", 2024-08-17T17:52:14.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:52:14.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:52:14.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872843+0000", 2024-08-17T17:52:14.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9350000000000001e-06 2024-08-17T17:52:14.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:52:14.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872847+0000", 2024-08-17T17:52:14.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8339999999999998e-06 2024-08-17T17:52:14.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:52:14.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872858+0000", 2024-08-17T17:52:14.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0899e-05 2024-08-17T17:52:14.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:52:14.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872865+0000", 2024-08-17T17:52:14.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3149999999999997e-06 2024-08-17T17:52:14.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872917+0000", 2024-08-17T17:52:14.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2454999999999998e-05 2024-08-17T17:52:14.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:52:14.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872926+0000", 2024-08-17T17:52:14.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1560000000000001e-06 2024-08-17T17:52:14.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:52:14.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218467+0000", 2024-08-17T17:52:14.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.34554088599999999 2024-08-17T17:52:14.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.288 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:52:14.288 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218550+0000", 2024-08-17T17:52:14.288 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2532000000000002e-05 2024-08-17T17:52:14.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.288 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:52:14.288 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218662+0000", 2024-08-17T17:52:14.288 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000112627 2024-08-17T17:52:14.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.289 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.289 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.289 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.224602+0000", 2024-08-17T17:52:14.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0059394620000000004 2024-08-17T17:52:14.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.289 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.289 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:14.289 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.224661+0000", 2024-08-17T17:52:14.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9670000000000003e-05 2024-08-17T17:52:14.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.289 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.289 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:52:14.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.224677+0000", 2024-08-17T17:52:14.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5271e-05 2024-08-17T17:52:14.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:52:14.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.224733+0000", 2024-08-17T17:52:14.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6258999999999998e-05 2024-08-17T17:52:14.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:52:14.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:52:14.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:52:14.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.224783+0000", 2024-08-17T17:52:14.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9855000000000003e-05 2024-08-17T17:52:14.291 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.291 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.291 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.291 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:52:14.291 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:52:14.291 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:52:14.313 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:14.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:14.415 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:14.437 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:14.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:14.539 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:14.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.563 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:14.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:14.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:14.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:14.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:14.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:14.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:14.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:14.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:14.935 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:14.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:14.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:15.036 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:15.058 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:15.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:15.151 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:15.160 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:15.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:15 smithi130 ceph-mon[105521]: pgmap v1743: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 19354/244116 objects degraded (7.928%); 14158/244116 objects misplaced (5.800%); 834 KiB/s, 47 objects/s recovering 2024-08-17T17:52:15.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:15 smithi130 ceph-mon[110068]: pgmap v1743: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 19354/244116 objects degraded (7.928%); 14158/244116 objects misplaced (5.800%); 834 KiB/s, 47 objects/s recovering 2024-08-17T17:52:15.182 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:15.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:15.248 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:52:15.249 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:52:15.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:15.308 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:15.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:15.410 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:15.431 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:15.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:15.533 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:15.555 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:15.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:15.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:15 smithi179 ceph-mon[93519]: pgmap v1743: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 19354/244116 objects degraded (7.928%); 14158/244116 objects misplaced (5.800%); 834 KiB/s, 47 objects/s recovering 2024-08-17T17:52:15.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:15.679 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:15.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:15.750 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:52:15.781 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:15.804 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:15.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:15.905 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:15.927 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:15.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:15.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:15.989 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:16.024 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:16.029 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:16.050 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:16.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:16.147 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:16 smithi179 ceph-mon[93519]: from='client.35529 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:52:16.147 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 19354/244116 objects degraded (7.928%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:16.152 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:16.174 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:16.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:16.276 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:16.298 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:16.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:16.400 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:16.413 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:16 smithi130 ceph-mon[105521]: from='client.35529 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:52:16.414 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 19354/244116 objects degraded (7.928%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:16.414 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:16 smithi130 ceph-mon[110068]: from='client.35529 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:52:16.414 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 19354/244116 objects degraded (7.928%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:16.423 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:16.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:16.524 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:16.546 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:16.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:16.647 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:16.676 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:16.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:16.778 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:16.800 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:16.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:16.901 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:16.923 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:16.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:16.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:17.025 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:17.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.047 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:17.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:17.148 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:17.171 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:17.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:17.273 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:17.296 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:17.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:17.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:17 smithi179 ceph-mon[93519]: pgmap v1744: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 103 op/s; 19304/243939 objects degraded (7.913%); 14124/243939 objects misplaced (5.790%); 819 KiB/s, 45 objects/s recovering 2024-08-17T17:52:17.398 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:17.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:17 smithi130 ceph-mon[105521]: pgmap v1744: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 103 op/s; 19304/243939 objects degraded (7.913%); 14124/243939 objects misplaced (5.790%); 819 KiB/s, 45 objects/s recovering 2024-08-17T17:52:17.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:17 smithi130 ceph-mon[110068]: pgmap v1744: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 103 op/s; 19304/243939 objects degraded (7.913%); 14124/243939 objects misplaced (5.790%); 819 KiB/s, 45 objects/s recovering 2024-08-17T17:52:17.429 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:17.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:17.521 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1743,"stamp":"2024-08-17T17:52:15.018709+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5331621861,"num_objects":81372,"num_object_clones":0,"num_object_copies":244116,"num_objects_missing_on_primary":4597,"num_objects_missing":4597,"num_objects_degraded":19354,"num_objects_misplaced":14158,"num_objects_unfound":0,"num_objects_dirty":81372,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":128542,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32938,"num_bytes_recovered":1980046349,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106345,"ondisk_log_size":106345,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":15065224,"kb_used_data":13744148,"kb_used_omap":295,"kb_used_meta":1320728,"kb_avail":547405688,"statfs":{"total":575970213888,"available":560543424512,"internally_reserved":0,"allocated":14074007552,"data_stored":14045133801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1352425935},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1,1,0,1,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-67305472,"num_objects":-1027,"num_object_clones":0,"num_object_copies":-3081,"num_objects_missing_on_primary":-68,"num_objects_missing":-68,"num_objects_degraded":-449,"num_objects_misplaced":-202,"num_objects_unfound":0,"num_objects_dirty":-1027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1027,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":458,"num_bytes_recovered":8323072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.741006"},"pg_stats":[{"pgid":"45.f","version":"1213'8023","reported_seq":18527,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:12.235430+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:52:12.235430+0000","last_peered":"2024-08-17T17:52:12.235430+0000","last_clean":"2024-08-17T17:52:12.235430+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:52:12.235430+0000","last_undegraded":"2024-08-17T17:52:12.235430+0000","last_fullsized":"2024-08-17T17:52:12.235430+0000","mapping_epoch":1194,"log_start":"1170'1131","ondisk_log_start":"1170'1131","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6892,"log_dups_size":0,"ondisk_log_size":6892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336920576,"num_objects":5141,"num_object_clones":0,"num_object_copies":15423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8023,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1213'7968","reported_seq":18421,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:12.953732+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:52:12.953732+0000","last_peered":"2024-08-17T17:52:12.953732+0000","last_clean":"2024-08-17T17:52:12.953732+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:52:12.953732+0000","last_undegraded":"2024-08-17T17:52:12.953732+0000","last_fullsized":"2024-08-17T17:52:12.953732+0000","mapping_epoch":1211,"log_start":"1170'932","ondisk_log_start":"1170'932","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7036,"log_dups_size":0,"ondisk_log_size":7036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329383936,"num_objects":5026,"num_object_clones":0,"num_object_copies":15078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7968,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1213'8210","reported_seq":19022,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:12.650899+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:52:12.650899+0000","last_peered":"2024-08-17T17:52:12.650899+0000","last_clean":"2024-08-17T17:52:12.650899+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:52:12.650899+0000","last_undegraded":"2024-08-17T17:52:12.650899+0000","last_fullsized":"2024-08-17T17:52:12.650899+0000","mapping_epoch":1205,"log_start":"1170'2268","ondisk_log_start":"1170'2268","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5942,"log_dups_size":0,"ondisk_log_size":5942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338624536,"num_objects":5168,"num_object_clones":0,"num_object_copies":15504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5168,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8210,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1213'7991","reported_seq":8045,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:13.010765+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:52:13.010765+0000","last_peered":"2024-08-17T17:52:13.010765+0000","last_clean":"2024-08-17T17:52:13.010765+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:52:13.010765+0000","last_undegraded":"2024-08-17T17:52:13.010765+0000","last_fullsized":"2024-08-17T17:52:13.010765+0000","mapping_epoch":1167,"log_start":"1170'1289","ondisk_log_start":"1170'1289","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6702,"log_dups_size":0,"ondisk_log_size":6702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328269824,"num_objects":5009,"num_object_clones":0,"num_object_copies":15027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7991,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1213'8065","reported_seq":13283,"reported_epoch":1213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:11.807106+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:52:11.807106+0000","last_peered":"2024-08-17T17:52:11.807106+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:52:11.807106+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1170'2126","ondisk_log_start":"1170'2126","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5939,"log_dups_size":0,"ondisk_log_size":5939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330891264,"num_objects":5049,"num_object_clones":0,"num_object_copies":15147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":5049,"num_objects_unfound":0,"num_objects_dirty":5049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8065,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":4531},{"shards":"1,4,5","objects":518}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1213'8065","reported_seq":8119,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:12.939627+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:52:12.939627+0000","last_peered":"2024-08-17T17:52:12.939627+0000","last_clean":"2024-08-17T17:52:12.939627+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:52:12.939627+0000","last_undegraded":"2024-08-17T17:52:12.939627+0000","last_fullsized":"2024-08-17T17:52:12.939627+0000","mapping_epoch":1167,"log_start":"1170'1377","ondisk_log_start":"1170'1377","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6688,"log_dups_size":0,"ondisk_log_size":6688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329580544,"num_objects":5029,"num_object_clones":0,"num_object_copies":15087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8065,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1213'8081","reported_seq":8135,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:12.964401+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:52:12.964401+0000","last_peered":"2024-08-17T17:52:12.964401+0000","last_clean":"2024-08-17T17:52:12.964401+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:52:12.964401+0000","last_undegraded":"2024-08-17T17:52:12.964401+0000","last_fullsized":"2024-08-17T17:52:12.964401+0000","mapping_epoch":1167,"log_start":"1170'1369","ondisk_log_start":"1170'1369","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6712,"log_dups_size":0,"ondisk_log_size":6712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337182720,"num_objects":5145,"num_object_clones":0,"num_object_copies":15435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8081,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1213'7977","reported_seq":12534,"reported_epoch":1213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:11.854733+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:52:11.854733+0000","last_peered":"2024-08-17T17:52:11.854733+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:52:11.854733+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1170'2075","ondisk_log_start":"1170'2075","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5902,"log_dups_size":0,"ondisk_log_size":5902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328531968,"num_objects":5013,"num_object_clones":0,"num_object_copies":15039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5319,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7977,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1482,"num_bytes_recovered":97124352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1626},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1213'7949","reported_seq":8013,"reported_epoch":1213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:12.712336+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:52:12.712336+0000","last_peered":"2024-08-17T17:52:12.712336+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:52:12.712336+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'1277","ondisk_log_start":"1170'1277","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6672,"log_dups_size":0,"ondisk_log_size":6672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336003072,"num_objects":5127,"num_object_clones":0,"num_object_copies":15381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7949,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":5127},{"shards":"1,6","objects":-706}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1213'7982","reported_seq":11033,"reported_epoch":1213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:12.986217+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:52:12.986217+0000","last_peered":"2024-08-17T17:52:12.986217+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:52:12.986217+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1170'2006","ondisk_log_start":"1170'2006","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5976,"log_dups_size":0,"ondisk_log_size":5976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340262912,"num_objects":5192,"num_object_clones":0,"num_object_copies":15576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7982,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":3631},{"shards":"1,3,5","objects":1561}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1213'8020","reported_seq":8126,"reported_epoch":1213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:12.971405+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:52:12.971405+0000","last_peered":"2024-08-17T17:52:12.971405+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:52:12.971405+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8020,"log_dups_size":0,"ondisk_log_size":8020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331218944,"num_objects":5054,"num_object_clones":0,"num_object_copies":15162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8020,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-99}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1213'7983","reported_seq":14150,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:13.547029+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:52:13.547029+0000","last_peered":"2024-08-17T17:52:13.547029+0000","last_clean":"2024-08-17T17:52:13.547029+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:52:13.547029+0000","last_undegraded":"2024-08-17T17:52:13.547029+0000","last_fullsized":"2024-08-17T17:52:13.547029+0000","mapping_epoch":1183,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7983,"log_dups_size":0,"ondisk_log_size":7983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331808768,"num_objects":5063,"num_object_clones":0,"num_object_copies":15189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7983,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1213'8015","reported_seq":8120,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:13.198919+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:52:13.198919+0000","last_peered":"2024-08-17T17:52:13.198919+0000","last_clean":"2024-08-17T17:52:13.198919+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:52:13.198919+0000","last_undegraded":"2024-08-17T17:52:13.198919+0000","last_fullsized":"2024-08-17T17:52:13.198919+0000","mapping_epoch":1167,"log_start":"1170'2091","ondisk_log_start":"1170'2091","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5924,"log_dups_size":0,"ondisk_log_size":5924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337182720,"num_objects":5145,"num_object_clones":0,"num_object_copies":15435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8015,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1213'7928","reported_seq":7982,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:12.960713+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:52:12.960713+0000","last_peered":"2024-08-17T17:52:12.960713+0000","last_clean":"2024-08-17T17:52:12.960713+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:52:12.960713+0000","last_undegraded":"2024-08-17T17:52:12.960713+0000","last_fullsized":"2024-08-17T17:52:12.960713+0000","mapping_epoch":1167,"log_start":"1170'925","ondisk_log_start":"1170'925","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7003,"log_dups_size":0,"ondisk_log_size":7003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328073216,"num_objects":5006,"num_object_clones":0,"num_object_copies":15018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7928,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1213'8125","reported_seq":16060,"reported_epoch":1213,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:12.347527+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:52:12.347527+0000","last_peered":"2024-08-17T17:52:12.347527+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:52:12.347527+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'1291","ondisk_log_start":"1170'1291","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6834,"log_dups_size":0,"ondisk_log_size":6834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343343104,"num_objects":5239,"num_object_clones":0,"num_object_copies":15717,"num_objects_missing_on_primary":4597,"num_objects_missing":4597,"num_objects_degraded":265,"num_objects_misplaced":9109,"num_objects_unfound":0,"num_objects_dirty":5239,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8125,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3549,"num_bytes_recovered":232587264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":13},{"shards":"0,4,6","objects":1356},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":3605}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1213'7882","reported_seq":21173,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:13.014526+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:52:13.014526+0000","last_peered":"2024-08-17T17:52:13.014526+0000","last_clean":"2024-08-17T17:52:13.014526+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:52:13.014526+0000","last_undegraded":"2024-08-17T17:52:13.014526+0000","last_fullsized":"2024-08-17T17:52:13.014526+0000","mapping_epoch":1196,"log_start":"1170'1969","ondisk_log_start":"1170'1969","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5913,"log_dups_size":0,"ondisk_log_size":5913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323747840,"num_objects":4940,"num_object_clones":0,"num_object_copies":14820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7882,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1881,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.346410+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:52:10.346410+0000","last_peered":"2024-08-17T17:52:10.346410+0000","last_clean":"2024-08-17T17:52:10.346410+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:52:10.346410+0000","last_undegraded":"2024-08-17T17:52:10.346410+0000","last_fullsized":"2024-08-17T17:52:10.346410+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":490,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280050+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:52:05.280050+0000","last_peered":"2024-08-17T17:52:05.280050+0000","last_clean":"2024-08-17T17:52:05.280050+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:52:05.280050+0000","last_undegraded":"2024-08-17T17:52:05.280050+0000","last_fullsized":"2024-08-17T17:52:05.280050+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1801,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.112293+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:52:10.112293+0000","last_peered":"2024-08-17T17:52:10.112293+0000","last_clean":"2024-08-17T17:52:10.112293+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:52:10.112293+0000","last_undegraded":"2024-08-17T17:52:10.112293+0000","last_fullsized":"2024-08-17T17:52:10.112293+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1825,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109379+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:52:10.109379+0000","last_peered":"2024-08-17T17:52:10.109379+0000","last_clean":"2024-08-17T17:52:10.109379+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:52:10.109379+0000","last_undegraded":"2024-08-17T17:52:10.109379+0000","last_fullsized":"2024-08-17T17:52:10.109379+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":52,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280633+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:52:05.280633+0000","last_peered":"2024-08-17T17:52:05.280633+0000","last_clean":"2024-08-17T17:52:05.280633+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:52:05.280633+0000","last_undegraded":"2024-08-17T17:52:05.280633+0000","last_fullsized":"2024-08-17T17:52:05.280633+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1218,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.110062+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:52:10.110062+0000","last_peered":"2024-08-17T17:52:10.110062+0000","last_clean":"2024-08-17T17:52:10.110062+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:52:10.110062+0000","last_undegraded":"2024-08-17T17:52:10.110062+0000","last_fullsized":"2024-08-17T17:52:10.110062+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":76,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.110678+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:52:10.110678+0000","last_peered":"2024-08-17T17:52:10.110678+0000","last_clean":"2024-08-17T17:52:10.110678+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:52:10.110678+0000","last_undegraded":"2024-08-17T17:52:10.110678+0000","last_fullsized":"2024-08-17T17:52:10.110678+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":596,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109612+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:52:10.109612+0000","last_peered":"2024-08-17T17:52:10.109612+0000","last_clean":"2024-08-17T17:52:10.109612+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:52:10.109612+0000","last_undegraded":"2024-08-17T17:52:10.109612+0000","last_fullsized":"2024-08-17T17:52:10.109612+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1840,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.346349+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:52:10.346349+0000","last_peered":"2024-08-17T17:52:10.346349+0000","last_clean":"2024-08-17T17:52:10.346349+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:52:10.346349+0000","last_undegraded":"2024-08-17T17:52:10.346349+0000","last_fullsized":"2024-08-17T17:52:10.346349+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1842,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.356787+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:52:10.356787+0000","last_peered":"2024-08-17T17:52:10.356787+0000","last_clean":"2024-08-17T17:52:10.356787+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:52:10.356787+0000","last_undegraded":"2024-08-17T17:52:10.356787+0000","last_fullsized":"2024-08-17T17:52:10.356787+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":61,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.110019+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:52:10.110019+0000","last_peered":"2024-08-17T17:52:10.110019+0000","last_clean":"2024-08-17T17:52:10.110019+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:52:10.110019+0000","last_undegraded":"2024-08-17T17:52:10.110019+0000","last_fullsized":"2024-08-17T17:52:10.110019+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1814,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.112058+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:52:10.112058+0000","last_peered":"2024-08-17T17:52:10.112058+0000","last_clean":"2024-08-17T17:52:10.112058+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:52:10.112058+0000","last_undegraded":"2024-08-17T17:52:10.112058+0000","last_fullsized":"2024-08-17T17:52:10.112058+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1868,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.108902+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:52:10.108902+0000","last_peered":"2024-08-17T17:52:10.108902+0000","last_clean":"2024-08-17T17:52:10.108902+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:52:10.108902+0000","last_undegraded":"2024-08-17T17:52:10.108902+0000","last_fullsized":"2024-08-17T17:52:10.108902+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1862,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.346477+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:52:10.346477+0000","last_peered":"2024-08-17T17:52:10.346477+0000","last_clean":"2024-08-17T17:52:10.346477+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:52:10.346477+0000","last_undegraded":"2024-08-17T17:52:10.346477+0000","last_fullsized":"2024-08-17T17:52:10.346477+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1836,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109517+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:52:10.109517+0000","last_peered":"2024-08-17T17:52:10.109517+0000","last_clean":"2024-08-17T17:52:10.109517+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:52:10.109517+0000","last_undegraded":"2024-08-17T17:52:10.109517+0000","last_fullsized":"2024-08-17T17:52:10.109517+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1187,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280515+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:52:05.280515+0000","last_peered":"2024-08-17T17:52:05.280515+0000","last_clean":"2024-08-17T17:52:05.280515+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:52:05.280515+0000","last_undegraded":"2024-08-17T17:52:05.280515+0000","last_fullsized":"2024-08-17T17:52:05.280515+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1807,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109267+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:52:10.109267+0000","last_peered":"2024-08-17T17:52:10.109267+0000","last_clean":"2024-08-17T17:52:10.109267+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:52:10.109267+0000","last_undegraded":"2024-08-17T17:52:10.109267+0000","last_fullsized":"2024-08-17T17:52:10.109267+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1857,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.346524+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:52:10.346524+0000","last_peered":"2024-08-17T17:52:10.346524+0000","last_clean":"2024-08-17T17:52:10.346524+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:52:10.346524+0000","last_undegraded":"2024-08-17T17:52:10.346524+0000","last_fullsized":"2024-08-17T17:52:10.346524+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1763,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109295+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:52:10.109295+0000","last_peered":"2024-08-17T17:52:10.109295+0000","last_clean":"2024-08-17T17:52:10.109295+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:52:10.109295+0000","last_undegraded":"2024-08-17T17:52:10.109295+0000","last_fullsized":"2024-08-17T17:52:10.109295+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":506,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.284179+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:52:05.284179+0000","last_peered":"2024-08-17T17:52:05.284179+0000","last_clean":"2024-08-17T17:52:05.284179+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:52:05.284179+0000","last_undegraded":"2024-08-17T17:52:05.284179+0000","last_fullsized":"2024-08-17T17:52:05.284179+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1183,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283814+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:52:05.283814+0000","last_peered":"2024-08-17T17:52:05.283814+0000","last_clean":"2024-08-17T17:52:05.283814+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:52:05.283814+0000","last_undegraded":"2024-08-17T17:52:05.283814+0000","last_fullsized":"2024-08-17T17:52:05.283814+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1862,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.110251+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:52:10.110251+0000","last_peered":"2024-08-17T17:52:10.110251+0000","last_clean":"2024-08-17T17:52:10.110251+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:52:10.110251+0000","last_undegraded":"2024-08-17T17:52:10.110251+0000","last_fullsized":"2024-08-17T17:52:10.110251+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":344,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283695+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:52:05.283695+0000","last_peered":"2024-08-17T17:52:05.283695+0000","last_clean":"2024-08-17T17:52:05.283695+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:52:05.283695+0000","last_undegraded":"2024-08-17T17:52:05.283695+0000","last_fullsized":"2024-08-17T17:52:05.283695+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":490,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280438+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:52:05.280438+0000","last_peered":"2024-08-17T17:52:05.280438+0000","last_clean":"2024-08-17T17:52:05.280438+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:52:05.280438+0000","last_undegraded":"2024-08-17T17:52:05.280438+0000","last_fullsized":"2024-08-17T17:52:05.280438+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1168,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280443+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:52:05.280443+0000","last_peered":"2024-08-17T17:52:05.280443+0000","last_clean":"2024-08-17T17:52:05.280443+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:52:05.280443+0000","last_undegraded":"2024-08-17T17:52:05.280443+0000","last_fullsized":"2024-08-17T17:52:05.280443+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1176,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280586+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:52:05.280586+0000","last_peered":"2024-08-17T17:52:05.280586+0000","last_clean":"2024-08-17T17:52:05.280586+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:52:05.280586+0000","last_undegraded":"2024-08-17T17:52:05.280586+0000","last_fullsized":"2024-08-17T17:52:05.280586+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1909,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.346310+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:52:10.346310+0000","last_peered":"2024-08-17T17:52:10.346310+0000","last_clean":"2024-08-17T17:52:10.346310+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:52:10.346310+0000","last_undegraded":"2024-08-17T17:52:10.346310+0000","last_fullsized":"2024-08-17T17:52:10.346310+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1793,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109436+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:52:10.109436+0000","last_peered":"2024-08-17T17:52:10.109436+0000","last_clean":"2024-08-17T17:52:10.109436+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:52:10.109436+0000","last_undegraded":"2024-08-17T17:52:10.109436+0000","last_fullsized":"2024-08-17T17:52:10.109436+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1867,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.110702+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:52:10.110702+0000","last_peered":"2024-08-17T17:52:10.110702+0000","last_clean":"2024-08-17T17:52:10.110702+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:52:10.110702+0000","last_undegraded":"2024-08-17T17:52:10.110702+0000","last_fullsized":"2024-08-17T17:52:10.110702+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1795,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109712+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:52:10.109712+0000","last_peered":"2024-08-17T17:52:10.109712+0000","last_clean":"2024-08-17T17:52:10.109712+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:52:10.109712+0000","last_undegraded":"2024-08-17T17:52:10.109712+0000","last_fullsized":"2024-08-17T17:52:10.109712+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1827,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.346445+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:52:10.346445+0000","last_peered":"2024-08-17T17:52:10.346445+0000","last_clean":"2024-08-17T17:52:10.346445+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:52:10.346445+0000","last_undegraded":"2024-08-17T17:52:10.346445+0000","last_fullsized":"2024-08-17T17:52:10.346445+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1203,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283903+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:52:05.283903+0000","last_peered":"2024-08-17T17:52:05.283903+0000","last_clean":"2024-08-17T17:52:05.283903+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:52:05.283903+0000","last_undegraded":"2024-08-17T17:52:05.283903+0000","last_fullsized":"2024-08-17T17:52:05.283903+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1874,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.110877+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:52:10.110877+0000","last_peered":"2024-08-17T17:52:10.110877+0000","last_clean":"2024-08-17T17:52:10.110877+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:52:10.110877+0000","last_undegraded":"2024-08-17T17:52:10.110877+0000","last_fullsized":"2024-08-17T17:52:10.110877+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":61,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109878+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:52:10.109878+0000","last_peered":"2024-08-17T17:52:10.109878+0000","last_clean":"2024-08-17T17:52:10.109878+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:52:10.109878+0000","last_undegraded":"2024-08-17T17:52:10.109878+0000","last_fullsized":"2024-08-17T17:52:10.109878+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1893,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.110784+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:52:10.110784+0000","last_peered":"2024-08-17T17:52:10.110784+0000","last_clean":"2024-08-17T17:52:10.110784+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:52:10.110784+0000","last_undegraded":"2024-08-17T17:52:10.110784+0000","last_fullsized":"2024-08-17T17:52:10.110784+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":629,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.111847+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:52:10.111847+0000","last_peered":"2024-08-17T17:52:10.111847+0000","last_clean":"2024-08-17T17:52:10.111847+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:52:10.111847+0000","last_undegraded":"2024-08-17T17:52:10.111847+0000","last_fullsized":"2024-08-17T17:52:10.111847+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1874,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.110961+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:52:10.110961+0000","last_peered":"2024-08-17T17:52:10.110961+0000","last_clean":"2024-08-17T17:52:10.110961+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:52:10.110961+0000","last_undegraded":"2024-08-17T17:52:10.110961+0000","last_fullsized":"2024-08-17T17:52:10.110961+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1203,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.346084+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:52:10.346084+0000","last_peered":"2024-08-17T17:52:10.346084+0000","last_clean":"2024-08-17T17:52:10.346084+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:52:10.346084+0000","last_undegraded":"2024-08-17T17:52:10.346084+0000","last_fullsized":"2024-08-17T17:52:10.346084+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":120,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.110163+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:52:10.110163+0000","last_peered":"2024-08-17T17:52:10.110163+0000","last_clean":"2024-08-17T17:52:10.110163+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:52:10.110163+0000","last_undegraded":"2024-08-17T17:52:10.110163+0000","last_fullsized":"2024-08-17T17:52:10.110163+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1884,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.110272+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:52:10.110272+0000","last_peered":"2024-08-17T17:52:10.110272+0000","last_clean":"2024-08-17T17:52:10.110272+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:52:10.110272+0000","last_undegraded":"2024-08-17T17:52:10.110272+0000","last_fullsized":"2024-08-17T17:52:10.110272+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1862,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109023+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:52:10.109023+0000","last_peered":"2024-08-17T17:52:10.109023+0000","last_clean":"2024-08-17T17:52:10.109023+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:52:10.109023+0000","last_undegraded":"2024-08-17T17:52:10.109023+0000","last_fullsized":"2024-08-17T17:52:10.109023+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1883,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.358168+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:52:10.358168+0000","last_peered":"2024-08-17T17:52:10.358168+0000","last_clean":"2024-08-17T17:52:10.358168+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:52:10.358168+0000","last_undegraded":"2024-08-17T17:52:10.358168+0000","last_fullsized":"2024-08-17T17:52:10.358168+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1888,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.346496+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:52:10.346496+0000","last_peered":"2024-08-17T17:52:10.346496+0000","last_clean":"2024-08-17T17:52:10.346496+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:52:10.346496+0000","last_undegraded":"2024-08-17T17:52:10.346496+0000","last_fullsized":"2024-08-17T17:52:10.346496+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1868,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.367716+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:52:10.367716+0000","last_peered":"2024-08-17T17:52:10.367716+0000","last_clean":"2024-08-17T17:52:10.367716+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:52:10.367716+0000","last_undegraded":"2024-08-17T17:52:10.367716+0000","last_fullsized":"2024-08-17T17:52:10.367716+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1210,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283492+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:52:05.283492+0000","last_peered":"2024-08-17T17:52:05.283492+0000","last_clean":"2024-08-17T17:52:05.283492+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:52:05.283492+0000","last_undegraded":"2024-08-17T17:52:05.283492+0000","last_fullsized":"2024-08-17T17:52:05.283492+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1156,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280652+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:52:05.280652+0000","last_peered":"2024-08-17T17:52:05.280652+0000","last_clean":"2024-08-17T17:52:05.280652+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:52:05.280652+0000","last_undegraded":"2024-08-17T17:52:05.280652+0000","last_fullsized":"2024-08-17T17:52:05.280652+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1777,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109988+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:52:10.109988+0000","last_peered":"2024-08-17T17:52:10.109988+0000","last_clean":"2024-08-17T17:52:10.109988+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:52:10.109988+0000","last_undegraded":"2024-08-17T17:52:10.109988+0000","last_fullsized":"2024-08-17T17:52:10.109988+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":108,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.356510+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:52:10.356510+0000","last_peered":"2024-08-17T17:52:10.356510+0000","last_clean":"2024-08-17T17:52:10.356510+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:52:10.356510+0000","last_undegraded":"2024-08-17T17:52:10.356510+0000","last_fullsized":"2024-08-17T17:52:10.356510+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1522,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283394+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:52:05.283394+0000","last_peered":"2024-08-17T17:52:05.283394+0000","last_clean":"2024-08-17T17:52:05.283394+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:52:05.283394+0000","last_undegraded":"2024-08-17T17:52:05.283394+0000","last_fullsized":"2024-08-17T17:52:05.283394+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1869,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.110384+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:52:10.110384+0000","last_peered":"2024-08-17T17:52:10.110384+0000","last_clean":"2024-08-17T17:52:10.110384+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:52:10.110384+0000","last_undegraded":"2024-08-17T17:52:10.110384+0000","last_fullsized":"2024-08-17T17:52:10.110384+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1809,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109347+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:52:10.109347+0000","last_peered":"2024-08-17T17:52:10.109347+0000","last_clean":"2024-08-17T17:52:10.109347+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:52:10.109347+0000","last_undegraded":"2024-08-17T17:52:10.109347+0000","last_fullsized":"2024-08-17T17:52:10.109347+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":490,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280510+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:52:05.280510+0000","last_peered":"2024-08-17T17:52:05.280510+0000","last_clean":"2024-08-17T17:52:05.280510+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:52:05.280510+0000","last_undegraded":"2024-08-17T17:52:05.280510+0000","last_fullsized":"2024-08-17T17:52:05.280510+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":548,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.358094+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:52:10.358094+0000","last_peered":"2024-08-17T17:52:10.358094+0000","last_clean":"2024-08-17T17:52:10.358094+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:52:10.358094+0000","last_undegraded":"2024-08-17T17:52:10.358094+0000","last_fullsized":"2024-08-17T17:52:10.358094+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":564,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.356566+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:52:10.356566+0000","last_peered":"2024-08-17T17:52:10.356566+0000","last_clean":"2024-08-17T17:52:10.356566+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:52:10.356566+0000","last_undegraded":"2024-08-17T17:52:10.356566+0000","last_fullsized":"2024-08-17T17:52:10.356566+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1174,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280643+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:52:05.280643+0000","last_peered":"2024-08-17T17:52:05.280643+0000","last_clean":"2024-08-17T17:52:05.280643+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:52:05.280643+0000","last_undegraded":"2024-08-17T17:52:05.280643+0000","last_fullsized":"2024-08-17T17:52:05.280643+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":580,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109679+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:52:10.109679+0000","last_peered":"2024-08-17T17:52:10.109679+0000","last_clean":"2024-08-17T17:52:10.109679+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:52:10.109679+0000","last_undegraded":"2024-08-17T17:52:10.109679+0000","last_fullsized":"2024-08-17T17:52:10.109679+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1806,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109715+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:52:10.109715+0000","last_peered":"2024-08-17T17:52:10.109715+0000","last_clean":"2024-08-17T17:52:10.109715+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:52:10.109715+0000","last_undegraded":"2024-08-17T17:52:10.109715+0000","last_fullsized":"2024-08-17T17:52:10.109715+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1901,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.346524+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:52:10.346524+0000","last_peered":"2024-08-17T17:52:10.346524+0000","last_clean":"2024-08-17T17:52:10.346524+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:52:10.346524+0000","last_undegraded":"2024-08-17T17:52:10.346524+0000","last_fullsized":"2024-08-17T17:52:10.346524+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":612,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109639+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:52:10.109639+0000","last_peered":"2024-08-17T17:52:10.109639+0000","last_clean":"2024-08-17T17:52:10.109639+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:52:10.109639+0000","last_undegraded":"2024-08-17T17:52:10.109639+0000","last_fullsized":"2024-08-17T17:52:10.109639+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1163,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280249+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:52:05.280249+0000","last_peered":"2024-08-17T17:52:05.280249+0000","last_clean":"2024-08-17T17:52:05.280249+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:52:05.280249+0000","last_undegraded":"2024-08-17T17:52:05.280249+0000","last_fullsized":"2024-08-17T17:52:05.280249+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1162,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283933+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:52:05.283933+0000","last_peered":"2024-08-17T17:52:05.283933+0000","last_clean":"2024-08-17T17:52:05.283933+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:52:05.283933+0000","last_undegraded":"2024-08-17T17:52:05.283933+0000","last_fullsized":"2024-08-17T17:52:05.283933+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":597,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109003+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:52:10.109003+0000","last_peered":"2024-08-17T17:52:10.109003+0000","last_clean":"2024-08-17T17:52:10.109003+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:52:10.109003+0000","last_undegraded":"2024-08-17T17:52:10.109003+0000","last_fullsized":"2024-08-17T17:52:10.109003+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1180,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.284256+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:52:05.284256+0000","last_peered":"2024-08-17T17:52:05.284256+0000","last_clean":"2024-08-17T17:52:05.284256+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:52:05.284256+0000","last_undegraded":"2024-08-17T17:52:05.284256+0000","last_fullsized":"2024-08-17T17:52:05.284256+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":64,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283364+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:52:05.283364+0000","last_peered":"2024-08-17T17:52:05.283364+0000","last_clean":"2024-08-17T17:52:05.283364+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:52:05.283364+0000","last_undegraded":"2024-08-17T17:52:05.283364+0000","last_fullsized":"2024-08-17T17:52:05.283364+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1196,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283994+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:52:05.283994+0000","last_peered":"2024-08-17T17:52:05.283994+0000","last_clean":"2024-08-17T17:52:05.283994+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:52:05.283994+0000","last_undegraded":"2024-08-17T17:52:05.283994+0000","last_fullsized":"2024-08-17T17:52:05.283994+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1213,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280219+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:52:05.280219+0000","last_peered":"2024-08-17T17:52:05.280219+0000","last_clean":"2024-08-17T17:52:05.280219+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:52:05.280219+0000","last_undegraded":"2024-08-17T17:52:05.280219+0000","last_fullsized":"2024-08-17T17:52:05.280219+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1807,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109605+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:52:10.109605+0000","last_peered":"2024-08-17T17:52:10.109605+0000","last_clean":"2024-08-17T17:52:10.109605+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:52:10.109605+0000","last_undegraded":"2024-08-17T17:52:10.109605+0000","last_fullsized":"2024-08-17T17:52:10.109605+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1157,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283673+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:52:05.283673+0000","last_peered":"2024-08-17T17:52:05.283673+0000","last_clean":"2024-08-17T17:52:05.283673+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:52:05.283673+0000","last_undegraded":"2024-08-17T17:52:05.283673+0000","last_fullsized":"2024-08-17T17:52:05.283673+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1860,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.346413+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:52:10.346413+0000","last_peered":"2024-08-17T17:52:10.346413+0000","last_clean":"2024-08-17T17:52:10.346413+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:52:10.346413+0000","last_undegraded":"2024-08-17T17:52:10.346413+0000","last_fullsized":"2024-08-17T17:52:10.346413+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1861,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.346133+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:52:10.346133+0000","last_peered":"2024-08-17T17:52:10.346133+0000","last_clean":"2024-08-17T17:52:10.346133+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:52:10.346133+0000","last_undegraded":"2024-08-17T17:52:10.346133+0000","last_fullsized":"2024-08-17T17:52:10.346133+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":570,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283766+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:52:05.283766+0000","last_peered":"2024-08-17T17:52:05.283766+0000","last_clean":"2024-08-17T17:52:05.283766+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:52:05.283766+0000","last_undegraded":"2024-08-17T17:52:05.283766+0000","last_fullsized":"2024-08-17T17:52:05.283766+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1182,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283881+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:52:05.283881+0000","last_peered":"2024-08-17T17:52:05.283881+0000","last_clean":"2024-08-17T17:52:05.283881+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:52:05.283881+0000","last_undegraded":"2024-08-17T17:52:05.283881+0000","last_fullsized":"2024-08-17T17:52:05.283881+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1807,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109782+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:52:10.109782+0000","last_peered":"2024-08-17T17:52:10.109782+0000","last_clean":"2024-08-17T17:52:10.109782+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:52:10.109782+0000","last_undegraded":"2024-08-17T17:52:10.109782+0000","last_fullsized":"2024-08-17T17:52:10.109782+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":5331025944,"num_objects":81346,"num_object_clones":0,"num_object_copies":244038,"num_objects_missing_on_primary":4597,"num_objects_missing":4597,"num_objects_degraded":19354,"num_objects_misplaced":14158,"num_objects_unfound":0,"num_objects_dirty":81346,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":128264,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32909,"num_bytes_recovered":1978269696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16648974336,"data_stored":16648962120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106138,"ondisk_log_size":106138,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234376,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2473664,"kb_used_data":2176312,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":91271488,"statfs":{"total":95995035648,"available":93462003712,"internally_reserved":0,"allocated":2228543488,"data_stored":2223723387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496011,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2813316,"kb_used_data":2515964,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":90931836,"statfs":{"total":95995035648,"available":93114200064,"internally_reserved":0,"allocated":2576347136,"data_stored":2571523440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757649,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326263,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3108716,"kb_used_data":2906148,"kb_used_omap":57,"kb_used_meta":202502,"kb_avail":90636436,"statfs":{"total":95995035648,"available":92811710464,"internally_reserved":0,"allocated":2975895552,"data_stored":2971090631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":207362811},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405502,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2807092,"kb_used_data":2604460,"kb_used_omap":61,"kb_used_meta":202562,"kb_avail":90938060,"statfs":{"total":95995035648,"available":93120573440,"internally_reserved":0,"allocated":2666967040,"data_stored":2662159091,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":207424440},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452036,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2121452,"kb_used_data":1967456,"kb_used_omap":54,"kb_used_meta":153929,"kb_avail":91623700,"statfs":{"total":95995035648,"available":93822668800,"internally_reserved":0,"allocated":2014674944,"data_stored":2009873984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":157623694},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474728,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1740984,"kb_used_data":1573808,"kb_used_omap":119,"kb_used_meta":167048,"kb_avail":92004168,"statfs":{"total":95995035648,"available":94212268032,"internally_reserved":0,"allocated":1611579392,"data_stored":1606763268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":171057886},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,1,0,1,1],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019284,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1596592128,"data_stored":1596588056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2000486400,"data_stored":2000486400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2653290496,"data_stored":2653290496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2962558976,"data_stored":2962554904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1102774272,"data_stored":1102774272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1557725184,"data_stored":1557725184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2561413120,"data_stored":2561409048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2214133760,"data_stored":2214133760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:52:17.578 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:17.599 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:17.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:17.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:17.725 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:17.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:17.827 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:17.854 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:17.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:17.955 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:17.977 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:17.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:17.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:18.040 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:52:18.040 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:52:18.040 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T17:52:18.040 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:52:18.080 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:18.102 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:18.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:18.204 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:18.226 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:18.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:18.328 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:18.349 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:18.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:18.450 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:18.478 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:18.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:18.579 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:18.601 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:18.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:18.703 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:18.725 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:18.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:18.735 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:52:18.827 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:18.848 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:18.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:18.950 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:18.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:18.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:18.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:19.039 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:19.078 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:19.100 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:19.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:19.202 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:19.224 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:19.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:19.325 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:19.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.349 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:19.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:19 smithi130 ceph-mon[105521]: pgmap v1745: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 19304/242868 objects degraded (7.948%); 14051/242868 objects misplaced (5.785%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:52:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:19 smithi130 ceph-mon[110068]: pgmap v1745: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 19304/242868 objects degraded (7.948%); 14051/242868 objects misplaced (5.785%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:52:19.449 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:19.472 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:19.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:19.490 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:19 smithi179 ceph-mon[93519]: pgmap v1745: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 19304/242868 objects degraded (7.948%); 14051/242868 objects misplaced (5.785%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:52:19.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:19.595 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:19.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:19.697 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:19.718 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:19.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:19.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:19.843 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:19.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:19.945 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:19.969 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:19.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:19.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:20.049 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:52:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:52:19] "GET /metrics HTTP/1.1" 200 36810 "" "Prometheus/2.51.0" 2024-08-17T17:52:20.070 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:20.092 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:20.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:20.194 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:20.217 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:20.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:20.225 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:52:20.226 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:52:20.243 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:52:20.243 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:52:20.257 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:52:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:52:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:52:20.261 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:52:20.261 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:52:20.278 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:52:20.278 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:52:20.294 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:52:20.294 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:52:20.310 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:52:20.310 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:52:20.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1875834075' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:20.311 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1875834075' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:20.319 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:20.326 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:52:20.326 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:52:20.340 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:20.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:20.341 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:52:20.342 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:52:20.441 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:20.464 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:20.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:20.521 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1875834075' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:20.566 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:20.578 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:52:20.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:20.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:20.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:20.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:20.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:20.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:20.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:20.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:20.907 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:52:20.907 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:52:20.936 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:20.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:20.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:20.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:21.060 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:21.082 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:21.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:21.184 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:21.205 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:21.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:21.265 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T17:52:21.306 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:21.328 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:21.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:21.430 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:21.451 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:21.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:21.552 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:21.574 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:21.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:21.586 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 19304/242868 objects degraded (7.948%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:21.586 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:21 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1454170045' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:21.587 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:21 smithi179 ceph-mon[93519]: pgmap v1746: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 19304/242682 objects degraded (7.954%); 14051/242682 objects misplaced (5.790%); 780 KiB/s, 12 objects/s recovering 2024-08-17T17:52:21.611 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:52:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 19304/242868 objects degraded (7.948%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:21 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1454170045' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:21 smithi130 ceph-mon[105521]: pgmap v1746: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 19304/242682 objects degraded (7.954%); 14051/242682 objects misplaced (5.790%); 780 KiB/s, 12 objects/s recovering 2024-08-17T17:52:21.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 19304/242868 objects degraded (7.948%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:21.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:21 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1454170045' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:21.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:21 smithi130 ceph-mon[110068]: pgmap v1746: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 91 op/s; 19304/242682 objects degraded (7.954%); 14051/242682 objects misplaced (5.790%); 780 KiB/s, 12 objects/s recovering 2024-08-17T17:52:21.676 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:21.714 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:21.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:21.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:21.839 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:21.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:21.918 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:21.941 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:21.963 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:21.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:21.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:22.065 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:22.086 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:22.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:22.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:22.210 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:22.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:22.237 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:22.312 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:22.333 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:22.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:22.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:22 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:52:22.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:22 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1454170045' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:22.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:22 smithi179 ceph-mon[93519]: osdmap e1214: 8 total, 8 up, 6 in 2024-08-17T17:52:22.434 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:22.456 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:22.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:22.500 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:22 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:52:22.500 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:22 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1454170045' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:22.500 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:22 smithi130 ceph-mon[105521]: osdmap e1214: 8 total, 8 up, 6 in 2024-08-17T17:52:22.500 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:22 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:52:22.500 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:22 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1454170045' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:22.500 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:22 smithi130 ceph-mon[110068]: osdmap e1214: 8 total, 8 up, 6 in 2024-08-17T17:52:22.558 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:22.580 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:22.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:22.681 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:22.703 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:22.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:22.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:22.826 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:22.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:22.927 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:22.949 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:22.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:22.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:23.050 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:23.071 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:23.173 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:23.195 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:23.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:23.297 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:23.318 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:23.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:23.387 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:52:23.420 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:23.441 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:23.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:23.506 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:23 smithi179 ceph-mon[93519]: pgmap v1748: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 19222/242289 objects degraded (7.934%); 13947/242289 objects misplaced (5.756%); 896 KiB/s, 13 objects/s recovering 2024-08-17T17:52:23.543 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:23.571 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:23.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:23.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:23 smithi130 ceph-mon[105521]: pgmap v1748: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 19222/242289 objects degraded (7.934%); 13947/242289 objects misplaced (5.756%); 896 KiB/s, 13 objects/s recovering 2024-08-17T17:52:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:23 smithi130 ceph-mon[110068]: pgmap v1748: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 19222/242289 objects degraded (7.934%); 13947/242289 objects misplaced (5.756%); 896 KiB/s, 13 objects/s recovering 2024-08-17T17:52:23.673 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:23.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:23.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:23.775 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:52:23.775 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:52:23.797 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:23.819 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:23.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:23.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:23.944 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:23.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:23.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:24.045 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:24.067 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:24.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:24.168 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:24.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.190 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:24.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:24.230 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:52:24.291 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:24.314 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:24.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:24.358 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:24.416 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:24.441 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:24.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:24.452 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:52:24.543 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:24.565 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:24.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:24.667 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:24.690 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:24.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:24.713 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:24.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:24.798 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:52:24.799 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:52:24.813 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:24.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:24.915 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:24.940 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:24.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:24.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:25.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:25.066 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:25.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:25.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:25 smithi130 ceph-mon[105521]: from='client.35547 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:52:25.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:25 smithi130 ceph-mon[105521]: pgmap v1749: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 19222/241137 objects degraded (7.971%); 13947/241137 objects misplaced (5.784%); 896 KiB/s, 13 objects/s recovering 2024-08-17T17:52:25.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:25 smithi130 ceph-mon[110068]: from='client.35547 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:52:25.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:25 smithi130 ceph-mon[110068]: pgmap v1749: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 19222/241137 objects degraded (7.971%); 13947/241137 objects misplaced (5.784%); 896 KiB/s, 13 objects/s recovering 2024-08-17T17:52:25.168 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:25.194 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:25.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:25.295 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:25.319 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:25.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:25.422 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:25.443 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:25.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:25.545 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:25.568 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:25.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:25.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:25 smithi179 ceph-mon[93519]: from='client.35547 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:52:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:25 smithi179 ceph-mon[93519]: pgmap v1749: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 19222/241137 objects degraded (7.971%); 13947/241137 objects misplaced (5.784%); 896 KiB/s, 13 objects/s recovering 2024-08-17T17:52:25.669 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:25.692 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:25.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:25.794 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:25.815 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:25.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:25.916 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:25.938 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:25.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:25.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:26.040 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:26.062 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:26.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:26.111 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1748,"stamp":"2024-08-17T17:52:23.020964+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5291710437,"num_objects":80763,"num_object_clones":0,"num_object_copies":242289,"num_objects_missing_on_primary":4522,"num_objects_missing":4522,"num_objects_degraded":19222,"num_objects_misplaced":13947,"num_objects_unfound":0,"num_objects_dirty":80763,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":129151,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33078,"num_bytes_recovered":1989221389,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106654,"ondisk_log_size":106654,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14962792,"kb_used_data":13641716,"kb_used_omap":295,"kb_used_meta":1320728,"kb_avail":547508120,"statfs":{"total":575970213888,"available":560648314880,"internally_reserved":0,"allocated":13969117184,"data_stored":13940228561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1352425935},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,0,0,0,0,0,0,0,0,2],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-64815104,"num_objects":-989,"num_object_clones":0,"num_object_copies":-2967,"num_objects_missing_on_primary":-75,"num_objects_missing":-75,"num_objects_degraded":-132,"num_objects_misplaced":-211,"num_objects_unfound":0,"num_objects_dirty":-989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":989,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":140,"num_bytes_recovered":9175040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002726"},"pg_stats":[{"pgid":"45.f","version":"1214'8083","reported_seq":18589,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:22.032445+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:52:22.032445+0000","last_peered":"2024-08-17T17:52:22.032445+0000","last_clean":"2024-08-17T17:52:22.032445+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:52:22.032445+0000","last_undegraded":"2024-08-17T17:52:22.032445+0000","last_fullsized":"2024-08-17T17:52:22.032445+0000","mapping_epoch":1194,"log_start":"1170'1231","ondisk_log_start":"1170'1231","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6852,"log_dups_size":0,"ondisk_log_size":6852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332988416,"num_objects":5081,"num_object_clones":0,"num_object_copies":15243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8083,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1213'7992","reported_seq":18445,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:18.157962+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:52:18.157962+0000","last_peered":"2024-08-17T17:52:18.157962+0000","last_clean":"2024-08-17T17:52:18.157962+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:52:18.157962+0000","last_undegraded":"2024-08-17T17:52:18.157962+0000","last_fullsized":"2024-08-17T17:52:18.157962+0000","mapping_epoch":1211,"log_start":"1170'932","ondisk_log_start":"1170'932","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7060,"log_dups_size":0,"ondisk_log_size":7060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327811072,"num_objects":5002,"num_object_clones":0,"num_object_copies":15006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7992,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1213'8241","reported_seq":19053,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:18.161365+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:52:18.161365+0000","last_peered":"2024-08-17T17:52:18.161365+0000","last_clean":"2024-08-17T17:52:18.161365+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:52:18.161365+0000","last_undegraded":"2024-08-17T17:52:18.161365+0000","last_fullsized":"2024-08-17T17:52:18.161365+0000","mapping_epoch":1205,"log_start":"1170'2268","ondisk_log_start":"1170'2268","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5973,"log_dups_size":0,"ondisk_log_size":5973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336592920,"num_objects":5137,"num_object_clones":0,"num_object_copies":15411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5137,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8241,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1213'8028","reported_seq":8082,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:17.485817+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:52:17.485817+0000","last_peered":"2024-08-17T17:52:17.485817+0000","last_clean":"2024-08-17T17:52:17.485817+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:52:17.485817+0000","last_undegraded":"2024-08-17T17:52:17.485817+0000","last_fullsized":"2024-08-17T17:52:17.485817+0000","mapping_epoch":1167,"log_start":"1170'1289","ondisk_log_start":"1170'1289","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6739,"log_dups_size":0,"ondisk_log_size":6739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325844992,"num_objects":4972,"num_object_clones":0,"num_object_copies":14916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8028,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1214'8127","reported_seq":13347,"reported_epoch":1214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:21.385306+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:52:21.385306+0000","last_peered":"2024-08-17T17:52:21.385306+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:52:21.385306+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1170'2226","ondisk_log_start":"1170'2226","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5901,"log_dups_size":0,"ondisk_log_size":5901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326828032,"num_objects":4987,"num_object_clones":0,"num_object_copies":14961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":4987,"num_objects_unfound":0,"num_objects_dirty":4987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8127,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":4469},{"shards":"1,4,5","objects":518}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1213'8094","reported_seq":8148,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:17.761228+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:52:17.761228+0000","last_peered":"2024-08-17T17:52:17.761228+0000","last_clean":"2024-08-17T17:52:17.761228+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:52:17.761228+0000","last_undegraded":"2024-08-17T17:52:17.761228+0000","last_fullsized":"2024-08-17T17:52:17.761228+0000","mapping_epoch":1167,"log_start":"1170'1377","ondisk_log_start":"1170'1377","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6717,"log_dups_size":0,"ondisk_log_size":6717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680000,"num_objects":5000,"num_object_clones":0,"num_object_copies":15000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8094,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1213'8104","reported_seq":8158,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:17.816434+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:52:17.816434+0000","last_peered":"2024-08-17T17:52:17.816434+0000","last_clean":"2024-08-17T17:52:17.816434+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:52:17.816434+0000","last_undegraded":"2024-08-17T17:52:17.816434+0000","last_fullsized":"2024-08-17T17:52:17.816434+0000","mapping_epoch":1167,"log_start":"1170'1369","ondisk_log_start":"1170'1369","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6735,"log_dups_size":0,"ondisk_log_size":6735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335675392,"num_objects":5122,"num_object_clones":0,"num_object_copies":15366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8104,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1214'8043","reported_seq":12800,"reported_epoch":1214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:21.830315+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:52:21.830315+0000","last_peered":"2024-08-17T17:52:21.830315+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:52:21.830315+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1170'2075","ondisk_log_start":"1170'2075","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5968,"log_dups_size":0,"ondisk_log_size":5968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324206592,"num_objects":4947,"num_object_clones":0,"num_object_copies":14841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5187,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8043,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1548,"num_bytes_recovered":101449728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1560},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1213'7976","reported_seq":8040,"reported_epoch":1213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:17.475304+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:52:17.475304+0000","last_peered":"2024-08-17T17:52:17.475304+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:52:17.475304+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'1277","ondisk_log_start":"1170'1277","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6699,"log_dups_size":0,"ondisk_log_size":6699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334233600,"num_objects":5100,"num_object_clones":0,"num_object_copies":15300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7976,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":5100},{"shards":"1,6","objects":-733}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1213'8016","reported_seq":11067,"reported_epoch":1213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:18.417946+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:52:18.417946+0000","last_peered":"2024-08-17T17:52:18.417946+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:52:18.417946+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1170'2106","ondisk_log_start":"1170'2106","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5910,"log_dups_size":0,"ondisk_log_size":5910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338034688,"num_objects":5158,"num_object_clones":0,"num_object_copies":15474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8016,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":3597},{"shards":"1,3,5","objects":1561}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1213'8050","reported_seq":8156,"reported_epoch":1213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:18.917373+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:52:18.917373+0000","last_peered":"2024-08-17T17:52:18.917373+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:52:18.917373+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8050,"log_dups_size":0,"ondisk_log_size":8050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329252864,"num_objects":5024,"num_object_clones":0,"num_object_copies":15072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8050,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-129}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1213'8015","reported_seq":14182,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:18.988003+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:52:18.988003+0000","last_peered":"2024-08-17T17:52:18.988003+0000","last_clean":"2024-08-17T17:52:18.988003+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:52:18.988003+0000","last_undegraded":"2024-08-17T17:52:18.988003+0000","last_fullsized":"2024-08-17T17:52:18.988003+0000","mapping_epoch":1183,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8015,"log_dups_size":0,"ondisk_log_size":8015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329711616,"num_objects":5031,"num_object_clones":0,"num_object_copies":15093,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5031,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8015,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1213'8041","reported_seq":8146,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:18.252034+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:52:18.252034+0000","last_peered":"2024-08-17T17:52:18.252034+0000","last_clean":"2024-08-17T17:52:18.252034+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:52:18.252034+0000","last_undegraded":"2024-08-17T17:52:18.252034+0000","last_fullsized":"2024-08-17T17:52:18.252034+0000","mapping_epoch":1167,"log_start":"1170'2091","ondisk_log_start":"1170'2091","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5950,"log_dups_size":0,"ondisk_log_size":5950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335478784,"num_objects":5119,"num_object_clones":0,"num_object_copies":15357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8041,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1213'7958","reported_seq":8012,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:17.705305+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:52:17.705305+0000","last_peered":"2024-08-17T17:52:17.705305+0000","last_clean":"2024-08-17T17:52:17.705305+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:52:17.705305+0000","last_undegraded":"2024-08-17T17:52:17.705305+0000","last_fullsized":"2024-08-17T17:52:17.705305+0000","mapping_epoch":1167,"log_start":"1170'925","ondisk_log_start":"1170'925","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7033,"log_dups_size":0,"ondisk_log_size":7033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326107136,"num_objects":4976,"num_object_clones":0,"num_object_copies":14928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7958,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1214'8198","reported_seq":16358,"reported_epoch":1214,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:22.522190+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:52:22.522190+0000","last_peered":"2024-08-17T17:52:22.522190+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:52:22.522190+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'1291","ondisk_log_start":"1170'1291","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6907,"log_dups_size":0,"ondisk_log_size":6907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338558976,"num_objects":5166,"num_object_clones":0,"num_object_copies":15498,"num_objects_missing_on_primary":4522,"num_objects_missing":4522,"num_objects_degraded":265,"num_objects_misplaced":8960,"num_objects_unfound":0,"num_objects_dirty":5166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8198,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3623,"num_bytes_recovered":237436928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":14},{"shards":"0,4,6","objects":1356},{"shards":"0,6","objects":1},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":3530}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1213'7907","reported_seq":21198,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:17.680001+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:52:17.680001+0000","last_peered":"2024-08-17T17:52:17.680001+0000","last_clean":"2024-08-17T17:52:17.680001+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:52:17.680001+0000","last_undegraded":"2024-08-17T17:52:17.680001+0000","last_fullsized":"2024-08-17T17:52:17.680001+0000","mapping_epoch":1196,"log_start":"1170'1969","ondisk_log_start":"1170'1969","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5938,"log_dups_size":0,"ondisk_log_size":5938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322109440,"num_objects":4915,"num_object_clones":0,"num_object_copies":14745,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4915,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7907,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1883,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.275053+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:52:21.275053+0000","last_peered":"2024-08-17T17:52:21.275053+0000","last_clean":"2024-08-17T17:52:21.275053+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:52:21.275053+0000","last_undegraded":"2024-08-17T17:52:21.275053+0000","last_fullsized":"2024-08-17T17:52:21.275053+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":490,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280050+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:52:05.280050+0000","last_peered":"2024-08-17T17:52:05.280050+0000","last_clean":"2024-08-17T17:52:05.280050+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:52:05.280050+0000","last_undegraded":"2024-08-17T17:52:05.280050+0000","last_fullsized":"2024-08-17T17:52:05.280050+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1801,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.112293+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:52:10.112293+0000","last_peered":"2024-08-17T17:52:10.112293+0000","last_clean":"2024-08-17T17:52:10.112293+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:52:10.112293+0000","last_undegraded":"2024-08-17T17:52:10.112293+0000","last_fullsized":"2024-08-17T17:52:10.112293+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1825,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109379+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:52:10.109379+0000","last_peered":"2024-08-17T17:52:10.109379+0000","last_clean":"2024-08-17T17:52:10.109379+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:52:10.109379+0000","last_undegraded":"2024-08-17T17:52:10.109379+0000","last_fullsized":"2024-08-17T17:52:10.109379+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":52,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280633+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:52:05.280633+0000","last_peered":"2024-08-17T17:52:05.280633+0000","last_clean":"2024-08-17T17:52:05.280633+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:52:05.280633+0000","last_undegraded":"2024-08-17T17:52:05.280633+0000","last_fullsized":"2024-08-17T17:52:05.280633+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1219,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.110091+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:52:10.110091+0000","last_peered":"2024-08-17T17:52:10.110091+0000","last_clean":"2024-08-17T17:52:10.110091+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:52:10.110091+0000","last_undegraded":"2024-08-17T17:52:10.110091+0000","last_fullsized":"2024-08-17T17:52:10.110091+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":79,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.385120+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:52:21.385120+0000","last_peered":"2024-08-17T17:52:21.385120+0000","last_clean":"2024-08-17T17:52:21.385120+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:52:21.385120+0000","last_undegraded":"2024-08-17T17:52:21.385120+0000","last_fullsized":"2024-08-17T17:52:21.385120+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":597,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109648+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:52:10.109648+0000","last_peered":"2024-08-17T17:52:10.109648+0000","last_clean":"2024-08-17T17:52:10.109648+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:52:10.109648+0000","last_undegraded":"2024-08-17T17:52:10.109648+0000","last_fullsized":"2024-08-17T17:52:10.109648+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1842,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.274627+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:52:21.274627+0000","last_peered":"2024-08-17T17:52:21.274627+0000","last_clean":"2024-08-17T17:52:21.274627+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:52:21.274627+0000","last_undegraded":"2024-08-17T17:52:21.274627+0000","last_fullsized":"2024-08-17T17:52:21.274627+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1842,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.356787+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:52:10.356787+0000","last_peered":"2024-08-17T17:52:10.356787+0000","last_clean":"2024-08-17T17:52:10.356787+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:52:10.356787+0000","last_undegraded":"2024-08-17T17:52:10.356787+0000","last_fullsized":"2024-08-17T17:52:10.356787+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":64,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.386431+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:52:21.386431+0000","last_peered":"2024-08-17T17:52:21.386431+0000","last_clean":"2024-08-17T17:52:21.386431+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:52:21.386431+0000","last_undegraded":"2024-08-17T17:52:21.386431+0000","last_fullsized":"2024-08-17T17:52:21.386431+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1814,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.112058+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:52:10.112058+0000","last_peered":"2024-08-17T17:52:10.112058+0000","last_clean":"2024-08-17T17:52:10.112058+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:52:10.112058+0000","last_undegraded":"2024-08-17T17:52:10.112058+0000","last_fullsized":"2024-08-17T17:52:10.112058+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1870,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.384215+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:52:21.384215+0000","last_peered":"2024-08-17T17:52:21.384215+0000","last_clean":"2024-08-17T17:52:21.384215+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:52:21.384215+0000","last_undegraded":"2024-08-17T17:52:21.384215+0000","last_fullsized":"2024-08-17T17:52:21.384215+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1864,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.275144+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:52:21.275144+0000","last_peered":"2024-08-17T17:52:21.275144+0000","last_clean":"2024-08-17T17:52:21.275144+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:52:21.275144+0000","last_undegraded":"2024-08-17T17:52:21.275144+0000","last_fullsized":"2024-08-17T17:52:21.275144+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1836,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109517+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:52:10.109517+0000","last_peered":"2024-08-17T17:52:10.109517+0000","last_clean":"2024-08-17T17:52:10.109517+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:52:10.109517+0000","last_undegraded":"2024-08-17T17:52:10.109517+0000","last_fullsized":"2024-08-17T17:52:10.109517+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1187,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280515+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:52:05.280515+0000","last_peered":"2024-08-17T17:52:05.280515+0000","last_clean":"2024-08-17T17:52:05.280515+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:52:05.280515+0000","last_undegraded":"2024-08-17T17:52:05.280515+0000","last_fullsized":"2024-08-17T17:52:05.280515+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1807,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109267+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:52:10.109267+0000","last_peered":"2024-08-17T17:52:10.109267+0000","last_clean":"2024-08-17T17:52:10.109267+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:52:10.109267+0000","last_undegraded":"2024-08-17T17:52:10.109267+0000","last_fullsized":"2024-08-17T17:52:10.109267+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1859,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.275114+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:52:21.275114+0000","last_peered":"2024-08-17T17:52:21.275114+0000","last_clean":"2024-08-17T17:52:21.275114+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:52:21.275114+0000","last_undegraded":"2024-08-17T17:52:21.275114+0000","last_fullsized":"2024-08-17T17:52:21.275114+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1763,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109295+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:52:10.109295+0000","last_peered":"2024-08-17T17:52:10.109295+0000","last_clean":"2024-08-17T17:52:10.109295+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:52:10.109295+0000","last_undegraded":"2024-08-17T17:52:10.109295+0000","last_fullsized":"2024-08-17T17:52:10.109295+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":506,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.284179+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:52:05.284179+0000","last_peered":"2024-08-17T17:52:05.284179+0000","last_clean":"2024-08-17T17:52:05.284179+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:52:05.284179+0000","last_undegraded":"2024-08-17T17:52:05.284179+0000","last_fullsized":"2024-08-17T17:52:05.284179+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1183,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283814+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:52:05.283814+0000","last_peered":"2024-08-17T17:52:05.283814+0000","last_clean":"2024-08-17T17:52:05.283814+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:52:05.283814+0000","last_undegraded":"2024-08-17T17:52:05.283814+0000","last_fullsized":"2024-08-17T17:52:05.283814+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1864,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.386635+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:52:21.386635+0000","last_peered":"2024-08-17T17:52:21.386635+0000","last_clean":"2024-08-17T17:52:21.386635+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:52:21.386635+0000","last_undegraded":"2024-08-17T17:52:21.386635+0000","last_fullsized":"2024-08-17T17:52:21.386635+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":344,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283695+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:52:05.283695+0000","last_peered":"2024-08-17T17:52:05.283695+0000","last_clean":"2024-08-17T17:52:05.283695+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:52:05.283695+0000","last_undegraded":"2024-08-17T17:52:05.283695+0000","last_fullsized":"2024-08-17T17:52:05.283695+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":490,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280438+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:52:05.280438+0000","last_peered":"2024-08-17T17:52:05.280438+0000","last_clean":"2024-08-17T17:52:05.280438+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:52:05.280438+0000","last_undegraded":"2024-08-17T17:52:05.280438+0000","last_fullsized":"2024-08-17T17:52:05.280438+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1168,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280443+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:52:05.280443+0000","last_peered":"2024-08-17T17:52:05.280443+0000","last_clean":"2024-08-17T17:52:05.280443+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:52:05.280443+0000","last_undegraded":"2024-08-17T17:52:05.280443+0000","last_fullsized":"2024-08-17T17:52:05.280443+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1176,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280586+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:52:05.280586+0000","last_peered":"2024-08-17T17:52:05.280586+0000","last_clean":"2024-08-17T17:52:05.280586+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:52:05.280586+0000","last_undegraded":"2024-08-17T17:52:05.280586+0000","last_fullsized":"2024-08-17T17:52:05.280586+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1911,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.274907+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:52:21.274907+0000","last_peered":"2024-08-17T17:52:21.274907+0000","last_clean":"2024-08-17T17:52:21.274907+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:52:21.274907+0000","last_undegraded":"2024-08-17T17:52:21.274907+0000","last_fullsized":"2024-08-17T17:52:21.274907+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1793,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109436+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:52:10.109436+0000","last_peered":"2024-08-17T17:52:10.109436+0000","last_clean":"2024-08-17T17:52:10.109436+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:52:10.109436+0000","last_undegraded":"2024-08-17T17:52:10.109436+0000","last_fullsized":"2024-08-17T17:52:10.109436+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1870,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.385279+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:52:21.385279+0000","last_peered":"2024-08-17T17:52:21.385279+0000","last_clean":"2024-08-17T17:52:21.385279+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:52:21.385279+0000","last_undegraded":"2024-08-17T17:52:21.385279+0000","last_fullsized":"2024-08-17T17:52:21.385279+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1795,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109712+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:52:10.109712+0000","last_peered":"2024-08-17T17:52:10.109712+0000","last_clean":"2024-08-17T17:52:10.109712+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:52:10.109712+0000","last_undegraded":"2024-08-17T17:52:10.109712+0000","last_fullsized":"2024-08-17T17:52:10.109712+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1829,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.274686+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:52:21.274686+0000","last_peered":"2024-08-17T17:52:21.274686+0000","last_clean":"2024-08-17T17:52:21.274686+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:52:21.274686+0000","last_undegraded":"2024-08-17T17:52:21.274686+0000","last_fullsized":"2024-08-17T17:52:21.274686+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1203,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283903+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:52:05.283903+0000","last_peered":"2024-08-17T17:52:05.283903+0000","last_clean":"2024-08-17T17:52:05.283903+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:52:05.283903+0000","last_undegraded":"2024-08-17T17:52:05.283903+0000","last_fullsized":"2024-08-17T17:52:05.283903+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1876,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.385299+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:52:21.385299+0000","last_peered":"2024-08-17T17:52:21.385299+0000","last_clean":"2024-08-17T17:52:21.385299+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:52:21.385299+0000","last_undegraded":"2024-08-17T17:52:21.385299+0000","last_fullsized":"2024-08-17T17:52:21.385299+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":63,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.384776+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:52:21.384776+0000","last_peered":"2024-08-17T17:52:21.384776+0000","last_clean":"2024-08-17T17:52:21.384776+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:52:21.384776+0000","last_undegraded":"2024-08-17T17:52:21.384776+0000","last_fullsized":"2024-08-17T17:52:21.384776+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1895,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.385218+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:52:21.385218+0000","last_peered":"2024-08-17T17:52:21.385218+0000","last_clean":"2024-08-17T17:52:21.385218+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:52:21.385218+0000","last_undegraded":"2024-08-17T17:52:21.385218+0000","last_fullsized":"2024-08-17T17:52:21.385218+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":630,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.111889+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:52:10.111889+0000","last_peered":"2024-08-17T17:52:10.111889+0000","last_clean":"2024-08-17T17:52:10.111889+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:52:10.111889+0000","last_undegraded":"2024-08-17T17:52:10.111889+0000","last_fullsized":"2024-08-17T17:52:10.111889+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1876,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.385381+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:52:21.385381+0000","last_peered":"2024-08-17T17:52:21.385381+0000","last_clean":"2024-08-17T17:52:21.385381+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:52:21.385381+0000","last_undegraded":"2024-08-17T17:52:21.385381+0000","last_fullsized":"2024-08-17T17:52:21.385381+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1205,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.274461+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:52:21.274461+0000","last_peered":"2024-08-17T17:52:21.274461+0000","last_clean":"2024-08-17T17:52:21.274461+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:52:21.274461+0000","last_undegraded":"2024-08-17T17:52:21.274461+0000","last_fullsized":"2024-08-17T17:52:21.274461+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":123,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.386530+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:52:21.386530+0000","last_peered":"2024-08-17T17:52:21.386530+0000","last_clean":"2024-08-17T17:52:21.386530+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:52:21.386530+0000","last_undegraded":"2024-08-17T17:52:21.386530+0000","last_fullsized":"2024-08-17T17:52:21.386530+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1887,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.385489+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:52:21.385489+0000","last_peered":"2024-08-17T17:52:21.385489+0000","last_clean":"2024-08-17T17:52:21.385489+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:52:21.385489+0000","last_undegraded":"2024-08-17T17:52:21.385489+0000","last_fullsized":"2024-08-17T17:52:21.385489+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1864,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.386117+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:52:21.386117+0000","last_peered":"2024-08-17T17:52:21.386117+0000","last_clean":"2024-08-17T17:52:21.386117+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:52:21.386117+0000","last_undegraded":"2024-08-17T17:52:21.386117+0000","last_fullsized":"2024-08-17T17:52:21.386117+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1883,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.358168+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:52:10.358168+0000","last_peered":"2024-08-17T17:52:10.358168+0000","last_clean":"2024-08-17T17:52:10.358168+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:52:10.358168+0000","last_undegraded":"2024-08-17T17:52:10.358168+0000","last_fullsized":"2024-08-17T17:52:10.358168+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1890,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.275226+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:52:21.275226+0000","last_peered":"2024-08-17T17:52:21.275226+0000","last_clean":"2024-08-17T17:52:21.275226+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:52:21.275226+0000","last_undegraded":"2024-08-17T17:52:21.275226+0000","last_fullsized":"2024-08-17T17:52:21.275226+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1870,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.346097+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:52:21.346097+0000","last_peered":"2024-08-17T17:52:21.346097+0000","last_clean":"2024-08-17T17:52:21.346097+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:52:21.346097+0000","last_undegraded":"2024-08-17T17:52:21.346097+0000","last_fullsized":"2024-08-17T17:52:21.346097+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1210,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283492+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:52:05.283492+0000","last_peered":"2024-08-17T17:52:05.283492+0000","last_clean":"2024-08-17T17:52:05.283492+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:52:05.283492+0000","last_undegraded":"2024-08-17T17:52:05.283492+0000","last_fullsized":"2024-08-17T17:52:05.283492+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1156,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280652+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:52:05.280652+0000","last_peered":"2024-08-17T17:52:05.280652+0000","last_clean":"2024-08-17T17:52:05.280652+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:52:05.280652+0000","last_undegraded":"2024-08-17T17:52:05.280652+0000","last_fullsized":"2024-08-17T17:52:05.280652+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1777,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109988+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:52:10.109988+0000","last_peered":"2024-08-17T17:52:10.109988+0000","last_clean":"2024-08-17T17:52:10.109988+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:52:10.109988+0000","last_undegraded":"2024-08-17T17:52:10.109988+0000","last_fullsized":"2024-08-17T17:52:10.109988+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":109,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.356563+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:52:10.356563+0000","last_peered":"2024-08-17T17:52:10.356563+0000","last_clean":"2024-08-17T17:52:10.356563+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:52:10.356563+0000","last_undegraded":"2024-08-17T17:52:10.356563+0000","last_fullsized":"2024-08-17T17:52:10.356563+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1522,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283394+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:52:05.283394+0000","last_peered":"2024-08-17T17:52:05.283394+0000","last_clean":"2024-08-17T17:52:05.283394+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:52:05.283394+0000","last_undegraded":"2024-08-17T17:52:05.283394+0000","last_fullsized":"2024-08-17T17:52:05.283394+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1871,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.385681+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:52:21.385681+0000","last_peered":"2024-08-17T17:52:21.385681+0000","last_clean":"2024-08-17T17:52:21.385681+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:52:21.385681+0000","last_undegraded":"2024-08-17T17:52:21.385681+0000","last_fullsized":"2024-08-17T17:52:21.385681+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1809,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109347+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:52:10.109347+0000","last_peered":"2024-08-17T17:52:10.109347+0000","last_clean":"2024-08-17T17:52:10.109347+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:52:10.109347+0000","last_undegraded":"2024-08-17T17:52:10.109347+0000","last_fullsized":"2024-08-17T17:52:10.109347+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":490,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280510+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:52:05.280510+0000","last_peered":"2024-08-17T17:52:05.280510+0000","last_clean":"2024-08-17T17:52:05.280510+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:52:05.280510+0000","last_undegraded":"2024-08-17T17:52:05.280510+0000","last_fullsized":"2024-08-17T17:52:05.280510+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":548,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.358094+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:52:10.358094+0000","last_peered":"2024-08-17T17:52:10.358094+0000","last_clean":"2024-08-17T17:52:10.358094+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:52:10.358094+0000","last_undegraded":"2024-08-17T17:52:10.358094+0000","last_fullsized":"2024-08-17T17:52:10.358094+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":564,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.356566+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:52:10.356566+0000","last_peered":"2024-08-17T17:52:10.356566+0000","last_clean":"2024-08-17T17:52:10.356566+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:52:10.356566+0000","last_undegraded":"2024-08-17T17:52:10.356566+0000","last_fullsized":"2024-08-17T17:52:10.356566+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1174,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280643+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:52:05.280643+0000","last_peered":"2024-08-17T17:52:05.280643+0000","last_clean":"2024-08-17T17:52:05.280643+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:52:05.280643+0000","last_undegraded":"2024-08-17T17:52:05.280643+0000","last_fullsized":"2024-08-17T17:52:05.280643+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":580,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109679+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:52:10.109679+0000","last_peered":"2024-08-17T17:52:10.109679+0000","last_clean":"2024-08-17T17:52:10.109679+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:52:10.109679+0000","last_undegraded":"2024-08-17T17:52:10.109679+0000","last_fullsized":"2024-08-17T17:52:10.109679+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1806,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109715+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:52:10.109715+0000","last_peered":"2024-08-17T17:52:10.109715+0000","last_clean":"2024-08-17T17:52:10.109715+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:52:10.109715+0000","last_undegraded":"2024-08-17T17:52:10.109715+0000","last_fullsized":"2024-08-17T17:52:10.109715+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1903,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.274711+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:52:21.274711+0000","last_peered":"2024-08-17T17:52:21.274711+0000","last_clean":"2024-08-17T17:52:21.274711+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:52:21.274711+0000","last_undegraded":"2024-08-17T17:52:21.274711+0000","last_fullsized":"2024-08-17T17:52:21.274711+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":612,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109639+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:52:10.109639+0000","last_peered":"2024-08-17T17:52:10.109639+0000","last_clean":"2024-08-17T17:52:10.109639+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:52:10.109639+0000","last_undegraded":"2024-08-17T17:52:10.109639+0000","last_fullsized":"2024-08-17T17:52:10.109639+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1163,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280249+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:52:05.280249+0000","last_peered":"2024-08-17T17:52:05.280249+0000","last_clean":"2024-08-17T17:52:05.280249+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:52:05.280249+0000","last_undegraded":"2024-08-17T17:52:05.280249+0000","last_fullsized":"2024-08-17T17:52:05.280249+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1162,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283933+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:52:05.283933+0000","last_peered":"2024-08-17T17:52:05.283933+0000","last_clean":"2024-08-17T17:52:05.283933+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:52:05.283933+0000","last_undegraded":"2024-08-17T17:52:05.283933+0000","last_fullsized":"2024-08-17T17:52:05.283933+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":599,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.384317+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:52:21.384317+0000","last_peered":"2024-08-17T17:52:21.384317+0000","last_clean":"2024-08-17T17:52:21.384317+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:52:21.384317+0000","last_undegraded":"2024-08-17T17:52:21.384317+0000","last_fullsized":"2024-08-17T17:52:21.384317+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1180,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.284256+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:52:05.284256+0000","last_peered":"2024-08-17T17:52:05.284256+0000","last_clean":"2024-08-17T17:52:05.284256+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:52:05.284256+0000","last_undegraded":"2024-08-17T17:52:05.284256+0000","last_fullsized":"2024-08-17T17:52:05.284256+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":64,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283364+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:52:05.283364+0000","last_peered":"2024-08-17T17:52:05.283364+0000","last_clean":"2024-08-17T17:52:05.283364+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:52:05.283364+0000","last_undegraded":"2024-08-17T17:52:05.283364+0000","last_fullsized":"2024-08-17T17:52:05.283364+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1196,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283994+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:52:05.283994+0000","last_peered":"2024-08-17T17:52:05.283994+0000","last_clean":"2024-08-17T17:52:05.283994+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:52:05.283994+0000","last_undegraded":"2024-08-17T17:52:05.283994+0000","last_fullsized":"2024-08-17T17:52:05.283994+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1213,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.280219+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:52:05.280219+0000","last_peered":"2024-08-17T17:52:05.280219+0000","last_clean":"2024-08-17T17:52:05.280219+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:52:05.280219+0000","last_undegraded":"2024-08-17T17:52:05.280219+0000","last_fullsized":"2024-08-17T17:52:05.280219+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1807,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109605+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:52:10.109605+0000","last_peered":"2024-08-17T17:52:10.109605+0000","last_clean":"2024-08-17T17:52:10.109605+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:52:10.109605+0000","last_undegraded":"2024-08-17T17:52:10.109605+0000","last_fullsized":"2024-08-17T17:52:10.109605+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1157,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283673+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:52:05.283673+0000","last_peered":"2024-08-17T17:52:05.283673+0000","last_clean":"2024-08-17T17:52:05.283673+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:52:05.283673+0000","last_undegraded":"2024-08-17T17:52:05.283673+0000","last_fullsized":"2024-08-17T17:52:05.283673+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1862,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.274997+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:52:21.274997+0000","last_peered":"2024-08-17T17:52:21.274997+0000","last_clean":"2024-08-17T17:52:21.274997+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:52:21.274997+0000","last_undegraded":"2024-08-17T17:52:21.274997+0000","last_fullsized":"2024-08-17T17:52:21.274997+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1863,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.274520+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:52:21.274520+0000","last_peered":"2024-08-17T17:52:21.274520+0000","last_clean":"2024-08-17T17:52:21.274520+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:52:21.274520+0000","last_undegraded":"2024-08-17T17:52:21.274520+0000","last_fullsized":"2024-08-17T17:52:21.274520+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":570,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283766+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:52:05.283766+0000","last_peered":"2024-08-17T17:52:05.283766+0000","last_clean":"2024-08-17T17:52:05.283766+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:52:05.283766+0000","last_undegraded":"2024-08-17T17:52:05.283766+0000","last_fullsized":"2024-08-17T17:52:05.283766+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1182,"reported_epoch":1212,"state":"active+clean","last_fresh":"2024-08-17T17:52:05.283881+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:52:05.283881+0000","last_peered":"2024-08-17T17:52:05.283881+0000","last_clean":"2024-08-17T17:52:05.283881+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:52:05.283881+0000","last_undegraded":"2024-08-17T17:52:05.283881+0000","last_fullsized":"2024-08-17T17:52:05.283881+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1807,"reported_epoch":1213,"state":"active+clean","last_fresh":"2024-08-17T17:52:10.109782+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:52:10.109782+0000","last_peered":"2024-08-17T17:52:10.109782+0000","last_clean":"2024-08-17T17:52:10.109782+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:52:10.109782+0000","last_undegraded":"2024-08-17T17:52:10.109782+0000","last_fullsized":"2024-08-17T17:52:10.109782+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":5291114520,"num_objects":80737,"num_object_clones":0,"num_object_copies":242211,"num_objects_missing_on_primary":4522,"num_objects_missing":4522,"num_objects_degraded":19222,"num_objects_misplaced":13947,"num_objects_unfound":0,"num_objects_dirty":80737,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":128873,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33049,"num_bytes_recovered":1987444736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16525766656,"data_stored":16525754440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106447,"ondisk_log_size":106447,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234377,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2466560,"kb_used_data":2165048,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91278592,"statfs":{"total":95995035648,"available":93469278208,"internally_reserved":0,"allocated":2217009152,"data_stored":2212189051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496012,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2801028,"kb_used_data":2499516,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":90944124,"statfs":{"total":95995035648,"available":93126782976,"internally_reserved":0,"allocated":2559504384,"data_stored":2554680688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757650,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326264,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3091436,"kb_used_data":2888868,"kb_used_omap":57,"kb_used_meta":202502,"kb_avail":90653716,"statfs":{"total":95995035648,"available":92829405184,"internally_reserved":0,"allocated":2958200832,"data_stored":2953395911,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":207362811},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405503,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2791092,"kb_used_data":2588460,"kb_used_omap":61,"kb_used_meta":202562,"kb_avail":90954060,"statfs":{"total":95995035648,"available":93136957440,"internally_reserved":0,"allocated":2650583040,"data_stored":2645775091,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":207424440},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452038,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2099448,"kb_used_data":1941296,"kb_used_omap":54,"kb_used_meta":158089,"kb_avail":91645704,"statfs":{"total":95995035648,"available":93845200896,"internally_reserved":0,"allocated":1987887104,"data_stored":1983078708,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":161883534},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474730,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1713228,"kb_used_data":1558528,"kb_used_omap":119,"kb_used_meta":154568,"kb_avail":92031924,"statfs":{"total":95995035648,"available":94240690176,"internally_reserved":0,"allocated":1595932672,"data_stored":1591109112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":158278366},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,2],"upper_bound":8192},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019285,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1581977600,"data_stored":1581973528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1974665216,"data_stored":1974665216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2637234176,"data_stored":2637234176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2944733184,"data_stored":2944729112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1100611584,"data_stored":1100611584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1538064384,"data_stored":1538064384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2545487872,"data_stored":2545483800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2202992640,"data_stored":2202992640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:52:26.115 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:52:26.164 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:26.185 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:26.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:26.287 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:26.307 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 19222/241137 objects degraded (7.971%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:26.310 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:26.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:26.412 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:26.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 19222/241137 objects degraded (7.971%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:26.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 19222/241137 objects degraded (7.971%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:26.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:26.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:26.535 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:26.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.561 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:26.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:26.591 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:52:26.591 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:52:26.662 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:26.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:26.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:26.788 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:26.809 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:26.912 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:26.949 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:26.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:26.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:27.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:27.073 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:27.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:27.175 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:27.186 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:27.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.197 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:27.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:27.297 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:27 smithi179 ceph-mon[93519]: pgmap v1750: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 19148/240927 objects degraded (7.948%); 13914/240927 objects misplaced (5.775%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:52:27.298 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:27.320 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:52:27.323 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:27.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:27 smithi130 ceph-mon[105521]: pgmap v1750: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 19148/240927 objects degraded (7.948%); 13914/240927 objects misplaced (5.775%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:52:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:27 smithi130 ceph-mon[110068]: pgmap v1750: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 19148/240927 objects degraded (7.948%); 13914/240927 objects misplaced (5.775%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:52:27.425 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:27.447 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:27.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:27.548 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:27.571 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:27.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:27.595 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:27.673 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:27.696 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:27.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:27.798 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:27.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:27.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:27.922 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:27.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:27.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:27.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:28.045 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:28.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.070 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:28.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:28.171 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:28.205 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:28.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:28.241 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:52:28.241 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:52:28.241 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:52:28.241 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:52:28.241 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:52:28.241 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:52:28.247 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:28.256 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:52:28.256 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:52:28.283 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:52:28.283 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:52:28.305 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:52:28.305 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:52:28.306 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:28.327 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:28.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:28.333 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:52:28.333 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:52:28.333 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:28.333 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:28.333 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:28 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2470726122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:28.334 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:28.334 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:28.334 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:28 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2470726122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:28.361 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:52:28.361 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:52:28.429 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:28.450 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:28.551 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:28.575 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:28.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:28 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2470726122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:28.677 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:28.703 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:28.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:28.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:28.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:28.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:28.928 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:28.952 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:28.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:28.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:29.054 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:29.076 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:29.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:29.177 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:29.200 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:29.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:29.301 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:29.327 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:29.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:29.418 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:52:29.418 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:52:29.419 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T17:52:29.419 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T17:52:29.419 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:52:29.419 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:29 smithi179 ceph-mon[93519]: pgmap v1751: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 19148/239508 objects degraded (7.995%); 13843/239508 objects misplaced (5.780%); 966 KiB/s, 15 objects/s recovering 2024-08-17T17:52:29.429 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:29.451 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:29.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:29.553 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:29.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:29.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:29.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:29 smithi130 ceph-mon[105521]: pgmap v1751: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 19148/239508 objects degraded (7.995%); 13843/239508 objects misplaced (5.780%); 966 KiB/s, 15 objects/s recovering 2024-08-17T17:52:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:29 smithi130 ceph-mon[110068]: pgmap v1751: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 19148/239508 objects degraded (7.995%); 13843/239508 objects misplaced (5.780%); 966 KiB/s, 15 objects/s recovering 2024-08-17T17:52:29.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:29.698 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:29.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:29.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:29.827 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:29.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:29.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:29.951 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:29.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:29.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:30.053 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:30.075 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:30.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:52:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:52:29] "GET /metrics HTTP/1.1" 200 36809 "" "Prometheus/2.51.0" 2024-08-17T17:52:30.177 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:30.199 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:30.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:30.299 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:52:30.301 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:30.321 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:30.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:30.423 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:30.446 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:30.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:30.548 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:30.561 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:30.572 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:30.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:30.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:52:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:52:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:52:30.674 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:30.697 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:30.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:30.798 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:30.820 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:30.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:30.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:30.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:30.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:30.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:30.995 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:30 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 19148/239508 objects degraded (7.995%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:31.052 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:31.073 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:31.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:31.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:30 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 19148/239508 objects degraded (7.995%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:31.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:30 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 19148/239508 objects degraded (7.995%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:31.174 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:31.198 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:31.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:31.300 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:31.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.322 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:31.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:31.423 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:31.444 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:31.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:31.546 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:31.569 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:31.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:31.671 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:31.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:31.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:31.740 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:52:31.741 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 19148/239259 objects degraded (8.003%), 6 pgs degraded, 6 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1472,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1214,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":79753,"data_bytes":5225519077,"bytes_used":15170244608,"bytes_avail":560799969280,"bytes_total":575970213888,"degraded_objects":19148,"degraded_total":239259,"degraded_ratio":0.080030427277552774,"misplaced_objects":13843,"misplaced_total":239259,"misplaced_ratio":0.057857802632293877,"recovering_objects_per_sec":15,"recovering_bytes_per_sec":989283,"recovering_keys_per_sec":0,"num_objects_recovered":151,"num_bytes_recovered":9895936,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":114},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":25,"modified":"2024-08-17T17:49:39.646037+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (4m)\n [==========================..] (remaining: 21s)","progress":0.93258428573608398,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:52:31.794 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:31.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:31.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:31.923 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:31.945 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:31.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:31.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:31.985 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:31 smithi130 ceph-mon[105521]: pgmap v1752: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 19148/239259 objects degraded (8.003%); 13843/239259 objects misplaced (5.786%); 966 KiB/s, 15 objects/s recovering 2024-08-17T17:52:31.986 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:31 smithi130 ceph-mon[110068]: pgmap v1752: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 19148/239259 objects degraded (8.003%); 13843/239259 objects misplaced (5.786%); 966 KiB/s, 15 objects/s recovering 2024-08-17T17:52:32.047 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:32.075 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:32.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:31 smithi179 ceph-mon[93519]: pgmap v1752: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 19148/239259 objects degraded (8.003%); 13843/239259 objects misplaced (5.786%); 966 KiB/s, 15 objects/s recovering 2024-08-17T17:52:32.176 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:32.198 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:32.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:32.300 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:32.323 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:32.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:32.384 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:52:32.384 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T17:52:32.385 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T17:52:32.385 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T17:52:32.385 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.385 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.44945.0:3069 44.11 e1135/1015 44:8acf66ad:::rbd_data.af91c8ed8b9b.000000000000000b:head v 1135'28, mlcod=1122'24)", 2024-08-17T17:52:32.385 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:45:48.880300+0000", 2024-08-17T17:52:32.385 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 403.47455399400002, 2024-08-17T17:52:32.385 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.04859496, 2024-08-17T17:52:32.385 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.385 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:32.385 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.385 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.385 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.385 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:48.880300+0000", 2024-08-17T17:52:32.385 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.386 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.386 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.386 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.386 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:48.880300+0000", 2024-08-17T17:52:32.386 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.386 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.386 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.386 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.386 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:48.880300+0000", 2024-08-17T17:52:32.386 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967296 2024-08-17T17:52:32.386 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.386 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.387 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.387 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:48.899987+0000", 2024-08-17T17:52:32.387 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.019686611999999999 2024-08-17T17:52:32.387 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.387 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.387 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.387 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:48.900003+0000", 2024-08-17T17:52:32.387 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.6702999999999999e-05 2024-08-17T17:52:32.387 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.387 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.387 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.387 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:48.900009+0000", 2024-08-17T17:52:32.387 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6890000000000001e-06 2024-08-17T17:52:32.387 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.388 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.388 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.388 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:48.900085+0000", 2024-08-17T17:52:32.388 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.5710999999999994e-05 2024-08-17T17:52:32.388 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.388 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.388 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:32.388 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:48.900112+0000", 2024-08-17T17:52:32.388 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.6761e-05 2024-08-17T17:52:32.388 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.388 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.388 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:32.388 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:48.928854+0000", 2024-08-17T17:52:32.389 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.028742051000000001 2024-08-17T17:52:32.389 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.389 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.389 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.389 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:48.928895+0000", 2024-08-17T17:52:32.389 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1637000000000001e-05 2024-08-17T17:52:32.389 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.389 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.389 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.389 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.389 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.389 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.44945.0:6942 44.6 44:64b77b5f:::rbd_data.af91fb6f432.0000000000000001:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1158)", 2024-08-17T17:52:32.389 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:46:11.755236+0000", 2024-08-17T17:52:32.390 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 380.59961877799998, 2024-08-17T17:52:32.390 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.044327476999999997, 2024-08-17T17:52:32.390 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.390 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:32.390 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:32.390 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.44945", 2024-08-17T17:52:32.390 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/735940947", 2024-08-17T17:52:32.390 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 6942 2024-08-17T17:52:32.390 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.390 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.390 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.390 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.390 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:46:11.755236+0000", 2024-08-17T17:52:32.391 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.391 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.391 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.391 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.391 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:46:11.755236+0000", 2024-08-17T17:52:32.391 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.391 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.391 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.391 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.391 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:46:11.755231+0000", 2024-08-17T17:52:32.391 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999957 2024-08-17T17:52:32.391 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.391 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.391 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.392 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:46:11.764390+0000", 2024-08-17T17:52:32.392 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0091589029999999995 2024-08-17T17:52:32.392 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.392 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.392 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.392 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:46:11.764434+0000", 2024-08-17T17:52:32.392 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3577e-05 2024-08-17T17:52:32.392 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.392 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.392 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.392 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:46:11.764446+0000", 2024-08-17T17:52:32.392 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2172e-05 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:46:11.764538+0000", 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.1638999999999995e-05 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:46:11.764702+0000", 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000164654 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:46:11.764847+0000", 2024-08-17T17:52:32.393 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000144592 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:46:11.772487+0000", 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.007639814 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:46:11.799421+0000", 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.026934288000000001 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:46:11.799475+0000", 2024-08-17T17:52:32.394 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.4326000000000001e-05 2024-08-17T17:52:32.395 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.395 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.395 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:32.395 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:46:11.799523+0000", 2024-08-17T17:52:32.395 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7332e-05 2024-08-17T17:52:32.395 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.395 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.395 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.395 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:46:11.799563+0000", 2024-08-17T17:52:32.395 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0377999999999999e-05 2024-08-17T17:52:32.395 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.395 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.395 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.395 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.45158.0:62426 45.1 e1175/1173 45:876b244c:::benchmark_data_smithi130_172551_object62425:head v 1175'3976, mlcod=1170'715)", 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:19.297389+0000", 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 253.05746583199999, 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.045999542999999997, 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297389+0000", 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.396 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297389+0000", 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297387+0000", 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297415+0000", 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.775e-05 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.397 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.398 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297428+0000", 2024-08-17T17:52:32.398 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2955999999999999e-05 2024-08-17T17:52:32.398 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.398 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.398 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.398 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297435+0000", 2024-08-17T17:52:32.398 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4139999999999999e-06 2024-08-17T17:52:32.398 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.398 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.398 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.398 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297484+0000", 2024-08-17T17:52:32.398 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9425999999999997e-05 2024-08-17T17:52:32.398 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.399 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.399 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:32.399 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297507+0000", 2024-08-17T17:52:32.399 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3479999999999999e-05 2024-08-17T17:52:32.399 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.399 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.399 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:32.399 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.343348+0000", 2024-08-17T17:52:32.399 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.045840076 2024-08-17T17:52:32.399 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.399 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.399 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.399 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.343388+0000", 2024-08-17T17:52:32.400 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0510999999999999e-05 2024-08-17T17:52:32.400 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.400 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.400 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.400 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.400 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.400 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.45158.0:62437 45.1 e1175/1173 45:86d1cc6a:::benchmark_data_smithi130_172551_object62436:head v 1175'3977, mlcod=1170'715)", 2024-08-17T17:52:32.400 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:19.297717+0000", 2024-08-17T17:52:32.400 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 253.05713752400001, 2024-08-17T17:52:32.400 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.049960840999999999, 2024-08-17T17:52:32.400 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.400 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:32.400 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.400 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.401 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.401 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297717+0000", 2024-08-17T17:52:32.401 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.401 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.401 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.401 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.401 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297717+0000", 2024-08-17T17:52:32.401 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.401 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.401 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.401 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.401 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297716+0000", 2024-08-17T17:52:32.401 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:52:32.402 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.402 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.402 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.402 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297741+0000", 2024-08-17T17:52:32.402 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4481000000000002e-05 2024-08-17T17:52:32.402 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.402 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.402 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.402 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297751+0000", 2024-08-17T17:52:32.402 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0504e-05 2024-08-17T17:52:32.402 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.402 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.402 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.403 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.297757+0000", 2024-08-17T17:52:32.403 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.4999999999999999e-06 2024-08-17T17:52:32.403 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.403 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.403 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.403 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.299279+0000", 2024-08-17T17:52:32.403 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0015218969999999999 2024-08-17T17:52:32.403 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.403 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.403 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:32.403 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.299300+0000", 2024-08-17T17:52:32.403 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.1257999999999999e-05 2024-08-17T17:52:32.403 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.404 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.404 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:32.404 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.347590+0000", 2024-08-17T17:52:32.404 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.048290080999999999 2024-08-17T17:52:32.404 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.404 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.404 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.404 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.347678+0000", 2024-08-17T17:52:32.404 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7810000000000001e-05 2024-08-17T17:52:32.404 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.404 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.404 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.404 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.405 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.405 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.45158.0:62441 45.0 e1175/1173 45:0d4f8b75:::benchmark_data_smithi130_172551_object62440:head v 1175'3825, mlcod=0'0)", 2024-08-17T17:52:32.405 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:19.299115+0000", 2024-08-17T17:52:32.405 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 253.05573955200001, 2024-08-17T17:52:32.405 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.045174164000000003, 2024-08-17T17:52:32.405 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.405 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:32.405 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.405 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.405 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.405 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.299115+0000", 2024-08-17T17:52:32.405 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.405 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.405 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.406 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.406 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.299115+0000", 2024-08-17T17:52:32.406 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.406 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.406 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.406 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.406 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.299114+0000", 2024-08-17T17:52:32.406 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:52:32.406 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.406 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.406 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.406 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.299191+0000", 2024-08-17T17:52:32.406 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7600000000000002e-05 2024-08-17T17:52:32.407 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.407 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.407 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.407 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.299206+0000", 2024-08-17T17:52:32.407 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4796e-05 2024-08-17T17:52:32.407 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.407 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.407 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.407 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.299212+0000", 2024-08-17T17:52:32.407 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6520000000000003e-06 2024-08-17T17:52:32.407 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.407 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.407 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.408 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.299258+0000", 2024-08-17T17:52:32.408 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6286999999999999e-05 2024-08-17T17:52:32.408 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.408 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.408 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:32.408 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.299281+0000", 2024-08-17T17:52:32.408 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3161000000000001e-05 2024-08-17T17:52:32.408 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.408 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.408 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:32.408 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.344211+0000", 2024-08-17T17:52:32.408 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.044930215000000003 2024-08-17T17:52:32.408 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.408 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.409 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.409 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:19.344289+0000", 2024-08-17T17:52:32.409 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7471000000000004e-05 2024-08-17T17:52:32.409 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.409 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.409 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.409 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.409 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.409 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1199/1189 cost 1049576)", 2024-08-17T17:52:32.409 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:50:54.781959+0000", 2024-08-17T17:52:32.409 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 97.572895411000005, 2024-08-17T17:52:32.409 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00078927299999999995, 2024-08-17T17:52:32.409 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.409 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.410 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.410 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.410 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.410 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:54.781959+0000", 2024-08-17T17:52:32.410 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.410 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.410 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.410 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.410 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:54.781959+0000", 2024-08-17T17:52:32.410 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.410 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.410 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.410 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.410 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:54.781957+0000", 2024-08-17T17:52:32.411 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:32.411 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.411 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.411 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.411 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:54.781982+0000", 2024-08-17T17:52:32.411 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4572000000000001e-05 2024-08-17T17:52:32.411 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.411 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.411 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.411 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:54.782011+0000", 2024-08-17T17:52:32.411 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9592999999999999e-05 2024-08-17T17:52:32.411 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.411 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.411 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.412 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:54.782024+0000", 2024-08-17T17:52:32.412 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2510000000000001e-05 2024-08-17T17:52:32.412 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.412 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.412 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.412 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:54.782125+0000", 2024-08-17T17:52:32.412 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000100857 2024-08-17T17:52:32.412 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.412 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.412 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.412 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:54.782748+0000", 2024-08-17T17:52:32.412 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00062365399999999996 2024-08-17T17:52:32.412 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.412 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.413 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.413 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.413 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.413 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1203/1189 cost 1049576)", 2024-08-17T17:52:32.413 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:51:17.857180+0000", 2024-08-17T17:52:32.413 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 74.497674638000007, 2024-08-17T17:52:32.413 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00070549299999999996, 2024-08-17T17:52:32.413 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.413 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.413 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.413 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.413 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.413 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:17.857180+0000", 2024-08-17T17:52:32.414 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.414 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.414 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.414 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.414 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:17.857180+0000", 2024-08-17T17:52:32.414 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.414 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.414 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.414 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.414 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:17.857177+0000", 2024-08-17T17:52:32.414 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999976 2024-08-17T17:52:32.414 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.414 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.415 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.415 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:17.857203+0000", 2024-08-17T17:52:32.415 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5536000000000001e-05 2024-08-17T17:52:32.415 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.415 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.415 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.415 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:17.857231+0000", 2024-08-17T17:52:32.415 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8416e-05 2024-08-17T17:52:32.415 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.415 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.415 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.415 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:17.857244+0000", 2024-08-17T17:52:32.415 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.238e-05 2024-08-17T17:52:32.415 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:17.857346+0000", 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000101858 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:17.857885+0000", 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00053966599999999995 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.416 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1204/1189 cost 1049576)", 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:51:19.498865+0000", 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 72.855989073000003, 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00073345699999999997, 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:19.498865+0000", 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:19.498865+0000", 2024-08-17T17:52:32.417 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.418 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.418 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.418 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.418 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:19.498863+0000", 2024-08-17T17:52:32.418 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:32.418 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.418 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.418 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.418 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:19.498892+0000", 2024-08-17T17:52:32.418 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8399e-05 2024-08-17T17:52:32.418 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.418 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.418 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:19.498920+0000", 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8309999999999998e-05 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:19.498932+0000", 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2023e-05 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:19.499034+0000", 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000101714 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.419 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:19.499599+0000", 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00056478400000000003 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1204/1189 cost 1049576)", 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:51:21.538507+0000", 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 70.816347711000006, 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00067240099999999999, 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.420 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:21.538507+0000", 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:21.538507+0000", 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:21.538505+0000", 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.421 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:21.538530+0000", 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5199e-05 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:21.538559+0000", 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8600000000000001e-05 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:21.538571+0000", 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2216e-05 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.422 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:21.538685+0000", 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000113966 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:21.539179+0000", 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00049421299999999997 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1209/1189 cost 1049576)", 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:51:44.108990+0000", 2024-08-17T17:52:32.423 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 48.245863843000002, 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00067501499999999995, 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:44.108990+0000", 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:44.108990+0000", 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.424 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:44.108988+0000", 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:44.109013+0000", 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4879e-05 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:44.109042+0000", 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8619000000000002e-05 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.425 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:44.109054+0000", 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2196e-05 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:44.109148+0000", 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.3374000000000004e-05 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:44.109666+0000", 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00051799300000000001 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.426 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.427 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.427 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.427 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.427 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1209/1189 cost 1049576)", 2024-08-17T17:52:32.427 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:51:46.957129+0000", 2024-08-17T17:52:32.427 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 45.397725620999999, 2024-08-17T17:52:32.427 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00066122700000000002, 2024-08-17T17:52:32.427 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.427 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.427 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.427 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.427 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.427 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:46.957129+0000", 2024-08-17T17:52:32.428 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.428 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.428 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.428 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.428 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:46.957129+0000", 2024-08-17T17:52:32.428 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.428 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.428 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.428 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.428 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:46.957127+0000", 2024-08-17T17:52:32.428 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:32.428 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.428 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.429 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.429 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:46.957153+0000", 2024-08-17T17:52:32.429 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.6302e-05 2024-08-17T17:52:32.429 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.429 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.429 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.429 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:46.957182+0000", 2024-08-17T17:52:32.429 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.942e-05 2024-08-17T17:52:32.429 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.429 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.429 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.429 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:46.957196+0000", 2024-08-17T17:52:32.429 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3259e-05 2024-08-17T17:52:32.429 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.430 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.430 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.430 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:46.957301+0000", 2024-08-17T17:52:32.430 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105895 2024-08-17T17:52:32.430 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.430 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.430 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.430 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:46.957790+0000", 2024-08-17T17:52:32.430 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00048849400000000002 2024-08-17T17:52:32.430 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.430 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.430 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.430 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.430 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.431 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1209/1189 cost 1049576)", 2024-08-17T17:52:32.431 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:51:52.969675+0000", 2024-08-17T17:52:32.431 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 39.385179620000002, 2024-08-17T17:52:32.431 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025244299999999998, 2024-08-17T17:52:32.431 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.431 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.431 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.431 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.431 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.431 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.969675+0000", 2024-08-17T17:52:32.431 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.431 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.432 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.432 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.432 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.969675+0000", 2024-08-17T17:52:32.432 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.432 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.432 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.432 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.432 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.969674+0000", 2024-08-17T17:52:32.432 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:52:32.432 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.432 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.432 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.432 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.969687+0000", 2024-08-17T17:52:32.433 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3352e-05 2024-08-17T17:52:32.433 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.433 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.433 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.433 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.969702+0000", 2024-08-17T17:52:32.433 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5099000000000001e-05 2024-08-17T17:52:32.433 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.433 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.433 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.433 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.969708+0000", 2024-08-17T17:52:32.433 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5840000000000002e-06 2024-08-17T17:52:32.433 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.433 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.433 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.434 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.969753+0000", 2024-08-17T17:52:32.434 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5194000000000001e-05 2024-08-17T17:52:32.434 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.434 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.434 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.434 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.972199+0000", 2024-08-17T17:52:32.434 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024463330000000002 2024-08-17T17:52:32.434 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.434 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.434 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.434 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.434 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.435 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1213/1189 cost 1049576)", 2024-08-17T17:52:32.435 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:52:10.954848+0000", 2024-08-17T17:52:32.435 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 21.400006647000001, 2024-08-17T17:52:32.435 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00067296700000000001, 2024-08-17T17:52:32.435 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.435 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.435 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.435 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.435 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.435 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:10.954848+0000", 2024-08-17T17:52:32.435 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.435 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.435 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.436 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.436 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:10.954848+0000", 2024-08-17T17:52:32.436 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.436 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.436 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.436 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.436 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:10.954846+0000", 2024-08-17T17:52:32.436 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:32.436 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.436 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.436 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.436 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:10.954864+0000", 2024-08-17T17:52:32.436 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.7924e-05 2024-08-17T17:52:32.437 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.437 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.437 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.437 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:10.954884+0000", 2024-08-17T17:52:32.437 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.9799000000000001e-05 2024-08-17T17:52:32.437 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.437 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.437 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.437 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:10.954893+0000", 2024-08-17T17:52:32.437 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.3009999999999995e-06 2024-08-17T17:52:32.437 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.437 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.437 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.437 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:10.954959+0000", 2024-08-17T17:52:32.438 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5937000000000002e-05 2024-08-17T17:52:32.438 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.438 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.438 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.438 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:10.955521+0000", 2024-08-17T17:52:32.438 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000561704 2024-08-17T17:52:32.438 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.438 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.438 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.438 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.438 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.438 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1213/1189 cost 1049576)", 2024-08-17T17:52:32.438 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:52:11.132452+0000", 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 21.222401994999998, 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00077957899999999997, 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:11.132452+0000", 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:11.132452+0000", 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.439 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.440 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.440 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.440 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:11.132450+0000", 2024-08-17T17:52:32.440 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999976 2024-08-17T17:52:32.440 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.440 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.440 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.440 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:11.132476+0000", 2024-08-17T17:52:32.440 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5772e-05 2024-08-17T17:52:32.440 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.440 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.440 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.440 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:11.132507+0000", 2024-08-17T17:52:32.440 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1412999999999999e-05 2024-08-17T17:52:32.441 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.441 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.441 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.441 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:11.132521+0000", 2024-08-17T17:52:32.441 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3436e-05 2024-08-17T17:52:32.441 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.441 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.441 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.441 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:11.132633+0000", 2024-08-17T17:52:32.441 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000112252 2024-08-17T17:52:32.441 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.441 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.441 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.442 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:11.133232+0000", 2024-08-17T17:52:32.442 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00059898199999999996 2024-08-17T17:52:32.442 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.442 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.442 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.442 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.442 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.442 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1213/1189 cost 1049576)", 2024-08-17T17:52:32.442 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:52:12.489193+0000", 2024-08-17T17:52:32.442 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 19.865661592999999, 2024-08-17T17:52:32.442 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00071147799999999998, 2024-08-17T17:52:32.442 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.442 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:12.489193+0000", 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:12.489193+0000", 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:12.489191+0000", 2024-08-17T17:52:32.443 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:12.489217+0000", 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.6506999999999999e-05 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:12.489248+0000", 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0559000000000003e-05 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:12.489261+0000", 2024-08-17T17:52:32.444 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.327e-05 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:12.489371+0000", 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000109671 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:12.489904+0000", 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00053351799999999997 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.445 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1213/1189 cost 1049576)", 2024-08-17T17:52:32.446 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:52:14.841426+0000", 2024-08-17T17:52:32.446 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 17.513427871000001, 2024-08-17T17:52:32.446 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002875641, 2024-08-17T17:52:32.446 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.446 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.446 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.446 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.446 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.446 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.841426+0000", 2024-08-17T17:52:32.446 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.446 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.446 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.446 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.446 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.841426+0000", 2024-08-17T17:52:32.447 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.447 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.447 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.447 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.447 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.841425+0000", 2024-08-17T17:52:32.447 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:52:32.447 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.447 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.447 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.447 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.841437+0000", 2024-08-17T17:52:32.447 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1294999999999999e-05 2024-08-17T17:52:32.447 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.447 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.448 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.448 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.841450+0000", 2024-08-17T17:52:32.448 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3532e-05 2024-08-17T17:52:32.448 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.448 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.448 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.448 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.841456+0000", 2024-08-17T17:52:32.448 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8510000000000004e-06 2024-08-17T17:52:32.448 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.448 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.448 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.448 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.841498+0000", 2024-08-17T17:52:32.448 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1644999999999999e-05 2024-08-17T17:52:32.449 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.449 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.449 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.449 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.844302+0000", 2024-08-17T17:52:32.449 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028043579999999999 2024-08-17T17:52:32.449 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.449 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.449 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.449 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.449 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.449 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1213/1189 cost 1049576)", 2024-08-17T17:52:32.449 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:52:15.277396+0000", 2024-08-17T17:52:32.449 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 17.077458821, 2024-08-17T17:52:32.449 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00074163899999999999, 2024-08-17T17:52:32.450 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.450 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.450 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.450 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.450 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.450 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:15.277396+0000", 2024-08-17T17:52:32.450 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.450 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.450 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.450 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.450 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:15.277396+0000", 2024-08-17T17:52:32.450 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.450 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.450 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:15.277394+0000", 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:15.277418+0000", 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4623000000000001e-05 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:15.277447+0000", 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8819e-05 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.451 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:15.277459+0000", 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.207e-05 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:15.277595+0000", 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013574999999999999 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:15.278137+0000", 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00054224999999999998 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.452 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.453 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.453 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.453 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1214/1189 cost 1049576)", 2024-08-17T17:52:32.453 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:52:29.293660+0000", 2024-08-17T17:52:32.453 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 3.0611943789999998, 2024-08-17T17:52:32.453 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00066698099999999998, 2024-08-17T17:52:32.453 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.453 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.453 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.453 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.453 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.453 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.293660+0000", 2024-08-17T17:52:32.453 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.453 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.454 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.454 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.454 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.293660+0000", 2024-08-17T17:52:32.454 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.454 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.454 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.454 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.454 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.293658+0000", 2024-08-17T17:52:32.454 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:32.454 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.454 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.454 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.293683+0000", 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4604999999999999e-05 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.293712+0000", 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8949999999999999e-05 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.293724+0000", 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.219e-05 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.455 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.293866+0000", 2024-08-17T17:52:32.456 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014202600000000001 2024-08-17T17:52:32.456 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.456 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.456 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.456 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.294327+0000", 2024-08-17T17:52:32.456 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00046097999999999999 2024-08-17T17:52:32.456 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.456 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.456 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.456 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.456 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.456 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1214/1189 cost 1049576)", 2024-08-17T17:52:32.456 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:52:29.506049+0000", 2024-08-17T17:52:32.456 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 2.8488052609999999, 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00064905300000000002, 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.506049+0000", 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.506049+0000", 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.457 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.458 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.458 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.506047+0000", 2024-08-17T17:52:32.458 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:32.458 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.458 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.458 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.458 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.506072+0000", 2024-08-17T17:52:32.458 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4936e-05 2024-08-17T17:52:32.458 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.458 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.458 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.458 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.506098+0000", 2024-08-17T17:52:32.458 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5843e-05 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.506113+0000", 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4577e-05 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.506207+0000", 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4419000000000005e-05 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:29.506698+0000", 2024-08-17T17:52:32.459 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00049104499999999996 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1214/1189 cost 1049576)", 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:52:32.152365+0000", 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.20248893900000001, 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000677112, 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:32.460 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:32.152365+0000", 2024-08-17T17:52:32.461 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.461 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.461 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.461 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:32.461 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:32.152365+0000", 2024-08-17T17:52:32.461 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:32.461 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.461 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.461 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:32.461 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:32.152364+0000", 2024-08-17T17:52:32.461 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:32.461 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.461 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.461 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:32.462 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:32.152388+0000", 2024-08-17T17:52:32.462 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4298999999999999e-05 2024-08-17T17:52:32.462 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.462 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.462 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:32.462 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:32.152416+0000", 2024-08-17T17:52:32.462 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8479e-05 2024-08-17T17:52:32.462 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.462 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.462 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:32.462 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:32.152428+0000", 2024-08-17T17:52:32.462 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1749999999999999e-05 2024-08-17T17:52:32.462 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.462 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.463 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:32.463 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:32.152582+0000", 2024-08-17T17:52:32.463 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000154244 2024-08-17T17:52:32.463 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:32.463 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:32.463 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:32.463 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:32.153043+0000", 2024-08-17T17:52:32.463 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00046019000000000002 2024-08-17T17:52:32.463 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.463 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.463 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.463 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:32.463 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:32.463 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:52:32.464 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:32.485 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:32.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:32.551 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:52:32.587 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:32.608 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:32.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:32.710 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:32.731 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:32.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:32.750 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:52:32.800 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:52:32.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:52:32.828 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/931731863' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:52:32.828 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/731031744' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:52:32.829 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:32 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:52:32.829 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/931731863' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:52:32.829 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/731031744' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:52:32.829 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:32 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:52:32.832 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:32.853 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:32.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:32.955 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:32.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:32.976 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:32.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:33.077 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:33.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/931731863' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:52:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/731031744' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:52:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:32 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:52:33.101 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:33.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:33.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:33.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:33.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:33.326 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:33.348 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:52:33.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.349 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:33.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:33.450 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:33.472 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:33.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:33.574 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:33.601 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:33.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:33.607 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:33.639 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:33.652 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:33.702 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:33.724 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:33.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:33.763 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:33 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:33.763 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:33 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:52:33.763 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:33 smithi130 ceph-mon[105521]: osdmap e1215: 8 total, 8 up, 6 in 2024-08-17T17:52:33.763 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:33 smithi130 ceph-mon[105521]: pgmap v1754: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 19076/238836 objects degraded (7.987%); 13736/238836 objects misplaced (5.751%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:52:33.764 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:33 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:33.764 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:33 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:52:33.764 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:33 smithi130 ceph-mon[110068]: osdmap e1215: 8 total, 8 up, 6 in 2024-08-17T17:52:33.764 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:33 smithi130 ceph-mon[110068]: pgmap v1754: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 19076/238836 objects degraded (7.987%); 13736/238836 objects misplaced (5.751%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:52:33.825 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:33.848 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:33.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:33.950 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:33.974 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:33.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:33.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:34.050 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:33 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:34.051 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:33 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:52:34.051 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:33 smithi179 ceph-mon[93519]: osdmap e1215: 8 total, 8 up, 6 in 2024-08-17T17:52:34.051 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:33 smithi179 ceph-mon[93519]: pgmap v1754: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 19076/238836 objects degraded (7.987%); 13736/238836 objects misplaced (5.751%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:52:34.076 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:34.097 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:34.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:34.199 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:34.220 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:34.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:34.322 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:34.343 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:34.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:34.444 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:34.465 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:34.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:34.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:34.587 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:34.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:34.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:34.710 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:34.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:34.812 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:34.833 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:34.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:34.934 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:34.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:34.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:34.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:35.058 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:35.080 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:35.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:35.181 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:35.208 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:35.303 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:35 smithi179 ceph-mon[93519]: pgmap v1755: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 19076/237633 objects degraded (8.028%); 13736/237633 objects misplaced (5.780%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:52:35.310 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:35.331 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:35.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:35.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:35 smithi130 ceph-mon[105521]: pgmap v1755: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 19076/237633 objects degraded (8.028%); 13736/237633 objects misplaced (5.780%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:52:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:35 smithi130 ceph-mon[110068]: pgmap v1755: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 19076/237633 objects degraded (8.028%); 13736/237633 objects misplaced (5.780%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:52:35.432 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:35.456 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:35.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:35.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:52:35.558 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:35.560 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:52:35.560 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:52:35.560 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T17:52:35.560 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:52:35.596 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:35.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:35.697 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:35.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:35.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:35.822 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:35.848 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:35.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:35.950 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:35.971 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:35.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:35.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:36.073 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:36.098 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:36.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:36.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 19076/237633 objects degraded (8.028%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:36.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 19076/237633 objects degraded (8.028%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:36.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:36.224 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:36.224 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 19076/237633 objects degraded (8.028%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:36.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:36.250 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:52:36.326 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:36.348 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:36.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:36.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:36.473 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:36.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:36.517 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:36.575 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:36.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.599 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:36.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:36.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:36.722 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:36.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:36.823 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:36.846 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:36.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:36.904 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:52:36.904 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:52:36.919 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:52:36.920 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:52:36.936 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:52:36.937 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:52:36.939 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:36.948 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:36.954 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:52:36.954 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:52:36.971 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:36.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:36.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:36.972 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:52:36.972 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:52:36.988 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:52:36.989 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:52:37.008 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:52:37.008 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:52:37.030 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:52:37.030 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:52:37.073 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:37.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:37.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:37.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:37 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3311444259' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:37.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:37 smithi130 ceph-mon[105521]: pgmap v1756: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 19002/237432 objects degraded (8.003%); 13706/237432 objects misplaced (5.773%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:52:37.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:37 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3311444259' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:37.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:37 smithi130 ceph-mon[110068]: pgmap v1756: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 19002/237432 objects degraded (8.003%); 13706/237432 objects misplaced (5.773%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:52:37.197 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:37.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:37.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:37.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:37.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:37 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3311444259' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:37.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:37 smithi179 ceph-mon[93519]: pgmap v1756: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 19002/237432 objects degraded (8.003%); 13706/237432 objects misplaced (5.773%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:52:37.349 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:37.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:37.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:37.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:37.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:37.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:37.595 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:37.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:37.697 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:37.703 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:52:37.704 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:52:37.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:37.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:37.823 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:37.844 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:37.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:37.946 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:37.967 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:37.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:37.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:38.069 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:38.090 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:38.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:38.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:38 smithi130 ceph-mon[105521]: from='client.35577 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:52:38.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:38 smithi130 ceph-mon[110068]: from='client.35577 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:52:38.192 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:38.213 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:38.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:38.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:38.341 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:38.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:38.407 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:38 smithi179 ceph-mon[93519]: from='client.35577 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:52:38.413 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:52:38.413 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:52:38.443 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:38.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.466 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:38.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:38.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:38.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:38.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:38.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:38.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:38.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:38.812 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:38.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.834 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:38.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:38.935 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:38.959 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:38.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:38.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:39.061 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:39.082 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:39.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:39.184 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:39.205 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:39.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:39.293 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:52:39.308 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:39.357 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:39.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:39.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:39 smithi130 ceph-mon[105521]: pgmap v1757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 109 op/s; 19002/236229 objects degraded (8.044%); 13632/236229 objects misplaced (5.771%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:52:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:39 smithi130 ceph-mon[110068]: pgmap v1757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 109 op/s; 19002/236229 objects degraded (8.044%); 13632/236229 objects misplaced (5.771%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:52:39.458 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:39.479 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:39.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:39.553 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:39.581 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:39.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:39 smithi179 ceph-mon[93519]: pgmap v1757: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 109 op/s; 19002/236229 objects degraded (8.044%); 13632/236229 objects misplaced (5.771%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:52:39.605 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:39.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:39.707 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:39.729 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:39.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:39.738 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1756,"stamp":"2024-08-17T17:52:37.025105+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5185607653,"num_objects":79144,"num_object_clones":0,"num_object_copies":237432,"num_objects_missing_on_primary":4450,"num_objects_missing":4450,"num_objects_degraded":19002,"num_objects_misplaced":13706,"num_objects_unfound":0,"num_objects_dirty":79144,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":130770,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33261,"num_bytes_recovered":2001214477,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106773,"ondisk_log_size":106773,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14689196,"kb_used_data":13384692,"kb_used_omap":295,"kb_used_meta":1304088,"kb_avail":547781716,"statfs":{"total":575970213888,"available":560928477184,"internally_reserved":0,"allocated":13705924608,"data_stored":13676976497,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1335386575},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,0,0,0,3,0,5,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-76349440,"num_objects":-1165,"num_object_clones":0,"num_object_copies":-3495,"num_objects_missing_on_primary":-72,"num_objects_missing":-72,"num_objects_degraded":-146,"num_objects_misplaced":-208,"num_objects_unfound":0,"num_objects_dirty":-1165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1165,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":146,"num_bytes_recovered":9568256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002796"},"pg_stats":[{"pgid":"45.f","version":"1214'8154","reported_seq":18660,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.615675+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:52:32.615675+0000","last_peered":"2024-08-17T17:52:32.615675+0000","last_clean":"2024-08-17T17:52:32.615675+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:52:32.615675+0000","last_undegraded":"2024-08-17T17:52:32.615675+0000","last_fullsized":"2024-08-17T17:52:32.615675+0000","mapping_epoch":1194,"log_start":"1170'1331","ondisk_log_start":"1170'1331","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6823,"log_dups_size":0,"ondisk_log_size":6823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328335360,"num_objects":5010,"num_object_clones":0,"num_object_copies":15030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8154,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1215'8094","reported_seq":18549,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.770235+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:52:32.770235+0000","last_peered":"2024-08-17T17:52:32.770235+0000","last_clean":"2024-08-17T17:52:32.770235+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:52:32.770235+0000","last_undegraded":"2024-08-17T17:52:32.770235+0000","last_fullsized":"2024-08-17T17:52:32.770235+0000","mapping_epoch":1211,"log_start":"1170'1032","ondisk_log_start":"1170'1032","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7062,"log_dups_size":0,"ondisk_log_size":7062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321126400,"num_objects":4900,"num_object_clones":0,"num_object_copies":14700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8094,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1215'8364","reported_seq":19180,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:33.546091+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:52:33.546091+0000","last_peered":"2024-08-17T17:52:33.546091+0000","last_clean":"2024-08-17T17:52:33.546091+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:52:33.546091+0000","last_undegraded":"2024-08-17T17:52:33.546091+0000","last_fullsized":"2024-08-17T17:52:33.546091+0000","mapping_epoch":1205,"log_start":"1170'2468","ondisk_log_start":"1170'2468","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5896,"log_dups_size":0,"ondisk_log_size":5896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328531992,"num_objects":5014,"num_object_clones":0,"num_object_copies":15042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5014,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8364,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1215'8118","reported_seq":8174,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.759911+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:52:32.759911+0000","last_peered":"2024-08-17T17:52:32.759911+0000","last_clean":"2024-08-17T17:52:32.759911+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:52:32.759911+0000","last_undegraded":"2024-08-17T17:52:32.759911+0000","last_fullsized":"2024-08-17T17:52:32.759911+0000","mapping_epoch":1167,"log_start":"1170'1389","ondisk_log_start":"1170'1389","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6729,"log_dups_size":0,"ondisk_log_size":6729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319946752,"num_objects":4882,"num_object_clones":0,"num_object_copies":14646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8118,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1215'8223","reported_seq":13445,"reported_epoch":1215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:36.556588+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:52:36.556588+0000","last_peered":"2024-08-17T17:52:36.556588+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:52:36.556588+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1170'2326","ondisk_log_start":"1170'2326","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5897,"log_dups_size":0,"ondisk_log_size":5897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320536576,"num_objects":4891,"num_object_clones":0,"num_object_copies":14673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":4891,"num_objects_unfound":0,"num_objects_dirty":4891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8223,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":4373},{"shards":"1,4,5","objects":518}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1215'8207","reported_seq":8263,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.827296+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:52:32.827296+0000","last_peered":"2024-08-17T17:52:32.827296+0000","last_clean":"2024-08-17T17:52:32.827296+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:52:32.827296+0000","last_undegraded":"2024-08-17T17:52:32.827296+0000","last_fullsized":"2024-08-17T17:52:32.827296+0000","mapping_epoch":1167,"log_start":"1170'1477","ondisk_log_start":"1170'1477","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6730,"log_dups_size":0,"ondisk_log_size":6730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320274432,"num_objects":4887,"num_object_clones":0,"num_object_copies":14661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8207,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1215'8221","reported_seq":8277,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.823261+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:52:32.823261+0000","last_peered":"2024-08-17T17:52:32.823261+0000","last_clean":"2024-08-17T17:52:32.823261+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:52:32.823261+0000","last_undegraded":"2024-08-17T17:52:32.823261+0000","last_fullsized":"2024-08-17T17:52:32.823261+0000","mapping_epoch":1167,"log_start":"1170'1469","ondisk_log_start":"1170'1469","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6752,"log_dups_size":0,"ondisk_log_size":6752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328007680,"num_objects":5005,"num_object_clones":0,"num_object_copies":15015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8221,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1215'8153","reported_seq":13242,"reported_epoch":1215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:36.687026+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:52:36.687026+0000","last_peered":"2024-08-17T17:52:36.687026+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:52:36.687026+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1170'2175","ondisk_log_start":"1170'2175","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5978,"log_dups_size":0,"ondisk_log_size":5978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316997632,"num_objects":4837,"num_object_clones":0,"num_object_copies":14511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4967,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8153,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1658,"num_bytes_recovered":108658688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1450},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1215'8084","reported_seq":8150,"reported_epoch":1215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:32.751835+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:52:32.751835+0000","last_peered":"2024-08-17T17:52:32.751835+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:52:32.751835+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'1377","ondisk_log_start":"1170'1377","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6707,"log_dups_size":0,"ondisk_log_size":6707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327155712,"num_objects":4992,"num_object_clones":0,"num_object_copies":14976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8084,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":4992},{"shards":"1,6","objects":-841}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1215'8124","reported_seq":11179,"reported_epoch":1215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:33.591036+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:52:33.591036+0000","last_peered":"2024-08-17T17:52:33.591036+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:52:33.591036+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1170'2206","ondisk_log_start":"1170'2206","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330956800,"num_objects":5050,"num_object_clones":0,"num_object_copies":15150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8124,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":3489},{"shards":"1,3,5","objects":1491}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1215'8149","reported_seq":8259,"reported_epoch":1215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:33.708274+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:52:33.708274+0000","last_peered":"2024-08-17T17:52:33.708274+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:52:33.708274+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8149,"log_dups_size":0,"ondisk_log_size":8149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322764800,"num_objects":4925,"num_object_clones":0,"num_object_copies":14775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8149,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-228}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1215'8120","reported_seq":14291,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.852588+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:52:32.852588+0000","last_peered":"2024-08-17T17:52:32.852588+0000","last_clean":"2024-08-17T17:52:32.852588+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:52:32.852588+0000","last_undegraded":"2024-08-17T17:52:32.852588+0000","last_fullsized":"2024-08-17T17:52:32.852588+0000","mapping_epoch":1183,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8120,"log_dups_size":0,"ondisk_log_size":8120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322830336,"num_objects":4926,"num_object_clones":0,"num_object_copies":14778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8120,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1215'8165","reported_seq":8274,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:33.414664+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:52:33.414664+0000","last_peered":"2024-08-17T17:52:33.414664+0000","last_clean":"2024-08-17T17:52:33.414664+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:52:33.414664+0000","last_undegraded":"2024-08-17T17:52:33.414664+0000","last_fullsized":"2024-08-17T17:52:33.414664+0000","mapping_epoch":1167,"log_start":"1170'2191","ondisk_log_start":"1170'2191","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5974,"log_dups_size":0,"ondisk_log_size":5974,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327352320,"num_objects":4995,"num_object_clones":0,"num_object_copies":14985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8165,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1215'8050","reported_seq":8106,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.780970+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:52:32.780970+0000","last_peered":"2024-08-17T17:52:32.780970+0000","last_clean":"2024-08-17T17:52:32.780970+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:52:32.780970+0000","last_undegraded":"2024-08-17T17:52:32.780970+0000","last_fullsized":"2024-08-17T17:52:32.780970+0000","mapping_epoch":1167,"log_start":"1170'1025","ondisk_log_start":"1170'1025","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7025,"log_dups_size":0,"ondisk_log_size":7025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320077824,"num_objects":4884,"num_object_clones":0,"num_object_copies":14652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8050,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1214'8272","reported_seq":16649,"reported_epoch":1214,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:32.605023+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:52:32.605023+0000","last_peered":"2024-08-17T17:52:32.605023+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:52:32.605023+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'1391","ondisk_log_start":"1170'1391","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6881,"log_dups_size":0,"ondisk_log_size":6881,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333709312,"num_objects":5092,"num_object_clones":0,"num_object_copies":15276,"num_objects_missing_on_primary":4450,"num_objects_missing":4450,"num_objects_degraded":265,"num_objects_misplaced":8815,"num_objects_unfound":0,"num_objects_dirty":5092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8272,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3696,"num_bytes_recovered":242221056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":13},{"shards":"0,4,6","objects":1356},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":3458}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1215'7994","reported_seq":21289,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:33.444680+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:52:33.444680+0000","last_peered":"2024-08-17T17:52:33.444680+0000","last_clean":"2024-08-17T17:52:33.444680+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:52:33.444680+0000","last_undegraded":"2024-08-17T17:52:33.444680+0000","last_fullsized":"2024-08-17T17:52:33.444680+0000","mapping_epoch":1196,"log_start":"1170'2069","ondisk_log_start":"1170'2069","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5925,"log_dups_size":0,"ondisk_log_size":5925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316407808,"num_objects":4828,"num_object_clones":0,"num_object_copies":14484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7994,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1883,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.275053+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:52:21.275053+0000","last_peered":"2024-08-17T17:52:21.275053+0000","last_clean":"2024-08-17T17:52:21.275053+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:52:21.275053+0000","last_undegraded":"2024-08-17T17:52:21.275053+0000","last_fullsized":"2024-08-17T17:52:21.275053+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":492,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.266677+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:52:21.266677+0000","last_peered":"2024-08-17T17:52:21.266677+0000","last_clean":"2024-08-17T17:52:21.266677+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:52:21.266677+0000","last_undegraded":"2024-08-17T17:52:21.266677+0000","last_fullsized":"2024-08-17T17:52:21.266677+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1805,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.769269+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:52:32.769269+0000","last_peered":"2024-08-17T17:52:32.769269+0000","last_clean":"2024-08-17T17:52:32.769269+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:52:32.769269+0000","last_undegraded":"2024-08-17T17:52:32.769269+0000","last_fullsized":"2024-08-17T17:52:32.769269+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1829,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.764441+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:52:32.764441+0000","last_peered":"2024-08-17T17:52:32.764441+0000","last_clean":"2024-08-17T17:52:32.764441+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:52:32.764441+0000","last_undegraded":"2024-08-17T17:52:32.764441+0000","last_fullsized":"2024-08-17T17:52:32.764441+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":54,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.267333+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:52:21.267333+0000","last_peered":"2024-08-17T17:52:21.267333+0000","last_clean":"2024-08-17T17:52:21.267333+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:52:21.267333+0000","last_undegraded":"2024-08-17T17:52:21.267333+0000","last_fullsized":"2024-08-17T17:52:21.267333+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1223,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.765922+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:52:32.765922+0000","last_peered":"2024-08-17T17:52:32.765922+0000","last_clean":"2024-08-17T17:52:32.765922+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:52:32.765922+0000","last_undegraded":"2024-08-17T17:52:32.765922+0000","last_fullsized":"2024-08-17T17:52:32.765922+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":81,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.771634+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:52:32.771634+0000","last_peered":"2024-08-17T17:52:32.771634+0000","last_clean":"2024-08-17T17:52:32.771634+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:52:32.771634+0000","last_undegraded":"2024-08-17T17:52:32.771634+0000","last_fullsized":"2024-08-17T17:52:32.771634+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":601,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.768010+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:52:32.768010+0000","last_peered":"2024-08-17T17:52:32.768010+0000","last_clean":"2024-08-17T17:52:32.768010+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:52:32.768010+0000","last_undegraded":"2024-08-17T17:52:32.768010+0000","last_fullsized":"2024-08-17T17:52:32.768010+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1842,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.274627+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:52:21.274627+0000","last_peered":"2024-08-17T17:52:21.274627+0000","last_clean":"2024-08-17T17:52:21.274627+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:52:21.274627+0000","last_undegraded":"2024-08-17T17:52:21.274627+0000","last_fullsized":"2024-08-17T17:52:21.274627+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1846,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.759585+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:52:32.759585+0000","last_peered":"2024-08-17T17:52:32.759585+0000","last_clean":"2024-08-17T17:52:32.759585+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:52:32.759585+0000","last_undegraded":"2024-08-17T17:52:32.759585+0000","last_fullsized":"2024-08-17T17:52:32.759585+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":66,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.772338+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:52:32.772338+0000","last_peered":"2024-08-17T17:52:32.772338+0000","last_clean":"2024-08-17T17:52:32.772338+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:52:32.772338+0000","last_undegraded":"2024-08-17T17:52:32.772338+0000","last_fullsized":"2024-08-17T17:52:32.772338+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1818,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.769071+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:52:32.769071+0000","last_peered":"2024-08-17T17:52:32.769071+0000","last_clean":"2024-08-17T17:52:32.769071+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:52:32.769071+0000","last_undegraded":"2024-08-17T17:52:32.769071+0000","last_fullsized":"2024-08-17T17:52:32.769071+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1873,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.770163+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:52:32.770163+0000","last_peered":"2024-08-17T17:52:32.770163+0000","last_clean":"2024-08-17T17:52:32.770163+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:52:32.770163+0000","last_undegraded":"2024-08-17T17:52:32.770163+0000","last_fullsized":"2024-08-17T17:52:32.770163+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1864,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.275144+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:52:21.275144+0000","last_peered":"2024-08-17T17:52:21.275144+0000","last_clean":"2024-08-17T17:52:21.275144+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:52:21.275144+0000","last_undegraded":"2024-08-17T17:52:21.275144+0000","last_fullsized":"2024-08-17T17:52:21.275144+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1840,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.764378+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:52:32.764378+0000","last_peered":"2024-08-17T17:52:32.764378+0000","last_clean":"2024-08-17T17:52:32.764378+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:52:32.764378+0000","last_undegraded":"2024-08-17T17:52:32.764378+0000","last_fullsized":"2024-08-17T17:52:32.764378+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1189,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.267285+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:52:21.267285+0000","last_peered":"2024-08-17T17:52:21.267285+0000","last_clean":"2024-08-17T17:52:21.267285+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:52:21.267285+0000","last_undegraded":"2024-08-17T17:52:21.267285+0000","last_fullsized":"2024-08-17T17:52:21.267285+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1811,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.763986+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:52:32.763986+0000","last_peered":"2024-08-17T17:52:32.763986+0000","last_clean":"2024-08-17T17:52:32.763986+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:52:32.763986+0000","last_undegraded":"2024-08-17T17:52:32.763986+0000","last_fullsized":"2024-08-17T17:52:32.763986+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1859,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.275114+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:52:21.275114+0000","last_peered":"2024-08-17T17:52:21.275114+0000","last_clean":"2024-08-17T17:52:21.275114+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:52:21.275114+0000","last_undegraded":"2024-08-17T17:52:21.275114+0000","last_fullsized":"2024-08-17T17:52:21.275114+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1767,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.764342+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:52:32.764342+0000","last_peered":"2024-08-17T17:52:32.764342+0000","last_clean":"2024-08-17T17:52:32.764342+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:52:32.764342+0000","last_undegraded":"2024-08-17T17:52:32.764342+0000","last_fullsized":"2024-08-17T17:52:32.764342+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":508,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.372437+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:52:21.372437+0000","last_peered":"2024-08-17T17:52:21.372437+0000","last_clean":"2024-08-17T17:52:21.372437+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:52:21.372437+0000","last_undegraded":"2024-08-17T17:52:21.372437+0000","last_fullsized":"2024-08-17T17:52:21.372437+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1185,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.369139+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:52:21.369139+0000","last_peered":"2024-08-17T17:52:21.369139+0000","last_clean":"2024-08-17T17:52:21.369139+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:52:21.369139+0000","last_undegraded":"2024-08-17T17:52:21.369139+0000","last_fullsized":"2024-08-17T17:52:21.369139+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1866,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.772565+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:52:32.772565+0000","last_peered":"2024-08-17T17:52:32.772565+0000","last_clean":"2024-08-17T17:52:32.772565+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:52:32.772565+0000","last_undegraded":"2024-08-17T17:52:32.772565+0000","last_fullsized":"2024-08-17T17:52:32.772565+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":346,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.369140+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:52:21.369140+0000","last_peered":"2024-08-17T17:52:21.369140+0000","last_clean":"2024-08-17T17:52:21.369140+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:52:21.369140+0000","last_undegraded":"2024-08-17T17:52:21.369140+0000","last_fullsized":"2024-08-17T17:52:21.369140+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":492,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.267045+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:52:21.267045+0000","last_peered":"2024-08-17T17:52:21.267045+0000","last_clean":"2024-08-17T17:52:21.267045+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:52:21.267045+0000","last_undegraded":"2024-08-17T17:52:21.267045+0000","last_fullsized":"2024-08-17T17:52:21.267045+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1170,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.267216+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:52:21.267216+0000","last_peered":"2024-08-17T17:52:21.267216+0000","last_clean":"2024-08-17T17:52:21.267216+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:52:21.267216+0000","last_undegraded":"2024-08-17T17:52:21.267216+0000","last_fullsized":"2024-08-17T17:52:21.267216+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1178,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.267367+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:52:21.267367+0000","last_peered":"2024-08-17T17:52:21.267367+0000","last_clean":"2024-08-17T17:52:21.267367+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:52:21.267367+0000","last_undegraded":"2024-08-17T17:52:21.267367+0000","last_fullsized":"2024-08-17T17:52:21.267367+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1911,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.274907+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:52:21.274907+0000","last_peered":"2024-08-17T17:52:21.274907+0000","last_clean":"2024-08-17T17:52:21.274907+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:52:21.274907+0000","last_undegraded":"2024-08-17T17:52:21.274907+0000","last_fullsized":"2024-08-17T17:52:21.274907+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1797,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.764164+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:52:32.764164+0000","last_peered":"2024-08-17T17:52:32.764164+0000","last_clean":"2024-08-17T17:52:32.764164+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:52:32.764164+0000","last_undegraded":"2024-08-17T17:52:32.764164+0000","last_fullsized":"2024-08-17T17:52:32.764164+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1872,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.770910+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:52:32.770910+0000","last_peered":"2024-08-17T17:52:32.770910+0000","last_clean":"2024-08-17T17:52:32.770910+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:52:32.770910+0000","last_undegraded":"2024-08-17T17:52:32.770910+0000","last_fullsized":"2024-08-17T17:52:32.770910+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1799,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.769924+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:52:32.769924+0000","last_peered":"2024-08-17T17:52:32.769924+0000","last_clean":"2024-08-17T17:52:32.769924+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:52:32.769924+0000","last_undegraded":"2024-08-17T17:52:32.769924+0000","last_fullsized":"2024-08-17T17:52:32.769924+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1829,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.274686+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:52:21.274686+0000","last_peered":"2024-08-17T17:52:21.274686+0000","last_clean":"2024-08-17T17:52:21.274686+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:52:21.274686+0000","last_undegraded":"2024-08-17T17:52:21.274686+0000","last_fullsized":"2024-08-17T17:52:21.274686+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1205,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.369328+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:52:21.369328+0000","last_peered":"2024-08-17T17:52:21.369328+0000","last_clean":"2024-08-17T17:52:21.369328+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:52:21.369328+0000","last_undegraded":"2024-08-17T17:52:21.369328+0000","last_fullsized":"2024-08-17T17:52:21.369328+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1878,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.771821+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:52:32.771821+0000","last_peered":"2024-08-17T17:52:32.771821+0000","last_clean":"2024-08-17T17:52:32.771821+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:52:32.771821+0000","last_undegraded":"2024-08-17T17:52:32.771821+0000","last_fullsized":"2024-08-17T17:52:32.771821+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":65,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.772185+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:52:32.772185+0000","last_peered":"2024-08-17T17:52:32.772185+0000","last_clean":"2024-08-17T17:52:32.772185+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:52:32.772185+0000","last_undegraded":"2024-08-17T17:52:32.772185+0000","last_fullsized":"2024-08-17T17:52:32.772185+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1897,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.771725+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:52:32.771725+0000","last_peered":"2024-08-17T17:52:32.771725+0000","last_clean":"2024-08-17T17:52:32.771725+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:52:32.771725+0000","last_undegraded":"2024-08-17T17:52:32.771725+0000","last_fullsized":"2024-08-17T17:52:32.771725+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":634,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.768888+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:52:32.768888+0000","last_peered":"2024-08-17T17:52:32.768888+0000","last_clean":"2024-08-17T17:52:32.768888+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:52:32.768888+0000","last_undegraded":"2024-08-17T17:52:32.768888+0000","last_fullsized":"2024-08-17T17:52:32.768888+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1878,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.771906+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:52:32.771906+0000","last_peered":"2024-08-17T17:52:32.771906+0000","last_clean":"2024-08-17T17:52:32.771906+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:52:32.771906+0000","last_undegraded":"2024-08-17T17:52:32.771906+0000","last_fullsized":"2024-08-17T17:52:32.771906+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1205,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.274461+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:52:21.274461+0000","last_peered":"2024-08-17T17:52:21.274461+0000","last_clean":"2024-08-17T17:52:21.274461+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:52:21.274461+0000","last_undegraded":"2024-08-17T17:52:21.274461+0000","last_fullsized":"2024-08-17T17:52:21.274461+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":125,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.772480+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:52:32.772480+0000","last_peered":"2024-08-17T17:52:32.772480+0000","last_clean":"2024-08-17T17:52:32.772480+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:52:32.772480+0000","last_undegraded":"2024-08-17T17:52:32.772480+0000","last_fullsized":"2024-08-17T17:52:32.772480+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1889,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.771198+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:52:32.771198+0000","last_peered":"2024-08-17T17:52:32.771198+0000","last_clean":"2024-08-17T17:52:32.771198+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:52:32.771198+0000","last_undegraded":"2024-08-17T17:52:32.771198+0000","last_fullsized":"2024-08-17T17:52:32.771198+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1866,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.773252+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:52:32.773252+0000","last_peered":"2024-08-17T17:52:32.773252+0000","last_clean":"2024-08-17T17:52:32.773252+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:52:32.773252+0000","last_undegraded":"2024-08-17T17:52:32.773252+0000","last_fullsized":"2024-08-17T17:52:32.773252+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1887,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757137+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:52:32.757137+0000","last_peered":"2024-08-17T17:52:32.757137+0000","last_clean":"2024-08-17T17:52:32.757137+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:52:32.757137+0000","last_undegraded":"2024-08-17T17:52:32.757137+0000","last_fullsized":"2024-08-17T17:52:32.757137+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1890,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.275226+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:52:21.275226+0000","last_peered":"2024-08-17T17:52:21.275226+0000","last_clean":"2024-08-17T17:52:21.275226+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:52:21.275226+0000","last_undegraded":"2024-08-17T17:52:21.275226+0000","last_fullsized":"2024-08-17T17:52:21.275226+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1870,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.346097+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:52:21.346097+0000","last_peered":"2024-08-17T17:52:21.346097+0000","last_clean":"2024-08-17T17:52:21.346097+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:52:21.346097+0000","last_undegraded":"2024-08-17T17:52:21.346097+0000","last_fullsized":"2024-08-17T17:52:21.346097+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1212,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.369021+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:52:21.369021+0000","last_peered":"2024-08-17T17:52:21.369021+0000","last_clean":"2024-08-17T17:52:21.369021+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:52:21.369021+0000","last_undegraded":"2024-08-17T17:52:21.369021+0000","last_fullsized":"2024-08-17T17:52:21.369021+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1158,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.267379+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:52:21.267379+0000","last_peered":"2024-08-17T17:52:21.267379+0000","last_clean":"2024-08-17T17:52:21.267379+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:52:21.267379+0000","last_undegraded":"2024-08-17T17:52:21.267379+0000","last_fullsized":"2024-08-17T17:52:21.267379+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1781,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.765841+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:52:32.765841+0000","last_peered":"2024-08-17T17:52:32.765841+0000","last_clean":"2024-08-17T17:52:32.765841+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:52:32.765841+0000","last_undegraded":"2024-08-17T17:52:32.765841+0000","last_fullsized":"2024-08-17T17:52:32.765841+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":113,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757420+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:52:32.757420+0000","last_peered":"2024-08-17T17:52:32.757420+0000","last_clean":"2024-08-17T17:52:32.757420+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:52:32.757420+0000","last_undegraded":"2024-08-17T17:52:32.757420+0000","last_fullsized":"2024-08-17T17:52:32.757420+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1524,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.368851+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:52:21.368851+0000","last_peered":"2024-08-17T17:52:21.368851+0000","last_clean":"2024-08-17T17:52:21.368851+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:52:21.368851+0000","last_undegraded":"2024-08-17T17:52:21.368851+0000","last_fullsized":"2024-08-17T17:52:21.368851+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1873,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.771323+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:52:32.771323+0000","last_peered":"2024-08-17T17:52:32.771323+0000","last_clean":"2024-08-17T17:52:32.771323+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:52:32.771323+0000","last_undegraded":"2024-08-17T17:52:32.771323+0000","last_fullsized":"2024-08-17T17:52:32.771323+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1813,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.764081+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:52:32.764081+0000","last_peered":"2024-08-17T17:52:32.764081+0000","last_clean":"2024-08-17T17:52:32.764081+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:52:32.764081+0000","last_undegraded":"2024-08-17T17:52:32.764081+0000","last_fullsized":"2024-08-17T17:52:32.764081+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":492,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.267127+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:52:21.267127+0000","last_peered":"2024-08-17T17:52:21.267127+0000","last_clean":"2024-08-17T17:52:21.267127+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:52:21.267127+0000","last_undegraded":"2024-08-17T17:52:21.267127+0000","last_fullsized":"2024-08-17T17:52:21.267127+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":552,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.756974+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:52:32.756974+0000","last_peered":"2024-08-17T17:52:32.756974+0000","last_clean":"2024-08-17T17:52:32.756974+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:52:32.756974+0000","last_undegraded":"2024-08-17T17:52:32.756974+0000","last_fullsized":"2024-08-17T17:52:32.756974+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":568,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757442+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:52:32.757442+0000","last_peered":"2024-08-17T17:52:32.757442+0000","last_clean":"2024-08-17T17:52:32.757442+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:52:32.757442+0000","last_undegraded":"2024-08-17T17:52:32.757442+0000","last_fullsized":"2024-08-17T17:52:32.757442+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1176,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.267020+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:52:21.267020+0000","last_peered":"2024-08-17T17:52:21.267020+0000","last_clean":"2024-08-17T17:52:21.267020+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:52:21.267020+0000","last_undegraded":"2024-08-17T17:52:21.267020+0000","last_fullsized":"2024-08-17T17:52:21.267020+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":584,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.765471+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:52:32.765471+0000","last_peered":"2024-08-17T17:52:32.765471+0000","last_clean":"2024-08-17T17:52:32.765471+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:52:32.765471+0000","last_undegraded":"2024-08-17T17:52:32.765471+0000","last_fullsized":"2024-08-17T17:52:32.765471+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1810,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.768084+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:52:32.768084+0000","last_peered":"2024-08-17T17:52:32.768084+0000","last_clean":"2024-08-17T17:52:32.768084+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:52:32.768084+0000","last_undegraded":"2024-08-17T17:52:32.768084+0000","last_fullsized":"2024-08-17T17:52:32.768084+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1903,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.274711+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:52:21.274711+0000","last_peered":"2024-08-17T17:52:21.274711+0000","last_clean":"2024-08-17T17:52:21.274711+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:52:21.274711+0000","last_undegraded":"2024-08-17T17:52:21.274711+0000","last_fullsized":"2024-08-17T17:52:21.274711+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":616,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.764748+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:52:32.764748+0000","last_peered":"2024-08-17T17:52:32.764748+0000","last_clean":"2024-08-17T17:52:32.764748+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:52:32.764748+0000","last_undegraded":"2024-08-17T17:52:32.764748+0000","last_fullsized":"2024-08-17T17:52:32.764748+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1165,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.266888+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:52:21.266888+0000","last_peered":"2024-08-17T17:52:21.266888+0000","last_clean":"2024-08-17T17:52:21.266888+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:52:21.266888+0000","last_undegraded":"2024-08-17T17:52:21.266888+0000","last_fullsized":"2024-08-17T17:52:21.266888+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1164,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.369224+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:52:21.369224+0000","last_peered":"2024-08-17T17:52:21.369224+0000","last_clean":"2024-08-17T17:52:21.369224+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:52:21.369224+0000","last_undegraded":"2024-08-17T17:52:21.369224+0000","last_fullsized":"2024-08-17T17:52:21.369224+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":601,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.770258+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:52:32.770258+0000","last_peered":"2024-08-17T17:52:32.770258+0000","last_clean":"2024-08-17T17:52:32.770258+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:52:32.770258+0000","last_undegraded":"2024-08-17T17:52:32.770258+0000","last_fullsized":"2024-08-17T17:52:32.770258+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1182,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.372520+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:52:21.372520+0000","last_peered":"2024-08-17T17:52:21.372520+0000","last_clean":"2024-08-17T17:52:21.372520+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:52:21.372520+0000","last_undegraded":"2024-08-17T17:52:21.372520+0000","last_fullsized":"2024-08-17T17:52:21.372520+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":66,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.368743+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:52:21.368743+0000","last_peered":"2024-08-17T17:52:21.368743+0000","last_clean":"2024-08-17T17:52:21.368743+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:52:21.368743+0000","last_undegraded":"2024-08-17T17:52:21.368743+0000","last_fullsized":"2024-08-17T17:52:21.368743+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1198,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.369298+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:52:21.369298+0000","last_peered":"2024-08-17T17:52:21.369298+0000","last_clean":"2024-08-17T17:52:21.369298+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:52:21.369298+0000","last_undegraded":"2024-08-17T17:52:21.369298+0000","last_fullsized":"2024-08-17T17:52:21.369298+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1215,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.266845+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:52:21.266845+0000","last_peered":"2024-08-17T17:52:21.266845+0000","last_clean":"2024-08-17T17:52:21.266845+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:52:21.266845+0000","last_undegraded":"2024-08-17T17:52:21.266845+0000","last_fullsized":"2024-08-17T17:52:21.266845+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1811,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.769830+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:52:32.769830+0000","last_peered":"2024-08-17T17:52:32.769830+0000","last_clean":"2024-08-17T17:52:32.769830+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:52:32.769830+0000","last_undegraded":"2024-08-17T17:52:32.769830+0000","last_fullsized":"2024-08-17T17:52:32.769830+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1159,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.368986+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:52:21.368986+0000","last_peered":"2024-08-17T17:52:21.368986+0000","last_clean":"2024-08-17T17:52:21.368986+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:52:21.368986+0000","last_undegraded":"2024-08-17T17:52:21.368986+0000","last_fullsized":"2024-08-17T17:52:21.368986+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1862,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.274997+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:52:21.274997+0000","last_peered":"2024-08-17T17:52:21.274997+0000","last_clean":"2024-08-17T17:52:21.274997+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:52:21.274997+0000","last_undegraded":"2024-08-17T17:52:21.274997+0000","last_fullsized":"2024-08-17T17:52:21.274997+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1863,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.274520+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:52:21.274520+0000","last_peered":"2024-08-17T17:52:21.274520+0000","last_clean":"2024-08-17T17:52:21.274520+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:52:21.274520+0000","last_undegraded":"2024-08-17T17:52:21.274520+0000","last_fullsized":"2024-08-17T17:52:21.274520+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":572,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.369046+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:52:21.369046+0000","last_peered":"2024-08-17T17:52:21.369046+0000","last_clean":"2024-08-17T17:52:21.369046+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:52:21.369046+0000","last_undegraded":"2024-08-17T17:52:21.369046+0000","last_fullsized":"2024-08-17T17:52:21.369046+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1184,"reported_epoch":1214,"state":"active+clean","last_fresh":"2024-08-17T17:52:21.369109+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:52:21.369109+0000","last_peered":"2024-08-17T17:52:21.369109+0000","last_clean":"2024-08-17T17:52:21.369109+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:52:21.369109+0000","last_undegraded":"2024-08-17T17:52:21.369109+0000","last_fullsized":"2024-08-17T17:52:21.369109+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1811,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.765598+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:52:32.765598+0000","last_peered":"2024-08-17T17:52:32.765598+0000","last_clean":"2024-08-17T17:52:32.765598+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:52:32.765598+0000","last_undegraded":"2024-08-17T17:52:32.765598+0000","last_fullsized":"2024-08-17T17:52:32.765598+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":5185011736,"num_objects":79118,"num_object_clones":0,"num_object_copies":237354,"num_objects_missing_on_primary":4450,"num_objects_missing":4450,"num_objects_degraded":19002,"num_objects_misplaced":13706,"num_objects_unfound":0,"num_objects_dirty":79118,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":130492,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33232,"num_bytes_recovered":1999437824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16191401984,"data_stored":16191389768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106566,"ondisk_log_size":106566,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234380,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2429988,"kb_used_data":2124312,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91315164,"statfs":{"total":95995035648,"available":93506727936,"internally_reserved":0,"allocated":2175295488,"data_stored":2170460515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496015,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2751508,"kb_used_data":2449932,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":90993644,"statfs":{"total":95995035648,"available":93177491456,"internally_reserved":0,"allocated":2508730368,"data_stored":2503899236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757653,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326267,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3019024,"kb_used_data":2828932,"kb_used_omap":57,"kb_used_meta":190022,"kb_avail":90726128,"statfs":{"total":95995035648,"available":92903555072,"internally_reserved":0,"allocated":2896826368,"data_stored":2892006575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":194583291},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405506,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2726164,"kb_used_data":2536012,"kb_used_omap":61,"kb_used_meta":190082,"kb_avail":91018988,"statfs":{"total":95995035648,"available":93203443712,"internally_reserved":0,"allocated":2596876288,"data_stored":2592053467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":194644920},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452041,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2061768,"kb_used_data":1903616,"kb_used_omap":54,"kb_used_meta":158089,"kb_avail":91683384,"statfs":{"total":95995035648,"available":93883785216,"internally_reserved":0,"allocated":1949302784,"data_stored":1944486952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":161883534},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474732,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1700744,"kb_used_data":1541888,"kb_used_omap":119,"kb_used_meta":158728,"kb_avail":92044408,"statfs":{"total":95995035648,"available":94253473792,"internally_reserved":0,"allocated":1578893312,"data_stored":1574069752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":162538206},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,0,0,0,3,0,5,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019288,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1562513408,"data_stored":1562509336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1933246464,"data_stored":1933246464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2583756800,"data_stored":2583756800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2882867200,"data_stored":2882863128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1094385664,"data_stored":1094385664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1479081984,"data_stored":1479081984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2493976576,"data_stored":2493972504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2161573888,"data_stored":2161573888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:52:39.831 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:39.855 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:39.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:39.957 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:39.978 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:39.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:39.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:40.080 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:40.102 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:40.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:52:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:52:39] "GET /metrics HTTP/1.1" 200 36809 "" "Prometheus/2.51.0" 2024-08-17T17:52:40.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:40.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:40.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:40.326 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:40.326 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:52:40.342 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:52:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:52:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:52:40.347 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:40.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:40.449 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:40.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.470 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:40.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:40.571 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:40.592 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:40.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:40.694 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:40.716 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:40.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:40.818 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:40.821 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:40 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 19002/236229 objects degraded (8.044%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:40.846 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:40.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:40.912 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:40.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:40 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 19002/236229 objects degraded (8.044%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:40 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 19002/236229 objects degraded (8.044%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:40.948 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:40.969 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:40.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:40.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:41.071 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:41.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.093 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:41.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:41.194 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:41.220 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:41.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:41.321 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:41.351 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:41.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:41.421 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:52:41.421 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:52:41.453 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:41.475 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:41.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:41.577 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:41.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.599 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:41.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:41.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:41.724 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:41.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:41.738 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T17:52:41.825 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:41.847 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:41.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:41.949 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:41.974 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:41.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:41.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:42.075 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:42.084 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:41 smithi179 ceph-mon[93519]: pgmap v1758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 19002/236058 objects degraded (8.050%); 13632/236058 objects misplaced (5.775%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:52:42.084 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:41 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3803350565' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:42.084 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:41 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:42.099 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:42.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:42.110 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:52:42.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:41 smithi130 ceph-mon[105521]: pgmap v1758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 19002/236058 objects degraded (8.050%); 13632/236058 objects misplaced (5.775%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:52:42.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:41 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3803350565' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:42.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:41 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:42.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:41 smithi130 ceph-mon[110068]: pgmap v1758: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 19002/236058 objects degraded (8.050%); 13632/236058 objects misplaced (5.775%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:52:42.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:41 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3803350565' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:42.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:41 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:52:42.201 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:42.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:42.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:42.325 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:42.348 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:42.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:42.382 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:42.449 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:42.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:42.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:42.573 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:42.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.595 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:42.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:42.696 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:42.719 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:42.742 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:52:42.765 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:42 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:42.765 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:42 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:42.765 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:42 smithi130 ceph-mon[105521]: osdmap e1216: 8 total, 8 up, 6 in 2024-08-17T17:52:42.765 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:42.765 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:42.765 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:42 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:42.765 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:42 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:42.765 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:42 smithi130 ceph-mon[110068]: osdmap e1216: 8 total, 8 up, 6 in 2024-08-17T17:52:42.766 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:42.766 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:42.821 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:42.838 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:42 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:42.838 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:42 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:52:42.838 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:42 smithi179 ceph-mon[93519]: osdmap e1216: 8 total, 8 up, 6 in 2024-08-17T17:52:42.839 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:52:42.839 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:42.846 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:42.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:42.948 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:42.969 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:42.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:42.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:43.071 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:43.099 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:43.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:43.201 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:43.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.223 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:43.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:43.324 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:43.345 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:43.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:43.447 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:43.473 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:43.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:43.574 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:43.598 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:43.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:43.630 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:52:43.630 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:52:43.637 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:52:43.637 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:52:43.658 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:52:43.659 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:52:43.673 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:43.676 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:52:43.677 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:52:43.693 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:52:43.693 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:52:43.701 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:43.710 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:52:43.710 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:52:43.724 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:43.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:43.727 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:52:43.728 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:52:43.756 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:52:43.757 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:52:43.826 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:43.848 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:43.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:43 smithi130 ceph-mon[105521]: pgmap v1760: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 18928/235620 objects degraded (8.033%); 13524/235620 objects misplaced (5.740%); 947 KiB/s, 14 objects/s recovering 2024-08-17T17:52:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:43 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2417240937' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:43 smithi130 ceph-mon[110068]: pgmap v1760: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 18928/235620 objects degraded (8.033%); 13524/235620 objects misplaced (5.740%); 947 KiB/s, 14 objects/s recovering 2024-08-17T17:52:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:43 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2417240937' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:43.950 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:43.972 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:43.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:43.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:44.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:43 smithi179 ceph-mon[93519]: pgmap v1760: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 18928/235620 objects degraded (8.033%); 13524/235620 objects misplaced (5.740%); 947 KiB/s, 14 objects/s recovering 2024-08-17T17:52:44.040 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:43 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2417240937' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:44.073 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:44.099 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:44.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:44.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:44.223 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:44.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:44.298 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:52:44.298 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:52:44.298 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T17:52:44.298 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:52:44.325 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:44.350 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:44.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:44.452 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:44.475 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:44.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:44.576 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:44.598 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:44.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:44.700 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:44.722 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:44.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:44.824 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:44.846 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:44.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:44.948 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:44.973 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:44.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:44.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:44.987 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:52:45.075 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:45.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:45.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:45.113 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:52:45.198 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:45.222 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:45.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:45.252 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:45.323 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:45.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:45 smithi179 ceph-mon[93519]: pgmap v1761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 18928/234396 objects degraded (8.075%); 13524/234396 objects misplaced (5.770%); 947 KiB/s, 14 objects/s recovering 2024-08-17T17:52:45.347 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:45.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:45.399 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:45 smithi130 ceph-mon[105521]: pgmap v1761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 18928/234396 objects degraded (8.075%); 13524/234396 objects misplaced (5.770%); 947 KiB/s, 14 objects/s recovering 2024-08-17T17:52:45.399 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:45 smithi130 ceph-mon[110068]: pgmap v1761: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 107 op/s; 18928/234396 objects degraded (8.075%); 13524/234396 objects misplaced (5.770%); 947 KiB/s, 14 objects/s recovering 2024-08-17T17:52:45.449 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:45.470 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:45.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:45.571 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:45.593 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:45.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:45.694 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:45.716 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:45.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:45.817 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:45.838 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:45.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:45.940 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:45.961 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:45.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:45.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:46.063 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:46.085 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:46.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:46.186 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:46.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:46.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:46.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:46.338 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:46.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:46.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18928/234396 objects degraded (8.075%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:46.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18928/234396 objects degraded (8.075%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:46.439 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:46.454 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18928/234396 objects degraded (8.075%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:46.459 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:46.460 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:46.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:46.562 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:46.586 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:46.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:46.642 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:52:46.642 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:52:46.688 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:46.710 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:46.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:46.811 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:46.834 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:46.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:46.935 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:46.959 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:46.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:46.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:47.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:47.088 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:47.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:47.152 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:52:47.152 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:52:47.152 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T17:52:47.152 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T17:52:47.152 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:52:47.190 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:47.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:47.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:47.315 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:47.340 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:47.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:47.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:47 smithi179 ceph-mon[93519]: from='client.45458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:52:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:47 smithi179 ceph-mon[93519]: pgmap v1762: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 18852/234177 objects degraded (8.050%); 13489/234177 objects misplaced (5.760%); 953 KiB/s, 14 objects/s recovering 2024-08-17T17:52:47.406 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:47 smithi130 ceph-mon[105521]: from='client.45458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:52:47.406 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:47 smithi130 ceph-mon[105521]: pgmap v1762: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 18852/234177 objects degraded (8.050%); 13489/234177 objects misplaced (5.760%); 953 KiB/s, 14 objects/s recovering 2024-08-17T17:52:47.407 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:47 smithi130 ceph-mon[110068]: from='client.45458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:52:47.407 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:47 smithi130 ceph-mon[110068]: pgmap v1762: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 108 op/s; 18852/234177 objects degraded (8.050%); 13489/234177 objects misplaced (5.760%); 953 KiB/s, 14 objects/s recovering 2024-08-17T17:52:47.431 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:52:47.442 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:47.496 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:47.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:47.598 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:47.625 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:47.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:47.727 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:47.749 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:47.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:47.850 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:47.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.874 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:47.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:47.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:52:47.975 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:47.996 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:47.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:47.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:48.098 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:48.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.121 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:48.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:48.162 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:52:48.222 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:48.243 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:48.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:48.345 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:48.367 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:48.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:48.468 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:48.491 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:48.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:48.593 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:48.611 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1761,"stamp":"2024-08-17T17:52:45.027315+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5119285221,"num_objects":78132,"num_object_clones":0,"num_object_copies":234396,"num_objects_missing_on_primary":4374,"num_objects_missing":4374,"num_objects_degraded":18928,"num_objects_misplaced":13524,"num_objects_unfound":0,"num_objects_dirty":78132,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":131782,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33372,"num_bytes_recovered":2008488973,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106785,"ondisk_log_size":106785,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14561076,"kb_used_data":13221940,"kb_used_omap":295,"kb_used_meta":1338776,"kb_avail":547909836,"statfs":{"total":575970213888,"available":561059672064,"internally_reserved":0,"allocated":13539266560,"data_stored":13510258961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1370907087},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-70713344,"num_objects":-1079,"num_object_clones":0,"num_object_copies":-3237,"num_objects_missing_on_primary":-76,"num_objects_missing":-76,"num_objects_degraded":-148,"num_objects_misplaced":-212,"num_objects_unfound":0,"num_objects_dirty":-1079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1079,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002733"},"pg_stats":[{"pgid":"45.f","version":"1216'8230","reported_seq":18740,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.711218+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:52:42.711218+0000","last_peered":"2024-08-17T17:52:42.711218+0000","last_clean":"2024-08-17T17:52:42.711218+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:52:42.711218+0000","last_undegraded":"2024-08-17T17:52:42.711218+0000","last_fullsized":"2024-08-17T17:52:42.711218+0000","mapping_epoch":1194,"log_start":"1170'1331","ondisk_log_start":"1170'1331","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6899,"log_dups_size":0,"ondisk_log_size":6899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323354624,"num_objects":4934,"num_object_clones":0,"num_object_copies":14802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8230,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1216'8168","reported_seq":18624,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:43.095591+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:52:43.095591+0000","last_peered":"2024-08-17T17:52:43.095591+0000","last_clean":"2024-08-17T17:52:43.095591+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:52:43.095591+0000","last_undegraded":"2024-08-17T17:52:43.095591+0000","last_fullsized":"2024-08-17T17:52:43.095591+0000","mapping_epoch":1211,"log_start":"1170'1132","ondisk_log_start":"1170'1132","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7036,"log_dups_size":0,"ondisk_log_size":7036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316276736,"num_objects":4826,"num_object_clones":0,"num_object_copies":14478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8168,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1216'8427","reported_seq":19245,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:43.239803+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:52:43.239803+0000","last_peered":"2024-08-17T17:52:43.239803+0000","last_clean":"2024-08-17T17:52:43.239803+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:52:43.239803+0000","last_undegraded":"2024-08-17T17:52:43.239803+0000","last_fullsized":"2024-08-17T17:52:43.239803+0000","mapping_epoch":1205,"log_start":"1170'2468","ondisk_log_start":"1170'2468","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5959,"log_dups_size":0,"ondisk_log_size":5959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324403224,"num_objects":4951,"num_object_clones":0,"num_object_copies":14853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4951,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8427,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1216'8182","reported_seq":8239,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:43.151443+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:52:43.151443+0000","last_peered":"2024-08-17T17:52:43.151443+0000","last_clean":"2024-08-17T17:52:43.151443+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:52:43.151443+0000","last_undegraded":"2024-08-17T17:52:43.151443+0000","last_fullsized":"2024-08-17T17:52:43.151443+0000","mapping_epoch":1167,"log_start":"1170'1489","ondisk_log_start":"1170'1489","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6693,"log_dups_size":0,"ondisk_log_size":6693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315752448,"num_objects":4818,"num_object_clones":0,"num_object_copies":14454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8182,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1215'8255","reported_seq":13479,"reported_epoch":1216,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:41.745293+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:52:41.745293+0000","last_peered":"2024-08-17T17:52:41.745293+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:52:41.745293+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1170'2326","ondisk_log_start":"1170'2326","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5929,"log_dups_size":0,"ondisk_log_size":5929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318439424,"num_objects":4859,"num_object_clones":0,"num_object_copies":14577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":4859,"num_objects_unfound":0,"num_objects_dirty":4859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8255,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":4341},{"shards":"1,4,5","objects":518}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1216'8267","reported_seq":8324,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.703290+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:52:42.703290+0000","last_peered":"2024-08-17T17:52:42.703290+0000","last_clean":"2024-08-17T17:52:42.703290+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:52:42.703290+0000","last_undegraded":"2024-08-17T17:52:42.703290+0000","last_fullsized":"2024-08-17T17:52:42.703290+0000","mapping_epoch":1167,"log_start":"1170'1577","ondisk_log_start":"1170'1577","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6690,"log_dups_size":0,"ondisk_log_size":6690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316342272,"num_objects":4827,"num_object_clones":0,"num_object_copies":14481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8267,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1216'8286","reported_seq":8343,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:43.091688+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:52:43.091688+0000","last_peered":"2024-08-17T17:52:43.091688+0000","last_clean":"2024-08-17T17:52:43.091688+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:52:43.091688+0000","last_undegraded":"2024-08-17T17:52:43.091688+0000","last_fullsized":"2024-08-17T17:52:43.091688+0000","mapping_epoch":1167,"log_start":"1170'1569","ondisk_log_start":"1170'1569","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6717,"log_dups_size":0,"ondisk_log_size":6717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323747840,"num_objects":4940,"num_object_clones":0,"num_object_copies":14820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8286,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1215'8190","reported_seq":13392,"reported_epoch":1216,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:41.740685+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:52:41.740685+0000","last_peered":"2024-08-17T17:52:41.740685+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:52:41.740685+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1170'2275","ondisk_log_start":"1170'2275","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5915,"log_dups_size":0,"ondisk_log_size":5915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314572800,"num_objects":4800,"num_object_clones":0,"num_object_copies":14400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4893,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8190,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1695,"num_bytes_recovered":111083520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1413},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1216'8151","reported_seq":8218,"reported_epoch":1216,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:43.105608+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:52:43.105608+0000","last_peered":"2024-08-17T17:52:43.105608+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:52:43.105608+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'1477","ondisk_log_start":"1170'1477","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6674,"log_dups_size":0,"ondisk_log_size":6674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322764800,"num_objects":4925,"num_object_clones":0,"num_object_copies":14775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8151,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":4925},{"shards":"1,6","objects":-908}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1216'8200","reported_seq":11257,"reported_epoch":1216,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:43.247049+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:52:43.247049+0000","last_peered":"2024-08-17T17:52:43.247049+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:52:43.247049+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1170'2306","ondisk_log_start":"1170'2306","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5894,"log_dups_size":0,"ondisk_log_size":5894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325976064,"num_objects":4974,"num_object_clones":0,"num_object_copies":14922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8200,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":3413},{"shards":"1,3,5","objects":1415}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1216'8209","reported_seq":8321,"reported_epoch":1216,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:43.460397+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:52:43.460397+0000","last_peered":"2024-08-17T17:52:43.460397+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:52:43.460397+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8209,"log_dups_size":0,"ondisk_log_size":8209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318832640,"num_objects":4865,"num_object_clones":0,"num_object_copies":14595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8209,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-288}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1216'8187","reported_seq":14360,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:44.010917+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:52:44.010917+0000","last_peered":"2024-08-17T17:52:44.010917+0000","last_clean":"2024-08-17T17:52:44.010917+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:52:44.010917+0000","last_undegraded":"2024-08-17T17:52:44.010917+0000","last_fullsized":"2024-08-17T17:52:44.010917+0000","mapping_epoch":1183,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8187,"log_dups_size":0,"ondisk_log_size":8187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318439424,"num_objects":4859,"num_object_clones":0,"num_object_copies":14577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8187,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1216'8240","reported_seq":8351,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:43.220529+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:52:43.220529+0000","last_peered":"2024-08-17T17:52:43.220529+0000","last_clean":"2024-08-17T17:52:43.220529+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:52:43.220529+0000","last_undegraded":"2024-08-17T17:52:43.220529+0000","last_fullsized":"2024-08-17T17:52:43.220529+0000","mapping_epoch":1167,"log_start":"1170'2291","ondisk_log_start":"1170'2291","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5949,"log_dups_size":0,"ondisk_log_size":5949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322437120,"num_objects":4920,"num_object_clones":0,"num_object_copies":14760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8240,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1216'8115","reported_seq":8172,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.520215+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:52:42.520215+0000","last_peered":"2024-08-17T17:52:42.520215+0000","last_clean":"2024-08-17T17:52:42.520215+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:52:42.520215+0000","last_undegraded":"2024-08-17T17:52:42.520215+0000","last_fullsized":"2024-08-17T17:52:42.520215+0000","mapping_epoch":1167,"log_start":"1170'1125","ondisk_log_start":"1170'1125","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6990,"log_dups_size":0,"ondisk_log_size":6990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315817984,"num_objects":4819,"num_object_clones":0,"num_object_copies":14457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8115,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1216'8346","reported_seq":16952,"reported_epoch":1216,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:42.811721+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:52:42.811721+0000","last_peered":"2024-08-17T17:52:42.811721+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:52:42.811721+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'1491","ondisk_log_start":"1170'1491","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6855,"log_dups_size":0,"ondisk_log_size":6855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328859648,"num_objects":5018,"num_object_clones":0,"num_object_copies":15054,"num_objects_missing_on_primary":4374,"num_objects_missing":4374,"num_objects_degraded":265,"num_objects_misplaced":8665,"num_objects_unfound":0,"num_objects_dirty":5018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8346,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3770,"num_bytes_recovered":247070720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":15},{"shards":"0,4,6","objects":1356},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":3382}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1216'8051","reported_seq":21348,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:43.205883+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:52:43.205883+0000","last_peered":"2024-08-17T17:52:43.205883+0000","last_clean":"2024-08-17T17:52:43.205883+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:52:43.205883+0000","last_undegraded":"2024-08-17T17:52:43.205883+0000","last_fullsized":"2024-08-17T17:52:43.205883+0000","mapping_epoch":1196,"log_start":"1170'2069","ondisk_log_start":"1170'2069","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5982,"log_dups_size":0,"ondisk_log_size":5982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312672256,"num_objects":4771,"num_object_clones":0,"num_object_copies":14313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8051,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1888,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.001708+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:52:42.001708+0000","last_peered":"2024-08-17T17:52:42.001708+0000","last_clean":"2024-08-17T17:52:42.001708+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:52:42.001708+0000","last_undegraded":"2024-08-17T17:52:42.001708+0000","last_fullsized":"2024-08-17T17:52:42.001708+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":493,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.748579+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:52:32.748579+0000","last_peered":"2024-08-17T17:52:32.748579+0000","last_clean":"2024-08-17T17:52:32.748579+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:52:32.748579+0000","last_undegraded":"2024-08-17T17:52:32.748579+0000","last_fullsized":"2024-08-17T17:52:32.748579+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1807,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.743536+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:52:41.743536+0000","last_peered":"2024-08-17T17:52:41.743536+0000","last_clean":"2024-08-17T17:52:41.743536+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:52:41.743536+0000","last_undegraded":"2024-08-17T17:52:41.743536+0000","last_fullsized":"2024-08-17T17:52:41.743536+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1831,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.740904+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:52:41.740904+0000","last_peered":"2024-08-17T17:52:41.740904+0000","last_clean":"2024-08-17T17:52:41.740904+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:52:41.740904+0000","last_undegraded":"2024-08-17T17:52:41.740904+0000","last_fullsized":"2024-08-17T17:52:41.740904+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":55,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.749166+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:52:32.749166+0000","last_peered":"2024-08-17T17:52:32.749166+0000","last_clean":"2024-08-17T17:52:32.749166+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:52:32.749166+0000","last_undegraded":"2024-08-17T17:52:32.749166+0000","last_fullsized":"2024-08-17T17:52:32.749166+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1225,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.741662+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:52:41.741662+0000","last_peered":"2024-08-17T17:52:41.741662+0000","last_clean":"2024-08-17T17:52:41.741662+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:52:41.741662+0000","last_undegraded":"2024-08-17T17:52:41.741662+0000","last_fullsized":"2024-08-17T17:52:41.741662+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":83,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.746253+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:52:41.746253+0000","last_peered":"2024-08-17T17:52:41.746253+0000","last_clean":"2024-08-17T17:52:41.746253+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:52:41.746253+0000","last_undegraded":"2024-08-17T17:52:41.746253+0000","last_fullsized":"2024-08-17T17:52:41.746253+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":603,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.745195+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:52:41.745195+0000","last_peered":"2024-08-17T17:52:41.745195+0000","last_clean":"2024-08-17T17:52:41.745195+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:52:41.745195+0000","last_undegraded":"2024-08-17T17:52:41.745195+0000","last_fullsized":"2024-08-17T17:52:41.745195+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1846,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.001280+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:52:42.001280+0000","last_peered":"2024-08-17T17:52:42.001280+0000","last_clean":"2024-08-17T17:52:42.001280+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:52:42.001280+0000","last_undegraded":"2024-08-17T17:52:42.001280+0000","last_fullsized":"2024-08-17T17:52:42.001280+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1848,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.965143+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:52:41.965143+0000","last_peered":"2024-08-17T17:52:41.965143+0000","last_clean":"2024-08-17T17:52:41.965143+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:52:41.965143+0000","last_undegraded":"2024-08-17T17:52:41.965143+0000","last_fullsized":"2024-08-17T17:52:41.965143+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":68,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.744696+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:52:41.744696+0000","last_peered":"2024-08-17T17:52:41.744696+0000","last_clean":"2024-08-17T17:52:41.744696+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:52:41.744696+0000","last_undegraded":"2024-08-17T17:52:41.744696+0000","last_fullsized":"2024-08-17T17:52:41.744696+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1820,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.746161+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:52:41.746161+0000","last_peered":"2024-08-17T17:52:41.746161+0000","last_clean":"2024-08-17T17:52:41.746161+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:52:41.746161+0000","last_undegraded":"2024-08-17T17:52:41.746161+0000","last_fullsized":"2024-08-17T17:52:41.746161+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1875,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.740257+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:52:41.740257+0000","last_peered":"2024-08-17T17:52:41.740257+0000","last_clean":"2024-08-17T17:52:41.740257+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:52:41.740257+0000","last_undegraded":"2024-08-17T17:52:41.740257+0000","last_fullsized":"2024-08-17T17:52:41.740257+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1869,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.001611+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:52:42.001611+0000","last_peered":"2024-08-17T17:52:42.001611+0000","last_clean":"2024-08-17T17:52:42.001611+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:52:42.001611+0000","last_undegraded":"2024-08-17T17:52:42.001611+0000","last_fullsized":"2024-08-17T17:52:42.001611+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1842,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.740709+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:52:41.740709+0000","last_peered":"2024-08-17T17:52:41.740709+0000","last_clean":"2024-08-17T17:52:41.740709+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:52:41.740709+0000","last_undegraded":"2024-08-17T17:52:41.740709+0000","last_fullsized":"2024-08-17T17:52:41.740709+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1190,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.749321+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:52:32.749321+0000","last_peered":"2024-08-17T17:52:32.749321+0000","last_clean":"2024-08-17T17:52:32.749321+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:52:32.749321+0000","last_undegraded":"2024-08-17T17:52:32.749321+0000","last_fullsized":"2024-08-17T17:52:32.749321+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1813,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.740478+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:52:41.740478+0000","last_peered":"2024-08-17T17:52:41.740478+0000","last_clean":"2024-08-17T17:52:41.740478+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:52:41.740478+0000","last_undegraded":"2024-08-17T17:52:41.740478+0000","last_fullsized":"2024-08-17T17:52:41.740478+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1863,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.001668+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:52:42.001668+0000","last_peered":"2024-08-17T17:52:42.001668+0000","last_clean":"2024-08-17T17:52:42.001668+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:52:42.001668+0000","last_undegraded":"2024-08-17T17:52:42.001668+0000","last_fullsized":"2024-08-17T17:52:42.001668+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1769,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.740898+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:52:41.740898+0000","last_peered":"2024-08-17T17:52:41.740898+0000","last_clean":"2024-08-17T17:52:41.740898+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:52:41.740898+0000","last_undegraded":"2024-08-17T17:52:41.740898+0000","last_fullsized":"2024-08-17T17:52:41.740898+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":509,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757635+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:52:32.757635+0000","last_peered":"2024-08-17T17:52:32.757635+0000","last_clean":"2024-08-17T17:52:32.757635+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:52:32.757635+0000","last_undegraded":"2024-08-17T17:52:32.757635+0000","last_fullsized":"2024-08-17T17:52:32.757635+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1186,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757529+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:52:32.757529+0000","last_peered":"2024-08-17T17:52:32.757529+0000","last_clean":"2024-08-17T17:52:32.757529+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:52:32.757529+0000","last_undegraded":"2024-08-17T17:52:32.757529+0000","last_fullsized":"2024-08-17T17:52:32.757529+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1868,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.744981+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:52:41.744981+0000","last_peered":"2024-08-17T17:52:41.744981+0000","last_clean":"2024-08-17T17:52:41.744981+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:52:41.744981+0000","last_undegraded":"2024-08-17T17:52:41.744981+0000","last_fullsized":"2024-08-17T17:52:41.744981+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":347,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.759072+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:52:32.759072+0000","last_peered":"2024-08-17T17:52:32.759072+0000","last_clean":"2024-08-17T17:52:32.759072+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:52:32.759072+0000","last_undegraded":"2024-08-17T17:52:32.759072+0000","last_fullsized":"2024-08-17T17:52:32.759072+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":493,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.748900+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:52:32.748900+0000","last_peered":"2024-08-17T17:52:32.748900+0000","last_clean":"2024-08-17T17:52:32.748900+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:52:32.748900+0000","last_undegraded":"2024-08-17T17:52:32.748900+0000","last_fullsized":"2024-08-17T17:52:32.748900+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1171,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.748762+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:52:32.748762+0000","last_peered":"2024-08-17T17:52:32.748762+0000","last_clean":"2024-08-17T17:52:32.748762+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:52:32.748762+0000","last_undegraded":"2024-08-17T17:52:32.748762+0000","last_fullsized":"2024-08-17T17:52:32.748762+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1179,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.749365+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:52:32.749365+0000","last_peered":"2024-08-17T17:52:32.749365+0000","last_clean":"2024-08-17T17:52:32.749365+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:52:32.749365+0000","last_undegraded":"2024-08-17T17:52:32.749365+0000","last_fullsized":"2024-08-17T17:52:32.749365+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1915,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.001447+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:52:42.001447+0000","last_peered":"2024-08-17T17:52:42.001447+0000","last_clean":"2024-08-17T17:52:42.001447+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:52:42.001447+0000","last_undegraded":"2024-08-17T17:52:42.001447+0000","last_fullsized":"2024-08-17T17:52:42.001447+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1799,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.740633+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:52:41.740633+0000","last_peered":"2024-08-17T17:52:41.740633+0000","last_clean":"2024-08-17T17:52:41.740633+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:52:41.740633+0000","last_undegraded":"2024-08-17T17:52:41.740633+0000","last_fullsized":"2024-08-17T17:52:41.740633+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1874,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.745771+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:52:41.745771+0000","last_peered":"2024-08-17T17:52:41.745771+0000","last_clean":"2024-08-17T17:52:41.745771+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:52:41.745771+0000","last_undegraded":"2024-08-17T17:52:41.745771+0000","last_fullsized":"2024-08-17T17:52:41.745771+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1801,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.747002+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:52:41.747002+0000","last_peered":"2024-08-17T17:52:41.747002+0000","last_clean":"2024-08-17T17:52:41.747002+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:52:41.747002+0000","last_undegraded":"2024-08-17T17:52:41.747002+0000","last_fullsized":"2024-08-17T17:52:41.747002+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1833,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.001311+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:52:42.001311+0000","last_peered":"2024-08-17T17:52:42.001311+0000","last_clean":"2024-08-17T17:52:42.001311+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:52:42.001311+0000","last_undegraded":"2024-08-17T17:52:42.001311+0000","last_fullsized":"2024-08-17T17:52:42.001311+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1206,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757747+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:52:32.757747+0000","last_peered":"2024-08-17T17:52:32.757747+0000","last_clean":"2024-08-17T17:52:32.757747+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:52:32.757747+0000","last_undegraded":"2024-08-17T17:52:32.757747+0000","last_fullsized":"2024-08-17T17:52:32.757747+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1880,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.746514+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:52:41.746514+0000","last_peered":"2024-08-17T17:52:41.746514+0000","last_clean":"2024-08-17T17:52:41.746514+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:52:41.746514+0000","last_undegraded":"2024-08-17T17:52:41.746514+0000","last_fullsized":"2024-08-17T17:52:41.746514+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":67,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.740324+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:52:41.740324+0000","last_peered":"2024-08-17T17:52:41.740324+0000","last_clean":"2024-08-17T17:52:41.740324+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:52:41.740324+0000","last_undegraded":"2024-08-17T17:52:41.740324+0000","last_fullsized":"2024-08-17T17:52:41.740324+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1899,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.746451+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:52:41.746451+0000","last_peered":"2024-08-17T17:52:41.746451+0000","last_clean":"2024-08-17T17:52:41.746451+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:52:41.746451+0000","last_undegraded":"2024-08-17T17:52:41.746451+0000","last_fullsized":"2024-08-17T17:52:41.746451+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":636,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.745955+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:52:41.745955+0000","last_peered":"2024-08-17T17:52:41.745955+0000","last_clean":"2024-08-17T17:52:41.745955+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:52:41.745955+0000","last_undegraded":"2024-08-17T17:52:41.745955+0000","last_fullsized":"2024-08-17T17:52:41.745955+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1880,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.746535+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:52:41.746535+0000","last_peered":"2024-08-17T17:52:41.746535+0000","last_clean":"2024-08-17T17:52:41.746535+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:52:41.746535+0000","last_undegraded":"2024-08-17T17:52:41.746535+0000","last_fullsized":"2024-08-17T17:52:41.746535+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1209,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.001207+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:52:42.001207+0000","last_peered":"2024-08-17T17:52:42.001207+0000","last_clean":"2024-08-17T17:52:42.001207+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:52:42.001207+0000","last_undegraded":"2024-08-17T17:52:42.001207+0000","last_fullsized":"2024-08-17T17:52:42.001207+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":127,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.744849+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:52:41.744849+0000","last_peered":"2024-08-17T17:52:41.744849+0000","last_clean":"2024-08-17T17:52:41.744849+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:52:41.744849+0000","last_undegraded":"2024-08-17T17:52:41.744849+0000","last_fullsized":"2024-08-17T17:52:41.744849+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1891,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.745253+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:52:41.745253+0000","last_peered":"2024-08-17T17:52:41.745253+0000","last_clean":"2024-08-17T17:52:41.745253+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:52:41.745253+0000","last_undegraded":"2024-08-17T17:52:41.745253+0000","last_fullsized":"2024-08-17T17:52:41.745253+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1869,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.743094+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:52:41.743094+0000","last_peered":"2024-08-17T17:52:41.743094+0000","last_clean":"2024-08-17T17:52:41.743094+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:52:41.743094+0000","last_undegraded":"2024-08-17T17:52:41.743094+0000","last_fullsized":"2024-08-17T17:52:41.743094+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1889,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.965192+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:52:41.965192+0000","last_peered":"2024-08-17T17:52:41.965192+0000","last_clean":"2024-08-17T17:52:41.965192+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:52:41.965192+0000","last_undegraded":"2024-08-17T17:52:41.965192+0000","last_fullsized":"2024-08-17T17:52:41.965192+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1894,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.001603+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:52:42.001603+0000","last_peered":"2024-08-17T17:52:42.001603+0000","last_clean":"2024-08-17T17:52:42.001603+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:52:42.001603+0000","last_undegraded":"2024-08-17T17:52:42.001603+0000","last_fullsized":"2024-08-17T17:52:42.001603+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1875,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.040200+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:52:42.040200+0000","last_peered":"2024-08-17T17:52:42.040200+0000","last_clean":"2024-08-17T17:52:42.040200+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:52:42.040200+0000","last_undegraded":"2024-08-17T17:52:42.040200+0000","last_fullsized":"2024-08-17T17:52:42.040200+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1213,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757395+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:52:32.757395+0000","last_peered":"2024-08-17T17:52:32.757395+0000","last_clean":"2024-08-17T17:52:32.757395+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:52:32.757395+0000","last_undegraded":"2024-08-17T17:52:32.757395+0000","last_fullsized":"2024-08-17T17:52:32.757395+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1159,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.749180+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:52:32.749180+0000","last_peered":"2024-08-17T17:52:32.749180+0000","last_clean":"2024-08-17T17:52:32.749180+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:52:32.749180+0000","last_undegraded":"2024-08-17T17:52:32.749180+0000","last_fullsized":"2024-08-17T17:52:32.749180+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1784,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.741583+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:52:41.741583+0000","last_peered":"2024-08-17T17:52:41.741583+0000","last_clean":"2024-08-17T17:52:41.741583+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:52:41.741583+0000","last_undegraded":"2024-08-17T17:52:41.741583+0000","last_fullsized":"2024-08-17T17:52:41.741583+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":115,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.966238+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:52:41.966238+0000","last_peered":"2024-08-17T17:52:41.966238+0000","last_clean":"2024-08-17T17:52:41.966238+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:52:41.966238+0000","last_undegraded":"2024-08-17T17:52:41.966238+0000","last_fullsized":"2024-08-17T17:52:41.966238+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1525,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757304+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:52:32.757304+0000","last_peered":"2024-08-17T17:52:32.757304+0000","last_clean":"2024-08-17T17:52:32.757304+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:52:32.757304+0000","last_undegraded":"2024-08-17T17:52:32.757304+0000","last_fullsized":"2024-08-17T17:52:32.757304+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1875,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.745393+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:52:41.745393+0000","last_peered":"2024-08-17T17:52:41.745393+0000","last_clean":"2024-08-17T17:52:41.745393+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:52:41.745393+0000","last_undegraded":"2024-08-17T17:52:41.745393+0000","last_fullsized":"2024-08-17T17:52:41.745393+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1815,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.740554+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:52:41.740554+0000","last_peered":"2024-08-17T17:52:41.740554+0000","last_clean":"2024-08-17T17:52:41.740554+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:52:41.740554+0000","last_undegraded":"2024-08-17T17:52:41.740554+0000","last_fullsized":"2024-08-17T17:52:41.740554+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":493,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.749107+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:52:32.749107+0000","last_peered":"2024-08-17T17:52:32.749107+0000","last_clean":"2024-08-17T17:52:32.749107+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:52:32.749107+0000","last_undegraded":"2024-08-17T17:52:32.749107+0000","last_fullsized":"2024-08-17T17:52:32.749107+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":554,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.964709+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:52:41.964709+0000","last_peered":"2024-08-17T17:52:41.964709+0000","last_clean":"2024-08-17T17:52:41.964709+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:52:41.964709+0000","last_undegraded":"2024-08-17T17:52:41.964709+0000","last_fullsized":"2024-08-17T17:52:41.964709+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":570,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.965278+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:52:41.965278+0000","last_peered":"2024-08-17T17:52:41.965278+0000","last_clean":"2024-08-17T17:52:41.965278+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:52:41.965278+0000","last_undegraded":"2024-08-17T17:52:41.965278+0000","last_fullsized":"2024-08-17T17:52:41.965278+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1177,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.748954+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:52:32.748954+0000","last_peered":"2024-08-17T17:52:32.748954+0000","last_clean":"2024-08-17T17:52:32.748954+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:52:32.748954+0000","last_undegraded":"2024-08-17T17:52:32.748954+0000","last_fullsized":"2024-08-17T17:52:32.748954+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":586,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.741270+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:52:41.741270+0000","last_peered":"2024-08-17T17:52:41.741270+0000","last_clean":"2024-08-17T17:52:41.741270+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:52:41.741270+0000","last_undegraded":"2024-08-17T17:52:41.741270+0000","last_fullsized":"2024-08-17T17:52:41.741270+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1812,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.745285+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:52:41.745285+0000","last_peered":"2024-08-17T17:52:41.745285+0000","last_clean":"2024-08-17T17:52:41.745285+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:52:41.745285+0000","last_undegraded":"2024-08-17T17:52:41.745285+0000","last_fullsized":"2024-08-17T17:52:41.745285+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1907,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.001414+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:52:42.001414+0000","last_peered":"2024-08-17T17:52:42.001414+0000","last_clean":"2024-08-17T17:52:42.001414+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:52:42.001414+0000","last_undegraded":"2024-08-17T17:52:42.001414+0000","last_fullsized":"2024-08-17T17:52:42.001414+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":619,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.741184+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:52:41.741184+0000","last_peered":"2024-08-17T17:52:41.741184+0000","last_clean":"2024-08-17T17:52:41.741184+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:52:41.741184+0000","last_undegraded":"2024-08-17T17:52:41.741184+0000","last_fullsized":"2024-08-17T17:52:41.741184+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1166,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.748811+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:52:32.748811+0000","last_peered":"2024-08-17T17:52:32.748811+0000","last_clean":"2024-08-17T17:52:32.748811+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:52:32.748811+0000","last_undegraded":"2024-08-17T17:52:32.748811+0000","last_fullsized":"2024-08-17T17:52:32.748811+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1165,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757673+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:52:32.757673+0000","last_peered":"2024-08-17T17:52:32.757673+0000","last_clean":"2024-08-17T17:52:32.757673+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:52:32.757673+0000","last_undegraded":"2024-08-17T17:52:32.757673+0000","last_fullsized":"2024-08-17T17:52:32.757673+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":603,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.740292+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:52:41.740292+0000","last_peered":"2024-08-17T17:52:41.740292+0000","last_clean":"2024-08-17T17:52:41.740292+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:52:41.740292+0000","last_undegraded":"2024-08-17T17:52:41.740292+0000","last_fullsized":"2024-08-17T17:52:41.740292+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1183,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757817+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:52:32.757817+0000","last_peered":"2024-08-17T17:52:32.757817+0000","last_clean":"2024-08-17T17:52:32.757817+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:52:32.757817+0000","last_undegraded":"2024-08-17T17:52:32.757817+0000","last_fullsized":"2024-08-17T17:52:32.757817+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":67,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757239+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:52:32.757239+0000","last_peered":"2024-08-17T17:52:32.757239+0000","last_clean":"2024-08-17T17:52:32.757239+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:52:32.757239+0000","last_undegraded":"2024-08-17T17:52:32.757239+0000","last_fullsized":"2024-08-17T17:52:32.757239+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1199,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757767+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:52:32.757767+0000","last_peered":"2024-08-17T17:52:32.757767+0000","last_clean":"2024-08-17T17:52:32.757767+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:52:32.757767+0000","last_undegraded":"2024-08-17T17:52:32.757767+0000","last_fullsized":"2024-08-17T17:52:32.757767+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1216,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.748724+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:52:32.748724+0000","last_peered":"2024-08-17T17:52:32.748724+0000","last_clean":"2024-08-17T17:52:32.748724+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:52:32.748724+0000","last_undegraded":"2024-08-17T17:52:32.748724+0000","last_fullsized":"2024-08-17T17:52:32.748724+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1813,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.746844+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:52:41.746844+0000","last_peered":"2024-08-17T17:52:41.746844+0000","last_clean":"2024-08-17T17:52:41.746844+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:52:41.746844+0000","last_undegraded":"2024-08-17T17:52:41.746844+0000","last_fullsized":"2024-08-17T17:52:41.746844+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1160,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757440+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:52:32.757440+0000","last_peered":"2024-08-17T17:52:32.757440+0000","last_clean":"2024-08-17T17:52:32.757440+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:52:32.757440+0000","last_undegraded":"2024-08-17T17:52:32.757440+0000","last_fullsized":"2024-08-17T17:52:32.757440+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1866,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.001496+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:52:42.001496+0000","last_peered":"2024-08-17T17:52:42.001496+0000","last_clean":"2024-08-17T17:52:42.001496+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:52:42.001496+0000","last_undegraded":"2024-08-17T17:52:42.001496+0000","last_fullsized":"2024-08-17T17:52:42.001496+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1867,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:42.001325+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:52:42.001325+0000","last_peered":"2024-08-17T17:52:42.001325+0000","last_clean":"2024-08-17T17:52:42.001325+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:52:42.001325+0000","last_undegraded":"2024-08-17T17:52:42.001325+0000","last_fullsized":"2024-08-17T17:52:42.001325+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":573,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757495+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:52:32.757495+0000","last_peered":"2024-08-17T17:52:32.757495+0000","last_clean":"2024-08-17T17:52:32.757495+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:52:32.757495+0000","last_undegraded":"2024-08-17T17:52:32.757495+0000","last_fullsized":"2024-08-17T17:52:32.757495+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1185,"reported_epoch":1215,"state":"active+clean","last_fresh":"2024-08-17T17:52:32.757699+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:52:32.757699+0000","last_peered":"2024-08-17T17:52:32.757699+0000","last_clean":"2024-08-17T17:52:32.757699+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:52:32.757699+0000","last_undegraded":"2024-08-17T17:52:32.757699+0000","last_fullsized":"2024-08-17T17:52:32.757699+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1813,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.741387+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:52:41.741387+0000","last_peered":"2024-08-17T17:52:41.741387+0000","last_clean":"2024-08-17T17:52:41.741387+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:52:41.741387+0000","last_undegraded":"2024-08-17T17:52:41.741387+0000","last_fullsized":"2024-08-17T17:52:41.741387+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":5118689304,"num_objects":78106,"num_object_clones":0,"num_object_copies":234318,"num_objects_missing_on_primary":4374,"num_objects_missing":4374,"num_objects_degraded":18928,"num_objects_misplaced":13524,"num_objects_unfound":0,"num_objects_dirty":78106,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131504,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33343,"num_bytes_recovered":2006712320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15985029120,"data_stored":15985016904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106578,"ondisk_log_size":106578,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234382,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2404340,"kb_used_data":2098664,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91340812,"statfs":{"total":95995035648,"available":93532991488,"internally_reserved":0,"allocated":2149031936,"data_stored":2144189527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496017,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2725816,"kb_used_data":2420140,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91019336,"statfs":{"total":95995035648,"available":93203800064,"internally_reserved":0,"allocated":2478223360,"data_stored":2473377356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757655,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326269,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3001756,"kb_used_data":2789460,"kb_used_omap":57,"kb_used_meta":212230,"kb_avail":90743396,"statfs":{"total":95995035648,"available":92921237504,"internally_reserved":0,"allocated":2856407040,"data_stored":2851579811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":217324283},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405508,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2697636,"kb_used_data":2503324,"kb_used_omap":61,"kb_used_meta":194242,"kb_avail":91047516,"statfs":{"total":95995035648,"available":93232656384,"internally_reserved":0,"allocated":2563403776,"data_stored":2558573519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":198904760},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452042,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2049692,"kb_used_data":1887376,"kb_used_omap":54,"kb_used_meta":162249,"kb_avail":91695460,"statfs":{"total":95995035648,"available":93896151040,"internally_reserved":0,"allocated":1932673024,"data_stored":1927849756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":166143374},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474734,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1681836,"kb_used_data":1522976,"kb_used_omap":119,"kb_used_meta":158728,"kb_avail":92063316,"statfs":{"total":95995035648,"available":94272835584,"internally_reserved":0,"allocated":1559527424,"data_stored":1554688992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":162538206},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019290,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1544491008,"data_stored":1544486936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1919549440,"data_stored":1919549440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2549350400,"data_stored":2549350400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2843217920,"data_stored":2843213848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1090191360,"data_stored":1090191360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1439760384,"data_stored":1439760384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2463567872,"data_stored":2463563800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2134900736,"data_stored":2134900736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:52:48.616 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:52:48.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.617 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:48.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:48.718 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:48.741 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:48.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:48.843 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:48.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.865 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:48.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:48.940 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:48.967 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:48.988 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:48.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:48.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:49.090 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:49.112 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:49.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:49.197 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:49 smithi130 ceph-mon[105521]: pgmap v1763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 18852/232866 objects degraded (8.096%); 13411/232866 objects misplaced (5.759%); 966 KiB/s, 15 objects/s recovering 2024-08-17T17:52:49.197 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:49 smithi130 ceph-mon[110068]: pgmap v1763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 18852/232866 objects degraded (8.096%); 13411/232866 objects misplaced (5.759%); 966 KiB/s, 15 objects/s recovering 2024-08-17T17:52:49.208 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:49 smithi179 ceph-mon[93519]: pgmap v1763: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 18852/232866 objects degraded (8.096%); 13411/232866 objects misplaced (5.759%); 966 KiB/s, 15 objects/s recovering 2024-08-17T17:52:49.214 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:49.236 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:49.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:49.245 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:49.338 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:49.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.361 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:49.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:49.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:49.491 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:49.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:49.593 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:49.620 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:49.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:49.722 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:49.747 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:49.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:49.848 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:49.871 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:49.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:49.972 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:49.997 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:49.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:49.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:50.098 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:50.114 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:52:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:52:49] "GET /metrics HTTP/1.1" 200 36807 "" "Prometheus/2.51.0" 2024-08-17T17:52:50.119 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T17:52:50.120 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:50.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:50.123 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:52:50.123 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T17:52:50.123 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T17:52:50.123 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T17:52:50.123 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.123 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.44945.0:2807 44.16 44:6bfa3af8:::rbd_data.af91c8ed8b9b.000000000000005c:head [write 2577372~1616932] snapc 72=[72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-08-17T17:52:50.123 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:45:43.833257+0000", 2024-08-17T17:52:50.123 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 426.28237842700003, 2024-08-17T17:52:50.124 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.14521906700000001, 2024-08-17T17:52:50.124 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.124 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.124 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.124 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.44945", 2024-08-17T17:52:50.124 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/735940947", 2024-08-17T17:52:50.124 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 2807 2024-08-17T17:52:50.124 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.124 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.124 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.124 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.124 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.833257+0000", 2024-08-17T17:52:50.124 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.124 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.833257+0000", 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.833255+0000", 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836081+0000", 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002825597 2024-08-17T17:52:50.125 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.126 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.126 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.126 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836109+0000", 2024-08-17T17:52:50.126 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8121999999999999e-05 2024-08-17T17:52:50.126 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.126 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.126 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.126 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836117+0000", 2024-08-17T17:52:50.126 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7970000000000001e-06 2024-08-17T17:52:50.126 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.126 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.126 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.126 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836182+0000", 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4968999999999994e-05 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op must wait for map", 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836190+0000", 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0229999999999992e-06 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960630+0000", 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.124440812 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.127 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960725+0000", 2024-08-17T17:52:50.128 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4609000000000002e-05 2024-08-17T17:52:50.128 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.128 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.128 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,5", 2024-08-17T17:52:50.128 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960919+0000", 2024-08-17T17:52:50.128 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000194291 2024-08-17T17:52:50.128 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.128 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.128 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.128 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.965209+0000", 2024-08-17T17:52:50.129 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0042898329999999998 2024-08-17T17:52:50.129 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.129 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.129 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.129 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978326+0000", 2024-08-17T17:52:50.129 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013117007999999999 2024-08-17T17:52:50.129 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.129 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.129 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.129 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978387+0000", 2024-08-17T17:52:50.129 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1228000000000004e-05 2024-08-17T17:52:50.129 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.129 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.129 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.130 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978438+0000", 2024-08-17T17:52:50.130 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0680000000000003e-05 2024-08-17T17:52:50.130 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.130 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.130 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.130 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978476+0000", 2024-08-17T17:52:50.130 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8074999999999997e-05 2024-08-17T17:52:50.130 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.130 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.130 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.130 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.130 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.44945.0:2809 44.19 44:9cf770fe:::rbd_data.af91c8ed8b9b.0000000000000014:head [write 4008407~185897] snapc 72=[72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:45:43.836130+0000", 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 426.279505255, 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.14255548400000001, 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.44945", 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/735940947", 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 2809 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.131 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836130+0000", 2024-08-17T17:52:50.132 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.132 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.132 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.132 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.132 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836130+0000", 2024-08-17T17:52:50.132 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.132 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.132 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.132 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.132 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836129+0000", 2024-08-17T17:52:50.132 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:50.132 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.132 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.132 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.133 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836269+0000", 2024-08-17T17:52:50.133 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014036699999999999 2024-08-17T17:52:50.133 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.133 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.133 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.133 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836279+0000", 2024-08-17T17:52:50.133 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.3649999999999993e-06 2024-08-17T17:52:50.133 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.133 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.133 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.133 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836284+0000", 2024-08-17T17:52:50.133 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7600000000000002e-06 2024-08-17T17:52:50.133 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.133 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.134 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.134 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836342+0000", 2024-08-17T17:52:50.134 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8489999999999997e-05 2024-08-17T17:52:50.134 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.134 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.134 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op must wait for map", 2024-08-17T17:52:50.134 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836347+0000", 2024-08-17T17:52:50.134 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1270000000000002e-06 2024-08-17T17:52:50.134 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.134 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.134 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.134 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.841100+0000", 2024-08-17T17:52:50.134 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0047532820000000002 2024-08-17T17:52:50.135 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.135 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.135 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.135 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.841242+0000", 2024-08-17T17:52:50.135 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014152500000000001 2024-08-17T17:52:50.135 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.135 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.135 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:52:50.135 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960848+0000", 2024-08-17T17:52:50.135 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.119605845 2024-08-17T17:52:50.135 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.136 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.136 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.136 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.962334+0000", 2024-08-17T17:52:50.136 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0014866269999999999 2024-08-17T17:52:50.136 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.136 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.136 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.136 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.962704+0000", 2024-08-17T17:52:50.136 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00036935800000000002 2024-08-17T17:52:50.136 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.136 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.136 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.136 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978608+0000", 2024-08-17T17:52:50.137 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.015903870000000001 2024-08-17T17:52:50.137 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.137 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.137 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.137 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978644+0000", 2024-08-17T17:52:50.137 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6739000000000002e-05 2024-08-17T17:52:50.137 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.137 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.137 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.137 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978686+0000", 2024-08-17T17:52:50.137 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1322000000000002e-05 2024-08-17T17:52:50.137 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.137 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.137 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.138 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.138 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.138 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.44945.0:2806 44.2 e1130/1015 44:46ff3400:::rbd_data.af91c8ed8b9b.000000000000005b:head v 1130'32, mlcod=1128'28)", 2024-08-17T17:52:50.138 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:45:43.838017+0000", 2024-08-17T17:52:50.138 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 426.27761810300001, 2024-08-17T17:52:50.138 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.140595952, 2024-08-17T17:52:50.138 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.138 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.138 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.138 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.138 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.138 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.838017+0000", 2024-08-17T17:52:50.138 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.139 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.139 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.139 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.139 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.838017+0000", 2024-08-17T17:52:50.139 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.139 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.139 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.139 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.139 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.838017+0000", 2024-08-17T17:52:50.139 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967296 2024-08-17T17:52:50.139 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.139 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.139 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.140 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.847265+0000", 2024-08-17T17:52:50.140 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0092478170000000002 2024-08-17T17:52:50.140 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.140 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.140 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.140 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960961+0000", 2024-08-17T17:52:50.140 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.113695542 2024-08-17T17:52:50.140 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.140 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.140 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.140 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960968+0000", 2024-08-17T17:52:50.140 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0559999999999999e-06 2024-08-17T17:52:50.140 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.141 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.141 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.141 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.961005+0000", 2024-08-17T17:52:50.141 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7635999999999997e-05 2024-08-17T17:52:50.141 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.141 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.141 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.141 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.961024+0000", 2024-08-17T17:52:50.141 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.9057999999999999e-05 2024-08-17T17:52:50.141 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.141 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.141 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.141 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978574+0000", 2024-08-17T17:52:50.141 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.017549828999999999 2024-08-17T17:52:50.142 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.142 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.142 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.142 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978613+0000", 2024-08-17T17:52:50.142 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9227999999999998e-05 2024-08-17T17:52:50.142 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.142 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.142 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.142 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.142 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.142 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.44945.0:2808 44.d e1130/1015 44:b7f70492:::rbd_data.af91c8ed8b9b.000000000000005d:head v 1130'43, mlcod=1125'39)", 2024-08-17T17:52:50.143 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:45:43.839267+0000", 2024-08-17T17:52:50.143 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 426.27636852199998, 2024-08-17T17:52:50.143 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.13564867799999999, 2024-08-17T17:52:50.143 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.143 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.143 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.143 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.143 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.143 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.839267+0000", 2024-08-17T17:52:50.143 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.143 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.143 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.144 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.144 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.839267+0000", 2024-08-17T17:52:50.144 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.144 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.144 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.144 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.144 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.839267+0000", 2024-08-17T17:52:50.145 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:52:50.145 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.145 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.145 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.145 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.842005+0000", 2024-08-17T17:52:50.145 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027385220000000002 2024-08-17T17:52:50.145 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.145 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.145 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.145 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960414+0000", 2024-08-17T17:52:50.145 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.118408872 2024-08-17T17:52:50.145 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.145 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.145 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.146 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960424+0000", 2024-08-17T17:52:50.146 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0019e-05 2024-08-17T17:52:50.146 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.146 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.146 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.146 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960492+0000", 2024-08-17T17:52:50.146 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7566999999999999e-05 2024-08-17T17:52:50.146 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.146 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.146 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.146 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960519+0000", 2024-08-17T17:52:50.146 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.7560999999999999e-05 2024-08-17T17:52:50.147 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.147 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.147 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.147 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.974868+0000", 2024-08-17T17:52:50.147 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.014348681 2024-08-17T17:52:50.147 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.147 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.147 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.147 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.974916+0000", 2024-08-17T17:52:50.147 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7874999999999998e-05 2024-08-17T17:52:50.147 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.147 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.147 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.147 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.148 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.148 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.44945.0:2819 44.a 44:51bcdcfb:::rbd_data.af91c8ed8b9b.0000000000000034:head [write 0~2394689] snapc 72=[72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-08-17T17:52:50.148 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:45:43.883032+0000", 2024-08-17T17:52:50.148 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 426.23260345400001, 2024-08-17T17:52:50.148 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.104906519, 2024-08-17T17:52:50.148 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.148 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.148 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.148 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.44945", 2024-08-17T17:52:50.148 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/735940947", 2024-08-17T17:52:50.148 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 2819 2024-08-17T17:52:50.148 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.149 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.149 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.149 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.149 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.883032+0000", 2024-08-17T17:52:50.149 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.149 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.149 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.149 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.149 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.883032+0000", 2024-08-17T17:52:50.149 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.149 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.149 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.149 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.883030+0000", 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.886244+0000", 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0032137709999999998 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960544+0000", 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.074300081000000004 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.150 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.151 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960554+0000", 2024-08-17T17:52:50.151 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4639999999999995e-06 2024-08-17T17:52:50.151 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.151 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.151 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.151 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960606+0000", 2024-08-17T17:52:50.151 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2080999999999998e-05 2024-08-17T17:52:50.151 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.151 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.151 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.151 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960722+0000", 2024-08-17T17:52:50.151 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000115952 2024-08-17T17:52:50.151 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.151 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960936+0000", 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00021386699999999999 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.972281+0000", 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.011345808000000001 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978276+0000", 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0059948190000000002 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.152 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.987870+0000", 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0095934569999999997 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.987908+0000", 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8858e-05 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.987939+0000", 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0091e-05 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.153 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.154 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.154 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.154 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.154 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83239 45.e 45:777b3534:::benchmark_data_smithi130_172551_object83238:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:52:50.154 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.640449+0000", 2024-08-17T17:52:50.154 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 251.47518636300001, 2024-08-17T17:52:50.154 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.429002718, 2024-08-17T17:52:50.154 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.154 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:52:50.154 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.154 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.154 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83239 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640449+0000", 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640449+0000", 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.155 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.156 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640447+0000", 2024-08-17T17:52:50.156 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:50.156 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.156 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.156 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.156 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640482+0000", 2024-08-17T17:52:50.156 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4570000000000003e-05 2024-08-17T17:52:50.156 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.156 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.156 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.156 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640492+0000", 2024-08-17T17:52:50.156 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0718e-05 2024-08-17T17:52:50.156 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.156 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.157 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.157 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640498+0000", 2024-08-17T17:52:50.157 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3009999999999997e-06 2024-08-17T17:52:50.157 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.157 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.157 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.157 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640600+0000", 2024-08-17T17:52:50.157 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000101969 2024-08-17T17:52:50.157 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.157 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.157 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.157 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640743+0000", 2024-08-17T17:52:50.158 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014374400000000001 2024-08-17T17:52:50.158 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.158 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.158 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:52:50.158 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640908+0000", 2024-08-17T17:52:50.158 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016489599999999999 2024-08-17T17:52:50.158 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.158 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.158 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.158 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.649642+0000", 2024-08-17T17:52:50.158 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0087341720000000001 2024-08-17T17:52:50.158 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.649653+0000", 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0495e-05 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.644929+0000", 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99527607200000001 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.644942+0000", 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2503e-05 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.159 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.069362+0000", 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.424420222 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.069452+0000", 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0087999999999995e-05 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83251 45.e 45:7cdda562:::benchmark_data_smithi130_172551_object83250:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:52:50.160 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.668406+0000", 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 251.44722991, 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4230686509999999, 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83251 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668406+0000", 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.161 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.162 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.162 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.162 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668406+0000", 2024-08-17T17:52:50.162 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.162 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.162 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.162 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.162 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668405+0000", 2024-08-17T17:52:50.162 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:52:50.162 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.162 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.162 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.162 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668461+0000", 2024-08-17T17:52:50.163 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6495e-05 2024-08-17T17:52:50.163 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.163 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.163 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.163 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668469+0000", 2024-08-17T17:52:50.163 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.1890000000000007e-06 2024-08-17T17:52:50.163 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.163 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.163 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.163 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668474+0000", 2024-08-17T17:52:50.163 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2010000000000001e-06 2024-08-17T17:52:50.163 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.163 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668532+0000", 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8044999999999997e-05 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668538+0000", 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3180000000000003e-06 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.644966+0000", 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97642784599999999 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.164 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:52:50.165 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.644973+0000", 2024-08-17T17:52:50.165 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.25e-06 2024-08-17T17:52:50.165 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.165 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.165 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.165 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.071858+0000", 2024-08-17T17:52:50.165 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.426884539 2024-08-17T17:52:50.165 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.165 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.165 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.165 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.071955+0000", 2024-08-17T17:52:50.165 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7349000000000006e-05 2024-08-17T17:52:50.165 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.165 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.166 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:52:50.166 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.072058+0000", 2024-08-17T17:52:50.166 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000103127 2024-08-17T17:52:50.166 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.166 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.166 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.075587+0000", 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0035286889999999998 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.078139+0000", 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025525970000000002 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.088422+0000", 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.010282217999999999 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.167 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.088469+0000", 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7750999999999997e-05 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.091474+0000", 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003004933 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83268 45.e 45:74a67c83:::benchmark_data_smithi130_172551_object83267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.700848+0000", 2024-08-17T17:52:50.168 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 251.41478744400001, 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.393204908, 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83268 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700848+0000", 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.169 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.170 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700848+0000", 2024-08-17T17:52:50.170 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.170 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.170 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.170 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.170 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700847+0000", 2024-08-17T17:52:50.170 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:50.170 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.170 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.170 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.170 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700894+0000", 2024-08-17T17:52:50.170 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6879000000000003e-05 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700906+0000", 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2201999999999999e-05 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700910+0000", 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4700000000000004e-06 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700963+0000", 2024-08-17T17:52:50.171 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2654000000000001e-05 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700973+0000", 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0511e-05 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.644997+0000", 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.94402362799999995 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.645004+0000", 2024-08-17T17:52:50.172 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6959999999999999e-06 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.074035+0000", 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4290308789999999 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.074125+0000", 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0348999999999998e-05 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.074233+0000", 2024-08-17T17:52:50.173 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010794300000000001 2024-08-17T17:52:50.174 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.174 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.174 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.174 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.075721+0000", 2024-08-17T17:52:50.174 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0014879299999999999 2024-08-17T17:52:50.174 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.174 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.174 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.174 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.078189+0000", 2024-08-17T17:52:50.174 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024685559999999998 2024-08-17T17:52:50.174 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.174 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.175 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.175 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.091539+0000", 2024-08-17T17:52:50.175 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013349152 2024-08-17T17:52:50.175 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.175 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.175 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.175 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.091607+0000", 2024-08-17T17:52:50.175 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.8219999999999994e-05 2024-08-17T17:52:50.175 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.175 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.175 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.175 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.094053+0000", 2024-08-17T17:52:50.175 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024461499999999998 2024-08-17T17:52:50.175 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.176 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.176 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.176 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.176 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.176 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:90848 45.e 45:78c2541e:::benchmark_data_smithi130_172551_object90847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:52:50.176 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:53.710358+0000", 2024-08-17T17:52:50.176 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 236.40527765300001, 2024-08-17T17:52:50.176 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.084649796999999999, 2024-08-17T17:52:50.176 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.176 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.176 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.176 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.176 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.176 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 90848 2024-08-17T17:52:50.177 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.177 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.177 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.177 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.177 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710358+0000", 2024-08-17T17:52:50.177 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.177 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.177 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.177 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.177 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710358+0000", 2024-08-17T17:52:50.177 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.177 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.177 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.177 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.178 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710356+0000", 2024-08-17T17:52:50.178 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:50.178 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.178 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.178 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.178 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710398+0000", 2024-08-17T17:52:50.178 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1658e-05 2024-08-17T17:52:50.178 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.178 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.178 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.178 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710415+0000", 2024-08-17T17:52:50.179 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.7419e-05 2024-08-17T17:52:50.179 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.179 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.179 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.179 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710422+0000", 2024-08-17T17:52:50.179 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.9589999999999998e-06 2024-08-17T17:52:50.179 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.179 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.179 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.179 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710464+0000", 2024-08-17T17:52:50.179 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1910999999999999e-05 2024-08-17T17:52:50.179 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.179 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.180 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.180 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710579+0000", 2024-08-17T17:52:50.180 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000114905 2024-08-17T17:52:50.180 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.180 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.180 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:52:50.180 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710693+0000", 2024-08-17T17:52:50.180 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00011406899999999999 2024-08-17T17:52:50.180 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.180 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.180 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.180 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.722128+0000", 2024-08-17T17:52:50.180 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.011434826 2024-08-17T17:52:50.181 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.181 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.181 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.181 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.722173+0000", 2024-08-17T17:52:50.181 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4889999999999999e-05 2024-08-17T17:52:50.181 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.181 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.181 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.181 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.792606+0000", 2024-08-17T17:52:50.181 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.070432967999999999 2024-08-17T17:52:50.181 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.181 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.181 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.181 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.792646+0000", 2024-08-17T17:52:50.182 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9838e-05 2024-08-17T17:52:50.182 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.182 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.182 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.182 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.795008+0000", 2024-08-17T17:52:50.182 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002361765 2024-08-17T17:52:50.182 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.182 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.182 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.182 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.182 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.183 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:90854 45.2 45:438ed877:::benchmark_data_smithi130_172551_object90853:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:52:50.183 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:53.715817+0000", 2024-08-17T17:52:50.183 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 236.39981800800001, 2024-08-17T17:52:50.183 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.084003866999999996, 2024-08-17T17:52:50.183 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.183 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.183 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.183 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.183 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.183 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 90854 2024-08-17T17:52:50.183 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.183 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.183 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.184 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.184 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.715817+0000", 2024-08-17T17:52:50.184 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.184 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.184 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.184 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.184 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.715817+0000", 2024-08-17T17:52:50.184 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.184 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.184 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.184 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.184 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.715816+0000", 2024-08-17T17:52:50.184 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:52:50.184 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.715876+0000", 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9901000000000001e-05 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.715888+0000", 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1861999999999999e-05 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.715893+0000", 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8760000000000004e-06 2024-08-17T17:52:50.185 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.186 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.186 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.186 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.715937+0000", 2024-08-17T17:52:50.186 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3560000000000003e-05 2024-08-17T17:52:50.186 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.186 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.186 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.186 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.716060+0000", 2024-08-17T17:52:50.186 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012357700000000001 2024-08-17T17:52:50.186 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.186 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.186 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:52:50.186 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.716199+0000", 2024-08-17T17:52:50.186 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013864500000000001 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.719149+0000", 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002950584 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.720519+0000", 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001369157 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.799745+0000", 2024-08-17T17:52:50.187 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.079226435999999997 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.799777+0000", 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2379000000000003e-05 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.799821+0000", 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4014999999999998e-05 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.188 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.189 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:101590 45.e 45:7e22289b:::benchmark_data_smithi130_172551_object101589:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1184)", 2024-08-17T17:52:50.189 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:13.080211+0000", 2024-08-17T17:52:50.189 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 217.03542442299999, 2024-08-17T17:52:50.189 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.090180265999999995, 2024-08-17T17:52:50.189 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.189 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.189 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.189 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.189 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.189 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 101590 2024-08-17T17:52:50.189 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.189 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.189 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.190 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.190 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080211+0000", 2024-08-17T17:52:50.190 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.190 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.190 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.190 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.190 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080211+0000", 2024-08-17T17:52:50.190 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.190 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.190 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.190 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.190 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080210+0000", 2024-08-17T17:52:50.190 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:52:50.190 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.191 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.191 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.191 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080268+0000", 2024-08-17T17:52:50.191 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8019999999999997e-05 2024-08-17T17:52:50.191 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.191 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.191 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.191 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080279+0000", 2024-08-17T17:52:50.191 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1412e-05 2024-08-17T17:52:50.191 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.191 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.191 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.191 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080284+0000", 2024-08-17T17:52:50.192 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4970000000000003e-06 2024-08-17T17:52:50.192 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.192 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.192 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.192 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080456+0000", 2024-08-17T17:52:50.192 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017256500000000001 2024-08-17T17:52:50.192 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.192 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.192 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.192 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080611+0000", 2024-08-17T17:52:50.192 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000154167 2024-08-17T17:52:50.192 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.192 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.192 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:52:50.193 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080755+0000", 2024-08-17T17:52:50.193 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000144752 2024-08-17T17:52:50.193 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.193 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.193 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.193 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.082747+0000", 2024-08-17T17:52:50.193 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019912559999999998 2024-08-17T17:52:50.193 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.193 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.193 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:50 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3991087592' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:50.194 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.194 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.098363+0000", 2024-08-17T17:52:50.194 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.015616260999999999 2024-08-17T17:52:50.194 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.195 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.195 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.195 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.167705+0000", 2024-08-17T17:52:50.195 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.069342564999999995 2024-08-17T17:52:50.195 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.195 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.195 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.195 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.167764+0000", 2024-08-17T17:52:50.195 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8885000000000003e-05 2024-08-17T17:52:50.195 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.195 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.195 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.195 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.170391+0000", 2024-08-17T17:52:50.195 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002627028 2024-08-17T17:52:50.196 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.196 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.196 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.196 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.196 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.196 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:101592 45.e 45:72eb4cf9:::benchmark_data_smithi130_172551_object101591:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1184)", 2024-08-17T17:52:50.196 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:13.083483+0000", 2024-08-17T17:52:50.196 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 217.03215228100001, 2024-08-17T17:52:50.196 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.088968239000000005, 2024-08-17T17:52:50.196 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.196 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.196 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.196 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.196 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 101592 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083483+0000", 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083483+0000", 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.197 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083482+0000", 2024-08-17T17:52:50.198 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:52:50.198 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.198 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.198 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.198 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083563+0000", 2024-08-17T17:52:50.198 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0604000000000003e-05 2024-08-17T17:52:50.198 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.198 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.198 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.198 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083573+0000", 2024-08-17T17:52:50.198 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.063e-05 2024-08-17T17:52:50.198 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.198 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083578+0000", 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0810000000000001e-06 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083615+0000", 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6106999999999998e-05 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083705+0000", 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0505000000000002e-05 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.199 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.200 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:52:50.200 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083808+0000", 2024-08-17T17:52:50.200 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000103218 2024-08-17T17:52:50.200 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.200 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.200 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.200 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.086192+0000", 2024-08-17T17:52:50.200 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0023838499999999999 2024-08-17T17:52:50.200 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.200 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.200 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.200 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.105096+0000", 2024-08-17T17:52:50.200 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.018904213999999999 2024-08-17T17:52:50.200 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.170445+0000", 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.065348385999999994 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.170486+0000", 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0969e-05 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.172451+0000", 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019656980000000001 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.201 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.202 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.202 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.202 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.202 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:101696 45.e 45:7d01e76d:::benchmark_data_smithi130_172551_object101695:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1184)", 2024-08-17T17:52:50.202 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:13.270443+0000", 2024-08-17T17:52:50.202 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 216.845192239, 2024-08-17T17:52:50.202 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.087804115000000002, 2024-08-17T17:52:50.202 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.202 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.202 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.202 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.202 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.203 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 101696 2024-08-17T17:52:50.203 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.203 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.203 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.203 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.203 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270443+0000", 2024-08-17T17:52:50.203 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.203 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.203 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.203 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.203 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270443+0000", 2024-08-17T17:52:50.203 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.203 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.204 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.204 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.204 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270442+0000", 2024-08-17T17:52:50.204 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:50.204 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.204 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.204 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.204 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270493+0000", 2024-08-17T17:52:50.204 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1311999999999999e-05 2024-08-17T17:52:50.204 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.204 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.204 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.204 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270506+0000", 2024-08-17T17:52:50.204 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2787e-05 2024-08-17T17:52:50.205 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.205 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.205 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.205 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270511+0000", 2024-08-17T17:52:50.205 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2390000000000002e-06 2024-08-17T17:52:50.205 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.205 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.205 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.205 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270544+0000", 2024-08-17T17:52:50.205 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2267000000000001e-05 2024-08-17T17:52:50.205 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.205 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.205 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.206 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270648+0000", 2024-08-17T17:52:50.206 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000104453 2024-08-17T17:52:50.206 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.206 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.206 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:52:50.206 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270753+0000", 2024-08-17T17:52:50.206 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105376 2024-08-17T17:52:50.206 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.206 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.206 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.206 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.273647+0000", 2024-08-17T17:52:50.206 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028938670000000001 2024-08-17T17:52:50.206 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.207 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.207 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.207 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.303717+0000", 2024-08-17T17:52:50.207 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.030070125999999999 2024-08-17T17:52:50.207 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.207 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.207 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.207 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.355466+0000", 2024-08-17T17:52:50.207 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.051748283999999999 2024-08-17T17:52:50.207 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.207 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.207 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.207 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.355525+0000", 2024-08-17T17:52:50.208 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9740000000000001e-05 2024-08-17T17:52:50.208 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.208 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.208 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.208 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.358247+0000", 2024-08-17T17:52:50.208 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002722005 2024-08-17T17:52:50.208 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.208 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.208 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.208 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.208 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.208 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:108186 45.e 45:76cb9ea5:::benchmark_data_smithi130_172551_object3379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:52:50.209 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:25.911036+0000", 2024-08-17T17:52:50.209 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 204.20459947200001, 2024-08-17T17:52:50.209 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.12617594500000001, 2024-08-17T17:52:50.209 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.209 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.209 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.209 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.209 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.209 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 108186 2024-08-17T17:52:50.209 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.209 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.209 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.209 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.209 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911036+0000", 2024-08-17T17:52:50.210 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.210 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.210 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.210 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.210 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911036+0000", 2024-08-17T17:52:50.210 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.210 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.210 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.210 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.210 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911034+0000", 2024-08-17T17:52:50.210 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:52:50.210 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.211 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.211 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.211 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911042+0000", 2024-08-17T17:52:50.211 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2099999999999993e-06 2024-08-17T17:52:50.211 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.211 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.211 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.211 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911059+0000", 2024-08-17T17:52:50.211 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.6358000000000002e-05 2024-08-17T17:52:50.211 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.211 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.211 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.211 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911066+0000", 2024-08-17T17:52:50.211 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7900000000000002e-06 2024-08-17T17:52:50.212 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.212 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.212 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.212 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911114+0000", 2024-08-17T17:52:50.212 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8751000000000001e-05 2024-08-17T17:52:50.212 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.212 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.212 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.212 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911281+0000", 2024-08-17T17:52:50.212 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016636600000000001 2024-08-17T17:52:50.212 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.212 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.212 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911446+0000", 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000165587 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.914259+0000", 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028124069999999998 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.917316+0000", 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0030570250000000001 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.213 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.214 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.034078+0000", 2024-08-17T17:52:50.214 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.116762218 2024-08-17T17:52:50.214 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.214 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.214 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.214 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.034131+0000", 2024-08-17T17:52:50.214 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3542000000000003e-05 2024-08-17T17:52:50.214 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.214 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.214 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.214 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.037212+0000", 2024-08-17T17:52:50.214 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0030805400000000001 2024-08-17T17:52:50.214 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.215 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.215 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.215 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.215 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.215 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109016 45.e 45:7392c310:::benchmark_data_smithi130_172551_object4209:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:52:50.215 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:27.871637+0000", 2024-08-17T17:52:50.215 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 202.243998197, 2024-08-17T17:52:50.215 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.13319884600000001, 2024-08-17T17:52:50.215 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.215 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.215 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.215 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.215 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.215 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109016 2024-08-17T17:52:50.216 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.216 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.216 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.216 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.216 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.871637+0000", 2024-08-17T17:52:50.216 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.216 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.216 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.216 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.216 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.871637+0000", 2024-08-17T17:52:50.216 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.216 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.216 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.216 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.217 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.871636+0000", 2024-08-17T17:52:50.217 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:50.217 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.217 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.217 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.217 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.871642+0000", 2024-08-17T17:52:50.217 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4119999999999998e-06 2024-08-17T17:52:50.217 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.217 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.217 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.217 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.871657+0000", 2024-08-17T17:52:50.217 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.434e-05 2024-08-17T17:52:50.217 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.217 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.218 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.218 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.871663+0000", 2024-08-17T17:52:50.218 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7619999999999998e-06 2024-08-17T17:52:50.218 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.218 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.218 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.218 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.874473+0000", 2024-08-17T17:52:50.218 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028094700000000001 2024-08-17T17:52:50.218 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.218 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.218 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.218 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.874633+0000", 2024-08-17T17:52:50.218 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00015979300000000001 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.874801+0000", 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000168479 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.877602+0000", 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028012639999999999 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.885005+0000", 2024-08-17T17:52:50.219 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0074027429999999998 2024-08-17T17:52:50.220 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.220 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.220 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.220 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.002257+0000", 2024-08-17T17:52:50.220 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.117251828 2024-08-17T17:52:50.220 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.220 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.220 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.220 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.002295+0000", 2024-08-17T17:52:50.220 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8457999999999997e-05 2024-08-17T17:52:50.220 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.220 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.220 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.221 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.004836+0000", 2024-08-17T17:52:50.221 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025407870000000001 2024-08-17T17:52:50.221 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.221 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.221 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.221 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.221 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.221 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109029 45.e 45:736e9cfe:::benchmark_data_smithi130_172551_object4222:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:52:50.221 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:27.888679+0000", 2024-08-17T17:52:50.221 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 202.22695642799999, 2024-08-17T17:52:50.221 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.13807940599999999, 2024-08-17T17:52:50.221 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.221 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.222 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.222 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.222 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.222 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109029 2024-08-17T17:52:50.222 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.222 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.222 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.222 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.222 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888679+0000", 2024-08-17T17:52:50.222 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.222 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.222 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.222 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.223 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888679+0000", 2024-08-17T17:52:50.223 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.223 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.223 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.223 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.223 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888678+0000", 2024-08-17T17:52:50.223 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:50.223 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.223 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.223 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.223 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888684+0000", 2024-08-17T17:52:50.223 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7729999999999996e-06 2024-08-17T17:52:50.224 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.224 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.224 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.224 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888695+0000", 2024-08-17T17:52:50.224 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1851000000000001e-05 2024-08-17T17:52:50.224 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.224 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.224 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.224 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888700+0000", 2024-08-17T17:52:50.224 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2749999999999997e-06 2024-08-17T17:52:50.224 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.224 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.224 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.224 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888733+0000", 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3581999999999998e-05 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888853+0000", 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00011956 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888951+0000", 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7683000000000006e-05 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.225 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.891481+0000", 2024-08-17T17:52:50.226 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025300269999999998 2024-08-17T17:52:50.226 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.226 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.226 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.226 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.894100+0000", 2024-08-17T17:52:50.226 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0026193810000000001 2024-08-17T17:52:50.226 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.226 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.226 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.226 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.024256+0000", 2024-08-17T17:52:50.226 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.13015583999999999 2024-08-17T17:52:50.226 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.226 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.024290+0000", 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4668000000000001e-05 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.026758+0000", 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024680219999999998 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109152 45.e 45:7c082e4c:::benchmark_data_smithi130_172551_object4345:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:52:50.227 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:28.221818+0000", 2024-08-17T17:52:50.228 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 201.893817066, 2024-08-17T17:52:50.228 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.087909323999999997, 2024-08-17T17:52:50.228 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.228 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.228 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.228 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.228 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.228 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109152 2024-08-17T17:52:50.228 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.228 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.228 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.228 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.228 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221818+0000", 2024-08-17T17:52:50.229 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.229 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.229 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.229 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.229 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221818+0000", 2024-08-17T17:52:50.229 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.229 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.229 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.229 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.229 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221817+0000", 2024-08-17T17:52:50.229 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:50.229 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.229 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.230 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.230 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221823+0000", 2024-08-17T17:52:50.230 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4330000000000001e-06 2024-08-17T17:52:50.230 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.230 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.230 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.230 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221837+0000", 2024-08-17T17:52:50.230 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3266e-05 2024-08-17T17:52:50.230 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.230 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.230 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.230 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221842+0000", 2024-08-17T17:52:50.230 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9259999999999997e-06 2024-08-17T17:52:50.230 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.231 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.231 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.231 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221888+0000", 2024-08-17T17:52:50.231 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5895e-05 2024-08-17T17:52:50.231 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.231 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.231 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.231 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221991+0000", 2024-08-17T17:52:50.231 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010257200000000001 2024-08-17T17:52:50.231 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.231 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.231 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:52:50.231 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.222081+0000", 2024-08-17T17:52:50.231 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.9705999999999998e-05 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.224601+0000", 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025198870000000002 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.230423+0000", 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0058221369999999998 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.307111+0000", 2024-08-17T17:52:50.232 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.076687986999999999 2024-08-17T17:52:50.233 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.233 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.233 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.233 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.307145+0000", 2024-08-17T17:52:50.233 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3918999999999998e-05 2024-08-17T17:52:50.233 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.233 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.233 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.233 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.309728+0000", 2024-08-17T17:52:50.233 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025831959999999998 2024-08-17T17:52:50.233 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.233 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.233 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.234 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.234 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.234 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109305 45.e 45:7c61d673:::benchmark_data_smithi130_172551_object4498:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:52:50.234 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:28.556071+0000", 2024-08-17T17:52:50.234 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 201.55956413999999, 2024-08-17T17:52:50.234 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.138312143, 2024-08-17T17:52:50.234 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.234 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.234 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.234 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.234 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.234 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109305 2024-08-17T17:52:50.234 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.234 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.235 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.235 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.235 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556071+0000", 2024-08-17T17:52:50.235 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.235 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.235 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.235 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.235 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556071+0000", 2024-08-17T17:52:50.235 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.235 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.235 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.235 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.236 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556070+0000", 2024-08-17T17:52:50.236 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:50.236 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.236 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.236 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.236 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556075+0000", 2024-08-17T17:52:50.236 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8550000000000001e-06 2024-08-17T17:52:50.236 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.236 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.236 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.236 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556085+0000", 2024-08-17T17:52:50.236 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0234000000000001e-05 2024-08-17T17:52:50.236 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556091+0000", 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7710000000000003e-06 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556267+0000", 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017659699999999999 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556384+0000", 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000116746 2024-08-17T17:52:50.237 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.238 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.238 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:52:50.238 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556490+0000", 2024-08-17T17:52:50.238 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105827 2024-08-17T17:52:50.238 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.238 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.238 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.238 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.559327+0000", 2024-08-17T17:52:50.238 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002837348 2024-08-17T17:52:50.238 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.238 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.238 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.238 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.566772+0000", 2024-08-17T17:52:50.239 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0074446429999999999 2024-08-17T17:52:50.239 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.239 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.239 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.239 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.691832+0000", 2024-08-17T17:52:50.239 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.125059854 2024-08-17T17:52:50.239 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.239 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.239 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.239 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.691893+0000", 2024-08-17T17:52:50.239 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1379999999999998e-05 2024-08-17T17:52:50.239 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.239 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.239 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.694383+0000", 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024904049999999998 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:112629 45.e 45:7a542bc8:::benchmark_data_smithi130_172551_object7822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:36.133587+0000", 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 193.98204859000001, 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.12507467, 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.240 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.241 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.241 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 112629 2024-08-17T17:52:50.241 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.241 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.241 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.241 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.241 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133587+0000", 2024-08-17T17:52:50.241 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.241 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.241 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.241 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.241 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133587+0000", 2024-08-17T17:52:50.241 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.242 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.242 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.242 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.242 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133586+0000", 2024-08-17T17:52:50.242 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:52:50.242 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.242 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.242 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.242 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133595+0000", 2024-08-17T17:52:50.242 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.8470000000000002e-06 2024-08-17T17:52:50.242 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.242 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.242 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.242 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133607+0000", 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2517999999999999e-05 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133612+0000", 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4449999999999998e-06 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133650+0000", 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8469e-05 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.243 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133816+0000", 2024-08-17T17:52:50.244 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000166105 2024-08-17T17:52:50.244 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.244 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.244 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:52:50.244 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133955+0000", 2024-08-17T17:52:50.244 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013925999999999999 2024-08-17T17:52:50.244 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.244 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.244 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.244 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.137212+0000", 2024-08-17T17:52:50.244 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003256671 2024-08-17T17:52:50.244 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.244 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.244 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.245 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.170283+0000", 2024-08-17T17:52:50.245 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.033070755 2024-08-17T17:52:50.245 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.245 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.245 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.245 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.256179+0000", 2024-08-17T17:52:50.245 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.085895972000000001 2024-08-17T17:52:50.245 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.245 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.245 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.245 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.256239+0000", 2024-08-17T17:52:50.245 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0436000000000002e-05 2024-08-17T17:52:50.245 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.245 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.246 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.246 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.258662+0000", 2024-08-17T17:52:50.246 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002422352 2024-08-17T17:52:50.246 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.246 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.246 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.246 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.246 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.246 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:112656 45.e 45:775998fc:::benchmark_data_smithi130_172551_object7849:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:52:50.246 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:36.177766+0000", 2024-08-17T17:52:50.246 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 193.937869224, 2024-08-17T17:52:50.246 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.099564008999999995, 2024-08-17T17:52:50.246 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:52:50.247 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:52:50.247 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:52:50.247 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:52:50.247 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:52:50.247 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 112656 2024-08-17T17:52:50.247 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.247 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:52:50.247 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.247 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:52:50.247 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.177766+0000", 2024-08-17T17:52:50.247 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.247 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.248 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.248 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:52:50.248 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.177766+0000", 2024-08-17T17:52:50.248 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:52:50.248 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.248 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.248 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:52:50.248 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.177765+0000", 2024-08-17T17:52:50.248 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:52:50.248 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.248 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.248 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:52:50.248 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.177771+0000", 2024-08-17T17:52:50.249 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0120000000000002e-06 2024-08-17T17:52:50.249 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.249 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.249 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:52:50.249 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.177789+0000", 2024-08-17T17:52:50.249 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.8116000000000001e-05 2024-08-17T17:52:50.249 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.249 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.249 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:52:50.249 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.177795+0000", 2024-08-17T17:52:50.249 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2729999999999999e-06 2024-08-17T17:52:50.249 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.250 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.250 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:52:50.250 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.177854+0000", 2024-08-17T17:52:50.250 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9225999999999998e-05 2024-08-17T17:52:50.250 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.250 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.250 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:52:50.250 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.178009+0000", 2024-08-17T17:52:50.250 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00015437200000000001 2024-08-17T17:52:50.250 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.250 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.250 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.178167+0000", 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000158453 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.181685+0000", 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0035177289999999998 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.187195+0000", 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0055097999999999996 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:52:50.251 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.274322+0000", 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.087127048999999998 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.274360+0000", 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7824000000000002e-05 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.277330+0000", 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002970622 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.252 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:52:50.253 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:52:50.253 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:52:50.253 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:50.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:50.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:50.376 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:50.396 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:50.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:50.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:50 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3991087592' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:50.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:50 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3991087592' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:52:50.498 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:50.523 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:50.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:52:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:52:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:52:50.625 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:50.647 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:50.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:50.749 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:50.770 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:50.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:50.872 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:50.894 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:50.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:50.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:50.996 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:51.019 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:51.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:51.121 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:51.145 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:51.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:51.156 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:52:51.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:51.274 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:51.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:51.376 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:51.402 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:51.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:51.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:51 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:51.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:51 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3991087592' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:52:51.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:51 smithi130 ceph-mon[105521]: osdmap e1217: 8 total, 8 up, 6 in 2024-08-17T17:52:51.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18852/232866 objects degraded (8.096%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:51.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:51 smithi130 ceph-mon[105521]: pgmap v1765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 18852/232710 objects degraded (8.101%); 13411/232710 objects misplaced (5.763%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:52:51.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:51 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:51.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:51 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3991087592' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:52:51.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:51 smithi130 ceph-mon[110068]: osdmap e1217: 8 total, 8 up, 6 in 2024-08-17T17:52:51.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18852/232866 objects degraded (8.096%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:51.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:51 smithi130 ceph-mon[110068]: pgmap v1765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 18852/232710 objects degraded (8.101%); 13411/232710 objects misplaced (5.763%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:52:51.504 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:51.525 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:51.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:51.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:51 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:52:51.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:51 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3991087592' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:52:51.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:51 smithi179 ceph-mon[93519]: osdmap e1217: 8 total, 8 up, 6 in 2024-08-17T17:52:51.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18852/232866 objects degraded (8.096%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:51.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:51 smithi179 ceph-mon[93519]: pgmap v1765: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 18852/232710 objects degraded (8.101%); 13411/232710 objects misplaced (5.763%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:52:51.627 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:51.649 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:51.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:51.751 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:51.773 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:51.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:51.874 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:51.897 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:51.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:51.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:51.998 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:52.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.020 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:52.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:52.121 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:52.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.143 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:52.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:52.244 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:52.270 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:52.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:52.372 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:52.399 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:52.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:52.500 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:52.522 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:52.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:52.624 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:52.631 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:52.651 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:52.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:52.752 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:52.776 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:52.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:52.878 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:52.907 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:52.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:52.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:53.009 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:53.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.030 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:53.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:53.131 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:53.152 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:53 smithi130 ceph-mon[110068]: pgmap v1766: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 18780/232488 objects degraded (8.078%); 13373/232488 objects misplaced (5.752%); 723 KiB/s, 11 objects/s recovering 2024-08-17T17:52:53.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:53 smithi130 ceph-mon[105521]: pgmap v1766: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 18780/232488 objects degraded (8.078%); 13373/232488 objects misplaced (5.752%); 723 KiB/s, 11 objects/s recovering 2024-08-17T17:52:53.160 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:53.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:53.261 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:53.283 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:53.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:53.292 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:52:53.384 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:53.410 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:53.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:53.512 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:53.541 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:53.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:53.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:53 smithi179 ceph-mon[93519]: pgmap v1766: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 104 op/s; 18780/232488 objects degraded (8.078%); 13373/232488 objects misplaced (5.752%); 723 KiB/s, 11 objects/s recovering 2024-08-17T17:52:53.598 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:52:53.598 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:52:53.618 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:52:53.618 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:52:53.636 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:52:53.636 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:52:53.642 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:53.652 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:52:53.653 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:52:53.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.664 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:53.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:53.669 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:52:53.669 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:52:53.688 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:52:53.688 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:52:53.704 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:52:53.704 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:52:53.719 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:52:53.720 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:52:53.766 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:53.787 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:53.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:53.889 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:53.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:53.911 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:53.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:54.012 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:54.033 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:54.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:54.135 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:54.152 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:52:54.152 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 18780/232488 objects degraded (8.078%), 6 pgs degraded, 6 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1495,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1217,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":77496,"data_bytes":5077604325,"bytes_used":14830129152,"bytes_avail":561140084736,"bytes_total":575970213888,"degraded_objects":18780,"degraded_total":232488,"degraded_ratio":0.080778362754206676,"misplaced_objects":13373,"misplaced_total":232488,"misplaced_ratio":0.057521248408520007,"recovering_objects_per_sec":11,"recovering_bytes_per_sec":740356,"recovering_keys_per_sec":0,"num_objects_recovered":113,"num_bytes_recovered":7405568,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":104},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":25,"modified":"2024-08-17T17:49:39.646037+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (5m)\n [==========================..] (remaining: 23s)","progress":0.93258428573608398,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:52:54.168 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:54.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:54.270 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:54.292 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:54.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:54.394 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:54.416 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:54.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/324090737' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:54.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/324090737' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:54.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:54.519 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:54.541 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:54.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:54.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/324090737' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:52:54.643 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:54.666 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:54.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:54.768 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:54.789 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:54.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:54.891 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:54.917 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:54.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:54.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:55.020 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:55.041 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:55.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:55.143 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:55.164 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:55.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:55.266 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:55.287 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:55.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:55.389 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:55.410 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:55.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:55.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:55 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/586936952' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:52:55.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:55 smithi130 ceph-mon[105521]: pgmap v1767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 18780/230805 objects degraded (8.137%); 13299/230805 objects misplaced (5.762%); 960 KiB/s, 14 objects/s recovering 2024-08-17T17:52:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:55 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/586936952' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:52:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:55 smithi130 ceph-mon[110068]: pgmap v1767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 18780/230805 objects degraded (8.137%); 13299/230805 objects misplaced (5.762%); 960 KiB/s, 14 objects/s recovering 2024-08-17T17:52:55.512 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:55.533 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:55.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:55.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:55 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/586936952' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:52:55.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:55 smithi179 ceph-mon[93519]: pgmap v1767: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 18780/230805 objects degraded (8.137%); 13299/230805 objects misplaced (5.762%); 960 KiB/s, 14 objects/s recovering 2024-08-17T17:52:55.635 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:55.656 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:55.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:55.758 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:55.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.780 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:55.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:55.881 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:55.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:55.907 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:55.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:56.008 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:56.029 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:56.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:56.131 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:56.152 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:56.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:56.250 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:52:56.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:52:56.254 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:56.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.319 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:56.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:56.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18780/230805 objects degraded (8.137%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:56.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18780/230805 objects degraded (8.137%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:56.420 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:56.441 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:56.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:56.543 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:56.566 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:56.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:56.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18780/230805 objects degraded (8.137%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:52:56.652 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:52:56.668 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:56.717 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:56.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:56.765 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:56.767 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:52:56.767 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:52:56.819 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:56.841 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:56.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:56.942 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:56.964 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:56.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:56.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:56.994 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:57.065 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:57.086 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:57.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:57.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:57.209 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:57.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:57.310 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:57.334 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:57.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:57.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:57 smithi130 ceph-mon[105521]: pgmap v1768: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 18710/230598 objects degraded (8.114%); 13265/230598 objects misplaced (5.752%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:52:57.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:57 smithi130 ceph-mon[110068]: pgmap v1768: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 18710/230598 objects degraded (8.114%); 13265/230598 objects misplaced (5.752%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:52:57.436 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:57.457 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:57.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:57.480 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:52:57.559 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:57.581 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:57.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:57 smithi179 ceph-mon[93519]: pgmap v1768: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 18710/230598 objects degraded (8.114%); 13265/230598 objects misplaced (5.752%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:52:57.682 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:57.709 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:57.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:57.810 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:57.833 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:57.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:57.934 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:57.956 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:57.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:57.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:58.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:58.080 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:58.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:58.182 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:58.203 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:58.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:58.303 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:58.304 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:58.305 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:58.335 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:58.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:58.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:52:58.458 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:58.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:52:58.560 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:52:58.580 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:58.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:52:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:52:58.682 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:58.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.704 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:58.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:58.805 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:58.827 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:58.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:58.928 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:52:58.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:58.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:58.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:52:59.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:52:59.072 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:59.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:52:59.174 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:59.197 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:59.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:59.298 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:59.319 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:59.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:59.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:52:59 smithi130 ceph-mon[105521]: pgmap v1769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 18710/229443 objects degraded (8.155%); 13191/229443 objects misplaced (5.749%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:52:59.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:52:59 smithi130 ceph-mon[110068]: pgmap v1769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 18710/229443 objects degraded (8.155%); 13191/229443 objects misplaced (5.749%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:52:59.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:52:59.443 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:59.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:52:59.545 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:59.566 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:59.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:52:59.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:52:59 smithi179 ceph-mon[93519]: pgmap v1769: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 18710/229443 objects degraded (8.155%); 13191/229443 objects misplaced (5.749%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:52:59.668 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:52:59.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:59.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:52:59.795 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:52:59.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.817 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:52:59.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:52:59.820 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:52:59.918 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:52:59.939 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:52:59.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:52:59.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:00.041 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:00.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.063 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:00.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:00.164 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:00.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:52:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:52:59] "GET /metrics HTTP/1.1" 200 36812 "" "Prometheus/2.51.0" 2024-08-17T17:53:00.186 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:00.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:00.287 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:00.308 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:00.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:00.410 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:00.432 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:00.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:00.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:00.557 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:00.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:53:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:53:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:53:00.659 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:00.687 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:00.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:00.789 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:00.810 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:00.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:00.912 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:00.946 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:00.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:00.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:01.048 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:01.069 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:01.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:01.075 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:00 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18710/229443 objects degraded (8.155%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:01.075 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:00 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18710/229443 objects degraded (8.155%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:01.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:00 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18710/229443 objects degraded (8.155%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:01.171 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:01.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.193 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:01.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:01.293 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:01.322 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:01.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:01.424 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:01.445 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:01.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:01.546 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:01.574 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:01.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:01.675 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:01.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.700 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:01.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:01.750 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T17:53:01.801 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:01.805 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:53:01.806 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:53:01.826 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:01.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:01.928 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:01.949 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:01.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:01.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:02.024 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:01 smithi130 ceph-mon[105521]: pgmap v1770: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 18710/229221 objects degraded (8.162%); 13191/229221 objects misplaced (5.755%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:53:02.025 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:01 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3767790403' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:02.025 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:01 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:02.025 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:01 smithi130 ceph-mon[110068]: pgmap v1770: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 18710/229221 objects degraded (8.162%); 13191/229221 objects misplaced (5.755%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:53:02.025 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:01 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3767790403' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:02.025 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:01 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:02.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:02.072 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:02.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:02.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:01 smithi179 ceph-mon[93519]: pgmap v1770: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 18710/229221 objects degraded (8.162%); 13191/229221 objects misplaced (5.755%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:53:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:01 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3767790403' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:01 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:02.173 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:02.200 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:02.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:02.302 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:02.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.324 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:02.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:02.425 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:02.435 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:02.455 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:02.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:02.557 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:02.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.581 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:02.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:02.682 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:02.705 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:02.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:02.807 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:02.830 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:02.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:02.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:02 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:53:02.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:02 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:02.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:02 smithi130 ceph-mon[105521]: osdmap e1218: 8 total, 8 up, 6 in 2024-08-17T17:53:02.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:02 smithi130 ceph-mon[105521]: from='client.55165 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:02.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:02 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:53:02.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:02 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:02.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:02 smithi130 ceph-mon[110068]: osdmap e1218: 8 total, 8 up, 6 in 2024-08-17T17:53:02.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:02 smithi130 ceph-mon[110068]: from='client.55165 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:02.932 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:02.960 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:02.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:02.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:03.062 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:03.084 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:03.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:03.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:02 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:53:03.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:02 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:03.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:02 smithi179 ceph-mon[93519]: osdmap e1218: 8 total, 8 up, 6 in 2024-08-17T17:53:03.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:02 smithi179 ceph-mon[93519]: from='client.55165 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:03.185 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:03.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:03.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:03.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:03.336 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:03.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:03.438 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:03.459 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:03.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:03.561 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:03.582 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:03.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:03.684 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:03.705 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:03.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:03.807 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:03.828 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:03.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:03 smithi130 ceph-mon[105521]: pgmap v1772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 18636/229032 objects degraded (8.137%); 13165/229032 objects misplaced (5.748%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:53:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:03 smithi130 ceph-mon[110068]: pgmap v1772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 18636/229032 objects degraded (8.137%); 13165/229032 objects misplaced (5.748%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:53:03.930 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:03.953 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:03.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:03.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:04.055 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:04.079 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:04.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:04.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:03 smithi179 ceph-mon[93519]: pgmap v1772: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 18636/229032 objects degraded (8.137%); 13165/229032 objects misplaced (5.748%); 934 KiB/s, 14 objects/s recovering 2024-08-17T17:53:04.170 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1771,"stamp":"2024-08-17T17:53:01.739196+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5006235621,"num_objects":76407,"num_object_clones":0,"num_object_copies":229221,"num_objects_missing_on_primary":4261,"num_objects_missing":4261,"num_objects_degraded":18710,"num_objects_misplaced":13191,"num_objects_unfound":0,"num_objects_dirty":76407,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":133507,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33594,"num_bytes_recovered":2023037965,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107110,"ondisk_log_size":107110,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14343940,"kb_used_data":12947924,"kb_used_omap":295,"kb_used_meta":1395672,"kb_avail":548126972,"statfs":{"total":575970213888,"available":561282019328,"internally_reserved":0,"allocated":13258674176,"data_stored":13229621961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1429168591},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-76218368,"num_objects":-1163,"num_object_clones":0,"num_object_copies":-3489,"num_objects_missing_on_primary":-74,"num_objects_missing":-74,"num_objects_degraded":-142,"num_objects_misplaced":-220,"num_objects_unfound":0,"num_objects_dirty":-1163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1163,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":145,"num_bytes_recovered":9502720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.710162"},"pg_stats":[{"pgid":"45.f","version":"1217'8334","reported_seq":18847,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:57.843328+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:52:57.843328+0000","last_peered":"2024-08-17T17:52:57.843328+0000","last_clean":"2024-08-17T17:52:57.843328+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:52:57.843328+0000","last_undegraded":"2024-08-17T17:52:57.843328+0000","last_fullsized":"2024-08-17T17:52:57.843328+0000","mapping_epoch":1194,"log_start":"1170'1431","ondisk_log_start":"1170'1431","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6903,"log_dups_size":0,"ondisk_log_size":6903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316538880,"num_objects":4830,"num_object_clones":0,"num_object_copies":14490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8334,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1217'8267","reported_seq":18724,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:57.933313+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:52:57.933313+0000","last_peered":"2024-08-17T17:52:57.933313+0000","last_clean":"2024-08-17T17:52:57.933313+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:52:57.933313+0000","last_undegraded":"2024-08-17T17:52:57.933313+0000","last_fullsized":"2024-08-17T17:52:57.933313+0000","mapping_epoch":1211,"log_start":"1170'1232","ondisk_log_start":"1170'1232","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7035,"log_dups_size":0,"ondisk_log_size":7035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309788672,"num_objects":4727,"num_object_clones":0,"num_object_copies":14181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8267,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1217'8540","reported_seq":19360,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:58.098601+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:52:58.098601+0000","last_peered":"2024-08-17T17:52:58.098601+0000","last_clean":"2024-08-17T17:52:58.098601+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:52:58.098601+0000","last_undegraded":"2024-08-17T17:52:58.098601+0000","last_fullsized":"2024-08-17T17:52:58.098601+0000","mapping_epoch":1205,"log_start":"1170'2568","ondisk_log_start":"1170'2568","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5972,"log_dups_size":0,"ondisk_log_size":5972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316997656,"num_objects":4838,"num_object_clones":0,"num_object_copies":14514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4838,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8540,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1217'8280","reported_seq":8338,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:57.819577+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:52:57.819577+0000","last_peered":"2024-08-17T17:52:57.819577+0000","last_clean":"2024-08-17T17:52:57.819577+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:52:57.819577+0000","last_undegraded":"2024-08-17T17:52:57.819577+0000","last_fullsized":"2024-08-17T17:52:57.819577+0000","mapping_epoch":1167,"log_start":"1170'1589","ondisk_log_start":"1170'1589","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6691,"log_dups_size":0,"ondisk_log_size":6691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309329920,"num_objects":4720,"num_object_clones":0,"num_object_copies":14160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8280,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1217'8362","reported_seq":13588,"reported_epoch":1217,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:56.357153+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:52:56.357153+0000","last_peered":"2024-08-17T17:52:56.357153+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:52:56.357153+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1170'2426","ondisk_log_start":"1170'2426","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5936,"log_dups_size":0,"ondisk_log_size":5936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311427072,"num_objects":4752,"num_object_clones":0,"num_object_copies":14256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":4752,"num_objects_unfound":0,"num_objects_dirty":4752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8362,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":4234},{"shards":"1,4,5","objects":518}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1217'8400","reported_seq":8458,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:57.835798+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:52:57.835798+0000","last_peered":"2024-08-17T17:52:57.835798+0000","last_clean":"2024-08-17T17:52:57.835798+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:52:57.835798+0000","last_undegraded":"2024-08-17T17:52:57.835798+0000","last_fullsized":"2024-08-17T17:52:57.835798+0000","mapping_epoch":1167,"log_start":"1170'1677","ondisk_log_start":"1170'1677","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6723,"log_dups_size":0,"ondisk_log_size":6723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307625984,"num_objects":4694,"num_object_clones":0,"num_object_copies":14082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8400,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1217'8399","reported_seq":8457,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:58.102010+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:52:58.102010+0000","last_peered":"2024-08-17T17:52:58.102010+0000","last_clean":"2024-08-17T17:52:58.102010+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:52:58.102010+0000","last_undegraded":"2024-08-17T17:52:58.102010+0000","last_fullsized":"2024-08-17T17:52:58.102010+0000","mapping_epoch":1167,"log_start":"1170'1669","ondisk_log_start":"1170'1669","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6730,"log_dups_size":0,"ondisk_log_size":6730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316342272,"num_objects":4827,"num_object_clones":0,"num_object_copies":14481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8399,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1217'8299","reported_seq":13830,"reported_epoch":1217,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:56.516561+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:52:56.516561+0000","last_peered":"2024-08-17T17:52:56.516561+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:52:56.516561+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1170'2375","ondisk_log_start":"1170'2375","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5924,"log_dups_size":0,"ondisk_log_size":5924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307429376,"num_objects":4691,"num_object_clones":0,"num_object_copies":14073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4675,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8299,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1804,"num_bytes_recovered":118226944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1304},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1217'8266","reported_seq":8334,"reported_epoch":1217,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:57.839591+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:52:57.839591+0000","last_peered":"2024-08-17T17:52:57.839591+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:52:57.839591+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'1577","ondisk_log_start":"1170'1577","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6689,"log_dups_size":0,"ondisk_log_size":6689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315228160,"num_objects":4810,"num_object_clones":0,"num_object_copies":14430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8266,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":4810},{"shards":"1,6","objects":-1023}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1217'8311","reported_seq":11370,"reported_epoch":1217,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:58.340103+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:52:58.340103+0000","last_peered":"2024-08-17T17:52:58.340103+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:52:58.340103+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1170'2406","ondisk_log_start":"1170'2406","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5905,"log_dups_size":0,"ondisk_log_size":5905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318701568,"num_objects":4863,"num_object_clones":0,"num_object_copies":14589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8311,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":3302},{"shards":"1,3,5","objects":1304}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1217'8316","reported_seq":8431,"reported_epoch":1217,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:59.098911+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:52:59.098911+0000","last_peered":"2024-08-17T17:52:59.098911+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:52:59.098911+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8316,"log_dups_size":0,"ondisk_log_size":8316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311820288,"num_objects":4758,"num_object_clones":0,"num_object_copies":14274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8316,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-395}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1217'8277","reported_seq":14452,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:58.095138+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:52:58.095138+0000","last_peered":"2024-08-17T17:52:58.095138+0000","last_clean":"2024-08-17T17:52:58.095138+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:52:58.095138+0000","last_undegraded":"2024-08-17T17:52:58.095138+0000","last_fullsized":"2024-08-17T17:52:58.095138+0000","mapping_epoch":1183,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8277,"log_dups_size":0,"ondisk_log_size":8277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312541184,"num_objects":4769,"num_object_clones":0,"num_object_copies":14307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8277,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1217'8339","reported_seq":8452,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:58.463658+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:52:58.463658+0000","last_peered":"2024-08-17T17:52:58.463658+0000","last_clean":"2024-08-17T17:52:58.463658+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:52:58.463658+0000","last_undegraded":"2024-08-17T17:52:58.463658+0000","last_fullsized":"2024-08-17T17:52:58.463658+0000","mapping_epoch":1167,"log_start":"1170'2391","ondisk_log_start":"1170'2391","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5948,"log_dups_size":0,"ondisk_log_size":5948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315949056,"num_objects":4821,"num_object_clones":0,"num_object_copies":14463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8339,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1217'8230","reported_seq":8288,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:57.483181+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:52:57.483181+0000","last_peered":"2024-08-17T17:52:57.483181+0000","last_clean":"2024-08-17T17:52:57.483181+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:52:57.483181+0000","last_undegraded":"2024-08-17T17:52:57.483181+0000","last_fullsized":"2024-08-17T17:52:57.483181+0000","mapping_epoch":1167,"log_start":"1170'1225","ondisk_log_start":"1170'1225","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7005,"log_dups_size":0,"ondisk_log_size":7005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308281344,"num_objects":4704,"num_object_clones":0,"num_object_copies":14112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8230,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1217'8459","reported_seq":17406,"reported_epoch":1217,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:52:58.079615+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:52:58.079615+0000","last_peered":"2024-08-17T17:52:58.079615+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:52:58.079615+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'1591","ondisk_log_start":"1170'1591","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6868,"log_dups_size":0,"ondisk_log_size":6868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321454080,"num_objects":4905,"num_object_clones":0,"num_object_copies":14715,"num_objects_missing_on_primary":4261,"num_objects_missing":4261,"num_objects_degraded":265,"num_objects_misplaced":8439,"num_objects_unfound":0,"num_objects_dirty":4905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8459,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3883,"num_bytes_recovered":254476288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":15},{"shards":"0,4,6","objects":1356},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":3269}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1217'8150","reported_seq":21449,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:57.729242+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:52:57.729242+0000","last_peered":"2024-08-17T17:52:57.729242+0000","last_clean":"2024-08-17T17:52:57.729242+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:52:57.729242+0000","last_undegraded":"2024-08-17T17:52:57.729242+0000","last_fullsized":"2024-08-17T17:52:57.729242+0000","mapping_epoch":1196,"log_start":"1170'2169","ondisk_log_start":"1170'2169","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5981,"log_dups_size":0,"ondisk_log_size":5981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306184192,"num_objects":4672,"num_object_clones":0,"num_object_copies":14016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8150,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1890,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.198227+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:52:50.198227+0000","last_peered":"2024-08-17T17:52:50.198227+0000","last_clean":"2024-08-17T17:52:50.198227+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:52:50.198227+0000","last_undegraded":"2024-08-17T17:52:50.198227+0000","last_fullsized":"2024-08-17T17:52:50.198227+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":494,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.968332+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:52:41.968332+0000","last_peered":"2024-08-17T17:52:41.968332+0000","last_clean":"2024-08-17T17:52:41.968332+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:52:41.968332+0000","last_undegraded":"2024-08-17T17:52:41.968332+0000","last_fullsized":"2024-08-17T17:52:41.968332+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1809,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.122766+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:52:50.122766+0000","last_peered":"2024-08-17T17:52:50.122766+0000","last_clean":"2024-08-17T17:52:50.122766+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:52:50.122766+0000","last_undegraded":"2024-08-17T17:52:50.122766+0000","last_fullsized":"2024-08-17T17:52:50.122766+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1833,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.121586+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:52:50.121586+0000","last_peered":"2024-08-17T17:52:50.121586+0000","last_clean":"2024-08-17T17:52:50.121586+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:52:50.121586+0000","last_undegraded":"2024-08-17T17:52:50.121586+0000","last_fullsized":"2024-08-17T17:52:50.121586+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":56,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.968914+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:52:41.968914+0000","last_peered":"2024-08-17T17:52:41.968914+0000","last_clean":"2024-08-17T17:52:41.968914+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:52:41.968914+0000","last_undegraded":"2024-08-17T17:52:41.968914+0000","last_fullsized":"2024-08-17T17:52:41.968914+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1227,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.121885+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:52:50.121885+0000","last_peered":"2024-08-17T17:52:50.121885+0000","last_clean":"2024-08-17T17:52:50.121885+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:52:50.121885+0000","last_undegraded":"2024-08-17T17:52:50.121885+0000","last_fullsized":"2024-08-17T17:52:50.121885+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":85,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.198171+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:52:50.198171+0000","last_peered":"2024-08-17T17:52:50.198171+0000","last_clean":"2024-08-17T17:52:50.198171+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:52:50.198171+0000","last_undegraded":"2024-08-17T17:52:50.198171+0000","last_fullsized":"2024-08-17T17:52:50.198171+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":605,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.122660+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:52:50.122660+0000","last_peered":"2024-08-17T17:52:50.122660+0000","last_clean":"2024-08-17T17:52:50.122660+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:52:50.122660+0000","last_undegraded":"2024-08-17T17:52:50.122660+0000","last_fullsized":"2024-08-17T17:52:50.122660+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1848,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.197664+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:52:50.197664+0000","last_peered":"2024-08-17T17:52:50.197664+0000","last_clean":"2024-08-17T17:52:50.197664+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:52:50.197664+0000","last_undegraded":"2024-08-17T17:52:50.197664+0000","last_fullsized":"2024-08-17T17:52:50.197664+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1850,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.206540+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:52:50.206540+0000","last_peered":"2024-08-17T17:52:50.206540+0000","last_clean":"2024-08-17T17:52:50.206540+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:52:50.206540+0000","last_undegraded":"2024-08-17T17:52:50.206540+0000","last_fullsized":"2024-08-17T17:52:50.206540+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":70,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.197052+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:52:50.197052+0000","last_peered":"2024-08-17T17:52:50.197052+0000","last_clean":"2024-08-17T17:52:50.197052+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:52:50.197052+0000","last_undegraded":"2024-08-17T17:52:50.197052+0000","last_fullsized":"2024-08-17T17:52:50.197052+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1822,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.122870+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:52:50.122870+0000","last_peered":"2024-08-17T17:52:50.122870+0000","last_clean":"2024-08-17T17:52:50.122870+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:52:50.122870+0000","last_undegraded":"2024-08-17T17:52:50.122870+0000","last_fullsized":"2024-08-17T17:52:50.122870+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1877,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.196467+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:52:50.196467+0000","last_peered":"2024-08-17T17:52:50.196467+0000","last_clean":"2024-08-17T17:52:50.196467+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:52:50.196467+0000","last_undegraded":"2024-08-17T17:52:50.196467+0000","last_fullsized":"2024-08-17T17:52:50.196467+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1871,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.198312+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:52:50.198312+0000","last_peered":"2024-08-17T17:52:50.198312+0000","last_clean":"2024-08-17T17:52:50.198312+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:52:50.198312+0000","last_undegraded":"2024-08-17T17:52:50.198312+0000","last_fullsized":"2024-08-17T17:52:50.198312+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1844,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.121728+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:52:50.121728+0000","last_peered":"2024-08-17T17:52:50.121728+0000","last_clean":"2024-08-17T17:52:50.121728+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:52:50.121728+0000","last_undegraded":"2024-08-17T17:52:50.121728+0000","last_fullsized":"2024-08-17T17:52:50.121728+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1191,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.968984+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:52:41.968984+0000","last_peered":"2024-08-17T17:52:41.968984+0000","last_clean":"2024-08-17T17:52:41.968984+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:52:41.968984+0000","last_undegraded":"2024-08-17T17:52:41.968984+0000","last_fullsized":"2024-08-17T17:52:41.968984+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1816,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.121495+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:52:50.121495+0000","last_peered":"2024-08-17T17:52:50.121495+0000","last_clean":"2024-08-17T17:52:50.121495+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:52:50.121495+0000","last_undegraded":"2024-08-17T17:52:50.121495+0000","last_fullsized":"2024-08-17T17:52:50.121495+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1865,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.198315+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:52:50.198315+0000","last_peered":"2024-08-17T17:52:50.198315+0000","last_clean":"2024-08-17T17:52:50.198315+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:52:50.198315+0000","last_undegraded":"2024-08-17T17:52:50.198315+0000","last_fullsized":"2024-08-17T17:52:50.198315+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1772,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.121494+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:52:50.121494+0000","last_peered":"2024-08-17T17:52:50.121494+0000","last_clean":"2024-08-17T17:52:50.121494+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:52:50.121494+0000","last_undegraded":"2024-08-17T17:52:50.121494+0000","last_fullsized":"2024-08-17T17:52:50.121494+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":510,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.965109+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:52:41.965109+0000","last_peered":"2024-08-17T17:52:41.965109+0000","last_clean":"2024-08-17T17:52:41.965109+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:52:41.965109+0000","last_undegraded":"2024-08-17T17:52:41.965109+0000","last_fullsized":"2024-08-17T17:52:41.965109+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1187,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.964987+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:52:41.964987+0000","last_peered":"2024-08-17T17:52:41.964987+0000","last_clean":"2024-08-17T17:52:41.964987+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:52:41.964987+0000","last_undegraded":"2024-08-17T17:52:41.964987+0000","last_fullsized":"2024-08-17T17:52:41.964987+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1870,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.197275+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:52:50.197275+0000","last_peered":"2024-08-17T17:52:50.197275+0000","last_clean":"2024-08-17T17:52:50.197275+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:52:50.197275+0000","last_undegraded":"2024-08-17T17:52:50.197275+0000","last_fullsized":"2024-08-17T17:52:50.197275+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":348,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.964917+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:52:41.964917+0000","last_peered":"2024-08-17T17:52:41.964917+0000","last_clean":"2024-08-17T17:52:41.964917+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:52:41.964917+0000","last_undegraded":"2024-08-17T17:52:41.964917+0000","last_fullsized":"2024-08-17T17:52:41.964917+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":494,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.968598+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:52:41.968598+0000","last_peered":"2024-08-17T17:52:41.968598+0000","last_clean":"2024-08-17T17:52:41.968598+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:52:41.968598+0000","last_undegraded":"2024-08-17T17:52:41.968598+0000","last_fullsized":"2024-08-17T17:52:41.968598+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1172,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.968674+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:52:41.968674+0000","last_peered":"2024-08-17T17:52:41.968674+0000","last_clean":"2024-08-17T17:52:41.968674+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:52:41.968674+0000","last_undegraded":"2024-08-17T17:52:41.968674+0000","last_fullsized":"2024-08-17T17:52:41.968674+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1180,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.969054+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:52:41.969054+0000","last_peered":"2024-08-17T17:52:41.969054+0000","last_clean":"2024-08-17T17:52:41.969054+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:52:41.969054+0000","last_undegraded":"2024-08-17T17:52:41.969054+0000","last_fullsized":"2024-08-17T17:52:41.969054+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1918,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.198134+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:52:50.198134+0000","last_peered":"2024-08-17T17:52:50.198134+0000","last_clean":"2024-08-17T17:52:50.198134+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:52:50.198134+0000","last_undegraded":"2024-08-17T17:52:50.198134+0000","last_fullsized":"2024-08-17T17:52:50.198134+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1801,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.121647+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:52:50.121647+0000","last_peered":"2024-08-17T17:52:50.121647+0000","last_clean":"2024-08-17T17:52:50.121647+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:52:50.121647+0000","last_undegraded":"2024-08-17T17:52:50.121647+0000","last_fullsized":"2024-08-17T17:52:50.121647+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1876,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.197587+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:52:50.197587+0000","last_peered":"2024-08-17T17:52:50.197587+0000","last_clean":"2024-08-17T17:52:50.197587+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:52:50.197587+0000","last_undegraded":"2024-08-17T17:52:50.197587+0000","last_fullsized":"2024-08-17T17:52:50.197587+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1803,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.122533+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:52:50.122533+0000","last_peered":"2024-08-17T17:52:50.122533+0000","last_clean":"2024-08-17T17:52:50.122533+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:52:50.122533+0000","last_undegraded":"2024-08-17T17:52:50.122533+0000","last_fullsized":"2024-08-17T17:52:50.122533+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1836,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.197758+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:52:50.197758+0000","last_peered":"2024-08-17T17:52:50.197758+0000","last_clean":"2024-08-17T17:52:50.197758+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:52:50.197758+0000","last_undegraded":"2024-08-17T17:52:50.197758+0000","last_fullsized":"2024-08-17T17:52:50.197758+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1207,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.965066+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:52:41.965066+0000","last_peered":"2024-08-17T17:52:41.965066+0000","last_clean":"2024-08-17T17:52:41.965066+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:52:41.965066+0000","last_undegraded":"2024-08-17T17:52:41.965066+0000","last_fullsized":"2024-08-17T17:52:41.965066+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1882,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.198333+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:52:50.198333+0000","last_peered":"2024-08-17T17:52:50.198333+0000","last_clean":"2024-08-17T17:52:50.198333+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:52:50.198333+0000","last_undegraded":"2024-08-17T17:52:50.198333+0000","last_fullsized":"2024-08-17T17:52:50.198333+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":69,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.196473+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:52:50.196473+0000","last_peered":"2024-08-17T17:52:50.196473+0000","last_clean":"2024-08-17T17:52:50.196473+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:52:50.196473+0000","last_undegraded":"2024-08-17T17:52:50.196473+0000","last_fullsized":"2024-08-17T17:52:50.196473+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1901,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.198257+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:52:50.198257+0000","last_peered":"2024-08-17T17:52:50.198257+0000","last_clean":"2024-08-17T17:52:50.198257+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:52:50.198257+0000","last_undegraded":"2024-08-17T17:52:50.198257+0000","last_fullsized":"2024-08-17T17:52:50.198257+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":638,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.122704+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:52:50.122704+0000","last_peered":"2024-08-17T17:52:50.122704+0000","last_clean":"2024-08-17T17:52:50.122704+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:52:50.122704+0000","last_undegraded":"2024-08-17T17:52:50.122704+0000","last_fullsized":"2024-08-17T17:52:50.122704+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1882,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.198420+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:52:50.198420+0000","last_peered":"2024-08-17T17:52:50.198420+0000","last_clean":"2024-08-17T17:52:50.198420+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:52:50.198420+0000","last_undegraded":"2024-08-17T17:52:50.198420+0000","last_fullsized":"2024-08-17T17:52:50.198420+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1211,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.197507+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:52:50.197507+0000","last_peered":"2024-08-17T17:52:50.197507+0000","last_clean":"2024-08-17T17:52:50.197507+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:52:50.197507+0000","last_undegraded":"2024-08-17T17:52:50.197507+0000","last_fullsized":"2024-08-17T17:52:50.197507+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":129,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.197192+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:52:50.197192+0000","last_peered":"2024-08-17T17:52:50.197192+0000","last_clean":"2024-08-17T17:52:50.197192+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:52:50.197192+0000","last_undegraded":"2024-08-17T17:52:50.197192+0000","last_fullsized":"2024-08-17T17:52:50.197192+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1893,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.198711+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:52:50.198711+0000","last_peered":"2024-08-17T17:52:50.198711+0000","last_clean":"2024-08-17T17:52:50.198711+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:52:50.198711+0000","last_undegraded":"2024-08-17T17:52:50.198711+0000","last_fullsized":"2024-08-17T17:52:50.198711+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1871,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.196681+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:52:50.196681+0000","last_peered":"2024-08-17T17:52:50.196681+0000","last_clean":"2024-08-17T17:52:50.196681+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:52:50.196681+0000","last_undegraded":"2024-08-17T17:52:50.196681+0000","last_fullsized":"2024-08-17T17:52:50.196681+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1891,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.206334+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:52:50.206334+0000","last_peered":"2024-08-17T17:52:50.206334+0000","last_clean":"2024-08-17T17:52:50.206334+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:52:50.206334+0000","last_undegraded":"2024-08-17T17:52:50.206334+0000","last_fullsized":"2024-08-17T17:52:50.206334+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1897,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.198425+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:52:50.198425+0000","last_peered":"2024-08-17T17:52:50.198425+0000","last_clean":"2024-08-17T17:52:50.198425+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:52:50.198425+0000","last_undegraded":"2024-08-17T17:52:50.198425+0000","last_fullsized":"2024-08-17T17:52:50.198425+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1877,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.299568+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:52:50.299568+0000","last_peered":"2024-08-17T17:52:50.299568+0000","last_clean":"2024-08-17T17:52:50.299568+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:52:50.299568+0000","last_undegraded":"2024-08-17T17:52:50.299568+0000","last_fullsized":"2024-08-17T17:52:50.299568+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1214,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.964951+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:52:41.964951+0000","last_peered":"2024-08-17T17:52:41.964951+0000","last_clean":"2024-08-17T17:52:41.964951+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:52:41.964951+0000","last_undegraded":"2024-08-17T17:52:41.964951+0000","last_fullsized":"2024-08-17T17:52:41.964951+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1160,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.968942+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:52:41.968942+0000","last_peered":"2024-08-17T17:52:41.968942+0000","last_clean":"2024-08-17T17:52:41.968942+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:52:41.968942+0000","last_undegraded":"2024-08-17T17:52:41.968942+0000","last_fullsized":"2024-08-17T17:52:41.968942+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1786,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.121778+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:52:50.121778+0000","last_peered":"2024-08-17T17:52:50.121778+0000","last_clean":"2024-08-17T17:52:50.121778+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:52:50.121778+0000","last_undegraded":"2024-08-17T17:52:50.121778+0000","last_fullsized":"2024-08-17T17:52:50.121778+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":117,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.207839+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:52:50.207839+0000","last_peered":"2024-08-17T17:52:50.207839+0000","last_clean":"2024-08-17T17:52:50.207839+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:52:50.207839+0000","last_undegraded":"2024-08-17T17:52:50.207839+0000","last_fullsized":"2024-08-17T17:52:50.207839+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1526,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.964725+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:52:41.964725+0000","last_peered":"2024-08-17T17:52:41.964725+0000","last_clean":"2024-08-17T17:52:41.964725+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:52:41.964725+0000","last_undegraded":"2024-08-17T17:52:41.964725+0000","last_fullsized":"2024-08-17T17:52:41.964725+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1877,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.198785+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:52:50.198785+0000","last_peered":"2024-08-17T17:52:50.198785+0000","last_clean":"2024-08-17T17:52:50.198785+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:52:50.198785+0000","last_undegraded":"2024-08-17T17:52:50.198785+0000","last_fullsized":"2024-08-17T17:52:50.198785+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1817,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.121552+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:52:50.121552+0000","last_peered":"2024-08-17T17:52:50.121552+0000","last_clean":"2024-08-17T17:52:50.121552+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:52:50.121552+0000","last_undegraded":"2024-08-17T17:52:50.121552+0000","last_fullsized":"2024-08-17T17:52:50.121552+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":494,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.968815+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:52:41.968815+0000","last_peered":"2024-08-17T17:52:41.968815+0000","last_clean":"2024-08-17T17:52:41.968815+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:52:41.968815+0000","last_undegraded":"2024-08-17T17:52:41.968815+0000","last_fullsized":"2024-08-17T17:52:41.968815+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":556,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.206268+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:52:50.206268+0000","last_peered":"2024-08-17T17:52:50.206268+0000","last_clean":"2024-08-17T17:52:50.206268+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:52:50.206268+0000","last_undegraded":"2024-08-17T17:52:50.206268+0000","last_fullsized":"2024-08-17T17:52:50.206268+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":572,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.206626+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:52:50.206626+0000","last_peered":"2024-08-17T17:52:50.206626+0000","last_clean":"2024-08-17T17:52:50.206626+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:52:50.206626+0000","last_undegraded":"2024-08-17T17:52:50.206626+0000","last_fullsized":"2024-08-17T17:52:50.206626+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1178,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.968788+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:52:41.968788+0000","last_peered":"2024-08-17T17:52:41.968788+0000","last_clean":"2024-08-17T17:52:41.968788+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:52:41.968788+0000","last_undegraded":"2024-08-17T17:52:41.968788+0000","last_fullsized":"2024-08-17T17:52:41.968788+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":588,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.122245+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:52:50.122245+0000","last_peered":"2024-08-17T17:52:50.122245+0000","last_clean":"2024-08-17T17:52:50.122245+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:52:50.122245+0000","last_undegraded":"2024-08-17T17:52:50.122245+0000","last_fullsized":"2024-08-17T17:52:50.122245+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1814,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.122743+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:52:50.122743+0000","last_peered":"2024-08-17T17:52:50.122743+0000","last_clean":"2024-08-17T17:52:50.122743+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:52:50.122743+0000","last_undegraded":"2024-08-17T17:52:50.122743+0000","last_fullsized":"2024-08-17T17:52:50.122743+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1909,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.197849+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:52:50.197849+0000","last_peered":"2024-08-17T17:52:50.197849+0000","last_clean":"2024-08-17T17:52:50.197849+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:52:50.197849+0000","last_undegraded":"2024-08-17T17:52:50.197849+0000","last_fullsized":"2024-08-17T17:52:50.197849+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":621,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.122146+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:52:50.122146+0000","last_peered":"2024-08-17T17:52:50.122146+0000","last_clean":"2024-08-17T17:52:50.122146+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:52:50.122146+0000","last_undegraded":"2024-08-17T17:52:50.122146+0000","last_fullsized":"2024-08-17T17:52:50.122146+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1167,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.968527+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:52:41.968527+0000","last_peered":"2024-08-17T17:52:41.968527+0000","last_clean":"2024-08-17T17:52:41.968527+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:52:41.968527+0000","last_undegraded":"2024-08-17T17:52:41.968527+0000","last_fullsized":"2024-08-17T17:52:41.968527+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1166,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.965069+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:52:41.965069+0000","last_peered":"2024-08-17T17:52:41.965069+0000","last_clean":"2024-08-17T17:52:41.965069+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:52:41.965069+0000","last_undegraded":"2024-08-17T17:52:41.965069+0000","last_fullsized":"2024-08-17T17:52:41.965069+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":605,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.196567+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:52:50.196567+0000","last_peered":"2024-08-17T17:52:50.196567+0000","last_clean":"2024-08-17T17:52:50.196567+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:52:50.196567+0000","last_undegraded":"2024-08-17T17:52:50.196567+0000","last_fullsized":"2024-08-17T17:52:50.196567+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1184,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.965213+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:52:41.965213+0000","last_peered":"2024-08-17T17:52:41.965213+0000","last_clean":"2024-08-17T17:52:41.965213+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:52:41.965213+0000","last_undegraded":"2024-08-17T17:52:41.965213+0000","last_fullsized":"2024-08-17T17:52:41.965213+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":68,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.964636+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:52:41.964636+0000","last_peered":"2024-08-17T17:52:41.964636+0000","last_clean":"2024-08-17T17:52:41.964636+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:52:41.964636+0000","last_undegraded":"2024-08-17T17:52:41.964636+0000","last_fullsized":"2024-08-17T17:52:41.964636+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1200,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.965165+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:52:41.965165+0000","last_peered":"2024-08-17T17:52:41.965165+0000","last_clean":"2024-08-17T17:52:41.965165+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:52:41.965165+0000","last_undegraded":"2024-08-17T17:52:41.965165+0000","last_fullsized":"2024-08-17T17:52:41.965165+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1217,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.968484+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:52:41.968484+0000","last_peered":"2024-08-17T17:52:41.968484+0000","last_clean":"2024-08-17T17:52:41.968484+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:52:41.968484+0000","last_undegraded":"2024-08-17T17:52:41.968484+0000","last_fullsized":"2024-08-17T17:52:41.968484+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1815,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.122393+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:52:50.122393+0000","last_peered":"2024-08-17T17:52:50.122393+0000","last_clean":"2024-08-17T17:52:50.122393+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:52:50.122393+0000","last_undegraded":"2024-08-17T17:52:50.122393+0000","last_fullsized":"2024-08-17T17:52:50.122393+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1161,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.964798+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:52:41.964798+0000","last_peered":"2024-08-17T17:52:41.964798+0000","last_clean":"2024-08-17T17:52:41.964798+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:52:41.964798+0000","last_undegraded":"2024-08-17T17:52:41.964798+0000","last_fullsized":"2024-08-17T17:52:41.964798+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1868,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.198181+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:52:50.198181+0000","last_peered":"2024-08-17T17:52:50.198181+0000","last_clean":"2024-08-17T17:52:50.198181+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:52:50.198181+0000","last_undegraded":"2024-08-17T17:52:50.198181+0000","last_fullsized":"2024-08-17T17:52:50.198181+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1869,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.198579+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:52:50.198579+0000","last_peered":"2024-08-17T17:52:50.198579+0000","last_clean":"2024-08-17T17:52:50.198579+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:52:50.198579+0000","last_undegraded":"2024-08-17T17:52:50.198579+0000","last_fullsized":"2024-08-17T17:52:50.198579+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":574,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.964869+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:52:41.964869+0000","last_peered":"2024-08-17T17:52:41.964869+0000","last_clean":"2024-08-17T17:52:41.964869+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:52:41.964869+0000","last_undegraded":"2024-08-17T17:52:41.964869+0000","last_fullsized":"2024-08-17T17:52:41.964869+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1186,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:52:41.965987+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:52:41.965987+0000","last_peered":"2024-08-17T17:52:41.965987+0000","last_clean":"2024-08-17T17:52:41.965987+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:52:41.965987+0000","last_undegraded":"2024-08-17T17:52:41.965987+0000","last_fullsized":"2024-08-17T17:52:41.965987+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1816,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.122374+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:52:50.122374+0000","last_peered":"2024-08-17T17:52:50.122374+0000","last_clean":"2024-08-17T17:52:50.122374+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:52:50.122374+0000","last_undegraded":"2024-08-17T17:52:50.122374+0000","last_fullsized":"2024-08-17T17:52:50.122374+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":5005639704,"num_objects":76381,"num_object_clones":0,"num_object_copies":229143,"num_objects_missing_on_primary":4261,"num_objects_missing":4261,"num_objects_degraded":18710,"num_objects_misplaced":13191,"num_objects_unfound":0,"num_objects_dirty":76381,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":133229,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33565,"num_bytes_recovered":2021261312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15637295104,"data_stored":15637282888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106903,"ondisk_log_size":106903,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234385,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2369664,"kb_used_data":2059832,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91375488,"statfs":{"total":95995035648,"available":93568499712,"internally_reserved":0,"allocated":2109267968,"data_stored":2104418123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496020,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2679108,"kb_used_data":2369276,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91066044,"statfs":{"total":95995035648,"available":93251629056,"internally_reserved":0,"allocated":2426138624,"data_stored":2421285184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757658,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326272,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2944428,"kb_used_data":2727972,"kb_used_omap":57,"kb_used_meta":216390,"kb_avail":90800724,"statfs":{"total":95995035648,"available":92979941376,"internally_reserved":0,"allocated":2793443328,"data_stored":2788608663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":221584123},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405511,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2649332,"kb_used_data":2450860,"kb_used_omap":61,"kb_used_meta":198402,"kb_avail":91095820,"statfs":{"total":95995035648,"available":93282119680,"internally_reserved":0,"allocated":2509680640,"data_stored":2504842947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":203164600},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452045,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2007656,"kb_used_data":1845344,"kb_used_omap":54,"kb_used_meta":162249,"kb_avail":91737496,"statfs":{"total":95995035648,"available":93939195904,"internally_reserved":0,"allocated":1889632256,"data_stored":1884801552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":166143374},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474737,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1693752,"kb_used_data":1494640,"kb_used_omap":119,"kb_used_meta":198984,"kb_avail":92051400,"statfs":{"total":95995035648,"available":94260633600,"internally_reserved":0,"allocated":1530511360,"data_stored":1525665492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":203760350},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019293,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1516638208,"data_stored":1516634136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1875181568,"data_stored":1875181568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2495610880,"data_stored":2495610880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2779844608,"data_stored":2779840536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1083572224,"data_stored":1083572224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1380777984,"data_stored":1380777984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2411204608,"data_stored":2411200536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2094465024,"data_stored":2094465024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:53:04.181 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:04.203 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:04.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:04.304 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:04.331 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:04.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:04.432 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:04.453 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:04.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:04.555 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:04.578 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:04.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:04.650 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:53:04.650 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:53:04.680 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:04.704 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:04.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:04.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:04.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:04.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:04.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:04.952 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:04.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:04.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:05.054 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:05.075 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:05.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:05.177 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:05.198 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:05.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:05.301 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:05.324 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:05.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:05.370 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:05 smithi130 ceph-mon[105521]: pgmap v1773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 18636/227427 objects degraded (8.194%); 13090/227427 objects misplaced (5.756%); 947 KiB/s, 14 objects/s recovering 2024-08-17T17:53:05.370 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:05 smithi130 ceph-mon[110068]: pgmap v1773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 18636/227427 objects degraded (8.194%); 13090/227427 objects misplaced (5.756%); 947 KiB/s, 14 objects/s recovering 2024-08-17T17:53:05.426 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:05.447 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:05.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:05.549 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:05.572 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:05.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:05 smithi179 ceph-mon[93519]: pgmap v1773: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 18636/227427 objects degraded (8.194%); 13090/227427 objects misplaced (5.756%); 947 KiB/s, 14 objects/s recovering 2024-08-17T17:53:05.674 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:05.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:05.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:05.790 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:53:05.797 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:05.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.819 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:05.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:05.920 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:05.942 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:05.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:05.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:06.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:06.065 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:06.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:06.167 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:06.192 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:06.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:06.293 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:06.315 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:06.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:06.416 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:06.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18636/227427 objects degraded (8.194%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:06.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18636/227427 objects degraded (8.194%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:06.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.439 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:06.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:06.540 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:06.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.570 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:06.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:06.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18636/227427 objects degraded (8.194%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:06.671 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:06.697 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:06.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:06.800 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:06.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:06.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:06.922 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:06.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:06.945 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:06.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:07.046 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:07.067 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:07.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:07.169 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:07.175 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:53:07.194 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:07.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:07.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:07 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:53:07.286 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:07 smithi179 ceph-mon[93519]: pgmap v1774: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 18544/227154 objects degraded (8.164%); 13045/227154 objects misplaced (5.743%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:53:07.289 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:07.296 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:07.317 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:07.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:07.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:07 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:53:07.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:07 smithi130 ceph-mon[105521]: pgmap v1774: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 18544/227154 objects degraded (8.164%); 13045/227154 objects misplaced (5.743%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:53:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:07 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:53:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:07 smithi130 ceph-mon[110068]: pgmap v1774: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 18544/227154 objects degraded (8.164%); 13045/227154 objects misplaced (5.743%); 1017 KiB/s, 15 objects/s recovering 2024-08-17T17:53:07.419 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:07.441 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:07.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:07.542 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:07.565 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:07.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:07.666 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:07.687 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:07.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:07.789 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:07.812 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:07.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:07.914 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:07.936 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:07.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:07.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:08.037 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:08.060 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:08.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:08.162 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:08.189 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:08.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:08.290 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:08.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.312 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:08.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:08.413 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:08.440 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:08.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:08.485 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:08.498 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:53:08.499 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:53:08.517 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:53:08.518 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:53:08.535 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:53:08.536 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:53:08.542 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:08.553 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:53:08.553 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:53:08.563 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:08.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:08.569 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:53:08.570 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:53:08.585 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:53:08.585 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:53:08.600 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:53:08.601 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:53:08.616 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:53:08.617 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:53:08.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:08.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:08 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3385722872' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:08 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3385722872' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:08.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:08.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:08.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:53:08.788 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:08.809 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:08.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:08.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:08 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3385722872' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:08.911 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:08.932 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:08.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:08.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:09.034 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:09.055 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:09.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:09.157 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:09.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.179 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:09.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:09.280 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:09.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.302 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:09.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:09.403 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:09.427 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:09.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:09.529 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:09.550 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:09.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:09.646 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:09 smithi130 ceph-mon[105521]: pgmap v1775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 18544/225741 objects degraded (8.215%); 12970/225741 objects misplaced (5.746%); 1024 KiB/s, 15 objects/s recovering 2024-08-17T17:53:09.646 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:09 smithi130 ceph-mon[110068]: pgmap v1775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 18544/225741 objects degraded (8.215%); 12970/225741 objects misplaced (5.746%); 1024 KiB/s, 15 objects/s recovering 2024-08-17T17:53:09.652 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:09.675 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:09.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:09.777 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:09.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.799 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:09.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:09 smithi179 ceph-mon[93519]: pgmap v1775: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 18544/225741 objects degraded (8.215%); 12970/225741 objects misplaced (5.746%); 1024 KiB/s, 15 objects/s recovering 2024-08-17T17:53:09.900 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:09.908 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:53:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:53:09] "GET /metrics HTTP/1.1" 200 36812 "" "Prometheus/2.51.0" 2024-08-17T17:53:09.930 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:09.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:09.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:10.032 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:10.059 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:10.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:10.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:10.145 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:53:10.145 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T17:53:10.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T17:53:10.145 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T17:53:10.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.146 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:116980 45.1 45:8799f42e:::benchmark_data_smithi130_172551_object12173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1191)", 2024-08-17T17:53:10.146 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:11.202055+0000", 2024-08-17T17:53:10.146 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 178.94004894299999, 2024-08-17T17:53:10.146 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0427133929999997, 2024-08-17T17:53:10.146 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.146 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.146 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.146 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.146 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.146 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.146 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 116980 2024-08-17T17:53:10.146 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.146 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.146 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.202055+0000", 2024-08-17T17:53:10.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.202055+0000", 2024-08-17T17:53:10.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.202057+0000", 2024-08-17T17:53:10.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9570000000000001e-06 2024-08-17T17:53:10.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.202064+0000", 2024-08-17T17:53:10.148 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4429999999999997e-06 2024-08-17T17:53:10.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.202080+0000", 2024-08-17T17:53:10.148 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5553000000000002e-05 2024-08-17T17:53:10.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.202088+0000", 2024-08-17T17:53:10.149 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1929999999999993e-06 2024-08-17T17:53:10.149 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.149 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.149 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.583665+0000", 2024-08-17T17:53:10.149 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.381576849 2024-08-17T17:53:10.149 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.149 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.149 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.601146+0000", 2024-08-17T17:53:10.149 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017480921 2024-08-17T17:53:10.149 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.150 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.150 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.821377+0000", 2024-08-17T17:53:10.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.220231552 2024-08-17T17:53:10.150 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.150 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.150 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.150 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.832291+0000", 2024-08-17T17:53:10.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010913156 2024-08-17T17:53:10.150 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.150 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.150 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.150 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.152620+0000", 2024-08-17T17:53:10.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.320328881 2024-08-17T17:53:10.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.160695+0000", 2024-08-17T17:53:10.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0080755629999999991 2024-08-17T17:53:10.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.178912+0000", 2024-08-17T17:53:10.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018216901000000001 2024-08-17T17:53:10.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.207906+0000", 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028994028000000002 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.210068+0000", 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021623110000000001 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.212964+0000", 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028953949999999998 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.153 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.219201+0000", 2024-08-17T17:53:10.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0062367990000000003 2024-08-17T17:53:10.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.153 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.153 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.244768+0000", 2024-08-17T17:53:10.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025567890999999999 2024-08-17T17:53:10.153 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.153 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.153 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.153 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117006 45.1 45:8e79452c:::benchmark_data_smithi130_172551_object12199:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1191)", 2024-08-17T17:53:10.153 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:12.051804+0000", 2024-08-17T17:53:10.154 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 178.090299539, 2024-08-17T17:53:10.154 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6324937070000001, 2024-08-17T17:53:10.154 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.154 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.154 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.154 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.154 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.154 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.154 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117006 2024-08-17T17:53:10.154 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.154 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.154 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.154 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.051804+0000", 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.051804+0000", 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.051807+0000", 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.261e-06 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.155 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.156 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.051817+0000", 2024-08-17T17:53:10.156 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0013000000000001e-05 2024-08-17T17:53:10.156 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.156 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.156 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.156 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.051843+0000", 2024-08-17T17:53:10.156 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6792999999999999e-05 2024-08-17T17:53:10.156 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.156 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.156 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.156 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.051851+0000", 2024-08-17T17:53:10.156 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7270000000000007e-06 2024-08-17T17:53:10.156 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.157 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.372425+0000", 2024-08-17T17:53:10.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.32057382899999998 2024-08-17T17:53:10.157 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.157 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.392911+0000", 2024-08-17T17:53:10.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020486397 2024-08-17T17:53:10.157 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.157 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.478544+0000", 2024-08-17T17:53:10.158 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.085632642999999994 2024-08-17T17:53:10.158 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.158 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.158 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.158 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.488558+0000", 2024-08-17T17:53:10.158 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010013538000000001 2024-08-17T17:53:10.158 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.158 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.158 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.158 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.495084+0000", 2024-08-17T17:53:10.158 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0065260469999999 2024-08-17T17:53:10.158 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.158 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.158 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.159 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.509554+0000", 2024-08-17T17:53:10.159 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014470264 2024-08-17T17:53:10.159 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.159 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.159 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.159 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.538572+0000", 2024-08-17T17:53:10.159 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.029018004 2024-08-17T17:53:10.159 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.159 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.160 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.160 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.651135+0000", 2024-08-17T17:53:10.160 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.11256314100000001 2024-08-17T17:53:10.160 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.160 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.160 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.160 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.653806+0000", 2024-08-17T17:53:10.160 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026705710000000001 2024-08-17T17:53:10.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.657210+0000", 2024-08-17T17:53:10.161 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003404168 2024-08-17T17:53:10.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.665389+0000", 2024-08-17T17:53:10.161 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0081789949999999997 2024-08-17T17:53:10.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.162 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.684298+0000", 2024-08-17T17:53:10.162 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018909315999999999 2024-08-17T17:53:10.162 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.162 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.162 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.162 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.162 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.162 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117051 45.1 45:87d9b5c3:::benchmark_data_smithi130_172551_object12244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1191)", 2024-08-17T17:53:10.162 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:12.410351+0000", 2024-08-17T17:53:10.162 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 177.73175282899999, 2024-08-17T17:53:10.162 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2018803760000001, 2024-08-17T17:53:10.162 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.162 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.163 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.163 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.163 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.163 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.163 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117051 2024-08-17T17:53:10.163 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.163 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.163 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.163 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.163 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.410351+0000", 2024-08-17T17:53:10.163 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.163 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.163 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.163 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.164 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.410351+0000", 2024-08-17T17:53:10.164 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.164 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.164 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.164 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.410353+0000", 2024-08-17T17:53:10.164 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0710000000000002e-06 2024-08-17T17:53:10.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.164 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.164 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.164 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.410361+0000", 2024-08-17T17:53:10.164 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0690000000000002e-06 2024-08-17T17:53:10.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.165 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.165 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.165 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.410377+0000", 2024-08-17T17:53:10.165 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5492999999999999e-05 2024-08-17T17:53:10.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.165 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.165 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.165 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.410386+0000", 2024-08-17T17:53:10.165 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8300000000000002e-06 2024-08-17T17:53:10.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.165 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.165 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.165 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.670860+0000", 2024-08-17T17:53:10.165 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.260474488 2024-08-17T17:53:10.166 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.689297+0000", 2024-08-17T17:53:10.166 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018437174000000001 2024-08-17T17:53:10.166 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.757380+0000", 2024-08-17T17:53:10.166 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.068083187000000003 2024-08-17T17:53:10.166 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.167 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.768726+0000", 2024-08-17T17:53:10.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011345696000000001 2024-08-17T17:53:10.167 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.167 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.167 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.923952+0000", 2024-08-17T17:53:10.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1552257940000001 2024-08-17T17:53:10.167 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.167 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.167 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.935961+0000", 2024-08-17T17:53:10.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012008959 2024-08-17T17:53:10.167 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.974312+0000", 2024-08-17T17:53:10.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.038351277000000003 2024-08-17T17:53:10.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.073288+0000", 2024-08-17T17:53:10.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.098975749000000002 2024-08-17T17:53:10.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.075449+0000", 2024-08-17T17:53:10.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021606849999999999 2024-08-17T17:53:10.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.583241+0000", 2024-08-17T17:53:10.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.50779248499999996 2024-08-17T17:53:10.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.589776+0000", 2024-08-17T17:53:10.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0065349500000000003 2024-08-17T17:53:10.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.612232+0000", 2024-08-17T17:53:10.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022455468999999999 2024-08-17T17:53:10.170 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.170 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.170 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.170 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.170 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.170 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117063 45.1 45:84049d28:::benchmark_data_smithi130_172551_object12256:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1191)", 2024-08-17T17:53:10.170 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:12.475601+0000", 2024-08-17T17:53:10.170 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 177.666502998, 2024-08-17T17:53:10.170 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.17575086, 2024-08-17T17:53:10.170 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.170 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.170 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117063 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.475601+0000", 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.475601+0000", 2024-08-17T17:53:10.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.172 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.172 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.172 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.475603+0000", 2024-08-17T17:53:10.172 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.77e-06 2024-08-17T17:53:10.172 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.172 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.172 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.475609+0000", 2024-08-17T17:53:10.172 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4980000000000003e-06 2024-08-17T17:53:10.172 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.172 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.475622+0000", 2024-08-17T17:53:10.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2917e-05 2024-08-17T17:53:10.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.475630+0000", 2024-08-17T17:53:10.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5109999999999997e-06 2024-08-17T17:53:10.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.691253+0000", 2024-08-17T17:53:10.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.21562358000000001 2024-08-17T17:53:10.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.708213+0000", 2024-08-17T17:53:10.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016959795 2024-08-17T17:53:10.174 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.891227+0000", 2024-08-17T17:53:10.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.18301373700000001 2024-08-17T17:53:10.174 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.905968+0000", 2024-08-17T17:53:10.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014741318 2024-08-17T17:53:10.175 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.175 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.175 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.028661+0000", 2024-08-17T17:53:10.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1226928090000001 2024-08-17T17:53:10.175 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.175 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.175 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.038437+0000", 2024-08-17T17:53:10.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0097758250000000001 2024-08-17T17:53:10.175 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.175 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.055169+0000", 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016732120999999999 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.201705+0000", 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14653565299999999 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.615029+0000", 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.41332463400000002 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.176 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.620582+0000", 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0055527340000000001 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.624922+0000", 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0043401409999999996 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.651352+0000", 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.026429817000000001 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.177 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.178 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117072 45.1 45:8383d11f:::benchmark_data_smithi130_172551_object12265:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1191)", 2024-08-17T17:53:10.178 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:12.694419+0000", 2024-08-17T17:53:10.178 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 177.44768506700001, 2024-08-17T17:53:10.178 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9983870160000001, 2024-08-17T17:53:10.178 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.178 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.178 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.178 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.178 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.178 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.178 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117072 2024-08-17T17:53:10.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.179 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.694419+0000", 2024-08-17T17:53:10.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.694419+0000", 2024-08-17T17:53:10.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.694421+0000", 2024-08-17T17:53:10.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0219999999999999e-06 2024-08-17T17:53:10.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.180 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.180 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.694430+0000", 2024-08-17T17:53:10.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6459999999999999e-06 2024-08-17T17:53:10.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.180 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.180 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.694445+0000", 2024-08-17T17:53:10.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.522e-05 2024-08-17T17:53:10.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.180 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.694454+0000", 2024-08-17T17:53:10.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2809999999999997e-06 2024-08-17T17:53:10.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.181 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.958017+0000", 2024-08-17T17:53:10.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.263563031 2024-08-17T17:53:10.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.181 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.977402+0000", 2024-08-17T17:53:10.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019385135000000001 2024-08-17T17:53:10.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.059271+0000", 2024-08-17T17:53:10.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.081868301000000004 2024-08-17T17:53:10.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.067839+0000", 2024-08-17T17:53:10.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0085682269999999994 2024-08-17T17:53:10.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.115160+0000", 2024-08-17T17:53:10.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0473214799999999 2024-08-17T17:53:10.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.122535+0000", 2024-08-17T17:53:10.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0073748700000000004 2024-08-17T17:53:10.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.140700+0000", 2024-08-17T17:53:10.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018164882 2024-08-17T17:53:10.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.288391+0000", 2024-08-17T17:53:10.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14769062099999999 2024-08-17T17:53:10.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.655373+0000", 2024-08-17T17:53:10.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.36698246899999998 2024-08-17T17:53:10.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.659842+0000", 2024-08-17T17:53:10.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0044693880000000004 2024-08-17T17:53:10.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.664773+0000", 2024-08-17T17:53:10.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0049301579999999996 2024-08-17T17:53:10.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.692806+0000", 2024-08-17T17:53:10.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028033285000000002 2024-08-17T17:53:10.185 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.185 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.186 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.186 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117087 45.1 45:8d1ad67f:::benchmark_data_smithi130_172551_object12280:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1191)", 2024-08-17T17:53:10.186 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:13.044755+0000", 2024-08-17T17:53:10.186 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 177.09734867, 2024-08-17T17:53:10.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6944316280000002, 2024-08-17T17:53:10.186 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.186 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.186 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.186 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.186 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.186 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.187 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117087 2024-08-17T17:53:10.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.187 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.044755+0000", 2024-08-17T17:53:10.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.044755+0000", 2024-08-17T17:53:10.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.044758+0000", 2024-08-17T17:53:10.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5299999999999999e-06 2024-08-17T17:53:10.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.044767+0000", 2024-08-17T17:53:10.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2690000000000005e-06 2024-08-17T17:53:10.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.044789+0000", 2024-08-17T17:53:10.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2064000000000001e-05 2024-08-17T17:53:10.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.044814+0000", 2024-08-17T17:53:10.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5258000000000001e-05 2024-08-17T17:53:10.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.156054+0000", 2024-08-17T17:53:10.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.111239721 2024-08-17T17:53:10.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.172700+0000", 2024-08-17T17:53:10.190 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016645750000000001 2024-08-17T17:53:10.190 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.190 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.190 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.190 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.266218+0000", 2024-08-17T17:53:10.190 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.093517811000000006 2024-08-17T17:53:10.190 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.190 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.190 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.190 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.278568+0000", 2024-08-17T17:53:10.190 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012350292000000001 2024-08-17T17:53:10.190 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.190 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.191 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.191 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.202811+0000", 2024-08-17T17:53:10.191 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9242432539999998 2024-08-17T17:53:10.191 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.191 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.191 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.191 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.211716+0000", 2024-08-17T17:53:10.191 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.008905095 2024-08-17T17:53:10.191 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.191 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.191 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.191 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.228505+0000", 2024-08-17T17:53:10.191 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016788964999999999 2024-08-17T17:53:10.191 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.192 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.372900+0000", 2024-08-17T17:53:10.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14439475399999999 2024-08-17T17:53:10.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.192 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.696610+0000", 2024-08-17T17:53:10.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.32371030299999998 2024-08-17T17:53:10.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.192 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.702116+0000", 2024-08-17T17:53:10.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0055056749999999998 2024-08-17T17:53:10.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.193 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.193 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.706734+0000", 2024-08-17T17:53:10.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046176840000000004 2024-08-17T17:53:10.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.193 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.193 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.739187+0000", 2024-08-17T17:53:10.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.032453203 2024-08-17T17:53:10.193 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.193 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.193 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.194 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117138 45.1 45:82c30d21:::benchmark_data_smithi130_172551_object12331:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:53:10.194 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:14.196732+0000", 2024-08-17T17:53:10.194 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 175.94537235999999, 2024-08-17T17:53:10.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.70372638, 2024-08-17T17:53:10.194 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.194 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.194 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.194 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.194 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.194 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.194 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117138 2024-08-17T17:53:10.194 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.194 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.195 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.196732+0000", 2024-08-17T17:53:10.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.195 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.196732+0000", 2024-08-17T17:53:10.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.195 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.196734+0000", 2024-08-17T17:53:10.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9479999999999998e-06 2024-08-17T17:53:10.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.196740+0000", 2024-08-17T17:53:10.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3280000000000002e-06 2024-08-17T17:53:10.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.196755+0000", 2024-08-17T17:53:10.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4874e-05 2024-08-17T17:53:10.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.196762+0000", 2024-08-17T17:53:10.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4900000000000003e-06 2024-08-17T17:53:10.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.313933+0000", 2024-08-17T17:53:10.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.117171022 2024-08-17T17:53:10.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.327698+0000", 2024-08-17T17:53:10.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013764609000000001 2024-08-17T17:53:10.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.198 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.198 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.384376+0000", 2024-08-17T17:53:10.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.056677727999999997 2024-08-17T17:53:10.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.198 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.198 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.198 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.393627+0000", 2024-08-17T17:53:10.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0092510240000000001 2024-08-17T17:53:10.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.198 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.198 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.198 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.668409+0000", 2024-08-17T17:53:10.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2747823719999998 2024-08-17T17:53:10.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.199 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.199 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.674408+0000", 2024-08-17T17:53:10.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0059989489999999999 2024-08-17T17:53:10.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.199 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.199 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.686181+0000", 2024-08-17T17:53:10.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011773336000000001 2024-08-17T17:53:10.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.199 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.199 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.794142+0000", 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.107960316 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.796752+0000", 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026106329999999998 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.861007+0000", 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.064255123999999997 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.871761+0000", 2024-08-17T17:53:10.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010753792999999999 2024-08-17T17:53:10.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.201 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.201 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.900458+0000", 2024-08-17T17:53:10.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028696834000000001 2024-08-17T17:53:10.201 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.201 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.201 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.201 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117183 45.1 45:802f39c4:::benchmark_data_smithi130_172551_object12376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:53:10.201 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:14.982725+0000", 2024-08-17T17:53:10.201 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 175.15937912800001, 2024-08-17T17:53:10.202 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8032687460000001, 2024-08-17T17:53:10.202 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.202 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:10.203 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.204 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.204 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.204 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.204 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.204 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117183 2024-08-17T17:53:10.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.204 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.982725+0000", 2024-08-17T17:53:10.204 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.982725+0000", 2024-08-17T17:53:10.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.982727+0000", 2024-08-17T17:53:10.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6950000000000001e-06 2024-08-17T17:53:10.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.982735+0000", 2024-08-17T17:53:10.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4659999999999999e-06 2024-08-17T17:53:10.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.206 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.206 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.982747+0000", 2024-08-17T17:53:10.206 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2388999999999999e-05 2024-08-17T17:53:10.206 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.206 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.206 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.982755+0000", 2024-08-17T17:53:10.206 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1160000000000001e-06 2024-08-17T17:53:10.206 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.206 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.207 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.306395+0000", 2024-08-17T17:53:10.207 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.32364099000000002 2024-08-17T17:53:10.207 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.207 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.207 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.207 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.318989+0000", 2024-08-17T17:53:10.207 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012593332 2024-08-17T17:53:10.207 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.207 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.207 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.207 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.411042+0000", 2024-08-17T17:53:10.207 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.092052758999999998 2024-08-17T17:53:10.207 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.207 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.208 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.208 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.427984+0000", 2024-08-17T17:53:10.208 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.01694201 2024-08-17T17:53:10.208 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.208 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.208 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.208 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.610241+0000", 2024-08-17T17:53:10.208 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1822570410000002 2024-08-17T17:53:10.208 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.208 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.208 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.208 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.623111+0000", 2024-08-17T17:53:10.208 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012870825000000001 2024-08-17T17:53:10.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.642243+0000", 2024-08-17T17:53:10.209 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019131657 2024-08-17T17:53:10.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.739757+0000", 2024-08-17T17:53:10.209 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.097514067999999995 2024-08-17T17:53:10.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.742367+0000", 2024-08-17T17:53:10.210 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026096829999999998 2024-08-17T17:53:10.210 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.210 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.210 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.745766+0000", 2024-08-17T17:53:10.210 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0033987079999999999 2024-08-17T17:53:10.210 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.210 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.210 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.754589+0000", 2024-08-17T17:53:10.210 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0088238429999999996 2024-08-17T17:53:10.210 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.210 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.211 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.211 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.785994+0000", 2024-08-17T17:53:10.211 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.031404163999999998 2024-08-17T17:53:10.211 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.211 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.211 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.211 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.211 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.211 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117271 45.1 45:86e9ffcd:::benchmark_data_smithi130_172551_object12464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:53:10.211 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:16.753226+0000", 2024-08-17T17:53:10.211 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 173.38887789099999, 2024-08-17T17:53:10.211 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9279494399999999, 2024-08-17T17:53:10.211 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.211 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117271 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.753226+0000", 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.212 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.753226+0000", 2024-08-17T17:53:10.213 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.213 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.213 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.213 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.213 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.753228+0000", 2024-08-17T17:53:10.213 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.731e-06 2024-08-17T17:53:10.213 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.213 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.213 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.213 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.753235+0000", 2024-08-17T17:53:10.213 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.481e-06 2024-08-17T17:53:10.213 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.213 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.214 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.214 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.753256+0000", 2024-08-17T17:53:10.214 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0707e-05 2024-08-17T17:53:10.214 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.214 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.214 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.214 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.753265+0000", 2024-08-17T17:53:10.214 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1160000000000005e-06 2024-08-17T17:53:10.214 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.214 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.214 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.214 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.182325+0000", 2024-08-17T17:53:10.214 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.42906007000000002 2024-08-17T17:53:10.214 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.215 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.215 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.215 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.192896+0000", 2024-08-17T17:53:10.215 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010570461999999999 2024-08-17T17:53:10.215 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.215 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.215 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.215 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.257048+0000", 2024-08-17T17:53:10.215 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.064152826999999996 2024-08-17T17:53:10.215 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.215 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.215 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.215 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.268122+0000", 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011073553 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:19.904981+0000", 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.636858691 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:19.917279+0000", 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012298335000000001 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.216 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:19.940310+0000", 2024-08-17T17:53:10.217 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023030508000000002 2024-08-17T17:53:10.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.217 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.217 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.217 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.047544+0000", 2024-08-17T17:53:10.217 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.107234048 2024-08-17T17:53:10.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.217 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.217 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.217 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.050156+0000", 2024-08-17T17:53:10.217 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026120269999999998 2024-08-17T17:53:10.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.217 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.217 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.218 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.654099+0000", 2024-08-17T17:53:10.218 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.603943652 2024-08-17T17:53:10.218 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.218 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.218 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.661398+0000", 2024-08-17T17:53:10.218 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072991360000000003 2024-08-17T17:53:10.218 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.218 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.218 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.681176+0000", 2024-08-17T17:53:10.218 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019777096000000001 2024-08-17T17:53:10.218 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.219 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.219 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.219 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.219 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.219 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117312 45.1 45:8106ae5d:::benchmark_data_smithi130_172551_object12505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:53:10.219 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:16.890803+0000", 2024-08-17T17:53:10.219 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 173.25130087700001, 2024-08-17T17:53:10.219 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8771543880000001, 2024-08-17T17:53:10.219 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.219 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.219 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.219 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.219 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.220 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.220 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117312 2024-08-17T17:53:10.220 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.220 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.220 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.220 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.220 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.890803+0000", 2024-08-17T17:53:10.220 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.220 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.220 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.220 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.220 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.890803+0000", 2024-08-17T17:53:10.220 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.220 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.221 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.221 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.221 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.890805+0000", 2024-08-17T17:53:10.221 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.79e-06 2024-08-17T17:53:10.221 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.221 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.221 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.221 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.890812+0000", 2024-08-17T17:53:10.221 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7009999999999999e-06 2024-08-17T17:53:10.221 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.221 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.221 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.221 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.890827+0000", 2024-08-17T17:53:10.221 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5054e-05 2024-08-17T17:53:10.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.890835+0000", 2024-08-17T17:53:10.222 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7479999999999999e-06 2024-08-17T17:53:10.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.194094+0000", 2024-08-17T17:53:10.222 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.303258156 2024-08-17T17:53:10.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.204820+0000", 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010726127 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.366433+0000", 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16161305300000001 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.379828+0000", 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013395504000000001 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.224 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.089528+0000", 2024-08-17T17:53:10.224 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.70969967 2024-08-17T17:53:10.224 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.224 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.224 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.224 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.100667+0000", 2024-08-17T17:53:10.224 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011139539 2024-08-17T17:53:10.224 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.224 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.224 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.224 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.120743+0000", 2024-08-17T17:53:10.224 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020075221000000001 2024-08-17T17:53:10.224 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.224 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.225 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.225 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.223059+0000", 2024-08-17T17:53:10.225 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.102316746 2024-08-17T17:53:10.225 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.225 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.225 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.225 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.728570+0000", 2024-08-17T17:53:10.225 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.505510864 2024-08-17T17:53:10.225 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.225 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.225 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.225 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.735815+0000", 2024-08-17T17:53:10.225 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072448340000000003 2024-08-17T17:53:10.225 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.226 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.226 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.226 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.745136+0000", 2024-08-17T17:53:10.226 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0093212420000000004 2024-08-17T17:53:10.226 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.226 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.226 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.226 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.767957+0000", 2024-08-17T17:53:10.226 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022821139000000001 2024-08-17T17:53:10.226 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.226 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.226 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.226 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.227 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117315 45.1 45:89f3983e:::benchmark_data_smithi130_172551_object12508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:53:10.227 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:16.898971+0000", 2024-08-17T17:53:10.227 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 173.24313253899999, 2024-08-17T17:53:10.227 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8238647210000001, 2024-08-17T17:53:10.227 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.227 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.227 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.227 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.227 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.227 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.227 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117315 2024-08-17T17:53:10.227 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.227 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.228 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.228 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.228 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.898971+0000", 2024-08-17T17:53:10.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.228 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.228 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.228 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.898971+0000", 2024-08-17T17:53:10.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.228 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.228 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.228 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.898973+0000", 2024-08-17T17:53:10.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.437e-06 2024-08-17T17:53:10.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.898987+0000", 2024-08-17T17:53:10.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3764000000000001e-05 2024-08-17T17:53:10.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.899000+0000", 2024-08-17T17:53:10.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3559e-05 2024-08-17T17:53:10.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.230 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.899007+0000", 2024-08-17T17:53:10.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.126e-06 2024-08-17T17:53:10.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.230 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.230 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.208678+0000", 2024-08-17T17:53:10.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.30967032500000002 2024-08-17T17:53:10.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.230 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.230 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.220331+0000", 2024-08-17T17:53:10.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011653596 2024-08-17T17:53:10.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.307207+0000", 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.086876244000000005 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.318268+0000", 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011060327 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.007325+0000", 2024-08-17T17:53:10.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6890573180000001 2024-08-17T17:53:10.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.232 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.232 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.016133+0000", 2024-08-17T17:53:10.232 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0088075940000000002 2024-08-17T17:53:10.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.232 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.232 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.031654+0000", 2024-08-17T17:53:10.232 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015521237 2024-08-17T17:53:10.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.232 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.233 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.136315+0000", 2024-08-17T17:53:10.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.104660695 2024-08-17T17:53:10.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.233 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.233 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.684865+0000", 2024-08-17T17:53:10.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.54855074199999998 2024-08-17T17:53:10.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.233 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.233 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.688834+0000", 2024-08-17T17:53:10.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039684760000000003 2024-08-17T17:53:10.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.694674+0000", 2024-08-17T17:53:10.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0058402920000000004 2024-08-17T17:53:10.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.722836+0000", 2024-08-17T17:53:10.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028161988999999998 2024-08-17T17:53:10.234 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.234 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.234 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.235 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117325 45.1 45:876cdfdb:::benchmark_data_smithi130_172551_object12518:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:53:10.235 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:16.936304+0000", 2024-08-17T17:53:10.235 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 173.20579980900001, 2024-08-17T17:53:10.235 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8880692140000002, 2024-08-17T17:53:10.235 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.235 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.235 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.235 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.235 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.235 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.235 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117325 2024-08-17T17:53:10.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.235 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.235 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.936304+0000", 2024-08-17T17:53:10.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.936304+0000", 2024-08-17T17:53:10.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.936306+0000", 2024-08-17T17:53:10.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1090000000000001e-06 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.936312+0000", 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1109999999999996e-06 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.936326+0000", 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3729e-05 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.936335+0000", 2024-08-17T17:53:10.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.132e-06 2024-08-17T17:53:10.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.383988+0000", 2024-08-17T17:53:10.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.44765318599999998 2024-08-17T17:53:10.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.398315+0000", 2024-08-17T17:53:10.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014326383 2024-08-17T17:53:10.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.475851+0000", 2024-08-17T17:53:10.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.077535779999999999 2024-08-17T17:53:10.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.490245+0000", 2024-08-17T17:53:10.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014393992 2024-08-17T17:53:10.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.183369+0000", 2024-08-17T17:53:10.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6931242759999998 2024-08-17T17:53:10.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.240 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.240 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.193862+0000", 2024-08-17T17:53:10.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010493186999999999 2024-08-17T17:53:10.240 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.240 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.240 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.240 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.208305+0000", 2024-08-17T17:53:10.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014442521999999999 2024-08-17T17:53:10.240 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.240 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.240 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.240 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.312432+0000", 2024-08-17T17:53:10.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.104126987 2024-08-17T17:53:10.240 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.241 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.241 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.241 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.773252+0000", 2024-08-17T17:53:10.241 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.46082050200000002 2024-08-17T17:53:10.241 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.241 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.241 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.241 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.783161+0000", 2024-08-17T17:53:10.241 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.009908867 2024-08-17T17:53:10.241 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.241 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.241 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.241 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.793591+0000", 2024-08-17T17:53:10.241 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010429946000000001 2024-08-17T17:53:10.242 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.242 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.824373+0000", 2024-08-17T17:53:10.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.030782505000000002 2024-08-17T17:53:10.242 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.242 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.242 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.242 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.242 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.242 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117345 45.1 45:8c7a643c:::benchmark_data_smithi130_172551_object12538:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:53:10.242 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:17.010755+0000", 2024-08-17T17:53:10.242 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 173.131349207, 2024-08-17T17:53:10.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9264242559999998, 2024-08-17T17:53:10.243 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.243 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.243 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.243 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.243 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.243 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.243 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117345 2024-08-17T17:53:10.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.243 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.010755+0000", 2024-08-17T17:53:10.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.010755+0000", 2024-08-17T17:53:10.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.010757+0000", 2024-08-17T17:53:10.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9379999999999999e-06 2024-08-17T17:53:10.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.010776+0000", 2024-08-17T17:53:10.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9273e-05 2024-08-17T17:53:10.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.010790+0000", 2024-08-17T17:53:10.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4409999999999999e-05 2024-08-17T17:53:10.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.010806+0000", 2024-08-17T17:53:10.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6113999999999999e-05 2024-08-17T17:53:10.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.246 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.246 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.399734+0000", 2024-08-17T17:53:10.246 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38892710800000002 2024-08-17T17:53:10.246 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.246 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.246 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.414221+0000", 2024-08-17T17:53:10.246 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014487619 2024-08-17T17:53:10.246 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.246 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.246 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.586260+0000", 2024-08-17T17:53:10.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.17203851100000001 2024-08-17T17:53:10.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.597691+0000", 2024-08-17T17:53:10.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011431619000000001 2024-08-17T17:53:10.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.354941+0000", 2024-08-17T17:53:10.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.7572492820000001 2024-08-17T17:53:10.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.248 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.363534+0000", 2024-08-17T17:53:10.248 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0085931940000000002 2024-08-17T17:53:10.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.248 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.248 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.380254+0000", 2024-08-17T17:53:10.248 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016719801999999999 2024-08-17T17:53:10.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.248 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.248 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.478132+0000", 2024-08-17T17:53:10.248 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.097878039999999999 2024-08-17T17:53:10.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.249 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.249 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.249 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.883521+0000", 2024-08-17T17:53:10.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.40538944999999998 2024-08-17T17:53:10.249 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.249 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.249 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.249 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.890390+0000", 2024-08-17T17:53:10.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0068690490000000003 2024-08-17T17:53:10.249 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.249 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.249 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.249 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.900699+0000", 2024-08-17T17:53:10.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010308552 2024-08-17T17:53:10.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.250 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.250 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.250 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.937179+0000", 2024-08-17T17:53:10.250 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.036480295000000003 2024-08-17T17:53:10.250 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.250 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.250 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.250 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.250 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117351 45.1 45:8354d1e0:::benchmark_data_smithi130_172551_object12544:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:53:10.250 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:17.049349+0000", 2024-08-17T17:53:10.250 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 173.092754996, 2024-08-17T17:53:10.250 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8289573089999998, 2024-08-17T17:53:10.251 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.251 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.251 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.251 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.251 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.251 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.251 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117351 2024-08-17T17:53:10.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.251 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.251 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.049349+0000", 2024-08-17T17:53:10.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.049349+0000", 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.049350+0000", 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.384e-06 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.049356+0000", 2024-08-17T17:53:10.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9390000000000002e-06 2024-08-17T17:53:10.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.049369+0000", 2024-08-17T17:53:10.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2716e-05 2024-08-17T17:53:10.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.049377+0000", 2024-08-17T17:53:10.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0879999999999995e-06 2024-08-17T17:53:10.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.418937+0000", 2024-08-17T17:53:10.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.36956013500000001 2024-08-17T17:53:10.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.432534+0000", 2024-08-17T17:53:10.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013596721000000001 2024-08-17T17:53:10.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.531230+0000", 2024-08-17T17:53:10.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.098696273000000001 2024-08-17T17:53:10.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.543385+0000", 2024-08-17T17:53:10.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012154999 2024-08-17T17:53:10.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.271977+0000", 2024-08-17T17:53:10.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.7285919249999999 2024-08-17T17:53:10.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.281763+0000", 2024-08-17T17:53:10.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0097861259999999992 2024-08-17T17:53:10.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.297762+0000", 2024-08-17T17:53:10.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015998931000000001 2024-08-17T17:53:10.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.392943+0000", 2024-08-17T17:53:10.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.095180734000000003 2024-08-17T17:53:10.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.828437+0000", 2024-08-17T17:53:10.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.43549367100000003 2024-08-17T17:53:10.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.257 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.834123+0000", 2024-08-17T17:53:10.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0056861419999999999 2024-08-17T17:53:10.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.257 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.840939+0000", 2024-08-17T17:53:10.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0068157790000000001 2024-08-17T17:53:10.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.878306+0000", 2024-08-17T17:53:10.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.037367746 2024-08-17T17:53:10.258 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.258 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.258 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.258 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117699 45.1 45:89818d88:::benchmark_data_smithi130_172551_object12892:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-08-17T17:53:10.258 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:21.161816+0000", 2024-08-17T17:53:10.258 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 168.98028805199999, 2024-08-17T17:53:10.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6034206969999998, 2024-08-17T17:53:10.259 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.259 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.259 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.259 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.259 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.259 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.259 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117699 2024-08-17T17:53:10.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.259 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.259 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.259 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.161816+0000", 2024-08-17T17:53:10.259 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.161816+0000", 2024-08-17T17:53:10.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.161818+0000", 2024-08-17T17:53:10.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.61e-06 2024-08-17T17:53:10.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.161823+0000", 2024-08-17T17:53:10.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8919999999999997e-06 2024-08-17T17:53:10.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.161835+0000", 2024-08-17T17:53:10.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1622e-05 2024-08-17T17:53:10.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.161842+0000", 2024-08-17T17:53:10.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6769999999999998e-06 2024-08-17T17:53:10.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.625179+0000", 2024-08-17T17:53:10.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.463337216 2024-08-17T17:53:10.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.634265+0000", 2024-08-17T17:53:10.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0090856689999999993 2024-08-17T17:53:10.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.772321+0000", 2024-08-17T17:53:10.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.13805646999999999 2024-08-17T17:53:10.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.782022+0000", 2024-08-17T17:53:10.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0097011969999999999 2024-08-17T17:53:10.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.660103+0000", 2024-08-17T17:53:10.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8780806889999999 2024-08-17T17:53:10.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.674652+0000", 2024-08-17T17:53:10.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014548991000000001 2024-08-17T17:53:10.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.697572+0000", 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022919695 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.723258+0000", 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025686356 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.726761+0000", 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0035028659999999999 2024-08-17T17:53:10.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.728435+0000", 2024-08-17T17:53:10.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001674236 2024-08-17T17:53:10.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.733754+0000", 2024-08-17T17:53:10.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0053185430000000002 2024-08-17T17:53:10.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.765237+0000", 2024-08-17T17:53:10.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.031482968 2024-08-17T17:53:10.266 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.266 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.266 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.266 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117710 45.1 45:8c275207:::benchmark_data_smithi130_172551_object12903:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-08-17T17:53:10.266 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:21.199072+0000", 2024-08-17T17:53:10.266 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 168.94303153300001, 2024-08-17T17:53:10.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.730388627, 2024-08-17T17:53:10.266 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.266 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.266 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.267 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.267 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.267 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.267 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117710 2024-08-17T17:53:10.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.267 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.199072+0000", 2024-08-17T17:53:10.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.199072+0000", 2024-08-17T17:53:10.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.268 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.199074+0000", 2024-08-17T17:53:10.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7260000000000001e-06 2024-08-17T17:53:10.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.268 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.199080+0000", 2024-08-17T17:53:10.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.665e-06 2024-08-17T17:53:10.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.199092+0000", 2024-08-17T17:53:10.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1727e-05 2024-08-17T17:53:10.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.199099+0000", 2024-08-17T17:53:10.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8800000000000008e-06 2024-08-17T17:53:10.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.785723+0000", 2024-08-17T17:53:10.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.58662361699999999 2024-08-17T17:53:10.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.795637+0000", 2024-08-17T17:53:10.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0099135660000000004 2024-08-17T17:53:10.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.849071+0000", 2024-08-17T17:53:10.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.053434118000000003 2024-08-17T17:53:10.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.857120+0000", 2024-08-17T17:53:10.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0080492189999999998 2024-08-17T17:53:10.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.828397+0000", 2024-08-17T17:53:10.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9712766269999999 2024-08-17T17:53:10.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.842426+0000", 2024-08-17T17:53:10.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014029741 2024-08-17T17:53:10.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.858962+0000", 2024-08-17T17:53:10.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016535516 2024-08-17T17:53:10.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.886263+0000", 2024-08-17T17:53:10.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.027301107000000002 2024-08-17T17:53:10.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.887898+0000", 2024-08-17T17:53:10.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001635077 2024-08-17T17:53:10.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.890177+0000", 2024-08-17T17:53:10.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0022786960000000002 2024-08-17T17:53:10.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.896455+0000", 2024-08-17T17:53:10.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.006277843 2024-08-17T17:53:10.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.929461+0000", 2024-08-17T17:53:10.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.033006502 2024-08-17T17:53:10.273 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.274 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.274 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.274 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117761 45.1 45:8c0eaaf2:::benchmark_data_smithi130_172551_object12954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-08-17T17:53:10.274 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:21.522576+0000", 2024-08-17T17:53:10.274 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 168.61952758999999, 2024-08-17T17:53:10.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7703922250000002, 2024-08-17T17:53:10.274 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.274 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.274 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.274 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.274 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.275 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.275 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117761 2024-08-17T17:53:10.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.275 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.275 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.275 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.522576+0000", 2024-08-17T17:53:10.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.275 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.275 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.522576+0000", 2024-08-17T17:53:10.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.522578+0000", 2024-08-17T17:53:10.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9420000000000002e-06 2024-08-17T17:53:10.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.522593+0000", 2024-08-17T17:53:10.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5051999999999999e-05 2024-08-17T17:53:10.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.522610+0000", 2024-08-17T17:53:10.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6605000000000001e-05 2024-08-17T17:53:10.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.522621+0000", 2024-08-17T17:53:10.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0654e-05 2024-08-17T17:53:10.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.796893+0000", 2024-08-17T17:53:10.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.27427218799999997 2024-08-17T17:53:10.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.806000+0000", 2024-08-17T17:53:10.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0091074329999999998 2024-08-17T17:53:10.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.969794+0000", 2024-08-17T17:53:10.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16379380599999999 2024-08-17T17:53:10.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.979236+0000", 2024-08-17T17:53:10.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0094422570000000008 2024-08-17T17:53:10.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.173034+0000", 2024-08-17T17:53:10.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1937977829999999 2024-08-17T17:53:10.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.188247+0000", 2024-08-17T17:53:10.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015213032 2024-08-17T17:53:10.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.214123+0000", 2024-08-17T17:53:10.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025876050000000001 2024-08-17T17:53:10.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.280 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.280 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.280 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.250644+0000", 2024-08-17T17:53:10.280 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.036521074000000001 2024-08-17T17:53:10.280 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.280 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.280 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.280 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.254884+0000", 2024-08-17T17:53:10.280 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0042393789999999997 2024-08-17T17:53:10.280 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.280 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.280 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.280 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.256566+0000", 2024-08-17T17:53:10.280 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001682317 2024-08-17T17:53:10.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.264351+0000", 2024-08-17T17:53:10.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0077850719999999997 2024-08-17T17:53:10.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.292969+0000", 2024-08-17T17:53:10.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028617581 2024-08-17T17:53:10.281 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.281 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.281 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.282 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117773 45.1 45:8ab76996:::benchmark_data_smithi130_172551_object12966:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-08-17T17:53:10.282 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:22.296464+0000", 2024-08-17T17:53:10.282 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 167.84564023600001, 2024-08-17T17:53:10.282 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6386252799999999, 2024-08-17T17:53:10.282 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.282 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.282 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.282 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.282 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.282 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.282 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117773 2024-08-17T17:53:10.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.296464+0000", 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.296464+0000", 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.296465+0000", 2024-08-17T17:53:10.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1689999999999999e-06 2024-08-17T17:53:10.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.296470+0000", 2024-08-17T17:53:10.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0379999999999999e-06 2024-08-17T17:53:10.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.296481+0000", 2024-08-17T17:53:10.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1504e-05 2024-08-17T17:53:10.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.296487+0000", 2024-08-17T17:53:10.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8590000000000001e-06 2024-08-17T17:53:10.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.401116+0000", 2024-08-17T17:53:10.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.10462840700000001 2024-08-17T17:53:10.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.415403+0000", 2024-08-17T17:53:10.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014287456 2024-08-17T17:53:10.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.557228+0000", 2024-08-17T17:53:10.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14182444499999999 2024-08-17T17:53:10.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.568859+0000", 2024-08-17T17:53:10.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011631466 2024-08-17T17:53:10.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.730508+0000", 2024-08-17T17:53:10.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1616493860000001 2024-08-17T17:53:10.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.738421+0000", 2024-08-17T17:53:10.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0079125879999999999 2024-08-17T17:53:10.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.752356+0000", 2024-08-17T17:53:10.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013934811 2024-08-17T17:53:10.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.288 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.289 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.850855+0000", 2024-08-17T17:53:10.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.098499574000000006 2024-08-17T17:53:10.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.289 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.289 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.289 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.887510+0000", 2024-08-17T17:53:10.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.036654478999999997 2024-08-17T17:53:10.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.896175+0000", 2024-08-17T17:53:10.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0086650410000000001 2024-08-17T17:53:10.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.904846+0000", 2024-08-17T17:53:10.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0086708459999999994 2024-08-17T17:53:10.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.935089+0000", 2024-08-17T17:53:10.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.030243210999999999 2024-08-17T17:53:10.291 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.291 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.291 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.291 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117795 45.1 45:86158ee1:::benchmark_data_smithi130_172551_object12988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-08-17T17:53:10.291 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:22.768871+0000", 2024-08-17T17:53:10.291 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 167.373233271, 2024-08-17T17:53:10.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7097064340000001, 2024-08-17T17:53:10.291 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.291 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.291 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.292 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.292 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.292 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.292 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117795 2024-08-17T17:53:10.292 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.292 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.292 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.292 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.292 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.768871+0000", 2024-08-17T17:53:10.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.292 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.292 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.292 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.292 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.768871+0000", 2024-08-17T17:53:10.293 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.293 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.293 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.768872+0000", 2024-08-17T17:53:10.293 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.345e-06 2024-08-17T17:53:10.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.293 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.293 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.768877+0000", 2024-08-17T17:53:10.293 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9590000000000003e-06 2024-08-17T17:53:10.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.294 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.294 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.768888+0000", 2024-08-17T17:53:10.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.102e-05 2024-08-17T17:53:10.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.294 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.294 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.768893+0000", 2024-08-17T17:53:10.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2170000000000002e-06 2024-08-17T17:53:10.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.294 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.294 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.132950+0000", 2024-08-17T17:53:10.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.36405708799999997 2024-08-17T17:53:10.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.151568+0000", 2024-08-17T17:53:10.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018618170999999999 2024-08-17T17:53:10.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.203989+0000", 2024-08-17T17:53:10.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.052420952999999999 2024-08-17T17:53:10.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.212712+0000", 2024-08-17T17:53:10.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0087221250000000007 2024-08-17T17:53:10.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.149119+0000", 2024-08-17T17:53:10.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9364075810000001 2024-08-17T17:53:10.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.161407+0000", 2024-08-17T17:53:10.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012288017 2024-08-17T17:53:10.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.297 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.176123+0000", 2024-08-17T17:53:10.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014715611 2024-08-17T17:53:10.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.297 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.297 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.273615+0000", 2024-08-17T17:53:10.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.097492032000000006 2024-08-17T17:53:10.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.297 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.297 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.275069+0000", 2024-08-17T17:53:10.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001453984 2024-08-17T17:53:10.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.439845+0000", 2024-08-17T17:53:10.298 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16477650299999999 2024-08-17T17:53:10.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.448492+0000", 2024-08-17T17:53:10.298 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0086466979999999995 2024-08-17T17:53:10.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.299 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.478577+0000", 2024-08-17T17:53:10.299 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.030085130000000002 2024-08-17T17:53:10.299 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.299 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.299 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.299 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.299 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.299 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117925 45.1 45:86b02470:::benchmark_data_smithi130_172551_object13118:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-08-17T17:53:10.299 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:23.946452+0000", 2024-08-17T17:53:10.299 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 166.19565225700001, 2024-08-17T17:53:10.299 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6290720809999999, 2024-08-17T17:53:10.299 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:10.299 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:10.299 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:10.300 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:10.300 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:10.300 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:10.300 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117925 2024-08-17T17:53:10.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.300 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:10.300 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.300 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:10.300 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.946452+0000", 2024-08-17T17:53:10.300 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.300 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.300 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:10.301 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.946452+0000", 2024-08-17T17:53:10.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:10.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.301 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:10.301 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.946453+0000", 2024-08-17T17:53:10.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.556e-06 2024-08-17T17:53:10.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.301 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:10.301 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.946461+0000", 2024-08-17T17:53:10.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6229999999999997e-06 2024-08-17T17:53:10.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:10.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.946471+0000", 2024-08-17T17:53:10.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0125999999999999e-05 2024-08-17T17:53:10.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:10.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.946477+0000", 2024-08-17T17:53:10.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7860000000000002e-06 2024-08-17T17:53:10.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.109726+0000", 2024-08-17T17:53:10.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.163249224 2024-08-17T17:53:10.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:53:10.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.127220+0000", 2024-08-17T17:53:10.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017493841 2024-08-17T17:53:10.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.399095+0000", 2024-08-17T17:53:10.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.27187506700000003 2024-08-17T17:53:10.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:53:10.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.417340+0000", 2024-08-17T17:53:10.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018244782000000001 2024-08-17T17:53:10.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:10.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.436576+0000", 2024-08-17T17:53:10.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0192360040000001 2024-08-17T17:53:10.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:10.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.443099+0000", 2024-08-17T17:53:10.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0065228339999999999 2024-08-17T17:53:10.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:53:10.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.456244+0000", 2024-08-17T17:53:10.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013145899000000001 2024-08-17T17:53:10.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:10.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.543652+0000", 2024-08-17T17:53:10.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.087407886000000004 2024-08-17T17:53:10.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.545856+0000", 2024-08-17T17:53:10.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0022041209999999999 2024-08-17T17:53:10.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:10.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.548716+0000", 2024-08-17T17:53:10.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028598740000000001 2024-08-17T17:53:10.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:10.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.553839+0000", 2024-08-17T17:53:10.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0051224970000000002 2024-08-17T17:53:10.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:10.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:10.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:10.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.575524+0000", 2024-08-17T17:53:10.307 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021684960999999999 2024-08-17T17:53:10.307 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.307 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.307 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.307 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:10.307 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:10.307 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:53:10.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:10.308 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:10.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:10.409 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:10.435 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:10.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:10.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:10.536 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:10.555 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:53:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:53:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:53:10.558 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:10.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:10.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:10.659 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:10.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:10.681 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:10.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:10.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:10.808 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:10.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:10.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:10.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:10 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18544/225741 objects degraded (8.215%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:10.911 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:10.936 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:10.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:10.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:11.038 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:11.064 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:11.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:11.166 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:10 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18544/225741 objects degraded (8.215%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:11.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:10 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18544/225741 objects degraded (8.215%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:11.189 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:11.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:11.291 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:11.312 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:11.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:11.414 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:11.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:11.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:11.539 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:11.562 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:11.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:11.664 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:11.686 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:11.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:11.788 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:11.811 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:11.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:11.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:11.940 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:11.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:11.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:12.042 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:12.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:12.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:12 smithi130 ceph-mon[105521]: pgmap v1776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 18544/225537 objects degraded (8.222%); 12970/225537 objects misplaced (5.751%); 1024 KiB/s, 15 objects/s recovering 2024-08-17T17:53:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:12 smithi130 ceph-mon[110068]: pgmap v1776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 18544/225537 objects degraded (8.222%); 12970/225537 objects misplaced (5.751%); 1024 KiB/s, 15 objects/s recovering 2024-08-17T17:53:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:12.167 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:12.188 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:12.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:12.290 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:12.314 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:12.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:12 smithi179 ceph-mon[93519]: pgmap v1776: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 18544/225537 objects degraded (8.222%); 12970/225537 objects misplaced (5.751%); 1024 KiB/s, 15 objects/s recovering 2024-08-17T17:53:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:12.415 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:12.437 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:12.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:12.539 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:12.560 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:12.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:12.662 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:12.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.684 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:12.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:12.785 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:12.807 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:12.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:12.908 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:12.935 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:12.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:12.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:13.037 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:13.041 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:53:13.058 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:13.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:13.160 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:13.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.182 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:13.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:13.283 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:13.305 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:13.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:13.407 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:13.429 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:13.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:13.530 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:13.553 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:13.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:13 smithi179 ceph-mon[93519]: pgmap v1777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 18494/225402 objects degraded (8.205%); 12950/225402 objects misplaced (5.745%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:13.655 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:13.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:13.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:13.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:13 smithi130 ceph-mon[105521]: pgmap v1777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 18494/225402 objects degraded (8.205%); 12950/225402 objects misplaced (5.745%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:13.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:13.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:13.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:13 smithi130 ceph-mon[110068]: pgmap v1777: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 18494/225402 objects degraded (8.205%); 12950/225402 objects misplaced (5.745%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:13.677 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:13.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:13.779 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:13.807 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:13.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:13.907 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:53:13.909 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:13.933 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:13.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:13.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:14.035 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:14.057 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:14.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:14.152 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:14.159 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:14.186 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:14.197 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:14.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:14.299 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:14.322 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:14.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:14.370 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:53:14.371 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:53:14.424 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:14.446 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:14.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:14.547 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:14.568 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:14.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:14.670 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:14.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:14.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:14.793 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:14.814 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:14.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:14.916 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:14.943 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:14.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:14.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:15.044 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:15.065 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:15.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:15.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:15 smithi130 ceph-mon[105521]: from='client.55174 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:15.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:15 smithi130 ceph-mon[105521]: pgmap v1778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 18494/224019 objects degraded (8.256%); 12870/224019 objects misplaced (5.745%); 992 KiB/s, 15 objects/s recovering 2024-08-17T17:53:15.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:15 smithi130 ceph-mon[110068]: from='client.55174 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:15.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:15 smithi130 ceph-mon[110068]: pgmap v1778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 18494/224019 objects degraded (8.256%); 12870/224019 objects misplaced (5.745%); 992 KiB/s, 15 objects/s recovering 2024-08-17T17:53:15.167 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:15.190 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:15.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:15.202 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:15.292 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:15.313 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:15.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:15.395 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:15 smithi179 ceph-mon[93519]: from='client.55174 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:15.395 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:15 smithi179 ceph-mon[93519]: pgmap v1778: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 18494/224019 objects degraded (8.256%); 12870/224019 objects misplaced (5.745%); 992 KiB/s, 15 objects/s recovering 2024-08-17T17:53:15.415 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:15.437 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:15.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:15.539 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:15.564 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:15.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:15.666 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:15.687 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:15.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:15.789 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:15.815 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:15.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:15.916 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:15.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:15.942 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:15.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:16.043 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:16.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:16.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:16.167 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:16.191 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:16.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:16.292 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:16.317 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:16.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:16.413 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18494/224019 objects degraded (8.256%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:16.414 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18494/224019 objects degraded (8.256%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:16.418 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:16.439 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:16.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:16.541 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:16.561 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:16.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18494/224019 objects degraded (8.256%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:16.663 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:16.687 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:16.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:16.789 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:16.811 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:16.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:16.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:16.935 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:16.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:16.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:17.036 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:17.063 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:17.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:17.165 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:17.186 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:17.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:17.288 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:17.314 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:17.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:17.397 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:17 smithi130 ceph-mon[105521]: pgmap v1779: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 18440/223836 objects degraded (8.238%); 12836/223836 objects misplaced (5.735%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:53:17.397 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:17 smithi130 ceph-mon[110068]: pgmap v1779: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 18440/223836 objects degraded (8.238%); 12836/223836 objects misplaced (5.735%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:53:17.416 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:17.437 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:17.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:17.539 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:17.564 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:17.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:17 smithi179 ceph-mon[93519]: pgmap v1779: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 18440/223836 objects degraded (8.238%); 12836/223836 objects misplaced (5.735%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:53:17.666 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:17.688 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:17.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:17.789 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:17.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.811 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:17.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:17.837 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1777,"stamp":"2024-08-17T17:53:13.034664+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4922808293,"num_objects":75134,"num_object_clones":0,"num_object_copies":225402,"num_objects_missing_on_primary":4188,"num_objects_missing":4188,"num_objects_degraded":18494,"num_objects_misplaced":12950,"num_objects_unfound":0,"num_objects_dirty":75134,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":134780,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33779,"num_bytes_recovered":2035162125,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107183,"ondisk_log_size":107183,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14151012,"kb_used_data":12742516,"kb_used_omap":295,"kb_used_meta":1408152,"kb_avail":548319900,"statfs":{"total":575970213888,"available":561479577600,"internally_reserved":0,"allocated":13048336384,"data_stored":13019239553,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1441948111},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-83427328,"num_objects":-1273,"num_object_clones":0,"num_object_copies":-3819,"num_objects_missing_on_primary":-73,"num_objects_missing":-73,"num_objects_degraded":-216,"num_objects_misplaced":-241,"num_objects_unfound":0,"num_objects_dirty":-1273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1273,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":185,"num_bytes_recovered":12124160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.295468"},"pg_stats":[{"pgid":"45.f","version":"1218'8408","reported_seq":18923,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:08.189652+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:53:08.189652+0000","last_peered":"2024-08-17T17:53:08.189652+0000","last_clean":"2024-08-17T17:53:08.189652+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:53:08.189652+0000","last_undegraded":"2024-08-17T17:53:08.189652+0000","last_fullsized":"2024-08-17T17:53:08.189652+0000","mapping_epoch":1194,"log_start":"1170'1531","ondisk_log_start":"1170'1531","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6877,"log_dups_size":0,"ondisk_log_size":6877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311689216,"num_objects":4756,"num_object_clones":0,"num_object_copies":14268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8408,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1218'8351","reported_seq":18809,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:07.766174+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:53:07.766174+0000","last_peered":"2024-08-17T17:53:07.766174+0000","last_clean":"2024-08-17T17:53:07.766174+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:53:07.766174+0000","last_undegraded":"2024-08-17T17:53:07.766174+0000","last_fullsized":"2024-08-17T17:53:07.766174+0000","mapping_epoch":1211,"log_start":"1170'1332","ondisk_log_start":"1170'1332","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7019,"log_dups_size":0,"ondisk_log_size":7019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304283648,"num_objects":4643,"num_object_clones":0,"num_object_copies":13929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8351,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1218'8614","reported_seq":19436,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:08.371393+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:53:08.371393+0000","last_peered":"2024-08-17T17:53:08.371393+0000","last_clean":"2024-08-17T17:53:08.371393+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:53:08.371393+0000","last_undegraded":"2024-08-17T17:53:08.371393+0000","last_fullsized":"2024-08-17T17:53:08.371393+0000","mapping_epoch":1205,"log_start":"1170'2668","ondisk_log_start":"1170'2668","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5946,"log_dups_size":0,"ondisk_log_size":5946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312147992,"num_objects":4764,"num_object_clones":0,"num_object_copies":14292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4764,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8614,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1218'8345","reported_seq":8404,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:07.372645+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:53:07.372645+0000","last_peered":"2024-08-17T17:53:07.372645+0000","last_clean":"2024-08-17T17:53:07.372645+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:53:07.372645+0000","last_undegraded":"2024-08-17T17:53:07.372645+0000","last_fullsized":"2024-08-17T17:53:07.372645+0000","mapping_epoch":1167,"log_start":"1170'1589","ondisk_log_start":"1170'1589","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6756,"log_dups_size":0,"ondisk_log_size":6756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305070080,"num_objects":4655,"num_object_clones":0,"num_object_copies":13965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8345,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1218'8453","reported_seq":13681,"reported_epoch":1218,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:11.093994+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:53:11.093994+0000","last_peered":"2024-08-17T17:53:11.093994+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:53:11.093994+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1170'2526","ondisk_log_start":"1170'2526","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5927,"log_dups_size":0,"ondisk_log_size":5927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305463296,"num_objects":4661,"num_object_clones":0,"num_object_copies":13983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":4661,"num_objects_unfound":0,"num_objects_dirty":4661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8453,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":4143},{"shards":"1,4,5","objects":518}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1218'8487","reported_seq":8546,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:07.748597+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:53:07.748597+0000","last_peered":"2024-08-17T17:53:07.748597+0000","last_clean":"2024-08-17T17:53:07.748597+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:53:07.748597+0000","last_undegraded":"2024-08-17T17:53:07.748597+0000","last_fullsized":"2024-08-17T17:53:07.748597+0000","mapping_epoch":1167,"log_start":"1170'1777","ondisk_log_start":"1170'1777","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6710,"log_dups_size":0,"ondisk_log_size":6710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301924352,"num_objects":4607,"num_object_clones":0,"num_object_copies":13821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8487,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1218'8472","reported_seq":8531,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:07.598687+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:53:07.598687+0000","last_peered":"2024-08-17T17:53:07.598687+0000","last_clean":"2024-08-17T17:53:07.598687+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:53:07.598687+0000","last_undegraded":"2024-08-17T17:53:07.598687+0000","last_fullsized":"2024-08-17T17:53:07.598687+0000","mapping_epoch":1167,"log_start":"1170'1769","ondisk_log_start":"1170'1769","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6703,"log_dups_size":0,"ondisk_log_size":6703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311558144,"num_objects":4754,"num_object_clones":0,"num_object_copies":14262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8472,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1218'8407","reported_seq":14264,"reported_epoch":1218,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:11.126162+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:53:11.126162+0000","last_peered":"2024-08-17T17:53:11.126162+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:53:11.126162+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1170'2475","ondisk_log_start":"1170'2475","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5932,"log_dups_size":0,"ondisk_log_size":5932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300351488,"num_objects":4583,"num_object_clones":0,"num_object_copies":13749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4459,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8407,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1912,"num_bytes_recovered":125304832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1196},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1218'8352","reported_seq":8421,"reported_epoch":1218,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:08.141725+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:53:08.141725+0000","last_peered":"2024-08-17T17:53:08.141725+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:53:08.141725+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'1677","ondisk_log_start":"1170'1677","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6675,"log_dups_size":0,"ondisk_log_size":6675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309592064,"num_objects":4724,"num_object_clones":0,"num_object_copies":14172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8352,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":4724},{"shards":"1,6","objects":-1109}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1218'8381","reported_seq":11442,"reported_epoch":1218,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:08.419736+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:53:08.419736+0000","last_peered":"2024-08-17T17:53:08.419736+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:53:08.419736+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1170'2406","ondisk_log_start":"1170'2406","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5975,"log_dups_size":0,"ondisk_log_size":5975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314114048,"num_objects":4793,"num_object_clones":0,"num_object_copies":14379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8381,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":3232},{"shards":"1,3,5","objects":1234}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1218'8401","reported_seq":8518,"reported_epoch":1218,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:08.981061+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:53:08.981061+0000","last_peered":"2024-08-17T17:53:08.981061+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:53:08.981061+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8401,"log_dups_size":0,"ondisk_log_size":8401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306249728,"num_objects":4673,"num_object_clones":0,"num_object_copies":14019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8401,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-480}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1218'8343","reported_seq":14520,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:09.008527+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:53:09.008527+0000","last_peered":"2024-08-17T17:53:09.008527+0000","last_clean":"2024-08-17T17:53:09.008527+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:53:09.008527+0000","last_undegraded":"2024-08-17T17:53:09.008527+0000","last_fullsized":"2024-08-17T17:53:09.008527+0000","mapping_epoch":1183,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8343,"log_dups_size":0,"ondisk_log_size":8343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308215808,"num_objects":4703,"num_object_clones":0,"num_object_copies":14109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8343,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1218'8417","reported_seq":8532,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:08.375192+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:53:08.375192+0000","last_peered":"2024-08-17T17:53:08.375192+0000","last_clean":"2024-08-17T17:53:08.375192+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:53:08.375192+0000","last_undegraded":"2024-08-17T17:53:08.375192+0000","last_fullsized":"2024-08-17T17:53:08.375192+0000","mapping_epoch":1167,"log_start":"1170'2491","ondisk_log_start":"1170'2491","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5926,"log_dups_size":0,"ondisk_log_size":5926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310837248,"num_objects":4743,"num_object_clones":0,"num_object_copies":14229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8417,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1218'8310","reported_seq":8369,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:08.123963+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:53:08.123963+0000","last_peered":"2024-08-17T17:53:08.123963+0000","last_clean":"2024-08-17T17:53:08.123963+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:53:08.123963+0000","last_undegraded":"2024-08-17T17:53:08.123963+0000","last_fullsized":"2024-08-17T17:53:08.123963+0000","mapping_epoch":1167,"log_start":"1170'1325","ondisk_log_start":"1170'1325","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6985,"log_dups_size":0,"ondisk_log_size":6985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303038464,"num_objects":4624,"num_object_clones":0,"num_object_copies":13872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8310,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1218'8536","reported_seq":17712,"reported_epoch":1218,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:08.364721+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:53:08.364721+0000","last_peered":"2024-08-17T17:53:08.364721+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:53:08.364721+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'1691","ondisk_log_start":"1170'1691","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6845,"log_dups_size":0,"ondisk_log_size":6845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316407808,"num_objects":4828,"num_object_clones":0,"num_object_copies":14484,"num_objects_missing_on_primary":4188,"num_objects_missing":4188,"num_objects_degraded":265,"num_objects_misplaced":8289,"num_objects_unfound":0,"num_objects_dirty":4828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8536,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3960,"num_bytes_recovered":259522560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":11},{"shards":"0,4,6","objects":1356},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":3196}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1218'8225","reported_seq":21526,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:08.196162+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:53:08.196162+0000","last_peered":"2024-08-17T17:53:08.196162+0000","last_clean":"2024-08-17T17:53:08.196162+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:53:08.196162+0000","last_undegraded":"2024-08-17T17:53:08.196162+0000","last_fullsized":"2024-08-17T17:53:08.196162+0000","mapping_epoch":1196,"log_start":"1170'2269","ondisk_log_start":"1170'2269","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5956,"log_dups_size":0,"ondisk_log_size":5956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301268992,"num_objects":4597,"num_object_clones":0,"num_object_copies":13791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8225,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1892,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.819772+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:53:01.819772+0000","last_peered":"2024-08-17T17:53:01.819772+0000","last_clean":"2024-08-17T17:53:01.819772+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:53:01.819772+0000","last_undegraded":"2024-08-17T17:53:01.819772+0000","last_fullsized":"2024-08-17T17:53:01.819772+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":495,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.193839+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:52:50.193839+0000","last_peered":"2024-08-17T17:52:50.193839+0000","last_clean":"2024-08-17T17:52:50.193839+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:52:50.193839+0000","last_undegraded":"2024-08-17T17:52:50.193839+0000","last_fullsized":"2024-08-17T17:52:50.193839+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1811,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.743199+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:53:01.743199+0000","last_peered":"2024-08-17T17:53:01.743199+0000","last_clean":"2024-08-17T17:53:01.743199+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:53:01.743199+0000","last_undegraded":"2024-08-17T17:53:01.743199+0000","last_fullsized":"2024-08-17T17:53:01.743199+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1835,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.741550+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:53:01.741550+0000","last_peered":"2024-08-17T17:53:01.741550+0000","last_clean":"2024-08-17T17:53:01.741550+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:53:01.741550+0000","last_undegraded":"2024-08-17T17:53:01.741550+0000","last_fullsized":"2024-08-17T17:53:01.741550+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":57,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.194343+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:52:50.194343+0000","last_peered":"2024-08-17T17:52:50.194343+0000","last_clean":"2024-08-17T17:52:50.194343+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:52:50.194343+0000","last_undegraded":"2024-08-17T17:52:50.194343+0000","last_fullsized":"2024-08-17T17:52:50.194343+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1229,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.741992+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:53:01.741992+0000","last_peered":"2024-08-17T17:53:01.741992+0000","last_clean":"2024-08-17T17:53:01.741992+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:53:01.741992+0000","last_undegraded":"2024-08-17T17:53:01.741992+0000","last_fullsized":"2024-08-17T17:53:01.741992+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":87,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.746276+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:53:01.746276+0000","last_peered":"2024-08-17T17:53:01.746276+0000","last_clean":"2024-08-17T17:53:01.746276+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:53:01.746276+0000","last_undegraded":"2024-08-17T17:53:01.746276+0000","last_fullsized":"2024-08-17T17:53:01.746276+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":607,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.742960+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:53:01.742960+0000","last_peered":"2024-08-17T17:53:01.742960+0000","last_clean":"2024-08-17T17:53:01.742960+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:53:01.742960+0000","last_undegraded":"2024-08-17T17:53:01.742960+0000","last_fullsized":"2024-08-17T17:53:01.742960+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1850,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.819046+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:53:01.819046+0000","last_peered":"2024-08-17T17:53:01.819046+0000","last_clean":"2024-08-17T17:53:01.819046+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:53:01.819046+0000","last_undegraded":"2024-08-17T17:53:01.819046+0000","last_fullsized":"2024-08-17T17:53:01.819046+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1852,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.819104+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:53:01.819104+0000","last_peered":"2024-08-17T17:53:01.819104+0000","last_clean":"2024-08-17T17:53:01.819104+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:53:01.819104+0000","last_undegraded":"2024-08-17T17:53:01.819104+0000","last_fullsized":"2024-08-17T17:53:01.819104+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":72,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.746952+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:53:01.746952+0000","last_peered":"2024-08-17T17:53:01.746952+0000","last_clean":"2024-08-17T17:53:01.746952+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:53:01.746952+0000","last_undegraded":"2024-08-17T17:53:01.746952+0000","last_fullsized":"2024-08-17T17:53:01.746952+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1824,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.743885+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:53:01.743885+0000","last_peered":"2024-08-17T17:53:01.743885+0000","last_clean":"2024-08-17T17:53:01.743885+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:53:01.743885+0000","last_undegraded":"2024-08-17T17:53:01.743885+0000","last_fullsized":"2024-08-17T17:53:01.743885+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1879,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.744412+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:53:01.744412+0000","last_peered":"2024-08-17T17:53:01.744412+0000","last_clean":"2024-08-17T17:53:01.744412+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:53:01.744412+0000","last_undegraded":"2024-08-17T17:53:01.744412+0000","last_fullsized":"2024-08-17T17:53:01.744412+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1873,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.819588+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:53:01.819588+0000","last_peered":"2024-08-17T17:53:01.819588+0000","last_clean":"2024-08-17T17:53:01.819588+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:53:01.819588+0000","last_undegraded":"2024-08-17T17:53:01.819588+0000","last_fullsized":"2024-08-17T17:53:01.819588+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1846,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.741528+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:53:01.741528+0000","last_peered":"2024-08-17T17:53:01.741528+0000","last_clean":"2024-08-17T17:53:01.741528+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:53:01.741528+0000","last_undegraded":"2024-08-17T17:53:01.741528+0000","last_fullsized":"2024-08-17T17:53:01.741528+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1192,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.194446+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:52:50.194446+0000","last_peered":"2024-08-17T17:52:50.194446+0000","last_clean":"2024-08-17T17:52:50.194446+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:52:50.194446+0000","last_undegraded":"2024-08-17T17:52:50.194446+0000","last_fullsized":"2024-08-17T17:52:50.194446+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1818,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.741317+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:53:01.741317+0000","last_peered":"2024-08-17T17:53:01.741317+0000","last_clean":"2024-08-17T17:53:01.741317+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:53:01.741317+0000","last_undegraded":"2024-08-17T17:53:01.741317+0000","last_fullsized":"2024-08-17T17:53:01.741317+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1867,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.819848+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:53:01.819848+0000","last_peered":"2024-08-17T17:53:01.819848+0000","last_clean":"2024-08-17T17:53:01.819848+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:53:01.819848+0000","last_undegraded":"2024-08-17T17:53:01.819848+0000","last_fullsized":"2024-08-17T17:53:01.819848+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1774,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.741457+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:53:01.741457+0000","last_peered":"2024-08-17T17:53:01.741457+0000","last_clean":"2024-08-17T17:53:01.741457+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:53:01.741457+0000","last_undegraded":"2024-08-17T17:53:01.741457+0000","last_fullsized":"2024-08-17T17:53:01.741457+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":511,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.205130+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:52:50.205130+0000","last_peered":"2024-08-17T17:52:50.205130+0000","last_clean":"2024-08-17T17:52:50.205130+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:52:50.205130+0000","last_undegraded":"2024-08-17T17:52:50.205130+0000","last_fullsized":"2024-08-17T17:52:50.205130+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1188,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.205015+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:52:50.205015+0000","last_peered":"2024-08-17T17:52:50.205015+0000","last_clean":"2024-08-17T17:52:50.205015+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:52:50.205015+0000","last_undegraded":"2024-08-17T17:52:50.205015+0000","last_fullsized":"2024-08-17T17:52:50.205015+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1872,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.747201+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:53:01.747201+0000","last_peered":"2024-08-17T17:53:01.747201+0000","last_clean":"2024-08-17T17:53:01.747201+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:53:01.747201+0000","last_undegraded":"2024-08-17T17:53:01.747201+0000","last_fullsized":"2024-08-17T17:53:01.747201+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":349,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.204980+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:52:50.204980+0000","last_peered":"2024-08-17T17:52:50.204980+0000","last_clean":"2024-08-17T17:52:50.204980+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:52:50.204980+0000","last_undegraded":"2024-08-17T17:52:50.204980+0000","last_fullsized":"2024-08-17T17:52:50.204980+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":495,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.194128+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:52:50.194128+0000","last_peered":"2024-08-17T17:52:50.194128+0000","last_clean":"2024-08-17T17:52:50.194128+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:52:50.194128+0000","last_undegraded":"2024-08-17T17:52:50.194128+0000","last_fullsized":"2024-08-17T17:52:50.194128+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1173,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.194009+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:52:50.194009+0000","last_peered":"2024-08-17T17:52:50.194009+0000","last_clean":"2024-08-17T17:52:50.194009+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:52:50.194009+0000","last_undegraded":"2024-08-17T17:52:50.194009+0000","last_fullsized":"2024-08-17T17:52:50.194009+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1181,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.194485+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:52:50.194485+0000","last_peered":"2024-08-17T17:52:50.194485+0000","last_clean":"2024-08-17T17:52:50.194485+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:52:50.194485+0000","last_undegraded":"2024-08-17T17:52:50.194485+0000","last_fullsized":"2024-08-17T17:52:50.194485+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1920,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.819466+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:53:01.819466+0000","last_peered":"2024-08-17T17:53:01.819466+0000","last_clean":"2024-08-17T17:53:01.819466+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:53:01.819466+0000","last_undegraded":"2024-08-17T17:53:01.819466+0000","last_fullsized":"2024-08-17T17:53:01.819466+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1803,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.741459+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:53:01.741459+0000","last_peered":"2024-08-17T17:53:01.741459+0000","last_clean":"2024-08-17T17:53:01.741459+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:53:01.741459+0000","last_undegraded":"2024-08-17T17:53:01.741459+0000","last_fullsized":"2024-08-17T17:53:01.741459+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1878,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.747757+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:53:01.747757+0000","last_peered":"2024-08-17T17:53:01.747757+0000","last_clean":"2024-08-17T17:53:01.747757+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:53:01.747757+0000","last_undegraded":"2024-08-17T17:53:01.747757+0000","last_fullsized":"2024-08-17T17:53:01.747757+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1805,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.742610+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:53:01.742610+0000","last_peered":"2024-08-17T17:53:01.742610+0000","last_clean":"2024-08-17T17:53:01.742610+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:53:01.742610+0000","last_undegraded":"2024-08-17T17:53:01.742610+0000","last_fullsized":"2024-08-17T17:53:01.742610+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1838,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.819166+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:53:01.819166+0000","last_peered":"2024-08-17T17:53:01.819166+0000","last_clean":"2024-08-17T17:53:01.819166+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:53:01.819166+0000","last_undegraded":"2024-08-17T17:53:01.819166+0000","last_fullsized":"2024-08-17T17:53:01.819166+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1208,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.205117+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:52:50.205117+0000","last_peered":"2024-08-17T17:52:50.205117+0000","last_clean":"2024-08-17T17:52:50.205117+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:52:50.205117+0000","last_undegraded":"2024-08-17T17:52:50.205117+0000","last_fullsized":"2024-08-17T17:52:50.205117+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1884,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.746454+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:53:01.746454+0000","last_peered":"2024-08-17T17:53:01.746454+0000","last_clean":"2024-08-17T17:53:01.746454+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:53:01.746454+0000","last_undegraded":"2024-08-17T17:53:01.746454+0000","last_fullsized":"2024-08-17T17:53:01.746454+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":71,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.746802+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:53:01.746802+0000","last_peered":"2024-08-17T17:53:01.746802+0000","last_clean":"2024-08-17T17:53:01.746802+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:53:01.746802+0000","last_undegraded":"2024-08-17T17:53:01.746802+0000","last_fullsized":"2024-08-17T17:53:01.746802+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1903,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.746372+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:53:01.746372+0000","last_peered":"2024-08-17T17:53:01.746372+0000","last_clean":"2024-08-17T17:53:01.746372+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:53:01.746372+0000","last_undegraded":"2024-08-17T17:53:01.746372+0000","last_fullsized":"2024-08-17T17:53:01.746372+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":640,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.743681+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:53:01.743681+0000","last_peered":"2024-08-17T17:53:01.743681+0000","last_clean":"2024-08-17T17:53:01.743681+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:53:01.743681+0000","last_undegraded":"2024-08-17T17:53:01.743681+0000","last_fullsized":"2024-08-17T17:53:01.743681+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1884,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.746540+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:53:01.746540+0000","last_peered":"2024-08-17T17:53:01.746540+0000","last_clean":"2024-08-17T17:53:01.746540+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:53:01.746540+0000","last_undegraded":"2024-08-17T17:53:01.746540+0000","last_fullsized":"2024-08-17T17:53:01.746540+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1213,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818865+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:53:01.818865+0000","last_peered":"2024-08-17T17:53:01.818865+0000","last_clean":"2024-08-17T17:53:01.818865+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:53:01.818865+0000","last_undegraded":"2024-08-17T17:53:01.818865+0000","last_fullsized":"2024-08-17T17:53:01.818865+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":131,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.747108+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:53:01.747108+0000","last_peered":"2024-08-17T17:53:01.747108+0000","last_clean":"2024-08-17T17:53:01.747108+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:53:01.747108+0000","last_undegraded":"2024-08-17T17:53:01.747108+0000","last_fullsized":"2024-08-17T17:53:01.747108+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1895,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.748724+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:53:01.748724+0000","last_peered":"2024-08-17T17:53:01.748724+0000","last_clean":"2024-08-17T17:53:01.748724+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:53:01.748724+0000","last_undegraded":"2024-08-17T17:53:01.748724+0000","last_fullsized":"2024-08-17T17:53:01.748724+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1873,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.744122+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:53:01.744122+0000","last_peered":"2024-08-17T17:53:01.744122+0000","last_clean":"2024-08-17T17:53:01.744122+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:53:01.744122+0000","last_undegraded":"2024-08-17T17:53:01.744122+0000","last_fullsized":"2024-08-17T17:53:01.744122+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1893,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818374+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:53:01.818374+0000","last_peered":"2024-08-17T17:53:01.818374+0000","last_clean":"2024-08-17T17:53:01.818374+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:53:01.818374+0000","last_undegraded":"2024-08-17T17:53:01.818374+0000","last_fullsized":"2024-08-17T17:53:01.818374+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1899,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.819636+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:53:01.819636+0000","last_peered":"2024-08-17T17:53:01.819636+0000","last_clean":"2024-08-17T17:53:01.819636+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:53:01.819636+0000","last_undegraded":"2024-08-17T17:53:01.819636+0000","last_fullsized":"2024-08-17T17:53:01.819636+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1879,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.901662+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:53:01.901662+0000","last_peered":"2024-08-17T17:53:01.901662+0000","last_clean":"2024-08-17T17:53:01.901662+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:53:01.901662+0000","last_undegraded":"2024-08-17T17:53:01.901662+0000","last_fullsized":"2024-08-17T17:53:01.901662+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1215,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.204799+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:52:50.204799+0000","last_peered":"2024-08-17T17:52:50.204799+0000","last_clean":"2024-08-17T17:52:50.204799+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:52:50.204799+0000","last_undegraded":"2024-08-17T17:52:50.204799+0000","last_fullsized":"2024-08-17T17:52:50.204799+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1161,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.194374+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:52:50.194374+0000","last_peered":"2024-08-17T17:52:50.194374+0000","last_clean":"2024-08-17T17:52:50.194374+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:52:50.194374+0000","last_undegraded":"2024-08-17T17:52:50.194374+0000","last_fullsized":"2024-08-17T17:52:50.194374+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1788,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.741909+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:53:01.741909+0000","last_peered":"2024-08-17T17:53:01.741909+0000","last_clean":"2024-08-17T17:53:01.741909+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:53:01.741909+0000","last_undegraded":"2024-08-17T17:53:01.741909+0000","last_fullsized":"2024-08-17T17:53:01.741909+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":119,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818312+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:53:01.818312+0000","last_peered":"2024-08-17T17:53:01.818312+0000","last_clean":"2024-08-17T17:53:01.818312+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:53:01.818312+0000","last_undegraded":"2024-08-17T17:53:01.818312+0000","last_fullsized":"2024-08-17T17:53:01.818312+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1527,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.204629+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:52:50.204629+0000","last_peered":"2024-08-17T17:52:50.204629+0000","last_clean":"2024-08-17T17:52:50.204629+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:52:50.204629+0000","last_undegraded":"2024-08-17T17:52:50.204629+0000","last_fullsized":"2024-08-17T17:52:50.204629+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1880,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.748854+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:53:01.748854+0000","last_peered":"2024-08-17T17:53:01.748854+0000","last_clean":"2024-08-17T17:53:01.748854+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:53:01.748854+0000","last_undegraded":"2024-08-17T17:53:01.748854+0000","last_fullsized":"2024-08-17T17:53:01.748854+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1819,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.741387+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:53:01.741387+0000","last_peered":"2024-08-17T17:53:01.741387+0000","last_clean":"2024-08-17T17:53:01.741387+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:53:01.741387+0000","last_undegraded":"2024-08-17T17:53:01.741387+0000","last_fullsized":"2024-08-17T17:53:01.741387+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":495,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.194287+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:52:50.194287+0000","last_peered":"2024-08-17T17:52:50.194287+0000","last_clean":"2024-08-17T17:52:50.194287+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:52:50.194287+0000","last_undegraded":"2024-08-17T17:52:50.194287+0000","last_fullsized":"2024-08-17T17:52:50.194287+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":558,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818226+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:53:01.818226+0000","last_peered":"2024-08-17T17:53:01.818226+0000","last_clean":"2024-08-17T17:53:01.818226+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:53:01.818226+0000","last_undegraded":"2024-08-17T17:53:01.818226+0000","last_fullsized":"2024-08-17T17:53:01.818226+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":574,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.820212+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:53:01.820212+0000","last_peered":"2024-08-17T17:53:01.820212+0000","last_clean":"2024-08-17T17:53:01.820212+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:53:01.820212+0000","last_undegraded":"2024-08-17T17:53:01.820212+0000","last_fullsized":"2024-08-17T17:53:01.820212+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1179,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.195400+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:52:50.195400+0000","last_peered":"2024-08-17T17:52:50.195400+0000","last_clean":"2024-08-17T17:52:50.195400+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:52:50.195400+0000","last_undegraded":"2024-08-17T17:52:50.195400+0000","last_fullsized":"2024-08-17T17:52:50.195400+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":590,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.742220+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:53:01.742220+0000","last_peered":"2024-08-17T17:53:01.742220+0000","last_clean":"2024-08-17T17:53:01.742220+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:53:01.742220+0000","last_undegraded":"2024-08-17T17:53:01.742220+0000","last_fullsized":"2024-08-17T17:53:01.742220+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1816,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.743042+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:53:01.743042+0000","last_peered":"2024-08-17T17:53:01.743042+0000","last_clean":"2024-08-17T17:53:01.743042+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:53:01.743042+0000","last_undegraded":"2024-08-17T17:53:01.743042+0000","last_fullsized":"2024-08-17T17:53:01.743042+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1912,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.819330+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:53:01.819330+0000","last_peered":"2024-08-17T17:53:01.819330+0000","last_clean":"2024-08-17T17:53:01.819330+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:53:01.819330+0000","last_undegraded":"2024-08-17T17:53:01.819330+0000","last_fullsized":"2024-08-17T17:53:01.819330+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":623,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.742153+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:53:01.742153+0000","last_peered":"2024-08-17T17:53:01.742153+0000","last_clean":"2024-08-17T17:53:01.742153+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:53:01.742153+0000","last_undegraded":"2024-08-17T17:53:01.742153+0000","last_fullsized":"2024-08-17T17:53:01.742153+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1168,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.195305+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:52:50.195305+0000","last_peered":"2024-08-17T17:52:50.195305+0000","last_clean":"2024-08-17T17:52:50.195305+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:52:50.195305+0000","last_undegraded":"2024-08-17T17:52:50.195305+0000","last_fullsized":"2024-08-17T17:52:50.195305+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1167,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.205125+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:52:50.205125+0000","last_peered":"2024-08-17T17:52:50.205125+0000","last_clean":"2024-08-17T17:52:50.205125+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:52:50.205125+0000","last_undegraded":"2024-08-17T17:52:50.205125+0000","last_fullsized":"2024-08-17T17:52:50.205125+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":607,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.744507+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:53:01.744507+0000","last_peered":"2024-08-17T17:53:01.744507+0000","last_clean":"2024-08-17T17:53:01.744507+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:53:01.744507+0000","last_undegraded":"2024-08-17T17:53:01.744507+0000","last_fullsized":"2024-08-17T17:53:01.744507+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1185,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.205246+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:52:50.205246+0000","last_peered":"2024-08-17T17:52:50.205246+0000","last_clean":"2024-08-17T17:52:50.205246+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:52:50.205246+0000","last_undegraded":"2024-08-17T17:52:50.205246+0000","last_fullsized":"2024-08-17T17:52:50.205246+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":69,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.204586+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:52:50.204586+0000","last_peered":"2024-08-17T17:52:50.204586+0000","last_clean":"2024-08-17T17:52:50.204586+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:52:50.204586+0000","last_undegraded":"2024-08-17T17:52:50.204586+0000","last_fullsized":"2024-08-17T17:52:50.204586+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1201,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.205262+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:52:50.205262+0000","last_peered":"2024-08-17T17:52:50.205262+0000","last_clean":"2024-08-17T17:52:50.205262+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:52:50.205262+0000","last_undegraded":"2024-08-17T17:52:50.205262+0000","last_fullsized":"2024-08-17T17:52:50.205262+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1218,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.194040+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:52:50.194040+0000","last_peered":"2024-08-17T17:52:50.194040+0000","last_clean":"2024-08-17T17:52:50.194040+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:52:50.194040+0000","last_undegraded":"2024-08-17T17:52:50.194040+0000","last_fullsized":"2024-08-17T17:52:50.194040+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1817,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.742503+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:53:01.742503+0000","last_peered":"2024-08-17T17:53:01.742503+0000","last_clean":"2024-08-17T17:53:01.742503+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:53:01.742503+0000","last_undegraded":"2024-08-17T17:53:01.742503+0000","last_fullsized":"2024-08-17T17:53:01.742503+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1162,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.204864+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:52:50.204864+0000","last_peered":"2024-08-17T17:52:50.204864+0000","last_clean":"2024-08-17T17:52:50.204864+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:52:50.204864+0000","last_undegraded":"2024-08-17T17:52:50.204864+0000","last_fullsized":"2024-08-17T17:52:50.204864+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1870,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.819493+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:53:01.819493+0000","last_peered":"2024-08-17T17:53:01.819493+0000","last_clean":"2024-08-17T17:53:01.819493+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:53:01.819493+0000","last_undegraded":"2024-08-17T17:53:01.819493+0000","last_fullsized":"2024-08-17T17:53:01.819493+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1871,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818677+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:53:01.818677+0000","last_peered":"2024-08-17T17:53:01.818677+0000","last_clean":"2024-08-17T17:53:01.818677+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:53:01.818677+0000","last_undegraded":"2024-08-17T17:53:01.818677+0000","last_fullsized":"2024-08-17T17:53:01.818677+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":575,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.204907+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:52:50.204907+0000","last_peered":"2024-08-17T17:52:50.204907+0000","last_clean":"2024-08-17T17:52:50.204907+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:52:50.204907+0000","last_undegraded":"2024-08-17T17:52:50.204907+0000","last_fullsized":"2024-08-17T17:52:50.204907+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1187,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:52:50.204860+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:52:50.204860+0000","last_peered":"2024-08-17T17:52:50.204860+0000","last_clean":"2024-08-17T17:52:50.204860+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:52:50.204860+0000","last_undegraded":"2024-08-17T17:52:50.204860+0000","last_fullsized":"2024-08-17T17:52:50.204860+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1818,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.742324+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:53:01.742324+0000","last_peered":"2024-08-17T17:53:01.742324+0000","last_clean":"2024-08-17T17:53:01.742324+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:53:01.742324+0000","last_undegraded":"2024-08-17T17:53:01.742324+0000","last_fullsized":"2024-08-17T17:53:01.742324+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":4922212376,"num_objects":75108,"num_object_clones":0,"num_object_copies":225324,"num_objects_missing_on_primary":4188,"num_objects_missing":4188,"num_objects_degraded":18494,"num_objects_misplaced":12950,"num_objects_unfound":0,"num_objects_dirty":75108,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":134502,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33750,"num_bytes_recovered":2033385472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15363289088,"data_stored":15363276872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106976,"ondisk_log_size":106976,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234387,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2341904,"kb_used_data":2032072,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91403248,"statfs":{"total":95995035648,"available":93596925952,"internally_reserved":0,"allocated":2080841728,"data_stored":2075984447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496022,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2644500,"kb_used_data":2334668,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91100652,"statfs":{"total":95995035648,"available":93287067648,"internally_reserved":0,"allocated":2390700032,"data_stored":2385839156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757661,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326274,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2900156,"kb_used_data":2683700,"kb_used_omap":57,"kb_used_meta":216390,"kb_avail":90844996,"statfs":{"total":95995035648,"available":93025275904,"internally_reserved":0,"allocated":2748108800,"data_stored":2743266699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":221584123},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405513,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2614916,"kb_used_data":2412284,"kb_used_omap":61,"kb_used_meta":202562,"kb_avail":91130236,"statfs":{"total":95995035648,"available":93317361664,"internally_reserved":0,"allocated":2470178816,"data_stored":2465333687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":207424440},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452048,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1970488,"kb_used_data":1804016,"kb_used_omap":54,"kb_used_meta":166409,"kb_avail":91774664,"statfs":{"total":95995035648,"available":93977255936,"internally_reserved":0,"allocated":1847312384,"data_stored":1842474244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":170403214},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474739,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1679048,"kb_used_data":1475776,"kb_used_omap":119,"kb_used_meta":203144,"kb_avail":92066104,"statfs":{"total":95995035648,"available":94275690496,"internally_reserved":0,"allocated":1511194624,"data_stored":1506341320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":208020190},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019295,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1495863296,"data_stored":1495859224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1833304064,"data_stored":1833304064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2456354816,"data_stored":2456354816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2734952448,"data_stored":2734948376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1079246848,"data_stored":1079246848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1321795584,"data_stored":1321795584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2376142848,"data_stored":2376138776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2065629184,"data_stored":2065629184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:53:17.841 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:53:17.912 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:17.933 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:17.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:17.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:18.035 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:18.056 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:18.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:18.158 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:18.179 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:18.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:18.281 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:18.302 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:18.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:18.404 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:18.425 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:18.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:18.528 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:18.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.551 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:18.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:18.653 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.675 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:18.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:18.776 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:18.797 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:18.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:18.899 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:18.921 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:18.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:18.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:19.023 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:19.045 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:19.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:19.147 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:19.170 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:19.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:19.272 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:19.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.294 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:19.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:19.395 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:19.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:19 smithi130 ceph-mon[105521]: pgmap v1780: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 18440/222195 objects degraded (8.299%); 12761/222195 objects misplaced (5.743%); 890 KiB/s, 13 objects/s recovering 2024-08-17T17:53:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:19 smithi130 ceph-mon[110068]: pgmap v1780: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 18440/222195 objects degraded (8.299%); 12761/222195 objects misplaced (5.743%); 890 KiB/s, 13 objects/s recovering 2024-08-17T17:53:19.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:19.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:19.519 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:19.540 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:19.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:19 smithi179 ceph-mon[93519]: pgmap v1780: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 18440/222195 objects degraded (8.299%); 12761/222195 objects misplaced (5.743%); 890 KiB/s, 13 objects/s recovering 2024-08-17T17:53:19.642 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:19.664 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:19.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:19.765 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:19.786 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:19.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:19.888 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:19.903 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:53:19.903 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:53:19.903 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 0 2024-08-17T17:53:19.904 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:53:19.912 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:19.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:19.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:20.014 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:20.035 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:20.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:20.137 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:20.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.158 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:20.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:53:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:53:19] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-08-17T17:53:20.260 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:20.282 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:20.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:20.384 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:20.408 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:20.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:20.510 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:20.532 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:20.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:20.535 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2630270343' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:53:20.535 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:20 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:53:20.535 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2630270343' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:53:20.535 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:20 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:53:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2630270343' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:53:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:20 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:53:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:53:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:53:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:53:20.634 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:20.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.656 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:20.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:20.757 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:20.782 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:20.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:20.884 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:20.912 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:20.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:20.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:21.014 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:21.035 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:21.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:21.137 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:21.159 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:21.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:21.260 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:21.283 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:21.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:21.369 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:53:21.385 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:21.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.407 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:21.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:21.508 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:21.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.531 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:21.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:21 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:21 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:53:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:21 smithi179 ceph-mon[93519]: osdmap e1219: 8 total, 8 up, 6 in 2024-08-17T17:53:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18440/222195 objects degraded (8.299%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:21 smithi179 ceph-mon[93519]: pgmap v1782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 18440/221949 objects degraded (8.308%); 12761/221949 objects misplaced (5.750%); 825 KiB/s, 12 objects/s recovering 2024-08-17T17:53:21.631 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:21.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:21.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:21.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:21 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:21 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:53:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:21 smithi130 ceph-mon[105521]: osdmap e1219: 8 total, 8 up, 6 in 2024-08-17T17:53:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18440/222195 objects degraded (8.299%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:21 smithi130 ceph-mon[105521]: pgmap v1782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 18440/221949 objects degraded (8.308%); 12761/221949 objects misplaced (5.750%); 825 KiB/s, 12 objects/s recovering 2024-08-17T17:53:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:21 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:21 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:53:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:21 smithi130 ceph-mon[110068]: osdmap e1219: 8 total, 8 up, 6 in 2024-08-17T17:53:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18440/222195 objects degraded (8.299%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:21 smithi130 ceph-mon[110068]: pgmap v1782: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 18440/221949 objects degraded (8.308%); 12761/221949 objects misplaced (5.750%); 825 KiB/s, 12 objects/s recovering 2024-08-17T17:53:21.758 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:21.782 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:21.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:21.883 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:21.908 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:21.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:21.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:21.970 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:53:21.971 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:53:21.989 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:53:21.990 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:53:22.008 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:53:22.008 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:53:22.010 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:22.030 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:53:22.030 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:53:22.032 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:22.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:22.057 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:53:22.057 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:53:22.083 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:53:22.083 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:53:22.110 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:53:22.110 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:53:22.132 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:53:22.132 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:53:22.132 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:53:22.134 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:22.154 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:22.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:22.256 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:22.277 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:22.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:22.379 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:22.406 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:22.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:22.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:22 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4047563530' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:22.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:22 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4047563530' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:22.508 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:22.530 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:22.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:22.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:22 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4047563530' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:22.632 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:22.657 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:22.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:22.759 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:22.784 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:22.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:22.886 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:22.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:22.910 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:22.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:23.011 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:23.032 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:23.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:23.134 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:23.155 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:23.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:23.257 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:23.278 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:23.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:23.380 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:23.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:23 smithi130 ceph-mon[105521]: pgmap v1783: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 18360/221739 objects degraded (8.280%); 12731/221739 objects misplaced (5.741%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:53:23.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:23 smithi130 ceph-mon[110068]: pgmap v1783: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 18360/221739 objects degraded (8.280%); 12731/221739 objects misplaced (5.741%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:53:23.401 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:23.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:23.446 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:23.482 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:23.503 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:23.530 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:23.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:23 smithi179 ceph-mon[93519]: pgmap v1783: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 122 op/s; 18360/221739 objects degraded (8.280%); 12731/221739 objects misplaced (5.741%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:53:23.631 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:23.652 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:23.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:23.754 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:23.775 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:23.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:23.877 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:23.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:23.898 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:23.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:23.999 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:24.025 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:24.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:24.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:24.148 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:24.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:24.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:24.274 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:24.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:24.376 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:24.398 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:24.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:24.500 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:24.521 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:24.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:24.623 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:24.645 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:24.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:24.747 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:24.773 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:24.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:24.803 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:53:24.875 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:24.896 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:24.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:24.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:24.998 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:25.019 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:25.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:25.121 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:25.149 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:25.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:25.251 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:25.274 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:25.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:25.376 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:25.397 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:25.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:25.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:25 smithi130 ceph-mon[105521]: pgmap v1784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 18360/220284 objects degraded (8.335%); 12653/220284 objects misplaced (5.744%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:53:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:25 smithi130 ceph-mon[110068]: pgmap v1784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 18360/220284 objects degraded (8.335%); 12653/220284 objects misplaced (5.744%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:53:25.422 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:53:25.500 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:25.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.521 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:25.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:25 smithi179 ceph-mon[93519]: pgmap v1784: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 18360/220284 objects degraded (8.335%); 12653/220284 objects misplaced (5.744%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:53:25.623 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:25.650 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:25.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:25.751 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:25.773 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:25.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:25.874 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:25.900 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:25.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:25.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:26.002 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:26.024 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:26.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:26.126 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:26.147 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:26.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:26.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18360/220284 objects degraded (8.335%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:26.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18360/220284 objects degraded (8.335%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:26.249 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:26.273 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:26.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:26.375 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:26.396 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:26.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:26.498 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:26.519 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:26.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:26.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18360/220284 objects degraded (8.335%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:26.621 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:26.642 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:26.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:26.744 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:26.769 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:26.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:26.871 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:26.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:26.893 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:26.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:26.994 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:27.016 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:27.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:27.119 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:27.144 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:27.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:27.245 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:27.270 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:27.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:27.336 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:27 smithi130 ceph-mon[105521]: pgmap v1785: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 18318/220137 objects degraded (8.321%); 12625/220137 objects misplaced (5.735%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:53:27.337 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:27 smithi130 ceph-mon[110068]: pgmap v1785: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 18318/220137 objects degraded (8.321%); 12625/220137 objects misplaced (5.735%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:53:27.372 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:27.394 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:27.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:27.496 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:27.517 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:27.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:27 smithi179 ceph-mon[93519]: pgmap v1785: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 18318/220137 objects degraded (8.321%); 12625/220137 objects misplaced (5.735%); 889 KiB/s, 13 objects/s recovering 2024-08-17T17:53:27.619 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:27.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.643 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:27.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:27.744 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:27.766 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:27.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:27.806 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:27.868 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:27.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:27.890 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:27.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:27.990 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:28.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.013 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:28.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:28.114 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:28.135 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:28.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:28.237 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:28.258 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:28.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:28.346 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:28.346 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:28.346 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:28.346 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:28.346 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:53:28.347 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:53:28.347 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:28.347 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:28.347 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:28.347 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:28.347 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:28.347 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:53:28.347 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:53:28.347 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:28.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:28.384 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:28.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:28.485 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:28.508 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:28.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:53:28.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:53:28.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:53:28.609 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:28.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.631 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:28.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:28.732 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:28.755 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:28.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:28.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:28.881 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:28.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:28.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:28.905 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:53:28.905 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 18318/220137 objects degraded (8.321%), 6 pgs degraded, 6 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1530,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1219,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":73379,"data_bytes":4807792613,"bytes_used":14158082048,"bytes_avail":561812131840,"bytes_total":575970213888,"degraded_objects":18318,"degraded_total":220137,"degraded_ratio":0.083211818095095327,"misplaced_objects":12625,"misplaced_total":220137,"misplaced_ratio":0.057350649822610462,"recovering_objects_per_sec":13,"recovering_bytes_per_sec":910715,"recovering_keys_per_sec":0,"num_objects_recovered":139,"num_bytes_recovered":9109504,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":123},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":25,"modified":"2024-08-17T17:49:39.646037+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (5m)\n [==========================..] (remaining: 25s)","progress":0.93258428573608398,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:53:28.983 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:29.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.005 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:29.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:29.106 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:29.127 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:29.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:29.229 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:29.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.251 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:29.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:29.352 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:29.375 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:29.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:29.477 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:29.503 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:29.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1511923160' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:53:29.662 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:29 smithi179 ceph-mon[93519]: pgmap v1786: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 18318/219039 objects degraded (8.363%); 12543/219039 objects misplaced (5.726%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:53:29.662 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:29.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1511923160' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:53:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:29 smithi130 ceph-mon[105521]: pgmap v1786: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 18318/219039 objects degraded (8.363%); 12543/219039 objects misplaced (5.726%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:53:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1511923160' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:53:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:29 smithi130 ceph-mon[110068]: pgmap v1786: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 105 op/s; 18318/219039 objects degraded (8.363%); 12543/219039 objects misplaced (5.726%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:53:29.687 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:29.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:29.789 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:29.810 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:29.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:29.912 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:29.933 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:29.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:29.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:30.035 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:30.056 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:30.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:30.093 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:53:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:53:29] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-08-17T17:53:30.126 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:53:30.126 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:53:30.158 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:30.193 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:30.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:30.295 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:30.324 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:30.325 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:30.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:30.369 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:53:30.369 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:53:30.427 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:30.449 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:30.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:30.550 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:30.573 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:30.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:53:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:53:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:53:30.674 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:30.697 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:30.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:30.798 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:30.820 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:30.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:30.923 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:30.944 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:30.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:30.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:31.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:31.047 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:30 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18318/219039 objects degraded (8.363%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:31.048 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:30 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18318/219039 objects degraded (8.363%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:31.078 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:31.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:31.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:30 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18318/219039 objects degraded (8.363%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:31.183 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:31.204 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:31.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:31.306 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:31.328 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:31.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:31.429 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:31.451 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:31.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:31.552 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:31.575 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:31.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:31.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:31.698 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:31.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:31.800 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:31.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:31.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:31 smithi130 ceph-mon[105521]: pgmap v1787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 18318/218838 objects degraded (8.371%); 12543/218838 objects misplaced (5.732%); 831 KiB/s, 12 objects/s recovering 2024-08-17T17:53:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:31 smithi130 ceph-mon[110068]: pgmap v1787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 18318/218838 objects degraded (8.371%); 12543/218838 objects misplaced (5.732%); 831 KiB/s, 12 objects/s recovering 2024-08-17T17:53:31.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:31.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:31.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:31.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:32.052 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:32.078 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:32.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:31 smithi179 ceph-mon[93519]: pgmap v1787: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 95 op/s; 18318/218838 objects degraded (8.371%); 12543/218838 objects misplaced (5.732%); 831 KiB/s, 12 objects/s recovering 2024-08-17T17:53:32.180 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:32.201 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:32.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:32.303 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:32.329 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:32.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:32.430 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:32.453 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:32.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:32.473 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:53:32.554 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:32.577 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:32.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:32.679 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:32.704 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:32.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:32.806 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:32.834 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:32.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:32.935 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:32.957 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:32.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:32.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:33.059 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:33.085 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:33.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:33.187 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:33.216 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:33.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:33.318 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:33.331 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:33.339 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:33.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:33.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:33 smithi130 ceph-mon[105521]: pgmap v1788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 18250/218631 objects degraded (8.347%); 12508/218631 objects misplaced (5.721%); 933 KiB/s, 14 objects/s recovering 2024-08-17T17:53:33.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:33 smithi130 ceph-mon[110068]: pgmap v1788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 18250/218631 objects degraded (8.347%); 12508/218631 objects misplaced (5.721%); 933 KiB/s, 14 objects/s recovering 2024-08-17T17:53:33.441 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:33.464 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:33.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:33.566 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:33.587 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:33.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:33.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:33 smithi179 ceph-mon[93519]: pgmap v1788: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 18250/218631 objects degraded (8.347%); 12508/218631 objects misplaced (5.721%); 933 KiB/s, 14 objects/s recovering 2024-08-17T17:53:33.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:33.710 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:33.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:33.812 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:33.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:33.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:33.863 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:53:33.864 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:53:33.882 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:53:33.882 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:53:33.902 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:53:33.903 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:53:33.920 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:53:33.921 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:53:33.936 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:33.937 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:53:33.937 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:53:33.954 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:53:33.954 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:53:33.958 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:33.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:33.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:33.970 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:53:33.970 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:53:33.986 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:53:33.986 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:53:34.060 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:34.081 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:34.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:34.128 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T17:53:34.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:34.205 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:34.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:34.307 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:34.330 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:34.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:34 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1335294561' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:34 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:34 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1714604328' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:34 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1335294561' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:34 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:34 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1714604328' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:34.432 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:34.458 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:34.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:34.560 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:34.581 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:34.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:34.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:34 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1335294561' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:34.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:34 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:34.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:34 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1714604328' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:34.683 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:34.704 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:34.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:34.806 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:34.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.830 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:34.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:34.931 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:34.953 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:34.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:34.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:35.054 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:35.077 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:35.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:35.178 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:35.200 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:35.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:35.302 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:35.324 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:35.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:35 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:35 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:35 smithi130 ceph-mon[105521]: osdmap e1220: 8 total, 8 up, 6 in 2024-08-17T17:53:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:35 smithi130 ceph-mon[105521]: pgmap v1790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 18250/216663 objects degraded (8.423%); 12434/216663 objects misplaced (5.739%); 838 KiB/s, 13 objects/s recovering 2024-08-17T17:53:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:35 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:35 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:35 smithi130 ceph-mon[110068]: osdmap e1220: 8 total, 8 up, 6 in 2024-08-17T17:53:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:35 smithi130 ceph-mon[110068]: pgmap v1790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 18250/216663 objects degraded (8.423%); 12434/216663 objects misplaced (5.739%); 838 KiB/s, 13 objects/s recovering 2024-08-17T17:53:35.428 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:35.450 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:35.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:35.551 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:35.573 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:35.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:35 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:35 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:35 smithi179 ceph-mon[93519]: osdmap e1220: 8 total, 8 up, 6 in 2024-08-17T17:53:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:35 smithi179 ceph-mon[93519]: pgmap v1790: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 18250/216663 objects degraded (8.423%); 12434/216663 objects misplaced (5.739%); 838 KiB/s, 13 objects/s recovering 2024-08-17T17:53:35.675 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:35.696 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:35.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:35.798 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:35.820 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:35.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:35.922 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:35.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:35.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:35.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:36.045 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:36.067 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:36.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:36.169 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:36.192 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:36.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:36.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18250/216663 objects degraded (8.423%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:36.244 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18250/216663 objects degraded (8.423%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:36.293 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:36.315 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:36.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:36.417 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:36.438 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:36.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:36.540 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:36.561 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:36.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:36.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18250/216663 objects degraded (8.423%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:36.663 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:36.684 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:36.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:36.786 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:36.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.810 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:36.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:36.912 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:36.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:36.934 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:36.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:37.035 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:37.056 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:37.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:37.158 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:37.179 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:37.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:37.281 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:37.302 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:37.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:37.404 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:37.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:37 smithi130 ceph-mon[105521]: pgmap v1791: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 18174/216393 objects degraded (8.399%); 12382/216393 objects misplaced (5.722%); 947 KiB/s, 14 objects/s recovering 2024-08-17T17:53:37.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:37 smithi130 ceph-mon[110068]: pgmap v1791: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 18174/216393 objects degraded (8.399%); 12382/216393 objects misplaced (5.722%); 947 KiB/s, 14 objects/s recovering 2024-08-17T17:53:37.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.426 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:37.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:37.526 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:37.548 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:37.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:37 smithi179 ceph-mon[93519]: pgmap v1791: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 124 op/s; 18174/216393 objects degraded (8.399%); 12382/216393 objects misplaced (5.722%); 947 KiB/s, 14 objects/s recovering 2024-08-17T17:53:37.650 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:37.672 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:37.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:37.683 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:37.774 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:37.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.795 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:37.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:37.870 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:53:37.871 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:53:37.896 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:37.920 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:37.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:37.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:38.022 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:38.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.044 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:38.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:38.145 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:38.168 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:38.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:38.270 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:38.275 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:53:38.292 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:38.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:38.301 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:38 smithi130 ceph-mon[105521]: from='client.55210 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:38.301 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:38 smithi130 ceph-mon[110068]: from='client.55210 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:38.394 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:38.420 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:38.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:38.522 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:38.543 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:38.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:38 smithi179 ceph-mon[93519]: from='client.55210 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:38.645 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:38.668 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:38.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:38.748 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1791,"stamp":"2024-08-17T17:53:37.040947+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4726003685,"num_objects":72131,"num_object_clones":0,"num_object_copies":216393,"num_objects_missing_on_primary":3991,"num_objects_missing":3991,"num_objects_degraded":18174,"num_objects_misplaced":12382,"num_objects_unfound":0,"num_objects_dirty":72131,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":137783,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34131,"num_bytes_recovered":2058230797,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107486,"ondisk_log_size":107486,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13624344,"kb_used_data":12269924,"kb_used_omap":295,"kb_used_meta":1354072,"kb_avail":548846568,"statfs":{"total":575970213888,"available":562018885632,"internally_reserved":0,"allocated":12564402176,"data_stored":12535253293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1386570191},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-81788928,"num_objects":-1248,"num_object_clones":0,"num_object_copies":-3744,"num_objects_missing_on_primary":-80,"num_objects_missing":-80,"num_objects_degraded":-144,"num_objects_misplaced":-243,"num_objects_unfound":0,"num_objects_dirty":-1248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1248,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":148,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002511"},"pg_stats":[{"pgid":"45.f","version":"1219'8609","reported_seq":19126,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:33.245885+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:53:33.245885+0000","last_peered":"2024-08-17T17:53:33.245885+0000","last_clean":"2024-08-17T17:53:33.245885+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:53:33.245885+0000","last_undegraded":"2024-08-17T17:53:33.245885+0000","last_fullsized":"2024-08-17T17:53:33.245885+0000","mapping_epoch":1194,"log_start":"1170'1731","ondisk_log_start":"1170'1731","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6878,"log_dups_size":0,"ondisk_log_size":6878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298516480,"num_objects":4555,"num_object_clones":0,"num_object_copies":13665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8609,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1219'8526","reported_seq":18985,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:33.007549+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:53:33.007549+0000","last_peered":"2024-08-17T17:53:33.007549+0000","last_clean":"2024-08-17T17:53:33.007549+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:53:33.007549+0000","last_undegraded":"2024-08-17T17:53:33.007549+0000","last_fullsized":"2024-08-17T17:53:33.007549+0000","mapping_epoch":1211,"log_start":"1170'1532","ondisk_log_start":"1170'1532","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6994,"log_dups_size":0,"ondisk_log_size":6994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292814848,"num_objects":4468,"num_object_clones":0,"num_object_copies":13404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8526,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1219'8806","reported_seq":19630,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:33.225617+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:53:33.225617+0000","last_peered":"2024-08-17T17:53:33.225617+0000","last_clean":"2024-08-17T17:53:33.225617+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:53:33.225617+0000","last_undegraded":"2024-08-17T17:53:33.225617+0000","last_fullsized":"2024-08-17T17:53:33.225617+0000","mapping_epoch":1205,"log_start":"1171'2868","ondisk_log_start":"1171'2868","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5938,"log_dups_size":0,"ondisk_log_size":5938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299565080,"num_objects":4572,"num_object_clones":0,"num_object_copies":13716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4572,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8806,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1219'8550","reported_seq":8610,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:33.126449+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:53:33.126449+0000","last_peered":"2024-08-17T17:53:33.126449+0000","last_clean":"2024-08-17T17:53:33.126449+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:53:33.126449+0000","last_undegraded":"2024-08-17T17:53:33.126449+0000","last_fullsized":"2024-08-17T17:53:33.126449+0000","mapping_epoch":1167,"log_start":"1170'1789","ondisk_log_start":"1170'1789","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6761,"log_dups_size":0,"ondisk_log_size":6761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291635200,"num_objects":4450,"num_object_clones":0,"num_object_copies":13350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8550,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1220'8632","reported_seq":13864,"reported_epoch":1220,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:36.798028+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:53:36.798028+0000","last_peered":"2024-08-17T17:53:36.798028+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:53:36.798028+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1171'2726","ondisk_log_start":"1171'2726","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5906,"log_dups_size":0,"ondisk_log_size":5906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293732352,"num_objects":4482,"num_object_clones":0,"num_object_copies":13446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":4482,"num_objects_unfound":0,"num_objects_dirty":4482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8632,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":3964},{"shards":"1,4,5","objects":518}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1219'8673","reported_seq":8733,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:33.050595+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:53:33.050595+0000","last_peered":"2024-08-17T17:53:33.050595+0000","last_clean":"2024-08-17T17:53:33.050595+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:53:33.050595+0000","last_undegraded":"2024-08-17T17:53:33.050595+0000","last_fullsized":"2024-08-17T17:53:33.050595+0000","mapping_epoch":1167,"log_start":"1170'1977","ondisk_log_start":"1170'1977","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6696,"log_dups_size":0,"ondisk_log_size":6696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289734656,"num_objects":4421,"num_object_clones":0,"num_object_copies":13263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8673,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1219'8645","reported_seq":8705,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:33.025738+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:53:33.025738+0000","last_peered":"2024-08-17T17:53:33.025738+0000","last_clean":"2024-08-17T17:53:33.025738+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:53:33.025738+0000","last_undegraded":"2024-08-17T17:53:33.025738+0000","last_fullsized":"2024-08-17T17:53:33.025738+0000","mapping_epoch":1167,"log_start":"1170'1969","ondisk_log_start":"1170'1969","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6676,"log_dups_size":0,"ondisk_log_size":6676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300220416,"num_objects":4581,"num_object_clones":0,"num_object_copies":13743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8645,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1220'8567","reported_seq":14908,"reported_epoch":1220,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:36.824225+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:53:36.824225+0000","last_peered":"2024-08-17T17:53:36.824225+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:53:36.824225+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1171'2675","ondisk_log_start":"1171'2675","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5892,"log_dups_size":0,"ondisk_log_size":5892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289865728,"num_objects":4423,"num_object_clones":0,"num_object_copies":13269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4139,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8567,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2072,"num_bytes_recovered":135790592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1036},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1219'8555","reported_seq":8625,"reported_epoch":1219,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:33.170986+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:53:33.170986+0000","last_peered":"2024-08-17T17:53:33.170986+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:53:33.170986+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'1877","ondisk_log_start":"1170'1877","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6678,"log_dups_size":0,"ondisk_log_size":6678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296288256,"num_objects":4521,"num_object_clones":0,"num_object_copies":13563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8555,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":4521},{"shards":"1,6","objects":-1312}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1219'8573","reported_seq":11636,"reported_epoch":1219,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:33.214899+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:53:33.214899+0000","last_peered":"2024-08-17T17:53:33.214899+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:53:33.214899+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1170'2606","ondisk_log_start":"1170'2606","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5967,"log_dups_size":0,"ondisk_log_size":5967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301531136,"num_objects":4601,"num_object_clones":0,"num_object_copies":13803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8573,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":3040},{"shards":"1,3,5","objects":1042}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1219'8565","reported_seq":8684,"reported_epoch":1219,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:33.177339+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:53:33.177339+0000","last_peered":"2024-08-17T17:53:33.177339+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:53:33.177339+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8565,"log_dups_size":0,"ondisk_log_size":8565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295501824,"num_objects":4509,"num_object_clones":0,"num_object_copies":13527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8565,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-644}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1219'8537","reported_seq":14716,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:33.241578+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:53:33.241578+0000","last_peered":"2024-08-17T17:53:33.241578+0000","last_clean":"2024-08-17T17:53:33.241578+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:53:33.241578+0000","last_undegraded":"2024-08-17T17:53:33.241578+0000","last_fullsized":"2024-08-17T17:53:33.241578+0000","mapping_epoch":1183,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8537,"log_dups_size":0,"ondisk_log_size":8537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295501824,"num_objects":4509,"num_object_clones":0,"num_object_copies":13527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8537,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1219'8636","reported_seq":8753,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:33.256896+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:53:33.256896+0000","last_peered":"2024-08-17T17:53:33.256896+0000","last_clean":"2024-08-17T17:53:33.256896+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:53:33.256896+0000","last_undegraded":"2024-08-17T17:53:33.256896+0000","last_fullsized":"2024-08-17T17:53:33.256896+0000","mapping_epoch":1167,"log_start":"1171'2691","ondisk_log_start":"1171'2691","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5945,"log_dups_size":0,"ondisk_log_size":5945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296484864,"num_objects":4524,"num_object_clones":0,"num_object_copies":13572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8636,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1219'8494","reported_seq":8554,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:33.154261+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:53:33.154261+0000","last_peered":"2024-08-17T17:53:33.154261+0000","last_clean":"2024-08-17T17:53:33.154261+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:53:33.154261+0000","last_undegraded":"2024-08-17T17:53:33.154261+0000","last_fullsized":"2024-08-17T17:53:33.154261+0000","mapping_epoch":1167,"log_start":"1170'1425","ondisk_log_start":"1170'1425","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7069,"log_dups_size":0,"ondisk_log_size":7069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290979840,"num_objects":4440,"num_object_clones":0,"num_object_copies":13320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8494,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1219'8728","reported_seq":18487,"reported_epoch":1219,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:33.766589+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:53:33.766589+0000","last_peered":"2024-08-17T17:53:33.766589+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:53:33.766589+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'1891","ondisk_log_start":"1170'1891","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6837,"log_dups_size":0,"ondisk_log_size":6837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303824896,"num_objects":4636,"num_object_clones":0,"num_object_copies":13908,"num_objects_missing_on_primary":3991,"num_objects_missing":3991,"num_objects_degraded":265,"num_objects_misplaced":7900,"num_objects_unfound":0,"num_objects_dirty":4636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8728,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4152,"num_bytes_recovered":272105472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":16},{"shards":"0,4,6","objects":1356},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":2999}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1219'8409","reported_seq":21712,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:33.249545+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:53:33.249545+0000","last_peered":"2024-08-17T17:53:33.249545+0000","last_clean":"2024-08-17T17:53:33.249545+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:53:33.249545+0000","last_undegraded":"2024-08-17T17:53:33.249545+0000","last_fullsized":"2024-08-17T17:53:33.249545+0000","mapping_epoch":1196,"log_start":"1170'2469","ondisk_log_start":"1170'2469","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5940,"log_dups_size":0,"ondisk_log_size":5940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289210368,"num_objects":4413,"num_object_clones":0,"num_object_copies":13239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8409,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1894,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.363791+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:53:20.363791+0000","last_peered":"2024-08-17T17:53:20.363791+0000","last_clean":"2024-08-17T17:53:20.363791+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:53:20.363791+0000","last_undegraded":"2024-08-17T17:53:20.363791+0000","last_fullsized":"2024-08-17T17:53:20.363791+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":496,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.749921+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:53:01.749921+0000","last_peered":"2024-08-17T17:53:01.749921+0000","last_clean":"2024-08-17T17:53:01.749921+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:53:01.749921+0000","last_undegraded":"2024-08-17T17:53:01.749921+0000","last_fullsized":"2024-08-17T17:53:01.749921+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1813,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532881+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:53:20.532881+0000","last_peered":"2024-08-17T17:53:20.532881+0000","last_clean":"2024-08-17T17:53:20.532881+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:53:20.532881+0000","last_undegraded":"2024-08-17T17:53:20.532881+0000","last_fullsized":"2024-08-17T17:53:20.532881+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1838,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532562+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:53:20.532562+0000","last_peered":"2024-08-17T17:53:20.532562+0000","last_clean":"2024-08-17T17:53:20.532562+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:53:20.532562+0000","last_undegraded":"2024-08-17T17:53:20.532562+0000","last_fullsized":"2024-08-17T17:53:20.532562+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":58,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.750447+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:53:01.750447+0000","last_peered":"2024-08-17T17:53:01.750447+0000","last_clean":"2024-08-17T17:53:01.750447+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:53:01.750447+0000","last_undegraded":"2024-08-17T17:53:01.750447+0000","last_fullsized":"2024-08-17T17:53:01.750447+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1231,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.534595+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:53:20.534595+0000","last_peered":"2024-08-17T17:53:20.534595+0000","last_clean":"2024-08-17T17:53:20.534595+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:53:20.534595+0000","last_undegraded":"2024-08-17T17:53:20.534595+0000","last_fullsized":"2024-08-17T17:53:20.534595+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":91,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222787+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:53:34.222787+0000","last_peered":"2024-08-17T17:53:34.222787+0000","last_clean":"2024-08-17T17:53:34.222787+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:53:34.222787+0000","last_undegraded":"2024-08-17T17:53:34.222787+0000","last_fullsized":"2024-08-17T17:53:34.222787+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":609,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532612+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:53:20.532612+0000","last_peered":"2024-08-17T17:53:20.532612+0000","last_clean":"2024-08-17T17:53:20.532612+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:53:20.532612+0000","last_undegraded":"2024-08-17T17:53:20.532612+0000","last_fullsized":"2024-08-17T17:53:20.532612+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1853,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.363428+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:53:20.363428+0000","last_peered":"2024-08-17T17:53:20.363428+0000","last_clean":"2024-08-17T17:53:20.363428+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:53:20.363428+0000","last_undegraded":"2024-08-17T17:53:20.363428+0000","last_fullsized":"2024-08-17T17:53:20.363428+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1854,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.438364+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:53:20.438364+0000","last_peered":"2024-08-17T17:53:20.438364+0000","last_clean":"2024-08-17T17:53:20.438364+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:53:20.438364+0000","last_undegraded":"2024-08-17T17:53:20.438364+0000","last_fullsized":"2024-08-17T17:53:20.438364+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":76,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222434+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:53:34.222434+0000","last_peered":"2024-08-17T17:53:34.222434+0000","last_clean":"2024-08-17T17:53:34.222434+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:53:34.222434+0000","last_undegraded":"2024-08-17T17:53:34.222434+0000","last_fullsized":"2024-08-17T17:53:34.222434+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1826,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532880+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:53:20.532880+0000","last_peered":"2024-08-17T17:53:20.532880+0000","last_clean":"2024-08-17T17:53:20.532880+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:53:20.532880+0000","last_undegraded":"2024-08-17T17:53:20.532880+0000","last_fullsized":"2024-08-17T17:53:20.532880+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1883,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222364+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:53:34.222364+0000","last_peered":"2024-08-17T17:53:34.222364+0000","last_clean":"2024-08-17T17:53:34.222364+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:53:34.222364+0000","last_undegraded":"2024-08-17T17:53:34.222364+0000","last_fullsized":"2024-08-17T17:53:34.222364+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1875,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.363689+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:53:20.363689+0000","last_peered":"2024-08-17T17:53:20.363689+0000","last_clean":"2024-08-17T17:53:20.363689+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:53:20.363689+0000","last_undegraded":"2024-08-17T17:53:20.363689+0000","last_fullsized":"2024-08-17T17:53:20.363689+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1849,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532554+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:53:20.532554+0000","last_peered":"2024-08-17T17:53:20.532554+0000","last_clean":"2024-08-17T17:53:20.532554+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:53:20.532554+0000","last_undegraded":"2024-08-17T17:53:20.532554+0000","last_fullsized":"2024-08-17T17:53:20.532554+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1193,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.750923+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:53:01.750923+0000","last_peered":"2024-08-17T17:53:01.750923+0000","last_clean":"2024-08-17T17:53:01.750923+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:53:01.750923+0000","last_undegraded":"2024-08-17T17:53:01.750923+0000","last_fullsized":"2024-08-17T17:53:01.750923+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1820,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532264+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:53:20.532264+0000","last_peered":"2024-08-17T17:53:20.532264+0000","last_clean":"2024-08-17T17:53:20.532264+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:53:20.532264+0000","last_undegraded":"2024-08-17T17:53:20.532264+0000","last_fullsized":"2024-08-17T17:53:20.532264+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1870,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.363825+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:53:20.363825+0000","last_peered":"2024-08-17T17:53:20.363825+0000","last_clean":"2024-08-17T17:53:20.363825+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:53:20.363825+0000","last_undegraded":"2024-08-17T17:53:20.363825+0000","last_fullsized":"2024-08-17T17:53:20.363825+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1776,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532271+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:53:20.532271+0000","last_peered":"2024-08-17T17:53:20.532271+0000","last_clean":"2024-08-17T17:53:20.532271+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:53:20.532271+0000","last_undegraded":"2024-08-17T17:53:20.532271+0000","last_fullsized":"2024-08-17T17:53:20.532271+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":512,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818688+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:53:01.818688+0000","last_peered":"2024-08-17T17:53:01.818688+0000","last_clean":"2024-08-17T17:53:01.818688+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:53:01.818688+0000","last_undegraded":"2024-08-17T17:53:01.818688+0000","last_fullsized":"2024-08-17T17:53:01.818688+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1189,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818654+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:53:01.818654+0000","last_peered":"2024-08-17T17:53:01.818654+0000","last_clean":"2024-08-17T17:53:01.818654+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:53:01.818654+0000","last_undegraded":"2024-08-17T17:53:01.818654+0000","last_fullsized":"2024-08-17T17:53:01.818654+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1876,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222584+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:53:34.222584+0000","last_peered":"2024-08-17T17:53:34.222584+0000","last_clean":"2024-08-17T17:53:34.222584+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:53:34.222584+0000","last_undegraded":"2024-08-17T17:53:34.222584+0000","last_fullsized":"2024-08-17T17:53:34.222584+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":350,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818599+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:53:01.818599+0000","last_peered":"2024-08-17T17:53:01.818599+0000","last_clean":"2024-08-17T17:53:01.818599+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:53:01.818599+0000","last_undegraded":"2024-08-17T17:53:01.818599+0000","last_fullsized":"2024-08-17T17:53:01.818599+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":496,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.750125+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:53:01.750125+0000","last_peered":"2024-08-17T17:53:01.750125+0000","last_clean":"2024-08-17T17:53:01.750125+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:53:01.750125+0000","last_undegraded":"2024-08-17T17:53:01.750125+0000","last_fullsized":"2024-08-17T17:53:01.750125+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1174,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.750229+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:53:01.750229+0000","last_peered":"2024-08-17T17:53:01.750229+0000","last_clean":"2024-08-17T17:53:01.750229+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:53:01.750229+0000","last_undegraded":"2024-08-17T17:53:01.750229+0000","last_fullsized":"2024-08-17T17:53:01.750229+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1182,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.750963+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:53:01.750963+0000","last_peered":"2024-08-17T17:53:01.750963+0000","last_clean":"2024-08-17T17:53:01.750963+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:53:01.750963+0000","last_undegraded":"2024-08-17T17:53:01.750963+0000","last_fullsized":"2024-08-17T17:53:01.750963+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1922,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.363522+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:53:20.363522+0000","last_peered":"2024-08-17T17:53:20.363522+0000","last_clean":"2024-08-17T17:53:20.363522+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:53:20.363522+0000","last_undegraded":"2024-08-17T17:53:20.363522+0000","last_fullsized":"2024-08-17T17:53:20.363522+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1805,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532358+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:53:20.532358+0000","last_peered":"2024-08-17T17:53:20.532358+0000","last_clean":"2024-08-17T17:53:20.532358+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:53:20.532358+0000","last_undegraded":"2024-08-17T17:53:20.532358+0000","last_fullsized":"2024-08-17T17:53:20.532358+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1882,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.223547+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:53:34.223547+0000","last_peered":"2024-08-17T17:53:34.223547+0000","last_clean":"2024-08-17T17:53:34.223547+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:53:34.223547+0000","last_undegraded":"2024-08-17T17:53:34.223547+0000","last_fullsized":"2024-08-17T17:53:34.223547+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1808,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.533085+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:53:20.533085+0000","last_peered":"2024-08-17T17:53:20.533085+0000","last_clean":"2024-08-17T17:53:20.533085+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:53:20.533085+0000","last_undegraded":"2024-08-17T17:53:20.533085+0000","last_fullsized":"2024-08-17T17:53:20.533085+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1840,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.363501+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:53:20.363501+0000","last_peered":"2024-08-17T17:53:20.363501+0000","last_clean":"2024-08-17T17:53:20.363501+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:53:20.363501+0000","last_undegraded":"2024-08-17T17:53:20.363501+0000","last_fullsized":"2024-08-17T17:53:20.363501+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1209,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818733+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:53:01.818733+0000","last_peered":"2024-08-17T17:53:01.818733+0000","last_clean":"2024-08-17T17:53:01.818733+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:53:01.818733+0000","last_undegraded":"2024-08-17T17:53:01.818733+0000","last_fullsized":"2024-08-17T17:53:01.818733+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1889,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.223038+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:53:34.223038+0000","last_peered":"2024-08-17T17:53:34.223038+0000","last_clean":"2024-08-17T17:53:34.223038+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:53:34.223038+0000","last_undegraded":"2024-08-17T17:53:34.223038+0000","last_fullsized":"2024-08-17T17:53:34.223038+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":75,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222332+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:53:34.222332+0000","last_peered":"2024-08-17T17:53:34.222332+0000","last_clean":"2024-08-17T17:53:34.222332+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:53:34.222332+0000","last_undegraded":"2024-08-17T17:53:34.222332+0000","last_fullsized":"2024-08-17T17:53:34.222332+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1908,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222918+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:53:34.222918+0000","last_peered":"2024-08-17T17:53:34.222918+0000","last_clean":"2024-08-17T17:53:34.222918+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:53:34.222918+0000","last_undegraded":"2024-08-17T17:53:34.222918+0000","last_fullsized":"2024-08-17T17:53:34.222918+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":642,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532755+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:53:20.532755+0000","last_peered":"2024-08-17T17:53:20.532755+0000","last_clean":"2024-08-17T17:53:20.532755+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:53:20.532755+0000","last_undegraded":"2024-08-17T17:53:20.532755+0000","last_fullsized":"2024-08-17T17:53:20.532755+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1889,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.223101+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:53:34.223101+0000","last_peered":"2024-08-17T17:53:34.223101+0000","last_clean":"2024-08-17T17:53:34.223101+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:53:34.223101+0000","last_undegraded":"2024-08-17T17:53:34.223101+0000","last_fullsized":"2024-08-17T17:53:34.223101+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1215,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.363259+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:53:20.363259+0000","last_peered":"2024-08-17T17:53:20.363259+0000","last_clean":"2024-08-17T17:53:20.363259+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:53:20.363259+0000","last_undegraded":"2024-08-17T17:53:20.363259+0000","last_fullsized":"2024-08-17T17:53:20.363259+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":135,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222581+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:53:34.222581+0000","last_peered":"2024-08-17T17:53:34.222581+0000","last_clean":"2024-08-17T17:53:34.222581+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:53:34.222581+0000","last_undegraded":"2024-08-17T17:53:34.222581+0000","last_fullsized":"2024-08-17T17:53:34.222581+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1899,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.223938+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:53:34.223938+0000","last_peered":"2024-08-17T17:53:34.223938+0000","last_clean":"2024-08-17T17:53:34.223938+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:53:34.223938+0000","last_undegraded":"2024-08-17T17:53:34.223938+0000","last_fullsized":"2024-08-17T17:53:34.223938+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1877,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222253+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:53:34.222253+0000","last_peered":"2024-08-17T17:53:34.222253+0000","last_clean":"2024-08-17T17:53:34.222253+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:53:34.222253+0000","last_undegraded":"2024-08-17T17:53:34.222253+0000","last_fullsized":"2024-08-17T17:53:34.222253+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1895,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.437868+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:53:20.437868+0000","last_peered":"2024-08-17T17:53:20.437868+0000","last_clean":"2024-08-17T17:53:20.437868+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:53:20.437868+0000","last_undegraded":"2024-08-17T17:53:20.437868+0000","last_fullsized":"2024-08-17T17:53:20.437868+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1901,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.363725+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:53:20.363725+0000","last_peered":"2024-08-17T17:53:20.363725+0000","last_clean":"2024-08-17T17:53:20.363725+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:53:20.363725+0000","last_undegraded":"2024-08-17T17:53:20.363725+0000","last_fullsized":"2024-08-17T17:53:20.363725+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1881,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.373870+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:53:20.373870+0000","last_peered":"2024-08-17T17:53:20.373870+0000","last_clean":"2024-08-17T17:53:20.373870+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:53:20.373870+0000","last_undegraded":"2024-08-17T17:53:20.373870+0000","last_fullsized":"2024-08-17T17:53:20.373870+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1216,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818344+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:53:01.818344+0000","last_peered":"2024-08-17T17:53:01.818344+0000","last_clean":"2024-08-17T17:53:01.818344+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:53:01.818344+0000","last_undegraded":"2024-08-17T17:53:01.818344+0000","last_fullsized":"2024-08-17T17:53:01.818344+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1162,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.750492+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:53:01.750492+0000","last_peered":"2024-08-17T17:53:01.750492+0000","last_clean":"2024-08-17T17:53:01.750492+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:53:01.750492+0000","last_undegraded":"2024-08-17T17:53:01.750492+0000","last_fullsized":"2024-08-17T17:53:01.750492+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1790,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.534520+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:53:20.534520+0000","last_peered":"2024-08-17T17:53:20.534520+0000","last_clean":"2024-08-17T17:53:20.534520+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:53:20.534520+0000","last_undegraded":"2024-08-17T17:53:20.534520+0000","last_fullsized":"2024-08-17T17:53:20.534520+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":121,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.438035+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:53:20.438035+0000","last_peered":"2024-08-17T17:53:20.438035+0000","last_clean":"2024-08-17T17:53:20.438035+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:53:20.438035+0000","last_undegraded":"2024-08-17T17:53:20.438035+0000","last_fullsized":"2024-08-17T17:53:20.438035+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1528,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818253+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:53:01.818253+0000","last_peered":"2024-08-17T17:53:01.818253+0000","last_clean":"2024-08-17T17:53:01.818253+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:53:01.818253+0000","last_undegraded":"2024-08-17T17:53:01.818253+0000","last_fullsized":"2024-08-17T17:53:01.818253+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1884,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.223865+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:53:34.223865+0000","last_peered":"2024-08-17T17:53:34.223865+0000","last_clean":"2024-08-17T17:53:34.223865+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:53:34.223865+0000","last_undegraded":"2024-08-17T17:53:34.223865+0000","last_fullsized":"2024-08-17T17:53:34.223865+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1822,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532383+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:53:20.532383+0000","last_peered":"2024-08-17T17:53:20.532383+0000","last_clean":"2024-08-17T17:53:20.532383+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:53:20.532383+0000","last_undegraded":"2024-08-17T17:53:20.532383+0000","last_fullsized":"2024-08-17T17:53:20.532383+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":496,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.750214+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:53:01.750214+0000","last_peered":"2024-08-17T17:53:01.750214+0000","last_clean":"2024-08-17T17:53:01.750214+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:53:01.750214+0000","last_undegraded":"2024-08-17T17:53:01.750214+0000","last_fullsized":"2024-08-17T17:53:01.750214+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":560,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.437778+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:53:20.437778+0000","last_peered":"2024-08-17T17:53:20.437778+0000","last_clean":"2024-08-17T17:53:20.437778+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:53:20.437778+0000","last_undegraded":"2024-08-17T17:53:20.437778+0000","last_fullsized":"2024-08-17T17:53:20.437778+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":576,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.438240+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:53:20.438240+0000","last_peered":"2024-08-17T17:53:20.438240+0000","last_clean":"2024-08-17T17:53:20.438240+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:53:20.438240+0000","last_undegraded":"2024-08-17T17:53:20.438240+0000","last_fullsized":"2024-08-17T17:53:20.438240+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1180,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.751302+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:53:01.751302+0000","last_peered":"2024-08-17T17:53:01.751302+0000","last_clean":"2024-08-17T17:53:01.751302+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:53:01.751302+0000","last_undegraded":"2024-08-17T17:53:01.751302+0000","last_fullsized":"2024-08-17T17:53:01.751302+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":592,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532571+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:53:20.532571+0000","last_peered":"2024-08-17T17:53:20.532571+0000","last_clean":"2024-08-17T17:53:20.532571+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:53:20.532571+0000","last_undegraded":"2024-08-17T17:53:20.532571+0000","last_fullsized":"2024-08-17T17:53:20.532571+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1818,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532679+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:53:20.532679+0000","last_peered":"2024-08-17T17:53:20.532679+0000","last_clean":"2024-08-17T17:53:20.532679+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:53:20.532679+0000","last_undegraded":"2024-08-17T17:53:20.532679+0000","last_fullsized":"2024-08-17T17:53:20.532679+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1914,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.363526+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:53:20.363526+0000","last_peered":"2024-08-17T17:53:20.363526+0000","last_clean":"2024-08-17T17:53:20.363526+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:53:20.363526+0000","last_undegraded":"2024-08-17T17:53:20.363526+0000","last_fullsized":"2024-08-17T17:53:20.363526+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":625,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532487+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:53:20.532487+0000","last_peered":"2024-08-17T17:53:20.532487+0000","last_clean":"2024-08-17T17:53:20.532487+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:53:20.532487+0000","last_undegraded":"2024-08-17T17:53:20.532487+0000","last_fullsized":"2024-08-17T17:53:20.532487+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1169,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.751198+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:53:01.751198+0000","last_peered":"2024-08-17T17:53:01.751198+0000","last_clean":"2024-08-17T17:53:01.751198+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:53:01.751198+0000","last_undegraded":"2024-08-17T17:53:01.751198+0000","last_fullsized":"2024-08-17T17:53:01.751198+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1168,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818703+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:53:01.818703+0000","last_peered":"2024-08-17T17:53:01.818703+0000","last_clean":"2024-08-17T17:53:01.818703+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:53:01.818703+0000","last_undegraded":"2024-08-17T17:53:01.818703+0000","last_fullsized":"2024-08-17T17:53:01.818703+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":611,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222447+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:53:34.222447+0000","last_peered":"2024-08-17T17:53:34.222447+0000","last_clean":"2024-08-17T17:53:34.222447+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:53:34.222447+0000","last_undegraded":"2024-08-17T17:53:34.222447+0000","last_fullsized":"2024-08-17T17:53:34.222447+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1186,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818789+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:53:01.818789+0000","last_peered":"2024-08-17T17:53:01.818789+0000","last_clean":"2024-08-17T17:53:01.818789+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:53:01.818789+0000","last_undegraded":"2024-08-17T17:53:01.818789+0000","last_fullsized":"2024-08-17T17:53:01.818789+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":70,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818206+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:53:01.818206+0000","last_peered":"2024-08-17T17:53:01.818206+0000","last_clean":"2024-08-17T17:53:01.818206+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:53:01.818206+0000","last_undegraded":"2024-08-17T17:53:01.818206+0000","last_fullsized":"2024-08-17T17:53:01.818206+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1202,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818788+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:53:01.818788+0000","last_peered":"2024-08-17T17:53:01.818788+0000","last_clean":"2024-08-17T17:53:01.818788+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:53:01.818788+0000","last_undegraded":"2024-08-17T17:53:01.818788+0000","last_fullsized":"2024-08-17T17:53:01.818788+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1219,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.750359+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:53:01.750359+0000","last_peered":"2024-08-17T17:53:01.750359+0000","last_clean":"2024-08-17T17:53:01.750359+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:53:01.750359+0000","last_undegraded":"2024-08-17T17:53:01.750359+0000","last_fullsized":"2024-08-17T17:53:01.750359+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1820,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532948+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:53:20.532948+0000","last_peered":"2024-08-17T17:53:20.532948+0000","last_clean":"2024-08-17T17:53:20.532948+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:53:20.532948+0000","last_undegraded":"2024-08-17T17:53:20.532948+0000","last_fullsized":"2024-08-17T17:53:20.532948+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1163,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818492+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:53:01.818492+0000","last_peered":"2024-08-17T17:53:01.818492+0000","last_clean":"2024-08-17T17:53:01.818492+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:53:01.818492+0000","last_undegraded":"2024-08-17T17:53:01.818492+0000","last_fullsized":"2024-08-17T17:53:01.818492+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1873,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.363659+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:53:20.363659+0000","last_peered":"2024-08-17T17:53:20.363659+0000","last_clean":"2024-08-17T17:53:20.363659+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:53:20.363659+0000","last_undegraded":"2024-08-17T17:53:20.363659+0000","last_fullsized":"2024-08-17T17:53:20.363659+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1874,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.363252+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:53:20.363252+0000","last_peered":"2024-08-17T17:53:20.363252+0000","last_clean":"2024-08-17T17:53:20.363252+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:53:20.363252+0000","last_undegraded":"2024-08-17T17:53:20.363252+0000","last_fullsized":"2024-08-17T17:53:20.363252+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":576,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818552+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:53:01.818552+0000","last_peered":"2024-08-17T17:53:01.818552+0000","last_clean":"2024-08-17T17:53:01.818552+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:53:01.818552+0000","last_undegraded":"2024-08-17T17:53:01.818552+0000","last_fullsized":"2024-08-17T17:53:01.818552+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1188,"reported_epoch":1218,"state":"active+clean","last_fresh":"2024-08-17T17:53:01.818499+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:53:01.818499+0000","last_peered":"2024-08-17T17:53:01.818499+0000","last_clean":"2024-08-17T17:53:01.818499+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:53:01.818499+0000","last_undegraded":"2024-08-17T17:53:01.818499+0000","last_fullsized":"2024-08-17T17:53:01.818499+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1820,"reported_epoch":1219,"state":"active+clean","last_fresh":"2024-08-17T17:53:20.532713+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:53:20.532713+0000","last_peered":"2024-08-17T17:53:20.532713+0000","last_clean":"2024-08-17T17:53:20.532713+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:53:20.532713+0000","last_undegraded":"2024-08-17T17:53:20.532713+0000","last_fullsized":"2024-08-17T17:53:20.532713+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":4725407768,"num_objects":72105,"num_object_clones":0,"num_object_copies":216315,"num_objects_missing_on_primary":3991,"num_objects_missing":3991,"num_objects_degraded":18174,"num_objects_misplaced":12382,"num_objects_unfound":0,"num_objects_dirty":72105,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":137505,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34102,"num_bytes_recovered":2056454144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14792142848,"data_stored":14792130632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107279,"ondisk_log_size":107279,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234392,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2263072,"kb_used_data":1961560,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91482080,"statfs":{"total":95995035648,"available":93677649920,"internally_reserved":0,"allocated":2008637440,"data_stored":2003772723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496027,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2554468,"kb_used_data":2252956,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91190684,"statfs":{"total":95995035648,"available":93379260416,"internally_reserved":0,"allocated":2307026944,"data_stored":2302158632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757665,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326279,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2787212,"kb_used_data":2579076,"kb_used_omap":57,"kb_used_meta":208070,"kb_avail":90957940,"statfs":{"total":95995035648,"available":93140930560,"internally_reserved":0,"allocated":2640973824,"data_stored":2636124287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":213064443},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405518,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2509460,"kb_used_data":2315148,"kb_used_omap":61,"kb_used_meta":194242,"kb_avail":91235692,"statfs":{"total":95995035648,"available":93425348608,"internally_reserved":0,"allocated":2370711552,"data_stored":2365858987,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":198904760},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452053,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1888668,"kb_used_data":1734672,"kb_used_omap":54,"kb_used_meta":153929,"kb_avail":91856484,"statfs":{"total":95995035648,"available":94061039616,"internally_reserved":0,"allocated":1776304128,"data_stored":1771451116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":157623694},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474744,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1621464,"kb_used_data":1426512,"kb_used_omap":119,"kb_used_meta":194824,"kb_avail":92123688,"statfs":{"total":95995035648,"available":94334656512,"internally_reserved":0,"allocated":1460748288,"data_stored":1455887548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":199500510},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019300,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1447170048,"data_stored":1447165976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1760690176,"data_stored":1760690176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2357133312,"data_stored":2357133312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2625245184,"data_stored":2625241112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1068761088,"data_stored":1068761088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1254555648,"data_stored":1254555648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2288521216,"data_stored":2288517144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1990066176,"data_stored":1990066176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:53:38.769 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:53:38.769 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:38.791 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:38.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:38.893 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:38.914 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:38.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:38.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:39.016 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:39.037 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:39.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:39.139 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:39.159 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:39.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:39.261 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:39.284 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:39.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:39.362 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:39 smithi130 ceph-mon[105521]: pgmap v1792: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 18174/215049 objects degraded (8.451%); 12303/215049 objects misplaced (5.721%); 953 KiB/s, 14 objects/s recovering 2024-08-17T17:53:39.362 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:39 smithi130 ceph-mon[110068]: pgmap v1792: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 18174/215049 objects degraded (8.451%); 12303/215049 objects misplaced (5.721%); 953 KiB/s, 14 objects/s recovering 2024-08-17T17:53:39.385 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:39.407 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:39.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:39.508 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:39.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.530 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:39.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:39 smithi179 ceph-mon[93519]: pgmap v1792: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 18174/215049 objects degraded (8.451%); 12303/215049 objects misplaced (5.721%); 953 KiB/s, 14 objects/s recovering 2024-08-17T17:53:39.631 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:39.653 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:39.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:39.755 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:39.776 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:39.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:39.878 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:39.900 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:39.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:39.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:39.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:53:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:53:39] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-08-17T17:53:40.002 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:40.023 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:40.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:40.125 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:40.149 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:40.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:40.249 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:53:40.249 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T17:53:40.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T17:53:40.249 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T17:53:40.249 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.249 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:83238 45.8 45:11a1a64f:::benchmark_data_smithi130_172551_object83237:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:53:40.249 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:38.638381+0000", 2024-08-17T17:53:40.249 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 301.586967808, 2024-08-17T17:53:40.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.406600589, 2024-08-17T17:53:40.249 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.250 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.250 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T17:53:40.250 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.250 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.250 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.250 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 83238 2024-08-17T17:53:40.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.250 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.250 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.250 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.250 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638381+0000", 2024-08-17T17:53:40.250 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.251 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638381+0000", 2024-08-17T17:53:40.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.251 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638383+0000", 2024-08-17T17:53:40.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4339999999999999e-06 2024-08-17T17:53:40.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.251 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638406+0000", 2024-08-17T17:53:40.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3493e-05 2024-08-17T17:53:40.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638416+0000", 2024-08-17T17:53:40.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0095000000000001e-05 2024-08-17T17:53:40.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638423+0000", 2024-08-17T17:53:40.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9949999999999999e-06 2024-08-17T17:53:40.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638664+0000", 2024-08-17T17:53:40.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000240688 2024-08-17T17:53:40.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638975+0000", 2024-08-17T17:53:40.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00031074400000000001 2024-08-17T17:53:40.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:53:40.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.639400+0000", 2024-08-17T17:53:40.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00042540200000000002 2024-08-17T17:53:40.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.646624+0000", 2024-08-17T17:53:40.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072241290000000001 2024-08-17T17:53:40.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.662508+0000", 2024-08-17T17:53:40.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015883544999999999 2024-08-17T17:53:40.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:53:40.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.662522+0000", 2024-08-17T17:53:40.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4402e-05 2024-08-17T17:53:40.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.650046+0000", 2024-08-17T17:53:40.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98752317000000001 2024-08-17T17:53:40.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:53:40.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.650063+0000", 2024-08-17T17:53:40.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7394e-05 2024-08-17T17:53:40.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.044811+0000", 2024-08-17T17:53:40.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3947477720000001 2024-08-17T17:53:40.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.044982+0000", 2024-08-17T17:53:40.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017132599999999999 2024-08-17T17:53:40.256 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.257 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.257 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.257 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:106932 45.8 45:18d24a25:::benchmark_data_smithi130_172551_object2125:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.257 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:23.289066+0000", 2024-08-17T17:53:40.257 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 256.93628334499999, 2024-08-17T17:53:40.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22737446, 2024-08-17T17:53:40.257 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.257 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.257 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.257 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.257 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.257 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.258 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 106932 2024-08-17T17:53:40.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.258 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289066+0000", 2024-08-17T17:53:40.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289066+0000", 2024-08-17T17:53:40.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.259 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.259 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289068+0000", 2024-08-17T17:53:40.259 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2670000000000001e-06 2024-08-17T17:53:40.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.259 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.259 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289074+0000", 2024-08-17T17:53:40.259 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1750000000000002e-06 2024-08-17T17:53:40.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289087+0000", 2024-08-17T17:53:40.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2638e-05 2024-08-17T17:53:40.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289094+0000", 2024-08-17T17:53:40.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7719999999999997e-06 2024-08-17T17:53:40.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.363175+0000", 2024-08-17T17:53:40.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.074080928000000004 2024-08-17T17:53:40.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.365629+0000", 2024-08-17T17:53:40.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002453864 2024-08-17T17:53:40.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.417171+0000", 2024-08-17T17:53:40.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.051542814999999999 2024-08-17T17:53:40.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.417273+0000", 2024-08-17T17:53:40.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000101457 2024-08-17T17:53:40.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.417423+0000", 2024-08-17T17:53:40.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000149846 2024-08-17T17:53:40.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.452410+0000", 2024-08-17T17:53:40.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.034987799 2024-08-17T17:53:40.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.511559+0000", 2024-08-17T17:53:40.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.059148575000000002 2024-08-17T17:53:40.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.511606+0000", 2024-08-17T17:53:40.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7194000000000002e-05 2024-08-17T17:53:40.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.264 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.511673+0000", 2024-08-17T17:53:40.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6995999999999993e-05 2024-08-17T17:53:40.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.264 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.516440+0000", 2024-08-17T17:53:40.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0047671340000000001 2024-08-17T17:53:40.264 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.264 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.264 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.265 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:106967 45.8 45:13e64e6c:::benchmark_data_smithi130_172551_object2160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.265 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:23.378357+0000", 2024-08-17T17:53:40.265 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 256.846992118, 2024-08-17T17:53:40.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.279186029, 2024-08-17T17:53:40.265 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.265 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.265 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.265 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.265 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.265 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.265 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 106967 2024-08-17T17:53:40.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.265 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378357+0000", 2024-08-17T17:53:40.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378357+0000", 2024-08-17T17:53:40.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378359+0000", 2024-08-17T17:53:40.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1780000000000002e-06 2024-08-17T17:53:40.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378367+0000", 2024-08-17T17:53:40.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3649999999999998e-06 2024-08-17T17:53:40.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378387+0000", 2024-08-17T17:53:40.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9945999999999998e-05 2024-08-17T17:53:40.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378396+0000", 2024-08-17T17:53:40.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6420000000000002e-06 2024-08-17T17:53:40.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.268 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.534504+0000", 2024-08-17T17:53:40.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.15610795299999999 2024-08-17T17:53:40.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.268 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.537672+0000", 2024-08-17T17:53:40.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0031681180000000001 2024-08-17T17:53:40.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.580200+0000", 2024-08-17T17:53:40.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042527593000000002 2024-08-17T17:53:40.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.580441+0000", 2024-08-17T17:53:40.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00024059999999999999 2024-08-17T17:53:40.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.580882+0000", 2024-08-17T17:53:40.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00044118599999999999 2024-08-17T17:53:40.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.612775+0000", 2024-08-17T17:53:40.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.031893171999999997 2024-08-17T17:53:40.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.653587+0000", 2024-08-17T17:53:40.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.040812290000000001 2024-08-17T17:53:40.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.653642+0000", 2024-08-17T17:53:40.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4500999999999999e-05 2024-08-17T17:53:40.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.653792+0000", 2024-08-17T17:53:40.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000150235 2024-08-17T17:53:40.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.657543+0000", 2024-08-17T17:53:40.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0037512499999999998 2024-08-17T17:53:40.274 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.274 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.274 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.274 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:106987 45.8 45:18bd4662:::benchmark_data_smithi130_172551_object2180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.274 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:23.445589+0000", 2024-08-17T17:53:40.274 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 256.77975988100002, 2024-08-17T17:53:40.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.21594681099999999, 2024-08-17T17:53:40.275 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.275 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.275 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.275 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.275 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.275 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.275 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 106987 2024-08-17T17:53:40.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.275 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.275 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445589+0000", 2024-08-17T17:53:40.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445589+0000", 2024-08-17T17:53:40.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445592+0000", 2024-08-17T17:53:40.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.418e-06 2024-08-17T17:53:40.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445599+0000", 2024-08-17T17:53:40.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.588e-06 2024-08-17T17:53:40.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445616+0000", 2024-08-17T17:53:40.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6401000000000001e-05 2024-08-17T17:53:40.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445626+0000", 2024-08-17T17:53:40.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0465000000000001e-05 2024-08-17T17:53:40.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.537729+0000", 2024-08-17T17:53:40.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.092102937999999995 2024-08-17T17:53:40.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.539000+0000", 2024-08-17T17:53:40.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0012709609999999999 2024-08-17T17:53:40.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.595265+0000", 2024-08-17T17:53:40.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.056264464 2024-08-17T17:53:40.280 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.280 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.280 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.280 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.595396+0000", 2024-08-17T17:53:40.280 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000130996 2024-08-17T17:53:40.280 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.280 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.280 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.280 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.595582+0000", 2024-08-17T17:53:40.280 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000186097 2024-08-17T17:53:40.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.612782+0000", 2024-08-17T17:53:40.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017199941 2024-08-17T17:53:40.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.657596+0000", 2024-08-17T17:53:40.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.044814639000000003 2024-08-17T17:53:40.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.282 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.282 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.657637+0000", 2024-08-17T17:53:40.282 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1032999999999998e-05 2024-08-17T17:53:40.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.282 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.282 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.657728+0000", 2024-08-17T17:53:40.282 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0958000000000006e-05 2024-08-17T17:53:40.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.282 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.282 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.661536+0000", 2024-08-17T17:53:40.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0038079120000000001 2024-08-17T17:53:40.283 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.283 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.283 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.283 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:107041 45.8 45:11e0ef02:::benchmark_data_smithi130_172551_object2234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.283 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:23.556499+0000", 2024-08-17T17:53:40.283 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 256.66884993000002, 2024-08-17T17:53:40.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.225671231, 2024-08-17T17:53:40.283 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.283 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.283 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.284 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.284 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.284 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.284 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 107041 2024-08-17T17:53:40.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.284 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556499+0000", 2024-08-17T17:53:40.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556499+0000", 2024-08-17T17:53:40.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556501+0000", 2024-08-17T17:53:40.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1670000000000002e-06 2024-08-17T17:53:40.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556508+0000", 2024-08-17T17:53:40.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9430000000000003e-06 2024-08-17T17:53:40.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556528+0000", 2024-08-17T17:53:40.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.999e-05 2024-08-17T17:53:40.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556537+0000", 2024-08-17T17:53:40.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5890000000000003e-06 2024-08-17T17:53:40.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.674728+0000", 2024-08-17T17:53:40.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.118190689 2024-08-17T17:53:40.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.675674+0000", 2024-08-17T17:53:40.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00094605799999999995 2024-08-17T17:53:40.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.709070+0000", 2024-08-17T17:53:40.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.033396239000000001 2024-08-17T17:53:40.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.288 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.709171+0000", 2024-08-17T17:53:40.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010085 2024-08-17T17:53:40.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.289 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.289 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.709302+0000", 2024-08-17T17:53:40.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013068800000000001 2024-08-17T17:53:40.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.724080+0000", 2024-08-17T17:53:40.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014778964 2024-08-17T17:53:40.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.727981+0000", 2024-08-17T17:53:40.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039007310000000002 2024-08-17T17:53:40.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.739624+0000", 2024-08-17T17:53:40.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011642857 2024-08-17T17:53:40.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.739859+0000", 2024-08-17T17:53:40.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000234959 2024-08-17T17:53:40.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.782171+0000", 2024-08-17T17:53:40.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042311506999999998 2024-08-17T17:53:40.292 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.292 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.292 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.292 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.292 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.292 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108281 45.8 45:17388508:::benchmark_data_smithi130_172551_object3474:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.292 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:26.195227+0000", 2024-08-17T17:53:40.292 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 254.03012213599999, 2024-08-17T17:53:40.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.232226402, 2024-08-17T17:53:40.292 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.292 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.292 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.293 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.293 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.293 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.293 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108281 2024-08-17T17:53:40.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.293 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.293 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.293 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195227+0000", 2024-08-17T17:53:40.293 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.293 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.293 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195227+0000", 2024-08-17T17:53:40.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.294 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.294 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195229+0000", 2024-08-17T17:53:40.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4449999999999999e-06 2024-08-17T17:53:40.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.294 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.294 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195235+0000", 2024-08-17T17:53:40.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5440000000000004e-06 2024-08-17T17:53:40.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195247+0000", 2024-08-17T17:53:40.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2245999999999999e-05 2024-08-17T17:53:40.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195255+0000", 2024-08-17T17:53:40.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6790000000000005e-06 2024-08-17T17:53:40.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.312742+0000", 2024-08-17T17:53:40.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.11748729500000001 2024-08-17T17:53:40.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.313545+0000", 2024-08-17T17:53:40.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00080311800000000002 2024-08-17T17:53:40.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.355977+0000", 2024-08-17T17:53:40.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042431218999999999 2024-08-17T17:53:40.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.356037+0000", 2024-08-17T17:53:40.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9966000000000002e-05 2024-08-17T17:53:40.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.297 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.297 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.356111+0000", 2024-08-17T17:53:40.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4802999999999995e-05 2024-08-17T17:53:40.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.297 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.297 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.377134+0000", 2024-08-17T17:53:40.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021022968 2024-08-17T17:53:40.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.297 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.425050+0000", 2024-08-17T17:53:40.298 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.047915439999999997 2024-08-17T17:53:40.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.425098+0000", 2024-08-17T17:53:40.298 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7867000000000001e-05 2024-08-17T17:53:40.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.425140+0000", 2024-08-17T17:53:40.298 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2126000000000003e-05 2024-08-17T17:53:40.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.299 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.299 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.299 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.427454+0000", 2024-08-17T17:53:40.299 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0023136860000000001 2024-08-17T17:53:40.299 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.299 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.299 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.299 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.299 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.299 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108681 45.8 45:10018309:::benchmark_data_smithi130_172551_object3874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.299 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:27.139549+0000", 2024-08-17T17:53:40.299 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 253.085799872, 2024-08-17T17:53:40.299 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.21731466799999999, 2024-08-17T17:53:40.300 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.300 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.300 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.300 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.300 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.300 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.300 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108681 2024-08-17T17:53:40.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.300 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.300 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.300 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.300 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139549+0000", 2024-08-17T17:53:40.300 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.301 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.301 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139549+0000", 2024-08-17T17:53:40.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.301 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.301 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139550+0000", 2024-08-17T17:53:40.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4600000000000005e-07 2024-08-17T17:53:40.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.301 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.301 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139553+0000", 2024-08-17T17:53:40.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.774e-06 2024-08-17T17:53:40.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139560+0000", 2024-08-17T17:53:40.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4089999999999997e-06 2024-08-17T17:53:40.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139565+0000", 2024-08-17T17:53:40.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3059999999999997e-06 2024-08-17T17:53:40.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.261331+0000", 2024-08-17T17:53:40.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.121765262 2024-08-17T17:53:40.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.262881+0000", 2024-08-17T17:53:40.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0015505829999999999 2024-08-17T17:53:40.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.294557+0000", 2024-08-17T17:53:40.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.031675523999999997 2024-08-17T17:53:40.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.294741+0000", 2024-08-17T17:53:40.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000183833 2024-08-17T17:53:40.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.294884+0000", 2024-08-17T17:53:40.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014387499999999999 2024-08-17T17:53:40.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.320518+0000", 2024-08-17T17:53:40.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025633501 2024-08-17T17:53:40.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.320533+0000", 2024-08-17T17:53:40.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4738e-05 2024-08-17T17:53:40.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.352852+0000", 2024-08-17T17:53:40.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.032319379000000002 2024-08-17T17:53:40.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.352917+0000", 2024-08-17T17:53:40.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4584000000000004e-05 2024-08-17T17:53:40.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.356864+0000", 2024-08-17T17:53:40.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039473540000000001 2024-08-17T17:53:40.306 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.307 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.307 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.307 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.307 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.307 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108685 45.8 45:188d0d10:::benchmark_data_smithi130_172551_object3878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.307 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:27.141094+0000", 2024-08-17T17:53:40.307 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 253.08425551600001, 2024-08-17T17:53:40.307 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22079505299999999, 2024-08-17T17:53:40.307 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.307 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.307 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.307 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.307 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.308 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.308 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108685 2024-08-17T17:53:40.308 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.308 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.308 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.308 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.308 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141094+0000", 2024-08-17T17:53:40.308 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.308 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.308 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.308 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.308 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141094+0000", 2024-08-17T17:53:40.308 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.308 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141095+0000", 2024-08-17T17:53:40.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.469e-06 2024-08-17T17:53:40.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141100+0000", 2024-08-17T17:53:40.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2519999999999999e-06 2024-08-17T17:53:40.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141115+0000", 2024-08-17T17:53:40.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4592e-05 2024-08-17T17:53:40.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.310 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.310 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141123+0000", 2024-08-17T17:53:40.310 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0369999999999995e-06 2024-08-17T17:53:40.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.310 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.310 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.262940+0000", 2024-08-17T17:53:40.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.12181718800000001 2024-08-17T17:53:40.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.311 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.265083+0000", 2024-08-17T17:53:40.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021428350000000001 2024-08-17T17:53:40.312 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.312 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.309940+0000", 2024-08-17T17:53:40.312 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.044856832999999999 2024-08-17T17:53:40.313 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.313 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.313 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.313 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.310048+0000", 2024-08-17T17:53:40.313 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000107886 2024-08-17T17:53:40.313 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.313 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.314 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.314 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.310277+0000", 2024-08-17T17:53:40.314 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00022885700000000001 2024-08-17T17:53:40.314 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.314 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.314 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.314 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.357133+0000", 2024-08-17T17:53:40.314 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.046856061999999997 2024-08-17T17:53:40.314 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.314 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.357191+0000", 2024-08-17T17:53:40.315 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8743999999999999e-05 2024-08-17T17:53:40.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.357325+0000", 2024-08-17T17:53:40.315 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013331899999999999 2024-08-17T17:53:40.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.357474+0000", 2024-08-17T17:53:40.315 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000148796 2024-08-17T17:53:40.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.361889+0000", 2024-08-17T17:53:40.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0044151829999999996 2024-08-17T17:53:40.316 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.316 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.316 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.316 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108963 45.8 45:1d275979:::benchmark_data_smithi130_172551_object4156:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.316 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:27.755594+0000", 2024-08-17T17:53:40.316 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 252.46975519, 2024-08-17T17:53:40.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.27754511999999998, 2024-08-17T17:53:40.317 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.317 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.317 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.317 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.317 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.317 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.317 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108963 2024-08-17T17:53:40.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.317 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755594+0000", 2024-08-17T17:53:40.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755594+0000", 2024-08-17T17:53:40.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755595+0000", 2024-08-17T17:53:40.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4799999999999997e-07 2024-08-17T17:53:40.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755598+0000", 2024-08-17T17:53:40.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5599999999999998e-06 2024-08-17T17:53:40.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755610+0000", 2024-08-17T17:53:40.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.184e-05 2024-08-17T17:53:40.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.320 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755617+0000", 2024-08-17T17:53:40.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5950000000000004e-06 2024-08-17T17:53:40.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.320 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.841456+0000", 2024-08-17T17:53:40.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.085839020000000002 2024-08-17T17:53:40.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.321 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.321 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.321 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.846882+0000", 2024-08-17T17:53:40.321 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0054257439999999997 2024-08-17T17:53:40.321 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.321 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.321 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.321 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.875006+0000", 2024-08-17T17:53:40.321 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028124712999999999 2024-08-17T17:53:40.321 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.321 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.321 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.321 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.875105+0000", 2024-08-17T17:53:40.322 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8784999999999997e-05 2024-08-17T17:53:40.322 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.322 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.322 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.322 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.875242+0000", 2024-08-17T17:53:40.322 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013735599999999999 2024-08-17T17:53:40.322 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.322 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.322 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.322 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.028166+0000", 2024-08-17T17:53:40.322 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.15292386199999999 2024-08-17T17:53:40.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.323 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.028216+0000", 2024-08-17T17:53:40.323 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9851000000000001e-05 2024-08-17T17:53:40.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.323 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.028272+0000", 2024-08-17T17:53:40.323 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5674999999999998e-05 2024-08-17T17:53:40.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.029209+0000", 2024-08-17T17:53:40.324 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00093682800000000001 2024-08-17T17:53:40.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.033139+0000", 2024-08-17T17:53:40.324 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039305429999999999 2024-08-17T17:53:40.324 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.324 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.324 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.324 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109021 45.8 45:14c0b812:::benchmark_data_smithi130_172551_object4214:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.325 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:27.872171+0000", 2024-08-17T17:53:40.325 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 252.353178136, 2024-08-17T17:53:40.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22437405799999999, 2024-08-17T17:53:40.325 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.325 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.325 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.325 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.325 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.325 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.325 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109021 2024-08-17T17:53:40.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.325 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.325 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.325 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872171+0000", 2024-08-17T17:53:40.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.326 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872171+0000", 2024-08-17T17:53:40.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.326 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872173+0000", 2024-08-17T17:53:40.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.922e-06 2024-08-17T17:53:40.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.327 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872180+0000", 2024-08-17T17:53:40.327 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5899999999999996e-06 2024-08-17T17:53:40.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.327 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872192+0000", 2024-08-17T17:53:40.327 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2275e-05 2024-08-17T17:53:40.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.327 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872199+0000", 2024-08-17T17:53:40.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7750000000000004e-06 2024-08-17T17:53:40.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.328 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.328 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.041304+0000", 2024-08-17T17:53:40.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16910571499999999 2024-08-17T17:53:40.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.328 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.328 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.044079+0000", 2024-08-17T17:53:40.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027747480000000001 2024-08-17T17:53:40.329 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.329 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.329 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.329 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.068075+0000", 2024-08-17T17:53:40.329 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023995408999999999 2024-08-17T17:53:40.329 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.329 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.329 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.329 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.068206+0000", 2024-08-17T17:53:40.329 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013108299999999999 2024-08-17T17:53:40.329 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.329 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.329 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.330 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.068334+0000", 2024-08-17T17:53:40.330 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000128773 2024-08-17T17:53:40.330 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.330 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.330 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.330 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.084130+0000", 2024-08-17T17:53:40.330 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015795113999999999 2024-08-17T17:53:40.330 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.330 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.331 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.331 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.084175+0000", 2024-08-17T17:53:40.331 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5321000000000002e-05 2024-08-17T17:53:40.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.331 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.331 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.331 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.092490+0000", 2024-08-17T17:53:40.331 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0083154240000000001 2024-08-17T17:53:40.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.331 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.092654+0000", 2024-08-17T17:53:40.332 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000163599 2024-08-17T17:53:40.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.332 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.096545+0000", 2024-08-17T17:53:40.333 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0038913099999999998 2024-08-17T17:53:40.333 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.333 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.333 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.333 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.334 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109270 45.8 45:15211971:::benchmark_data_smithi130_172551_object4463:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.334 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.480079+0000", 2024-08-17T17:53:40.334 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 251.74526976600001, 2024-08-17T17:53:40.334 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.29654451799999998, 2024-08-17T17:53:40.334 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.335 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.335 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.335 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.335 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.335 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.335 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109270 2024-08-17T17:53:40.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.336 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480079+0000", 2024-08-17T17:53:40.336 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.337 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.337 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.337 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480079+0000", 2024-08-17T17:53:40.337 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.337 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.337 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.338 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.338 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480081+0000", 2024-08-17T17:53:40.338 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3659999999999999e-06 2024-08-17T17:53:40.338 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.338 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.339 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.339 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480086+0000", 2024-08-17T17:53:40.339 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2290000000000002e-06 2024-08-17T17:53:40.339 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.339 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.339 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.340 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480102+0000", 2024-08-17T17:53:40.340 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6399e-05 2024-08-17T17:53:40.340 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.340 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.340 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.340 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480110+0000", 2024-08-17T17:53:40.341 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8390000000000007e-06 2024-08-17T17:53:40.341 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.341 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.341 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.341 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.574022+0000", 2024-08-17T17:53:40.342 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.093911290999999994 2024-08-17T17:53:40.342 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.342 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.342 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.342 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.574696+0000", 2024-08-17T17:53:40.342 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00067411700000000001 2024-08-17T17:53:40.343 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.343 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.343 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.343 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.719056+0000", 2024-08-17T17:53:40.343 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.144360194 2024-08-17T17:53:40.344 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.344 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.344 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.344 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.719205+0000", 2024-08-17T17:53:40.344 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014905499999999999 2024-08-17T17:53:40.344 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.345 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.345 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.345 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.719356+0000", 2024-08-17T17:53:40.345 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015102599999999999 2024-08-17T17:53:40.345 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.345 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.346 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.346 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.738239+0000", 2024-08-17T17:53:40.346 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018882638 2024-08-17T17:53:40.346 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.346 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.346 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.347 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.772530+0000", 2024-08-17T17:53:40.347 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.034291508999999998 2024-08-17T17:53:40.347 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.347 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.347 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.347 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.772581+0000", 2024-08-17T17:53:40.348 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0745000000000003e-05 2024-08-17T17:53:40.348 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.348 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.348 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.348 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.772702+0000", 2024-08-17T17:53:40.348 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012133600000000001 2024-08-17T17:53:40.348 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.349 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.349 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.349 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.776624+0000", 2024-08-17T17:53:40.349 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039217740000000003 2024-08-17T17:53:40.349 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.349 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.350 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.350 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.350 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.350 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109286 45.8 45:1465a8ca:::benchmark_data_smithi130_172551_object4479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.350 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.517695+0000", 2024-08-17T17:53:40.350 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 251.70765431300001, 2024-08-17T17:53:40.350 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26377572999999999, 2024-08-17T17:53:40.351 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.351 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.351 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.351 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.351 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.351 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.351 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109286 2024-08-17T17:53:40.351 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.351 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.351 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.351 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.351 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517695+0000", 2024-08-17T17:53:40.351 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.352 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.352 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.352 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.352 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517695+0000", 2024-08-17T17:53:40.352 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.352 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.352 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.352 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.352 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517697+0000", 2024-08-17T17:53:40.352 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7290000000000001e-06 2024-08-17T17:53:40.352 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.352 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.352 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.353 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517702+0000", 2024-08-17T17:53:40.353 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0780000000000003e-06 2024-08-17T17:53:40.353 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.353 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.353 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.353 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517710+0000", 2024-08-17T17:53:40.353 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7439999999999996e-06 2024-08-17T17:53:40.353 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.353 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.353 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.353 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517717+0000", 2024-08-17T17:53:40.353 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0769999999999996e-06 2024-08-17T17:53:40.353 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.353 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.354 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.354 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.574731+0000", 2024-08-17T17:53:40.354 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.057014505 2024-08-17T17:53:40.354 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.354 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.354 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.354 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.575221+0000", 2024-08-17T17:53:40.354 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00049007399999999996 2024-08-17T17:53:40.354 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.355 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.738278+0000", 2024-08-17T17:53:40.355 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16305646800000001 2024-08-17T17:53:40.355 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.355 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.738388+0000", 2024-08-17T17:53:40.355 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010992700000000001 2024-08-17T17:53:40.355 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.355 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.738536+0000", 2024-08-17T17:53:40.356 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014820799999999999 2024-08-17T17:53:40.356 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.356 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:40.357 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.358 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.753488+0000", 2024-08-17T17:53:40.358 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014951951999999999 2024-08-17T17:53:40.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.358 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.358 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.776680+0000", 2024-08-17T17:53:40.358 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023192260999999999 2024-08-17T17:53:40.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.358 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.358 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.776719+0000", 2024-08-17T17:53:40.359 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8858e-05 2024-08-17T17:53:40.359 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.359 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.359 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.359 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.776829+0000", 2024-08-17T17:53:40.359 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011035399999999999 2024-08-17T17:53:40.359 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.359 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.359 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.359 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.781471+0000", 2024-08-17T17:53:40.359 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046414949999999998 2024-08-17T17:53:40.359 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.359 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.360 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.360 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.360 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.360 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109287 45.8 45:1ce0b8c0:::benchmark_data_smithi130_172551_object4480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.360 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.517973+0000", 2024-08-17T17:53:40.360 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 251.70737652400001, 2024-08-17T17:53:40.360 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.27116035399999999, 2024-08-17T17:53:40.360 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.360 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.360 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.360 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.360 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.360 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.361 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109287 2024-08-17T17:53:40.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.361 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.361 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517973+0000", 2024-08-17T17:53:40.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.361 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517973+0000", 2024-08-17T17:53:40.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.362 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517974+0000", 2024-08-17T17:53:40.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0780000000000001e-06 2024-08-17T17:53:40.362 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.362 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517979+0000", 2024-08-17T17:53:40.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.967e-06 2024-08-17T17:53:40.362 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517995+0000", 2024-08-17T17:53:40.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6325e-05 2024-08-17T17:53:40.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.518003+0000", 2024-08-17T17:53:40.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9559999999999997e-06 2024-08-17T17:53:40.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.575256+0000", 2024-08-17T17:53:40.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.057252589 2024-08-17T17:53:40.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.704722+0000", 2024-08-17T17:53:40.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.12946607399999999 2024-08-17T17:53:40.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.753525+0000", 2024-08-17T17:53:40.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.048803226999999998 2024-08-17T17:53:40.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.753627+0000", 2024-08-17T17:53:40.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000102113 2024-08-17T17:53:40.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.753767+0000", 2024-08-17T17:53:40.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000140106 2024-08-17T17:53:40.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.768534+0000", 2024-08-17T17:53:40.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014766359 2024-08-17T17:53:40.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.366 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.366 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.781525+0000", 2024-08-17T17:53:40.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012991669000000001 2024-08-17T17:53:40.366 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.366 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.366 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.781568+0000", 2024-08-17T17:53:40.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2818000000000002e-05 2024-08-17T17:53:40.366 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.366 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.366 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.781626+0000", 2024-08-17T17:53:40.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8424999999999997e-05 2024-08-17T17:53:40.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.367 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.367 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.367 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789133+0000", 2024-08-17T17:53:40.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0075066480000000003 2024-08-17T17:53:40.367 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.367 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.367 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.367 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.367 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109293 45.8 45:166b0a81:::benchmark_data_smithi130_172551_object4486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.367 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.533090+0000", 2024-08-17T17:53:40.368 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 251.692259073, 2024-08-17T17:53:40.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26005087900000001, 2024-08-17T17:53:40.368 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.368 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.368 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.368 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.368 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.368 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.368 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109293 2024-08-17T17:53:40.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.368 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533090+0000", 2024-08-17T17:53:40.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.369 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533090+0000", 2024-08-17T17:53:40.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.369 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533092+0000", 2024-08-17T17:53:40.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0719999999999998e-06 2024-08-17T17:53:40.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533100+0000", 2024-08-17T17:53:40.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2699999999999999e-06 2024-08-17T17:53:40.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533113+0000", 2024-08-17T17:53:40.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3903999999999999e-05 2024-08-17T17:53:40.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533121+0000", 2024-08-17T17:53:40.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9160000000000001e-06 2024-08-17T17:53:40.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.371 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.371 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.371 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.704763+0000", 2024-08-17T17:53:40.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.17164132700000001 2024-08-17T17:53:40.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.371 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.371 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.371 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.707453+0000", 2024-08-17T17:53:40.372 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026901220000000001 2024-08-17T17:53:40.372 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.372 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.372 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.372 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.768583+0000", 2024-08-17T17:53:40.372 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.061130048999999999 2024-08-17T17:53:40.372 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.372 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.372 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.372 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.768675+0000", 2024-08-17T17:53:40.372 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1698000000000002e-05 2024-08-17T17:53:40.372 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.372 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.373 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.373 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.768806+0000", 2024-08-17T17:53:40.373 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013118900000000001 2024-08-17T17:53:40.373 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.373 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.373 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.373 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789153+0000", 2024-08-17T17:53:40.373 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020347371999999999 2024-08-17T17:53:40.373 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.374 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.374 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.374 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789212+0000", 2024-08-17T17:53:40.374 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9163000000000003e-05 2024-08-17T17:53:40.374 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.374 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.374 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.375 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789274+0000", 2024-08-17T17:53:40.375 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1339000000000003e-05 2024-08-17T17:53:40.375 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.375 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.375 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.375 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789347+0000", 2024-08-17T17:53:40.375 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3458e-05 2024-08-17T17:53:40.375 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.375 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.376 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.376 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.793141+0000", 2024-08-17T17:53:40.376 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0037940000000000001 2024-08-17T17:53:40.376 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.376 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.376 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.376 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.376 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.376 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109294 45.8 45:16984eff:::benchmark_data_smithi130_172551_object4487:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.376 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.534050+0000", 2024-08-17T17:53:40.376 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 251.69129953500001, 2024-08-17T17:53:40.376 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.33223749499999999, 2024-08-17T17:53:40.376 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.377 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.377 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.377 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.377 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.377 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.377 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109294 2024-08-17T17:53:40.377 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.377 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.377 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.377 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.377 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534050+0000", 2024-08-17T17:53:40.377 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.377 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.377 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.378 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.378 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534050+0000", 2024-08-17T17:53:40.378 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.378 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.378 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.378 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.378 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534051+0000", 2024-08-17T17:53:40.378 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4789999999999999e-06 2024-08-17T17:53:40.378 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.379 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.379 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.379 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534057+0000", 2024-08-17T17:53:40.379 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5489999999999997e-06 2024-08-17T17:53:40.379 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.379 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.379 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.379 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534075+0000", 2024-08-17T17:53:40.379 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7782999999999999e-05 2024-08-17T17:53:40.379 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.379 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.379 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.379 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534083+0000", 2024-08-17T17:53:40.380 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1089999999999998e-06 2024-08-17T17:53:40.380 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.380 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.380 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.380 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.793348+0000", 2024-08-17T17:53:40.380 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.259265264 2024-08-17T17:53:40.380 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.380 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.380 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.381 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.794825+0000", 2024-08-17T17:53:40.381 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0014772260000000001 2024-08-17T17:53:40.381 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.381 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.381 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.381 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.820020+0000", 2024-08-17T17:53:40.381 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025194431 2024-08-17T17:53:40.381 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.381 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.381 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.381 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.820117+0000", 2024-08-17T17:53:40.381 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7385000000000004e-05 2024-08-17T17:53:40.381 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.381 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.382 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.382 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.820245+0000", 2024-08-17T17:53:40.382 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012771000000000001 2024-08-17T17:53:40.382 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.382 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.382 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.382 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.842202+0000", 2024-08-17T17:53:40.382 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021957300999999999 2024-08-17T17:53:40.382 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.382 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.382 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.382 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.842217+0000", 2024-08-17T17:53:40.383 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5438999999999999e-05 2024-08-17T17:53:40.383 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.383 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.383 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.383 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.862154+0000", 2024-08-17T17:53:40.383 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019936878000000002 2024-08-17T17:53:40.383 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.383 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.383 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.383 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.862230+0000", 2024-08-17T17:53:40.383 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6091e-05 2024-08-17T17:53:40.383 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.383 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.383 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.384 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.866287+0000", 2024-08-17T17:53:40.384 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0040568499999999999 2024-08-17T17:53:40.384 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.384 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.384 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.384 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.384 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.384 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109309 45.8 45:12b009b3:::benchmark_data_smithi130_172551_object4502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.384 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.570036+0000", 2024-08-17T17:53:40.384 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 251.655313178, 2024-08-17T17:53:40.384 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.300907009, 2024-08-17T17:53:40.384 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.384 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.385 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.385 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.385 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.385 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.385 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109309 2024-08-17T17:53:40.385 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.385 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.385 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.385 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.385 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570036+0000", 2024-08-17T17:53:40.385 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.385 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.385 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.386 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.386 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570036+0000", 2024-08-17T17:53:40.386 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.386 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.386 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.386 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.386 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570038+0000", 2024-08-17T17:53:40.386 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.688e-06 2024-08-17T17:53:40.386 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.386 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.386 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.386 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570046+0000", 2024-08-17T17:53:40.387 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3899999999999993e-06 2024-08-17T17:53:40.387 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.387 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.387 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.387 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570056+0000", 2024-08-17T17:53:40.387 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0217999999999999e-05 2024-08-17T17:53:40.387 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.387 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.387 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.387 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570064+0000", 2024-08-17T17:53:40.387 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.892e-06 2024-08-17T17:53:40.387 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.387 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.387 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.388 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.794964+0000", 2024-08-17T17:53:40.388 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22489930799999999 2024-08-17T17:53:40.388 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.388 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.388 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.388 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.796783+0000", 2024-08-17T17:53:40.388 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001819443 2024-08-17T17:53:40.388 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.388 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.388 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.388 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.836565+0000", 2024-08-17T17:53:40.388 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.039782114 2024-08-17T17:53:40.388 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.389 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.389 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.389 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.836800+0000", 2024-08-17T17:53:40.389 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00023482700000000001 2024-08-17T17:53:40.389 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.389 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.389 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.389 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.837157+0000", 2024-08-17T17:53:40.389 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00035710700000000001 2024-08-17T17:53:40.389 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.389 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.389 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.389 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.842249+0000", 2024-08-17T17:53:40.390 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005092315 2024-08-17T17:53:40.390 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.390 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.390 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.390 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.842261+0000", 2024-08-17T17:53:40.390 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.189e-05 2024-08-17T17:53:40.390 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.390 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.390 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.390 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.866457+0000", 2024-08-17T17:53:40.390 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.024195761 2024-08-17T17:53:40.390 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.390 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.866646+0000", 2024-08-17T17:53:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018857100000000001 2024-08-17T17:53:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.870943+0000", 2024-08-17T17:53:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0042974850000000002 2024-08-17T17:53:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.391 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109672 45.8 45:19e4da43:::benchmark_data_smithi130_172551_object4865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.392 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:29.496933+0000", 2024-08-17T17:53:40.392 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 250.72841576600001, 2024-08-17T17:53:40.392 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.221123546, 2024-08-17T17:53:40.392 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.392 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.392 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.392 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.392 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.392 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.392 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109672 2024-08-17T17:53:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496933+0000", 2024-08-17T17:53:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496933+0000", 2024-08-17T17:53:40.393 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.394 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.394 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.394 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.394 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496934+0000", 2024-08-17T17:53:40.394 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2899999999999999e-07 2024-08-17T17:53:40.394 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.394 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.394 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.394 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496936+0000", 2024-08-17T17:53:40.394 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5320000000000001e-06 2024-08-17T17:53:40.394 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.395 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.395 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.395 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496945+0000", 2024-08-17T17:53:40.395 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6160000000000002e-06 2024-08-17T17:53:40.395 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.395 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.395 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.395 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496950+0000", 2024-08-17T17:53:40.395 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4140000000000002e-06 2024-08-17T17:53:40.395 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.395 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.395 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.395 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.568388+0000", 2024-08-17T17:53:40.396 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.071437234000000002 2024-08-17T17:53:40.396 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.396 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.396 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.396 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.569153+0000", 2024-08-17T17:53:40.396 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00076482600000000003 2024-08-17T17:53:40.396 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.396 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.396 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.396 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.697807+0000", 2024-08-17T17:53:40.396 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.128654886 2024-08-17T17:53:40.396 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.396 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.397 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.397 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.697938+0000", 2024-08-17T17:53:40.397 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013016899999999999 2024-08-17T17:53:40.397 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.397 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.397 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.397 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.698100+0000", 2024-08-17T17:53:40.397 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016217099999999999 2024-08-17T17:53:40.397 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.397 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.397 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.397 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.713384+0000", 2024-08-17T17:53:40.397 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015284588 2024-08-17T17:53:40.397 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.398 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.398 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.398 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.713429+0000", 2024-08-17T17:53:40.398 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4895000000000003e-05 2024-08-17T17:53:40.398 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.398 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.398 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.398 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.713504+0000", 2024-08-17T17:53:40.398 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4994999999999996e-05 2024-08-17T17:53:40.398 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.398 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.398 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.399 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.713752+0000", 2024-08-17T17:53:40.399 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00024759199999999998 2024-08-17T17:53:40.399 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.399 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.399 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.399 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.718057+0000", 2024-08-17T17:53:40.399 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.004305199 2024-08-17T17:53:40.399 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.399 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.399 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.399 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.399 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.399 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109679 45.8 45:13e4c37d:::benchmark_data_smithi130_172551_object4872:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.400 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:29.510400+0000", 2024-08-17T17:53:40.400 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 250.71494941, 2024-08-17T17:53:40.400 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22616212299999999, 2024-08-17T17:53:40.400 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.400 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.400 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.400 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.400 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.400 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.400 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109679 2024-08-17T17:53:40.400 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.400 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.400 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.400 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.401 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510400+0000", 2024-08-17T17:53:40.401 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.401 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.401 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.401 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.401 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510400+0000", 2024-08-17T17:53:40.401 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.401 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.401 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.401 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.401 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510402+0000", 2024-08-17T17:53:40.401 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.736e-06 2024-08-17T17:53:40.401 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.402 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.402 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.402 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510407+0000", 2024-08-17T17:53:40.402 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2140000000000004e-06 2024-08-17T17:53:40.402 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.402 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.402 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.402 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510423+0000", 2024-08-17T17:53:40.402 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6066000000000002e-05 2024-08-17T17:53:40.402 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.402 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.402 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.403 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510430+0000", 2024-08-17T17:53:40.403 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1590000000000004e-06 2024-08-17T17:53:40.403 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.403 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.403 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.403 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.569188+0000", 2024-08-17T17:53:40.403 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.058757566999999997 2024-08-17T17:53:40.403 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.403 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.403 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.403 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.570491+0000", 2024-08-17T17:53:40.404 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001303228 2024-08-17T17:53:40.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.404 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.404 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.404 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.708557+0000", 2024-08-17T17:53:40.404 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.13806636799999999 2024-08-17T17:53:40.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.404 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.404 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.404 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.708713+0000", 2024-08-17T17:53:40.404 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015618000000000001 2024-08-17T17:53:40.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.404 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.404 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.405 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.708867+0000", 2024-08-17T17:53:40.405 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015330699999999999 2024-08-17T17:53:40.405 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.405 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.405 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.405 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.721271+0000", 2024-08-17T17:53:40.405 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012404178 2024-08-17T17:53:40.405 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.405 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.405 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.405 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.724727+0000", 2024-08-17T17:53:40.405 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003456083 2024-08-17T17:53:40.405 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.406 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.406 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.406 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.727154+0000", 2024-08-17T17:53:40.406 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0024270379999999999 2024-08-17T17:53:40.406 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.406 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.406 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.406 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.727215+0000", 2024-08-17T17:53:40.406 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1525999999999993e-05 2024-08-17T17:53:40.406 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.406 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.406 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.736562+0000", 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0093464729999999992 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109712 45.8 45:1acdc34f:::benchmark_data_smithi130_172551_object4905:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:29.603003+0000", 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 250.622346695, 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.21928672099999999, 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.407 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.408 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.408 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.408 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109712 2024-08-17T17:53:40.408 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.408 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.408 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.408 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.408 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603003+0000", 2024-08-17T17:53:40.408 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.408 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.408 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.408 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.408 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603003+0000", 2024-08-17T17:53:40.409 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.409 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.409 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.409 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.409 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603005+0000", 2024-08-17T17:53:40.409 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3209999999999999e-06 2024-08-17T17:53:40.409 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.409 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.409 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.409 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603013+0000", 2024-08-17T17:53:40.409 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4610000000000008e-06 2024-08-17T17:53:40.409 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.409 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.409 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.410 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603033+0000", 2024-08-17T17:53:40.410 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9606000000000002e-05 2024-08-17T17:53:40.410 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.410 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.410 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.410 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603046+0000", 2024-08-17T17:53:40.410 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2634e-05 2024-08-17T17:53:40.410 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.410 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.410 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.410 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.718108+0000", 2024-08-17T17:53:40.410 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.115062581 2024-08-17T17:53:40.411 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.411 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.411 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.411 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.721208+0000", 2024-08-17T17:53:40.411 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0030999180000000001 2024-08-17T17:53:40.411 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.411 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.411 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.411 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.758271+0000", 2024-08-17T17:53:40.411 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.037063082999999997 2024-08-17T17:53:40.411 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.411 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.411 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.412 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.758375+0000", 2024-08-17T17:53:40.412 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000103467 2024-08-17T17:53:40.412 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.412 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.412 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.412 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.758517+0000", 2024-08-17T17:53:40.412 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014272699999999999 2024-08-17T17:53:40.412 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.412 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.412 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.412 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.772488+0000", 2024-08-17T17:53:40.412 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013970419 2024-08-17T17:53:40.412 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.413 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.413 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.413 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.788268+0000", 2024-08-17T17:53:40.413 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015780076000000001 2024-08-17T17:53:40.413 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.413 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.413 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.413 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.816948+0000", 2024-08-17T17:53:40.413 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028680039000000001 2024-08-17T17:53:40.413 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.413 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.414 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.414 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.817013+0000", 2024-08-17T17:53:40.414 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5081999999999998e-05 2024-08-17T17:53:40.414 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.414 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.414 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.414 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.822289+0000", 2024-08-17T17:53:40.414 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005276307 2024-08-17T17:53:40.414 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.415 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.415 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.415 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.415 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.415 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109714 45.8 45:16437e11:::benchmark_data_smithi130_172551_object4907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:53:40.415 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:29.605847+0000", 2024-08-17T17:53:40.416 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 250.61950201100001, 2024-08-17T17:53:40.416 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26878229799999998, 2024-08-17T17:53:40.416 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:53:40.416 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:53:40.416 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:53:40.416 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:53:40.416 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:53:40.416 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:53:40.416 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109714 2024-08-17T17:53:40.416 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.417 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:53:40.417 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.417 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:53:40.417 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605847+0000", 2024-08-17T17:53:40.417 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.417 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.418 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.418 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:53:40.418 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605847+0000", 2024-08-17T17:53:40.418 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:53:40.418 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.418 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.418 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:53:40.418 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605849+0000", 2024-08-17T17:53:40.418 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9209999999999999e-06 2024-08-17T17:53:40.418 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.419 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.419 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:53:40.419 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605856+0000", 2024-08-17T17:53:40.419 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6209999999999998e-06 2024-08-17T17:53:40.419 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.419 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.419 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:53:40.420 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605870+0000", 2024-08-17T17:53:40.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4395e-05 2024-08-17T17:53:40.420 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.420 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.420 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:53:40.420 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605879+0000", 2024-08-17T17:53:40.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4239999999999993e-06 2024-08-17T17:53:40.420 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.420 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.420 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.420 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.721311+0000", 2024-08-17T17:53:40.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.115432828 2024-08-17T17:53:40.421 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.421 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.421 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:53:40.421 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.722135+0000", 2024-08-17T17:53:40.421 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00082366000000000004 2024-08-17T17:53:40.421 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.421 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.421 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:53:40.421 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.783886+0000", 2024-08-17T17:53:40.421 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.061750939999999997 2024-08-17T17:53:40.422 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.422 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.422 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:53:40.422 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.784004+0000", 2024-08-17T17:53:40.422 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011846899999999999 2024-08-17T17:53:40.422 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.422 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.422 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:53:40.422 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.784179+0000", 2024-08-17T17:53:40.422 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017467599999999999 2024-08-17T17:53:40.422 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.422 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.423 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:53:40.423 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.791605+0000", 2024-08-17T17:53:40.423 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0074256260000000003 2024-08-17T17:53:40.423 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.423 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.423 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.423 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.791767+0000", 2024-08-17T17:53:40.423 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016228 2024-08-17T17:53:40.423 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.424 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.424 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:53:40.424 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.869906+0000", 2024-08-17T17:53:40.424 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.078138847999999997 2024-08-17T17:53:40.424 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.424 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.424 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:53:40.424 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.870304+0000", 2024-08-17T17:53:40.424 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00039816299999999999 2024-08-17T17:53:40.424 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:53:40.424 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:53:40.424 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:53:40.424 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.874630+0000", 2024-08-17T17:53:40.425 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0043254469999999996 2024-08-17T17:53:40.425 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.425 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.425 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.425 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:53:40.425 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:53:40.425 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:53:40.446 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:40.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:40.548 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:40.572 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:40.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:53:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:53:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:53:40.636 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:40.675 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:40.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:40.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:40.798 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:40.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:40.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:40.924 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:40.953 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:40.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:40.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:41.005 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:40 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18174/215049 objects degraded (8.451%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:41.005 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:40 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18174/215049 objects degraded (8.451%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:41.055 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:41.077 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:41.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:41.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:40 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18174/215049 objects degraded (8.451%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:41.179 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:41.200 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:41.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:41.302 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:41.323 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:41.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:41.425 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:41.451 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:41.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:41.552 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:41.574 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:41.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:41.676 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:41.699 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:41.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:41.754 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:53:41.801 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:41.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.823 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:41.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:41 smithi130 ceph-mon[105521]: pgmap v1793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 18174/214797 objects degraded (8.461%); 12303/214797 objects misplaced (5.728%); 953 KiB/s, 14 objects/s recovering 2024-08-17T17:53:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:41 smithi130 ceph-mon[110068]: pgmap v1793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 18174/214797 objects degraded (8.461%); 12303/214797 objects misplaced (5.728%); 953 KiB/s, 14 objects/s recovering 2024-08-17T17:53:41.923 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:41.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:41.947 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:41.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:42.048 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:42.069 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:42.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:42.085 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:53:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:41 smithi179 ceph-mon[93519]: pgmap v1793: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 18174/214797 objects degraded (8.461%); 12303/214797 objects misplaced (5.728%); 953 KiB/s, 14 objects/s recovering 2024-08-17T17:53:42.103 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:42.170 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:42.192 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:42.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:42.294 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:42.320 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:42.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:42.421 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:42.443 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:42.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:42.545 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:42.567 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:42.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:42.669 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:42.690 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:42.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:42.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:42.813 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:42.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:42.915 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:42.936 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:42.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:42.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:43.039 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:43.061 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:43.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:43.079 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:43.079 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:43.101 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:43.163 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:43.186 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:43.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:43.288 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:43.312 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:43.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:43.414 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:43.438 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:43.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:43.540 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:43.561 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:43.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:43.663 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:43.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:43.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:43.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:43.815 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:43.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:43.867 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:43 smithi130 ceph-mon[105521]: pgmap v1794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 136 op/s; 18076/214539 objects degraded (8.426%); 12266/214539 objects misplaced (5.717%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:43.868 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:43 smithi130 ceph-mon[110068]: pgmap v1794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 136 op/s; 18076/214539 objects degraded (8.426%); 12266/214539 objects misplaced (5.717%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:43.896 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:53:43.897 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:53:43.897 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:53:43.897 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:53:43.897 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:53:43.897 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:53:43.897 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:53:43.897 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:53:43.897 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:53:43.897 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:53:43.904 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:53:43.904 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:53:43.917 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:43.920 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:53:43.920 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:53:43.936 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:53:43.936 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:53:43.938 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:43.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:43.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:44.040 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:44.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.062 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:44.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:43 smithi179 ceph-mon[93519]: pgmap v1794: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 136 op/s; 18076/214539 objects degraded (8.426%); 12266/214539 objects misplaced (5.717%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:44.164 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:44.185 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:44.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:44.287 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:44.308 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:44.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:44.409 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:44.435 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:44.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:44.537 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:44.561 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:44.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:44.663 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:44.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:44.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:44.786 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:44.809 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:44.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:44.911 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:44.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:44.934 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:44.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:45.036 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:45.060 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:45.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:44 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2203251225' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:45.162 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:45.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:44 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2203251225' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:44 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2203251225' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:45.183 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:45.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:45.285 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:45.306 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:45.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:45.409 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:45.431 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:45.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:45.532 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:45.553 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:45.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:45.655 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:45.685 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:45.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:45.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:45.808 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:45.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:45.910 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:45.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:45 smithi130 ceph-mon[110068]: pgmap v1795: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 18076/212937 objects degraded (8.489%); 12184/212937 objects misplaced (5.722%); 991 KiB/s, 15 objects/s recovering 2024-08-17T17:53:45.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:45 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18076/214539 objects degraded (8.426%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:45 smithi130 ceph-mon[105521]: pgmap v1795: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 18076/212937 objects degraded (8.489%); 12184/212937 objects misplaced (5.722%); 991 KiB/s, 15 objects/s recovering 2024-08-17T17:53:45.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:45 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18076/214539 objects degraded (8.426%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:45.931 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:45.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:45.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:46.033 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:46.058 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:46.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:45 smithi179 ceph-mon[93519]: pgmap v1795: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 18076/212937 objects degraded (8.489%); 12184/212937 objects misplaced (5.722%); 991 KiB/s, 15 objects/s recovering 2024-08-17T17:53:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:45 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18076/214539 objects degraded (8.426%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:46.160 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:46.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.182 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:46.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:46.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:46.307 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:46.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:46.409 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:46.433 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:46.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:46.536 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:46.566 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:46.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:46.668 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:46.689 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:46.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:46.766 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T17:53:46.791 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:46.814 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:46.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:46.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:46 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2559511064' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:53:46.870 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:46 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2559511064' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:53:46.917 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:46.938 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:46.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:46.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:47.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:47.071 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:47.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:46 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2559511064' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:53:47.174 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:47.198 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:47.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:47.300 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:47.321 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:47.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:47.423 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:47.444 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:47.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:47.546 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:47.568 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:47.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:47.670 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:47.696 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:47.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:47.798 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:47.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:47.864 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:47 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:47.864 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:47 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2559511064' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:53:47.864 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:47 smithi130 ceph-mon[105521]: osdmap e1221: 8 total, 8 up, 6 in 2024-08-17T17:53:47.864 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:47 smithi130 ceph-mon[105521]: pgmap v1797: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 121 op/s; 18008/212733 objects degraded (8.465%); 12150/212733 objects misplaced (5.711%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:47.865 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:47 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:47.865 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:47 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2559511064' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:53:47.865 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:47 smithi130 ceph-mon[110068]: osdmap e1221: 8 total, 8 up, 6 in 2024-08-17T17:53:47.865 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:47 smithi130 ceph-mon[110068]: pgmap v1797: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 121 op/s; 18008/212733 objects degraded (8.465%); 12150/212733 objects misplaced (5.711%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:47.923 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:47.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:47.945 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:47.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:48.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:48.070 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:48.072 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:48.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:47 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:53:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:47 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2559511064' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:53:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:47 smithi179 ceph-mon[93519]: osdmap e1221: 8 total, 8 up, 6 in 2024-08-17T17:53:48.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:47 smithi179 ceph-mon[93519]: pgmap v1797: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 121 op/s; 18008/212733 objects degraded (8.465%); 12150/212733 objects misplaced (5.711%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:48.174 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:48.195 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:48.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:48.297 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:48.319 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:48.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:48.421 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:48.445 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:48.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:48.547 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:48.574 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:48.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:48.676 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:48.698 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:48.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:48.800 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:48.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:48.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:48.898 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:53:48.923 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:48.944 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:48.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:48.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:49.046 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:49.069 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:53:49.070 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:53:49.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.072 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:49.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:49.173 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:49.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.195 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:49.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:49.296 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:49.316 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:49.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:49.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:49 smithi130 ceph-mon[105521]: pgmap v1798: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 125 op/s; 18008/211290 objects degraded (8.523%); 12070/211290 objects misplaced (5.713%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:49.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:49 smithi130 ceph-mon[105521]: from='client.55228 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:49.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:49 smithi130 ceph-mon[110068]: pgmap v1798: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 125 op/s; 18008/211290 objects degraded (8.523%); 12070/211290 objects misplaced (5.713%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:49.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:49 smithi130 ceph-mon[110068]: from='client.55228 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:49.419 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:49.443 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:49.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:49.544 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:49.568 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:49.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:49 smithi179 ceph-mon[93519]: pgmap v1798: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 125 op/s; 18008/211290 objects degraded (8.523%); 12070/211290 objects misplaced (5.713%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:49 smithi179 ceph-mon[93519]: from='client.55228 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:53:49.670 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:49.694 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:49.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:49.796 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:49.817 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:49.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:49.919 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:49.942 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:49.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:49.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:50.044 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:50.065 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:50.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:50.090 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:53:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:53:49] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-08-17T17:53:50.091 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1798,"stamp":"2024-08-17T17:53:49.044186+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4614526949,"num_objects":70430,"num_object_clones":0,"num_object_copies":211290,"num_objects_missing_on_primary":3872,"num_objects_missing":3872,"num_objects_degraded":18008,"num_objects_misplaced":12070,"num_objects_unfound":0,"num_objects_dirty":70430,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":139484,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34336,"num_bytes_recovered":2071665677,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107687,"ondisk_log_size":107687,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13378992,"kb_used_data":12003764,"kb_used_omap":295,"kb_used_meta":1374872,"kb_avail":549091920,"statfs":{"total":575970213888,"available":562270126080,"internally_reserved":0,"allocated":12291854336,"data_stored":12262638529,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1407869391},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-82116608,"num_objects":-1253,"num_object_clones":0,"num_object_copies":-3759,"num_objects_missing_on_primary":-82,"num_objects_missing":-82,"num_objects_degraded":-166,"num_objects_misplaced":-233,"num_objects_unfound":0,"num_objects_dirty":-1253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1253,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":163,"num_bytes_recovered":10682368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002798"},"pg_stats":[{"pgid":"45.f","version":"1221'8717","reported_seq":19238,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:48.918783+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:53:48.918783+0000","last_peered":"2024-08-17T17:53:48.918783+0000","last_clean":"2024-08-17T17:53:48.918783+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:53:48.918783+0000","last_undegraded":"2024-08-17T17:53:48.918783+0000","last_fullsized":"2024-08-17T17:53:48.918783+0000","mapping_epoch":1194,"log_start":"1170'1831","ondisk_log_start":"1170'1831","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6886,"log_dups_size":0,"ondisk_log_size":6886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291438592,"num_objects":4447,"num_object_clones":0,"num_object_copies":13341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8717,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1221'8623","reported_seq":19084,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:47.846291+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:53:47.846291+0000","last_peered":"2024-08-17T17:53:47.846291+0000","last_clean":"2024-08-17T17:53:47.846291+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:53:47.846291+0000","last_undegraded":"2024-08-17T17:53:47.846291+0000","last_fullsized":"2024-08-17T17:53:47.846291+0000","mapping_epoch":1211,"log_start":"1170'1632","ondisk_log_start":"1170'1632","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6991,"log_dups_size":0,"ondisk_log_size":6991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286457856,"num_objects":4371,"num_object_clones":0,"num_object_copies":13113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8623,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1221'8933","reported_seq":19761,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:48.523497+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:53:48.523497+0000","last_peered":"2024-08-17T17:53:48.523497+0000","last_clean":"2024-08-17T17:53:48.523497+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:53:48.523497+0000","last_undegraded":"2024-08-17T17:53:48.523497+0000","last_fullsized":"2024-08-17T17:53:48.523497+0000","mapping_epoch":1205,"log_start":"1171'2968","ondisk_log_start":"1171'2968","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5965,"log_dups_size":0,"ondisk_log_size":5965,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291242008,"num_objects":4445,"num_object_clones":0,"num_object_copies":13335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4445,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8933,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1221'8665","reported_seq":8727,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:47.870237+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:53:47.870237+0000","last_peered":"2024-08-17T17:53:47.870237+0000","last_clean":"2024-08-17T17:53:47.870237+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:53:47.870237+0000","last_undegraded":"2024-08-17T17:53:47.870237+0000","last_fullsized":"2024-08-17T17:53:47.870237+0000","mapping_epoch":1167,"log_start":"1170'1989","ondisk_log_start":"1170'1989","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6676,"log_dups_size":0,"ondisk_log_size":6676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284098560,"num_objects":4335,"num_object_clones":0,"num_object_copies":13005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8665,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1220'8703","reported_seq":13935,"reported_epoch":1220,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:46.631207+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:53:46.631207+0000","last_peered":"2024-08-17T17:53:46.631207+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:53:46.631207+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1171'2726","ondisk_log_start":"1171'2726","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5977,"log_dups_size":0,"ondisk_log_size":5977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289079296,"num_objects":4411,"num_object_clones":0,"num_object_copies":13233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":4411,"num_objects_unfound":0,"num_objects_dirty":4411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8703,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":3893},{"shards":"1,4,5","objects":518}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1221'8801","reported_seq":8863,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:47.664020+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:53:47.664020+0000","last_peered":"2024-08-17T17:53:47.664020+0000","last_clean":"2024-08-17T17:53:47.664020+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:53:47.664020+0000","last_undegraded":"2024-08-17T17:53:47.664020+0000","last_fullsized":"2024-08-17T17:53:47.664020+0000","mapping_epoch":1167,"log_start":"1170'2077","ondisk_log_start":"1170'2077","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6724,"log_dups_size":0,"ondisk_log_size":6724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281346048,"num_objects":4293,"num_object_clones":0,"num_object_copies":12879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8801,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1221'8768","reported_seq":8830,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:47.849668+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:53:47.849668+0000","last_peered":"2024-08-17T17:53:47.849668+0000","last_clean":"2024-08-17T17:53:47.849668+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:53:47.849668+0000","last_undegraded":"2024-08-17T17:53:47.849668+0000","last_fullsized":"2024-08-17T17:53:47.849668+0000","mapping_epoch":1167,"log_start":"1170'2069","ondisk_log_start":"1170'2069","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6699,"log_dups_size":0,"ondisk_log_size":6699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292159488,"num_objects":4458,"num_object_clones":0,"num_object_copies":13374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8768,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1220'8650","reported_seq":15240,"reported_epoch":1220,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:46.717279+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:53:46.717279+0000","last_peered":"2024-08-17T17:53:46.717279+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:53:46.717279+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1171'2675","ondisk_log_start":"1171'2675","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5975,"log_dups_size":0,"ondisk_log_size":5975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284426240,"num_objects":4340,"num_object_clones":0,"num_object_copies":13020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3973,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8650,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2155,"num_bytes_recovered":141230080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":953},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1221'8659","reported_seq":8731,"reported_epoch":1221,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:47.856967+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:53:47.856967+0000","last_peered":"2024-08-17T17:53:47.856967+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:53:47.856967+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'1977","ondisk_log_start":"1170'1977","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6682,"log_dups_size":0,"ondisk_log_size":6682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289472512,"num_objects":4417,"num_object_clones":0,"num_object_copies":13251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8659,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":4417},{"shards":"1,6","objects":-1416}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1221'8693","reported_seq":11760,"reported_epoch":1221,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:48.578604+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:53:48.578604+0000","last_peered":"2024-08-17T17:53:48.578604+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:53:48.578604+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1171'2806","ondisk_log_start":"1171'2806","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5887,"log_dups_size":0,"ondisk_log_size":5887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293666816,"num_objects":4481,"num_object_clones":0,"num_object_copies":13443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8693,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":2920},{"shards":"1,3,5","objects":922}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1220'8628","reported_seq":8749,"reported_epoch":1220,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:43.842482+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:53:43.842482+0000","last_peered":"2024-08-17T17:53:43.842482+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:53:43.842482+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8628,"log_dups_size":0,"ondisk_log_size":8628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291373056,"num_objects":4446,"num_object_clones":0,"num_object_copies":13338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8628,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-707}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1220'8619","reported_seq":14800,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:43.426488+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:53:43.426488+0000","last_peered":"2024-08-17T17:53:43.426488+0000","last_clean":"2024-08-17T17:53:43.426488+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:53:43.426488+0000","last_undegraded":"2024-08-17T17:53:43.426488+0000","last_fullsized":"2024-08-17T17:53:43.426488+0000","mapping_epoch":1183,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8619,"log_dups_size":0,"ondisk_log_size":8619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290127872,"num_objects":4427,"num_object_clones":0,"num_object_copies":13281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8619,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1221'8764","reported_seq":8885,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:48.577208+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:53:48.577208+0000","last_peered":"2024-08-17T17:53:48.577208+0000","last_clean":"2024-08-17T17:53:48.577208+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:53:48.577208+0000","last_undegraded":"2024-08-17T17:53:48.577208+0000","last_fullsized":"2024-08-17T17:53:48.577208+0000","mapping_epoch":1167,"log_start":"1171'2791","ondisk_log_start":"1171'2791","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5973,"log_dups_size":0,"ondisk_log_size":5973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288096256,"num_objects":4396,"num_object_clones":0,"num_object_copies":13188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8764,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1221'8612","reported_seq":8674,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:47.612467+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:53:47.612467+0000","last_peered":"2024-08-17T17:53:47.612467+0000","last_clean":"2024-08-17T17:53:47.612467+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:53:47.612467+0000","last_undegraded":"2024-08-17T17:53:47.612467+0000","last_fullsized":"2024-08-17T17:53:47.612467+0000","mapping_epoch":1167,"log_start":"1170'1625","ondisk_log_start":"1170'1625","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6987,"log_dups_size":0,"ondisk_log_size":6987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283246592,"num_objects":4322,"num_object_clones":0,"num_object_copies":12966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8612,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1221'8849","reported_seq":18975,"reported_epoch":1221,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:53:48.975116+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:53:48.975116+0000","last_peered":"2024-08-17T17:53:48.975116+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:53:48.975116+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'1991","ondisk_log_start":"1170'1991","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6858,"log_dups_size":0,"ondisk_log_size":6858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295895040,"num_objects":4515,"num_object_clones":0,"num_object_copies":13545,"num_objects_missing_on_primary":3872,"num_objects_missing":3872,"num_objects_degraded":265,"num_objects_misplaced":7659,"num_objects_unfound":0,"num_objects_dirty":4515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8849,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4274,"num_bytes_recovered":280100864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":13},{"shards":"0,4,6","objects":1356},{"shards":"0,6","objects":1},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":2880}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1221'8522","reported_seq":21830,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:48.527318+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:53:48.527318+0000","last_peered":"2024-08-17T17:53:48.527318+0000","last_clean":"2024-08-17T17:53:48.527318+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:53:48.527318+0000","last_undegraded":"2024-08-17T17:53:48.527318+0000","last_fullsized":"2024-08-17T17:53:48.527318+0000","mapping_epoch":1196,"log_start":"1170'2569","ondisk_log_start":"1170'2569","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5953,"log_dups_size":0,"ondisk_log_size":5953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281804800,"num_objects":4300,"num_object_clones":0,"num_object_copies":12900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8522,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1898,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.836315+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:53:46.836315+0000","last_peered":"2024-08-17T17:53:46.836315+0000","last_clean":"2024-08-17T17:53:46.836315+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:53:46.836315+0000","last_undegraded":"2024-08-17T17:53:46.836315+0000","last_fullsized":"2024-08-17T17:53:46.836315+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":498,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.124734+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:53:34.124734+0000","last_peered":"2024-08-17T17:53:34.124734+0000","last_clean":"2024-08-17T17:53:34.124734+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:53:34.124734+0000","last_undegraded":"2024-08-17T17:53:34.124734+0000","last_fullsized":"2024-08-17T17:53:34.124734+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1818,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.974225+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:53:46.974225+0000","last_peered":"2024-08-17T17:53:46.974225+0000","last_clean":"2024-08-17T17:53:46.974225+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:53:46.974225+0000","last_undegraded":"2024-08-17T17:53:46.974225+0000","last_fullsized":"2024-08-17T17:53:46.974225+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1842,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973268+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:53:46.973268+0000","last_peered":"2024-08-17T17:53:46.973268+0000","last_clean":"2024-08-17T17:53:46.973268+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:53:46.973268+0000","last_undegraded":"2024-08-17T17:53:46.973268+0000","last_fullsized":"2024-08-17T17:53:46.973268+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":60,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.125351+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:53:34.125351+0000","last_peered":"2024-08-17T17:53:34.125351+0000","last_clean":"2024-08-17T17:53:34.125351+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:53:34.125351+0000","last_undegraded":"2024-08-17T17:53:34.125351+0000","last_fullsized":"2024-08-17T17:53:34.125351+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1235,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973471+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:53:46.973471+0000","last_peered":"2024-08-17T17:53:46.973471+0000","last_clean":"2024-08-17T17:53:46.973471+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:53:46.973471+0000","last_undegraded":"2024-08-17T17:53:46.973471+0000","last_fullsized":"2024-08-17T17:53:46.973471+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":91,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222787+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:53:34.222787+0000","last_peered":"2024-08-17T17:53:34.222787+0000","last_clean":"2024-08-17T17:53:34.222787+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:53:34.222787+0000","last_undegraded":"2024-08-17T17:53:34.222787+0000","last_fullsized":"2024-08-17T17:53:34.222787+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":613,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973658+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:53:46.973658+0000","last_peered":"2024-08-17T17:53:46.973658+0000","last_clean":"2024-08-17T17:53:46.973658+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:53:46.973658+0000","last_undegraded":"2024-08-17T17:53:46.973658+0000","last_fullsized":"2024-08-17T17:53:46.973658+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1857,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.836019+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:53:46.836019+0000","last_peered":"2024-08-17T17:53:46.836019+0000","last_clean":"2024-08-17T17:53:46.836019+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:53:46.836019+0000","last_undegraded":"2024-08-17T17:53:46.836019+0000","last_fullsized":"2024-08-17T17:53:46.836019+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1857,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.254651+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:53:34.254651+0000","last_peered":"2024-08-17T17:53:34.254651+0000","last_clean":"2024-08-17T17:53:34.254651+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:53:34.254651+0000","last_undegraded":"2024-08-17T17:53:34.254651+0000","last_fullsized":"2024-08-17T17:53:34.254651+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":76,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222434+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:53:34.222434+0000","last_peered":"2024-08-17T17:53:34.222434+0000","last_clean":"2024-08-17T17:53:34.222434+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:53:34.222434+0000","last_undegraded":"2024-08-17T17:53:34.222434+0000","last_fullsized":"2024-08-17T17:53:34.222434+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1830,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.974002+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:53:46.974002+0000","last_peered":"2024-08-17T17:53:46.974002+0000","last_clean":"2024-08-17T17:53:46.974002+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:53:46.974002+0000","last_undegraded":"2024-08-17T17:53:46.974002+0000","last_fullsized":"2024-08-17T17:53:46.974002+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1883,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222364+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:53:34.222364+0000","last_peered":"2024-08-17T17:53:34.222364+0000","last_clean":"2024-08-17T17:53:34.222364+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:53:34.222364+0000","last_undegraded":"2024-08-17T17:53:34.222364+0000","last_fullsized":"2024-08-17T17:53:34.222364+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1879,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.836364+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:53:46.836364+0000","last_peered":"2024-08-17T17:53:46.836364+0000","last_clean":"2024-08-17T17:53:46.836364+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:53:46.836364+0000","last_undegraded":"2024-08-17T17:53:46.836364+0000","last_fullsized":"2024-08-17T17:53:46.836364+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1853,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973356+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:53:46.973356+0000","last_peered":"2024-08-17T17:53:46.973356+0000","last_clean":"2024-08-17T17:53:46.973356+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:53:46.973356+0000","last_undegraded":"2024-08-17T17:53:46.973356+0000","last_fullsized":"2024-08-17T17:53:46.973356+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1195,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.125396+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:53:34.125396+0000","last_peered":"2024-08-17T17:53:34.125396+0000","last_clean":"2024-08-17T17:53:34.125396+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:53:34.125396+0000","last_undegraded":"2024-08-17T17:53:34.125396+0000","last_fullsized":"2024-08-17T17:53:34.125396+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1824,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973225+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:53:46.973225+0000","last_peered":"2024-08-17T17:53:46.973225+0000","last_clean":"2024-08-17T17:53:46.973225+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:53:46.973225+0000","last_undegraded":"2024-08-17T17:53:46.973225+0000","last_fullsized":"2024-08-17T17:53:46.973225+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1874,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.836324+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:53:46.836324+0000","last_peered":"2024-08-17T17:53:46.836324+0000","last_clean":"2024-08-17T17:53:46.836324+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:53:46.836324+0000","last_undegraded":"2024-08-17T17:53:46.836324+0000","last_fullsized":"2024-08-17T17:53:46.836324+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1780,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973168+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:53:46.973168+0000","last_peered":"2024-08-17T17:53:46.973168+0000","last_clean":"2024-08-17T17:53:46.973168+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:53:46.973168+0000","last_undegraded":"2024-08-17T17:53:46.973168+0000","last_fullsized":"2024-08-17T17:53:46.973168+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":514,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.260567+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:53:34.260567+0000","last_peered":"2024-08-17T17:53:34.260567+0000","last_clean":"2024-08-17T17:53:34.260567+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:53:34.260567+0000","last_undegraded":"2024-08-17T17:53:34.260567+0000","last_fullsized":"2024-08-17T17:53:34.260567+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1191,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.260322+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:53:34.260322+0000","last_peered":"2024-08-17T17:53:34.260322+0000","last_clean":"2024-08-17T17:53:34.260322+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:53:34.260322+0000","last_undegraded":"2024-08-17T17:53:34.260322+0000","last_fullsized":"2024-08-17T17:53:34.260322+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1876,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222584+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:53:34.222584+0000","last_peered":"2024-08-17T17:53:34.222584+0000","last_clean":"2024-08-17T17:53:34.222584+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:53:34.222584+0000","last_undegraded":"2024-08-17T17:53:34.222584+0000","last_fullsized":"2024-08-17T17:53:34.222584+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":352,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.261839+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:53:34.261839+0000","last_peered":"2024-08-17T17:53:34.261839+0000","last_clean":"2024-08-17T17:53:34.261839+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:53:34.261839+0000","last_undegraded":"2024-08-17T17:53:34.261839+0000","last_fullsized":"2024-08-17T17:53:34.261839+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":498,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.125028+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:53:34.125028+0000","last_peered":"2024-08-17T17:53:34.125028+0000","last_clean":"2024-08-17T17:53:34.125028+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:53:34.125028+0000","last_undegraded":"2024-08-17T17:53:34.125028+0000","last_fullsized":"2024-08-17T17:53:34.125028+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1176,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.124910+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:53:34.124910+0000","last_peered":"2024-08-17T17:53:34.124910+0000","last_clean":"2024-08-17T17:53:34.124910+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:53:34.124910+0000","last_undegraded":"2024-08-17T17:53:34.124910+0000","last_fullsized":"2024-08-17T17:53:34.124910+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1184,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.125440+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:53:34.125440+0000","last_peered":"2024-08-17T17:53:34.125440+0000","last_clean":"2024-08-17T17:53:34.125440+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:53:34.125440+0000","last_undegraded":"2024-08-17T17:53:34.125440+0000","last_fullsized":"2024-08-17T17:53:34.125440+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1926,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.836139+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:53:46.836139+0000","last_peered":"2024-08-17T17:53:46.836139+0000","last_clean":"2024-08-17T17:53:46.836139+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:53:46.836139+0000","last_undegraded":"2024-08-17T17:53:46.836139+0000","last_fullsized":"2024-08-17T17:53:46.836139+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1810,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973350+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:53:46.973350+0000","last_peered":"2024-08-17T17:53:46.973350+0000","last_clean":"2024-08-17T17:53:46.973350+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:53:46.973350+0000","last_undegraded":"2024-08-17T17:53:46.973350+0000","last_fullsized":"2024-08-17T17:53:46.973350+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1882,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.223547+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:53:34.223547+0000","last_peered":"2024-08-17T17:53:34.223547+0000","last_clean":"2024-08-17T17:53:34.223547+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:53:34.223547+0000","last_undegraded":"2024-08-17T17:53:34.223547+0000","last_fullsized":"2024-08-17T17:53:34.223547+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1812,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973916+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:53:46.973916+0000","last_peered":"2024-08-17T17:53:46.973916+0000","last_clean":"2024-08-17T17:53:46.973916+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:53:46.973916+0000","last_undegraded":"2024-08-17T17:53:46.973916+0000","last_fullsized":"2024-08-17T17:53:46.973916+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1844,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.836062+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:53:46.836062+0000","last_peered":"2024-08-17T17:53:46.836062+0000","last_clean":"2024-08-17T17:53:46.836062+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:53:46.836062+0000","last_undegraded":"2024-08-17T17:53:46.836062+0000","last_fullsized":"2024-08-17T17:53:46.836062+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1211,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.260517+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:53:34.260517+0000","last_peered":"2024-08-17T17:53:34.260517+0000","last_clean":"2024-08-17T17:53:34.260517+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:53:34.260517+0000","last_undegraded":"2024-08-17T17:53:34.260517+0000","last_fullsized":"2024-08-17T17:53:34.260517+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1889,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.223038+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:53:34.223038+0000","last_peered":"2024-08-17T17:53:34.223038+0000","last_clean":"2024-08-17T17:53:34.223038+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:53:34.223038+0000","last_undegraded":"2024-08-17T17:53:34.223038+0000","last_fullsized":"2024-08-17T17:53:34.223038+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":75,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222332+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:53:34.222332+0000","last_peered":"2024-08-17T17:53:34.222332+0000","last_clean":"2024-08-17T17:53:34.222332+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:53:34.222332+0000","last_undegraded":"2024-08-17T17:53:34.222332+0000","last_fullsized":"2024-08-17T17:53:34.222332+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1908,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222918+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:53:34.222918+0000","last_peered":"2024-08-17T17:53:34.222918+0000","last_clean":"2024-08-17T17:53:34.222918+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:53:34.222918+0000","last_undegraded":"2024-08-17T17:53:34.222918+0000","last_fullsized":"2024-08-17T17:53:34.222918+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":646,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973770+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:53:46.973770+0000","last_peered":"2024-08-17T17:53:46.973770+0000","last_clean":"2024-08-17T17:53:46.973770+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:53:46.973770+0000","last_undegraded":"2024-08-17T17:53:46.973770+0000","last_fullsized":"2024-08-17T17:53:46.973770+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1889,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.223101+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:53:34.223101+0000","last_peered":"2024-08-17T17:53:34.223101+0000","last_clean":"2024-08-17T17:53:34.223101+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:53:34.223101+0000","last_undegraded":"2024-08-17T17:53:34.223101+0000","last_fullsized":"2024-08-17T17:53:34.223101+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1219,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.835925+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:53:46.835925+0000","last_peered":"2024-08-17T17:53:46.835925+0000","last_clean":"2024-08-17T17:53:46.835925+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:53:46.835925+0000","last_undegraded":"2024-08-17T17:53:46.835925+0000","last_fullsized":"2024-08-17T17:53:46.835925+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":135,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222581+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:53:34.222581+0000","last_peered":"2024-08-17T17:53:34.222581+0000","last_clean":"2024-08-17T17:53:34.222581+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:53:34.222581+0000","last_undegraded":"2024-08-17T17:53:34.222581+0000","last_fullsized":"2024-08-17T17:53:34.222581+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1899,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.223938+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:53:34.223938+0000","last_peered":"2024-08-17T17:53:34.223938+0000","last_clean":"2024-08-17T17:53:34.223938+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:53:34.223938+0000","last_undegraded":"2024-08-17T17:53:34.223938+0000","last_fullsized":"2024-08-17T17:53:34.223938+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1877,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222253+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:53:34.222253+0000","last_peered":"2024-08-17T17:53:34.222253+0000","last_clean":"2024-08-17T17:53:34.222253+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:53:34.222253+0000","last_undegraded":"2024-08-17T17:53:34.222253+0000","last_fullsized":"2024-08-17T17:53:34.222253+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1897,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.254284+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:53:34.254284+0000","last_peered":"2024-08-17T17:53:34.254284+0000","last_clean":"2024-08-17T17:53:34.254284+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:53:34.254284+0000","last_undegraded":"2024-08-17T17:53:34.254284+0000","last_fullsized":"2024-08-17T17:53:34.254284+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1905,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.836370+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:53:46.836370+0000","last_peered":"2024-08-17T17:53:46.836370+0000","last_clean":"2024-08-17T17:53:46.836370+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:53:46.836370+0000","last_undegraded":"2024-08-17T17:53:46.836370+0000","last_fullsized":"2024-08-17T17:53:46.836370+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1885,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.842476+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:53:46.842476+0000","last_peered":"2024-08-17T17:53:46.842476+0000","last_clean":"2024-08-17T17:53:46.842476+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:53:46.842476+0000","last_undegraded":"2024-08-17T17:53:46.842476+0000","last_fullsized":"2024-08-17T17:53:46.842476+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1218,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.260557+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:53:34.260557+0000","last_peered":"2024-08-17T17:53:34.260557+0000","last_clean":"2024-08-17T17:53:34.260557+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:53:34.260557+0000","last_undegraded":"2024-08-17T17:53:34.260557+0000","last_fullsized":"2024-08-17T17:53:34.260557+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1164,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.125378+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:53:34.125378+0000","last_peered":"2024-08-17T17:53:34.125378+0000","last_clean":"2024-08-17T17:53:34.125378+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:53:34.125378+0000","last_undegraded":"2024-08-17T17:53:34.125378+0000","last_fullsized":"2024-08-17T17:53:34.125378+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1794,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973372+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:53:46.973372+0000","last_peered":"2024-08-17T17:53:46.973372+0000","last_clean":"2024-08-17T17:53:46.973372+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:53:46.973372+0000","last_undegraded":"2024-08-17T17:53:46.973372+0000","last_fullsized":"2024-08-17T17:53:46.973372+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":123,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.254414+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:53:34.254414+0000","last_peered":"2024-08-17T17:53:34.254414+0000","last_clean":"2024-08-17T17:53:34.254414+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:53:34.254414+0000","last_undegraded":"2024-08-17T17:53:34.254414+0000","last_fullsized":"2024-08-17T17:53:34.254414+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1530,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.260381+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:53:34.260381+0000","last_peered":"2024-08-17T17:53:34.260381+0000","last_clean":"2024-08-17T17:53:34.260381+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:53:34.260381+0000","last_undegraded":"2024-08-17T17:53:34.260381+0000","last_fullsized":"2024-08-17T17:53:34.260381+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1884,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.223865+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:53:34.223865+0000","last_peered":"2024-08-17T17:53:34.223865+0000","last_clean":"2024-08-17T17:53:34.223865+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:53:34.223865+0000","last_undegraded":"2024-08-17T17:53:34.223865+0000","last_fullsized":"2024-08-17T17:53:34.223865+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1826,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973270+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:53:46.973270+0000","last_peered":"2024-08-17T17:53:46.973270+0000","last_clean":"2024-08-17T17:53:46.973270+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:53:46.973270+0000","last_undegraded":"2024-08-17T17:53:46.973270+0000","last_fullsized":"2024-08-17T17:53:46.973270+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":498,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.125263+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:53:34.125263+0000","last_peered":"2024-08-17T17:53:34.125263+0000","last_clean":"2024-08-17T17:53:34.125263+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:53:34.125263+0000","last_undegraded":"2024-08-17T17:53:34.125263+0000","last_fullsized":"2024-08-17T17:53:34.125263+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":562,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.254172+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:53:34.254172+0000","last_peered":"2024-08-17T17:53:34.254172+0000","last_clean":"2024-08-17T17:53:34.254172+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:53:34.254172+0000","last_undegraded":"2024-08-17T17:53:34.254172+0000","last_fullsized":"2024-08-17T17:53:34.254172+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":578,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.254522+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:53:34.254522+0000","last_peered":"2024-08-17T17:53:34.254522+0000","last_clean":"2024-08-17T17:53:34.254522+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:53:34.254522+0000","last_undegraded":"2024-08-17T17:53:34.254522+0000","last_fullsized":"2024-08-17T17:53:34.254522+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1182,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.125133+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:53:34.125133+0000","last_peered":"2024-08-17T17:53:34.125133+0000","last_clean":"2024-08-17T17:53:34.125133+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:53:34.125133+0000","last_undegraded":"2024-08-17T17:53:34.125133+0000","last_fullsized":"2024-08-17T17:53:34.125133+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":596,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973710+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:53:46.973710+0000","last_peered":"2024-08-17T17:53:46.973710+0000","last_clean":"2024-08-17T17:53:46.973710+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:53:46.973710+0000","last_undegraded":"2024-08-17T17:53:46.973710+0000","last_fullsized":"2024-08-17T17:53:46.973710+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1822,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.974366+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:53:46.974366+0000","last_peered":"2024-08-17T17:53:46.974366+0000","last_clean":"2024-08-17T17:53:46.974366+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:53:46.974366+0000","last_undegraded":"2024-08-17T17:53:46.974366+0000","last_fullsized":"2024-08-17T17:53:46.974366+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1918,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.836137+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:53:46.836137+0000","last_peered":"2024-08-17T17:53:46.836137+0000","last_clean":"2024-08-17T17:53:46.836137+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:53:46.836137+0000","last_undegraded":"2024-08-17T17:53:46.836137+0000","last_fullsized":"2024-08-17T17:53:46.836137+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":629,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973413+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:53:46.973413+0000","last_peered":"2024-08-17T17:53:46.973413+0000","last_clean":"2024-08-17T17:53:46.973413+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:53:46.973413+0000","last_undegraded":"2024-08-17T17:53:46.973413+0000","last_fullsized":"2024-08-17T17:53:46.973413+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1171,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.124993+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:53:34.124993+0000","last_peered":"2024-08-17T17:53:34.124993+0000","last_clean":"2024-08-17T17:53:34.124993+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:53:34.124993+0000","last_undegraded":"2024-08-17T17:53:34.124993+0000","last_fullsized":"2024-08-17T17:53:34.124993+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1170,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.260433+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:53:34.260433+0000","last_peered":"2024-08-17T17:53:34.260433+0000","last_clean":"2024-08-17T17:53:34.260433+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:53:34.260433+0000","last_undegraded":"2024-08-17T17:53:34.260433+0000","last_fullsized":"2024-08-17T17:53:34.260433+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":611,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.222447+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:53:34.222447+0000","last_peered":"2024-08-17T17:53:34.222447+0000","last_clean":"2024-08-17T17:53:34.222447+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:53:34.222447+0000","last_undegraded":"2024-08-17T17:53:34.222447+0000","last_fullsized":"2024-08-17T17:53:34.222447+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1188,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.260747+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:53:34.260747+0000","last_peered":"2024-08-17T17:53:34.260747+0000","last_clean":"2024-08-17T17:53:34.260747+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:53:34.260747+0000","last_undegraded":"2024-08-17T17:53:34.260747+0000","last_fullsized":"2024-08-17T17:53:34.260747+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":72,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.260122+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:53:34.260122+0000","last_peered":"2024-08-17T17:53:34.260122+0000","last_clean":"2024-08-17T17:53:34.260122+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:53:34.260122+0000","last_undegraded":"2024-08-17T17:53:34.260122+0000","last_fullsized":"2024-08-17T17:53:34.260122+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1204,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.260911+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:53:34.260911+0000","last_peered":"2024-08-17T17:53:34.260911+0000","last_clean":"2024-08-17T17:53:34.260911+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:53:34.260911+0000","last_undegraded":"2024-08-17T17:53:34.260911+0000","last_fullsized":"2024-08-17T17:53:34.260911+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1221,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.124901+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:53:34.124901+0000","last_peered":"2024-08-17T17:53:34.124901+0000","last_clean":"2024-08-17T17:53:34.124901+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:53:34.124901+0000","last_undegraded":"2024-08-17T17:53:34.124901+0000","last_fullsized":"2024-08-17T17:53:34.124901+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1824,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973825+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:53:46.973825+0000","last_peered":"2024-08-17T17:53:46.973825+0000","last_clean":"2024-08-17T17:53:46.973825+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:53:46.973825+0000","last_undegraded":"2024-08-17T17:53:46.973825+0000","last_fullsized":"2024-08-17T17:53:46.973825+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1165,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.260342+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:53:34.260342+0000","last_peered":"2024-08-17T17:53:34.260342+0000","last_clean":"2024-08-17T17:53:34.260342+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:53:34.260342+0000","last_undegraded":"2024-08-17T17:53:34.260342+0000","last_fullsized":"2024-08-17T17:53:34.260342+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1877,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.836139+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:53:46.836139+0000","last_peered":"2024-08-17T17:53:46.836139+0000","last_clean":"2024-08-17T17:53:46.836139+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:53:46.836139+0000","last_undegraded":"2024-08-17T17:53:46.836139+0000","last_fullsized":"2024-08-17T17:53:46.836139+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1878,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.835951+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:53:46.835951+0000","last_peered":"2024-08-17T17:53:46.835951+0000","last_clean":"2024-08-17T17:53:46.835951+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:53:46.835951+0000","last_undegraded":"2024-08-17T17:53:46.835951+0000","last_fullsized":"2024-08-17T17:53:46.835951+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":578,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.260334+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:53:34.260334+0000","last_peered":"2024-08-17T17:53:34.260334+0000","last_clean":"2024-08-17T17:53:34.260334+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:53:34.260334+0000","last_undegraded":"2024-08-17T17:53:34.260334+0000","last_fullsized":"2024-08-17T17:53:34.260334+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1190,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:53:34.260429+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:53:34.260429+0000","last_peered":"2024-08-17T17:53:34.260429+0000","last_clean":"2024-08-17T17:53:34.260429+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:53:34.260429+0000","last_undegraded":"2024-08-17T17:53:34.260429+0000","last_fullsized":"2024-08-17T17:53:34.260429+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1824,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.973628+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:53:46.973628+0000","last_peered":"2024-08-17T17:53:46.973628+0000","last_clean":"2024-08-17T17:53:46.973628+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:53:46.973628+0000","last_undegraded":"2024-08-17T17:53:46.973628+0000","last_fullsized":"2024-08-17T17:53:46.973628+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":4613931032,"num_objects":70404,"num_object_clones":0,"num_object_copies":211212,"num_objects_missing_on_primary":3872,"num_objects_missing":3872,"num_objects_degraded":18008,"num_objects_misplaced":12070,"num_objects_unfound":0,"num_objects_dirty":70404,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":139206,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34307,"num_bytes_recovered":2069889024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14483992576,"data_stored":14483980360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107480,"ondisk_log_size":107480,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234395,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2220484,"kb_used_data":1914808,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91524668,"statfs":{"total":95995035648,"available":93721260032,"internally_reserved":0,"allocated":1960763392,"data_stored":1955883803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496030,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2502276,"kb_used_data":2196604,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91242876,"statfs":{"total":95995035648,"available":93432705024,"internally_reserved":0,"allocated":2249322496,"data_stored":2244439312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757668,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326282,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2725616,"kb_used_data":2513316,"kb_used_omap":57,"kb_used_meta":212230,"kb_avail":91019536,"statfs":{"total":95995035648,"available":93204004864,"internally_reserved":0,"allocated":2573635584,"data_stored":2568771175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":217324283},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405520,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2474596,"kb_used_data":2276124,"kb_used_omap":61,"kb_used_meta":198402,"kb_avail":91270556,"statfs":{"total":95995035648,"available":93461049344,"internally_reserved":0,"allocated":2330750976,"data_stored":2325890975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":203164600},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452055,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1857368,"kb_used_data":1703376,"kb_used_omap":54,"kb_used_meta":153929,"kb_avail":91887784,"statfs":{"total":95995035648,"available":94093090816,"internally_reserved":0,"allocated":1744257024,"data_stored":1739404012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":157623694},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474747,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1598652,"kb_used_data":1399536,"kb_used_omap":119,"kb_used_meta":198984,"kb_avail":92146500,"statfs":{"total":95995035648,"available":94358016000,"internally_reserved":0,"allocated":1433124864,"data_stored":1428249252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":203760350},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019303,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1416499200,"data_stored":1416495128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1729429504,"data_stored":1729429504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2317484032,"data_stored":2317484032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2555514880,"data_stored":2555510808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1061027840,"data_stored":1061027840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1225064448,"data_stored":1225064448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2234191872,"data_stored":2234187800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1944780800,"data_stored":1944780800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:53:50.095 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:53:50.100 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:53:50.166 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:50.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.205 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:50.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:50.306 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:50.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.328 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:50.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:50.429 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:50.452 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:50.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:50.554 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:50.576 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:50.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:50.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:53:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:53:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:53:50.678 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:50.699 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:50.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:50.801 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:50.822 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:50.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:50.891 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:50 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18008/211290 objects degraded (8.523%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:50.891 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:50 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18008/211290 objects degraded (8.523%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:50.924 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:50.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:50.953 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:50.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:51.054 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:51.075 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:51.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:50 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18008/211290 objects degraded (8.523%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:51.152 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:53:51.152 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:53:51.177 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:51.201 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:51.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:51.302 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:51.324 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:51.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:51.425 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:51.447 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:51.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:51.549 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:51.570 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:51.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:51.672 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:51.693 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:51.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:51.788 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:51.789 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:51.793 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:51.794 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:51.816 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:51.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:51.837 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:53:51.918 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:51.940 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:51.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:51.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:52.000 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:51 smithi130 ceph-mon[105521]: pgmap v1799: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 125 op/s; 18008/211044 objects degraded (8.533%); 12070/211044 objects misplaced (5.719%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:52.000 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:51 smithi130 ceph-mon[110068]: pgmap v1799: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 125 op/s; 18008/211044 objects degraded (8.533%); 12070/211044 objects misplaced (5.719%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:52.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:52.065 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:51 smithi179 ceph-mon[93519]: pgmap v1799: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 125 op/s; 18008/211044 objects degraded (8.533%); 12070/211044 objects misplaced (5.719%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:53:52.166 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:52.189 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:52.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:52.291 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:52.311 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:52.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:52.413 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:52.439 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:52.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:52.541 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:52.569 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:52.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:52.671 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:52.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:52.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:52.794 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:52.815 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:52.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:52.916 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:52.940 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:52.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:52.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:53.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:53.065 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:53.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:53.167 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:53.189 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:53.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:53.290 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:53.312 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:53.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:53.413 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:53.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:53 smithi130 ceph-mon[105521]: pgmap v1800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 17948/210840 objects degraded (8.513%); 12032/210840 objects misplaced (5.707%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:53:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:53 smithi130 ceph-mon[110068]: pgmap v1800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 17948/210840 objects degraded (8.513%); 12032/210840 objects misplaced (5.707%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:53:53.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:53.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:53.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:53.557 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:53.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:53 smithi179 ceph-mon[93519]: pgmap v1800: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 17948/210840 objects degraded (8.513%); 12032/210840 objects misplaced (5.707%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:53:53.660 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:53.681 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:53.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:53.783 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:53.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.811 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:53.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:53.912 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:53.933 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:53.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:53.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:54.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:54.056 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:54.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:54.158 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:54.186 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:54.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:54.288 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:54.309 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:54.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:54.410 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:54.433 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:54.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:54.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:54.558 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:54.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:54.660 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:54.682 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:54.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:54.784 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:54.805 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:54.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:54.907 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:54.931 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:54.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:54.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:55.033 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:55.054 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:55.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:55.155 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:55.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:55 smithi130 ceph-mon[105521]: pgmap v1801: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 17948/208980 objects degraded (8.588%); 11951/208980 objects misplaced (5.719%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:53:55.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:55 smithi130 ceph-mon[110068]: pgmap v1801: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 17948/208980 objects degraded (8.588%); 11951/208980 objects misplaced (5.719%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:53:55.178 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:55.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:55.280 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:55.306 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:55.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:55.408 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:55.434 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:55.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:55.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:55.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.559 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:55.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:55.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:55 smithi179 ceph-mon[93519]: pgmap v1801: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 17948/208980 objects degraded (8.588%); 11951/208980 objects misplaced (5.719%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:53:55.660 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:55.682 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:55.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:55.784 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:55.805 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:55.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:55.907 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:55.936 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:55.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:55.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:56.037 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:56.067 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:56.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:56.169 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:56.190 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:56.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:56.291 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:53:56.291 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:53:56.291 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:53:56.292 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:53:56.292 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:53:56.292 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:53:56.292 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:56.312 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:53:56.313 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:53:56.325 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:56.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:56.341 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:53:56.342 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:53:56.366 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:53:56.367 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:53:56.367 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17948/208980 objects degraded (8.588%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:56.367 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17948/208980 objects degraded (8.588%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:56.381 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:56.394 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:53:56.394 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:53:56.418 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:53:56.418 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:53:56.427 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:56.461 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:56.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:56.497 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:53:56.498 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 17948/208980 objects degraded (8.588%), 6 pgs degraded, 6 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1557,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1221,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":69660,"data_bytes":4564064229,"bytes_used":13568380928,"bytes_avail":562401832960,"bytes_total":575970213888,"degraded_objects":17948,"degraded_total":208980,"degraded_ratio":0.085883816633170629,"misplaced_objects":11951,"misplaced_total":208980,"misplaced_ratio":0.057187290649822947,"recovering_objects_per_sec":14,"recovering_bytes_per_sec":943439,"recovering_keys_per_sec":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":131},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":25,"modified":"2024-08-17T17:49:39.646037+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (6m)\n [==========================..] (remaining: 27s)","progress":0.93258428573608398,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:53:56.563 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:56.584 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:56.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:56.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17948/208980 objects degraded (8.588%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:53:56.686 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:56.709 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:56.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:56.811 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:56.832 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:56.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:56.934 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:56.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:56.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:56.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:57.059 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:53:57.080 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:57.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:53:57.182 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:57.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.204 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:57.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:57.305 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:57.329 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:57.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:57.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:57 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1820198383' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:57.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:57 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3570213813' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:53:57.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:57 smithi130 ceph-mon[105521]: pgmap v1802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 17842/208668 objects degraded (8.550%); 11900/208668 objects misplaced (5.703%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:53:57.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:57 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1820198383' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:57.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:57 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3570213813' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:53:57.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:57 smithi130 ceph-mon[110068]: pgmap v1802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 17842/208668 objects degraded (8.550%); 11900/208668 objects misplaced (5.703%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:53:57.431 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:57.454 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:57.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:57.499 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:53:57.500 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:53:57.557 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:57.577 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:57.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:57.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:57 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1820198383' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:53:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:57 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3570213813' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:53:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:57 smithi179 ceph-mon[93519]: pgmap v1802: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 17842/208668 objects degraded (8.550%); 11900/208668 objects misplaced (5.703%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:53:57.679 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:57.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.710 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:57.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:57.811 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:53:57.833 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:57.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:53:57.935 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:57.956 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:57.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:57.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:58.058 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:58.079 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:58.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:58.143 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T17:53:58.181 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:58.203 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:58.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:58.305 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:58.326 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:58.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:58 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2721458099' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:58 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:58.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:58.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:58 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2721458099' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:58.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:58 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:58.428 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:58.451 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:58.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:58.553 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:53:58.574 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:58.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:53:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:53:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:58 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2721458099' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:58 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:53:58.675 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:58.696 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:58.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:58.798 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:58.819 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:58.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:58.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:53:58.943 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:58.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:58.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:53:59.045 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:59.068 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:59.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:59.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:59 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:53:59.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:59 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:59.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:59 smithi130 ceph-mon[105521]: osdmap e1222: 8 total, 8 up, 6 in 2024-08-17T17:53:59.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:53:59 smithi130 ceph-mon[105521]: pgmap v1804: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 17842/207015 objects degraded (8.619%); 11900/207015 objects misplaced (5.748%); 787 KiB/s, 12 objects/s recovering 2024-08-17T17:53:59.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:59 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:53:59.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:59 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:59.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:59 smithi130 ceph-mon[110068]: osdmap e1222: 8 total, 8 up, 6 in 2024-08-17T17:53:59.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:53:59 smithi130 ceph-mon[110068]: pgmap v1804: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 17842/207015 objects degraded (8.619%); 11900/207015 objects misplaced (5.748%); 787 KiB/s, 12 objects/s recovering 2024-08-17T17:53:59.170 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:59.191 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:59.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:59.292 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:53:59.321 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:59.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:53:59.423 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:53:59.446 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:59.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:53:59.547 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:59.568 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:59.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:59.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:59 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:53:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:59 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:53:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:59 smithi179 ceph-mon[93519]: osdmap e1222: 8 total, 8 up, 6 in 2024-08-17T17:53:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:53:59 smithi179 ceph-mon[93519]: pgmap v1804: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 17842/207015 objects degraded (8.619%); 11900/207015 objects misplaced (5.748%); 787 KiB/s, 12 objects/s recovering 2024-08-17T17:53:59.670 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:59.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:59.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:53:59.745 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:53:59.794 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:53:59.818 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:53:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:53:59.920 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:53:59.941 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:53:59.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:53:59.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:00.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:00.063 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:00.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:00.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:00.165 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:00.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:53:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:53:59] "GET /metrics HTTP/1.1" 200 36808 "" "Prometheus/2.51.0" 2024-08-17T17:54:00.186 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:00.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:00.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:00.196 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:54:00.288 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:00.382 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:00.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:00.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:00.484 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:00.505 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:00.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:00.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:54:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:54:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:54:00.607 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:00.629 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:00.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:00.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:00.730 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:00.752 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:00.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:00.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:00.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:00.876 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:00.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:00.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:00.949 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:54:00.977 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:00.996 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:00 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17842/207015 objects degraded (8.619%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:00.996 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:00 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17842/207015 objects degraded (8.619%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:00.999 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:00.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:00 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17842/207015 objects degraded (8.619%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:01.100 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:01.123 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:01.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:01.224 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:01.246 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:01.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:01.348 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:01.369 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:01.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:01.471 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:01.496 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:01.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:01.598 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:01.619 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:01.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:01.721 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:01.742 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:01.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:01.844 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:01.865 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:01.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:01.932 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:01.966 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:01.973 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:01.992 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:01.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:01.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:02.061 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:01 smithi130 ceph-mon[105521]: pgmap v1805: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 17842/206490 objects degraded (8.641%); 11822/206490 objects misplaced (5.725%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:02.061 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:01 smithi130 ceph-mon[110068]: pgmap v1805: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 17842/206490 objects degraded (8.641%); 11822/206490 objects misplaced (5.725%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:01 smithi179 ceph-mon[93519]: pgmap v1805: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 17842/206490 objects degraded (8.641%); 11822/206490 objects misplaced (5.725%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:02.094 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:02.115 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:02.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:02.217 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:02.238 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:02.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:02.340 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:02.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.362 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:02.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:02.463 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:02.484 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:02.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:02.585 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:02.609 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:02.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:02.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:02.736 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:02.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:02.838 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:02.859 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:02.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:02.960 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:02.987 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:02.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:02.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:03.089 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:03.110 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:03.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:03.212 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:03.234 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:03.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:03.335 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:03.356 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:03.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:03.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:03 smithi130 ceph-mon[105521]: pgmap v1806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 17778/206301 objects degraded (8.618%); 11791/206301 objects misplaced (5.715%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:03.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:03 smithi130 ceph-mon[110068]: pgmap v1806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 17778/206301 objects degraded (8.618%); 11791/206301 objects misplaced (5.715%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:03.458 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:03.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:03.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:03.581 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:03.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:03 smithi179 ceph-mon[93519]: pgmap v1806: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 17778/206301 objects degraded (8.618%); 11791/206301 objects misplaced (5.715%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:03.608 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:03.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:03.710 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:03.732 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:03.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:03.834 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:03.855 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:03.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:03.957 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:03.981 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:03.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:03.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:04.082 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:04.104 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:04.206 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:04.228 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:04.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:04.331 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:04.352 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:04.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:04.454 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:04.479 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:04.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:04.581 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:04.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:04.704 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:04.729 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:04.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:04.758 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:54:04.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:04.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.855 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:04.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:04.956 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:04.978 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:04.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:04.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:05.002 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:04 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/525889435' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:05.002 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:04 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:05.002 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:04 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:05.002 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:04 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/525889435' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:05.002 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:04 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:05.002 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:04 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:05.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:05.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:04 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/525889435' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:05.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:04 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:05.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:04 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:05.102 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:05.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:05.204 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:05.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.226 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:05.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:05.327 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:05.349 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:05.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:05.451 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:05.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:05.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:05.574 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:05.598 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:05.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:05.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:05.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:05.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:05.823 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:05.844 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:05.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:05.946 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:05.978 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:05.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:05.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:06.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:05 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:54:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:05 smithi179 ceph-mon[93519]: osdmap e1223: 8 total, 8 up, 6 in 2024-08-17T17:54:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:05 smithi179 ceph-mon[93519]: pgmap v1808: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 156 op/s; 17778/204912 objects degraded (8.676%); 11713/204912 objects misplaced (5.716%); 880 KiB/s, 13 objects/s recovering 2024-08-17T17:54:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:05 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17778/206301 objects degraded (8.618%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:06.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.102 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:06.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:05 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:54:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:05 smithi130 ceph-mon[105521]: osdmap e1223: 8 total, 8 up, 6 in 2024-08-17T17:54:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:05 smithi130 ceph-mon[105521]: pgmap v1808: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 156 op/s; 17778/204912 objects degraded (8.676%); 11713/204912 objects misplaced (5.716%); 880 KiB/s, 13 objects/s recovering 2024-08-17T17:54:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:05 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17778/206301 objects degraded (8.618%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:05 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:54:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:05 smithi130 ceph-mon[110068]: osdmap e1223: 8 total, 8 up, 6 in 2024-08-17T17:54:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:05 smithi130 ceph-mon[110068]: pgmap v1808: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 156 op/s; 17778/204912 objects degraded (8.676%); 11713/204912 objects misplaced (5.716%); 880 KiB/s, 13 objects/s recovering 2024-08-17T17:54:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:05 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17778/206301 objects degraded (8.618%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:06.171 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:54:06.171 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:54:06.203 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:06.226 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:06.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:06.328 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:06.353 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:06.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:06.354 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:54:06.355 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:54:06.355 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T17:54:06.355 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T17:54:06.355 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:54:06.454 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:06.476 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:06.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:06.578 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:06.600 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:06.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:06.702 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:06.723 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:06.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:06.825 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:06.847 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:06.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:06.949 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:06.970 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:06.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:06.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:07.072 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:07.101 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:07.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:07.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:07 smithi130 ceph-mon[110068]: from='client.45584 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:54:07.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:07 smithi130 ceph-mon[110068]: pgmap v1809: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 17716/204714 objects degraded (8.654%); 11678/204714 objects misplaced (5.705%); 1013 KiB/s, 15 objects/s recovering 2024-08-17T17:54:07.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:07 smithi130 ceph-mon[105521]: from='client.45584 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:54:07.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:07 smithi130 ceph-mon[105521]: pgmap v1809: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 17716/204714 objects degraded (8.654%); 11678/204714 objects misplaced (5.705%); 1013 KiB/s, 15 objects/s recovering 2024-08-17T17:54:07.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:07.223 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:07.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:07.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:54:07.325 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:07.388 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:07.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:07.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:07.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.515 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:07.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:07.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:07 smithi179 ceph-mon[93519]: from='client.45584 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:54:07.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:07 smithi179 ceph-mon[93519]: pgmap v1809: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 17716/204714 objects degraded (8.654%); 11678/204714 objects misplaced (5.705%); 1013 KiB/s, 15 objects/s recovering 2024-08-17T17:54:07.616 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:07.642 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:07.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:07.744 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:07.765 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:07.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:07.867 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:07.889 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:07.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:07.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:07.991 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:08.014 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:08.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:08.116 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:08.137 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:08.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:08.239 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:08.264 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:08.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:08.366 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:08.387 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:08.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:08.489 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:08.510 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:08.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:08.612 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:08.625 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1808,"stamp":"2024-08-17T17:54:05.048514+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4475197413,"num_objects":68304,"num_object_clones":0,"num_object_copies":204912,"num_objects_missing_on_primary":3753,"num_objects_missing":3753,"num_objects_degraded":17778,"num_objects_misplaced":11713,"num_objects_unfound":0,"num_objects_dirty":68304,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":141610,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34569,"num_bytes_recovered":2086935565,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107913,"ondisk_log_size":107913,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13064036,"kb_used_data":11645812,"kb_used_omap":295,"kb_used_meta":1417880,"kb_avail":549406876,"statfs":{"total":575970213888,"available":562592641024,"internally_reserved":0,"allocated":11925311488,"data_stored":11896036193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1451909583},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,2,2,0,2,4,8],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-82051072,"num_objects":-1252,"num_object_clones":0,"num_object_copies":-3756,"num_objects_missing_on_primary":-78,"num_objects_missing":-78,"num_objects_degraded":-64,"num_objects_misplaced":-187,"num_objects_unfound":0,"num_objects_dirty":-1252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1252,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":110,"num_bytes_recovered":7208960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002125"},"pg_stats":[{"pgid":"45.f","version":"1222'8841","reported_seq":19364,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.074774+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:54:04.074774+0000","last_peered":"2024-08-17T17:54:04.074774+0000","last_clean":"2024-08-17T17:54:04.074774+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:54:04.074774+0000","last_undegraded":"2024-08-17T17:54:04.074774+0000","last_fullsized":"2024-08-17T17:54:04.074774+0000","mapping_epoch":1194,"log_start":"1170'1931","ondisk_log_start":"1170'1931","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6910,"log_dups_size":0,"ondisk_log_size":6910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283312128,"num_objects":4323,"num_object_clones":0,"num_object_copies":12969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8841,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1222'8764","reported_seq":19226,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:54:03.035674+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:54:03.035674+0000","last_peered":"2024-08-17T17:54:03.035674+0000","last_clean":"2024-08-17T17:54:03.035674+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:54:03.035674+0000","last_undegraded":"2024-08-17T17:54:03.035674+0000","last_fullsized":"2024-08-17T17:54:03.035674+0000","mapping_epoch":1211,"log_start":"1170'1732","ondisk_log_start":"1170'1732","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7032,"log_dups_size":0,"ondisk_log_size":7032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277217280,"num_objects":4230,"num_object_clones":0,"num_object_copies":12690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8764,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1222'9060","reported_seq":19890,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:54:03.633877+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:54:03.633877+0000","last_peered":"2024-08-17T17:54:03.633877+0000","last_clean":"2024-08-17T17:54:03.633877+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:54:03.633877+0000","last_undegraded":"2024-08-17T17:54:03.633877+0000","last_fullsized":"2024-08-17T17:54:03.633877+0000","mapping_epoch":1205,"log_start":"1174'3168","ondisk_log_start":"1174'3168","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5892,"log_dups_size":0,"ondisk_log_size":5892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282918936,"num_objects":4318,"num_object_clones":0,"num_object_copies":12954,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4318,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9060,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1222'8793","reported_seq":8856,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:54:03.031706+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:54:03.031706+0000","last_peered":"2024-08-17T17:54:03.031706+0000","last_clean":"2024-08-17T17:54:03.031706+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:54:03.031706+0000","last_undegraded":"2024-08-17T17:54:03.031706+0000","last_fullsized":"2024-08-17T17:54:03.031706+0000","mapping_epoch":1167,"log_start":"1170'2089","ondisk_log_start":"1170'2089","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6704,"log_dups_size":0,"ondisk_log_size":6704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275709952,"num_objects":4207,"num_object_clones":0,"num_object_copies":12621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8793,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1222'8823","reported_seq":14059,"reported_epoch":1222,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:01.370993+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:54:01.370993+0000","last_peered":"2024-08-17T17:54:01.370993+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:54:01.370993+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1171'2926","ondisk_log_start":"1171'2926","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5897,"log_dups_size":0,"ondisk_log_size":5897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281214976,"num_objects":4291,"num_object_clones":0,"num_object_copies":12873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":4291,"num_objects_unfound":0,"num_objects_dirty":4291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8823,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":3773},{"shards":"1,4,5","objects":518}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1222'8921","reported_seq":8984,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:54:03.011631+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:54:03.011631+0000","last_peered":"2024-08-17T17:54:03.011631+0000","last_clean":"2024-08-17T17:54:03.011631+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:54:03.011631+0000","last_undegraded":"2024-08-17T17:54:03.011631+0000","last_fullsized":"2024-08-17T17:54:03.011631+0000","mapping_epoch":1167,"log_start":"1170'2177","ondisk_log_start":"1170'2177","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6744,"log_dups_size":0,"ondisk_log_size":6744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273481728,"num_objects":4173,"num_object_clones":0,"num_object_copies":12519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8921,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1222'8898","reported_seq":8961,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:54:02.332415+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:54:02.332415+0000","last_peered":"2024-08-17T17:54:02.332415+0000","last_clean":"2024-08-17T17:54:02.332415+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:54:02.332415+0000","last_undegraded":"2024-08-17T17:54:02.332415+0000","last_fullsized":"2024-08-17T17:54:02.332415+0000","mapping_epoch":1167,"log_start":"1170'2169","ondisk_log_start":"1170'2169","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6729,"log_dups_size":0,"ondisk_log_size":6729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283639808,"num_objects":4328,"num_object_clones":0,"num_object_copies":12984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8898,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1222'8765","reported_seq":15704,"reported_epoch":1222,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:01.094169+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:54:01.094169+0000","last_peered":"2024-08-17T17:54:01.094169+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:54:01.094169+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1171'2875","ondisk_log_start":"1171'2875","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5890,"log_dups_size":0,"ondisk_log_size":5890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276889600,"num_objects":4225,"num_object_clones":0,"num_object_copies":12675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3743,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8765,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2270,"num_bytes_recovered":148766720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":838},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1222'8783","reported_seq":8856,"reported_epoch":1222,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:03.034541+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:54:03.034541+0000","last_peered":"2024-08-17T17:54:03.034541+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:54:03.034541+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'2077","ondisk_log_start":"1170'2077","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6706,"log_dups_size":0,"ondisk_log_size":6706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281346048,"num_objects":4293,"num_object_clones":0,"num_object_copies":12879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8783,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":4293},{"shards":"1,6","objects":-1540}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1222'8828","reported_seq":11897,"reported_epoch":1222,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:03.649789+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:54:03.649789+0000","last_peered":"2024-08-17T17:54:03.649789+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:54:03.649789+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1171'2906","ondisk_log_start":"1171'2906","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5922,"log_dups_size":0,"ondisk_log_size":5922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284819456,"num_objects":4346,"num_object_clones":0,"num_object_copies":13038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8828,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":2785},{"shards":"1,3,5","objects":787}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1222'8802","reported_seq":8927,"reported_epoch":1222,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:03.994194+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:54:03.994194+0000","last_peered":"2024-08-17T17:54:03.994194+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:54:03.994194+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8802,"log_dups_size":0,"ondisk_log_size":8802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279969792,"num_objects":4272,"num_object_clones":0,"num_object_copies":12816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8802,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-881}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1222'8786","reported_seq":14972,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:54:03.829818+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:54:03.829818+0000","last_peered":"2024-08-17T17:54:03.829818+0000","last_clean":"2024-08-17T17:54:03.829818+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:54:03.829818+0000","last_undegraded":"2024-08-17T17:54:03.829818+0000","last_fullsized":"2024-08-17T17:54:03.829818+0000","mapping_epoch":1183,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8786,"log_dups_size":0,"ondisk_log_size":8786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279183360,"num_objects":4260,"num_object_clones":0,"num_object_copies":12780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8786,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1222'8891","reported_seq":9015,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:54:03.546676+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:54:03.546676+0000","last_peered":"2024-08-17T17:54:03.546676+0000","last_clean":"2024-08-17T17:54:03.546676+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:54:03.546676+0000","last_undegraded":"2024-08-17T17:54:03.546676+0000","last_fullsized":"2024-08-17T17:54:03.546676+0000","mapping_epoch":1167,"log_start":"1171'2991","ondisk_log_start":"1171'2991","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5900,"log_dups_size":0,"ondisk_log_size":5900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279773184,"num_objects":4269,"num_object_clones":0,"num_object_copies":12807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8891,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1222'8756","reported_seq":8819,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:54:02.987257+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:54:02.987257+0000","last_peered":"2024-08-17T17:54:02.987257+0000","last_clean":"2024-08-17T17:54:02.987257+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:54:02.987257+0000","last_undegraded":"2024-08-17T17:54:02.987257+0000","last_fullsized":"2024-08-17T17:54:02.987257+0000","mapping_epoch":1167,"log_start":"1170'1725","ondisk_log_start":"1170'1725","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7031,"log_dups_size":0,"ondisk_log_size":7031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273809408,"num_objects":4178,"num_object_clones":0,"num_object_copies":12534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8756,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1222'8968","reported_seq":19451,"reported_epoch":1222,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:04.165894+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:54:04.165894+0000","last_peered":"2024-08-17T17:54:04.165894+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:54:04.165894+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'2091","ondisk_log_start":"1170'2091","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6877,"log_dups_size":0,"ondisk_log_size":6877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288096256,"num_objects":4396,"num_object_clones":0,"num_object_copies":13188,"num_objects_missing_on_primary":3753,"num_objects_missing":3753,"num_objects_degraded":265,"num_objects_misplaced":7422,"num_objects_unfound":0,"num_objects_dirty":4396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8968,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4392,"num_bytes_recovered":287834112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":14},{"shards":"0,4,6","objects":1356},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":2761}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1222'8653","reported_seq":21963,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:54:03.694265+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:54:03.694265+0000","last_peered":"2024-08-17T17:54:03.694265+0000","last_clean":"2024-08-17T17:54:03.694265+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:54:03.694265+0000","last_undegraded":"2024-08-17T17:54:03.694265+0000","last_fullsized":"2024-08-17T17:54:03.694265+0000","mapping_epoch":1196,"log_start":"1171'2769","ondisk_log_start":"1171'2769","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5884,"log_dups_size":0,"ondisk_log_size":5884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273219584,"num_objects":4169,"num_object_clones":0,"num_object_copies":12507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8653,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1900,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.253145+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:53:58.253145+0000","last_peered":"2024-08-17T17:53:58.253145+0000","last_clean":"2024-08-17T17:53:58.253145+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:53:58.253145+0000","last_undegraded":"2024-08-17T17:53:58.253145+0000","last_fullsized":"2024-08-17T17:53:58.253145+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":499,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.771153+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:53:46.771153+0000","last_peered":"2024-08-17T17:53:46.771153+0000","last_clean":"2024-08-17T17:53:46.771153+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:53:46.771153+0000","last_undegraded":"2024-08-17T17:53:46.771153+0000","last_fullsized":"2024-08-17T17:53:46.771153+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1820,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.142367+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:53:58.142367+0000","last_peered":"2024-08-17T17:53:58.142367+0000","last_clean":"2024-08-17T17:53:58.142367+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:53:58.142367+0000","last_undegraded":"2024-08-17T17:53:58.142367+0000","last_fullsized":"2024-08-17T17:53:58.142367+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1844,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.141506+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:53:58.141506+0000","last_peered":"2024-08-17T17:53:58.141506+0000","last_clean":"2024-08-17T17:53:58.141506+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:53:58.141506+0000","last_undegraded":"2024-08-17T17:53:58.141506+0000","last_fullsized":"2024-08-17T17:53:58.141506+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":61,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.771717+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:53:46.771717+0000","last_peered":"2024-08-17T17:53:46.771717+0000","last_clean":"2024-08-17T17:53:46.771717+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:53:46.771717+0000","last_undegraded":"2024-08-17T17:53:46.771717+0000","last_fullsized":"2024-08-17T17:53:46.771717+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1237,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.141793+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:53:58.141793+0000","last_peered":"2024-08-17T17:53:58.141793+0000","last_clean":"2024-08-17T17:53:58.141793+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:53:58.141793+0000","last_undegraded":"2024-08-17T17:53:58.141793+0000","last_fullsized":"2024-08-17T17:53:58.141793+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":95,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.243150+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:53:58.243150+0000","last_peered":"2024-08-17T17:53:58.243150+0000","last_clean":"2024-08-17T17:53:58.243150+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:53:58.243150+0000","last_undegraded":"2024-08-17T17:53:58.243150+0000","last_fullsized":"2024-08-17T17:53:58.243150+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":615,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.142173+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:53:58.142173+0000","last_peered":"2024-08-17T17:53:58.142173+0000","last_clean":"2024-08-17T17:53:58.142173+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:53:58.142173+0000","last_undegraded":"2024-08-17T17:53:58.142173+0000","last_fullsized":"2024-08-17T17:53:58.142173+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1859,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.252457+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:53:58.252457+0000","last_peered":"2024-08-17T17:53:58.252457+0000","last_clean":"2024-08-17T17:53:58.252457+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:53:58.252457+0000","last_undegraded":"2024-08-17T17:53:58.252457+0000","last_fullsized":"2024-08-17T17:53:58.252457+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1861,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.246646+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:53:58.246646+0000","last_peered":"2024-08-17T17:53:58.246646+0000","last_clean":"2024-08-17T17:53:58.246646+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:53:58.246646+0000","last_undegraded":"2024-08-17T17:53:58.246646+0000","last_fullsized":"2024-08-17T17:53:58.246646+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":80,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.242418+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:53:58.242418+0000","last_peered":"2024-08-17T17:53:58.242418+0000","last_clean":"2024-08-17T17:53:58.242418+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:53:58.242418+0000","last_undegraded":"2024-08-17T17:53:58.242418+0000","last_fullsized":"2024-08-17T17:53:58.242418+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1832,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.142150+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:53:58.142150+0000","last_peered":"2024-08-17T17:53:58.142150+0000","last_clean":"2024-08-17T17:53:58.142150+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:53:58.142150+0000","last_undegraded":"2024-08-17T17:53:58.142150+0000","last_fullsized":"2024-08-17T17:53:58.142150+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1887,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.242184+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:53:58.242184+0000","last_peered":"2024-08-17T17:53:58.242184+0000","last_clean":"2024-08-17T17:53:58.242184+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:53:58.242184+0000","last_undegraded":"2024-08-17T17:53:58.242184+0000","last_fullsized":"2024-08-17T17:53:58.242184+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1881,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.252953+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:53:58.252953+0000","last_peered":"2024-08-17T17:53:58.252953+0000","last_clean":"2024-08-17T17:53:58.252953+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:53:58.252953+0000","last_undegraded":"2024-08-17T17:53:58.252953+0000","last_fullsized":"2024-08-17T17:53:58.252953+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1855,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.141659+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:53:58.141659+0000","last_peered":"2024-08-17T17:53:58.141659+0000","last_clean":"2024-08-17T17:53:58.141659+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:53:58.141659+0000","last_undegraded":"2024-08-17T17:53:58.141659+0000","last_fullsized":"2024-08-17T17:53:58.141659+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1196,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.771914+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:53:46.771914+0000","last_peered":"2024-08-17T17:53:46.771914+0000","last_clean":"2024-08-17T17:53:46.771914+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:53:46.771914+0000","last_undegraded":"2024-08-17T17:53:46.771914+0000","last_fullsized":"2024-08-17T17:53:46.771914+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1826,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.141478+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:53:58.141478+0000","last_peered":"2024-08-17T17:53:58.141478+0000","last_clean":"2024-08-17T17:53:58.141478+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:53:58.141478+0000","last_undegraded":"2024-08-17T17:53:58.141478+0000","last_fullsized":"2024-08-17T17:53:58.141478+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1876,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.253158+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:53:58.253158+0000","last_peered":"2024-08-17T17:53:58.253158+0000","last_clean":"2024-08-17T17:53:58.253158+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:53:58.253158+0000","last_undegraded":"2024-08-17T17:53:58.253158+0000","last_fullsized":"2024-08-17T17:53:58.253158+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1782,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.141434+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:53:58.141434+0000","last_peered":"2024-08-17T17:53:58.141434+0000","last_clean":"2024-08-17T17:53:58.141434+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:53:58.141434+0000","last_undegraded":"2024-08-17T17:53:58.141434+0000","last_fullsized":"2024-08-17T17:53:58.141434+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":515,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.965601+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:53:46.965601+0000","last_peered":"2024-08-17T17:53:46.965601+0000","last_clean":"2024-08-17T17:53:46.965601+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:53:46.965601+0000","last_undegraded":"2024-08-17T17:53:46.965601+0000","last_fullsized":"2024-08-17T17:53:46.965601+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1192,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.965367+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:53:46.965367+0000","last_peered":"2024-08-17T17:53:46.965367+0000","last_clean":"2024-08-17T17:53:46.965367+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:53:46.965367+0000","last_undegraded":"2024-08-17T17:53:46.965367+0000","last_fullsized":"2024-08-17T17:53:46.965367+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1881,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.242527+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:53:58.242527+0000","last_peered":"2024-08-17T17:53:58.242527+0000","last_clean":"2024-08-17T17:53:58.242527+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:53:58.242527+0000","last_undegraded":"2024-08-17T17:53:58.242527+0000","last_fullsized":"2024-08-17T17:53:58.242527+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":353,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.967622+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:53:46.967622+0000","last_peered":"2024-08-17T17:53:46.967622+0000","last_clean":"2024-08-17T17:53:46.967622+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:53:46.967622+0000","last_undegraded":"2024-08-17T17:53:46.967622+0000","last_fullsized":"2024-08-17T17:53:46.967622+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":499,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.771452+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:53:46.771452+0000","last_peered":"2024-08-17T17:53:46.771452+0000","last_clean":"2024-08-17T17:53:46.771452+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:53:46.771452+0000","last_undegraded":"2024-08-17T17:53:46.771452+0000","last_fullsized":"2024-08-17T17:53:46.771452+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1177,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.771574+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:53:46.771574+0000","last_peered":"2024-08-17T17:53:46.771574+0000","last_clean":"2024-08-17T17:53:46.771574+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:53:46.771574+0000","last_undegraded":"2024-08-17T17:53:46.771574+0000","last_fullsized":"2024-08-17T17:53:46.771574+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1185,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.772005+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:53:46.772005+0000","last_peered":"2024-08-17T17:53:46.772005+0000","last_clean":"2024-08-17T17:53:46.772005+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:53:46.772005+0000","last_undegraded":"2024-08-17T17:53:46.772005+0000","last_fullsized":"2024-08-17T17:53:46.772005+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1928,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.252712+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:53:58.252712+0000","last_peered":"2024-08-17T17:53:58.252712+0000","last_clean":"2024-08-17T17:53:58.252712+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:53:58.252712+0000","last_undegraded":"2024-08-17T17:53:58.252712+0000","last_fullsized":"2024-08-17T17:53:58.252712+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1812,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.141571+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:53:58.141571+0000","last_peered":"2024-08-17T17:53:58.141571+0000","last_clean":"2024-08-17T17:53:58.141571+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:53:58.141571+0000","last_undegraded":"2024-08-17T17:53:58.141571+0000","last_fullsized":"2024-08-17T17:53:58.141571+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1886,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.243353+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:53:58.243353+0000","last_peered":"2024-08-17T17:53:58.243353+0000","last_clean":"2024-08-17T17:53:58.243353+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:53:58.243353+0000","last_undegraded":"2024-08-17T17:53:58.243353+0000","last_fullsized":"2024-08-17T17:53:58.243353+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1814,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.142291+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:53:58.142291+0000","last_peered":"2024-08-17T17:53:58.142291+0000","last_clean":"2024-08-17T17:53:58.142291+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:53:58.142291+0000","last_undegraded":"2024-08-17T17:53:58.142291+0000","last_fullsized":"2024-08-17T17:53:58.142291+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1846,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.252592+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:53:58.252592+0000","last_peered":"2024-08-17T17:53:58.252592+0000","last_clean":"2024-08-17T17:53:58.252592+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:53:58.252592+0000","last_undegraded":"2024-08-17T17:53:58.252592+0000","last_fullsized":"2024-08-17T17:53:58.252592+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1212,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.965500+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:53:46.965500+0000","last_peered":"2024-08-17T17:53:46.965500+0000","last_clean":"2024-08-17T17:53:46.965500+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:53:46.965500+0000","last_undegraded":"2024-08-17T17:53:46.965500+0000","last_fullsized":"2024-08-17T17:53:46.965500+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1893,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.243315+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:53:58.243315+0000","last_peered":"2024-08-17T17:53:58.243315+0000","last_clean":"2024-08-17T17:53:58.243315+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:53:58.243315+0000","last_undegraded":"2024-08-17T17:53:58.243315+0000","last_fullsized":"2024-08-17T17:53:58.243315+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":80,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.242323+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:53:58.242323+0000","last_peered":"2024-08-17T17:53:58.242323+0000","last_clean":"2024-08-17T17:53:58.242323+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:53:58.242323+0000","last_undegraded":"2024-08-17T17:53:58.242323+0000","last_fullsized":"2024-08-17T17:53:58.242323+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1912,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.243239+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:53:58.243239+0000","last_peered":"2024-08-17T17:53:58.243239+0000","last_clean":"2024-08-17T17:53:58.243239+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:53:58.243239+0000","last_undegraded":"2024-08-17T17:53:58.243239+0000","last_fullsized":"2024-08-17T17:53:58.243239+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":648,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.142050+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:53:58.142050+0000","last_peered":"2024-08-17T17:53:58.142050+0000","last_clean":"2024-08-17T17:53:58.142050+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:53:58.142050+0000","last_undegraded":"2024-08-17T17:53:58.142050+0000","last_fullsized":"2024-08-17T17:53:58.142050+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1893,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.243393+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:53:58.243393+0000","last_peered":"2024-08-17T17:53:58.243393+0000","last_clean":"2024-08-17T17:53:58.243393+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:53:58.243393+0000","last_undegraded":"2024-08-17T17:53:58.243393+0000","last_fullsized":"2024-08-17T17:53:58.243393+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1221,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.252356+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:53:58.252356+0000","last_peered":"2024-08-17T17:53:58.252356+0000","last_clean":"2024-08-17T17:53:58.252356+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:53:58.252356+0000","last_undegraded":"2024-08-17T17:53:58.252356+0000","last_fullsized":"2024-08-17T17:53:58.252356+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":139,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.242459+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:53:58.242459+0000","last_peered":"2024-08-17T17:53:58.242459+0000","last_clean":"2024-08-17T17:53:58.242459+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:53:58.242459+0000","last_undegraded":"2024-08-17T17:53:58.242459+0000","last_fullsized":"2024-08-17T17:53:58.242459+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1903,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.242834+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:53:58.242834+0000","last_peered":"2024-08-17T17:53:58.242834+0000","last_clean":"2024-08-17T17:53:58.242834+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:53:58.242834+0000","last_undegraded":"2024-08-17T17:53:58.242834+0000","last_fullsized":"2024-08-17T17:53:58.242834+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1881,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.242062+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:53:58.242062+0000","last_peered":"2024-08-17T17:53:58.242062+0000","last_clean":"2024-08-17T17:53:58.242062+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:53:58.242062+0000","last_undegraded":"2024-08-17T17:53:58.242062+0000","last_fullsized":"2024-08-17T17:53:58.242062+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1901,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.246143+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:53:58.246143+0000","last_peered":"2024-08-17T17:53:58.246143+0000","last_clean":"2024-08-17T17:53:58.246143+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:53:58.246143+0000","last_undegraded":"2024-08-17T17:53:58.246143+0000","last_fullsized":"2024-08-17T17:53:58.246143+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1907,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.253030+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:53:58.253030+0000","last_peered":"2024-08-17T17:53:58.253030+0000","last_clean":"2024-08-17T17:53:58.253030+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:53:58.253030+0000","last_undegraded":"2024-08-17T17:53:58.253030+0000","last_fullsized":"2024-08-17T17:53:58.253030+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1887,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.265917+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:53:58.265917+0000","last_peered":"2024-08-17T17:53:58.265917+0000","last_clean":"2024-08-17T17:53:58.265917+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:53:58.265917+0000","last_undegraded":"2024-08-17T17:53:58.265917+0000","last_fullsized":"2024-08-17T17:53:58.265917+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1219,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.965228+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:53:46.965228+0000","last_peered":"2024-08-17T17:53:46.965228+0000","last_clean":"2024-08-17T17:53:46.965228+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:53:46.965228+0000","last_undegraded":"2024-08-17T17:53:46.965228+0000","last_fullsized":"2024-08-17T17:53:46.965228+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1165,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.771774+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:53:46.771774+0000","last_peered":"2024-08-17T17:53:46.771774+0000","last_clean":"2024-08-17T17:53:46.771774+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:53:46.771774+0000","last_undegraded":"2024-08-17T17:53:46.771774+0000","last_fullsized":"2024-08-17T17:53:46.771774+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1796,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.141747+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:53:58.141747+0000","last_peered":"2024-08-17T17:53:58.141747+0000","last_clean":"2024-08-17T17:53:58.141747+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:53:58.141747+0000","last_undegraded":"2024-08-17T17:53:58.141747+0000","last_fullsized":"2024-08-17T17:53:58.141747+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":127,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.246292+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:53:58.246292+0000","last_peered":"2024-08-17T17:53:58.246292+0000","last_clean":"2024-08-17T17:53:58.246292+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:53:58.246292+0000","last_undegraded":"2024-08-17T17:53:58.246292+0000","last_fullsized":"2024-08-17T17:53:58.246292+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1531,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.965149+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:53:46.965149+0000","last_peered":"2024-08-17T17:53:46.965149+0000","last_clean":"2024-08-17T17:53:46.965149+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:53:46.965149+0000","last_undegraded":"2024-08-17T17:53:46.965149+0000","last_fullsized":"2024-08-17T17:53:46.965149+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1888,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.242960+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:53:58.242960+0000","last_peered":"2024-08-17T17:53:58.242960+0000","last_clean":"2024-08-17T17:53:58.242960+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:53:58.242960+0000","last_undegraded":"2024-08-17T17:53:58.242960+0000","last_fullsized":"2024-08-17T17:53:58.242960+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1828,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.141544+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:53:58.141544+0000","last_peered":"2024-08-17T17:53:58.141544+0000","last_clean":"2024-08-17T17:53:58.141544+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:53:58.141544+0000","last_undegraded":"2024-08-17T17:53:58.141544+0000","last_fullsized":"2024-08-17T17:53:58.141544+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":499,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.771691+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:53:46.771691+0000","last_peered":"2024-08-17T17:53:46.771691+0000","last_clean":"2024-08-17T17:53:46.771691+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:53:46.771691+0000","last_undegraded":"2024-08-17T17:53:46.771691+0000","last_fullsized":"2024-08-17T17:53:46.771691+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":566,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.246605+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:53:58.246605+0000","last_peered":"2024-08-17T17:53:58.246605+0000","last_clean":"2024-08-17T17:53:58.246605+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:53:58.246605+0000","last_undegraded":"2024-08-17T17:53:58.246605+0000","last_fullsized":"2024-08-17T17:53:58.246605+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":582,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.246408+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:53:58.246408+0000","last_peered":"2024-08-17T17:53:58.246408+0000","last_clean":"2024-08-17T17:53:58.246408+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:53:58.246408+0000","last_undegraded":"2024-08-17T17:53:58.246408+0000","last_fullsized":"2024-08-17T17:53:58.246408+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1183,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.771568+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:53:46.771568+0000","last_peered":"2024-08-17T17:53:46.771568+0000","last_clean":"2024-08-17T17:53:46.771568+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:53:46.771568+0000","last_undegraded":"2024-08-17T17:53:46.771568+0000","last_fullsized":"2024-08-17T17:53:46.771568+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":598,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.142100+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:53:58.142100+0000","last_peered":"2024-08-17T17:53:58.142100+0000","last_clean":"2024-08-17T17:53:58.142100+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:53:58.142100+0000","last_undegraded":"2024-08-17T17:53:58.142100+0000","last_fullsized":"2024-08-17T17:53:58.142100+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1824,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.142238+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:53:58.142238+0000","last_peered":"2024-08-17T17:53:58.142238+0000","last_clean":"2024-08-17T17:53:58.142238+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:53:58.142238+0000","last_undegraded":"2024-08-17T17:53:58.142238+0000","last_fullsized":"2024-08-17T17:53:58.142238+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1920,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.252564+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:53:58.252564+0000","last_peered":"2024-08-17T17:53:58.252564+0000","last_clean":"2024-08-17T17:53:58.252564+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:53:58.252564+0000","last_undegraded":"2024-08-17T17:53:58.252564+0000","last_fullsized":"2024-08-17T17:53:58.252564+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":631,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.141863+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:53:58.141863+0000","last_peered":"2024-08-17T17:53:58.141863+0000","last_clean":"2024-08-17T17:53:58.141863+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:53:58.141863+0000","last_undegraded":"2024-08-17T17:53:58.141863+0000","last_fullsized":"2024-08-17T17:53:58.141863+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1172,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.771366+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:53:46.771366+0000","last_peered":"2024-08-17T17:53:46.771366+0000","last_clean":"2024-08-17T17:53:46.771366+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:53:46.771366+0000","last_undegraded":"2024-08-17T17:53:46.771366+0000","last_fullsized":"2024-08-17T17:53:46.771366+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1171,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.965472+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:53:46.965472+0000","last_peered":"2024-08-17T17:53:46.965472+0000","last_clean":"2024-08-17T17:53:46.965472+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:53:46.965472+0000","last_undegraded":"2024-08-17T17:53:46.965472+0000","last_fullsized":"2024-08-17T17:53:46.965472+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":615,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.242234+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:53:58.242234+0000","last_peered":"2024-08-17T17:53:58.242234+0000","last_clean":"2024-08-17T17:53:58.242234+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:53:58.242234+0000","last_undegraded":"2024-08-17T17:53:58.242234+0000","last_fullsized":"2024-08-17T17:53:58.242234+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1189,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.965704+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:53:46.965704+0000","last_peered":"2024-08-17T17:53:46.965704+0000","last_clean":"2024-08-17T17:53:46.965704+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:53:46.965704+0000","last_undegraded":"2024-08-17T17:53:46.965704+0000","last_fullsized":"2024-08-17T17:53:46.965704+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":73,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.965056+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:53:46.965056+0000","last_peered":"2024-08-17T17:53:46.965056+0000","last_clean":"2024-08-17T17:53:46.965056+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:53:46.965056+0000","last_undegraded":"2024-08-17T17:53:46.965056+0000","last_fullsized":"2024-08-17T17:53:46.965056+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1205,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.965717+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:53:46.965717+0000","last_peered":"2024-08-17T17:53:46.965717+0000","last_clean":"2024-08-17T17:53:46.965717+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:53:46.965717+0000","last_undegraded":"2024-08-17T17:53:46.965717+0000","last_fullsized":"2024-08-17T17:53:46.965717+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1222,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.771343+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:53:46.771343+0000","last_peered":"2024-08-17T17:53:46.771343+0000","last_clean":"2024-08-17T17:53:46.771343+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:53:46.771343+0000","last_undegraded":"2024-08-17T17:53:46.771343+0000","last_fullsized":"2024-08-17T17:53:46.771343+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1826,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.142242+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:53:58.142242+0000","last_peered":"2024-08-17T17:53:58.142242+0000","last_clean":"2024-08-17T17:53:58.142242+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:53:58.142242+0000","last_undegraded":"2024-08-17T17:53:58.142242+0000","last_fullsized":"2024-08-17T17:53:58.142242+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1166,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.965324+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:53:46.965324+0000","last_peered":"2024-08-17T17:53:46.965324+0000","last_clean":"2024-08-17T17:53:46.965324+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:53:46.965324+0000","last_undegraded":"2024-08-17T17:53:46.965324+0000","last_fullsized":"2024-08-17T17:53:46.965324+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1879,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.252818+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:53:58.252818+0000","last_peered":"2024-08-17T17:53:58.252818+0000","last_clean":"2024-08-17T17:53:58.252818+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:53:58.252818+0000","last_undegraded":"2024-08-17T17:53:58.252818+0000","last_fullsized":"2024-08-17T17:53:58.252818+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1880,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.252343+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:53:58.252343+0000","last_peered":"2024-08-17T17:53:58.252343+0000","last_clean":"2024-08-17T17:53:58.252343+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:53:58.252343+0000","last_undegraded":"2024-08-17T17:53:58.252343+0000","last_fullsized":"2024-08-17T17:53:58.252343+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":579,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.965277+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:53:46.965277+0000","last_peered":"2024-08-17T17:53:46.965277+0000","last_clean":"2024-08-17T17:53:46.965277+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:53:46.965277+0000","last_undegraded":"2024-08-17T17:53:46.965277+0000","last_fullsized":"2024-08-17T17:53:46.965277+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1191,"reported_epoch":1221,"state":"active+clean","last_fresh":"2024-08-17T17:53:46.965427+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:53:46.965427+0000","last_peered":"2024-08-17T17:53:46.965427+0000","last_clean":"2024-08-17T17:53:46.965427+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:53:46.965427+0000","last_undegraded":"2024-08-17T17:53:46.965427+0000","last_fullsized":"2024-08-17T17:53:46.965427+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1826,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.142021+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:53:58.142021+0000","last_peered":"2024-08-17T17:53:58.142021+0000","last_clean":"2024-08-17T17:53:58.142021+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:53:58.142021+0000","last_undegraded":"2024-08-17T17:53:58.142021+0000","last_fullsized":"2024-08-17T17:53:58.142021+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":4474601496,"num_objects":68278,"num_object_clones":0,"num_object_copies":204834,"num_objects_missing_on_primary":3753,"num_objects_missing":3753,"num_objects_degraded":17778,"num_objects_misplaced":11713,"num_objects_unfound":0,"num_objects_dirty":68278,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":141332,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34540,"num_bytes_recovered":2085158912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14086254592,"data_stored":14086242376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107706,"ondisk_log_size":107706,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234398,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2173200,"kb_used_data":1863368,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91571952,"statfs":{"total":95995035648,"available":93769678848,"internally_reserved":0,"allocated":1908088832,"data_stored":1903201807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496033,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2449556,"kb_used_data":2139724,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91295596,"statfs":{"total":95995035648,"available":93486690304,"internally_reserved":0,"allocated":2191077376,"data_stored":2186186756,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757671,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326285,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2655548,"kb_used_data":2439092,"kb_used_omap":57,"kb_used_meta":216390,"kb_avail":91089604,"statfs":{"total":95995035648,"available":93275754496,"internally_reserved":0,"allocated":2497630208,"data_stored":2492758363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":221584123},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405524,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2407108,"kb_used_data":2186428,"kb_used_omap":61,"kb_used_meta":220610,"kb_avail":91338044,"statfs":{"total":95995035648,"available":93530157056,"internally_reserved":0,"allocated":2238902272,"data_stored":2234027399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":225905592},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452058,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1812152,"kb_used_data":1654000,"kb_used_omap":54,"kb_used_meta":158089,"kb_avail":91933000,"statfs":{"total":95995035648,"available":94139392000,"internally_reserved":0,"allocated":1693696000,"data_stored":1688828116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":161883534},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474750,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1566472,"kb_used_data":1363200,"kb_used_omap":119,"kb_used_meta":203144,"kb_avail":92178680,"statfs":{"total":95995035648,"available":94390968320,"internally_reserved":0,"allocated":1395916800,"data_stored":1391033752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":208020190},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,2,2,0,2,4,8],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019306,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1381961728,"data_stored":1381957656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1680343040,"data_stored":1680343040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2225733632,"data_stored":2225733632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2481786880,"data_stored":2481782808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1053360128,"data_stored":1053360128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1195573248,"data_stored":1195573248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2175668224,"data_stored":2175664152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1891827712,"data_stored":1891827712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:54:08.634 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:08.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:08.736 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:08.757 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:08.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:08.846 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:54:08.847 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:54:08.859 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:08.871 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:54:08.871 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:54:08.873 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:08.890 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:08.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:08.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:08.899 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:54:08.899 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:54:08.917 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:54:08.918 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:54:08.933 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:54:08.934 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:54:08.949 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:54:08.949 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:54:08.965 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:54:08.965 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:54:08.987 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:54:08.987 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:54:08.991 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:09.013 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:09.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:09.114 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:09.135 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:09.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:09.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:08 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/14817083' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:09.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:08 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/14817083' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:09.237 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:09.258 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:09.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:08 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/14817083' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:09.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:09.382 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:09.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:09.484 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:09.506 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:09.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:09.607 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:09.629 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:09.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:09.731 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:09.752 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:09.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:09.853 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:09.875 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:09.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:09.977 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:09.998 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:09.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:09.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:10.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:54:10.071 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:09 smithi130 ceph-mon[105521]: pgmap v1810: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 17716/203466 objects degraded (8.707%); 11678/203466 objects misplaced (5.740%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:54:10.072 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:09 smithi130 ceph-mon[110068]: pgmap v1810: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 17716/203466 objects degraded (8.707%); 11678/203466 objects misplaced (5.740%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:54:10.072 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:54:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:54:09] "GET /metrics HTTP/1.1" 200 36808 "" "Prometheus/2.51.0" 2024-08-17T17:54:10.100 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:10.122 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:10.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:10.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:09 smithi179 ceph-mon[93519]: pgmap v1810: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 17716/203466 objects degraded (8.707%); 11678/203466 objects misplaced (5.740%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:54:10.224 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:10.245 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:10.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:10.347 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:10.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.370 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:10.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:10.471 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:10.495 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:10.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:54:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:54:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:54:10.597 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:10.626 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:10.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:10.728 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:10.749 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:10.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:10.850 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:10.873 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:10.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:10.881 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:10.975 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:10.996 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:10.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:10.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:11.098 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:11.124 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:11.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:11.155 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:10 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17716/203466 objects degraded (8.707%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:11.155 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:10 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17716/203466 objects degraded (8.707%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:11.227 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:11.251 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:11.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:10 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17716/203466 objects degraded (8.707%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:11.352 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:11.374 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:11.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:11.476 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:11.497 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:11.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:11.599 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:11.625 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:11.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:11.629 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:54:11.725 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:54:11.725 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:54:11.726 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:11.751 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:11.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:11.853 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:11.873 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:11.964 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:54:11.975 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:11.987 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:11 smithi130 ceph-mon[105521]: pgmap v1811: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 17716/202956 objects degraded (8.729%); 11598/202956 objects misplaced (5.715%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:54:11.987 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:11 smithi130 ceph-mon[110068]: pgmap v1811: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 17716/202956 objects degraded (8.729%); 11598/202956 objects misplaced (5.715%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:54:11.996 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:11.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:11.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:12.098 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:12.119 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:12.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:12.220 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:12.241 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:12.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:11 smithi179 ceph-mon[93519]: pgmap v1811: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 117 op/s; 17716/202956 objects degraded (8.729%); 11598/202956 objects misplaced (5.715%); 915 KiB/s, 14 objects/s recovering 2024-08-17T17:54:12.343 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:12.366 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:12.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:12.468 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:12.489 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:12.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:12.590 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:12.611 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:12.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:12.713 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:12.735 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:12.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:12.821 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:54:12.837 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:12.857 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:12.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:12.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:12.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:12.959 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:12.984 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:12.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:12.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:13.086 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:13.111 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:13.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:13.213 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:13.248 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:13.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:13.250 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:13.302 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:13.350 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:13.371 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:13.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:13.473 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:13.500 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:13.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:13.602 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:13.623 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:13.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:13.725 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:13.746 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:13.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:13.848 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:13.869 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:13.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:13.971 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:13.996 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:13.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:13.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:14.098 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:14.127 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:13 smithi130 ceph-mon[105521]: pgmap v1812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 17602/202677 objects degraded (8.685%); 11562/202677 objects misplaced (5.705%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:14.127 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:13 smithi130 ceph-mon[110068]: pgmap v1812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 17602/202677 objects degraded (8.685%); 11562/202677 objects misplaced (5.705%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:14.128 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:14.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:14.230 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:14.251 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:14.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:14.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:13 smithi179 ceph-mon[93519]: pgmap v1812: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 17602/202677 objects degraded (8.685%); 11562/202677 objects misplaced (5.705%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:14.353 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:14.377 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:14.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:14.476 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:14.479 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:14.505 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:14.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:14.606 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:14.627 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:14.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:14.729 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:14.753 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:14.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:14.855 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:14.881 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:14.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:14.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:14.901 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T17:54:14.983 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:15.005 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:15.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:15.106 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:15.127 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:15.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:15.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:14 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/731364018' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:15.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:14 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/731364018' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:15.229 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:15.244 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:14 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/731364018' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:15.250 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:15.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:15.352 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:15.374 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:15.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:15.476 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:15.498 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:15.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:15.600 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:15.621 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:15.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:15.723 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:15.747 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:15.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:15.849 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:15.870 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:15.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:15.971 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:15.993 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:15.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:15.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:16.095 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:16.116 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:16.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:16.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:15 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:16.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:15 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/731364018' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:54:16.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:15 smithi130 ceph-mon[105521]: osdmap e1224: 8 total, 8 up, 6 in 2024-08-17T17:54:16.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:15 smithi130 ceph-mon[105521]: pgmap v1814: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 17602/201090 objects degraded (8.753%); 11482/201090 objects misplaced (5.710%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:54:16.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:15 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17602/202677 objects degraded (8.685%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:16.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:15 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:16.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:15 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/731364018' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:54:16.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:15 smithi130 ceph-mon[110068]: osdmap e1224: 8 total, 8 up, 6 in 2024-08-17T17:54:16.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:15 smithi130 ceph-mon[110068]: pgmap v1814: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 17602/201090 objects degraded (8.753%); 11482/201090 objects misplaced (5.710%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:54:16.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:15 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17602/202677 objects degraded (8.685%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:16.218 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:16.246 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:16.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:15 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:15 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/731364018' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:54:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:15 smithi179 ceph-mon[93519]: osdmap e1224: 8 total, 8 up, 6 in 2024-08-17T17:54:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:15 smithi179 ceph-mon[93519]: pgmap v1814: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 17602/201090 objects degraded (8.753%); 11482/201090 objects misplaced (5.710%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:54:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:15 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17602/202677 objects degraded (8.685%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:16.347 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:16.370 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:16.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:16.471 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:16.494 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:16.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:16.596 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:16.618 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:16.719 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:16.740 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:16.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:16.842 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:16.864 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:16.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:16.966 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:16.987 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:16.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:16.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:17.089 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:17.114 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:17.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:17.216 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:17.239 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:17.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:17.341 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:17.345 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:17 smithi130 ceph-mon[105521]: pgmap v1815: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 17538/200889 objects degraded (8.730%); 11447/200889 objects misplaced (5.698%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:54:17.345 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:17 smithi130 ceph-mon[110068]: pgmap v1815: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 17538/200889 objects degraded (8.730%); 11447/200889 objects misplaced (5.698%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:54:17.362 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:17.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:17.465 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:17.487 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:17.589 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:17 smithi179 ceph-mon[93519]: pgmap v1815: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 127 op/s; 17538/200889 objects degraded (8.730%); 11447/200889 objects misplaced (5.698%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:54:17.614 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:17.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:17.715 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:17.736 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:17.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:17.810 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:54:17.811 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:54:17.838 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:17.859 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:17.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:17.960 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:17.983 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:17.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:17.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:18.085 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:18.106 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:18.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:18.208 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:18.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.229 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:18.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:18.330 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:18.351 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:18.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:18.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:18 smithi130 ceph-mon[105521]: from='client.35739 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:54:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:18 smithi130 ceph-mon[110068]: from='client.35739 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:54:18.454 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:18.477 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:18.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:18.579 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:18 smithi179 ceph-mon[93519]: from='client.35739 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:54:18.600 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:18.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:18.702 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:18.724 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:18.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:18.826 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:18.847 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:18.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:18.949 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:18.970 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:18.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:18.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:19.072 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:19.099 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:19.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:19.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:19.226 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:19.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:19.328 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:19.354 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:19.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:19.375 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:54:19.376 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:19 smithi130 ceph-mon[110068]: pgmap v1816: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 17538/199899 objects degraded (8.773%); 11447/199899 objects misplaced (5.726%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:54:19.376 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:19 smithi130 ceph-mon[105521]: pgmap v1816: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 17538/199899 objects degraded (8.773%); 11447/199899 objects misplaced (5.726%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:54:19.456 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:19.479 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:19.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:19.580 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:19 smithi179 ceph-mon[93519]: pgmap v1816: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 17538/199899 objects degraded (8.773%); 11447/199899 objects misplaced (5.726%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:54:19.605 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:19.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:19.707 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:19.729 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:19.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:19.830 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:19.852 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:19.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:19.954 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:19.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:19.976 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:19.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:20.077 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:20.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.100 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:20.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:20.125 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:54:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:54:19] "GET /metrics HTTP/1.1" 200 36810 "" "Prometheus/2.51.0" 2024-08-17T17:54:20.155 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1815,"stamp":"2024-08-17T17:54:17.051586+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4387313637,"num_objects":66963,"num_object_clones":0,"num_object_copies":200889,"num_objects_missing_on_primary":3674,"num_objects_missing":3674,"num_objects_degraded":17538,"num_objects_misplaced":11447,"num_objects_unfound":0,"num_objects_dirty":66963,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":142951,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34770,"num_bytes_recovered":2100108301,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108054,"ondisk_log_size":108054,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12835220,"kb_used_data":11433636,"kb_used_omap":295,"kb_used_meta":1401240,"kb_avail":549635692,"statfs":{"total":575970213888,"available":562826948608,"internally_reserved":0,"allocated":11708043264,"data_stored":11678715917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1434870223},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-83558400,"num_objects":-1275,"num_object_clones":0,"num_object_copies":-3825,"num_objects_missing_on_primary":-79,"num_objects_missing":-79,"num_objects_degraded":-178,"num_objects_misplaced":-231,"num_objects_unfound":0,"num_objects_dirty":-1275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1275,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":170,"num_bytes_recovered":11141120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002606"},"pg_stats":[{"pgid":"45.f","version":"1223'8922","reported_seq":19447,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.081612+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:54:14.081612+0000","last_peered":"2024-08-17T17:54:14.081612+0000","last_clean":"2024-08-17T17:54:14.081612+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:54:14.081612+0000","last_undegraded":"2024-08-17T17:54:14.081612+0000","last_fullsized":"2024-08-17T17:54:14.081612+0000","mapping_epoch":1194,"log_start":"1170'2031","ondisk_log_start":"1170'2031","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6891,"log_dups_size":0,"ondisk_log_size":6891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278003712,"num_objects":4242,"num_object_clones":0,"num_object_copies":12726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8922,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1223'8851","reported_seq":19314,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:13.097931+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:54:13.097931+0000","last_peered":"2024-08-17T17:54:13.097931+0000","last_clean":"2024-08-17T17:54:13.097931+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:54:13.097931+0000","last_undegraded":"2024-08-17T17:54:13.097931+0000","last_fullsized":"2024-08-17T17:54:13.097931+0000","mapping_epoch":1211,"log_start":"1170'1832","ondisk_log_start":"1170'1832","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7019,"log_dups_size":0,"ondisk_log_size":7019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271515648,"num_objects":4143,"num_object_clones":0,"num_object_copies":12429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8851,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1223'9150","reported_seq":19982,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:13.487316+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:54:13.487316+0000","last_peered":"2024-08-17T17:54:13.487316+0000","last_clean":"2024-08-17T17:54:13.487316+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:54:13.487316+0000","last_undegraded":"2024-08-17T17:54:13.487316+0000","last_fullsized":"2024-08-17T17:54:13.487316+0000","mapping_epoch":1205,"log_start":"1174'3168","ondisk_log_start":"1174'3168","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5982,"log_dups_size":0,"ondisk_log_size":5982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277020696,"num_objects":4228,"num_object_clones":0,"num_object_copies":12684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4228,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9150,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1223'8867","reported_seq":8931,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:13.157517+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:54:13.157517+0000","last_peered":"2024-08-17T17:54:13.157517+0000","last_clean":"2024-08-17T17:54:13.157517+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:54:13.157517+0000","last_undegraded":"2024-08-17T17:54:13.157517+0000","last_fullsized":"2024-08-17T17:54:13.157517+0000","mapping_epoch":1167,"log_start":"1170'2189","ondisk_log_start":"1170'2189","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6678,"log_dups_size":0,"ondisk_log_size":6678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270860288,"num_objects":4133,"num_object_clones":0,"num_object_copies":12399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8867,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1224'8929","reported_seq":14169,"reported_epoch":1224,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:16.466527+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:54:16.466527+0000","last_peered":"2024-08-17T17:54:16.466527+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:54:16.466527+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1171'3026","ondisk_log_start":"1171'3026","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5903,"log_dups_size":0,"ondisk_log_size":5903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274268160,"num_objects":4185,"num_object_clones":0,"num_object_copies":12555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":4185,"num_objects_unfound":0,"num_objects_dirty":4185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8929,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":3667},{"shards":"1,4,5","objects":518}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1223'8999","reported_seq":9063,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:13.169289+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:54:13.169289+0000","last_peered":"2024-08-17T17:54:13.169289+0000","last_clean":"2024-08-17T17:54:13.169289+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:54:13.169289+0000","last_undegraded":"2024-08-17T17:54:13.169289+0000","last_fullsized":"2024-08-17T17:54:13.169289+0000","mapping_epoch":1167,"log_start":"1170'2277","ondisk_log_start":"1170'2277","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6722,"log_dups_size":0,"ondisk_log_size":6722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268369920,"num_objects":4095,"num_object_clones":0,"num_object_copies":12285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8999,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1223'8971","reported_seq":9035,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:12.932594+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:54:12.932594+0000","last_peered":"2024-08-17T17:54:12.932594+0000","last_clean":"2024-08-17T17:54:12.932594+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:54:12.932594+0000","last_undegraded":"2024-08-17T17:54:12.932594+0000","last_fullsized":"2024-08-17T17:54:12.932594+0000","mapping_epoch":1167,"log_start":"1170'2269","ondisk_log_start":"1170'2269","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6702,"log_dups_size":0,"ondisk_log_size":6702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278855680,"num_objects":4255,"num_object_clones":0,"num_object_copies":12765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8971,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1224'8885","reported_seq":16189,"reported_epoch":1224,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:16.236530+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:54:16.236530+0000","last_peered":"2024-08-17T17:54:16.236530+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:54:16.236530+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1171'2975","ondisk_log_start":"1171'2975","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5910,"log_dups_size":0,"ondisk_log_size":5910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269025280,"num_objects":4105,"num_object_clones":0,"num_object_copies":12315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3503,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8885,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2390,"num_bytes_recovered":156631040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":718},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1223'8857","reported_seq":8931,"reported_epoch":1223,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:13.159507+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:54:13.159507+0000","last_peered":"2024-08-17T17:54:13.159507+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:54:13.159507+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'2177","ondisk_log_start":"1170'2177","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6680,"log_dups_size":0,"ondisk_log_size":6680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276496384,"num_objects":4219,"num_object_clones":0,"num_object_copies":12657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8857,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":4219},{"shards":"1,6","objects":-1614}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1223'8914","reported_seq":11986,"reported_epoch":1223,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:13.219598+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:54:13.219598+0000","last_peered":"2024-08-17T17:54:13.219598+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:54:13.219598+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1171'3006","ondisk_log_start":"1171'3006","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5908,"log_dups_size":0,"ondisk_log_size":5908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279183360,"num_objects":4260,"num_object_clones":0,"num_object_copies":12780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8914,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":2699},{"shards":"1,3,5","objects":701}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1223'8874","reported_seq":9001,"reported_epoch":1223,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:14.054244+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:54:14.054244+0000","last_peered":"2024-08-17T17:54:14.054244+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:54:14.054244+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8874,"log_dups_size":0,"ondisk_log_size":8874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275251200,"num_objects":4200,"num_object_clones":0,"num_object_copies":12600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8874,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-953}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1223'8856","reported_seq":15044,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:13.970138+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:54:13.970138+0000","last_peered":"2024-08-17T17:54:13.970138+0000","last_clean":"2024-08-17T17:54:13.970138+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:54:13.970138+0000","last_undegraded":"2024-08-17T17:54:13.970138+0000","last_fullsized":"2024-08-17T17:54:13.970138+0000","mapping_epoch":1183,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":8856,"log_dups_size":0,"ondisk_log_size":8856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274595840,"num_objects":4190,"num_object_clones":0,"num_object_copies":12570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8856,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1223'8981","reported_seq":9107,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:13.532073+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:54:13.532073+0000","last_peered":"2024-08-17T17:54:13.532073+0000","last_clean":"2024-08-17T17:54:13.532073+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:54:13.532073+0000","last_undegraded":"2024-08-17T17:54:13.532073+0000","last_fullsized":"2024-08-17T17:54:13.532073+0000","mapping_epoch":1167,"log_start":"1174'3091","ondisk_log_start":"1174'3091","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5890,"log_dups_size":0,"ondisk_log_size":5890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273874944,"num_objects":4179,"num_object_clones":0,"num_object_copies":12537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8981,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1223'8833","reported_seq":8897,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:13.168973+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:54:13.168973+0000","last_peered":"2024-08-17T17:54:13.168973+0000","last_clean":"2024-08-17T17:54:13.168973+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:54:13.168973+0000","last_undegraded":"2024-08-17T17:54:13.168973+0000","last_fullsized":"2024-08-17T17:54:13.168973+0000","mapping_epoch":1167,"log_start":"1170'1825","ondisk_log_start":"1170'1825","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7008,"log_dups_size":0,"ondisk_log_size":7008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268763136,"num_objects":4101,"num_object_clones":0,"num_object_copies":12303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8833,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1223'9049","reported_seq":19776,"reported_epoch":1223,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:14.283379+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:54:14.283379+0000","last_peered":"2024-08-17T17:54:14.283379+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:54:14.283379+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'2191","ondisk_log_start":"1170'2191","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6858,"log_dups_size":0,"ondisk_log_size":6858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282787840,"num_objects":4315,"num_object_clones":0,"num_object_copies":12945,"num_objects_missing_on_primary":3674,"num_objects_missing":3674,"num_objects_degraded":265,"num_objects_misplaced":7262,"num_objects_unfound":0,"num_objects_dirty":4315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9049,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4473,"num_bytes_recovered":293142528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":12},{"shards":"0,4,6","objects":1356},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":2682}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1223'8735","reported_seq":22047,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:13.509342+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:54:13.509342+0000","last_peered":"2024-08-17T17:54:13.509342+0000","last_clean":"2024-08-17T17:54:13.509342+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:54:13.509342+0000","last_undegraded":"2024-08-17T17:54:13.509342+0000","last_fullsized":"2024-08-17T17:54:13.509342+0000","mapping_epoch":1196,"log_start":"1171'2769","ondisk_log_start":"1171'2769","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5966,"log_dups_size":0,"ondisk_log_size":5966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267845632,"num_objects":4087,"num_object_clones":0,"num_object_copies":12261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8735,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1902,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.816359+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:54:04.816359+0000","last_peered":"2024-08-17T17:54:04.816359+0000","last_clean":"2024-08-17T17:54:04.816359+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:54:04.816359+0000","last_undegraded":"2024-08-17T17:54:04.816359+0000","last_fullsized":"2024-08-17T17:54:04.816359+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":500,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.247558+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:53:58.247558+0000","last_peered":"2024-08-17T17:53:58.247558+0000","last_clean":"2024-08-17T17:53:58.247558+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:53:58.247558+0000","last_undegraded":"2024-08-17T17:53:58.247558+0000","last_fullsized":"2024-08-17T17:53:58.247558+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1822,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.809253+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:54:04.809253+0000","last_peered":"2024-08-17T17:54:04.809253+0000","last_clean":"2024-08-17T17:54:04.809253+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:54:04.809253+0000","last_undegraded":"2024-08-17T17:54:04.809253+0000","last_fullsized":"2024-08-17T17:54:04.809253+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1846,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808143+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:54:04.808143+0000","last_peered":"2024-08-17T17:54:04.808143+0000","last_clean":"2024-08-17T17:54:04.808143+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:54:04.808143+0000","last_undegraded":"2024-08-17T17:54:04.808143+0000","last_fullsized":"2024-08-17T17:54:04.808143+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":62,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.248170+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:53:58.248170+0000","last_peered":"2024-08-17T17:53:58.248170+0000","last_clean":"2024-08-17T17:53:58.248170+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:53:58.248170+0000","last_undegraded":"2024-08-17T17:53:58.248170+0000","last_fullsized":"2024-08-17T17:53:58.248170+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1239,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808396+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:54:04.808396+0000","last_peered":"2024-08-17T17:54:04.808396+0000","last_clean":"2024-08-17T17:54:04.808396+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:54:04.808396+0000","last_undegraded":"2024-08-17T17:54:04.808396+0000","last_fullsized":"2024-08-17T17:54:04.808396+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":99,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.905955+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:54:14.905955+0000","last_peered":"2024-08-17T17:54:14.905955+0000","last_clean":"2024-08-17T17:54:14.905955+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:54:14.905955+0000","last_undegraded":"2024-08-17T17:54:14.905955+0000","last_fullsized":"2024-08-17T17:54:14.905955+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":617,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808423+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:54:04.808423+0000","last_peered":"2024-08-17T17:54:04.808423+0000","last_clean":"2024-08-17T17:54:04.808423+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:54:04.808423+0000","last_undegraded":"2024-08-17T17:54:04.808423+0000","last_fullsized":"2024-08-17T17:54:04.808423+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1861,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.816015+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:54:04.816015+0000","last_peered":"2024-08-17T17:54:04.816015+0000","last_clean":"2024-08-17T17:54:04.816015+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:54:04.816015+0000","last_undegraded":"2024-08-17T17:54:04.816015+0000","last_fullsized":"2024-08-17T17:54:04.816015+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1863,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.868512+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:54:04.868512+0000","last_peered":"2024-08-17T17:54:04.868512+0000","last_clean":"2024-08-17T17:54:04.868512+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:54:04.868512+0000","last_undegraded":"2024-08-17T17:54:04.868512+0000","last_fullsized":"2024-08-17T17:54:04.868512+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":84,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.905288+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:54:14.905288+0000","last_peered":"2024-08-17T17:54:14.905288+0000","last_clean":"2024-08-17T17:54:14.905288+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:54:14.905288+0000","last_undegraded":"2024-08-17T17:54:14.905288+0000","last_fullsized":"2024-08-17T17:54:14.905288+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1834,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.809148+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:54:04.809148+0000","last_peered":"2024-08-17T17:54:04.809148+0000","last_clean":"2024-08-17T17:54:04.809148+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:54:04.809148+0000","last_undegraded":"2024-08-17T17:54:04.809148+0000","last_fullsized":"2024-08-17T17:54:04.809148+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1891,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.904953+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:54:14.904953+0000","last_peered":"2024-08-17T17:54:14.904953+0000","last_clean":"2024-08-17T17:54:14.904953+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:54:14.904953+0000","last_undegraded":"2024-08-17T17:54:14.904953+0000","last_fullsized":"2024-08-17T17:54:14.904953+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1883,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.816394+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:54:04.816394+0000","last_peered":"2024-08-17T17:54:04.816394+0000","last_clean":"2024-08-17T17:54:04.816394+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:54:04.816394+0000","last_undegraded":"2024-08-17T17:54:04.816394+0000","last_fullsized":"2024-08-17T17:54:04.816394+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1857,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808436+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:54:04.808436+0000","last_peered":"2024-08-17T17:54:04.808436+0000","last_clean":"2024-08-17T17:54:04.808436+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:54:04.808436+0000","last_undegraded":"2024-08-17T17:54:04.808436+0000","last_fullsized":"2024-08-17T17:54:04.808436+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1197,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.248123+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:53:58.248123+0000","last_peered":"2024-08-17T17:53:58.248123+0000","last_clean":"2024-08-17T17:53:58.248123+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:53:58.248123+0000","last_undegraded":"2024-08-17T17:53:58.248123+0000","last_fullsized":"2024-08-17T17:53:58.248123+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1828,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808150+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:54:04.808150+0000","last_peered":"2024-08-17T17:54:04.808150+0000","last_clean":"2024-08-17T17:54:04.808150+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:54:04.808150+0000","last_undegraded":"2024-08-17T17:54:04.808150+0000","last_fullsized":"2024-08-17T17:54:04.808150+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1878,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.816436+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:54:04.816436+0000","last_peered":"2024-08-17T17:54:04.816436+0000","last_clean":"2024-08-17T17:54:04.816436+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:54:04.816436+0000","last_undegraded":"2024-08-17T17:54:04.816436+0000","last_fullsized":"2024-08-17T17:54:04.816436+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1784,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808085+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:54:04.808085+0000","last_peered":"2024-08-17T17:54:04.808085+0000","last_clean":"2024-08-17T17:54:04.808085+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:54:04.808085+0000","last_undegraded":"2024-08-17T17:54:04.808085+0000","last_fullsized":"2024-08-17T17:54:04.808085+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":516,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.139152+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:53:58.139152+0000","last_peered":"2024-08-17T17:53:58.139152+0000","last_clean":"2024-08-17T17:53:58.139152+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:53:58.139152+0000","last_undegraded":"2024-08-17T17:53:58.139152+0000","last_fullsized":"2024-08-17T17:53:58.139152+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1193,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.139023+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:53:58.139023+0000","last_peered":"2024-08-17T17:53:58.139023+0000","last_clean":"2024-08-17T17:53:58.139023+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:53:58.139023+0000","last_undegraded":"2024-08-17T17:53:58.139023+0000","last_fullsized":"2024-08-17T17:53:58.139023+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1885,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.905727+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:54:14.905727+0000","last_peered":"2024-08-17T17:54:14.905727+0000","last_clean":"2024-08-17T17:54:14.905727+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:54:14.905727+0000","last_undegraded":"2024-08-17T17:54:14.905727+0000","last_fullsized":"2024-08-17T17:54:14.905727+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":354,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.139122+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:53:58.139122+0000","last_peered":"2024-08-17T17:53:58.139122+0000","last_clean":"2024-08-17T17:53:58.139122+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:53:58.139122+0000","last_undegraded":"2024-08-17T17:53:58.139122+0000","last_fullsized":"2024-08-17T17:53:58.139122+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":500,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.247854+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:53:58.247854+0000","last_peered":"2024-08-17T17:53:58.247854+0000","last_clean":"2024-08-17T17:53:58.247854+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:53:58.247854+0000","last_undegraded":"2024-08-17T17:53:58.247854+0000","last_fullsized":"2024-08-17T17:53:58.247854+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1178,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.247803+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:53:58.247803+0000","last_peered":"2024-08-17T17:53:58.247803+0000","last_clean":"2024-08-17T17:53:58.247803+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:53:58.247803+0000","last_undegraded":"2024-08-17T17:53:58.247803+0000","last_fullsized":"2024-08-17T17:53:58.247803+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1186,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.248197+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:53:58.248197+0000","last_peered":"2024-08-17T17:53:58.248197+0000","last_clean":"2024-08-17T17:53:58.248197+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:53:58.248197+0000","last_undegraded":"2024-08-17T17:53:58.248197+0000","last_fullsized":"2024-08-17T17:53:58.248197+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1930,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.816270+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:54:04.816270+0000","last_peered":"2024-08-17T17:54:04.816270+0000","last_clean":"2024-08-17T17:54:04.816270+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:54:04.816270+0000","last_undegraded":"2024-08-17T17:54:04.816270+0000","last_fullsized":"2024-08-17T17:54:04.816270+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1814,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808357+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:54:04.808357+0000","last_peered":"2024-08-17T17:54:04.808357+0000","last_clean":"2024-08-17T17:54:04.808357+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:54:04.808357+0000","last_undegraded":"2024-08-17T17:54:04.808357+0000","last_fullsized":"2024-08-17T17:54:04.808357+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1890,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.905600+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:54:14.905600+0000","last_peered":"2024-08-17T17:54:14.905600+0000","last_clean":"2024-08-17T17:54:14.905600+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:54:14.905600+0000","last_undegraded":"2024-08-17T17:54:14.905600+0000","last_fullsized":"2024-08-17T17:54:14.905600+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1816,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808569+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:54:04.808569+0000","last_peered":"2024-08-17T17:54:04.808569+0000","last_clean":"2024-08-17T17:54:04.808569+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:54:04.808569+0000","last_undegraded":"2024-08-17T17:54:04.808569+0000","last_fullsized":"2024-08-17T17:54:04.808569+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1848,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.816069+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:54:04.816069+0000","last_peered":"2024-08-17T17:54:04.816069+0000","last_clean":"2024-08-17T17:54:04.816069+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:54:04.816069+0000","last_undegraded":"2024-08-17T17:54:04.816069+0000","last_fullsized":"2024-08-17T17:54:04.816069+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1213,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.139143+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:53:58.139143+0000","last_peered":"2024-08-17T17:53:58.139143+0000","last_clean":"2024-08-17T17:53:58.139143+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:53:58.139143+0000","last_undegraded":"2024-08-17T17:53:58.139143+0000","last_fullsized":"2024-08-17T17:53:58.139143+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1897,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.906130+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:54:14.906130+0000","last_peered":"2024-08-17T17:54:14.906130+0000","last_clean":"2024-08-17T17:54:14.906130+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:54:14.906130+0000","last_undegraded":"2024-08-17T17:54:14.906130+0000","last_fullsized":"2024-08-17T17:54:14.906130+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":84,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.905131+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:54:14.905131+0000","last_peered":"2024-08-17T17:54:14.905131+0000","last_clean":"2024-08-17T17:54:14.905131+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:54:14.905131+0000","last_undegraded":"2024-08-17T17:54:14.905131+0000","last_fullsized":"2024-08-17T17:54:14.905131+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1916,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.906048+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:54:14.906048+0000","last_peered":"2024-08-17T17:54:14.906048+0000","last_clean":"2024-08-17T17:54:14.906048+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:54:14.906048+0000","last_undegraded":"2024-08-17T17:54:14.906048+0000","last_fullsized":"2024-08-17T17:54:14.906048+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":650,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808935+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:54:04.808935+0000","last_peered":"2024-08-17T17:54:04.808935+0000","last_clean":"2024-08-17T17:54:04.808935+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:54:04.808935+0000","last_undegraded":"2024-08-17T17:54:04.808935+0000","last_fullsized":"2024-08-17T17:54:04.808935+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1897,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.906214+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:54:14.906214+0000","last_peered":"2024-08-17T17:54:14.906214+0000","last_clean":"2024-08-17T17:54:14.906214+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:54:14.906214+0000","last_undegraded":"2024-08-17T17:54:14.906214+0000","last_fullsized":"2024-08-17T17:54:14.906214+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1224,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.815951+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:54:04.815951+0000","last_peered":"2024-08-17T17:54:04.815951+0000","last_clean":"2024-08-17T17:54:04.815951+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:54:04.815951+0000","last_undegraded":"2024-08-17T17:54:04.815951+0000","last_fullsized":"2024-08-17T17:54:04.815951+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":143,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.905636+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:54:14.905636+0000","last_peered":"2024-08-17T17:54:14.905636+0000","last_clean":"2024-08-17T17:54:14.905636+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:54:14.905636+0000","last_undegraded":"2024-08-17T17:54:14.905636+0000","last_fullsized":"2024-08-17T17:54:14.905636+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1907,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.905606+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:54:14.905606+0000","last_peered":"2024-08-17T17:54:14.905606+0000","last_clean":"2024-08-17T17:54:14.905606+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:54:14.905606+0000","last_undegraded":"2024-08-17T17:54:14.905606+0000","last_fullsized":"2024-08-17T17:54:14.905606+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1885,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.904894+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:54:14.904894+0000","last_peered":"2024-08-17T17:54:14.904894+0000","last_clean":"2024-08-17T17:54:14.904894+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:54:14.904894+0000","last_undegraded":"2024-08-17T17:54:14.904894+0000","last_fullsized":"2024-08-17T17:54:14.904894+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1904,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.868142+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:54:04.868142+0000","last_peered":"2024-08-17T17:54:04.868142+0000","last_clean":"2024-08-17T17:54:04.868142+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:54:04.868142+0000","last_undegraded":"2024-08-17T17:54:04.868142+0000","last_fullsized":"2024-08-17T17:54:04.868142+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1909,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.816448+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:54:04.816448+0000","last_peered":"2024-08-17T17:54:04.816448+0000","last_clean":"2024-08-17T17:54:04.816448+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:54:04.816448+0000","last_undegraded":"2024-08-17T17:54:04.816448+0000","last_fullsized":"2024-08-17T17:54:04.816448+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1889,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.824983+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:54:04.824983+0000","last_peered":"2024-08-17T17:54:04.824983+0000","last_clean":"2024-08-17T17:54:04.824983+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:54:04.824983+0000","last_undegraded":"2024-08-17T17:54:04.824983+0000","last_fullsized":"2024-08-17T17:54:04.824983+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1220,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.139027+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:53:58.139027+0000","last_peered":"2024-08-17T17:53:58.139027+0000","last_clean":"2024-08-17T17:53:58.139027+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:53:58.139027+0000","last_undegraded":"2024-08-17T17:53:58.139027+0000","last_fullsized":"2024-08-17T17:53:58.139027+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1166,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.248151+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:53:58.248151+0000","last_peered":"2024-08-17T17:53:58.248151+0000","last_clean":"2024-08-17T17:53:58.248151+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:53:58.248151+0000","last_undegraded":"2024-08-17T17:53:58.248151+0000","last_fullsized":"2024-08-17T17:53:58.248151+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1798,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808292+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:54:04.808292+0000","last_peered":"2024-08-17T17:54:04.808292+0000","last_clean":"2024-08-17T17:54:04.808292+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:54:04.808292+0000","last_undegraded":"2024-08-17T17:54:04.808292+0000","last_fullsized":"2024-08-17T17:54:04.808292+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":129,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.868430+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:54:04.868430+0000","last_peered":"2024-08-17T17:54:04.868430+0000","last_clean":"2024-08-17T17:54:04.868430+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:54:04.868430+0000","last_undegraded":"2024-08-17T17:54:04.868430+0000","last_fullsized":"2024-08-17T17:54:04.868430+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1532,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.138846+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:53:58.138846+0000","last_peered":"2024-08-17T17:53:58.138846+0000","last_clean":"2024-08-17T17:53:58.138846+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:53:58.138846+0000","last_undegraded":"2024-08-17T17:53:58.138846+0000","last_fullsized":"2024-08-17T17:53:58.138846+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1892,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.906389+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:54:14.906389+0000","last_peered":"2024-08-17T17:54:14.906389+0000","last_clean":"2024-08-17T17:54:14.906389+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:54:14.906389+0000","last_undegraded":"2024-08-17T17:54:14.906389+0000","last_fullsized":"2024-08-17T17:54:14.906389+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1830,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808241+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:54:04.808241+0000","last_peered":"2024-08-17T17:54:04.808241+0000","last_clean":"2024-08-17T17:54:04.808241+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:54:04.808241+0000","last_undegraded":"2024-08-17T17:54:04.808241+0000","last_fullsized":"2024-08-17T17:54:04.808241+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":500,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.250468+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:53:58.250468+0000","last_peered":"2024-08-17T17:53:58.250468+0000","last_clean":"2024-08-17T17:53:58.250468+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:53:58.250468+0000","last_undegraded":"2024-08-17T17:53:58.250468+0000","last_fullsized":"2024-08-17T17:53:58.250468+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":569,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.868029+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:54:04.868029+0000","last_peered":"2024-08-17T17:54:04.868029+0000","last_clean":"2024-08-17T17:54:04.868029+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:54:04.868029+0000","last_undegraded":"2024-08-17T17:54:04.868029+0000","last_fullsized":"2024-08-17T17:54:04.868029+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":584,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.868651+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:54:04.868651+0000","last_peered":"2024-08-17T17:54:04.868651+0000","last_clean":"2024-08-17T17:54:04.868651+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:54:04.868651+0000","last_undegraded":"2024-08-17T17:54:04.868651+0000","last_fullsized":"2024-08-17T17:54:04.868651+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1184,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.247975+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:53:58.247975+0000","last_peered":"2024-08-17T17:53:58.247975+0000","last_clean":"2024-08-17T17:53:58.247975+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:53:58.247975+0000","last_undegraded":"2024-08-17T17:53:58.247975+0000","last_fullsized":"2024-08-17T17:53:58.247975+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":600,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808490+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:54:04.808490+0000","last_peered":"2024-08-17T17:54:04.808490+0000","last_clean":"2024-08-17T17:54:04.808490+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:54:04.808490+0000","last_undegraded":"2024-08-17T17:54:04.808490+0000","last_fullsized":"2024-08-17T17:54:04.808490+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1826,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808521+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:54:04.808521+0000","last_peered":"2024-08-17T17:54:04.808521+0000","last_clean":"2024-08-17T17:54:04.808521+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:54:04.808521+0000","last_undegraded":"2024-08-17T17:54:04.808521+0000","last_fullsized":"2024-08-17T17:54:04.808521+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1922,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.816112+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:54:04.816112+0000","last_peered":"2024-08-17T17:54:04.816112+0000","last_clean":"2024-08-17T17:54:04.816112+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:54:04.816112+0000","last_undegraded":"2024-08-17T17:54:04.816112+0000","last_fullsized":"2024-08-17T17:54:04.816112+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":633,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808442+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:54:04.808442+0000","last_peered":"2024-08-17T17:54:04.808442+0000","last_clean":"2024-08-17T17:54:04.808442+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:54:04.808442+0000","last_undegraded":"2024-08-17T17:54:04.808442+0000","last_fullsized":"2024-08-17T17:54:04.808442+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1173,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.247750+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:53:58.247750+0000","last_peered":"2024-08-17T17:53:58.247750+0000","last_clean":"2024-08-17T17:53:58.247750+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:53:58.247750+0000","last_undegraded":"2024-08-17T17:53:58.247750+0000","last_fullsized":"2024-08-17T17:53:58.247750+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1172,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.139206+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:53:58.139206+0000","last_peered":"2024-08-17T17:53:58.139206+0000","last_clean":"2024-08-17T17:53:58.139206+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:53:58.139206+0000","last_undegraded":"2024-08-17T17:53:58.139206+0000","last_fullsized":"2024-08-17T17:53:58.139206+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":620,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:14.905320+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:54:14.905320+0000","last_peered":"2024-08-17T17:54:14.905320+0000","last_clean":"2024-08-17T17:54:14.905320+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:54:14.905320+0000","last_undegraded":"2024-08-17T17:54:14.905320+0000","last_fullsized":"2024-08-17T17:54:14.905320+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1190,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.139251+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:53:58.139251+0000","last_peered":"2024-08-17T17:53:58.139251+0000","last_clean":"2024-08-17T17:53:58.139251+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:53:58.139251+0000","last_undegraded":"2024-08-17T17:53:58.139251+0000","last_fullsized":"2024-08-17T17:53:58.139251+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":74,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.138759+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:53:58.138759+0000","last_peered":"2024-08-17T17:53:58.138759+0000","last_clean":"2024-08-17T17:53:58.138759+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:53:58.138759+0000","last_undegraded":"2024-08-17T17:53:58.138759+0000","last_fullsized":"2024-08-17T17:53:58.138759+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1206,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.139218+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:53:58.139218+0000","last_peered":"2024-08-17T17:53:58.139218+0000","last_clean":"2024-08-17T17:53:58.139218+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:53:58.139218+0000","last_undegraded":"2024-08-17T17:53:58.139218+0000","last_fullsized":"2024-08-17T17:53:58.139218+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1223,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.247695+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:53:58.247695+0000","last_peered":"2024-08-17T17:53:58.247695+0000","last_clean":"2024-08-17T17:53:58.247695+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:53:58.247695+0000","last_undegraded":"2024-08-17T17:53:58.247695+0000","last_fullsized":"2024-08-17T17:53:58.247695+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1828,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808460+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:54:04.808460+0000","last_peered":"2024-08-17T17:54:04.808460+0000","last_clean":"2024-08-17T17:54:04.808460+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:54:04.808460+0000","last_undegraded":"2024-08-17T17:54:04.808460+0000","last_fullsized":"2024-08-17T17:54:04.808460+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1167,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.139053+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:53:58.139053+0000","last_peered":"2024-08-17T17:53:58.139053+0000","last_clean":"2024-08-17T17:53:58.139053+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:53:58.139053+0000","last_undegraded":"2024-08-17T17:53:58.139053+0000","last_fullsized":"2024-08-17T17:53:58.139053+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1881,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.816308+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:54:04.816308+0000","last_peered":"2024-08-17T17:54:04.816308+0000","last_clean":"2024-08-17T17:54:04.816308+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:54:04.816308+0000","last_undegraded":"2024-08-17T17:54:04.816308+0000","last_fullsized":"2024-08-17T17:54:04.816308+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1882,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.815892+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:54:04.815892+0000","last_peered":"2024-08-17T17:54:04.815892+0000","last_clean":"2024-08-17T17:54:04.815892+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:54:04.815892+0000","last_undegraded":"2024-08-17T17:54:04.815892+0000","last_fullsized":"2024-08-17T17:54:04.815892+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":580,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.138971+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:53:58.138971+0000","last_peered":"2024-08-17T17:53:58.138971+0000","last_clean":"2024-08-17T17:53:58.138971+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:53:58.138971+0000","last_undegraded":"2024-08-17T17:53:58.138971+0000","last_fullsized":"2024-08-17T17:53:58.138971+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1192,"reported_epoch":1222,"state":"active+clean","last_fresh":"2024-08-17T17:53:58.139350+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:53:58.139350+0000","last_peered":"2024-08-17T17:53:58.139350+0000","last_clean":"2024-08-17T17:53:58.139350+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:53:58.139350+0000","last_undegraded":"2024-08-17T17:53:58.139350+0000","last_fullsized":"2024-08-17T17:53:58.139350+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1828,"reported_epoch":1223,"state":"active+clean","last_fresh":"2024-08-17T17:54:04.808637+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:54:04.808637+0000","last_peered":"2024-08-17T17:54:04.808637+0000","last_clean":"2024-08-17T17:54:04.808637+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:54:04.808637+0000","last_undegraded":"2024-08-17T17:54:04.808637+0000","last_fullsized":"2024-08-17T17:54:04.808637+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":4386717720,"num_objects":66937,"num_object_clones":0,"num_object_copies":200811,"num_objects_missing_on_primary":3674,"num_objects_missing":3674,"num_objects_degraded":17538,"num_objects_misplaced":11447,"num_objects_unfound":0,"num_objects_dirty":66937,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142673,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34741,"num_bytes_recovered":2098331648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13840429056,"data_stored":13840416840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107847,"ondisk_log_size":107847,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234400,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2143456,"kb_used_data":1833624,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91601696,"statfs":{"total":95995035648,"available":93800136704,"internally_reserved":0,"allocated":1877630976,"data_stored":1872736515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496035,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2414564,"kb_used_data":2104732,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91330588,"statfs":{"total":95995035648,"available":93522522112,"internally_reserved":0,"allocated":2155245568,"data_stored":2150347512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757673,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326287,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2614732,"kb_used_data":2394116,"kb_used_omap":57,"kb_used_meta":220550,"kb_avail":91130420,"statfs":{"total":95995035648,"available":93317550080,"internally_reserved":0,"allocated":2451574784,"data_stored":2446695503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":225843963},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405526,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2356948,"kb_used_data":2148748,"kb_used_omap":61,"kb_used_meta":208130,"kb_avail":91388204,"statfs":{"total":95995035648,"available":93581520896,"internally_reserved":0,"allocated":2200317952,"data_stored":2195435643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":213126072},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452061,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1773144,"kb_used_data":1610832,"kb_used_omap":54,"kb_used_meta":162249,"kb_avail":91972008,"statfs":{"total":95995035648,"available":94179336192,"internally_reserved":0,"allocated":1649491968,"data_stored":1644609212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":166143374},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474752,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1532376,"kb_used_data":1341584,"kb_used_omap":119,"kb_used_meta":190664,"kb_avail":92212776,"statfs":{"total":95995035648,"available":94425882624,"internally_reserved":0,"allocated":1373782016,"data_stored":1368891532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":195240670},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019308,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1359745024,"data_stored":1359740952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1636171776,"data_stored":1636171776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2186346496,"data_stored":2186346496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2432897024,"data_stored":2432892952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1048248320,"data_stored":1048248320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1175912448,"data_stored":1175912448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2140147712,"data_stored":2140143640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1860960256,"data_stored":1860960256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:54:20.159 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:54:20.200 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:20.221 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:20.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:20.323 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:20.344 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:20.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:20.394 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:54:20.394 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:54:20.412 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:54:20.412 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:54:20.442 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:54:20.442 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:54:20.446 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:20.460 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:54:20.460 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:54:20.467 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:20.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:20.486 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:54:20.487 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:54:20.506 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:54:20.506 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:54:20.524 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:54:20.524 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:54:20.545 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:54:20.545 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:54:20.569 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3725187049' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:54:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:54:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:54:20.597 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:20.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:20.648 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:54:20.648 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:54:20.648 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 0 2024-08-17T17:54:20.648 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:54:20.649 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3725187049' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:20.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3725187049' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:20.698 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:20.725 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:20.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:20.826 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:20.847 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:20.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:20.949 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:20.970 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:20.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:20.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:21.072 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:21.094 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:21.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:21.195 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:21.218 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:21.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:21.320 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:21.341 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:21.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:21.443 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:21.464 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:21.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:21.566 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:21.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:21.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:21.688 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17538/199899 objects degraded (8.773%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:21.688 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:21 smithi130 ceph-mon[105521]: pgmap v1817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 17538/199461 objects degraded (8.793%); 11359/199461 objects misplaced (5.695%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:21.688 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17538/199899 objects degraded (8.773%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:21.688 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:21 smithi130 ceph-mon[110068]: pgmap v1817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 17538/199461 objects degraded (8.793%); 11359/199461 objects misplaced (5.695%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:21.690 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:21.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:21.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:21.812 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:21.833 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:21.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:21.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17538/199899 objects degraded (8.773%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:21.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:21 smithi179 ceph-mon[93519]: pgmap v1817: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 17538/199461 objects degraded (8.793%); 11359/199461 objects misplaced (5.695%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:21.935 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:21.958 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:21.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:21.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:22.060 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:22.081 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:22.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:22.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:22.209 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:22.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:22.311 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:22.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:54:22.333 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:22.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:22.369 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:22.434 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:22.460 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:22.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:22.563 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:22.586 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:22.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:22.687 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:22.713 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:22.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:22.814 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:22.836 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:22.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:22.937 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:22.960 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:22.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:22.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:23.062 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:23.083 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:23.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:23.118 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:23.162 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:23.184 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:23.206 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:23.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:23.244 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:23 smithi130 ceph-mon[105521]: pgmap v1818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 17464/199239 objects degraded (8.765%); 11322/199239 objects misplaced (5.683%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:54:23.245 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:23 smithi130 ceph-mon[110068]: pgmap v1818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 17464/199239 objects degraded (8.765%); 11322/199239 objects misplaced (5.683%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:54:23.308 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:23.334 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:23.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:23.436 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:23.457 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:23.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:23.559 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:23.582 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:23.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:23.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:23 smithi179 ceph-mon[93519]: pgmap v1818: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 114 op/s; 17464/199239 objects degraded (8.765%); 11322/199239 objects misplaced (5.683%); 973 KiB/s, 15 objects/s recovering 2024-08-17T17:54:23.637 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:54:23.685 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:23.709 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:23.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:23.811 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:23.834 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:23.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:23.935 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:23.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:23.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:23.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:24.059 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:24.083 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:24.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:24.184 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:24.206 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:24.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:24.308 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:24.331 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:24.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:24.433 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:24.454 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:24.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:24.555 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:24.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:24.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:24.678 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:24.699 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:24.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:24.801 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:24.822 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:24.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:24.924 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:24.945 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:24.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:24.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:25.046 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:25.067 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:25.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:25.169 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:25.191 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:25.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:25.293 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:25.321 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:25.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:25.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:25 smithi130 ceph-mon[105521]: pgmap v1819: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 110 op/s; 17464/197730 objects degraded (8.832%); 11237/197730 objects misplaced (5.683%); 977 KiB/s, 15 objects/s recovering 2024-08-17T17:54:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:25 smithi130 ceph-mon[110068]: pgmap v1819: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 110 op/s; 17464/197730 objects degraded (8.832%); 11237/197730 objects misplaced (5.683%); 977 KiB/s, 15 objects/s recovering 2024-08-17T17:54:25.423 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:25.445 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:25.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:25.546 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:25.568 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:25.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:25.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:25 smithi179 ceph-mon[93519]: pgmap v1819: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 110 op/s; 17464/197730 objects degraded (8.832%); 11237/197730 objects misplaced (5.683%); 977 KiB/s, 15 objects/s recovering 2024-08-17T17:54:25.671 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:25.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:25.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:25.795 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:25.817 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:25.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:25.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:25.940 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:25.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:25.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:26.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:26.070 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:26.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:26.173 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:26.195 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:26.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:26.231 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17464/197730 objects degraded (8.832%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:26.232 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17464/197730 objects degraded (8.832%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:26.296 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:26.325 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:26.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:26.427 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:26.453 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:26.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:26.556 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:26.577 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:26.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17464/197730 objects degraded (8.832%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:26.678 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:26.701 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:26.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:26.804 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:26.825 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:26.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:26.927 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:26.948 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:26.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:26.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:27.049 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:27.071 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:27.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:27.173 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:27.195 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:27.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:27.296 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:27.318 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:27.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:27 smithi130 ceph-mon[105521]: pgmap v1820: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 17384/197517 objects degraded (8.801%); 11206/197517 objects misplaced (5.673%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:27 smithi130 ceph-mon[110068]: pgmap v1820: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 17384/197517 objects degraded (8.801%); 11206/197517 objects misplaced (5.673%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:27.419 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:27.440 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:27.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:27.508 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:54:27.508 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 17384/197517 objects degraded (8.801%), 6 pgs degraded, 6 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1588,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1224,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":65839,"data_bytes":4313651173,"bytes_used":12943724544,"bytes_avail":563026489344,"bytes_total":575970213888,"degraded_objects":17384,"degraded_total":197517,"degraded_ratio":0.088012677389794292,"misplaced_objects":11206,"misplaced_total":197517,"misplaced_ratio":0.056734357042684931,"recovering_objects_per_sec":16,"recovering_bytes_per_sec":1064664,"recovering_keys_per_sec":0,"num_objects_recovered":195,"num_bytes_recovered":12779520,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":99},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":25,"modified":"2024-08-17T17:49:39.646037+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (6m)\n [==========================..] (remaining: 29s)","progress":0.93258428573608398,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:54:27.542 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:27.563 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:27.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:27.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:27 smithi179 ceph-mon[93519]: pgmap v1820: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 99 op/s; 17384/197517 objects degraded (8.801%); 11206/197517 objects misplaced (5.673%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:27.664 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:27.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:27.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:27.739 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:27.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:27.809 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:27.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:27.911 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:27.932 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:27.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:27.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:28.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:28.055 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:28.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:28.156 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:28.183 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:28.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:28.285 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:28.306 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:28.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:28.408 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:28.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:28.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:28 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2406070797' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:54:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:28 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2406070797' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:54:28.432 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:28.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:28.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:28.558 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:28.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:28 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2406070797' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:54:28.660 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:28.681 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:28.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:28.783 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:28.805 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:28.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:28.907 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:28.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:28.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:28.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:29.029 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:29.051 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:29.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:29.148 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T17:54:29.152 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:29.183 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:29.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:29.285 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:29.310 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:29.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:29.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/508468451' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:54:29.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:29 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:54:29.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:29 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:54:29.312 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:29 smithi130 ceph-mon[105521]: pgmap v1821: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 17384/196425 objects degraded (8.850%); 11206/196425 objects misplaced (5.705%); 869 KiB/s, 13 objects/s recovering 2024-08-17T17:54:29.312 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/508468451' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:54:29.312 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:29 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:54:29.312 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:29 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:54:29.312 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:29 smithi130 ceph-mon[110068]: pgmap v1821: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 17384/196425 objects degraded (8.850%); 11206/196425 objects misplaced (5.705%); 869 KiB/s, 13 objects/s recovering 2024-08-17T17:54:29.332 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/508468451' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:54:29.333 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:29 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:54:29.333 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:29 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:54:29.333 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:29 smithi179 ceph-mon[93519]: pgmap v1821: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 17384/196425 objects degraded (8.850%); 11206/196425 objects misplaced (5.705%); 869 KiB/s, 13 objects/s recovering 2024-08-17T17:54:29.346 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:54:29.346 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:54:29.412 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:29.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:29.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:29.536 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:29.558 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:29.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:29.660 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:29.682 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:29.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:29.784 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:29.805 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:29.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:29.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:29.931 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:29.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:29.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:30.033 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:30.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.055 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:30.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:30.065 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:54:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:54:29] "GET /metrics HTTP/1.1" 200 36813 "" "Prometheus/2.51.0" 2024-08-17T17:54:30.156 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:30.177 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:30.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:30.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:30 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:30.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:30 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:54:30.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:30 smithi179 ceph-mon[93519]: osdmap e1225: 8 total, 8 up, 6 in 2024-08-17T17:54:30.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:30.285 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:54:30.285 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:54:30.285 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T17:54:30.285 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T17:54:30.285 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:54:30.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:30 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:30.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:30 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:54:30.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:30 smithi130 ceph-mon[110068]: osdmap e1225: 8 total, 8 up, 6 in 2024-08-17T17:54:30.287 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:30 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:30.287 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:30 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:54:30.287 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:30 smithi130 ceph-mon[105521]: osdmap e1225: 8 total, 8 up, 6 in 2024-08-17T17:54:30.301 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:30.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:30.404 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:30.426 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:30.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:30.528 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:30.554 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:30.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:54:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:54:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:54:30.655 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:30.680 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:30.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:30.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:30.804 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:30.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:30.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:30.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:30.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:30.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:30.955 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:54:31.030 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:31.099 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:31.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:31.201 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:31.222 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:31.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:31.319 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17384/196425 objects degraded (8.850%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:31.319 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:31 smithi130 ceph-mon[105521]: pgmap v1823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 17384/195909 objects degraded (8.874%); 11121/195909 objects misplaced (5.677%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:31.319 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17384/196425 objects degraded (8.850%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:31.319 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:31 smithi130 ceph-mon[110068]: pgmap v1823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 17384/195909 objects degraded (8.874%); 11121/195909 objects misplaced (5.677%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:31.324 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:31.352 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:31.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:31.454 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:31.478 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:31.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:31.579 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:31.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17384/196425 objects degraded (8.850%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:31 smithi179 ceph-mon[93519]: pgmap v1823: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 118 op/s; 17384/195909 objects degraded (8.874%); 11121/195909 objects misplaced (5.677%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:54:31.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:31.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:31.704 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:31.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.726 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:31.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:31.827 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:31.849 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:31.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:31.950 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:31.973 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:31.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:32.075 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:32.093 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:54:32.098 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:32.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:32.200 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:32.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.222 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:32.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:32.323 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:32.344 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:32.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:32.357 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:54:32.357 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:54:32.367 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:32.393 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:54:32.393 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:54:32.420 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:54:32.421 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:54:32.432 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:32.445 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:32.449 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:54:32.449 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:54:32.466 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:32.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:32.475 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:54:32.475 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:54:32.501 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:54:32.502 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:54:32.502 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2990407423' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:32.502 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2990407423' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:32.521 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:54:32.522 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:54:32.538 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:54:32.539 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:54:32.568 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:32.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.590 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:32.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:32.661 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2990407423' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:32.691 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:32.712 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:32.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:32.814 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:32.838 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:32.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:32.940 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:32.962 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:32.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:32.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:33.063 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:33.085 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:33.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:33.187 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:33.211 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:33.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:33.313 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:33.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:33.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:33.436 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:33.465 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:33.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:33.568 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:33.589 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:33.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:33.690 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:33.712 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:33.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:33.814 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:33.835 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:33.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:33.875 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:33.937 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:33.959 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:33.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:33.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:34.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:34.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.084 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:34.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:34.185 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:34.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.219 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:34.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:34.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:33 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:34.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:33 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:34.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:33 smithi130 ceph-mon[105521]: pgmap v1824: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 17290/195645 objects degraded (8.837%); 11080/195645 objects misplaced (5.663%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:34.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:33 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:34.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:33 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:34.298 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:33 smithi130 ceph-mon[110068]: pgmap v1824: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 17290/195645 objects degraded (8.837%); 11080/195645 objects misplaced (5.663%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:34.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:34.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:33 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:34.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:33 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:34.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:33 smithi179 ceph-mon[93519]: pgmap v1824: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 119 op/s; 17290/195645 objects degraded (8.837%); 11080/195645 objects misplaced (5.663%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:34.345 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:34.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:34.447 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:34.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.475 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:34.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:34.576 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:34.598 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:34.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:34.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:34.722 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:34.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:34.824 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:34.845 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:34.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:34.947 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:34.968 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:34.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:34.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:35.070 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:35.096 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:35.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:35.198 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:35.221 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:35.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:35.323 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:35.344 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:35.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:35.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:35 smithi130 ceph-mon[105521]: pgmap v1825: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 17290/193944 objects degraded (8.915%); 10999/193944 objects misplaced (5.671%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:35 smithi130 ceph-mon[110068]: pgmap v1825: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 17290/193944 objects degraded (8.915%); 10999/193944 objects misplaced (5.671%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:35.446 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:35.471 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:35.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:35.477 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:54:35.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:35.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:35 smithi179 ceph-mon[93519]: pgmap v1825: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 126 op/s; 17290/193944 objects degraded (8.915%); 10999/193944 objects misplaced (5.671%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:35.599 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:35.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:35.701 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:35.736 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:35.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:35.838 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:35.859 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:35.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:35.961 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:35.983 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:35.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:35.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:36.085 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:36.113 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:36.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:36.215 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:36.237 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:36.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:36.339 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:36.364 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:36.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:36.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17290/193944 objects degraded (8.915%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:36.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17290/193944 objects degraded (8.915%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:36.466 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:36.490 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:36.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:36.591 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:36.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17290/193944 objects degraded (8.915%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:36.612 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:36.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:36.714 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:36.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.738 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:36.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:36.839 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:36.860 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:36.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:36.962 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:36.983 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:36.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:37.085 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:37.111 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:37.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:37.213 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:37.234 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:37.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:37.335 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:37.357 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:37.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:37.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:37 smithi130 ceph-mon[105521]: pgmap v1826: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 17178/193626 objects degraded (8.872%); 10949/193626 objects misplaced (5.655%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:54:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:37 smithi130 ceph-mon[110068]: pgmap v1826: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 17178/193626 objects degraded (8.872%); 10949/193626 objects misplaced (5.655%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:54:37.459 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:37.481 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:37.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:37.583 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:37 smithi179 ceph-mon[93519]: pgmap v1826: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 17178/193626 objects degraded (8.872%); 10949/193626 objects misplaced (5.655%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:54:37.612 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:37.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:37.714 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:37.739 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:37.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:37.841 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:37.863 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:37.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:37.904 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:54:37.905 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:54:37.964 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:37.987 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:37.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:37.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:38.089 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:38.111 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:38.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:38.213 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:38.242 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:38.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:38.344 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:38.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.370 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:38.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:38.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:38 smithi130 ceph-mon[105521]: from='client.55294 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:54:38.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:38 smithi130 ceph-mon[110068]: from='client.55294 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:54:38.471 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:38.492 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:38.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:38 smithi179 ceph-mon[93519]: from='client.55294 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:54:38.594 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:38.615 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:38.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:38.717 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:38.740 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:38.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:38.842 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:38.869 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:38.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:38.928 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:38.936 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:54:38.936 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:54:38.970 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:38.991 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:38.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:38.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:39.093 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:39.115 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:39.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:39.216 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:39.238 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:39.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:39.340 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:39.365 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:39.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:39.381 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:39 smithi130 ceph-mon[105521]: pgmap v1827: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 148 op/s; 17178/191973 objects degraded (8.948%); 10949/191973 objects misplaced (5.703%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:54:39.381 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:39 smithi130 ceph-mon[110068]: pgmap v1827: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 148 op/s; 17178/191973 objects degraded (8.948%); 10949/191973 objects misplaced (5.703%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:54:39.418 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1826,"stamp":"2024-08-17T17:54:37.057213+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4228650981,"num_objects":64542,"num_object_clones":0,"num_object_copies":193626,"num_objects_missing_on_primary":3505,"num_objects_missing":3505,"num_objects_degraded":17178,"num_objects_misplaced":10949,"num_objects_unfound":0,"num_objects_dirty":64542,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":145372,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35120,"num_bytes_recovered":2123045901,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108475,"ondisk_log_size":108475,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12463108,"kb_used_data":11051796,"kb_used_omap":295,"kb_used_meta":1410968,"kb_avail":550007804,"statfs":{"total":575970213888,"available":563207991296,"internally_reserved":0,"allocated":11317039104,"data_stored":11287629961,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1444831695},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,1,0,3,1,1,3,4,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-85000192,"num_objects":-1297,"num_object_clones":0,"num_object_copies":-3891,"num_objects_missing_on_primary":-82,"num_objects_missing":-82,"num_objects_degraded":-206,"num_objects_misplaced":-257,"num_objects_unfound":0,"num_objects_dirty":-1297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1297,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":187,"num_bytes_recovered":12255232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002800"},"pg_stats":[{"pgid":"45.f","version":"1225'9087","reported_seq":19616,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:34.506109+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:54:34.506109+0000","last_peered":"2024-08-17T17:54:34.506109+0000","last_clean":"2024-08-17T17:54:34.506109+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:54:34.506109+0000","last_undegraded":"2024-08-17T17:54:34.506109+0000","last_fullsized":"2024-08-17T17:54:34.506109+0000","mapping_epoch":1194,"log_start":"1170'2231","ondisk_log_start":"1170'2231","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6856,"log_dups_size":0,"ondisk_log_size":6856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267190272,"num_objects":4077,"num_object_clones":0,"num_object_copies":12231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9087,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1225'8986","reported_seq":19451,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:32.878349+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:54:32.878349+0000","last_peered":"2024-08-17T17:54:32.878349+0000","last_clean":"2024-08-17T17:54:32.878349+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:54:32.878349+0000","last_undegraded":"2024-08-17T17:54:32.878349+0000","last_fullsized":"2024-08-17T17:54:32.878349+0000","mapping_epoch":1211,"log_start":"1170'1932","ondisk_log_start":"1170'1932","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7054,"log_dups_size":0,"ondisk_log_size":7054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262668288,"num_objects":4008,"num_object_clones":0,"num_object_copies":12024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8986,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1225'9285","reported_seq":20122,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:33.588233+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:54:33.588233+0000","last_peered":"2024-08-17T17:54:33.588233+0000","last_clean":"2024-08-17T17:54:33.588233+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:54:33.588233+0000","last_undegraded":"2024-08-17T17:54:33.588233+0000","last_fullsized":"2024-08-17T17:54:33.588233+0000","mapping_epoch":1205,"log_start":"1174'3368","ondisk_log_start":"1174'3368","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5917,"log_dups_size":0,"ondisk_log_size":5917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268173336,"num_objects":4093,"num_object_clones":0,"num_object_copies":12279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4093,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9285,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1225'9012","reported_seq":9078,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:32.797957+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:54:32.797957+0000","last_peered":"2024-08-17T17:54:32.797957+0000","last_clean":"2024-08-17T17:54:32.797957+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:54:32.797957+0000","last_undegraded":"2024-08-17T17:54:32.797957+0000","last_fullsized":"2024-08-17T17:54:32.797957+0000","mapping_epoch":1167,"log_start":"1170'2289","ondisk_log_start":"1170'2289","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6723,"log_dups_size":0,"ondisk_log_size":6723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261357568,"num_objects":3988,"num_object_clones":0,"num_object_copies":11964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9012,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1225'9088","reported_seq":14331,"reported_epoch":1225,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:36.623823+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:54:36.623823+0000","last_peered":"2024-08-17T17:54:36.623823+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:54:36.623823+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1174'3126","ondisk_log_start":"1174'3126","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5962,"log_dups_size":0,"ondisk_log_size":5962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263847936,"num_objects":4026,"num_object_clones":0,"num_object_copies":12078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":4026,"num_objects_unfound":0,"num_objects_dirty":4026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9088,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":3508},{"shards":"1,4,5","objects":518}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1225'9154","reported_seq":9220,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:32.780821+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:54:32.780821+0000","last_peered":"2024-08-17T17:54:32.780821+0000","last_clean":"2024-08-17T17:54:32.780821+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:54:32.780821+0000","last_undegraded":"2024-08-17T17:54:32.780821+0000","last_fullsized":"2024-08-17T17:54:32.780821+0000","mapping_epoch":1167,"log_start":"1170'2477","ondisk_log_start":"1170'2477","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6677,"log_dups_size":0,"ondisk_log_size":6677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258211840,"num_objects":3940,"num_object_clones":0,"num_object_copies":11820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9154,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1225'9106","reported_seq":9172,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:33.104086+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:54:33.104086+0000","last_peered":"2024-08-17T17:54:33.104086+0000","last_clean":"2024-08-17T17:54:33.104086+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:54:33.104086+0000","last_undegraded":"2024-08-17T17:54:33.104086+0000","last_fullsized":"2024-08-17T17:54:33.104086+0000","mapping_epoch":1167,"log_start":"1170'2369","ondisk_log_start":"1170'2369","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6737,"log_dups_size":0,"ondisk_log_size":6737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270008320,"num_objects":4120,"num_object_clones":0,"num_object_copies":12360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9106,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1225'9065","reported_seq":16911,"reported_epoch":1225,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:36.504393+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:54:36.504393+0000","last_peered":"2024-08-17T17:54:36.504393+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:54:36.504393+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1174'3175","ondisk_log_start":"1174'3175","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5890,"log_dups_size":0,"ondisk_log_size":5890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257228800,"num_objects":3925,"num_object_clones":0,"num_object_copies":11775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3143,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9065,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2570,"num_bytes_recovered":168427520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":538},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1225'8998","reported_seq":9074,"reported_epoch":1225,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:32.656435+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:54:32.656435+0000","last_peered":"2024-08-17T17:54:32.656435+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:54:32.656435+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'2277","ondisk_log_start":"1170'2277","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6721,"log_dups_size":0,"ondisk_log_size":6721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267255808,"num_objects":4078,"num_object_clones":0,"num_object_copies":12234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4078,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8998,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":4078},{"shards":"1,6","objects":-1755}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1225'9069","reported_seq":12145,"reported_epoch":1225,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:33.523116+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:54:33.523116+0000","last_peered":"2024-08-17T17:54:33.523116+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:54:33.523116+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1174'3106","ondisk_log_start":"1174'3106","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5963,"log_dups_size":0,"ondisk_log_size":5963,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269025280,"num_objects":4105,"num_object_clones":0,"num_object_copies":12315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9069,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":2544},{"shards":"1,3,5","objects":546}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1225'9014","reported_seq":9145,"reported_epoch":1225,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:33.557000+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:54:33.557000+0000","last_peered":"2024-08-17T17:54:33.557000+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:54:33.557000+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9014,"log_dups_size":0,"ondisk_log_size":9014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266076160,"num_objects":4060,"num_object_clones":0,"num_object_copies":12180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9014,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-1093}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1225'9012","reported_seq":15204,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:33.297108+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:54:33.297108+0000","last_peered":"2024-08-17T17:54:33.297108+0000","last_clean":"2024-08-17T17:54:33.297108+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:54:33.297108+0000","last_undegraded":"2024-08-17T17:54:33.297108+0000","last_fullsized":"2024-08-17T17:54:33.297108+0000","mapping_epoch":1183,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9012,"log_dups_size":0,"ondisk_log_size":9012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264372224,"num_objects":4034,"num_object_clones":0,"num_object_copies":12102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9012,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1225'9132","reported_seq":9262,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:33.620036+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:54:33.620036+0000","last_peered":"2024-08-17T17:54:33.620036+0000","last_clean":"2024-08-17T17:54:33.620036+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:54:33.620036+0000","last_undegraded":"2024-08-17T17:54:33.620036+0000","last_fullsized":"2024-08-17T17:54:33.620036+0000","mapping_epoch":1167,"log_start":"1174'3191","ondisk_log_start":"1174'3191","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5941,"log_dups_size":0,"ondisk_log_size":5941,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263979008,"num_objects":4028,"num_object_clones":0,"num_object_copies":12084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9132,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1225'8990","reported_seq":9056,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:33.173546+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:54:33.173546+0000","last_peered":"2024-08-17T17:54:33.173546+0000","last_clean":"2024-08-17T17:54:33.173546+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:54:33.173546+0000","last_undegraded":"2024-08-17T17:54:33.173546+0000","last_fullsized":"2024-08-17T17:54:33.173546+0000","mapping_epoch":1167,"log_start":"1170'1925","ondisk_log_start":"1170'1925","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7065,"log_dups_size":0,"ondisk_log_size":7065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258473984,"num_objects":3944,"num_object_clones":0,"num_object_copies":11832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8990,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1225'9218","reported_seq":20457,"reported_epoch":1225,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:34.530007+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:54:34.530007+0000","last_peered":"2024-08-17T17:54:34.530007+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:54:34.530007+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'2391","ondisk_log_start":"1170'2391","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6827,"log_dups_size":0,"ondisk_log_size":6827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271712256,"num_objects":4146,"num_object_clones":0,"num_object_copies":12438,"num_objects_missing_on_primary":3505,"num_objects_missing":3505,"num_objects_degraded":265,"num_objects_misplaced":6923,"num_objects_unfound":0,"num_objects_dirty":4146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9218,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4643,"num_bytes_recovered":304283648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":11},{"shards":"0,4,6","objects":1356},{"shards":"0,6","objects":1},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":2513}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1225'8878","reported_seq":22194,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:33.577945+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:54:33.577945+0000","last_peered":"2024-08-17T17:54:33.577945+0000","last_clean":"2024-08-17T17:54:33.577945+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:54:33.577945+0000","last_undegraded":"2024-08-17T17:54:33.577945+0000","last_fullsized":"2024-08-17T17:54:33.577945+0000","mapping_epoch":1196,"log_start":"1174'2969","ondisk_log_start":"1174'2969","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5909,"log_dups_size":0,"ondisk_log_size":5909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258473984,"num_objects":3944,"num_object_clones":0,"num_object_copies":11832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8878,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1906,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.162966+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:54:29.162966+0000","last_peered":"2024-08-17T17:54:29.162966+0000","last_clean":"2024-08-17T17:54:29.162966+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:54:29.162966+0000","last_undegraded":"2024-08-17T17:54:29.162966+0000","last_fullsized":"2024-08-17T17:54:29.162966+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":502,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.018865+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:54:15.018865+0000","last_peered":"2024-08-17T17:54:15.018865+0000","last_clean":"2024-08-17T17:54:15.018865+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:54:15.018865+0000","last_undegraded":"2024-08-17T17:54:15.018865+0000","last_fullsized":"2024-08-17T17:54:15.018865+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1826,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.144423+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:54:29.144423+0000","last_peered":"2024-08-17T17:54:29.144423+0000","last_clean":"2024-08-17T17:54:29.144423+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:54:29.144423+0000","last_undegraded":"2024-08-17T17:54:29.144423+0000","last_fullsized":"2024-08-17T17:54:29.144423+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1850,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.143235+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:54:29.143235+0000","last_peered":"2024-08-17T17:54:29.143235+0000","last_clean":"2024-08-17T17:54:29.143235+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:54:29.143235+0000","last_undegraded":"2024-08-17T17:54:29.143235+0000","last_fullsized":"2024-08-17T17:54:29.143235+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":64,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.019430+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:54:15.019430+0000","last_peered":"2024-08-17T17:54:15.019430+0000","last_clean":"2024-08-17T17:54:15.019430+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:54:15.019430+0000","last_undegraded":"2024-08-17T17:54:15.019430+0000","last_fullsized":"2024-08-17T17:54:15.019430+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1243,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.144028+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:54:29.144028+0000","last_peered":"2024-08-17T17:54:29.144028+0000","last_clean":"2024-08-17T17:54:29.144028+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:54:29.144028+0000","last_undegraded":"2024-08-17T17:54:29.144028+0000","last_fullsized":"2024-08-17T17:54:29.144028+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":101,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.145631+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:54:29.145631+0000","last_peered":"2024-08-17T17:54:29.145631+0000","last_clean":"2024-08-17T17:54:29.145631+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:54:29.145631+0000","last_undegraded":"2024-08-17T17:54:29.145631+0000","last_fullsized":"2024-08-17T17:54:29.145631+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":621,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.147740+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:54:29.147740+0000","last_peered":"2024-08-17T17:54:29.147740+0000","last_clean":"2024-08-17T17:54:29.147740+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:54:29.147740+0000","last_undegraded":"2024-08-17T17:54:29.147740+0000","last_fullsized":"2024-08-17T17:54:29.147740+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1865,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.162468+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:54:29.162468+0000","last_peered":"2024-08-17T17:54:29.162468+0000","last_clean":"2024-08-17T17:54:29.162468+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:54:29.162468+0000","last_undegraded":"2024-08-17T17:54:29.162468+0000","last_fullsized":"2024-08-17T17:54:29.162468+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1867,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.167168+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:54:29.167168+0000","last_peered":"2024-08-17T17:54:29.167168+0000","last_clean":"2024-08-17T17:54:29.167168+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:54:29.167168+0000","last_undegraded":"2024-08-17T17:54:29.167168+0000","last_fullsized":"2024-08-17T17:54:29.167168+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":86,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.145318+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:54:29.145318+0000","last_peered":"2024-08-17T17:54:29.145318+0000","last_clean":"2024-08-17T17:54:29.145318+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:54:29.145318+0000","last_undegraded":"2024-08-17T17:54:29.145318+0000","last_fullsized":"2024-08-17T17:54:29.145318+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1839,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.144298+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:54:29.144298+0000","last_peered":"2024-08-17T17:54:29.144298+0000","last_clean":"2024-08-17T17:54:29.144298+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:54:29.144298+0000","last_undegraded":"2024-08-17T17:54:29.144298+0000","last_fullsized":"2024-08-17T17:54:29.144298+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1893,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.145143+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:54:29.145143+0000","last_peered":"2024-08-17T17:54:29.145143+0000","last_clean":"2024-08-17T17:54:29.145143+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:54:29.145143+0000","last_undegraded":"2024-08-17T17:54:29.145143+0000","last_fullsized":"2024-08-17T17:54:29.145143+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1887,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.164409+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:54:29.164409+0000","last_peered":"2024-08-17T17:54:29.164409+0000","last_clean":"2024-08-17T17:54:29.164409+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:54:29.164409+0000","last_undegraded":"2024-08-17T17:54:29.164409+0000","last_fullsized":"2024-08-17T17:54:29.164409+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1861,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.143687+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:54:29.143687+0000","last_peered":"2024-08-17T17:54:29.143687+0000","last_clean":"2024-08-17T17:54:29.143687+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:54:29.143687+0000","last_undegraded":"2024-08-17T17:54:29.143687+0000","last_fullsized":"2024-08-17T17:54:29.143687+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1199,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.019528+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:54:15.019528+0000","last_peered":"2024-08-17T17:54:15.019528+0000","last_clean":"2024-08-17T17:54:15.019528+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:54:15.019528+0000","last_undegraded":"2024-08-17T17:54:15.019528+0000","last_fullsized":"2024-08-17T17:54:15.019528+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1832,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.143212+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:54:29.143212+0000","last_peered":"2024-08-17T17:54:29.143212+0000","last_clean":"2024-08-17T17:54:29.143212+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:54:29.143212+0000","last_undegraded":"2024-08-17T17:54:29.143212+0000","last_fullsized":"2024-08-17T17:54:29.143212+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1882,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.163068+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:54:29.163068+0000","last_peered":"2024-08-17T17:54:29.163068+0000","last_clean":"2024-08-17T17:54:29.163068+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:54:29.163068+0000","last_undegraded":"2024-08-17T17:54:29.163068+0000","last_fullsized":"2024-08-17T17:54:29.163068+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1788,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.143153+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:54:29.143153+0000","last_peered":"2024-08-17T17:54:29.143153+0000","last_clean":"2024-08-17T17:54:29.143153+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:54:29.143153+0000","last_undegraded":"2024-08-17T17:54:29.143153+0000","last_fullsized":"2024-08-17T17:54:29.143153+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":518,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.032693+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:54:15.032693+0000","last_peered":"2024-08-17T17:54:15.032693+0000","last_clean":"2024-08-17T17:54:15.032693+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:54:15.032693+0000","last_undegraded":"2024-08-17T17:54:15.032693+0000","last_fullsized":"2024-08-17T17:54:15.032693+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1195,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.032603+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:54:15.032603+0000","last_peered":"2024-08-17T17:54:15.032603+0000","last_clean":"2024-08-17T17:54:15.032603+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:54:15.032603+0000","last_undegraded":"2024-08-17T17:54:15.032603+0000","last_fullsized":"2024-08-17T17:54:15.032603+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1887,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.145418+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:54:29.145418+0000","last_peered":"2024-08-17T17:54:29.145418+0000","last_clean":"2024-08-17T17:54:29.145418+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:54:29.145418+0000","last_undegraded":"2024-08-17T17:54:29.145418+0000","last_fullsized":"2024-08-17T17:54:29.145418+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":356,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.034754+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:54:15.034754+0000","last_peered":"2024-08-17T17:54:15.034754+0000","last_clean":"2024-08-17T17:54:15.034754+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:54:15.034754+0000","last_undegraded":"2024-08-17T17:54:15.034754+0000","last_fullsized":"2024-08-17T17:54:15.034754+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":502,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.019222+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:54:15.019222+0000","last_peered":"2024-08-17T17:54:15.019222+0000","last_clean":"2024-08-17T17:54:15.019222+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:54:15.019222+0000","last_undegraded":"2024-08-17T17:54:15.019222+0000","last_fullsized":"2024-08-17T17:54:15.019222+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1180,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.019100+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:54:15.019100+0000","last_peered":"2024-08-17T17:54:15.019100+0000","last_clean":"2024-08-17T17:54:15.019100+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:54:15.019100+0000","last_undegraded":"2024-08-17T17:54:15.019100+0000","last_fullsized":"2024-08-17T17:54:15.019100+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1188,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.019589+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:54:15.019589+0000","last_peered":"2024-08-17T17:54:15.019589+0000","last_clean":"2024-08-17T17:54:15.019589+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:54:15.019589+0000","last_undegraded":"2024-08-17T17:54:15.019589+0000","last_fullsized":"2024-08-17T17:54:15.019589+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1934,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.164167+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:54:29.164167+0000","last_peered":"2024-08-17T17:54:29.164167+0000","last_clean":"2024-08-17T17:54:29.164167+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:54:29.164167+0000","last_undegraded":"2024-08-17T17:54:29.164167+0000","last_fullsized":"2024-08-17T17:54:29.164167+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1818,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.143592+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:54:29.143592+0000","last_peered":"2024-08-17T17:54:29.143592+0000","last_clean":"2024-08-17T17:54:29.143592+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:54:29.143592+0000","last_undegraded":"2024-08-17T17:54:29.143592+0000","last_fullsized":"2024-08-17T17:54:29.143592+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1893,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.146338+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:54:29.146338+0000","last_peered":"2024-08-17T17:54:29.146338+0000","last_clean":"2024-08-17T17:54:29.146338+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:54:29.146338+0000","last_undegraded":"2024-08-17T17:54:29.146338+0000","last_fullsized":"2024-08-17T17:54:29.146338+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1820,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.143677+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:54:29.143677+0000","last_peered":"2024-08-17T17:54:29.143677+0000","last_clean":"2024-08-17T17:54:29.143677+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:54:29.143677+0000","last_undegraded":"2024-08-17T17:54:29.143677+0000","last_fullsized":"2024-08-17T17:54:29.143677+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1852,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.162547+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:54:29.162547+0000","last_peered":"2024-08-17T17:54:29.162547+0000","last_clean":"2024-08-17T17:54:29.162547+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:54:29.162547+0000","last_undegraded":"2024-08-17T17:54:29.162547+0000","last_fullsized":"2024-08-17T17:54:29.162547+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1215,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.032729+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:54:15.032729+0000","last_peered":"2024-08-17T17:54:15.032729+0000","last_clean":"2024-08-17T17:54:15.032729+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:54:15.032729+0000","last_undegraded":"2024-08-17T17:54:15.032729+0000","last_fullsized":"2024-08-17T17:54:15.032729+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1899,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.145827+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:54:29.145827+0000","last_peered":"2024-08-17T17:54:29.145827+0000","last_clean":"2024-08-17T17:54:29.145827+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:54:29.145827+0000","last_undegraded":"2024-08-17T17:54:29.145827+0000","last_fullsized":"2024-08-17T17:54:29.145827+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":86,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.145157+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:54:29.145157+0000","last_peered":"2024-08-17T17:54:29.145157+0000","last_clean":"2024-08-17T17:54:29.145157+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:54:29.145157+0000","last_undegraded":"2024-08-17T17:54:29.145157+0000","last_fullsized":"2024-08-17T17:54:29.145157+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1918,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.145731+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:54:29.145731+0000","last_peered":"2024-08-17T17:54:29.145731+0000","last_clean":"2024-08-17T17:54:29.145731+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:54:29.145731+0000","last_undegraded":"2024-08-17T17:54:29.145731+0000","last_fullsized":"2024-08-17T17:54:29.145731+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":655,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.144243+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:54:29.144243+0000","last_peered":"2024-08-17T17:54:29.144243+0000","last_clean":"2024-08-17T17:54:29.144243+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:54:29.144243+0000","last_undegraded":"2024-08-17T17:54:29.144243+0000","last_fullsized":"2024-08-17T17:54:29.144243+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1899,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.145878+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:54:29.145878+0000","last_peered":"2024-08-17T17:54:29.145878+0000","last_clean":"2024-08-17T17:54:29.145878+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:54:29.145878+0000","last_undegraded":"2024-08-17T17:54:29.145878+0000","last_fullsized":"2024-08-17T17:54:29.145878+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1228,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.162332+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:54:29.162332+0000","last_peered":"2024-08-17T17:54:29.162332+0000","last_clean":"2024-08-17T17:54:29.162332+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:54:29.162332+0000","last_undegraded":"2024-08-17T17:54:29.162332+0000","last_fullsized":"2024-08-17T17:54:29.162332+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":146,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.145417+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:54:29.145417+0000","last_peered":"2024-08-17T17:54:29.145417+0000","last_clean":"2024-08-17T17:54:29.145417+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:54:29.145417+0000","last_undegraded":"2024-08-17T17:54:29.145417+0000","last_fullsized":"2024-08-17T17:54:29.145417+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1910,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.146646+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:54:29.146646+0000","last_peered":"2024-08-17T17:54:29.146646+0000","last_clean":"2024-08-17T17:54:29.146646+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:54:29.146646+0000","last_undegraded":"2024-08-17T17:54:29.146646+0000","last_fullsized":"2024-08-17T17:54:29.146646+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1887,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.146052+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:54:29.146052+0000","last_peered":"2024-08-17T17:54:29.146052+0000","last_clean":"2024-08-17T17:54:29.146052+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:54:29.146052+0000","last_undegraded":"2024-08-17T17:54:29.146052+0000","last_fullsized":"2024-08-17T17:54:29.146052+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1908,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.166137+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:54:29.166137+0000","last_peered":"2024-08-17T17:54:29.166137+0000","last_clean":"2024-08-17T17:54:29.166137+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:54:29.166137+0000","last_undegraded":"2024-08-17T17:54:29.166137+0000","last_fullsized":"2024-08-17T17:54:29.166137+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1913,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.164491+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:54:29.164491+0000","last_peered":"2024-08-17T17:54:29.164491+0000","last_clean":"2024-08-17T17:54:29.164491+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:54:29.164491+0000","last_undegraded":"2024-08-17T17:54:29.164491+0000","last_fullsized":"2024-08-17T17:54:29.164491+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1893,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.272519+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:54:29.272519+0000","last_peered":"2024-08-17T17:54:29.272519+0000","last_clean":"2024-08-17T17:54:29.272519+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:54:29.272519+0000","last_undegraded":"2024-08-17T17:54:29.272519+0000","last_fullsized":"2024-08-17T17:54:29.272519+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1222,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.032401+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:54:15.032401+0000","last_peered":"2024-08-17T17:54:15.032401+0000","last_clean":"2024-08-17T17:54:15.032401+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:54:15.032401+0000","last_undegraded":"2024-08-17T17:54:15.032401+0000","last_fullsized":"2024-08-17T17:54:15.032401+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1168,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.019409+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:54:15.019409+0000","last_peered":"2024-08-17T17:54:15.019409+0000","last_clean":"2024-08-17T17:54:15.019409+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:54:15.019409+0000","last_undegraded":"2024-08-17T17:54:15.019409+0000","last_fullsized":"2024-08-17T17:54:15.019409+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1802,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.144004+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:54:29.144004+0000","last_peered":"2024-08-17T17:54:29.144004+0000","last_clean":"2024-08-17T17:54:29.144004+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:54:29.144004+0000","last_undegraded":"2024-08-17T17:54:29.144004+0000","last_fullsized":"2024-08-17T17:54:29.144004+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":134,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.167727+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:54:29.167727+0000","last_peered":"2024-08-17T17:54:29.167727+0000","last_clean":"2024-08-17T17:54:29.167727+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:54:29.167727+0000","last_undegraded":"2024-08-17T17:54:29.167727+0000","last_fullsized":"2024-08-17T17:54:29.167727+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1534,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.032326+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:54:15.032326+0000","last_peered":"2024-08-17T17:54:15.032326+0000","last_clean":"2024-08-17T17:54:15.032326+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:54:15.032326+0000","last_undegraded":"2024-08-17T17:54:15.032326+0000","last_fullsized":"2024-08-17T17:54:15.032326+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1894,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.146735+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:54:29.146735+0000","last_peered":"2024-08-17T17:54:29.146735+0000","last_clean":"2024-08-17T17:54:29.146735+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:54:29.146735+0000","last_undegraded":"2024-08-17T17:54:29.146735+0000","last_fullsized":"2024-08-17T17:54:29.146735+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1834,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.143295+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:54:29.143295+0000","last_peered":"2024-08-17T17:54:29.143295+0000","last_clean":"2024-08-17T17:54:29.143295+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:54:29.143295+0000","last_undegraded":"2024-08-17T17:54:29.143295+0000","last_fullsized":"2024-08-17T17:54:29.143295+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":502,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.019247+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:54:15.019247+0000","last_peered":"2024-08-17T17:54:15.019247+0000","last_clean":"2024-08-17T17:54:15.019247+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:54:15.019247+0000","last_undegraded":"2024-08-17T17:54:15.019247+0000","last_fullsized":"2024-08-17T17:54:15.019247+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":573,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.166040+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:54:29.166040+0000","last_peered":"2024-08-17T17:54:29.166040+0000","last_clean":"2024-08-17T17:54:29.166040+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:54:29.166040+0000","last_undegraded":"2024-08-17T17:54:29.166040+0000","last_fullsized":"2024-08-17T17:54:29.166040+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":589,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.167328+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:54:29.167328+0000","last_peered":"2024-08-17T17:54:29.167328+0000","last_clean":"2024-08-17T17:54:29.167328+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:54:29.167328+0000","last_undegraded":"2024-08-17T17:54:29.167328+0000","last_fullsized":"2024-08-17T17:54:29.167328+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1186,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.019511+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:54:15.019511+0000","last_peered":"2024-08-17T17:54:15.019511+0000","last_clean":"2024-08-17T17:54:15.019511+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:54:15.019511+0000","last_undegraded":"2024-08-17T17:54:15.019511+0000","last_fullsized":"2024-08-17T17:54:15.019511+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":605,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.143615+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:54:29.143615+0000","last_peered":"2024-08-17T17:54:29.143615+0000","last_clean":"2024-08-17T17:54:29.143615+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:54:29.143615+0000","last_undegraded":"2024-08-17T17:54:29.143615+0000","last_fullsized":"2024-08-17T17:54:29.143615+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1831,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.147867+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:54:29.147867+0000","last_peered":"2024-08-17T17:54:29.147867+0000","last_clean":"2024-08-17T17:54:29.147867+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:54:29.147867+0000","last_undegraded":"2024-08-17T17:54:29.147867+0000","last_fullsized":"2024-08-17T17:54:29.147867+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1926,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.162617+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:54:29.162617+0000","last_peered":"2024-08-17T17:54:29.162617+0000","last_clean":"2024-08-17T17:54:29.162617+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:54:29.162617+0000","last_undegraded":"2024-08-17T17:54:29.162617+0000","last_fullsized":"2024-08-17T17:54:29.162617+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":637,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.143531+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:54:29.143531+0000","last_peered":"2024-08-17T17:54:29.143531+0000","last_clean":"2024-08-17T17:54:29.143531+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:54:29.143531+0000","last_undegraded":"2024-08-17T17:54:29.143531+0000","last_fullsized":"2024-08-17T17:54:29.143531+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1175,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.019428+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:54:15.019428+0000","last_peered":"2024-08-17T17:54:15.019428+0000","last_clean":"2024-08-17T17:54:15.019428+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:54:15.019428+0000","last_undegraded":"2024-08-17T17:54:15.019428+0000","last_fullsized":"2024-08-17T17:54:15.019428+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1174,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.032718+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:54:15.032718+0000","last_peered":"2024-08-17T17:54:15.032718+0000","last_clean":"2024-08-17T17:54:15.032718+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:54:15.032718+0000","last_undegraded":"2024-08-17T17:54:15.032718+0000","last_fullsized":"2024-08-17T17:54:15.032718+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":622,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.145233+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:54:29.145233+0000","last_peered":"2024-08-17T17:54:29.145233+0000","last_clean":"2024-08-17T17:54:29.145233+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:54:29.145233+0000","last_undegraded":"2024-08-17T17:54:29.145233+0000","last_fullsized":"2024-08-17T17:54:29.145233+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1192,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.032828+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:54:15.032828+0000","last_peered":"2024-08-17T17:54:15.032828+0000","last_clean":"2024-08-17T17:54:15.032828+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:54:15.032828+0000","last_undegraded":"2024-08-17T17:54:15.032828+0000","last_fullsized":"2024-08-17T17:54:15.032828+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":76,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.032252+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:54:15.032252+0000","last_peered":"2024-08-17T17:54:15.032252+0000","last_clean":"2024-08-17T17:54:15.032252+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:54:15.032252+0000","last_undegraded":"2024-08-17T17:54:15.032252+0000","last_fullsized":"2024-08-17T17:54:15.032252+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1208,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.032771+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:54:15.032771+0000","last_peered":"2024-08-17T17:54:15.032771+0000","last_clean":"2024-08-17T17:54:15.032771+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:54:15.032771+0000","last_undegraded":"2024-08-17T17:54:15.032771+0000","last_fullsized":"2024-08-17T17:54:15.032771+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1225,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.019018+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:54:15.019018+0000","last_peered":"2024-08-17T17:54:15.019018+0000","last_clean":"2024-08-17T17:54:15.019018+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:54:15.019018+0000","last_undegraded":"2024-08-17T17:54:15.019018+0000","last_fullsized":"2024-08-17T17:54:15.019018+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1832,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.143544+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:54:29.143544+0000","last_peered":"2024-08-17T17:54:29.143544+0000","last_clean":"2024-08-17T17:54:29.143544+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:54:29.143544+0000","last_undegraded":"2024-08-17T17:54:29.143544+0000","last_fullsized":"2024-08-17T17:54:29.143544+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1169,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.032414+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:54:15.032414+0000","last_peered":"2024-08-17T17:54:15.032414+0000","last_clean":"2024-08-17T17:54:15.032414+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:54:15.032414+0000","last_undegraded":"2024-08-17T17:54:15.032414+0000","last_fullsized":"2024-08-17T17:54:15.032414+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1885,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.164259+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:54:29.164259+0000","last_peered":"2024-08-17T17:54:29.164259+0000","last_clean":"2024-08-17T17:54:29.164259+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:54:29.164259+0000","last_undegraded":"2024-08-17T17:54:29.164259+0000","last_fullsized":"2024-08-17T17:54:29.164259+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1886,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.162187+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:54:29.162187+0000","last_peered":"2024-08-17T17:54:29.162187+0000","last_clean":"2024-08-17T17:54:29.162187+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:54:29.162187+0000","last_undegraded":"2024-08-17T17:54:29.162187+0000","last_fullsized":"2024-08-17T17:54:29.162187+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":582,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.032571+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:54:15.032571+0000","last_peered":"2024-08-17T17:54:15.032571+0000","last_clean":"2024-08-17T17:54:15.032571+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:54:15.032571+0000","last_undegraded":"2024-08-17T17:54:15.032571+0000","last_fullsized":"2024-08-17T17:54:15.032571+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1194,"reported_epoch":1224,"state":"active+clean","last_fresh":"2024-08-17T17:54:15.032562+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:54:15.032562+0000","last_peered":"2024-08-17T17:54:15.032562+0000","last_clean":"2024-08-17T17:54:15.032562+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:54:15.032562+0000","last_undegraded":"2024-08-17T17:54:15.032562+0000","last_fullsized":"2024-08-17T17:54:15.032562+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1832,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.143743+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:54:29.143743+0000","last_peered":"2024-08-17T17:54:29.143743+0000","last_clean":"2024-08-17T17:54:29.143743+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:54:29.143743+0000","last_undegraded":"2024-08-17T17:54:29.143743+0000","last_fullsized":"2024-08-17T17:54:29.143743+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":4228055064,"num_objects":64516,"num_object_clones":0,"num_object_copies":193548,"num_objects_missing_on_primary":3505,"num_objects_missing":3505,"num_objects_degraded":17178,"num_objects_misplaced":10949,"num_objects_unfound":0,"num_objects_dirty":64516,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":145094,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35091,"num_bytes_recovered":2121269248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13397929984,"data_stored":13397917768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108268,"ondisk_log_size":108268,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234404,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2078848,"kb_used_data":1777336,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91666304,"statfs":{"total":95995035648,"available":93866295296,"internally_reserved":0,"allocated":1819992064,"data_stored":1815082731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496039,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2340100,"kb_used_data":2038588,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91405052,"statfs":{"total":95995035648,"available":93598773248,"internally_reserved":0,"allocated":2087514112,"data_stored":2082601184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757677,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326291,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2515948,"kb_used_data":2307812,"kb_used_omap":57,"kb_used_meta":208070,"kb_avail":91229204,"statfs":{"total":95995035648,"available":93418704896,"internally_reserved":0,"allocated":2363199488,"data_stored":2358305335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":213064443},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405530,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2292660,"kb_used_data":2076140,"kb_used_omap":61,"kb_used_meta":216450,"kb_avail":91452492,"statfs":{"total":95995035648,"available":93647351808,"internally_reserved":0,"allocated":2125967360,"data_stored":2121070179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":221645752},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452065,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1730792,"kb_used_data":1546272,"kb_used_omap":54,"kb_used_meta":184457,"kb_avail":92014360,"statfs":{"total":95995035648,"available":94222704640,"internally_reserved":0,"allocated":1583382528,"data_stored":1578492336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":188884366},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474756,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1504760,"kb_used_data":1305648,"kb_used_omap":119,"kb_used_meta":198984,"kb_avail":92240392,"statfs":{"total":95995035648,"available":94454161408,"internally_reserved":0,"allocated":1336983552,"data_stored":1332078196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":203760350},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,3,1,1,3,4,7],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019312,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1321275392,"data_stored":1321271320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1569783808,"data_stored":1569783808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2108882944,"data_stored":2108882944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2346061824,"data_stored":2346057752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1039335424,"data_stored":1039335424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1136590848,"data_stored":1136590848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2072449024,"data_stored":2072444952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1803550720,"data_stored":1803550720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:54:39.467 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:39.489 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:39.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:39.591 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:39.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:39 smithi179 ceph-mon[93519]: pgmap v1827: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 148 op/s; 17178/191973 objects degraded (8.948%); 10949/191973 objects misplaced (5.703%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:54:39.612 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:39.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:39.713 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:39.735 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:39.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:39.836 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:39.860 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:39.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:39.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:54:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:54:39] "GET /metrics HTTP/1.1" 200 36813 "" "Prometheus/2.51.0" 2024-08-17T17:54:39.962 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:39.985 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:39.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:39.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:40.087 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:40.109 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:40.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:40.211 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:40.232 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:40.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:40.334 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:40.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:40.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:40.457 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:40.479 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:40.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:40.580 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:54:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:54:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:54:40.601 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:40.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:40.703 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:40.716 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:54:40.727 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:40.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:40.828 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.850 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:40.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:40.935 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:40 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17178/191973 objects degraded (8.948%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:40.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:40 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17178/191973 objects degraded (8.948%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:40.951 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:40.972 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:54:40.973 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:40.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:40.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:41.075 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:41.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:40 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17178/191973 objects degraded (8.948%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:41.100 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:41.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:41.202 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:41.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:41.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:41.326 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:41.347 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:41.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:41.449 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:41.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.473 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:41.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:41.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:41.595 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:41.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:41.697 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:41.719 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:41.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:41.821 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:41.842 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:41.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:41.943 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:41.964 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:41.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:41.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:42.001 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:41 smithi130 ceph-mon[105521]: pgmap v1828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 17178/191355 objects degraded (8.977%); 10866/191355 objects misplaced (5.678%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:54:42.001 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:41 smithi130 ceph-mon[110068]: pgmap v1828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 17178/191355 objects degraded (8.977%); 10866/191355 objects misplaced (5.678%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:54:42.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:42.089 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:42.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:42.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:41 smithi179 ceph-mon[93519]: pgmap v1828: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 17178/191355 objects degraded (8.977%); 10866/191355 objects misplaced (5.678%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:54:42.191 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:42.212 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:42.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:42.314 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:42.336 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:42.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:42.424 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:54:42.438 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:42.459 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:42.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:42.525 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:54:42.525 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:54:42.555 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:54:42.556 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:54:42.561 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:42.575 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:54:42.575 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:54:42.582 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:42.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:42.593 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:54:42.593 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:54:42.616 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:54:42.617 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:54:42.637 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:54:42.637 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:54:42.661 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:54:42.662 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:54:42.683 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:42.687 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:54:42.687 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:54:42.706 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:42.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:42.773 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T17:54:42.808 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:42.829 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:42.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:42.914 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:42 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1737029696' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:42.915 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:42 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:42.915 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:42.915 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:42 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3127388512' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:42.915 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:42 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1737029696' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:42.915 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:42 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:42.915 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:42.915 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:42 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3127388512' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:42.930 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:42 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1737029696' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:42.930 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:42 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:54:42.930 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:42.930 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:42 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3127388512' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:42.932 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:42.953 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:54:42.953 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:54:42.953 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T17:54:42.953 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:54:42.955 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:42.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:42.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:43.057 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:43.077 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:43.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:43.179 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:43.201 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:43.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:43.303 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:43.323 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:43.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:43.425 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:43.446 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:43.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:43.548 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:43.569 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:43.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:43.671 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:43.673 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:54:43.694 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:43.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:43.796 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:43.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:43.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:43.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:43.935 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:54:43.945 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:43.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:43.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:44.047 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:44.058 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:43 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:54:44.058 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:43 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:54:44.058 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:43 smithi179 ceph-mon[93519]: osdmap e1226: 8 total, 8 up, 6 in 2024-08-17T17:54:44.058 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:43 smithi179 ceph-mon[93519]: pgmap v1830: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 150 op/s; 17098/191127 objects degraded (8.946%); 10830/191127 objects misplaced (5.666%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:44.068 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:44.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:43 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:54:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:43 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:54:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:43 smithi130 ceph-mon[105521]: osdmap e1226: 8 total, 8 up, 6 in 2024-08-17T17:54:44.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:43 smithi130 ceph-mon[105521]: pgmap v1830: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 150 op/s; 17098/191127 objects degraded (8.946%); 10830/191127 objects misplaced (5.666%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:44.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:43 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:54:44.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:43 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:54:44.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:43 smithi130 ceph-mon[110068]: osdmap e1226: 8 total, 8 up, 6 in 2024-08-17T17:54:44.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:43 smithi130 ceph-mon[110068]: pgmap v1830: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 150 op/s; 17098/191127 objects degraded (8.946%); 10830/191127 objects misplaced (5.666%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:44.170 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:44.191 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:44.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:44.293 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:44.314 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:44.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:44.416 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:44.439 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:44.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:44.541 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:44.562 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:44.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:44.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:44.687 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:44.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:44.789 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:44.812 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:44.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:44.914 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:44.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:44.936 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:44.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:45.037 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:45.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.064 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:45.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:45.165 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:45.188 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:45.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:45.243 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:45 smithi179 ceph-mon[93519]: pgmap v1831: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 17098/189663 objects degraded (9.015%); 10738/189663 objects misplaced (5.662%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:54:45.286 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:45 smithi130 ceph-mon[105521]: pgmap v1831: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 17098/189663 objects degraded (9.015%); 10738/189663 objects misplaced (5.662%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:54:45.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:45 smithi130 ceph-mon[110068]: pgmap v1831: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 17098/189663 objects degraded (9.015%); 10738/189663 objects misplaced (5.662%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:54:45.290 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:45.315 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:45.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:45.418 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:45.439 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:45.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:45.541 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:45.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.570 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:45.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:45.671 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:45.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:45.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:45.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:54:45.770 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:45.795 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:45.823 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:45.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:45.875 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:54:45.875 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:54:45.875 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T17:54:45.875 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T17:54:45.875 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:54:45.925 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:45.947 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:45.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:45.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:46.049 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:46.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.072 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:46.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:46.173 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:46.196 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:46.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:46.298 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:46.320 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:46.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:46.353 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17098/189663 objects degraded (9.015%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:46.354 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17098/189663 objects degraded (9.015%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:46.422 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:46.446 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:46.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:46.499 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17098/189663 objects degraded (9.015%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:46.548 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:46.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.571 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:46.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:46.631 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:54:46.672 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:46.694 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:46.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:46.796 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:46.819 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:46.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:46.902 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:54:46.921 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:46.945 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:46.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:46.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:47.005 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:54:47.047 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:47.072 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:47.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:47.174 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:47.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.211 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:47.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:47.281 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:47 smithi179 ceph-mon[93519]: pgmap v1832: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 17002/189384 objects degraded (8.978%); 10693/189384 objects misplaced (5.646%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:47.312 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:47.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:47.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:47.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:47 smithi130 ceph-mon[105521]: pgmap v1832: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 17002/189384 objects degraded (8.978%); 10693/189384 objects misplaced (5.646%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:47.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:47 smithi130 ceph-mon[110068]: pgmap v1832: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 17002/189384 objects degraded (8.978%); 10693/189384 objects misplaced (5.646%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:47.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:47.459 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:47.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:47.561 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:47.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.583 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:47.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:47.684 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:47.708 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:47.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:47.810 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:47.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:47.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:47.933 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:47.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:47.955 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:47.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:48.056 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:48.078 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:48.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:48.098 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:48.180 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:48.208 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:48.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:48.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:48 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:48.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:48 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:48.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:48 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:54:48.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:48 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:54:48.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:48 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:48.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:48 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:48.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:48 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:48.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:48 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:54:48.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:48 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:54:48.307 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:48 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:48.310 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:48.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.338 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:48.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:48.439 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:48.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:48.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:48.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:48 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:48.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:48 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:48.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:48 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:54:48.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:48 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:54:48.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:48 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:54:48.562 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:48.586 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:48.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:48.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:48.712 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:48.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:48.814 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:48.835 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:48.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:48.912 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:48.937 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:48.963 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:48.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:48.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:48.980 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:54:48.981 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T17:54:48.981 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T17:54:48.981 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T17:54:48.981 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.981 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83241 45.1 45:82a6944f:::benchmark_data_smithi130_172551_object83240:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:54:48.981 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.647075+0000", 2024-08-17T17:54:48.981 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 370.32785323500002, 2024-08-17T17:54:48.981 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4851373099999998, 2024-08-17T17:54:48.981 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:48.981 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:48.981 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:48.982 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:48.982 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:48.982 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83241 2024-08-17T17:54:48.982 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.982 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:48.982 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.982 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:48.982 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647075+0000", 2024-08-17T17:54:48.982 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:48.982 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.982 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.982 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:48.983 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647075+0000", 2024-08-17T17:54:48.983 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:48.983 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.983 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.983 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:48.983 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647073+0000", 2024-08-17T17:54:48.983 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:54:48.983 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.983 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.983 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:48.983 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647109+0000", 2024-08-17T17:54:48.983 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6040000000000001e-05 2024-08-17T17:54:48.983 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.983 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.984 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:48.984 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647128+0000", 2024-08-17T17:54:48.984 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.9045000000000001e-05 2024-08-17T17:54:48.984 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.984 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.984 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:48.984 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647136+0000", 2024-08-17T17:54:48.984 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9589999999999995e-06 2024-08-17T17:54:48.984 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.984 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.984 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:48.984 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647230+0000", 2024-08-17T17:54:48.984 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4471999999999999e-05 2024-08-17T17:54:48.985 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.985 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.985 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:54:48.985 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647244+0000", 2024-08-17T17:54:48.985 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4146e-05 2024-08-17T17:54:48.985 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.985 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.985 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:48.985 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636452+0000", 2024-08-17T17:54:48.985 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98920747099999995 2024-08-17T17:54:48.985 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.985 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.985 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:54:48.986 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636467+0000", 2024-08-17T17:54:48.986 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5031999999999999e-05 2024-08-17T17:54:48.986 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.986 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.986 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:48.986 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.102636+0000", 2024-08-17T17:54:48.986 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4661687780000001 2024-08-17T17:54:48.986 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.986 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.986 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:48.986 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.102811+0000", 2024-08-17T17:54:48.986 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017539199999999999 2024-08-17T17:54:48.986 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.987 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.987 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:48.987 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.102997+0000", 2024-08-17T17:54:48.987 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000185706 2024-08-17T17:54:48.987 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.987 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.987 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:48.987 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.109961+0000", 2024-08-17T17:54:48.987 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0069645180000000003 2024-08-17T17:54:48.987 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.987 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.987 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:48.987 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.109972+0000", 2024-08-17T17:54:48.988 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.093e-05 2024-08-17T17:54:48.988 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.988 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.988 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:48.988 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.129457+0000", 2024-08-17T17:54:48.988 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.019484379999999999 2024-08-17T17:54:48.988 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.988 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.988 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:48.988 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.129554+0000", 2024-08-17T17:54:48.988 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7465000000000003e-05 2024-08-17T17:54:48.988 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.989 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.989 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:48.989 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.132212+0000", 2024-08-17T17:54:48.989 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002657867 2024-08-17T17:54:48.989 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:48.989 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:48.989 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:48.989 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.989 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.990 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83244 45.1 45:81aa3e3d:::benchmark_data_smithi130_172551_object83243:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:54:48.990 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.658809+0000", 2024-08-17T17:54:48.990 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 370.31611900000001, 2024-08-17T17:54:48.990 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4758367149999998, 2024-08-17T17:54:48.990 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:48.990 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:48.990 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:48.990 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:48.990 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:48.991 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83244 2024-08-17T17:54:48.991 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.991 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:48.991 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.991 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:48.991 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658809+0000", 2024-08-17T17:54:48.991 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:48.991 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.991 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.991 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:48.991 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658809+0000", 2024-08-17T17:54:48.991 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:48.992 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.992 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.992 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:48.992 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658807+0000", 2024-08-17T17:54:48.992 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:54:48.992 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.992 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.992 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:48.992 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658846+0000", 2024-08-17T17:54:48.992 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8951e-05 2024-08-17T17:54:48.992 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.993 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.993 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:48.993 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658857+0000", 2024-08-17T17:54:48.993 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0569e-05 2024-08-17T17:54:48.993 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.993 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.993 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:48.993 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658861+0000", 2024-08-17T17:54:48.993 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0130000000000004e-06 2024-08-17T17:54:48.993 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.994 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.994 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:48.994 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658909+0000", 2024-08-17T17:54:48.994 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7958999999999999e-05 2024-08-17T17:54:48.994 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.994 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.994 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:54:48.994 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658922+0000", 2024-08-17T17:54:48.994 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3519e-05 2024-08-17T17:54:48.994 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.994 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.994 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:48.994 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636515+0000", 2024-08-17T17:54:48.995 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97759250399999997 2024-08-17T17:54:48.995 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.995 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.995 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:54:48.995 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636526+0000", 2024-08-17T17:54:48.995 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1593000000000001e-05 2024-08-17T17:54:48.995 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.995 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.995 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:48.995 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.107864+0000", 2024-08-17T17:54:48.995 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4713374130000001 2024-08-17T17:54:48.995 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.995 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.996 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:48.996 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.107963+0000", 2024-08-17T17:54:48.996 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9687999999999998e-05 2024-08-17T17:54:48.996 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.996 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.996 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:48.996 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.108070+0000", 2024-08-17T17:54:48.996 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010689200000000001 2024-08-17T17:54:48.996 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.996 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.996 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:48.996 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112059+0000", 2024-08-17T17:54:48.996 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0039884810000000003 2024-08-17T17:54:48.997 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.997 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.997 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:48.997 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112120+0000", 2024-08-17T17:54:48.997 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1134000000000001e-05 2024-08-17T17:54:48.997 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.997 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.997 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:48.997 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.132267+0000", 2024-08-17T17:54:48.997 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.020146641999999999 2024-08-17T17:54:48.997 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.997 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.997 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:48.997 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.132303+0000", 2024-08-17T17:54:48.998 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6529000000000002e-05 2024-08-17T17:54:48.998 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.998 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.998 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:48.998 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.134646+0000", 2024-08-17T17:54:48.998 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0023425199999999998 2024-08-17T17:54:48.998 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:48.998 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:48.998 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:48.998 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.998 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.998 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83246 45.1 45:8e7e3f88:::benchmark_data_smithi130_172551_object83245:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:54:48.999 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.659302+0000", 2024-08-17T17:54:48.999 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 370.31562595100002, 2024-08-17T17:54:48.999 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.500952952, 2024-08-17T17:54:48.999 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:48.999 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:48.999 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:48.999 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:48.999 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:48.999 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83246 2024-08-17T17:54:48.999 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:48.999 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:48.999 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:48.999 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:48.999 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659302+0000", 2024-08-17T17:54:49.000 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.000 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.000 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.000 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.000 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659302+0000", 2024-08-17T17:54:49.000 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.000 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.000 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.000 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.000 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659301+0000", 2024-08-17T17:54:49.000 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:54:49.000 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.000 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.000 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.001 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659364+0000", 2024-08-17T17:54:49.001 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2366000000000006e-05 2024-08-17T17:54:49.001 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.001 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.001 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.001 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659374+0000", 2024-08-17T17:54:49.001 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0380999999999999e-05 2024-08-17T17:54:49.001 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.001 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.001 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.001 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659379+0000", 2024-08-17T17:54:49.001 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5950000000000001e-06 2024-08-17T17:54:49.001 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.002 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.002 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.002 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659426+0000", 2024-08-17T17:54:49.002 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7775000000000003e-05 2024-08-17T17:54:49.002 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.002 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.002 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:54:49.002 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659438+0000", 2024-08-17T17:54:49.002 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1267000000000001e-05 2024-08-17T17:54:49.002 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.002 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.002 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.003 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636564+0000", 2024-08-17T17:54:49.005 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97712642100000002 2024-08-17T17:54:49.005 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.005 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.005 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:54:49.005 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636570+0000", 2024-08-17T17:54:49.006 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9120000000000003e-06 2024-08-17T17:54:49.006 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.006 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.006 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.006 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.109992+0000", 2024-08-17T17:54:49.006 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.473421493 2024-08-17T17:54:49.006 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.006 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.006 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.006 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.110115+0000", 2024-08-17T17:54:49.006 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000122934 2024-08-17T17:54:49.006 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.007 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.007 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.007 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.110209+0000", 2024-08-17T17:54:49.007 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4676e-05 2024-08-17T17:54:49.007 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.007 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.007 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.007 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112110+0000", 2024-08-17T17:54:49.007 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019009809999999999 2024-08-17T17:54:49.007 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.007 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.008 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.008 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.116034+0000", 2024-08-17T17:54:49.008 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0039233669999999997 2024-08-17T17:54:49.008 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.008 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.008 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.008 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.157325+0000", 2024-08-17T17:54:49.008 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.041291094 2024-08-17T17:54:49.008 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.008 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.008 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.008 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.157364+0000", 2024-08-17T17:54:49.008 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9023999999999998e-05 2024-08-17T17:54:49.009 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.009 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.009 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.009 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.160255+0000", 2024-08-17T17:54:49.009 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028912460000000001 2024-08-17T17:54:49.009 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.009 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.009 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.009 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.009 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.009 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83252 45.1 45:87ce6d24:::benchmark_data_smithi130_172551_object83251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:54:49.009 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.670101+0000", 2024-08-17T17:54:49.009 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 370.30482690100001, 2024-08-17T17:54:49.009 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.493777863, 2024-08-17T17:54:49.010 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.010 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.010 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.010 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.010 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.010 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83252 2024-08-17T17:54:49.010 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.010 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.010 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.010 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.010 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670101+0000", 2024-08-17T17:54:49.010 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.010 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.011 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.011 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.011 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670101+0000", 2024-08-17T17:54:49.011 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.011 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.011 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.011 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.011 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670100+0000", 2024-08-17T17:54:49.011 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:54:49.011 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.011 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.011 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.011 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670158+0000", 2024-08-17T17:54:49.011 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8179e-05 2024-08-17T17:54:49.012 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.012 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.012 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.012 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670170+0000", 2024-08-17T17:54:49.012 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2e-05 2024-08-17T17:54:49.012 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.012 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.012 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.012 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670175+0000", 2024-08-17T17:54:49.012 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3290000000000001e-06 2024-08-17T17:54:49.012 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.012 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.012 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.013 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670225+0000", 2024-08-17T17:54:49.013 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0126000000000001e-05 2024-08-17T17:54:49.013 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.013 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.013 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:54:49.013 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670237+0000", 2024-08-17T17:54:49.013 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1832e-05 2024-08-17T17:54:49.013 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.013 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.013 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.013 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636594+0000", 2024-08-17T17:54:49.013 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.96635708200000003 2024-08-17T17:54:49.013 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.014 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.014 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:54:49.014 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636603+0000", 2024-08-17T17:54:49.014 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7139999999999999e-06 2024-08-17T17:54:49.014 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.014 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.014 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.014 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112146+0000", 2024-08-17T17:54:49.014 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.475543396 2024-08-17T17:54:49.014 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.014 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.014 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.014 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112248+0000", 2024-08-17T17:54:49.015 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0001016 2024-08-17T17:54:49.015 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.015 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.015 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.015 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112350+0000", 2024-08-17T17:54:49.015 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000102067 2024-08-17T17:54:49.015 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.015 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.015 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.015 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.116088+0000", 2024-08-17T17:54:49.015 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037383759999999999 2024-08-17T17:54:49.015 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.015 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.016 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.016 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.117798+0000", 2024-08-17T17:54:49.016 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0017097239999999999 2024-08-17T17:54:49.016 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.016 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.016 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.016 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.160341+0000", 2024-08-17T17:54:49.016 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.042542888000000001 2024-08-17T17:54:49.016 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.016 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.016 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.016 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.160400+0000", 2024-08-17T17:54:49.016 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9277000000000002e-05 2024-08-17T17:54:49.017 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.017 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.017 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.017 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.163879+0000", 2024-08-17T17:54:49.017 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0034785689999999999 2024-08-17T17:54:49.017 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.017 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.017 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.017 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.017 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.017 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:88238 45.1 45:8de64967:::benchmark_data_smithi130_172551_object88237:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1180)", 2024-08-17T17:54:49.017 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:49.366656+0000", 2024-08-17T17:54:49.017 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 359.60827241200002, 2024-08-17T17:54:49.017 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.079152609999999998, 2024-08-17T17:54:49.018 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.018 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.018 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.018 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.018 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.018 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 88238 2024-08-17T17:54:49.018 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.018 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.018 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.018 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.018 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366656+0000", 2024-08-17T17:54:49.018 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.018 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.019 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.019 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.019 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366656+0000", 2024-08-17T17:54:49.019 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.019 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.019 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.019 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.019 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366655+0000", 2024-08-17T17:54:49.019 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:54:49.019 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.019 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.019 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.019 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366709+0000", 2024-08-17T17:54:49.019 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.4617999999999998e-05 2024-08-17T17:54:49.020 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.020 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.020 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.020 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366720+0000", 2024-08-17T17:54:49.020 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.009e-05 2024-08-17T17:54:49.020 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.020 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.020 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.020 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366723+0000", 2024-08-17T17:54:49.020 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8800000000000001e-06 2024-08-17T17:54:49.020 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.020 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.020 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.021 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.380121+0000", 2024-08-17T17:54:49.021 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013397166 2024-08-17T17:54:49.021 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.021 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.021 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.021 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.380213+0000", 2024-08-17T17:54:49.021 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2111e-05 2024-08-17T17:54:49.021 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.021 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.021 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.021 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.380314+0000", 2024-08-17T17:54:49.021 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000101746 2024-08-17T17:54:49.021 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.021 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.022 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.022 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.383932+0000", 2024-08-17T17:54:49.022 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00361722 2024-08-17T17:54:49.022 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.022 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.022 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.022 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.383975+0000", 2024-08-17T17:54:49.022 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3120000000000001e-05 2024-08-17T17:54:49.022 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.022 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.022 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.022 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.443093+0000", 2024-08-17T17:54:49.023 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.059118638000000001 2024-08-17T17:54:49.023 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.023 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.023 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.023 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.443138+0000", 2024-08-17T17:54:49.023 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4335999999999997e-05 2024-08-17T17:54:49.023 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.023 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.023 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.023 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.445808+0000", 2024-08-17T17:54:49.024 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002670389 2024-08-17T17:54:49.024 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.024 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.024 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.024 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.024 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.024 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:88246 45.1 45:8c252ee5:::benchmark_data_smithi130_172551_object88245:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1180)", 2024-08-17T17:54:49.024 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:49.370303+0000", 2024-08-17T17:54:49.024 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 359.604624726, 2024-08-17T17:54:49.024 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.078072420000000003, 2024-08-17T17:54:49.024 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.024 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.025 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.025 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.025 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.025 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 88246 2024-08-17T17:54:49.025 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.025 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.025 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.025 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.025 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370303+0000", 2024-08-17T17:54:49.025 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.025 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.026 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.026 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.026 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370303+0000", 2024-08-17T17:54:49.026 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.026 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.026 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.026 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.026 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370302+0000", 2024-08-17T17:54:49.026 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:54:49.026 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.026 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.026 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.026 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370372+0000", 2024-08-17T17:54:49.027 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0137999999999998e-05 2024-08-17T17:54:49.027 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.027 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.027 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.027 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370386+0000", 2024-08-17T17:54:49.027 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4062e-05 2024-08-17T17:54:49.027 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.027 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.027 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.027 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370392+0000", 2024-08-17T17:54:49.027 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8690000000000001e-06 2024-08-17T17:54:49.027 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.027 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.027 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.028 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.381694+0000", 2024-08-17T17:54:49.028 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.011302395999999999 2024-08-17T17:54:49.028 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.028 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.028 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.029 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.381800+0000", 2024-08-17T17:54:49.029 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105775 2024-08-17T17:54:49.029 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.030 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.030 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.030 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.381899+0000", 2024-08-17T17:54:49.030 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.8529999999999993e-05 2024-08-17T17:54:49.030 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.030 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.030 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.030 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.383968+0000", 2024-08-17T17:54:49.030 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020689520000000002 2024-08-17T17:54:49.030 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.030 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.030 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.030 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.384010+0000", 2024-08-17T17:54:49.031 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2639999999999998e-05 2024-08-17T17:54:49.031 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.031 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.031 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.031 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.445861+0000", 2024-08-17T17:54:49.031 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.061850694999999997 2024-08-17T17:54:49.031 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.031 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.031 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.031 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.445900+0000", 2024-08-17T17:54:49.031 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8890999999999997e-05 2024-08-17T17:54:49.031 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.031 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.031 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.032 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.448376+0000", 2024-08-17T17:54:49.032 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024758029999999999 2024-08-17T17:54:49.032 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.032 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.032 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.032 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.032 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.032 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:88852 45.1 45:8bcd85bb:::benchmark_data_smithi130_172551_object88851:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:54:49.032 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:50.423711+0000", 2024-08-17T17:54:49.032 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 358.55121729000001, 2024-08-17T17:54:49.032 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.067301394, 2024-08-17T17:54:49.032 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.032 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.033 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.033 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.033 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.033 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 88852 2024-08-17T17:54:49.033 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.033 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.033 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.033 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.033 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423711+0000", 2024-08-17T17:54:49.033 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.033 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.033 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.033 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.033 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423711+0000", 2024-08-17T17:54:49.034 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.034 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.034 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.034 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.034 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423703+0000", 2024-08-17T17:54:49.034 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999928 2024-08-17T17:54:49.034 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.034 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.034 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.034 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423743+0000", 2024-08-17T17:54:49.034 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9957999999999999e-05 2024-08-17T17:54:49.034 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.034 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.035 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.035 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423758+0000", 2024-08-17T17:54:49.035 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4294000000000001e-05 2024-08-17T17:54:49.035 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.035 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.035 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.035 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423764+0000", 2024-08-17T17:54:49.035 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2419999999999997e-06 2024-08-17T17:54:49.036 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.036 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.036 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.036 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423809+0000", 2024-08-17T17:54:49.036 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5754000000000003e-05 2024-08-17T17:54:49.036 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.037 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.037 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.037 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423915+0000", 2024-08-17T17:54:49.037 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105951 2024-08-17T17:54:49.037 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.037 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.037 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.037 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.424027+0000", 2024-08-17T17:54:49.037 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000111777 2024-08-17T17:54:49.037 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.037 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.038 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.038 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.430304+0000", 2024-08-17T17:54:49.038 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0062763059999999997 2024-08-17T17:54:49.038 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.038 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.039 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.039 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.430329+0000", 2024-08-17T17:54:49.039 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5187000000000001e-05 2024-08-17T17:54:49.039 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.039 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.039 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.039 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.488439+0000", 2024-08-17T17:54:49.039 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.058110436000000001 2024-08-17T17:54:49.039 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.039 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.040 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.040 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.488489+0000", 2024-08-17T17:54:49.040 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0201000000000003e-05 2024-08-17T17:54:49.040 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.040 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.040 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.040 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.491012+0000", 2024-08-17T17:54:49.040 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025226620000000002 2024-08-17T17:54:49.040 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.040 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.041 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.041 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.041 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.041 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:99564 45.1 45:877a8ac3:::benchmark_data_smithi130_172551_object99563:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1182)", 2024-08-17T17:54:49.041 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:08.821429+0000", 2024-08-17T17:54:49.041 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 340.15349936500002, 2024-08-17T17:54:49.041 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.069042615000000002, 2024-08-17T17:54:49.041 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.041 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.041 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.041 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.041 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.042 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 99564 2024-08-17T17:54:49.042 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.042 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.042 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.042 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.042 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821429+0000", 2024-08-17T17:54:49.042 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.043 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.043 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.043 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.043 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821429+0000", 2024-08-17T17:54:49.043 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.043 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.044 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.044 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.044 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821427+0000", 2024-08-17T17:54:49.044 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:54:49.044 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.044 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.045 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.045 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821522+0000", 2024-08-17T17:54:49.045 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5486e-05 2024-08-17T17:54:49.045 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.045 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.045 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.046 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821543+0000", 2024-08-17T17:54:49.046 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.1069000000000001e-05 2024-08-17T17:54:49.046 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.046 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.046 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.046 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821552+0000", 2024-08-17T17:54:49.047 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.4780000000000008e-06 2024-08-17T17:54:49.047 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.047 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.047 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.047 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821602+0000", 2024-08-17T17:54:49.047 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0368999999999998e-05 2024-08-17T17:54:49.048 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.048 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.048 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.048 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821791+0000", 2024-08-17T17:54:49.048 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00018891099999999999 2024-08-17T17:54:49.048 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.049 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.049 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.049 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821980+0000", 2024-08-17T17:54:49.049 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00018913400000000001 2024-08-17T17:54:49.049 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.049 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.050 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.050 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827561+0000", 2024-08-17T17:54:49.050 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0055811910000000001 2024-08-17T17:54:49.050 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.050 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.051 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.051 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830393+0000", 2024-08-17T17:54:49.051 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028316940000000001 2024-08-17T17:54:49.051 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.051 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.051 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.052 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.887201+0000", 2024-08-17T17:54:49.052 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.056807923000000003 2024-08-17T17:54:49.052 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.052 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.052 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.053 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.887243+0000", 2024-08-17T17:54:49.053 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.163e-05 2024-08-17T17:54:49.053 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.053 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.053 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.053 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.890471+0000", 2024-08-17T17:54:49.053 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003228608 2024-08-17T17:54:49.054 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.054 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.054 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.054 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.054 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.055 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:99567 45.1 45:86b126dd:::benchmark_data_smithi130_172551_object99566:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1182)", 2024-08-17T17:54:49.055 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:08.821921+0000", 2024-08-17T17:54:49.055 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 340.15300665400002, 2024-08-17T17:54:49.055 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.071874980000000005, 2024-08-17T17:54:49.055 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.055 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.056 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.056 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.056 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.056 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 99567 2024-08-17T17:54:49.056 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.056 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.056 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.056 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.057 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821921+0000", 2024-08-17T17:54:49.057 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.057 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.057 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.057 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.057 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821921+0000", 2024-08-17T17:54:49.057 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.057 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.057 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.057 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.057 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821920+0000", 2024-08-17T17:54:49.057 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:54:49.057 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.057 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.058 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.058 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821993+0000", 2024-08-17T17:54:49.058 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.2989000000000002e-05 2024-08-17T17:54:49.058 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.058 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.058 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.058 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.822006+0000", 2024-08-17T17:54:49.058 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2729e-05 2024-08-17T17:54:49.058 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.058 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.058 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.058 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.822012+0000", 2024-08-17T17:54:49.058 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.003e-06 2024-08-17T17:54:49.059 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.059 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.059 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.059 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.824701+0000", 2024-08-17T17:54:49.059 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0026888110000000002 2024-08-17T17:54:49.059 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.059 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.059 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.059 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.824892+0000", 2024-08-17T17:54:49.059 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019161699999999999 2024-08-17T17:54:49.059 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.059 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.059 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.060 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.825092+0000", 2024-08-17T17:54:49.060 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019969200000000001 2024-08-17T17:54:49.060 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.060 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.060 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.060 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830339+0000", 2024-08-17T17:54:49.060 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00524714 2024-08-17T17:54:49.060 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.060 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.060 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.060 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830563+0000", 2024-08-17T17:54:49.060 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00022378 2024-08-17T17:54:49.060 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.061 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.061 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.061 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.890526+0000", 2024-08-17T17:54:49.061 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.059962847 2024-08-17T17:54:49.061 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.061 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.061 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.061 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.890569+0000", 2024-08-17T17:54:49.061 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2808000000000001e-05 2024-08-17T17:54:49.061 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.061 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.061 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.061 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.893796+0000", 2024-08-17T17:54:49.062 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0032274970000000002 2024-08-17T17:54:49.062 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.062 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.062 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.062 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.062 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.062 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:99571 45.1 45:89cfe7f4:::benchmark_data_smithi130_172551_object99570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1182)", 2024-08-17T17:54:49.062 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:08.827641+0000", 2024-08-17T17:54:49.062 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 340.14728654300001, 2024-08-17T17:54:49.062 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.069045208999999996, 2024-08-17T17:54:49.062 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.062 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.062 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.063 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.063 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.063 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 99571 2024-08-17T17:54:49.063 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.063 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.063 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.063 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.063 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827641+0000", 2024-08-17T17:54:49.063 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.063 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.063 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.063 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.063 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827641+0000", 2024-08-17T17:54:49.063 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.064 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.064 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.064 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.064 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827640+0000", 2024-08-17T17:54:49.064 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:54:49.064 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.064 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.064 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.064 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827677+0000", 2024-08-17T17:54:49.064 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7431000000000001e-05 2024-08-17T17:54:49.064 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.064 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.064 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.064 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827689+0000", 2024-08-17T17:54:49.065 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1399999999999999e-05 2024-08-17T17:54:49.065 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.065 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.065 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.065 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827694+0000", 2024-08-17T17:54:49.065 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0180000000000001e-06 2024-08-17T17:54:49.065 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.065 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.065 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.065 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827736+0000", 2024-08-17T17:54:49.065 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1878000000000001e-05 2024-08-17T17:54:49.065 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.065 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.065 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.066 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827845+0000", 2024-08-17T17:54:49.066 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000109201 2024-08-17T17:54:49.066 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.066 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.066 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.066 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827950+0000", 2024-08-17T17:54:49.066 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105064 2024-08-17T17:54:49.066 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.066 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.066 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.066 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830518+0000", 2024-08-17T17:54:49.066 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025678049999999998 2024-08-17T17:54:49.066 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.067 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.067 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.067 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830749+0000", 2024-08-17T17:54:49.067 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00023170999999999999 2024-08-17T17:54:49.067 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.067 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.067 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.067 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.893864+0000", 2024-08-17T17:54:49.067 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.063114049000000005 2024-08-17T17:54:49.067 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.067 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.067 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.067 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.893911+0000", 2024-08-17T17:54:49.067 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7811e-05 2024-08-17T17:54:49.068 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.068 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.068 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.068 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.896687+0000", 2024-08-17T17:54:49.068 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002775226 2024-08-17T17:54:49.068 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.068 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.068 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.068 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.068 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.068 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:107884 45.1 45:89ca3498:::benchmark_data_smithi130_172551_object3077:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:54:49.068 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:25.288810+0000", 2024-08-17T17:54:49.068 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 323.68611812699999, 2024-08-17T17:54:49.069 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.069553105000000004, 2024-08-17T17:54:49.069 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.069 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.069 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.069 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.069 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.069 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 107884 2024-08-17T17:54:49.069 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.069 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.069 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.069 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.069 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.288810+0000", 2024-08-17T17:54:49.069 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.069 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.070 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.070 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.070 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.288810+0000", 2024-08-17T17:54:49.070 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.070 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.070 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.070 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.070 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.288808+0000", 2024-08-17T17:54:49.070 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:54:49.070 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.070 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.070 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.070 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.288817+0000", 2024-08-17T17:54:49.071 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0529999999999996e-06 2024-08-17T17:54:49.071 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.071 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.071 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.071 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.288830+0000", 2024-08-17T17:54:49.071 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2822e-05 2024-08-17T17:54:49.071 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.071 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.071 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.071 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.288836+0000", 2024-08-17T17:54:49.071 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8490000000000002e-06 2024-08-17T17:54:49.071 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.071 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.071 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.072 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.291429+0000", 2024-08-17T17:54:49.072 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025927120000000001 2024-08-17T17:54:49.072 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.072 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.072 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.072 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.291599+0000", 2024-08-17T17:54:49.072 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017012700000000001 2024-08-17T17:54:49.072 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.072 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.072 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.072 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.291763+0000", 2024-08-17T17:54:49.072 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016383200000000001 2024-08-17T17:54:49.072 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.072 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.073 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.073 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.294832+0000", 2024-08-17T17:54:49.073 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0030692250000000001 2024-08-17T17:54:49.073 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.073 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.073 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.073 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.295407+0000", 2024-08-17T17:54:49.073 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00057506500000000004 2024-08-17T17:54:49.073 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.073 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.073 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.073 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.354379+0000", 2024-08-17T17:54:49.073 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.058971509999999998 2024-08-17T17:54:49.073 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.074 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.074 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.074 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.354445+0000", 2024-08-17T17:54:49.074 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6694999999999998e-05 2024-08-17T17:54:49.074 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.074 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.074 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.074 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.358363+0000", 2024-08-17T17:54:49.074 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0039176549999999999 2024-08-17T17:54:49.074 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.074 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.074 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.074 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.075 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.075 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:111958 45.1 45:82b6ab59:::benchmark_data_smithi130_172551_object7151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:54:49.075 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:34.695989+0000", 2024-08-17T17:54:49.075 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 314.27893905399998, 2024-08-17T17:54:49.075 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.124334234, 2024-08-17T17:54:49.075 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.075 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.075 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.075 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.075 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.075 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 111958 2024-08-17T17:54:49.075 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.075 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.075 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.076 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.076 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.695989+0000", 2024-08-17T17:54:49.076 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.076 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.076 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.076 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.076 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.695989+0000", 2024-08-17T17:54:49.076 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.076 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.076 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.076 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.076 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.695987+0000", 2024-08-17T17:54:49.076 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:54:49.076 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.077 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.077 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.077 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.695994+0000", 2024-08-17T17:54:49.077 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3509999999999996e-06 2024-08-17T17:54:49.077 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.077 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.077 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.077 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696005+0000", 2024-08-17T17:54:49.077 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0891e-05 2024-08-17T17:54:49.077 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.077 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.077 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.077 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696010+0000", 2024-08-17T17:54:49.077 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8949999999999997e-06 2024-08-17T17:54:49.078 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.078 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.078 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.078 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.698750+0000", 2024-08-17T17:54:49.078 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027400459999999999 2024-08-17T17:54:49.078 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.078 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.078 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.078 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.698933+0000", 2024-08-17T17:54:49.078 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00018272300000000001 2024-08-17T17:54:49.078 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.078 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.078 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.079 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.699099+0000", 2024-08-17T17:54:49.079 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000166587 2024-08-17T17:54:49.079 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.079 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.079 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.079 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.705849+0000", 2024-08-17T17:54:49.079 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0067498280000000003 2024-08-17T17:54:49.079 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.079 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.079 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.079 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.705896+0000", 2024-08-17T17:54:49.079 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7012000000000003e-05 2024-08-17T17:54:49.079 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.079 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.080 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.080 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.817997+0000", 2024-08-17T17:54:49.080 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.112101241 2024-08-17T17:54:49.080 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.080 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.080 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.080 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.818038+0000", 2024-08-17T17:54:49.080 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0970000000000002e-05 2024-08-17T17:54:49.080 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.080 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.080 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.080 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.820323+0000", 2024-08-17T17:54:49.080 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002284669 2024-08-17T17:54:49.081 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.081 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.081 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.081 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.081 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.081 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:111959 45.1 45:85d2b159:::benchmark_data_smithi130_172551_object7152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:54:49.081 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:34.696016+0000", 2024-08-17T17:54:49.081 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 314.27891197100001, 2024-08-17T17:54:49.081 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.12563115999999999, 2024-08-17T17:54:49.081 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.081 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.081 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.081 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.081 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.082 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 111959 2024-08-17T17:54:49.082 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.082 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.082 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.082 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.082 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696016+0000", 2024-08-17T17:54:49.082 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.082 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.082 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.082 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.082 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696016+0000", 2024-08-17T17:54:49.082 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.082 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.083 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.083 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.083 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696016+0000", 2024-08-17T17:54:49.083 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:54:49.083 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.083 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.083 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.083 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696018+0000", 2024-08-17T17:54:49.083 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3719999999999999e-06 2024-08-17T17:54:49.083 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.083 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.083 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.083 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696024+0000", 2024-08-17T17:54:49.084 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0569999999999998e-06 2024-08-17T17:54:49.084 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.084 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.084 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.084 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696028+0000", 2024-08-17T17:54:49.084 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.002e-06 2024-08-17T17:54:49.084 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.084 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.084 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.084 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.702259+0000", 2024-08-17T17:54:49.084 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0062313639999999997 2024-08-17T17:54:49.084 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.084 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.084 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.085 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.702447+0000", 2024-08-17T17:54:49.085 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00018781000000000001 2024-08-17T17:54:49.085 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.085 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.085 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.085 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.702577+0000", 2024-08-17T17:54:49.085 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000129954 2024-08-17T17:54:49.085 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.085 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.085 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.085 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.705926+0000", 2024-08-17T17:54:49.085 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0033483889999999998 2024-08-17T17:54:49.085 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.085 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.086 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.086 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.705970+0000", 2024-08-17T17:54:49.086 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4419999999999998e-05 2024-08-17T17:54:49.086 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.086 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.086 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.086 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.820352+0000", 2024-08-17T17:54:49.086 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.114382285 2024-08-17T17:54:49.086 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.086 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.086 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.086 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.820385+0000", 2024-08-17T17:54:49.086 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2809999999999999e-05 2024-08-17T17:54:49.087 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.087 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.087 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.087 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.821647+0000", 2024-08-17T17:54:49.087 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0012620489999999999 2024-08-17T17:54:49.087 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.087 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.087 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.087 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.087 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.087 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:111967 45.1 45:838e5df7:::benchmark_data_smithi130_172551_object7160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:54:49.087 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:34.708154+0000", 2024-08-17T17:54:49.087 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 314.26677355100003, 2024-08-17T17:54:49.088 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.11479482100000001, 2024-08-17T17:54:49.088 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.088 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.088 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.088 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.088 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.088 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 111967 2024-08-17T17:54:49.088 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.088 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.088 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.088 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.088 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708154+0000", 2024-08-17T17:54:49.088 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.089 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.089 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.089 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.089 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708154+0000", 2024-08-17T17:54:49.089 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.089 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.089 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.089 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.089 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708153+0000", 2024-08-17T17:54:49.089 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:54:49.089 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.089 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.089 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.089 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708163+0000", 2024-08-17T17:54:49.090 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0693000000000001e-05 2024-08-17T17:54:49.090 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.090 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.090 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.090 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708175+0000", 2024-08-17T17:54:49.090 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1677e-05 2024-08-17T17:54:49.090 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.090 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.090 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.090 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708181+0000", 2024-08-17T17:54:49.090 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6840000000000001e-06 2024-08-17T17:54:49.090 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.090 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.091 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.091 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708229+0000", 2024-08-17T17:54:49.091 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8049e-05 2024-08-17T17:54:49.091 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.091 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.091 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.091 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708393+0000", 2024-08-17T17:54:49.091 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000164394 2024-08-17T17:54:49.091 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.091 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.091 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.091 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708553+0000", 2024-08-17T17:54:49.091 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00015941600000000001 2024-08-17T17:54:49.092 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.092 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.092 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.092 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.712190+0000", 2024-08-17T17:54:49.092 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0036373170000000002 2024-08-17T17:54:49.092 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.092 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.092 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.092 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.712200+0000", 2024-08-17T17:54:49.092 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6039999999999999e-06 2024-08-17T17:54:49.092 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.092 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.092 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.092 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.821672+0000", 2024-08-17T17:54:49.093 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.10947284 2024-08-17T17:54:49.093 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.093 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.093 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.093 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.821701+0000", 2024-08-17T17:54:49.093 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9023e-05 2024-08-17T17:54:49.093 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.093 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.093 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.093 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.822949+0000", 2024-08-17T17:54:49.093 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0012476950000000001 2024-08-17T17:54:49.093 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.093 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.093 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.094 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.094 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.094 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:111985 45.1 45:837e7695:::benchmark_data_smithi130_172551_object7178:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:54:49.094 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:34.747055+0000", 2024-08-17T17:54:49.094 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 314.22787256999999, 2024-08-17T17:54:49.094 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.078433560999999999, 2024-08-17T17:54:49.094 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.094 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.094 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.094 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.094 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.094 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 111985 2024-08-17T17:54:49.094 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.095 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.095 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.095 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.095 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747055+0000", 2024-08-17T17:54:49.095 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.095 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.095 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.095 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.095 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747055+0000", 2024-08-17T17:54:49.095 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.095 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.095 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.095 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.095 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747053+0000", 2024-08-17T17:54:49.096 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999976 2024-08-17T17:54:49.096 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.096 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.096 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.096 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747063+0000", 2024-08-17T17:54:49.096 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.005e-05 2024-08-17T17:54:49.096 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.096 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.096 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.096 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747082+0000", 2024-08-17T17:54:49.096 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.9391000000000001e-05 2024-08-17T17:54:49.096 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.096 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.097 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.097 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747090+0000", 2024-08-17T17:54:49.097 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0989999999999999e-06 2024-08-17T17:54:49.097 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.097 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.097 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.097 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747135+0000", 2024-08-17T17:54:49.097 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5169000000000001e-05 2024-08-17T17:54:49.097 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.097 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.097 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.097 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747265+0000", 2024-08-17T17:54:49.097 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012957100000000001 2024-08-17T17:54:49.097 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.098 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.098 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.098 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747367+0000", 2024-08-17T17:54:49.098 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000101648 2024-08-17T17:54:49.098 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.098 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.098 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.098 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.750636+0000", 2024-08-17T17:54:49.098 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0032697749999999999 2024-08-17T17:54:49.098 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.098 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.098 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.098 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.751406+0000", 2024-08-17T17:54:49.099 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00076938899999999999 2024-08-17T17:54:49.099 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.099 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.099 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.099 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.823006+0000", 2024-08-17T17:54:49.099 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.071600330000000004 2024-08-17T17:54:49.099 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.099 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.099 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.099 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.823056+0000", 2024-08-17T17:54:49.099 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9913000000000001e-05 2024-08-17T17:54:49.099 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.099 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.099 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.100 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.825489+0000", 2024-08-17T17:54:49.100 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024327950000000002 2024-08-17T17:54:49.100 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.100 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.100 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.100 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.100 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.100 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:111989 45.1 45:8ed4177b:::benchmark_data_smithi130_172551_object7182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:54:49.100 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:34.761421+0000", 2024-08-17T17:54:49.100 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 314.21350702500001, 2024-08-17T17:54:49.100 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.067313382000000005, 2024-08-17T17:54:49.100 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.100 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.101 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.101 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.101 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.101 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 111989 2024-08-17T17:54:49.101 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.101 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.101 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.101 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.101 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761421+0000", 2024-08-17T17:54:49.101 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.101 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.101 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.101 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.101 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761421+0000", 2024-08-17T17:54:49.102 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.102 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.102 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.102 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.102 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761418+0000", 2024-08-17T17:54:49.102 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999971 2024-08-17T17:54:49.102 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.102 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.102 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.102 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761433+0000", 2024-08-17T17:54:49.102 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4643e-05 2024-08-17T17:54:49.102 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.102 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.103 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.103 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761465+0000", 2024-08-17T17:54:49.103 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2753000000000003e-05 2024-08-17T17:54:49.103 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.103 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.103 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.103 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761478+0000", 2024-08-17T17:54:49.103 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2446000000000001e-05 2024-08-17T17:54:49.103 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.103 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.103 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.103 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761567+0000", 2024-08-17T17:54:49.103 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.8905000000000003e-05 2024-08-17T17:54:49.103 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.104 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.104 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.104 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761765+0000", 2024-08-17T17:54:49.104 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019786200000000001 2024-08-17T17:54:49.104 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.104 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.104 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.104 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761877+0000", 2024-08-17T17:54:49.104 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00011220100000000001 2024-08-17T17:54:49.104 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.104 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.104 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.104 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.765275+0000", 2024-08-17T17:54:49.104 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0033986089999999999 2024-08-17T17:54:49.105 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.105 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.105 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.105 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.765348+0000", 2024-08-17T17:54:49.105 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.2794999999999997e-05 2024-08-17T17:54:49.105 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.105 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.105 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.105 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.825563+0000", 2024-08-17T17:54:49.105 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.060214700000000003 2024-08-17T17:54:49.105 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.105 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.105 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.105 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.825679+0000", 2024-08-17T17:54:49.106 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000115665 2024-08-17T17:54:49.106 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.106 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.106 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.106 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.828734+0000", 2024-08-17T17:54:49.106 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0030556670000000002 2024-08-17T17:54:49.106 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.106 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.106 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.106 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.106 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.106 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:113030 45.1 45:8293cabe:::benchmark_data_smithi130_172551_object8223:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:54:49.106 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:37.005519+0000", 2024-08-17T17:54:49.106 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 311.96940910900003, 2024-08-17T17:54:49.107 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.084215829000000006, 2024-08-17T17:54:49.107 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.107 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.107 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.107 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.107 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.107 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 113030 2024-08-17T17:54:49.107 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.107 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.107 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.107 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.107 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005519+0000", 2024-08-17T17:54:49.107 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.108 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.108 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.108 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.108 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005519+0000", 2024-08-17T17:54:49.108 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.108 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.108 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.108 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.108 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005517+0000", 2024-08-17T17:54:49.108 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:54:49.108 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.108 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.108 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.108 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005523+0000", 2024-08-17T17:54:49.109 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5010000000000004e-06 2024-08-17T17:54:49.109 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.109 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.109 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.109 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005534+0000", 2024-08-17T17:54:49.109 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.13e-05 2024-08-17T17:54:49.109 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.109 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.109 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.109 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005539+0000", 2024-08-17T17:54:49.109 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0560000000000004e-06 2024-08-17T17:54:49.109 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.109 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.109 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.110 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.008372+0000", 2024-08-17T17:54:49.110 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002833225 2024-08-17T17:54:49.110 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.110 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.110 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.110 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.008492+0000", 2024-08-17T17:54:49.110 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000119633 2024-08-17T17:54:49.110 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.110 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.110 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.110 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.008597+0000", 2024-08-17T17:54:49.110 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105491 2024-08-17T17:54:49.110 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.111 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.111 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.111 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.014259+0000", 2024-08-17T17:54:49.111 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0056618279999999998 2024-08-17T17:54:49.111 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.111 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.111 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.111 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.014291+0000", 2024-08-17T17:54:49.111 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1934000000000003e-05 2024-08-17T17:54:49.111 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.111 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.111 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.111 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.087093+0000", 2024-08-17T17:54:49.112 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.072801433999999998 2024-08-17T17:54:49.112 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.112 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.112 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.112 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.087151+0000", 2024-08-17T17:54:49.112 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8109999999999998e-05 2024-08-17T17:54:49.112 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.112 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.112 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.112 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.089735+0000", 2024-08-17T17:54:49.112 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025838850000000002 2024-08-17T17:54:49.112 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.112 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.112 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.113 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.113 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.113 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:113033 45.1 45:874b46c2:::benchmark_data_smithi130_172551_object8226:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:54:49.113 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:37.005608+0000", 2024-08-17T17:54:49.113 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 311.96931997000002, 2024-08-17T17:54:49.113 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.086365193000000007, 2024-08-17T17:54:49.113 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.113 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.113 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.113 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.113 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.113 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 113033 2024-08-17T17:54:49.113 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.113 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.114 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.114 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.114 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005608+0000", 2024-08-17T17:54:49.114 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.114 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.114 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.114 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.114 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005608+0000", 2024-08-17T17:54:49.114 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.114 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.114 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.114 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.114 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005607+0000", 2024-08-17T17:54:49.115 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:54:49.115 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.115 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.115 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.115 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005611+0000", 2024-08-17T17:54:49.115 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4680000000000001e-06 2024-08-17T17:54:49.115 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.115 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.115 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.115 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005619+0000", 2024-08-17T17:54:49.115 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2409999999999995e-06 2024-08-17T17:54:49.115 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.115 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.115 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.116 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005623+0000", 2024-08-17T17:54:49.116 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7249999999999999e-06 2024-08-17T17:54:49.116 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.116 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.116 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.116 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.011289+0000", 2024-08-17T17:54:49.116 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0056665539999999999 2024-08-17T17:54:49.116 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.116 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.116 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.116 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.011465+0000", 2024-08-17T17:54:49.116 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017524399999999999 2024-08-17T17:54:49.116 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.116 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.117 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.117 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.011636+0000", 2024-08-17T17:54:49.117 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017170099999999999 2024-08-17T17:54:49.117 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.117 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.117 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.117 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.014286+0000", 2024-08-17T17:54:49.117 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0026499499999999999 2024-08-17T17:54:49.117 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.117 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.117 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.117 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.014351+0000", 2024-08-17T17:54:49.117 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4957999999999997e-05 2024-08-17T17:54:49.117 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.118 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.118 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.118 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.089799+0000", 2024-08-17T17:54:49.118 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.075448276999999994 2024-08-17T17:54:49.118 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.118 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.118 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.118 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.089855+0000", 2024-08-17T17:54:49.118 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5294000000000003e-05 2024-08-17T17:54:49.118 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.118 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.118 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.118 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.091973+0000", 2024-08-17T17:54:49.118 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002118417 2024-08-17T17:54:49.119 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.119 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.119 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.119 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.119 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.119 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:113039 45.1 45:82b52502:::benchmark_data_smithi130_172551_object8232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:54:49.119 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:37.021375+0000", 2024-08-17T17:54:49.119 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 311.95355275100002, 2024-08-17T17:54:49.119 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.072155133999999996, 2024-08-17T17:54:49.119 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.119 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.119 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.119 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.119 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.120 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 113039 2024-08-17T17:54:49.120 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.120 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.120 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.120 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.120 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021375+0000", 2024-08-17T17:54:49.120 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.120 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.120 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.120 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.120 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021375+0000", 2024-08-17T17:54:49.120 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.120 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.120 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.121 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.121 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021374+0000", 2024-08-17T17:54:49.121 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:54:49.121 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.121 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.121 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.121 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021380+0000", 2024-08-17T17:54:49.121 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1909999999999997e-06 2024-08-17T17:54:49.121 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.121 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.121 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.121 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021392+0000", 2024-08-17T17:54:49.121 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1912e-05 2024-08-17T17:54:49.121 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.122 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.122 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.122 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021396+0000", 2024-08-17T17:54:49.122 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5170000000000001e-06 2024-08-17T17:54:49.122 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.122 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.122 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.122 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021435+0000", 2024-08-17T17:54:49.122 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8065000000000002e-05 2024-08-17T17:54:49.122 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.122 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.122 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.122 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021544+0000", 2024-08-17T17:54:49.123 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010904300000000001 2024-08-17T17:54:49.123 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.123 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.123 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.123 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021674+0000", 2024-08-17T17:54:49.123 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013044700000000001 2024-08-17T17:54:49.123 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.123 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.123 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.123 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.027616+0000", 2024-08-17T17:54:49.123 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0059417929999999999 2024-08-17T17:54:49.123 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.123 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.124 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.124 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.027649+0000", 2024-08-17T17:54:49.124 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3655999999999998e-05 2024-08-17T17:54:49.124 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.124 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.124 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.124 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.092072+0000", 2024-08-17T17:54:49.124 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.064422191000000004 2024-08-17T17:54:49.124 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.124 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.124 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.124 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.092108+0000", 2024-08-17T17:54:49.124 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6560000000000002e-05 2024-08-17T17:54:49.125 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.125 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.125 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.125 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.093530+0000", 2024-08-17T17:54:49.125 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001422092 2024-08-17T17:54:49.125 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.125 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.125 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.125 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.125 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.125 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:113040 45.1 45:86caa578:::benchmark_data_smithi130_172551_object8233:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:54:49.125 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:37.023080+0000", 2024-08-17T17:54:49.125 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 311.951848273, 2024-08-17T17:54:49.125 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.072093230999999994, 2024-08-17T17:54:49.126 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:54:49.126 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:54:49.126 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:54:49.126 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:54:49.126 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:54:49.126 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 113040 2024-08-17T17:54:49.126 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.126 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:54:49.126 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.126 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:54:49.126 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.023080+0000", 2024-08-17T17:54:49.126 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.126 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.127 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.127 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:54:49.127 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.023080+0000", 2024-08-17T17:54:49.127 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:54:49.127 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.127 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.127 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:54:49.127 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.023078+0000", 2024-08-17T17:54:49.127 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:54:49.127 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.127 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.127 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:54:49.127 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.023088+0000", 2024-08-17T17:54:49.127 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5200000000000003e-06 2024-08-17T17:54:49.128 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.128 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.128 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:54:49.128 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.023104+0000", 2024-08-17T17:54:49.128 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5951e-05 2024-08-17T17:54:49.128 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.128 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.128 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:54:49.128 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.023111+0000", 2024-08-17T17:54:49.128 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.2810000000000003e-06 2024-08-17T17:54:49.128 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.128 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.128 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:54:49.128 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.025056+0000", 2024-08-17T17:54:49.129 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001944763 2024-08-17T17:54:49.129 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.129 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.129 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:54:49.129 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.025197+0000", 2024-08-17T17:54:49.129 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014151100000000001 2024-08-17T17:54:49.129 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.129 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.129 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:54:49.129 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.025296+0000", 2024-08-17T17:54:49.129 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9264000000000004e-05 2024-08-17T17:54:49.129 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.129 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.129 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:54:49.130 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.027753+0000", 2024-08-17T17:54:49.130 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024564270000000002 2024-08-17T17:54:49.130 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.130 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.130 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.130 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.027853+0000", 2024-08-17T17:54:49.130 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9986000000000001e-05 2024-08-17T17:54:49.130 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.130 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.130 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:54:49.130 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.093580+0000", 2024-08-17T17:54:49.130 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.065727168000000002 2024-08-17T17:54:49.130 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.093619+0000", 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8968999999999999e-05 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.095173+0000", 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0015538380000000001 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:54:49.131 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:54:49.132 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:49.134 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:54:49.135 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:54:49.201 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:49.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:49.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:49.303 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:49.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:49.327 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:49.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:49.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:49 smithi179 ceph-mon[93519]: from='client.45647 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:54:49.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:49 smithi179 ceph-mon[93519]: pgmap v1833: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 17002/187950 objects degraded (9.046%); 10693/187950 objects misplaced (5.689%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:49.428 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:49.450 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:49.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:49.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:49.552 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:49.570 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:49 smithi130 ceph-mon[105521]: from='client.45647 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:54:49.570 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:49 smithi130 ceph-mon[105521]: pgmap v1833: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 17002/187950 objects degraded (9.046%); 10693/187950 objects misplaced (5.689%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:49.570 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:49 smithi130 ceph-mon[110068]: from='client.45647 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:54:49.570 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:49 smithi130 ceph-mon[110068]: pgmap v1833: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 17002/187950 objects degraded (9.046%); 10693/187950 objects misplaced (5.689%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:49.573 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:49.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:49.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:49.675 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:49.697 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:49.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:49.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:49.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:49.820 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:49.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:49.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:49.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:54:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:54:49] "GET /metrics HTTP/1.1" 200 36812 "" "Prometheus/2.51.0" 2024-08-17T17:54:49.922 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:49.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:49.944 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:49.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:50.045 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:50.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.068 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:50.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:50.169 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:50.190 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:50.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:50.292 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:50.314 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:50.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:50.316 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:54:50.416 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:50.437 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:50.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:50.539 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:50.561 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:50.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:54:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:54:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:54:50.613 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:54:50.662 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:50.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:50.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:50.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:50.811 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:50.913 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:50 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17002/187950 objects degraded (9.046%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:50.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:50 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17002/187950 objects degraded (9.046%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:50.938 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:50.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:50.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:51.040 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:51.061 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:51.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:51.076 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:50 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17002/187950 objects degraded (9.046%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:51.163 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:51.192 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:51.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:51.294 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:51.316 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:51.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:51.417 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:51.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:51.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:51.540 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:51.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.562 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:51.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:51.663 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:51.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:51.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:51.787 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:51.808 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:51.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:51 smithi179 ceph-mon[93519]: pgmap v1834: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 17002/187410 objects degraded (9.072%); 10608/187410 objects misplaced (5.660%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:51.910 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:51.935 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:51.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:51.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:52.036 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:52.059 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:52.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:52.065 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:51 smithi130 ceph-mon[105521]: pgmap v1834: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 17002/187410 objects degraded (9.072%); 10608/187410 objects misplaced (5.660%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:52.065 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:51 smithi130 ceph-mon[110068]: pgmap v1834: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 17002/187410 objects degraded (9.072%); 10608/187410 objects misplaced (5.660%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:52.161 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:52.186 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:52.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:52.288 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:52.313 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:52.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:52.415 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:52.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:52.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:52.451 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:54:52.452 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:54:52.471 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:54:52.471 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:54:52.487 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:54:52.487 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:54:52.487 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T17:54:52.487 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:54:52.491 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:54:52.491 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:54:52.509 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:54:52.510 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:54:52.525 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:54:52.526 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:54:52.540 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:52.541 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:54:52.541 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:54:52.560 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:54:52.560 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:54:52.561 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:52.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:52.576 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:54:52.576 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:54:52.663 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:52.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:52.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:52.780 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:54:52.786 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:52.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:52.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:52.912 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:52.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:52 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/154892885' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:52.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:52 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:52 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3043546817' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:52 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/154892885' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:52 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:52 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3043546817' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:52.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:52.935 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:52.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:53.036 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:53.059 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:53.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:52 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/154892885' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:52 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:54:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:52 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3043546817' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:54:53.125 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1832,"stamp":"2024-08-17T17:54:47.059959+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4135983077,"num_objects":63128,"num_object_clones":0,"num_object_copies":189384,"num_objects_missing_on_primary":3416,"num_objects_missing":3416,"num_objects_degraded":17002,"num_objects_misplaced":10693,"num_objects_unfound":0,"num_objects_dirty":63128,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":146786,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35294,"num_bytes_recovered":2134449165,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108789,"ondisk_log_size":108789,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12245516,"kb_used_data":10825876,"kb_used_omap":295,"kb_used_meta":1419288,"kb_avail":550225396,"statfs":{"total":575970213888,"available":563430805504,"internally_reserved":0,"allocated":11085697024,"data_stored":11056258137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1453351375},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,0,2,1,1,2,1,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-92667904,"num_objects":-1414,"num_object_clones":0,"num_object_copies":-4242,"num_objects_missing_on_primary":-89,"num_objects_missing":-89,"num_objects_degraded":-176,"num_objects_misplaced":-256,"num_objects_unfound":0,"num_objects_dirty":-1414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1414,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":11403264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002746"},"pg_stats":[{"pgid":"45.f","version":"1226'9160","reported_seq":19691,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:44.352923+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:54:44.352923+0000","last_peered":"2024-08-17T17:54:44.352923+0000","last_clean":"2024-08-17T17:54:44.352923+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:54:44.352923+0000","last_undegraded":"2024-08-17T17:54:44.352923+0000","last_fullsized":"2024-08-17T17:54:44.352923+0000","mapping_epoch":1194,"log_start":"1170'2331","ondisk_log_start":"1170'2331","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6829,"log_dups_size":0,"ondisk_log_size":6829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262406144,"num_objects":4004,"num_object_clones":0,"num_object_copies":12012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9160,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1226'9072","reported_seq":19538,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:43.022016+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:54:43.022016+0000","last_peered":"2024-08-17T17:54:43.022016+0000","last_clean":"2024-08-17T17:54:43.022016+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:54:43.022016+0000","last_undegraded":"2024-08-17T17:54:43.022016+0000","last_fullsized":"2024-08-17T17:54:43.022016+0000","mapping_epoch":1211,"log_start":"1170'2032","ondisk_log_start":"1170'2032","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7040,"log_dups_size":0,"ondisk_log_size":7040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257032192,"num_objects":3922,"num_object_clones":0,"num_object_copies":11766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9072,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1226'9378","reported_seq":20217,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:43.384967+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:54:43.384967+0000","last_peered":"2024-08-17T17:54:43.384967+0000","last_clean":"2024-08-17T17:54:43.384967+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:54:43.384967+0000","last_undegraded":"2024-08-17T17:54:43.384967+0000","last_fullsized":"2024-08-17T17:54:43.384967+0000","mapping_epoch":1205,"log_start":"1174'3468","ondisk_log_start":"1174'3468","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5910,"log_dups_size":0,"ondisk_log_size":5910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262078488,"num_objects":4000,"num_object_clones":0,"num_object_copies":12000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4000,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9378,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1226'9106","reported_seq":9173,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.982894+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:54:42.982894+0000","last_peered":"2024-08-17T17:54:42.982894+0000","last_clean":"2024-08-17T17:54:42.982894+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:54:42.982894+0000","last_undegraded":"2024-08-17T17:54:42.982894+0000","last_fullsized":"2024-08-17T17:54:42.982894+0000","mapping_epoch":1167,"log_start":"1170'2389","ondisk_log_start":"1170'2389","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6717,"log_dups_size":0,"ondisk_log_size":6717,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255197184,"num_objects":3894,"num_object_clones":0,"num_object_copies":11682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9106,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1226'9169","reported_seq":14414,"reported_epoch":1226,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:46.365246+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:54:46.365246+0000","last_peered":"2024-08-17T17:54:46.365246+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:54:46.365246+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1174'3226","ondisk_log_start":"1174'3226","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258539520,"num_objects":3945,"num_object_clones":0,"num_object_copies":11835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":3945,"num_objects_unfound":0,"num_objects_dirty":3945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9169,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":3427},{"shards":"1,4,5","objects":459}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1225'9230","reported_seq":9297,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.445286+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:54:42.445286+0000","last_peered":"2024-08-17T17:54:42.445286+0000","last_clean":"2024-08-17T17:54:42.445286+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:54:42.445286+0000","last_undegraded":"2024-08-17T17:54:42.445286+0000","last_fullsized":"2024-08-17T17:54:42.445286+0000","mapping_epoch":1167,"log_start":"1170'2477","ondisk_log_start":"1170'2477","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6753,"log_dups_size":0,"ondisk_log_size":6753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253231104,"num_objects":3864,"num_object_clones":0,"num_object_copies":11592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9230,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1226'9211","reported_seq":9278,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:43.096067+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:54:43.096067+0000","last_peered":"2024-08-17T17:54:43.096067+0000","last_clean":"2024-08-17T17:54:43.096067+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:54:43.096067+0000","last_undegraded":"2024-08-17T17:54:43.096067+0000","last_fullsized":"2024-08-17T17:54:43.096067+0000","mapping_epoch":1167,"log_start":"1170'2469","ondisk_log_start":"1170'2469","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6742,"log_dups_size":0,"ondisk_log_size":6742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263127040,"num_objects":4015,"num_object_clones":0,"num_object_copies":12045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9211,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1226'9153","reported_seq":17265,"reported_epoch":1226,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:46.854212+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:54:46.854212+0000","last_peered":"2024-08-17T17:54:46.854212+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:54:46.854212+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1174'3175","ondisk_log_start":"1174'3175","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5978,"log_dups_size":0,"ondisk_log_size":5978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251461632,"num_objects":3837,"num_object_clones":0,"num_object_copies":11511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2967,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9153,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2658,"num_bytes_recovered":174194688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":450},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1226'9093","reported_seq":9170,"reported_epoch":1226,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:43.098003+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:54:43.098003+0000","last_peered":"2024-08-17T17:54:43.098003+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:54:43.098003+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'2377","ondisk_log_start":"1170'2377","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6716,"log_dups_size":0,"ondisk_log_size":6716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261029888,"num_objects":3983,"num_object_clones":0,"num_object_copies":11949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9093,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":3983},{"shards":"1,6","objects":-1850}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1226'9149","reported_seq":12227,"reported_epoch":1226,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:43.238680+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:54:43.238680+0000","last_peered":"2024-08-17T17:54:43.238680+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:54:43.238680+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1174'3206","ondisk_log_start":"1174'3206","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263782400,"num_objects":4025,"num_object_clones":0,"num_object_copies":12075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9149,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":2464},{"shards":"1,3,5","objects":466}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1226'9098","reported_seq":9231,"reported_epoch":1226,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:43.910557+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:54:43.910557+0000","last_peered":"2024-08-17T17:54:43.910557+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:54:43.910557+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9098,"log_dups_size":0,"ondisk_log_size":9098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260571136,"num_objects":3976,"num_object_clones":0,"num_object_copies":11928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9098,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-1177}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1226'9111","reported_seq":15305,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:43.846954+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:54:43.846954+0000","last_peered":"2024-08-17T17:54:43.846954+0000","last_clean":"2024-08-17T17:54:43.846954+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:54:43.846954+0000","last_undegraded":"2024-08-17T17:54:43.846954+0000","last_fullsized":"2024-08-17T17:54:43.846954+0000","mapping_epoch":1183,"log_start":"0'0","ondisk_log_start":"0'0","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9111,"log_dups_size":0,"ondisk_log_size":9111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257884160,"num_objects":3935,"num_object_clones":0,"num_object_copies":11805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9111,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1226'9235","reported_seq":9367,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:43.241889+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:54:43.241889+0000","last_peered":"2024-08-17T17:54:43.241889+0000","last_clean":"2024-08-17T17:54:43.241889+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:54:43.241889+0000","last_undegraded":"2024-08-17T17:54:43.241889+0000","last_fullsized":"2024-08-17T17:54:43.241889+0000","mapping_epoch":1167,"log_start":"1174'3291","ondisk_log_start":"1174'3291","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5944,"log_dups_size":0,"ondisk_log_size":5944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257228800,"num_objects":3925,"num_object_clones":0,"num_object_copies":11775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9235,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1226'9080","reported_seq":9147,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.846346+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:54:42.846346+0000","last_peered":"2024-08-17T17:54:42.846346+0000","last_clean":"2024-08-17T17:54:42.846346+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:54:42.846346+0000","last_undegraded":"2024-08-17T17:54:42.846346+0000","last_fullsized":"2024-08-17T17:54:42.846346+0000","mapping_epoch":1167,"log_start":"1170'2025","ondisk_log_start":"1170'2025","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252575744,"num_objects":3854,"num_object_clones":0,"num_object_copies":11562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9080,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1226'9304","reported_seq":20806,"reported_epoch":1226,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:44.629413+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:54:44.629413+0000","last_peered":"2024-08-17T17:54:44.629413+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:54:44.629413+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'2391","ondisk_log_start":"1170'2391","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6913,"log_dups_size":0,"ondisk_log_size":6913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266076160,"num_objects":4060,"num_object_clones":0,"num_object_copies":12180,"num_objects_missing_on_primary":3416,"num_objects_missing":3416,"num_objects_degraded":265,"num_objects_misplaced":6748,"num_objects_unfound":0,"num_objects_dirty":4060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9304,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4729,"num_bytes_recovered":309919744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":14},{"shards":"0,4,6","objects":1356},{"shards":"0,6","objects":1},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":2424}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1226'8959","reported_seq":22277,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:43.119715+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:54:43.119715+0000","last_peered":"2024-08-17T17:54:43.119715+0000","last_clean":"2024-08-17T17:54:43.119715+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:54:43.119715+0000","last_undegraded":"2024-08-17T17:54:43.119715+0000","last_fullsized":"2024-08-17T17:54:43.119715+0000","mapping_epoch":1196,"log_start":"1174'3069","ondisk_log_start":"1174'3069","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5890,"log_dups_size":0,"ondisk_log_size":5890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253165568,"num_objects":3863,"num_object_clones":0,"num_object_copies":11589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8959,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1909,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.850542+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:54:42.850542+0000","last_peered":"2024-08-17T17:54:42.850542+0000","last_clean":"2024-08-17T17:54:42.850542+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:54:42.850542+0000","last_undegraded":"2024-08-17T17:54:42.850542+0000","last_fullsized":"2024-08-17T17:54:42.850542+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":503,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.157713+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:54:29.157713+0000","last_peered":"2024-08-17T17:54:29.157713+0000","last_clean":"2024-08-17T17:54:29.157713+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:54:29.157713+0000","last_undegraded":"2024-08-17T17:54:29.157713+0000","last_fullsized":"2024-08-17T17:54:29.157713+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1828,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.843379+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:54:42.843379+0000","last_peered":"2024-08-17T17:54:42.843379+0000","last_clean":"2024-08-17T17:54:42.843379+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:54:42.843379+0000","last_undegraded":"2024-08-17T17:54:42.843379+0000","last_fullsized":"2024-08-17T17:54:42.843379+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1852,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.841708+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:54:42.841708+0000","last_peered":"2024-08-17T17:54:42.841708+0000","last_clean":"2024-08-17T17:54:42.841708+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:54:42.841708+0000","last_undegraded":"2024-08-17T17:54:42.841708+0000","last_fullsized":"2024-08-17T17:54:42.841708+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":65,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.158346+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:54:29.158346+0000","last_peered":"2024-08-17T17:54:29.158346+0000","last_clean":"2024-08-17T17:54:29.158346+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:54:29.158346+0000","last_undegraded":"2024-08-17T17:54:29.158346+0000","last_fullsized":"2024-08-17T17:54:29.158346+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1245,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.842132+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:54:42.842132+0000","last_peered":"2024-08-17T17:54:42.842132+0000","last_clean":"2024-08-17T17:54:42.842132+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:54:42.842132+0000","last_undegraded":"2024-08-17T17:54:42.842132+0000","last_fullsized":"2024-08-17T17:54:42.842132+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":104,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.838967+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:54:42.838967+0000","last_peered":"2024-08-17T17:54:42.838967+0000","last_clean":"2024-08-17T17:54:42.838967+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:54:42.838967+0000","last_undegraded":"2024-08-17T17:54:42.838967+0000","last_fullsized":"2024-08-17T17:54:42.838967+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":623,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.842261+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:54:42.842261+0000","last_peered":"2024-08-17T17:54:42.842261+0000","last_clean":"2024-08-17T17:54:42.842261+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:54:42.842261+0000","last_undegraded":"2024-08-17T17:54:42.842261+0000","last_fullsized":"2024-08-17T17:54:42.842261+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1867,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.849987+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:54:42.849987+0000","last_peered":"2024-08-17T17:54:42.849987+0000","last_clean":"2024-08-17T17:54:42.849987+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:54:42.849987+0000","last_undegraded":"2024-08-17T17:54:42.849987+0000","last_fullsized":"2024-08-17T17:54:42.849987+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1869,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.843109+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:54:42.843109+0000","last_peered":"2024-08-17T17:54:42.843109+0000","last_clean":"2024-08-17T17:54:42.843109+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:54:42.843109+0000","last_undegraded":"2024-08-17T17:54:42.843109+0000","last_fullsized":"2024-08-17T17:54:42.843109+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":89,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.838537+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:54:42.838537+0000","last_peered":"2024-08-17T17:54:42.838537+0000","last_clean":"2024-08-17T17:54:42.838537+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:54:42.838537+0000","last_undegraded":"2024-08-17T17:54:42.838537+0000","last_fullsized":"2024-08-17T17:54:42.838537+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1841,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.843227+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:54:42.843227+0000","last_peered":"2024-08-17T17:54:42.843227+0000","last_clean":"2024-08-17T17:54:42.843227+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:54:42.843227+0000","last_undegraded":"2024-08-17T17:54:42.843227+0000","last_fullsized":"2024-08-17T17:54:42.843227+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1895,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.836852+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:54:42.836852+0000","last_peered":"2024-08-17T17:54:42.836852+0000","last_clean":"2024-08-17T17:54:42.836852+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:54:42.836852+0000","last_undegraded":"2024-08-17T17:54:42.836852+0000","last_fullsized":"2024-08-17T17:54:42.836852+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1890,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.850315+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:54:42.850315+0000","last_peered":"2024-08-17T17:54:42.850315+0000","last_clean":"2024-08-17T17:54:42.850315+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:54:42.850315+0000","last_undegraded":"2024-08-17T17:54:42.850315+0000","last_fullsized":"2024-08-17T17:54:42.850315+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1863,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.841686+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:54:42.841686+0000","last_peered":"2024-08-17T17:54:42.841686+0000","last_clean":"2024-08-17T17:54:42.841686+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:54:42.841686+0000","last_undegraded":"2024-08-17T17:54:42.841686+0000","last_fullsized":"2024-08-17T17:54:42.841686+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1200,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.158300+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:54:29.158300+0000","last_peered":"2024-08-17T17:54:29.158300+0000","last_clean":"2024-08-17T17:54:29.158300+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:54:29.158300+0000","last_undegraded":"2024-08-17T17:54:29.158300+0000","last_fullsized":"2024-08-17T17:54:29.158300+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1834,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.841438+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:54:42.841438+0000","last_peered":"2024-08-17T17:54:42.841438+0000","last_clean":"2024-08-17T17:54:42.841438+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:54:42.841438+0000","last_undegraded":"2024-08-17T17:54:42.841438+0000","last_fullsized":"2024-08-17T17:54:42.841438+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1884,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.850586+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:54:42.850586+0000","last_peered":"2024-08-17T17:54:42.850586+0000","last_clean":"2024-08-17T17:54:42.850586+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:54:42.850586+0000","last_undegraded":"2024-08-17T17:54:42.850586+0000","last_fullsized":"2024-08-17T17:54:42.850586+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1790,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.841619+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:54:42.841619+0000","last_peered":"2024-08-17T17:54:42.841619+0000","last_clean":"2024-08-17T17:54:42.841619+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:54:42.841619+0000","last_undegraded":"2024-08-17T17:54:42.841619+0000","last_fullsized":"2024-08-17T17:54:42.841619+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":519,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.160254+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:54:29.160254+0000","last_peered":"2024-08-17T17:54:29.160254+0000","last_clean":"2024-08-17T17:54:29.160254+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:54:29.160254+0000","last_undegraded":"2024-08-17T17:54:29.160254+0000","last_fullsized":"2024-08-17T17:54:29.160254+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1196,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.160298+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:54:29.160298+0000","last_peered":"2024-08-17T17:54:29.160298+0000","last_clean":"2024-08-17T17:54:29.160298+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:54:29.160298+0000","last_undegraded":"2024-08-17T17:54:29.160298+0000","last_fullsized":"2024-08-17T17:54:29.160298+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1889,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.838655+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:54:42.838655+0000","last_peered":"2024-08-17T17:54:42.838655+0000","last_clean":"2024-08-17T17:54:42.838655+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:54:42.838655+0000","last_undegraded":"2024-08-17T17:54:42.838655+0000","last_fullsized":"2024-08-17T17:54:42.838655+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":357,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.160169+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:54:29.160169+0000","last_peered":"2024-08-17T17:54:29.160169+0000","last_clean":"2024-08-17T17:54:29.160169+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:54:29.160169+0000","last_undegraded":"2024-08-17T17:54:29.160169+0000","last_fullsized":"2024-08-17T17:54:29.160169+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":503,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.158026+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:54:29.158026+0000","last_peered":"2024-08-17T17:54:29.158026+0000","last_clean":"2024-08-17T17:54:29.158026+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:54:29.158026+0000","last_undegraded":"2024-08-17T17:54:29.158026+0000","last_fullsized":"2024-08-17T17:54:29.158026+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1181,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.157906+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:54:29.157906+0000","last_peered":"2024-08-17T17:54:29.157906+0000","last_clean":"2024-08-17T17:54:29.157906+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:54:29.157906+0000","last_undegraded":"2024-08-17T17:54:29.157906+0000","last_fullsized":"2024-08-17T17:54:29.157906+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1189,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.158343+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:54:29.158343+0000","last_peered":"2024-08-17T17:54:29.158343+0000","last_clean":"2024-08-17T17:54:29.158343+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:54:29.158343+0000","last_undegraded":"2024-08-17T17:54:29.158343+0000","last_fullsized":"2024-08-17T17:54:29.158343+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1936,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.850157+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:54:42.850157+0000","last_peered":"2024-08-17T17:54:42.850157+0000","last_clean":"2024-08-17T17:54:42.850157+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:54:42.850157+0000","last_undegraded":"2024-08-17T17:54:42.850157+0000","last_fullsized":"2024-08-17T17:54:42.850157+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1820,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.841604+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:54:42.841604+0000","last_peered":"2024-08-17T17:54:42.841604+0000","last_clean":"2024-08-17T17:54:42.841604+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:54:42.841604+0000","last_undegraded":"2024-08-17T17:54:42.841604+0000","last_fullsized":"2024-08-17T17:54:42.841604+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1895,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.839213+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:54:42.839213+0000","last_peered":"2024-08-17T17:54:42.839213+0000","last_clean":"2024-08-17T17:54:42.839213+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:54:42.839213+0000","last_undegraded":"2024-08-17T17:54:42.839213+0000","last_fullsized":"2024-08-17T17:54:42.839213+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1822,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.842351+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:54:42.842351+0000","last_peered":"2024-08-17T17:54:42.842351+0000","last_clean":"2024-08-17T17:54:42.842351+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:54:42.842351+0000","last_undegraded":"2024-08-17T17:54:42.842351+0000","last_fullsized":"2024-08-17T17:54:42.842351+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1854,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.849988+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:54:42.849988+0000","last_peered":"2024-08-17T17:54:42.849988+0000","last_clean":"2024-08-17T17:54:42.849988+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:54:42.849988+0000","last_undegraded":"2024-08-17T17:54:42.849988+0000","last_fullsized":"2024-08-17T17:54:42.849988+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1216,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.160337+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:54:29.160337+0000","last_peered":"2024-08-17T17:54:29.160337+0000","last_clean":"2024-08-17T17:54:29.160337+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:54:29.160337+0000","last_undegraded":"2024-08-17T17:54:29.160337+0000","last_fullsized":"2024-08-17T17:54:29.160337+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1901,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.839027+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:54:42.839027+0000","last_peered":"2024-08-17T17:54:42.839027+0000","last_clean":"2024-08-17T17:54:42.839027+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:54:42.839027+0000","last_undegraded":"2024-08-17T17:54:42.839027+0000","last_fullsized":"2024-08-17T17:54:42.839027+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":88,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.838386+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:54:42.838386+0000","last_peered":"2024-08-17T17:54:42.838386+0000","last_clean":"2024-08-17T17:54:42.838386+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:54:42.838386+0000","last_undegraded":"2024-08-17T17:54:42.838386+0000","last_fullsized":"2024-08-17T17:54:42.838386+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1920,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.838944+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:54:42.838944+0000","last_peered":"2024-08-17T17:54:42.838944+0000","last_clean":"2024-08-17T17:54:42.838944+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:54:42.838944+0000","last_undegraded":"2024-08-17T17:54:42.838944+0000","last_fullsized":"2024-08-17T17:54:42.838944+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":657,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.843016+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:54:42.843016+0000","last_peered":"2024-08-17T17:54:42.843016+0000","last_clean":"2024-08-17T17:54:42.843016+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:54:42.843016+0000","last_undegraded":"2024-08-17T17:54:42.843016+0000","last_fullsized":"2024-08-17T17:54:42.843016+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1901,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.839110+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:54:42.839110+0000","last_peered":"2024-08-17T17:54:42.839110+0000","last_clean":"2024-08-17T17:54:42.839110+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:54:42.839110+0000","last_undegraded":"2024-08-17T17:54:42.839110+0000","last_fullsized":"2024-08-17T17:54:42.839110+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1230,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.849596+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:54:42.849596+0000","last_peered":"2024-08-17T17:54:42.849596+0000","last_clean":"2024-08-17T17:54:42.849596+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:54:42.849596+0000","last_undegraded":"2024-08-17T17:54:42.849596+0000","last_fullsized":"2024-08-17T17:54:42.849596+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":148,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.838614+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:54:42.838614+0000","last_peered":"2024-08-17T17:54:42.838614+0000","last_clean":"2024-08-17T17:54:42.838614+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:54:42.838614+0000","last_undegraded":"2024-08-17T17:54:42.838614+0000","last_fullsized":"2024-08-17T17:54:42.838614+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1912,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.839221+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:54:42.839221+0000","last_peered":"2024-08-17T17:54:42.839221+0000","last_clean":"2024-08-17T17:54:42.839221+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:54:42.839221+0000","last_undegraded":"2024-08-17T17:54:42.839221+0000","last_fullsized":"2024-08-17T17:54:42.839221+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1890,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.836739+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:54:42.836739+0000","last_peered":"2024-08-17T17:54:42.836739+0000","last_clean":"2024-08-17T17:54:42.836739+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:54:42.836739+0000","last_undegraded":"2024-08-17T17:54:42.836739+0000","last_fullsized":"2024-08-17T17:54:42.836739+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1910,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.841639+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:54:42.841639+0000","last_peered":"2024-08-17T17:54:42.841639+0000","last_clean":"2024-08-17T17:54:42.841639+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:54:42.841639+0000","last_undegraded":"2024-08-17T17:54:42.841639+0000","last_fullsized":"2024-08-17T17:54:42.841639+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1915,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.850337+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:54:42.850337+0000","last_peered":"2024-08-17T17:54:42.850337+0000","last_clean":"2024-08-17T17:54:42.850337+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:54:42.850337+0000","last_undegraded":"2024-08-17T17:54:42.850337+0000","last_fullsized":"2024-08-17T17:54:42.850337+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1896,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.860134+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:54:42.860134+0000","last_peered":"2024-08-17T17:54:42.860134+0000","last_clean":"2024-08-17T17:54:42.860134+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:54:42.860134+0000","last_undegraded":"2024-08-17T17:54:42.860134+0000","last_fullsized":"2024-08-17T17:54:42.860134+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1223,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.159989+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:54:29.159989+0000","last_peered":"2024-08-17T17:54:29.159989+0000","last_clean":"2024-08-17T17:54:29.159989+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:54:29.159989+0000","last_undegraded":"2024-08-17T17:54:29.159989+0000","last_fullsized":"2024-08-17T17:54:29.159989+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1169,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.158346+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:54:29.158346+0000","last_peered":"2024-08-17T17:54:29.158346+0000","last_clean":"2024-08-17T17:54:29.158346+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:54:29.158346+0000","last_undegraded":"2024-08-17T17:54:29.158346+0000","last_fullsized":"2024-08-17T17:54:29.158346+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1805,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.841955+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:54:42.841955+0000","last_peered":"2024-08-17T17:54:42.841955+0000","last_clean":"2024-08-17T17:54:42.841955+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:54:42.841955+0000","last_undegraded":"2024-08-17T17:54:42.841955+0000","last_fullsized":"2024-08-17T17:54:42.841955+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":136,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.842656+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:54:42.842656+0000","last_peered":"2024-08-17T17:54:42.842656+0000","last_clean":"2024-08-17T17:54:42.842656+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:54:42.842656+0000","last_undegraded":"2024-08-17T17:54:42.842656+0000","last_fullsized":"2024-08-17T17:54:42.842656+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1535,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.160035+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:54:29.160035+0000","last_peered":"2024-08-17T17:54:29.160035+0000","last_clean":"2024-08-17T17:54:29.160035+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:54:29.160035+0000","last_undegraded":"2024-08-17T17:54:29.160035+0000","last_fullsized":"2024-08-17T17:54:29.160035+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1896,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.839324+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:54:42.839324+0000","last_peered":"2024-08-17T17:54:42.839324+0000","last_clean":"2024-08-17T17:54:42.839324+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:54:42.839324+0000","last_undegraded":"2024-08-17T17:54:42.839324+0000","last_fullsized":"2024-08-17T17:54:42.839324+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1836,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.841520+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:54:42.841520+0000","last_peered":"2024-08-17T17:54:42.841520+0000","last_clean":"2024-08-17T17:54:42.841520+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:54:42.841520+0000","last_undegraded":"2024-08-17T17:54:42.841520+0000","last_fullsized":"2024-08-17T17:54:42.841520+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":503,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.160881+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:54:29.160881+0000","last_peered":"2024-08-17T17:54:29.160881+0000","last_clean":"2024-08-17T17:54:29.160881+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:54:29.160881+0000","last_undegraded":"2024-08-17T17:54:29.160881+0000","last_fullsized":"2024-08-17T17:54:29.160881+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":575,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.841486+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:54:42.841486+0000","last_peered":"2024-08-17T17:54:42.841486+0000","last_clean":"2024-08-17T17:54:42.841486+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:54:42.841486+0000","last_undegraded":"2024-08-17T17:54:42.841486+0000","last_fullsized":"2024-08-17T17:54:42.841486+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":591,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.843261+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:54:42.843261+0000","last_peered":"2024-08-17T17:54:42.843261+0000","last_clean":"2024-08-17T17:54:42.843261+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:54:42.843261+0000","last_undegraded":"2024-08-17T17:54:42.843261+0000","last_fullsized":"2024-08-17T17:54:42.843261+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1187,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.158232+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:54:29.158232+0000","last_peered":"2024-08-17T17:54:29.158232+0000","last_clean":"2024-08-17T17:54:29.158232+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:54:29.158232+0000","last_undegraded":"2024-08-17T17:54:29.158232+0000","last_fullsized":"2024-08-17T17:54:29.158232+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":607,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.842026+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:54:42.842026+0000","last_peered":"2024-08-17T17:54:42.842026+0000","last_clean":"2024-08-17T17:54:42.842026+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:54:42.842026+0000","last_undegraded":"2024-08-17T17:54:42.842026+0000","last_fullsized":"2024-08-17T17:54:42.842026+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1833,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.842336+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:54:42.842336+0000","last_peered":"2024-08-17T17:54:42.842336+0000","last_clean":"2024-08-17T17:54:42.842336+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:54:42.842336+0000","last_undegraded":"2024-08-17T17:54:42.842336+0000","last_fullsized":"2024-08-17T17:54:42.842336+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1928,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.850076+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:54:42.850076+0000","last_peered":"2024-08-17T17:54:42.850076+0000","last_clean":"2024-08-17T17:54:42.850076+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:54:42.850076+0000","last_undegraded":"2024-08-17T17:54:42.850076+0000","last_fullsized":"2024-08-17T17:54:42.850076+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":639,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.841921+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:54:42.841921+0000","last_peered":"2024-08-17T17:54:42.841921+0000","last_clean":"2024-08-17T17:54:42.841921+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:54:42.841921+0000","last_undegraded":"2024-08-17T17:54:42.841921+0000","last_fullsized":"2024-08-17T17:54:42.841921+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1176,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.158129+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:54:29.158129+0000","last_peered":"2024-08-17T17:54:29.158129+0000","last_clean":"2024-08-17T17:54:29.158129+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:54:29.158129+0000","last_undegraded":"2024-08-17T17:54:29.158129+0000","last_fullsized":"2024-08-17T17:54:29.158129+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1175,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.160231+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:54:29.160231+0000","last_peered":"2024-08-17T17:54:29.160231+0000","last_clean":"2024-08-17T17:54:29.160231+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:54:29.160231+0000","last_undegraded":"2024-08-17T17:54:29.160231+0000","last_fullsized":"2024-08-17T17:54:29.160231+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":624,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.836935+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:54:42.836935+0000","last_peered":"2024-08-17T17:54:42.836935+0000","last_clean":"2024-08-17T17:54:42.836935+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:54:42.836935+0000","last_undegraded":"2024-08-17T17:54:42.836935+0000","last_fullsized":"2024-08-17T17:54:42.836935+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1193,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.160448+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:54:29.160448+0000","last_peered":"2024-08-17T17:54:29.160448+0000","last_clean":"2024-08-17T17:54:29.160448+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:54:29.160448+0000","last_undegraded":"2024-08-17T17:54:29.160448+0000","last_fullsized":"2024-08-17T17:54:29.160448+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":77,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.159810+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:54:29.159810+0000","last_peered":"2024-08-17T17:54:29.159810+0000","last_clean":"2024-08-17T17:54:29.159810+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:54:29.159810+0000","last_undegraded":"2024-08-17T17:54:29.159810+0000","last_fullsized":"2024-08-17T17:54:29.159810+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1209,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.160428+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:54:29.160428+0000","last_peered":"2024-08-17T17:54:29.160428+0000","last_clean":"2024-08-17T17:54:29.160428+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:54:29.160428+0000","last_undegraded":"2024-08-17T17:54:29.160428+0000","last_fullsized":"2024-08-17T17:54:29.160428+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1226,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.157899+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:54:29.157899+0000","last_peered":"2024-08-17T17:54:29.157899+0000","last_clean":"2024-08-17T17:54:29.157899+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:54:29.157899+0000","last_undegraded":"2024-08-17T17:54:29.157899+0000","last_fullsized":"2024-08-17T17:54:29.157899+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1834,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.842253+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:54:42.842253+0000","last_peered":"2024-08-17T17:54:42.842253+0000","last_clean":"2024-08-17T17:54:42.842253+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:54:42.842253+0000","last_undegraded":"2024-08-17T17:54:42.842253+0000","last_fullsized":"2024-08-17T17:54:42.842253+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1170,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.160025+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:54:29.160025+0000","last_peered":"2024-08-17T17:54:29.160025+0000","last_clean":"2024-08-17T17:54:29.160025+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:54:29.160025+0000","last_undegraded":"2024-08-17T17:54:29.160025+0000","last_fullsized":"2024-08-17T17:54:29.160025+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1887,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.850193+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:54:42.850193+0000","last_peered":"2024-08-17T17:54:42.850193+0000","last_clean":"2024-08-17T17:54:42.850193+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:54:42.850193+0000","last_undegraded":"2024-08-17T17:54:42.850193+0000","last_fullsized":"2024-08-17T17:54:42.850193+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1888,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.849760+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:54:42.849760+0000","last_peered":"2024-08-17T17:54:42.849760+0000","last_clean":"2024-08-17T17:54:42.849760+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:54:42.849760+0000","last_undegraded":"2024-08-17T17:54:42.849760+0000","last_fullsized":"2024-08-17T17:54:42.849760+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":583,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.160092+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:54:29.160092+0000","last_peered":"2024-08-17T17:54:29.160092+0000","last_clean":"2024-08-17T17:54:29.160092+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:54:29.160092+0000","last_undegraded":"2024-08-17T17:54:29.160092+0000","last_fullsized":"2024-08-17T17:54:29.160092+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1195,"reported_epoch":1225,"state":"active+clean","last_fresh":"2024-08-17T17:54:29.160179+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:54:29.160179+0000","last_peered":"2024-08-17T17:54:29.160179+0000","last_clean":"2024-08-17T17:54:29.160179+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:54:29.160179+0000","last_undegraded":"2024-08-17T17:54:29.160179+0000","last_fullsized":"2024-08-17T17:54:29.160179+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1834,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.842178+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:54:42.842178+0000","last_peered":"2024-08-17T17:54:42.842178+0000","last_clean":"2024-08-17T17:54:42.842178+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:54:42.842178+0000","last_undegraded":"2024-08-17T17:54:42.842178+0000","last_fullsized":"2024-08-17T17:54:42.842178+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":4135387160,"num_objects":63102,"num_object_clones":0,"num_object_copies":189306,"num_objects_missing_on_primary":3416,"num_objects_missing":3416,"num_objects_degraded":17002,"num_objects_misplaced":10693,"num_objects_unfound":0,"num_objects_dirty":63102,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":146508,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35265,"num_bytes_recovered":2132672512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13143584768,"data_stored":13143572552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108582,"ondisk_log_size":108582,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234406,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2041792,"kb_used_data":1740280,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91703360,"statfs":{"total":95995035648,"available":93904240640,"internally_reserved":0,"allocated":1782046720,"data_stored":1777137387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496041,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2299332,"kb_used_data":1997820,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91445820,"statfs":{"total":95995035648,"available":93640519680,"internally_reserved":0,"allocated":2045767680,"data_stored":2040854752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757679,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326293,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2472384,"kb_used_data":2260084,"kb_used_omap":57,"kb_used_meta":212230,"kb_avail":91272768,"statfs":{"total":95995035648,"available":93463314432,"internally_reserved":0,"allocated":2314326016,"data_stored":2309424427,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":217324283},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405532,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2248900,"kb_used_data":2028220,"kb_used_omap":61,"kb_used_meta":220610,"kb_avail":91496252,"statfs":{"total":95995035648,"available":93692162048,"internally_reserved":0,"allocated":2076897280,"data_stored":2071992663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":225905592},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452067,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1699576,"kb_used_data":1515056,"kb_used_omap":54,"kb_used_meta":184457,"kb_avail":92045576,"statfs":{"total":95995035648,"available":94254669824,"internally_reserved":0,"allocated":1551417344,"data_stored":1546519716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":188884366},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474758,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1483532,"kb_used_data":1284416,"kb_used_omap":119,"kb_used_meta":198984,"kb_avail":92261620,"statfs":{"total":95995035648,"available":94475898880,"internally_reserved":0,"allocated":1315241984,"data_stored":1310329192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":203760350},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2,1,1,2,1,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019314,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1300500480,"data_stored":1300496408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1537736704,"data_stored":1537736704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2060976128,"data_stored":2060976128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2298220544,"data_stored":2298216472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1033240576,"data_stored":1033240576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1116930048,"data_stored":1116930048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2030309376,"data_stored":2030305304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1765670912,"data_stored":1765670912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:54:53.129 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:54:53.162 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:53.194 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:53.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:53.227 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:54:53.295 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:53.317 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:53.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:53.419 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:53.441 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:53.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:53.490 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:54:53.543 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:53.564 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:53.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:53.666 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:53.687 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:53.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:53.789 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:53.810 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:53.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:53.886 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:53 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:53.886 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:53 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:54:53.886 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:53 smithi179 ceph-mon[93519]: osdmap e1227: 8 total, 8 up, 6 in 2024-08-17T17:54:53.886 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:53 smithi179 ceph-mon[93519]: pgmap v1836: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 16926/187170 objects degraded (9.043%); 10566/187170 objects misplaced (5.645%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:53.912 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:53.933 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:53.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:53.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:53.989 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:53 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:53.989 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:53 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:54:53.990 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:53 smithi130 ceph-mon[105521]: osdmap e1227: 8 total, 8 up, 6 in 2024-08-17T17:54:53.990 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:53 smithi130 ceph-mon[105521]: pgmap v1836: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 16926/187170 objects degraded (9.043%); 10566/187170 objects misplaced (5.645%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:53.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:53 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:54:53.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:53 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:54:53.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:53 smithi130 ceph-mon[110068]: osdmap e1227: 8 total, 8 up, 6 in 2024-08-17T17:54:53.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:53 smithi130 ceph-mon[110068]: pgmap v1836: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 131 op/s; 16926/187170 objects degraded (9.043%); 10566/187170 objects misplaced (5.645%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:54.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:54.056 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:54.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:54.158 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:54.180 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:54.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:54.282 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:54.303 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:54.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:54.404 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:54.426 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:54.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:54.528 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:54.553 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:54.553 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:54.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:54.655 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:54.676 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:54.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:54.778 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:54.802 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:54.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:54.904 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:54.926 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:54.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:54.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:55.027 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:55.052 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:55.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:55.154 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:55.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.177 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:55.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:55.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:55 smithi179 ceph-mon[93519]: pgmap v1837: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 16926/185418 objects degraded (9.129%); 10476/185418 objects misplaced (5.650%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:55.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:55.300 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:55.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:55.402 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:55.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:55 smithi130 ceph-mon[105521]: pgmap v1837: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 16926/185418 objects degraded (9.129%); 10476/185418 objects misplaced (5.650%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:55 smithi130 ceph-mon[110068]: pgmap v1837: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 16926/185418 objects degraded (9.129%); 10476/185418 objects misplaced (5.650%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:54:55.423 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:55.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:55.508 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:54:55.508 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:54:55.526 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:55.549 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:55.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:55.651 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:55.673 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:55.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:55.775 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:55.800 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:55.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:55.903 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:55.923 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:55.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:55.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:56.018 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:54:56.025 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:56.085 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:56.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:56.180 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16926/185418 objects degraded (9.129%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:56.187 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:56.204 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:54:56.212 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16926/185418 objects degraded (9.129%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:56.212 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16926/185418 objects degraded (9.129%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:54:56.215 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:56.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:56.317 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:56.340 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:56.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:56.442 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:56.468 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:56.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:56.478 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:54:56.543 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:54:56.570 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:56.619 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:56.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:56.721 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:56.743 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:56.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:56.845 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:56.858 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:56.884 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:56.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:56.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:56.986 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:57.008 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:57.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:57.110 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:57.131 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:57.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:57.158 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:54:57.158 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 16802/185070 objects degraded (9.079%), 6 pgs degraded, 6 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1618,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1227,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":61690,"data_bytes":4041742309,"bytes_used":12321341440,"bytes_avail":563648872448,"bytes_total":575970213888,"degraded_objects":16802,"degraded_total":185070,"degraded_ratio":0.090787269681742042,"misplaced_objects":10422,"misplaced_total":185070,"misplaced_ratio":0.056313827200518723,"recovering_objects_per_sec":18,"recovering_bytes_per_sec":1225166,"recovering_keys_per_sec":0,"num_objects_recovered":187,"num_bytes_recovered":12255232,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":143},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":25,"modified":"2024-08-17T17:49:39.646037+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (7m)\n [==========================..] (remaining: 31s)","progress":0.93258428573608398,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:54:57.233 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:57.255 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:57.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:57 smithi179 ceph-mon[93519]: pgmap v1838: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 143 op/s; 16802/185070 objects degraded (9.079%); 10422/185070 objects misplaced (5.631%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:54:57.356 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:57.378 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:57.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:57.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:57 smithi130 ceph-mon[110068]: pgmap v1838: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 143 op/s; 16802/185070 objects degraded (9.079%); 10422/185070 objects misplaced (5.631%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:54:57.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:57 smithi130 ceph-mon[105521]: pgmap v1838: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 143 op/s; 16802/185070 objects degraded (9.079%); 10422/185070 objects misplaced (5.631%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:54:57.480 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:57.502 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:57.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:57.603 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:57.624 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:57.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:57.726 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:57.747 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:57.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:57.822 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:54:57.849 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.871 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:57.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:57.972 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:57.995 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:57.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:57.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:58.097 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:58.119 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:58.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:58.221 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:58.243 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:58.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:58 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3225339504' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:54:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:58.345 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:58.366 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:58.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:58.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:58 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3225339504' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:54:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:58.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:58 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3225339504' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:54:58.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:54:58.469 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:54:58.489 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:58.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:54:58.591 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:54:58.603 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:54:58.603 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:54:58.612 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:58.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:54:58.648 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:54:58.649 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:54:58.714 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:58.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.736 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:58.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:58.837 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:58.862 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:58.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:54:58.963 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:58.985 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:58.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:58.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:59.086 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:54:59.112 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:59.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:54:59.214 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:54:59.235 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:59.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:54:59.309 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:54:59 smithi179 ceph-mon[93519]: pgmap v1839: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 146 op/s; 16802/183549 objects degraded (9.154%); 10422/183549 objects misplaced (5.678%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:54:59.332 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:54:59.337 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:59.348 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:54:59 smithi130 ceph-mon[105521]: pgmap v1839: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 146 op/s; 16802/183549 objects degraded (9.154%); 10422/183549 objects misplaced (5.678%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:54:59.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:54:59 smithi130 ceph-mon[110068]: pgmap v1839: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 146 op/s; 16802/183549 objects degraded (9.154%); 10422/183549 objects misplaced (5.678%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:54:59.360 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:59.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:59.461 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:54:59.482 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:59.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:54:59.584 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:59.592 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:54:59.607 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:59.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:59.709 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:54:59.731 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:54:59.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:54:59.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:54:59.862 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:59.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:54:59.874 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:54:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:54:59] "GET /metrics HTTP/1.1" 200 36815 "" "Prometheus/2.51.0" 2024-08-17T17:54:59.964 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:54:59.987 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:54:59.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:54:59.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:00.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:00.110 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:00.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:00.212 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:00.219 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:55:00.219 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:55:00.219 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:55:00.219 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:55:00.219 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:55:00.219 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:55:00.219 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:55:00.219 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:55:00.219 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:55:00.220 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:55:00.220 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:55:00.220 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:55:00.220 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:55:00.220 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:55:00.220 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:55:00.220 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:55:00.233 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:00.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:00.335 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:00.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:00 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/583920874' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:00.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:55:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:55:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:55:00.356 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:00.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:00.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:00 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/583920874' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:00.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:00 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/583920874' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:00.458 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:00.486 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:00.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:00.588 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:00.609 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:00.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:00.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:00.738 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:00.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:00.839 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:00.866 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:00.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:00.968 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:00.990 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:00.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:00.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:01.091 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:01.115 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:01.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:01.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16802/183549 objects degraded (9.154%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:01 smithi130 ceph-mon[110068]: pgmap v1840: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 16802/183042 objects degraded (9.179%); 10334/183042 objects misplaced (5.646%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:55:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16802/183549 objects degraded (9.154%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:01 smithi130 ceph-mon[105521]: pgmap v1840: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 16802/183042 objects degraded (9.179%); 10334/183042 objects misplaced (5.646%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:55:01.216 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:01.238 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:01.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:01.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:01.341 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:01.363 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:01.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:01.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:01.457 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16802/183549 objects degraded (9.154%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:01.457 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:01 smithi179 ceph-mon[93519]: pgmap v1840: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 16802/183042 objects degraded (9.179%); 10334/183042 objects misplaced (5.646%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:55:01.458 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:01.465 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:01.488 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:01.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:01.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:01.505 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:55:01.505 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:55:01.506 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T17:55:01.506 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:55:01.589 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:01.611 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:01.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:01.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:01.713 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:01.740 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:01.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:01.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:01.882 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:01.904 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:01.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:01.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:02.006 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:02.028 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:02.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:02.130 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:02.151 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:02.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:02.252 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:02.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:02.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:02.311 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:55:02.375 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:02.398 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:02.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:02.500 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:02.520 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:02.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:02.605 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:55:02.623 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:02.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:02.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:02.727 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T17:55:02.758 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:02.786 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:02.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:02.888 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:02.909 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:02.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:02.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:02.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:02 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/279891598' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:02.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:02 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/279891598' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:02.945 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:55:03.010 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:03.032 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:03.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:03.050 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:02 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/279891598' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:03.134 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:03.155 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:03.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:03.257 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:03.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.280 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:03.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:03.381 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:03.403 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:03.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:03.505 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:03.526 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:03.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:03.628 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:03.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.650 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:03.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:03.751 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:03.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.773 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:03.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:03 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:03 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/279891598' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:03 smithi179 ceph-mon[93519]: osdmap e1228: 8 total, 8 up, 6 in 2024-08-17T17:55:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:03 smithi179 ceph-mon[93519]: pgmap v1842: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 16704/182742 objects degraded (9.141%); 10283/182742 objects misplaced (5.627%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:55:03.874 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:03.897 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:03.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:03.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:03.999 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:04.023 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:04.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:04.125 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:04.147 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:04.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:04.161 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:03 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:04.161 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:03 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/279891598' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:04.161 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:03 smithi130 ceph-mon[105521]: osdmap e1228: 8 total, 8 up, 6 in 2024-08-17T17:55:04.161 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:03 smithi130 ceph-mon[105521]: pgmap v1842: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 16704/182742 objects degraded (9.141%); 10283/182742 objects misplaced (5.627%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:55:04.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:03 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:04.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:03 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/279891598' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:04.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:03 smithi130 ceph-mon[110068]: osdmap e1228: 8 total, 8 up, 6 in 2024-08-17T17:55:04.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:03 smithi130 ceph-mon[110068]: pgmap v1842: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 16704/182742 objects degraded (9.141%); 10283/182742 objects misplaced (5.627%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:55:04.249 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:04.270 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:04.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:04.373 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:04.395 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:04.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:04.490 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:55:04.490 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:55:04.490 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T17:55:04.490 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T17:55:04.490 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:55:04.498 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:04.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.522 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:04.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:04.603 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:04.623 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:04.644 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:04.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:04.746 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:04.785 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:04.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:04.819 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:55:04.820 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:55:04.887 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:04.913 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:04.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:04.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:05.015 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:05.037 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:05.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:05.139 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:05.162 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:05.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:05.218 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:55:05.264 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:05.286 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:05.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:05.388 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:05.410 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:05.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:05.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:05 smithi130 ceph-mon[105521]: from='client.45683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:55:05.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:05 smithi130 ceph-mon[105521]: pgmap v1843: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 16704/181137 objects degraded (9.222%); 10194/181137 objects misplaced (5.628%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:55:05.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:05 smithi130 ceph-mon[110068]: from='client.45683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:55:05.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:05 smithi130 ceph-mon[110068]: pgmap v1843: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 16704/181137 objects degraded (9.222%); 10194/181137 objects misplaced (5.628%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:55:05.485 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:55:05.505 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:55:05.512 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:05.576 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:05.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:05 smithi179 ceph-mon[93519]: from='client.45683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:55:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:05 smithi179 ceph-mon[93519]: pgmap v1843: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 142 op/s; 16704/181137 objects degraded (9.222%); 10194/181137 objects misplaced (5.628%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:55:05.677 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:05.681 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1842,"stamp":"2024-08-17T17:55:03.064249+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3990886373,"num_objects":60914,"num_object_clones":0,"num_object_copies":182742,"num_objects_missing_on_primary":3285,"num_objects_missing":3285,"num_objects_degraded":16704,"num_objects_misplaced":10283,"num_objects_unfound":0,"num_objects_dirty":60914,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":149000,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35575,"num_bytes_recovered":2152864781,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108503,"ondisk_log_size":108503,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11884868,"kb_used_data":10481876,"kb_used_omap":295,"kb_used_meta":1402648,"kb_avail":550586044,"statfs":{"total":575970213888,"available":563800109056,"internally_reserved":0,"allocated":10733441024,"data_stored":10703942649,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1436312015},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-96731136,"num_objects":-1476,"num_object_clones":0,"num_object_copies":-4428,"num_objects_missing_on_primary":-88,"num_objects_missing":-88,"num_objects_degraded":-222,"num_objects_misplaced":-283,"num_objects_unfound":0,"num_objects_dirty":-1476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1476,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":201,"num_bytes_recovered":13172736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002439"},"pg_stats":[{"pgid":"45.f","version":"1227'9295","reported_seq":19828,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:59.706615+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:54:59.706615+0000","last_peered":"2024-08-17T17:54:59.706615+0000","last_clean":"2024-08-17T17:54:59.706615+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:54:59.706615+0000","last_undegraded":"2024-08-17T17:54:59.706615+0000","last_fullsized":"2024-08-17T17:54:59.706615+0000","mapping_epoch":1194,"log_start":"1170'2431","ondisk_log_start":"1170'2431","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6864,"log_dups_size":0,"ondisk_log_size":6864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253558784,"num_objects":3869,"num_object_clones":0,"num_object_copies":11607,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3869,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9295,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1227'9229","reported_seq":19696,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:58.171282+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:54:58.171282+0000","last_peered":"2024-08-17T17:54:58.171282+0000","last_clean":"2024-08-17T17:54:58.171282+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:54:58.171282+0000","last_undegraded":"2024-08-17T17:54:58.171282+0000","last_fullsized":"2024-08-17T17:54:58.171282+0000","mapping_epoch":1211,"log_start":"1170'2232","ondisk_log_start":"1170'2232","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6997,"log_dups_size":0,"ondisk_log_size":6997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246743040,"num_objects":3765,"num_object_clones":0,"num_object_copies":11295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9229,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1227'9526","reported_seq":20367,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:58.671057+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:54:58.671057+0000","last_peered":"2024-08-17T17:54:58.671057+0000","last_clean":"2024-08-17T17:54:58.671057+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:54:58.671057+0000","last_undegraded":"2024-08-17T17:54:58.671057+0000","last_fullsized":"2024-08-17T17:54:58.671057+0000","mapping_epoch":1205,"log_start":"1174'3568","ondisk_log_start":"1174'3568","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5958,"log_dups_size":0,"ondisk_log_size":5958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252379160,"num_objects":3852,"num_object_clones":0,"num_object_copies":11556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3852,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9526,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1227'9281","reported_seq":9349,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:58.019280+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:54:58.019280+0000","last_peered":"2024-08-17T17:54:58.019280+0000","last_clean":"2024-08-17T17:54:58.019280+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:54:58.019280+0000","last_undegraded":"2024-08-17T17:54:58.019280+0000","last_fullsized":"2024-08-17T17:54:58.019280+0000","mapping_epoch":1167,"log_start":"1170'2589","ondisk_log_start":"1170'2589","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6692,"log_dups_size":0,"ondisk_log_size":6692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243728384,"num_objects":3719,"num_object_clones":0,"num_object_copies":11157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9281,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1227'9316","reported_seq":14563,"reported_epoch":1227,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:01.562281+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:55:01.562281+0000","last_peered":"2024-08-17T17:55:01.562281+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:55:01.562281+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1174'3426","ondisk_log_start":"1174'3426","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5890,"log_dups_size":0,"ondisk_log_size":5890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248905728,"num_objects":3798,"num_object_clones":0,"num_object_copies":11394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":3798,"num_objects_unfound":0,"num_objects_dirty":3798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9316,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":3280},{"shards":"1,4,5","objects":312}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1227'9356","reported_seq":9424,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:57.889824+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:54:57.889824+0000","last_peered":"2024-08-17T17:54:57.889824+0000","last_clean":"2024-08-17T17:54:57.889824+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:54:57.889824+0000","last_undegraded":"2024-08-17T17:54:57.889824+0000","last_fullsized":"2024-08-17T17:54:57.889824+0000","mapping_epoch":1167,"log_start":"1170'2677","ondisk_log_start":"1170'2677","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6679,"log_dups_size":0,"ondisk_log_size":6679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244973568,"num_objects":3738,"num_object_clones":0,"num_object_copies":11214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9356,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1227'9342","reported_seq":9410,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:57.236723+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:54:57.236723+0000","last_peered":"2024-08-17T17:54:57.236723+0000","last_clean":"2024-08-17T17:54:57.236723+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:54:57.236723+0000","last_undegraded":"2024-08-17T17:54:57.236723+0000","last_fullsized":"2024-08-17T17:54:57.236723+0000","mapping_epoch":1167,"log_start":"1171'2669","ondisk_log_start":"1171'2669","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6673,"log_dups_size":0,"ondisk_log_size":6673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254541824,"num_objects":3884,"num_object_clones":0,"num_object_copies":11652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9342,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1227'9302","reported_seq":17863,"reported_epoch":1227,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:01.799352+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:55:01.799352+0000","last_peered":"2024-08-17T17:55:01.799352+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:55:01.799352+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1174'3375","ondisk_log_start":"1174'3375","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5927,"log_dups_size":0,"ondisk_log_size":5927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241696768,"num_objects":3688,"num_object_clones":0,"num_object_copies":11064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2669,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9302,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2807,"num_bytes_recovered":183959552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":301},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1227'9215","reported_seq":9293,"reported_epoch":1227,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:57.912494+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:54:57.912494+0000","last_peered":"2024-08-17T17:54:57.912494+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:54:57.912494+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'2477","ondisk_log_start":"1170'2477","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6738,"log_dups_size":0,"ondisk_log_size":6738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253034496,"num_objects":3861,"num_object_clones":0,"num_object_copies":11583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9215,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":3861},{"shards":"1,6","objects":-1972}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1227'9306","reported_seq":12386,"reported_epoch":1227,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:58.675486+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:54:58.675486+0000","last_peered":"2024-08-17T17:54:58.675486+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:54:58.675486+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1174'3406","ondisk_log_start":"1174'3406","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5900,"log_dups_size":0,"ondisk_log_size":5900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253493248,"num_objects":3868,"num_object_clones":0,"num_object_copies":11604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9306,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":2307},{"shards":"1,3,5","objects":309}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1227'9225","reported_seq":9361,"reported_epoch":1227,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:58.847175+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:54:58.847175+0000","last_peered":"2024-08-17T17:54:58.847175+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:54:58.847175+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1170'100","ondisk_log_start":"1170'100","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9125,"log_dups_size":0,"ondisk_log_size":9125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252248064,"num_objects":3849,"num_object_clones":0,"num_object_copies":11547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9225,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-1304}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1227'9234","reported_seq":15430,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:58.471441+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:54:58.471441+0000","last_peered":"2024-08-17T17:54:58.471441+0000","last_clean":"2024-08-17T17:54:58.471441+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:54:58.471441+0000","last_undegraded":"2024-08-17T17:54:58.471441+0000","last_fullsized":"2024-08-17T17:54:58.471441+0000","mapping_epoch":1183,"log_start":"1170'100","ondisk_log_start":"1170'100","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9134,"log_dups_size":0,"ondisk_log_size":9134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249823232,"num_objects":3812,"num_object_clones":0,"num_object_copies":11436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9234,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1227'9364","reported_seq":9498,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:58.689191+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:54:58.689191+0000","last_peered":"2024-08-17T17:54:58.689191+0000","last_clean":"2024-08-17T17:54:58.689191+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:54:58.689191+0000","last_undegraded":"2024-08-17T17:54:58.689191+0000","last_fullsized":"2024-08-17T17:54:58.689191+0000","mapping_epoch":1167,"log_start":"1174'3391","ondisk_log_start":"1174'3391","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5973,"log_dups_size":0,"ondisk_log_size":5973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248774656,"num_objects":3796,"num_object_clones":0,"num_object_copies":11388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9364,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1227'9214","reported_seq":9282,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:58.035821+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:54:58.035821+0000","last_peered":"2024-08-17T17:54:58.035821+0000","last_clean":"2024-08-17T17:54:58.035821+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:54:58.035821+0000","last_undegraded":"2024-08-17T17:54:58.035821+0000","last_fullsized":"2024-08-17T17:54:58.035821+0000","mapping_epoch":1167,"log_start":"1170'2225","ondisk_log_start":"1170'2225","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6989,"log_dups_size":0,"ondisk_log_size":6989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243793920,"num_objects":3720,"num_object_clones":0,"num_object_copies":11160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9214,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1227'9437","reported_seq":21336,"reported_epoch":1227,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:54:59.763057+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:54:59.763057+0000","last_peered":"2024-08-17T17:54:59.763057+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:54:59.763057+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1170'2591","ondisk_log_start":"1170'2591","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6846,"log_dups_size":0,"ondisk_log_size":6846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257359872,"num_objects":3927,"num_object_clones":0,"num_object_copies":11781,"num_objects_missing_on_primary":3285,"num_objects_missing":3285,"num_objects_degraded":265,"num_objects_misplaced":6485,"num_objects_unfound":0,"num_objects_dirty":3927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9437,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4861,"num_bytes_recovered":318570496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,6","objects":13},{"shards":"0,4,6","objects":1356},{"shards":"4,5","objects":265},{"shards":"4,5,6","objects":2293}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1227'9080","reported_seq":22400,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:58.685469+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:54:58.685469+0000","last_peered":"2024-08-17T17:54:58.685469+0000","last_clean":"2024-08-17T17:54:58.685469+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:54:58.685469+0000","last_undegraded":"2024-08-17T17:54:58.685469+0000","last_fullsized":"2024-08-17T17:54:58.685469+0000","mapping_epoch":1196,"log_start":"1174'3169","ondisk_log_start":"1174'3169","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5911,"log_dups_size":0,"ondisk_log_size":5911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245235712,"num_objects":3742,"num_object_clones":0,"num_object_copies":11226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9080,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1911,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.814272+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:54:52.814272+0000","last_peered":"2024-08-17T17:54:52.814272+0000","last_clean":"2024-08-17T17:54:52.814272+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:54:52.814272+0000","last_undegraded":"2024-08-17T17:54:52.814272+0000","last_fullsized":"2024-08-17T17:54:52.814272+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":504,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.866810+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:54:42.866810+0000","last_peered":"2024-08-17T17:54:42.866810+0000","last_clean":"2024-08-17T17:54:42.866810+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:54:42.866810+0000","last_undegraded":"2024-08-17T17:54:42.866810+0000","last_fullsized":"2024-08-17T17:54:42.866810+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1830,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.855573+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:54:52.855573+0000","last_peered":"2024-08-17T17:54:52.855573+0000","last_clean":"2024-08-17T17:54:52.855573+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:54:52.855573+0000","last_undegraded":"2024-08-17T17:54:52.855573+0000","last_fullsized":"2024-08-17T17:54:52.855573+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1854,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.854474+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:54:52.854474+0000","last_peered":"2024-08-17T17:54:52.854474+0000","last_clean":"2024-08-17T17:54:52.854474+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:54:52.854474+0000","last_undegraded":"2024-08-17T17:54:52.854474+0000","last_fullsized":"2024-08-17T17:54:52.854474+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":66,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.867403+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:54:42.867403+0000","last_peered":"2024-08-17T17:54:42.867403+0000","last_clean":"2024-08-17T17:54:42.867403+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:54:42.867403+0000","last_undegraded":"2024-08-17T17:54:42.867403+0000","last_fullsized":"2024-08-17T17:54:42.867403+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1248,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.854531+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:54:52.854531+0000","last_peered":"2024-08-17T17:54:52.854531+0000","last_clean":"2024-08-17T17:54:52.854531+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:54:52.854531+0000","last_undegraded":"2024-08-17T17:54:52.854531+0000","last_fullsized":"2024-08-17T17:54:52.854531+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":106,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.874141+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:54:52.874141+0000","last_peered":"2024-08-17T17:54:52.874141+0000","last_clean":"2024-08-17T17:54:52.874141+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:54:52.874141+0000","last_undegraded":"2024-08-17T17:54:52.874141+0000","last_fullsized":"2024-08-17T17:54:52.874141+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":626,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.854834+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:54:52.854834+0000","last_peered":"2024-08-17T17:54:52.854834+0000","last_clean":"2024-08-17T17:54:52.854834+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:54:52.854834+0000","last_undegraded":"2024-08-17T17:54:52.854834+0000","last_fullsized":"2024-08-17T17:54:52.854834+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1869,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.814149+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:54:52.814149+0000","last_peered":"2024-08-17T17:54:52.814149+0000","last_clean":"2024-08-17T17:54:52.814149+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:54:52.814149+0000","last_undegraded":"2024-08-17T17:54:52.814149+0000","last_fullsized":"2024-08-17T17:54:52.814149+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1871,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.777890+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:54:52.777890+0000","last_peered":"2024-08-17T17:54:52.777890+0000","last_clean":"2024-08-17T17:54:52.777890+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:54:52.777890+0000","last_undegraded":"2024-08-17T17:54:52.777890+0000","last_fullsized":"2024-08-17T17:54:52.777890+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":91,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.874123+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:54:52.874123+0000","last_peered":"2024-08-17T17:54:52.874123+0000","last_clean":"2024-08-17T17:54:52.874123+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:54:52.874123+0000","last_undegraded":"2024-08-17T17:54:52.874123+0000","last_fullsized":"2024-08-17T17:54:52.874123+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1843,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.855459+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:54:52.855459+0000","last_peered":"2024-08-17T17:54:52.855459+0000","last_clean":"2024-08-17T17:54:52.855459+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:54:52.855459+0000","last_undegraded":"2024-08-17T17:54:52.855459+0000","last_fullsized":"2024-08-17T17:54:52.855459+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1898,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.873707+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:54:52.873707+0000","last_peered":"2024-08-17T17:54:52.873707+0000","last_clean":"2024-08-17T17:54:52.873707+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:54:52.873707+0000","last_undegraded":"2024-08-17T17:54:52.873707+0000","last_fullsized":"2024-08-17T17:54:52.873707+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1892,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.814269+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:54:52.814269+0000","last_peered":"2024-08-17T17:54:52.814269+0000","last_clean":"2024-08-17T17:54:52.814269+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:54:52.814269+0000","last_undegraded":"2024-08-17T17:54:52.814269+0000","last_fullsized":"2024-08-17T17:54:52.814269+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1865,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.856029+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:54:52.856029+0000","last_peered":"2024-08-17T17:54:52.856029+0000","last_clean":"2024-08-17T17:54:52.856029+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:54:52.856029+0000","last_undegraded":"2024-08-17T17:54:52.856029+0000","last_fullsized":"2024-08-17T17:54:52.856029+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1201,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.867609+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:54:42.867609+0000","last_peered":"2024-08-17T17:54:42.867609+0000","last_clean":"2024-08-17T17:54:42.867609+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:54:42.867609+0000","last_undegraded":"2024-08-17T17:54:42.867609+0000","last_fullsized":"2024-08-17T17:54:42.867609+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1836,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.855886+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:54:52.855886+0000","last_peered":"2024-08-17T17:54:52.855886+0000","last_clean":"2024-08-17T17:54:52.855886+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:54:52.855886+0000","last_undegraded":"2024-08-17T17:54:52.855886+0000","last_fullsized":"2024-08-17T17:54:52.855886+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1886,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.814378+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:54:52.814378+0000","last_peered":"2024-08-17T17:54:52.814378+0000","last_clean":"2024-08-17T17:54:52.814378+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:54:52.814378+0000","last_undegraded":"2024-08-17T17:54:52.814378+0000","last_fullsized":"2024-08-17T17:54:52.814378+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1792,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.854384+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:54:52.854384+0000","last_peered":"2024-08-17T17:54:52.854384+0000","last_clean":"2024-08-17T17:54:52.854384+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:54:52.854384+0000","last_undegraded":"2024-08-17T17:54:52.854384+0000","last_fullsized":"2024-08-17T17:54:52.854384+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":520,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.848526+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:54:42.848526+0000","last_peered":"2024-08-17T17:54:42.848526+0000","last_clean":"2024-08-17T17:54:42.848526+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:54:42.848526+0000","last_undegraded":"2024-08-17T17:54:42.848526+0000","last_fullsized":"2024-08-17T17:54:42.848526+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1197,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.846391+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:54:42.846391+0000","last_peered":"2024-08-17T17:54:42.846391+0000","last_clean":"2024-08-17T17:54:42.846391+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:54:42.846391+0000","last_undegraded":"2024-08-17T17:54:42.846391+0000","last_fullsized":"2024-08-17T17:54:42.846391+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1891,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.873982+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:54:52.873982+0000","last_peered":"2024-08-17T17:54:52.873982+0000","last_clean":"2024-08-17T17:54:52.873982+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:54:52.873982+0000","last_undegraded":"2024-08-17T17:54:52.873982+0000","last_fullsized":"2024-08-17T17:54:52.873982+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":358,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.846499+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:54:42.846499+0000","last_peered":"2024-08-17T17:54:42.846499+0000","last_clean":"2024-08-17T17:54:42.846499+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:54:42.846499+0000","last_undegraded":"2024-08-17T17:54:42.846499+0000","last_fullsized":"2024-08-17T17:54:42.846499+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":504,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.867098+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:54:42.867098+0000","last_peered":"2024-08-17T17:54:42.867098+0000","last_clean":"2024-08-17T17:54:42.867098+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:54:42.867098+0000","last_undegraded":"2024-08-17T17:54:42.867098+0000","last_fullsized":"2024-08-17T17:54:42.867098+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1182,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.866946+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:54:42.866946+0000","last_peered":"2024-08-17T17:54:42.866946+0000","last_clean":"2024-08-17T17:54:42.866946+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:54:42.866946+0000","last_undegraded":"2024-08-17T17:54:42.866946+0000","last_fullsized":"2024-08-17T17:54:42.866946+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1190,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.867676+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:54:42.867676+0000","last_peered":"2024-08-17T17:54:42.867676+0000","last_clean":"2024-08-17T17:54:42.867676+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:54:42.867676+0000","last_undegraded":"2024-08-17T17:54:42.867676+0000","last_fullsized":"2024-08-17T17:54:42.867676+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1939,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.814121+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:54:52.814121+0000","last_peered":"2024-08-17T17:54:52.814121+0000","last_clean":"2024-08-17T17:54:52.814121+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:54:52.814121+0000","last_undegraded":"2024-08-17T17:54:52.814121+0000","last_fullsized":"2024-08-17T17:54:52.814121+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1822,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.855993+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:54:52.855993+0000","last_peered":"2024-08-17T17:54:52.855993+0000","last_clean":"2024-08-17T17:54:52.855993+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:54:52.855993+0000","last_undegraded":"2024-08-17T17:54:52.855993+0000","last_fullsized":"2024-08-17T17:54:52.855993+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1897,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.874425+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:54:52.874425+0000","last_peered":"2024-08-17T17:54:52.874425+0000","last_clean":"2024-08-17T17:54:52.874425+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:54:52.874425+0000","last_undegraded":"2024-08-17T17:54:52.874425+0000","last_fullsized":"2024-08-17T17:54:52.874425+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1824,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.855224+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:54:52.855224+0000","last_peered":"2024-08-17T17:54:52.855224+0000","last_clean":"2024-08-17T17:54:52.855224+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:54:52.855224+0000","last_undegraded":"2024-08-17T17:54:52.855224+0000","last_fullsized":"2024-08-17T17:54:52.855224+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1857,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.814303+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:54:52.814303+0000","last_peered":"2024-08-17T17:54:52.814303+0000","last_clean":"2024-08-17T17:54:52.814303+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:54:52.814303+0000","last_undegraded":"2024-08-17T17:54:52.814303+0000","last_fullsized":"2024-08-17T17:54:52.814303+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1217,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.846589+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:54:42.846589+0000","last_peered":"2024-08-17T17:54:42.846589+0000","last_clean":"2024-08-17T17:54:42.846589+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:54:42.846589+0000","last_undegraded":"2024-08-17T17:54:42.846589+0000","last_fullsized":"2024-08-17T17:54:42.846589+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1903,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.874327+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:54:52.874327+0000","last_peered":"2024-08-17T17:54:52.874327+0000","last_clean":"2024-08-17T17:54:52.874327+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:54:52.874327+0000","last_undegraded":"2024-08-17T17:54:52.874327+0000","last_fullsized":"2024-08-17T17:54:52.874327+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":90,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.873736+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:54:52.873736+0000","last_peered":"2024-08-17T17:54:52.873736+0000","last_clean":"2024-08-17T17:54:52.873736+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:54:52.873736+0000","last_undegraded":"2024-08-17T17:54:52.873736+0000","last_fullsized":"2024-08-17T17:54:52.873736+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1922,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.874245+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:54:52.874245+0000","last_peered":"2024-08-17T17:54:52.874245+0000","last_clean":"2024-08-17T17:54:52.874245+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:54:52.874245+0000","last_undegraded":"2024-08-17T17:54:52.874245+0000","last_fullsized":"2024-08-17T17:54:52.874245+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":659,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.855368+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:54:52.855368+0000","last_peered":"2024-08-17T17:54:52.855368+0000","last_clean":"2024-08-17T17:54:52.855368+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:54:52.855368+0000","last_undegraded":"2024-08-17T17:54:52.855368+0000","last_fullsized":"2024-08-17T17:54:52.855368+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1903,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.874406+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:54:52.874406+0000","last_peered":"2024-08-17T17:54:52.874406+0000","last_clean":"2024-08-17T17:54:52.874406+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:54:52.874406+0000","last_undegraded":"2024-08-17T17:54:52.874406+0000","last_fullsized":"2024-08-17T17:54:52.874406+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1232,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.813926+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:54:52.813926+0000","last_peered":"2024-08-17T17:54:52.813926+0000","last_clean":"2024-08-17T17:54:52.813926+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:54:52.813926+0000","last_undegraded":"2024-08-17T17:54:52.813926+0000","last_fullsized":"2024-08-17T17:54:52.813926+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":150,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.873903+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:54:52.873903+0000","last_peered":"2024-08-17T17:54:52.873903+0000","last_clean":"2024-08-17T17:54:52.873903+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:54:52.873903+0000","last_undegraded":"2024-08-17T17:54:52.873903+0000","last_fullsized":"2024-08-17T17:54:52.873903+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1914,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.874424+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:54:52.874424+0000","last_peered":"2024-08-17T17:54:52.874424+0000","last_clean":"2024-08-17T17:54:52.874424+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:54:52.874424+0000","last_undegraded":"2024-08-17T17:54:52.874424+0000","last_fullsized":"2024-08-17T17:54:52.874424+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1892,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.873651+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:54:52.873651+0000","last_peered":"2024-08-17T17:54:52.873651+0000","last_clean":"2024-08-17T17:54:52.873651+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:54:52.873651+0000","last_undegraded":"2024-08-17T17:54:52.873651+0000","last_fullsized":"2024-08-17T17:54:52.873651+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1912,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.777647+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:54:52.777647+0000","last_peered":"2024-08-17T17:54:52.777647+0000","last_clean":"2024-08-17T17:54:52.777647+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:54:52.777647+0000","last_undegraded":"2024-08-17T17:54:52.777647+0000","last_fullsized":"2024-08-17T17:54:52.777647+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1918,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.814403+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:54:52.814403+0000","last_peered":"2024-08-17T17:54:52.814403+0000","last_clean":"2024-08-17T17:54:52.814403+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:54:52.814403+0000","last_undegraded":"2024-08-17T17:54:52.814403+0000","last_fullsized":"2024-08-17T17:54:52.814403+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1898,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.847299+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:54:52.847299+0000","last_peered":"2024-08-17T17:54:52.847299+0000","last_clean":"2024-08-17T17:54:52.847299+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:54:52.847299+0000","last_undegraded":"2024-08-17T17:54:52.847299+0000","last_fullsized":"2024-08-17T17:54:52.847299+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1224,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.846508+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:54:42.846508+0000","last_peered":"2024-08-17T17:54:42.846508+0000","last_clean":"2024-08-17T17:54:42.846508+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:54:42.846508+0000","last_undegraded":"2024-08-17T17:54:42.846508+0000","last_fullsized":"2024-08-17T17:54:42.846508+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1170,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.867403+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:54:42.867403+0000","last_peered":"2024-08-17T17:54:42.867403+0000","last_clean":"2024-08-17T17:54:42.867403+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:54:42.867403+0000","last_undegraded":"2024-08-17T17:54:42.867403+0000","last_fullsized":"2024-08-17T17:54:42.867403+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1807,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.854659+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:54:52.854659+0000","last_peered":"2024-08-17T17:54:52.854659+0000","last_clean":"2024-08-17T17:54:52.854659+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:54:52.854659+0000","last_undegraded":"2024-08-17T17:54:52.854659+0000","last_fullsized":"2024-08-17T17:54:52.854659+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":138,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.777763+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:54:52.777763+0000","last_peered":"2024-08-17T17:54:52.777763+0000","last_clean":"2024-08-17T17:54:52.777763+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:54:52.777763+0000","last_undegraded":"2024-08-17T17:54:52.777763+0000","last_fullsized":"2024-08-17T17:54:52.777763+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1536,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.846228+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:54:42.846228+0000","last_peered":"2024-08-17T17:54:42.846228+0000","last_clean":"2024-08-17T17:54:42.846228+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:54:42.846228+0000","last_undegraded":"2024-08-17T17:54:42.846228+0000","last_fullsized":"2024-08-17T17:54:42.846228+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1898,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.874502+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:54:52.874502+0000","last_peered":"2024-08-17T17:54:52.874502+0000","last_clean":"2024-08-17T17:54:52.874502+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:54:52.874502+0000","last_undegraded":"2024-08-17T17:54:52.874502+0000","last_fullsized":"2024-08-17T17:54:52.874502+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1838,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.855952+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:54:52.855952+0000","last_peered":"2024-08-17T17:54:52.855952+0000","last_clean":"2024-08-17T17:54:52.855952+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:54:52.855952+0000","last_undegraded":"2024-08-17T17:54:52.855952+0000","last_fullsized":"2024-08-17T17:54:52.855952+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":504,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.867384+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:54:42.867384+0000","last_peered":"2024-08-17T17:54:42.867384+0000","last_clean":"2024-08-17T17:54:42.867384+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:54:42.867384+0000","last_undegraded":"2024-08-17T17:54:42.867384+0000","last_fullsized":"2024-08-17T17:54:42.867384+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":577,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.777465+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:54:52.777465+0000","last_peered":"2024-08-17T17:54:52.777465+0000","last_clean":"2024-08-17T17:54:52.777465+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:54:52.777465+0000","last_undegraded":"2024-08-17T17:54:52.777465+0000","last_fullsized":"2024-08-17T17:54:52.777465+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":593,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.777998+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:54:52.777998+0000","last_peered":"2024-08-17T17:54:52.777998+0000","last_clean":"2024-08-17T17:54:52.777998+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:54:52.777998+0000","last_undegraded":"2024-08-17T17:54:52.777998+0000","last_fullsized":"2024-08-17T17:54:52.777998+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1188,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.867342+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:54:42.867342+0000","last_peered":"2024-08-17T17:54:42.867342+0000","last_clean":"2024-08-17T17:54:42.867342+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:54:42.867342+0000","last_undegraded":"2024-08-17T17:54:42.867342+0000","last_fullsized":"2024-08-17T17:54:42.867342+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":609,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.854969+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:54:52.854969+0000","last_peered":"2024-08-17T17:54:52.854969+0000","last_clean":"2024-08-17T17:54:52.854969+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:54:52.854969+0000","last_undegraded":"2024-08-17T17:54:52.854969+0000","last_fullsized":"2024-08-17T17:54:52.854969+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1835,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.854861+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:54:52.854861+0000","last_peered":"2024-08-17T17:54:52.854861+0000","last_clean":"2024-08-17T17:54:52.854861+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:54:52.854861+0000","last_undegraded":"2024-08-17T17:54:52.854861+0000","last_fullsized":"2024-08-17T17:54:52.854861+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1930,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.814350+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:54:52.814350+0000","last_peered":"2024-08-17T17:54:52.814350+0000","last_clean":"2024-08-17T17:54:52.814350+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:54:52.814350+0000","last_undegraded":"2024-08-17T17:54:52.814350+0000","last_fullsized":"2024-08-17T17:54:52.814350+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":642,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.854926+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:54:52.854926+0000","last_peered":"2024-08-17T17:54:52.854926+0000","last_clean":"2024-08-17T17:54:52.854926+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:54:52.854926+0000","last_undegraded":"2024-08-17T17:54:52.854926+0000","last_fullsized":"2024-08-17T17:54:52.854926+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1177,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.867157+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:54:42.867157+0000","last_peered":"2024-08-17T17:54:42.867157+0000","last_clean":"2024-08-17T17:54:42.867157+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:54:42.867157+0000","last_undegraded":"2024-08-17T17:54:42.867157+0000","last_fullsized":"2024-08-17T17:54:42.867157+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1176,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.846529+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:54:42.846529+0000","last_peered":"2024-08-17T17:54:42.846529+0000","last_clean":"2024-08-17T17:54:42.846529+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:54:42.846529+0000","last_undegraded":"2024-08-17T17:54:42.846529+0000","last_fullsized":"2024-08-17T17:54:42.846529+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":626,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.873782+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:54:52.873782+0000","last_peered":"2024-08-17T17:54:52.873782+0000","last_clean":"2024-08-17T17:54:52.873782+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:54:52.873782+0000","last_undegraded":"2024-08-17T17:54:52.873782+0000","last_fullsized":"2024-08-17T17:54:52.873782+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1194,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.848686+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:54:42.848686+0000","last_peered":"2024-08-17T17:54:42.848686+0000","last_clean":"2024-08-17T17:54:42.848686+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:54:42.848686+0000","last_undegraded":"2024-08-17T17:54:42.848686+0000","last_fullsized":"2024-08-17T17:54:42.848686+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":78,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.846156+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:54:42.846156+0000","last_peered":"2024-08-17T17:54:42.846156+0000","last_clean":"2024-08-17T17:54:42.846156+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:54:42.846156+0000","last_undegraded":"2024-08-17T17:54:42.846156+0000","last_fullsized":"2024-08-17T17:54:42.846156+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1210,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.846617+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:54:42.846617+0000","last_peered":"2024-08-17T17:54:42.846617+0000","last_clean":"2024-08-17T17:54:42.846617+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:54:42.846617+0000","last_undegraded":"2024-08-17T17:54:42.846617+0000","last_fullsized":"2024-08-17T17:54:42.846617+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1227,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.867005+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:54:42.867005+0000","last_peered":"2024-08-17T17:54:42.867005+0000","last_clean":"2024-08-17T17:54:42.867005+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:54:42.867005+0000","last_undegraded":"2024-08-17T17:54:42.867005+0000","last_fullsized":"2024-08-17T17:54:42.867005+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1836,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.855080+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:54:52.855080+0000","last_peered":"2024-08-17T17:54:52.855080+0000","last_clean":"2024-08-17T17:54:52.855080+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:54:52.855080+0000","last_undegraded":"2024-08-17T17:54:52.855080+0000","last_fullsized":"2024-08-17T17:54:52.855080+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1171,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.846500+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:54:42.846500+0000","last_peered":"2024-08-17T17:54:42.846500+0000","last_clean":"2024-08-17T17:54:42.846500+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:54:42.846500+0000","last_undegraded":"2024-08-17T17:54:42.846500+0000","last_fullsized":"2024-08-17T17:54:42.846500+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1889,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.814169+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:54:52.814169+0000","last_peered":"2024-08-17T17:54:52.814169+0000","last_clean":"2024-08-17T17:54:52.814169+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:54:52.814169+0000","last_undegraded":"2024-08-17T17:54:52.814169+0000","last_fullsized":"2024-08-17T17:54:52.814169+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1890,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.813966+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:54:52.813966+0000","last_peered":"2024-08-17T17:54:52.813966+0000","last_clean":"2024-08-17T17:54:52.813966+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:54:52.813966+0000","last_undegraded":"2024-08-17T17:54:52.813966+0000","last_fullsized":"2024-08-17T17:54:52.813966+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":584,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.846500+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:54:42.846500+0000","last_peered":"2024-08-17T17:54:42.846500+0000","last_clean":"2024-08-17T17:54:42.846500+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:54:42.846500+0000","last_undegraded":"2024-08-17T17:54:42.846500+0000","last_fullsized":"2024-08-17T17:54:42.846500+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1196,"reported_epoch":1226,"state":"active+clean","last_fresh":"2024-08-17T17:54:42.846801+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:54:42.846801+0000","last_peered":"2024-08-17T17:54:42.846801+0000","last_clean":"2024-08-17T17:54:42.846801+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:54:42.846801+0000","last_undegraded":"2024-08-17T17:54:42.846801+0000","last_fullsized":"2024-08-17T17:54:42.846801+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1836,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.855091+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:54:52.855091+0000","last_peered":"2024-08-17T17:54:52.855091+0000","last_clean":"2024-08-17T17:54:52.855091+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:54:52.855091+0000","last_undegraded":"2024-08-17T17:54:52.855091+0000","last_fullsized":"2024-08-17T17:54:52.855091+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":3990290456,"num_objects":60888,"num_object_clones":0,"num_object_copies":182664,"num_objects_missing_on_primary":3285,"num_objects_missing":3285,"num_objects_degraded":16704,"num_objects_misplaced":10283,"num_objects_unfound":0,"num_objects_dirty":60888,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":148722,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35546,"num_bytes_recovered":2151088128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12753580032,"data_stored":12753567816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108296,"ondisk_log_size":108296,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234409,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1990752,"kb_used_data":1685080,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91754400,"statfs":{"total":95995035648,"available":93956505600,"internally_reserved":0,"allocated":1725521920,"data_stored":1720597715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496044,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2244772,"kb_used_data":1934940,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91500380,"statfs":{"total":95995035648,"available":93696389120,"internally_reserved":0,"allocated":1981378560,"data_stored":1976450760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757683,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326296,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2407436,"kb_used_data":2190980,"kb_used_omap":57,"kb_used_meta":216390,"kb_avail":91337716,"statfs":{"total":95995035648,"available":93529821184,"internally_reserved":0,"allocated":2243563520,"data_stored":2238654495,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":221584123},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405535,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2167828,"kb_used_data":1959628,"kb_used_omap":61,"kb_used_meta":208130,"kb_avail":91577324,"statfs":{"total":95995035648,"available":93775179776,"internally_reserved":0,"allocated":2006659072,"data_stored":2001747019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":213126072},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452070,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1634376,"kb_used_data":1462336,"kb_used_omap":54,"kb_used_meta":171977,"kb_avail":92110776,"statfs":{"total":95995035648,"available":94321434624,"internally_reserved":0,"allocated":1497432064,"data_stored":1492527000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":176104846},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474761,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1439704,"kb_used_data":1248912,"kb_used_omap":119,"kb_used_meta":190664,"kb_avail":92305448,"statfs":{"total":95995035648,"available":94520778752,"internally_reserved":0,"allocated":1278885888,"data_stored":1273965660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":195240670},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019317,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1263538176,"data_stored":1263534104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1483407360,"data_stored":1483407360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1991507968,"data_stored":1991507968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2226065408,"data_stored":2226061336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1024327680,"data_stored":1024327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1966936064,"data_stored":1966931992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1709899776,"data_stored":1709899776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:55:05.702 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:05.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:05.804 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:05.828 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:05.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:05.930 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:05.953 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:05.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:05.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:06.054 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:06.077 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:06.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:06.178 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:06.200 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:06.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:06.302 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:06.324 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:06.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:06.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16704/181137 objects degraded (9.222%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:06.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16704/181137 objects degraded (9.222%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:06.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16704/181137 objects degraded (9.222%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:06.425 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:06.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.448 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:06.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:06.549 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:06.573 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:06.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:06.675 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:06.698 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:06.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:06.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:06.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:06.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:06.923 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:06.944 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:06.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:06.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:07.019 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:07.046 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:07.068 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:07.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:07.170 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:07.196 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:07.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:07.275 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:55:07.275 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:55:07.276 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:07 smithi130 ceph-mon[105521]: pgmap v1844: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 16630/180915 objects degraded (9.192%); 10157/180915 objects misplaced (5.614%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:55:07.276 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:07 smithi130 ceph-mon[110068]: pgmap v1844: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 16630/180915 objects degraded (9.192%); 10157/180915 objects misplaced (5.614%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:55:07.277 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:55:07.277 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:55:07.294 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:55:07.294 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:55:07.298 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:07.312 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:55:07.312 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:55:07.323 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:07.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:07.333 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:55:07.334 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:55:07.352 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:55:07.352 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:55:07.373 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:55:07.374 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:55:07.387 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:07 smithi179 ceph-mon[93519]: pgmap v1844: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 16630/180915 objects degraded (9.192%); 10157/180915 objects misplaced (5.614%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:55:07.401 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:55:07.402 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:55:07.425 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:07.426 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:55:07.426 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:55:07.446 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:07.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:07.548 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:07.575 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:07.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:07.676 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:07.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:07.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:07.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:07.824 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:07.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:07.926 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:07.947 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:07.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:07.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:08.049 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:08.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.075 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:08.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:08.176 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:08.198 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:08.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:08.221 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:55:08.300 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:08.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.326 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:08.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:08.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:08 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4262744841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:08.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:08 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4262744841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:08.427 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:08.453 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:08.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:08.530 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:55:08.555 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:08.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:08.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:08.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:08 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4262744841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:08.678 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:08.687 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:55:08.701 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:08.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:08.803 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:08.825 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:08.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:08.927 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:08.949 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:08.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:08.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:09.050 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:09.082 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:09.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:09.183 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:09.205 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:09.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:09.307 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:09.330 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:09.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:09 smithi179 ceph-mon[93519]: pgmap v1845: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 16630/179574 objects degraded (9.261%); 10157/179574 objects misplaced (5.656%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:55:09.408 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:09.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:09 smithi130 ceph-mon[105521]: pgmap v1845: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 16630/179574 objects degraded (9.261%); 10157/179574 objects misplaced (5.656%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:55:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:09 smithi130 ceph-mon[110068]: pgmap v1845: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 16630/179574 objects degraded (9.261%); 10157/179574 objects misplaced (5.656%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:55:09.432 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:09.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.454 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:09.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:09.555 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:09.584 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:09.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:09.686 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:09.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:09.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:09.812 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:09.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:09.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:09.937 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:09.959 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:09.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:09.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:10.061 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:10.076 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:55:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:55:09] "GET /metrics HTTP/1.1" 200 36815 "" "Prometheus/2.51.0" 2024-08-17T17:55:10.087 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:10.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:10.151 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T17:55:10.189 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:10.217 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:10.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:10.320 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:10.342 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:10.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:10.358 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:10 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2173142640' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:10.358 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:55:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:55:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:55:10.375 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:55:10.375 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:55:10.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:10 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2173142640' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:10.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:10 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2173142640' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:10.444 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:10.466 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:10.568 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:10.582 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:55:10.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.594 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:10.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:10.695 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:10.718 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:10.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:10.820 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:10.841 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:10.943 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:10.965 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:10.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:10.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:11.067 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:11.087 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:55:11.090 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:11.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:11.192 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:11.213 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:11.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:11.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:11.339 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:11.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:11.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:11 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:11 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2173142640' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:55:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:11 smithi179 ceph-mon[93519]: osdmap e1229: 8 total, 8 up, 6 in 2024-08-17T17:55:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16630/179574 objects degraded (9.261%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:11 smithi179 ceph-mon[93519]: pgmap v1847: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 16606/179007 objects degraded (9.277%); 10081/179007 objects misplaced (5.632%); 967 KiB/s, 15 objects/s recovering 2024-08-17T17:55:11.360 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:55:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:11 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:11 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2173142640' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:55:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:11 smithi130 ceph-mon[105521]: osdmap e1229: 8 total, 8 up, 6 in 2024-08-17T17:55:11.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16630/179574 objects degraded (9.261%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:11.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:11 smithi130 ceph-mon[105521]: pgmap v1847: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 16606/179007 objects degraded (9.277%); 10081/179007 objects misplaced (5.632%); 967 KiB/s, 15 objects/s recovering 2024-08-17T17:55:11.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:11 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:11.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:11 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2173142640' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:55:11.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:11 smithi130 ceph-mon[110068]: osdmap e1229: 8 total, 8 up, 6 in 2024-08-17T17:55:11.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16630/179574 objects degraded (9.261%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:11.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:11 smithi130 ceph-mon[110068]: pgmap v1847: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 149 op/s; 16606/179007 objects degraded (9.277%); 10081/179007 objects misplaced (5.632%); 967 KiB/s, 15 objects/s recovering 2024-08-17T17:55:11.441 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:11.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:11.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:11.563 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:11.584 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:11.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:11.686 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:11.708 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:11.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:11.810 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:11.832 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:11.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:11.933 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:11.956 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:11.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:12.058 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:12.080 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:12.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:12.181 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:12.197 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:12.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.204 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:12.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:12.305 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:12.327 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:12.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:12.356 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:55:12.357 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:55:12.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:12.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:12.428 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:12.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.453 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:12.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:12.554 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:12.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.580 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:12.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:12.675 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:12.681 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:12.702 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:12.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:12.804 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:12.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.826 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:12.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:12.927 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:12.949 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:12.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:12.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:13.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:13.072 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:13.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:13.174 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:13.192 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:55:13.199 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:13.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:13.245 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 17:55:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=cleanup t=2024-08-17T17:55:13.134504Z level=info msg="Completed cleanup jobs" duration=12.629652ms 2024-08-17T17:55:13.261 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:55:13.262 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:55:13.301 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:13.325 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:13.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:13.355 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1847,"stamp":"2024-08-17T17:55:11.066661+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3909294053,"num_objects":59669,"num_object_clones":0,"num_object_copies":179007,"num_objects_missing_on_primary":3195,"num_objects_missing":3195,"num_objects_degraded":16606,"num_objects_misplaced":10081,"num_objects_unfound":0,"num_objects_dirty":59669,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":150245,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35701,"num_bytes_recovered":2161122317,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108648,"ondisk_log_size":108648,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11712292,"kb_used_data":10292660,"kb_used_omap":295,"kb_used_meta":1419288,"kb_avail":550758620,"statfs":{"total":575970213888,"available":563976826880,"internally_reserved":0,"allocated":10539683840,"data_stored":10510140849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1453351375},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-81592320,"num_objects":-1245,"num_object_clones":0,"num_object_copies":-3735,"num_objects_missing_on_primary":-90,"num_objects_missing":-90,"num_objects_degraded":-98,"num_objects_misplaced":-202,"num_objects_unfound":0,"num_objects_dirty":-1245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1245,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":126,"num_bytes_recovered":8257536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.336473"},"pg_stats":[{"pgid":"45.f","version":"1228'9400","reported_seq":19936,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:09.754092+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:55:09.754092+0000","last_peered":"2024-08-17T17:55:09.754092+0000","last_clean":"2024-08-17T17:55:09.754092+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:55:09.754092+0000","last_undegraded":"2024-08-17T17:55:09.754092+0000","last_fullsized":"2024-08-17T17:55:09.754092+0000","mapping_epoch":1194,"log_start":"1170'2531","ondisk_log_start":"1170'2531","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6869,"log_dups_size":0,"ondisk_log_size":6869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246677504,"num_objects":3764,"num_object_clones":0,"num_object_copies":11292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9400,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1228'9314","reported_seq":19782,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:08.088175+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:55:08.088175+0000","last_peered":"2024-08-17T17:55:08.088175+0000","last_clean":"2024-08-17T17:55:08.088175+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:55:08.088175+0000","last_undegraded":"2024-08-17T17:55:08.088175+0000","last_fullsized":"2024-08-17T17:55:08.088175+0000","mapping_epoch":1211,"log_start":"1170'2332","ondisk_log_start":"1170'2332","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6982,"log_dups_size":0,"ondisk_log_size":6982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241172480,"num_objects":3680,"num_object_clones":0,"num_object_copies":11040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9314,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1228'9605","reported_seq":20448,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:08.371752+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:55:08.371752+0000","last_peered":"2024-08-17T17:55:08.371752+0000","last_clean":"2024-08-17T17:55:08.371752+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:55:08.371752+0000","last_undegraded":"2024-08-17T17:55:08.371752+0000","last_fullsized":"2024-08-17T17:55:08.371752+0000","mapping_epoch":1205,"log_start":"1174'3668","ondisk_log_start":"1174'3668","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5937,"log_dups_size":0,"ondisk_log_size":5937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247201816,"num_objects":3773,"num_object_clones":0,"num_object_copies":11319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3773,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9605,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1228'9343","reported_seq":9412,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:08.044289+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:55:08.044289+0000","last_peered":"2024-08-17T17:55:08.044289+0000","last_clean":"2024-08-17T17:55:08.044289+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:55:08.044289+0000","last_undegraded":"2024-08-17T17:55:08.044289+0000","last_fullsized":"2024-08-17T17:55:08.044289+0000","mapping_epoch":1167,"log_start":"1170'2589","ondisk_log_start":"1170'2589","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6754,"log_dups_size":0,"ondisk_log_size":6754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239665152,"num_objects":3657,"num_object_clones":0,"num_object_copies":10971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9343,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1228'9353","reported_seq":14602,"reported_epoch":1228,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:06.585683+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:55:06.585683+0000","last_peered":"2024-08-17T17:55:06.585683+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:55:06.585683+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1174'3426","ondisk_log_start":"1174'3426","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5927,"log_dups_size":0,"ondisk_log_size":5927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246480896,"num_objects":3761,"num_object_clones":0,"num_object_copies":11283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":3761,"num_objects_unfound":0,"num_objects_dirty":3761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9353,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":3243},{"shards":"1,4,5","objects":275}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1228'9450","reported_seq":9519,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:07.886639+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:55:07.886639+0000","last_peered":"2024-08-17T17:55:07.886639+0000","last_clean":"2024-08-17T17:55:07.886639+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:55:07.886639+0000","last_undegraded":"2024-08-17T17:55:07.886639+0000","last_fullsized":"2024-08-17T17:55:07.886639+0000","mapping_epoch":1167,"log_start":"1171'2777","ondisk_log_start":"1171'2777","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6673,"log_dups_size":0,"ondisk_log_size":6673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238813184,"num_objects":3644,"num_object_clones":0,"num_object_copies":10932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9450,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1228'9422","reported_seq":9491,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:08.176161+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:55:08.176161+0000","last_peered":"2024-08-17T17:55:08.176161+0000","last_clean":"2024-08-17T17:55:08.176161+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:55:08.176161+0000","last_undegraded":"2024-08-17T17:55:08.176161+0000","last_fullsized":"2024-08-17T17:55:08.176161+0000","mapping_epoch":1167,"log_start":"1171'2669","ondisk_log_start":"1171'2669","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6753,"log_dups_size":0,"ondisk_log_size":6753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249298944,"num_objects":3804,"num_object_clones":0,"num_object_copies":11412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9422,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1228'9339","reported_seq":18013,"reported_epoch":1228,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:06.594925+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:55:06.594925+0000","last_peered":"2024-08-17T17:55:06.594925+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:55:06.594925+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1174'3375","ondisk_log_start":"1174'3375","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5964,"log_dups_size":0,"ondisk_log_size":5964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239271936,"num_objects":3651,"num_object_clones":0,"num_object_copies":10953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2595,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9339,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2844,"num_bytes_recovered":186384384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":264},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1228'9312","reported_seq":9391,"reported_epoch":1228,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:08.029444+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:55:08.029444+0000","last_peered":"2024-08-17T17:55:08.029444+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:55:08.029444+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1170'2577","ondisk_log_start":"1170'2577","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6735,"log_dups_size":0,"ondisk_log_size":6735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246677504,"num_objects":3764,"num_object_clones":0,"num_object_copies":11292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9312,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":3764},{"shards":"1,6","objects":-2069}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1228'9395","reported_seq":12477,"reported_epoch":1228,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:08.313528+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:55:08.313528+0000","last_peered":"2024-08-17T17:55:08.313528+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:55:08.313528+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1174'3506","ondisk_log_start":"1174'3506","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5889,"log_dups_size":0,"ondisk_log_size":5889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247660544,"num_objects":3779,"num_object_clones":0,"num_object_copies":11337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9395,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":2218},{"shards":"1,3,5","objects":220}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1228'9308","reported_seq":9446,"reported_epoch":1228,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:09.019776+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:55:09.019776+0000","last_peered":"2024-08-17T17:55:09.019776+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:55:09.019776+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1170'200","ondisk_log_start":"1170'200","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9108,"log_dups_size":0,"ondisk_log_size":9108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246808576,"num_objects":3766,"num_object_clones":0,"num_object_copies":11298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9308,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-1387}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1228'9333","reported_seq":15531,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:08.914320+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:55:08.914320+0000","last_peered":"2024-08-17T17:55:08.914320+0000","last_clean":"2024-08-17T17:55:08.914320+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:55:08.914320+0000","last_undegraded":"2024-08-17T17:55:08.914320+0000","last_fullsized":"2024-08-17T17:55:08.914320+0000","mapping_epoch":1183,"log_start":"1170'200","ondisk_log_start":"1170'200","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9133,"log_dups_size":0,"ondisk_log_size":9133,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243335168,"num_objects":3713,"num_object_clones":0,"num_object_copies":11139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9333,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1228'9438","reported_seq":9574,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:08.417555+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:55:08.417555+0000","last_peered":"2024-08-17T17:55:08.417555+0000","last_clean":"2024-08-17T17:55:08.417555+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:55:08.417555+0000","last_undegraded":"2024-08-17T17:55:08.417555+0000","last_fullsized":"2024-08-17T17:55:08.417555+0000","mapping_epoch":1167,"log_start":"1174'3491","ondisk_log_start":"1174'3491","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5947,"log_dups_size":0,"ondisk_log_size":5947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243924992,"num_objects":3722,"num_object_clones":0,"num_object_copies":11166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9438,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1228'9275","reported_seq":9344,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:08.079523+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:55:08.079523+0000","last_peered":"2024-08-17T17:55:08.079523+0000","last_clean":"2024-08-17T17:55:08.079523+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:55:08.079523+0000","last_undegraded":"2024-08-17T17:55:08.079523+0000","last_fullsized":"2024-08-17T17:55:08.079523+0000","mapping_epoch":1167,"log_start":"1170'2225","ondisk_log_start":"1170'2225","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7050,"log_dups_size":0,"ondisk_log_size":7050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239796224,"num_objects":3659,"num_object_clones":0,"num_object_copies":10977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9275,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1228'9526","reported_seq":21706,"reported_epoch":1228,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:10.020468+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:55:10.020468+0000","last_peered":"2024-08-17T17:55:10.020468+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:55:10.020468+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1171'2691","ondisk_log_start":"1171'2691","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6835,"log_dups_size":0,"ondisk_log_size":6835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251527168,"num_objects":3838,"num_object_clones":0,"num_object_copies":11514,"num_objects_missing_on_primary":3195,"num_objects_missing":3195,"num_objects_degraded":241,"num_objects_misplaced":6320,"num_objects_unfound":0,"num_objects_dirty":3838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9526,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4950,"num_bytes_recovered":324403200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5","objects":14},{"shards":"0,4,6","objects":1356},{"shards":"4,5","objects":241},{"shards":"4,5,6","objects":2227}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1228'9154","reported_seq":22476,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:08.425515+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:55:08.425515+0000","last_peered":"2024-08-17T17:55:08.425515+0000","last_clean":"2024-08-17T17:55:08.425515+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:55:08.425515+0000","last_undegraded":"2024-08-17T17:55:08.425515+0000","last_fullsized":"2024-08-17T17:55:08.425515+0000","mapping_epoch":1196,"log_start":"1174'3269","ondisk_log_start":"1174'3269","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5885,"log_dups_size":0,"ondisk_log_size":5885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240386048,"num_objects":3668,"num_object_clones":0,"num_object_copies":11004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9154,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1913,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.850516+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:55:02.850516+0000","last_peered":"2024-08-17T17:55:02.850516+0000","last_clean":"2024-08-17T17:55:02.850516+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:55:02.850516+0000","last_undegraded":"2024-08-17T17:55:02.850516+0000","last_fullsized":"2024-08-17T17:55:02.850516+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":505,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.896150+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:54:52.896150+0000","last_peered":"2024-08-17T17:54:52.896150+0000","last_clean":"2024-08-17T17:54:52.896150+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:54:52.896150+0000","last_undegraded":"2024-08-17T17:54:52.896150+0000","last_fullsized":"2024-08-17T17:54:52.896150+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1832,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.827295+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:55:02.827295+0000","last_peered":"2024-08-17T17:55:02.827295+0000","last_clean":"2024-08-17T17:55:02.827295+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:55:02.827295+0000","last_undegraded":"2024-08-17T17:55:02.827295+0000","last_fullsized":"2024-08-17T17:55:02.827295+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1856,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825300+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:55:02.825300+0000","last_peered":"2024-08-17T17:55:02.825300+0000","last_clean":"2024-08-17T17:55:02.825300+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:55:02.825300+0000","last_undegraded":"2024-08-17T17:55:02.825300+0000","last_fullsized":"2024-08-17T17:55:02.825300+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":67,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.896726+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:54:52.896726+0000","last_peered":"2024-08-17T17:54:52.896726+0000","last_clean":"2024-08-17T17:54:52.896726+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:54:52.896726+0000","last_undegraded":"2024-08-17T17:54:52.896726+0000","last_fullsized":"2024-08-17T17:54:52.896726+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1250,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825849+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:55:02.825849+0000","last_peered":"2024-08-17T17:55:02.825849+0000","last_clean":"2024-08-17T17:55:02.825849+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:55:02.825849+0000","last_undegraded":"2024-08-17T17:55:02.825849+0000","last_fullsized":"2024-08-17T17:55:02.825849+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":108,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.820471+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:55:02.820471+0000","last_peered":"2024-08-17T17:55:02.820471+0000","last_clean":"2024-08-17T17:55:02.820471+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:55:02.820471+0000","last_undegraded":"2024-08-17T17:55:02.820471+0000","last_fullsized":"2024-08-17T17:55:02.820471+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":628,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825776+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:55:02.825776+0000","last_peered":"2024-08-17T17:55:02.825776+0000","last_clean":"2024-08-17T17:55:02.825776+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:55:02.825776+0000","last_undegraded":"2024-08-17T17:55:02.825776+0000","last_fullsized":"2024-08-17T17:55:02.825776+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1871,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.850064+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:55:02.850064+0000","last_peered":"2024-08-17T17:55:02.850064+0000","last_clean":"2024-08-17T17:55:02.850064+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:55:02.850064+0000","last_undegraded":"2024-08-17T17:55:02.850064+0000","last_fullsized":"2024-08-17T17:55:02.850064+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1873,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.826547+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:55:02.826547+0000","last_peered":"2024-08-17T17:55:02.826547+0000","last_clean":"2024-08-17T17:55:02.826547+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:55:02.826547+0000","last_undegraded":"2024-08-17T17:55:02.826547+0000","last_fullsized":"2024-08-17T17:55:02.826547+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":93,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.820189+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:55:02.820189+0000","last_peered":"2024-08-17T17:55:02.820189+0000","last_clean":"2024-08-17T17:55:02.820189+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:55:02.820189+0000","last_undegraded":"2024-08-17T17:55:02.820189+0000","last_fullsized":"2024-08-17T17:55:02.820189+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1845,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.826659+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:55:02.826659+0000","last_peered":"2024-08-17T17:55:02.826659+0000","last_clean":"2024-08-17T17:55:02.826659+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:55:02.826659+0000","last_undegraded":"2024-08-17T17:55:02.826659+0000","last_fullsized":"2024-08-17T17:55:02.826659+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1900,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.819826+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:55:02.819826+0000","last_peered":"2024-08-17T17:55:02.819826+0000","last_clean":"2024-08-17T17:55:02.819826+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:55:02.819826+0000","last_undegraded":"2024-08-17T17:55:02.819826+0000","last_fullsized":"2024-08-17T17:55:02.819826+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1894,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.850502+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:55:02.850502+0000","last_peered":"2024-08-17T17:55:02.850502+0000","last_clean":"2024-08-17T17:55:02.850502+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:55:02.850502+0000","last_undegraded":"2024-08-17T17:55:02.850502+0000","last_fullsized":"2024-08-17T17:55:02.850502+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1867,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825371+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:55:02.825371+0000","last_peered":"2024-08-17T17:55:02.825371+0000","last_clean":"2024-08-17T17:55:02.825371+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:55:02.825371+0000","last_undegraded":"2024-08-17T17:55:02.825371+0000","last_fullsized":"2024-08-17T17:55:02.825371+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1202,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.896762+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:54:52.896762+0000","last_peered":"2024-08-17T17:54:52.896762+0000","last_clean":"2024-08-17T17:54:52.896762+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:54:52.896762+0000","last_undegraded":"2024-08-17T17:54:52.896762+0000","last_fullsized":"2024-08-17T17:54:52.896762+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1839,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825170+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:55:02.825170+0000","last_peered":"2024-08-17T17:55:02.825170+0000","last_clean":"2024-08-17T17:55:02.825170+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:55:02.825170+0000","last_undegraded":"2024-08-17T17:55:02.825170+0000","last_fullsized":"2024-08-17T17:55:02.825170+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1888,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.850577+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:55:02.850577+0000","last_peered":"2024-08-17T17:55:02.850577+0000","last_clean":"2024-08-17T17:55:02.850577+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:55:02.850577+0000","last_undegraded":"2024-08-17T17:55:02.850577+0000","last_fullsized":"2024-08-17T17:55:02.850577+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1795,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825091+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:55:02.825091+0000","last_peered":"2024-08-17T17:55:02.825091+0000","last_clean":"2024-08-17T17:55:02.825091+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:55:02.825091+0000","last_undegraded":"2024-08-17T17:55:02.825091+0000","last_fullsized":"2024-08-17T17:55:02.825091+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":521,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.859923+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:54:52.859923+0000","last_peered":"2024-08-17T17:54:52.859923+0000","last_clean":"2024-08-17T17:54:52.859923+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:54:52.859923+0000","last_undegraded":"2024-08-17T17:54:52.859923+0000","last_fullsized":"2024-08-17T17:54:52.859923+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1198,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.858797+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:54:52.858797+0000","last_peered":"2024-08-17T17:54:52.858797+0000","last_clean":"2024-08-17T17:54:52.858797+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:54:52.858797+0000","last_undegraded":"2024-08-17T17:54:52.858797+0000","last_fullsized":"2024-08-17T17:54:52.858797+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1893,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.820380+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:55:02.820380+0000","last_peered":"2024-08-17T17:55:02.820380+0000","last_clean":"2024-08-17T17:55:02.820380+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:55:02.820380+0000","last_undegraded":"2024-08-17T17:55:02.820380+0000","last_fullsized":"2024-08-17T17:55:02.820380+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":359,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.858754+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:54:52.858754+0000","last_peered":"2024-08-17T17:54:52.858754+0000","last_clean":"2024-08-17T17:54:52.858754+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:54:52.858754+0000","last_undegraded":"2024-08-17T17:54:52.858754+0000","last_fullsized":"2024-08-17T17:54:52.858754+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":505,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.896448+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:54:52.896448+0000","last_peered":"2024-08-17T17:54:52.896448+0000","last_clean":"2024-08-17T17:54:52.896448+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:54:52.896448+0000","last_undegraded":"2024-08-17T17:54:52.896448+0000","last_fullsized":"2024-08-17T17:54:52.896448+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1183,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.896315+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:54:52.896315+0000","last_peered":"2024-08-17T17:54:52.896315+0000","last_clean":"2024-08-17T17:54:52.896315+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:54:52.896315+0000","last_undegraded":"2024-08-17T17:54:52.896315+0000","last_fullsized":"2024-08-17T17:54:52.896315+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1191,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.896818+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:54:52.896818+0000","last_peered":"2024-08-17T17:54:52.896818+0000","last_clean":"2024-08-17T17:54:52.896818+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:54:52.896818+0000","last_undegraded":"2024-08-17T17:54:52.896818+0000","last_fullsized":"2024-08-17T17:54:52.896818+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1941,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.850379+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:55:02.850379+0000","last_peered":"2024-08-17T17:55:02.850379+0000","last_clean":"2024-08-17T17:55:02.850379+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:55:02.850379+0000","last_undegraded":"2024-08-17T17:55:02.850379+0000","last_fullsized":"2024-08-17T17:55:02.850379+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1824,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825263+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:55:02.825263+0000","last_peered":"2024-08-17T17:55:02.825263+0000","last_clean":"2024-08-17T17:55:02.825263+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:55:02.825263+0000","last_undegraded":"2024-08-17T17:55:02.825263+0000","last_fullsized":"2024-08-17T17:55:02.825263+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1899,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.821332+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:55:02.821332+0000","last_peered":"2024-08-17T17:55:02.821332+0000","last_clean":"2024-08-17T17:55:02.821332+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:55:02.821332+0000","last_undegraded":"2024-08-17T17:55:02.821332+0000","last_fullsized":"2024-08-17T17:55:02.821332+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1826,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825740+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:55:02.825740+0000","last_peered":"2024-08-17T17:55:02.825740+0000","last_clean":"2024-08-17T17:55:02.825740+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:55:02.825740+0000","last_undegraded":"2024-08-17T17:55:02.825740+0000","last_fullsized":"2024-08-17T17:55:02.825740+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1859,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.850097+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:55:02.850097+0000","last_peered":"2024-08-17T17:55:02.850097+0000","last_clean":"2024-08-17T17:55:02.850097+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:55:02.850097+0000","last_undegraded":"2024-08-17T17:55:02.850097+0000","last_fullsized":"2024-08-17T17:55:02.850097+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1218,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.858897+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:54:52.858897+0000","last_peered":"2024-08-17T17:54:52.858897+0000","last_clean":"2024-08-17T17:54:52.858897+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:54:52.858897+0000","last_undegraded":"2024-08-17T17:54:52.858897+0000","last_fullsized":"2024-08-17T17:54:52.858897+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1905,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.820650+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:55:02.820650+0000","last_peered":"2024-08-17T17:55:02.820650+0000","last_clean":"2024-08-17T17:55:02.820650+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:55:02.820650+0000","last_undegraded":"2024-08-17T17:55:02.820650+0000","last_fullsized":"2024-08-17T17:55:02.820650+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":92,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.820070+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:55:02.820070+0000","last_peered":"2024-08-17T17:55:02.820070+0000","last_clean":"2024-08-17T17:55:02.820070+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:55:02.820070+0000","last_undegraded":"2024-08-17T17:55:02.820070+0000","last_fullsized":"2024-08-17T17:55:02.820070+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1924,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.820570+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:55:02.820570+0000","last_peered":"2024-08-17T17:55:02.820570+0000","last_clean":"2024-08-17T17:55:02.820570+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:55:02.820570+0000","last_undegraded":"2024-08-17T17:55:02.820570+0000","last_fullsized":"2024-08-17T17:55:02.820570+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":661,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.826485+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:55:02.826485+0000","last_peered":"2024-08-17T17:55:02.826485+0000","last_clean":"2024-08-17T17:55:02.826485+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:55:02.826485+0000","last_undegraded":"2024-08-17T17:55:02.826485+0000","last_fullsized":"2024-08-17T17:55:02.826485+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1905,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.820735+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:55:02.820735+0000","last_peered":"2024-08-17T17:55:02.820735+0000","last_clean":"2024-08-17T17:55:02.820735+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:55:02.820735+0000","last_undegraded":"2024-08-17T17:55:02.820735+0000","last_fullsized":"2024-08-17T17:55:02.820735+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1234,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.849957+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:55:02.849957+0000","last_peered":"2024-08-17T17:55:02.849957+0000","last_clean":"2024-08-17T17:55:02.849957+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:55:02.849957+0000","last_undegraded":"2024-08-17T17:55:02.849957+0000","last_fullsized":"2024-08-17T17:55:02.849957+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":152,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.820307+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:55:02.820307+0000","last_peered":"2024-08-17T17:55:02.820307+0000","last_clean":"2024-08-17T17:55:02.820307+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:55:02.820307+0000","last_undegraded":"2024-08-17T17:55:02.820307+0000","last_fullsized":"2024-08-17T17:55:02.820307+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1916,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.821636+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:55:02.821636+0000","last_peered":"2024-08-17T17:55:02.821636+0000","last_clean":"2024-08-17T17:55:02.821636+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:55:02.821636+0000","last_undegraded":"2024-08-17T17:55:02.821636+0000","last_fullsized":"2024-08-17T17:55:02.821636+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1894,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.820512+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:55:02.820512+0000","last_peered":"2024-08-17T17:55:02.820512+0000","last_clean":"2024-08-17T17:55:02.820512+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:55:02.820512+0000","last_undegraded":"2024-08-17T17:55:02.820512+0000","last_fullsized":"2024-08-17T17:55:02.820512+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1914,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825972+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:55:02.825972+0000","last_peered":"2024-08-17T17:55:02.825972+0000","last_clean":"2024-08-17T17:55:02.825972+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:55:02.825972+0000","last_undegraded":"2024-08-17T17:55:02.825972+0000","last_fullsized":"2024-08-17T17:55:02.825972+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1920,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.850569+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:55:02.850569+0000","last_peered":"2024-08-17T17:55:02.850569+0000","last_clean":"2024-08-17T17:55:02.850569+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:55:02.850569+0000","last_undegraded":"2024-08-17T17:55:02.850569+0000","last_fullsized":"2024-08-17T17:55:02.850569+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1900,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.858448+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:55:02.858448+0000","last_peered":"2024-08-17T17:55:02.858448+0000","last_clean":"2024-08-17T17:55:02.858448+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:55:02.858448+0000","last_undegraded":"2024-08-17T17:55:02.858448+0000","last_fullsized":"2024-08-17T17:55:02.858448+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1225,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.858514+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:54:52.858514+0000","last_peered":"2024-08-17T17:54:52.858514+0000","last_clean":"2024-08-17T17:54:52.858514+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:54:52.858514+0000","last_undegraded":"2024-08-17T17:54:52.858514+0000","last_fullsized":"2024-08-17T17:54:52.858514+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1171,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.896766+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:54:52.896766+0000","last_peered":"2024-08-17T17:54:52.896766+0000","last_clean":"2024-08-17T17:54:52.896766+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:54:52.896766+0000","last_undegraded":"2024-08-17T17:54:52.896766+0000","last_fullsized":"2024-08-17T17:54:52.896766+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1809,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825753+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:55:02.825753+0000","last_peered":"2024-08-17T17:55:02.825753+0000","last_clean":"2024-08-17T17:55:02.825753+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:55:02.825753+0000","last_undegraded":"2024-08-17T17:55:02.825753+0000","last_fullsized":"2024-08-17T17:55:02.825753+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":140,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.826090+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:55:02.826090+0000","last_peered":"2024-08-17T17:55:02.826090+0000","last_clean":"2024-08-17T17:55:02.826090+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:55:02.826090+0000","last_undegraded":"2024-08-17T17:55:02.826090+0000","last_fullsized":"2024-08-17T17:55:02.826090+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1537,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.858438+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:54:52.858438+0000","last_peered":"2024-08-17T17:54:52.858438+0000","last_clean":"2024-08-17T17:54:52.858438+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:54:52.858438+0000","last_undegraded":"2024-08-17T17:54:52.858438+0000","last_fullsized":"2024-08-17T17:54:52.858438+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1900,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.821845+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:55:02.821845+0000","last_peered":"2024-08-17T17:55:02.821845+0000","last_clean":"2024-08-17T17:55:02.821845+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:55:02.821845+0000","last_undegraded":"2024-08-17T17:55:02.821845+0000","last_fullsized":"2024-08-17T17:55:02.821845+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1840,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825933+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:55:02.825933+0000","last_peered":"2024-08-17T17:55:02.825933+0000","last_clean":"2024-08-17T17:55:02.825933+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:55:02.825933+0000","last_undegraded":"2024-08-17T17:55:02.825933+0000","last_fullsized":"2024-08-17T17:55:02.825933+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":505,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.896664+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:54:52.896664+0000","last_peered":"2024-08-17T17:54:52.896664+0000","last_clean":"2024-08-17T17:54:52.896664+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:54:52.896664+0000","last_undegraded":"2024-08-17T17:54:52.896664+0000","last_fullsized":"2024-08-17T17:54:52.896664+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":579,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825824+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:55:02.825824+0000","last_peered":"2024-08-17T17:55:02.825824+0000","last_clean":"2024-08-17T17:55:02.825824+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:55:02.825824+0000","last_undegraded":"2024-08-17T17:55:02.825824+0000","last_fullsized":"2024-08-17T17:55:02.825824+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":595,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.827169+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:55:02.827169+0000","last_peered":"2024-08-17T17:55:02.827169+0000","last_clean":"2024-08-17T17:55:02.827169+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:55:02.827169+0000","last_undegraded":"2024-08-17T17:55:02.827169+0000","last_fullsized":"2024-08-17T17:55:02.827169+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1189,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.896461+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:54:52.896461+0000","last_peered":"2024-08-17T17:54:52.896461+0000","last_clean":"2024-08-17T17:54:52.896461+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:54:52.896461+0000","last_undegraded":"2024-08-17T17:54:52.896461+0000","last_fullsized":"2024-08-17T17:54:52.896461+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":611,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825312+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:55:02.825312+0000","last_peered":"2024-08-17T17:55:02.825312+0000","last_clean":"2024-08-17T17:55:02.825312+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:55:02.825312+0000","last_undegraded":"2024-08-17T17:55:02.825312+0000","last_fullsized":"2024-08-17T17:55:02.825312+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1837,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825892+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:55:02.825892+0000","last_peered":"2024-08-17T17:55:02.825892+0000","last_clean":"2024-08-17T17:55:02.825892+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:55:02.825892+0000","last_undegraded":"2024-08-17T17:55:02.825892+0000","last_fullsized":"2024-08-17T17:55:02.825892+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1932,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.850180+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:55:02.850180+0000","last_peered":"2024-08-17T17:55:02.850180+0000","last_clean":"2024-08-17T17:55:02.850180+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:55:02.850180+0000","last_undegraded":"2024-08-17T17:55:02.850180+0000","last_fullsized":"2024-08-17T17:55:02.850180+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":644,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825206+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:55:02.825206+0000","last_peered":"2024-08-17T17:55:02.825206+0000","last_clean":"2024-08-17T17:55:02.825206+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:55:02.825206+0000","last_undegraded":"2024-08-17T17:55:02.825206+0000","last_fullsized":"2024-08-17T17:55:02.825206+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1178,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.896373+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:54:52.896373+0000","last_peered":"2024-08-17T17:54:52.896373+0000","last_clean":"2024-08-17T17:54:52.896373+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:54:52.896373+0000","last_undegraded":"2024-08-17T17:54:52.896373+0000","last_fullsized":"2024-08-17T17:54:52.896373+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1177,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.858859+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:54:52.858859+0000","last_peered":"2024-08-17T17:54:52.858859+0000","last_clean":"2024-08-17T17:54:52.858859+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:54:52.858859+0000","last_undegraded":"2024-08-17T17:54:52.858859+0000","last_fullsized":"2024-08-17T17:54:52.858859+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":628,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.819927+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:55:02.819927+0000","last_peered":"2024-08-17T17:55:02.819927+0000","last_clean":"2024-08-17T17:55:02.819927+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:55:02.819927+0000","last_undegraded":"2024-08-17T17:55:02.819927+0000","last_fullsized":"2024-08-17T17:55:02.819927+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1195,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.860004+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:54:52.860004+0000","last_peered":"2024-08-17T17:54:52.860004+0000","last_clean":"2024-08-17T17:54:52.860004+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:54:52.860004+0000","last_undegraded":"2024-08-17T17:54:52.860004+0000","last_fullsized":"2024-08-17T17:54:52.860004+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":79,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.858357+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:54:52.858357+0000","last_peered":"2024-08-17T17:54:52.858357+0000","last_clean":"2024-08-17T17:54:52.858357+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:54:52.858357+0000","last_undegraded":"2024-08-17T17:54:52.858357+0000","last_fullsized":"2024-08-17T17:54:52.858357+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1211,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.858892+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:54:52.858892+0000","last_peered":"2024-08-17T17:54:52.858892+0000","last_clean":"2024-08-17T17:54:52.858892+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:54:52.858892+0000","last_undegraded":"2024-08-17T17:54:52.858892+0000","last_fullsized":"2024-08-17T17:54:52.858892+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1228,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.896257+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:54:52.896257+0000","last_peered":"2024-08-17T17:54:52.896257+0000","last_clean":"2024-08-17T17:54:52.896257+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:54:52.896257+0000","last_undegraded":"2024-08-17T17:54:52.896257+0000","last_fullsized":"2024-08-17T17:54:52.896257+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1838,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825533+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:55:02.825533+0000","last_peered":"2024-08-17T17:55:02.825533+0000","last_clean":"2024-08-17T17:55:02.825533+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:55:02.825533+0000","last_undegraded":"2024-08-17T17:55:02.825533+0000","last_fullsized":"2024-08-17T17:55:02.825533+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1172,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.858517+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:54:52.858517+0000","last_peered":"2024-08-17T17:54:52.858517+0000","last_clean":"2024-08-17T17:54:52.858517+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:54:52.858517+0000","last_undegraded":"2024-08-17T17:54:52.858517+0000","last_fullsized":"2024-08-17T17:54:52.858517+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1891,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.850409+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:55:02.850409+0000","last_peered":"2024-08-17T17:55:02.850409+0000","last_clean":"2024-08-17T17:55:02.850409+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:55:02.850409+0000","last_undegraded":"2024-08-17T17:55:02.850409+0000","last_fullsized":"2024-08-17T17:55:02.850409+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1892,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.850058+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:55:02.850058+0000","last_peered":"2024-08-17T17:55:02.850058+0000","last_clean":"2024-08-17T17:55:02.850058+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:55:02.850058+0000","last_undegraded":"2024-08-17T17:55:02.850058+0000","last_fullsized":"2024-08-17T17:55:02.850058+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":585,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.858711+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:54:52.858711+0000","last_peered":"2024-08-17T17:54:52.858711+0000","last_clean":"2024-08-17T17:54:52.858711+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:54:52.858711+0000","last_undegraded":"2024-08-17T17:54:52.858711+0000","last_fullsized":"2024-08-17T17:54:52.858711+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1197,"reported_epoch":1227,"state":"active+clean","last_fresh":"2024-08-17T17:54:52.858674+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:54:52.858674+0000","last_peered":"2024-08-17T17:54:52.858674+0000","last_clean":"2024-08-17T17:54:52.858674+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:54:52.858674+0000","last_undegraded":"2024-08-17T17:54:52.858674+0000","last_fullsized":"2024-08-17T17:54:52.858674+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1839,"reported_epoch":1228,"state":"active+clean","last_fresh":"2024-08-17T17:55:02.825620+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:55:02.825620+0000","last_peered":"2024-08-17T17:55:02.825620+0000","last_clean":"2024-08-17T17:55:02.825620+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:55:02.825620+0000","last_undegraded":"2024-08-17T17:55:02.825620+0000","last_fullsized":"2024-08-17T17:55:02.825620+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":3908698136,"num_objects":59643,"num_object_clones":0,"num_object_copies":178929,"num_objects_missing_on_primary":3195,"num_objects_missing":3195,"num_objects_degraded":16606,"num_objects_misplaced":10081,"num_objects_unfound":0,"num_objects_dirty":59643,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":149967,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35672,"num_bytes_recovered":2159345664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12550549504,"data_stored":12550537288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108441,"ondisk_log_size":108441,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234411,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1966384,"kb_used_data":1656552,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91778768,"statfs":{"total":95995035648,"available":93981458432,"internally_reserved":0,"allocated":1696309248,"data_stored":1691377607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496046,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2209524,"kb_used_data":1899692,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91535628,"statfs":{"total":95995035648,"available":93732483072,"internally_reserved":0,"allocated":1945284608,"data_stored":1940349372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757684,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326298,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2367580,"kb_used_data":2146964,"kb_used_omap":57,"kb_used_meta":220550,"kb_avail":91377572,"statfs":{"total":95995035648,"available":93570633728,"internally_reserved":0,"allocated":2198491136,"data_stored":2193574675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":225843963},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405537,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2131236,"kb_used_data":1918876,"kb_used_omap":61,"kb_used_meta":212290,"kb_avail":91613916,"statfs":{"total":95995035648,"available":93812649984,"internally_reserved":0,"allocated":1964929024,"data_stored":1960009535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":217385912},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452071,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1617176,"kb_used_data":1445136,"kb_used_omap":54,"kb_used_meta":171977,"kb_avail":92127976,"statfs":{"total":95995035648,"available":94339047424,"internally_reserved":0,"allocated":1479819264,"data_stored":1474906764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":176104846},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474763,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1420392,"kb_used_data":1225440,"kb_used_omap":119,"kb_used_meta":194824,"kb_avail":92324760,"statfs":{"total":95995035648,"available":94540554240,"internally_reserved":0,"allocated":1254850560,"data_stored":1249922896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":199500510},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019319,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1239420928,"data_stored":1239416856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1465712640,"data_stored":1465712640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1947533312,"data_stored":1947533312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2181369856,"data_stored":2181365784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1018429440,"data_stored":1018429440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1930301440,"data_stored":1930297368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1679884288,"data_stored":1679884288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:55:13.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:55:13.428 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:13.469 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:13.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:13.483 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:13 smithi130 ceph-mon[105521]: from='client.45701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:55:13.484 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:13 smithi130 ceph-mon[105521]: pgmap v1848: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 133 op/s; 16526/178734 objects degraded (9.246%); 10030/178734 objects misplaced (5.612%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:55:13.484 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:13 smithi130 ceph-mon[110068]: from='client.45701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:55:13.484 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:13 smithi130 ceph-mon[110068]: pgmap v1848: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 133 op/s; 16526/178734 objects degraded (9.246%); 10030/178734 objects misplaced (5.612%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:55:13.572 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:13 smithi179 ceph-mon[93519]: from='client.45701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:55:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:13 smithi179 ceph-mon[93519]: pgmap v1848: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 133 op/s; 16526/178734 objects degraded (9.246%); 10030/178734 objects misplaced (5.612%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:55:13.596 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:13.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:13.698 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:13.721 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:13.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:13.823 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:13.847 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:13.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:13.950 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:55:13.951 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:13.972 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:13.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:13.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:14.074 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:14.096 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:14.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:14.198 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:14.207 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:55:14.222 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:14.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:14.324 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:14.350 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:14.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:14.427 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:14.437 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:14.453 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:14.479 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:14.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:14.580 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:14.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:14.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:14.704 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:14.725 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:14.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:14.830 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:14.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.852 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:14.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:14.953 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:14.977 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:14.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:14.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:15.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:15.110 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:15.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:15.212 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:15.232 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:15.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:15.334 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:15.359 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:15.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:15.376 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:15 smithi130 ceph-mon[105521]: pgmap v1849: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 16485/176958 objects degraded (9.316%); 9946/176958 objects misplaced (5.621%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:55:15.376 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:15 smithi130 ceph-mon[110068]: pgmap v1849: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 16485/176958 objects degraded (9.316%); 9946/176958 objects misplaced (5.621%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:55:15.462 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:15.470 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:15 smithi179 ceph-mon[93519]: pgmap v1849: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 16485/176958 objects degraded (9.316%); 9946/176958 objects misplaced (5.621%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:55:15.484 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:15.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:15.500 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:55:15.501 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:55:15.524 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:55:15.525 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:55:15.542 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:55:15.543 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:55:15.562 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:55:15.562 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:55:15.580 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:55:15.580 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:55:15.585 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:15.602 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:55:15.602 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:55:15.606 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:15.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:15.622 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:55:15.622 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:55:15.646 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:55:15.646 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:55:15.708 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:15.731 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:15.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:15.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:15.853 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:15.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:15.955 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:15.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:15.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:15.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:16.078 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:16.101 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:16.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:16.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:16.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:16.225 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:16.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:16.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:16.325 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:16 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2417488525' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:16.325 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16485/176958 objects degraded (9.316%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:16.327 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:16.349 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:16.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:16.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:16.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:16 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2417488525' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:16.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16485/176958 objects degraded (9.316%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:16.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:16 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2417488525' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:16.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16485/176958 objects degraded (9.316%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:16.451 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:55:16.451 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T17:55:16.451 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T17:55:16.451 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T17:55:16.451 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.451 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.44945.0:2807 44.16 44:6bfa3af8:::rbd_data.af91c8ed8b9b.000000000000005c:head [write 2577372~1616932] snapc 72=[72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-08-17T17:55:16.451 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:45:43.833257+0000", 2024-08-17T17:55:16.451 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 572.59142557600001, 2024-08-17T17:55:16.451 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.14521906700000001, 2024-08-17T17:55:16.452 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.452 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.452 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.452 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.44945", 2024-08-17T17:55:16.452 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/735940947", 2024-08-17T17:55:16.452 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 2807 2024-08-17T17:55:16.452 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.452 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.452 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.452 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.452 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.833257+0000", 2024-08-17T17:55:16.452 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.453 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.453 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.453 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.453 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.833257+0000", 2024-08-17T17:55:16.453 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.453 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.453 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.453 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.453 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.833255+0000", 2024-08-17T17:55:16.453 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:55:16.453 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.453 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.453 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.453 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836081+0000", 2024-08-17T17:55:16.454 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002825597 2024-08-17T17:55:16.454 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.454 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.454 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.454 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836109+0000", 2024-08-17T17:55:16.454 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8121999999999999e-05 2024-08-17T17:55:16.454 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.454 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.454 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.454 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836117+0000", 2024-08-17T17:55:16.454 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7970000000000001e-06 2024-08-17T17:55:16.454 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.454 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.455 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.455 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836182+0000", 2024-08-17T17:55:16.455 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4968999999999994e-05 2024-08-17T17:55:16.455 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.455 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.455 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op must wait for map", 2024-08-17T17:55:16.455 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836190+0000", 2024-08-17T17:55:16.455 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0229999999999992e-06 2024-08-17T17:55:16.455 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.455 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.455 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.455 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960630+0000", 2024-08-17T17:55:16.455 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.124440812 2024-08-17T17:55:16.456 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.456 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.456 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.456 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960725+0000", 2024-08-17T17:55:16.456 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4609000000000002e-05 2024-08-17T17:55:16.456 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.456 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.456 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,5", 2024-08-17T17:55:16.456 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960919+0000", 2024-08-17T17:55:16.456 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000194291 2024-08-17T17:55:16.456 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.456 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.457 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.457 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.965209+0000", 2024-08-17T17:55:16.457 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0042898329999999998 2024-08-17T17:55:16.457 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.457 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.457 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.457 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978326+0000", 2024-08-17T17:55:16.457 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013117007999999999 2024-08-17T17:55:16.457 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.457 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.457 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.457 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978387+0000", 2024-08-17T17:55:16.457 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1228000000000004e-05 2024-08-17T17:55:16.457 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.458 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.458 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.458 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978438+0000", 2024-08-17T17:55:16.458 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0680000000000003e-05 2024-08-17T17:55:16.458 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.458 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.458 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.458 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978476+0000", 2024-08-17T17:55:16.458 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8074999999999997e-05 2024-08-17T17:55:16.458 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.458 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.458 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.459 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.459 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.459 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.44945.0:2809 44.19 44:9cf770fe:::rbd_data.af91c8ed8b9b.0000000000000014:head [write 4008407~185897] snapc 72=[72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-08-17T17:55:16.459 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:45:43.836130+0000", 2024-08-17T17:55:16.459 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 572.58855240399998, 2024-08-17T17:55:16.459 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.14255548400000001, 2024-08-17T17:55:16.459 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.459 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.459 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.459 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.44945", 2024-08-17T17:55:16.459 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/735940947", 2024-08-17T17:55:16.459 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 2809 2024-08-17T17:55:16.459 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.459 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.460 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.460 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.460 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836130+0000", 2024-08-17T17:55:16.460 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.460 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.460 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.460 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.460 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836130+0000", 2024-08-17T17:55:16.460 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.460 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.460 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.460 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.461 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836129+0000", 2024-08-17T17:55:16.461 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:16.461 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.461 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.461 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.461 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836269+0000", 2024-08-17T17:55:16.461 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014036699999999999 2024-08-17T17:55:16.461 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.461 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.461 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.461 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836279+0000", 2024-08-17T17:55:16.461 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.3649999999999993e-06 2024-08-17T17:55:16.461 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.462 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.462 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.462 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836284+0000", 2024-08-17T17:55:16.462 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7600000000000002e-06 2024-08-17T17:55:16.462 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.462 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.462 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.462 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836342+0000", 2024-08-17T17:55:16.462 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8489999999999997e-05 2024-08-17T17:55:16.462 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.462 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.462 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op must wait for map", 2024-08-17T17:55:16.462 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.836347+0000", 2024-08-17T17:55:16.462 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1270000000000002e-06 2024-08-17T17:55:16.463 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.463 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.463 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.463 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.841100+0000", 2024-08-17T17:55:16.463 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0047532820000000002 2024-08-17T17:55:16.463 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.463 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.463 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.463 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.841242+0000", 2024-08-17T17:55:16.463 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014152500000000001 2024-08-17T17:55:16.463 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.463 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.463 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:55:16.464 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960848+0000", 2024-08-17T17:55:16.464 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.119605845 2024-08-17T17:55:16.464 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.464 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.464 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.464 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.962334+0000", 2024-08-17T17:55:16.464 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0014866269999999999 2024-08-17T17:55:16.464 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.464 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.464 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.464 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.962704+0000", 2024-08-17T17:55:16.464 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00036935800000000002 2024-08-17T17:55:16.464 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.464 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.465 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.465 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978608+0000", 2024-08-17T17:55:16.465 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.015903870000000001 2024-08-17T17:55:16.465 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.465 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.465 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.465 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978644+0000", 2024-08-17T17:55:16.465 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6739000000000002e-05 2024-08-17T17:55:16.465 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.465 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.465 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.465 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978686+0000", 2024-08-17T17:55:16.466 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1322000000000002e-05 2024-08-17T17:55:16.466 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.466 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.466 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.466 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.466 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.466 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.44945.0:2806 44.2 e1130/1015 44:46ff3400:::rbd_data.af91c8ed8b9b.000000000000005b:head v 1130'32, mlcod=1128'28)", 2024-08-17T17:55:16.466 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:45:43.838017+0000", 2024-08-17T17:55:16.466 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 572.58666525199999, 2024-08-17T17:55:16.470 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.140595952, 2024-08-17T17:55:16.470 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.470 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.470 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.470 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.470 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.470 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.838017+0000", 2024-08-17T17:55:16.470 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.470 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.471 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.471 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.471 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.838017+0000", 2024-08-17T17:55:16.471 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.471 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.471 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.471 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.471 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.838017+0000", 2024-08-17T17:55:16.471 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967296 2024-08-17T17:55:16.471 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.471 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.471 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.471 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.847265+0000", 2024-08-17T17:55:16.472 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0092478170000000002 2024-08-17T17:55:16.472 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.472 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.472 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.472 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960961+0000", 2024-08-17T17:55:16.472 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.113695542 2024-08-17T17:55:16.472 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.472 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.472 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.472 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960968+0000", 2024-08-17T17:55:16.472 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0559999999999999e-06 2024-08-17T17:55:16.472 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.472 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.472 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.474 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.961005+0000", 2024-08-17T17:55:16.474 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7635999999999997e-05 2024-08-17T17:55:16.474 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.474 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.474 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.474 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.961024+0000", 2024-08-17T17:55:16.474 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.9057999999999999e-05 2024-08-17T17:55:16.474 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.474 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.474 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.474 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978574+0000", 2024-08-17T17:55:16.475 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.017549828999999999 2024-08-17T17:55:16.475 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.475 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.475 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.475 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978613+0000", 2024-08-17T17:55:16.475 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9227999999999998e-05 2024-08-17T17:55:16.475 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.475 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.475 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.475 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.475 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.475 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.44945.0:2808 44.d e1130/1015 44:b7f70492:::rbd_data.af91c8ed8b9b.000000000000005d:head v 1130'43, mlcod=1125'39)", 2024-08-17T17:55:16.476 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:45:43.839267+0000", 2024-08-17T17:55:16.476 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 572.58541567099996, 2024-08-17T17:55:16.476 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.13564867799999999, 2024-08-17T17:55:16.476 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.476 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.476 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.476 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.476 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.476 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.839267+0000", 2024-08-17T17:55:16.476 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.476 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.476 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.476 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.477 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.839267+0000", 2024-08-17T17:55:16.477 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.477 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.477 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.477 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.477 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.839267+0000", 2024-08-17T17:55:16.477 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:55:16.477 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.477 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.477 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.477 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.842005+0000", 2024-08-17T17:55:16.477 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027385220000000002 2024-08-17T17:55:16.477 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.478 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.478 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.478 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960414+0000", 2024-08-17T17:55:16.478 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.118408872 2024-08-17T17:55:16.478 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.478 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.478 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.478 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960424+0000", 2024-08-17T17:55:16.478 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0019e-05 2024-08-17T17:55:16.478 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.478 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.478 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.478 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960492+0000", 2024-08-17T17:55:16.479 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7566999999999999e-05 2024-08-17T17:55:16.479 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.479 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.479 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.479 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960519+0000", 2024-08-17T17:55:16.479 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.7560999999999999e-05 2024-08-17T17:55:16.479 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.479 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.479 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.479 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.974868+0000", 2024-08-17T17:55:16.479 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.014348681 2024-08-17T17:55:16.479 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.479 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.480 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.480 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.974916+0000", 2024-08-17T17:55:16.480 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7874999999999998e-05 2024-08-17T17:55:16.480 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.480 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.480 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.480 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.480 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.480 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.44945.0:2819 44.a 44:51bcdcfb:::rbd_data.af91c8ed8b9b.0000000000000034:head [write 0~2394689] snapc 72=[72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1130)", 2024-08-17T17:55:16.480 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:45:43.883032+0000", 2024-08-17T17:55:16.480 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 572.54165060299999, 2024-08-17T17:55:16.480 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.104906519, 2024-08-17T17:55:16.480 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.481 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.481 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.481 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.44945", 2024-08-17T17:55:16.481 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/735940947", 2024-08-17T17:55:16.481 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 2819 2024-08-17T17:55:16.481 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.481 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.481 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.481 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.481 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.883032+0000", 2024-08-17T17:55:16.481 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.481 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.481 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.481 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.482 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.883032+0000", 2024-08-17T17:55:16.482 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.482 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.482 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.482 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.482 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.883030+0000", 2024-08-17T17:55:16.482 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:55:16.482 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.482 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.482 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.482 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.886244+0000", 2024-08-17T17:55:16.482 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0032137709999999998 2024-08-17T17:55:16.483 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.483 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.483 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.483 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960544+0000", 2024-08-17T17:55:16.483 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.074300081000000004 2024-08-17T17:55:16.483 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.483 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.483 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.483 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960554+0000", 2024-08-17T17:55:16.483 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4639999999999995e-06 2024-08-17T17:55:16.483 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.483 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.483 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.484 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960606+0000", 2024-08-17T17:55:16.484 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2080999999999998e-05 2024-08-17T17:55:16.484 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.484 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.484 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.484 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960722+0000", 2024-08-17T17:55:16.484 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000115952 2024-08-17T17:55:16.484 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.484 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.484 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:55:16.484 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.960936+0000", 2024-08-17T17:55:16.484 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00021386699999999999 2024-08-17T17:55:16.484 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.484 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.485 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.485 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.972281+0000", 2024-08-17T17:55:16.485 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.011345808000000001 2024-08-17T17:55:16.485 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.485 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.485 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.485 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.978276+0000", 2024-08-17T17:55:16.485 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0059948190000000002 2024-08-17T17:55:16.485 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.485 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.486 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.486 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.987870+0000", 2024-08-17T17:55:16.486 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0095934569999999997 2024-08-17T17:55:16.486 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.486 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.486 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.486 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.987908+0000", 2024-08-17T17:55:16.486 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8858e-05 2024-08-17T17:55:16.486 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.486 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.486 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.487 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:45:43.987939+0000", 2024-08-17T17:55:16.487 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0091e-05 2024-08-17T17:55:16.487 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.487 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.487 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.487 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.487 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.487 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83239 45.e 45:777b3534:::benchmark_data_smithi130_172551_object83238:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:55:16.487 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.640449+0000", 2024-08-17T17:55:16.487 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 397.78423351200001, 2024-08-17T17:55:16.487 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.429002718, 2024-08-17T17:55:16.487 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.488 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T17:55:16.488 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.488 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.488 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.488 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83239 2024-08-17T17:55:16.488 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.488 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.488 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.488 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.488 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640449+0000", 2024-08-17T17:55:16.488 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.488 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.488 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.489 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.489 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640449+0000", 2024-08-17T17:55:16.489 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.489 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.489 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.489 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.489 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640447+0000", 2024-08-17T17:55:16.489 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:55:16.489 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.489 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.489 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.489 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640482+0000", 2024-08-17T17:55:16.489 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4570000000000003e-05 2024-08-17T17:55:16.489 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.490 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.490 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.490 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640492+0000", 2024-08-17T17:55:16.490 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0718e-05 2024-08-17T17:55:16.490 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.490 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.490 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.490 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640498+0000", 2024-08-17T17:55:16.490 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3009999999999997e-06 2024-08-17T17:55:16.490 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.490 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.490 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.490 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640600+0000", 2024-08-17T17:55:16.491 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000101969 2024-08-17T17:55:16.491 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.491 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.491 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.491 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640743+0000", 2024-08-17T17:55:16.491 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014374400000000001 2024-08-17T17:55:16.491 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.491 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.491 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:55:16.491 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.640908+0000", 2024-08-17T17:55:16.491 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016489599999999999 2024-08-17T17:55:16.491 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.491 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.492 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.492 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.649642+0000", 2024-08-17T17:55:16.492 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0087341720000000001 2024-08-17T17:55:16.492 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.492 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.492 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:55:16.492 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.649653+0000", 2024-08-17T17:55:16.492 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0495e-05 2024-08-17T17:55:16.492 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.492 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.492 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.493 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.644929+0000", 2024-08-17T17:55:16.493 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99527607200000001 2024-08-17T17:55:16.493 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.493 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.493 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:55:16.493 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.644942+0000", 2024-08-17T17:55:16.493 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2503e-05 2024-08-17T17:55:16.493 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.493 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.493 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.493 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.069362+0000", 2024-08-17T17:55:16.493 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.424420222 2024-08-17T17:55:16.493 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.494 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.494 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.494 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.069452+0000", 2024-08-17T17:55:16.494 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0087999999999995e-05 2024-08-17T17:55:16.494 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.494 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.494 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.494 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.494 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.494 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83251 45.e 45:7cdda562:::benchmark_data_smithi130_172551_object83250:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:55:16.495 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.668406+0000", 2024-08-17T17:55:16.495 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 397.75627705900001, 2024-08-17T17:55:16.495 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4230686509999999, 2024-08-17T17:55:16.495 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.495 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.495 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.495 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.495 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.495 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83251 2024-08-17T17:55:16.495 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.495 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.495 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.495 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.495 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668406+0000", 2024-08-17T17:55:16.496 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.496 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.496 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.496 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.496 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668406+0000", 2024-08-17T17:55:16.496 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.496 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.496 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.496 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.496 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668405+0000", 2024-08-17T17:55:16.496 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:55:16.496 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.496 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.497 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.497 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668461+0000", 2024-08-17T17:55:16.497 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6495e-05 2024-08-17T17:55:16.497 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.497 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.497 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.497 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668469+0000", 2024-08-17T17:55:16.497 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.1890000000000007e-06 2024-08-17T17:55:16.497 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.497 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.497 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.497 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668474+0000", 2024-08-17T17:55:16.497 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2010000000000001e-06 2024-08-17T17:55:16.498 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.498 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.498 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.498 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668532+0000", 2024-08-17T17:55:16.498 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8044999999999997e-05 2024-08-17T17:55:16.498 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.498 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.498 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:55:16.498 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.668538+0000", 2024-08-17T17:55:16.498 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3180000000000003e-06 2024-08-17T17:55:16.498 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.498 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.499 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.499 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.644966+0000", 2024-08-17T17:55:16.499 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97642784599999999 2024-08-17T17:55:16.499 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.499 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.499 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:55:16.499 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.644973+0000", 2024-08-17T17:55:16.499 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.25e-06 2024-08-17T17:55:16.499 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.499 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.500 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.500 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.071858+0000", 2024-08-17T17:55:16.500 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.426884539 2024-08-17T17:55:16.500 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.500 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.500 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.500 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.071955+0000", 2024-08-17T17:55:16.500 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7349000000000006e-05 2024-08-17T17:55:16.500 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.500 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.500 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:55:16.500 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.072058+0000", 2024-08-17T17:55:16.501 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000103127 2024-08-17T17:55:16.501 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.501 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.501 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.501 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.075587+0000", 2024-08-17T17:55:16.501 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0035286889999999998 2024-08-17T17:55:16.501 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.501 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.501 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.501 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.078139+0000", 2024-08-17T17:55:16.501 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025525970000000002 2024-08-17T17:55:16.501 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.501 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.502 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.502 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.088422+0000", 2024-08-17T17:55:16.502 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.010282217999999999 2024-08-17T17:55:16.502 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.502 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.502 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.502 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.088469+0000", 2024-08-17T17:55:16.502 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7750999999999997e-05 2024-08-17T17:55:16.502 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.502 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.502 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.502 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.091474+0000", 2024-08-17T17:55:16.502 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003004933 2024-08-17T17:55:16.503 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.503 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.503 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.503 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.503 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.503 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83268 45.e 45:74a67c83:::benchmark_data_smithi130_172551_object83267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:55:16.503 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.700848+0000", 2024-08-17T17:55:16.503 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 397.72383459299999, 2024-08-17T17:55:16.503 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.393204908, 2024-08-17T17:55:16.503 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.503 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.503 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.503 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.504 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.504 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83268 2024-08-17T17:55:16.504 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.504 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.504 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.504 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.504 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700848+0000", 2024-08-17T17:55:16.504 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.504 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.504 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.504 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.504 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700848+0000", 2024-08-17T17:55:16.504 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.504 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.505 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.505 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.505 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700847+0000", 2024-08-17T17:55:16.505 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:16.505 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.505 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.505 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.505 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700894+0000", 2024-08-17T17:55:16.505 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6879000000000003e-05 2024-08-17T17:55:16.505 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.505 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.505 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.505 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700906+0000", 2024-08-17T17:55:16.506 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2201999999999999e-05 2024-08-17T17:55:16.506 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.506 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.506 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.506 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700910+0000", 2024-08-17T17:55:16.506 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4700000000000004e-06 2024-08-17T17:55:16.506 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.506 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.506 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.506 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700963+0000", 2024-08-17T17:55:16.506 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2654000000000001e-05 2024-08-17T17:55:16.506 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.507 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.507 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:55:16.507 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.700973+0000", 2024-08-17T17:55:16.507 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0511e-05 2024-08-17T17:55:16.507 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.507 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.507 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.507 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.644997+0000", 2024-08-17T17:55:16.507 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.94402362799999995 2024-08-17T17:55:16.507 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.507 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.507 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:55:16.507 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.645004+0000", 2024-08-17T17:55:16.507 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6959999999999999e-06 2024-08-17T17:55:16.508 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.508 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.508 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.508 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.074035+0000", 2024-08-17T17:55:16.508 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4290308789999999 2024-08-17T17:55:16.508 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.508 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.508 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.508 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.074125+0000", 2024-08-17T17:55:16.508 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0348999999999998e-05 2024-08-17T17:55:16.508 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.508 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.509 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:55:16.509 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.074233+0000", 2024-08-17T17:55:16.509 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010794300000000001 2024-08-17T17:55:16.509 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.509 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.509 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.509 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.075721+0000", 2024-08-17T17:55:16.509 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0014879299999999999 2024-08-17T17:55:16.509 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.509 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.509 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.509 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.078189+0000", 2024-08-17T17:55:16.509 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024685559999999998 2024-08-17T17:55:16.509 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.510 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.510 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.510 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.091539+0000", 2024-08-17T17:55:16.510 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013349152 2024-08-17T17:55:16.510 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.510 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.510 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.510 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.091607+0000", 2024-08-17T17:55:16.510 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.8219999999999994e-05 2024-08-17T17:55:16.510 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.510 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.510 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.510 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.094053+0000", 2024-08-17T17:55:16.511 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024461499999999998 2024-08-17T17:55:16.511 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.511 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.511 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.511 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.511 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.511 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:90848 45.e 45:78c2541e:::benchmark_data_smithi130_172551_object90847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:55:16.511 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:53.710358+0000", 2024-08-17T17:55:16.511 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 382.71432480200002, 2024-08-17T17:55:16.511 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.084649796999999999, 2024-08-17T17:55:16.511 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.511 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.511 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.511 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.512 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.512 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 90848 2024-08-17T17:55:16.512 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.512 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.512 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.512 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.512 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710358+0000", 2024-08-17T17:55:16.512 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.512 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.512 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.512 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.512 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710358+0000", 2024-08-17T17:55:16.513 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.513 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.513 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.513 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.513 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710356+0000", 2024-08-17T17:55:16.513 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:16.513 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.513 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.513 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.513 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710398+0000", 2024-08-17T17:55:16.513 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1658e-05 2024-08-17T17:55:16.513 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.513 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.514 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.514 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710415+0000", 2024-08-17T17:55:16.514 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.7419e-05 2024-08-17T17:55:16.514 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.514 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.514 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.514 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710422+0000", 2024-08-17T17:55:16.514 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.9589999999999998e-06 2024-08-17T17:55:16.514 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.514 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.514 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.514 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710464+0000", 2024-08-17T17:55:16.514 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1910999999999999e-05 2024-08-17T17:55:16.515 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.515 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.515 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.515 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710579+0000", 2024-08-17T17:55:16.515 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000114905 2024-08-17T17:55:16.515 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.515 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.515 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:55:16.515 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.710693+0000", 2024-08-17T17:55:16.515 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00011406899999999999 2024-08-17T17:55:16.515 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.515 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.515 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.516 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.722128+0000", 2024-08-17T17:55:16.516 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.011434826 2024-08-17T17:55:16.516 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.516 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.516 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.516 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.722173+0000", 2024-08-17T17:55:16.516 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4889999999999999e-05 2024-08-17T17:55:16.516 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.516 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.516 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.516 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.792606+0000", 2024-08-17T17:55:16.516 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.070432967999999999 2024-08-17T17:55:16.516 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.516 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.517 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.517 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.792646+0000", 2024-08-17T17:55:16.517 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9838e-05 2024-08-17T17:55:16.517 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.517 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.517 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.517 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.795008+0000", 2024-08-17T17:55:16.517 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002361765 2024-08-17T17:55:16.517 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.517 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.517 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.517 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.517 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.518 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:90854 45.2 45:438ed877:::benchmark_data_smithi130_172551_object90853:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:55:16.518 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:53.715817+0000", 2024-08-17T17:55:16.518 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 382.70886515699999, 2024-08-17T17:55:16.518 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.084003866999999996, 2024-08-17T17:55:16.518 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.518 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.518 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.518 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.518 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.518 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 90854 2024-08-17T17:55:16.518 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.518 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.518 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.518 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.519 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.715817+0000", 2024-08-17T17:55:16.519 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.519 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.519 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.519 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.519 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.715817+0000", 2024-08-17T17:55:16.519 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.519 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.519 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.519 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.519 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.715816+0000", 2024-08-17T17:55:16.519 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:55:16.519 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.520 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.520 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.520 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.715876+0000", 2024-08-17T17:55:16.520 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9901000000000001e-05 2024-08-17T17:55:16.520 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.520 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.520 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.520 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.715888+0000", 2024-08-17T17:55:16.520 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1861999999999999e-05 2024-08-17T17:55:16.520 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.520 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.520 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.521 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.715893+0000", 2024-08-17T17:55:16.521 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8760000000000004e-06 2024-08-17T17:55:16.521 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.521 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.521 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.521 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.715937+0000", 2024-08-17T17:55:16.521 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3560000000000003e-05 2024-08-17T17:55:16.521 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.521 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.521 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.521 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.716060+0000", 2024-08-17T17:55:16.521 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012357700000000001 2024-08-17T17:55:16.521 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.521 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.522 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:16.522 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.716199+0000", 2024-08-17T17:55:16.522 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013864500000000001 2024-08-17T17:55:16.522 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.522 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.522 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.522 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.719149+0000", 2024-08-17T17:55:16.522 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002950584 2024-08-17T17:55:16.522 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.522 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.522 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.522 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.720519+0000", 2024-08-17T17:55:16.522 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001369157 2024-08-17T17:55:16.522 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.799745+0000", 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.079226435999999997 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.799777+0000", 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2379000000000003e-05 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.799821+0000", 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4014999999999998e-05 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.523 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:101590 45.e 45:7e22289b:::benchmark_data_smithi130_172551_object101589:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1184)", 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:13.080211+0000", 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 363.34447157199997, 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.090180265999999995, 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 101590 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.524 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.525 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.525 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.525 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080211+0000", 2024-08-17T17:55:16.525 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.525 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.525 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.525 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.525 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080211+0000", 2024-08-17T17:55:16.525 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.525 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.525 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.525 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.525 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080210+0000", 2024-08-17T17:55:16.525 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:55:16.526 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.526 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.526 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.526 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080268+0000", 2024-08-17T17:55:16.526 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8019999999999997e-05 2024-08-17T17:55:16.526 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.526 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.526 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.526 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080279+0000", 2024-08-17T17:55:16.526 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1412e-05 2024-08-17T17:55:16.526 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.526 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.526 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080284+0000", 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4970000000000003e-06 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080456+0000", 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017256500000000001 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080611+0000", 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000154167 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:55:16.527 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.080755+0000", 2024-08-17T17:55:16.528 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000144752 2024-08-17T17:55:16.528 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.528 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.528 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.528 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.082747+0000", 2024-08-17T17:55:16.528 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019912559999999998 2024-08-17T17:55:16.528 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.528 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.528 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.528 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.098363+0000", 2024-08-17T17:55:16.528 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.015616260999999999 2024-08-17T17:55:16.528 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.528 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.167705+0000", 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.069342564999999995 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.167764+0000", 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8885000000000003e-05 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.170391+0000", 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002627028 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.529 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.530 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.530 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.530 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.530 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:101592 45.e 45:72eb4cf9:::benchmark_data_smithi130_172551_object101591:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1184)", 2024-08-17T17:55:16.530 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:13.083483+0000", 2024-08-17T17:55:16.530 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 363.34119943000002, 2024-08-17T17:55:16.530 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.088968239000000005, 2024-08-17T17:55:16.530 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.530 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.530 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.530 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.530 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.530 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 101592 2024-08-17T17:55:16.531 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.531 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.531 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.531 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.531 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083483+0000", 2024-08-17T17:55:16.531 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.531 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.531 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.531 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.531 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083483+0000", 2024-08-17T17:55:16.531 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.531 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.531 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083482+0000", 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083563+0000", 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0604000000000003e-05 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083573+0000", 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.063e-05 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.532 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083578+0000", 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0810000000000001e-06 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083615+0000", 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6106999999999998e-05 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083705+0000", 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0505000000000002e-05 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.533 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:55:16.534 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.083808+0000", 2024-08-17T17:55:16.534 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000103218 2024-08-17T17:55:16.534 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.534 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.534 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.534 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.086192+0000", 2024-08-17T17:55:16.534 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0023838499999999999 2024-08-17T17:55:16.534 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.534 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.534 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.534 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.105096+0000", 2024-08-17T17:55:16.534 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.018904213999999999 2024-08-17T17:55:16.534 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.170445+0000", 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.065348385999999994 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.170486+0000", 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0969e-05 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.172451+0000", 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019656980000000001 2024-08-17T17:55:16.535 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:101696 45.e 45:7d01e76d:::benchmark_data_smithi130_172551_object101695:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1184)", 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:13.270443+0000", 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 363.15423938800001, 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.087804115000000002, 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 101696 2024-08-17T17:55:16.536 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270443+0000", 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270443+0000", 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270442+0000", 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:16.537 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.538 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.538 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.538 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270493+0000", 2024-08-17T17:55:16.538 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1311999999999999e-05 2024-08-17T17:55:16.538 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.538 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.538 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.538 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270506+0000", 2024-08-17T17:55:16.538 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2787e-05 2024-08-17T17:55:16.538 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.538 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.538 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.538 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270511+0000", 2024-08-17T17:55:16.539 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2390000000000002e-06 2024-08-17T17:55:16.539 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.539 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.539 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.539 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270544+0000", 2024-08-17T17:55:16.539 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2267000000000001e-05 2024-08-17T17:55:16.539 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.539 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.539 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.539 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270648+0000", 2024-08-17T17:55:16.539 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000104453 2024-08-17T17:55:16.539 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.539 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.539 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:55:16.540 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.270753+0000", 2024-08-17T17:55:16.540 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105376 2024-08-17T17:55:16.540 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.540 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.540 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.540 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.273647+0000", 2024-08-17T17:55:16.540 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028938670000000001 2024-08-17T17:55:16.540 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.540 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.540 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.540 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.303717+0000", 2024-08-17T17:55:16.540 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.030070125999999999 2024-08-17T17:55:16.540 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.540 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.355466+0000", 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.051748283999999999 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.355525+0000", 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9740000000000001e-05 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:13.358247+0000", 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002722005 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.541 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.542 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.542 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.542 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.542 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:108186 45.e 45:76cb9ea5:::benchmark_data_smithi130_172551_object3379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:55:16.542 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:25.911036+0000", 2024-08-17T17:55:16.542 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 350.51364662100002, 2024-08-17T17:55:16.542 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.12617594500000001, 2024-08-17T17:55:16.542 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.542 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.542 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.542 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.542 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.542 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 108186 2024-08-17T17:55:16.542 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911036+0000", 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911036+0000", 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911034+0000", 2024-08-17T17:55:16.543 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911042+0000", 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2099999999999993e-06 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911059+0000", 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.6358000000000002e-05 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911066+0000", 2024-08-17T17:55:16.544 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7900000000000002e-06 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911114+0000", 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8751000000000001e-05 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911281+0000", 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016636600000000001 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.911446+0000", 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000165587 2024-08-17T17:55:16.545 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.546 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.546 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.546 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.914259+0000", 2024-08-17T17:55:16.546 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028124069999999998 2024-08-17T17:55:16.546 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.546 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.546 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.546 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.917316+0000", 2024-08-17T17:55:16.546 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0030570250000000001 2024-08-17T17:55:16.546 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.546 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.546 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.546 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.034078+0000", 2024-08-17T17:55:16.547 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.116762218 2024-08-17T17:55:16.547 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.547 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.547 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.547 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.034131+0000", 2024-08-17T17:55:16.547 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3542000000000003e-05 2024-08-17T17:55:16.547 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.547 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.547 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.547 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.037212+0000", 2024-08-17T17:55:16.547 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0030805400000000001 2024-08-17T17:55:16.547 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.547 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.548 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.548 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.548 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.548 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109016 45.e 45:7392c310:::benchmark_data_smithi130_172551_object4209:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:55:16.548 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:27.871637+0000", 2024-08-17T17:55:16.548 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 348.55304534599998, 2024-08-17T17:55:16.548 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.13319884600000001, 2024-08-17T17:55:16.548 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.548 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.548 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.548 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.548 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.548 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109016 2024-08-17T17:55:16.549 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.549 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.549 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.549 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.549 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.871637+0000", 2024-08-17T17:55:16.549 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.549 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.549 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.549 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.549 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.871637+0000", 2024-08-17T17:55:16.549 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.549 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.549 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.549 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.550 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.871636+0000", 2024-08-17T17:55:16.550 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:16.550 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.550 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.550 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.550 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.871642+0000", 2024-08-17T17:55:16.550 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4119999999999998e-06 2024-08-17T17:55:16.550 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.550 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.550 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.550 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.871657+0000", 2024-08-17T17:55:16.550 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.434e-05 2024-08-17T17:55:16.550 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.550 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.551 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.551 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.871663+0000", 2024-08-17T17:55:16.551 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7619999999999998e-06 2024-08-17T17:55:16.551 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.551 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.551 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.551 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.874473+0000", 2024-08-17T17:55:16.551 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028094700000000001 2024-08-17T17:55:16.551 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.551 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.551 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.551 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.874633+0000", 2024-08-17T17:55:16.551 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00015979300000000001 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.874801+0000", 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000168479 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.877602+0000", 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028012639999999999 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.885005+0000", 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0074027429999999998 2024-08-17T17:55:16.552 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.002257+0000", 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.117251828 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.002295+0000", 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8457999999999997e-05 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.004836+0000", 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025407870000000001 2024-08-17T17:55:16.553 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.554 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.554 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.554 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.554 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.554 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109029 45.e 45:736e9cfe:::benchmark_data_smithi130_172551_object4222:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:55:16.554 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:27.888679+0000", 2024-08-17T17:55:16.554 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 348.53600357699997, 2024-08-17T17:55:16.554 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.13807940599999999, 2024-08-17T17:55:16.554 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.554 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.554 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.554 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.555 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:16.556 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.556 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109029 2024-08-17T17:55:16.556 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.556 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888679+0000", 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888679+0000", 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888678+0000", 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:16.557 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.558 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.558 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.558 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888684+0000", 2024-08-17T17:55:16.558 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7729999999999996e-06 2024-08-17T17:55:16.558 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.558 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.558 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.558 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888695+0000", 2024-08-17T17:55:16.558 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1851000000000001e-05 2024-08-17T17:55:16.558 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.558 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.558 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.558 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888700+0000", 2024-08-17T17:55:16.559 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2749999999999997e-06 2024-08-17T17:55:16.559 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.559 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.559 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.559 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888733+0000", 2024-08-17T17:55:16.559 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3581999999999998e-05 2024-08-17T17:55:16.559 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.559 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.559 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.559 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888853+0000", 2024-08-17T17:55:16.559 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00011956 2024-08-17T17:55:16.559 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.559 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.559 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.888951+0000", 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7683000000000006e-05 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.891481+0000", 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025300269999999998 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.894100+0000", 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0026193810000000001 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.560 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.561 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.024256+0000", 2024-08-17T17:55:16.561 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.13015583999999999 2024-08-17T17:55:16.561 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.561 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.561 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.561 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.024290+0000", 2024-08-17T17:55:16.561 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4668000000000001e-05 2024-08-17T17:55:16.561 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.561 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.561 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.561 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.026758+0000", 2024-08-17T17:55:16.561 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024680219999999998 2024-08-17T17:55:16.561 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.562 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.562 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.562 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.562 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.562 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109152 45.e 45:7c082e4c:::benchmark_data_smithi130_172551_object4345:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:55:16.562 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:28.221818+0000", 2024-08-17T17:55:16.562 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 348.20286421499998, 2024-08-17T17:55:16.562 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.087909323999999997, 2024-08-17T17:55:16.562 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.562 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.562 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.562 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.562 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.563 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109152 2024-08-17T17:55:16.563 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.563 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.563 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.563 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.563 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221818+0000", 2024-08-17T17:55:16.563 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.563 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.563 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.563 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.563 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221818+0000", 2024-08-17T17:55:16.563 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.563 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.563 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.564 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.564 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221817+0000", 2024-08-17T17:55:16.564 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:16.564 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.564 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.564 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.564 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221823+0000", 2024-08-17T17:55:16.564 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4330000000000001e-06 2024-08-17T17:55:16.564 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.564 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.564 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.564 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221837+0000", 2024-08-17T17:55:16.564 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3266e-05 2024-08-17T17:55:16.564 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.565 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.565 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.565 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221842+0000", 2024-08-17T17:55:16.565 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9259999999999997e-06 2024-08-17T17:55:16.565 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.565 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.565 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.565 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221888+0000", 2024-08-17T17:55:16.565 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5895e-05 2024-08-17T17:55:16.565 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.565 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.565 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.565 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221991+0000", 2024-08-17T17:55:16.565 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010257200000000001 2024-08-17T17:55:16.566 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.566 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.566 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:55:16.566 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.222081+0000", 2024-08-17T17:55:16.566 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.9705999999999998e-05 2024-08-17T17:55:16.566 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.566 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.566 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.566 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.224601+0000", 2024-08-17T17:55:16.566 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025198870000000002 2024-08-17T17:55:16.566 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.566 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.566 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.567 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.230423+0000", 2024-08-17T17:55:16.567 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0058221369999999998 2024-08-17T17:55:16.567 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.567 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.567 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.567 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.307111+0000", 2024-08-17T17:55:16.567 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.076687986999999999 2024-08-17T17:55:16.567 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.567 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.567 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.567 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.307145+0000", 2024-08-17T17:55:16.567 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3918999999999998e-05 2024-08-17T17:55:16.567 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.568 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.568 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.568 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.309728+0000", 2024-08-17T17:55:16.568 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025831959999999998 2024-08-17T17:55:16.568 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.568 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.568 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.568 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.568 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.568 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109305 45.e 45:7c61d673:::benchmark_data_smithi130_172551_object4498:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:55:16.568 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:28.556071+0000", 2024-08-17T17:55:16.568 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 347.868611289, 2024-08-17T17:55:16.568 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.138312143, 2024-08-17T17:55:16.568 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109305 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556071+0000", 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556071+0000", 2024-08-17T17:55:16.569 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.570 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.570 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.570 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.570 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556070+0000", 2024-08-17T17:55:16.570 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:16.570 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.570 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.570 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.570 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556075+0000", 2024-08-17T17:55:16.570 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8550000000000001e-06 2024-08-17T17:55:16.570 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.570 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.570 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.571 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556085+0000", 2024-08-17T17:55:16.571 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0234000000000001e-05 2024-08-17T17:55:16.571 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.571 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.571 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.571 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556091+0000", 2024-08-17T17:55:16.571 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7710000000000003e-06 2024-08-17T17:55:16.571 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.571 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.571 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.571 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556267+0000", 2024-08-17T17:55:16.571 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017659699999999999 2024-08-17T17:55:16.571 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.571 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.572 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.572 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556384+0000", 2024-08-17T17:55:16.572 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000116746 2024-08-17T17:55:16.572 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.572 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.572 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:55:16.572 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.556490+0000", 2024-08-17T17:55:16.572 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105827 2024-08-17T17:55:16.572 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.572 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.572 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.572 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.559327+0000", 2024-08-17T17:55:16.572 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002837348 2024-08-17T17:55:16.572 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.573 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.573 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.573 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.566772+0000", 2024-08-17T17:55:16.573 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0074446429999999999 2024-08-17T17:55:16.573 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.573 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.573 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.573 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.691832+0000", 2024-08-17T17:55:16.573 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.125059854 2024-08-17T17:55:16.573 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.573 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.573 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.573 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.691893+0000", 2024-08-17T17:55:16.573 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1379999999999998e-05 2024-08-17T17:55:16.574 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.574 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.574 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.574 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.694383+0000", 2024-08-17T17:55:16.574 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024904049999999998 2024-08-17T17:55:16.574 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.574 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.574 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.574 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.574 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.574 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:112629 45.e 45:7a542bc8:::benchmark_data_smithi130_172551_object7822:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:55:16.574 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:36.133587+0000", 2024-08-17T17:55:16.574 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 340.29109573900001, 2024-08-17T17:55:16.575 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.12507467, 2024-08-17T17:55:16.575 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.575 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.575 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.575 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.575 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.575 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 112629 2024-08-17T17:55:16.575 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.575 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.575 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.575 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.575 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133587+0000", 2024-08-17T17:55:16.575 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133587+0000", 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133586+0000", 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133595+0000", 2024-08-17T17:55:16.576 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.8470000000000002e-06 2024-08-17T17:55:16.577 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.577 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.577 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.577 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133607+0000", 2024-08-17T17:55:16.577 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2517999999999999e-05 2024-08-17T17:55:16.577 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.577 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.577 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.577 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133612+0000", 2024-08-17T17:55:16.577 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4449999999999998e-06 2024-08-17T17:55:16.577 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.577 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.577 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.577 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133650+0000", 2024-08-17T17:55:16.578 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8469e-05 2024-08-17T17:55:16.578 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.578 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.578 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.578 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133816+0000", 2024-08-17T17:55:16.578 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000166105 2024-08-17T17:55:16.578 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.578 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.578 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:55:16.578 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.133955+0000", 2024-08-17T17:55:16.578 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013925999999999999 2024-08-17T17:55:16.578 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.578 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.137212+0000", 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003256671 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.170283+0000", 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.033070755 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.256179+0000", 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.085895972000000001 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.579 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.580 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.256239+0000", 2024-08-17T17:55:16.580 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0436000000000002e-05 2024-08-17T17:55:16.580 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.580 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.580 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.580 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.258662+0000", 2024-08-17T17:55:16.580 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002422352 2024-08-17T17:55:16.580 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.580 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.580 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.580 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.580 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.580 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:112656 45.e 45:775998fc:::benchmark_data_smithi130_172551_object7849:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:55:16.580 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:36.177766+0000", 2024-08-17T17:55:16.581 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 340.24691637299998, 2024-08-17T17:55:16.581 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.099564008999999995, 2024-08-17T17:55:16.581 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:55:16.581 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:16.581 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:55:16.581 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:55:16.581 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:16.581 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 112656 2024-08-17T17:55:16.581 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.581 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:55:16.581 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.581 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:55:16.581 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.177766+0000", 2024-08-17T17:55:16.581 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.582 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.582 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.582 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:55:16.582 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.177766+0000", 2024-08-17T17:55:16.582 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:55:16.582 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.582 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.582 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:55:16.582 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.177765+0000", 2024-08-17T17:55:16.582 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:55:16.582 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.582 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.582 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:55:16.583 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.177771+0000", 2024-08-17T17:55:16.583 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0120000000000002e-06 2024-08-17T17:55:16.583 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.583 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.583 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:55:16.583 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.177789+0000", 2024-08-17T17:55:16.583 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.8116000000000001e-05 2024-08-17T17:55:16.583 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.583 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.583 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:55:16.583 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.177795+0000", 2024-08-17T17:55:16.583 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2729999999999999e-06 2024-08-17T17:55:16.583 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.583 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.177854+0000", 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9225999999999998e-05 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.178009+0000", 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00015437200000000001 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.178167+0000", 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000158453 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.584 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.585 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:55:16.585 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.181685+0000", 2024-08-17T17:55:16.585 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0035177289999999998 2024-08-17T17:55:16.585 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.585 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.585 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.585 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.187195+0000", 2024-08-17T17:55:16.585 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0055097999999999996 2024-08-17T17:55:16.585 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.585 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.585 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:16.585 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.274322+0000", 2024-08-17T17:55:16.585 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.087127048999999998 2024-08-17T17:55:16.585 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.586 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.586 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:55:16.586 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.274360+0000", 2024-08-17T17:55:16.586 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7824000000000002e-05 2024-08-17T17:55:16.586 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:55:16.586 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:55:16.586 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:55:16.586 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.277330+0000", 2024-08-17T17:55:16.586 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002970622 2024-08-17T17:55:16.586 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.586 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.586 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.586 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:55:16.587 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:55:16.587 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:55:16.609 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:16.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:16.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:16.711 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:16.733 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:16.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:16.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:16.835 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:16.856 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:16.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:16.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:16.957 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:16.980 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:16.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:16.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:17.082 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:17.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.109 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:17.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:17.210 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:17.231 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:17.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:17.282 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:17 smithi179 ceph-mon[93519]: pgmap v1850: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 16395/176679 objects degraded (9.280%); 9898/176679 objects misplaced (5.602%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:55:17.308 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:55:17.330 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:17 smithi130 ceph-mon[105521]: pgmap v1850: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 16395/176679 objects degraded (9.280%); 9898/176679 objects misplaced (5.602%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:55:17.331 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:17 smithi130 ceph-mon[110068]: pgmap v1850: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 16395/176679 objects degraded (9.280%); 9898/176679 objects misplaced (5.602%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:55:17.333 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:17.354 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:17.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:17.456 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:17.480 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:17.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:17.582 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:17.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.605 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:17.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:17.706 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:17.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.729 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:17.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:17.830 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:17.851 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:17.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:17.953 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:17.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:17.975 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:17.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:18.076 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:18.101 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:18.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:18.202 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:18.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:18.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:18.326 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:18.349 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:18.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:18.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:18.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:18.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:18.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:18.588 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:55:18.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.600 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:18.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:18.701 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:18.711 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T17:55:18.727 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:18.738 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:18.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:18.840 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:18.864 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:18.865 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1494203731' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:18.865 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:18 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:18.866 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1494203731' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:18.866 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:18 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:18.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.866 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:55:18.866 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 16395/176679 objects degraded (9.280%), 6 pgs degraded, 6 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1639,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1230,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":58893,"data_bytes":3858438117,"bytes_used":11879714816,"bytes_avail":564090499072,"bytes_total":575970213888,"degraded_objects":16395,"degraded_total":176679,"degraded_ratio":0.092795408622416919,"misplaced_objects":9898,"misplaced_total":176679,"misplaced_ratio":0.056022504089337156,"recovering_objects_per_sec":16,"recovering_bytes_per_sec":1113792,"recovering_keys_per_sec":0,"num_objects_recovered":170,"num_bytes_recovered":11141120,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":141},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":25,"modified":"2024-08-17T17:49:39.646037+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (7m)\n [==========================..] (remaining: 33s)","progress":0.93258428573608398,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:55:18.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:18.967 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:18.988 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:18.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:18.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:19.090 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1494203731' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:18 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:19.115 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:19.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:19.217 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:19.238 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:19.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:19.340 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:19.360 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:19.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:19.462 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:19.482 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:19.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:19.584 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:19.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.607 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:19.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:19.708 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:19.741 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:19.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:19 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:55:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:19 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:19 smithi130 ceph-mon[110068]: osdmap e1230: 8 total, 8 up, 6 in 2024-08-17T17:55:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1614956175' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:55:19.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:19 smithi130 ceph-mon[110068]: pgmap v1852: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 128 op/s; 16395/175563 objects degraded (9.339%); 9898/175563 objects misplaced (5.638%); 912 KiB/s, 14 objects/s recovering 2024-08-17T17:55:19.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:19 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:55:19.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:19 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:19.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:19 smithi130 ceph-mon[105521]: osdmap e1230: 8 total, 8 up, 6 in 2024-08-17T17:55:19.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1614956175' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:55:19.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:19 smithi130 ceph-mon[105521]: pgmap v1852: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 128 op/s; 16395/175563 objects degraded (9.339%); 9898/175563 objects misplaced (5.638%); 912 KiB/s, 14 objects/s recovering 2024-08-17T17:55:19.843 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:19.865 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:19.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:19.966 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:19.987 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:19.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:19.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:20.089 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:19 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:55:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:19 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:19 smithi179 ceph-mon[93519]: osdmap e1230: 8 total, 8 up, 6 in 2024-08-17T17:55:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1614956175' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:55:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:19 smithi179 ceph-mon[93519]: pgmap v1852: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 128 op/s; 16395/175563 objects degraded (9.339%); 9898/175563 objects misplaced (5.638%); 912 KiB/s, 14 objects/s recovering 2024-08-17T17:55:20.112 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:20.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:55:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:55:19] "GET /metrics HTTP/1.1" 200 36808 "" "Prometheus/2.51.0" 2024-08-17T17:55:20.214 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:20.236 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:20.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:20.338 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:20.359 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:20.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:20.461 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:20.482 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:20.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:20.584 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:20.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:55:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:55:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:55:20.605 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:20.707 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:20.730 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:20.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:20.824 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:55:20.824 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:55:20.832 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:20.853 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:20.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:20.955 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:20.976 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:20.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:20.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:20.987 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:20 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16395/175563 objects degraded (9.339%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:20.987 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:20 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16395/175563 objects degraded (9.339%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:21.077 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:21.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:20 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16395/175563 objects degraded (9.339%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:21.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.099 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:21.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:21.200 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:21.221 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:21.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:21.323 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:21.344 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:21.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:21.446 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:21.467 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:21.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:21.569 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:21.591 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:21.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:21.625 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:21.693 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:21.714 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:21.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:21.816 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:21.837 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:21.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:21.856 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:21 smithi130 ceph-mon[105521]: pgmap v1853: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 16351/175041 objects degraded (9.341%); 9814/175041 objects misplaced (5.607%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:55:21.856 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:21 smithi130 ceph-mon[110068]: pgmap v1853: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 16351/175041 objects degraded (9.341%); 9814/175041 objects misplaced (5.607%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:55:21.939 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:21.960 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:21.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:21.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:22.062 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:22.083 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:22.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:21 smithi179 ceph-mon[93519]: pgmap v1853: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 16351/175041 objects degraded (9.341%); 9814/175041 objects misplaced (5.607%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:55:22.185 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:22.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.207 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:22.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:22.309 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:22.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.331 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:22.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:22.398 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:55:22.432 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:22.465 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:22.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:22.567 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:22.596 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:22.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:22.698 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:22.719 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:22.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:22.821 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:22.844 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:22.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:22.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:22.919 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:55:22.919 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T17:55:22.919 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.919 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151583 45.1 45:8f20bd87:::benchmark_data_smithi130_172551_object46776:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.919 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:20.995105+0000", 2024-08-17T17:55:22.919 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 1.3508873210000001, 2024-08-17T17:55:22.919 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.350925047, 2024-08-17T17:55:22.919 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.919 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.919 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.920 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.920 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.920 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.920 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151583 2024-08-17T17:55:22.920 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.920 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.920 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.920 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.920 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.995105+0000", 2024-08-17T17:55:22.920 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.920 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.920 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.920 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.921 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.995105+0000", 2024-08-17T17:55:22.921 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.921 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.921 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.921 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.921 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.995107+0000", 2024-08-17T17:55:22.921 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1789999999999998e-06 2024-08-17T17:55:22.921 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.921 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.921 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.921 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.995115+0000", 2024-08-17T17:55:22.921 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9699999999999999e-06 2024-08-17T17:55:22.921 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.922 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.922 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.922 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.995132+0000", 2024-08-17T17:55:22.922 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7187999999999999e-05 2024-08-17T17:55:22.922 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.922 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.922 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.922 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.995144+0000", 2024-08-17T17:55:22.922 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1212e-05 2024-08-17T17:55:22.922 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.922 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.922 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.922 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.019846+0000", 2024-08-17T17:55:22.923 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.024702622 2024-08-17T17:55:22.923 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.923 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.923 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.923 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.024752+0000", 2024-08-17T17:55:22.923 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0049058269999999998 2024-08-17T17:55:22.923 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.923 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.923 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.923 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.061866+0000", 2024-08-17T17:55:22.923 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.037113785000000003 2024-08-17T17:55:22.923 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.923 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.924 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.924 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.064660+0000", 2024-08-17T17:55:22.924 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027943040000000001 2024-08-17T17:55:22.924 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.924 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.924 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.924 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.924 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.924 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151643 45.1 45:85dce169:::benchmark_data_smithi130_172551_object46836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.924 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:21.618249+0000", 2024-08-17T17:55:22.924 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.72774296000000005, 2024-08-17T17:55:22.924 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.72786387299999999, 2024-08-17T17:55:22.924 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.925 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.925 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.925 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.925 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.925 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.925 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151643 2024-08-17T17:55:22.925 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.925 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.925 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.925 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.925 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.618249+0000", 2024-08-17T17:55:22.925 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.925 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.926 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.926 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.926 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.618249+0000", 2024-08-17T17:55:22.926 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.926 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.926 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.926 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.926 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.618251+0000", 2024-08-17T17:55:22.926 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9630000000000001e-06 2024-08-17T17:55:22.926 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.926 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.926 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.926 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.618259+0000", 2024-08-17T17:55:22.927 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3760000000000002e-06 2024-08-17T17:55:22.927 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.927 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.927 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.927 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.618279+0000", 2024-08-17T17:55:22.927 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0044999999999999e-05 2024-08-17T17:55:22.927 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.927 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.927 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.927 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.618288+0000", 2024-08-17T17:55:22.927 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6160000000000008e-06 2024-08-17T17:55:22.927 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.928 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.928 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.928 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.640772+0000", 2024-08-17T17:55:22.928 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022483500999999999 2024-08-17T17:55:22.928 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.928 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.928 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.928 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.646039+0000", 2024-08-17T17:55:22.928 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0052670549999999997 2024-08-17T17:55:22.928 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.928 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.928 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.928 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.711983+0000", 2024-08-17T17:55:22.929 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.065944108000000001 2024-08-17T17:55:22.929 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.929 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.929 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.929 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.714791+0000", 2024-08-17T17:55:22.929 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028082979999999999 2024-08-17T17:55:22.929 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.929 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.929 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.929 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.929 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.929 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151671 45.1 45:8ebf42a6:::benchmark_data_smithi130_172551_object46864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.929 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:21.948287+0000", 2024-08-17T17:55:22.929 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.39770517100000002, 2024-08-17T17:55:22.930 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.39789922700000002, 2024-08-17T17:55:22.930 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.930 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.930 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.930 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.930 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.930 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.930 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151671 2024-08-17T17:55:22.930 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.930 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.930 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.930 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.930 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.948287+0000", 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.948287+0000", 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.948289+0000", 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4050000000000001e-06 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.931 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.948295+0000", 2024-08-17T17:55:22.932 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4450000000000001e-06 2024-08-17T17:55:22.932 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.932 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.932 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.932 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.948307+0000", 2024-08-17T17:55:22.932 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1673e-05 2024-08-17T17:55:22.932 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.932 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.932 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.932 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.948315+0000", 2024-08-17T17:55:22.932 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0669999999999992e-06 2024-08-17T17:55:22.932 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.932 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.933 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.933 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.192192+0000", 2024-08-17T17:55:22.933 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.24387695000000001 2024-08-17T17:55:22.933 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.933 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.933 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.933 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.201198+0000", 2024-08-17T17:55:22.933 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0090065260000000008 2024-08-17T17:55:22.933 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.933 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.933 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.933 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.234882+0000", 2024-08-17T17:55:22.933 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.033683688000000003 2024-08-17T17:55:22.934 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.934 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.934 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.934 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.237553+0000", 2024-08-17T17:55:22.934 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002670923 2024-08-17T17:55:22.934 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.934 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.934 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.934 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.934 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.934 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151647 45.1 45:83de3d77:::benchmark_data_smithi130_172551_object46840:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.934 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:21.638445+0000", 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.70754724499999999, 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.70781453999999999, 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151647 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.638445+0000", 2024-08-17T17:55:22.935 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.936 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.936 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.936 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.936 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.638445+0000", 2024-08-17T17:55:22.936 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.936 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.936 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.936 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.936 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.638447+0000", 2024-08-17T17:55:22.936 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6109999999999999e-06 2024-08-17T17:55:22.936 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.936 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.936 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.937 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.638455+0000", 2024-08-17T17:55:22.937 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5280000000000003e-06 2024-08-17T17:55:22.937 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.937 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.937 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.937 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.638470+0000", 2024-08-17T17:55:22.937 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4404e-05 2024-08-17T17:55:22.937 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.937 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.937 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.937 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.638478+0000", 2024-08-17T17:55:22.937 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.96e-06 2024-08-17T17:55:22.937 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.937 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.938 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.938 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.715446+0000", 2024-08-17T17:55:22.938 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.076968811999999998 2024-08-17T17:55:22.938 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.938 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.938 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.938 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.720304+0000", 2024-08-17T17:55:22.938 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0048574209999999998 2024-08-17T17:55:22.938 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.938 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.938 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.938 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.756146+0000", 2024-08-17T17:55:22.938 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.035841668 2024-08-17T17:55:22.939 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.939 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.939 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.939 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.759289+0000", 2024-08-17T17:55:22.939 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003143453 2024-08-17T17:55:22.939 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.939 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.939 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.939 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.939 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.939 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151589 45.1 45:882d4ac9:::benchmark_data_smithi130_172551_object46782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.939 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:21.029060+0000", 2024-08-17T17:55:22.939 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 1.316932448, 2024-08-17T17:55:22.940 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.317269198, 2024-08-17T17:55:22.940 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.940 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.940 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.940 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.940 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.940 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.940 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151589 2024-08-17T17:55:22.940 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.940 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.940 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.940 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.940 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.029060+0000", 2024-08-17T17:55:22.940 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.941 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.941 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.941 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.941 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.029060+0000", 2024-08-17T17:55:22.941 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.941 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.941 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.941 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.941 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.029062+0000", 2024-08-17T17:55:22.941 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5910000000000001e-06 2024-08-17T17:55:22.941 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.941 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.941 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.942 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.029067+0000", 2024-08-17T17:55:22.942 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5709999999999997e-06 2024-08-17T17:55:22.942 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.942 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.942 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.942 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.029079+0000", 2024-08-17T17:55:22.942 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1897e-05 2024-08-17T17:55:22.942 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.942 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.942 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.942 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.029085+0000", 2024-08-17T17:55:22.942 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.472e-06 2024-08-17T17:55:22.942 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.943 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.943 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.943 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.200788+0000", 2024-08-17T17:55:22.943 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.171702674 2024-08-17T17:55:22.943 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.943 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.943 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.943 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.206509+0000", 2024-08-17T17:55:22.943 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0057208620000000002 2024-08-17T17:55:22.943 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.943 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.943 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.943 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.238970+0000", 2024-08-17T17:55:22.944 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.032460989000000003 2024-08-17T17:55:22.944 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.944 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.944 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.944 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.241600+0000", 2024-08-17T17:55:22.944 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026296449999999999 2024-08-17T17:55:22.944 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.944 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.944 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.944 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.944 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.944 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151554 45.1 45:8f2a1042:::benchmark_data_smithi130_172551_object46747:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.944 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:20.883998+0000", 2024-08-17T17:55:22.944 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 1.461994086, 2024-08-17T17:55:22.945 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4624003189999999, 2024-08-17T17:55:22.945 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.945 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.945 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.945 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.945 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.945 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.945 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151554 2024-08-17T17:55:22.945 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.945 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.945 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.945 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.945 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.883998+0000", 2024-08-17T17:55:22.946 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.946 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.946 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.946 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.946 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.883998+0000", 2024-08-17T17:55:22.946 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.946 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.946 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.946 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.946 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.884000+0000", 2024-08-17T17:55:22.946 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.792e-06 2024-08-17T17:55:22.946 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.946 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.947 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.947 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.884008+0000", 2024-08-17T17:55:22.947 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9209999999999992e-06 2024-08-17T17:55:22.947 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.947 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.947 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.947 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.884021+0000", 2024-08-17T17:55:22.947 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3064999999999999e-05 2024-08-17T17:55:22.947 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.947 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.947 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.947 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.884031+0000", 2024-08-17T17:55:22.947 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0095000000000001e-05 2024-08-17T17:55:22.948 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.948 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.948 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.948 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.958592+0000", 2024-08-17T17:55:22.948 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.074561180000000005 2024-08-17T17:55:22.948 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.948 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.948 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.948 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.964375+0000", 2024-08-17T17:55:22.948 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0057830980000000004 2024-08-17T17:55:22.948 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.948 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.948 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.949 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.016599+0000", 2024-08-17T17:55:22.949 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.052223825000000001 2024-08-17T17:55:22.949 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.949 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.949 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.949 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.019360+0000", 2024-08-17T17:55:22.949 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027608020000000001 2024-08-17T17:55:22.949 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.949 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.949 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.949 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.950 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.950 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151620 45.1 45:8da11e4c:::benchmark_data_smithi130_172551_object46813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.950 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:21.449168+0000", 2024-08-17T17:55:22.950 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.89682425700000001, 2024-08-17T17:55:22.950 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.89729541800000001, 2024-08-17T17:55:22.950 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.950 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.950 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.950 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.950 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.950 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.950 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151620 2024-08-17T17:55:22.950 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.951 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.951 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.951 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.951 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.449168+0000", 2024-08-17T17:55:22.951 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.951 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.951 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.951 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.951 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.449168+0000", 2024-08-17T17:55:22.951 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.952 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.952 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.952 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.952 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.449170+0000", 2024-08-17T17:55:22.952 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9530000000000002e-06 2024-08-17T17:55:22.952 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.952 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.952 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.952 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.449175+0000", 2024-08-17T17:55:22.952 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3689999999999998e-06 2024-08-17T17:55:22.952 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.952 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.952 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.952 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.449188+0000", 2024-08-17T17:55:22.953 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2445e-05 2024-08-17T17:55:22.953 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.953 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.953 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.953 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.449196+0000", 2024-08-17T17:55:22.953 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3149999999999992e-06 2024-08-17T17:55:22.953 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.953 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.953 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.953 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.614499+0000", 2024-08-17T17:55:22.953 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16530298600000001 2024-08-17T17:55:22.953 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.953 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.954 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.954 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.626908+0000", 2024-08-17T17:55:22.954 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012408653 2024-08-17T17:55:22.954 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.954 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.954 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.954 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.665415+0000", 2024-08-17T17:55:22.954 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.038506869999999999 2024-08-17T17:55:22.954 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.954 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.954 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.954 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.668211+0000", 2024-08-17T17:55:22.955 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002796396 2024-08-17T17:55:22.955 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.955 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.955 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.955 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.955 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.955 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151732 45.1 45:825ceb28:::benchmark_data_smithi130_172551_object46925:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.955 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:22.178141+0000", 2024-08-17T17:55:22.955 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.167851375, 2024-08-17T17:55:22.955 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16838386499999999, 2024-08-17T17:55:22.955 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.955 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.955 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.955 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.956 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.956 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.956 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151732 2024-08-17T17:55:22.956 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.956 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.956 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.956 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.956 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.178141+0000", 2024-08-17T17:55:22.956 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.956 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.956 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.956 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.956 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.178141+0000", 2024-08-17T17:55:22.957 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.957 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.957 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.957 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.957 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.178143+0000", 2024-08-17T17:55:22.957 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1320000000000001e-06 2024-08-17T17:55:22.957 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.957 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.957 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.957 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.178151+0000", 2024-08-17T17:55:22.957 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4479999999999997e-06 2024-08-17T17:55:22.957 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.957 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.958 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.958 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.178163+0000", 2024-08-17T17:55:22.958 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2475e-05 2024-08-17T17:55:22.958 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.958 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.958 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.958 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.178173+0000", 2024-08-17T17:55:22.958 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7550000000000006e-06 2024-08-17T17:55:22.958 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.958 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.958 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.958 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.201997+0000", 2024-08-17T17:55:22.958 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023824603 2024-08-17T17:55:22.959 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.959 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.959 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.959 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.208500+0000", 2024-08-17T17:55:22.959 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0065023269999999996 2024-08-17T17:55:22.959 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.959 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.959 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.959 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.258653+0000", 2024-08-17T17:55:22.959 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.050153283999999999 2024-08-17T17:55:22.959 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.959 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.960 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.960 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.261373+0000", 2024-08-17T17:55:22.960 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027203930000000002 2024-08-17T17:55:22.960 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.960 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.960 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.960 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.960 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.960 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151733 45.1 45:867add8d:::benchmark_data_smithi130_172551_object46926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.960 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:22.179080+0000", 2024-08-17T17:55:22.960 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.16691218399999999, 2024-08-17T17:55:22.960 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16750633100000001, 2024-08-17T17:55:22.961 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.961 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.961 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.961 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.961 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.961 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.961 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151733 2024-08-17T17:55:22.961 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.961 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.961 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.961 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.961 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.179080+0000", 2024-08-17T17:55:22.961 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.961 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.962 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.962 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.962 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.179080+0000", 2024-08-17T17:55:22.962 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.962 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.962 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.962 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.962 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.179081+0000", 2024-08-17T17:55:22.962 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4799999999999997e-07 2024-08-17T17:55:22.962 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.962 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.962 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.963 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.179085+0000", 2024-08-17T17:55:22.963 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2300000000000002e-06 2024-08-17T17:55:22.963 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.963 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.963 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.963 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.179093+0000", 2024-08-17T17:55:22.963 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0320000000000003e-06 2024-08-17T17:55:22.963 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.963 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.963 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.963 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.179099+0000", 2024-08-17T17:55:22.963 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1850000000000003e-06 2024-08-17T17:55:22.963 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.963 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.964 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.964 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.208992+0000", 2024-08-17T17:55:22.964 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.029893006 2024-08-17T17:55:22.964 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.964 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.964 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.964 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.213934+0000", 2024-08-17T17:55:22.964 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0049427280000000004 2024-08-17T17:55:22.964 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.964 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.964 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.964 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.281669+0000", 2024-08-17T17:55:22.964 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.067735060999999999 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.284403+0000", 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027331439999999999 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151592 45.1 45:8f4205d7:::benchmark_data_smithi130_172551_object46785:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:21.180125+0000", 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 1.1658677879999999, 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1665216890000001, 2024-08-17T17:55:22.965 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.966 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.966 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.966 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.966 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.966 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.966 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151592 2024-08-17T17:55:22.966 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.966 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.966 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.966 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.966 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.180125+0000", 2024-08-17T17:55:22.966 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.966 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.967 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.967 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.967 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.180125+0000", 2024-08-17T17:55:22.967 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.967 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.967 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.967 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.967 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.180126+0000", 2024-08-17T17:55:22.967 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.28e-06 2024-08-17T17:55:22.967 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.967 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.967 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.967 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.180132+0000", 2024-08-17T17:55:22.968 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6330000000000001e-06 2024-08-17T17:55:22.968 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.968 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.968 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.968 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.180144+0000", 2024-08-17T17:55:22.968 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2395000000000001e-05 2024-08-17T17:55:22.968 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.968 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.968 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.968 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.180152+0000", 2024-08-17T17:55:22.968 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0309999999999998e-06 2024-08-17T17:55:22.968 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.968 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.969 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.969 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.207138+0000", 2024-08-17T17:55:22.969 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.026986452000000001 2024-08-17T17:55:22.969 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.969 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.969 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.969 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.212468+0000", 2024-08-17T17:55:22.969 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0053300489999999999 2024-08-17T17:55:22.969 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.969 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.969 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.969 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.271745+0000", 2024-08-17T17:55:22.969 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.059276263000000003 2024-08-17T17:55:22.970 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.970 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.970 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.970 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.274361+0000", 2024-08-17T17:55:22.970 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026160979999999999 2024-08-17T17:55:22.970 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.970 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.970 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.970 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.970 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.970 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151738 45.1 45:838383c6:::benchmark_data_smithi130_172551_object46931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.970 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:22.198379+0000", 2024-08-17T17:55:22.970 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.14761312500000001, 2024-08-17T17:55:22.971 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14833166, 2024-08-17T17:55:22.971 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.971 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.971 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.971 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.971 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.971 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.971 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151738 2024-08-17T17:55:22.971 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.971 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.971 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.971 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.971 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.198379+0000", 2024-08-17T17:55:22.972 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.972 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.972 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.972 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.972 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.198379+0000", 2024-08-17T17:55:22.972 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.972 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.972 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.972 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.972 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.198381+0000", 2024-08-17T17:55:22.972 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9439999999999999e-06 2024-08-17T17:55:22.972 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.972 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.973 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.973 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.198388+0000", 2024-08-17T17:55:22.973 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0929999999999997e-06 2024-08-17T17:55:22.973 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.973 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.973 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.973 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.198404+0000", 2024-08-17T17:55:22.973 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5271999999999999e-05 2024-08-17T17:55:22.973 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.973 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.973 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.973 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.198412+0000", 2024-08-17T17:55:22.973 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7949999999999996e-06 2024-08-17T17:55:22.973 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.974 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.974 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.974 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.285385+0000", 2024-08-17T17:55:22.974 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.086972770000000005 2024-08-17T17:55:22.974 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.974 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.974 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.974 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.290410+0000", 2024-08-17T17:55:22.974 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0050250199999999998 2024-08-17T17:55:22.974 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.974 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.974 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.974 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.318122+0000", 2024-08-17T17:55:22.975 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.027712176000000002 2024-08-17T17:55:22.975 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.975 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.975 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.975 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.320897+0000", 2024-08-17T17:55:22.975 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002774323 2024-08-17T17:55:22.975 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.975 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.975 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.975 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.975 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.975 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151600 45.1 45:812b1b78:::benchmark_data_smithi130_172551_object46793:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.975 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:21.215423+0000", 2024-08-17T17:55:22.975 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 1.1305696679999999, 2024-08-17T17:55:22.976 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.13135016, 2024-08-17T17:55:22.976 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.976 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.976 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.976 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.976 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.976 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.976 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151600 2024-08-17T17:55:22.976 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.976 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.976 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.976 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.976 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.215423+0000", 2024-08-17T17:55:22.976 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.977 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.977 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.977 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.977 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.215423+0000", 2024-08-17T17:55:22.977 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.977 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.977 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.977 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.977 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.215424+0000", 2024-08-17T17:55:22.977 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4249999999999999e-06 2024-08-17T17:55:22.977 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.977 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.977 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.978 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.215429+0000", 2024-08-17T17:55:22.978 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2059999999999998e-06 2024-08-17T17:55:22.978 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.978 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.978 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.978 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.215441+0000", 2024-08-17T17:55:22.978 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1395e-05 2024-08-17T17:55:22.978 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.978 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.978 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.978 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.215447+0000", 2024-08-17T17:55:22.978 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9699999999999996e-06 2024-08-17T17:55:22.978 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.315131+0000", 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.099684475999999994 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.319954+0000", 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0048231860000000001 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.357939+0000", 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.037984146000000003 2024-08-17T17:55:22.979 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.980 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.980 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.980 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.360697+0000", 2024-08-17T17:55:22.980 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002758913 2024-08-17T17:55:22.980 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.980 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.980 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.980 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.980 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.980 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151546 45.1 45:8f1282da:::benchmark_data_smithi130_172551_object46739:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.980 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:20.636378+0000", 2024-08-17T17:55:22.980 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 1.7096148630000001, 2024-08-17T17:55:22.980 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.710471326, 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151546 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.636378+0000", 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.981 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.982 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.982 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.636378+0000", 2024-08-17T17:55:22.982 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.982 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.982 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.982 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.982 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.636379+0000", 2024-08-17T17:55:22.982 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.009e-06 2024-08-17T17:55:22.982 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.982 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.982 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.982 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.636384+0000", 2024-08-17T17:55:22.982 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.164e-06 2024-08-17T17:55:22.983 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.983 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.983 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.983 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.636394+0000", 2024-08-17T17:55:22.983 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0631e-05 2024-08-17T17:55:22.983 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.983 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.983 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.983 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.636400+0000", 2024-08-17T17:55:22.983 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4999999999999999e-06 2024-08-17T17:55:22.983 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.983 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.983 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.984 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.656605+0000", 2024-08-17T17:55:22.984 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020205451999999999 2024-08-17T17:55:22.984 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.984 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.984 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.984 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.661500+0000", 2024-08-17T17:55:22.984 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0048951100000000003 2024-08-17T17:55:22.984 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.984 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.984 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.984 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.695107+0000", 2024-08-17T17:55:22.984 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.033606279000000003 2024-08-17T17:55:22.984 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.984 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.985 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.985 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.697888+0000", 2024-08-17T17:55:22.985 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027812750000000002 2024-08-17T17:55:22.985 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.985 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.985 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.985 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.985 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.985 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151741 45.1 45:86ae3c57:::benchmark_data_smithi130_172551_object46934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.985 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:22.211853+0000", 2024-08-17T17:55:22.985 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.13413909600000001, 2024-08-17T17:55:22.985 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.13505773600000001, 2024-08-17T17:55:22.985 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.985 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.986 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.986 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.986 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.986 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.986 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151741 2024-08-17T17:55:22.986 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.986 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.986 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.986 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.986 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.211853+0000", 2024-08-17T17:55:22.986 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.986 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.986 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.211853+0000", 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.211855+0000", 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2500000000000001e-06 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.211860+0000", 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1240000000000004e-06 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.987 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.988 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.988 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.211871+0000", 2024-08-17T17:55:22.988 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0818999999999999e-05 2024-08-17T17:55:22.988 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.988 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.988 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.988 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.211877+0000", 2024-08-17T17:55:22.988 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0510000000000002e-06 2024-08-17T17:55:22.988 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.988 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.988 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.988 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.290899+0000", 2024-08-17T17:55:22.988 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.079022226000000001 2024-08-17T17:55:22.988 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.295893+0000", 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0049946019999999999 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.342427+0000", 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.046533165000000001 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:22.346369+0000", 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039420640000000003 2024-08-17T17:55:22.989 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.990 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.990 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.990 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.990 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.990 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151548 45.1 45:81778a0e:::benchmark_data_smithi130_172551_object46741:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.990 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:20.640712+0000", 2024-08-17T17:55:22.990 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 1.7052807109999999, 2024-08-17T17:55:22.990 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.706265173, 2024-08-17T17:55:22.990 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.990 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.990 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.990 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.990 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.991 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.991 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151548 2024-08-17T17:55:22.991 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.991 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.991 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.991 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.991 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.640712+0000", 2024-08-17T17:55:22.991 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.991 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.991 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.991 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.991 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.640712+0000", 2024-08-17T17:55:22.991 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.640713+0000", 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3900000000000003e-07 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.640717+0000", 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0589999999999996e-06 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.640727+0000", 2024-08-17T17:55:22.992 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9150000000000008e-06 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.640732+0000", 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0560000000000004e-06 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.662145+0000", 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021413262999999998 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.667092+0000", 2024-08-17T17:55:22.993 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0049470920000000002 2024-08-17T17:55:22.994 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.994 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.994 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.994 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.728606+0000", 2024-08-17T17:55:22.994 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.061514399999999997 2024-08-17T17:55:22.994 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.994 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.994 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.994 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:20.731774+0000", 2024-08-17T17:55:22.994 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0031674659999999999 2024-08-17T17:55:22.994 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.994 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.994 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:151633 45.1 45:89c823eb:::benchmark_data_smithi130_172551_object46826:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1230)", 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:55:21.586724+0000", 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.75926863600000005, 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.760318999, 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for degraded object", 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 151633 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.995 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:22.996 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.996 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:22.996 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.586724+0000", 2024-08-17T17:55:22.996 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.996 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.996 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.996 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:22.996 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.586724+0000", 2024-08-17T17:55:22.996 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:22.996 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.996 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.996 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:22.996 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.586726+0000", 2024-08-17T17:55:22.997 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9e-06 2024-08-17T17:55:22.997 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.997 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.997 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:22.997 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.586732+0000", 2024-08-17T17:55:22.997 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8129999999999998e-06 2024-08-17T17:55:22.997 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.997 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.997 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:22.997 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.586750+0000", 2024-08-17T17:55:22.997 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7858000000000001e-05 2024-08-17T17:55:22.997 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.997 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.998 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:22.998 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.586761+0000", 2024-08-17T17:55:22.998 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0407999999999999e-05 2024-08-17T17:55:22.998 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.998 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.998 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.998 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.627959+0000", 2024-08-17T17:55:22.998 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.041198510000000001 2024-08-17T17:55:22.998 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.998 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.998 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:22.998 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.640135+0000", 2024-08-17T17:55:22.998 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012175467000000001 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.688547+0000", 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.048412660000000003 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:55:21.691148+0000", 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026002669999999999 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T17:55:22.999 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 16 2024-08-17T17:55:23.000 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:55:23.000 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:23.021 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:23.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:23.123 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:23.144 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:23.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:23.246 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:23.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.268 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:23.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:23.329 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:23 smithi130 ceph-mon[105521]: pgmap v1854: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 16297/174837 objects degraded (9.321%); 9773/174837 objects misplaced (5.590%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:55:23.330 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:23 smithi130 ceph-mon[110068]: pgmap v1854: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 16297/174837 objects degraded (9.321%); 9773/174837 objects misplaced (5.590%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:55:23.369 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:23.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.392 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:23.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:23.493 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:23.514 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:23.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:23 smithi179 ceph-mon[93519]: pgmap v1854: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 16297/174837 objects degraded (9.321%); 9773/174837 objects misplaced (5.590%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:55:23.616 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:23.637 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:23.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:23.739 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:23.760 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:23.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:23.791 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:55:23.862 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:23.883 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:23.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:23.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:23.985 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:24.006 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:24.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:24.108 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:24.128 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:24.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:24.230 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:24.268 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:24.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:24.269 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:24.328 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:24.370 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:24.393 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:24.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:24.495 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:24.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.517 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:24.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:24.618 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:24.640 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:24.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:24.742 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:24.763 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:24.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:24.865 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:24.887 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:24.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:24.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:24.988 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:25.017 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:25.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:25.120 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:25.151 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:25.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:25.190 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:25 smithi130 ceph-mon[105521]: pgmap v1855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 16297/173592 objects degraded (9.388%); 9773/173592 objects misplaced (5.630%); 729 KiB/s, 11 objects/s recovering 2024-08-17T17:55:25.190 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:25 smithi130 ceph-mon[110068]: pgmap v1855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 16297/173592 objects degraded (9.388%); 9773/173592 objects misplaced (5.630%); 729 KiB/s, 11 objects/s recovering 2024-08-17T17:55:25.254 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:25.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.277 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:25.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:25.314 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:25.378 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:25.399 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:25.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:25.501 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:25.530 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:25.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:25.577 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:55:25.577 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:55:25.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:25 smithi179 ceph-mon[93519]: pgmap v1855: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 112 op/s; 16297/173592 objects degraded (9.388%); 9773/173592 objects misplaced (5.630%); 729 KiB/s, 11 objects/s recovering 2024-08-17T17:55:25.605 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:55:25.605 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:55:25.621 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:55:25.622 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:55:25.632 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:25.646 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:55:25.647 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:55:25.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.654 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:25.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:25.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:55:25.666 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:55:25.684 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:55:25.685 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:55:25.704 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:55:25.704 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:55:25.721 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:55:25.722 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:55:25.754 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:25.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.778 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:25.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:25.878 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:25.903 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:25.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:25.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:26.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:26.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.027 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:26.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:26.128 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:26.151 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:26.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:26.253 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:26.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:26.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:26.376 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:26.398 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:26.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:26.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:26 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4036522166' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:26.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16297/173592 objects degraded (9.388%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:26.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:26 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4036522166' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:26.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16297/173592 objects degraded (9.388%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:26.500 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:26.521 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:26.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:26 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4036522166' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16297/173592 objects degraded (9.388%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:26.623 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:26.644 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:26.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:26.746 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:26.772 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:26.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:26.875 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:26.896 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:26.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:26.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:26.998 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:27.019 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:27.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:27.120 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:27.143 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:27.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:27.245 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:27.267 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:27.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:27 smithi179 ceph-mon[93519]: pgmap v1856: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 16149/172986 objects degraded (9.335%); 9634/172986 objects misplaced (5.569%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:55:27.369 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:27.391 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:27.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:27.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:27 smithi130 ceph-mon[105521]: pgmap v1856: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 16149/172986 objects degraded (9.335%); 9634/172986 objects misplaced (5.569%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:55:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:27 smithi130 ceph-mon[110068]: pgmap v1856: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 123 op/s; 16149/172986 objects degraded (9.335%); 9634/172986 objects misplaced (5.569%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:55:27.493 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:27.519 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:27.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:27.622 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:27.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.644 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:27.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:27.745 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:27.767 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:27.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:27.869 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:27.890 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:27.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:27.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:27.991 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:28.012 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:28.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:28.114 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:28.143 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:28.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:28.245 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:28.269 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:28.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:28.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:28.392 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:28.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:28.494 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:28.516 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:28.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:55:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:28.618 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:28.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.641 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:28.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:28.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:55:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:55:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:28.742 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:28.764 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:28.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:28.865 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:28.886 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:28.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:28.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:28.988 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:29.009 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:29.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:29.111 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:29.132 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:29.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:29.234 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:29.255 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:29.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:29.356 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:29.377 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:29.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:29.479 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:29.500 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:29.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:29 smithi179 ceph-mon[93519]: pgmap v1857: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 16149/171648 objects degraded (9.408%); 9634/171648 objects misplaced (5.613%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:55:29.602 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:29.613 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:29 smithi130 ceph-mon[105521]: pgmap v1857: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 16149/171648 objects degraded (9.408%); 9634/171648 objects misplaced (5.613%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:55:29.613 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:29 smithi130 ceph-mon[110068]: pgmap v1857: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 16149/171648 objects degraded (9.408%); 9634/171648 objects misplaced (5.613%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:55:29.627 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:29.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:29.729 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:29.753 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:29.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:29.854 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:29.876 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:55:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:55:29] "GET /metrics HTTP/1.1" 200 36810 "" "Prometheus/2.51.0" 2024-08-17T17:55:29.881 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:29.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:29.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:29.983 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:30.004 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:30.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:30.106 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:30.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.128 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:30.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:30.229 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:30.252 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:30.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:30.315 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:55:30.353 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:30.387 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:30.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:30.489 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:30.518 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:30.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:30.561 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:30 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2133075546' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:55:30.562 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:30 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2133075546' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:55:30.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:30 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2133075546' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:55:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:55:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:55:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:55:30.620 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:30.642 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:30.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:30.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:55:30.744 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:30.775 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:30.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:30.877 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:30.900 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:30.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:30.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:31.001 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:31.023 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:31.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:31.124 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:31.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.147 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:31.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:31.248 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:31.269 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:31.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:31.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:31.392 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:31.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:31.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:31 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:31.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2133075546' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:55:31.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:31 smithi130 ceph-mon[105521]: osdmap e1231: 8 total, 8 up, 6 in 2024-08-17T17:55:31.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16149/171648 objects degraded (9.408%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:31.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:31 smithi130 ceph-mon[105521]: pgmap v1859: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 16105/171141 objects degraded (9.410%); 9542/171141 objects misplaced (5.576%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:55:31.488 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:31 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:31.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2133075546' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:55:31.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:31 smithi130 ceph-mon[110068]: osdmap e1231: 8 total, 8 up, 6 in 2024-08-17T17:55:31.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16149/171648 objects degraded (9.408%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:31.489 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:31 smithi130 ceph-mon[110068]: pgmap v1859: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 16105/171141 objects degraded (9.410%); 9542/171141 objects misplaced (5.576%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:55:31.494 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:31.515 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:31.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:31 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2133075546' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:55:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:31 smithi179 ceph-mon[93519]: osdmap e1231: 8 total, 8 up, 6 in 2024-08-17T17:55:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16149/171648 objects degraded (9.408%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:31 smithi179 ceph-mon[93519]: pgmap v1859: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 129 op/s; 16105/171141 objects degraded (9.410%); 9542/171141 objects misplaced (5.576%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:55:31.616 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:31.646 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:31.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:31.748 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:31.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.770 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:31.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:31.871 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:31.892 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:31.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:31.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:31.993 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:32.014 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:32.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:32.116 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:32.134 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:55:32.134 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:55:32.137 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:32.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:32.239 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:32.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.262 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:32.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:32.283 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:55:32.284 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:55:32.363 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:32.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.385 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:32.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:32.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:32.525 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:32.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:32.627 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:32.649 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:32.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:32.751 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:32.772 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:32.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:32.874 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:32.895 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:32.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:32.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:32.997 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:33.017 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:33.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:33.119 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:33.141 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:33.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:33.241 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:55:33.243 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:33.254 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:55:33.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.266 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:33.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:33.367 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:33.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.388 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:33.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:33.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:33 smithi130 ceph-mon[105521]: from='client.45737 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:55:33.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:33 smithi130 ceph-mon[105521]: pgmap v1860: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 133 op/s; 15993/170832 objects degraded (9.362%); 9495/170832 objects misplaced (5.558%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:55:33.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:33 smithi130 ceph-mon[110068]: from='client.45737 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:55:33.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:33 smithi130 ceph-mon[110068]: pgmap v1860: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 133 op/s; 15993/170832 objects degraded (9.362%); 9495/170832 objects misplaced (5.558%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:55:33.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:33.511 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:33.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:33.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:33 smithi179 ceph-mon[93519]: from='client.45737 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:55:33.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:33 smithi179 ceph-mon[93519]: pgmap v1860: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 133 op/s; 15993/170832 objects degraded (9.362%); 9495/170832 objects misplaced (5.558%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:55:33.613 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:33.634 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:33.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:33.736 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:33.761 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:33.762 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1859,"stamp":"2024-08-17T17:55:31.072066+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3737458661,"num_objects":57047,"num_object_clones":0,"num_object_copies":171141,"num_objects_missing_on_primary":3024,"num_objects_missing":3024,"num_objects_degraded":16105,"num_objects_misplaced":9542,"num_objects_unfound":0,"num_objects_dirty":57047,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":152867,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36040,"num_bytes_recovered":2183339021,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108670,"ondisk_log_size":108670,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11295396,"kb_used_data":9892404,"kb_used_omap":295,"kb_used_meta":1402648,"kb_avail":551175516,"statfs":{"total":575970213888,"available":564403728384,"internally_reserved":0,"allocated":10129821696,"data_stored":10100189473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1436312015},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-85196800,"num_objects":-1300,"num_object_clones":0,"num_object_copies":-3900,"num_objects_missing_on_primary":-86,"num_objects_missing":-86,"num_objects_degraded":-246,"num_objects_misplaced":-272,"num_objects_unfound":0,"num_objects_dirty":-1300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1300,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":170,"num_bytes_recovered":11141120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002707"},"pg_stats":[{"pgid":"45.f","version":"1230'9595","reported_seq":20135,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.096808+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:55:30.096808+0000","last_peered":"2024-08-17T17:55:30.096808+0000","last_clean":"2024-08-17T17:55:30.096808+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:55:30.096808+0000","last_undegraded":"2024-08-17T17:55:30.096808+0000","last_fullsized":"2024-08-17T17:55:30.096808+0000","mapping_epoch":1194,"log_start":"1171'2731","ondisk_log_start":"1171'2731","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6864,"log_dups_size":0,"ondisk_log_size":6864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233897984,"num_objects":3569,"num_object_clones":0,"num_object_copies":10707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9595,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1230'9450","reported_seq":19920,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:28.105598+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:55:28.105598+0000","last_peered":"2024-08-17T17:55:28.105598+0000","last_clean":"2024-08-17T17:55:28.105598+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:55:28.105598+0000","last_undegraded":"2024-08-17T17:55:28.105598+0000","last_fullsized":"2024-08-17T17:55:28.105598+0000","mapping_epoch":1211,"log_start":"1170'2432","ondisk_log_start":"1170'2432","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232259584,"num_objects":3544,"num_object_clones":0,"num_object_copies":10632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9450,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1230'9759","reported_seq":20606,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:28.450376+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:55:28.450376+0000","last_peered":"2024-08-17T17:55:28.450376+0000","last_clean":"2024-08-17T17:55:28.450376+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:55:28.450376+0000","last_undegraded":"2024-08-17T17:55:28.450376+0000","last_fullsized":"2024-08-17T17:55:28.450376+0000","mapping_epoch":1205,"log_start":"1175'3868","ondisk_log_start":"1175'3868","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5891,"log_dups_size":0,"ondisk_log_size":5891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237109272,"num_objects":3619,"num_object_clones":0,"num_object_copies":10857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3619,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9759,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1230'9514","reported_seq":9585,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:28.126183+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:55:28.126183+0000","last_peered":"2024-08-17T17:55:28.126183+0000","last_clean":"2024-08-17T17:55:28.126183+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:55:28.126183+0000","last_undegraded":"2024-08-17T17:55:28.126183+0000","last_fullsized":"2024-08-17T17:55:28.126183+0000","mapping_epoch":1167,"log_start":"1171'2789","ondisk_log_start":"1171'2789","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6725,"log_dups_size":0,"ondisk_log_size":6725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228458496,"num_objects":3486,"num_object_clones":0,"num_object_copies":10458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9514,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1230'9544","reported_seq":14797,"reported_epoch":1230,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:26.594048+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:55:26.594048+0000","last_peered":"2024-08-17T17:55:26.594048+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:55:26.594048+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1175'3626","ondisk_log_start":"1175'3626","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233963520,"num_objects":3570,"num_object_clones":0,"num_object_copies":10710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":3570,"num_objects_unfound":0,"num_objects_dirty":3570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9544,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":3052},{"shards":"1,4,5","objects":84}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1230'9629","reported_seq":9700,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:27.859161+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:55:27.859161+0000","last_peered":"2024-08-17T17:55:27.859161+0000","last_clean":"2024-08-17T17:55:27.859161+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:55:27.859161+0000","last_undegraded":"2024-08-17T17:55:27.859161+0000","last_fullsized":"2024-08-17T17:55:27.859161+0000","mapping_epoch":1167,"log_start":"1171'2877","ondisk_log_start":"1171'2877","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6752,"log_dups_size":0,"ondisk_log_size":6752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227082240,"num_objects":3465,"num_object_clones":0,"num_object_copies":10395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9629,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1230'9600","reported_seq":9671,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:28.166415+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:55:28.166415+0000","last_peered":"2024-08-17T17:55:28.166415+0000","last_clean":"2024-08-17T17:55:28.166415+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:55:28.166415+0000","last_undegraded":"2024-08-17T17:55:28.166415+0000","last_fullsized":"2024-08-17T17:55:28.166415+0000","mapping_epoch":1167,"log_start":"1171'2869","ondisk_log_start":"1171'2869","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6731,"log_dups_size":0,"ondisk_log_size":6731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237633536,"num_objects":3626,"num_object_clones":0,"num_object_copies":10878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9600,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1230'9504","reported_seq":18677,"reported_epoch":1230,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:26.764065+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:55:26.764065+0000","last_peered":"2024-08-17T17:55:26.764065+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:55:26.764065+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1174'3575","ondisk_log_start":"1174'3575","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5929,"log_dups_size":0,"ondisk_log_size":5929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228458496,"num_objects":3486,"num_object_clones":0,"num_object_copies":10458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2265,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9504,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3009,"num_bytes_recovered":197197824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":99},{"shards":"1,5,7","objects":2067},{"shards":"1,7","objects":1320}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1230'9452","reported_seq":9533,"reported_epoch":1230,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:27.787769+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:55:27.787769+0000","last_peered":"2024-08-17T17:55:27.787769+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:55:27.787769+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1171'2777","ondisk_log_start":"1171'2777","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6675,"log_dups_size":0,"ondisk_log_size":6675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237502464,"num_objects":3624,"num_object_clones":0,"num_object_copies":10872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9452,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":3624},{"shards":"1,6","objects":-2209}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1230'9551","reported_seq":12637,"reported_epoch":1230,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:28.557219+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:55:28.557219+0000","last_peered":"2024-08-17T17:55:28.557219+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:55:28.557219+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1175'3606","ondisk_log_start":"1175'3606","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5945,"log_dups_size":0,"ondisk_log_size":5945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237436928,"num_objects":3623,"num_object_clones":0,"num_object_copies":10869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9551,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":2062},{"shards":"1,3,5","objects":64}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1230'9452","reported_seq":9594,"reported_epoch":1230,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:28.718331+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:55:28.718331+0000","last_peered":"2024-08-17T17:55:28.718331+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:55:28.718331+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1170'300","ondisk_log_start":"1170'300","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9152,"log_dups_size":0,"ondisk_log_size":9152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237371392,"num_objects":3622,"num_object_clones":0,"num_object_copies":10866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9452,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-1531}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1230'9496","reported_seq":15698,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:28.964643+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:55:28.964643+0000","last_peered":"2024-08-17T17:55:28.964643+0000","last_clean":"2024-08-17T17:55:28.964643+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:55:28.964643+0000","last_undegraded":"2024-08-17T17:55:28.964643+0000","last_fullsized":"2024-08-17T17:55:28.964643+0000","mapping_epoch":1183,"log_start":"1170'400","ondisk_log_start":"1170'400","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9096,"log_dups_size":0,"ondisk_log_size":9096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232652800,"num_objects":3550,"num_object_clones":0,"num_object_copies":10650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9496,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1230'9608","reported_seq":9748,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:28.470150+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:55:28.470150+0000","last_peered":"2024-08-17T17:55:28.470150+0000","last_clean":"2024-08-17T17:55:28.470150+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:55:28.470150+0000","last_undegraded":"2024-08-17T17:55:28.470150+0000","last_fullsized":"2024-08-17T17:55:28.470150+0000","mapping_epoch":1167,"log_start":"1175'3691","ondisk_log_start":"1175'3691","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5917,"log_dups_size":0,"ondisk_log_size":5917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232783872,"num_objects":3552,"num_object_clones":0,"num_object_copies":10656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9608,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1230'9422","reported_seq":9493,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:27.799248+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:55:27.799248+0000","last_peered":"2024-08-17T17:55:27.799248+0000","last_clean":"2024-08-17T17:55:27.799248+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:55:27.799248+0000","last_undegraded":"2024-08-17T17:55:27.799248+0000","last_fullsized":"2024-08-17T17:55:27.799248+0000","mapping_epoch":1167,"log_start":"1170'2425","ondisk_log_start":"1170'2425","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6997,"log_dups_size":0,"ondisk_log_size":6997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230162432,"num_objects":3512,"num_object_clones":0,"num_object_copies":10536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9422,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1230'9699","reported_seq":22574,"reported_epoch":1230,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:30.231960+0000","last_change":"2024-08-17T17:51:24.150404+0000","last_active":"2024-08-17T17:55:30.231960+0000","last_peered":"2024-08-17T17:55:30.231960+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:55:30.231960+0000","last_undegraded":"2024-08-17T17:50:03.462270+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1171'2791","ondisk_log_start":"1171'2791","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6908,"log_dups_size":0,"ondisk_log_size":6908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240189440,"num_objects":3665,"num_object_clones":0,"num_object_copies":10995,"num_objects_missing_on_primary":3024,"num_objects_missing":3024,"num_objects_degraded":70,"num_objects_misplaced":5972,"num_objects_unfound":0,"num_objects_dirty":3665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9699,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5124,"num_bytes_recovered":335806464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5","objects":11},{"shards":"0,4,6","objects":1356},{"shards":"0,6","objects":1},{"shards":"4,5","objects":70},{"shards":"4,5,6","objects":2227}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1230'9314","reported_seq":22640,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:28.546277+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:55:28.546277+0000","last_peered":"2024-08-17T17:55:28.546277+0000","last_clean":"2024-08-17T17:55:28.546277+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:55:28.546277+0000","last_undegraded":"2024-08-17T17:55:28.546277+0000","last_fullsized":"2024-08-17T17:55:28.546277+0000","mapping_epoch":1196,"log_start":"1174'3369","ondisk_log_start":"1174'3369","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5945,"log_dups_size":0,"ondisk_log_size":5945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229900288,"num_objects":3508,"num_object_clones":0,"num_object_copies":10524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9314,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1917,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.807154+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:55:18.807154+0000","last_peered":"2024-08-17T17:55:18.807154+0000","last_clean":"2024-08-17T17:55:18.807154+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:55:18.807154+0000","last_undegraded":"2024-08-17T17:55:18.807154+0000","last_fullsized":"2024-08-17T17:55:18.807154+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":507,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.173015+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:55:10.173015+0000","last_peered":"2024-08-17T17:55:10.173015+0000","last_clean":"2024-08-17T17:55:10.173015+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:55:10.173015+0000","last_undegraded":"2024-08-17T17:55:10.173015+0000","last_fullsized":"2024-08-17T17:55:10.173015+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1836,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.711106+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:55:18.711106+0000","last_peered":"2024-08-17T17:55:18.711106+0000","last_clean":"2024-08-17T17:55:18.711106+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:55:18.711106+0000","last_undegraded":"2024-08-17T17:55:18.711106+0000","last_fullsized":"2024-08-17T17:55:18.711106+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1860,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.707791+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:55:18.707791+0000","last_peered":"2024-08-17T17:55:18.707791+0000","last_clean":"2024-08-17T17:55:18.707791+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:55:18.707791+0000","last_undegraded":"2024-08-17T17:55:18.707791+0000","last_fullsized":"2024-08-17T17:55:18.707791+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":69,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.173600+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:55:10.173600+0000","last_peered":"2024-08-17T17:55:10.173600+0000","last_clean":"2024-08-17T17:55:10.173600+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:55:10.173600+0000","last_undegraded":"2024-08-17T17:55:10.173600+0000","last_fullsized":"2024-08-17T17:55:10.173600+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1254,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.708827+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:55:18.708827+0000","last_peered":"2024-08-17T17:55:18.708827+0000","last_clean":"2024-08-17T17:55:18.708827+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:55:18.708827+0000","last_undegraded":"2024-08-17T17:55:18.708827+0000","last_fullsized":"2024-08-17T17:55:18.708827+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":112,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.710327+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:55:18.710327+0000","last_peered":"2024-08-17T17:55:18.710327+0000","last_clean":"2024-08-17T17:55:18.710327+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:55:18.710327+0000","last_undegraded":"2024-08-17T17:55:18.710327+0000","last_fullsized":"2024-08-17T17:55:18.710327+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":632,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.708356+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:55:18.708356+0000","last_peered":"2024-08-17T17:55:18.708356+0000","last_clean":"2024-08-17T17:55:18.708356+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:55:18.708356+0000","last_undegraded":"2024-08-17T17:55:18.708356+0000","last_fullsized":"2024-08-17T17:55:18.708356+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1875,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.806809+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:55:18.806809+0000","last_peered":"2024-08-17T17:55:18.806809+0000","last_clean":"2024-08-17T17:55:18.806809+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:55:18.806809+0000","last_undegraded":"2024-08-17T17:55:18.806809+0000","last_fullsized":"2024-08-17T17:55:18.806809+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1877,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.751330+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:55:18.751330+0000","last_peered":"2024-08-17T17:55:18.751330+0000","last_clean":"2024-08-17T17:55:18.751330+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:55:18.751330+0000","last_undegraded":"2024-08-17T17:55:18.751330+0000","last_fullsized":"2024-08-17T17:55:18.751330+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":97,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.708384+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:55:18.708384+0000","last_peered":"2024-08-17T17:55:18.708384+0000","last_clean":"2024-08-17T17:55:18.708384+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:55:18.708384+0000","last_undegraded":"2024-08-17T17:55:18.708384+0000","last_fullsized":"2024-08-17T17:55:18.708384+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1849,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.709542+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:55:18.709542+0000","last_peered":"2024-08-17T17:55:18.709542+0000","last_clean":"2024-08-17T17:55:18.709542+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:55:18.709542+0000","last_undegraded":"2024-08-17T17:55:18.709542+0000","last_fullsized":"2024-08-17T17:55:18.709542+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1904,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.710549+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:55:18.710549+0000","last_peered":"2024-08-17T17:55:18.710549+0000","last_clean":"2024-08-17T17:55:18.710549+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:55:18.710549+0000","last_undegraded":"2024-08-17T17:55:18.710549+0000","last_fullsized":"2024-08-17T17:55:18.710549+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1898,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.807439+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:55:18.807439+0000","last_peered":"2024-08-17T17:55:18.807439+0000","last_clean":"2024-08-17T17:55:18.807439+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:55:18.807439+0000","last_undegraded":"2024-08-17T17:55:18.807439+0000","last_fullsized":"2024-08-17T17:55:18.807439+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1871,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.710305+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:55:18.710305+0000","last_peered":"2024-08-17T17:55:18.710305+0000","last_clean":"2024-08-17T17:55:18.710305+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:55:18.710305+0000","last_undegraded":"2024-08-17T17:55:18.710305+0000","last_fullsized":"2024-08-17T17:55:18.710305+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1204,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.173724+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:55:10.173724+0000","last_peered":"2024-08-17T17:55:10.173724+0000","last_clean":"2024-08-17T17:55:10.173724+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:55:10.173724+0000","last_undegraded":"2024-08-17T17:55:10.173724+0000","last_fullsized":"2024-08-17T17:55:10.173724+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1843,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.709999+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:55:18.709999+0000","last_peered":"2024-08-17T17:55:18.709999+0000","last_clean":"2024-08-17T17:55:18.709999+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:55:18.709999+0000","last_undegraded":"2024-08-17T17:55:18.709999+0000","last_fullsized":"2024-08-17T17:55:18.709999+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1892,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.807260+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:55:18.807260+0000","last_peered":"2024-08-17T17:55:18.807260+0000","last_clean":"2024-08-17T17:55:18.807260+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:55:18.807260+0000","last_undegraded":"2024-08-17T17:55:18.807260+0000","last_fullsized":"2024-08-17T17:55:18.807260+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1799,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.707716+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:55:18.707716+0000","last_peered":"2024-08-17T17:55:18.707716+0000","last_clean":"2024-08-17T17:55:18.707716+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:55:18.707716+0000","last_undegraded":"2024-08-17T17:55:18.707716+0000","last_fullsized":"2024-08-17T17:55:18.707716+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":523,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.155671+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:55:10.155671+0000","last_peered":"2024-08-17T17:55:10.155671+0000","last_clean":"2024-08-17T17:55:10.155671+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:55:10.155671+0000","last_undegraded":"2024-08-17T17:55:10.155671+0000","last_fullsized":"2024-08-17T17:55:10.155671+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1200,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.155557+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:55:10.155557+0000","last_peered":"2024-08-17T17:55:10.155557+0000","last_clean":"2024-08-17T17:55:10.155557+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:55:10.155557+0000","last_undegraded":"2024-08-17T17:55:10.155557+0000","last_fullsized":"2024-08-17T17:55:10.155557+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1897,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.708631+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:55:18.708631+0000","last_peered":"2024-08-17T17:55:18.708631+0000","last_clean":"2024-08-17T17:55:18.708631+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:55:18.708631+0000","last_undegraded":"2024-08-17T17:55:18.708631+0000","last_fullsized":"2024-08-17T17:55:18.708631+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":361,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.155639+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:55:10.155639+0000","last_peered":"2024-08-17T17:55:10.155639+0000","last_clean":"2024-08-17T17:55:10.155639+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:55:10.155639+0000","last_undegraded":"2024-08-17T17:55:10.155639+0000","last_fullsized":"2024-08-17T17:55:10.155639+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":507,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.173314+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:55:10.173314+0000","last_peered":"2024-08-17T17:55:10.173314+0000","last_clean":"2024-08-17T17:55:10.173314+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:55:10.173314+0000","last_undegraded":"2024-08-17T17:55:10.173314+0000","last_fullsized":"2024-08-17T17:55:10.173314+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1185,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.173218+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:55:10.173218+0000","last_peered":"2024-08-17T17:55:10.173218+0000","last_clean":"2024-08-17T17:55:10.173218+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:55:10.173218+0000","last_undegraded":"2024-08-17T17:55:10.173218+0000","last_fullsized":"2024-08-17T17:55:10.173218+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1193,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.173762+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:55:10.173762+0000","last_peered":"2024-08-17T17:55:10.173762+0000","last_clean":"2024-08-17T17:55:10.173762+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:55:10.173762+0000","last_undegraded":"2024-08-17T17:55:10.173762+0000","last_fullsized":"2024-08-17T17:55:10.173762+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1945,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.806881+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:55:18.806881+0000","last_peered":"2024-08-17T17:55:18.806881+0000","last_clean":"2024-08-17T17:55:18.806881+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:55:18.806881+0000","last_undegraded":"2024-08-17T17:55:18.806881+0000","last_fullsized":"2024-08-17T17:55:18.806881+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1828,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.710219+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:55:18.710219+0000","last_peered":"2024-08-17T17:55:18.710219+0000","last_clean":"2024-08-17T17:55:18.710219+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:55:18.710219+0000","last_undegraded":"2024-08-17T17:55:18.710219+0000","last_fullsized":"2024-08-17T17:55:18.710219+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1903,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.711193+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:55:18.711193+0000","last_peered":"2024-08-17T17:55:18.711193+0000","last_clean":"2024-08-17T17:55:18.711193+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:55:18.711193+0000","last_undegraded":"2024-08-17T17:55:18.711193+0000","last_fullsized":"2024-08-17T17:55:18.711193+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1830,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.708820+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:55:18.708820+0000","last_peered":"2024-08-17T17:55:18.708820+0000","last_clean":"2024-08-17T17:55:18.708820+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:55:18.708820+0000","last_undegraded":"2024-08-17T17:55:18.708820+0000","last_fullsized":"2024-08-17T17:55:18.708820+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1863,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.806843+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:55:18.806843+0000","last_peered":"2024-08-17T17:55:18.806843+0000","last_clean":"2024-08-17T17:55:18.806843+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:55:18.806843+0000","last_undegraded":"2024-08-17T17:55:18.806843+0000","last_fullsized":"2024-08-17T17:55:18.806843+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1220,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.155655+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:55:10.155655+0000","last_peered":"2024-08-17T17:55:10.155655+0000","last_clean":"2024-08-17T17:55:10.155655+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:55:10.155655+0000","last_undegraded":"2024-08-17T17:55:10.155655+0000","last_fullsized":"2024-08-17T17:55:10.155655+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1909,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.710560+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:55:18.710560+0000","last_peered":"2024-08-17T17:55:18.710560+0000","last_clean":"2024-08-17T17:55:18.710560+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:55:18.710560+0000","last_undegraded":"2024-08-17T17:55:18.710560+0000","last_fullsized":"2024-08-17T17:55:18.710560+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":96,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.708220+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:55:18.708220+0000","last_peered":"2024-08-17T17:55:18.708220+0000","last_clean":"2024-08-17T17:55:18.708220+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:55:18.708220+0000","last_undegraded":"2024-08-17T17:55:18.708220+0000","last_fullsized":"2024-08-17T17:55:18.708220+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1928,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.710477+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:55:18.710477+0000","last_peered":"2024-08-17T17:55:18.710477+0000","last_clean":"2024-08-17T17:55:18.710477+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:55:18.710477+0000","last_undegraded":"2024-08-17T17:55:18.710477+0000","last_fullsized":"2024-08-17T17:55:18.710477+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":665,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.709324+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:55:18.709324+0000","last_peered":"2024-08-17T17:55:18.709324+0000","last_clean":"2024-08-17T17:55:18.709324+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:55:18.709324+0000","last_undegraded":"2024-08-17T17:55:18.709324+0000","last_fullsized":"2024-08-17T17:55:18.709324+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1909,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.710637+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:55:18.710637+0000","last_peered":"2024-08-17T17:55:18.710637+0000","last_clean":"2024-08-17T17:55:18.710637+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:55:18.710637+0000","last_undegraded":"2024-08-17T17:55:18.710637+0000","last_fullsized":"2024-08-17T17:55:18.710637+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1238,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.806588+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:55:18.806588+0000","last_peered":"2024-08-17T17:55:18.806588+0000","last_clean":"2024-08-17T17:55:18.806588+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:55:18.806588+0000","last_undegraded":"2024-08-17T17:55:18.806588+0000","last_fullsized":"2024-08-17T17:55:18.806588+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":156,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.708543+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:55:18.708543+0000","last_peered":"2024-08-17T17:55:18.708543+0000","last_clean":"2024-08-17T17:55:18.708543+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:55:18.708543+0000","last_undegraded":"2024-08-17T17:55:18.708543+0000","last_fullsized":"2024-08-17T17:55:18.708543+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1920,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.707740+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:55:18.707740+0000","last_peered":"2024-08-17T17:55:18.707740+0000","last_clean":"2024-08-17T17:55:18.707740+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:55:18.707740+0000","last_undegraded":"2024-08-17T17:55:18.707740+0000","last_fullsized":"2024-08-17T17:55:18.707740+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1898,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.711456+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:55:18.711456+0000","last_peered":"2024-08-17T17:55:18.711456+0000","last_clean":"2024-08-17T17:55:18.711456+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:55:18.711456+0000","last_undegraded":"2024-08-17T17:55:18.711456+0000","last_fullsized":"2024-08-17T17:55:18.711456+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1918,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.750193+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:55:18.750193+0000","last_peered":"2024-08-17T17:55:18.750193+0000","last_clean":"2024-08-17T17:55:18.750193+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:55:18.750193+0000","last_undegraded":"2024-08-17T17:55:18.750193+0000","last_fullsized":"2024-08-17T17:55:18.750193+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1924,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.807139+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:55:18.807139+0000","last_peered":"2024-08-17T17:55:18.807139+0000","last_clean":"2024-08-17T17:55:18.807139+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:55:18.807139+0000","last_undegraded":"2024-08-17T17:55:18.807139+0000","last_fullsized":"2024-08-17T17:55:18.807139+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1904,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.825306+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:55:18.825306+0000","last_peered":"2024-08-17T17:55:18.825306+0000","last_clean":"2024-08-17T17:55:18.825306+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:55:18.825306+0000","last_undegraded":"2024-08-17T17:55:18.825306+0000","last_fullsized":"2024-08-17T17:55:18.825306+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1227,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.155367+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:55:10.155367+0000","last_peered":"2024-08-17T17:55:10.155367+0000","last_clean":"2024-08-17T17:55:10.155367+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:55:10.155367+0000","last_undegraded":"2024-08-17T17:55:10.155367+0000","last_fullsized":"2024-08-17T17:55:10.155367+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1173,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.173609+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:55:10.173609+0000","last_peered":"2024-08-17T17:55:10.173609+0000","last_clean":"2024-08-17T17:55:10.173609+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:55:10.173609+0000","last_undegraded":"2024-08-17T17:55:10.173609+0000","last_fullsized":"2024-08-17T17:55:10.173609+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1813,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.708720+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:55:18.708720+0000","last_peered":"2024-08-17T17:55:18.708720+0000","last_clean":"2024-08-17T17:55:18.708720+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:55:18.708720+0000","last_undegraded":"2024-08-17T17:55:18.708720+0000","last_fullsized":"2024-08-17T17:55:18.708720+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":144,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.750497+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:55:18.750497+0000","last_peered":"2024-08-17T17:55:18.750497+0000","last_clean":"2024-08-17T17:55:18.750497+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:55:18.750497+0000","last_undegraded":"2024-08-17T17:55:18.750497+0000","last_fullsized":"2024-08-17T17:55:18.750497+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1539,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.155279+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:55:10.155279+0000","last_peered":"2024-08-17T17:55:10.155279+0000","last_clean":"2024-08-17T17:55:10.155279+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:55:10.155279+0000","last_undegraded":"2024-08-17T17:55:10.155279+0000","last_fullsized":"2024-08-17T17:55:10.155279+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1905,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.707899+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:55:18.707899+0000","last_peered":"2024-08-17T17:55:18.707899+0000","last_clean":"2024-08-17T17:55:18.707899+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:55:18.707899+0000","last_undegraded":"2024-08-17T17:55:18.707899+0000","last_fullsized":"2024-08-17T17:55:18.707899+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1844,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.710114+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:55:18.710114+0000","last_peered":"2024-08-17T17:55:18.710114+0000","last_clean":"2024-08-17T17:55:18.710114+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:55:18.710114+0000","last_undegraded":"2024-08-17T17:55:18.710114+0000","last_fullsized":"2024-08-17T17:55:18.710114+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":507,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.175701+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:55:10.175701+0000","last_peered":"2024-08-17T17:55:10.175701+0000","last_clean":"2024-08-17T17:55:10.175701+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:55:10.175701+0000","last_undegraded":"2024-08-17T17:55:10.175701+0000","last_fullsized":"2024-08-17T17:55:10.175701+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":583,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.749983+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:55:18.749983+0000","last_peered":"2024-08-17T17:55:18.749983+0000","last_clean":"2024-08-17T17:55:18.749983+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:55:18.749983+0000","last_undegraded":"2024-08-17T17:55:18.749983+0000","last_fullsized":"2024-08-17T17:55:18.749983+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":599,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.751960+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:55:18.751960+0000","last_peered":"2024-08-17T17:55:18.751960+0000","last_clean":"2024-08-17T17:55:18.751960+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:55:18.751960+0000","last_undegraded":"2024-08-17T17:55:18.751960+0000","last_fullsized":"2024-08-17T17:55:18.751960+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1191,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.173504+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:55:10.173504+0000","last_peered":"2024-08-17T17:55:10.173504+0000","last_clean":"2024-08-17T17:55:10.173504+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:55:10.173504+0000","last_undegraded":"2024-08-17T17:55:10.173504+0000","last_fullsized":"2024-08-17T17:55:10.173504+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":615,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.708252+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:55:18.708252+0000","last_peered":"2024-08-17T17:55:18.708252+0000","last_clean":"2024-08-17T17:55:18.708252+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:55:18.708252+0000","last_undegraded":"2024-08-17T17:55:18.708252+0000","last_fullsized":"2024-08-17T17:55:18.708252+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1841,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.708462+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:55:18.708462+0000","last_peered":"2024-08-17T17:55:18.708462+0000","last_clean":"2024-08-17T17:55:18.708462+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:55:18.708462+0000","last_undegraded":"2024-08-17T17:55:18.708462+0000","last_fullsized":"2024-08-17T17:55:18.708462+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1937,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.806924+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:55:18.806924+0000","last_peered":"2024-08-17T17:55:18.806924+0000","last_clean":"2024-08-17T17:55:18.806924+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:55:18.806924+0000","last_undegraded":"2024-08-17T17:55:18.806924+0000","last_fullsized":"2024-08-17T17:55:18.806924+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":648,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.707688+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:55:18.707688+0000","last_peered":"2024-08-17T17:55:18.707688+0000","last_clean":"2024-08-17T17:55:18.707688+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:55:18.707688+0000","last_undegraded":"2024-08-17T17:55:18.707688+0000","last_fullsized":"2024-08-17T17:55:18.707688+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1180,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.173322+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:55:10.173322+0000","last_peered":"2024-08-17T17:55:10.173322+0000","last_clean":"2024-08-17T17:55:10.173322+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:55:10.173322+0000","last_undegraded":"2024-08-17T17:55:10.173322+0000","last_fullsized":"2024-08-17T17:55:10.173322+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1179,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.155657+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:55:10.155657+0000","last_peered":"2024-08-17T17:55:10.155657+0000","last_clean":"2024-08-17T17:55:10.155657+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:55:10.155657+0000","last_undegraded":"2024-08-17T17:55:10.155657+0000","last_fullsized":"2024-08-17T17:55:10.155657+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":632,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.710677+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:55:18.710677+0000","last_peered":"2024-08-17T17:55:18.710677+0000","last_clean":"2024-08-17T17:55:18.710677+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:55:18.710677+0000","last_undegraded":"2024-08-17T17:55:18.710677+0000","last_fullsized":"2024-08-17T17:55:18.710677+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1197,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.155749+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:55:10.155749+0000","last_peered":"2024-08-17T17:55:10.155749+0000","last_clean":"2024-08-17T17:55:10.155749+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:55:10.155749+0000","last_undegraded":"2024-08-17T17:55:10.155749+0000","last_fullsized":"2024-08-17T17:55:10.155749+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":81,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.155188+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:55:10.155188+0000","last_peered":"2024-08-17T17:55:10.155188+0000","last_clean":"2024-08-17T17:55:10.155188+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:55:10.155188+0000","last_undegraded":"2024-08-17T17:55:10.155188+0000","last_fullsized":"2024-08-17T17:55:10.155188+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1213,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.155787+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:55:10.155787+0000","last_peered":"2024-08-17T17:55:10.155787+0000","last_clean":"2024-08-17T17:55:10.155787+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:55:10.155787+0000","last_undegraded":"2024-08-17T17:55:10.155787+0000","last_fullsized":"2024-08-17T17:55:10.155787+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1230,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.173172+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:55:10.173172+0000","last_peered":"2024-08-17T17:55:10.173172+0000","last_clean":"2024-08-17T17:55:10.173172+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:55:10.173172+0000","last_undegraded":"2024-08-17T17:55:10.173172+0000","last_fullsized":"2024-08-17T17:55:10.173172+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1842,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.708643+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:55:18.708643+0000","last_peered":"2024-08-17T17:55:18.708643+0000","last_clean":"2024-08-17T17:55:18.708643+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:55:18.708643+0000","last_undegraded":"2024-08-17T17:55:18.708643+0000","last_fullsized":"2024-08-17T17:55:18.708643+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1174,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.155470+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:55:10.155470+0000","last_peered":"2024-08-17T17:55:10.155470+0000","last_clean":"2024-08-17T17:55:10.155470+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:55:10.155470+0000","last_undegraded":"2024-08-17T17:55:10.155470+0000","last_fullsized":"2024-08-17T17:55:10.155470+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1895,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.807036+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:55:18.807036+0000","last_peered":"2024-08-17T17:55:18.807036+0000","last_clean":"2024-08-17T17:55:18.807036+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:55:18.807036+0000","last_undegraded":"2024-08-17T17:55:18.807036+0000","last_fullsized":"2024-08-17T17:55:18.807036+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1896,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.806583+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:55:18.806583+0000","last_peered":"2024-08-17T17:55:18.806583+0000","last_clean":"2024-08-17T17:55:18.806583+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:55:18.806583+0000","last_undegraded":"2024-08-17T17:55:18.806583+0000","last_fullsized":"2024-08-17T17:55:18.806583+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":587,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.155483+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:55:10.155483+0000","last_peered":"2024-08-17T17:55:10.155483+0000","last_clean":"2024-08-17T17:55:10.155483+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:55:10.155483+0000","last_undegraded":"2024-08-17T17:55:10.155483+0000","last_fullsized":"2024-08-17T17:55:10.155483+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1199,"reported_epoch":1229,"state":"active+clean","last_fresh":"2024-08-17T17:55:10.155525+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:55:10.155525+0000","last_peered":"2024-08-17T17:55:10.155525+0000","last_clean":"2024-08-17T17:55:10.155525+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:55:10.155525+0000","last_undegraded":"2024-08-17T17:55:10.155525+0000","last_fullsized":"2024-08-17T17:55:10.155525+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1843,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.708430+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:55:18.708430+0000","last_peered":"2024-08-17T17:55:18.708430+0000","last_clean":"2024-08-17T17:55:18.708430+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:55:18.708430+0000","last_undegraded":"2024-08-17T17:55:18.708430+0000","last_fullsized":"2024-08-17T17:55:18.708430+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":3736862744,"num_objects":57021,"num_object_clones":0,"num_object_copies":171063,"num_objects_missing_on_primary":3024,"num_objects_missing":3024,"num_objects_degraded":16105,"num_objects_misplaced":9542,"num_objects_unfound":0,"num_objects_dirty":57021,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152589,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36011,"num_bytes_recovered":2181562368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12132626432,"data_stored":12132614216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108463,"ondisk_log_size":108463,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234415,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1892368,"kb_used_data":1595016,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":91852784,"statfs":{"total":95995035648,"available":94057250816,"internally_reserved":0,"allocated":1633296384,"data_stored":1628349871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496050,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2138324,"kb_used_data":1836812,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91606828,"statfs":{"total":95995035648,"available":93805391872,"internally_reserved":0,"allocated":1880895488,"data_stored":1875945380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757688,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326302,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2268668,"kb_used_data":2060532,"kb_used_omap":57,"kb_used_meta":208070,"kb_avail":91476484,"statfs":{"total":95995035648,"available":93671919616,"internally_reserved":0,"allocated":2109984768,"data_stored":2105053435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":213064443},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405541,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2055876,"kb_used_data":1835196,"kb_used_omap":61,"kb_used_meta":220610,"kb_avail":91689276,"statfs":{"total":95995035648,"available":93889818624,"internally_reserved":0,"allocated":1879240704,"data_stored":1874306343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":225905592},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452075,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1557176,"kb_used_data":1380976,"kb_used_omap":54,"kb_used_meta":176137,"kb_avail":92187976,"statfs":{"total":95995035648,"available":94400487424,"internally_reserved":0,"allocated":1414119424,"data_stored":1409192052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":180364686},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474767,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1382984,"kb_used_data":1183872,"kb_used_omap":119,"kb_used_meta":198984,"kb_avail":92362168,"statfs":{"total":95995035648,"available":94578860032,"internally_reserved":0,"allocated":1212284928,"data_stored":1207342392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":203760350},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019323,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1197543424,"data_stored":1197539352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1400766464,"data_stored":1400766464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1860960256,"data_stored":1860960256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2095976448,"data_stored":2095972376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1005715456,"data_stored":1005715456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1866534912,"data_stored":1866530840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1617231872,"data_stored":1617231872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:55:33.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:33.799 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:33.867 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:33.888 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:33.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:33.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:33.990 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:34.012 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:34.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:34.114 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:34.135 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:34.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:34.237 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:34.261 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:34.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:34.362 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:34.383 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:34.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:34.485 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:34.506 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:34.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:34.587 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:34.607 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:34.636 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:34.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:34.738 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:34.759 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:34.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:34.861 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:34.882 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:34.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:34.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:34.985 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:35.006 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:35.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:35.107 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:35.129 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:35.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:35.157 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:35.231 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:35.252 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:35.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:35.354 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:35.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.376 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:35.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:35.393 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:35 smithi130 ceph-mon[105521]: pgmap v1861: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 15993/169476 objects degraded (9.437%); 9495/169476 objects misplaced (5.603%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:55:35.393 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:35 smithi130 ceph-mon[110068]: pgmap v1861: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 15993/169476 objects degraded (9.437%); 9495/169476 objects misplaced (5.603%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:55:35.477 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:35.498 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:35.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:35 smithi179 ceph-mon[93519]: pgmap v1861: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 137 op/s; 15993/169476 objects degraded (9.437%); 9495/169476 objects misplaced (5.603%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:55:35.599 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:35.620 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:35.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:35.722 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:35.743 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:35.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:35.845 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:35.867 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:35.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:35.969 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:35.990 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:35.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:35.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:36.092 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:36.114 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:36.216 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:36.237 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:36.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:36.339 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:36.361 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:36.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:36.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15993/169476 objects degraded (9.437%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:36.391 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15993/169476 objects degraded (9.437%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:36.464 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:36.484 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:36.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:36.586 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:36.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15993/169476 objects degraded (9.437%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:36.615 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:36.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:36.618 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:55:36.619 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:55:36.643 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:55:36.644 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:55:36.669 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:55:36.669 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:55:36.692 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:55:36.693 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:55:36.711 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:55:36.712 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:55:36.717 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:36.735 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:55:36.735 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:55:36.738 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:36.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:36.751 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:55:36.751 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:55:36.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:55:36.767 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:55:36.768 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:55:36.840 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:36.861 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:36.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:36.963 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:36.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:36.985 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:36.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:37.086 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:37.110 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:37.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:37.212 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:37.233 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:37.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:37.335 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:37.356 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:37.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:37.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:37 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2331926099' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:37.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:37 smithi130 ceph-mon[105521]: pgmap v1862: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 15868/169023 objects degraded (9.388%); 9378/169023 objects misplaced (5.548%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:55:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:37 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2331926099' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:37 smithi130 ceph-mon[110068]: pgmap v1862: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 15868/169023 objects degraded (9.388%); 9378/169023 objects misplaced (5.548%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:55:37.459 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:37.483 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:37.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:37.585 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:37.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:37 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2331926099' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:37 smithi179 ceph-mon[93519]: pgmap v1862: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 132 op/s; 15868/169023 objects degraded (9.388%); 9378/169023 objects misplaced (5.548%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:55:37.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.607 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:37.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:37.708 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:37.729 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:37.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:37.830 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:37.853 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:37.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:37.955 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:37.978 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:37.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:37.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:38.081 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:38.102 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:38.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:38.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:38.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.225 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:38.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:38.326 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:38.347 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:38.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:38.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:38.451 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:38.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:38.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:38.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:38.598 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:38.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:38.700 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:38.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.722 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:38.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:38.823 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:38.844 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:38.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:38.946 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:38.972 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:38.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:38.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:39.074 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:39.095 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:39.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:39.197 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:39.218 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:39.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:39.246 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:39 smithi130 ceph-mon[105521]: pgmap v1863: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 15868/168042 objects degraded (9.443%); 9378/168042 objects misplaced (5.581%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:55:39.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:39 smithi130 ceph-mon[110068]: pgmap v1863: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 15868/168042 objects degraded (9.443%); 9378/168042 objects misplaced (5.581%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:55:39.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:39.341 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:39.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:39.443 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:39.469 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:39.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:39.571 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:39 smithi179 ceph-mon[93519]: pgmap v1863: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 120 op/s; 15868/168042 objects degraded (9.443%); 9378/168042 objects misplaced (5.581%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:55:39.595 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:39.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:39.697 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:39.726 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:39.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:39.828 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:39.854 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:39.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:39.956 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:39.977 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:39.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:39.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:40.012 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:55:40.040 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:55:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:55:39] "GET /metrics HTTP/1.1" 200 36810 "" "Prometheus/2.51.0" 2024-08-17T17:55:40.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:40.105 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:40.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:40.207 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:40.228 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:40.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:40.329 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:40.350 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:40.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:40.452 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:40.475 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:40.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:40.576 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:55:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:55:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:55:40.603 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:40.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:40.674 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:55:40.674 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T17:55:40.675 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T17:55:40.675 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T17:55:40.675 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.675 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:116980 45.1 45:8799f42e:::benchmark_data_smithi130_172551_object12173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1191)", 2024-08-17T17:55:40.675 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:11.202055+0000", 2024-08-17T17:55:40.675 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 328.60800731199998, 2024-08-17T17:55:40.675 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0427133929999997, 2024-08-17T17:55:40.675 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.675 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.675 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.675 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.675 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.675 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.675 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 116980 2024-08-17T17:55:40.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.676 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.676 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.676 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.676 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.202055+0000", 2024-08-17T17:55:40.676 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.676 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.676 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.676 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.202055+0000", 2024-08-17T17:55:40.676 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.677 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.677 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.677 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.202057+0000", 2024-08-17T17:55:40.677 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9570000000000001e-06 2024-08-17T17:55:40.677 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.677 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.677 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.677 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.202064+0000", 2024-08-17T17:55:40.677 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4429999999999997e-06 2024-08-17T17:55:40.677 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.677 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.677 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.677 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.202080+0000", 2024-08-17T17:55:40.677 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5553000000000002e-05 2024-08-17T17:55:40.678 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.678 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.678 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.678 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.202088+0000", 2024-08-17T17:55:40.678 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1929999999999993e-06 2024-08-17T17:55:40.678 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.678 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.678 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.678 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.583665+0000", 2024-08-17T17:55:40.678 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.381576849 2024-08-17T17:55:40.678 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.678 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.679 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.679 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.601146+0000", 2024-08-17T17:55:40.679 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017480921 2024-08-17T17:55:40.679 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.679 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.679 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.679 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.821377+0000", 2024-08-17T17:55:40.679 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.220231552 2024-08-17T17:55:40.679 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.679 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.679 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.679 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:11.832291+0000", 2024-08-17T17:55:40.679 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010913156 2024-08-17T17:55:40.680 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.680 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.680 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.680 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.152620+0000", 2024-08-17T17:55:40.680 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.320328881 2024-08-17T17:55:40.680 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.680 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.680 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.680 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.160695+0000", 2024-08-17T17:55:40.680 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0080755629999999991 2024-08-17T17:55:40.680 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.680 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.680 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.681 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.178912+0000", 2024-08-17T17:55:40.681 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018216901000000001 2024-08-17T17:55:40.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.681 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.681 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.681 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.207906+0000", 2024-08-17T17:55:40.681 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028994028000000002 2024-08-17T17:55:40.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.681 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.681 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.681 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.210068+0000", 2024-08-17T17:55:40.681 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021623110000000001 2024-08-17T17:55:40.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.681 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.212964+0000", 2024-08-17T17:55:40.682 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028953949999999998 2024-08-17T17:55:40.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.682 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.219201+0000", 2024-08-17T17:55:40.682 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0062367990000000003 2024-08-17T17:55:40.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.682 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.244768+0000", 2024-08-17T17:55:40.683 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025567890999999999 2024-08-17T17:55:40.683 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.683 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.683 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.683 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.683 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117006 45.1 45:8e79452c:::benchmark_data_smithi130_172551_object12199:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1191)", 2024-08-17T17:55:40.683 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:12.051804+0000", 2024-08-17T17:55:40.683 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 327.75825790800002, 2024-08-17T17:55:40.683 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6324937070000001, 2024-08-17T17:55:40.683 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.683 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.684 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.684 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.684 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.684 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.684 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117006 2024-08-17T17:55:40.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.684 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.684 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.684 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.051804+0000", 2024-08-17T17:55:40.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.051804+0000", 2024-08-17T17:55:40.685 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.051807+0000", 2024-08-17T17:55:40.685 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.261e-06 2024-08-17T17:55:40.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.051817+0000", 2024-08-17T17:55:40.685 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0013000000000001e-05 2024-08-17T17:55:40.686 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.686 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.686 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.686 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.051843+0000", 2024-08-17T17:55:40.686 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6792999999999999e-05 2024-08-17T17:55:40.686 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.686 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.686 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.686 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.051851+0000", 2024-08-17T17:55:40.686 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7270000000000007e-06 2024-08-17T17:55:40.686 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.686 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.686 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.687 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.372425+0000", 2024-08-17T17:55:40.687 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.32057382899999998 2024-08-17T17:55:40.687 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.687 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.687 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.687 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.392911+0000", 2024-08-17T17:55:40.687 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020486397 2024-08-17T17:55:40.687 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.687 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.687 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.687 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.478544+0000", 2024-08-17T17:55:40.687 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.085632642999999994 2024-08-17T17:55:40.688 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.688 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.688 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.688 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.488558+0000", 2024-08-17T17:55:40.688 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010013538000000001 2024-08-17T17:55:40.688 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.688 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.688 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.688 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.495084+0000", 2024-08-17T17:55:40.688 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0065260469999999 2024-08-17T17:55:40.688 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.688 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.688 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.509554+0000", 2024-08-17T17:55:40.689 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014470264 2024-08-17T17:55:40.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.538572+0000", 2024-08-17T17:55:40.689 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.029018004 2024-08-17T17:55:40.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.651135+0000", 2024-08-17T17:55:40.689 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.11256314100000001 2024-08-17T17:55:40.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.690 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.690 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.653806+0000", 2024-08-17T17:55:40.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026705710000000001 2024-08-17T17:55:40.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.690 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.690 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.690 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.657210+0000", 2024-08-17T17:55:40.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003404168 2024-08-17T17:55:40.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.690 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.690 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.690 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.665389+0000", 2024-08-17T17:55:40.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0081789949999999997 2024-08-17T17:55:40.691 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.691 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.691 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.691 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.684298+0000", 2024-08-17T17:55:40.691 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018909315999999999 2024-08-17T17:55:40.691 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.691 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.691 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.691 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.691 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.691 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117051 45.1 45:87d9b5c3:::benchmark_data_smithi130_172551_object12244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1191)", 2024-08-17T17:55:40.691 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:12.410351+0000", 2024-08-17T17:55:40.691 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 327.39971119799998, 2024-08-17T17:55:40.691 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2018803760000001, 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117051 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.410351+0000", 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.692 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.693 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.693 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.410351+0000", 2024-08-17T17:55:40.693 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.693 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.693 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.693 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.693 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.410353+0000", 2024-08-17T17:55:40.693 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0710000000000002e-06 2024-08-17T17:55:40.693 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.693 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.693 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.693 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.410361+0000", 2024-08-17T17:55:40.693 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0690000000000002e-06 2024-08-17T17:55:40.693 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.694 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.694 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.694 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.410377+0000", 2024-08-17T17:55:40.694 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5492999999999999e-05 2024-08-17T17:55:40.694 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.694 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.694 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.694 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.410386+0000", 2024-08-17T17:55:40.694 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8300000000000002e-06 2024-08-17T17:55:40.694 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.694 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.694 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.694 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.670860+0000", 2024-08-17T17:55:40.694 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.260474488 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.689297+0000", 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018437174000000001 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.757380+0000", 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.068083187000000003 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.768726+0000", 2024-08-17T17:55:40.695 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011345696000000001 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.923952+0000", 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1552257940000001 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.935961+0000", 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012008959 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.974312+0000", 2024-08-17T17:55:40.696 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.038351277000000003 2024-08-17T17:55:40.697 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.697 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.697 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.697 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.073288+0000", 2024-08-17T17:55:40.697 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.098975749000000002 2024-08-17T17:55:40.697 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.697 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.697 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.697 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.075449+0000", 2024-08-17T17:55:40.697 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021606849999999999 2024-08-17T17:55:40.697 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.697 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.697 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.697 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.583241+0000", 2024-08-17T17:55:40.698 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.50779248499999996 2024-08-17T17:55:40.698 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.698 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.698 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.698 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.589776+0000", 2024-08-17T17:55:40.698 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0065349500000000003 2024-08-17T17:55:40.698 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.698 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.698 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.698 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.612232+0000", 2024-08-17T17:55:40.698 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022455468999999999 2024-08-17T17:55:40.698 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.698 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.699 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.699 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.699 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.699 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117063 45.1 45:84049d28:::benchmark_data_smithi130_172551_object12256:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1191)", 2024-08-17T17:55:40.699 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:12.475601+0000", 2024-08-17T17:55:40.699 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 327.33446136700002, 2024-08-17T17:55:40.699 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.17575086, 2024-08-17T17:55:40.699 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.699 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.699 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.699 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.699 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.699 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.700 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117063 2024-08-17T17:55:40.700 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.700 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.700 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.700 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.700 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.475601+0000", 2024-08-17T17:55:40.700 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.700 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.700 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.700 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.700 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.475601+0000", 2024-08-17T17:55:40.700 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.700 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.701 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.701 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.701 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.475603+0000", 2024-08-17T17:55:40.701 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.77e-06 2024-08-17T17:55:40.701 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.701 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.701 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.701 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.475609+0000", 2024-08-17T17:55:40.701 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4980000000000003e-06 2024-08-17T17:55:40.701 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.701 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.701 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.701 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.475622+0000", 2024-08-17T17:55:40.701 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2917e-05 2024-08-17T17:55:40.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.702 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.702 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.702 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.475630+0000", 2024-08-17T17:55:40.702 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5109999999999997e-06 2024-08-17T17:55:40.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.702 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.702 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.702 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.691253+0000", 2024-08-17T17:55:40.702 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.21562358000000001 2024-08-17T17:55:40.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.702 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.703 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.708213+0000", 2024-08-17T17:55:40.703 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016959795 2024-08-17T17:55:40.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.703 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.891227+0000", 2024-08-17T17:55:40.703 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.18301373700000001 2024-08-17T17:55:40.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.703 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.905968+0000", 2024-08-17T17:55:40.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014741318 2024-08-17T17:55:40.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.704 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.028661+0000", 2024-08-17T17:55:40.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1226928090000001 2024-08-17T17:55:40.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.704 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.038437+0000", 2024-08-17T17:55:40.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0097758250000000001 2024-08-17T17:55:40.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.705 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.705 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.055169+0000", 2024-08-17T17:55:40.705 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016732120999999999 2024-08-17T17:55:40.705 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.705 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.705 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.705 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.201705+0000", 2024-08-17T17:55:40.705 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14653565299999999 2024-08-17T17:55:40.705 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.705 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.705 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.705 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.615029+0000", 2024-08-17T17:55:40.706 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.41332463400000002 2024-08-17T17:55:40.706 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.706 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.706 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.706 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.620582+0000", 2024-08-17T17:55:40.706 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0055527340000000001 2024-08-17T17:55:40.706 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.706 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.706 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.706 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.624922+0000", 2024-08-17T17:55:40.706 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0043401409999999996 2024-08-17T17:55:40.706 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.707 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.707 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.707 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.651352+0000", 2024-08-17T17:55:40.707 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.026429817000000001 2024-08-17T17:55:40.707 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.707 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.707 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.707 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.707 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.707 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117072 45.1 45:8383d11f:::benchmark_data_smithi130_172551_object12265:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1191)", 2024-08-17T17:55:40.707 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:12.694419+0000", 2024-08-17T17:55:40.707 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 327.11564343600003, 2024-08-17T17:55:40.708 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9983870160000001, 2024-08-17T17:55:40.708 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.708 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.708 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.708 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.708 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.708 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.708 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117072 2024-08-17T17:55:40.708 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.708 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.708 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.708 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.709 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.694419+0000", 2024-08-17T17:55:40.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.709 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.709 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.709 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.694419+0000", 2024-08-17T17:55:40.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.709 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.709 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.709 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.694421+0000", 2024-08-17T17:55:40.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0219999999999999e-06 2024-08-17T17:55:40.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.710 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.710 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.694430+0000", 2024-08-17T17:55:40.710 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6459999999999999e-06 2024-08-17T17:55:40.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.710 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.710 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.694445+0000", 2024-08-17T17:55:40.710 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.522e-05 2024-08-17T17:55:40.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.710 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.710 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.694454+0000", 2024-08-17T17:55:40.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2809999999999997e-06 2024-08-17T17:55:40.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.711 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.958017+0000", 2024-08-17T17:55:40.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.263563031 2024-08-17T17:55:40.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.711 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:12.977402+0000", 2024-08-17T17:55:40.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019385135000000001 2024-08-17T17:55:40.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.059271+0000", 2024-08-17T17:55:40.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.081868301000000004 2024-08-17T17:55:40.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.712 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.067839+0000", 2024-08-17T17:55:40.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0085682269999999994 2024-08-17T17:55:40.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.712 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.115160+0000", 2024-08-17T17:55:40.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0473214799999999 2024-08-17T17:55:40.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.712 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.713 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.713 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.122535+0000", 2024-08-17T17:55:40.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0073748700000000004 2024-08-17T17:55:40.713 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.713 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.713 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.140700+0000", 2024-08-17T17:55:40.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018164882 2024-08-17T17:55:40.713 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.713 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.713 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.288391+0000", 2024-08-17T17:55:40.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14769062099999999 2024-08-17T17:55:40.713 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.655373+0000", 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.36698246899999998 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.659842+0000", 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0044693880000000004 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.664773+0000", 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0049301579999999996 2024-08-17T17:55:40.714 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.715 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.715 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.715 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.692806+0000", 2024-08-17T17:55:40.715 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028033285000000002 2024-08-17T17:55:40.715 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.715 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.715 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.715 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.715 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.715 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117087 45.1 45:8d1ad67f:::benchmark_data_smithi130_172551_object12280:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1191)", 2024-08-17T17:55:40.715 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:13.044755+0000", 2024-08-17T17:55:40.715 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 326.76530703899999, 2024-08-17T17:55:40.715 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6944316280000002, 2024-08-17T17:55:40.715 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117087 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.044755+0000", 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.716 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.044755+0000", 2024-08-17T17:55:40.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.717 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.044758+0000", 2024-08-17T17:55:40.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5299999999999999e-06 2024-08-17T17:55:40.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.717 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.044767+0000", 2024-08-17T17:55:40.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2690000000000005e-06 2024-08-17T17:55:40.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.718 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.044789+0000", 2024-08-17T17:55:40.718 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2064000000000001e-05 2024-08-17T17:55:40.718 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.718 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.718 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.044814+0000", 2024-08-17T17:55:40.718 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5258000000000001e-05 2024-08-17T17:55:40.718 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.718 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.718 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.156054+0000", 2024-08-17T17:55:40.719 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.111239721 2024-08-17T17:55:40.719 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.719 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.719 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.719 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.172700+0000", 2024-08-17T17:55:40.719 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016645750000000001 2024-08-17T17:55:40.719 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.719 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.719 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.719 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.266218+0000", 2024-08-17T17:55:40.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.093517811000000006 2024-08-17T17:55:40.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.720 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.720 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.720 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:13.278568+0000", 2024-08-17T17:55:40.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012350292000000001 2024-08-17T17:55:40.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.720 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.720 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.720 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.202811+0000", 2024-08-17T17:55:40.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9242432539999998 2024-08-17T17:55:40.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.720 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.211716+0000", 2024-08-17T17:55:40.721 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.008905095 2024-08-17T17:55:40.721 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.228505+0000", 2024-08-17T17:55:40.721 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016788964999999999 2024-08-17T17:55:40.721 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.372900+0000", 2024-08-17T17:55:40.721 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14439475399999999 2024-08-17T17:55:40.722 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.722 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.722 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.722 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.696610+0000", 2024-08-17T17:55:40.722 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.32371030299999998 2024-08-17T17:55:40.722 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.722 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.722 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.722 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.702116+0000", 2024-08-17T17:55:40.722 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0055056749999999998 2024-08-17T17:55:40.722 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.722 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.722 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.722 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.706734+0000", 2024-08-17T17:55:40.723 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046176840000000004 2024-08-17T17:55:40.723 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.723 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.723 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.723 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.739187+0000", 2024-08-17T17:55:40.723 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.032453203 2024-08-17T17:55:40.723 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.723 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.723 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.723 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.723 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.723 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117138 45.1 45:82c30d21:::benchmark_data_smithi130_172551_object12331:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:55:40.723 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:14.196732+0000", 2024-08-17T17:55:40.723 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 325.61333072899998, 2024-08-17T17:55:40.724 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.70372638, 2024-08-17T17:55:40.724 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.724 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.724 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.724 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.724 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.724 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.724 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117138 2024-08-17T17:55:40.724 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.724 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.724 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.724 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.724 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.196732+0000", 2024-08-17T17:55:40.725 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.725 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.725 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.725 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.725 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.196732+0000", 2024-08-17T17:55:40.725 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.725 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.725 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.725 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.725 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.196734+0000", 2024-08-17T17:55:40.725 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9479999999999998e-06 2024-08-17T17:55:40.725 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.725 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.725 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.196740+0000", 2024-08-17T17:55:40.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3280000000000002e-06 2024-08-17T17:55:40.726 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.726 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.726 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.196755+0000", 2024-08-17T17:55:40.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4874e-05 2024-08-17T17:55:40.726 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.726 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.726 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.196762+0000", 2024-08-17T17:55:40.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4900000000000003e-06 2024-08-17T17:55:40.726 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.313933+0000", 2024-08-17T17:55:40.727 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.117171022 2024-08-17T17:55:40.727 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.327698+0000", 2024-08-17T17:55:40.727 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013764609000000001 2024-08-17T17:55:40.727 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.384376+0000", 2024-08-17T17:55:40.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.056677727999999997 2024-08-17T17:55:40.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.728 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.393627+0000", 2024-08-17T17:55:40.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0092510240000000001 2024-08-17T17:55:40.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.728 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.668409+0000", 2024-08-17T17:55:40.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2747823719999998 2024-08-17T17:55:40.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.729 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.729 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.674408+0000", 2024-08-17T17:55:40.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0059989489999999999 2024-08-17T17:55:40.729 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.729 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.729 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.729 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.686181+0000", 2024-08-17T17:55:40.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011773336000000001 2024-08-17T17:55:40.729 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.729 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.729 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.729 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.794142+0000", 2024-08-17T17:55:40.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.107960316 2024-08-17T17:55:40.729 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.730 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.730 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.730 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.796752+0000", 2024-08-17T17:55:40.730 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026106329999999998 2024-08-17T17:55:40.730 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.730 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.730 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.730 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.861007+0000", 2024-08-17T17:55:40.730 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.064255123999999997 2024-08-17T17:55:40.730 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.730 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.730 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.731 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.871761+0000", 2024-08-17T17:55:40.731 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010753792999999999 2024-08-17T17:55:40.731 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.731 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.731 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.731 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.900458+0000", 2024-08-17T17:55:40.731 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028696834000000001 2024-08-17T17:55:40.731 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.731 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.731 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.731 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.731 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.731 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117183 45.1 45:802f39c4:::benchmark_data_smithi130_172551_object12376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:55:40.732 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:14.982725+0000", 2024-08-17T17:55:40.732 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 324.82733749699997, 2024-08-17T17:55:40.732 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8032687460000001, 2024-08-17T17:55:40.732 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.732 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.732 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.732 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.732 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.732 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.732 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117183 2024-08-17T17:55:40.732 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.732 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.732 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.982725+0000", 2024-08-17T17:55:40.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.733 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.982725+0000", 2024-08-17T17:55:40.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.733 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.982727+0000", 2024-08-17T17:55:40.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6950000000000001e-06 2024-08-17T17:55:40.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.734 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.734 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.734 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.982735+0000", 2024-08-17T17:55:40.734 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4659999999999999e-06 2024-08-17T17:55:40.734 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.734 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.734 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.734 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.982747+0000", 2024-08-17T17:55:40.734 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2388999999999999e-05 2024-08-17T17:55:40.734 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.734 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.734 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.734 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:14.982755+0000", 2024-08-17T17:55:40.734 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1160000000000001e-06 2024-08-17T17:55:40.735 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.735 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.735 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.735 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.306395+0000", 2024-08-17T17:55:40.735 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.32364099000000002 2024-08-17T17:55:40.735 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.735 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.735 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.735 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.318989+0000", 2024-08-17T17:55:40.735 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012593332 2024-08-17T17:55:40.735 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.735 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.735 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.735 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.411042+0000", 2024-08-17T17:55:40.736 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.092052758999999998 2024-08-17T17:55:40.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.736 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.736 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.736 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:15.427984+0000", 2024-08-17T17:55:40.736 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.01694201 2024-08-17T17:55:40.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.736 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.736 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.736 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.610241+0000", 2024-08-17T17:55:40.736 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1822570410000002 2024-08-17T17:55:40.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.736 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.737 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.737 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.623111+0000", 2024-08-17T17:55:40.737 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012870825000000001 2024-08-17T17:55:40.737 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.737 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.737 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.737 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.642243+0000", 2024-08-17T17:55:40.737 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019131657 2024-08-17T17:55:40.737 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.737 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.737 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.737 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.739757+0000", 2024-08-17T17:55:40.737 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.097514067999999995 2024-08-17T17:55:40.738 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.738 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.738 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.738 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.742367+0000", 2024-08-17T17:55:40.738 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026096829999999998 2024-08-17T17:55:40.738 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.738 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.738 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.738 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.745766+0000", 2024-08-17T17:55:40.738 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0033987079999999999 2024-08-17T17:55:40.738 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.738 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.738 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.739 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.754589+0000", 2024-08-17T17:55:40.739 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0088238429999999996 2024-08-17T17:55:40.739 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.739 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.739 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.739 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:18.785994+0000", 2024-08-17T17:55:40.739 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.031404163999999998 2024-08-17T17:55:40.739 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.739 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.739 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.739 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.739 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.740 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117271 45.1 45:86e9ffcd:::benchmark_data_smithi130_172551_object12464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:55:40.740 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:16.753226+0000", 2024-08-17T17:55:40.740 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 323.05683626000001, 2024-08-17T17:55:40.740 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9279494399999999, 2024-08-17T17:55:40.740 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.740 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.740 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.740 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.740 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.740 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.740 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117271 2024-08-17T17:55:40.740 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.740 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.741 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.741 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.753226+0000", 2024-08-17T17:55:40.741 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.741 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.741 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.741 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.753226+0000", 2024-08-17T17:55:40.741 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.741 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.741 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.741 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.753228+0000", 2024-08-17T17:55:40.742 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.731e-06 2024-08-17T17:55:40.742 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.742 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.742 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.742 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.753235+0000", 2024-08-17T17:55:40.742 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.481e-06 2024-08-17T17:55:40.742 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.742 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.742 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.742 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.753256+0000", 2024-08-17T17:55:40.742 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0707e-05 2024-08-17T17:55:40.742 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.743 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.743 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.753265+0000", 2024-08-17T17:55:40.743 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1160000000000005e-06 2024-08-17T17:55:40.743 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.743 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.743 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.182325+0000", 2024-08-17T17:55:40.743 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.42906007000000002 2024-08-17T17:55:40.743 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.743 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.743 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.192896+0000", 2024-08-17T17:55:40.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010570461999999999 2024-08-17T17:55:40.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.744 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.257048+0000", 2024-08-17T17:55:40.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.064152826999999996 2024-08-17T17:55:40.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.744 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.268122+0000", 2024-08-17T17:55:40.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011073553 2024-08-17T17:55:40.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.745 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:19.904981+0000", 2024-08-17T17:55:40.745 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.636858691 2024-08-17T17:55:40.745 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.745 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:19.917279+0000", 2024-08-17T17:55:40.745 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012298335000000001 2024-08-17T17:55:40.745 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.745 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:19.940310+0000", 2024-08-17T17:55:40.745 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023030508000000002 2024-08-17T17:55:40.746 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.746 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.746 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.746 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.047544+0000", 2024-08-17T17:55:40.746 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.107234048 2024-08-17T17:55:40.746 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.746 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.746 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.746 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.050156+0000", 2024-08-17T17:55:40.746 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026120269999999998 2024-08-17T17:55:40.746 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.746 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.746 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.747 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.654099+0000", 2024-08-17T17:55:40.747 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.603943652 2024-08-17T17:55:40.747 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.747 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.747 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.747 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.661398+0000", 2024-08-17T17:55:40.747 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072991360000000003 2024-08-17T17:55:40.747 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.747 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.747 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.747 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.681176+0000", 2024-08-17T17:55:40.747 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019777096000000001 2024-08-17T17:55:40.747 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.747 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.748 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.748 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.748 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.748 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117312 45.1 45:8106ae5d:::benchmark_data_smithi130_172551_object12505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:55:40.748 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:16.890803+0000", 2024-08-17T17:55:40.748 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 322.91925924600002, 2024-08-17T17:55:40.748 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8771543880000001, 2024-08-17T17:55:40.748 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.748 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.748 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.748 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.748 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.748 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.749 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117312 2024-08-17T17:55:40.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.749 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.749 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.749 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.890803+0000", 2024-08-17T17:55:40.749 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.749 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.749 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.890803+0000", 2024-08-17T17:55:40.749 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.890805+0000", 2024-08-17T17:55:40.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.79e-06 2024-08-17T17:55:40.750 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.750 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.890812+0000", 2024-08-17T17:55:40.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7009999999999999e-06 2024-08-17T17:55:40.750 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.750 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.890827+0000", 2024-08-17T17:55:40.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5054e-05 2024-08-17T17:55:40.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.890835+0000", 2024-08-17T17:55:40.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7479999999999999e-06 2024-08-17T17:55:40.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.194094+0000", 2024-08-17T17:55:40.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.303258156 2024-08-17T17:55:40.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.204820+0000", 2024-08-17T17:55:40.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010726127 2024-08-17T17:55:40.752 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.752 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.752 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.752 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.366433+0000", 2024-08-17T17:55:40.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16161305300000001 2024-08-17T17:55:40.752 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.752 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.752 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.752 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.379828+0000", 2024-08-17T17:55:40.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013395504000000001 2024-08-17T17:55:40.752 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.752 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.752 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.753 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.089528+0000", 2024-08-17T17:55:40.753 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.70969967 2024-08-17T17:55:40.753 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.753 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.753 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.753 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.100667+0000", 2024-08-17T17:55:40.753 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011139539 2024-08-17T17:55:40.753 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.753 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.753 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.753 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.120743+0000", 2024-08-17T17:55:40.753 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020075221000000001 2024-08-17T17:55:40.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.754 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.754 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.223059+0000", 2024-08-17T17:55:40.754 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.102316746 2024-08-17T17:55:40.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.754 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.754 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.728570+0000", 2024-08-17T17:55:40.754 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.505510864 2024-08-17T17:55:40.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.754 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.755 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.735815+0000", 2024-08-17T17:55:40.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072448340000000003 2024-08-17T17:55:40.755 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.755 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.755 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.755 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.745136+0000", 2024-08-17T17:55:40.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0093212420000000004 2024-08-17T17:55:40.755 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.755 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.755 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.755 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.767957+0000", 2024-08-17T17:55:40.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022821139000000001 2024-08-17T17:55:40.755 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.755 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.756 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.756 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117315 45.1 45:89f3983e:::benchmark_data_smithi130_172551_object12508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:55:40.756 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:16.898971+0000", 2024-08-17T17:55:40.756 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 322.91109090800001, 2024-08-17T17:55:40.756 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8238647210000001, 2024-08-17T17:55:40.756 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.756 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.756 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.756 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.756 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.756 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.756 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117315 2024-08-17T17:55:40.757 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.757 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.757 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.757 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.757 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.898971+0000", 2024-08-17T17:55:40.757 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.757 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.757 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.757 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.757 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.898971+0000", 2024-08-17T17:55:40.757 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.757 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.757 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.757 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.758 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.898973+0000", 2024-08-17T17:55:40.758 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.437e-06 2024-08-17T17:55:40.758 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.758 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.758 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.758 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.898987+0000", 2024-08-17T17:55:40.758 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3764000000000001e-05 2024-08-17T17:55:40.758 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.758 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.758 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.758 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.899000+0000", 2024-08-17T17:55:40.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3559e-05 2024-08-17T17:55:40.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.759 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.899007+0000", 2024-08-17T17:55:40.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.126e-06 2024-08-17T17:55:40.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.759 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.208678+0000", 2024-08-17T17:55:40.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.30967032500000002 2024-08-17T17:55:40.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.759 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.760 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.220331+0000", 2024-08-17T17:55:40.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011653596 2024-08-17T17:55:40.760 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.760 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.760 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.760 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.307207+0000", 2024-08-17T17:55:40.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.086876244000000005 2024-08-17T17:55:40.760 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.760 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.760 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.760 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.318268+0000", 2024-08-17T17:55:40.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011060327 2024-08-17T17:55:40.760 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.760 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.761 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.761 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.007325+0000", 2024-08-17T17:55:40.761 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6890573180000001 2024-08-17T17:55:40.761 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.761 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.761 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.761 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.016133+0000", 2024-08-17T17:55:40.761 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0088075940000000002 2024-08-17T17:55:40.761 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.761 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.761 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.761 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.031654+0000", 2024-08-17T17:55:40.761 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015521237 2024-08-17T17:55:40.761 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.762 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.762 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.762 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.136315+0000", 2024-08-17T17:55:40.762 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.104660695 2024-08-17T17:55:40.762 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.762 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.762 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.762 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.684865+0000", 2024-08-17T17:55:40.762 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.54855074199999998 2024-08-17T17:55:40.762 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.762 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.762 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.763 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.688834+0000", 2024-08-17T17:55:40.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039684760000000003 2024-08-17T17:55:40.763 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.763 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.763 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.763 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.694674+0000", 2024-08-17T17:55:40.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0058402920000000004 2024-08-17T17:55:40.763 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.763 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.763 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.763 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.722836+0000", 2024-08-17T17:55:40.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028161988999999998 2024-08-17T17:55:40.764 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.764 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.764 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.764 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.764 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.764 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117325 45.1 45:876cdfdb:::benchmark_data_smithi130_172551_object12518:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:55:40.764 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:16.936304+0000", 2024-08-17T17:55:40.764 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 322.873758178, 2024-08-17T17:55:40.764 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8880692140000002, 2024-08-17T17:55:40.764 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.764 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.764 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.764 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.764 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117325 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.936304+0000", 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.936304+0000", 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.936306+0000", 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1090000000000001e-06 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.936312+0000", 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1109999999999996e-06 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.936326+0000", 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3729e-05 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.766 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.767 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:16.936335+0000", 2024-08-17T17:55:40.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.132e-06 2024-08-17T17:55:40.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.767 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.383988+0000", 2024-08-17T17:55:40.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.44765318599999998 2024-08-17T17:55:40.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.767 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.398315+0000", 2024-08-17T17:55:40.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014326383 2024-08-17T17:55:40.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.768 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.768 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.768 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.475851+0000", 2024-08-17T17:55:40.768 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.077535779999999999 2024-08-17T17:55:40.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.768 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.768 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.768 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.490245+0000", 2024-08-17T17:55:40.768 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014393992 2024-08-17T17:55:40.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.768 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.768 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.769 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.183369+0000", 2024-08-17T17:55:40.769 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6931242759999998 2024-08-17T17:55:40.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.769 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.769 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.769 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.193862+0000", 2024-08-17T17:55:40.769 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010493186999999999 2024-08-17T17:55:40.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.769 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.769 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.769 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.208305+0000", 2024-08-17T17:55:40.769 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014442521999999999 2024-08-17T17:55:40.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.769 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.770 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.770 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.312432+0000", 2024-08-17T17:55:40.770 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.104126987 2024-08-17T17:55:40.770 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.770 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.770 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.770 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.773252+0000", 2024-08-17T17:55:40.770 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.46082050200000002 2024-08-17T17:55:40.770 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.770 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.770 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.770 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.783161+0000", 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.009908867 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.793591+0000", 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010429946000000001 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.824373+0000", 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.030782505000000002 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117345 45.1 45:8c7a643c:::benchmark_data_smithi130_172551_object12538:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:17.010755+0000", 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 322.79930757599999, 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9264242559999998, 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117345 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.010755+0000", 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.010755+0000", 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.010757+0000", 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9379999999999999e-06 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.010776+0000", 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9273e-05 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.010790+0000", 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4409999999999999e-05 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.010806+0000", 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6113999999999999e-05 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.775 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.399734+0000", 2024-08-17T17:55:40.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38892710800000002 2024-08-17T17:55:40.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.775 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.414221+0000", 2024-08-17T17:55:40.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014487619 2024-08-17T17:55:40.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.775 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.586260+0000", 2024-08-17T17:55:40.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.17203851100000001 2024-08-17T17:55:40.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.776 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.776 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.776 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.597691+0000", 2024-08-17T17:55:40.776 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011431619000000001 2024-08-17T17:55:40.776 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.776 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.776 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.776 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.354941+0000", 2024-08-17T17:55:40.776 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.7572492820000001 2024-08-17T17:55:40.776 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.776 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.776 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.776 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.363534+0000", 2024-08-17T17:55:40.776 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0085931940000000002 2024-08-17T17:55:40.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.777 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.380254+0000", 2024-08-17T17:55:40.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016719801999999999 2024-08-17T17:55:40.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.777 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.478132+0000", 2024-08-17T17:55:40.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.097878039999999999 2024-08-17T17:55:40.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.777 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.883521+0000", 2024-08-17T17:55:40.778 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.40538944999999998 2024-08-17T17:55:40.778 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.778 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.778 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.778 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.890390+0000", 2024-08-17T17:55:40.778 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0068690490000000003 2024-08-17T17:55:40.778 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.778 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.778 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.778 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.900699+0000", 2024-08-17T17:55:40.778 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010308552 2024-08-17T17:55:40.778 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.779 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.779 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.779 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.937179+0000", 2024-08-17T17:55:40.779 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.036480295000000003 2024-08-17T17:55:40.779 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.779 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.779 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.779 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.779 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117351 45.1 45:8354d1e0:::benchmark_data_smithi130_172551_object12544:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1192)", 2024-08-17T17:55:40.779 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:17.049349+0000", 2024-08-17T17:55:40.779 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 322.76071336500002, 2024-08-17T17:55:40.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8289573089999998, 2024-08-17T17:55:40.780 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.780 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.780 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.780 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.780 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.780 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.780 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117351 2024-08-17T17:55:40.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.780 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.780 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.780 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.780 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.049349+0000", 2024-08-17T17:55:40.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.049349+0000", 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.049350+0000", 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.384e-06 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.049356+0000", 2024-08-17T17:55:40.781 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9390000000000002e-06 2024-08-17T17:55:40.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.782 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.782 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.782 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.049369+0000", 2024-08-17T17:55:40.782 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2716e-05 2024-08-17T17:55:40.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.782 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.782 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.782 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.049377+0000", 2024-08-17T17:55:40.782 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0879999999999995e-06 2024-08-17T17:55:40.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.782 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.782 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.783 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.418937+0000", 2024-08-17T17:55:40.783 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.36956013500000001 2024-08-17T17:55:40.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.783 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.783 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.783 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.432534+0000", 2024-08-17T17:55:40.783 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013596721000000001 2024-08-17T17:55:40.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.783 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.783 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.783 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.531230+0000", 2024-08-17T17:55:40.783 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.098696273000000001 2024-08-17T17:55:40.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.783 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.784 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.784 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:17.543385+0000", 2024-08-17T17:55:40.784 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012154999 2024-08-17T17:55:40.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.784 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.784 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.784 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.271977+0000", 2024-08-17T17:55:40.784 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.7285919249999999 2024-08-17T17:55:40.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.784 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.784 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.784 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.281763+0000", 2024-08-17T17:55:40.784 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0097861259999999992 2024-08-17T17:55:40.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.785 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.785 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.785 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.297762+0000", 2024-08-17T17:55:40.785 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015998931000000001 2024-08-17T17:55:40.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.785 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.785 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.785 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.392943+0000", 2024-08-17T17:55:40.785 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.095180734000000003 2024-08-17T17:55:40.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.785 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.785 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.786 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.828437+0000", 2024-08-17T17:55:40.786 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.43549367100000003 2024-08-17T17:55:40.786 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.786 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.786 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.786 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.834123+0000", 2024-08-17T17:55:40.786 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0056861419999999999 2024-08-17T17:55:40.786 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.786 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.786 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.786 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.840939+0000", 2024-08-17T17:55:40.786 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0068157790000000001 2024-08-17T17:55:40.787 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.787 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.787 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.787 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:20.878306+0000", 2024-08-17T17:55:40.787 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.037367746 2024-08-17T17:55:40.787 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.787 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.787 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.787 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.787 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.787 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117699 45.1 45:89818d88:::benchmark_data_smithi130_172551_object12892:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-08-17T17:55:40.787 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:21.161816+0000", 2024-08-17T17:55:40.787 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 318.64824642100001, 2024-08-17T17:55:40.788 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6034206969999998, 2024-08-17T17:55:40.788 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.788 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.788 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.788 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.788 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.788 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.788 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117699 2024-08-17T17:55:40.788 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.788 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.788 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.788 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.788 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.161816+0000", 2024-08-17T17:55:40.789 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.789 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.789 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.789 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.161816+0000", 2024-08-17T17:55:40.789 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.789 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.789 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.789 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.161818+0000", 2024-08-17T17:55:40.789 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.61e-06 2024-08-17T17:55:40.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.790 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.790 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.790 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.161823+0000", 2024-08-17T17:55:40.790 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8919999999999997e-06 2024-08-17T17:55:40.790 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.790 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.790 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.790 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.161835+0000", 2024-08-17T17:55:40.790 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1622e-05 2024-08-17T17:55:40.790 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.790 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.790 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.791 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.161842+0000", 2024-08-17T17:55:40.791 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6769999999999998e-06 2024-08-17T17:55:40.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.791 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.791 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.625179+0000", 2024-08-17T17:55:40.791 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.463337216 2024-08-17T17:55:40.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.791 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.791 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.634265+0000", 2024-08-17T17:55:40.791 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0090856689999999993 2024-08-17T17:55:40.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.772321+0000", 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.13805646999999999 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.782022+0000", 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0097011969999999999 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.660103+0000", 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8780806889999999 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.792 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.793 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.793 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.674652+0000", 2024-08-17T17:55:40.793 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014548991000000001 2024-08-17T17:55:40.793 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.793 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.793 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.793 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.697572+0000", 2024-08-17T17:55:40.793 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022919695 2024-08-17T17:55:40.793 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.793 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.793 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.793 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.723258+0000", 2024-08-17T17:55:40.793 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025686356 2024-08-17T17:55:40.793 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.794 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.794 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.794 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.726761+0000", 2024-08-17T17:55:40.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0035028659999999999 2024-08-17T17:55:40.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.794 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.794 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.794 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.728435+0000", 2024-08-17T17:55:40.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001674236 2024-08-17T17:55:40.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.794 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.794 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.794 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.733754+0000", 2024-08-17T17:55:40.795 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0053185430000000002 2024-08-17T17:55:40.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.795 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.795 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.795 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.765237+0000", 2024-08-17T17:55:40.795 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.031482968 2024-08-17T17:55:40.795 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.795 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.795 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.795 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.795 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117710 45.1 45:8c275207:::benchmark_data_smithi130_172551_object12903:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-08-17T17:55:40.796 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:21.199072+0000", 2024-08-17T17:55:40.796 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 318.61098990200003, 2024-08-17T17:55:40.796 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.730388627, 2024-08-17T17:55:40.796 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.796 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.796 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.796 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.796 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.796 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.796 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117710 2024-08-17T17:55:40.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.796 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.796 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.797 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.199072+0000", 2024-08-17T17:55:40.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.797 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.797 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.797 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.797 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.199072+0000", 2024-08-17T17:55:40.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.797 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.797 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.797 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.797 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.199074+0000", 2024-08-17T17:55:40.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7260000000000001e-06 2024-08-17T17:55:40.797 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.798 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.798 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.798 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.199080+0000", 2024-08-17T17:55:40.798 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.665e-06 2024-08-17T17:55:40.798 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.798 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.798 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.798 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.199092+0000", 2024-08-17T17:55:40.798 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1727e-05 2024-08-17T17:55:40.798 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.798 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.798 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.798 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.199099+0000", 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8800000000000008e-06 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.785723+0000", 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.58662361699999999 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.795637+0000", 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0099135660000000004 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.799 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.849071+0000", 2024-08-17T17:55:40.800 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.053434118000000003 2024-08-17T17:55:40.800 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.800 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.800 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.800 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.857120+0000", 2024-08-17T17:55:40.800 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0080492189999999998 2024-08-17T17:55:40.800 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.800 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.800 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.800 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.828397+0000", 2024-08-17T17:55:40.800 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9712766269999999 2024-08-17T17:55:40.800 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.800 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.801 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.801 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.842426+0000", 2024-08-17T17:55:40.801 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014029741 2024-08-17T17:55:40.801 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.801 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.801 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.801 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.858962+0000", 2024-08-17T17:55:40.801 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016535516 2024-08-17T17:55:40.801 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.801 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.801 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.801 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.886263+0000", 2024-08-17T17:55:40.801 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.027301107000000002 2024-08-17T17:55:40.801 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.802 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.802 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.802 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.887898+0000", 2024-08-17T17:55:40.802 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001635077 2024-08-17T17:55:40.802 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.802 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.802 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.802 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.890177+0000", 2024-08-17T17:55:40.802 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0022786960000000002 2024-08-17T17:55:40.802 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.802 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.802 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.802 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.896455+0000", 2024-08-17T17:55:40.802 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.006277843 2024-08-17T17:55:40.803 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.803 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.803 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.803 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.929461+0000", 2024-08-17T17:55:40.803 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.033006502 2024-08-17T17:55:40.803 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.803 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.803 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.803 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.803 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.803 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117761 45.1 45:8c0eaaf2:::benchmark_data_smithi130_172551_object12954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-08-17T17:55:40.803 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:21.522576+0000", 2024-08-17T17:55:40.804 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 318.28748595899998, 2024-08-17T17:55:40.804 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7703922250000002, 2024-08-17T17:55:40.804 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.804 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.804 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.804 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.804 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.804 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.804 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117761 2024-08-17T17:55:40.804 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.804 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.804 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.804 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.804 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.522576+0000", 2024-08-17T17:55:40.805 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.805 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.805 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.805 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.805 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.522576+0000", 2024-08-17T17:55:40.805 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.805 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.805 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.805 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.805 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.522578+0000", 2024-08-17T17:55:40.805 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9420000000000002e-06 2024-08-17T17:55:40.805 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.805 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.806 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.806 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.522593+0000", 2024-08-17T17:55:40.806 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5051999999999999e-05 2024-08-17T17:55:40.806 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.806 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.806 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.806 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.522610+0000", 2024-08-17T17:55:40.806 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6605000000000001e-05 2024-08-17T17:55:40.806 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.806 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.806 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.806 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.522621+0000", 2024-08-17T17:55:40.806 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0654e-05 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.796893+0000", 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.27427218799999997 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.806000+0000", 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0091074329999999998 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.969794+0000", 2024-08-17T17:55:40.807 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16379380599999999 2024-08-17T17:55:40.808 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.808 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.808 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.808 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:21.979236+0000", 2024-08-17T17:55:40.808 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0094422570000000008 2024-08-17T17:55:40.808 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.808 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.808 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.808 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.173034+0000", 2024-08-17T17:55:40.808 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1937977829999999 2024-08-17T17:55:40.808 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.808 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.809 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.809 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.188247+0000", 2024-08-17T17:55:40.809 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015213032 2024-08-17T17:55:40.809 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.809 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.809 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.809 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.214123+0000", 2024-08-17T17:55:40.809 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025876050000000001 2024-08-17T17:55:40.809 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.809 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.809 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.809 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.250644+0000", 2024-08-17T17:55:40.809 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.036521074000000001 2024-08-17T17:55:40.809 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.810 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.810 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.810 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.254884+0000", 2024-08-17T17:55:40.810 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0042393789999999997 2024-08-17T17:55:40.810 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.810 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.810 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.810 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.256566+0000", 2024-08-17T17:55:40.810 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001682317 2024-08-17T17:55:40.810 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.810 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.810 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.810 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.264351+0000", 2024-08-17T17:55:40.811 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0077850719999999997 2024-08-17T17:55:40.811 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.811 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.811 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.811 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.292969+0000", 2024-08-17T17:55:40.811 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028617581 2024-08-17T17:55:40.811 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.811 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.811 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.811 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.811 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.811 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117773 45.1 45:8ab76996:::benchmark_data_smithi130_172551_object12966:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-08-17T17:55:40.812 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:22.296464+0000", 2024-08-17T17:55:40.812 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 317.51359860500003, 2024-08-17T17:55:40.812 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6386252799999999, 2024-08-17T17:55:40.812 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.812 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.812 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.812 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.812 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.812 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.812 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117773 2024-08-17T17:55:40.812 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.812 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.812 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.296464+0000", 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.296464+0000", 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.296465+0000", 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1689999999999999e-06 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.813 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.814 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.814 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.296470+0000", 2024-08-17T17:55:40.814 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0379999999999999e-06 2024-08-17T17:55:40.814 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.814 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.814 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.814 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.296481+0000", 2024-08-17T17:55:40.814 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1504e-05 2024-08-17T17:55:40.814 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.814 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.814 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.814 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.296487+0000", 2024-08-17T17:55:40.814 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8590000000000001e-06 2024-08-17T17:55:40.815 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.815 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.815 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.815 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.401116+0000", 2024-08-17T17:55:40.815 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.10462840700000001 2024-08-17T17:55:40.815 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.815 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.815 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.815 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.415403+0000", 2024-08-17T17:55:40.815 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014287456 2024-08-17T17:55:40.815 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.815 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.815 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.815 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.557228+0000", 2024-08-17T17:55:40.816 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14182444499999999 2024-08-17T17:55:40.816 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.816 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.816 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.816 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.568859+0000", 2024-08-17T17:55:40.816 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011631466 2024-08-17T17:55:40.816 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.816 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.816 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.816 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.730508+0000", 2024-08-17T17:55:40.816 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1616493860000001 2024-08-17T17:55:40.816 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.816 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.816 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.817 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.738421+0000", 2024-08-17T17:55:40.817 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0079125879999999999 2024-08-17T17:55:40.817 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.817 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.817 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.817 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.752356+0000", 2024-08-17T17:55:40.817 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013934811 2024-08-17T17:55:40.817 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.817 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.817 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.817 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.850855+0000", 2024-08-17T17:55:40.817 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.098499574000000006 2024-08-17T17:55:40.817 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.817 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.818 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.818 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.887510+0000", 2024-08-17T17:55:40.818 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.036654478999999997 2024-08-17T17:55:40.818 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.818 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.818 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.818 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.896175+0000", 2024-08-17T17:55:40.818 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0086650410000000001 2024-08-17T17:55:40.818 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.818 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.818 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.818 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.904846+0000", 2024-08-17T17:55:40.819 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0086708459999999994 2024-08-17T17:55:40.819 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.819 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.819 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.819 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:25.935089+0000", 2024-08-17T17:55:40.819 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.030243210999999999 2024-08-17T17:55:40.819 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.819 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.819 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.819 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117795 45.1 45:86158ee1:::benchmark_data_smithi130_172551_object12988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:22.768871+0000", 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 317.04119164000002, 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7097064340000001, 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117795 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.820 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.821 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.821 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.768871+0000", 2024-08-17T17:55:40.821 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.821 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.821 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.821 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.821 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.768871+0000", 2024-08-17T17:55:40.821 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.821 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.821 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.821 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.821 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.768872+0000", 2024-08-17T17:55:40.821 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.345e-06 2024-08-17T17:55:40.822 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.822 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.822 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.822 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.768877+0000", 2024-08-17T17:55:40.822 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9590000000000003e-06 2024-08-17T17:55:40.822 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.822 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.822 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.822 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.768888+0000", 2024-08-17T17:55:40.822 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.102e-05 2024-08-17T17:55:40.822 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.822 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.822 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.823 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:22.768893+0000", 2024-08-17T17:55:40.825 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2170000000000002e-06 2024-08-17T17:55:40.825 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.825 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.825 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.825 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.132950+0000", 2024-08-17T17:55:40.826 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.36405708799999997 2024-08-17T17:55:40.826 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.826 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.826 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.826 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.151568+0000", 2024-08-17T17:55:40.826 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018618170999999999 2024-08-17T17:55:40.826 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.826 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.826 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.826 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.203989+0000", 2024-08-17T17:55:40.826 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.052420952999999999 2024-08-17T17:55:40.826 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.826 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.827 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.827 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.212712+0000", 2024-08-17T17:55:40.827 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0087221250000000007 2024-08-17T17:55:40.827 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.827 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.827 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.827 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.149119+0000", 2024-08-17T17:55:40.827 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9364075810000001 2024-08-17T17:55:40.827 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.827 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.827 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.827 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.161407+0000", 2024-08-17T17:55:40.827 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012288017 2024-08-17T17:55:40.828 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.828 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.828 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.828 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.176123+0000", 2024-08-17T17:55:40.828 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014715611 2024-08-17T17:55:40.828 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.828 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.828 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.828 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.273615+0000", 2024-08-17T17:55:40.828 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.097492032000000006 2024-08-17T17:55:40.828 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.828 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.828 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.829 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.275069+0000", 2024-08-17T17:55:40.829 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001453984 2024-08-17T17:55:40.829 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.829 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.829 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.829 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.439845+0000", 2024-08-17T17:55:40.829 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16477650299999999 2024-08-17T17:55:40.829 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.829 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.829 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.829 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.448492+0000", 2024-08-17T17:55:40.829 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0086466979999999995 2024-08-17T17:55:40.829 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.830 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.830 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.830 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:26.478577+0000", 2024-08-17T17:55:40.830 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.030085130000000002 2024-08-17T17:55:40.830 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.830 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.830 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.830 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.830 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.830 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:117925 45.1 45:86b02470:::benchmark_data_smithi130_172551_object13118:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-08-17T17:55:40.830 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:50:23.946452+0000", 2024-08-17T17:55:40.830 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 315.86361062600002, 2024-08-17T17:55:40.830 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6290720809999999, 2024-08-17T17:55:40.831 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:55:40.831 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:55:40.831 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:55:40.831 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:55:40.831 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:55:40.831 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:55:40.831 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 117925 2024-08-17T17:55:40.831 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.831 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:55:40.831 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.831 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:55:40.831 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.946452+0000", 2024-08-17T17:55:40.831 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.831 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.832 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.832 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:55:40.832 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.946452+0000", 2024-08-17T17:55:40.832 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:55:40.832 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.832 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.832 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:55:40.832 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.946453+0000", 2024-08-17T17:55:40.832 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.556e-06 2024-08-17T17:55:40.832 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.832 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.832 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:55:40.832 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.946461+0000", 2024-08-17T17:55:40.832 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6229999999999997e-06 2024-08-17T17:55:40.833 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.833 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.833 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:55:40.833 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.946471+0000", 2024-08-17T17:55:40.833 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0125999999999999e-05 2024-08-17T17:55:40.833 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.833 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.833 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:55:40.833 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:23.946477+0000", 2024-08-17T17:55:40.833 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7860000000000002e-06 2024-08-17T17:55:40.833 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.833 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.833 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.833 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.109726+0000", 2024-08-17T17:55:40.834 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.163249224 2024-08-17T17:55:40.834 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.834 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.834 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:55:40.834 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.127220+0000", 2024-08-17T17:55:40.834 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017493841 2024-08-17T17:55:40.834 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.834 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.834 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.834 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.399095+0000", 2024-08-17T17:55:40.834 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.27187506700000003 2024-08-17T17:55:40.834 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.835 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.835 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:55:40.835 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:24.417340+0000", 2024-08-17T17:55:40.835 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018244782000000001 2024-08-17T17:55:40.835 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.835 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.835 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:55:40.835 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.436576+0000", 2024-08-17T17:55:40.835 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0192360040000001 2024-08-17T17:55:40.835 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.835 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.835 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:55:40.835 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.443099+0000", 2024-08-17T17:55:40.836 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0065228339999999999 2024-08-17T17:55:40.836 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.836 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.836 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:55:40.836 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.456244+0000", 2024-08-17T17:55:40.836 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013145899000000001 2024-08-17T17:55:40.836 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.836 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.836 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:55:40.836 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.543652+0000", 2024-08-17T17:55:40.836 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.087407886000000004 2024-08-17T17:55:40.836 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.836 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.836 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.837 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.545856+0000", 2024-08-17T17:55:40.837 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0022041209999999999 2024-08-17T17:55:40.837 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.837 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.837 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:55:40.837 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.548716+0000", 2024-08-17T17:55:40.837 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028598740000000001 2024-08-17T17:55:40.837 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.837 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.837 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:55:40.837 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.553839+0000", 2024-08-17T17:55:40.837 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0051224970000000002 2024-08-17T17:55:40.837 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:55:40.838 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:55:40.838 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:55:40.838 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:50:27.575524+0000", 2024-08-17T17:55:40.838 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021684960999999999 2024-08-17T17:55:40.838 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.838 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.838 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.838 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:55:40.838 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:55:40.838 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:55:40.839 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:40.860 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:40.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:40.961 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:40.990 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:40.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:40.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:41.092 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:40 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15868/168042 objects degraded (9.443%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:41.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:41.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:41.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:40 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15868/168042 objects degraded (9.443%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:40 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15868/168042 objects degraded (9.443%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:41.214 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:41.235 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:41.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:41.337 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:41.359 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:41.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:41.460 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:41.483 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:41.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:41.584 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:41.606 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:41.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:41.658 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:55:41.708 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:41.737 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:41.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:41.839 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:41.854 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T17:55:41.862 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:41.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:41 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3172827784' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:41 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:41 smithi130 ceph-mon[105521]: pgmap v1864: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 15843/167556 objects degraded (9.455%); 9271/167556 objects misplaced (5.533%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:55:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:41 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3172827784' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:41 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:41 smithi130 ceph-mon[110068]: pgmap v1864: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 15843/167556 objects degraded (9.455%); 9271/167556 objects misplaced (5.533%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:55:41.964 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:41.985 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:41.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:41.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:42.087 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:42.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:41 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3172827784' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:42.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:41 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:55:42.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:41 smithi179 ceph-mon[93519]: pgmap v1864: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 111 op/s; 15843/167556 objects degraded (9.455%); 9271/167556 objects misplaced (5.533%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:55:42.108 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:42.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:42.209 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:42.230 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:42.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:42.332 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:42.353 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:42.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:42.455 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:42.477 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:42.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:42.560 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:42.578 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:42.606 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:42.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:42.708 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:42.729 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:42.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:42.831 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:42.854 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:42.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:42.955 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:42.977 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:42.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:42.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:43.079 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:42 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:42 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:42 smithi179 ceph-mon[93519]: osdmap e1232: 8 total, 8 up, 6 in 2024-08-17T17:55:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:55:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:43.099 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:43.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:43.112 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:42 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:43.113 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:42 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:43.113 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:42 smithi130 ceph-mon[105521]: osdmap e1232: 8 total, 8 up, 6 in 2024-08-17T17:55:43.113 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:55:43.113 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:43.113 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:42 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:43.113 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:42 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:55:43.113 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:42 smithi130 ceph-mon[110068]: osdmap e1232: 8 total, 8 up, 6 in 2024-08-17T17:55:43.113 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:55:43.113 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:43.201 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:43.221 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:43.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:43.324 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:43.349 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:43.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:43.451 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:43.478 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:43.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:43.580 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:43.599 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:43.601 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:43.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:43.703 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:43.724 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:43.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:43.817 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:55:43.818 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:55:43.826 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:43.853 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:43.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:43.955 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:43.982 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:43.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:43.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:43.992 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:43 smithi130 ceph-mon[105521]: pgmap v1866: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 15837/167322 objects degraded (9.465%); 9225/167322 objects misplaced (5.513%); 845 KiB/s, 13 objects/s recovering 2024-08-17T17:55:43.992 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:43 smithi130 ceph-mon[110068]: pgmap v1866: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 15837/167322 objects degraded (9.465%); 9225/167322 objects misplaced (5.513%); 845 KiB/s, 13 objects/s recovering 2024-08-17T17:55:44.084 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:44.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:43 smithi179 ceph-mon[93519]: pgmap v1866: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 116 op/s; 15837/167322 objects degraded (9.465%); 9225/167322 objects misplaced (5.513%); 845 KiB/s, 13 objects/s recovering 2024-08-17T17:55:44.105 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:44.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:44.206 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:44.227 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:44.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:44.329 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:44.350 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:44.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:44.452 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:44.473 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:44.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:44.575 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:44.604 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:44.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:44.706 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:44.727 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:44.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:44.829 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:44.850 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:44.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:44.951 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:44.973 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:44.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:44.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:44.999 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:55:45.040 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:44 smithi130 ceph-mon[105521]: from='client.45755 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:55:45.040 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:44 smithi130 ceph-mon[110068]: from='client.45755 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:55:45.076 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:45.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:44 smithi179 ceph-mon[93519]: from='client.45755 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:55:45.098 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:45.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:45.200 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:45.221 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:45.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:45.323 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:45.347 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:45.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:45.449 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:45.470 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:45.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:45.537 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1866,"stamp":"2024-08-17T17:55:43.075232+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3654031333,"num_objects":55774,"num_object_clones":0,"num_object_copies":167322,"num_objects_missing_on_primary":2932,"num_objects_missing":2932,"num_objects_degraded":15837,"num_objects_misplaced":9225,"num_objects_unfound":0,"num_objects_dirty":55774,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":154140,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36228,"num_bytes_recovered":2195659789,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108843,"ondisk_log_size":108843,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11108484,"kb_used_data":9701332,"kb_used_omap":295,"kb_used_meta":1406808,"kb_avail":551362428,"statfs":{"total":575970213888,"available":564595126272,"internally_reserved":0,"allocated":9934163968,"data_stored":9904487129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1440571855},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,1,1,0,1,0,3,5,5],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-76677120,"num_objects":-1170,"num_object_clones":0,"num_object_copies":-3510,"num_objects_missing_on_primary":-92,"num_objects_missing":-92,"num_objects_degraded":-156,"num_objects_misplaced":-270,"num_objects_unfound":0,"num_objects_dirty":-1170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1170,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":132,"num_bytes_recovered":8650752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002775"},"pg_stats":[{"pgid":"45.f","version":"1231'9668","reported_seq":20210,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:40.051872+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:55:40.051872+0000","last_peered":"2024-08-17T17:55:40.051872+0000","last_clean":"2024-08-17T17:55:40.051872+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:55:40.051872+0000","last_undegraded":"2024-08-17T17:55:40.051872+0000","last_fullsized":"2024-08-17T17:55:40.051872+0000","mapping_epoch":1194,"log_start":"1171'2831","ondisk_log_start":"1171'2831","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6837,"log_dups_size":0,"ondisk_log_size":6837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229113856,"num_objects":3496,"num_object_clones":0,"num_object_copies":10488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9668,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1231'9527","reported_seq":19998,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:38.145411+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:55:38.145411+0000","last_peered":"2024-08-17T17:55:38.145411+0000","last_clean":"2024-08-17T17:55:38.145411+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:55:38.145411+0000","last_undegraded":"2024-08-17T17:55:38.145411+0000","last_fullsized":"2024-08-17T17:55:38.145411+0000","mapping_epoch":1211,"log_start":"1170'2532","ondisk_log_start":"1170'2532","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227213312,"num_objects":3467,"num_object_clones":0,"num_object_copies":10401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9527,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1231'9839","reported_seq":20688,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:38.206842+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:55:38.206842+0000","last_peered":"2024-08-17T17:55:38.206842+0000","last_clean":"2024-08-17T17:55:38.206842+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:55:38.206842+0000","last_undegraded":"2024-08-17T17:55:38.206842+0000","last_fullsized":"2024-08-17T17:55:38.206842+0000","mapping_epoch":1205,"log_start":"1175'3868","ondisk_log_start":"1175'3868","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5971,"log_dups_size":0,"ondisk_log_size":5971,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231866392,"num_objects":3539,"num_object_clones":0,"num_object_copies":10617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3539,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9839,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1231'9572","reported_seq":9644,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:38.104272+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:55:38.104272+0000","last_peered":"2024-08-17T17:55:38.104272+0000","last_clean":"2024-08-17T17:55:38.104272+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:55:38.104272+0000","last_undegraded":"2024-08-17T17:55:38.104272+0000","last_fullsized":"2024-08-17T17:55:38.104272+0000","mapping_epoch":1167,"log_start":"1171'2889","ondisk_log_start":"1171'2889","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6683,"log_dups_size":0,"ondisk_log_size":6683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224657408,"num_objects":3428,"num_object_clones":0,"num_object_copies":10284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9572,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1231'9669","reported_seq":14926,"reported_epoch":1232,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:41.779072+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:55:41.779072+0000","last_peered":"2024-08-17T17:55:41.779072+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:55:41.779072+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1175'3726","ondisk_log_start":"1175'3726","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225771520,"num_objects":3445,"num_object_clones":0,"num_object_copies":10335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":3445,"num_objects_unfound":0,"num_objects_dirty":3445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9669,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":2927}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1231'9699","reported_seq":9771,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:37.950471+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:55:37.950471+0000","last_peered":"2024-08-17T17:55:37.950471+0000","last_clean":"2024-08-17T17:55:37.950471+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:55:37.950471+0000","last_undegraded":"2024-08-17T17:55:37.950471+0000","last_fullsized":"2024-08-17T17:55:37.950471+0000","mapping_epoch":1167,"log_start":"1171'2977","ondisk_log_start":"1171'2977","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6722,"log_dups_size":0,"ondisk_log_size":6722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222494720,"num_objects":3395,"num_object_clones":0,"num_object_copies":10185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9699,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1231'9680","reported_seq":9752,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:38.161897+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:55:38.161897+0000","last_peered":"2024-08-17T17:55:38.161897+0000","last_clean":"2024-08-17T17:55:38.161897+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:55:38.161897+0000","last_undegraded":"2024-08-17T17:55:38.161897+0000","last_fullsized":"2024-08-17T17:55:38.161897+0000","mapping_epoch":1167,"log_start":"1171'2969","ondisk_log_start":"1171'2969","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6711,"log_dups_size":0,"ondisk_log_size":6711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232390656,"num_objects":3546,"num_object_clones":0,"num_object_copies":10638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9680,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1231'9632","reported_seq":19106,"reported_epoch":1232,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:41.778193+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:55:41.778193+0000","last_peered":"2024-08-17T17:55:41.778193+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:55:41.778193+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1175'3675","ondisk_log_start":"1175'3675","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5957,"log_dups_size":0,"ondisk_log_size":5957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220069888,"num_objects":3358,"num_object_clones":0,"num_object_copies":10074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9632,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3108,"num_bytes_recovered":203685888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":2038},{"shards":"1,7","objects":1291}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1231'9534","reported_seq":9616,"reported_epoch":1231,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:38.123050+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:55:38.123050+0000","last_peered":"2024-08-17T17:55:38.123050+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:55:38.123050+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1171'2777","ondisk_log_start":"1171'2777","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6757,"log_dups_size":0,"ondisk_log_size":6757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232128512,"num_objects":3542,"num_object_clones":0,"num_object_copies":10626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9534,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":3542},{"shards":"1,6","objects":-2291}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1231'9624","reported_seq":12712,"reported_epoch":1231,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:38.288600+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:55:38.288600+0000","last_peered":"2024-08-17T17:55:38.288600+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:55:38.288600+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1175'3706","ondisk_log_start":"1175'3706","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232652800,"num_objects":3550,"num_object_clones":0,"num_object_copies":10650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9624,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":1989}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1231'9535","reported_seq":9679,"reported_epoch":1231,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:38.923659+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:55:38.923659+0000","last_peered":"2024-08-17T17:55:38.923659+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:55:38.923659+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1170'400","ondisk_log_start":"1170'400","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9135,"log_dups_size":0,"ondisk_log_size":9135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231931904,"num_objects":3539,"num_object_clones":0,"num_object_copies":10617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9535,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-1614}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1231'9559","reported_seq":15763,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:38.963275+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:55:38.963275+0000","last_peered":"2024-08-17T17:55:38.963275+0000","last_clean":"2024-08-17T17:55:38.963275+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:55:38.963275+0000","last_undegraded":"2024-08-17T17:55:38.963275+0000","last_fullsized":"2024-08-17T17:55:38.963275+0000","mapping_epoch":1183,"log_start":"1170'400","ondisk_log_start":"1170'400","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9159,"log_dups_size":0,"ondisk_log_size":9159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228524032,"num_objects":3487,"num_object_clones":0,"num_object_copies":10461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9559,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1231'9668","reported_seq":9810,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:38.295183+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:55:38.295183+0000","last_peered":"2024-08-17T17:55:38.295183+0000","last_clean":"2024-08-17T17:55:38.295183+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:55:38.295183+0000","last_undegraded":"2024-08-17T17:55:38.295183+0000","last_fullsized":"2024-08-17T17:55:38.295183+0000","mapping_epoch":1167,"log_start":"1175'3691","ondisk_log_start":"1175'3691","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5977,"log_dups_size":0,"ondisk_log_size":5977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228851712,"num_objects":3492,"num_object_clones":0,"num_object_copies":10476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9668,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1231'9492","reported_seq":9564,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:37.720468+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:55:37.720468+0000","last_peered":"2024-08-17T17:55:37.720468+0000","last_clean":"2024-08-17T17:55:37.720468+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:55:37.720468+0000","last_undegraded":"2024-08-17T17:55:37.720468+0000","last_fullsized":"2024-08-17T17:55:37.720468+0000","mapping_epoch":1167,"log_start":"1170'2425","ondisk_log_start":"1170'2425","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7067,"log_dups_size":0,"ondisk_log_size":7067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225574912,"num_objects":3442,"num_object_clones":0,"num_object_copies":10326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9492,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1231'9787","reported_seq":23017,"reported_epoch":1231,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:55:40.304339+0000","last_change":"2024-08-17T17:55:37.941989+0000","last_active":"2024-08-17T17:55:40.304339+0000","last_peered":"2024-08-17T17:55:40.304339+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:55:40.304339+0000","last_undegraded":"2024-08-17T17:55:40.304339+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1171'2891","ondisk_log_start":"1171'2891","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6896,"log_dups_size":0,"ondisk_log_size":6896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234422272,"num_objects":3577,"num_object_clones":0,"num_object_copies":10731,"num_objects_missing_on_primary":2932,"num_objects_missing":2932,"num_objects_degraded":0,"num_objects_misplaced":5780,"num_objects_unfound":0,"num_objects_dirty":3577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9787,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5213,"num_bytes_recovered":341639168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":14},{"shards":"0,4,6","objects":1356},{"shards":"0,6","objects":2},{"shards":"4,5,6","objects":2205}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1231'9377","reported_seq":22706,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:38.268679+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:55:38.268679+0000","last_peered":"2024-08-17T17:55:38.268679+0000","last_clean":"2024-08-17T17:55:38.268679+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:55:38.268679+0000","last_undegraded":"2024-08-17T17:55:38.268679+0000","last_fullsized":"2024-08-17T17:55:38.268679+0000","mapping_epoch":1196,"log_start":"1174'3469","ondisk_log_start":"1174'3469","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5908,"log_dups_size":0,"ondisk_log_size":5908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225771520,"num_objects":3445,"num_object_clones":0,"num_object_copies":10335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9377,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1919,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.335502+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:55:30.335502+0000","last_peered":"2024-08-17T17:55:30.335502+0000","last_clean":"2024-08-17T17:55:30.335502+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:55:30.335502+0000","last_undegraded":"2024-08-17T17:55:30.335502+0000","last_fullsized":"2024-08-17T17:55:30.335502+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":508,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.741970+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:55:18.741970+0000","last_peered":"2024-08-17T17:55:18.741970+0000","last_clean":"2024-08-17T17:55:18.741970+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:55:18.741970+0000","last_undegraded":"2024-08-17T17:55:18.741970+0000","last_fullsized":"2024-08-17T17:55:18.741970+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1838,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.339172+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:55:30.339172+0000","last_peered":"2024-08-17T17:55:30.339172+0000","last_clean":"2024-08-17T17:55:30.339172+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:55:30.339172+0000","last_undegraded":"2024-08-17T17:55:30.339172+0000","last_fullsized":"2024-08-17T17:55:30.339172+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1862,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.337889+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:55:30.337889+0000","last_peered":"2024-08-17T17:55:30.337889+0000","last_clean":"2024-08-17T17:55:30.337889+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:55:30.337889+0000","last_undegraded":"2024-08-17T17:55:30.337889+0000","last_fullsized":"2024-08-17T17:55:30.337889+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":70,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.742566+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:55:18.742566+0000","last_peered":"2024-08-17T17:55:18.742566+0000","last_clean":"2024-08-17T17:55:18.742566+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:55:18.742566+0000","last_undegraded":"2024-08-17T17:55:18.742566+0000","last_fullsized":"2024-08-17T17:55:18.742566+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1256,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.338151+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:55:30.338151+0000","last_peered":"2024-08-17T17:55:30.338151+0000","last_clean":"2024-08-17T17:55:30.338151+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:55:30.338151+0000","last_undegraded":"2024-08-17T17:55:30.338151+0000","last_fullsized":"2024-08-17T17:55:30.338151+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":116,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.777131+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:55:41.777131+0000","last_peered":"2024-08-17T17:55:41.777131+0000","last_clean":"2024-08-17T17:55:41.777131+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:55:41.777131+0000","last_undegraded":"2024-08-17T17:55:41.777131+0000","last_fullsized":"2024-08-17T17:55:41.777131+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":634,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.338201+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:55:30.338201+0000","last_peered":"2024-08-17T17:55:30.338201+0000","last_clean":"2024-08-17T17:55:30.338201+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:55:30.338201+0000","last_undegraded":"2024-08-17T17:55:30.338201+0000","last_fullsized":"2024-08-17T17:55:30.338201+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1877,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.334355+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:55:30.334355+0000","last_peered":"2024-08-17T17:55:30.334355+0000","last_clean":"2024-08-17T17:55:30.334355+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:55:30.334355+0000","last_undegraded":"2024-08-17T17:55:30.334355+0000","last_fullsized":"2024-08-17T17:55:30.334355+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1879,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.348207+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:55:30.348207+0000","last_peered":"2024-08-17T17:55:30.348207+0000","last_clean":"2024-08-17T17:55:30.348207+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:55:30.348207+0000","last_undegraded":"2024-08-17T17:55:30.348207+0000","last_fullsized":"2024-08-17T17:55:30.348207+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":101,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.777779+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:55:41.777779+0000","last_peered":"2024-08-17T17:55:41.777779+0000","last_clean":"2024-08-17T17:55:41.777779+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:55:41.777779+0000","last_undegraded":"2024-08-17T17:55:41.777779+0000","last_fullsized":"2024-08-17T17:55:41.777779+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1851,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.339879+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:55:30.339879+0000","last_peered":"2024-08-17T17:55:30.339879+0000","last_clean":"2024-08-17T17:55:30.339879+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:55:30.339879+0000","last_undegraded":"2024-08-17T17:55:30.339879+0000","last_fullsized":"2024-08-17T17:55:30.339879+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1908,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.776111+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:55:41.776111+0000","last_peered":"2024-08-17T17:55:41.776111+0000","last_clean":"2024-08-17T17:55:41.776111+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:55:41.776111+0000","last_undegraded":"2024-08-17T17:55:41.776111+0000","last_fullsized":"2024-08-17T17:55:41.776111+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1900,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.335192+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:55:30.335192+0000","last_peered":"2024-08-17T17:55:30.335192+0000","last_clean":"2024-08-17T17:55:30.335192+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:55:30.335192+0000","last_undegraded":"2024-08-17T17:55:30.335192+0000","last_fullsized":"2024-08-17T17:55:30.335192+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1873,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.338025+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:55:30.338025+0000","last_peered":"2024-08-17T17:55:30.338025+0000","last_clean":"2024-08-17T17:55:30.338025+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:55:30.338025+0000","last_undegraded":"2024-08-17T17:55:30.338025+0000","last_fullsized":"2024-08-17T17:55:30.338025+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1205,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.742667+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:55:18.742667+0000","last_peered":"2024-08-17T17:55:18.742667+0000","last_clean":"2024-08-17T17:55:18.742667+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:55:18.742667+0000","last_undegraded":"2024-08-17T17:55:18.742667+0000","last_fullsized":"2024-08-17T17:55:18.742667+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1845,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.337816+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:55:30.337816+0000","last_peered":"2024-08-17T17:55:30.337816+0000","last_clean":"2024-08-17T17:55:30.337816+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:55:30.337816+0000","last_undegraded":"2024-08-17T17:55:30.337816+0000","last_fullsized":"2024-08-17T17:55:30.337816+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1894,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.335596+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:55:30.335596+0000","last_peered":"2024-08-17T17:55:30.335596+0000","last_clean":"2024-08-17T17:55:30.335596+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:55:30.335596+0000","last_undegraded":"2024-08-17T17:55:30.335596+0000","last_fullsized":"2024-08-17T17:55:30.335596+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1801,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.337949+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:55:30.337949+0000","last_peered":"2024-08-17T17:55:30.337949+0000","last_clean":"2024-08-17T17:55:30.337949+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:55:30.337949+0000","last_undegraded":"2024-08-17T17:55:30.337949+0000","last_fullsized":"2024-08-17T17:55:30.337949+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":524,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.756075+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:55:18.756075+0000","last_peered":"2024-08-17T17:55:18.756075+0000","last_clean":"2024-08-17T17:55:18.756075+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:55:18.756075+0000","last_undegraded":"2024-08-17T17:55:18.756075+0000","last_fullsized":"2024-08-17T17:55:18.756075+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1201,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.755815+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:55:18.755815+0000","last_peered":"2024-08-17T17:55:18.755815+0000","last_clean":"2024-08-17T17:55:18.755815+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:55:18.755815+0000","last_undegraded":"2024-08-17T17:55:18.755815+0000","last_fullsized":"2024-08-17T17:55:18.755815+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1901,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.778025+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:55:41.778025+0000","last_peered":"2024-08-17T17:55:41.778025+0000","last_clean":"2024-08-17T17:55:41.778025+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:55:41.778025+0000","last_undegraded":"2024-08-17T17:55:41.778025+0000","last_fullsized":"2024-08-17T17:55:41.778025+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":362,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.757726+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:55:18.757726+0000","last_peered":"2024-08-17T17:55:18.757726+0000","last_clean":"2024-08-17T17:55:18.757726+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:55:18.757726+0000","last_undegraded":"2024-08-17T17:55:18.757726+0000","last_fullsized":"2024-08-17T17:55:18.757726+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":508,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.742283+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:55:18.742283+0000","last_peered":"2024-08-17T17:55:18.742283+0000","last_clean":"2024-08-17T17:55:18.742283+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:55:18.742283+0000","last_undegraded":"2024-08-17T17:55:18.742283+0000","last_fullsized":"2024-08-17T17:55:18.742283+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1186,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.742442+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:55:18.742442+0000","last_peered":"2024-08-17T17:55:18.742442+0000","last_clean":"2024-08-17T17:55:18.742442+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:55:18.742442+0000","last_undegraded":"2024-08-17T17:55:18.742442+0000","last_fullsized":"2024-08-17T17:55:18.742442+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1194,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.742733+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:55:18.742733+0000","last_peered":"2024-08-17T17:55:18.742733+0000","last_clean":"2024-08-17T17:55:18.742733+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:55:18.742733+0000","last_undegraded":"2024-08-17T17:55:18.742733+0000","last_fullsized":"2024-08-17T17:55:18.742733+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1947,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.334969+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:55:30.334969+0000","last_peered":"2024-08-17T17:55:30.334969+0000","last_clean":"2024-08-17T17:55:30.334969+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:55:30.334969+0000","last_undegraded":"2024-08-17T17:55:30.334969+0000","last_fullsized":"2024-08-17T17:55:30.334969+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1830,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.337923+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:55:30.337923+0000","last_peered":"2024-08-17T17:55:30.337923+0000","last_clean":"2024-08-17T17:55:30.337923+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:55:30.337923+0000","last_undegraded":"2024-08-17T17:55:30.337923+0000","last_fullsized":"2024-08-17T17:55:30.337923+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1907,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.776739+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:55:41.776739+0000","last_peered":"2024-08-17T17:55:41.776739+0000","last_clean":"2024-08-17T17:55:41.776739+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:55:41.776739+0000","last_undegraded":"2024-08-17T17:55:41.776739+0000","last_fullsized":"2024-08-17T17:55:41.776739+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1833,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.339049+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:55:30.339049+0000","last_peered":"2024-08-17T17:55:30.339049+0000","last_clean":"2024-08-17T17:55:30.339049+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:55:30.339049+0000","last_undegraded":"2024-08-17T17:55:30.339049+0000","last_fullsized":"2024-08-17T17:55:30.339049+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1865,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.334543+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:55:30.334543+0000","last_peered":"2024-08-17T17:55:30.334543+0000","last_clean":"2024-08-17T17:55:30.334543+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:55:30.334543+0000","last_undegraded":"2024-08-17T17:55:30.334543+0000","last_fullsized":"2024-08-17T17:55:30.334543+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1221,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.756008+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:55:18.756008+0000","last_peered":"2024-08-17T17:55:18.756008+0000","last_clean":"2024-08-17T17:55:18.756008+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:55:18.756008+0000","last_undegraded":"2024-08-17T17:55:18.756008+0000","last_fullsized":"2024-08-17T17:55:18.756008+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1914,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.777341+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:55:41.777341+0000","last_peered":"2024-08-17T17:55:41.777341+0000","last_clean":"2024-08-17T17:55:41.777341+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:55:41.777341+0000","last_undegraded":"2024-08-17T17:55:41.777341+0000","last_fullsized":"2024-08-17T17:55:41.777341+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":100,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.777639+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:55:41.777639+0000","last_peered":"2024-08-17T17:55:41.777639+0000","last_clean":"2024-08-17T17:55:41.777639+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:55:41.777639+0000","last_undegraded":"2024-08-17T17:55:41.777639+0000","last_fullsized":"2024-08-17T17:55:41.777639+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1933,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.777257+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:55:41.777257+0000","last_peered":"2024-08-17T17:55:41.777257+0000","last_clean":"2024-08-17T17:55:41.777257+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:55:41.777257+0000","last_undegraded":"2024-08-17T17:55:41.777257+0000","last_fullsized":"2024-08-17T17:55:41.777257+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":667,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.339748+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:55:30.339748+0000","last_peered":"2024-08-17T17:55:30.339748+0000","last_clean":"2024-08-17T17:55:30.339748+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:55:30.339748+0000","last_undegraded":"2024-08-17T17:55:30.339748+0000","last_fullsized":"2024-08-17T17:55:30.339748+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1914,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.777424+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:55:41.777424+0000","last_peered":"2024-08-17T17:55:41.777424+0000","last_clean":"2024-08-17T17:55:41.777424+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:55:41.777424+0000","last_undegraded":"2024-08-17T17:55:41.777424+0000","last_fullsized":"2024-08-17T17:55:41.777424+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1240,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.334214+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:55:30.334214+0000","last_peered":"2024-08-17T17:55:30.334214+0000","last_clean":"2024-08-17T17:55:30.334214+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:55:30.334214+0000","last_undegraded":"2024-08-17T17:55:30.334214+0000","last_fullsized":"2024-08-17T17:55:30.334214+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":160,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.777934+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:55:41.777934+0000","last_peered":"2024-08-17T17:55:41.777934+0000","last_clean":"2024-08-17T17:55:41.777934+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:55:41.777934+0000","last_undegraded":"2024-08-17T17:55:41.777934+0000","last_fullsized":"2024-08-17T17:55:41.777934+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1924,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.778806+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:55:41.778806+0000","last_peered":"2024-08-17T17:55:41.778806+0000","last_clean":"2024-08-17T17:55:41.778806+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:55:41.778806+0000","last_undegraded":"2024-08-17T17:55:41.778806+0000","last_fullsized":"2024-08-17T17:55:41.778806+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1902,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.779073+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:55:41.779073+0000","last_peered":"2024-08-17T17:55:41.779073+0000","last_clean":"2024-08-17T17:55:41.779073+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:55:41.779073+0000","last_undegraded":"2024-08-17T17:55:41.779073+0000","last_fullsized":"2024-08-17T17:55:41.779073+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1920,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.347983+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:55:30.347983+0000","last_peered":"2024-08-17T17:55:30.347983+0000","last_clean":"2024-08-17T17:55:30.347983+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:55:30.347983+0000","last_undegraded":"2024-08-17T17:55:30.347983+0000","last_fullsized":"2024-08-17T17:55:30.347983+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1926,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.335264+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:55:30.335264+0000","last_peered":"2024-08-17T17:55:30.335264+0000","last_clean":"2024-08-17T17:55:30.335264+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:55:30.335264+0000","last_undegraded":"2024-08-17T17:55:30.335264+0000","last_fullsized":"2024-08-17T17:55:30.335264+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1906,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.375032+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:55:30.375032+0000","last_peered":"2024-08-17T17:55:30.375032+0000","last_clean":"2024-08-17T17:55:30.375032+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:55:30.375032+0000","last_undegraded":"2024-08-17T17:55:30.375032+0000","last_fullsized":"2024-08-17T17:55:30.375032+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1228,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.755813+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:55:18.755813+0000","last_peered":"2024-08-17T17:55:18.755813+0000","last_clean":"2024-08-17T17:55:18.755813+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:55:18.755813+0000","last_undegraded":"2024-08-17T17:55:18.755813+0000","last_fullsized":"2024-08-17T17:55:18.755813+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1174,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.742677+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:55:18.742677+0000","last_peered":"2024-08-17T17:55:18.742677+0000","last_clean":"2024-08-17T17:55:18.742677+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:55:18.742677+0000","last_undegraded":"2024-08-17T17:55:18.742677+0000","last_fullsized":"2024-08-17T17:55:18.742677+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1815,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.338047+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:55:30.338047+0000","last_peered":"2024-08-17T17:55:30.338047+0000","last_clean":"2024-08-17T17:55:30.338047+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:55:30.338047+0000","last_undegraded":"2024-08-17T17:55:30.338047+0000","last_fullsized":"2024-08-17T17:55:30.338047+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":146,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.348105+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:55:30.348105+0000","last_peered":"2024-08-17T17:55:30.348105+0000","last_clean":"2024-08-17T17:55:30.348105+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:55:30.348105+0000","last_undegraded":"2024-08-17T17:55:30.348105+0000","last_fullsized":"2024-08-17T17:55:30.348105+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1540,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.755556+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:55:18.755556+0000","last_peered":"2024-08-17T17:55:18.755556+0000","last_clean":"2024-08-17T17:55:18.755556+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:55:18.755556+0000","last_undegraded":"2024-08-17T17:55:18.755556+0000","last_fullsized":"2024-08-17T17:55:18.755556+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1909,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.778955+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:55:41.778955+0000","last_peered":"2024-08-17T17:55:41.778955+0000","last_clean":"2024-08-17T17:55:41.778955+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:55:41.778955+0000","last_undegraded":"2024-08-17T17:55:41.778955+0000","last_fullsized":"2024-08-17T17:55:41.778955+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1847,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.338088+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:55:30.338088+0000","last_peered":"2024-08-17T17:55:30.338088+0000","last_clean":"2024-08-17T17:55:30.338088+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:55:30.338088+0000","last_undegraded":"2024-08-17T17:55:30.338088+0000","last_fullsized":"2024-08-17T17:55:30.338088+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":508,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.744498+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:55:18.744498+0000","last_peered":"2024-08-17T17:55:18.744498+0000","last_clean":"2024-08-17T17:55:18.744498+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:55:18.744498+0000","last_undegraded":"2024-08-17T17:55:18.744498+0000","last_fullsized":"2024-08-17T17:55:18.744498+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":585,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.347843+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:55:30.347843+0000","last_peered":"2024-08-17T17:55:30.347843+0000","last_clean":"2024-08-17T17:55:30.347843+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:55:30.347843+0000","last_undegraded":"2024-08-17T17:55:30.347843+0000","last_fullsized":"2024-08-17T17:55:30.347843+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":601,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.348516+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:55:30.348516+0000","last_peered":"2024-08-17T17:55:30.348516+0000","last_clean":"2024-08-17T17:55:30.348516+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:55:30.348516+0000","last_undegraded":"2024-08-17T17:55:30.348516+0000","last_fullsized":"2024-08-17T17:55:30.348516+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1192,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.742435+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:55:18.742435+0000","last_peered":"2024-08-17T17:55:18.742435+0000","last_clean":"2024-08-17T17:55:18.742435+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:55:18.742435+0000","last_undegraded":"2024-08-17T17:55:18.742435+0000","last_fullsized":"2024-08-17T17:55:18.742435+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":617,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.338369+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:55:30.338369+0000","last_peered":"2024-08-17T17:55:30.338369+0000","last_clean":"2024-08-17T17:55:30.338369+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:55:30.338369+0000","last_undegraded":"2024-08-17T17:55:30.338369+0000","last_fullsized":"2024-08-17T17:55:30.338369+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1843,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.338277+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:55:30.338277+0000","last_peered":"2024-08-17T17:55:30.338277+0000","last_clean":"2024-08-17T17:55:30.338277+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:55:30.338277+0000","last_undegraded":"2024-08-17T17:55:30.338277+0000","last_fullsized":"2024-08-17T17:55:30.338277+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1939,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.334602+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:55:30.334602+0000","last_peered":"2024-08-17T17:55:30.334602+0000","last_clean":"2024-08-17T17:55:30.334602+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:55:30.334602+0000","last_undegraded":"2024-08-17T17:55:30.334602+0000","last_fullsized":"2024-08-17T17:55:30.334602+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":650,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.338273+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:55:30.338273+0000","last_peered":"2024-08-17T17:55:30.338273+0000","last_clean":"2024-08-17T17:55:30.338273+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:55:30.338273+0000","last_undegraded":"2024-08-17T17:55:30.338273+0000","last_fullsized":"2024-08-17T17:55:30.338273+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1181,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.742217+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:55:18.742217+0000","last_peered":"2024-08-17T17:55:18.742217+0000","last_clean":"2024-08-17T17:55:18.742217+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:55:18.742217+0000","last_undegraded":"2024-08-17T17:55:18.742217+0000","last_fullsized":"2024-08-17T17:55:18.742217+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1180,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.755934+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:55:18.755934+0000","last_peered":"2024-08-17T17:55:18.755934+0000","last_clean":"2024-08-17T17:55:18.755934+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:55:18.755934+0000","last_undegraded":"2024-08-17T17:55:18.755934+0000","last_fullsized":"2024-08-17T17:55:18.755934+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":636,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.776204+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:55:41.776204+0000","last_peered":"2024-08-17T17:55:41.776204+0000","last_clean":"2024-08-17T17:55:41.776204+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:55:41.776204+0000","last_undegraded":"2024-08-17T17:55:41.776204+0000","last_fullsized":"2024-08-17T17:55:41.776204+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1198,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.756199+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:55:18.756199+0000","last_peered":"2024-08-17T17:55:18.756199+0000","last_clean":"2024-08-17T17:55:18.756199+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:55:18.756199+0000","last_undegraded":"2024-08-17T17:55:18.756199+0000","last_fullsized":"2024-08-17T17:55:18.756199+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":82,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.755525+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:55:18.755525+0000","last_peered":"2024-08-17T17:55:18.755525+0000","last_clean":"2024-08-17T17:55:18.755525+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:55:18.755525+0000","last_undegraded":"2024-08-17T17:55:18.755525+0000","last_fullsized":"2024-08-17T17:55:18.755525+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1214,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.756146+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:55:18.756146+0000","last_peered":"2024-08-17T17:55:18.756146+0000","last_clean":"2024-08-17T17:55:18.756146+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:55:18.756146+0000","last_undegraded":"2024-08-17T17:55:18.756146+0000","last_fullsized":"2024-08-17T17:55:18.756146+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1231,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.742125+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:55:18.742125+0000","last_peered":"2024-08-17T17:55:18.742125+0000","last_clean":"2024-08-17T17:55:18.742125+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:55:18.742125+0000","last_undegraded":"2024-08-17T17:55:18.742125+0000","last_fullsized":"2024-08-17T17:55:18.742125+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1844,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.338867+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:55:30.338867+0000","last_peered":"2024-08-17T17:55:30.338867+0000","last_clean":"2024-08-17T17:55:30.338867+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:55:30.338867+0000","last_undegraded":"2024-08-17T17:55:30.338867+0000","last_fullsized":"2024-08-17T17:55:30.338867+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1175,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.755904+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:55:18.755904+0000","last_peered":"2024-08-17T17:55:18.755904+0000","last_clean":"2024-08-17T17:55:18.755904+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:55:18.755904+0000","last_undegraded":"2024-08-17T17:55:18.755904+0000","last_fullsized":"2024-08-17T17:55:18.755904+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1897,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.335064+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:55:30.335064+0000","last_peered":"2024-08-17T17:55:30.335064+0000","last_clean":"2024-08-17T17:55:30.335064+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:55:30.335064+0000","last_undegraded":"2024-08-17T17:55:30.335064+0000","last_fullsized":"2024-08-17T17:55:30.335064+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1899,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.334773+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:55:30.334773+0000","last_peered":"2024-08-17T17:55:30.334773+0000","last_clean":"2024-08-17T17:55:30.334773+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:55:30.334773+0000","last_undegraded":"2024-08-17T17:55:30.334773+0000","last_fullsized":"2024-08-17T17:55:30.334773+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":588,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.755793+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:55:18.755793+0000","last_peered":"2024-08-17T17:55:18.755793+0000","last_clean":"2024-08-17T17:55:18.755793+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:55:18.755793+0000","last_undegraded":"2024-08-17T17:55:18.755793+0000","last_fullsized":"2024-08-17T17:55:18.755793+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1200,"reported_epoch":1230,"state":"active+clean","last_fresh":"2024-08-17T17:55:18.755851+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:55:18.755851+0000","last_peered":"2024-08-17T17:55:18.755851+0000","last_clean":"2024-08-17T17:55:18.755851+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:55:18.755851+0000","last_undegraded":"2024-08-17T17:55:18.755851+0000","last_fullsized":"2024-08-17T17:55:18.755851+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1845,"reported_epoch":1231,"state":"active+clean","last_fresh":"2024-08-17T17:55:30.338499+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:55:30.338499+0000","last_peered":"2024-08-17T17:55:30.338499+0000","last_clean":"2024-08-17T17:55:30.338499+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:55:30.338499+0000","last_undegraded":"2024-08-17T17:55:30.338499+0000","last_fullsized":"2024-08-17T17:55:30.338499+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":3653435416,"num_objects":55748,"num_object_clones":0,"num_object_copies":167244,"num_objects_missing_on_primary":2932,"num_objects_missing":2932,"num_objects_degraded":15837,"num_objects_misplaced":9225,"num_objects_unfound":0,"num_objects_dirty":55748,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153862,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36199,"num_bytes_recovered":2193883136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11930710016,"data_stored":11930697800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108636,"ondisk_log_size":108636,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234417,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1869536,"kb_used_data":1568024,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91875616,"statfs":{"total":95995035648,"available":94080630784,"internally_reserved":0,"allocated":1605656576,"data_stored":1600702627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496052,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2110628,"kb_used_data":1809116,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91634524,"statfs":{"total":95995035648,"available":93833752576,"internally_reserved":0,"allocated":1852534784,"data_stored":1847577240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757691,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326304,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2236492,"kb_used_data":2024196,"kb_used_omap":57,"kb_used_meta":212230,"kb_avail":91508660,"statfs":{"total":95995035648,"available":93704867840,"internally_reserved":0,"allocated":2072776704,"data_stored":2067837935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":217324283},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405543,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2008276,"kb_used_data":1800076,"kb_used_omap":61,"kb_used_meta":208130,"kb_avail":91736876,"statfs":{"total":95995035648,"available":93938561024,"internally_reserved":0,"allocated":1843277824,"data_stored":1838336027,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":213126072},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452078,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1516040,"kb_used_data":1335680,"kb_used_omap":54,"kb_used_meta":180297,"kb_avail":92229112,"statfs":{"total":95995035648,"available":94442610688,"internally_reserved":0,"allocated":1367736320,"data_stored":1362801512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":184624526},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474769,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1367512,"kb_used_data":1164240,"kb_used_omap":119,"kb_used_meta":203144,"kb_avail":92377640,"statfs":{"total":95995035648,"available":94594703360,"internally_reserved":0,"allocated":1192181760,"data_stored":1187231788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":208020190},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,1,1,0,1,0,3,5,5],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019325,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1176965120,"data_stored":1176961048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1353777152,"data_stored":1353777152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1825439744,"data_stored":1825439744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2059014144,"data_stored":2059010072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1000800256,"data_stored":1000800256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1836978176,"data_stored":1836974104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1589837824,"data_stored":1589837824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:55:45.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:55:45.572 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:45.593 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:45.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:45.695 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:45.720 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:45.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:45.822 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:45.844 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:45.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:45.946 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:45.966 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:45.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:45.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:46.068 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:46.081 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:45 smithi130 ceph-mon[105521]: pgmap v1867: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 133 op/s; 15837/165471 objects degraded (9.571%); 9225/165471 objects misplaced (5.575%); 845 KiB/s, 13 objects/s recovering 2024-08-17T17:55:46.081 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:45 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/167322 objects degraded (9.465%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:46.081 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:45 smithi130 ceph-mon[110068]: pgmap v1867: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 133 op/s; 15837/165471 objects degraded (9.571%); 9225/165471 objects misplaced (5.575%); 845 KiB/s, 13 objects/s recovering 2024-08-17T17:55:46.082 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:45 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/167322 objects degraded (9.465%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:46.089 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:46.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:46.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:45 smithi179 ceph-mon[93519]: pgmap v1867: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 133 op/s; 15837/165471 objects degraded (9.571%); 9225/165471 objects misplaced (5.575%); 845 KiB/s, 13 objects/s recovering 2024-08-17T17:55:46.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:45 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/167322 objects degraded (9.465%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:46.190 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:46.216 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:46.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:46.317 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:46.338 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:46.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:46.443 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:46.464 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:46.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:46.566 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:46.589 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:46.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:46.690 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:46.726 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:46.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:46.827 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:46.849 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:46.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:46.950 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:46.988 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:46.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:46.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:47.090 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:47.114 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:47.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:47.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:47.215 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:47.235 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:47 smithi130 ceph-mon[105521]: pgmap v1868: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 15837/164865 objects degraded (9.606%); 9078/164865 objects misplaced (5.506%); 646 KiB/s, 10 objects/s recovering 2024-08-17T17:55:47.235 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:47 smithi130 ceph-mon[110068]: pgmap v1868: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 15837/164865 objects degraded (9.606%); 9078/164865 objects misplaced (5.506%); 646 KiB/s, 10 objects/s recovering 2024-08-17T17:55:47.236 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:47.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:47.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:47.338 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:47.367 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:47.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:47.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:47.469 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:47.490 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:47.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:47.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:47.592 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:47.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:47 smithi179 ceph-mon[93519]: pgmap v1868: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 138 op/s; 15837/164865 objects degraded (9.606%); 9078/164865 objects misplaced (5.506%); 646 KiB/s, 10 objects/s recovering 2024-08-17T17:55:47.613 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:47.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:47.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:47.715 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:47.719 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:55:47.719 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:55:47.719 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:55:47.719 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:55:47.719 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:55:47.719 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:55:47.719 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:55:47.719 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:55:47.720 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:55:47.720 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:55:47.720 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:55:47.720 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:55:47.720 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:55:47.720 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:55:47.720 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:55:47.720 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:55:47.739 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:47.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:47.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:47.841 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:47.862 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:47.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:47.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:47.964 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:47.985 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:47.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:47.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:48.087 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:48.108 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:48.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:48.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:48.210 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:48.231 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:48.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:48.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:48.272 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:55:48.272 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:55:48.272 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 0 2024-08-17T17:55:48.272 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:55:48.333 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:48.354 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:48.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:48.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:48.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:48 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1840195189' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:48.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:48 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1840195189' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:48.456 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:48.480 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:48.480 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:48.487 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:48.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:48.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:48.589 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:48 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1840195189' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:48.613 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:48.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:48.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:48.715 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:48.737 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:48.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:48.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:48.838 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:48.860 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:48.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:48.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:48.962 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:48.984 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:48.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:48.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:49.086 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:49.107 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:49.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:49.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:49.209 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:49.231 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:49.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:49.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:49.333 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:49.356 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:49.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:49.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:49.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:49 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:55:49.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:49 smithi130 ceph-mon[105521]: pgmap v1869: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 161 op/s; 15837/163185 objects degraded (9.705%); 9078/163185 objects misplaced (5.563%); 646 KiB/s, 10 objects/s recovering 2024-08-17T17:55:49.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:49 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:55:49.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:49 smithi130 ceph-mon[110068]: pgmap v1869: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 161 op/s; 15837/163185 objects degraded (9.705%); 9078/163185 objects misplaced (5.563%); 646 KiB/s, 10 objects/s recovering 2024-08-17T17:55:49.423 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:49 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:55:49.423 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:49 smithi179 ceph-mon[93519]: pgmap v1869: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 161 op/s; 15837/163185 objects degraded (9.705%); 9078/163185 objects misplaced (5.563%); 646 KiB/s, 10 objects/s recovering 2024-08-17T17:55:49.458 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:49.483 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:49.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:49.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:49.585 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:49.606 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:49.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:49.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:49.708 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:49.731 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:49.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:49.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:49.833 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:49.855 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:49.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:49.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:49.956 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:49.979 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:55:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:55:49] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-08-17T17:55:49.981 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:49.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:49.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:50.083 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:50.105 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:50.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:50.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:50.206 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:50.229 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:50.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:50.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:50.330 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:50.342 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:55:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:55:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:55:50.353 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:50.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:50.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:50.455 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:50.476 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:50.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:50.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:50.578 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:50.592 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:55:50.601 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:50.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:50.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:50.703 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:50.726 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:50.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:50.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:50.828 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:50.853 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:50.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:50.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:50.955 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:50.981 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:50.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:50.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:51.071 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:50 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/163185 objects degraded (9.705%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:51.071 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:50 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/163185 objects degraded (9.705%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:51.083 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:51.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:50 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/163185 objects degraded (9.705%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:51.104 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:51.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:51.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:51.206 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:51.238 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:51.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:51.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:51.340 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:51.361 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:51.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:51.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:51.463 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:51.486 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:51.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:51.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:51.588 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:51.611 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:51.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:51.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:51.713 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:51.734 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:51.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:51.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:51.739 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:55:51.835 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:51.857 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:51.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:51.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:51.959 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:51.985 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:51.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:51.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:52.028 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:51 smithi130 ceph-mon[105521]: pgmap v1870: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 166 op/s; 15837/162570 objects degraded (9.742%); 8975/162570 objects misplaced (5.521%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:55:52.029 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:51 smithi130 ceph-mon[110068]: pgmap v1870: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 166 op/s; 15837/162570 objects degraded (9.742%); 8975/162570 objects misplaced (5.521%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:55:52.044 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:51 smithi179 ceph-mon[93519]: pgmap v1870: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 166 op/s; 15837/162570 objects degraded (9.742%); 8975/162570 objects misplaced (5.521%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:55:52.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:52.113 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:52.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:52.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:52.214 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:52.241 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:52.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:52.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:52.342 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:52.373 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:52.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:52.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:52.475 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:52.483 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:55:52.483 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15837/162570 objects degraded (9.742%), 5 pgs degraded, 6 pgs undersized","count":11},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1673,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1232,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":54190,"data_bytes":3550222309,"bytes_used":11133816832,"bytes_avail":564836397056,"bytes_total":575970213888,"degraded_objects":15837,"degraded_total":162570,"degraded_ratio":0.097416497508765451,"misplaced_objects":8975,"misplaced_total":162570,"misplaced_ratio":0.055206987759119149,"recovering_objects_per_sec":10,"recovering_bytes_per_sec":694483,"recovering_keys_per_sec":0,"num_objects_recovered":106,"num_bytes_recovered":6946816,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":166},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":25,"modified":"2024-08-17T17:49:39.646037+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (8m)\n [==========================..] (remaining: 35s)","progress":0.93258428573608398,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:55:52.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:52.496 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:52.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:52.597 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:52.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:52.618 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:52.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:52.719 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:52.743 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:52.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:52.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:52.845 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:52.867 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:52.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:52.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:52.968 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:52.990 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:52.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:53.093 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:53.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:52 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2752227211' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:55:53.104 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:52 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2752227211' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:55:53.104 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:52 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2752227211' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:55:53.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:53.116 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:53.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:53.217 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:53.245 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:53.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:53.346 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:53.367 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:53.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:53.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:53.469 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:53.491 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:53.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:53.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:53.593 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:53.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:53.616 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:53.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:53.717 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:53.742 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:53.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:53.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:53.844 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:53.864 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:53.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:53.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:53.935 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:53 smithi130 ceph-mon[105521]: pgmap v1871: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 154 op/s; 15837/162300 objects degraded (9.758%); 8935/162300 objects misplaced (5.505%); 600 KiB/s, 9 objects/s recovering 2024-08-17T17:55:53.935 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:53 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:55:53.935 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:53 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:55:53.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:53 smithi130 ceph-mon[110068]: pgmap v1871: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 154 op/s; 15837/162300 objects degraded (9.758%); 8935/162300 objects misplaced (5.505%); 600 KiB/s, 9 objects/s recovering 2024-08-17T17:55:53.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:53 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:55:53.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:53 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:55:53.966 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:53.987 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:53.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:53.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:54.089 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:53 smithi179 ceph-mon[93519]: pgmap v1871: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 154 op/s; 15837/162300 objects degraded (9.758%); 8935/162300 objects misplaced (5.505%); 600 KiB/s, 9 objects/s recovering 2024-08-17T17:55:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:53 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:55:54.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:53 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:55:54.110 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:54.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:54.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:54.212 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:54.234 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:54.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:54.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:54.336 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:54.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:54.358 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:54.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:54.459 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:54.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:54.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:54.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:54.582 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:54.603 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:54.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:54.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:54.706 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:54.727 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:54.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:54.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:54.791 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T17:55:54.829 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:54.853 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:54.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:54.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:54.954 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:54.975 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:54.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:54.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:55.077 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/702565610' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:54 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:55.098 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:55.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:55.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:55.120 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/702565610' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:55.121 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:54 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:55.121 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/702565610' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:55.121 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:54 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:55:55.200 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:55.223 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:55.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:55.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:55.325 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:55.350 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:55.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:55.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:55.452 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:55.473 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:55.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:55.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:55.575 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:55.596 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:55.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:55.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:55.698 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:55.719 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:55.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:55.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:55.749 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:55:55.749 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:55:55.770 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:55.821 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:55.824 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:55.842 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:55.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:55.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:55.943 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:55.979 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:55.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:55.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:55 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:56.044 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:55 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:55:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:55 smithi130 ceph-mon[110068]: osdmap e1233: 8 total, 8 up, 6 in 2024-08-17T17:55:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:55 smithi130 ceph-mon[110068]: pgmap v1873: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 161 op/s; 15837/160635 objects degraded (9.859%); 8935/160635 objects misplaced (5.562%); 659 KiB/s, 10 objects/s recovering 2024-08-17T17:55:56.045 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:55 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/162300 objects degraded (9.758%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:56.045 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:55 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:56.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:55 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:55:56.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:55 smithi130 ceph-mon[105521]: osdmap e1233: 8 total, 8 up, 6 in 2024-08-17T17:55:56.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:55 smithi130 ceph-mon[105521]: pgmap v1873: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 161 op/s; 15837/160635 objects degraded (9.859%); 8935/160635 objects misplaced (5.562%); 659 KiB/s, 10 objects/s recovering 2024-08-17T17:55:56.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:55 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/162300 objects degraded (9.758%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:56.082 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:55 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:55:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:55 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:55:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:55 smithi179 ceph-mon[93519]: osdmap e1233: 8 total, 8 up, 6 in 2024-08-17T17:55:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:55 smithi179 ceph-mon[93519]: pgmap v1873: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 161 op/s; 15837/160635 objects degraded (9.859%); 8935/160635 objects misplaced (5.562%); 659 KiB/s, 10 objects/s recovering 2024-08-17T17:55:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:55 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/162300 objects degraded (9.758%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:55:56.103 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:56.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:56.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:56.205 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:56.227 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:56.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:56.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:56.328 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:56.354 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:56.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:56.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:56.456 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:56.482 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:56.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:56.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:56.584 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:56.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:56.608 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:56.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:56.709 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:56.732 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:56.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:56.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:56.834 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:56.856 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:56.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:56.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:56.958 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:56.979 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:56.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:56.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:57.081 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:57.102 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:57.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:57.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:57.204 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:57.226 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:57.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:57.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:57.280 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:57 smithi130 ceph-mon[105521]: pgmap v1874: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 158 op/s; 15837/160098 objects degraded (9.892%); 8781/160098 objects misplaced (5.485%); 665 KiB/s, 10 objects/s recovering 2024-08-17T17:55:57.281 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:57 smithi130 ceph-mon[110068]: pgmap v1874: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 158 op/s; 15837/160098 objects degraded (9.892%); 8781/160098 objects misplaced (5.485%); 665 KiB/s, 10 objects/s recovering 2024-08-17T17:55:57.327 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:57.353 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:57.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:57.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:57.455 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:57.459 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:55:57.482 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:57.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:57.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:57.584 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:57.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:57 smithi179 ceph-mon[93519]: pgmap v1874: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 158 op/s; 15837/160098 objects degraded (9.892%); 8781/160098 objects misplaced (5.485%); 665 KiB/s, 10 objects/s recovering 2024-08-17T17:55:57.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:57.606 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:57.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:57.707 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:57.730 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:57.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:57.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:57.831 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:55:57.853 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:57.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:57.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:55:57.955 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:57.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:57.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:57.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:58.078 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:58.099 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:58.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:58.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:58.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:58.203 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:55:58.243 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:58.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:58.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:58.345 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:55:58.366 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:58.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:58.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:55:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:58.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:58.468 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:58.499 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:58.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:58.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:55:58.601 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:58.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:58.624 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:58.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:58.725 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:58.759 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:58.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:58.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:58.860 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:55:58.881 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:58.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:58.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:55:58.983 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:55:59.006 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:59.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:59.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:55:59.107 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:59.130 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:59.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:59.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:59.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:59.244 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:55:59.244 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:55:59.244 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:55:59.244 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:55:59.245 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:55:59.245 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:55:59.245 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:55:59.245 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:55:59.245 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:55:59.245 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:55:59.245 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:55:59.245 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:55:59.245 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:55:59.245 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:55:59.245 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:55:59.246 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:55:59.254 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:59.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:59.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:59.356 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:59.381 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:59.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:59.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:59.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:59 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/12989518' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:59.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:55:59 smithi130 ceph-mon[105521]: pgmap v1875: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 15837/158829 objects degraded (9.971%); 8781/158829 objects misplaced (5.529%); 665 KiB/s, 10 objects/s recovering 2024-08-17T17:55:59.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:59 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/12989518' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:59.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:55:59 smithi130 ceph-mon[110068]: pgmap v1875: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 15837/158829 objects degraded (9.971%); 8781/158829 objects misplaced (5.529%); 665 KiB/s, 10 objects/s recovering 2024-08-17T17:55:59.483 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:55:59.508 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:59.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:59.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:55:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:59 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/12989518' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:55:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:55:59 smithi179 ceph-mon[93519]: pgmap v1875: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 145 op/s; 15837/158829 objects degraded (9.971%); 8781/158829 objects misplaced (5.529%); 665 KiB/s, 10 objects/s recovering 2024-08-17T17:55:59.609 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:55:59.631 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:59.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:59.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:55:59.733 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:55:59.754 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:55:59.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:59.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:55:59.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:55:59.878 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:55:59.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:55:59.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:55:59.980 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:00.005 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:00.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:00.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:00.107 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:00.110 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:55:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:55:59] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-08-17T17:56:00.130 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:00.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:00.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:00.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:00.255 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:00.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:00.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:00.357 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:00.377 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:00.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:00.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:00.479 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:00.501 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:00.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:00.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:56:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:56:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:56:00.603 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:00.624 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:00.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:00.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:00.726 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:00.749 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:00.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:00.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:00.756 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:56:00.756 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:56:00.756 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T17:56:00.756 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T17:56:00.756 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:56:00.852 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:00.873 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:00.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:00.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:00.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:00 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/158829 objects degraded (9.971%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:00.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:00 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/158829 objects degraded (9.971%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:00.975 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:00.997 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:00.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:00.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:00 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/158829 objects degraded (9.971%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:01.098 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:01.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:01.121 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:01.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:01.222 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:01.244 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:01.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:01.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:01.346 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:01.367 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:01.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:01.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:01.469 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:01.490 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:01.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:01.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:01.592 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:01.614 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:01.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:01.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:01.658 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:56:01.716 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:01.776 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:01.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:01.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:01.878 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:01.901 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:01.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:01.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:01.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:01 smithi130 ceph-mon[105521]: pgmap v1876: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 144 op/s; 15837/158247 objects degraded (10.008%); 8676/158247 objects misplaced (5.483%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:56:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:01 smithi130 ceph-mon[110068]: pgmap v1876: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 144 op/s; 15837/158247 objects degraded (10.008%); 8676/158247 objects misplaced (5.483%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:56:02.002 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:02.026 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:02.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:02.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:02.039 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:56:02.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:01 smithi179 ceph-mon[93519]: pgmap v1876: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 144 op/s; 15837/158247 objects degraded (10.008%); 8676/158247 objects misplaced (5.483%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:56:02.129 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:02.142 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:02.152 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:02.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:02.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:02.254 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:02.282 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:02.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:02.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:02.384 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:02.406 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:02.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:02.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:02.508 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:02.538 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:02.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:02.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:02.640 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:02.664 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:02.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:02.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:02.766 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:02.792 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:02.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:02.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:02.894 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:02.920 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:02.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:02.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:03.022 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:03.047 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:03.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:03.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:03.149 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:03.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:03.264 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:03.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:03.279 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:03 smithi130 ceph-mon[105521]: pgmap v1877: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 15837/158061 objects degraded (10.020%); 8635/158061 objects misplaced (5.463%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:56:03.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:03 smithi130 ceph-mon[110068]: pgmap v1877: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 15837/158061 objects degraded (10.020%); 8635/158061 objects misplaced (5.463%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:56:03.365 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:03.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:03.388 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:03.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:03.489 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:03.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:03.512 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:03.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:03.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:03 smithi179 ceph-mon[93519]: pgmap v1877: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 141 op/s; 15837/158061 objects degraded (10.020%); 8635/158061 objects misplaced (5.463%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:56:03.613 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:03.635 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:03.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:03.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:03.736 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:03.758 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:03.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:03.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:03.860 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:03.885 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:03.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:03.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:03.987 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:04.009 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:04.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:04.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:04.101 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:56:04.103 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:56:04.111 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:04.138 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:04.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:04.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:04.240 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:04.261 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:04.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:04.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:04.363 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:04.385 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:04.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:04.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:04.487 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:04.508 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:04.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:04.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:04.609 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:04.631 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:04.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:04.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:04.733 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:04.755 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:04.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:04.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:04.857 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:04.878 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:04.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:04.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:04.980 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:05.007 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:05.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:05.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:05.054 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1877,"stamp":"2024-08-17T17:56:03.080358+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3451721701,"num_objects":52687,"num_object_clones":0,"num_object_copies":158061,"num_objects_missing_on_primary":2726,"num_objects_missing":2726,"num_objects_degraded":15837,"num_objects_misplaced":8635,"num_objects_unfound":0,"num_objects_dirty":52687,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":157227,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36437,"num_bytes_recovered":2209356813,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108730,"ondisk_log_size":108730,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":10636740,"kb_used_data":9180628,"kb_used_omap":295,"kb_used_meta":1455704,"kb_avail":551834172,"statfs":{"total":575970213888,"available":565078192128,"internally_reserved":0,"allocated":9400963072,"data_stored":9371197001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1490641359},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-92602368,"num_objects":-1413,"num_object_clones":0,"num_object_copies":-4239,"num_objects_missing_on_primary":-104,"num_objects_missing":-104,"num_objects_degraded":0,"num_objects_misplaced":-300,"num_objects_unfound":0,"num_objects_dirty":-1413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1413,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":106,"num_bytes_recovered":6946816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002292"},"pg_stats":[{"pgid":"45.f","version":"1233'9858","reported_seq":20404,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:56:00.537695+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:56:00.537695+0000","last_peered":"2024-08-17T17:56:00.537695+0000","last_clean":"2024-08-17T17:56:00.537695+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:56:00.537695+0000","last_undegraded":"2024-08-17T17:56:00.537695+0000","last_fullsized":"2024-08-17T17:56:00.537695+0000","mapping_epoch":1194,"log_start":"1171'3031","ondisk_log_start":"1171'3031","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6827,"log_dups_size":0,"ondisk_log_size":6827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216662016,"num_objects":3306,"num_object_clones":0,"num_object_copies":9918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9858,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1233'9725","reported_seq":20198,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:57.721042+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:55:57.721042+0000","last_peered":"2024-08-17T17:55:57.721042+0000","last_clean":"2024-08-17T17:55:57.721042+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:55:57.721042+0000","last_undegraded":"2024-08-17T17:55:57.721042+0000","last_fullsized":"2024-08-17T17:55:57.721042+0000","mapping_epoch":1211,"log_start":"1171'2732","ondisk_log_start":"1171'2732","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6993,"log_dups_size":0,"ondisk_log_size":6993,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214237184,"num_objects":3269,"num_object_clones":0,"num_object_copies":9807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9725,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1233'10033","reported_seq":20886,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:58.238316+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:55:58.238316+0000","last_peered":"2024-08-17T17:55:58.238316+0000","last_clean":"2024-08-17T17:55:58.238316+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:55:58.238316+0000","last_undegraded":"2024-08-17T17:55:58.238316+0000","last_fullsized":"2024-08-17T17:55:58.238316+0000","mapping_epoch":1205,"log_start":"1175'4068","ondisk_log_start":"1175'4068","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5965,"log_dups_size":0,"ondisk_log_size":5965,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219152408,"num_objects":3345,"num_object_clones":0,"num_object_copies":10035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3345,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10033,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1233'9765","reported_seq":9839,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:58.131959+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:55:58.131959+0000","last_peered":"2024-08-17T17:55:58.131959+0000","last_clean":"2024-08-17T17:55:58.131959+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:55:58.131959+0000","last_undegraded":"2024-08-17T17:55:58.131959+0000","last_fullsized":"2024-08-17T17:55:58.131959+0000","mapping_epoch":1167,"log_start":"1174'3089","ondisk_log_start":"1174'3089","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6676,"log_dups_size":0,"ondisk_log_size":6676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212008960,"num_objects":3235,"num_object_clones":0,"num_object_copies":9705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9765,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1233'9844","reported_seq":15103,"reported_epoch":1233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:01.779569+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:56:01.779569+0000","last_peered":"2024-08-17T17:56:01.779569+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:56:01.779569+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1175'3926","ondisk_log_start":"1175'3926","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214302720,"num_objects":3270,"num_object_clones":0,"num_object_copies":9810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":3270,"num_objects_unfound":0,"num_objects_dirty":3270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9844,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":2752}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1233'9883","reported_seq":9957,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:57.721872+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:55:57.721872+0000","last_peered":"2024-08-17T17:55:57.721872+0000","last_clean":"2024-08-17T17:55:57.721872+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:55:57.721872+0000","last_undegraded":"2024-08-17T17:55:57.721872+0000","last_fullsized":"2024-08-17T17:55:57.721872+0000","mapping_epoch":1167,"log_start":"1174'3177","ondisk_log_start":"1174'3177","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6706,"log_dups_size":0,"ondisk_log_size":6706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210436096,"num_objects":3211,"num_object_clones":0,"num_object_copies":9633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9883,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1233'9880","reported_seq":9954,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:58.153361+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:55:58.153361+0000","last_peered":"2024-08-17T17:55:58.153361+0000","last_clean":"2024-08-17T17:55:58.153361+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:55:58.153361+0000","last_undegraded":"2024-08-17T17:55:58.153361+0000","last_fullsized":"2024-08-17T17:55:58.153361+0000","mapping_epoch":1167,"log_start":"1174'3169","ondisk_log_start":"1174'3169","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6711,"log_dups_size":0,"ondisk_log_size":6711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219283456,"num_objects":3346,"num_object_clones":0,"num_object_copies":10038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9880,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1233'9806","reported_seq":19282,"reported_epoch":1233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:01.450397+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:56:01.450397+0000","last_peered":"2024-08-17T17:56:01.450397+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:56:01.450397+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1175'3875","ondisk_log_start":"1175'3875","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5931,"log_dups_size":0,"ondisk_log_size":5931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208666624,"num_objects":3184,"num_object_clones":0,"num_object_copies":9552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9806,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3108,"num_bytes_recovered":203685888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":1864},{"shards":"1,7","objects":1117}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1233'9736","reported_seq":9820,"reported_epoch":1233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:58.163690+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:55:58.163690+0000","last_peered":"2024-08-17T17:55:58.163690+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:55:58.163690+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1171'2977","ondisk_log_start":"1171'2977","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6759,"log_dups_size":0,"ondisk_log_size":6759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218890240,"num_objects":3340,"num_object_clones":0,"num_object_copies":10020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9736,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":3340},{"shards":"1,6","objects":-2493}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1233'9820","reported_seq":12912,"reported_epoch":1233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:58.177675+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:55:58.177675+0000","last_peered":"2024-08-17T17:55:58.177675+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:55:58.177675+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1175'3906","ondisk_log_start":"1175'3906","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5914,"log_dups_size":0,"ondisk_log_size":5914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219807744,"num_objects":3354,"num_object_clones":0,"num_object_copies":10062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9820,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":1793}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1233'9724","reported_seq":9872,"reported_epoch":1233,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:55:59.063469+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:55:59.063469+0000","last_peered":"2024-08-17T17:55:59.063469+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:55:59.063469+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1170'600","ondisk_log_start":"1170'600","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9124,"log_dups_size":0,"ondisk_log_size":9124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219545600,"num_objects":3350,"num_object_clones":0,"num_object_copies":10050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9724,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-1803}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1233'9757","reported_seq":15966,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:58.850692+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:55:58.850692+0000","last_peered":"2024-08-17T17:55:58.850692+0000","last_clean":"2024-08-17T17:55:58.850692+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:55:58.850692+0000","last_undegraded":"2024-08-17T17:55:58.850692+0000","last_fullsized":"2024-08-17T17:55:58.850692+0000","mapping_epoch":1183,"log_start":"1170'600","ondisk_log_start":"1170'600","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9157,"log_dups_size":0,"ondisk_log_size":9157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215547904,"num_objects":3289,"num_object_clones":0,"num_object_copies":9867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9757,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1233'9844","reported_seq":9991,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:58.248364+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:55:58.248364+0000","last_peered":"2024-08-17T17:55:58.248364+0000","last_clean":"2024-08-17T17:55:58.248364+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:55:58.248364+0000","last_undegraded":"2024-08-17T17:55:58.248364+0000","last_fullsized":"2024-08-17T17:55:58.248364+0000","mapping_epoch":1167,"log_start":"1175'3891","ondisk_log_start":"1175'3891","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5953,"log_dups_size":0,"ondisk_log_size":5953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217317376,"num_objects":3316,"num_object_clones":0,"num_object_copies":9948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9844,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1233'9699","reported_seq":9773,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:57.690289+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:55:57.690289+0000","last_peered":"2024-08-17T17:55:57.690289+0000","last_clean":"2024-08-17T17:55:57.690289+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:55:57.690289+0000","last_undegraded":"2024-08-17T17:55:57.690289+0000","last_fullsized":"2024-08-17T17:55:57.690289+0000","mapping_epoch":1167,"log_start":"1170'2625","ondisk_log_start":"1170'2625","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7074,"log_dups_size":0,"ondisk_log_size":7074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212008960,"num_objects":3235,"num_object_clones":0,"num_object_copies":9705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9699,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1233'9997","reported_seq":23854,"reported_epoch":1233,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:56:00.598982+0000","last_change":"2024-08-17T17:55:37.941989+0000","last_active":"2024-08-17T17:56:00.598982+0000","last_peered":"2024-08-17T17:56:00.598982+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:56:00.598982+0000","last_undegraded":"2024-08-17T17:56:00.598982+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1171'3091","ondisk_log_start":"1171'3091","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6906,"log_dups_size":0,"ondisk_log_size":6906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220659712,"num_objects":3367,"num_object_clones":0,"num_object_copies":10101,"num_objects_missing_on_primary":2726,"num_objects_missing":2726,"num_objects_degraded":0,"num_objects_misplaced":5365,"num_objects_unfound":0,"num_objects_dirty":3367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9997,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5422,"num_bytes_recovered":355336192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":11},{"shards":"0,4,6","objects":1356},{"shards":"0,6","objects":1},{"shards":"4,5,6","objects":1999}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1233'9578","reported_seq":22911,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:58.140359+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:55:58.140359+0000","last_peered":"2024-08-17T17:55:58.140359+0000","last_clean":"2024-08-17T17:55:58.140359+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:55:58.140359+0000","last_undegraded":"2024-08-17T17:55:58.140359+0000","last_fullsized":"2024-08-17T17:55:58.140359+0000","mapping_epoch":1196,"log_start":"1175'3669","ondisk_log_start":"1175'3669","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5909,"log_dups_size":0,"ondisk_log_size":5909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212598784,"num_objects":3244,"num_object_clones":0,"num_object_copies":9732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9578,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1923,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.796134+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:55:54.796134+0000","last_peered":"2024-08-17T17:55:54.796134+0000","last_clean":"2024-08-17T17:55:54.796134+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:55:54.796134+0000","last_undegraded":"2024-08-17T17:55:54.796134+0000","last_fullsized":"2024-08-17T17:55:54.796134+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":510,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.774524+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:55:41.774524+0000","last_peered":"2024-08-17T17:55:41.774524+0000","last_clean":"2024-08-17T17:55:41.774524+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:55:41.774524+0000","last_undegraded":"2024-08-17T17:55:41.774524+0000","last_fullsized":"2024-08-17T17:55:41.774524+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1843,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789959+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:55:54.789959+0000","last_peered":"2024-08-17T17:55:54.789959+0000","last_clean":"2024-08-17T17:55:54.789959+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:55:54.789959+0000","last_undegraded":"2024-08-17T17:55:54.789959+0000","last_fullsized":"2024-08-17T17:55:54.789959+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1867,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789300+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:55:54.789300+0000","last_peered":"2024-08-17T17:55:54.789300+0000","last_clean":"2024-08-17T17:55:54.789300+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:55:54.789300+0000","last_undegraded":"2024-08-17T17:55:54.789300+0000","last_fullsized":"2024-08-17T17:55:54.789300+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":72,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.775068+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:55:41.775068+0000","last_peered":"2024-08-17T17:55:41.775068+0000","last_clean":"2024-08-17T17:55:41.775068+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:55:41.775068+0000","last_undegraded":"2024-08-17T17:55:41.775068+0000","last_fullsized":"2024-08-17T17:55:41.775068+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1260,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789407+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:55:54.789407+0000","last_peered":"2024-08-17T17:55:54.789407+0000","last_clean":"2024-08-17T17:55:54.789407+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:55:54.789407+0000","last_undegraded":"2024-08-17T17:55:54.789407+0000","last_fullsized":"2024-08-17T17:55:54.789407+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":118,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.803897+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:55:54.803897+0000","last_peered":"2024-08-17T17:55:54.803897+0000","last_clean":"2024-08-17T17:55:54.803897+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:55:54.803897+0000","last_undegraded":"2024-08-17T17:55:54.803897+0000","last_fullsized":"2024-08-17T17:55:54.803897+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":638,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789643+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:55:54.789643+0000","last_peered":"2024-08-17T17:55:54.789643+0000","last_clean":"2024-08-17T17:55:54.789643+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:55:54.789643+0000","last_undegraded":"2024-08-17T17:55:54.789643+0000","last_fullsized":"2024-08-17T17:55:54.789643+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1881,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.795669+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:55:54.795669+0000","last_peered":"2024-08-17T17:55:54.795669+0000","last_clean":"2024-08-17T17:55:54.795669+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:55:54.795669+0000","last_undegraded":"2024-08-17T17:55:54.795669+0000","last_fullsized":"2024-08-17T17:55:54.795669+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1883,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.798711+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:55:54.798711+0000","last_peered":"2024-08-17T17:55:54.798711+0000","last_clean":"2024-08-17T17:55:54.798711+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:55:54.798711+0000","last_undegraded":"2024-08-17T17:55:54.798711+0000","last_fullsized":"2024-08-17T17:55:54.798711+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":103,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.803604+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:55:54.803604+0000","last_peered":"2024-08-17T17:55:54.803604+0000","last_clean":"2024-08-17T17:55:54.803604+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:55:54.803604+0000","last_undegraded":"2024-08-17T17:55:54.803604+0000","last_fullsized":"2024-08-17T17:55:54.803604+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1855,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789952+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:55:54.789952+0000","last_peered":"2024-08-17T17:55:54.789952+0000","last_clean":"2024-08-17T17:55:54.789952+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:55:54.789952+0000","last_undegraded":"2024-08-17T17:55:54.789952+0000","last_fullsized":"2024-08-17T17:55:54.789952+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1910,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.803109+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:55:54.803109+0000","last_peered":"2024-08-17T17:55:54.803109+0000","last_clean":"2024-08-17T17:55:54.803109+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:55:54.803109+0000","last_undegraded":"2024-08-17T17:55:54.803109+0000","last_fullsized":"2024-08-17T17:55:54.803109+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1904,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.795895+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:55:54.795895+0000","last_peered":"2024-08-17T17:55:54.795895+0000","last_clean":"2024-08-17T17:55:54.795895+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:55:54.795895+0000","last_undegraded":"2024-08-17T17:55:54.795895+0000","last_fullsized":"2024-08-17T17:55:54.795895+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1878,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789419+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:55:54.789419+0000","last_peered":"2024-08-17T17:55:54.789419+0000","last_clean":"2024-08-17T17:55:54.789419+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:55:54.789419+0000","last_undegraded":"2024-08-17T17:55:54.789419+0000","last_fullsized":"2024-08-17T17:55:54.789419+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1207,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.777086+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:55:41.777086+0000","last_peered":"2024-08-17T17:55:41.777086+0000","last_clean":"2024-08-17T17:55:41.777086+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:55:41.777086+0000","last_undegraded":"2024-08-17T17:55:41.777086+0000","last_fullsized":"2024-08-17T17:55:41.777086+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1849,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789177+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:55:54.789177+0000","last_peered":"2024-08-17T17:55:54.789177+0000","last_clean":"2024-08-17T17:55:54.789177+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:55:54.789177+0000","last_undegraded":"2024-08-17T17:55:54.789177+0000","last_fullsized":"2024-08-17T17:55:54.789177+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1899,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.796285+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:55:54.796285+0000","last_peered":"2024-08-17T17:55:54.796285+0000","last_clean":"2024-08-17T17:55:54.796285+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:55:54.796285+0000","last_undegraded":"2024-08-17T17:55:54.796285+0000","last_fullsized":"2024-08-17T17:55:54.796285+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1805,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789167+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:55:54.789167+0000","last_peered":"2024-08-17T17:55:54.789167+0000","last_clean":"2024-08-17T17:55:54.789167+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:55:54.789167+0000","last_undegraded":"2024-08-17T17:55:54.789167+0000","last_fullsized":"2024-08-17T17:55:54.789167+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":526,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.800367+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:55:41.800367+0000","last_peered":"2024-08-17T17:55:41.800367+0000","last_clean":"2024-08-17T17:55:41.800367+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:55:41.800367+0000","last_undegraded":"2024-08-17T17:55:41.800367+0000","last_fullsized":"2024-08-17T17:55:41.800367+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1203,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.800200+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:55:41.800200+0000","last_peered":"2024-08-17T17:55:41.800200+0000","last_clean":"2024-08-17T17:55:41.800200+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:55:41.800200+0000","last_undegraded":"2024-08-17T17:55:41.800200+0000","last_fullsized":"2024-08-17T17:55:41.800200+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1903,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.803821+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:55:54.803821+0000","last_peered":"2024-08-17T17:55:54.803821+0000","last_clean":"2024-08-17T17:55:54.803821+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:55:54.803821+0000","last_undegraded":"2024-08-17T17:55:54.803821+0000","last_fullsized":"2024-08-17T17:55:54.803821+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":364,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.802512+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:55:41.802512+0000","last_peered":"2024-08-17T17:55:41.802512+0000","last_clean":"2024-08-17T17:55:41.802512+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:55:41.802512+0000","last_undegraded":"2024-08-17T17:55:41.802512+0000","last_fullsized":"2024-08-17T17:55:41.802512+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":510,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.774819+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:55:41.774819+0000","last_peered":"2024-08-17T17:55:41.774819+0000","last_clean":"2024-08-17T17:55:41.774819+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:55:41.774819+0000","last_undegraded":"2024-08-17T17:55:41.774819+0000","last_fullsized":"2024-08-17T17:55:41.774819+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1188,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.774678+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:55:41.774678+0000","last_peered":"2024-08-17T17:55:41.774678+0000","last_clean":"2024-08-17T17:55:41.774678+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:55:41.774678+0000","last_undegraded":"2024-08-17T17:55:41.774678+0000","last_fullsized":"2024-08-17T17:55:41.774678+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1196,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.777150+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:55:41.777150+0000","last_peered":"2024-08-17T17:55:41.777150+0000","last_clean":"2024-08-17T17:55:41.777150+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:55:41.777150+0000","last_undegraded":"2024-08-17T17:55:41.777150+0000","last_fullsized":"2024-08-17T17:55:41.777150+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1951,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.795633+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:55:54.795633+0000","last_peered":"2024-08-17T17:55:54.795633+0000","last_clean":"2024-08-17T17:55:54.795633+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:55:54.795633+0000","last_undegraded":"2024-08-17T17:55:54.795633+0000","last_fullsized":"2024-08-17T17:55:54.795633+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1835,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789267+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:55:54.789267+0000","last_peered":"2024-08-17T17:55:54.789267+0000","last_clean":"2024-08-17T17:55:54.789267+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:55:54.789267+0000","last_undegraded":"2024-08-17T17:55:54.789267+0000","last_fullsized":"2024-08-17T17:55:54.789267+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1909,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.804556+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:55:54.804556+0000","last_peered":"2024-08-17T17:55:54.804556+0000","last_clean":"2024-08-17T17:55:54.804556+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:55:54.804556+0000","last_undegraded":"2024-08-17T17:55:54.804556+0000","last_fullsized":"2024-08-17T17:55:54.804556+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1837,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.790006+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:55:54.790006+0000","last_peered":"2024-08-17T17:55:54.790006+0000","last_clean":"2024-08-17T17:55:54.790006+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:55:54.790006+0000","last_undegraded":"2024-08-17T17:55:54.790006+0000","last_fullsized":"2024-08-17T17:55:54.790006+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1869,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.795671+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:55:54.795671+0000","last_peered":"2024-08-17T17:55:54.795671+0000","last_clean":"2024-08-17T17:55:54.795671+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:55:54.795671+0000","last_undegraded":"2024-08-17T17:55:54.795671+0000","last_fullsized":"2024-08-17T17:55:54.795671+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1223,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.800275+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:55:41.800275+0000","last_peered":"2024-08-17T17:55:41.800275+0000","last_clean":"2024-08-17T17:55:41.800275+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:55:41.800275+0000","last_undegraded":"2024-08-17T17:55:41.800275+0000","last_fullsized":"2024-08-17T17:55:41.800275+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1916,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.804065+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:55:54.804065+0000","last_peered":"2024-08-17T17:55:54.804065+0000","last_clean":"2024-08-17T17:55:54.804065+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:55:54.804065+0000","last_undegraded":"2024-08-17T17:55:54.804065+0000","last_fullsized":"2024-08-17T17:55:54.804065+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":103,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.803502+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:55:54.803502+0000","last_peered":"2024-08-17T17:55:54.803502+0000","last_clean":"2024-08-17T17:55:54.803502+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:55:54.803502+0000","last_undegraded":"2024-08-17T17:55:54.803502+0000","last_fullsized":"2024-08-17T17:55:54.803502+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1935,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.803987+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:55:54.803987+0000","last_peered":"2024-08-17T17:55:54.803987+0000","last_clean":"2024-08-17T17:55:54.803987+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:55:54.803987+0000","last_undegraded":"2024-08-17T17:55:54.803987+0000","last_fullsized":"2024-08-17T17:55:54.803987+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":671,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789842+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:55:54.789842+0000","last_peered":"2024-08-17T17:55:54.789842+0000","last_clean":"2024-08-17T17:55:54.789842+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:55:54.789842+0000","last_undegraded":"2024-08-17T17:55:54.789842+0000","last_fullsized":"2024-08-17T17:55:54.789842+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1916,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.804141+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:55:54.804141+0000","last_peered":"2024-08-17T17:55:54.804141+0000","last_clean":"2024-08-17T17:55:54.804141+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:55:54.804141+0000","last_undegraded":"2024-08-17T17:55:54.804141+0000","last_fullsized":"2024-08-17T17:55:54.804141+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1244,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.795588+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:55:54.795588+0000","last_peered":"2024-08-17T17:55:54.795588+0000","last_clean":"2024-08-17T17:55:54.795588+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:55:54.795588+0000","last_undegraded":"2024-08-17T17:55:54.795588+0000","last_fullsized":"2024-08-17T17:55:54.795588+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":162,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.803731+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:55:54.803731+0000","last_peered":"2024-08-17T17:55:54.803731+0000","last_clean":"2024-08-17T17:55:54.803731+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:55:54.803731+0000","last_undegraded":"2024-08-17T17:55:54.803731+0000","last_fullsized":"2024-08-17T17:55:54.803731+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1926,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.805632+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:55:54.805632+0000","last_peered":"2024-08-17T17:55:54.805632+0000","last_clean":"2024-08-17T17:55:54.805632+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:55:54.805632+0000","last_undegraded":"2024-08-17T17:55:54.805632+0000","last_fullsized":"2024-08-17T17:55:54.805632+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1904,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.803047+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:55:54.803047+0000","last_peered":"2024-08-17T17:55:54.803047+0000","last_clean":"2024-08-17T17:55:54.803047+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:55:54.803047+0000","last_undegraded":"2024-08-17T17:55:54.803047+0000","last_fullsized":"2024-08-17T17:55:54.803047+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1924,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.801053+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:55:54.801053+0000","last_peered":"2024-08-17T17:55:54.801053+0000","last_clean":"2024-08-17T17:55:54.801053+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:55:54.801053+0000","last_undegraded":"2024-08-17T17:55:54.801053+0000","last_fullsized":"2024-08-17T17:55:54.801053+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1930,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.795973+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:55:54.795973+0000","last_peered":"2024-08-17T17:55:54.795973+0000","last_clean":"2024-08-17T17:55:54.795973+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:55:54.795973+0000","last_undegraded":"2024-08-17T17:55:54.795973+0000","last_fullsized":"2024-08-17T17:55:54.795973+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1910,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.833412+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:55:54.833412+0000","last_peered":"2024-08-17T17:55:54.833412+0000","last_clean":"2024-08-17T17:55:54.833412+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:55:54.833412+0000","last_undegraded":"2024-08-17T17:55:54.833412+0000","last_fullsized":"2024-08-17T17:55:54.833412+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1230,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.800105+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:55:41.800105+0000","last_peered":"2024-08-17T17:55:41.800105+0000","last_clean":"2024-08-17T17:55:41.800105+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:55:41.800105+0000","last_undegraded":"2024-08-17T17:55:41.800105+0000","last_fullsized":"2024-08-17T17:55:41.800105+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1176,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.775135+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:55:41.775135+0000","last_peered":"2024-08-17T17:55:41.775135+0000","last_clean":"2024-08-17T17:55:41.775135+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:55:41.775135+0000","last_undegraded":"2024-08-17T17:55:41.775135+0000","last_fullsized":"2024-08-17T17:55:41.775135+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1819,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789240+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:55:54.789240+0000","last_peered":"2024-08-17T17:55:54.789240+0000","last_clean":"2024-08-17T17:55:54.789240+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:55:54.789240+0000","last_undegraded":"2024-08-17T17:55:54.789240+0000","last_fullsized":"2024-08-17T17:55:54.789240+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":150,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.798476+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:55:54.798476+0000","last_peered":"2024-08-17T17:55:54.798476+0000","last_clean":"2024-08-17T17:55:54.798476+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:55:54.798476+0000","last_undegraded":"2024-08-17T17:55:54.798476+0000","last_fullsized":"2024-08-17T17:55:54.798476+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1542,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.799996+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:55:41.799996+0000","last_peered":"2024-08-17T17:55:41.799996+0000","last_clean":"2024-08-17T17:55:41.799996+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:55:41.799996+0000","last_undegraded":"2024-08-17T17:55:41.799996+0000","last_fullsized":"2024-08-17T17:55:41.799996+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1911,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.805727+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:55:54.805727+0000","last_peered":"2024-08-17T17:55:54.805727+0000","last_clean":"2024-08-17T17:55:54.805727+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:55:54.805727+0000","last_undegraded":"2024-08-17T17:55:54.805727+0000","last_fullsized":"2024-08-17T17:55:54.805727+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1851,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789241+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:55:54.789241+0000","last_peered":"2024-08-17T17:55:54.789241+0000","last_clean":"2024-08-17T17:55:54.789241+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:55:54.789241+0000","last_undegraded":"2024-08-17T17:55:54.789241+0000","last_fullsized":"2024-08-17T17:55:54.789241+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":510,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.775060+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:55:41.775060+0000","last_peered":"2024-08-17T17:55:41.775060+0000","last_clean":"2024-08-17T17:55:41.775060+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:55:41.775060+0000","last_undegraded":"2024-08-17T17:55:41.775060+0000","last_fullsized":"2024-08-17T17:55:41.775060+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":589,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.800955+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:55:54.800955+0000","last_peered":"2024-08-17T17:55:54.800955+0000","last_clean":"2024-08-17T17:55:54.800955+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:55:54.800955+0000","last_undegraded":"2024-08-17T17:55:54.800955+0000","last_fullsized":"2024-08-17T17:55:54.800955+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":605,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.798672+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:55:54.798672+0000","last_peered":"2024-08-17T17:55:54.798672+0000","last_clean":"2024-08-17T17:55:54.798672+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:55:54.798672+0000","last_undegraded":"2024-08-17T17:55:54.798672+0000","last_fullsized":"2024-08-17T17:55:54.798672+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1194,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.775208+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:55:41.775208+0000","last_peered":"2024-08-17T17:55:41.775208+0000","last_clean":"2024-08-17T17:55:41.775208+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:55:41.775208+0000","last_undegraded":"2024-08-17T17:55:41.775208+0000","last_fullsized":"2024-08-17T17:55:41.775208+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":621,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789480+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:55:54.789480+0000","last_peered":"2024-08-17T17:55:54.789480+0000","last_clean":"2024-08-17T17:55:54.789480+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:55:54.789480+0000","last_undegraded":"2024-08-17T17:55:54.789480+0000","last_fullsized":"2024-08-17T17:55:54.789480+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1847,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789719+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:55:54.789719+0000","last_peered":"2024-08-17T17:55:54.789719+0000","last_clean":"2024-08-17T17:55:54.789719+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:55:54.789719+0000","last_undegraded":"2024-08-17T17:55:54.789719+0000","last_fullsized":"2024-08-17T17:55:54.789719+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1943,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.795757+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:55:54.795757+0000","last_peered":"2024-08-17T17:55:54.795757+0000","last_clean":"2024-08-17T17:55:54.795757+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:55:54.795757+0000","last_undegraded":"2024-08-17T17:55:54.795757+0000","last_fullsized":"2024-08-17T17:55:54.795757+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":654,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789423+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:55:54.789423+0000","last_peered":"2024-08-17T17:55:54.789423+0000","last_clean":"2024-08-17T17:55:54.789423+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:55:54.789423+0000","last_undegraded":"2024-08-17T17:55:54.789423+0000","last_fullsized":"2024-08-17T17:55:54.789423+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1183,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.775164+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:55:41.775164+0000","last_peered":"2024-08-17T17:55:41.775164+0000","last_clean":"2024-08-17T17:55:41.775164+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:55:41.775164+0000","last_undegraded":"2024-08-17T17:55:41.775164+0000","last_fullsized":"2024-08-17T17:55:41.775164+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1182,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.800266+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:55:41.800266+0000","last_peered":"2024-08-17T17:55:41.800266+0000","last_clean":"2024-08-17T17:55:41.800266+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:55:41.800266+0000","last_undegraded":"2024-08-17T17:55:41.800266+0000","last_fullsized":"2024-08-17T17:55:41.800266+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":638,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.803218+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:55:54.803218+0000","last_peered":"2024-08-17T17:55:54.803218+0000","last_clean":"2024-08-17T17:55:54.803218+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:55:54.803218+0000","last_undegraded":"2024-08-17T17:55:54.803218+0000","last_fullsized":"2024-08-17T17:55:54.803218+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1200,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.800542+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:55:41.800542+0000","last_peered":"2024-08-17T17:55:41.800542+0000","last_clean":"2024-08-17T17:55:41.800542+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:55:41.800542+0000","last_undegraded":"2024-08-17T17:55:41.800542+0000","last_fullsized":"2024-08-17T17:55:41.800542+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":84,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.799944+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:55:41.799944+0000","last_peered":"2024-08-17T17:55:41.799944+0000","last_clean":"2024-08-17T17:55:41.799944+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:55:41.799944+0000","last_undegraded":"2024-08-17T17:55:41.799944+0000","last_fullsized":"2024-08-17T17:55:41.799944+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1216,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.800422+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:55:41.800422+0000","last_peered":"2024-08-17T17:55:41.800422+0000","last_clean":"2024-08-17T17:55:41.800422+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:55:41.800422+0000","last_undegraded":"2024-08-17T17:55:41.800422+0000","last_fullsized":"2024-08-17T17:55:41.800422+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1233,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.774746+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:55:41.774746+0000","last_peered":"2024-08-17T17:55:41.774746+0000","last_clean":"2024-08-17T17:55:41.774746+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:55:41.774746+0000","last_undegraded":"2024-08-17T17:55:41.774746+0000","last_fullsized":"2024-08-17T17:55:41.774746+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1849,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789913+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:55:54.789913+0000","last_peered":"2024-08-17T17:55:54.789913+0000","last_clean":"2024-08-17T17:55:54.789913+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:55:54.789913+0000","last_undegraded":"2024-08-17T17:55:54.789913+0000","last_fullsized":"2024-08-17T17:55:54.789913+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1177,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.800171+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:55:41.800171+0000","last_peered":"2024-08-17T17:55:41.800171+0000","last_clean":"2024-08-17T17:55:41.800171+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:55:41.800171+0000","last_undegraded":"2024-08-17T17:55:41.800171+0000","last_fullsized":"2024-08-17T17:55:41.800171+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1902,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.795738+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:55:54.795738+0000","last_peered":"2024-08-17T17:55:54.795738+0000","last_clean":"2024-08-17T17:55:54.795738+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:55:54.795738+0000","last_undegraded":"2024-08-17T17:55:54.795738+0000","last_fullsized":"2024-08-17T17:55:54.795738+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1903,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.795419+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:55:54.795419+0000","last_peered":"2024-08-17T17:55:54.795419+0000","last_clean":"2024-08-17T17:55:54.795419+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:55:54.795419+0000","last_undegraded":"2024-08-17T17:55:54.795419+0000","last_fullsized":"2024-08-17T17:55:54.795419+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":590,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.800127+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:55:41.800127+0000","last_peered":"2024-08-17T17:55:41.800127+0000","last_clean":"2024-08-17T17:55:41.800127+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:55:41.800127+0000","last_undegraded":"2024-08-17T17:55:41.800127+0000","last_fullsized":"2024-08-17T17:55:41.800127+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1202,"reported_epoch":1232,"state":"active+clean","last_fresh":"2024-08-17T17:55:41.800424+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:55:41.800424+0000","last_peered":"2024-08-17T17:55:41.800424+0000","last_clean":"2024-08-17T17:55:41.800424+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:55:41.800424+0000","last_undegraded":"2024-08-17T17:55:41.800424+0000","last_fullsized":"2024-08-17T17:55:41.800424+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1849,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789606+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:55:54.789606+0000","last_peered":"2024-08-17T17:55:54.789606+0000","last_clean":"2024-08-17T17:55:54.789606+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:55:54.789606+0000","last_undegraded":"2024-08-17T17:55:54.789606+0000","last_fullsized":"2024-08-17T17:55:54.789606+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":3451125784,"num_objects":52661,"num_object_clones":0,"num_object_copies":157983,"num_objects_missing_on_primary":2726,"num_objects_missing":2726,"num_objects_degraded":15837,"num_objects_misplaced":8635,"num_objects_unfound":0,"num_objects_dirty":52661,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":156949,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36408,"num_bytes_recovered":2207580160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11388268544,"data_stored":11388256328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108523,"ondisk_log_size":108523,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234421,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1785920,"kb_used_data":1480248,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91959232,"statfs":{"total":95995035648,"available":94166253568,"internally_reserved":0,"allocated":1515773952,"data_stored":1510805131,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496056,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2018180,"kb_used_data":1708348,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91726972,"statfs":{"total":95995035648,"available":93928419328,"internally_reserved":0,"allocated":1749348352,"data_stored":1744375936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757695,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326308,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2135916,"kb_used_data":1915300,"kb_used_omap":57,"kb_used_meta":220550,"kb_avail":91609236,"statfs":{"total":95995035648,"available":93807857664,"internally_reserved":0,"allocated":1961267200,"data_stored":1956313559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":225843963},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405547,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1915572,"kb_used_data":1699052,"kb_used_omap":61,"kb_used_meta":216450,"kb_avail":91829580,"statfs":{"total":95995035648,"available":94033489920,"internally_reserved":0,"allocated":1739829248,"data_stored":1734872579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":221645752},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452082,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1448680,"kb_used_data":1264160,"kb_used_omap":54,"kb_used_meta":184457,"kb_avail":92296472,"statfs":{"total":95995035648,"available":94511587328,"internally_reserved":0,"allocated":1294499840,"data_stored":1289550160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":188884366},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474773,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1332472,"kb_used_data":1113520,"kb_used_omap":119,"kb_used_meta":218760,"kb_avail":92412680,"statfs":{"total":95995035648,"available":94630584320,"internally_reserved":0,"allocated":1140244480,"data_stored":1135279636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":224010974},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019329,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1124470784,"data_stored":1124466712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1281097728,"data_stored":1281097728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1723990016,"data_stored":1723990016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1947602944,"data_stored":1947598872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":988676096,"data_stored":988676096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1734152192,"data_stored":1734148120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1500381184,"data_stored":1500381184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:56:05.068 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:05.109 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:05.131 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:05.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:05.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:05.233 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:05.254 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:05.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:05.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:05.356 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:05.377 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:05.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:05.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:05.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:05 smithi130 ceph-mon[105521]: from='client.55444 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:56:05.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:05 smithi130 ceph-mon[105521]: pgmap v1878: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 15837/156327 objects degraded (10.131%); 8635/156327 objects misplaced (5.524%); 659 KiB/s, 10 objects/s recovering 2024-08-17T17:56:05.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:05 smithi130 ceph-mon[110068]: from='client.55444 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:56:05.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:05 smithi130 ceph-mon[110068]: pgmap v1878: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 15837/156327 objects degraded (10.131%); 8635/156327 objects misplaced (5.524%); 659 KiB/s, 10 objects/s recovering 2024-08-17T17:56:05.479 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:05.500 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:05.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:05.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:05 smithi179 ceph-mon[93519]: from='client.55444 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:56:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:05 smithi179 ceph-mon[93519]: pgmap v1878: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 15837/156327 objects degraded (10.131%); 8635/156327 objects misplaced (5.524%); 659 KiB/s, 10 objects/s recovering 2024-08-17T17:56:05.602 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:05.623 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:05.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:05.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:05.725 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:05.752 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:05.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:05.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:05.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:05.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:05.875 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:05.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:05.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:06.000 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:06.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:06.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:06.102 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:06.128 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:06.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:06.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:06.230 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:06.252 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:06.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:06.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:06.291 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/156327 objects degraded (10.131%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:06.291 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/156327 objects degraded (10.131%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:06.354 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:06.375 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:06.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:06.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:06.477 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:06.498 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:06.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:06.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/156327 objects degraded (10.131%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:06.599 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:06.622 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:06.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:06.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:06.724 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:06.745 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:06.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:06.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:06.847 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:06.871 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:06.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:06.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:06.973 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:06.994 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:06.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:06.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:07.096 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:07.117 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:07.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:07.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:07.219 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:07.243 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:07.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:07.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:07.345 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:07.366 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:07.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:07.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:07.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:07 smithi130 ceph-mon[105521]: pgmap v1879: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 15837/155628 objects degraded (10.176%); 8466/155628 objects misplaced (5.440%); 842 KiB/s, 13 objects/s recovering 2024-08-17T17:56:07.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:07 smithi130 ceph-mon[110068]: pgmap v1879: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 15837/155628 objects degraded (10.176%); 8466/155628 objects misplaced (5.440%); 842 KiB/s, 13 objects/s recovering 2024-08-17T17:56:07.467 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:07.489 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:07.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:07.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:07.590 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:07.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:07 smithi179 ceph-mon[93519]: pgmap v1879: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.2 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 139 op/s; 15837/155628 objects degraded (10.176%); 8466/155628 objects misplaced (5.440%); 842 KiB/s, 13 objects/s recovering 2024-08-17T17:56:07.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:07.614 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:07.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:07.715 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:07.737 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:07.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:07.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:07.840 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:07.861 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:07.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:07.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:07.949 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:07.964 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:07.995 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:07.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:07.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:08.059 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:56:08.097 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:08.124 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:08.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:08.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:08.163 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T17:56:08.226 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:08.248 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:08.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:08.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:08.350 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:08.372 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:08.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:08.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:08.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:08 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1996398615' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:08.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:08 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:08.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:08 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1996398615' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:08.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:08 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:08.475 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:08.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:08.497 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:08.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:08 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1996398615' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:08 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:08.598 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:08.622 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:08.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:08.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:08.724 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:08.747 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:08.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:08.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:08.849 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:08.870 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:08.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:08.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:08.972 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:08.993 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:08.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:08.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:09.095 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:09.116 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:09.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:09.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:09.218 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:09.239 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:09.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:09.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:09.341 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:09.362 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:09.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:09.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:09.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:09 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:56:09.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:09 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:56:09.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:09 smithi130 ceph-mon[105521]: osdmap e1234: 8 total, 8 up, 6 in 2024-08-17T17:56:09.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:09 smithi130 ceph-mon[105521]: pgmap v1881: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 159 op/s; 15837/154044 objects degraded (10.281%); 8466/154044 objects misplaced (5.496%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:56:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:09 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:56:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:09 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:56:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:09 smithi130 ceph-mon[110068]: osdmap e1234: 8 total, 8 up, 6 in 2024-08-17T17:56:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:09 smithi130 ceph-mon[110068]: pgmap v1881: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 159 op/s; 15837/154044 objects degraded (10.281%); 8466/154044 objects misplaced (5.496%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:56:09.464 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:09.487 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:09.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:09.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:09.590 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:09.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:09 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:56:09.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:09 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:56:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:09 smithi179 ceph-mon[93519]: osdmap e1234: 8 total, 8 up, 6 in 2024-08-17T17:56:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:09 smithi179 ceph-mon[93519]: pgmap v1881: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 159 op/s; 15837/154044 objects degraded (10.281%); 8466/154044 objects misplaced (5.496%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:56:09.611 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:09.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:09.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:09.712 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:09.733 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:09.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:09.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:09.835 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:09.859 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:09.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:09.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:09.961 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:09.984 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:09.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:09.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:10.079 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:56:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:56:09] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-08-17T17:56:10.086 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:10.108 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:10.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:10.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:10.209 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:10.232 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:10.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:10.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:10.314 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:56:10.314 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T17:56:10.314 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T17:56:10.314 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T17:56:10.314 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.314 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:83238 45.8 45:11a1a64f:::benchmark_data_smithi130_172551_object83237:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:56:10.314 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:48:38.638381+0000", 2024-08-17T17:56:10.314 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 451.67301262199999, 2024-08-17T17:56:10.314 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.406600589, 2024-08-17T17:56:10.315 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.315 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.315 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T17:56:10.315 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.315 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.315 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.315 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 83238 2024-08-17T17:56:10.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.315 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638381+0000", 2024-08-17T17:56:10.315 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638381+0000", 2024-08-17T17:56:10.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638383+0000", 2024-08-17T17:56:10.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4339999999999999e-06 2024-08-17T17:56:10.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638406+0000", 2024-08-17T17:56:10.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3493e-05 2024-08-17T17:56:10.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638416+0000", 2024-08-17T17:56:10.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0095000000000001e-05 2024-08-17T17:56:10.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638423+0000", 2024-08-17T17:56:10.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9949999999999999e-06 2024-08-17T17:56:10.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638664+0000", 2024-08-17T17:56:10.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000240688 2024-08-17T17:56:10.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.638975+0000", 2024-08-17T17:56:10.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00031074400000000001 2024-08-17T17:56:10.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:56:10.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.639400+0000", 2024-08-17T17:56:10.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00042540200000000002 2024-08-17T17:56:10.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.646624+0000", 2024-08-17T17:56:10.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072241290000000001 2024-08-17T17:56:10.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.662508+0000", 2024-08-17T17:56:10.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015883544999999999 2024-08-17T17:56:10.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:56:10.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:38.662522+0000", 2024-08-17T17:56:10.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4402e-05 2024-08-17T17:56:10.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.320 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.650046+0000", 2024-08-17T17:56:10.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98752317000000001 2024-08-17T17:56:10.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.320 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T17:56:10.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:39.650063+0000", 2024-08-17T17:56:10.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7394e-05 2024-08-17T17:56:10.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.321 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.321 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.044811+0000", 2024-08-17T17:56:10.321 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3947477720000001 2024-08-17T17:56:10.321 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.321 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.321 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.321 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:48:41.044982+0000", 2024-08-17T17:56:10.321 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017132599999999999 2024-08-17T17:56:10.321 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.321 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.321 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.321 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.321 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.322 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:106932 45.8 45:18d24a25:::benchmark_data_smithi130_172551_object2125:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.322 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:23.289066+0000", 2024-08-17T17:56:10.322 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 407.02232815899998, 2024-08-17T17:56:10.322 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22737446, 2024-08-17T17:56:10.322 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.322 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.322 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.322 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.322 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.322 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.322 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 106932 2024-08-17T17:56:10.322 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.322 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.323 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289066+0000", 2024-08-17T17:56:10.323 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.323 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289066+0000", 2024-08-17T17:56:10.323 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289068+0000", 2024-08-17T17:56:10.324 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2670000000000001e-06 2024-08-17T17:56:10.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289074+0000", 2024-08-17T17:56:10.324 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1750000000000002e-06 2024-08-17T17:56:10.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289087+0000", 2024-08-17T17:56:10.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2638e-05 2024-08-17T17:56:10.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.325 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.325 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.325 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289094+0000", 2024-08-17T17:56:10.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7719999999999997e-06 2024-08-17T17:56:10.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.325 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.325 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.325 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.363175+0000", 2024-08-17T17:56:10.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.074080928000000004 2024-08-17T17:56:10.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.325 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.365629+0000", 2024-08-17T17:56:10.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002453864 2024-08-17T17:56:10.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.326 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.417171+0000", 2024-08-17T17:56:10.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.051542814999999999 2024-08-17T17:56:10.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.326 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.417273+0000", 2024-08-17T17:56:10.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000101457 2024-08-17T17:56:10.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.327 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.417423+0000", 2024-08-17T17:56:10.327 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000149846 2024-08-17T17:56:10.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.327 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.452410+0000", 2024-08-17T17:56:10.327 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.034987799 2024-08-17T17:56:10.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.327 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.511559+0000", 2024-08-17T17:56:10.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.059148575000000002 2024-08-17T17:56:10.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.328 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.328 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.511606+0000", 2024-08-17T17:56:10.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7194000000000002e-05 2024-08-17T17:56:10.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.328 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.328 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.511673+0000", 2024-08-17T17:56:10.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6995999999999993e-05 2024-08-17T17:56:10.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.329 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.329 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.516440+0000", 2024-08-17T17:56:10.330 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0047671340000000001 2024-08-17T17:56:10.330 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.330 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.330 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.330 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.330 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.330 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:106967 45.8 45:13e64e6c:::benchmark_data_smithi130_172551_object2160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.330 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:23.378357+0000", 2024-08-17T17:56:10.330 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 406.93303693199999, 2024-08-17T17:56:10.330 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.279186029, 2024-08-17T17:56:10.330 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.330 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.331 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.331 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.331 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.331 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.331 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 106967 2024-08-17T17:56:10.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.331 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.331 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.331 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.331 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378357+0000", 2024-08-17T17:56:10.331 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.331 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.331 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378357+0000", 2024-08-17T17:56:10.332 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.332 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378359+0000", 2024-08-17T17:56:10.332 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1780000000000002e-06 2024-08-17T17:56:10.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.332 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378367+0000", 2024-08-17T17:56:10.332 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3649999999999998e-06 2024-08-17T17:56:10.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.333 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.333 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378387+0000", 2024-08-17T17:56:10.333 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9945999999999998e-05 2024-08-17T17:56:10.333 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.333 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.333 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378396+0000", 2024-08-17T17:56:10.333 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6420000000000002e-06 2024-08-17T17:56:10.333 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.333 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.333 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.534504+0000", 2024-08-17T17:56:10.334 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.15610795299999999 2024-08-17T17:56:10.334 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.334 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.334 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.334 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.537672+0000", 2024-08-17T17:56:10.334 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0031681180000000001 2024-08-17T17:56:10.334 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.334 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.334 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.334 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.580200+0000", 2024-08-17T17:56:10.334 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042527593000000002 2024-08-17T17:56:10.334 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.334 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.335 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.335 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.580441+0000", 2024-08-17T17:56:10.335 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00024059999999999999 2024-08-17T17:56:10.335 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.335 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.335 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.335 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.580882+0000", 2024-08-17T17:56:10.335 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00044118599999999999 2024-08-17T17:56:10.335 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.335 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.335 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.335 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.612775+0000", 2024-08-17T17:56:10.335 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.031893171999999997 2024-08-17T17:56:10.335 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.653587+0000", 2024-08-17T17:56:10.336 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.040812290000000001 2024-08-17T17:56:10.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.653642+0000", 2024-08-17T17:56:10.336 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4500999999999999e-05 2024-08-17T17:56:10.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.653792+0000", 2024-08-17T17:56:10.337 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000150235 2024-08-17T17:56:10.337 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.337 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.337 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.337 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.657543+0000", 2024-08-17T17:56:10.337 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0037512499999999998 2024-08-17T17:56:10.337 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.337 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.337 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.337 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.337 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.337 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:106987 45.8 45:18bd4662:::benchmark_data_smithi130_172551_object2180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.337 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:23.445589+0000", 2024-08-17T17:56:10.338 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 406.86580469500001, 2024-08-17T17:56:10.338 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.21594681099999999, 2024-08-17T17:56:10.338 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.338 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.338 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.338 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.338 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.338 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.338 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 106987 2024-08-17T17:56:10.338 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.338 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.338 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.338 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.339 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445589+0000", 2024-08-17T17:56:10.339 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.339 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.339 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.339 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.339 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445589+0000", 2024-08-17T17:56:10.339 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.339 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.339 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.339 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.339 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445592+0000", 2024-08-17T17:56:10.339 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.418e-06 2024-08-17T17:56:10.339 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.340 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.340 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.340 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445599+0000", 2024-08-17T17:56:10.340 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.588e-06 2024-08-17T17:56:10.340 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.340 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.340 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.340 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445616+0000", 2024-08-17T17:56:10.340 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6401000000000001e-05 2024-08-17T17:56:10.340 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.340 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.340 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.340 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445626+0000", 2024-08-17T17:56:10.340 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0465000000000001e-05 2024-08-17T17:56:10.341 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.341 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.341 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.341 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.537729+0000", 2024-08-17T17:56:10.341 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.092102937999999995 2024-08-17T17:56:10.341 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.341 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.341 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.341 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.539000+0000", 2024-08-17T17:56:10.341 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0012709609999999999 2024-08-17T17:56:10.341 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.341 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.341 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.342 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.595265+0000", 2024-08-17T17:56:10.342 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.056264464 2024-08-17T17:56:10.342 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.342 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.342 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.342 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.595396+0000", 2024-08-17T17:56:10.342 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000130996 2024-08-17T17:56:10.342 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.342 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.342 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.342 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.595582+0000", 2024-08-17T17:56:10.342 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000186097 2024-08-17T17:56:10.342 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.343 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.343 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.343 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.612782+0000", 2024-08-17T17:56:10.343 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017199941 2024-08-17T17:56:10.343 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.343 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.343 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.343 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.657596+0000", 2024-08-17T17:56:10.343 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.044814639000000003 2024-08-17T17:56:10.343 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.343 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.343 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.343 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.657637+0000", 2024-08-17T17:56:10.344 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1032999999999998e-05 2024-08-17T17:56:10.344 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.344 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.344 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.344 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.657728+0000", 2024-08-17T17:56:10.344 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0958000000000006e-05 2024-08-17T17:56:10.344 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.344 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.344 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.344 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.661536+0000", 2024-08-17T17:56:10.344 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0038079120000000001 2024-08-17T17:56:10.344 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.344 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.345 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.345 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.345 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.345 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:107041 45.8 45:11e0ef02:::benchmark_data_smithi130_172551_object2234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.345 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:23.556499+0000", 2024-08-17T17:56:10.345 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 406.75489474400001, 2024-08-17T17:56:10.345 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.225671231, 2024-08-17T17:56:10.345 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.345 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.345 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.345 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.345 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.345 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.345 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 107041 2024-08-17T17:56:10.346 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.346 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.346 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.346 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.346 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556499+0000", 2024-08-17T17:56:10.346 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.346 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.346 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.346 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.346 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556499+0000", 2024-08-17T17:56:10.346 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.346 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.346 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.347 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.347 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556501+0000", 2024-08-17T17:56:10.347 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1670000000000002e-06 2024-08-17T17:56:10.347 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.347 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.347 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.347 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556508+0000", 2024-08-17T17:56:10.347 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9430000000000003e-06 2024-08-17T17:56:10.347 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.347 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.347 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.347 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556528+0000", 2024-08-17T17:56:10.347 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.999e-05 2024-08-17T17:56:10.347 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.348 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.348 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.348 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556537+0000", 2024-08-17T17:56:10.348 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5890000000000003e-06 2024-08-17T17:56:10.348 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.348 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.348 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.348 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.674728+0000", 2024-08-17T17:56:10.348 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.118190689 2024-08-17T17:56:10.348 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.348 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.348 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.348 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.675674+0000", 2024-08-17T17:56:10.348 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00094605799999999995 2024-08-17T17:56:10.349 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.349 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.349 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.349 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.709070+0000", 2024-08-17T17:56:10.349 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.033396239000000001 2024-08-17T17:56:10.349 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.349 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.349 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.349 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.709171+0000", 2024-08-17T17:56:10.349 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010085 2024-08-17T17:56:10.349 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.349 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.349 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.349 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.709302+0000", 2024-08-17T17:56:10.350 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013068800000000001 2024-08-17T17:56:10.350 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.350 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.350 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.350 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.724080+0000", 2024-08-17T17:56:10.350 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014778964 2024-08-17T17:56:10.350 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.350 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.350 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.350 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.727981+0000", 2024-08-17T17:56:10.350 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039007310000000002 2024-08-17T17:56:10.350 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.350 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.351 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.351 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.739624+0000", 2024-08-17T17:56:10.351 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011642857 2024-08-17T17:56:10.351 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.351 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.351 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.351 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.739859+0000", 2024-08-17T17:56:10.351 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000234959 2024-08-17T17:56:10.351 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.351 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.351 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.351 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.782171+0000", 2024-08-17T17:56:10.351 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042311506999999998 2024-08-17T17:56:10.351 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.352 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.352 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.352 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.352 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.352 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108281 45.8 45:17388508:::benchmark_data_smithi130_172551_object3474:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.352 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:26.195227+0000", 2024-08-17T17:56:10.352 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 404.11616694999998, 2024-08-17T17:56:10.352 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.232226402, 2024-08-17T17:56:10.352 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.352 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.352 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.352 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.352 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.353 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.353 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108281 2024-08-17T17:56:10.353 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.353 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.353 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.353 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.353 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195227+0000", 2024-08-17T17:56:10.353 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.353 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.353 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.353 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.353 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195227+0000", 2024-08-17T17:56:10.353 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.353 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.354 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.354 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.354 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195229+0000", 2024-08-17T17:56:10.354 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4449999999999999e-06 2024-08-17T17:56:10.354 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.354 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.354 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.354 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195235+0000", 2024-08-17T17:56:10.354 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5440000000000004e-06 2024-08-17T17:56:10.354 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.354 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.354 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.354 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195247+0000", 2024-08-17T17:56:10.354 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2245999999999999e-05 2024-08-17T17:56:10.355 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.355 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195255+0000", 2024-08-17T17:56:10.355 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6790000000000005e-06 2024-08-17T17:56:10.355 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.355 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.312742+0000", 2024-08-17T17:56:10.355 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.11748729500000001 2024-08-17T17:56:10.355 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.356 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.356 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.313545+0000", 2024-08-17T17:56:10.356 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00080311800000000002 2024-08-17T17:56:10.356 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.356 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.356 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.356 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.355977+0000", 2024-08-17T17:56:10.356 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042431218999999999 2024-08-17T17:56:10.356 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.356 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.356 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.356 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.356037+0000", 2024-08-17T17:56:10.356 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9966000000000002e-05 2024-08-17T17:56:10.356 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.357 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.357 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.357 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.356111+0000", 2024-08-17T17:56:10.357 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4802999999999995e-05 2024-08-17T17:56:10.357 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.357 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.357 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.357 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.377134+0000", 2024-08-17T17:56:10.357 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021022968 2024-08-17T17:56:10.357 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.357 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.357 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.357 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.425050+0000", 2024-08-17T17:56:10.357 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.047915439999999997 2024-08-17T17:56:10.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.358 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.358 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.425098+0000", 2024-08-17T17:56:10.358 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7867000000000001e-05 2024-08-17T17:56:10.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.358 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.358 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.425140+0000", 2024-08-17T17:56:10.358 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2126000000000003e-05 2024-08-17T17:56:10.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.358 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.359 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.427454+0000", 2024-08-17T17:56:10.359 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0023136860000000001 2024-08-17T17:56:10.359 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.359 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.359 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.359 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.359 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.359 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108681 45.8 45:10018309:::benchmark_data_smithi130_172551_object3874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.359 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:27.139549+0000", 2024-08-17T17:56:10.359 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 403.17184468599999, 2024-08-17T17:56:10.359 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.21731466799999999, 2024-08-17T17:56:10.359 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.359 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.359 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.360 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.360 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.360 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.360 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108681 2024-08-17T17:56:10.360 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.360 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.360 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.360 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.360 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139549+0000", 2024-08-17T17:56:10.360 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.360 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.360 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.360 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139549+0000", 2024-08-17T17:56:10.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.361 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139550+0000", 2024-08-17T17:56:10.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4600000000000005e-07 2024-08-17T17:56:10.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.361 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139553+0000", 2024-08-17T17:56:10.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.774e-06 2024-08-17T17:56:10.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.362 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139560+0000", 2024-08-17T17:56:10.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4089999999999997e-06 2024-08-17T17:56:10.362 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.362 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139565+0000", 2024-08-17T17:56:10.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3059999999999997e-06 2024-08-17T17:56:10.362 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.362 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.261331+0000", 2024-08-17T17:56:10.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.121765262 2024-08-17T17:56:10.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.262881+0000", 2024-08-17T17:56:10.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0015505829999999999 2024-08-17T17:56:10.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.294557+0000", 2024-08-17T17:56:10.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.031675523999999997 2024-08-17T17:56:10.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.294741+0000", 2024-08-17T17:56:10.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000183833 2024-08-17T17:56:10.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.294884+0000", 2024-08-17T17:56:10.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014387499999999999 2024-08-17T17:56:10.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.320518+0000", 2024-08-17T17:56:10.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025633501 2024-08-17T17:56:10.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.320533+0000", 2024-08-17T17:56:10.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4738e-05 2024-08-17T17:56:10.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.352852+0000", 2024-08-17T17:56:10.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.032319379000000002 2024-08-17T17:56:10.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.352917+0000", 2024-08-17T17:56:10.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4584000000000004e-05 2024-08-17T17:56:10.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.366 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.366 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.356864+0000", 2024-08-17T17:56:10.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039473540000000001 2024-08-17T17:56:10.366 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.366 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.366 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.366 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.366 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108685 45.8 45:188d0d10:::benchmark_data_smithi130_172551_object3878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.366 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:27.141094+0000", 2024-08-17T17:56:10.366 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 403.17030032999998, 2024-08-17T17:56:10.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22079505299999999, 2024-08-17T17:56:10.367 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.367 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.367 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.367 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.367 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.367 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.367 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108685 2024-08-17T17:56:10.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.367 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.367 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.367 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.367 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141094+0000", 2024-08-17T17:56:10.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141094+0000", 2024-08-17T17:56:10.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141095+0000", 2024-08-17T17:56:10.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.469e-06 2024-08-17T17:56:10.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141100+0000", 2024-08-17T17:56:10.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2519999999999999e-06 2024-08-17T17:56:10.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.369 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141115+0000", 2024-08-17T17:56:10.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4592e-05 2024-08-17T17:56:10.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.369 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141123+0000", 2024-08-17T17:56:10.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0369999999999995e-06 2024-08-17T17:56:10.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.369 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.262940+0000", 2024-08-17T17:56:10.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.12181718800000001 2024-08-17T17:56:10.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.265083+0000", 2024-08-17T17:56:10.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021428350000000001 2024-08-17T17:56:10.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.309940+0000", 2024-08-17T17:56:10.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.044856832999999999 2024-08-17T17:56:10.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.371 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.371 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.310048+0000", 2024-08-17T17:56:10.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000107886 2024-08-17T17:56:10.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.371 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.371 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.371 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.310277+0000", 2024-08-17T17:56:10.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00022885700000000001 2024-08-17T17:56:10.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.371 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.371 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.371 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.357133+0000", 2024-08-17T17:56:10.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.046856061999999997 2024-08-17T17:56:10.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.372 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.372 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.372 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.357191+0000", 2024-08-17T17:56:10.372 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8743999999999999e-05 2024-08-17T17:56:10.372 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.372 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.372 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.372 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.357325+0000", 2024-08-17T17:56:10.372 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013331899999999999 2024-08-17T17:56:10.372 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.372 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.372 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.372 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.357474+0000", 2024-08-17T17:56:10.372 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000148796 2024-08-17T17:56:10.373 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.373 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.373 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.373 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.361889+0000", 2024-08-17T17:56:10.373 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0044151829999999996 2024-08-17T17:56:10.373 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.373 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.373 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.373 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.373 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.373 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108963 45.8 45:1d275979:::benchmark_data_smithi130_172551_object4156:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.373 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:27.755594+0000", 2024-08-17T17:56:10.373 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 402.55580000399999, 2024-08-17T17:56:10.374 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.27754511999999998, 2024-08-17T17:56:10.374 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.374 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.374 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.374 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.374 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.374 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.374 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108963 2024-08-17T17:56:10.374 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.374 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.374 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.374 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.374 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755594+0000", 2024-08-17T17:56:10.375 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.375 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.375 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.375 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.375 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755594+0000", 2024-08-17T17:56:10.375 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.375 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.375 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.375 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.375 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755595+0000", 2024-08-17T17:56:10.375 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4799999999999997e-07 2024-08-17T17:56:10.375 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.375 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.375 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.376 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755598+0000", 2024-08-17T17:56:10.376 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5599999999999998e-06 2024-08-17T17:56:10.376 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.376 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.376 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.376 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755610+0000", 2024-08-17T17:56:10.376 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.184e-05 2024-08-17T17:56:10.376 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.376 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.376 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.376 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755617+0000", 2024-08-17T17:56:10.376 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5950000000000004e-06 2024-08-17T17:56:10.376 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.377 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.377 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.377 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.841456+0000", 2024-08-17T17:56:10.377 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.085839020000000002 2024-08-17T17:56:10.377 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.377 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.377 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.377 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.846882+0000", 2024-08-17T17:56:10.377 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0054257439999999997 2024-08-17T17:56:10.377 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.377 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.377 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.377 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.875006+0000", 2024-08-17T17:56:10.377 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028124712999999999 2024-08-17T17:56:10.378 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.378 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.378 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.378 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.875105+0000", 2024-08-17T17:56:10.378 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8784999999999997e-05 2024-08-17T17:56:10.378 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.378 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.378 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.378 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.875242+0000", 2024-08-17T17:56:10.378 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013735599999999999 2024-08-17T17:56:10.378 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.378 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.378 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.378 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.028166+0000", 2024-08-17T17:56:10.379 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.15292386199999999 2024-08-17T17:56:10.379 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.379 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.379 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.379 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.028216+0000", 2024-08-17T17:56:10.379 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9851000000000001e-05 2024-08-17T17:56:10.379 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.379 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.379 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.379 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.028272+0000", 2024-08-17T17:56:10.379 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5674999999999998e-05 2024-08-17T17:56:10.379 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.379 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.379 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.380 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.029209+0000", 2024-08-17T17:56:10.380 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00093682800000000001 2024-08-17T17:56:10.380 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.380 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.380 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.380 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.033139+0000", 2024-08-17T17:56:10.380 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039305429999999999 2024-08-17T17:56:10.380 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.380 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.380 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.380 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.380 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.380 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109021 45.8 45:14c0b812:::benchmark_data_smithi130_172551_object4214:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.381 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:27.872171+0000", 2024-08-17T17:56:10.381 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 402.43922294999999, 2024-08-17T17:56:10.381 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22437405799999999, 2024-08-17T17:56:10.381 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.381 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.381 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.381 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.381 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.381 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.381 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109021 2024-08-17T17:56:10.381 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.381 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.381 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.381 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.382 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872171+0000", 2024-08-17T17:56:10.382 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.382 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.382 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.382 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.382 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872171+0000", 2024-08-17T17:56:10.382 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.382 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.382 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.382 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.382 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872173+0000", 2024-08-17T17:56:10.382 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.922e-06 2024-08-17T17:56:10.382 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.383 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.383 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.383 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872180+0000", 2024-08-17T17:56:10.383 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5899999999999996e-06 2024-08-17T17:56:10.383 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.383 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.383 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.383 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872192+0000", 2024-08-17T17:56:10.383 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2275e-05 2024-08-17T17:56:10.383 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.383 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.383 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.383 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872199+0000", 2024-08-17T17:56:10.384 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7750000000000004e-06 2024-08-17T17:56:10.384 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.384 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.384 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.384 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.041304+0000", 2024-08-17T17:56:10.384 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16910571499999999 2024-08-17T17:56:10.384 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.384 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.384 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.384 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.044079+0000", 2024-08-17T17:56:10.384 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027747480000000001 2024-08-17T17:56:10.384 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.384 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.384 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.385 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.068075+0000", 2024-08-17T17:56:10.385 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023995408999999999 2024-08-17T17:56:10.385 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.385 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.385 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.385 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.068206+0000", 2024-08-17T17:56:10.385 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013108299999999999 2024-08-17T17:56:10.385 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.385 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.385 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.385 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.068334+0000", 2024-08-17T17:56:10.385 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000128773 2024-08-17T17:56:10.385 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.385 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.386 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.386 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.084130+0000", 2024-08-17T17:56:10.386 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015795113999999999 2024-08-17T17:56:10.386 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:10.387 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.387 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.388 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.388 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.084175+0000", 2024-08-17T17:56:10.388 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5321000000000002e-05 2024-08-17T17:56:10.388 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.388 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.388 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.388 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.092490+0000", 2024-08-17T17:56:10.388 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0083154240000000001 2024-08-17T17:56:10.388 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.388 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.389 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.389 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.092654+0000", 2024-08-17T17:56:10.389 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000163599 2024-08-17T17:56:10.389 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.389 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.389 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.389 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.096545+0000", 2024-08-17T17:56:10.389 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0038913099999999998 2024-08-17T17:56:10.389 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.389 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.389 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.389 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.389 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.390 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109270 45.8 45:15211971:::benchmark_data_smithi130_172551_object4463:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.390 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.480079+0000", 2024-08-17T17:56:10.390 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 401.83131458000003, 2024-08-17T17:56:10.390 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.29654451799999998, 2024-08-17T17:56:10.390 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.390 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.390 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.390 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.390 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.390 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.390 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109270 2024-08-17T17:56:10.390 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.390 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.391 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.391 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480079+0000", 2024-08-17T17:56:10.391 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.391 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.391 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.391 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480079+0000", 2024-08-17T17:56:10.391 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.391 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.391 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.391 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480081+0000", 2024-08-17T17:56:10.391 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3659999999999999e-06 2024-08-17T17:56:10.392 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.392 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.392 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.392 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480086+0000", 2024-08-17T17:56:10.392 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2290000000000002e-06 2024-08-17T17:56:10.392 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.392 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.392 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.392 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480102+0000", 2024-08-17T17:56:10.392 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6399e-05 2024-08-17T17:56:10.392 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.392 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.392 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.393 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480110+0000", 2024-08-17T17:56:10.393 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8390000000000007e-06 2024-08-17T17:56:10.393 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.393 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.393 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.393 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.574022+0000", 2024-08-17T17:56:10.393 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.093911290999999994 2024-08-17T17:56:10.393 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.393 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.393 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.393 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.574696+0000", 2024-08-17T17:56:10.393 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00067411700000000001 2024-08-17T17:56:10.393 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.393 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.394 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.394 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.719056+0000", 2024-08-17T17:56:10.394 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.144360194 2024-08-17T17:56:10.394 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.394 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.394 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.394 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.719205+0000", 2024-08-17T17:56:10.394 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014905499999999999 2024-08-17T17:56:10.394 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.394 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.394 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.394 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.719356+0000", 2024-08-17T17:56:10.394 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015102599999999999 2024-08-17T17:56:10.395 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.395 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.395 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.395 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.738239+0000", 2024-08-17T17:56:10.395 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018882638 2024-08-17T17:56:10.395 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.395 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.395 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.395 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.772530+0000", 2024-08-17T17:56:10.395 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.034291508999999998 2024-08-17T17:56:10.395 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.395 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.395 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.396 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.772581+0000", 2024-08-17T17:56:10.396 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0745000000000003e-05 2024-08-17T17:56:10.396 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.396 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.396 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.396 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.772702+0000", 2024-08-17T17:56:10.396 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012133600000000001 2024-08-17T17:56:10.396 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.396 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.396 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.396 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.776624+0000", 2024-08-17T17:56:10.396 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039217740000000003 2024-08-17T17:56:10.396 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.396 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.397 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.397 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.397 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.397 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109286 45.8 45:1465a8ca:::benchmark_data_smithi130_172551_object4479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.397 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.517695+0000", 2024-08-17T17:56:10.397 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 401.79369912700002, 2024-08-17T17:56:10.397 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26377572999999999, 2024-08-17T17:56:10.397 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.397 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.397 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.397 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.397 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.397 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.397 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109286 2024-08-17T17:56:10.398 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.398 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.398 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.398 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.398 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517695+0000", 2024-08-17T17:56:10.398 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.398 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.398 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.398 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.398 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517695+0000", 2024-08-17T17:56:10.398 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.398 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.398 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.399 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.399 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517697+0000", 2024-08-17T17:56:10.399 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7290000000000001e-06 2024-08-17T17:56:10.399 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.399 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.399 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.399 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517702+0000", 2024-08-17T17:56:10.399 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0780000000000003e-06 2024-08-17T17:56:10.399 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.399 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.399 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.399 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517710+0000", 2024-08-17T17:56:10.399 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7439999999999996e-06 2024-08-17T17:56:10.399 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.400 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.400 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.400 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517717+0000", 2024-08-17T17:56:10.400 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0769999999999996e-06 2024-08-17T17:56:10.400 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.400 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.400 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.400 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.574731+0000", 2024-08-17T17:56:10.400 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.057014505 2024-08-17T17:56:10.400 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.400 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.400 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.400 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.575221+0000", 2024-08-17T17:56:10.400 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00049007399999999996 2024-08-17T17:56:10.401 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.401 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.401 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.401 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.738278+0000", 2024-08-17T17:56:10.401 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16305646800000001 2024-08-17T17:56:10.401 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.401 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.401 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.401 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.738388+0000", 2024-08-17T17:56:10.401 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010992700000000001 2024-08-17T17:56:10.401 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.401 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.402 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.402 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.738536+0000", 2024-08-17T17:56:10.402 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014820799999999999 2024-08-17T17:56:10.402 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.402 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.402 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.402 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.753488+0000", 2024-08-17T17:56:10.402 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014951951999999999 2024-08-17T17:56:10.402 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.402 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.402 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.402 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.776680+0000", 2024-08-17T17:56:10.402 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023192260999999999 2024-08-17T17:56:10.403 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.403 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.403 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.403 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.776719+0000", 2024-08-17T17:56:10.403 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8858e-05 2024-08-17T17:56:10.403 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.403 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.403 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.403 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.776829+0000", 2024-08-17T17:56:10.403 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011035399999999999 2024-08-17T17:56:10.403 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.403 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.403 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.404 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.781471+0000", 2024-08-17T17:56:10.404 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046414949999999998 2024-08-17T17:56:10.404 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.404 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.404 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.404 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.404 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109287 45.8 45:1ce0b8c0:::benchmark_data_smithi130_172551_object4480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.404 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.517973+0000", 2024-08-17T17:56:10.404 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 401.79342133799997, 2024-08-17T17:56:10.404 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.27116035399999999, 2024-08-17T17:56:10.404 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.404 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.404 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.405 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.405 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.405 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.405 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109287 2024-08-17T17:56:10.405 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.405 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.405 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.405 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.405 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517973+0000", 2024-08-17T17:56:10.405 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.405 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.405 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.405 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.406 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517973+0000", 2024-08-17T17:56:10.406 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.406 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.406 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.406 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.406 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517974+0000", 2024-08-17T17:56:10.406 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0780000000000001e-06 2024-08-17T17:56:10.406 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.406 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.406 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.406 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517979+0000", 2024-08-17T17:56:10.406 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.967e-06 2024-08-17T17:56:10.406 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.406 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.407 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.407 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517995+0000", 2024-08-17T17:56:10.407 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6325e-05 2024-08-17T17:56:10.407 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.407 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.407 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.407 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.518003+0000", 2024-08-17T17:56:10.407 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9559999999999997e-06 2024-08-17T17:56:10.407 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.407 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.407 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.407 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.575256+0000", 2024-08-17T17:56:10.407 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.057252589 2024-08-17T17:56:10.408 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.408 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.408 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.408 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.704722+0000", 2024-08-17T17:56:10.408 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.12946607399999999 2024-08-17T17:56:10.408 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.408 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.408 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.408 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.753525+0000", 2024-08-17T17:56:10.408 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.048803226999999998 2024-08-17T17:56:10.408 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.408 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.408 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.408 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.753627+0000", 2024-08-17T17:56:10.409 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000102113 2024-08-17T17:56:10.409 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.409 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.409 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.409 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.753767+0000", 2024-08-17T17:56:10.409 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000140106 2024-08-17T17:56:10.409 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.409 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.409 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.409 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.768534+0000", 2024-08-17T17:56:10.409 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014766359 2024-08-17T17:56:10.409 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.409 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.410 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.410 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.781525+0000", 2024-08-17T17:56:10.410 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012991669000000001 2024-08-17T17:56:10.410 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.410 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.410 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.410 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.781568+0000", 2024-08-17T17:56:10.410 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2818000000000002e-05 2024-08-17T17:56:10.410 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.410 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.410 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.410 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.781626+0000", 2024-08-17T17:56:10.410 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8424999999999997e-05 2024-08-17T17:56:10.410 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.411 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.411 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.411 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789133+0000", 2024-08-17T17:56:10.411 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0075066480000000003 2024-08-17T17:56:10.411 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.411 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.411 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.411 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.411 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.411 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109293 45.8 45:166b0a81:::benchmark_data_smithi130_172551_object4486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.411 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.533090+0000", 2024-08-17T17:56:10.411 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 401.77830388699999, 2024-08-17T17:56:10.411 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26005087900000001, 2024-08-17T17:56:10.412 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.412 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.412 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.412 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.412 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.412 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.412 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109293 2024-08-17T17:56:10.412 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.412 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.412 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.412 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.412 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533090+0000", 2024-08-17T17:56:10.412 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.412 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.413 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.413 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.413 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533090+0000", 2024-08-17T17:56:10.413 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.413 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.413 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.413 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.413 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533092+0000", 2024-08-17T17:56:10.413 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0719999999999998e-06 2024-08-17T17:56:10.413 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.413 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.413 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.413 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533100+0000", 2024-08-17T17:56:10.413 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2699999999999999e-06 2024-08-17T17:56:10.414 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.414 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.414 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.414 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533113+0000", 2024-08-17T17:56:10.414 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3903999999999999e-05 2024-08-17T17:56:10.414 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.414 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.414 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.414 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533121+0000", 2024-08-17T17:56:10.414 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9160000000000001e-06 2024-08-17T17:56:10.414 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.414 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.414 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.414 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.704763+0000", 2024-08-17T17:56:10.415 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.17164132700000001 2024-08-17T17:56:10.415 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.415 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.415 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.415 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.707453+0000", 2024-08-17T17:56:10.415 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026901220000000001 2024-08-17T17:56:10.415 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.415 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.415 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.415 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.768583+0000", 2024-08-17T17:56:10.415 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.061130048999999999 2024-08-17T17:56:10.415 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.415 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.416 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.416 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.768675+0000", 2024-08-17T17:56:10.416 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1698000000000002e-05 2024-08-17T17:56:10.416 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.416 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.416 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.416 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.768806+0000", 2024-08-17T17:56:10.416 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013118900000000001 2024-08-17T17:56:10.416 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.416 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.416 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.416 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789153+0000", 2024-08-17T17:56:10.416 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020347371999999999 2024-08-17T17:56:10.416 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.417 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.417 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.417 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789212+0000", 2024-08-17T17:56:10.417 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9163000000000003e-05 2024-08-17T17:56:10.417 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.417 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.417 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.417 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789274+0000", 2024-08-17T17:56:10.417 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1339000000000003e-05 2024-08-17T17:56:10.417 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.417 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.417 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.417 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789347+0000", 2024-08-17T17:56:10.418 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3458e-05 2024-08-17T17:56:10.418 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.418 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.418 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.418 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.793141+0000", 2024-08-17T17:56:10.418 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0037940000000000001 2024-08-17T17:56:10.418 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.418 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.418 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.418 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.418 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.418 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109294 45.8 45:16984eff:::benchmark_data_smithi130_172551_object4487:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.418 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.534050+0000", 2024-08-17T17:56:10.418 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 401.77734434899997, 2024-08-17T17:56:10.419 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.33223749499999999, 2024-08-17T17:56:10.419 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.419 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.419 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.419 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.419 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.419 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.419 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109294 2024-08-17T17:56:10.419 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.419 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.419 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.419 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.419 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534050+0000", 2024-08-17T17:56:10.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.420 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.420 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.420 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.420 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534050+0000", 2024-08-17T17:56:10.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.420 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.420 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.420 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.420 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534051+0000", 2024-08-17T17:56:10.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4789999999999999e-06 2024-08-17T17:56:10.420 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.420 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.420 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.421 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534057+0000", 2024-08-17T17:56:10.421 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5489999999999997e-06 2024-08-17T17:56:10.421 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.421 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.421 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.421 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534075+0000", 2024-08-17T17:56:10.421 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7782999999999999e-05 2024-08-17T17:56:10.421 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.421 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.421 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.421 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534083+0000", 2024-08-17T17:56:10.421 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1089999999999998e-06 2024-08-17T17:56:10.421 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.421 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.422 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.422 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.793348+0000", 2024-08-17T17:56:10.422 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.259265264 2024-08-17T17:56:10.422 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.422 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.422 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.422 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.794825+0000", 2024-08-17T17:56:10.422 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0014772260000000001 2024-08-17T17:56:10.422 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.422 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.422 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.422 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.820020+0000", 2024-08-17T17:56:10.422 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025194431 2024-08-17T17:56:10.423 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.423 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.423 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.423 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.820117+0000", 2024-08-17T17:56:10.423 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7385000000000004e-05 2024-08-17T17:56:10.423 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.423 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.423 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.423 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.820245+0000", 2024-08-17T17:56:10.423 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012771000000000001 2024-08-17T17:56:10.423 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.423 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.423 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.424 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.842202+0000", 2024-08-17T17:56:10.424 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021957300999999999 2024-08-17T17:56:10.424 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.424 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.424 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.424 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.842217+0000", 2024-08-17T17:56:10.424 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5438999999999999e-05 2024-08-17T17:56:10.424 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.424 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.424 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.424 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.862154+0000", 2024-08-17T17:56:10.424 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019936878000000002 2024-08-17T17:56:10.424 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.424 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.425 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.425 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.862230+0000", 2024-08-17T17:56:10.425 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6091e-05 2024-08-17T17:56:10.425 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.425 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.425 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.425 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.866287+0000", 2024-08-17T17:56:10.425 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0040568499999999999 2024-08-17T17:56:10.425 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.425 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.425 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.425 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.425 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.425 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109309 45.8 45:12b009b3:::benchmark_data_smithi130_172551_object4502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.426 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.570036+0000", 2024-08-17T17:56:10.426 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 401.74135799200002, 2024-08-17T17:56:10.426 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.300907009, 2024-08-17T17:56:10.426 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.426 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.426 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.426 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.426 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.426 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.426 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109309 2024-08-17T17:56:10.426 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.426 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.426 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.427 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.427 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570036+0000", 2024-08-17T17:56:10.427 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.427 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.427 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.427 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.427 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570036+0000", 2024-08-17T17:56:10.427 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.427 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.427 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.427 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.427 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570038+0000", 2024-08-17T17:56:10.427 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.688e-06 2024-08-17T17:56:10.427 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.428 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.428 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.428 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570046+0000", 2024-08-17T17:56:10.428 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3899999999999993e-06 2024-08-17T17:56:10.428 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.428 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.428 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.428 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570056+0000", 2024-08-17T17:56:10.428 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0217999999999999e-05 2024-08-17T17:56:10.428 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.428 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.428 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.428 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570064+0000", 2024-08-17T17:56:10.428 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.892e-06 2024-08-17T17:56:10.429 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.429 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.429 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.429 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.794964+0000", 2024-08-17T17:56:10.429 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22489930799999999 2024-08-17T17:56:10.429 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.429 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.429 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.429 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.796783+0000", 2024-08-17T17:56:10.429 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001819443 2024-08-17T17:56:10.429 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.429 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.429 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.430 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.836565+0000", 2024-08-17T17:56:10.430 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.039782114 2024-08-17T17:56:10.430 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.430 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.430 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.430 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.836800+0000", 2024-08-17T17:56:10.430 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00023482700000000001 2024-08-17T17:56:10.430 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.430 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.430 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.430 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.837157+0000", 2024-08-17T17:56:10.430 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00035710700000000001 2024-08-17T17:56:10.430 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.431 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.431 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.431 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.842249+0000", 2024-08-17T17:56:10.431 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005092315 2024-08-17T17:56:10.431 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.431 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.431 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.431 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.842261+0000", 2024-08-17T17:56:10.431 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.189e-05 2024-08-17T17:56:10.431 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.431 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.431 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.431 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.866457+0000", 2024-08-17T17:56:10.432 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.024195761 2024-08-17T17:56:10.432 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.432 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.432 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.432 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.866646+0000", 2024-08-17T17:56:10.432 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018857100000000001 2024-08-17T17:56:10.432 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.432 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.432 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.432 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.870943+0000", 2024-08-17T17:56:10.432 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0042974850000000002 2024-08-17T17:56:10.432 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.432 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.432 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.433 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.433 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.433 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109672 45.8 45:19e4da43:::benchmark_data_smithi130_172551_object4865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.433 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:29.496933+0000", 2024-08-17T17:56:10.433 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 400.81446058, 2024-08-17T17:56:10.433 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.221123546, 2024-08-17T17:56:10.433 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.433 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.433 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.433 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.433 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.433 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.433 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109672 2024-08-17T17:56:10.434 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.434 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.434 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.434 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.434 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496933+0000", 2024-08-17T17:56:10.434 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.434 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.434 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.434 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.434 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496933+0000", 2024-08-17T17:56:10.434 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.434 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.434 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.435 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.435 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496934+0000", 2024-08-17T17:56:10.435 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2899999999999999e-07 2024-08-17T17:56:10.435 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.435 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.435 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.435 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496936+0000", 2024-08-17T17:56:10.435 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5320000000000001e-06 2024-08-17T17:56:10.435 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.435 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.435 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.435 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496945+0000", 2024-08-17T17:56:10.435 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6160000000000002e-06 2024-08-17T17:56:10.436 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.436 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.436 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.436 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496950+0000", 2024-08-17T17:56:10.436 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4140000000000002e-06 2024-08-17T17:56:10.436 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.436 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.436 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.436 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.568388+0000", 2024-08-17T17:56:10.436 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.071437234000000002 2024-08-17T17:56:10.436 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.436 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.436 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.437 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.569153+0000", 2024-08-17T17:56:10.437 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00076482600000000003 2024-08-17T17:56:10.437 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.437 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.437 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.437 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.697807+0000", 2024-08-17T17:56:10.437 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.128654886 2024-08-17T17:56:10.437 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.437 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.437 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.437 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.697938+0000", 2024-08-17T17:56:10.437 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013016899999999999 2024-08-17T17:56:10.437 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.437 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.438 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.438 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.698100+0000", 2024-08-17T17:56:10.438 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016217099999999999 2024-08-17T17:56:10.438 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.438 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.438 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.438 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.713384+0000", 2024-08-17T17:56:10.438 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015284588 2024-08-17T17:56:10.438 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.438 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.438 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.438 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.713429+0000", 2024-08-17T17:56:10.438 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4895000000000003e-05 2024-08-17T17:56:10.439 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.439 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.439 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.439 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.713504+0000", 2024-08-17T17:56:10.439 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4994999999999996e-05 2024-08-17T17:56:10.439 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.439 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.439 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.439 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.713752+0000", 2024-08-17T17:56:10.439 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00024759199999999998 2024-08-17T17:56:10.439 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.439 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.439 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.440 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.718057+0000", 2024-08-17T17:56:10.440 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.004305199 2024-08-17T17:56:10.440 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.440 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.440 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.440 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.440 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.440 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109679 45.8 45:13e4c37d:::benchmark_data_smithi130_172551_object4872:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.440 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:29.510400+0000", 2024-08-17T17:56:10.440 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 400.80099422400002, 2024-08-17T17:56:10.440 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22616212299999999, 2024-08-17T17:56:10.440 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.440 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.441 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.441 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.441 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.441 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.441 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109679 2024-08-17T17:56:10.441 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.441 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.441 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.441 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.441 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510400+0000", 2024-08-17T17:56:10.441 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.441 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.441 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.442 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.442 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510400+0000", 2024-08-17T17:56:10.442 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.442 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.442 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.442 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.442 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510402+0000", 2024-08-17T17:56:10.442 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.736e-06 2024-08-17T17:56:10.442 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.442 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.442 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.442 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510407+0000", 2024-08-17T17:56:10.442 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2140000000000004e-06 2024-08-17T17:56:10.443 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.443 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.443 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.443 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510423+0000", 2024-08-17T17:56:10.443 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6066000000000002e-05 2024-08-17T17:56:10.443 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.443 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.443 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.443 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510430+0000", 2024-08-17T17:56:10.443 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1590000000000004e-06 2024-08-17T17:56:10.443 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.443 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.443 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.443 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.569188+0000", 2024-08-17T17:56:10.444 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.058757566999999997 2024-08-17T17:56:10.444 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.444 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.444 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.444 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.570491+0000", 2024-08-17T17:56:10.444 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001303228 2024-08-17T17:56:10.444 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.444 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.444 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.444 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.708557+0000", 2024-08-17T17:56:10.444 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.13806636799999999 2024-08-17T17:56:10.444 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.445 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.445 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.445 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.708713+0000", 2024-08-17T17:56:10.445 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015618000000000001 2024-08-17T17:56:10.445 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.445 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.445 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.445 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.708867+0000", 2024-08-17T17:56:10.445 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015330699999999999 2024-08-17T17:56:10.445 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.445 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.445 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.445 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.721271+0000", 2024-08-17T17:56:10.446 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012404178 2024-08-17T17:56:10.446 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.446 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.446 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.446 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.724727+0000", 2024-08-17T17:56:10.446 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003456083 2024-08-17T17:56:10.446 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.446 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.446 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.446 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.727154+0000", 2024-08-17T17:56:10.446 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0024270379999999999 2024-08-17T17:56:10.446 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.446 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.447 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.447 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.727215+0000", 2024-08-17T17:56:10.447 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1525999999999993e-05 2024-08-17T17:56:10.447 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.447 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.447 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.447 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.736562+0000", 2024-08-17T17:56:10.447 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0093464729999999992 2024-08-17T17:56:10.447 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.447 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.447 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.447 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.447 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.447 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109712 45.8 45:1acdc34f:::benchmark_data_smithi130_172551_object4905:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.448 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:29.603003+0000", 2024-08-17T17:56:10.448 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 400.70839150900002, 2024-08-17T17:56:10.448 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.21928672099999999, 2024-08-17T17:56:10.448 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.448 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.448 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.448 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.448 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.448 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.448 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109712 2024-08-17T17:56:10.448 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.448 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.448 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.449 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.449 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603003+0000", 2024-08-17T17:56:10.449 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.449 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.449 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.449 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.449 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603003+0000", 2024-08-17T17:56:10.449 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.449 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.449 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.449 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.449 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603005+0000", 2024-08-17T17:56:10.449 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3209999999999999e-06 2024-08-17T17:56:10.450 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.450 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.450 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.450 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603013+0000", 2024-08-17T17:56:10.450 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4610000000000008e-06 2024-08-17T17:56:10.450 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.450 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.450 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.450 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603033+0000", 2024-08-17T17:56:10.450 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9606000000000002e-05 2024-08-17T17:56:10.450 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.450 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.450 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.451 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603046+0000", 2024-08-17T17:56:10.451 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2634e-05 2024-08-17T17:56:10.451 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.451 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.451 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.451 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.718108+0000", 2024-08-17T17:56:10.451 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.115062581 2024-08-17T17:56:10.451 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.451 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.451 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.451 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.721208+0000", 2024-08-17T17:56:10.451 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0030999180000000001 2024-08-17T17:56:10.451 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.451 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.452 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.452 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.758271+0000", 2024-08-17T17:56:10.452 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.037063082999999997 2024-08-17T17:56:10.452 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.452 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.452 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.452 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.758375+0000", 2024-08-17T17:56:10.452 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000103467 2024-08-17T17:56:10.452 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.452 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.452 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.452 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.758517+0000", 2024-08-17T17:56:10.452 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014272699999999999 2024-08-17T17:56:10.453 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.453 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.453 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.453 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.772488+0000", 2024-08-17T17:56:10.453 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013970419 2024-08-17T17:56:10.453 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.453 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.453 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.453 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.788268+0000", 2024-08-17T17:56:10.453 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015780076000000001 2024-08-17T17:56:10.453 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.453 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.453 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.454 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.816948+0000", 2024-08-17T17:56:10.454 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028680039000000001 2024-08-17T17:56:10.454 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.454 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.454 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.454 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.817013+0000", 2024-08-17T17:56:10.454 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5081999999999998e-05 2024-08-17T17:56:10.454 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.454 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.454 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.454 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.822289+0000", 2024-08-17T17:56:10.454 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005276307 2024-08-17T17:56:10.454 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.455 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.455 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.455 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.455 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.455 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109714 45.8 45:16437e11:::benchmark_data_smithi130_172551_object4907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:56:10.455 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:29.605847+0000", 2024-08-17T17:56:10.455 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 400.705546825, 2024-08-17T17:56:10.455 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26878229799999998, 2024-08-17T17:56:10.455 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:56:10.455 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:56:10.455 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:56:10.455 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:56:10.455 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:56:10.456 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:56:10.456 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109714 2024-08-17T17:56:10.456 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.456 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:56:10.456 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.456 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:56:10.456 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605847+0000", 2024-08-17T17:56:10.456 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.456 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.456 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.456 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:56:10.456 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605847+0000", 2024-08-17T17:56:10.456 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:56:10.456 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.457 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.457 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:56:10.457 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605849+0000", 2024-08-17T17:56:10.457 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9209999999999999e-06 2024-08-17T17:56:10.457 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.457 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.457 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:56:10.457 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605856+0000", 2024-08-17T17:56:10.457 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6209999999999998e-06 2024-08-17T17:56:10.457 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.457 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.457 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:56:10.457 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605870+0000", 2024-08-17T17:56:10.457 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4395e-05 2024-08-17T17:56:10.458 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.458 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.458 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:56:10.458 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605879+0000", 2024-08-17T17:56:10.458 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4239999999999993e-06 2024-08-17T17:56:10.458 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.458 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.458 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.458 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.721311+0000", 2024-08-17T17:56:10.458 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.115432828 2024-08-17T17:56:10.458 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.458 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.458 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:56:10.459 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.722135+0000", 2024-08-17T17:56:10.459 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00082366000000000004 2024-08-17T17:56:10.459 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.459 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.459 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:56:10.459 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.783886+0000", 2024-08-17T17:56:10.459 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.061750939999999997 2024-08-17T17:56:10.459 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.459 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.459 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:56:10.459 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.784004+0000", 2024-08-17T17:56:10.459 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011846899999999999 2024-08-17T17:56:10.459 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.460 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.460 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:56:10.460 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.784179+0000", 2024-08-17T17:56:10.460 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017467599999999999 2024-08-17T17:56:10.460 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.460 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.460 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:56:10.460 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.791605+0000", 2024-08-17T17:56:10.460 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0074256260000000003 2024-08-17T17:56:10.460 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.460 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.460 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.460 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.791767+0000", 2024-08-17T17:56:10.460 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016228 2024-08-17T17:56:10.461 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.461 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.461 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:56:10.461 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.869906+0000", 2024-08-17T17:56:10.461 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.078138847999999997 2024-08-17T17:56:10.461 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.461 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.461 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:56:10.461 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.870304+0000", 2024-08-17T17:56:10.461 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00039816299999999999 2024-08-17T17:56:10.461 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:56:10.461 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:56:10.461 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:56:10.462 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.874630+0000", 2024-08-17T17:56:10.462 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0043254469999999996 2024-08-17T17:56:10.462 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.462 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.462 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.462 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:56:10.462 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:56:10.462 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:56:10.483 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:10.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:10.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:10.587 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:56:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:56:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:56:10.608 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:10.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:10.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:10.710 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:10.737 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:10.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:10.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:10.838 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:10.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:10.861 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:10.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:10 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/154044 objects degraded (10.281%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:10 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/154044 objects degraded (10.281%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:10.962 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:10.984 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:10.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:10.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:11.086 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:11.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:10 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/154044 objects degraded (10.281%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:11.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:11.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:11.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:11.209 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:11.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:11.232 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:11.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:11.334 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:11.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:11.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:11.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:11.457 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:11.479 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:11.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:11.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:11.553 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:56:11.580 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:11.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:11.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:11.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:11.704 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:11.725 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:11.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:11.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:11.827 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:11.861 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:11.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:11.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:11.963 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:11.985 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:11.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:11.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:12.086 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:12.089 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:11 smithi130 ceph-mon[105521]: pgmap v1882: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 159 op/s; 15837/153468 objects degraded (10.319%); 8355/153468 objects misplaced (5.444%); 685 KiB/s, 10 objects/s recovering 2024-08-17T17:56:12.089 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:11 smithi130 ceph-mon[110068]: pgmap v1882: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 159 op/s; 15837/153468 objects degraded (10.319%); 8355/153468 objects misplaced (5.444%); 685 KiB/s, 10 objects/s recovering 2024-08-17T17:56:12.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:11 smithi179 ceph-mon[93519]: pgmap v1882: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 159 op/s; 15837/153468 objects degraded (10.319%); 8355/153468 objects misplaced (5.444%); 685 KiB/s, 10 objects/s recovering 2024-08-17T17:56:12.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:12.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:12.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:12.209 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:12.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:12.231 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:12.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:12.332 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:12.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:12.354 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:12.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:12.455 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:12.476 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:12.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:12.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:12.578 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:12.603 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:12.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:12.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:12.705 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:12.731 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:12.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:12.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:12.834 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:12.836 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:56:12.855 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:12.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:12.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:12.956 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:12.977 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:12.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:12.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:13.079 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:13.099 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:13.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:13.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:13.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:13.225 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:13.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:13.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:13.327 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:13.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:13.349 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:13.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:13.450 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:13.471 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:13.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:13.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:13.506 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:13.574 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:13 smithi179 ceph-mon[93519]: pgmap v1883: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 191 op/s; 15837/152316 objects degraded (10.397%); 8322/152316 objects misplaced (5.464%); 685 KiB/s, 10 objects/s recovering 2024-08-17T17:56:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:13.596 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:13.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:13.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:13.639 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:13.639 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:13.639 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:13 smithi130 ceph-mon[105521]: pgmap v1883: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 191 op/s; 15837/152316 objects degraded (10.397%); 8322/152316 objects misplaced (5.464%); 685 KiB/s, 10 objects/s recovering 2024-08-17T17:56:13.639 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:13.639 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:13.639 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:13.639 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:13.640 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:13 smithi130 ceph-mon[110068]: pgmap v1883: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 191 op/s; 15837/152316 objects degraded (10.397%); 8322/152316 objects misplaced (5.464%); 685 KiB/s, 10 objects/s recovering 2024-08-17T17:56:13.640 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:13.640 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:13.698 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:13.726 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:13.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:13.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:13.799 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:13.799 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:13.810 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:56:13.811 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:56:13.828 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:13.839 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:56:13.839 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:56:13.853 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:13.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:13.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:13.858 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:56:13.858 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:56:13.877 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:56:13.877 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:56:13.894 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:56:13.895 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:56:13.910 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:56:13.911 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:56:13.926 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:56:13.926 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:56:13.942 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:56:13.942 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:56:13.955 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:13.976 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:13.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:13.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:14.078 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:14.100 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:14.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:14.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:14.202 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:14.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:14.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:14.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:14.326 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:14.351 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:14.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:14.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:14.453 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:14.482 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:14.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:14.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:14.584 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:14 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:56:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:14 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:56:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:14 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3567364633' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:56:14.603 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:14 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:56:14.603 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:14 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:56:14.603 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:14.603 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:14 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3567364633' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:56:14.603 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:14 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:56:14.603 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:14 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:56:14.603 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:14.604 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:14 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3567364633' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:56:14.608 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:14.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:14.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:14.710 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:14.731 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:14.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:14.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:14.833 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:14.854 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:14.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:14.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:14.955 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:14.976 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:14.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:14.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:15.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:15.099 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:15.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:15.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:15.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:15.223 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:15.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:15.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:15.325 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:15.346 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:15.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:15.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:15.448 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:15.469 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:15.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:15.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:15.571 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:15.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:15 smithi179 ceph-mon[93519]: pgmap v1884: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 161 op/s; 15837/151470 objects degraded (10.456%); 8322/151470 objects misplaced (5.494%); 685 KiB/s, 10 objects/s recovering 2024-08-17T17:56:15.596 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:15.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:15.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:15.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:15 smithi130 ceph-mon[105521]: pgmap v1884: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 161 op/s; 15837/151470 objects degraded (10.456%); 8322/151470 objects misplaced (5.494%); 685 KiB/s, 10 objects/s recovering 2024-08-17T17:56:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:15 smithi130 ceph-mon[110068]: pgmap v1884: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 161 op/s; 15837/151470 objects degraded (10.456%); 8322/151470 objects misplaced (5.494%); 685 KiB/s, 10 objects/s recovering 2024-08-17T17:56:15.698 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:15.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:15.720 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:15.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:15.821 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:15.842 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:15.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:15.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:15.944 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:15.965 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:15.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:15.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:16.067 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:16.092 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:16.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:16.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:16.194 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:16.215 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:16.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:16.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:16.317 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:16.339 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:16.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:16.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:16.441 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:16.463 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:16.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:16.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:16.565 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:16.588 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:16.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:16.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:16.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/151470 objects degraded (10.456%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:16.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/151470 objects degraded (10.456%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:16.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/151470 objects degraded (10.456%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:16.691 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:16.712 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:16.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:16.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:16.814 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:16.834 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:16.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:16.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:16.939 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:16.962 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:16.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:17.064 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:17.086 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:17.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:17.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:17.188 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:17.215 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:17.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:17.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:17.264 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:56:17.317 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:17.338 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:17.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:17.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:17.440 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:17.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:17.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:17.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:17.532 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:17 smithi130 ceph-mon[105521]: pgmap v1885: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 15837/150801 objects degraded (10.502%); 8152/150801 objects misplaced (5.406%); 697 KiB/s, 10 objects/s recovering 2024-08-17T17:56:17.532 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:17 smithi130 ceph-mon[110068]: pgmap v1885: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 15837/150801 objects degraded (10.502%); 8152/150801 objects misplaced (5.406%); 697 KiB/s, 10 objects/s recovering 2024-08-17T17:56:17.563 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:17.584 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:17.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:17.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:17 smithi179 ceph-mon[93519]: pgmap v1885: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.1 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 160 op/s; 15837/150801 objects degraded (10.502%); 8152/150801 objects misplaced (5.406%); 697 KiB/s, 10 objects/s recovering 2024-08-17T17:56:17.685 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:17.711 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:17.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:17.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:17.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:17.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:17.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:17.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:17.937 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:17.958 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:17.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:17.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:18.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:18.083 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:18.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:18.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:18.185 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:18.206 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:18.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:18.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:18.274 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:18.308 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:18.329 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:18.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:18.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:18.337 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:56:18.431 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:18.452 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:18.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:18.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:18.554 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:18.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:18.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:18.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:18.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1678417043' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:56:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:18 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:56:18.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1678417043' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:56:18.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:18 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:56:18.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1678417043' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:56:18.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:18 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:56:18.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:18.698 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:18.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:18.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:18.800 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:18.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:18.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:18.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:18.923 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:18.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:18.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:18.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:19.045 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:19.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:19.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:19.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:19.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:19.191 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:19.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:19.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:19.293 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:19.314 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:19.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:19.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:19.416 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:19.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:19 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:56:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:19 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:56:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:19 smithi130 ceph-mon[110068]: osdmap e1235: 8 total, 8 up, 6 in 2024-08-17T17:56:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:19 smithi130 ceph-mon[110068]: pgmap v1887: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 15837/149625 objects degraded (10.584%); 8152/149625 objects misplaced (5.448%); 697 KiB/s, 10 objects/s recovering 2024-08-17T17:56:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:19 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:56:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:19 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:56:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:19 smithi130 ceph-mon[105521]: osdmap e1235: 8 total, 8 up, 6 in 2024-08-17T17:56:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:19 smithi130 ceph-mon[105521]: pgmap v1887: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 15837/149625 objects degraded (10.584%); 8152/149625 objects misplaced (5.448%); 697 KiB/s, 10 objects/s recovering 2024-08-17T17:56:19.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:19.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:19.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:19.540 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:19.563 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:19.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:19.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:19 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:56:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:19 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:56:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:19 smithi179 ceph-mon[93519]: osdmap e1235: 8 total, 8 up, 6 in 2024-08-17T17:56:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:19 smithi179 ceph-mon[93519]: pgmap v1887: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 147 op/s; 15837/149625 objects degraded (10.584%); 8152/149625 objects misplaced (5.448%); 697 KiB/s, 10 objects/s recovering 2024-08-17T17:56:19.665 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:19.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:19.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:19.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:19.795 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:19.818 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:19.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:19.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:19.920 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:19.941 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:19.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:19.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:20.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:20.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:20.065 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:20.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:20.166 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:20.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:56:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:56:19] "GET /metrics HTTP/1.1" 200 36815 "" "Prometheus/2.51.0" 2024-08-17T17:56:20.193 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:20.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:20.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:20.295 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:20.317 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:20.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:20.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:20.418 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:20.444 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:20.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:20.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:20.546 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:20.567 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:20.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:20.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:20.572 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:56:20.573 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:56:20.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:56:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:56:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:56:20.669 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:20.697 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:20.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:20.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:20.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:20.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:20.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:20.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:20.923 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:20.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:20.945 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:20.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:21.046 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:21.057 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:20 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/149625 objects degraded (10.584%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:21.057 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:20 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/149625 objects degraded (10.584%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:21.068 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:21.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:21.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:21.070 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:56:21.070 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:56:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:20 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/149625 objects degraded (10.584%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:21.170 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:21.195 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:21.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:21.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:21.297 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:21.318 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:21.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:21.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:21.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:21.447 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:21.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:21.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:21.549 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:21.570 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:21.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:21.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:21.672 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:21.693 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:21.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:21.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:21.795 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:21.823 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:21.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:21.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:21.925 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:21.947 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:21.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:21.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:22.049 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:22.070 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:22.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:22.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:22.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:21 smithi179 ceph-mon[93519]: from='client.45809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:56:22.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:21 smithi179 ceph-mon[93519]: pgmap v1888: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 150 op/s; 15837/148962 objects degraded (10.632%); 8037/148962 objects misplaced (5.395%); 717 KiB/s, 11 objects/s recovering 2024-08-17T17:56:22.142 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:21 smithi130 ceph-mon[105521]: from='client.45809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:56:22.142 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:21 smithi130 ceph-mon[105521]: pgmap v1888: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 150 op/s; 15837/148962 objects degraded (10.632%); 8037/148962 objects misplaced (5.395%); 717 KiB/s, 11 objects/s recovering 2024-08-17T17:56:22.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:21 smithi130 ceph-mon[110068]: from='client.45809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:56:22.142 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:21 smithi130 ceph-mon[110068]: pgmap v1888: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 150 op/s; 15837/148962 objects degraded (10.632%); 8037/148962 objects misplaced (5.395%); 717 KiB/s, 11 objects/s recovering 2024-08-17T17:56:22.172 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:22.195 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:22.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:22.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:22.296 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:22.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:22.319 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:22.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:22.420 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:22.441 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:22.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:22.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:22.543 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:22.566 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:22.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:22.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:22.671 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:22.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:22.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:22.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:22.727 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1887,"stamp":"2024-08-17T17:56:19.085171+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3267434469,"num_objects":49875,"num_object_clones":0,"num_object_copies":149625,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":15837,"num_objects_misplaced":8152,"num_objects_unfound":0,"num_objects_dirty":49875,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":160039,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36598,"num_bytes_recovered":2219908109,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108742,"ondisk_log_size":108742,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":10153060,"kb_used_data":8717748,"kb_used_omap":295,"kb_used_meta":1434904,"kb_avail":552317852,"statfs":{"total":575970213888,"available":565573480448,"internally_reserved":0,"allocated":8926973952,"data_stored":8897163265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1469342159},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,0,2,3,1,4,5,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-96534528,"num_objects":-1473,"num_object_clones":0,"num_object_copies":-4419,"num_objects_missing_on_primary":-109,"num_objects_missing":-109,"num_objects_degraded":0,"num_objects_misplaced":-314,"num_objects_unfound":0,"num_objects_dirty":-1473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1473,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":109,"num_bytes_recovered":7143424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003081"},"pg_stats":[{"pgid":"45.f","version":"1234'10007","reported_seq":20555,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:15.647480+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:56:15.647480+0000","last_peered":"2024-08-17T17:56:15.647480+0000","last_clean":"2024-08-17T17:56:15.647480+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:56:15.647480+0000","last_undegraded":"2024-08-17T17:56:15.647480+0000","last_fullsized":"2024-08-17T17:56:15.647480+0000","mapping_epoch":1194,"log_start":"1174'3131","ondisk_log_start":"1174'3131","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6876,"log_dups_size":0,"ondisk_log_size":6876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206897152,"num_objects":3157,"num_object_clones":0,"num_object_copies":9471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10007,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1234'9923","reported_seq":20397,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:17.726975+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:56:17.726975+0000","last_peered":"2024-08-17T17:56:17.726975+0000","last_clean":"2024-08-17T17:56:17.726975+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:56:17.726975+0000","last_undegraded":"2024-08-17T17:56:17.726975+0000","last_fullsized":"2024-08-17T17:56:17.726975+0000","mapping_epoch":1211,"log_start":"1171'2932","ondisk_log_start":"1171'2932","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6991,"log_dups_size":0,"ondisk_log_size":6991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201261056,"num_objects":3071,"num_object_clones":0,"num_object_copies":9213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9923,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1235'10239","reported_seq":21096,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.676244+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:56:18.676244+0000","last_peered":"2024-08-17T17:56:18.676244+0000","last_clean":"2024-08-17T17:56:18.676244+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:56:18.676244+0000","last_undegraded":"2024-08-17T17:56:18.676244+0000","last_fullsized":"2024-08-17T17:56:18.676244+0000","mapping_epoch":1205,"log_start":"1175'4268","ondisk_log_start":"1175'4268","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5971,"log_dups_size":0,"ondisk_log_size":5971,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205651992,"num_objects":3139,"num_object_clones":0,"num_object_copies":9417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3139,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10239,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1234'9947","reported_seq":10022,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:17.217241+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:56:17.217241+0000","last_peered":"2024-08-17T17:56:17.217241+0000","last_clean":"2024-08-17T17:56:17.217241+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:56:17.217241+0000","last_undegraded":"2024-08-17T17:56:17.217241+0000","last_fullsized":"2024-08-17T17:56:17.217241+0000","mapping_epoch":1167,"log_start":"1174'3189","ondisk_log_start":"1174'3189","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6758,"log_dups_size":0,"ondisk_log_size":6758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200081408,"num_objects":3053,"num_object_clones":0,"num_object_copies":9159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9947,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1234'10003","reported_seq":15264,"reported_epoch":1234,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:16.505599+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:56:16.505599+0000","last_peered":"2024-08-17T17:56:16.505599+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:56:16.505599+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1175'4026","ondisk_log_start":"1175'4026","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5977,"log_dups_size":0,"ondisk_log_size":5977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203882496,"num_objects":3111,"num_object_clones":0,"num_object_copies":9333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":3111,"num_objects_unfound":0,"num_objects_dirty":3111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10003,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":2593}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1234'10075","reported_seq":10150,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:17.640262+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:56:17.640262+0000","last_peered":"2024-08-17T17:56:17.640262+0000","last_clean":"2024-08-17T17:56:17.640262+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:56:17.640262+0000","last_undegraded":"2024-08-17T17:56:17.640262+0000","last_fullsized":"2024-08-17T17:56:17.640262+0000","mapping_epoch":1167,"log_start":"1174'3377","ondisk_log_start":"1174'3377","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6698,"log_dups_size":0,"ondisk_log_size":6698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197853184,"num_objects":3019,"num_object_clones":0,"num_object_copies":9057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10075,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1234'10066","reported_seq":10141,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:17.337743+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:56:17.337743+0000","last_peered":"2024-08-17T17:56:17.337743+0000","last_clean":"2024-08-17T17:56:17.337743+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:56:17.337743+0000","last_undegraded":"2024-08-17T17:56:17.337743+0000","last_fullsized":"2024-08-17T17:56:17.337743+0000","mapping_epoch":1167,"log_start":"1174'3369","ondisk_log_start":"1174'3369","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207093760,"num_objects":3160,"num_object_clones":0,"num_object_copies":9480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10066,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1234'9968","reported_seq":19447,"reported_epoch":1234,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:16.912299+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:56:16.912299+0000","last_peered":"2024-08-17T17:56:16.912299+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:56:16.912299+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1175'4075","ondisk_log_start":"1175'4075","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5893,"log_dups_size":0,"ondisk_log_size":5893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198049792,"num_objects":3022,"num_object_clones":0,"num_object_copies":9066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9968,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3108,"num_bytes_recovered":203685888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":1702},{"shards":"1,7","objects":955}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1234'9916","reported_seq":10001,"reported_epoch":1234,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:17.280709+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:56:17.280709+0000","last_peered":"2024-08-17T17:56:17.280709+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:56:17.280709+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1174'3177","ondisk_log_start":"1174'3177","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6739,"log_dups_size":0,"ondisk_log_size":6739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207093760,"num_objects":3160,"num_object_clones":0,"num_object_copies":9480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9916,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":3160},{"shards":"1,6","objects":-2673}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1235'10016","reported_seq":13113,"reported_epoch":1235,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:18.733738+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:56:18.733738+0000","last_peered":"2024-08-17T17:56:18.733738+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:56:18.733738+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1175'4106","ondisk_log_start":"1175'4106","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5910,"log_dups_size":0,"ondisk_log_size":5910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206962688,"num_objects":3158,"num_object_clones":0,"num_object_copies":9474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10016,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":1597}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1234'9869","reported_seq":10019,"reported_epoch":1234,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:13.784754+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:56:13.784754+0000","last_peered":"2024-08-17T17:56:13.784754+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:56:13.784754+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1170'700","ondisk_log_start":"1170'700","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9169,"log_dups_size":0,"ondisk_log_size":9169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210042880,"num_objects":3205,"num_object_clones":0,"num_object_copies":9615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9869,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-1948}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1234'9897","reported_seq":16108,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:13.292399+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:56:13.292399+0000","last_peered":"2024-08-17T17:56:13.292399+0000","last_clean":"2024-08-17T17:56:13.292399+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:56:13.292399+0000","last_undegraded":"2024-08-17T17:56:13.292399+0000","last_fullsized":"2024-08-17T17:56:13.292399+0000","mapping_epoch":1183,"log_start":"1170'800","ondisk_log_start":"1170'800","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9097,"log_dups_size":0,"ondisk_log_size":9097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206372864,"num_objects":3149,"num_object_clones":0,"num_object_copies":9447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9897,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1235'10002","reported_seq":10153,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.333286+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:56:18.333286+0000","last_peered":"2024-08-17T17:56:18.333286+0000","last_clean":"2024-08-17T17:56:18.333286+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:56:18.333286+0000","last_undegraded":"2024-08-17T17:56:18.333286+0000","last_fullsized":"2024-08-17T17:56:18.333286+0000","mapping_epoch":1167,"log_start":"1175'4091","ondisk_log_start":"1175'4091","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5911,"log_dups_size":0,"ondisk_log_size":5911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206962688,"num_objects":3158,"num_object_clones":0,"num_object_copies":9474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10002,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1234'9887","reported_seq":9962,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:17.534796+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:56:17.534796+0000","last_peered":"2024-08-17T17:56:17.534796+0000","last_clean":"2024-08-17T17:56:17.534796+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:56:17.534796+0000","last_undegraded":"2024-08-17T17:56:17.534796+0000","last_fullsized":"2024-08-17T17:56:17.534796+0000","mapping_epoch":1167,"log_start":"1171'2825","ondisk_log_start":"1171'2825","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7062,"log_dups_size":0,"ondisk_log_size":7062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199688192,"num_objects":3047,"num_object_clones":0,"num_object_copies":9141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9887,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1234'10159","reported_seq":24505,"reported_epoch":1234,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:56:15.696791+0000","last_change":"2024-08-17T17:55:37.941989+0000","last_active":"2024-08-17T17:56:15.696791+0000","last_peered":"2024-08-17T17:56:15.696791+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:56:15.696791+0000","last_undegraded":"2024-08-17T17:56:15.696791+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1174'3291","ondisk_log_start":"1174'3291","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6868,"log_dups_size":0,"ondisk_log_size":6868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210042880,"num_objects":3205,"num_object_clones":0,"num_object_copies":9615,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":0,"num_objects_misplaced":5041,"num_objects_unfound":0,"num_objects_dirty":3205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10159,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5583,"num_bytes_recovered":365887488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":13},{"shards":"0,4,6","objects":1356},{"shards":"4,5,6","objects":1836}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1235'9787","reported_seq":23124,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.678678+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:56:18.678678+0000","last_peered":"2024-08-17T17:56:18.678678+0000","last_clean":"2024-08-17T17:56:18.678678+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:56:18.678678+0000","last_undegraded":"2024-08-17T17:56:18.678678+0000","last_fullsized":"2024-08-17T17:56:18.678678+0000","mapping_epoch":1196,"log_start":"1175'3869","ondisk_log_start":"1175'3869","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198901760,"num_objects":3035,"num_object_clones":0,"num_object_copies":9105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9787,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1925,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.201060+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:56:08.201060+0000","last_peered":"2024-08-17T17:56:08.201060+0000","last_clean":"2024-08-17T17:56:08.201060+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:56:08.201060+0000","last_undegraded":"2024-08-17T17:56:08.201060+0000","last_fullsized":"2024-08-17T17:56:08.201060+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":511,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789051+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:55:54.789051+0000","last_peered":"2024-08-17T17:55:54.789051+0000","last_clean":"2024-08-17T17:55:54.789051+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:55:54.789051+0000","last_undegraded":"2024-08-17T17:55:54.789051+0000","last_fullsized":"2024-08-17T17:55:54.789051+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1847,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.333277+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:56:18.333277+0000","last_peered":"2024-08-17T17:56:18.333277+0000","last_clean":"2024-08-17T17:56:18.333277+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:56:18.333277+0000","last_undegraded":"2024-08-17T17:56:18.333277+0000","last_fullsized":"2024-08-17T17:56:18.333277+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1871,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.331777+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:56:18.331777+0000","last_peered":"2024-08-17T17:56:18.331777+0000","last_clean":"2024-08-17T17:56:18.331777+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:56:18.331777+0000","last_undegraded":"2024-08-17T17:56:18.331777+0000","last_fullsized":"2024-08-17T17:56:18.331777+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":73,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789590+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:55:54.789590+0000","last_peered":"2024-08-17T17:55:54.789590+0000","last_clean":"2024-08-17T17:55:54.789590+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:55:54.789590+0000","last_undegraded":"2024-08-17T17:55:54.789590+0000","last_fullsized":"2024-08-17T17:55:54.789590+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1264,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.332458+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:56:18.332458+0000","last_peered":"2024-08-17T17:56:18.332458+0000","last_clean":"2024-08-17T17:56:18.332458+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:56:18.332458+0000","last_undegraded":"2024-08-17T17:56:18.332458+0000","last_fullsized":"2024-08-17T17:56:18.332458+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":120,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.296110+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:56:08.296110+0000","last_peered":"2024-08-17T17:56:08.296110+0000","last_clean":"2024-08-17T17:56:08.296110+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:56:08.296110+0000","last_undegraded":"2024-08-17T17:56:08.296110+0000","last_fullsized":"2024-08-17T17:56:08.296110+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":642,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.333337+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:56:18.333337+0000","last_peered":"2024-08-17T17:56:18.333337+0000","last_clean":"2024-08-17T17:56:18.333337+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:56:18.333337+0000","last_undegraded":"2024-08-17T17:56:18.333337+0000","last_fullsized":"2024-08-17T17:56:18.333337+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1884,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.200834+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:56:08.200834+0000","last_peered":"2024-08-17T17:56:08.200834+0000","last_clean":"2024-08-17T17:56:08.200834+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:56:08.200834+0000","last_undegraded":"2024-08-17T17:56:08.200834+0000","last_fullsized":"2024-08-17T17:56:08.200834+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1886,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.160618+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:56:08.160618+0000","last_peered":"2024-08-17T17:56:08.160618+0000","last_clean":"2024-08-17T17:56:08.160618+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:56:08.160618+0000","last_undegraded":"2024-08-17T17:56:08.160618+0000","last_fullsized":"2024-08-17T17:56:08.160618+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":105,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.295618+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:56:08.295618+0000","last_peered":"2024-08-17T17:56:08.295618+0000","last_clean":"2024-08-17T17:56:08.295618+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:56:08.295618+0000","last_undegraded":"2024-08-17T17:56:08.295618+0000","last_fullsized":"2024-08-17T17:56:08.295618+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1859,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.333141+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:56:18.333141+0000","last_peered":"2024-08-17T17:56:18.333141+0000","last_clean":"2024-08-17T17:56:18.333141+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:56:18.333141+0000","last_undegraded":"2024-08-17T17:56:18.333141+0000","last_fullsized":"2024-08-17T17:56:18.333141+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1912,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.295552+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:56:08.295552+0000","last_peered":"2024-08-17T17:56:08.295552+0000","last_clean":"2024-08-17T17:56:08.295552+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:56:08.295552+0000","last_undegraded":"2024-08-17T17:56:08.295552+0000","last_fullsized":"2024-08-17T17:56:08.295552+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1906,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.201147+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:56:08.201147+0000","last_peered":"2024-08-17T17:56:08.201147+0000","last_clean":"2024-08-17T17:56:08.201147+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:56:08.201147+0000","last_undegraded":"2024-08-17T17:56:08.201147+0000","last_fullsized":"2024-08-17T17:56:08.201147+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1882,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.331888+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:56:18.331888+0000","last_peered":"2024-08-17T17:56:18.331888+0000","last_clean":"2024-08-17T17:56:18.331888+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:56:18.331888+0000","last_undegraded":"2024-08-17T17:56:18.331888+0000","last_fullsized":"2024-08-17T17:56:18.331888+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1208,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789644+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:55:54.789644+0000","last_peered":"2024-08-17T17:55:54.789644+0000","last_clean":"2024-08-17T17:55:54.789644+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:55:54.789644+0000","last_undegraded":"2024-08-17T17:55:54.789644+0000","last_fullsized":"2024-08-17T17:55:54.789644+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1853,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.331708+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:56:18.331708+0000","last_peered":"2024-08-17T17:56:18.331708+0000","last_clean":"2024-08-17T17:56:18.331708+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:56:18.331708+0000","last_undegraded":"2024-08-17T17:56:18.331708+0000","last_fullsized":"2024-08-17T17:56:18.331708+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1901,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.201147+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:56:08.201147+0000","last_peered":"2024-08-17T17:56:08.201147+0000","last_clean":"2024-08-17T17:56:08.201147+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:56:08.201147+0000","last_undegraded":"2024-08-17T17:56:08.201147+0000","last_fullsized":"2024-08-17T17:56:08.201147+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1809,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.331622+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:56:18.331622+0000","last_peered":"2024-08-17T17:56:18.331622+0000","last_clean":"2024-08-17T17:56:18.331622+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:56:18.331622+0000","last_undegraded":"2024-08-17T17:56:18.331622+0000","last_fullsized":"2024-08-17T17:56:18.331622+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":527,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.799509+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:55:54.799509+0000","last_peered":"2024-08-17T17:55:54.799509+0000","last_clean":"2024-08-17T17:55:54.799509+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:55:54.799509+0000","last_undegraded":"2024-08-17T17:55:54.799509+0000","last_fullsized":"2024-08-17T17:55:54.799509+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1204,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.799373+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:55:54.799373+0000","last_peered":"2024-08-17T17:55:54.799373+0000","last_clean":"2024-08-17T17:55:54.799373+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:55:54.799373+0000","last_undegraded":"2024-08-17T17:55:54.799373+0000","last_fullsized":"2024-08-17T17:55:54.799373+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1905,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.296013+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:56:08.296013+0000","last_peered":"2024-08-17T17:56:08.296013+0000","last_clean":"2024-08-17T17:56:08.296013+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:56:08.296013+0000","last_undegraded":"2024-08-17T17:56:08.296013+0000","last_fullsized":"2024-08-17T17:56:08.296013+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":365,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.799391+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:55:54.799391+0000","last_peered":"2024-08-17T17:55:54.799391+0000","last_clean":"2024-08-17T17:55:54.799391+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:55:54.799391+0000","last_undegraded":"2024-08-17T17:55:54.799391+0000","last_fullsized":"2024-08-17T17:55:54.799391+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":511,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789356+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:55:54.789356+0000","last_peered":"2024-08-17T17:55:54.789356+0000","last_clean":"2024-08-17T17:55:54.789356+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:55:54.789356+0000","last_undegraded":"2024-08-17T17:55:54.789356+0000","last_fullsized":"2024-08-17T17:55:54.789356+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1189,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789452+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:55:54.789452+0000","last_peered":"2024-08-17T17:55:54.789452+0000","last_clean":"2024-08-17T17:55:54.789452+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:55:54.789452+0000","last_undegraded":"2024-08-17T17:55:54.789452+0000","last_fullsized":"2024-08-17T17:55:54.789452+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1197,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789684+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:55:54.789684+0000","last_peered":"2024-08-17T17:55:54.789684+0000","last_clean":"2024-08-17T17:55:54.789684+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:55:54.789684+0000","last_undegraded":"2024-08-17T17:55:54.789684+0000","last_fullsized":"2024-08-17T17:55:54.789684+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1953,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.200878+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:56:08.200878+0000","last_peered":"2024-08-17T17:56:08.200878+0000","last_clean":"2024-08-17T17:56:08.200878+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:56:08.200878+0000","last_undegraded":"2024-08-17T17:56:08.200878+0000","last_fullsized":"2024-08-17T17:56:08.200878+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1839,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.331817+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:56:18.331817+0000","last_peered":"2024-08-17T17:56:18.331817+0000","last_clean":"2024-08-17T17:56:18.331817+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:56:18.331817+0000","last_undegraded":"2024-08-17T17:56:18.331817+0000","last_fullsized":"2024-08-17T17:56:18.331817+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1911,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.295949+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:56:08.295949+0000","last_peered":"2024-08-17T17:56:08.295949+0000","last_clean":"2024-08-17T17:56:08.295949+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:56:08.295949+0000","last_undegraded":"2024-08-17T17:56:08.295949+0000","last_fullsized":"2024-08-17T17:56:08.295949+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1841,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.333115+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:56:18.333115+0000","last_peered":"2024-08-17T17:56:18.333115+0000","last_clean":"2024-08-17T17:56:18.333115+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:56:18.333115+0000","last_undegraded":"2024-08-17T17:56:18.333115+0000","last_fullsized":"2024-08-17T17:56:18.333115+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1871,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.200927+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:56:08.200927+0000","last_peered":"2024-08-17T17:56:08.200927+0000","last_clean":"2024-08-17T17:56:08.200927+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:56:08.200927+0000","last_undegraded":"2024-08-17T17:56:08.200927+0000","last_fullsized":"2024-08-17T17:56:08.200927+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1224,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.799475+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:55:54.799475+0000","last_peered":"2024-08-17T17:55:54.799475+0000","last_clean":"2024-08-17T17:55:54.799475+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:55:54.799475+0000","last_undegraded":"2024-08-17T17:55:54.799475+0000","last_fullsized":"2024-08-17T17:55:54.799475+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1918,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.296280+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:56:08.296280+0000","last_peered":"2024-08-17T17:56:08.296280+0000","last_clean":"2024-08-17T17:56:08.296280+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:56:08.296280+0000","last_undegraded":"2024-08-17T17:56:08.296280+0000","last_fullsized":"2024-08-17T17:56:08.296280+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":105,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.295480+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:56:08.295480+0000","last_peered":"2024-08-17T17:56:08.295480+0000","last_clean":"2024-08-17T17:56:08.295480+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:56:08.295480+0000","last_undegraded":"2024-08-17T17:56:08.295480+0000","last_fullsized":"2024-08-17T17:56:08.295480+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1937,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.296200+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:56:08.296200+0000","last_peered":"2024-08-17T17:56:08.296200+0000","last_clean":"2024-08-17T17:56:08.296200+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:56:08.296200+0000","last_undegraded":"2024-08-17T17:56:08.296200+0000","last_fullsized":"2024-08-17T17:56:08.296200+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":675,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.332851+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:56:18.332851+0000","last_peered":"2024-08-17T17:56:18.332851+0000","last_clean":"2024-08-17T17:56:18.332851+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:56:18.332851+0000","last_undegraded":"2024-08-17T17:56:18.332851+0000","last_fullsized":"2024-08-17T17:56:18.332851+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1918,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.296358+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:56:08.296358+0000","last_peered":"2024-08-17T17:56:08.296358+0000","last_clean":"2024-08-17T17:56:08.296358+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:56:08.296358+0000","last_undegraded":"2024-08-17T17:56:08.296358+0000","last_fullsized":"2024-08-17T17:56:08.296358+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1246,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.200677+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:56:08.200677+0000","last_peered":"2024-08-17T17:56:08.200677+0000","last_clean":"2024-08-17T17:56:08.200677+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:56:08.200677+0000","last_undegraded":"2024-08-17T17:56:08.200677+0000","last_fullsized":"2024-08-17T17:56:08.200677+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":164,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.295750+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:56:08.295750+0000","last_peered":"2024-08-17T17:56:08.295750+0000","last_clean":"2024-08-17T17:56:08.295750+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:56:08.295750+0000","last_undegraded":"2024-08-17T17:56:08.295750+0000","last_fullsized":"2024-08-17T17:56:08.295750+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1928,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.297754+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:56:08.297754+0000","last_peered":"2024-08-17T17:56:08.297754+0000","last_clean":"2024-08-17T17:56:08.297754+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:56:08.297754+0000","last_undegraded":"2024-08-17T17:56:08.297754+0000","last_fullsized":"2024-08-17T17:56:08.297754+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1906,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.295404+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:56:08.295404+0000","last_peered":"2024-08-17T17:56:08.295404+0000","last_clean":"2024-08-17T17:56:08.295404+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:56:08.295404+0000","last_undegraded":"2024-08-17T17:56:08.295404+0000","last_fullsized":"2024-08-17T17:56:08.295404+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1927,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.160869+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:56:08.160869+0000","last_peered":"2024-08-17T17:56:08.160869+0000","last_clean":"2024-08-17T17:56:08.160869+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:56:08.160869+0000","last_undegraded":"2024-08-17T17:56:08.160869+0000","last_fullsized":"2024-08-17T17:56:08.160869+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1932,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.201244+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:56:08.201244+0000","last_peered":"2024-08-17T17:56:08.201244+0000","last_clean":"2024-08-17T17:56:08.201244+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:56:08.201244+0000","last_undegraded":"2024-08-17T17:56:08.201244+0000","last_fullsized":"2024-08-17T17:56:08.201244+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1912,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.264629+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:56:08.264629+0000","last_peered":"2024-08-17T17:56:08.264629+0000","last_clean":"2024-08-17T17:56:08.264629+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:56:08.264629+0000","last_undegraded":"2024-08-17T17:56:08.264629+0000","last_fullsized":"2024-08-17T17:56:08.264629+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1231,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.799206+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:55:54.799206+0000","last_peered":"2024-08-17T17:55:54.799206+0000","last_clean":"2024-08-17T17:55:54.799206+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:55:54.799206+0000","last_undegraded":"2024-08-17T17:55:54.799206+0000","last_fullsized":"2024-08-17T17:55:54.799206+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1177,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789616+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:55:54.789616+0000","last_peered":"2024-08-17T17:55:54.789616+0000","last_clean":"2024-08-17T17:55:54.789616+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:55:54.789616+0000","last_undegraded":"2024-08-17T17:55:54.789616+0000","last_fullsized":"2024-08-17T17:55:54.789616+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1823,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.332374+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:56:18.332374+0000","last_peered":"2024-08-17T17:56:18.332374+0000","last_clean":"2024-08-17T17:56:18.332374+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:56:18.332374+0000","last_undegraded":"2024-08-17T17:56:18.332374+0000","last_fullsized":"2024-08-17T17:56:18.332374+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":152,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.160485+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:56:08.160485+0000","last_peered":"2024-08-17T17:56:08.160485+0000","last_clean":"2024-08-17T17:56:08.160485+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:56:08.160485+0000","last_undegraded":"2024-08-17T17:56:08.160485+0000","last_fullsized":"2024-08-17T17:56:08.160485+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1543,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.799105+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:55:54.799105+0000","last_peered":"2024-08-17T17:55:54.799105+0000","last_clean":"2024-08-17T17:55:54.799105+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:55:54.799105+0000","last_undegraded":"2024-08-17T17:55:54.799105+0000","last_fullsized":"2024-08-17T17:55:54.799105+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1913,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.297835+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:56:08.297835+0000","last_peered":"2024-08-17T17:56:08.297835+0000","last_clean":"2024-08-17T17:56:08.297835+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:56:08.297835+0000","last_undegraded":"2024-08-17T17:56:08.297835+0000","last_fullsized":"2024-08-17T17:56:08.297835+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1855,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.331752+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:56:18.331752+0000","last_peered":"2024-08-17T17:56:18.331752+0000","last_clean":"2024-08-17T17:56:18.331752+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:56:18.331752+0000","last_undegraded":"2024-08-17T17:56:18.331752+0000","last_fullsized":"2024-08-17T17:56:18.331752+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":511,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789530+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:55:54.789530+0000","last_peered":"2024-08-17T17:55:54.789530+0000","last_clean":"2024-08-17T17:55:54.789530+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:55:54.789530+0000","last_undegraded":"2024-08-17T17:55:54.789530+0000","last_fullsized":"2024-08-17T17:55:54.789530+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":591,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.160675+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:56:08.160675+0000","last_peered":"2024-08-17T17:56:08.160675+0000","last_clean":"2024-08-17T17:56:08.160675+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:56:08.160675+0000","last_undegraded":"2024-08-17T17:56:08.160675+0000","last_fullsized":"2024-08-17T17:56:08.160675+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":607,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.160860+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:56:08.160860+0000","last_peered":"2024-08-17T17:56:08.160860+0000","last_clean":"2024-08-17T17:56:08.160860+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:56:08.160860+0000","last_undegraded":"2024-08-17T17:56:08.160860+0000","last_fullsized":"2024-08-17T17:56:08.160860+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1195,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789447+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:55:54.789447+0000","last_peered":"2024-08-17T17:55:54.789447+0000","last_clean":"2024-08-17T17:55:54.789447+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:55:54.789447+0000","last_undegraded":"2024-08-17T17:55:54.789447+0000","last_fullsized":"2024-08-17T17:55:54.789447+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":625,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.332149+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:56:18.332149+0000","last_peered":"2024-08-17T17:56:18.332149+0000","last_clean":"2024-08-17T17:56:18.332149+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:56:18.332149+0000","last_undegraded":"2024-08-17T17:56:18.332149+0000","last_fullsized":"2024-08-17T17:56:18.332149+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1851,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.333443+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:56:18.333443+0000","last_peered":"2024-08-17T17:56:18.333443+0000","last_clean":"2024-08-17T17:56:18.333443+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:56:18.333443+0000","last_undegraded":"2024-08-17T17:56:18.333443+0000","last_fullsized":"2024-08-17T17:56:18.333443+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1945,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.201347+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:56:08.201347+0000","last_peered":"2024-08-17T17:56:08.201347+0000","last_clean":"2024-08-17T17:56:08.201347+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:56:08.201347+0000","last_undegraded":"2024-08-17T17:56:08.201347+0000","last_fullsized":"2024-08-17T17:56:08.201347+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":658,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.332053+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:56:18.332053+0000","last_peered":"2024-08-17T17:56:18.332053+0000","last_clean":"2024-08-17T17:56:18.332053+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:56:18.332053+0000","last_undegraded":"2024-08-17T17:56:18.332053+0000","last_fullsized":"2024-08-17T17:56:18.332053+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1184,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789233+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:55:54.789233+0000","last_peered":"2024-08-17T17:55:54.789233+0000","last_clean":"2024-08-17T17:55:54.789233+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:55:54.789233+0000","last_undegraded":"2024-08-17T17:55:54.789233+0000","last_fullsized":"2024-08-17T17:55:54.789233+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1183,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.799458+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:55:54.799458+0000","last_peered":"2024-08-17T17:55:54.799458+0000","last_clean":"2024-08-17T17:55:54.799458+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:55:54.799458+0000","last_undegraded":"2024-08-17T17:55:54.799458+0000","last_fullsized":"2024-08-17T17:55:54.799458+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":641,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.295514+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:56:08.295514+0000","last_peered":"2024-08-17T17:56:08.295514+0000","last_clean":"2024-08-17T17:56:08.295514+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:56:08.295514+0000","last_undegraded":"2024-08-17T17:56:08.295514+0000","last_fullsized":"2024-08-17T17:56:08.295514+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1201,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.799658+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:55:54.799658+0000","last_peered":"2024-08-17T17:55:54.799658+0000","last_clean":"2024-08-17T17:55:54.799658+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:55:54.799658+0000","last_undegraded":"2024-08-17T17:55:54.799658+0000","last_fullsized":"2024-08-17T17:55:54.799658+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":85,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.799055+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:55:54.799055+0000","last_peered":"2024-08-17T17:55:54.799055+0000","last_clean":"2024-08-17T17:55:54.799055+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:55:54.799055+0000","last_undegraded":"2024-08-17T17:55:54.799055+0000","last_fullsized":"2024-08-17T17:55:54.799055+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1217,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.799602+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:55:54.799602+0000","last_peered":"2024-08-17T17:55:54.799602+0000","last_clean":"2024-08-17T17:55:54.799602+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:55:54.799602+0000","last_undegraded":"2024-08-17T17:55:54.799602+0000","last_fullsized":"2024-08-17T17:55:54.799602+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1234,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.789187+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:55:54.789187+0000","last_peered":"2024-08-17T17:55:54.789187+0000","last_clean":"2024-08-17T17:55:54.789187+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:55:54.789187+0000","last_undegraded":"2024-08-17T17:55:54.789187+0000","last_fullsized":"2024-08-17T17:55:54.789187+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1853,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.333119+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:56:18.333119+0000","last_peered":"2024-08-17T17:56:18.333119+0000","last_clean":"2024-08-17T17:56:18.333119+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:56:18.333119+0000","last_undegraded":"2024-08-17T17:56:18.333119+0000","last_fullsized":"2024-08-17T17:56:18.333119+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1178,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.801001+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:55:54.801001+0000","last_peered":"2024-08-17T17:55:54.801001+0000","last_clean":"2024-08-17T17:55:54.801001+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:55:54.801001+0000","last_undegraded":"2024-08-17T17:55:54.801001+0000","last_fullsized":"2024-08-17T17:55:54.801001+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1904,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.200997+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:56:08.200997+0000","last_peered":"2024-08-17T17:56:08.200997+0000","last_clean":"2024-08-17T17:56:08.200997+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:56:08.200997+0000","last_undegraded":"2024-08-17T17:56:08.200997+0000","last_fullsized":"2024-08-17T17:56:08.200997+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1905,"reported_epoch":1234,"state":"active+clean","last_fresh":"2024-08-17T17:56:08.200485+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:56:08.200485+0000","last_peered":"2024-08-17T17:56:08.200485+0000","last_clean":"2024-08-17T17:56:08.200485+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:56:08.200485+0000","last_undegraded":"2024-08-17T17:56:08.200485+0000","last_fullsized":"2024-08-17T17:56:08.200485+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":591,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.799340+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:55:54.799340+0000","last_peered":"2024-08-17T17:55:54.799340+0000","last_clean":"2024-08-17T17:55:54.799340+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:55:54.799340+0000","last_undegraded":"2024-08-17T17:55:54.799340+0000","last_fullsized":"2024-08-17T17:55:54.799340+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1203,"reported_epoch":1233,"state":"active+clean","last_fresh":"2024-08-17T17:55:54.799524+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:55:54.799524+0000","last_peered":"2024-08-17T17:55:54.799524+0000","last_clean":"2024-08-17T17:55:54.799524+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:55:54.799524+0000","last_undegraded":"2024-08-17T17:55:54.799524+0000","last_fullsized":"2024-08-17T17:55:54.799524+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1853,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.332277+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:56:18.332277+0000","last_peered":"2024-08-17T17:56:18.332277+0000","last_clean":"2024-08-17T17:56:18.332277+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:56:18.332277+0000","last_undegraded":"2024-08-17T17:56:18.332277+0000","last_fullsized":"2024-08-17T17:56:18.332277+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":3266838552,"num_objects":49849,"num_object_clones":0,"num_object_copies":149547,"num_objects_missing_on_primary":2563,"num_objects_missing":2563,"num_objects_degraded":15837,"num_objects_misplaced":8152,"num_objects_unfound":0,"num_objects_dirty":49849,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":159761,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36569,"num_bytes_recovered":2218131456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10899107840,"data_stored":10899095624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108535,"ondisk_log_size":108535,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234425,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1711568,"kb_used_data":1397576,"kb_used_omap":1,"kb_used_meta":313982,"kb_avail":92033584,"statfs":{"total":95995035648,"available":94242390016,"internally_reserved":0,"allocated":1431117824,"data_stored":1426141567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":321517912},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496060,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1906836,"kb_used_data":1609484,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":91838316,"statfs":{"total":95995035648,"available":94042435584,"internally_reserved":0,"allocated":1648111616,"data_stored":1643131764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757698,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326312,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2011516,"kb_used_data":1803316,"kb_used_omap":57,"kb_used_meta":208070,"kb_avail":91733636,"statfs":{"total":95995035648,"available":93935243264,"internally_reserved":0,"allocated":1846595584,"data_stored":1841634507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":213064443},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405550,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1843588,"kb_used_data":1622908,"kb_used_omap":61,"kb_used_meta":220610,"kb_avail":91901564,"statfs":{"total":95995035648,"available":94107201536,"internally_reserved":0,"allocated":1661857792,"data_stored":1656893687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":225905592},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452085,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1380856,"kb_used_data":1208816,"kb_used_omap":54,"kb_used_meta":171977,"kb_avail":92364296,"statfs":{"total":95995035648,"available":94581039104,"internally_reserved":0,"allocated":1237827584,"data_stored":1232870468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":176104846},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474776,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1298696,"kb_used_data":1075648,"kb_used_omap":119,"kb_used_meta":222920,"kb_avail":92446456,"statfs":{"total":95995035648,"available":94665170944,"internally_reserved":0,"allocated":1101463552,"data_stored":1096491272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":228270814},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,0,2,3,1,4,5,6],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019333,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1084035072,"data_stored":1084031000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1221394432,"data_stored":1221394432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1648164864,"data_stored":1648164864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1830883328,"data_stored":1830879256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":975568896,"data_stored":975568896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1634078720,"data_stored":1634074648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1417084928,"data_stored":1417084928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:56:22.731 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:56:22.793 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:22.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:22.816 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:22.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:22.917 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:22.945 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:22.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:22.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:23.047 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:23.070 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:23.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:23.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:23.172 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:23.196 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:23.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:23.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:23.298 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:23.319 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:23.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:23.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:23.421 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:23.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:56:23.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:23.443 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:23.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:23.544 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:23.565 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:23.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:23.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:23.667 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:23.688 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:56:23.688 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:56:23.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:23.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:23.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:23.711 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:56:23.711 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:56:23.733 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:56:23.734 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:56:23.743 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:23.764 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:56:23.765 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:56:23.769 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:56:23.782 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:23.788 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:56:23.788 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:56:23.796 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:23.809 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:56:23.810 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:56:23.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:23.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:23.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:23.828 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:56:23.828 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:56:23.851 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:56:23.851 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:56:23.925 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:23.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:23.949 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:23.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:24.050 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:24.071 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:24.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:24.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:24.173 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:24.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:24.197 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:24.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:24.297 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:24.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:24.320 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:24.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:24.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:24 smithi130 ceph-mon[110068]: pgmap v1889: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 15837/147783 objects degraded (10.716%); 7993/147783 objects misplaced (5.409%); 717 KiB/s, 11 objects/s recovering 2024-08-17T17:56:24.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:24 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1881338311' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:56:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:24 smithi130 ceph-mon[105521]: pgmap v1889: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 15837/147783 objects degraded (10.716%); 7993/147783 objects misplaced (5.409%); 717 KiB/s, 11 objects/s recovering 2024-08-17T17:56:24.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:24 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1881338311' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:56:24.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:24.444 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:24.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:24.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:24.546 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:24.569 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:24.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:24.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:24 smithi179 ceph-mon[93519]: pgmap v1889: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 151 op/s; 15837/147783 objects degraded (10.716%); 7993/147783 objects misplaced (5.409%); 717 KiB/s, 11 objects/s recovering 2024-08-17T17:56:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:24 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1881338311' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:56:24.670 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:24.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:24.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:24.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:24.793 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:24.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:24.818 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:24.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:24.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:24.940 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:24.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:24.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:25.042 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:25.064 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:25.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:25.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:25.165 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:25.186 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:25.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:25.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:25.288 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:25.309 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:25.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:25.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:25.411 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:25.431 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:25.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:25.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:25.533 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:25.555 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:25.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:25.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:25.657 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:25.682 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:25.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:25.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:25.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:25.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:25.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:25.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:25.912 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:25.936 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:25.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:25.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:26.037 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:26.058 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:26.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:26.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:26.160 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:26.181 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:26.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:26.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:26.282 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:26.305 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:26.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:26.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:26.393 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:26 smithi179 ceph-mon[93519]: pgmap v1890: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 15837/146895 objects degraded (10.781%); 7993/146895 objects misplaced (5.441%); 717 KiB/s, 11 objects/s recovering 2024-08-17T17:56:26.393 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/146895 objects degraded (10.781%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:26.405 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:26 smithi130 ceph-mon[105521]: pgmap v1890: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 15837/146895 objects degraded (10.781%); 7993/146895 objects misplaced (5.441%); 717 KiB/s, 11 objects/s recovering 2024-08-17T17:56:26.405 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/146895 objects degraded (10.781%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:26.406 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:26 smithi130 ceph-mon[110068]: pgmap v1890: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 15837/146895 objects degraded (10.781%); 7993/146895 objects misplaced (5.441%); 717 KiB/s, 11 objects/s recovering 2024-08-17T17:56:26.406 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/146895 objects degraded (10.781%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:26.407 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:26.436 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:26.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:26.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:26.537 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:26.562 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:26.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:26.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:26.659 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:26.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:26.689 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:26.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:26.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:26.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:26.812 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:26.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:26.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:26.914 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:26.935 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:26.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:26.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:27.037 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:27.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:27.059 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:27.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:27.160 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:27.183 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:27.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:27.285 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:27.311 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:27.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:27.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:27.413 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:27.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:27.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:27.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:27.536 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:27.560 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:27.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:27.661 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:27.683 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:27.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:27.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:27.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:27.801 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:56:27.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:27.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:27.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:27.912 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:27.942 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:27.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:27.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:28.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:28.068 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:28.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:28.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:28.170 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:28.191 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:28.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:28.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:28.293 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:28.314 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:28.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:28.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:28.416 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:28 smithi130 ceph-mon[105521]: pgmap v1891: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 15837/146238 objects degraded (10.830%); 7822/146238 objects misplaced (5.349%); 729 KiB/s, 11 objects/s recovering 2024-08-17T17:56:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:28 smithi130 ceph-mon[110068]: pgmap v1891: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 15837/146238 objects degraded (10.830%); 7822/146238 objects misplaced (5.349%); 729 KiB/s, 11 objects/s recovering 2024-08-17T17:56:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:28.428 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:56:28.428 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15837/146238 objects degraded (10.830%), 5 pgs degraded, 6 pgs undersized","count":11},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1709,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1235,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":48746,"data_bytes":3193444325,"bytes_used":10192490496,"bytes_avail":565777723392,"bytes_total":575970213888,"degraded_objects":15837,"degraded_total":146238,"degraded_ratio":0.10829606531817997,"misplaced_objects":7822,"misplaced_total":146238,"misplaced_ratio":0.053488149454998017,"recovering_objects_per_sec":11,"recovering_bytes_per_sec":746901,"recovering_keys_per_sec":0,"num_objects_recovered":114,"num_bytes_recovered":7471104,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":152},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":25,"modified":"2024-08-17T17:49:39.646037+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (8m)\n [==========================..] (remaining: 38s)","progress":0.93258428573608398,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:56:28.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:28.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:28.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:28.540 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:28.562 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:28.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:28.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:28 smithi179 ceph-mon[93519]: pgmap v1891: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 3.0 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 152 op/s; 15837/146238 objects degraded (10.830%); 7822/146238 objects misplaced (5.349%); 729 KiB/s, 11 objects/s recovering 2024-08-17T17:56:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:28.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:28.688 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:28.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:28.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:28.790 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:28.812 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:28.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:28.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:28.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:28.940 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:28.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:28.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:29.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:29.065 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:29.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:29.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:29.167 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:29.189 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:29.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:29.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:29.291 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:29.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:29.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:29.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:29.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:29.443 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:29.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:29.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:29.545 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:29.567 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:29.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:29.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:29.570 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2067818089' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:56:29.570 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2067818089' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:56:29.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2067818089' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:56:29.670 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:29.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:29.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:29.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:29.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:29.813 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:29.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:29.829 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:56:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:56:29] "GET /metrics HTTP/1.1" 200 36810 "" "Prometheus/2.51.0" 2024-08-17T17:56:29.829 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:56:29.829 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:56:29.829 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T17:56:29.829 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T17:56:29.830 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:56:29.915 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:29.936 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:29.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:29.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:30.038 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:30.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:30.062 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:30.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:30.163 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:30.185 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:30.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:30.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:30.287 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:30.308 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:30.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:30.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:30.410 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:30.431 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:30.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:30.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:30.533 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:30.554 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:30.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:30.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:30 smithi179 ceph-mon[93519]: pgmap v1892: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 170 op/s; 15837/144606 objects degraded (10.952%); 7822/144606 objects misplaced (5.409%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:56:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:56:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:56:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:56:30.656 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:30.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:30 smithi130 ceph-mon[105521]: pgmap v1892: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 170 op/s; 15837/144606 objects degraded (10.952%); 7822/144606 objects misplaced (5.409%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:56:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:30 smithi130 ceph-mon[110068]: pgmap v1892: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 170 op/s; 15837/144606 objects degraded (10.952%); 7822/144606 objects misplaced (5.409%); 678 KiB/s, 10 objects/s recovering 2024-08-17T17:56:30.678 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:30.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:30.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:30.780 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:30.801 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:30.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:30.903 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:30.924 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:30.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:30.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:31.000 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:56:31.026 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:31.047 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:31.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:31.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:31.149 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:31.172 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:31.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:31.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:31.277 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:31.299 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:31.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:31.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:31.401 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:31.425 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:31.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:31.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:31.527 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:31.548 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:31.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:31.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/144606 objects degraded (10.952%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:31.650 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/144606 objects degraded (10.952%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/144606 objects degraded (10.952%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:31.671 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:31.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:31.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:31.722 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:56:31.722 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:56:31.773 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:31.801 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:31.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:31.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:31.903 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:31.931 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:31.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:31.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:32.033 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:32.053 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:32.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:32.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:32.081 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:32.085 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:32.155 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:32.176 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:32.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:32.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:32.277 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:32.299 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:32.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:32.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:32.322 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T17:56:32.400 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:32.421 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:32.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:32.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:32.523 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:32.549 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:32.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:32.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:32 smithi179 ceph-mon[93519]: pgmap v1893: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 157 op/s; 15837/143952 objects degraded (11.002%); 7707/143952 objects misplaced (5.354%); 917 KiB/s, 14 objects/s recovering 2024-08-17T17:56:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2932777313' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:32 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:32.650 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:32.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:32 smithi130 ceph-mon[105521]: pgmap v1893: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 157 op/s; 15837/143952 objects degraded (11.002%); 7707/143952 objects misplaced (5.354%); 917 KiB/s, 14 objects/s recovering 2024-08-17T17:56:32.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2932777313' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:32.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:32 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:32 smithi130 ceph-mon[110068]: pgmap v1893: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 157 op/s; 15837/143952 objects degraded (11.002%); 7707/143952 objects misplaced (5.354%); 917 KiB/s, 14 objects/s recovering 2024-08-17T17:56:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2932777313' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:32 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:32.672 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:32.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:32.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:32.774 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:32.796 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:32.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:32.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:32.898 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:32.927 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:32.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:32.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:33.029 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:33.050 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:33.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:33.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:33.152 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:33.174 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:33.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:33.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:33.276 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:33.296 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:33.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:33.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:33.398 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:33.427 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:33.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:33.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:33.529 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:33.550 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:33.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:33.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:33.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:33 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:56:33.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:33 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:56:33.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:33 smithi179 ceph-mon[93519]: osdmap e1236: 8 total, 8 up, 6 in 2024-08-17T17:56:33.596 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:33 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:56:33.596 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:33 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:56:33.596 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:33 smithi130 ceph-mon[105521]: osdmap e1236: 8 total, 8 up, 6 in 2024-08-17T17:56:33.596 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:33 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:56:33.596 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:33 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:56:33.596 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:33 smithi130 ceph-mon[110068]: osdmap e1236: 8 total, 8 up, 6 in 2024-08-17T17:56:33.653 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:33.675 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:33.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:33.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:33.776 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:33.805 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:33.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:33.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:33.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:34.048 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:34.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:34.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:34.150 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:34.171 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:34.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:34.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:34.272 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:34.295 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:34.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:34.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:34.397 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:34.423 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:34.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:34.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:34.524 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:34.545 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:34.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:34.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:34.585 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:34 smithi130 ceph-mon[105521]: pgmap v1895: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 15837/142704 objects degraded (11.098%); 7655/142704 objects misplaced (5.364%); 729 KiB/s, 11 objects/s recovering 2024-08-17T17:56:34.585 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:34 smithi130 ceph-mon[110068]: pgmap v1895: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 15837/142704 objects degraded (11.098%); 7655/142704 objects misplaced (5.364%); 729 KiB/s, 11 objects/s recovering 2024-08-17T17:56:34.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:34 smithi179 ceph-mon[93519]: pgmap v1895: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 15837/142704 objects degraded (11.098%); 7655/142704 objects misplaced (5.364%); 729 KiB/s, 11 objects/s recovering 2024-08-17T17:56:34.650 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:34.671 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:34.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:34.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:34.773 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:34.794 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:34.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:34.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:34.801 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:34.896 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:34.923 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:34.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:34.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:35.024 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:35.048 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:35.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:35.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:35.150 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:35.171 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:35.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:35.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:35.272 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:35.293 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:35.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:35.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:35.395 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:35.416 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:35.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:35.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:35.481 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:56:35.517 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:35.546 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:35.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:35.648 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:35.679 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:35.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:35.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:35.780 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:35.804 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:35.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:35.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:35.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:35.927 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:35.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:35.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:36.028 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:36.053 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:36.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:36.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:36.155 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:36.182 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:36.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:36.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:36.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:36.312 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:36.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:36.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:36.414 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:36.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:36 smithi130 ceph-mon[105521]: pgmap v1896: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 15837/141810 objects degraded (11.168%); 7655/141810 objects misplaced (5.398%); 729 KiB/s, 11 objects/s recovering 2024-08-17T17:56:36.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/141810 objects degraded (11.168%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:36.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:36 smithi130 ceph-mon[110068]: pgmap v1896: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 15837/141810 objects degraded (11.168%); 7655/141810 objects misplaced (5.398%); 729 KiB/s, 11 objects/s recovering 2024-08-17T17:56:36.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/141810 objects degraded (11.168%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:36.435 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:36.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:36.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:36.537 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:36.558 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:36.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:36.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:36 smithi179 ceph-mon[93519]: pgmap v1896: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 169 op/s; 15837/141810 objects degraded (11.168%); 7655/141810 objects misplaced (5.398%); 729 KiB/s, 11 objects/s recovering 2024-08-17T17:56:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/141810 objects degraded (11.168%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:36.660 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:36.681 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:36.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:36.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:36.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:36.806 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:36.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:36.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:36.908 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:36.929 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:36.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:36.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:37.031 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:37.053 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:37.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:37.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:37.107 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:56:37.109 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:56:37.155 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:37.177 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:37.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:37.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:37.278 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:37.300 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:37.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:37.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:37.402 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:37.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:37 smithi130 ceph-mon[105521]: from='client.45833 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:56:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:37 smithi130 ceph-mon[110068]: from='client.45833 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:56:37.424 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:37.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:37.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:37.526 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:37.547 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:37.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:37.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:37.649 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:37.670 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:37.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:37.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:37.772 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:37.797 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:37.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:37.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:37 smithi179 ceph-mon[93519]: from='client.45833 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:56:37.846 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1896,"stamp":"2024-08-17T17:56:35.089396+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3096713189,"num_objects":47270,"num_object_clones":0,"num_object_copies":141810,"num_objects_missing_on_primary":2391,"num_objects_missing":2391,"num_objects_degraded":15837,"num_objects_misplaced":7655,"num_objects_unfound":0,"num_objects_dirty":47270,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":162644,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36770,"num_bytes_recovered":2231180301,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108647,"ondisk_log_size":108647,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9727820,"kb_used_data":8280084,"kb_used_omap":295,"kb_used_meta":1447384,"kb_avail":552743092,"statfs":{"total":575970213888,"available":566008926208,"internally_reserved":0,"allocated":8478806016,"data_stored":8448920969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1482121679},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,0,1,0,0,1,4,6,2],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-111083520,"num_objects":-1695,"num_object_clones":0,"num_object_copies":-5085,"num_objects_missing_on_primary":-115,"num_objects_missing":-115,"num_objects_degraded":0,"num_objects_misplaced":-338,"num_objects_unfound":0,"num_objects_dirty":-1695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1695,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":114,"num_bytes_recovered":7471104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002547"},"pg_stats":[{"pgid":"45.f","version":"1235'10162","reported_seq":20712,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:30.729068+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:56:30.729068+0000","last_peered":"2024-08-17T17:56:30.729068+0000","last_clean":"2024-08-17T17:56:30.729068+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:56:30.729068+0000","last_undegraded":"2024-08-17T17:56:30.729068+0000","last_fullsized":"2024-08-17T17:56:30.729068+0000","mapping_epoch":1194,"log_start":"1174'3331","ondisk_log_start":"1174'3331","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6831,"log_dups_size":0,"ondisk_log_size":6831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196739072,"num_objects":3002,"num_object_clones":0,"num_object_copies":9006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10162,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1236'10082","reported_seq":20558,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.815437+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:56:32.815437+0000","last_peered":"2024-08-17T17:56:32.815437+0000","last_clean":"2024-08-17T17:56:32.815437+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:56:32.815437+0000","last_undegraded":"2024-08-17T17:56:32.815437+0000","last_fullsized":"2024-08-17T17:56:32.815437+0000","mapping_epoch":1211,"log_start":"1174'3032","ondisk_log_start":"1174'3032","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7050,"log_dups_size":0,"ondisk_log_size":7050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190840832,"num_objects":2912,"num_object_clones":0,"num_object_copies":8736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10082,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1236'10394","reported_seq":21253,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:33.575476+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:56:33.575476+0000","last_peered":"2024-08-17T17:56:33.575476+0000","last_clean":"2024-08-17T17:56:33.575476+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:56:33.575476+0000","last_undegraded":"2024-08-17T17:56:33.575476+0000","last_fullsized":"2024-08-17T17:56:33.575476+0000","mapping_epoch":1205,"log_start":"1175'4468","ondisk_log_start":"1175'4468","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5926,"log_dups_size":0,"ondisk_log_size":5926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195493912,"num_objects":2984,"num_object_clones":0,"num_object_copies":8952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2984,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10394,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1236'10097","reported_seq":10174,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.891867+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:56:32.891867+0000","last_peered":"2024-08-17T17:56:32.891867+0000","last_clean":"2024-08-17T17:56:32.891867+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:56:32.891867+0000","last_undegraded":"2024-08-17T17:56:32.891867+0000","last_fullsized":"2024-08-17T17:56:32.891867+0000","mapping_epoch":1167,"log_start":"1174'3389","ondisk_log_start":"1174'3389","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6708,"log_dups_size":0,"ondisk_log_size":6708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190251008,"num_objects":2903,"num_object_clones":0,"num_object_copies":8709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10097,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1235'10156","reported_seq":15419,"reported_epoch":1235,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:31.420476+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:56:31.420476+0000","last_peered":"2024-08-17T17:56:31.420476+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:56:31.420476+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1175'4226","ondisk_log_start":"1175'4226","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5930,"log_dups_size":0,"ondisk_log_size":5930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193855488,"num_objects":2958,"num_object_clones":0,"num_object_copies":8874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":2958,"num_objects_unfound":0,"num_objects_dirty":2958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10156,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":2440}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1236'10247","reported_seq":10324,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.904849+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:56:32.904849+0000","last_peered":"2024-08-17T17:56:32.904849+0000","last_clean":"2024-08-17T17:56:32.904849+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:56:32.904849+0000","last_undegraded":"2024-08-17T17:56:32.904849+0000","last_fullsized":"2024-08-17T17:56:32.904849+0000","mapping_epoch":1167,"log_start":"1174'3577","ondisk_log_start":"1174'3577","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6670,"log_dups_size":0,"ondisk_log_size":6670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186580992,"num_objects":2847,"num_object_clones":0,"num_object_copies":8541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10247,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1236'10222","reported_seq":10299,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.863631+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:56:32.863631+0000","last_peered":"2024-08-17T17:56:32.863631+0000","last_clean":"2024-08-17T17:56:32.863631+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:56:32.863631+0000","last_undegraded":"2024-08-17T17:56:32.863631+0000","last_fullsized":"2024-08-17T17:56:32.863631+0000","mapping_epoch":1167,"log_start":"1174'3469","ondisk_log_start":"1174'3469","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6753,"log_dups_size":0,"ondisk_log_size":6753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196870144,"num_objects":3004,"num_object_clones":0,"num_object_copies":9012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10222,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1235'10107","reported_seq":19588,"reported_epoch":1235,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:31.833844+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:56:31.833844+0000","last_peered":"2024-08-17T17:56:31.833844+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:56:31.833844+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1175'4175","ondisk_log_start":"1175'4175","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5932,"log_dups_size":0,"ondisk_log_size":5932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188940288,"num_objects":2883,"num_object_clones":0,"num_object_copies":8649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2883,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10107,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3108,"num_bytes_recovered":203685888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":1563},{"shards":"1,7","objects":816}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1236'10084","reported_seq":10171,"reported_epoch":1236,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:32.895815+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:56:32.895815+0000","last_peered":"2024-08-17T17:56:32.895815+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:56:32.895815+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1174'3377","ondisk_log_start":"1174'3377","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6707,"log_dups_size":0,"ondisk_log_size":6707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196083712,"num_objects":2992,"num_object_clones":0,"num_object_copies":8976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10084,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":2992},{"shards":"1,6","objects":-2841}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1236'10179","reported_seq":13278,"reported_epoch":1236,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:33.553018+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:56:33.553018+0000","last_peered":"2024-08-17T17:56:33.553018+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:56:33.553018+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1175'4206","ondisk_log_start":"1175'4206","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5973,"log_dups_size":0,"ondisk_log_size":5973,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196280320,"num_objects":2995,"num_object_clones":0,"num_object_copies":8985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10179,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":1434}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1236'10091","reported_seq":10245,"reported_epoch":1236,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:33.805001+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:56:33.805001+0000","last_peered":"2024-08-17T17:56:33.805001+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:56:33.805001+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1170'1000","ondisk_log_start":"1170'1000","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9091,"log_dups_size":0,"ondisk_log_size":9091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195493888,"num_objects":2983,"num_object_clones":0,"num_object_copies":8949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10091,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-2170}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1236'10102","reported_seq":16317,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:33.786294+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:56:33.786294+0000","last_peered":"2024-08-17T17:56:33.786294+0000","last_clean":"2024-08-17T17:56:33.786294+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:56:33.786294+0000","last_undegraded":"2024-08-17T17:56:33.786294+0000","last_fullsized":"2024-08-17T17:56:33.786294+0000","mapping_epoch":1183,"log_start":"1170'1000","ondisk_log_start":"1170'1000","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9102,"log_dups_size":0,"ondisk_log_size":9102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192937984,"num_objects":2944,"num_object_clones":0,"num_object_copies":8832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10102,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1236'10154","reported_seq":10307,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:33.572429+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:56:33.572429+0000","last_peered":"2024-08-17T17:56:33.572429+0000","last_clean":"2024-08-17T17:56:33.572429+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:56:33.572429+0000","last_undegraded":"2024-08-17T17:56:33.572429+0000","last_fullsized":"2024-08-17T17:56:33.572429+0000","mapping_epoch":1167,"log_start":"1175'4191","ondisk_log_start":"1175'4191","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5963,"log_dups_size":0,"ondisk_log_size":5963,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197001216,"num_objects":3006,"num_object_clones":0,"num_object_copies":9018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10154,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1236'10018","reported_seq":10095,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.919195+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:56:32.919195+0000","last_peered":"2024-08-17T17:56:32.919195+0000","last_clean":"2024-08-17T17:56:32.919195+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:56:32.919195+0000","last_undegraded":"2024-08-17T17:56:32.919195+0000","last_fullsized":"2024-08-17T17:56:32.919195+0000","mapping_epoch":1167,"log_start":"1172'3025","ondisk_log_start":"1172'3025","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6993,"log_dups_size":0,"ondisk_log_size":6993,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191102976,"num_objects":2916,"num_object_clones":0,"num_object_copies":8748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2916,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10018,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1235'10330","reported_seq":25194,"reported_epoch":1235,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:56:30.897457+0000","last_change":"2024-08-17T17:55:37.941989+0000","last_active":"2024-08-17T17:56:30.897457+0000","last_peered":"2024-08-17T17:56:30.897457+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:56:30.897457+0000","last_undegraded":"2024-08-17T17:56:30.897457+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1174'3491","ondisk_log_start":"1174'3491","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6839,"log_dups_size":0,"ondisk_log_size":6839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198836224,"num_objects":3034,"num_object_clones":0,"num_object_copies":9102,"num_objects_missing_on_primary":2391,"num_objects_missing":2391,"num_objects_degraded":0,"num_objects_misplaced":4697,"num_objects_unfound":0,"num_objects_dirty":3034,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10330,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5755,"num_bytes_recovered":377159680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":13},{"shards":"0,4,6","objects":1356},{"shards":"0,6","objects":1},{"shards":"4,5,6","objects":1664}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1236'9941","reported_seq":23280,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:33.397482+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:56:33.397482+0000","last_peered":"2024-08-17T17:56:33.397482+0000","last_clean":"2024-08-17T17:56:33.397482+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:56:33.397482+0000","last_undegraded":"2024-08-17T17:56:33.397482+0000","last_fullsized":"2024-08-17T17:56:33.397482+0000","mapping_epoch":1196,"log_start":"1175'3969","ondisk_log_start":"1175'3969","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5972,"log_dups_size":0,"ondisk_log_size":5972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188809216,"num_objects":2881,"num_object_clones":0,"num_object_copies":8643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9941,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1927,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.346513+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:56:18.346513+0000","last_peered":"2024-08-17T17:56:18.346513+0000","last_clean":"2024-08-17T17:56:18.346513+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:56:18.346513+0000","last_undegraded":"2024-08-17T17:56:18.346513+0000","last_fullsized":"2024-08-17T17:56:18.346513+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":513,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.337337+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:56:18.337337+0000","last_peered":"2024-08-17T17:56:18.337337+0000","last_clean":"2024-08-17T17:56:18.337337+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:56:18.337337+0000","last_undegraded":"2024-08-17T17:56:18.337337+0000","last_fullsized":"2024-08-17T17:56:18.337337+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1849,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.322729+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:56:32.322729+0000","last_peered":"2024-08-17T17:56:32.322729+0000","last_clean":"2024-08-17T17:56:32.322729+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:56:32.322729+0000","last_undegraded":"2024-08-17T17:56:32.322729+0000","last_fullsized":"2024-08-17T17:56:32.322729+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1873,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.320360+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:56:32.320360+0000","last_peered":"2024-08-17T17:56:32.320360+0000","last_clean":"2024-08-17T17:56:32.320360+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:56:32.320360+0000","last_undegraded":"2024-08-17T17:56:32.320360+0000","last_fullsized":"2024-08-17T17:56:32.320360+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":75,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.337869+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:56:18.337869+0000","last_peered":"2024-08-17T17:56:18.337869+0000","last_clean":"2024-08-17T17:56:18.337869+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:56:18.337869+0000","last_undegraded":"2024-08-17T17:56:18.337869+0000","last_fullsized":"2024-08-17T17:56:18.337869+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1266,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.320828+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:56:32.320828+0000","last_peered":"2024-08-17T17:56:32.320828+0000","last_clean":"2024-08-17T17:56:32.320828+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:56:32.320828+0000","last_undegraded":"2024-08-17T17:56:32.320828+0000","last_fullsized":"2024-08-17T17:56:32.320828+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":122,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.335439+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:56:18.335439+0000","last_peered":"2024-08-17T17:56:18.335439+0000","last_clean":"2024-08-17T17:56:18.335439+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:56:18.335439+0000","last_undegraded":"2024-08-17T17:56:18.335439+0000","last_fullsized":"2024-08-17T17:56:18.335439+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":644,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.322465+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:56:32.322465+0000","last_peered":"2024-08-17T17:56:32.322465+0000","last_clean":"2024-08-17T17:56:32.322465+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:56:32.322465+0000","last_undegraded":"2024-08-17T17:56:32.322465+0000","last_fullsized":"2024-08-17T17:56:32.322465+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1886,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.345833+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:56:18.345833+0000","last_peered":"2024-08-17T17:56:18.345833+0000","last_clean":"2024-08-17T17:56:18.345833+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:56:18.345833+0000","last_undegraded":"2024-08-17T17:56:18.345833+0000","last_fullsized":"2024-08-17T17:56:18.345833+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1890,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.329457+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:56:32.329457+0000","last_peered":"2024-08-17T17:56:32.329457+0000","last_clean":"2024-08-17T17:56:32.329457+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:56:32.329457+0000","last_undegraded":"2024-08-17T17:56:32.329457+0000","last_fullsized":"2024-08-17T17:56:32.329457+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":107,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.338569+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:56:18.338569+0000","last_peered":"2024-08-17T17:56:18.338569+0000","last_clean":"2024-08-17T17:56:18.338569+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:56:18.338569+0000","last_undegraded":"2024-08-17T17:56:18.338569+0000","last_fullsized":"2024-08-17T17:56:18.338569+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1861,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.322692+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:56:32.322692+0000","last_peered":"2024-08-17T17:56:32.322692+0000","last_clean":"2024-08-17T17:56:32.322692+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:56:32.322692+0000","last_undegraded":"2024-08-17T17:56:32.322692+0000","last_fullsized":"2024-08-17T17:56:32.322692+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1914,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.335093+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:56:18.335093+0000","last_peered":"2024-08-17T17:56:18.335093+0000","last_clean":"2024-08-17T17:56:18.335093+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:56:18.335093+0000","last_undegraded":"2024-08-17T17:56:18.335093+0000","last_fullsized":"2024-08-17T17:56:18.335093+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1908,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.346325+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:56:18.346325+0000","last_peered":"2024-08-17T17:56:18.346325+0000","last_clean":"2024-08-17T17:56:18.346325+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:56:18.346325+0000","last_undegraded":"2024-08-17T17:56:18.346325+0000","last_fullsized":"2024-08-17T17:56:18.346325+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1884,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.320637+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:56:32.320637+0000","last_peered":"2024-08-17T17:56:32.320637+0000","last_clean":"2024-08-17T17:56:32.320637+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:56:32.320637+0000","last_undegraded":"2024-08-17T17:56:32.320637+0000","last_fullsized":"2024-08-17T17:56:32.320637+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1210,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.337982+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:56:18.337982+0000","last_peered":"2024-08-17T17:56:18.337982+0000","last_clean":"2024-08-17T17:56:18.337982+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:56:18.337982+0000","last_undegraded":"2024-08-17T17:56:18.337982+0000","last_fullsized":"2024-08-17T17:56:18.337982+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1855,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.320405+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:56:32.320405+0000","last_peered":"2024-08-17T17:56:32.320405+0000","last_clean":"2024-08-17T17:56:32.320405+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:56:32.320405+0000","last_undegraded":"2024-08-17T17:56:32.320405+0000","last_fullsized":"2024-08-17T17:56:32.320405+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1903,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.346563+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:56:18.346563+0000","last_peered":"2024-08-17T17:56:18.346563+0000","last_clean":"2024-08-17T17:56:18.346563+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:56:18.346563+0000","last_undegraded":"2024-08-17T17:56:18.346563+0000","last_fullsized":"2024-08-17T17:56:18.346563+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1811,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.320265+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:56:32.320265+0000","last_peered":"2024-08-17T17:56:32.320265+0000","last_clean":"2024-08-17T17:56:32.320265+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:56:32.320265+0000","last_undegraded":"2024-08-17T17:56:32.320265+0000","last_fullsized":"2024-08-17T17:56:32.320265+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":529,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.331996+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:56:18.331996+0000","last_peered":"2024-08-17T17:56:18.331996+0000","last_clean":"2024-08-17T17:56:18.331996+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:56:18.331996+0000","last_undegraded":"2024-08-17T17:56:18.331996+0000","last_fullsized":"2024-08-17T17:56:18.331996+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1206,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.331900+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:56:18.331900+0000","last_peered":"2024-08-17T17:56:18.331900+0000","last_clean":"2024-08-17T17:56:18.331900+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:56:18.331900+0000","last_undegraded":"2024-08-17T17:56:18.331900+0000","last_fullsized":"2024-08-17T17:56:18.331900+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1908,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.338841+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:56:18.338841+0000","last_peered":"2024-08-17T17:56:18.338841+0000","last_clean":"2024-08-17T17:56:18.338841+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:56:18.338841+0000","last_undegraded":"2024-08-17T17:56:18.338841+0000","last_fullsized":"2024-08-17T17:56:18.338841+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":367,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.332089+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:56:18.332089+0000","last_peered":"2024-08-17T17:56:18.332089+0000","last_clean":"2024-08-17T17:56:18.332089+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:56:18.332089+0000","last_undegraded":"2024-08-17T17:56:18.332089+0000","last_fullsized":"2024-08-17T17:56:18.332089+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":513,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.337647+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:56:18.337647+0000","last_peered":"2024-08-17T17:56:18.337647+0000","last_clean":"2024-08-17T17:56:18.337647+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:56:18.337647+0000","last_undegraded":"2024-08-17T17:56:18.337647+0000","last_fullsized":"2024-08-17T17:56:18.337647+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1191,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.337558+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:56:18.337558+0000","last_peered":"2024-08-17T17:56:18.337558+0000","last_clean":"2024-08-17T17:56:18.337558+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:56:18.337558+0000","last_undegraded":"2024-08-17T17:56:18.337558+0000","last_fullsized":"2024-08-17T17:56:18.337558+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1199,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.338053+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:56:18.338053+0000","last_peered":"2024-08-17T17:56:18.338053+0000","last_clean":"2024-08-17T17:56:18.338053+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:56:18.338053+0000","last_undegraded":"2024-08-17T17:56:18.338053+0000","last_fullsized":"2024-08-17T17:56:18.338053+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1955,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.346221+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:56:18.346221+0000","last_peered":"2024-08-17T17:56:18.346221+0000","last_clean":"2024-08-17T17:56:18.346221+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:56:18.346221+0000","last_undegraded":"2024-08-17T17:56:18.346221+0000","last_fullsized":"2024-08-17T17:56:18.346221+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1841,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.320562+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:56:32.320562+0000","last_peered":"2024-08-17T17:56:32.320562+0000","last_clean":"2024-08-17T17:56:32.320562+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:56:32.320562+0000","last_undegraded":"2024-08-17T17:56:32.320562+0000","last_fullsized":"2024-08-17T17:56:32.320562+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1913,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.335559+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:56:18.335559+0000","last_peered":"2024-08-17T17:56:18.335559+0000","last_clean":"2024-08-17T17:56:18.335559+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:56:18.335559+0000","last_undegraded":"2024-08-17T17:56:18.335559+0000","last_fullsized":"2024-08-17T17:56:18.335559+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1843,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.321485+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:56:32.321485+0000","last_peered":"2024-08-17T17:56:32.321485+0000","last_clean":"2024-08-17T17:56:32.321485+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:56:32.321485+0000","last_undegraded":"2024-08-17T17:56:32.321485+0000","last_fullsized":"2024-08-17T17:56:32.321485+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1873,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.346029+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:56:18.346029+0000","last_peered":"2024-08-17T17:56:18.346029+0000","last_clean":"2024-08-17T17:56:18.346029+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:56:18.346029+0000","last_undegraded":"2024-08-17T17:56:18.346029+0000","last_fullsized":"2024-08-17T17:56:18.346029+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1226,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.332071+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:56:18.332071+0000","last_peered":"2024-08-17T17:56:18.332071+0000","last_clean":"2024-08-17T17:56:18.332071+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:56:18.332071+0000","last_undegraded":"2024-08-17T17:56:18.332071+0000","last_fullsized":"2024-08-17T17:56:18.332071+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1920,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.335615+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:56:18.335615+0000","last_peered":"2024-08-17T17:56:18.335615+0000","last_clean":"2024-08-17T17:56:18.335615+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:56:18.335615+0000","last_undegraded":"2024-08-17T17:56:18.335615+0000","last_fullsized":"2024-08-17T17:56:18.335615+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":107,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.335280+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:56:18.335280+0000","last_peered":"2024-08-17T17:56:18.335280+0000","last_clean":"2024-08-17T17:56:18.335280+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:56:18.335280+0000","last_undegraded":"2024-08-17T17:56:18.335280+0000","last_fullsized":"2024-08-17T17:56:18.335280+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1939,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.335535+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:56:18.335535+0000","last_peered":"2024-08-17T17:56:18.335535+0000","last_clean":"2024-08-17T17:56:18.335535+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:56:18.335535+0000","last_undegraded":"2024-08-17T17:56:18.335535+0000","last_fullsized":"2024-08-17T17:56:18.335535+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":678,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.321850+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:56:32.321850+0000","last_peered":"2024-08-17T17:56:32.321850+0000","last_clean":"2024-08-17T17:56:32.321850+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:56:32.321850+0000","last_undegraded":"2024-08-17T17:56:32.321850+0000","last_fullsized":"2024-08-17T17:56:32.321850+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1920,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.335695+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:56:18.335695+0000","last_peered":"2024-08-17T17:56:18.335695+0000","last_clean":"2024-08-17T17:56:18.335695+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:56:18.335695+0000","last_undegraded":"2024-08-17T17:56:18.335695+0000","last_fullsized":"2024-08-17T17:56:18.335695+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1249,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.345714+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:56:18.345714+0000","last_peered":"2024-08-17T17:56:18.345714+0000","last_clean":"2024-08-17T17:56:18.345714+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:56:18.345714+0000","last_undegraded":"2024-08-17T17:56:18.345714+0000","last_fullsized":"2024-08-17T17:56:18.345714+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":167,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.338711+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:56:18.338711+0000","last_peered":"2024-08-17T17:56:18.338711+0000","last_clean":"2024-08-17T17:56:18.338711+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:56:18.338711+0000","last_undegraded":"2024-08-17T17:56:18.338711+0000","last_fullsized":"2024-08-17T17:56:18.338711+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1930,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.335553+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:56:18.335553+0000","last_peered":"2024-08-17T17:56:18.335553+0000","last_clean":"2024-08-17T17:56:18.335553+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:56:18.335553+0000","last_undegraded":"2024-08-17T17:56:18.335553+0000","last_fullsized":"2024-08-17T17:56:18.335553+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1908,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.335968+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:56:18.335968+0000","last_peered":"2024-08-17T17:56:18.335968+0000","last_clean":"2024-08-17T17:56:18.335968+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:56:18.335968+0000","last_undegraded":"2024-08-17T17:56:18.335968+0000","last_fullsized":"2024-08-17T17:56:18.335968+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1931,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.329951+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:56:32.329951+0000","last_peered":"2024-08-17T17:56:32.329951+0000","last_clean":"2024-08-17T17:56:32.329951+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:56:32.329951+0000","last_undegraded":"2024-08-17T17:56:32.329951+0000","last_fullsized":"2024-08-17T17:56:32.329951+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1934,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.346388+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:56:18.346388+0000","last_peered":"2024-08-17T17:56:18.346388+0000","last_clean":"2024-08-17T17:56:18.346388+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:56:18.346388+0000","last_undegraded":"2024-08-17T17:56:18.346388+0000","last_fullsized":"2024-08-17T17:56:18.346388+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1914,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.402167+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:56:18.402167+0000","last_peered":"2024-08-17T17:56:18.402167+0000","last_clean":"2024-08-17T17:56:18.402167+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:56:18.402167+0000","last_undegraded":"2024-08-17T17:56:18.402167+0000","last_fullsized":"2024-08-17T17:56:18.402167+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1233,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.331761+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:56:18.331761+0000","last_peered":"2024-08-17T17:56:18.331761+0000","last_clean":"2024-08-17T17:56:18.331761+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:56:18.331761+0000","last_undegraded":"2024-08-17T17:56:18.331761+0000","last_fullsized":"2024-08-17T17:56:18.331761+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1179,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.337888+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:56:18.337888+0000","last_peered":"2024-08-17T17:56:18.337888+0000","last_clean":"2024-08-17T17:56:18.337888+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:56:18.337888+0000","last_undegraded":"2024-08-17T17:56:18.337888+0000","last_fullsized":"2024-08-17T17:56:18.337888+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1825,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.320730+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:56:32.320730+0000","last_peered":"2024-08-17T17:56:32.320730+0000","last_clean":"2024-08-17T17:56:32.320730+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:56:32.320730+0000","last_undegraded":"2024-08-17T17:56:32.320730+0000","last_fullsized":"2024-08-17T17:56:32.320730+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":157,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.329028+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:56:32.329028+0000","last_peered":"2024-08-17T17:56:32.329028+0000","last_clean":"2024-08-17T17:56:32.329028+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:56:32.329028+0000","last_undegraded":"2024-08-17T17:56:32.329028+0000","last_fullsized":"2024-08-17T17:56:32.329028+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1545,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.331663+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:56:18.331663+0000","last_peered":"2024-08-17T17:56:18.331663+0000","last_clean":"2024-08-17T17:56:18.331663+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:56:18.331663+0000","last_undegraded":"2024-08-17T17:56:18.331663+0000","last_fullsized":"2024-08-17T17:56:18.331663+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1915,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.335691+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:56:18.335691+0000","last_peered":"2024-08-17T17:56:18.335691+0000","last_clean":"2024-08-17T17:56:18.335691+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:56:18.335691+0000","last_undegraded":"2024-08-17T17:56:18.335691+0000","last_fullsized":"2024-08-17T17:56:18.335691+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1857,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.320482+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:56:32.320482+0000","last_peered":"2024-08-17T17:56:32.320482+0000","last_clean":"2024-08-17T17:56:32.320482+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:56:32.320482+0000","last_undegraded":"2024-08-17T17:56:32.320482+0000","last_fullsized":"2024-08-17T17:56:32.320482+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":513,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.337867+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:56:18.337867+0000","last_peered":"2024-08-17T17:56:18.337867+0000","last_clean":"2024-08-17T17:56:18.337867+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:56:18.337867+0000","last_undegraded":"2024-08-17T17:56:18.337867+0000","last_fullsized":"2024-08-17T17:56:18.337867+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":596,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.329747+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:56:32.329747+0000","last_peered":"2024-08-17T17:56:32.329747+0000","last_clean":"2024-08-17T17:56:32.329747+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:56:32.329747+0000","last_undegraded":"2024-08-17T17:56:32.329747+0000","last_fullsized":"2024-08-17T17:56:32.329747+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":612,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.335085+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:56:32.335085+0000","last_peered":"2024-08-17T17:56:32.335085+0000","last_clean":"2024-08-17T17:56:32.335085+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:56:32.335085+0000","last_undegraded":"2024-08-17T17:56:32.335085+0000","last_fullsized":"2024-08-17T17:56:32.335085+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1197,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.337813+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:56:18.337813+0000","last_peered":"2024-08-17T17:56:18.337813+0000","last_clean":"2024-08-17T17:56:18.337813+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:56:18.337813+0000","last_undegraded":"2024-08-17T17:56:18.337813+0000","last_fullsized":"2024-08-17T17:56:18.337813+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":628,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.321123+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:56:32.321123+0000","last_peered":"2024-08-17T17:56:32.321123+0000","last_clean":"2024-08-17T17:56:32.321123+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:56:32.321123+0000","last_undegraded":"2024-08-17T17:56:32.321123+0000","last_fullsized":"2024-08-17T17:56:32.321123+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1854,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.322598+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:56:32.322598+0000","last_peered":"2024-08-17T17:56:32.322598+0000","last_clean":"2024-08-17T17:56:32.322598+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:56:32.322598+0000","last_undegraded":"2024-08-17T17:56:32.322598+0000","last_fullsized":"2024-08-17T17:56:32.322598+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1947,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.346111+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:56:18.346111+0000","last_peered":"2024-08-17T17:56:18.346111+0000","last_clean":"2024-08-17T17:56:18.346111+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:56:18.346111+0000","last_undegraded":"2024-08-17T17:56:18.346111+0000","last_fullsized":"2024-08-17T17:56:18.346111+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":660,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.321010+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:56:32.321010+0000","last_peered":"2024-08-17T17:56:32.321010+0000","last_clean":"2024-08-17T17:56:32.321010+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:56:32.321010+0000","last_undegraded":"2024-08-17T17:56:32.321010+0000","last_fullsized":"2024-08-17T17:56:32.321010+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1186,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.337702+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:56:18.337702+0000","last_peered":"2024-08-17T17:56:18.337702+0000","last_clean":"2024-08-17T17:56:18.337702+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:56:18.337702+0000","last_undegraded":"2024-08-17T17:56:18.337702+0000","last_fullsized":"2024-08-17T17:56:18.337702+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1185,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.332011+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:56:18.332011+0000","last_peered":"2024-08-17T17:56:18.332011+0000","last_clean":"2024-08-17T17:56:18.332011+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:56:18.332011+0000","last_undegraded":"2024-08-17T17:56:18.332011+0000","last_fullsized":"2024-08-17T17:56:18.332011+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":643,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.335181+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:56:18.335181+0000","last_peered":"2024-08-17T17:56:18.335181+0000","last_clean":"2024-08-17T17:56:18.335181+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:56:18.335181+0000","last_undegraded":"2024-08-17T17:56:18.335181+0000","last_fullsized":"2024-08-17T17:56:18.335181+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1203,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.332171+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:56:18.332171+0000","last_peered":"2024-08-17T17:56:18.332171+0000","last_clean":"2024-08-17T17:56:18.332171+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:56:18.332171+0000","last_undegraded":"2024-08-17T17:56:18.332171+0000","last_fullsized":"2024-08-17T17:56:18.332171+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":87,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.331564+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:56:18.331564+0000","last_peered":"2024-08-17T17:56:18.331564+0000","last_clean":"2024-08-17T17:56:18.331564+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:56:18.331564+0000","last_undegraded":"2024-08-17T17:56:18.331564+0000","last_fullsized":"2024-08-17T17:56:18.331564+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1219,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.332117+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:56:18.332117+0000","last_peered":"2024-08-17T17:56:18.332117+0000","last_clean":"2024-08-17T17:56:18.332117+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:56:18.332117+0000","last_undegraded":"2024-08-17T17:56:18.332117+0000","last_fullsized":"2024-08-17T17:56:18.332117+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1236,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.337482+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:56:18.337482+0000","last_peered":"2024-08-17T17:56:18.337482+0000","last_clean":"2024-08-17T17:56:18.337482+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:56:18.337482+0000","last_undegraded":"2024-08-17T17:56:18.337482+0000","last_fullsized":"2024-08-17T17:56:18.337482+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1855,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.321379+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:56:32.321379+0000","last_peered":"2024-08-17T17:56:32.321379+0000","last_clean":"2024-08-17T17:56:32.321379+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:56:32.321379+0000","last_undegraded":"2024-08-17T17:56:32.321379+0000","last_fullsized":"2024-08-17T17:56:32.321379+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1180,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.331775+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:56:18.331775+0000","last_peered":"2024-08-17T17:56:18.331775+0000","last_clean":"2024-08-17T17:56:18.331775+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:56:18.331775+0000","last_undegraded":"2024-08-17T17:56:18.331775+0000","last_fullsized":"2024-08-17T17:56:18.331775+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1906,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.346244+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:56:18.346244+0000","last_peered":"2024-08-17T17:56:18.346244+0000","last_clean":"2024-08-17T17:56:18.346244+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:56:18.346244+0000","last_undegraded":"2024-08-17T17:56:18.346244+0000","last_fullsized":"2024-08-17T17:56:18.346244+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1907,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.345862+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:56:18.345862+0000","last_peered":"2024-08-17T17:56:18.345862+0000","last_clean":"2024-08-17T17:56:18.345862+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:56:18.345862+0000","last_undegraded":"2024-08-17T17:56:18.345862+0000","last_fullsized":"2024-08-17T17:56:18.345862+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":593,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.331803+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:56:18.331803+0000","last_peered":"2024-08-17T17:56:18.331803+0000","last_clean":"2024-08-17T17:56:18.331803+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:56:18.331803+0000","last_undegraded":"2024-08-17T17:56:18.331803+0000","last_fullsized":"2024-08-17T17:56:18.331803+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1205,"reported_epoch":1235,"state":"active+clean","last_fresh":"2024-08-17T17:56:18.331889+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:56:18.331889+0000","last_peered":"2024-08-17T17:56:18.331889+0000","last_clean":"2024-08-17T17:56:18.331889+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:56:18.331889+0000","last_undegraded":"2024-08-17T17:56:18.331889+0000","last_fullsized":"2024-08-17T17:56:18.331889+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1855,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.321203+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:56:32.321203+0000","last_peered":"2024-08-17T17:56:32.321203+0000","last_clean":"2024-08-17T17:56:32.321203+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:56:32.321203+0000","last_undegraded":"2024-08-17T17:56:32.321203+0000","last_fullsized":"2024-08-17T17:56:32.321203+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":3096117272,"num_objects":47244,"num_object_clones":0,"num_object_copies":141732,"num_objects_missing_on_primary":2391,"num_objects_missing":2391,"num_objects_degraded":15837,"num_objects_misplaced":7655,"num_objects_unfound":0,"num_objects_dirty":47244,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":162366,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36741,"num_bytes_recovered":2229403648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10439569408,"data_stored":10439557192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108440,"ondisk_log_size":108440,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234428,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1633712,"kb_used_data":1332200,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92111440,"statfs":{"total":95995035648,"available":94322114560,"internally_reserved":0,"allocated":1364172800,"data_stored":1359181671,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496063,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1837172,"kb_used_data":1531500,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91907980,"statfs":{"total":95995035648,"available":94113771520,"internally_reserved":0,"allocated":1568256000,"data_stored":1563261276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757701,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326315,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1927264,"kb_used_data":1710804,"kb_used_omap":57,"kb_used_meta":216390,"kb_avail":91817888,"statfs":{"total":95995035648,"available":94021517312,"internally_reserved":0,"allocated":1751863296,"data_stored":1746887347,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":221584123},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405554,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1738216,"kb_used_data":1521692,"kb_used_omap":61,"kb_used_meta":216450,"kb_avail":92006936,"statfs":{"total":95995035648,"available":94215102464,"internally_reserved":0,"allocated":1558212608,"data_stored":1553233631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":221645752},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452088,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1325704,"kb_used_data":1149504,"kb_used_omap":54,"kb_used_meta":176137,"kb_avail":92419448,"statfs":{"total":95995035648,"available":94637514752,"internally_reserved":0,"allocated":1177092096,"data_stored":1172127544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":180364686},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474779,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1265752,"kb_used_data":1034384,"kb_used_omap":119,"kb_used_meta":231240,"kb_avail":92479400,"statfs":{"total":95995035648,"available":94698905600,"internally_reserved":0,"allocated":1059209216,"data_stored":1054229500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":236790494},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,0,1,0,0,1,4,6,2],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019336,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1043927040,"data_stored":1043922968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1160314880,"data_stored":1160314880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1544683520,"data_stored":1544683520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1737035776,"data_stored":1737031704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":965410816,"data_stored":965410816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1551503360,"data_stored":1551499288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1348796416,"data_stored":1348796416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:56:37.899 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:37.921 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:37.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:37.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:38.023 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:38.044 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:38.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:38.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:38.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:38.167 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:38.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:38.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:38.269 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:38.289 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:38.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:38.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:38.391 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:38.422 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:38.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:38.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:38.448 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:38.524 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:38.546 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:38.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:38.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:38.615 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:38 smithi130 ceph-mon[105521]: pgmap v1897: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 166 op/s; 15837/141231 objects degraded (11.214%); 7479/141231 objects misplaced (5.296%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:56:38.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:38 smithi130 ceph-mon[110068]: pgmap v1897: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 166 op/s; 15837/141231 objects degraded (11.214%); 7479/141231 objects misplaced (5.296%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:56:38.648 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:38.669 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:38.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:38.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:38.771 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:38.795 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:38.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:38.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:38 smithi179 ceph-mon[93519]: pgmap v1897: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.9 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 166 op/s; 15837/141231 objects degraded (11.214%); 7479/141231 objects misplaced (5.296%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:56:38.897 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:38.927 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:38.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:38.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:39.029 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:39.054 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:39.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:39.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:39.156 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:39.186 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:39.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:39.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:39.290 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:39.311 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:39.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:39.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:39.414 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:39.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:39.440 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:39.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:39.454 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:56:39.454 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:56:39.474 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:56:39.474 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:56:39.496 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:56:39.497 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:56:39.519 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:56:39.519 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:56:39.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:39.543 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:56:39.543 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:56:39.561 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:56:39.562 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:56:39.566 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:39.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:39.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:39.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:56:39.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:56:39.602 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:56:39.602 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:56:39.668 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:39.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:39.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:39.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:39.792 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:39.815 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:39.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:39.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:39.917 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:39.939 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:39.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:39.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:40.040 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:40.062 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:40.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:40.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:40.164 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:40.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:56:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:56:39] "GET /metrics HTTP/1.1" 200 36810 "" "Prometheus/2.51.0" 2024-08-17T17:56:40.190 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:40.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:40.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:40.292 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:40.313 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:40.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:40.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:40.414 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:40.438 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:40.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:40.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:40.540 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:40.569 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:40.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:40.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:40 smithi179 ceph-mon[93519]: pgmap v1898: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 162 op/s; 15837/139743 objects degraded (11.333%); 7479/139743 objects misplaced (5.352%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:56:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:40 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1763121470' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:56:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:56:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:56:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:56:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:40 smithi130 ceph-mon[105521]: pgmap v1898: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 162 op/s; 15837/139743 objects degraded (11.333%); 7479/139743 objects misplaced (5.352%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:56:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:40 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1763121470' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:56:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:40 smithi130 ceph-mon[110068]: pgmap v1898: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.2 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 162 op/s; 15837/139743 objects degraded (11.333%); 7479/139743 objects misplaced (5.352%); 742 KiB/s, 11 objects/s recovering 2024-08-17T17:56:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:40 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1763121470' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:56:40.670 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:40.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:40.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:40.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:40.793 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:40.814 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:40.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:40.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:40.851 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:56:40.916 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:40.937 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:40.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:40.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:41.039 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:41.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:41.060 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:41.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:41.161 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:41.187 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:41.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:41.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:41.288 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:41.311 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:41.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:41.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:41.378 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:56:41.378 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:56:41.412 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:41.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:41 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/139743 objects degraded (11.333%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:41.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:41 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/139743 objects degraded (11.333%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:41.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:41.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:41.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:41.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:41.557 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:41.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:41.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:41.658 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:41.680 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:41.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:41.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:41.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:41.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:41.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:41.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:41.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:41 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/139743 objects degraded (11.333%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:41.912 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:41.936 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:41.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:41.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:42.038 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:42.059 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:42.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:42.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:42.161 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:42.183 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:42.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:42.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:42.285 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:42.306 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:42.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:42.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:42.408 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:42.412 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:56:42.429 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:42.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:42.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:42.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:42.556 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:42.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:42.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:42.657 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:42.661 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:42 smithi130 ceph-mon[105521]: pgmap v1899: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 163 op/s; 15837/139050 objects degraded (11.389%); 7359/139050 objects misplaced (5.292%); 755 KiB/s, 11 objects/s recovering 2024-08-17T17:56:42.661 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:42.662 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:42 smithi130 ceph-mon[110068]: pgmap v1899: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 163 op/s; 15837/139050 objects degraded (11.389%); 7359/139050 objects misplaced (5.292%); 755 KiB/s, 11 objects/s recovering 2024-08-17T17:56:42.662 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:42.689 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:42.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:42.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:42.712 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:56:42.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:42.804 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:42.813 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:42.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:42.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:42.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:42 smithi179 ceph-mon[93519]: pgmap v1899: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 163 op/s; 15837/139050 objects degraded (11.389%); 7359/139050 objects misplaced (5.292%); 755 KiB/s, 11 objects/s recovering 2024-08-17T17:56:42.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:42.849 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:42.915 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:42.937 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:42.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:42.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:43.039 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:43.060 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:43.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:43.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:43.161 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:43.184 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:43.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:43.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:43.286 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:43.313 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:43.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:43.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:43.358 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T17:56:43.414 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:43.436 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:43.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:43.538 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:43.565 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:43.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:43.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:43.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:43 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/668566341' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:56:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:43 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:56:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:43 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/668566341' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:56:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:43 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:56:43.667 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:43.690 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:43.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:43.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:43.792 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:43.813 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:43.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:43.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:43 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/668566341' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:56:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:43 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:56:43.915 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:43.936 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:43.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:43.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:44.038 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:44.061 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:44.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:44.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:44.163 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:44.188 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:44.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:44.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:44.289 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:44.311 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:44.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:44.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:44.412 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:44.437 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:44.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:44.448 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:44.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:44.549 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:44.570 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:44.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:44.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:44.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:44 smithi130 ceph-mon[105521]: pgmap v1900: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 192 op/s; 15837/137454 objects degraded (11.522%); 7295/137454 objects misplaced (5.307%); 701 KiB/s, 10 objects/s recovering 2024-08-17T17:56:44.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:44 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:56:44.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:56:44.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:44 smithi130 ceph-mon[105521]: osdmap e1237: 8 total, 8 up, 6 in 2024-08-17T17:56:44.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:44 smithi130 ceph-mon[110068]: pgmap v1900: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 192 op/s; 15837/137454 objects degraded (11.522%); 7295/137454 objects misplaced (5.307%); 701 KiB/s, 10 objects/s recovering 2024-08-17T17:56:44.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:44 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:56:44.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:56:44.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:44 smithi130 ceph-mon[110068]: osdmap e1237: 8 total, 8 up, 6 in 2024-08-17T17:56:44.672 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:44.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:44.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:44.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:44.794 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:44.818 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:44.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:44.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:44.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:44 smithi179 ceph-mon[93519]: pgmap v1900: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 192 op/s; 15837/137454 objects degraded (11.522%); 7295/137454 objects misplaced (5.307%); 701 KiB/s, 10 objects/s recovering 2024-08-17T17:56:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:44 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:56:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:56:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:44 smithi179 ceph-mon[93519]: osdmap e1237: 8 total, 8 up, 6 in 2024-08-17T17:56:44.920 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:44.942 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:44.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:44.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:45.043 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:45.065 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:45.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:45.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:45.166 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:45.187 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:45.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:45.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:45.289 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:45.312 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:45.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:45.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:45.414 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:45.440 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:45.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:45.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:45.542 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:45.564 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:45.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:45.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:45.666 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:45.689 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:45.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:45.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:45.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:45.813 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:45.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:45.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:45.915 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:45.938 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:45.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:45.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:46.040 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:46.062 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:46.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:46.163 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:46.185 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:46.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:46.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:46.287 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:46.309 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:46.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:46.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:46.410 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:46.432 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:46.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:46.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:46.484 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:46 smithi130 ceph-mon[105521]: pgmap v1902: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 185 op/s; 15837/136236 objects degraded (11.625%); 7295/136236 objects misplaced (5.355%); 755 KiB/s, 11 objects/s recovering 2024-08-17T17:56:46.484 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/136236 objects degraded (11.625%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:46.484 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:46 smithi130 ceph-mon[110068]: pgmap v1902: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 185 op/s; 15837/136236 objects degraded (11.625%); 7295/136236 objects misplaced (5.355%); 755 KiB/s, 11 objects/s recovering 2024-08-17T17:56:46.484 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/136236 objects degraded (11.625%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:46.533 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:46.555 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:46.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:46.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:46.645 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:46 smithi179 ceph-mon[93519]: pgmap v1902: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 185 op/s; 15837/136236 objects degraded (11.625%); 7295/136236 objects misplaced (5.355%); 755 KiB/s, 11 objects/s recovering 2024-08-17T17:56:46.645 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/136236 objects degraded (11.625%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:46.657 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:46.679 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:46.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:46.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:46.780 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:46.802 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:46.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:46.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:46.904 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:46.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:46.927 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:46.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:46.931 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:56:47.028 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:47.049 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:47.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:47.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:47.151 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:47.174 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:47.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:47.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:47.276 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:47.302 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:47.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:47.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:47.403 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:47.433 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:47.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:47.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:47.445 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:56:47.445 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:56:47.535 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:47.563 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:47.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:47.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:47.665 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:47.686 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:47.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:47.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:47.788 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:47.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:47.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:47.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:47.912 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:47.939 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:47.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:47.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:48.041 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:48.062 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:48.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:48.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:48.163 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:48.184 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:48.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:48.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:48.286 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:48.307 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:48.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:48.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:48.409 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:48.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:48 smithi130 ceph-mon[105521]: pgmap v1903: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 191 op/s; 15837/135477 objects degraded (11.690%); 7104/135477 objects misplaced (5.244%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:56:48.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:48 smithi130 ceph-mon[110068]: pgmap v1903: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 191 op/s; 15837/135477 objects degraded (11.690%); 7104/135477 objects misplaced (5.244%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:56:48.430 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:48.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:48.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:48.531 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:48.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:48.553 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:48.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:48.654 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:48.675 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:48.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:48.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:48.777 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:48.800 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:48.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:48.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:48 smithi179 ceph-mon[93519]: pgmap v1903: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 191 op/s; 15837/135477 objects degraded (11.690%); 7104/135477 objects misplaced (5.244%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:56:48.902 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:48.923 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:56:48.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:48.925 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:48.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:49.026 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:49.050 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:49.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:49.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:49.152 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:49.192 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:49.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:49.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:49.294 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:49.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:49.317 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:49.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:49.418 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:49.442 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:56:49.442 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:56:49.443 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:49.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:49.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:49.462 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:56:49.462 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:56:49.479 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:56:49.479 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:56:49.496 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:56:49.496 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:56:49.512 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:56:49.512 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:56:49.528 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:56:49.529 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:56:49.544 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:49.545 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:56:49.545 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:56:49.560 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:56:49.561 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:56:49.570 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:49.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:49.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:49.672 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:49.693 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:49.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:49.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:49.794 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:49.827 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:49.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:49.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:49.928 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:49.950 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:49.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:49.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:50.033 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:50.044 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:56:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:56:49] "GET /metrics HTTP/1.1" 200 36806 "" "Prometheus/2.51.0" 2024-08-17T17:56:50.045 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:56:50.046 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:50.048 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:56:50.051 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:50.086 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:50.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:50.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:50.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:50.210 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:50.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:50.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:50.312 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:50.333 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:50.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:50.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:50.434 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:50.455 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:50.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:50.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:50.557 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:50.578 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:50.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:50.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:50 smithi179 ceph-mon[93519]: pgmap v1904: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 213 op/s; 15837/133323 objects degraded (11.879%); 7104/133323 objects misplaced (5.328%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:56:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:50 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2155749032' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:56:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:50 smithi179 ceph-mon[93519]: from='client.45857 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:56:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:56:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:56:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:56:50.627 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:50 smithi130 ceph-mon[105521]: pgmap v1904: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 213 op/s; 15837/133323 objects degraded (11.879%); 7104/133323 objects misplaced (5.328%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:56:50.627 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:50 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2155749032' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:56:50.627 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:50 smithi130 ceph-mon[105521]: from='client.45857 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:56:50.627 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:50 smithi130 ceph-mon[110068]: pgmap v1904: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 213 op/s; 15837/133323 objects degraded (11.879%); 7104/133323 objects misplaced (5.328%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:56:50.627 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:50 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2155749032' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:56:50.627 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:50 smithi130 ceph-mon[110068]: from='client.45857 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:56:50.680 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:50.701 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:50.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:50.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:50.803 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:50.823 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:50.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:50.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:50.925 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:50.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:50.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:50.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:51.052 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:51.074 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:51.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:51.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:51.175 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:51.196 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:51.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:51.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:51.298 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:51.320 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:51.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:51.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:51.422 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:51.443 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:51.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:51.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:51.545 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:51.567 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:51.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:51.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:51.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/133323 objects degraded (11.879%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:51.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/133323 objects degraded (11.879%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:51.669 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:51.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:51.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:51.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:51.793 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:51.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:51.815 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:51.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/133323 objects degraded (11.879%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:51.916 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:51.946 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:51.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:51.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:52.048 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:52.070 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:52.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:52.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:52.172 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:52.193 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:52.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:52.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:52.295 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:52.319 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:52.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:52.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:52.421 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:52.450 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:52.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:52.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:52.551 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:52.573 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:52.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:52.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:52.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:52 smithi130 ceph-mon[105521]: pgmap v1905: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 203 op/s; 15837/132954 objects degraded (11.912%); 7104/132954 objects misplaced (5.343%); 390 KiB/s, 6 objects/s recovering 2024-08-17T17:56:52.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:52 smithi130 ceph-mon[110068]: pgmap v1905: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 203 op/s; 15837/132954 objects degraded (11.912%); 7104/132954 objects misplaced (5.343%); 390 KiB/s, 6 objects/s recovering 2024-08-17T17:56:52.667 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:56:52.675 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:52.736 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:52.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:52.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:52.820 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1904,"stamp":"2024-08-17T17:56:49.093062+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2911311845,"num_objects":44441,"num_object_clones":0,"num_object_copies":133323,"num_objects_missing_on_primary":2215,"num_objects_missing":2215,"num_objects_degraded":15837,"num_objects_misplaced":7104,"num_objects_unfound":0,"num_objects_dirty":44441,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":165473,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36949,"num_bytes_recovered":2242911245,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108776,"ondisk_log_size":108776,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9300624,"kb_used_data":7821780,"kb_used_omap":295,"kb_used_meta":1478488,"kb_avail":553170288,"statfs":{"total":575970213888,"available":566446374912,"internally_reserved":0,"allocated":8009502720,"data_stored":7979558185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1513972175},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-140247040,"num_objects":-2140,"num_object_clones":0,"num_object_copies":-6420,"num_objects_missing_on_primary":-117,"num_objects_missing":-117,"num_objects_degraded":0,"num_objects_misplaced":-375,"num_objects_unfound":0,"num_objects_dirty":-2140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2140,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":121,"num_bytes_recovered":7929856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002737"},"pg_stats":[{"pgid":"45.f","version":"1237'10320","reported_seq":20874,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:46.020983+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:56:46.020983+0000","last_peered":"2024-08-17T17:56:46.020983+0000","last_clean":"2024-08-17T17:56:46.020983+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:56:46.020983+0000","last_undegraded":"2024-08-17T17:56:46.020983+0000","last_fullsized":"2024-08-17T17:56:46.020983+0000","mapping_epoch":1194,"log_start":"1174'3431","ondisk_log_start":"1174'3431","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6889,"log_dups_size":0,"ondisk_log_size":6889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186384384,"num_objects":2844,"num_object_clones":0,"num_object_copies":8532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10320,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1237'10271","reported_seq":20748,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:47.935571+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:56:47.935571+0000","last_peered":"2024-08-17T17:56:47.935571+0000","last_clean":"2024-08-17T17:56:47.935571+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:56:47.935571+0000","last_undegraded":"2024-08-17T17:56:47.935571+0000","last_fullsized":"2024-08-17T17:56:47.935571+0000","mapping_epoch":1211,"log_start":"1174'3232","ondisk_log_start":"1174'3232","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7039,"log_dups_size":0,"ondisk_log_size":7039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178454528,"num_objects":2723,"num_object_clones":0,"num_object_copies":8169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10271,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1237'10590","reported_seq":21452,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:48.594698+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:56:48.594698+0000","last_peered":"2024-08-17T17:56:48.594698+0000","last_clean":"2024-08-17T17:56:48.594698+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:56:48.594698+0000","last_undegraded":"2024-08-17T17:56:48.594698+0000","last_fullsized":"2024-08-17T17:56:48.594698+0000","mapping_epoch":1205,"log_start":"1175'4668","ondisk_log_start":"1175'4668","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5922,"log_dups_size":0,"ondisk_log_size":5922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182648856,"num_objects":2788,"num_object_clones":0,"num_object_copies":8364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2788,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10590,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1237'10278","reported_seq":10356,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:47.896927+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:56:47.896927+0000","last_peered":"2024-08-17T17:56:47.896927+0000","last_clean":"2024-08-17T17:56:47.896927+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:56:47.896927+0000","last_undegraded":"2024-08-17T17:56:47.896927+0000","last_fullsized":"2024-08-17T17:56:47.896927+0000","mapping_epoch":1167,"log_start":"1174'3589","ondisk_log_start":"1174'3589","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6689,"log_dups_size":0,"ondisk_log_size":6689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178388992,"num_objects":2722,"num_object_clones":0,"num_object_copies":8166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10278,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1237'10352","reported_seq":15620,"reported_epoch":1237,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:46.694889+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:56:46.694889+0000","last_peered":"2024-08-17T17:56:46.694889+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:56:46.694889+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1175'4426","ondisk_log_start":"1175'4426","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5926,"log_dups_size":0,"ondisk_log_size":5926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181010432,"num_objects":2762,"num_object_clones":0,"num_object_copies":8286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":2762,"num_objects_unfound":0,"num_objects_dirty":2762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10352,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":2244}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1237'10413","reported_seq":10491,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:47.885321+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:56:47.885321+0000","last_peered":"2024-08-17T17:56:47.885321+0000","last_clean":"2024-08-17T17:56:47.885321+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:56:47.885321+0000","last_undegraded":"2024-08-17T17:56:47.885321+0000","last_fullsized":"2024-08-17T17:56:47.885321+0000","mapping_epoch":1167,"log_start":"1174'3677","ondisk_log_start":"1174'3677","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6736,"log_dups_size":0,"ondisk_log_size":6736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175702016,"num_objects":2681,"num_object_clones":0,"num_object_copies":8043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10413,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1237'10423","reported_seq":10501,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:47.889139+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:56:47.889139+0000","last_peered":"2024-08-17T17:56:47.889139+0000","last_clean":"2024-08-17T17:56:47.889139+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:56:47.889139+0000","last_undegraded":"2024-08-17T17:56:47.889139+0000","last_fullsized":"2024-08-17T17:56:47.889139+0000","mapping_epoch":1167,"log_start":"1175'3669","ondisk_log_start":"1175'3669","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6754,"log_dups_size":0,"ondisk_log_size":6754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183697408,"num_objects":2803,"num_object_clones":0,"num_object_copies":8409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10423,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1237'10267","reported_seq":19752,"reported_epoch":1237,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:46.690664+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:56:46.690664+0000","last_peered":"2024-08-17T17:56:46.690664+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:56:46.690664+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1175'4375","ondisk_log_start":"1175'4375","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5892,"log_dups_size":0,"ondisk_log_size":5892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178454528,"num_objects":2723,"num_object_clones":0,"num_object_copies":8169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10267,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3108,"num_bytes_recovered":203685888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":1403},{"shards":"1,7","objects":656}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1237'10287","reported_seq":10375,"reported_epoch":1237,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:47.901683+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:56:47.901683+0000","last_peered":"2024-08-17T17:56:47.901683+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:56:47.901683+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1175'3577","ondisk_log_start":"1175'3577","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6710,"log_dups_size":0,"ondisk_log_size":6710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182779904,"num_objects":2789,"num_object_clones":0,"num_object_copies":8367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10287,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":2789},{"shards":"1,6","objects":-3044}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1237'10372","reported_seq":13473,"reported_epoch":1237,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:48.205418+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:56:48.205418+0000","last_peered":"2024-08-17T17:56:48.205418+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:56:48.205418+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1175'4406","ondisk_log_start":"1175'4406","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5966,"log_dups_size":0,"ondisk_log_size":5966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183631872,"num_objects":2802,"num_object_clones":0,"num_object_copies":8406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10372,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":1241}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1237'10206","reported_seq":10362,"reported_epoch":1237,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:56:43.780917+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:56:43.780917+0000","last_peered":"2024-08-17T17:56:43.780917+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:56:43.780917+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1170'1100","ondisk_log_start":"1170'1100","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9106,"log_dups_size":0,"ondisk_log_size":9106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187957248,"num_objects":2868,"num_object_clones":0,"num_object_copies":8604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10206,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-2285}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1237'10232","reported_seq":16449,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:44.050783+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:56:44.050783+0000","last_peered":"2024-08-17T17:56:44.050783+0000","last_clean":"2024-08-17T17:56:44.050783+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:56:44.050783+0000","last_undegraded":"2024-08-17T17:56:44.050783+0000","last_fullsized":"2024-08-17T17:56:44.050783+0000","mapping_epoch":1183,"log_start":"1170'1100","ondisk_log_start":"1170'1100","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9132,"log_dups_size":0,"ondisk_log_size":9132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184418304,"num_objects":2814,"num_object_clones":0,"num_object_copies":8442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10232,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1237'10342","reported_seq":10497,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:48.749904+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:56:48.749904+0000","last_peered":"2024-08-17T17:56:48.749904+0000","last_clean":"2024-08-17T17:56:48.749904+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:56:48.749904+0000","last_undegraded":"2024-08-17T17:56:48.749904+0000","last_fullsized":"2024-08-17T17:56:48.749904+0000","mapping_epoch":1167,"log_start":"1175'4391","ondisk_log_start":"1175'4391","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5951,"log_dups_size":0,"ondisk_log_size":5951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184680448,"num_objects":2818,"num_object_clones":0,"num_object_copies":8454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10342,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1237'10210","reported_seq":10288,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:47.931391+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:56:47.931391+0000","last_peered":"2024-08-17T17:56:47.931391+0000","last_clean":"2024-08-17T17:56:47.931391+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:56:47.931391+0000","last_undegraded":"2024-08-17T17:56:47.931391+0000","last_fullsized":"2024-08-17T17:56:47.931391+0000","mapping_epoch":1167,"log_start":"1174'3225","ondisk_log_start":"1174'3225","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6985,"log_dups_size":0,"ondisk_log_size":6985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178520064,"num_objects":2724,"num_object_clones":0,"num_object_copies":8172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10210,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1237'10509","reported_seq":25910,"reported_epoch":1237,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:56:46.021540+0000","last_change":"2024-08-17T17:55:37.941989+0000","last_active":"2024-08-17T17:56:46.021540+0000","last_peered":"2024-08-17T17:56:46.021540+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:56:46.021540+0000","last_undegraded":"2024-08-17T17:56:46.021540+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1174'3591","ondisk_log_start":"1174'3591","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6918,"log_dups_size":0,"ondisk_log_size":6918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187105280,"num_objects":2855,"num_object_clones":0,"num_object_copies":8565,"num_objects_missing_on_primary":2215,"num_objects_missing":2215,"num_objects_degraded":0,"num_objects_misplaced":4342,"num_objects_unfound":0,"num_objects_dirty":2855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10509,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5934,"num_bytes_recovered":388890624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":10},{"shards":"0,4,6","objects":1356},{"shards":"0,6","objects":1},{"shards":"4,5,6","objects":1488}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1237'10123","reported_seq":23464,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:48.714930+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:56:48.714930+0000","last_peered":"2024-08-17T17:56:48.714930+0000","last_clean":"2024-08-17T17:56:48.714930+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:56:48.714930+0000","last_undegraded":"2024-08-17T17:56:48.714930+0000","last_fullsized":"2024-08-17T17:56:48.714930+0000","mapping_epoch":1196,"log_start":"1175'4169","ondisk_log_start":"1175'4169","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5954,"log_dups_size":0,"ondisk_log_size":5954,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176881664,"num_objects":2699,"num_object_clones":0,"num_object_copies":8097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10123,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1932,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.370284+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:56:43.370284+0000","last_peered":"2024-08-17T17:56:43.370284+0000","last_clean":"2024-08-17T17:56:43.370284+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:56:43.370284+0000","last_undegraded":"2024-08-17T17:56:43.370284+0000","last_fullsized":"2024-08-17T17:56:43.370284+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":514,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.356709+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:56:32.356709+0000","last_peered":"2024-08-17T17:56:32.356709+0000","last_clean":"2024-08-17T17:56:32.356709+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:56:32.356709+0000","last_undegraded":"2024-08-17T17:56:32.356709+0000","last_fullsized":"2024-08-17T17:56:32.356709+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1851,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.362806+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:56:43.362806+0000","last_peered":"2024-08-17T17:56:43.362806+0000","last_clean":"2024-08-17T17:56:43.362806+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:56:43.362806+0000","last_undegraded":"2024-08-17T17:56:43.362806+0000","last_fullsized":"2024-08-17T17:56:43.362806+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1875,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.360555+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:56:43.360555+0000","last_peered":"2024-08-17T17:56:43.360555+0000","last_clean":"2024-08-17T17:56:43.360555+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:56:43.360555+0000","last_undegraded":"2024-08-17T17:56:43.360555+0000","last_fullsized":"2024-08-17T17:56:43.360555+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":76,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.357348+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:56:32.357348+0000","last_peered":"2024-08-17T17:56:32.357348+0000","last_clean":"2024-08-17T17:56:32.357348+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:56:32.357348+0000","last_undegraded":"2024-08-17T17:56:32.357348+0000","last_fullsized":"2024-08-17T17:56:32.357348+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1268,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.360672+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:56:43.360672+0000","last_peered":"2024-08-17T17:56:43.360672+0000","last_clean":"2024-08-17T17:56:43.360672+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:56:43.360672+0000","last_undegraded":"2024-08-17T17:56:43.360672+0000","last_fullsized":"2024-08-17T17:56:43.360672+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":126,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.370552+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:56:43.370552+0000","last_peered":"2024-08-17T17:56:43.370552+0000","last_clean":"2024-08-17T17:56:43.370552+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:56:43.370552+0000","last_undegraded":"2024-08-17T17:56:43.370552+0000","last_fullsized":"2024-08-17T17:56:43.370552+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":646,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.360865+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:56:43.360865+0000","last_peered":"2024-08-17T17:56:43.360865+0000","last_clean":"2024-08-17T17:56:43.360865+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:56:43.360865+0000","last_undegraded":"2024-08-17T17:56:43.360865+0000","last_fullsized":"2024-08-17T17:56:43.360865+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1890,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.368893+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:56:43.368893+0000","last_peered":"2024-08-17T17:56:43.368893+0000","last_clean":"2024-08-17T17:56:43.368893+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:56:43.368893+0000","last_undegraded":"2024-08-17T17:56:43.368893+0000","last_fullsized":"2024-08-17T17:56:43.368893+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1892,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.374197+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:56:43.374197+0000","last_peered":"2024-08-17T17:56:43.374197+0000","last_clean":"2024-08-17T17:56:43.374197+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:56:43.374197+0000","last_undegraded":"2024-08-17T17:56:43.374197+0000","last_fullsized":"2024-08-17T17:56:43.374197+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":111,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.369630+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:56:43.369630+0000","last_peered":"2024-08-17T17:56:43.369630+0000","last_clean":"2024-08-17T17:56:43.369630+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:56:43.369630+0000","last_undegraded":"2024-08-17T17:56:43.369630+0000","last_fullsized":"2024-08-17T17:56:43.369630+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1864,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.362170+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:56:43.362170+0000","last_peered":"2024-08-17T17:56:43.362170+0000","last_clean":"2024-08-17T17:56:43.362170+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:56:43.362170+0000","last_undegraded":"2024-08-17T17:56:43.362170+0000","last_fullsized":"2024-08-17T17:56:43.362170+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1918,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.369354+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:56:43.369354+0000","last_peered":"2024-08-17T17:56:43.369354+0000","last_clean":"2024-08-17T17:56:43.369354+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:56:43.369354+0000","last_undegraded":"2024-08-17T17:56:43.369354+0000","last_fullsized":"2024-08-17T17:56:43.369354+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1913,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.372078+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:56:43.372078+0000","last_peered":"2024-08-17T17:56:43.372078+0000","last_clean":"2024-08-17T17:56:43.372078+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:56:43.372078+0000","last_undegraded":"2024-08-17T17:56:43.372078+0000","last_fullsized":"2024-08-17T17:56:43.372078+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1886,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.361212+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:56:43.361212+0000","last_peered":"2024-08-17T17:56:43.361212+0000","last_clean":"2024-08-17T17:56:43.361212+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:56:43.361212+0000","last_undegraded":"2024-08-17T17:56:43.361212+0000","last_fullsized":"2024-08-17T17:56:43.361212+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1211,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.357372+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:56:32.357372+0000","last_peered":"2024-08-17T17:56:32.357372+0000","last_clean":"2024-08-17T17:56:32.357372+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:56:32.357372+0000","last_undegraded":"2024-08-17T17:56:32.357372+0000","last_fullsized":"2024-08-17T17:56:32.357372+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1857,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.361000+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:56:43.361000+0000","last_peered":"2024-08-17T17:56:43.361000+0000","last_clean":"2024-08-17T17:56:43.361000+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:56:43.361000+0000","last_undegraded":"2024-08-17T17:56:43.361000+0000","last_fullsized":"2024-08-17T17:56:43.361000+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1907,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.370340+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:56:43.370340+0000","last_peered":"2024-08-17T17:56:43.370340+0000","last_clean":"2024-08-17T17:56:43.370340+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:56:43.370340+0000","last_undegraded":"2024-08-17T17:56:43.370340+0000","last_fullsized":"2024-08-17T17:56:43.370340+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1813,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.360462+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:56:43.360462+0000","last_peered":"2024-08-17T17:56:43.360462+0000","last_clean":"2024-08-17T17:56:43.360462+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:56:43.360462+0000","last_undegraded":"2024-08-17T17:56:43.360462+0000","last_fullsized":"2024-08-17T17:56:43.360462+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":530,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.337726+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:56:32.337726+0000","last_peered":"2024-08-17T17:56:32.337726+0000","last_clean":"2024-08-17T17:56:32.337726+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:56:32.337726+0000","last_undegraded":"2024-08-17T17:56:32.337726+0000","last_fullsized":"2024-08-17T17:56:32.337726+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1207,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.337583+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:56:32.337583+0000","last_peered":"2024-08-17T17:56:32.337583+0000","last_clean":"2024-08-17T17:56:32.337583+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:56:32.337583+0000","last_undegraded":"2024-08-17T17:56:32.337583+0000","last_fullsized":"2024-08-17T17:56:32.337583+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1912,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.369729+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:56:43.369729+0000","last_peered":"2024-08-17T17:56:43.369729+0000","last_clean":"2024-08-17T17:56:43.369729+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:56:43.369729+0000","last_undegraded":"2024-08-17T17:56:43.369729+0000","last_fullsized":"2024-08-17T17:56:43.369729+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":368,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.339876+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:56:32.339876+0000","last_peered":"2024-08-17T17:56:32.339876+0000","last_clean":"2024-08-17T17:56:32.339876+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:56:32.339876+0000","last_undegraded":"2024-08-17T17:56:32.339876+0000","last_fullsized":"2024-08-17T17:56:32.339876+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":514,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.357064+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:56:32.357064+0000","last_peered":"2024-08-17T17:56:32.357064+0000","last_clean":"2024-08-17T17:56:32.357064+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:56:32.357064+0000","last_undegraded":"2024-08-17T17:56:32.357064+0000","last_fullsized":"2024-08-17T17:56:32.357064+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1192,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.356917+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:56:32.356917+0000","last_peered":"2024-08-17T17:56:32.356917+0000","last_clean":"2024-08-17T17:56:32.356917+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:56:32.356917+0000","last_undegraded":"2024-08-17T17:56:32.356917+0000","last_fullsized":"2024-08-17T17:56:32.356917+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1200,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.357445+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:56:32.357445+0000","last_peered":"2024-08-17T17:56:32.357445+0000","last_clean":"2024-08-17T17:56:32.357445+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:56:32.357445+0000","last_undegraded":"2024-08-17T17:56:32.357445+0000","last_fullsized":"2024-08-17T17:56:32.357445+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1959,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.371729+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:56:43.371729+0000","last_peered":"2024-08-17T17:56:43.371729+0000","last_clean":"2024-08-17T17:56:43.371729+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:56:43.371729+0000","last_undegraded":"2024-08-17T17:56:43.371729+0000","last_fullsized":"2024-08-17T17:56:43.371729+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1843,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.361142+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:56:43.361142+0000","last_peered":"2024-08-17T17:56:43.361142+0000","last_clean":"2024-08-17T17:56:43.361142+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:56:43.361142+0000","last_undegraded":"2024-08-17T17:56:43.361142+0000","last_fullsized":"2024-08-17T17:56:43.361142+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1918,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.371118+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:56:43.371118+0000","last_peered":"2024-08-17T17:56:43.371118+0000","last_clean":"2024-08-17T17:56:43.371118+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:56:43.371118+0000","last_undegraded":"2024-08-17T17:56:43.371118+0000","last_fullsized":"2024-08-17T17:56:43.371118+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1845,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.361840+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:56:43.361840+0000","last_peered":"2024-08-17T17:56:43.361840+0000","last_clean":"2024-08-17T17:56:43.361840+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:56:43.361840+0000","last_undegraded":"2024-08-17T17:56:43.361840+0000","last_fullsized":"2024-08-17T17:56:43.361840+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1877,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.368998+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:56:43.368998+0000","last_peered":"2024-08-17T17:56:43.368998+0000","last_clean":"2024-08-17T17:56:43.368998+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:56:43.368998+0000","last_undegraded":"2024-08-17T17:56:43.368998+0000","last_fullsized":"2024-08-17T17:56:43.368998+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1227,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.337741+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:56:32.337741+0000","last_peered":"2024-08-17T17:56:32.337741+0000","last_clean":"2024-08-17T17:56:32.337741+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:56:32.337741+0000","last_undegraded":"2024-08-17T17:56:32.337741+0000","last_fullsized":"2024-08-17T17:56:32.337741+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1924,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.370714+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:56:43.370714+0000","last_peered":"2024-08-17T17:56:43.370714+0000","last_clean":"2024-08-17T17:56:43.370714+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:56:43.370714+0000","last_undegraded":"2024-08-17T17:56:43.370714+0000","last_fullsized":"2024-08-17T17:56:43.370714+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":111,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.369306+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:56:43.369306+0000","last_peered":"2024-08-17T17:56:43.369306+0000","last_clean":"2024-08-17T17:56:43.369306+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:56:43.369306+0000","last_undegraded":"2024-08-17T17:56:43.369306+0000","last_fullsized":"2024-08-17T17:56:43.369306+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1943,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.370638+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:56:43.370638+0000","last_peered":"2024-08-17T17:56:43.370638+0000","last_clean":"2024-08-17T17:56:43.370638+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:56:43.370638+0000","last_undegraded":"2024-08-17T17:56:43.370638+0000","last_fullsized":"2024-08-17T17:56:43.370638+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":680,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.361959+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:56:43.361959+0000","last_peered":"2024-08-17T17:56:43.361959+0000","last_clean":"2024-08-17T17:56:43.361959+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:56:43.361959+0000","last_undegraded":"2024-08-17T17:56:43.361959+0000","last_fullsized":"2024-08-17T17:56:43.361959+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1924,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.370793+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:56:43.370793+0000","last_peered":"2024-08-17T17:56:43.370793+0000","last_clean":"2024-08-17T17:56:43.370793+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:56:43.370793+0000","last_undegraded":"2024-08-17T17:56:43.370793+0000","last_fullsized":"2024-08-17T17:56:43.370793+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1253,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.368702+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:56:43.368702+0000","last_peered":"2024-08-17T17:56:43.368702+0000","last_clean":"2024-08-17T17:56:43.368702+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:56:43.368702+0000","last_undegraded":"2024-08-17T17:56:43.368702+0000","last_fullsized":"2024-08-17T17:56:43.368702+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":171,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.370721+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:56:43.370721+0000","last_peered":"2024-08-17T17:56:43.370721+0000","last_clean":"2024-08-17T17:56:43.370721+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:56:43.370721+0000","last_undegraded":"2024-08-17T17:56:43.370721+0000","last_fullsized":"2024-08-17T17:56:43.370721+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1935,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.370195+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:56:43.370195+0000","last_peered":"2024-08-17T17:56:43.370195+0000","last_clean":"2024-08-17T17:56:43.370195+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:56:43.370195+0000","last_undegraded":"2024-08-17T17:56:43.370195+0000","last_fullsized":"2024-08-17T17:56:43.370195+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1913,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.372248+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:56:43.372248+0000","last_peered":"2024-08-17T17:56:43.372248+0000","last_clean":"2024-08-17T17:56:43.372248+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:56:43.372248+0000","last_undegraded":"2024-08-17T17:56:43.372248+0000","last_fullsized":"2024-08-17T17:56:43.372248+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1933,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.373595+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:56:43.373595+0000","last_peered":"2024-08-17T17:56:43.373595+0000","last_clean":"2024-08-17T17:56:43.373595+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:56:43.373595+0000","last_undegraded":"2024-08-17T17:56:43.373595+0000","last_fullsized":"2024-08-17T17:56:43.373595+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1938,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.372122+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:56:43.372122+0000","last_peered":"2024-08-17T17:56:43.372122+0000","last_clean":"2024-08-17T17:56:43.372122+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:56:43.372122+0000","last_undegraded":"2024-08-17T17:56:43.372122+0000","last_fullsized":"2024-08-17T17:56:43.372122+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1919,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.437130+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:56:43.437130+0000","last_peered":"2024-08-17T17:56:43.437130+0000","last_clean":"2024-08-17T17:56:43.437130+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:56:43.437130+0000","last_undegraded":"2024-08-17T17:56:43.437130+0000","last_fullsized":"2024-08-17T17:56:43.437130+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1234,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.337445+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:56:32.337445+0000","last_peered":"2024-08-17T17:56:32.337445+0000","last_clean":"2024-08-17T17:56:32.337445+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:56:32.337445+0000","last_undegraded":"2024-08-17T17:56:32.337445+0000","last_fullsized":"2024-08-17T17:56:32.337445+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1180,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.357357+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:56:32.357357+0000","last_peered":"2024-08-17T17:56:32.357357+0000","last_clean":"2024-08-17T17:56:32.357357+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:56:32.357357+0000","last_undegraded":"2024-08-17T17:56:32.357357+0000","last_fullsized":"2024-08-17T17:56:32.357357+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1828,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.360622+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:56:43.360622+0000","last_peered":"2024-08-17T17:56:43.360622+0000","last_clean":"2024-08-17T17:56:43.360622+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:56:43.360622+0000","last_undegraded":"2024-08-17T17:56:43.360622+0000","last_fullsized":"2024-08-17T17:56:43.360622+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":159,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.374034+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:56:43.374034+0000","last_peered":"2024-08-17T17:56:43.374034+0000","last_clean":"2024-08-17T17:56:43.374034+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:56:43.374034+0000","last_undegraded":"2024-08-17T17:56:43.374034+0000","last_fullsized":"2024-08-17T17:56:43.374034+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1546,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.337343+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:56:32.337343+0000","last_peered":"2024-08-17T17:56:32.337343+0000","last_clean":"2024-08-17T17:56:32.337343+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:56:32.337343+0000","last_undegraded":"2024-08-17T17:56:32.337343+0000","last_fullsized":"2024-08-17T17:56:32.337343+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1919,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.370316+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:56:43.370316+0000","last_peered":"2024-08-17T17:56:43.370316+0000","last_clean":"2024-08-17T17:56:43.370316+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:56:43.370316+0000","last_undegraded":"2024-08-17T17:56:43.370316+0000","last_fullsized":"2024-08-17T17:56:43.370316+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1859,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.361062+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:56:43.361062+0000","last_peered":"2024-08-17T17:56:43.361062+0000","last_clean":"2024-08-17T17:56:43.361062+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:56:43.361062+0000","last_undegraded":"2024-08-17T17:56:43.361062+0000","last_fullsized":"2024-08-17T17:56:43.361062+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":514,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.357286+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:56:32.357286+0000","last_peered":"2024-08-17T17:56:32.357286+0000","last_clean":"2024-08-17T17:56:32.357286+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:56:32.357286+0000","last_undegraded":"2024-08-17T17:56:32.357286+0000","last_fullsized":"2024-08-17T17:56:32.357286+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":598,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.373529+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:56:43.373529+0000","last_peered":"2024-08-17T17:56:43.373529+0000","last_clean":"2024-08-17T17:56:43.373529+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:56:43.373529+0000","last_undegraded":"2024-08-17T17:56:43.373529+0000","last_fullsized":"2024-08-17T17:56:43.373529+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":614,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.374180+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:56:43.374180+0000","last_peered":"2024-08-17T17:56:43.374180+0000","last_clean":"2024-08-17T17:56:43.374180+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:56:43.374180+0000","last_undegraded":"2024-08-17T17:56:43.374180+0000","last_fullsized":"2024-08-17T17:56:43.374180+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1198,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.357199+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:56:32.357199+0000","last_peered":"2024-08-17T17:56:32.357199+0000","last_clean":"2024-08-17T17:56:32.357199+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:56:32.357199+0000","last_undegraded":"2024-08-17T17:56:32.357199+0000","last_fullsized":"2024-08-17T17:56:32.357199+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":630,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.361478+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:56:43.361478+0000","last_peered":"2024-08-17T17:56:43.361478+0000","last_clean":"2024-08-17T17:56:43.361478+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:56:43.361478+0000","last_undegraded":"2024-08-17T17:56:43.361478+0000","last_fullsized":"2024-08-17T17:56:43.361478+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1856,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.360912+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:56:43.360912+0000","last_peered":"2024-08-17T17:56:43.360912+0000","last_clean":"2024-08-17T17:56:43.360912+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:56:43.360912+0000","last_undegraded":"2024-08-17T17:56:43.360912+0000","last_fullsized":"2024-08-17T17:56:43.360912+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1951,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.369081+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:56:43.369081+0000","last_peered":"2024-08-17T17:56:43.369081+0000","last_clean":"2024-08-17T17:56:43.369081+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:56:43.369081+0000","last_undegraded":"2024-08-17T17:56:43.369081+0000","last_fullsized":"2024-08-17T17:56:43.369081+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":662,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.360468+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:56:43.360468+0000","last_peered":"2024-08-17T17:56:43.360468+0000","last_clean":"2024-08-17T17:56:43.360468+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:56:43.360468+0000","last_undegraded":"2024-08-17T17:56:43.360468+0000","last_fullsized":"2024-08-17T17:56:43.360468+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1187,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.357029+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:56:32.357029+0000","last_peered":"2024-08-17T17:56:32.357029+0000","last_clean":"2024-08-17T17:56:32.357029+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:56:32.357029+0000","last_undegraded":"2024-08-17T17:56:32.357029+0000","last_fullsized":"2024-08-17T17:56:32.357029+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1186,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.337668+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:56:32.337668+0000","last_peered":"2024-08-17T17:56:32.337668+0000","last_clean":"2024-08-17T17:56:32.337668+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:56:32.337668+0000","last_undegraded":"2024-08-17T17:56:32.337668+0000","last_fullsized":"2024-08-17T17:56:32.337668+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":647,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.369445+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:56:43.369445+0000","last_peered":"2024-08-17T17:56:43.369445+0000","last_clean":"2024-08-17T17:56:43.369445+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:56:43.369445+0000","last_undegraded":"2024-08-17T17:56:43.369445+0000","last_fullsized":"2024-08-17T17:56:43.369445+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1204,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.337857+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:56:32.337857+0000","last_peered":"2024-08-17T17:56:32.337857+0000","last_clean":"2024-08-17T17:56:32.337857+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:56:32.337857+0000","last_undegraded":"2024-08-17T17:56:32.337857+0000","last_fullsized":"2024-08-17T17:56:32.337857+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":88,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.337261+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:56:32.337261+0000","last_peered":"2024-08-17T17:56:32.337261+0000","last_clean":"2024-08-17T17:56:32.337261+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:56:32.337261+0000","last_undegraded":"2024-08-17T17:56:32.337261+0000","last_fullsized":"2024-08-17T17:56:32.337261+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1220,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.337707+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:56:32.337707+0000","last_peered":"2024-08-17T17:56:32.337707+0000","last_clean":"2024-08-17T17:56:32.337707+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:56:32.337707+0000","last_undegraded":"2024-08-17T17:56:32.337707+0000","last_fullsized":"2024-08-17T17:56:32.337707+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1237,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.356888+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:56:32.356888+0000","last_peered":"2024-08-17T17:56:32.356888+0000","last_clean":"2024-08-17T17:56:32.356888+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:56:32.356888+0000","last_undegraded":"2024-08-17T17:56:32.356888+0000","last_fullsized":"2024-08-17T17:56:32.356888+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1857,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.361738+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:56:43.361738+0000","last_peered":"2024-08-17T17:56:43.361738+0000","last_clean":"2024-08-17T17:56:43.361738+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:56:43.361738+0000","last_undegraded":"2024-08-17T17:56:43.361738+0000","last_fullsized":"2024-08-17T17:56:43.361738+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1181,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.337496+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:56:32.337496+0000","last_peered":"2024-08-17T17:56:32.337496+0000","last_clean":"2024-08-17T17:56:32.337496+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:56:32.337496+0000","last_undegraded":"2024-08-17T17:56:32.337496+0000","last_fullsized":"2024-08-17T17:56:32.337496+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1910,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.371838+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:56:43.371838+0000","last_peered":"2024-08-17T17:56:43.371838+0000","last_clean":"2024-08-17T17:56:43.371838+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:56:43.371838+0000","last_undegraded":"2024-08-17T17:56:43.371838+0000","last_fullsized":"2024-08-17T17:56:43.371838+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1911,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.368544+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:56:43.368544+0000","last_peered":"2024-08-17T17:56:43.368544+0000","last_clean":"2024-08-17T17:56:43.368544+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:56:43.368544+0000","last_undegraded":"2024-08-17T17:56:43.368544+0000","last_fullsized":"2024-08-17T17:56:43.368544+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":594,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.337496+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:56:32.337496+0000","last_peered":"2024-08-17T17:56:32.337496+0000","last_clean":"2024-08-17T17:56:32.337496+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:56:32.337496+0000","last_undegraded":"2024-08-17T17:56:32.337496+0000","last_fullsized":"2024-08-17T17:56:32.337496+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1206,"reported_epoch":1236,"state":"active+clean","last_fresh":"2024-08-17T17:56:32.337682+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:56:32.337682+0000","last_peered":"2024-08-17T17:56:32.337682+0000","last_clean":"2024-08-17T17:56:32.337682+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:56:32.337682+0000","last_undegraded":"2024-08-17T17:56:32.337682+0000","last_fullsized":"2024-08-17T17:56:32.337682+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1857,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.361594+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:56:43.361594+0000","last_peered":"2024-08-17T17:56:43.361594+0000","last_clean":"2024-08-17T17:56:43.361594+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:56:43.361594+0000","last_undegraded":"2024-08-17T17:56:43.361594+0000","last_fullsized":"2024-08-17T17:56:43.361594+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":2910715928,"num_objects":44415,"num_object_clones":0,"num_object_copies":133245,"num_objects_missing_on_primary":2215,"num_objects_missing":2215,"num_objects_degraded":15837,"num_objects_misplaced":7104,"num_objects_unfound":0,"num_objects_dirty":44415,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":165195,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36920,"num_bytes_recovered":2241134592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9950867456,"data_stored":9950855240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108569,"ondisk_log_size":108569,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234431,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1555200,"kb_used_data":1249528,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":92189952,"statfs":{"total":95995035648,"available":94402510848,"internally_reserved":0,"allocated":1279516672,"data_stored":1274518107,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496066,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1742916,"kb_used_data":1433084,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92002236,"statfs":{"total":95995035648,"available":94210289664,"internally_reserved":0,"allocated":1467478016,"data_stored":1462475856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757704,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326318,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1849516,"kb_used_data":1606116,"kb_used_omap":57,"kb_used_meta":243334,"kb_avail":91895636,"statfs":{"total":95995035648,"available":94101131264,"internally_reserved":0,"allocated":1644662784,"data_stored":1639679399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":249174779},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405556,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1680952,"kb_used_data":1460268,"kb_used_omap":61,"kb_used_meta":220610,"kb_avail":92064200,"statfs":{"total":95995035648,"available":94273740800,"internally_reserved":0,"allocated":1495314432,"data_stored":1490328019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":225905592},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452091,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1260652,"kb_used_data":1080288,"kb_used_omap":54,"kb_used_meta":180297,"kb_avail":92484500,"statfs":{"total":95995035648,"available":94704128000,"internally_reserved":0,"allocated":1106214912,"data_stored":1101235488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":184624526},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474782,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1211388,"kb_used_data":992496,"kb_used_omap":119,"kb_used_meta":218760,"kb_avail":92533764,"statfs":{"total":95995035648,"available":94754574336,"internally_reserved":0,"allocated":1016315904,"data_stored":1011321316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":224010974},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019339,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":997855232,"data_stored":997851160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1087045632,"data_stored":1087045632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1480916992,"data_stored":1480916992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1628246016,"data_stored":1628241944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":952500224,"data_stored":952500224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1452150784,"data_stored":1452146712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1264254976,"data_stored":1264254976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:56:52.824 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:56:52.839 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:52.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:52 smithi179 ceph-mon[93519]: pgmap v1905: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 203 op/s; 15837/132954 objects degraded (11.912%); 7104/132954 objects misplaced (5.343%); 390 KiB/s, 6 objects/s recovering 2024-08-17T17:56:52.859 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:52.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:52.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:52.961 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:52.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:52.983 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:52.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:53.084 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:53.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:53.105 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:53.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:53.206 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:53.230 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:53.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:53.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:53.332 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:53.353 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:53.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:53.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:53.455 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:53.476 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:53.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:53.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:53.578 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:53.599 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:53.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:53.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:53.701 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:53.727 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:53.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:53.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:53.828 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:53.850 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:53.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:53.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:53.952 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:53.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:53.973 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:53.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:54.074 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:54.104 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:54.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:54.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:54.206 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:54.235 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:54.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:54.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:54.337 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:54.357 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:54.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:54.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:54.459 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:54.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:54.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:54.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:54.582 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:54.588 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:54 smithi130 ceph-mon[105521]: pgmap v1906: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 15837/131028 objects degraded (12.087%); 6923/131028 objects misplaced (5.284%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:56:54.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:54 smithi130 ceph-mon[110068]: pgmap v1906: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 15837/131028 objects degraded (12.087%); 6923/131028 objects misplaced (5.284%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:56:54.603 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:54.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:54.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:54.705 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:54.727 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:54.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:54.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:54.829 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:54 smithi179 ceph-mon[93519]: pgmap v1906: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.7 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 214 op/s; 15837/131028 objects degraded (12.087%); 6923/131028 objects misplaced (5.284%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:56:54.850 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:54.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:54.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:54.952 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:54.972 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:54.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:54.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:55.074 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:55.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:55.099 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:55.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:55.185 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:55.191 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:56:55.200 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:55.221 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:55.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:55.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:55.323 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:55.347 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:55.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:55.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:55.395 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T17:56:55.449 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:55.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:55.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:55.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:55.572 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:55.593 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:55.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:55.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:55.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:55 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3365427787' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:55 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:55 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3365427787' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:55 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:55.695 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:55.716 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:55.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:55.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:55.818 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:55.839 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:55.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:55.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:55 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3365427787' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:55.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:55 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:56:55.941 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:55.965 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:55.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:55.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:56.067 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:56.089 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:56.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:56.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:56.191 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:56.214 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:56.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:56.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:56.316 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:56.349 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:56.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:56.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:56.398 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:56 smithi130 ceph-mon[105521]: pgmap v1907: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 15837/129834 objects degraded (12.198%); 6923/129834 objects misplaced (5.332%); 660 KiB/s, 10 objects/s recovering 2024-08-17T17:56:56.398 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:56 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:56:56.398 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:56 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:56:56.398 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:56 smithi130 ceph-mon[105521]: osdmap e1238: 8 total, 8 up, 6 in 2024-08-17T17:56:56.399 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/129834 objects degraded (12.198%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:56.399 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:56 smithi130 ceph-mon[110068]: pgmap v1907: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 15837/129834 objects degraded (12.198%); 6923/129834 objects misplaced (5.332%); 660 KiB/s, 10 objects/s recovering 2024-08-17T17:56:56.399 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:56 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:56:56.399 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:56 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:56:56.399 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:56 smithi130 ceph-mon[110068]: osdmap e1238: 8 total, 8 up, 6 in 2024-08-17T17:56:56.399 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/129834 objects degraded (12.198%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:56.451 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:56.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:56.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:56.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:56.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:56.598 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:56.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:56.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:56.701 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:56.722 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:56.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:56.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:56.824 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:56 smithi179 ceph-mon[93519]: pgmap v1907: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 216 op/s; 15837/129834 objects degraded (12.198%); 6923/129834 objects misplaced (5.332%); 660 KiB/s, 10 objects/s recovering 2024-08-17T17:56:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:56 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:56:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:56 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:56:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:56 smithi179 ceph-mon[93519]: osdmap e1238: 8 total, 8 up, 6 in 2024-08-17T17:56:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/129834 objects degraded (12.198%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:56:56.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:56.851 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:56.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:56.900 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:56:56.900 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:56:56.900 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T17:56:56.901 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T17:56:56.901 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:56:56.952 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:56.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:56.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:56.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:57.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:57.099 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:57.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:57.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:57.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:57.222 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:57.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:57.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:57.324 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:57.345 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:57.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:57.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:57.447 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:57.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:57.469 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:57.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:57.570 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:57.592 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:57.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:57.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:57.693 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:57.714 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:57.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:57.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:57.817 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:56:57.838 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:57.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:57.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:56:57.940 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:57.950 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:56:57.967 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:57.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:57.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:56:58.069 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:58.094 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:58.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:58.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:58.174 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:56:58.196 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:58.216 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:58.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:58.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:58.318 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:58.346 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:58.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:58.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:58.447 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:56:58.462 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:58 smithi130 ceph-mon[105521]: pgmap v1909: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 213 op/s; 15837/129084 objects degraded (12.269%); 6744/129084 objects misplaced (5.225%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:56:58.462 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:58.463 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:56:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:58.463 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:58 smithi130 ceph-mon[110068]: pgmap v1909: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 213 op/s; 15837/129084 objects degraded (12.269%); 6744/129084 objects misplaced (5.225%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:56:58.463 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:58.463 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:56:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:58.469 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:58.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:56:58.570 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:58.591 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:58.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:58.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:58 smithi179 ceph-mon[93519]: pgmap v1909: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 213 op/s; 15837/129084 objects degraded (12.269%); 6744/129084 objects misplaced (5.225%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:56:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:56:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:56:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:56:58.694 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:58.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:58.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:58.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:58.823 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:58.844 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:58.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:58.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:58.946 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:58.968 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:58.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:58.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:59.069 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:56:59.090 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:59.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:59.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:56:59.192 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:59.219 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:59.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:59.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:59.321 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:56:59.349 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:59.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:59.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:56:59.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:59.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:59.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:59.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:59.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:56:59.596 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:56:59.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:59.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:56:59.697 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:56:59.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:59.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:59.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:56:59.822 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:56:59.848 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:59.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:59.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:56:59.950 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:56:59.974 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:56:59.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:56:59.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:00.045 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:00.076 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:00.097 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:00.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:00.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:00.145 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:56:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:56:59] "GET /metrics HTTP/1.1" 200 36802 "" "Prometheus/2.51.0" 2024-08-17T17:57:00.199 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:00.224 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:00.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:00.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:00.326 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:00.347 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:00.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:00.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:00.411 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:00 smithi130 ceph-mon[105521]: pgmap v1910: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 202 op/s; 15837/127233 objects degraded (12.447%); 6744/127233 objects misplaced (5.301%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:57:00.411 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:00 smithi130 ceph-mon[110068]: pgmap v1910: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 202 op/s; 15837/127233 objects degraded (12.447%); 6744/127233 objects misplaced (5.301%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:57:00.449 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:00.470 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:00.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:00.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:00.572 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:00 smithi179 ceph-mon[93519]: pgmap v1910: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 202 op/s; 15837/127233 objects degraded (12.447%); 6744/127233 objects misplaced (5.301%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:57:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:57:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:57:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:57:00.595 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:00.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:00.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:00.696 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:00.717 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:00.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:00.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:00.819 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:00.841 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:00.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:00.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:00.942 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:00.963 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:00.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:00.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:01.065 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:01.087 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:01.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:01.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:01.099 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:01.189 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:01.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:01.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:01.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:01.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:01.338 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:01.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:01.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:01.343 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/127233 objects degraded (12.447%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:01.343 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/127233 objects degraded (12.447%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:01.362 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:57:01.362 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:57:01.394 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:57:01.395 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:57:01.419 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:57:01.420 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:57:01.429 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:57:01.429 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15837/127233 objects degraded (12.447%), 5 pgs degraded, 6 pgs undersized","count":11},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1742,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1238,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":42411,"data_bytes":2778273765,"bytes_used":9166155776,"bytes_avail":566804058112,"bytes_total":575970213888,"degraded_objects":15837,"degraded_total":127233,"degraded_ratio":0.1244724246068237,"misplaced_objects":6744,"misplaced_total":127233,"misplaced_ratio":0.053005116597109242,"recovering_objects_per_sec":12,"recovering_bytes_per_sec":792752,"recovering_keys_per_sec":0,"num_objects_recovered":121,"num_bytes_recovered":7929856,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":202},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":25,"modified":"2024-08-17T17:49:39.646037+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (9m)\n [==========================..] (remaining: 40s)","progress":0.93258428573608398,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:57:01.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:01.445 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:57:01.446 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:57:01.465 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:57:01.465 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:57:01.465 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:01.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:01.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:01.484 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:57:01.484 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:57:01.500 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:57:01.500 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:57:01.517 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:57:01.517 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:57:01.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:01.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:01.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:01.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:01.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/127233 objects degraded (12.447%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:01.690 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:01.714 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:01.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:01.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:01.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:01.838 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:01.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:01.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:01.940 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:01.962 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:01.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:01.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:02.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:02.089 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:02.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:02.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:02.191 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:02.212 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:02.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:02.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:02.314 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:02.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:02.336 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:02.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:02.377 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:02 smithi130 ceph-mon[105521]: pgmap v1911: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 204 op/s; 15837/126819 objects degraded (12.488%); 6744/126819 objects misplaced (5.318%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:57:02.377 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:02 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4283889181' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:02.377 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:02 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/832781321' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:57:02.377 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:02 smithi130 ceph-mon[110068]: pgmap v1911: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 204 op/s; 15837/126819 objects degraded (12.488%); 6744/126819 objects misplaced (5.318%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:57:02.377 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:02 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4283889181' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:02.377 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:02 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/832781321' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:57:02.438 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:02.462 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:02.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:02.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:02.563 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:02.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:02.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:02.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:02.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:02 smithi179 ceph-mon[93519]: pgmap v1911: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.6 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 204 op/s; 15837/126819 objects degraded (12.488%); 6744/126819 objects misplaced (5.318%); 774 KiB/s, 12 objects/s recovering 2024-08-17T17:57:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:02 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4283889181' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:02 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/832781321' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:57:02.687 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:02.709 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:02.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:02.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:02.810 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:02.832 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:02.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:02.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:02.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:02.958 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:02.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:02.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:03.060 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:03.080 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:03.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:03.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:03.182 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:03.204 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:03.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:03.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:03.306 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:03.328 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:03.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:03.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:03.429 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:03.452 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:03.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:03.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:03.554 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:03.588 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:03.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:03.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:03.690 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:03.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:03.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:03.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:03.812 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:03.833 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:03.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:03.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:03.935 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:03.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:03.957 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:03.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:04.059 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:04.080 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:04.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:04.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:04.182 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:04.204 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:04.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:04.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:04.306 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:04.329 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:04.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:04.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:04.431 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:04.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:04.454 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:04.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:04.542 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:57:04.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:57:04.555 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:04.566 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:04 smithi130 ceph-mon[105521]: pgmap v1912: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 200 op/s; 15837/125019 objects degraded (12.668%); 6561/125019 objects misplaced (5.248%); 800 KiB/s, 12 objects/s recovering 2024-08-17T17:57:04.566 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:04 smithi130 ceph-mon[110068]: pgmap v1912: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 200 op/s; 15837/125019 objects degraded (12.668%); 6561/125019 objects misplaced (5.248%); 800 KiB/s, 12 objects/s recovering 2024-08-17T17:57:04.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:04 smithi179 ceph-mon[93519]: pgmap v1912: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 200 op/s; 15837/125019 objects degraded (12.668%); 6561/125019 objects misplaced (5.248%); 800 KiB/s, 12 objects/s recovering 2024-08-17T17:57:04.594 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:04.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:04.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:04.696 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:04.719 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:04.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:04.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:04.821 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:04.847 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:04.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:04.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:04.949 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:04.970 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:04.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:04.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:05.072 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:05.099 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:05.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:05.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:05.200 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:05.222 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:05.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:05.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:05.324 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:05.345 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:05.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:05.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:05.447 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:05.468 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:05.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:05.570 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:05.595 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:05.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:05.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:05.697 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:05.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:05.719 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:05.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:05.752 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:57:05.753 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:57:05.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:05.841 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:05.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:05.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:05.943 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:05.964 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:05.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:05.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:06.008 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:57:06.066 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:06.087 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:06.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:06.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:06.189 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:06.211 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:06.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:06.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:06.313 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:06.334 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:06.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:06.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:06.436 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:06.440 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:06.446 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:57:06.457 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:06.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:06.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:06.559 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:06.577 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:06 smithi130 ceph-mon[105521]: pgmap v1913: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 15837/123960 objects degraded (12.776%); 6561/123960 objects misplaced (5.293%); 800 KiB/s, 12 objects/s recovering 2024-08-17T17:57:06.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/123960 objects degraded (12.776%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:06.578 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:06 smithi130 ceph-mon[110068]: pgmap v1913: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 15837/123960 objects degraded (12.776%); 6561/123960 objects misplaced (5.293%); 800 KiB/s, 12 objects/s recovering 2024-08-17T17:57:06.578 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/123960 objects degraded (12.776%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:06.580 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:06.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:06.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:06 smithi179 ceph-mon[93519]: pgmap v1913: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 195 op/s; 15837/123960 objects degraded (12.776%); 6561/123960 objects misplaced (5.293%); 800 KiB/s, 12 objects/s recovering 2024-08-17T17:57:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/123960 objects degraded (12.776%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:06.681 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:06.707 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:06.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:06.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:06.759 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:57:06.809 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:06.834 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:06.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:06.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:06.937 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:06.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:06.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:06.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:07.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:07.090 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:07.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:07.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:07.191 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:07.217 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:07.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:07.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:07.318 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:07.344 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:07.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:07.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:07.445 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:07.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:07.468 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:07.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:07.569 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:07.591 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:07.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:07.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:07.692 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:07.714 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:07.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:07.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:07.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:07.845 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:07.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:07.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:07.947 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:07.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:07.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:07.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:08.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:08.100 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:08.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:08.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:08.202 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:08.225 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:08.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:08.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:08.307 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:08.321 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:57:08.327 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:08.348 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:08.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:08.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:08.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:08.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:08.473 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:08.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:08.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:08.580 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:08 smithi179 ceph-mon[93519]: pgmap v1914: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 189 op/s; 15837/123195 objects degraded (12.855%); 6360/123195 objects misplaced (5.163%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:57:08.580 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:08 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1264562847' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:57:08.580 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:08 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:57:08.597 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:08.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:08.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:08.603 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:57:08.604 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:57:08.644 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:08 smithi130 ceph-mon[105521]: pgmap v1914: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 189 op/s; 15837/123195 objects degraded (12.855%); 6360/123195 objects misplaced (5.163%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:57:08.644 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:08 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1264562847' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:57:08.644 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:08 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:57:08.644 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:08 smithi130 ceph-mon[110068]: pgmap v1914: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 189 op/s; 15837/123195 objects degraded (12.855%); 6360/123195 objects misplaced (5.163%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:57:08.644 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:08 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1264562847' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:57:08.644 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:08 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:57:08.700 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:08.721 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:08.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:08.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:08.822 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:08.843 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:08.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:08.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:08.945 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:08.967 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:08.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:08.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:09.069 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:09.091 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:09.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:09.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:09.193 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:09.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:09.215 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:09.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:09.302 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:57:09.316 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:09.341 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:09.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:09.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:09.443 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:09.463 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:09.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:09.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:09.565 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:09.567 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:57:09.588 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:09.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:09.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:09 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:57:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:09 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:57:09.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:09 smithi179 ceph-mon[93519]: osdmap e1239: 8 total, 8 up, 6 in 2024-08-17T17:57:09.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:09 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:57:09.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:09 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:57:09.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:09 smithi130 ceph-mon[105521]: osdmap e1239: 8 total, 8 up, 6 in 2024-08-17T17:57:09.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:09 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:57:09.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:09 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:57:09.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:09 smithi130 ceph-mon[110068]: osdmap e1239: 8 total, 8 up, 6 in 2024-08-17T17:57:09.690 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:09.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:09.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:09.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:09.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:09.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:09.844 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:09.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:09.945 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:09.967 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:09.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:09.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:10.069 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:10.090 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:10.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:10.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:57:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:57:09] "GET /metrics HTTP/1.1" 200 36802 "" "Prometheus/2.51.0" 2024-08-17T17:57:10.192 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:10.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:10.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:10.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:10.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:10.320 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:57:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:57:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:57:10.343 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:10.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:10.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:10.445 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:10.467 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:10.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:10.569 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:10.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:10.591 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:10.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:10.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:10 smithi179 ceph-mon[93519]: pgmap v1916: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 203 op/s; 15837/121134 objects degraded (13.074%); 6360/121134 objects misplaced (5.250%); 845 KiB/s, 13 objects/s recovering 2024-08-17T17:57:10.615 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:57:10.615 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:10 smithi130 ceph-mon[105521]: pgmap v1916: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 203 op/s; 15837/121134 objects degraded (13.074%); 6360/121134 objects misplaced (5.250%); 845 KiB/s, 13 objects/s recovering 2024-08-17T17:57:10.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:10 smithi130 ceph-mon[110068]: pgmap v1916: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 203 op/s; 15837/121134 objects degraded (13.074%); 6360/121134 objects misplaced (5.250%); 845 KiB/s, 13 objects/s recovering 2024-08-17T17:57:10.618 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:57:10.692 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:10.713 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:10.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:10.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:10.815 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:10.836 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:10.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:10.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:10.938 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:10.964 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:10.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:11.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:11.087 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:11.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:11.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:11.189 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:11.210 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:11.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:11.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:11.312 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:11.333 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:11.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:11.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:11.425 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:57:11.434 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:11.496 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:11.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:11.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:11 smithi179 ceph-mon[93519]: from='client.36021 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:57:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/121134 objects degraded (13.074%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:11.598 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:11.628 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:11.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:11.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:11.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:11 smithi130 ceph-mon[105521]: from='client.36021 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:57:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/121134 objects degraded (13.074%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:11 smithi130 ceph-mon[110068]: from='client.36021 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:57:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/121134 objects degraded (13.074%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:11.731 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:11.768 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:11.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:11.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:11.869 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:11.891 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:11.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:11.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:11.993 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:12.016 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:12.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:12.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:12.040 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:57:12.040 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:57:12.069 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:12.070 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:57:12.070 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:57:12.100 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:57:12.101 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:57:12.101 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:57:12.101 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:57:12.118 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:12.121 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:57:12.122 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:57:12.139 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:12.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:12.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:12.142 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:57:12.143 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:57:12.160 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:57:12.160 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:57:12.180 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:57:12.180 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:57:12.203 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:57:12.203 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:57:12.240 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:12.261 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:12.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:12.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:12.363 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:12.385 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:12.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:12.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:12.487 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:12.510 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:12.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:12.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:12 smithi179 ceph-mon[93519]: pgmap v1917: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 15837/120759 objects degraded (13.115%); 6360/120759 objects misplaced (5.267%); 845 KiB/s, 13 objects/s recovering 2024-08-17T17:57:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:12 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2239879312' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:12.611 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:12.624 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:12 smithi130 ceph-mon[105521]: pgmap v1917: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 15837/120759 objects degraded (13.115%); 6360/120759 objects misplaced (5.267%); 845 KiB/s, 13 objects/s recovering 2024-08-17T17:57:12.624 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:12 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2239879312' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:12.625 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:12.625 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:12.625 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:12 smithi130 ceph-mon[110068]: pgmap v1917: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.5 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 201 op/s; 15837/120759 objects degraded (13.115%); 6360/120759 objects misplaced (5.267%); 845 KiB/s, 13 objects/s recovering 2024-08-17T17:57:12.625 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:12 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2239879312' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:12.625 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:12.625 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:12.634 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:12.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:12.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:12.658 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1916,"stamp":"2024-08-17T17:57:09.098764+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2645039077,"num_objects":40378,"num_object_clones":0,"num_object_copies":121134,"num_objects_missing_on_primary":1962,"num_objects_missing":1962,"num_objects_degraded":15837,"num_objects_misplaced":6360,"num_objects_unfound":0,"num_objects_dirty":40378,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":169536,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37202,"num_bytes_recovered":2259491853,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108639,"ondisk_log_size":108639,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8641080,"kb_used_data":7130052,"kb_used_omap":295,"kb_used_meta":1510680,"kb_avail":553829832,"statfs":{"total":575970213888,"available":567121747968,"internally_reserved":0,"allocated":7301173248,"data_stored":7271176661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1546936783},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-133234688,"num_objects":-2033,"num_object_clones":0,"num_object_copies":-6099,"num_objects_missing_on_primary":-127,"num_objects_missing":-127,"num_objects_degraded":0,"num_objects_misplaced":-384,"num_objects_unfound":0,"num_objects_dirty":-2033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2033,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":132,"num_bytes_recovered":8650752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002766"},"pg_stats":[{"pgid":"45.f","version":"1238'10592","reported_seq":21148,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:57:06.238371+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:57:06.238371+0000","last_peered":"2024-08-17T17:57:06.238371+0000","last_clean":"2024-08-17T17:57:06.238371+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:57:06.238371+0000","last_undegraded":"2024-08-17T17:57:06.238371+0000","last_fullsized":"2024-08-17T17:57:06.238371+0000","mapping_epoch":1194,"log_start":"1175'3731","ondisk_log_start":"1175'3731","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6861,"log_dups_size":0,"ondisk_log_size":6861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168558592,"num_objects":2572,"num_object_clones":0,"num_object_copies":7716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10592,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1238'10523","reported_seq":21001,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:57:07.881908+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:57:07.881908+0000","last_peered":"2024-08-17T17:57:07.881908+0000","last_clean":"2024-08-17T17:57:07.881908+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:57:07.881908+0000","last_undegraded":"2024-08-17T17:57:07.881908+0000","last_fullsized":"2024-08-17T17:57:07.881908+0000","mapping_epoch":1211,"log_start":"1174'3532","ondisk_log_start":"1174'3532","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6991,"log_dups_size":0,"ondisk_log_size":6991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161939456,"num_objects":2471,"num_object_clones":0,"num_object_copies":7413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10523,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1239'10865","reported_seq":21731,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.620112+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:57:08.620112+0000","last_peered":"2024-08-17T17:57:08.620112+0000","last_clean":"2024-08-17T17:57:08.620112+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:57:08.620112+0000","last_undegraded":"2024-08-17T17:57:08.620112+0000","last_fullsized":"2024-08-17T17:57:08.620112+0000","mapping_epoch":1205,"log_start":"1175'4968","ondisk_log_start":"1175'4968","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5897,"log_dups_size":0,"ondisk_log_size":5897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164626456,"num_objects":2513,"num_object_clones":0,"num_object_copies":7539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2513,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10865,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1238'10510","reported_seq":10589,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:57:07.788693+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:57:07.788693+0000","last_peered":"2024-08-17T17:57:07.788693+0000","last_clean":"2024-08-17T17:57:07.788693+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:57:07.788693+0000","last_undegraded":"2024-08-17T17:57:07.788693+0000","last_fullsized":"2024-08-17T17:57:07.788693+0000","mapping_epoch":1167,"log_start":"1175'3789","ondisk_log_start":"1175'3789","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6721,"log_dups_size":0,"ondisk_log_size":6721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163184640,"num_objects":2490,"num_object_clones":0,"num_object_copies":7470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10510,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1238'10590","reported_seq":15860,"reported_epoch":1238,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:06.858415+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:57:06.858415+0000","last_peered":"2024-08-17T17:57:06.858415+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:57:06.858415+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1175'4626","ondisk_log_start":"1175'4626","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5964,"log_dups_size":0,"ondisk_log_size":5964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165412864,"num_objects":2524,"num_object_clones":0,"num_object_copies":7572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":2524,"num_objects_unfound":0,"num_objects_dirty":2524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10590,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":2006}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1238'10642","reported_seq":10721,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:57:07.863277+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:57:07.863277+0000","last_peered":"2024-08-17T17:57:07.863277+0000","last_clean":"2024-08-17T17:57:07.863277+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:57:07.863277+0000","last_undegraded":"2024-08-17T17:57:07.863277+0000","last_fullsized":"2024-08-17T17:57:07.863277+0000","mapping_epoch":1167,"log_start":"1175'3877","ondisk_log_start":"1175'3877","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6765,"log_dups_size":0,"ondisk_log_size":6765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160694272,"num_objects":2452,"num_object_clones":0,"num_object_copies":7356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10642,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1238'10696","reported_seq":10775,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:57:07.848717+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:57:07.848717+0000","last_peered":"2024-08-17T17:57:07.848717+0000","last_clean":"2024-08-17T17:57:07.848717+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:57:07.848717+0000","last_undegraded":"2024-08-17T17:57:07.848717+0000","last_fullsized":"2024-08-17T17:57:07.848717+0000","mapping_epoch":1167,"log_start":"1175'3969","ondisk_log_start":"1175'3969","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6727,"log_dups_size":0,"ondisk_log_size":6727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165806080,"num_objects":2530,"num_object_clones":0,"num_object_copies":7590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10696,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1238'10494","reported_seq":19981,"reported_epoch":1238,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:06.772053+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:57:06.772053+0000","last_peered":"2024-08-17T17:57:06.772053+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:57:06.772053+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1175'4575","ondisk_log_start":"1175'4575","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5919,"log_dups_size":0,"ondisk_log_size":5919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163577856,"num_objects":2496,"num_object_clones":0,"num_object_copies":7488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10494,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3108,"num_bytes_recovered":203685888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":1176},{"shards":"1,7","objects":429}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1238'10546","reported_seq":10635,"reported_epoch":1238,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:07.816939+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:57:07.816939+0000","last_peered":"2024-08-17T17:57:07.816939+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:57:07.816939+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1175'3877","ondisk_log_start":"1175'3877","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6669,"log_dups_size":0,"ondisk_log_size":6669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165806080,"num_objects":2530,"num_object_clones":0,"num_object_copies":7590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10546,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":2530},{"shards":"1,6","objects":-3303}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1239'10640","reported_seq":13745,"reported_epoch":1239,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:08.652630+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:57:08.652630+0000","last_peered":"2024-08-17T17:57:08.652630+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:57:08.652630+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1175'4706","ondisk_log_start":"1175'4706","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5934,"log_dups_size":0,"ondisk_log_size":5934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166068224,"num_objects":2534,"num_object_clones":0,"num_object_copies":7602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10640,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":973}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1238'10458","reported_seq":10617,"reported_epoch":1238,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:04.155514+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:57:04.155514+0000","last_peered":"2024-08-17T17:57:04.155514+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:57:04.155514+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1170'1300","ondisk_log_start":"1170'1300","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9158,"log_dups_size":0,"ondisk_log_size":9158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171442176,"num_objects":2616,"num_object_clones":0,"num_object_copies":7848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10458,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-2537}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1238'10500","reported_seq":16719,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:57:04.121763+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:57:04.121763+0000","last_peered":"2024-08-17T17:57:04.121763+0000","last_clean":"2024-08-17T17:57:04.121763+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:57:04.121763+0000","last_undegraded":"2024-08-17T17:57:04.121763+0000","last_fullsized":"2024-08-17T17:57:04.121763+0000","mapping_epoch":1183,"log_start":"1170'1400","ondisk_log_start":"1170'1400","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9100,"log_dups_size":0,"ondisk_log_size":9100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166854656,"num_objects":2546,"num_object_clones":0,"num_object_copies":7638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10500,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1239'10611","reported_seq":10770,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.496733+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:57:08.496733+0000","last_peered":"2024-08-17T17:57:08.496733+0000","last_clean":"2024-08-17T17:57:08.496733+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:57:08.496733+0000","last_undegraded":"2024-08-17T17:57:08.496733+0000","last_fullsized":"2024-08-17T17:57:08.496733+0000","mapping_epoch":1167,"log_start":"1175'4691","ondisk_log_start":"1175'4691","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5920,"log_dups_size":0,"ondisk_log_size":5920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167051264,"num_objects":2549,"num_object_clones":0,"num_object_copies":7647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10611,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1238'10468","reported_seq":10547,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:57:07.761602+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:57:07.761602+0000","last_peered":"2024-08-17T17:57:07.761602+0000","last_clean":"2024-08-17T17:57:07.761602+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:57:07.761602+0000","last_undegraded":"2024-08-17T17:57:07.761602+0000","last_fullsized":"2024-08-17T17:57:07.761602+0000","mapping_epoch":1167,"log_start":"1174'3425","ondisk_log_start":"1174'3425","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7043,"log_dups_size":0,"ondisk_log_size":7043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161611776,"num_objects":2466,"num_object_clones":0,"num_object_copies":7398,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2466,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10468,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1238'10762","reported_seq":26923,"reported_epoch":1238,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:57:06.263203+0000","last_change":"2024-08-17T17:55:37.941989+0000","last_active":"2024-08-17T17:57:06.263203+0000","last_peered":"2024-08-17T17:57:06.263203+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:57:06.263203+0000","last_undegraded":"2024-08-17T17:57:06.263203+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1175'3891","ondisk_log_start":"1175'3891","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6871,"log_dups_size":0,"ondisk_log_size":6871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170524672,"num_objects":2602,"num_object_clones":0,"num_object_copies":7806,"num_objects_missing_on_primary":1962,"num_objects_missing":1962,"num_objects_degraded":0,"num_objects_misplaced":3836,"num_objects_unfound":0,"num_objects_dirty":2602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10762,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6187,"num_bytes_recovered":405471232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":10},{"shards":"0,4,6","objects":1356},{"shards":"0,6","objects":1},{"shards":"4,5,6","objects":1235}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1239'10361","reported_seq":23706,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.491009+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:57:08.491009+0000","last_peered":"2024-08-17T17:57:08.491009+0000","last_clean":"2024-08-17T17:57:08.491009+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:57:08.491009+0000","last_undegraded":"2024-08-17T17:57:08.491009+0000","last_fullsized":"2024-08-17T17:57:08.491009+0000","mapping_epoch":1196,"log_start":"1175'4469","ondisk_log_start":"1175'4469","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5892,"log_dups_size":0,"ondisk_log_size":5892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161284096,"num_objects":2461,"num_object_clones":0,"num_object_copies":7383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10361,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1934,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.399128+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:56:55.399128+0000","last_peered":"2024-08-17T17:56:55.399128+0000","last_clean":"2024-08-17T17:56:55.399128+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:56:55.399128+0000","last_undegraded":"2024-08-17T17:56:55.399128+0000","last_fullsized":"2024-08-17T17:56:55.399128+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":515,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.380527+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:56:43.380527+0000","last_peered":"2024-08-17T17:56:43.380527+0000","last_clean":"2024-08-17T17:56:43.380527+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:56:43.380527+0000","last_undegraded":"2024-08-17T17:56:43.380527+0000","last_fullsized":"2024-08-17T17:56:43.380527+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1855,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.321372+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:57:08.321372+0000","last_peered":"2024-08-17T17:57:08.321372+0000","last_clean":"2024-08-17T17:57:08.321372+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:57:08.321372+0000","last_undegraded":"2024-08-17T17:57:08.321372+0000","last_fullsized":"2024-08-17T17:57:08.321372+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1879,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.318675+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:57:08.318675+0000","last_peered":"2024-08-17T17:57:08.318675+0000","last_clean":"2024-08-17T17:57:08.318675+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:57:08.318675+0000","last_undegraded":"2024-08-17T17:57:08.318675+0000","last_fullsized":"2024-08-17T17:57:08.318675+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":77,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.381028+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:56:43.381028+0000","last_peered":"2024-08-17T17:56:43.381028+0000","last_clean":"2024-08-17T17:56:43.381028+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:56:43.381028+0000","last_undegraded":"2024-08-17T17:56:43.381028+0000","last_fullsized":"2024-08-17T17:56:43.381028+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1272,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.319076+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:57:08.319076+0000","last_peered":"2024-08-17T17:57:08.319076+0000","last_clean":"2024-08-17T17:57:08.319076+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:57:08.319076+0000","last_undegraded":"2024-08-17T17:57:08.319076+0000","last_fullsized":"2024-08-17T17:57:08.319076+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":128,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.440487+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:56:55.440487+0000","last_peered":"2024-08-17T17:56:55.440487+0000","last_clean":"2024-08-17T17:56:55.440487+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:56:55.440487+0000","last_undegraded":"2024-08-17T17:56:55.440487+0000","last_fullsized":"2024-08-17T17:56:55.440487+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":650,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.320650+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:57:08.320650+0000","last_peered":"2024-08-17T17:57:08.320650+0000","last_clean":"2024-08-17T17:57:08.320650+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:57:08.320650+0000","last_undegraded":"2024-08-17T17:57:08.320650+0000","last_fullsized":"2024-08-17T17:57:08.320650+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1892,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.398257+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:56:55.398257+0000","last_peered":"2024-08-17T17:56:55.398257+0000","last_clean":"2024-08-17T17:56:55.398257+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:56:55.398257+0000","last_undegraded":"2024-08-17T17:56:55.398257+0000","last_fullsized":"2024-08-17T17:56:55.398257+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1894,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.405895+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:56:55.405895+0000","last_peered":"2024-08-17T17:56:55.405895+0000","last_clean":"2024-08-17T17:56:55.405895+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:56:55.405895+0000","last_undegraded":"2024-08-17T17:56:55.405895+0000","last_fullsized":"2024-08-17T17:56:55.405895+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":113,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.440475+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:56:55.440475+0000","last_peered":"2024-08-17T17:56:55.440475+0000","last_clean":"2024-08-17T17:56:55.440475+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:56:55.440475+0000","last_undegraded":"2024-08-17T17:56:55.440475+0000","last_fullsized":"2024-08-17T17:56:55.440475+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1868,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.321101+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:57:08.321101+0000","last_peered":"2024-08-17T17:57:08.321101+0000","last_clean":"2024-08-17T17:57:08.321101+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:57:08.321101+0000","last_undegraded":"2024-08-17T17:57:08.321101+0000","last_fullsized":"2024-08-17T17:57:08.321101+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1920,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.440007+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:56:55.440007+0000","last_peered":"2024-08-17T17:56:55.440007+0000","last_clean":"2024-08-17T17:56:55.440007+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:56:55.440007+0000","last_undegraded":"2024-08-17T17:56:55.440007+0000","last_fullsized":"2024-08-17T17:56:55.440007+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1915,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.400020+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:56:55.400020+0000","last_peered":"2024-08-17T17:56:55.400020+0000","last_clean":"2024-08-17T17:56:55.400020+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:56:55.400020+0000","last_undegraded":"2024-08-17T17:56:55.400020+0000","last_fullsized":"2024-08-17T17:56:55.400020+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1890,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.318542+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:57:08.318542+0000","last_peered":"2024-08-17T17:57:08.318542+0000","last_clean":"2024-08-17T17:57:08.318542+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:57:08.318542+0000","last_undegraded":"2024-08-17T17:57:08.318542+0000","last_fullsized":"2024-08-17T17:57:08.318542+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1212,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.381121+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:56:43.381121+0000","last_peered":"2024-08-17T17:56:43.381121+0000","last_clean":"2024-08-17T17:56:43.381121+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:56:43.381121+0000","last_undegraded":"2024-08-17T17:56:43.381121+0000","last_fullsized":"2024-08-17T17:56:43.381121+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1861,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.318249+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:57:08.318249+0000","last_peered":"2024-08-17T17:57:08.318249+0000","last_clean":"2024-08-17T17:57:08.318249+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:57:08.318249+0000","last_undegraded":"2024-08-17T17:57:08.318249+0000","last_fullsized":"2024-08-17T17:57:08.318249+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1909,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.398654+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:56:55.398654+0000","last_peered":"2024-08-17T17:56:55.398654+0000","last_clean":"2024-08-17T17:56:55.398654+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:56:55.398654+0000","last_undegraded":"2024-08-17T17:56:55.398654+0000","last_fullsized":"2024-08-17T17:56:55.398654+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1817,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.318668+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:57:08.318668+0000","last_peered":"2024-08-17T17:57:08.318668+0000","last_clean":"2024-08-17T17:57:08.318668+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:57:08.318668+0000","last_undegraded":"2024-08-17T17:57:08.318668+0000","last_fullsized":"2024-08-17T17:57:08.318668+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":531,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.364856+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:56:43.364856+0000","last_peered":"2024-08-17T17:56:43.364856+0000","last_clean":"2024-08-17T17:56:43.364856+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:56:43.364856+0000","last_undegraded":"2024-08-17T17:56:43.364856+0000","last_fullsized":"2024-08-17T17:56:43.364856+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1208,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.364410+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:56:43.364410+0000","last_peered":"2024-08-17T17:56:43.364410+0000","last_clean":"2024-08-17T17:56:43.364410+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:56:43.364410+0000","last_undegraded":"2024-08-17T17:56:43.364410+0000","last_fullsized":"2024-08-17T17:56:43.364410+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1914,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.440404+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:56:55.440404+0000","last_peered":"2024-08-17T17:56:55.440404+0000","last_clean":"2024-08-17T17:56:55.440404+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:56:55.440404+0000","last_undegraded":"2024-08-17T17:56:55.440404+0000","last_fullsized":"2024-08-17T17:56:55.440404+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":369,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.364708+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:56:43.364708+0000","last_peered":"2024-08-17T17:56:43.364708+0000","last_clean":"2024-08-17T17:56:43.364708+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:56:43.364708+0000","last_undegraded":"2024-08-17T17:56:43.364708+0000","last_fullsized":"2024-08-17T17:56:43.364708+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":515,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.380914+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:56:43.380914+0000","last_peered":"2024-08-17T17:56:43.380914+0000","last_clean":"2024-08-17T17:56:43.380914+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:56:43.380914+0000","last_undegraded":"2024-08-17T17:56:43.380914+0000","last_fullsized":"2024-08-17T17:56:43.380914+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1193,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.380713+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:56:43.380713+0000","last_peered":"2024-08-17T17:56:43.380713+0000","last_clean":"2024-08-17T17:56:43.380713+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:56:43.380713+0000","last_undegraded":"2024-08-17T17:56:43.380713+0000","last_fullsized":"2024-08-17T17:56:43.380713+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1201,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.381172+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:56:43.381172+0000","last_peered":"2024-08-17T17:56:43.381172+0000","last_clean":"2024-08-17T17:56:43.381172+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:56:43.381172+0000","last_undegraded":"2024-08-17T17:56:43.381172+0000","last_fullsized":"2024-08-17T17:56:43.381172+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1962,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.399888+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:56:55.399888+0000","last_peered":"2024-08-17T17:56:55.399888+0000","last_clean":"2024-08-17T17:56:55.399888+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:56:55.399888+0000","last_undegraded":"2024-08-17T17:56:55.399888+0000","last_fullsized":"2024-08-17T17:56:55.399888+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1847,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.318450+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:57:08.318450+0000","last_peered":"2024-08-17T17:57:08.318450+0000","last_clean":"2024-08-17T17:57:08.318450+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:57:08.318450+0000","last_undegraded":"2024-08-17T17:57:08.318450+0000","last_fullsized":"2024-08-17T17:57:08.318450+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1920,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.440909+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:56:55.440909+0000","last_peered":"2024-08-17T17:56:55.440909+0000","last_clean":"2024-08-17T17:56:55.440909+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:56:55.440909+0000","last_undegraded":"2024-08-17T17:56:55.440909+0000","last_fullsized":"2024-08-17T17:56:55.440909+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1849,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.321419+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:57:08.321419+0000","last_peered":"2024-08-17T17:57:08.321419+0000","last_clean":"2024-08-17T17:57:08.321419+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:57:08.321419+0000","last_undegraded":"2024-08-17T17:57:08.321419+0000","last_fullsized":"2024-08-17T17:57:08.321419+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1880,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.398369+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:56:55.398369+0000","last_peered":"2024-08-17T17:56:55.398369+0000","last_clean":"2024-08-17T17:56:55.398369+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:56:55.398369+0000","last_undegraded":"2024-08-17T17:56:55.398369+0000","last_fullsized":"2024-08-17T17:56:55.398369+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1228,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.364671+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:56:43.364671+0000","last_peered":"2024-08-17T17:56:43.364671+0000","last_clean":"2024-08-17T17:56:43.364671+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:56:43.364671+0000","last_undegraded":"2024-08-17T17:56:43.364671+0000","last_fullsized":"2024-08-17T17:56:43.364671+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1926,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.440598+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:56:55.440598+0000","last_peered":"2024-08-17T17:56:55.440598+0000","last_clean":"2024-08-17T17:56:55.440598+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:56:55.440598+0000","last_undegraded":"2024-08-17T17:56:55.440598+0000","last_fullsized":"2024-08-17T17:56:55.440598+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":113,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.440109+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:56:55.440109+0000","last_peered":"2024-08-17T17:56:55.440109+0000","last_clean":"2024-08-17T17:56:55.440109+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:56:55.440109+0000","last_undegraded":"2024-08-17T17:56:55.440109+0000","last_fullsized":"2024-08-17T17:56:55.440109+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1945,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.440653+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:56:55.440653+0000","last_peered":"2024-08-17T17:56:55.440653+0000","last_clean":"2024-08-17T17:56:55.440653+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:56:55.440653+0000","last_undegraded":"2024-08-17T17:56:55.440653+0000","last_fullsized":"2024-08-17T17:56:55.440653+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":684,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.320908+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:57:08.320908+0000","last_peered":"2024-08-17T17:57:08.320908+0000","last_clean":"2024-08-17T17:57:08.320908+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:57:08.320908+0000","last_undegraded":"2024-08-17T17:57:08.320908+0000","last_fullsized":"2024-08-17T17:57:08.320908+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1926,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.440703+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:56:55.440703+0000","last_peered":"2024-08-17T17:56:55.440703+0000","last_clean":"2024-08-17T17:56:55.440703+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:56:55.440703+0000","last_undegraded":"2024-08-17T17:56:55.440703+0000","last_fullsized":"2024-08-17T17:56:55.440703+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1255,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.398056+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:56:55.398056+0000","last_peered":"2024-08-17T17:56:55.398056+0000","last_clean":"2024-08-17T17:56:55.398056+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:56:55.398056+0000","last_undegraded":"2024-08-17T17:56:55.398056+0000","last_fullsized":"2024-08-17T17:56:55.398056+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":173,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.440317+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:56:55.440317+0000","last_peered":"2024-08-17T17:56:55.440317+0000","last_clean":"2024-08-17T17:56:55.440317+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:56:55.440317+0000","last_undegraded":"2024-08-17T17:56:55.440317+0000","last_fullsized":"2024-08-17T17:56:55.440317+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1937,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.443103+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:56:55.443103+0000","last_peered":"2024-08-17T17:56:55.443103+0000","last_clean":"2024-08-17T17:56:55.443103+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:56:55.443103+0000","last_undegraded":"2024-08-17T17:56:55.443103+0000","last_fullsized":"2024-08-17T17:56:55.443103+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1915,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.439889+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:56:55.439889+0000","last_peered":"2024-08-17T17:56:55.439889+0000","last_clean":"2024-08-17T17:56:55.439889+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:56:55.439889+0000","last_undegraded":"2024-08-17T17:56:55.439889+0000","last_fullsized":"2024-08-17T17:56:55.439889+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1935,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.403207+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:56:55.403207+0000","last_peered":"2024-08-17T17:56:55.403207+0000","last_clean":"2024-08-17T17:56:55.403207+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:56:55.403207+0000","last_undegraded":"2024-08-17T17:56:55.403207+0000","last_fullsized":"2024-08-17T17:56:55.403207+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1941,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.400076+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:56:55.400076+0000","last_peered":"2024-08-17T17:56:55.400076+0000","last_clean":"2024-08-17T17:56:55.400076+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:56:55.400076+0000","last_undegraded":"2024-08-17T17:56:55.400076+0000","last_fullsized":"2024-08-17T17:56:55.400076+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1921,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.445625+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:56:55.445625+0000","last_peered":"2024-08-17T17:56:55.445625+0000","last_clean":"2024-08-17T17:56:55.445625+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:56:55.445625+0000","last_undegraded":"2024-08-17T17:56:55.445625+0000","last_fullsized":"2024-08-17T17:56:55.445625+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1235,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.364436+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:56:43.364436+0000","last_peered":"2024-08-17T17:56:43.364436+0000","last_clean":"2024-08-17T17:56:43.364436+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:56:43.364436+0000","last_undegraded":"2024-08-17T17:56:43.364436+0000","last_fullsized":"2024-08-17T17:56:43.364436+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1181,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.381048+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:56:43.381048+0000","last_peered":"2024-08-17T17:56:43.381048+0000","last_clean":"2024-08-17T17:56:43.381048+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:56:43.381048+0000","last_undegraded":"2024-08-17T17:56:43.381048+0000","last_fullsized":"2024-08-17T17:56:43.381048+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1832,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.318980+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:57:08.318980+0000","last_peered":"2024-08-17T17:57:08.318980+0000","last_clean":"2024-08-17T17:57:08.318980+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:57:08.318980+0000","last_undegraded":"2024-08-17T17:57:08.318980+0000","last_fullsized":"2024-08-17T17:57:08.318980+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":161,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.403441+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:56:55.403441+0000","last_peered":"2024-08-17T17:56:55.403441+0000","last_clean":"2024-08-17T17:56:55.403441+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:56:55.403441+0000","last_undegraded":"2024-08-17T17:56:55.403441+0000","last_fullsized":"2024-08-17T17:56:55.403441+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1547,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.364714+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:56:43.364714+0000","last_peered":"2024-08-17T17:56:43.364714+0000","last_clean":"2024-08-17T17:56:43.364714+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:56:43.364714+0000","last_undegraded":"2024-08-17T17:56:43.364714+0000","last_fullsized":"2024-08-17T17:56:43.364714+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1921,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.443222+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:56:55.443222+0000","last_peered":"2024-08-17T17:56:55.443222+0000","last_clean":"2024-08-17T17:56:55.443222+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:56:55.443222+0000","last_undegraded":"2024-08-17T17:56:55.443222+0000","last_fullsized":"2024-08-17T17:56:55.443222+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1863,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.318345+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:57:08.318345+0000","last_peered":"2024-08-17T17:57:08.318345+0000","last_clean":"2024-08-17T17:57:08.318345+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:57:08.318345+0000","last_undegraded":"2024-08-17T17:57:08.318345+0000","last_fullsized":"2024-08-17T17:57:08.318345+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":515,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.381023+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:56:43.381023+0000","last_peered":"2024-08-17T17:56:43.381023+0000","last_clean":"2024-08-17T17:56:43.381023+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:56:43.381023+0000","last_undegraded":"2024-08-17T17:56:43.381023+0000","last_fullsized":"2024-08-17T17:56:43.381023+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":600,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.403046+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:56:55.403046+0000","last_peered":"2024-08-17T17:56:55.403046+0000","last_clean":"2024-08-17T17:56:55.403046+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:56:55.403046+0000","last_undegraded":"2024-08-17T17:56:55.403046+0000","last_fullsized":"2024-08-17T17:56:55.403046+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":616,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.403649+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:56:55.403649+0000","last_peered":"2024-08-17T17:56:55.403649+0000","last_clean":"2024-08-17T17:56:55.403649+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:56:55.403649+0000","last_undegraded":"2024-08-17T17:56:55.403649+0000","last_fullsized":"2024-08-17T17:56:55.403649+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1199,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.380913+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:56:43.380913+0000","last_peered":"2024-08-17T17:56:43.380913+0000","last_clean":"2024-08-17T17:56:43.380913+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:56:43.380913+0000","last_undegraded":"2024-08-17T17:56:43.380913+0000","last_fullsized":"2024-08-17T17:56:43.380913+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":634,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.320465+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:57:08.320465+0000","last_peered":"2024-08-17T17:57:08.320465+0000","last_clean":"2024-08-17T17:57:08.320465+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:57:08.320465+0000","last_undegraded":"2024-08-17T17:57:08.320465+0000","last_fullsized":"2024-08-17T17:57:08.320465+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1860,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.320740+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:57:08.320740+0000","last_peered":"2024-08-17T17:57:08.320740+0000","last_clean":"2024-08-17T17:57:08.320740+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:57:08.320740+0000","last_undegraded":"2024-08-17T17:57:08.320740+0000","last_fullsized":"2024-08-17T17:57:08.320740+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1953,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.398362+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:56:55.398362+0000","last_peered":"2024-08-17T17:56:55.398362+0000","last_clean":"2024-08-17T17:56:55.398362+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:56:55.398362+0000","last_undegraded":"2024-08-17T17:56:55.398362+0000","last_fullsized":"2024-08-17T17:56:55.398362+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":667,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.320462+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:57:08.320462+0000","last_peered":"2024-08-17T17:57:08.320462+0000","last_clean":"2024-08-17T17:57:08.320462+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:57:08.320462+0000","last_undegraded":"2024-08-17T17:57:08.320462+0000","last_fullsized":"2024-08-17T17:57:08.320462+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1188,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.380720+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:56:43.380720+0000","last_peered":"2024-08-17T17:56:43.380720+0000","last_clean":"2024-08-17T17:56:43.380720+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:56:43.380720+0000","last_undegraded":"2024-08-17T17:56:43.380720+0000","last_fullsized":"2024-08-17T17:56:43.380720+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1187,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.364571+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:56:43.364571+0000","last_peered":"2024-08-17T17:56:43.364571+0000","last_clean":"2024-08-17T17:56:43.364571+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:56:43.364571+0000","last_undegraded":"2024-08-17T17:56:43.364571+0000","last_fullsized":"2024-08-17T17:56:43.364571+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":649,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.440106+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:56:55.440106+0000","last_peered":"2024-08-17T17:56:55.440106+0000","last_clean":"2024-08-17T17:56:55.440106+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:56:55.440106+0000","last_undegraded":"2024-08-17T17:56:55.440106+0000","last_fullsized":"2024-08-17T17:56:55.440106+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1205,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.364979+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:56:43.364979+0000","last_peered":"2024-08-17T17:56:43.364979+0000","last_clean":"2024-08-17T17:56:43.364979+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:56:43.364979+0000","last_undegraded":"2024-08-17T17:56:43.364979+0000","last_fullsized":"2024-08-17T17:56:43.364979+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":89,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.364224+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:56:43.364224+0000","last_peered":"2024-08-17T17:56:43.364224+0000","last_clean":"2024-08-17T17:56:43.364224+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:56:43.364224+0000","last_undegraded":"2024-08-17T17:56:43.364224+0000","last_fullsized":"2024-08-17T17:56:43.364224+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1221,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.364630+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:56:43.364630+0000","last_peered":"2024-08-17T17:56:43.364630+0000","last_clean":"2024-08-17T17:56:43.364630+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:56:43.364630+0000","last_undegraded":"2024-08-17T17:56:43.364630+0000","last_fullsized":"2024-08-17T17:56:43.364630+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1238,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.380684+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:56:43.380684+0000","last_peered":"2024-08-17T17:56:43.380684+0000","last_clean":"2024-08-17T17:56:43.380684+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:56:43.380684+0000","last_undegraded":"2024-08-17T17:56:43.380684+0000","last_fullsized":"2024-08-17T17:56:43.380684+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1861,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.321335+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:57:08.321335+0000","last_peered":"2024-08-17T17:57:08.321335+0000","last_clean":"2024-08-17T17:57:08.321335+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:57:08.321335+0000","last_undegraded":"2024-08-17T17:57:08.321335+0000","last_fullsized":"2024-08-17T17:57:08.321335+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1182,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.364462+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:56:43.364462+0000","last_peered":"2024-08-17T17:56:43.364462+0000","last_clean":"2024-08-17T17:56:43.364462+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:56:43.364462+0000","last_undegraded":"2024-08-17T17:56:43.364462+0000","last_fullsized":"2024-08-17T17:56:43.364462+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1912,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.399940+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:56:55.399940+0000","last_peered":"2024-08-17T17:56:55.399940+0000","last_clean":"2024-08-17T17:56:55.399940+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:56:55.399940+0000","last_undegraded":"2024-08-17T17:56:55.399940+0000","last_fullsized":"2024-08-17T17:56:55.399940+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1913,"reported_epoch":1238,"state":"active+clean","last_fresh":"2024-08-17T17:56:55.398198+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:56:55.398198+0000","last_peered":"2024-08-17T17:56:55.398198+0000","last_clean":"2024-08-17T17:56:55.398198+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:56:55.398198+0000","last_undegraded":"2024-08-17T17:56:55.398198+0000","last_fullsized":"2024-08-17T17:56:55.398198+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":595,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.364812+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:56:43.364812+0000","last_peered":"2024-08-17T17:56:43.364812+0000","last_clean":"2024-08-17T17:56:43.364812+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:56:43.364812+0000","last_undegraded":"2024-08-17T17:56:43.364812+0000","last_fullsized":"2024-08-17T17:56:43.364812+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1207,"reported_epoch":1237,"state":"active+clean","last_fresh":"2024-08-17T17:56:43.364760+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:56:43.364760+0000","last_peered":"2024-08-17T17:56:43.364760+0000","last_clean":"2024-08-17T17:56:43.364760+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:56:43.364760+0000","last_undegraded":"2024-08-17T17:56:43.364760+0000","last_fullsized":"2024-08-17T17:56:43.364760+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1861,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.320566+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:57:08.320566+0000","last_peered":"2024-08-17T17:57:08.320566+0000","last_clean":"2024-08-17T17:57:08.320566+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:57:08.320566+0000","last_undegraded":"2024-08-17T17:57:08.320566+0000","last_fullsized":"2024-08-17T17:57:08.320566+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":2644443160,"num_objects":40352,"num_object_clones":0,"num_object_copies":121056,"num_objects_missing_on_primary":1962,"num_objects_missing":1962,"num_objects_degraded":15837,"num_objects_misplaced":6360,"num_objects_unfound":0,"num_objects_dirty":40352,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":169258,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37173,"num_bytes_recovered":2257715200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9233379328,"data_stored":9233367112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108432,"ondisk_log_size":108432,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234435,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1447568,"kb_used_data":1133576,"kb_used_omap":1,"kb_used_meta":313982,"kb_avail":92297584,"statfs":{"total":95995035648,"available":94512726016,"internally_reserved":0,"allocated":1160781824,"data_stored":1155775823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":321517912},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496070,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1605204,"kb_used_data":1303692,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92139948,"statfs":{"total":95995035648,"available":94351306752,"internally_reserved":0,"allocated":1334980608,"data_stored":1329971012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757708,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326322,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1698448,"kb_used_data":1459204,"kb_used_omap":57,"kb_used_meta":239174,"kb_avail":92046704,"statfs":{"total":95995035648,"available":94255824896,"internally_reserved":0,"allocated":1494224896,"data_stored":1489226639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":244914939},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405560,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1569156,"kb_used_data":1328764,"kb_used_omap":61,"kb_used_meta":240322,"kb_avail":92175996,"statfs":{"total":95995035648,"available":94388219904,"internally_reserved":0,"allocated":1360654336,"data_stored":1355660487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":246090680},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452095,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1169208,"kb_used_data":980528,"kb_used_omap":54,"kb_used_meta":188617,"kb_avail":92575944,"statfs":{"total":95995035648,"available":94797766656,"internally_reserved":0,"allocated":1004060672,"data_stored":999073812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":193144206},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474786,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1151496,"kb_used_data":924288,"kb_used_omap":119,"kb_used_meta":227080,"kb_avail":92593656,"statfs":{"total":95995035648,"available":94815903744,"internally_reserved":0,"allocated":946470912,"data_stored":941468888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":232530654},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019343,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":929894400,"data_stored":929890328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":990511104,"data_stored":990511104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1346830336,"data_stored":1346830336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1476988928,"data_stored":1476984856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":936706048,"data_stored":936706048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1318457344,"data_stored":1318453272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1146093568,"data_stored":1146093568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:57:12.736 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:12.755 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:57:12.757 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:12.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:12.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:12.860 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:12.881 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:12.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:12.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:12.983 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:13.012 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:13.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:13.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:13.027 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:57:13.114 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:13.136 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:13.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:13.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:13.238 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:13.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:13.260 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:13.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:13.361 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:13.386 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:13.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:13.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:13.488 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:13.510 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:13.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:13.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:13.612 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:13.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:13.635 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:13.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:13.736 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:13.756 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:13.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:13.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:13.858 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:13.881 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:13.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:13.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:13.983 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:14.004 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:14.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:14.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:14.106 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:14.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:14.128 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:14.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:14.229 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:14.256 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:14.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:14.358 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:14.381 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:14.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:14.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:14.483 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:14.505 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:14.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:14.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:14 smithi179 ceph-mon[93519]: pgmap v1918: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 15837/118791 objects degraded (13.332%); 6161/118791 objects misplaced (5.186%); 864 KiB/s, 13 objects/s recovering 2024-08-17T17:57:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:14 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:57:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:14.606 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:14.626 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:14 smithi130 ceph-mon[110068]: pgmap v1918: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 15837/118791 objects degraded (13.332%); 6161/118791 objects misplaced (5.186%); 864 KiB/s, 13 objects/s recovering 2024-08-17T17:57:14.626 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:14 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:57:14.626 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:14.626 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:14.626 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:14 smithi130 ceph-mon[105521]: pgmap v1918: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 207 op/s; 15837/118791 objects degraded (13.332%); 6161/118791 objects misplaced (5.186%); 864 KiB/s, 13 objects/s recovering 2024-08-17T17:57:14.626 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:14 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:57:14.626 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:14.627 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:14.628 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:14.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:14.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:14.729 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:14.763 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:14.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:14.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:14.865 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:14.889 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:14.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:14.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:14.991 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:57:14.991 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T17:57:14.991 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T17:57:14.992 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T17:57:14.992 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.992 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83241 45.1 45:82a6944f:::benchmark_data_smithi130_172551_object83240:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:57:14.992 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.647075+0000", 2024-08-17T17:57:14.992 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 516.33710231600003, 2024-08-17T17:57:14.992 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4851373099999998, 2024-08-17T17:57:14.992 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:14.992 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:14.992 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:14.992 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:14.992 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:14.992 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83241 2024-08-17T17:57:14.993 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.993 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:14.993 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.993 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:14.993 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647075+0000", 2024-08-17T17:57:14.993 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:14.993 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.993 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.993 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:14.993 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647075+0000", 2024-08-17T17:57:14.993 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:14.993 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.993 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.993 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:14.994 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647073+0000", 2024-08-17T17:57:14.994 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:57:14.994 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.994 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.994 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:14.994 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647109+0000", 2024-08-17T17:57:14.994 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6040000000000001e-05 2024-08-17T17:57:14.994 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.994 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.994 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:14.994 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647128+0000", 2024-08-17T17:57:14.994 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.9045000000000001e-05 2024-08-17T17:57:14.994 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.995 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.995 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:14.995 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647136+0000", 2024-08-17T17:57:14.995 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9589999999999995e-06 2024-08-17T17:57:14.995 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.995 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.995 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:14.995 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647230+0000", 2024-08-17T17:57:14.995 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4471999999999999e-05 2024-08-17T17:57:14.995 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.995 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.995 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:57:14.995 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.647244+0000", 2024-08-17T17:57:14.996 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4146e-05 2024-08-17T17:57:14.996 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.996 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.996 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:14.996 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636452+0000", 2024-08-17T17:57:14.996 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98920747099999995 2024-08-17T17:57:14.996 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.996 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.996 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:57:14.996 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636467+0000", 2024-08-17T17:57:14.996 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5031999999999999e-05 2024-08-17T17:57:14.996 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.996 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.996 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:14.997 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.102636+0000", 2024-08-17T17:57:14.997 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4661687780000001 2024-08-17T17:57:14.997 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.997 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.997 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:14.997 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.102811+0000", 2024-08-17T17:57:14.997 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017539199999999999 2024-08-17T17:57:14.997 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.997 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.997 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:14.997 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.102997+0000", 2024-08-17T17:57:14.997 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000185706 2024-08-17T17:57:14.998 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.998 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.998 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:14.998 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.109961+0000", 2024-08-17T17:57:14.998 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0069645180000000003 2024-08-17T17:57:14.998 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.998 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.998 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:14.998 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.109972+0000", 2024-08-17T17:57:14.998 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.093e-05 2024-08-17T17:57:14.998 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.998 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.998 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:14.999 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.129457+0000", 2024-08-17T17:57:14.999 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.019484379999999999 2024-08-17T17:57:14.999 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.999 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.999 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:14.999 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.129554+0000", 2024-08-17T17:57:14.999 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7465000000000003e-05 2024-08-17T17:57:14.999 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:14.999 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:14.999 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:14.999 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.132212+0000", 2024-08-17T17:57:14.999 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002657867 2024-08-17T17:57:14.999 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:14.999 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.000 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.000 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.000 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.000 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83244 45.1 45:81aa3e3d:::benchmark_data_smithi130_172551_object83243:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:57:15.000 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.658809+0000", 2024-08-17T17:57:15.000 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 516.32536808099997, 2024-08-17T17:57:15.000 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4758367149999998, 2024-08-17T17:57:15.000 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.000 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.000 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.000 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.000 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.000 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83244 2024-08-17T17:57:15.000 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.001 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.001 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.001 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.001 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658809+0000", 2024-08-17T17:57:15.001 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.001 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.001 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.001 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.001 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658809+0000", 2024-08-17T17:57:15.001 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.001 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.001 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.001 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.002 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658807+0000", 2024-08-17T17:57:15.002 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:57:15.002 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.002 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.002 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.002 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658846+0000", 2024-08-17T17:57:15.002 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8951e-05 2024-08-17T17:57:15.002 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.002 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.002 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.002 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658857+0000", 2024-08-17T17:57:15.002 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0569e-05 2024-08-17T17:57:15.002 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.002 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.003 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.003 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658861+0000", 2024-08-17T17:57:15.003 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0130000000000004e-06 2024-08-17T17:57:15.003 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.003 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.003 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.003 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658909+0000", 2024-08-17T17:57:15.003 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7958999999999999e-05 2024-08-17T17:57:15.003 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.003 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.003 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:57:15.003 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.658922+0000", 2024-08-17T17:57:15.003 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3519e-05 2024-08-17T17:57:15.004 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.004 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.004 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.004 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636515+0000", 2024-08-17T17:57:15.004 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97759250399999997 2024-08-17T17:57:15.004 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.004 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.004 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:57:15.004 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636526+0000", 2024-08-17T17:57:15.004 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1593000000000001e-05 2024-08-17T17:57:15.004 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.004 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.004 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.004 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.107864+0000", 2024-08-17T17:57:15.005 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4713374130000001 2024-08-17T17:57:15.005 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.005 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.005 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.005 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.107963+0000", 2024-08-17T17:57:15.005 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9687999999999998e-05 2024-08-17T17:57:15.005 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.005 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.005 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.005 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.108070+0000", 2024-08-17T17:57:15.005 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010689200000000001 2024-08-17T17:57:15.005 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.005 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.006 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.006 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112059+0000", 2024-08-17T17:57:15.006 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0039884810000000003 2024-08-17T17:57:15.006 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.006 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.006 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.006 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112120+0000", 2024-08-17T17:57:15.006 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1134000000000001e-05 2024-08-17T17:57:15.006 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.006 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.006 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.006 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.132267+0000", 2024-08-17T17:57:15.006 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.020146641999999999 2024-08-17T17:57:15.007 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.007 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.007 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.007 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.132303+0000", 2024-08-17T17:57:15.007 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6529000000000002e-05 2024-08-17T17:57:15.007 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.007 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.007 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.007 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.134646+0000", 2024-08-17T17:57:15.007 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0023425199999999998 2024-08-17T17:57:15.007 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.007 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.007 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.007 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.008 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.008 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83246 45.1 45:8e7e3f88:::benchmark_data_smithi130_172551_object83245:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:57:15.008 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.659302+0000", 2024-08-17T17:57:15.008 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 516.32487503200002, 2024-08-17T17:57:15.008 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.500952952, 2024-08-17T17:57:15.008 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.008 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.008 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.008 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.008 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.008 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83246 2024-08-17T17:57:15.008 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.008 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.009 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.009 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.009 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659302+0000", 2024-08-17T17:57:15.009 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.009 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.009 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.009 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.009 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659302+0000", 2024-08-17T17:57:15.009 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.009 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.009 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.009 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.009 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659301+0000", 2024-08-17T17:57:15.009 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:57:15.010 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.010 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.010 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.010 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659364+0000", 2024-08-17T17:57:15.010 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2366000000000006e-05 2024-08-17T17:57:15.010 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.010 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.010 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.010 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659374+0000", 2024-08-17T17:57:15.010 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0380999999999999e-05 2024-08-17T17:57:15.010 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.010 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.010 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.011 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659379+0000", 2024-08-17T17:57:15.011 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5950000000000001e-06 2024-08-17T17:57:15.011 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.011 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.011 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.011 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659426+0000", 2024-08-17T17:57:15.011 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7775000000000003e-05 2024-08-17T17:57:15.011 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.011 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.011 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:57:15.011 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.659438+0000", 2024-08-17T17:57:15.011 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1267000000000001e-05 2024-08-17T17:57:15.011 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.011 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.012 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.012 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636564+0000", 2024-08-17T17:57:15.012 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97712642100000002 2024-08-17T17:57:15.012 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.012 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.012 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:57:15.012 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636570+0000", 2024-08-17T17:57:15.012 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9120000000000003e-06 2024-08-17T17:57:15.012 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.012 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.012 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.012 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.109992+0000", 2024-08-17T17:57:15.012 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.473421493 2024-08-17T17:57:15.012 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.013 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.013 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.013 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.110115+0000", 2024-08-17T17:57:15.013 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000122934 2024-08-17T17:57:15.013 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.013 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.013 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.013 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.110209+0000", 2024-08-17T17:57:15.013 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4676e-05 2024-08-17T17:57:15.013 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.013 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.013 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.013 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112110+0000", 2024-08-17T17:57:15.013 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019009809999999999 2024-08-17T17:57:15.014 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.014 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.014 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.014 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.116034+0000", 2024-08-17T17:57:15.014 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0039233669999999997 2024-08-17T17:57:15.014 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.014 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.014 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.014 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.157325+0000", 2024-08-17T17:57:15.014 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.041291094 2024-08-17T17:57:15.014 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.014 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.015 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.015 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.157364+0000", 2024-08-17T17:57:15.015 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9023999999999998e-05 2024-08-17T17:57:15.015 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.015 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.015 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.015 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.160255+0000", 2024-08-17T17:57:15.015 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028912460000000001 2024-08-17T17:57:15.015 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.015 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.015 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.015 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.015 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:83252 45.1 45:87ce6d24:::benchmark_data_smithi130_172551_object83251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1178)", 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.670101+0000", 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 516.31407598199996, 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.493777863, 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 83252 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.016 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670101+0000", 2024-08-17T17:57:15.017 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.017 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.017 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.017 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.017 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670101+0000", 2024-08-17T17:57:15.017 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.017 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.017 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.017 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.017 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670100+0000", 2024-08-17T17:57:15.017 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:57:15.017 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.017 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.017 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.018 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670158+0000", 2024-08-17T17:57:15.018 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8179e-05 2024-08-17T17:57:15.018 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.018 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.018 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.018 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670170+0000", 2024-08-17T17:57:15.018 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2e-05 2024-08-17T17:57:15.018 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.018 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.018 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.018 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670175+0000", 2024-08-17T17:57:15.018 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3290000000000001e-06 2024-08-17T17:57:15.018 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.019 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.019 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.019 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670225+0000", 2024-08-17T17:57:15.019 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0126000000000001e-05 2024-08-17T17:57:15.019 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.019 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.019 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:57:15.019 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.670237+0000", 2024-08-17T17:57:15.019 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1832e-05 2024-08-17T17:57:15.019 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.019 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.019 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.019 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636594+0000", 2024-08-17T17:57:15.020 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.96635708200000003 2024-08-17T17:57:15.020 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.020 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.020 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T17:57:15.020 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:39.636603+0000", 2024-08-17T17:57:15.020 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7139999999999999e-06 2024-08-17T17:57:15.020 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.020 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.020 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.020 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112146+0000", 2024-08-17T17:57:15.020 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.475543396 2024-08-17T17:57:15.020 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.020 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.020 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.021 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112248+0000", 2024-08-17T17:57:15.021 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0001016 2024-08-17T17:57:15.021 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.021 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.021 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.021 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.112350+0000", 2024-08-17T17:57:15.021 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000102067 2024-08-17T17:57:15.021 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.021 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.021 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.021 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.116088+0000", 2024-08-17T17:57:15.021 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037383759999999999 2024-08-17T17:57:15.021 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.021 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.022 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.022 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.117798+0000", 2024-08-17T17:57:15.022 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0017097239999999999 2024-08-17T17:57:15.022 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.022 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.022 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.022 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.160341+0000", 2024-08-17T17:57:15.022 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.042542888000000001 2024-08-17T17:57:15.022 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.022 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.022 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.023 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.160400+0000", 2024-08-17T17:57:15.023 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9277000000000002e-05 2024-08-17T17:57:15.023 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.023 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.023 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.023 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:41.163879+0000", 2024-08-17T17:57:15.023 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0034785689999999999 2024-08-17T17:57:15.023 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.023 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.023 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.023 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.023 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.023 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:88238 45.1 45:8de64967:::benchmark_data_smithi130_172551_object88237:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1180)", 2024-08-17T17:57:15.024 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:49.366656+0000", 2024-08-17T17:57:15.024 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 505.61752149300003, 2024-08-17T17:57:15.024 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.079152609999999998, 2024-08-17T17:57:15.024 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.024 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.024 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.024 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.024 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.024 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 88238 2024-08-17T17:57:15.024 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.024 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.024 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.024 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.025 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366656+0000", 2024-08-17T17:57:15.025 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.025 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.025 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.025 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.025 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366656+0000", 2024-08-17T17:57:15.025 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.025 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.025 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.025 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.025 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366655+0000", 2024-08-17T17:57:15.025 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:57:15.025 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.026 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.026 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.026 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366709+0000", 2024-08-17T17:57:15.026 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.4617999999999998e-05 2024-08-17T17:57:15.026 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.026 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.026 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.026 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366720+0000", 2024-08-17T17:57:15.026 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.009e-05 2024-08-17T17:57:15.026 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.026 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.026 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.026 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.366723+0000", 2024-08-17T17:57:15.027 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8800000000000001e-06 2024-08-17T17:57:15.027 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.027 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.027 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.027 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.380121+0000", 2024-08-17T17:57:15.027 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013397166 2024-08-17T17:57:15.027 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.027 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.027 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.027 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.380213+0000", 2024-08-17T17:57:15.027 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2111e-05 2024-08-17T17:57:15.027 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.027 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.027 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.028 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.380314+0000", 2024-08-17T17:57:15.028 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000101746 2024-08-17T17:57:15.028 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.028 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.028 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.028 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.383932+0000", 2024-08-17T17:57:15.028 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00361722 2024-08-17T17:57:15.028 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.028 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.028 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.028 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.383975+0000", 2024-08-17T17:57:15.028 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3120000000000001e-05 2024-08-17T17:57:15.028 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.029 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.029 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.029 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.443093+0000", 2024-08-17T17:57:15.029 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.059118638000000001 2024-08-17T17:57:15.029 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.029 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.029 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.029 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.443138+0000", 2024-08-17T17:57:15.029 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4335999999999997e-05 2024-08-17T17:57:15.029 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.029 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.029 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.029 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.445808+0000", 2024-08-17T17:57:15.029 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002670389 2024-08-17T17:57:15.030 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.030 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.030 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.030 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.030 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.030 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:88246 45.1 45:8c252ee5:::benchmark_data_smithi130_172551_object88245:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1180)", 2024-08-17T17:57:15.030 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:49.370303+0000", 2024-08-17T17:57:15.030 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 505.613873807, 2024-08-17T17:57:15.030 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.078072420000000003, 2024-08-17T17:57:15.030 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.030 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.030 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.030 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.031 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.031 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 88246 2024-08-17T17:57:15.031 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.031 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.031 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.031 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.031 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370303+0000", 2024-08-17T17:57:15.031 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.031 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.031 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.031 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.031 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370303+0000", 2024-08-17T17:57:15.031 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.031 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.032 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.032 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.032 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370302+0000", 2024-08-17T17:57:15.032 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:57:15.032 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.032 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.032 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.032 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370372+0000", 2024-08-17T17:57:15.032 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0137999999999998e-05 2024-08-17T17:57:15.032 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.032 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.032 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.032 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370386+0000", 2024-08-17T17:57:15.032 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4062e-05 2024-08-17T17:57:15.033 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.033 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.033 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.033 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.370392+0000", 2024-08-17T17:57:15.033 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8690000000000001e-06 2024-08-17T17:57:15.033 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.033 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.033 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.033 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.381694+0000", 2024-08-17T17:57:15.033 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.011302395999999999 2024-08-17T17:57:15.033 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.033 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.033 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:15.035 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.381800+0000", 2024-08-17T17:57:15.035 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105775 2024-08-17T17:57:15.036 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.036 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.036 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.036 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.381899+0000", 2024-08-17T17:57:15.036 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.8529999999999993e-05 2024-08-17T17:57:15.036 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.036 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.036 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.036 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.383968+0000", 2024-08-17T17:57:15.036 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020689520000000002 2024-08-17T17:57:15.036 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.036 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.036 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.037 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.384010+0000", 2024-08-17T17:57:15.037 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2639999999999998e-05 2024-08-17T17:57:15.037 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.037 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.037 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.037 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.445861+0000", 2024-08-17T17:57:15.037 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.061850694999999997 2024-08-17T17:57:15.037 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.037 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.037 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.037 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.445900+0000", 2024-08-17T17:57:15.037 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8890999999999997e-05 2024-08-17T17:57:15.037 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.037 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.038 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.038 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:49.448376+0000", 2024-08-17T17:57:15.038 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024758029999999999 2024-08-17T17:57:15.038 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.038 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.038 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.038 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.038 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.038 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:88852 45.1 45:8bcd85bb:::benchmark_data_smithi130_172551_object88851:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:57:15.038 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:50.423711+0000", 2024-08-17T17:57:15.038 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 504.56046637100002, 2024-08-17T17:57:15.038 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.067301394, 2024-08-17T17:57:15.038 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.038 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.039 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.039 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.039 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.039 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 88852 2024-08-17T17:57:15.039 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.039 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.039 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.039 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.039 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423711+0000", 2024-08-17T17:57:15.039 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.039 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.039 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.039 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.040 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423711+0000", 2024-08-17T17:57:15.040 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.040 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.040 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.040 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.040 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423703+0000", 2024-08-17T17:57:15.040 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999928 2024-08-17T17:57:15.040 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.040 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.040 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.040 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423743+0000", 2024-08-17T17:57:15.040 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9957999999999999e-05 2024-08-17T17:57:15.040 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.041 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.041 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.041 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423758+0000", 2024-08-17T17:57:15.041 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4294000000000001e-05 2024-08-17T17:57:15.041 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.041 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.041 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.041 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423764+0000", 2024-08-17T17:57:15.041 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2419999999999997e-06 2024-08-17T17:57:15.041 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.041 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.041 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.041 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423809+0000", 2024-08-17T17:57:15.041 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5754000000000003e-05 2024-08-17T17:57:15.042 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.042 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.042 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.042 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.423915+0000", 2024-08-17T17:57:15.042 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105951 2024-08-17T17:57:15.042 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.042 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.042 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.042 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.424027+0000", 2024-08-17T17:57:15.042 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000111777 2024-08-17T17:57:15.042 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.044 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.044 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.044 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.430304+0000", 2024-08-17T17:57:15.044 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0062763059999999997 2024-08-17T17:57:15.044 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.044 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.044 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.044 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.430329+0000", 2024-08-17T17:57:15.044 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5187000000000001e-05 2024-08-17T17:57:15.044 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.044 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.044 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.045 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.488439+0000", 2024-08-17T17:57:15.045 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.058110436000000001 2024-08-17T17:57:15.045 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.045 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.045 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.045 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.488489+0000", 2024-08-17T17:57:15.045 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0201000000000003e-05 2024-08-17T17:57:15.045 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.045 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.045 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.045 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:50.491012+0000", 2024-08-17T17:57:15.045 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025226620000000002 2024-08-17T17:57:15.045 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.045 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.046 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.046 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.046 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.046 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:99564 45.1 45:877a8ac3:::benchmark_data_smithi130_172551_object99563:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1182)", 2024-08-17T17:57:15.046 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:08.821429+0000", 2024-08-17T17:57:15.046 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 486.16274844600002, 2024-08-17T17:57:15.046 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.069042615000000002, 2024-08-17T17:57:15.046 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.046 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.046 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.046 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.046 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.046 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 99564 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821429+0000", 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821429+0000", 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.047 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821427+0000", 2024-08-17T17:57:15.048 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:57:15.048 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.048 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.048 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.048 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821522+0000", 2024-08-17T17:57:15.048 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5486e-05 2024-08-17T17:57:15.048 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.048 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.048 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.048 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821543+0000", 2024-08-17T17:57:15.048 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.1069000000000001e-05 2024-08-17T17:57:15.048 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.048 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.048 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821552+0000", 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.4780000000000008e-06 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821602+0000", 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0368999999999998e-05 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821791+0000", 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00018891099999999999 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.049 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.050 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821980+0000", 2024-08-17T17:57:15.050 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00018913400000000001 2024-08-17T17:57:15.050 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.050 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.050 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.050 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827561+0000", 2024-08-17T17:57:15.050 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0055811910000000001 2024-08-17T17:57:15.050 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.050 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.050 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.050 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830393+0000", 2024-08-17T17:57:15.050 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028316940000000001 2024-08-17T17:57:15.050 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.050 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.051 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.051 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.887201+0000", 2024-08-17T17:57:15.051 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.056807923000000003 2024-08-17T17:57:15.051 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.051 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.051 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.051 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.887243+0000", 2024-08-17T17:57:15.051 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.163e-05 2024-08-17T17:57:15.051 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.051 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.051 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.051 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.890471+0000", 2024-08-17T17:57:15.051 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003228608 2024-08-17T17:57:15.052 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.052 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.052 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.052 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.052 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.052 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:99567 45.1 45:86b126dd:::benchmark_data_smithi130_172551_object99566:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1182)", 2024-08-17T17:57:15.052 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:08.821921+0000", 2024-08-17T17:57:15.052 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 486.16225573499997, 2024-08-17T17:57:15.052 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.071874980000000005, 2024-08-17T17:57:15.052 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.052 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.052 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.052 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.052 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.053 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 99567 2024-08-17T17:57:15.053 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.053 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.053 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.053 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.053 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821921+0000", 2024-08-17T17:57:15.053 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.053 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.053 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.053 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.053 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821921+0000", 2024-08-17T17:57:15.053 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.053 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.054 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.054 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.054 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821920+0000", 2024-08-17T17:57:15.054 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:57:15.054 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.054 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.054 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.054 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.821993+0000", 2024-08-17T17:57:15.054 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.2989000000000002e-05 2024-08-17T17:57:15.054 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.054 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.054 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.054 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.822006+0000", 2024-08-17T17:57:15.055 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2729e-05 2024-08-17T17:57:15.055 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.055 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.055 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.055 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.822012+0000", 2024-08-17T17:57:15.055 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.003e-06 2024-08-17T17:57:15.055 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.055 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.055 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.055 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.824701+0000", 2024-08-17T17:57:15.055 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0026888110000000002 2024-08-17T17:57:15.055 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.055 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.055 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.056 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.824892+0000", 2024-08-17T17:57:15.056 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019161699999999999 2024-08-17T17:57:15.056 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.056 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.056 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.056 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.825092+0000", 2024-08-17T17:57:15.056 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019969200000000001 2024-08-17T17:57:15.056 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.056 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.056 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.056 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830339+0000", 2024-08-17T17:57:15.056 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00524714 2024-08-17T17:57:15.056 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.056 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.057 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.057 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830563+0000", 2024-08-17T17:57:15.057 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00022378 2024-08-17T17:57:15.057 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.057 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.057 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.057 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.890526+0000", 2024-08-17T17:57:15.057 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.059962847 2024-08-17T17:57:15.057 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.057 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.057 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.057 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.890569+0000", 2024-08-17T17:57:15.057 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2808000000000001e-05 2024-08-17T17:57:15.057 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.058 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.058 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.058 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.893796+0000", 2024-08-17T17:57:15.058 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0032274970000000002 2024-08-17T17:57:15.058 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.058 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.058 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.058 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.058 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.058 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:99571 45.1 45:89cfe7f4:::benchmark_data_smithi130_172551_object99570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1182)", 2024-08-17T17:57:15.058 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:08.827641+0000", 2024-08-17T17:57:15.058 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 486.15653562400001, 2024-08-17T17:57:15.058 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.069045208999999996, 2024-08-17T17:57:15.059 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.059 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.059 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.059 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.059 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.059 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 99571 2024-08-17T17:57:15.059 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.059 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.059 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.059 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.059 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827641+0000", 2024-08-17T17:57:15.059 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.059 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.059 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.060 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.060 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827641+0000", 2024-08-17T17:57:15.060 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.060 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.060 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.060 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.060 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827640+0000", 2024-08-17T17:57:15.060 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:57:15.060 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.060 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.060 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.060 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827677+0000", 2024-08-17T17:57:15.060 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7431000000000001e-05 2024-08-17T17:57:15.061 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.061 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.061 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.061 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827689+0000", 2024-08-17T17:57:15.061 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1399999999999999e-05 2024-08-17T17:57:15.061 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.061 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.061 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.061 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827694+0000", 2024-08-17T17:57:15.061 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0180000000000001e-06 2024-08-17T17:57:15.061 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.061 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.061 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.061 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827736+0000", 2024-08-17T17:57:15.062 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1878000000000001e-05 2024-08-17T17:57:15.062 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.062 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.062 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.063 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827845+0000", 2024-08-17T17:57:15.063 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000109201 2024-08-17T17:57:15.063 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.063 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.063 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.063 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.827950+0000", 2024-08-17T17:57:15.063 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105064 2024-08-17T17:57:15.063 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.063 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.064 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.064 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830518+0000", 2024-08-17T17:57:15.064 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025678049999999998 2024-08-17T17:57:15.064 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.065 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.065 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.065 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.830749+0000", 2024-08-17T17:57:15.065 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00023170999999999999 2024-08-17T17:57:15.065 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.065 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.065 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.065 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.893864+0000", 2024-08-17T17:57:15.065 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.063114049000000005 2024-08-17T17:57:15.065 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.065 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.065 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.065 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.893911+0000", 2024-08-17T17:57:15.066 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7811e-05 2024-08-17T17:57:15.066 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.066 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.066 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.066 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:08.896687+0000", 2024-08-17T17:57:15.066 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002775226 2024-08-17T17:57:15.066 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.066 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.066 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.066 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.066 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.066 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:107884 45.1 45:89ca3498:::benchmark_data_smithi130_172551_object3077:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:57:15.066 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:25.288810+0000", 2024-08-17T17:57:15.067 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 469.69536720799999, 2024-08-17T17:57:15.067 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.069553105000000004, 2024-08-17T17:57:15.067 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.067 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.067 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.067 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.067 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.067 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 107884 2024-08-17T17:57:15.067 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.067 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.067 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.067 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.067 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.288810+0000", 2024-08-17T17:57:15.068 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.068 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.068 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.068 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.068 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.288810+0000", 2024-08-17T17:57:15.068 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.068 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.068 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.068 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.068 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.288808+0000", 2024-08-17T17:57:15.068 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:57:15.068 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.068 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.068 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.069 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.288817+0000", 2024-08-17T17:57:15.069 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0529999999999996e-06 2024-08-17T17:57:15.069 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.069 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.069 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.069 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.288830+0000", 2024-08-17T17:57:15.069 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2822e-05 2024-08-17T17:57:15.069 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.069 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.069 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.069 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.288836+0000", 2024-08-17T17:57:15.069 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8490000000000002e-06 2024-08-17T17:57:15.069 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.069 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.070 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.070 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.291429+0000", 2024-08-17T17:57:15.070 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025927120000000001 2024-08-17T17:57:15.070 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.070 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.070 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.070 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.291599+0000", 2024-08-17T17:57:15.070 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017012700000000001 2024-08-17T17:57:15.070 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.070 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.070 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.070 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.291763+0000", 2024-08-17T17:57:15.070 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016383200000000001 2024-08-17T17:57:15.070 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.071 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.071 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.071 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.294832+0000", 2024-08-17T17:57:15.071 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0030692250000000001 2024-08-17T17:57:15.071 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.071 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.071 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.071 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.295407+0000", 2024-08-17T17:57:15.071 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00057506500000000004 2024-08-17T17:57:15.071 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.071 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.071 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.071 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.354379+0000", 2024-08-17T17:57:15.072 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.058971509999999998 2024-08-17T17:57:15.072 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.072 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.072 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.072 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.354445+0000", 2024-08-17T17:57:15.072 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6694999999999998e-05 2024-08-17T17:57:15.072 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.072 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.072 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.072 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.358363+0000", 2024-08-17T17:57:15.072 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0039176549999999999 2024-08-17T17:57:15.072 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.072 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.072 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.073 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.073 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.073 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:111958 45.1 45:82b6ab59:::benchmark_data_smithi130_172551_object7151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:57:15.073 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:34.695989+0000", 2024-08-17T17:57:15.073 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 460.28818813499998, 2024-08-17T17:57:15.073 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.124334234, 2024-08-17T17:57:15.073 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.073 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.073 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.073 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.073 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.073 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 111958 2024-08-17T17:57:15.073 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.073 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.074 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.074 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.074 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.695989+0000", 2024-08-17T17:57:15.074 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.074 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.074 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.074 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.074 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.695989+0000", 2024-08-17T17:57:15.074 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.074 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.074 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.074 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.074 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.695987+0000", 2024-08-17T17:57:15.075 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:57:15.075 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.075 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.075 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.075 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.695994+0000", 2024-08-17T17:57:15.075 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3509999999999996e-06 2024-08-17T17:57:15.075 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.075 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.075 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.075 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696005+0000", 2024-08-17T17:57:15.075 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0891e-05 2024-08-17T17:57:15.075 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.075 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.075 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.076 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696010+0000", 2024-08-17T17:57:15.076 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8949999999999997e-06 2024-08-17T17:57:15.076 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.076 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.076 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.076 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.698750+0000", 2024-08-17T17:57:15.076 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027400459999999999 2024-08-17T17:57:15.076 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.076 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.076 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.076 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.698933+0000", 2024-08-17T17:57:15.076 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00018272300000000001 2024-08-17T17:57:15.076 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.076 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.699099+0000", 2024-08-17T17:57:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000166587 2024-08-17T17:57:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.705849+0000", 2024-08-17T17:57:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0067498280000000003 2024-08-17T17:57:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.705896+0000", 2024-08-17T17:57:15.077 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7012000000000003e-05 2024-08-17T17:57:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.817997+0000", 2024-08-17T17:57:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.112101241 2024-08-17T17:57:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.818038+0000", 2024-08-17T17:57:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0970000000000002e-05 2024-08-17T17:57:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.078 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.820323+0000", 2024-08-17T17:57:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002284669 2024-08-17T17:57:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:111959 45.1 45:85d2b159:::benchmark_data_smithi130_172551_object7152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:57:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:34.696016+0000", 2024-08-17T17:57:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 460.28816105200002, 2024-08-17T17:57:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.12563115999999999, 2024-08-17T17:57:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.079 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 111959 2024-08-17T17:57:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696016+0000", 2024-08-17T17:57:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.080 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696016+0000", 2024-08-17T17:57:15.081 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.081 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.081 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.081 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.081 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696016+0000", 2024-08-17T17:57:15.081 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:57:15.081 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.081 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.081 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.081 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696018+0000", 2024-08-17T17:57:15.081 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3719999999999999e-06 2024-08-17T17:57:15.081 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.081 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.082 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.082 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696024+0000", 2024-08-17T17:57:15.082 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0569999999999998e-06 2024-08-17T17:57:15.082 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.082 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.082 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.082 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.696028+0000", 2024-08-17T17:57:15.082 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.002e-06 2024-08-17T17:57:15.082 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.082 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.082 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.082 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.702259+0000", 2024-08-17T17:57:15.082 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0062313639999999997 2024-08-17T17:57:15.082 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.083 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.083 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.083 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.702447+0000", 2024-08-17T17:57:15.083 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00018781000000000001 2024-08-17T17:57:15.083 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.083 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.083 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.083 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.702577+0000", 2024-08-17T17:57:15.083 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000129954 2024-08-17T17:57:15.083 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.083 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.083 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.083 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.705926+0000", 2024-08-17T17:57:15.083 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0033483889999999998 2024-08-17T17:57:15.084 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.084 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.084 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.084 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.705970+0000", 2024-08-17T17:57:15.084 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4419999999999998e-05 2024-08-17T17:57:15.084 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.084 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.084 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.084 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.820352+0000", 2024-08-17T17:57:15.084 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.114382285 2024-08-17T17:57:15.084 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.084 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.084 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.085 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.820385+0000", 2024-08-17T17:57:15.085 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2809999999999999e-05 2024-08-17T17:57:15.085 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.085 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.085 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.085 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.821647+0000", 2024-08-17T17:57:15.085 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0012620489999999999 2024-08-17T17:57:15.085 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.085 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.085 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.085 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.085 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.085 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:111967 45.1 45:838e5df7:::benchmark_data_smithi130_172551_object7160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:57:15.085 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:34.708154+0000", 2024-08-17T17:57:15.086 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 460.27602263199998, 2024-08-17T17:57:15.086 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.11479482100000001, 2024-08-17T17:57:15.086 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.086 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.086 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.086 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.086 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.086 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 111967 2024-08-17T17:57:15.086 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.086 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.086 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.086 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.086 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708154+0000", 2024-08-17T17:57:15.087 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.087 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.087 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.087 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.087 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708154+0000", 2024-08-17T17:57:15.087 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.087 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.087 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.087 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.087 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708153+0000", 2024-08-17T17:57:15.087 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:57:15.087 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.087 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.088 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.088 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708163+0000", 2024-08-17T17:57:15.088 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0693000000000001e-05 2024-08-17T17:57:15.088 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.088 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.088 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.088 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708175+0000", 2024-08-17T17:57:15.088 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1677e-05 2024-08-17T17:57:15.088 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.088 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.088 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.088 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708181+0000", 2024-08-17T17:57:15.088 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6840000000000001e-06 2024-08-17T17:57:15.088 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.089 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.089 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.089 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708229+0000", 2024-08-17T17:57:15.089 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8049e-05 2024-08-17T17:57:15.089 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.089 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.089 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.089 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708393+0000", 2024-08-17T17:57:15.089 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000164394 2024-08-17T17:57:15.089 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.089 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.089 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.089 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.708553+0000", 2024-08-17T17:57:15.090 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00015941600000000001 2024-08-17T17:57:15.090 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.090 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.090 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.090 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.712190+0000", 2024-08-17T17:57:15.090 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0036373170000000002 2024-08-17T17:57:15.090 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.090 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.090 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.090 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.712200+0000", 2024-08-17T17:57:15.090 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6039999999999999e-06 2024-08-17T17:57:15.090 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.090 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.090 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.091 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.821672+0000", 2024-08-17T17:57:15.091 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.10947284 2024-08-17T17:57:15.091 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.091 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.091 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.091 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.821701+0000", 2024-08-17T17:57:15.091 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9023e-05 2024-08-17T17:57:15.091 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.091 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.091 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.091 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.822949+0000", 2024-08-17T17:57:15.091 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0012476950000000001 2024-08-17T17:57:15.091 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.091 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.092 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.092 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.092 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.092 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:111985 45.1 45:837e7695:::benchmark_data_smithi130_172551_object7178:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:57:15.092 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:34.747055+0000", 2024-08-17T17:57:15.092 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 460.237121651, 2024-08-17T17:57:15.092 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.078433560999999999, 2024-08-17T17:57:15.092 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.092 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.092 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.092 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.092 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.092 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 111985 2024-08-17T17:57:15.093 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.093 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.093 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.093 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.093 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747055+0000", 2024-08-17T17:57:15.093 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.093 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.093 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.093 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.093 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747055+0000", 2024-08-17T17:57:15.093 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.093 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.093 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.093 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.094 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747053+0000", 2024-08-17T17:57:15.094 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999976 2024-08-17T17:57:15.094 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.094 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.094 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.094 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747063+0000", 2024-08-17T17:57:15.094 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.005e-05 2024-08-17T17:57:15.094 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.094 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.094 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.094 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747082+0000", 2024-08-17T17:57:15.094 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.9391000000000001e-05 2024-08-17T17:57:15.094 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.095 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.095 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.095 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747090+0000", 2024-08-17T17:57:15.095 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0989999999999999e-06 2024-08-17T17:57:15.095 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.095 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.095 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.095 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747135+0000", 2024-08-17T17:57:15.095 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5169000000000001e-05 2024-08-17T17:57:15.095 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.095 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.095 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.095 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747265+0000", 2024-08-17T17:57:15.095 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012957100000000001 2024-08-17T17:57:15.096 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.096 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.096 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.096 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.747367+0000", 2024-08-17T17:57:15.096 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000101648 2024-08-17T17:57:15.096 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.096 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.096 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.096 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.750636+0000", 2024-08-17T17:57:15.096 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0032697749999999999 2024-08-17T17:57:15.096 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.096 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.096 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.097 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.751406+0000", 2024-08-17T17:57:15.097 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00076938899999999999 2024-08-17T17:57:15.097 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.097 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.097 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.097 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.823006+0000", 2024-08-17T17:57:15.097 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.071600330000000004 2024-08-17T17:57:15.097 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.097 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.097 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.097 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.823056+0000", 2024-08-17T17:57:15.097 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9913000000000001e-05 2024-08-17T17:57:15.097 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.097 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.098 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.098 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.825489+0000", 2024-08-17T17:57:15.098 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024327950000000002 2024-08-17T17:57:15.098 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.098 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.098 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.098 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.098 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.098 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:111989 45.1 45:8ed4177b:::benchmark_data_smithi130_172551_object7182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:57:15.098 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:34.761421+0000", 2024-08-17T17:57:15.098 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 460.22275610600002, 2024-08-17T17:57:15.098 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.067313382000000005, 2024-08-17T17:57:15.098 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.098 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.099 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.099 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.099 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.099 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 111989 2024-08-17T17:57:15.099 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.099 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.099 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.099 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.099 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761421+0000", 2024-08-17T17:57:15.099 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.099 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.099 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.099 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.100 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761421+0000", 2024-08-17T17:57:15.100 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.100 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.100 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.100 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.100 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761418+0000", 2024-08-17T17:57:15.100 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999971 2024-08-17T17:57:15.100 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.100 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.100 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.100 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761433+0000", 2024-08-17T17:57:15.100 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4643e-05 2024-08-17T17:57:15.100 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.100 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.101 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.101 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761465+0000", 2024-08-17T17:57:15.101 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2753000000000003e-05 2024-08-17T17:57:15.101 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.101 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.101 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.101 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761478+0000", 2024-08-17T17:57:15.101 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2446000000000001e-05 2024-08-17T17:57:15.101 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.101 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.101 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.101 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761567+0000", 2024-08-17T17:57:15.101 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.8905000000000003e-05 2024-08-17T17:57:15.101 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.102 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.102 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.102 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761765+0000", 2024-08-17T17:57:15.102 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019786200000000001 2024-08-17T17:57:15.102 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.102 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.102 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.102 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.761877+0000", 2024-08-17T17:57:15.102 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00011220100000000001 2024-08-17T17:57:15.102 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.102 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.102 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.102 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.765275+0000", 2024-08-17T17:57:15.103 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0033986089999999999 2024-08-17T17:57:15.103 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.103 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.103 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.103 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.765348+0000", 2024-08-17T17:57:15.103 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.2794999999999997e-05 2024-08-17T17:57:15.103 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.103 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.103 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.103 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.825563+0000", 2024-08-17T17:57:15.103 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.060214700000000003 2024-08-17T17:57:15.103 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.103 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.103 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.104 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.825679+0000", 2024-08-17T17:57:15.104 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000115665 2024-08-17T17:57:15.104 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.104 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.104 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.104 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:34.828734+0000", 2024-08-17T17:57:15.104 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0030556670000000002 2024-08-17T17:57:15.104 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.104 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.104 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.104 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.104 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.104 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:113030 45.1 45:8293cabe:::benchmark_data_smithi130_172551_object8223:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:57:15.104 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:37.005519+0000", 2024-08-17T17:57:15.105 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 457.97865818999998, 2024-08-17T17:57:15.105 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.084215829000000006, 2024-08-17T17:57:15.105 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.105 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.105 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.105 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.105 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.105 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 113030 2024-08-17T17:57:15.105 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.105 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.105 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.105 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.105 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005519+0000", 2024-08-17T17:57:15.106 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.106 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.106 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.106 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.106 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005519+0000", 2024-08-17T17:57:15.106 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.106 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.106 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.106 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.106 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005517+0000", 2024-08-17T17:57:15.106 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:57:15.106 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.106 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.107 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.107 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005523+0000", 2024-08-17T17:57:15.107 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5010000000000004e-06 2024-08-17T17:57:15.107 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.107 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.107 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.107 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005534+0000", 2024-08-17T17:57:15.107 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.13e-05 2024-08-17T17:57:15.107 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.107 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.107 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.107 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005539+0000", 2024-08-17T17:57:15.107 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0560000000000004e-06 2024-08-17T17:57:15.107 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.108 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.108 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.108 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.008372+0000", 2024-08-17T17:57:15.108 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002833225 2024-08-17T17:57:15.108 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.108 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.108 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.108 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.008492+0000", 2024-08-17T17:57:15.108 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000119633 2024-08-17T17:57:15.108 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.108 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.108 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.108 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.008597+0000", 2024-08-17T17:57:15.108 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105491 2024-08-17T17:57:15.109 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.109 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.109 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.109 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.014259+0000", 2024-08-17T17:57:15.109 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0056618279999999998 2024-08-17T17:57:15.109 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.109 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.109 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.109 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.014291+0000", 2024-08-17T17:57:15.109 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1934000000000003e-05 2024-08-17T17:57:15.109 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.109 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.109 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.110 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.087093+0000", 2024-08-17T17:57:15.110 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.072801433999999998 2024-08-17T17:57:15.110 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.110 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.110 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.110 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.087151+0000", 2024-08-17T17:57:15.110 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8109999999999998e-05 2024-08-17T17:57:15.110 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.110 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.110 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.110 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.089735+0000", 2024-08-17T17:57:15.110 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025838850000000002 2024-08-17T17:57:15.110 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.111 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.111 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.111 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.111 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.111 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:113033 45.1 45:874b46c2:::benchmark_data_smithi130_172551_object8226:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:57:15.111 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:37.005608+0000", 2024-08-17T17:57:15.111 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 457.97856905100002, 2024-08-17T17:57:15.111 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.086365193000000007, 2024-08-17T17:57:15.111 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.111 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.111 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.111 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.111 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.112 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 113033 2024-08-17T17:57:15.112 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.112 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.112 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.112 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.112 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005608+0000", 2024-08-17T17:57:15.112 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.112 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.112 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.112 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.112 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005608+0000", 2024-08-17T17:57:15.112 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.112 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.113 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.113 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.113 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005607+0000", 2024-08-17T17:57:15.113 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:57:15.113 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.113 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.113 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.113 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005611+0000", 2024-08-17T17:57:15.113 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4680000000000001e-06 2024-08-17T17:57:15.113 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.113 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.113 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.113 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005619+0000", 2024-08-17T17:57:15.114 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2409999999999995e-06 2024-08-17T17:57:15.114 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.114 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.114 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.114 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.005623+0000", 2024-08-17T17:57:15.114 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7249999999999999e-06 2024-08-17T17:57:15.114 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.114 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.114 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.114 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.011289+0000", 2024-08-17T17:57:15.114 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0056665539999999999 2024-08-17T17:57:15.114 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.114 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.115 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.115 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.011465+0000", 2024-08-17T17:57:15.115 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017524399999999999 2024-08-17T17:57:15.115 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.115 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.115 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.115 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.011636+0000", 2024-08-17T17:57:15.115 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017170099999999999 2024-08-17T17:57:15.115 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.115 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.115 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.115 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.014286+0000", 2024-08-17T17:57:15.115 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0026499499999999999 2024-08-17T17:57:15.115 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.116 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.116 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.116 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.014351+0000", 2024-08-17T17:57:15.116 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4957999999999997e-05 2024-08-17T17:57:15.116 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.116 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.116 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.116 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.089799+0000", 2024-08-17T17:57:15.116 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.075448276999999994 2024-08-17T17:57:15.116 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.116 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.116 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.116 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.089855+0000", 2024-08-17T17:57:15.117 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5294000000000003e-05 2024-08-17T17:57:15.117 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.117 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.117 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.117 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.091973+0000", 2024-08-17T17:57:15.117 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002118417 2024-08-17T17:57:15.117 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.117 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.117 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.117 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.117 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.117 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:113039 45.1 45:82b52502:::benchmark_data_smithi130_172551_object8232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:57:15.117 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:37.021375+0000", 2024-08-17T17:57:15.117 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 457.96280183200003, 2024-08-17T17:57:15.118 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.072155133999999996, 2024-08-17T17:57:15.118 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.118 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.118 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.118 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.118 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.118 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 113039 2024-08-17T17:57:15.118 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.118 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.118 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.118 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.118 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021375+0000", 2024-08-17T17:57:15.118 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.118 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.119 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.119 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.119 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021375+0000", 2024-08-17T17:57:15.119 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.119 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.119 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.119 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.119 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021374+0000", 2024-08-17T17:57:15.119 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:57:15.119 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.119 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.119 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.119 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021380+0000", 2024-08-17T17:57:15.120 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1909999999999997e-06 2024-08-17T17:57:15.120 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.120 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.120 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.120 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021392+0000", 2024-08-17T17:57:15.120 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1912e-05 2024-08-17T17:57:15.120 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.120 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.120 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.120 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021396+0000", 2024-08-17T17:57:15.120 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5170000000000001e-06 2024-08-17T17:57:15.120 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.120 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.120 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.121 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021435+0000", 2024-08-17T17:57:15.121 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8065000000000002e-05 2024-08-17T17:57:15.121 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.121 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.121 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.121 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021544+0000", 2024-08-17T17:57:15.121 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010904300000000001 2024-08-17T17:57:15.121 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.121 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.121 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.121 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.021674+0000", 2024-08-17T17:57:15.121 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013044700000000001 2024-08-17T17:57:15.121 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.122 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.122 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.122 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.027616+0000", 2024-08-17T17:57:15.122 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0059417929999999999 2024-08-17T17:57:15.122 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.122 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.122 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.122 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.027649+0000", 2024-08-17T17:57:15.122 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3655999999999998e-05 2024-08-17T17:57:15.122 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.122 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.122 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.122 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.092072+0000", 2024-08-17T17:57:15.123 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.064422191000000004 2024-08-17T17:57:15.123 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.123 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.123 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.123 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.092108+0000", 2024-08-17T17:57:15.123 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6560000000000002e-05 2024-08-17T17:57:15.123 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.123 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.123 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.123 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.093530+0000", 2024-08-17T17:57:15.123 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001422092 2024-08-17T17:57:15.123 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.123 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.123 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.124 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.124 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.124 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:113040 45.1 45:86caa578:::benchmark_data_smithi130_172551_object8233:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:57:15.124 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:37.023080+0000", 2024-08-17T17:57:15.124 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 457.961097354, 2024-08-17T17:57:15.124 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.072093230999999994, 2024-08-17T17:57:15.124 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:15.124 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:15.124 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:15.124 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:15.124 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:15.124 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 113040 2024-08-17T17:57:15.124 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.124 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:15.125 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.125 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:15.125 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.023080+0000", 2024-08-17T17:57:15.125 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.125 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.125 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.125 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:15.125 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.023080+0000", 2024-08-17T17:57:15.125 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:15.125 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.125 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.125 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:15.125 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.023078+0000", 2024-08-17T17:57:15.126 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:57:15.126 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.126 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.126 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:15.126 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.023088+0000", 2024-08-17T17:57:15.126 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5200000000000003e-06 2024-08-17T17:57:15.126 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.126 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.126 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:15.126 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.023104+0000", 2024-08-17T17:57:15.126 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5951e-05 2024-08-17T17:57:15.126 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.126 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.126 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:15.127 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.023111+0000", 2024-08-17T17:57:15.127 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.2810000000000003e-06 2024-08-17T17:57:15.127 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.127 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.127 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:15.127 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.025056+0000", 2024-08-17T17:57:15.127 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001944763 2024-08-17T17:57:15.127 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.127 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.127 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:15.127 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.025197+0000", 2024-08-17T17:57:15.127 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014151100000000001 2024-08-17T17:57:15.128 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.128 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.128 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6", 2024-08-17T17:57:15.128 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.025296+0000", 2024-08-17T17:57:15.128 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9264000000000004e-05 2024-08-17T17:57:15.128 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.128 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.128 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:15.128 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.027753+0000", 2024-08-17T17:57:15.128 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024564270000000002 2024-08-17T17:57:15.128 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.128 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.128 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.128 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.027853+0000", 2024-08-17T17:57:15.129 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9986000000000001e-05 2024-08-17T17:57:15.129 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.129 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.129 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:15.129 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.093580+0000", 2024-08-17T17:57:15.129 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.065727168000000002 2024-08-17T17:57:15.129 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.129 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.129 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:15.129 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.093619+0000", 2024-08-17T17:57:15.129 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8968999999999999e-05 2024-08-17T17:57:15.129 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:15.129 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:15.129 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:15.130 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:37.095173+0000", 2024-08-17T17:57:15.130 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0015538380000000001 2024-08-17T17:57:15.130 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.130 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.130 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.130 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:15.130 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:15.130 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:57:15.153 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:15.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:15.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:15.254 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:15.284 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:15.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:15.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:15.300 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:57:15.387 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:15.408 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:15.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:15.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:15.510 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:15.515 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:15 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:15.515 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:15 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:15.515 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:15 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:15.515 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:15 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:15.531 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:15.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:15.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:15.632 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:15.661 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:15.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:15.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:15.662 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:57:15.763 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:15.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:15.784 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:15.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:15 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:15 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:15.885 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:15.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:15.907 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:15.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:16.008 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:16.039 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:16.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:16.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:16.058 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:16.061 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:57:16.065 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:16.141 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:16.162 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:16.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:16.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:16.264 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:16.285 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:16.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:16.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:16.387 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:16.412 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:16.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:16.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:16.454 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T17:57:16.515 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:16.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:16.542 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:16.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:16.644 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:16.669 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:16.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:16.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:16.700 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:16 smithi130 ceph-mon[105521]: pgmap v1919: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 15837/117648 objects degraded (13.461%); 6161/117648 objects misplaced (5.237%); 864 KiB/s, 13 objects/s recovering 2024-08-17T17:57:16.700 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:16 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1932474291' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:57:16.700 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/117648 objects degraded (13.461%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:16.700 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:16 smithi130 ceph-mon[110068]: pgmap v1919: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 15837/117648 objects degraded (13.461%); 6161/117648 objects misplaced (5.237%); 864 KiB/s, 13 objects/s recovering 2024-08-17T17:57:16.700 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:16 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1932474291' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:57:16.700 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/117648 objects degraded (13.461%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:16.771 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:16.793 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:16.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:16.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:16 smithi179 ceph-mon[93519]: pgmap v1919: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 210 op/s; 15837/117648 objects degraded (13.461%); 6161/117648 objects misplaced (5.237%); 864 KiB/s, 13 objects/s recovering 2024-08-17T17:57:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:16 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1932474291' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:57:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/117648 objects degraded (13.461%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:16.866 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:57:16.896 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:16.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:16.919 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:16.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:17.020 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:17.044 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:17.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:17.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:17.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:17.171 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:17.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:17.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:17.273 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:17.296 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:17.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:17.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:17.398 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:17.421 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:17.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:17.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:17.522 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:17.545 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:17.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:17.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:17.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:17 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:57:17.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:17 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1932474291' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:57:17.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:17 smithi130 ceph-mon[110068]: osdmap e1240: 8 total, 8 up, 6 in 2024-08-17T17:57:17.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:17 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:57:17.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:17 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1932474291' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:57:17.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:17 smithi130 ceph-mon[105521]: osdmap e1240: 8 total, 8 up, 6 in 2024-08-17T17:57:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:17 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:57:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:17 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1932474291' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:57:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:17 smithi179 ceph-mon[93519]: osdmap e1240: 8 total, 8 up, 6 in 2024-08-17T17:57:17.647 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:17.668 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:17.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:17.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:17.770 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:17.791 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:17.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:17.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:17.893 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:17.915 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:17.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:17.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:18.017 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:18.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:18.039 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:18.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:18.140 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:18.160 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:18.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:18.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:18.262 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:18.283 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:18.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:18.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:18.385 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:18.408 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:18.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:18.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:18.510 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:18.537 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:18.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:18.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:18.579 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:18 smithi179 ceph-mon[93519]: pgmap v1921: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 192 op/s; 15837/116943 objects degraded (13.542%); 5968/116943 objects misplaced (5.103%); 969 KiB/s, 15 objects/s recovering 2024-08-17T17:57:18.638 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:18.651 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:18 smithi130 ceph-mon[105521]: pgmap v1921: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 192 op/s; 15837/116943 objects degraded (13.542%); 5968/116943 objects misplaced (5.103%); 969 KiB/s, 15 objects/s recovering 2024-08-17T17:57:18.651 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:18 smithi130 ceph-mon[110068]: pgmap v1921: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.4 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 192 op/s; 15837/116943 objects degraded (13.542%); 5968/116943 objects misplaced (5.103%); 969 KiB/s, 15 objects/s recovering 2024-08-17T17:57:18.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:18.661 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:18.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:18.762 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:18.785 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:18.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:18.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:18.817 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:57:18.817 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:57:18.817 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T17:57:18.817 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:57:18.887 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:18.909 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:18.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:18.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:19.011 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:19.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:19.038 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:19.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:19.139 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:19.160 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:19.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:19.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:19.262 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:19.284 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:19.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:19.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:19.385 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:19.412 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:19.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:19.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:19.514 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:19.537 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:19.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:19.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:19.639 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:19.658 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:57:19.662 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:19.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:19.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:19.764 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:19.785 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:19.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:19.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:19.887 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:19.914 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:19.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:19.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:20.007 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:57:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:57:19] "GET /metrics HTTP/1.1" 200 36801 "" "Prometheus/2.51.0" 2024-08-17T17:57:20.016 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:20.039 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:20.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:20.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:20.141 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:20.163 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:20.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:20.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:20.266 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:20.290 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:20.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:20.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:20.326 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:57:20.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:57:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:57:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:57:20.393 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:20.416 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:20.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:20.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:20.518 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:20.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:20.540 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:20.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:20.642 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:20.663 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:20.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:20.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:20.765 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:20.773 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:20 smithi130 ceph-mon[105521]: pgmap v1922: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 15837/115164 objects degraded (13.752%); 5968/115164 objects misplaced (5.182%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:57:20.774 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:20 smithi130 ceph-mon[110068]: pgmap v1922: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 15837/115164 objects degraded (13.752%); 5968/115164 objects misplaced (5.182%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:57:20.780 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:20 smithi179 ceph-mon[93519]: pgmap v1922: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 15837/115164 objects degraded (13.752%); 5968/115164 objects misplaced (5.182%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:57:20.789 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:20.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:20.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:20.874 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:57:20.874 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:57:20.874 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:57:20.875 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:57:20.875 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:57:20.875 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:57:20.875 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:57:20.875 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:57:20.887 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:57:20.887 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:57:20.892 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:20.905 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:57:20.905 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:57:20.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:20.918 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:20.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:20.921 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:57:20.921 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:57:20.927 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:57:20.927 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:57:20.944 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:57:20.944 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:57:21.020 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:21.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:21.043 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:21.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:21.144 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:21.168 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:21.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:21.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:21.270 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:21.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:21.293 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:21.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:21.393 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:21.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:21.416 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:21.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:21.503 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:57:21.516 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:21.567 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:21.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:21.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/115164 objects degraded (13.752%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:21 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/77864437' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:21 smithi179 ceph-mon[93519]: from='client.36039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:57:21.669 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:21.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:21.702 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:21.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:21.803 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:21.826 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:21.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:21.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/115164 objects degraded (13.752%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:21 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/77864437' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:21 smithi130 ceph-mon[105521]: from='client.36039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:57:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/115164 objects degraded (13.752%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:21 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/77864437' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:21 smithi130 ceph-mon[110068]: from='client.36039 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:57:21.928 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:21.948 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:21.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:21.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:21.990 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1922,"stamp":"2024-08-17T17:57:19.101273+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2514622437,"num_objects":38388,"num_object_clones":0,"num_object_copies":115164,"num_objects_missing_on_primary":1826,"num_objects_missing":1826,"num_objects_degraded":15837,"num_objects_misplaced":5968,"num_objects_unfound":0,"num_objects_dirty":38388,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":171526,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37335,"num_bytes_recovered":2268208141,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108729,"ondisk_log_size":108729,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":8278200,"kb_used_data":6792132,"kb_used_omap":295,"kb_used_meta":1485720,"kb_avail":554192712,"statfs":{"total":575970213888,"available":567493337088,"internally_reserved":0,"allocated":6955143168,"data_stored":6925087093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1521377743},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,2,0,1,0,1,4,8,1],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-130416640,"num_objects":-1990,"num_object_clones":0,"num_object_copies":-5970,"num_objects_missing_on_primary":-136,"num_objects_missing":-136,"num_objects_degraded":0,"num_objects_misplaced":-392,"num_objects_unfound":0,"num_objects_dirty":-1990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1990,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":133,"num_bytes_recovered":8716288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002509"},"pg_stats":[{"pgid":"45.f","version":"1239'10714","reported_seq":21272,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.349596+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:57:16.349596+0000","last_peered":"2024-08-17T17:57:16.349596+0000","last_clean":"2024-08-17T17:57:16.349596+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:57:16.349596+0000","last_undegraded":"2024-08-17T17:57:16.349596+0000","last_fullsized":"2024-08-17T17:57:16.349596+0000","mapping_epoch":1194,"log_start":"1175'3831","ondisk_log_start":"1175'3831","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6883,"log_dups_size":0,"ondisk_log_size":6883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160563200,"num_objects":2450,"num_object_clones":0,"num_object_copies":7350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10714,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1240'10653","reported_seq":21133,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:17.581616+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:57:17.581616+0000","last_peered":"2024-08-17T17:57:17.581616+0000","last_clean":"2024-08-17T17:57:17.581616+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:57:17.581616+0000","last_undegraded":"2024-08-17T17:57:17.581616+0000","last_fullsized":"2024-08-17T17:57:17.581616+0000","mapping_epoch":1211,"log_start":"1175'3632","ondisk_log_start":"1175'3632","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7021,"log_dups_size":0,"ondisk_log_size":7021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153419776,"num_objects":2341,"num_object_clones":0,"num_object_copies":7023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10653,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1240'11006","reported_seq":21874,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:18.543525+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:57:18.543525+0000","last_peered":"2024-08-17T17:57:18.543525+0000","last_clean":"2024-08-17T17:57:18.543525+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:57:18.543525+0000","last_undegraded":"2024-08-17T17:57:18.543525+0000","last_fullsized":"2024-08-17T17:57:18.543525+0000","mapping_epoch":1205,"log_start":"1176'5068","ondisk_log_start":"1176'5068","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5938,"log_dups_size":0,"ondisk_log_size":5938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155385880,"num_objects":2372,"num_object_clones":0,"num_object_copies":7116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2372,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11006,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1240'10622","reported_seq":10703,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:17.531985+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:57:17.531985+0000","last_peered":"2024-08-17T17:57:17.531985+0000","last_clean":"2024-08-17T17:57:17.531985+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:57:17.531985+0000","last_undegraded":"2024-08-17T17:57:17.531985+0000","last_fullsized":"2024-08-17T17:57:17.531985+0000","mapping_epoch":1167,"log_start":"1175'3889","ondisk_log_start":"1175'3889","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6733,"log_dups_size":0,"ondisk_log_size":6733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155844608,"num_objects":2378,"num_object_clones":0,"num_object_copies":7134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10622,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1240'10713","reported_seq":15987,"reported_epoch":1240,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:16.767588+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:57:16.767588+0000","last_peered":"2024-08-17T17:57:16.767588+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:57:16.767588+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1175'4826","ondisk_log_start":"1175'4826","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5887,"log_dups_size":0,"ondisk_log_size":5887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157351936,"num_objects":2401,"num_object_clones":0,"num_object_copies":7203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":2401,"num_objects_unfound":0,"num_objects_dirty":2401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10713,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":1883}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1240'10762","reported_seq":10843,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:17.811635+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:57:17.811635+0000","last_peered":"2024-08-17T17:57:17.811635+0000","last_clean":"2024-08-17T17:57:17.811635+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:57:17.811635+0000","last_undegraded":"2024-08-17T17:57:17.811635+0000","last_fullsized":"2024-08-17T17:57:17.811635+0000","mapping_epoch":1167,"log_start":"1175'4077","ondisk_log_start":"1175'4077","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6685,"log_dups_size":0,"ondisk_log_size":6685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152829952,"num_objects":2332,"num_object_clones":0,"num_object_copies":6996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10762,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1240'10820","reported_seq":10901,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:17.694827+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:57:17.694827+0000","last_peered":"2024-08-17T17:57:17.694827+0000","last_clean":"2024-08-17T17:57:17.694827+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:57:17.694827+0000","last_undegraded":"2024-08-17T17:57:17.694827+0000","last_fullsized":"2024-08-17T17:57:17.694827+0000","mapping_epoch":1167,"log_start":"1175'4069","ondisk_log_start":"1175'4069","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6751,"log_dups_size":0,"ondisk_log_size":6751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157679616,"num_objects":2406,"num_object_clones":0,"num_object_copies":7218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10820,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1240'10617","reported_seq":20108,"reported_epoch":1240,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:16.725142+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:57:16.725142+0000","last_peered":"2024-08-17T17:57:16.725142+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:57:16.725142+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1175'4675","ondisk_log_start":"1175'4675","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5942,"log_dups_size":0,"ondisk_log_size":5942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155516928,"num_objects":2373,"num_object_clones":0,"num_object_copies":7119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10617,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3108,"num_bytes_recovered":203685888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":1053},{"shards":"1,7","objects":306}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1240'10669","reported_seq":10760,"reported_epoch":1240,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:17.565558+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:57:17.565558+0000","last_peered":"2024-08-17T17:57:17.565558+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:57:17.565558+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1175'3977","ondisk_log_start":"1175'3977","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6692,"log_dups_size":0,"ondisk_log_size":6692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157745152,"num_objects":2407,"num_object_clones":0,"num_object_copies":7221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10669,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":2407},{"shards":"1,6","objects":-3426}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1240'10765","reported_seq":13872,"reported_epoch":1240,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:17.863485+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:57:17.863485+0000","last_peered":"2024-08-17T17:57:17.863485+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:57:17.863485+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1175'4806","ondisk_log_start":"1175'4806","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5959,"log_dups_size":0,"ondisk_log_size":5959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157876224,"num_objects":2409,"num_object_clones":0,"num_object_copies":7227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10765,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":848}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1239'10587","reported_seq":10748,"reported_epoch":1239,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:14.158418+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:57:14.158418+0000","last_peered":"2024-08-17T17:57:14.158418+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:57:14.158418+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1170'1400","ondisk_log_start":"1170'1400","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9187,"log_dups_size":0,"ondisk_log_size":9187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162988032,"num_objects":2487,"num_object_clones":0,"num_object_copies":7461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10587,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-2666}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1239'10618","reported_seq":16839,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:14.004423+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:57:14.004423+0000","last_peered":"2024-08-17T17:57:14.004423+0000","last_clean":"2024-08-17T17:57:14.004423+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:57:14.004423+0000","last_undegraded":"2024-08-17T17:57:14.004423+0000","last_fullsized":"2024-08-17T17:57:14.004423+0000","mapping_epoch":1183,"log_start":"1170'1500","ondisk_log_start":"1170'1500","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9118,"log_dups_size":0,"ondisk_log_size":9118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159121408,"num_objects":2428,"num_object_clones":0,"num_object_copies":7284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10618,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1240'10734","reported_seq":10895,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:18.470243+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:57:18.470243+0000","last_peered":"2024-08-17T17:57:18.470243+0000","last_clean":"2024-08-17T17:57:18.470243+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:57:18.470243+0000","last_undegraded":"2024-08-17T17:57:18.470243+0000","last_fullsized":"2024-08-17T17:57:18.470243+0000","mapping_epoch":1167,"log_start":"1175'4791","ondisk_log_start":"1175'4791","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158990336,"num_objects":2426,"num_object_clones":0,"num_object_copies":7278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10734,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1240'10610","reported_seq":10691,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:17.559559+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:57:17.559559+0000","last_peered":"2024-08-17T17:57:17.559559+0000","last_clean":"2024-08-17T17:57:17.559559+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:57:17.559559+0000","last_undegraded":"2024-08-17T17:57:17.559559+0000","last_fullsized":"2024-08-17T17:57:17.559559+0000","mapping_epoch":1167,"log_start":"1175'3625","ondisk_log_start":"1175'3625","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6985,"log_dups_size":0,"ondisk_log_size":6985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152305664,"num_objects":2324,"num_object_clones":0,"num_object_copies":6972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10610,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1239'10896","reported_seq":27462,"reported_epoch":1239,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:57:16.391624+0000","last_change":"2024-08-17T17:55:37.941989+0000","last_active":"2024-08-17T17:57:16.391624+0000","last_peered":"2024-08-17T17:57:16.391624+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:57:16.391624+0000","last_undegraded":"2024-08-17T17:57:16.391624+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1175'3991","ondisk_log_start":"1175'3991","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6905,"log_dups_size":0,"ondisk_log_size":6905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161742848,"num_objects":2468,"num_object_clones":0,"num_object_copies":7404,"num_objects_missing_on_primary":1826,"num_objects_missing":1826,"num_objects_degraded":0,"num_objects_misplaced":3567,"num_objects_unfound":0,"num_objects_dirty":2468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10896,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6320,"num_bytes_recovered":414187520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":13},{"shards":"0,4,6","objects":1356},{"shards":"4,5,6","objects":1099}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1240'10462","reported_seq":23809,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:18.518391+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:57:18.518391+0000","last_peered":"2024-08-17T17:57:18.518391+0000","last_clean":"2024-08-17T17:57:18.518391+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:57:18.518391+0000","last_undegraded":"2024-08-17T17:57:18.518391+0000","last_fullsized":"2024-08-17T17:57:18.518391+0000","mapping_epoch":1196,"log_start":"1175'4569","ondisk_log_start":"1175'4569","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5893,"log_dups_size":0,"ondisk_log_size":5893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154664960,"num_objects":2360,"num_object_clones":0,"num_object_copies":7080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10462,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1936,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.368445+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:57:08.368445+0000","last_peered":"2024-08-17T17:57:08.368445+0000","last_clean":"2024-08-17T17:57:08.368445+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:57:08.368445+0000","last_undegraded":"2024-08-17T17:57:08.368445+0000","last_fullsized":"2024-08-17T17:57:08.368445+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":517,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.370938+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:57:08.370938+0000","last_peered":"2024-08-17T17:57:08.370938+0000","last_clean":"2024-08-17T17:57:08.370938+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:57:08.370938+0000","last_undegraded":"2024-08-17T17:57:08.370938+0000","last_fullsized":"2024-08-17T17:57:08.370938+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1857,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.475238+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:57:16.475238+0000","last_peered":"2024-08-17T17:57:16.475238+0000","last_clean":"2024-08-17T17:57:16.475238+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:57:16.475238+0000","last_undegraded":"2024-08-17T17:57:16.475238+0000","last_fullsized":"2024-08-17T17:57:16.475238+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1881,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470294+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:57:16.470294+0000","last_peered":"2024-08-17T17:57:16.470294+0000","last_clean":"2024-08-17T17:57:16.470294+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:57:16.470294+0000","last_undegraded":"2024-08-17T17:57:16.470294+0000","last_fullsized":"2024-08-17T17:57:16.470294+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":79,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.371480+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:57:08.371480+0000","last_peered":"2024-08-17T17:57:08.371480+0000","last_clean":"2024-08-17T17:57:08.371480+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:57:08.371480+0000","last_undegraded":"2024-08-17T17:57:08.371480+0000","last_fullsized":"2024-08-17T17:57:08.371480+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1275,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470736+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:57:16.470736+0000","last_peered":"2024-08-17T17:57:16.470736+0000","last_clean":"2024-08-17T17:57:16.470736+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:57:16.470736+0000","last_undegraded":"2024-08-17T17:57:16.470736+0000","last_fullsized":"2024-08-17T17:57:16.470736+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":133,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.477948+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:57:16.477948+0000","last_peered":"2024-08-17T17:57:16.477948+0000","last_clean":"2024-08-17T17:57:16.477948+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:57:16.477948+0000","last_undegraded":"2024-08-17T17:57:16.477948+0000","last_fullsized":"2024-08-17T17:57:16.477948+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":653,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.474694+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:57:16.474694+0000","last_peered":"2024-08-17T17:57:16.474694+0000","last_clean":"2024-08-17T17:57:16.474694+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:57:16.474694+0000","last_undegraded":"2024-08-17T17:57:16.474694+0000","last_fullsized":"2024-08-17T17:57:16.474694+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1894,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.368094+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:57:08.368094+0000","last_peered":"2024-08-17T17:57:08.368094+0000","last_clean":"2024-08-17T17:57:08.368094+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:57:08.368094+0000","last_undegraded":"2024-08-17T17:57:08.368094+0000","last_fullsized":"2024-08-17T17:57:08.368094+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1896,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.371255+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:57:08.371255+0000","last_peered":"2024-08-17T17:57:08.371255+0000","last_clean":"2024-08-17T17:57:08.371255+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:57:08.371255+0000","last_undegraded":"2024-08-17T17:57:08.371255+0000","last_fullsized":"2024-08-17T17:57:08.371255+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":118,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.476483+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:57:16.476483+0000","last_peered":"2024-08-17T17:57:16.476483+0000","last_clean":"2024-08-17T17:57:16.476483+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:57:16.476483+0000","last_undegraded":"2024-08-17T17:57:16.476483+0000","last_fullsized":"2024-08-17T17:57:16.476483+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1870,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.476046+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:57:16.476046+0000","last_peered":"2024-08-17T17:57:16.476046+0000","last_clean":"2024-08-17T17:57:16.476046+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:57:16.476046+0000","last_undegraded":"2024-08-17T17:57:16.476046+0000","last_fullsized":"2024-08-17T17:57:16.476046+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1925,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.475964+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:57:16.475964+0000","last_peered":"2024-08-17T17:57:16.475964+0000","last_clean":"2024-08-17T17:57:16.475964+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:57:16.475964+0000","last_undegraded":"2024-08-17T17:57:16.475964+0000","last_fullsized":"2024-08-17T17:57:16.475964+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1917,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.368383+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:57:08.368383+0000","last_peered":"2024-08-17T17:57:08.368383+0000","last_clean":"2024-08-17T17:57:08.368383+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:57:08.368383+0000","last_undegraded":"2024-08-17T17:57:08.368383+0000","last_fullsized":"2024-08-17T17:57:08.368383+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1892,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.471300+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:57:16.471300+0000","last_peered":"2024-08-17T17:57:16.471300+0000","last_clean":"2024-08-17T17:57:16.471300+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:57:16.471300+0000","last_undegraded":"2024-08-17T17:57:16.471300+0000","last_fullsized":"2024-08-17T17:57:16.471300+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1214,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.371498+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:57:08.371498+0000","last_peered":"2024-08-17T17:57:08.371498+0000","last_clean":"2024-08-17T17:57:08.371498+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:57:08.371498+0000","last_undegraded":"2024-08-17T17:57:08.371498+0000","last_fullsized":"2024-08-17T17:57:08.371498+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1863,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.471072+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:57:16.471072+0000","last_peered":"2024-08-17T17:57:16.471072+0000","last_clean":"2024-08-17T17:57:16.471072+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:57:16.471072+0000","last_undegraded":"2024-08-17T17:57:16.471072+0000","last_fullsized":"2024-08-17T17:57:16.471072+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1911,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.368534+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:57:08.368534+0000","last_peered":"2024-08-17T17:57:08.368534+0000","last_clean":"2024-08-17T17:57:08.368534+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:57:08.368534+0000","last_undegraded":"2024-08-17T17:57:08.368534+0000","last_fullsized":"2024-08-17T17:57:08.368534+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1819,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470188+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:57:16.470188+0000","last_peered":"2024-08-17T17:57:16.470188+0000","last_clean":"2024-08-17T17:57:16.470188+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:57:16.470188+0000","last_undegraded":"2024-08-17T17:57:16.470188+0000","last_fullsized":"2024-08-17T17:57:16.470188+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":533,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.322501+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:57:08.322501+0000","last_peered":"2024-08-17T17:57:08.322501+0000","last_clean":"2024-08-17T17:57:08.322501+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:57:08.322501+0000","last_undegraded":"2024-08-17T17:57:08.322501+0000","last_fullsized":"2024-08-17T17:57:08.322501+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1210,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.322373+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:57:08.322373+0000","last_peered":"2024-08-17T17:57:08.322373+0000","last_clean":"2024-08-17T17:57:08.322373+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:57:08.322373+0000","last_undegraded":"2024-08-17T17:57:08.322373+0000","last_fullsized":"2024-08-17T17:57:08.322373+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1918,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.476693+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:57:16.476693+0000","last_peered":"2024-08-17T17:57:16.476693+0000","last_clean":"2024-08-17T17:57:16.476693+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:57:16.476693+0000","last_undegraded":"2024-08-17T17:57:16.476693+0000","last_fullsized":"2024-08-17T17:57:16.476693+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":371,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.322491+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:57:08.322491+0000","last_peered":"2024-08-17T17:57:08.322491+0000","last_clean":"2024-08-17T17:57:08.322491+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:57:08.322491+0000","last_undegraded":"2024-08-17T17:57:08.322491+0000","last_fullsized":"2024-08-17T17:57:08.322491+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":517,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.371205+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:57:08.371205+0000","last_peered":"2024-08-17T17:57:08.371205+0000","last_clean":"2024-08-17T17:57:08.371205+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:57:08.371205+0000","last_undegraded":"2024-08-17T17:57:08.371205+0000","last_fullsized":"2024-08-17T17:57:08.371205+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1195,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.373046+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:57:08.373046+0000","last_peered":"2024-08-17T17:57:08.373046+0000","last_clean":"2024-08-17T17:57:08.373046+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:57:08.373046+0000","last_undegraded":"2024-08-17T17:57:08.373046+0000","last_fullsized":"2024-08-17T17:57:08.373046+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1203,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.371563+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:57:08.371563+0000","last_peered":"2024-08-17T17:57:08.371563+0000","last_clean":"2024-08-17T17:57:08.371563+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:57:08.371563+0000","last_undegraded":"2024-08-17T17:57:08.371563+0000","last_fullsized":"2024-08-17T17:57:08.371563+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1964,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.368266+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:57:08.368266+0000","last_peered":"2024-08-17T17:57:08.368266+0000","last_clean":"2024-08-17T17:57:08.368266+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:57:08.368266+0000","last_undegraded":"2024-08-17T17:57:08.368266+0000","last_fullsized":"2024-08-17T17:57:08.368266+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1849,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.471225+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:57:16.471225+0000","last_peered":"2024-08-17T17:57:16.471225+0000","last_clean":"2024-08-17T17:57:16.471225+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:57:16.471225+0000","last_undegraded":"2024-08-17T17:57:16.471225+0000","last_fullsized":"2024-08-17T17:57:16.471225+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1924,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.476619+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:57:16.476619+0000","last_peered":"2024-08-17T17:57:16.476619+0000","last_clean":"2024-08-17T17:57:16.476619+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:57:16.476619+0000","last_undegraded":"2024-08-17T17:57:16.476619+0000","last_fullsized":"2024-08-17T17:57:16.476619+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1851,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.475116+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:57:16.475116+0000","last_peered":"2024-08-17T17:57:16.475116+0000","last_clean":"2024-08-17T17:57:16.475116+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:57:16.475116+0000","last_undegraded":"2024-08-17T17:57:16.475116+0000","last_fullsized":"2024-08-17T17:57:16.475116+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1882,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.368160+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:57:08.368160+0000","last_peered":"2024-08-17T17:57:08.368160+0000","last_clean":"2024-08-17T17:57:08.368160+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:57:08.368160+0000","last_undegraded":"2024-08-17T17:57:08.368160+0000","last_fullsized":"2024-08-17T17:57:08.368160+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1230,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.322585+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:57:08.322585+0000","last_peered":"2024-08-17T17:57:08.322585+0000","last_clean":"2024-08-17T17:57:08.322585+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:57:08.322585+0000","last_undegraded":"2024-08-17T17:57:08.322585+0000","last_fullsized":"2024-08-17T17:57:08.322585+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1930,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.478070+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:57:16.478070+0000","last_peered":"2024-08-17T17:57:16.478070+0000","last_clean":"2024-08-17T17:57:16.478070+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:57:16.478070+0000","last_undegraded":"2024-08-17T17:57:16.478070+0000","last_fullsized":"2024-08-17T17:57:16.478070+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":117,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.476307+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:57:16.476307+0000","last_peered":"2024-08-17T17:57:16.476307+0000","last_clean":"2024-08-17T17:57:16.476307+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:57:16.476307+0000","last_undegraded":"2024-08-17T17:57:16.476307+0000","last_fullsized":"2024-08-17T17:57:16.476307+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1949,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.477994+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:57:16.477994+0000","last_peered":"2024-08-17T17:57:16.477994+0000","last_clean":"2024-08-17T17:57:16.477994+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:57:16.477994+0000","last_undegraded":"2024-08-17T17:57:16.477994+0000","last_fullsized":"2024-08-17T17:57:16.477994+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":686,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.475839+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:57:16.475839+0000","last_peered":"2024-08-17T17:57:16.475839+0000","last_clean":"2024-08-17T17:57:16.475839+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:57:16.475839+0000","last_undegraded":"2024-08-17T17:57:16.475839+0000","last_fullsized":"2024-08-17T17:57:16.475839+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1930,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.478146+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:57:16.478146+0000","last_peered":"2024-08-17T17:57:16.478146+0000","last_clean":"2024-08-17T17:57:16.478146+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:57:16.478146+0000","last_undegraded":"2024-08-17T17:57:16.478146+0000","last_fullsized":"2024-08-17T17:57:16.478146+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1257,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.367898+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:57:08.367898+0000","last_peered":"2024-08-17T17:57:08.367898+0000","last_clean":"2024-08-17T17:57:08.367898+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:57:08.367898+0000","last_undegraded":"2024-08-17T17:57:08.367898+0000","last_fullsized":"2024-08-17T17:57:08.367898+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":177,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.476599+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:57:16.476599+0000","last_peered":"2024-08-17T17:57:16.476599+0000","last_clean":"2024-08-17T17:57:16.476599+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:57:16.476599+0000","last_undegraded":"2024-08-17T17:57:16.476599+0000","last_fullsized":"2024-08-17T17:57:16.476599+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1941,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.480186+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:57:16.480186+0000","last_peered":"2024-08-17T17:57:16.480186+0000","last_clean":"2024-08-17T17:57:16.480186+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:57:16.480186+0000","last_undegraded":"2024-08-17T17:57:16.480186+0000","last_fullsized":"2024-08-17T17:57:16.480186+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1919,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.475732+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:57:16.475732+0000","last_peered":"2024-08-17T17:57:16.475732+0000","last_clean":"2024-08-17T17:57:16.475732+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:57:16.475732+0000","last_undegraded":"2024-08-17T17:57:16.475732+0000","last_fullsized":"2024-08-17T17:57:16.475732+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1937,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.375181+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:57:08.375181+0000","last_peered":"2024-08-17T17:57:08.375181+0000","last_clean":"2024-08-17T17:57:08.375181+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:57:08.375181+0000","last_undegraded":"2024-08-17T17:57:08.375181+0000","last_fullsized":"2024-08-17T17:57:08.375181+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1943,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.368438+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:57:08.368438+0000","last_peered":"2024-08-17T17:57:08.368438+0000","last_clean":"2024-08-17T17:57:08.368438+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:57:08.368438+0000","last_undegraded":"2024-08-17T17:57:08.368438+0000","last_fullsized":"2024-08-17T17:57:08.368438+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1923,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.392275+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:57:08.392275+0000","last_peered":"2024-08-17T17:57:08.392275+0000","last_clean":"2024-08-17T17:57:08.392275+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:57:08.392275+0000","last_undegraded":"2024-08-17T17:57:08.392275+0000","last_fullsized":"2024-08-17T17:57:08.392275+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1237,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.322309+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:57:08.322309+0000","last_peered":"2024-08-17T17:57:08.322309+0000","last_clean":"2024-08-17T17:57:08.322309+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:57:08.322309+0000","last_undegraded":"2024-08-17T17:57:08.322309+0000","last_fullsized":"2024-08-17T17:57:08.322309+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1183,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.371505+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:57:08.371505+0000","last_peered":"2024-08-17T17:57:08.371505+0000","last_clean":"2024-08-17T17:57:08.371505+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:57:08.371505+0000","last_undegraded":"2024-08-17T17:57:08.371505+0000","last_fullsized":"2024-08-17T17:57:08.371505+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1834,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470634+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:57:16.470634+0000","last_peered":"2024-08-17T17:57:16.470634+0000","last_clean":"2024-08-17T17:57:16.470634+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:57:16.470634+0000","last_undegraded":"2024-08-17T17:57:16.470634+0000","last_fullsized":"2024-08-17T17:57:16.470634+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":163,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.371073+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:57:08.371073+0000","last_peered":"2024-08-17T17:57:08.371073+0000","last_clean":"2024-08-17T17:57:08.371073+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:57:08.371073+0000","last_undegraded":"2024-08-17T17:57:08.371073+0000","last_fullsized":"2024-08-17T17:57:08.371073+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1549,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.322048+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:57:08.322048+0000","last_peered":"2024-08-17T17:57:08.322048+0000","last_clean":"2024-08-17T17:57:08.322048+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:57:08.322048+0000","last_undegraded":"2024-08-17T17:57:08.322048+0000","last_fullsized":"2024-08-17T17:57:08.322048+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1925,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.480305+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:57:16.480305+0000","last_peered":"2024-08-17T17:57:16.480305+0000","last_clean":"2024-08-17T17:57:16.480305+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:57:16.480305+0000","last_undegraded":"2024-08-17T17:57:16.480305+0000","last_fullsized":"2024-08-17T17:57:16.480305+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1865,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.471148+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:57:16.471148+0000","last_peered":"2024-08-17T17:57:16.471148+0000","last_clean":"2024-08-17T17:57:16.471148+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:57:16.471148+0000","last_undegraded":"2024-08-17T17:57:16.471148+0000","last_fullsized":"2024-08-17T17:57:16.471148+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":517,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.371390+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:57:08.371390+0000","last_peered":"2024-08-17T17:57:08.371390+0000","last_clean":"2024-08-17T17:57:08.371390+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:57:08.371390+0000","last_undegraded":"2024-08-17T17:57:08.371390+0000","last_fullsized":"2024-08-17T17:57:08.371390+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":602,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.375078+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:57:08.375078+0000","last_peered":"2024-08-17T17:57:08.375078+0000","last_clean":"2024-08-17T17:57:08.375078+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:57:08.375078+0000","last_undegraded":"2024-08-17T17:57:08.375078+0000","last_fullsized":"2024-08-17T17:57:08.375078+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":618,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.371427+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:57:08.371427+0000","last_peered":"2024-08-17T17:57:08.371427+0000","last_clean":"2024-08-17T17:57:08.371427+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:57:08.371427+0000","last_undegraded":"2024-08-17T17:57:08.371427+0000","last_fullsized":"2024-08-17T17:57:08.371427+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1201,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.371210+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:57:08.371210+0000","last_peered":"2024-08-17T17:57:08.371210+0000","last_clean":"2024-08-17T17:57:08.371210+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:57:08.371210+0000","last_undegraded":"2024-08-17T17:57:08.371210+0000","last_fullsized":"2024-08-17T17:57:08.371210+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":636,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.474662+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:57:16.474662+0000","last_peered":"2024-08-17T17:57:16.474662+0000","last_clean":"2024-08-17T17:57:16.474662+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:57:16.474662+0000","last_undegraded":"2024-08-17T17:57:16.474662+0000","last_fullsized":"2024-08-17T17:57:16.474662+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1862,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.474701+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:57:16.474701+0000","last_peered":"2024-08-17T17:57:16.474701+0000","last_clean":"2024-08-17T17:57:16.474701+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:57:16.474701+0000","last_undegraded":"2024-08-17T17:57:16.474701+0000","last_fullsized":"2024-08-17T17:57:16.474701+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1955,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.368257+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:57:08.368257+0000","last_peered":"2024-08-17T17:57:08.368257+0000","last_clean":"2024-08-17T17:57:08.368257+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:57:08.368257+0000","last_undegraded":"2024-08-17T17:57:08.368257+0000","last_fullsized":"2024-08-17T17:57:08.368257+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":669,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.474524+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:57:16.474524+0000","last_peered":"2024-08-17T17:57:16.474524+0000","last_clean":"2024-08-17T17:57:16.474524+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:57:16.474524+0000","last_undegraded":"2024-08-17T17:57:16.474524+0000","last_fullsized":"2024-08-17T17:57:16.474524+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1190,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.371108+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:57:08.371108+0000","last_peered":"2024-08-17T17:57:08.371108+0000","last_clean":"2024-08-17T17:57:08.371108+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:57:08.371108+0000","last_undegraded":"2024-08-17T17:57:08.371108+0000","last_fullsized":"2024-08-17T17:57:08.371108+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1189,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.322508+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:57:08.322508+0000","last_peered":"2024-08-17T17:57:08.322508+0000","last_clean":"2024-08-17T17:57:08.322508+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:57:08.322508+0000","last_undegraded":"2024-08-17T17:57:08.322508+0000","last_fullsized":"2024-08-17T17:57:08.322508+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":653,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.476012+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:57:16.476012+0000","last_peered":"2024-08-17T17:57:16.476012+0000","last_clean":"2024-08-17T17:57:16.476012+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:57:16.476012+0000","last_undegraded":"2024-08-17T17:57:16.476012+0000","last_fullsized":"2024-08-17T17:57:16.476012+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1207,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.322569+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:57:08.322569+0000","last_peered":"2024-08-17T17:57:08.322569+0000","last_clean":"2024-08-17T17:57:08.322569+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:57:08.322569+0000","last_undegraded":"2024-08-17T17:57:08.322569+0000","last_fullsized":"2024-08-17T17:57:08.322569+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":91,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.321974+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:57:08.321974+0000","last_peered":"2024-08-17T17:57:08.321974+0000","last_clean":"2024-08-17T17:57:08.321974+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:57:08.321974+0000","last_undegraded":"2024-08-17T17:57:08.321974+0000","last_fullsized":"2024-08-17T17:57:08.321974+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1223,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.322568+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:57:08.322568+0000","last_peered":"2024-08-17T17:57:08.322568+0000","last_clean":"2024-08-17T17:57:08.322568+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:57:08.322568+0000","last_undegraded":"2024-08-17T17:57:08.322568+0000","last_fullsized":"2024-08-17T17:57:08.322568+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1240,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.371107+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:57:08.371107+0000","last_peered":"2024-08-17T17:57:08.371107+0000","last_clean":"2024-08-17T17:57:08.371107+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:57:08.371107+0000","last_undegraded":"2024-08-17T17:57:08.371107+0000","last_fullsized":"2024-08-17T17:57:08.371107+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1863,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.474985+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:57:16.474985+0000","last_peered":"2024-08-17T17:57:16.474985+0000","last_clean":"2024-08-17T17:57:16.474985+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:57:16.474985+0000","last_undegraded":"2024-08-17T17:57:16.474985+0000","last_fullsized":"2024-08-17T17:57:16.474985+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1184,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.322182+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:57:08.322182+0000","last_peered":"2024-08-17T17:57:08.322182+0000","last_clean":"2024-08-17T17:57:08.322182+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:57:08.322182+0000","last_undegraded":"2024-08-17T17:57:08.322182+0000","last_fullsized":"2024-08-17T17:57:08.322182+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1914,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.368296+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:57:08.368296+0000","last_peered":"2024-08-17T17:57:08.368296+0000","last_clean":"2024-08-17T17:57:08.368296+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:57:08.368296+0000","last_undegraded":"2024-08-17T17:57:08.368296+0000","last_fullsized":"2024-08-17T17:57:08.368296+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1915,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.367979+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:57:08.367979+0000","last_peered":"2024-08-17T17:57:08.367979+0000","last_clean":"2024-08-17T17:57:08.367979+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:57:08.367979+0000","last_undegraded":"2024-08-17T17:57:08.367979+0000","last_fullsized":"2024-08-17T17:57:08.367979+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":597,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.322418+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:57:08.322418+0000","last_peered":"2024-08-17T17:57:08.322418+0000","last_clean":"2024-08-17T17:57:08.322418+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:57:08.322418+0000","last_undegraded":"2024-08-17T17:57:08.322418+0000","last_fullsized":"2024-08-17T17:57:08.322418+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1209,"reported_epoch":1239,"state":"active+clean","last_fresh":"2024-08-17T17:57:08.322245+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:57:08.322245+0000","last_peered":"2024-08-17T17:57:08.322245+0000","last_clean":"2024-08-17T17:57:08.322245+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:57:08.322245+0000","last_undegraded":"2024-08-17T17:57:08.322245+0000","last_fullsized":"2024-08-17T17:57:08.322245+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1863,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.474788+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:57:16.474788+0000","last_peered":"2024-08-17T17:57:16.474788+0000","last_clean":"2024-08-17T17:57:16.474788+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:57:16.474788+0000","last_undegraded":"2024-08-17T17:57:16.474788+0000","last_fullsized":"2024-08-17T17:57:16.474788+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":2514026520,"num_objects":38362,"num_object_clones":0,"num_object_copies":115086,"num_objects_missing_on_primary":1826,"num_objects_missing":1826,"num_objects_degraded":15837,"num_objects_misplaced":5968,"num_objects_unfound":0,"num_objects_dirty":38362,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":171248,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37306,"num_bytes_recovered":2266431488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8880795648,"data_stored":8880783432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108522,"ondisk_log_size":108522,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234437,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1379440,"kb_used_data":1077928,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92365712,"statfs":{"total":95995035648,"available":94582489088,"internally_reserved":0,"allocated":1103798272,"data_stored":1098777399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496072,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1541236,"kb_used_data":1235564,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":92203916,"statfs":{"total":95995035648,"available":94416809984,"internally_reserved":0,"allocated":1265217536,"data_stored":1260193068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757710,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326324,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1632224,"kb_used_data":1388820,"kb_used_omap":57,"kb_used_meta":243334,"kb_avail":92112928,"statfs":{"total":95995035648,"available":94323638272,"internally_reserved":0,"allocated":1422151680,"data_stored":1417145987,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":249174779},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405562,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1508436,"kb_used_data":1263884,"kb_used_omap":61,"kb_used_meta":244482,"kb_avail":92236716,"statfs":{"total":95995035648,"available":94450397184,"internally_reserved":0,"allocated":1294217216,"data_stored":1289215931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":250350520},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452097,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1108360,"kb_used_data":936320,"kb_used_omap":54,"kb_used_meta":171977,"kb_avail":92636792,"statfs":{"total":95995035648,"available":94860075008,"internally_reserved":0,"allocated":958791680,"data_stored":953797384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":176104846},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474788,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1108504,"kb_used_data":889616,"kb_used_omap":119,"kb_used_meta":218760,"kb_avail":92636648,"statfs":{"total":95995035648,"available":94859927552,"internally_reserved":0,"allocated":910966784,"data_stored":905957324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":224010974},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,2,0,1,0,1,4,8,1],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019345,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":896208896,"data_stored":896204824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":940703744,"data_stored":940703744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1280311296,"data_stored":1280311296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1406537728,"data_stored":1406533656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":929038336,"data_stored":929038336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1250562048,"data_stored":1250557976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1089536000,"data_stored":1089536000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:57:21.994 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:57:22.010 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:22.051 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:22.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:22.075 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:22.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:22.175 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:22.199 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:22.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:22.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:22.300 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:22.330 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:22.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:22.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:22.348 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:57:22.348 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:57:22.432 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:22.454 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:22.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:22.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:22.556 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:22.577 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:22.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:22.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:22.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:22 smithi179 ceph-mon[93519]: pgmap v1923: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 199 op/s; 15837/114777 objects degraded (13.798%); 5968/114777 objects misplaced (5.200%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:57:22.678 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:22.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:22.700 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:22.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:22.801 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:22.823 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:22.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:22.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:22.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:22 smithi130 ceph-mon[105521]: pgmap v1923: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 199 op/s; 15837/114777 objects degraded (13.798%); 5968/114777 objects misplaced (5.200%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:57:22.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:22 smithi130 ceph-mon[110068]: pgmap v1923: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 199 op/s; 15837/114777 objects degraded (13.798%); 5968/114777 objects misplaced (5.200%); 851 KiB/s, 13 objects/s recovering 2024-08-17T17:57:22.924 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:22.945 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:22.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:22.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:23.047 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:23.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:23.068 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:23.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:23.138 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:57:23.170 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:23.198 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:23.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:23.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:23.300 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:23.321 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:23.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:23.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:23.423 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:23.446 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:23.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:23.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:23.548 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:23.569 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:23.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:23.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:23.671 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:23.675 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:23.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:23.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:23.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:23.794 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:23.815 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:23.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:23.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:23.908 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:57:23.916 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:23.952 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:23.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:23.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:24.054 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:24.076 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:24.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:24.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:24.178 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:24.204 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:24.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:24.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:24.306 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:24.327 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:24.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:24.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:24.429 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:24.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:24.452 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:24.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:24.553 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:24.564 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:57:24.581 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:24.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:24.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:24.683 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:24.715 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:24.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:24.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:24.788 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:24 smithi130 ceph-mon[105521]: pgmap v1924: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 15837/112968 objects degraded (14.019%); 5765/112968 objects misplaced (5.103%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:57:24.788 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:24 smithi130 ceph-mon[110068]: pgmap v1924: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 15837/112968 objects degraded (14.019%); 5765/112968 objects misplaced (5.103%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:57:24.817 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:24.837 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:24.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:24.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:24.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:24 smithi179 ceph-mon[93519]: pgmap v1924: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 15837/112968 objects degraded (14.019%); 5765/112968 objects misplaced (5.103%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:57:24.939 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:24.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:24.962 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:24.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:25.063 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:25.084 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:25.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:25.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:25.186 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:25.208 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:25.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:25.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:25.309 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:25.342 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:25.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:25.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:25.444 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:25.473 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:25.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:25.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:25.575 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:25.599 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:25.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:25.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:25.701 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:25.723 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:25.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:25.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:25.824 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:25.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:25.847 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:25.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:25.948 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:25.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:25.972 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:25.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:26.073 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:26.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:26.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:26.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:26.139 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:57:26.139 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:57:26.198 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:26.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:26.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:26.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:26.321 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:26.343 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:26.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:26.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:26.444 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:26.474 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:26.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:26.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:26.575 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:26.607 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:26.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:26.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:26.709 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:26.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:26.732 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:26.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:26.833 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:26.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:26 smithi179 ceph-mon[93519]: pgmap v1925: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 192 op/s; 15837/111864 objects degraded (14.157%); 5765/111864 objects misplaced (5.154%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:57:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/111864 objects degraded (14.157%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:26.857 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:26.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:26.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:26.885 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:26 smithi130 ceph-mon[105521]: pgmap v1925: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 192 op/s; 15837/111864 objects degraded (14.157%); 5765/111864 objects misplaced (5.154%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:57:26.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/111864 objects degraded (14.157%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:26.886 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:26 smithi130 ceph-mon[110068]: pgmap v1925: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 192 op/s; 15837/111864 objects degraded (14.157%); 5765/111864 objects misplaced (5.154%); 870 KiB/s, 13 objects/s recovering 2024-08-17T17:57:26.886 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/111864 objects degraded (14.157%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:26.933 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:57:26.958 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:26.985 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:26.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:26.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:27.087 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:27.111 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:27.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:27.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:27.213 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:27.235 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:27.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:27.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:27.323 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:57:27.337 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:27.359 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:27.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:27.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:27.461 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:27.482 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T17:57:27.483 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:27.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:27.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:27.585 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:27.606 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:27.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:27.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:27 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2343638716' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:57:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:27 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2343638716' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:57:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:27.707 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:27.708 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:27.752 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:27.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:27.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:27.776 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:27 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2343638716' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:57:27.776 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:27.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:27.876 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:27.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:27.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:27.978 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:27.999 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:27.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:28.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:28.102 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:28.124 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:28.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:28.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:28.226 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:28.247 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:28.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:28.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:28.349 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:28.371 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:28.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:28.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:28.473 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:28.495 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:28.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:28.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:28.597 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:28.621 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:28.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:28.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:28 smithi130 ceph-mon[110068]: pgmap v1926: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 15837/110940 objects degraded (14.275%); 5551/110940 objects misplaced (5.004%); 848 KiB/s, 13 objects/s recovering 2024-08-17T17:57:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:28 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:57:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:28 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2343638716' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:57:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:28 smithi130 ceph-mon[110068]: osdmap e1241: 8 total, 8 up, 6 in 2024-08-17T17:57:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:28 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2614075718' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:57:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:28 smithi130 ceph-mon[105521]: pgmap v1926: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 15837/110940 objects degraded (14.275%); 5551/110940 objects misplaced (5.004%); 848 KiB/s, 13 objects/s recovering 2024-08-17T17:57:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:28 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:57:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:28 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2343638716' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:57:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:28 smithi130 ceph-mon[105521]: osdmap e1241: 8 total, 8 up, 6 in 2024-08-17T17:57:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:28 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2614075718' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:57:28.723 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:28.748 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:28.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:28.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:28 smithi179 ceph-mon[93519]: pgmap v1926: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.3 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 198 op/s; 15837/110940 objects degraded (14.275%); 5551/110940 objects misplaced (5.004%); 848 KiB/s, 13 objects/s recovering 2024-08-17T17:57:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:28 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:57:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:28 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2343638716' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:57:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:28 smithi179 ceph-mon[93519]: osdmap e1241: 8 total, 8 up, 6 in 2024-08-17T17:57:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:28 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2614075718' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:57:28.850 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:28.876 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:28.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:28.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:28.977 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:29.004 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:29.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:29.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:29.106 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:29.127 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:29.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:29.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:29.178 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:57:29.178 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15837/110940 objects degraded (14.275%), 5 pgs degraded, 6 pgs undersized","count":11},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1769,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1241,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":36980,"data_bytes":2422347749,"bytes_used":8234569728,"bytes_avail":567735644160,"bytes_total":575970213888,"degraded_objects":15837,"degraded_total":110940,"degraded_ratio":0.14275283937263386,"misplaced_objects":5551,"misplaced_total":110940,"misplaced_ratio":0.050036055525509292,"recovering_objects_per_sec":13,"recovering_bytes_per_sec":868105,"recovering_keys_per_sec":0,"num_objects_recovered":141,"num_bytes_recovered":9240576,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":198},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":26,"modified":"2024-08-17T17:57:13.099933+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (9m)\n [==========================..] (remaining: 42s)","progress":0.93258428573608398,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:57:29.229 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:29.252 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:29.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:29.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:29.354 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:29.376 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:29.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:29.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:29.478 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:29.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:29.501 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:29.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:29.602 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:29.625 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:29.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:29.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:29.727 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:29.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:29.749 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:29.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:29.850 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:29.874 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:29.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:29.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:29.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:29.999 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:29.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:30.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:30.025 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:57:30.025 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:57:30.061 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:57:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:57:29] "GET /metrics HTTP/1.1" 200 36803 "" "Prometheus/2.51.0" 2024-08-17T17:57:30.099 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:57:30.099 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:57:30.100 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:30.123 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:30.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:30.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:30.225 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:30.250 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:30.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:30.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:30.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:57:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:57:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:57:30.351 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:30.373 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:30.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:30.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:30.475 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:30.496 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:30.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:30.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:30.598 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:30.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:30 smithi130 ceph-mon[105521]: pgmap v1928: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 221 op/s; 15837/108525 objects degraded (14.593%); 5551/108525 objects misplaced (5.115%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:57:30.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:30 smithi130 ceph-mon[110068]: pgmap v1928: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 221 op/s; 15837/108525 objects degraded (14.593%); 5551/108525 objects misplaced (5.115%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:57:30.621 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:30.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:30.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:30.723 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:30.744 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:30.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:30.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:30.821 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:57:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:30 smithi179 ceph-mon[93519]: pgmap v1928: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 221 op/s; 15837/108525 objects degraded (14.593%); 5551/108525 objects misplaced (5.115%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:57:30.846 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:30.897 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:30.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:30.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:30.998 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:31.020 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:31.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:31.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:31.120 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:57:31.122 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:31.147 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:31.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:31.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:31.249 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:31.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:31.270 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:31.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:31.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:31.394 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:31.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:31.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:31.399 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:57:31.496 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:31.517 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:31.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:31.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:31.619 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:31.644 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:31.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:31.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:31.746 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:31.767 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:31.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:31.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:31.820 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:31 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:31 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/108525 objects degraded (14.593%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:31.869 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:31.892 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:31.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:31.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:31.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:31 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:31 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/108525 objects degraded (14.593%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:31 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:31 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/108525 objects degraded (14.593%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:31.994 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:32.015 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:32.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:32.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:32.117 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:32.139 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:32.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:32.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:32.241 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:32.262 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:32.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:32.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:32.363 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:32.386 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:32.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:32.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:32.487 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:32.509 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:32.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:32.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:32.599 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:57:32.600 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:57:32.610 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:32.622 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:57:32.622 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:57:32.632 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:32.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:32.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:32.645 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:57:32.646 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:57:32.674 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:57:32.675 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:57:32.701 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:57:32.701 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:57:32.720 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:57:32.720 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:57:32.734 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:32.739 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:57:32.740 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:57:32.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:32.763 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:32.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:32.770 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:57:32.770 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:57:32.836 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:32 smithi130 ceph-mon[105521]: pgmap v1929: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 15837/108057 objects degraded (14.656%); 5551/108057 objects misplaced (5.137%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:57:32.836 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/761476179' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:32.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:32 smithi130 ceph-mon[110068]: pgmap v1929: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 15837/108057 objects degraded (14.656%); 5551/108057 objects misplaced (5.137%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:57:32.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/761476179' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:32.865 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:32.886 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:32.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:32.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:32.972 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:32 smithi179 ceph-mon[93519]: pgmap v1929: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 223 op/s; 15837/108057 objects degraded (14.656%); 5551/108057 objects misplaced (5.137%); 902 KiB/s, 14 objects/s recovering 2024-08-17T17:57:32.972 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/761476179' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:32.987 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:33.008 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:33.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:33.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:33.110 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:33.139 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:33.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:33.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:33.241 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:33.268 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:33.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:33.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:33.370 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:33.397 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:33.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:33.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:33.405 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:57:33.406 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:57:33.406 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T17:57:33.406 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T17:57:33.406 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:57:33.499 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:33.525 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:33.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:33.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:33.627 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:33.655 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:33.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:33.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:33.756 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:33.779 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:33.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:33.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:33.881 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:33.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:33.904 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:33.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:34.005 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:34.028 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:34.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:34.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:34.130 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:34.148 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:57:34.153 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:34.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:34.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:34.255 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:34.278 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:34.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:34.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:34.380 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:34.406 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:34.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:34.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:34.437 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:57:34.508 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:34.530 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:34.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:34.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:34.632 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:34.653 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:34.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:34.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:34.755 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:34.777 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:34.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:34.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:34 smithi179 ceph-mon[93519]: pgmap v1930: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 230 op/s; 15837/106038 objects degraded (14.935%); 5339/106038 objects misplaced (5.035%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:57:34.879 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:34.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:34 smithi130 ceph-mon[110068]: pgmap v1930: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 230 op/s; 15837/106038 objects degraded (14.935%); 5339/106038 objects misplaced (5.035%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:57:34.895 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:34 smithi130 ceph-mon[105521]: pgmap v1930: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 230 op/s; 15837/106038 objects degraded (14.935%); 5339/106038 objects misplaced (5.035%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:57:34.902 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:34.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:34.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:35.004 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:35.033 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:35.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:35.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:35.105 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:35.134 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:35.162 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:35.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:35.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:35.264 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:35.288 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:35.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:35.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:35.389 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:35.413 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:35.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:35.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:35.515 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:35.537 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:35.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:35.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:35.639 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:35.662 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:35.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:35.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:35.763 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:35.787 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:35.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:35.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:35.889 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:35.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:35.911 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:35.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:36.012 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:36.035 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:36.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:36.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:36.136 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:36.141 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:57:36.157 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:36.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:36.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:36.206 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:36 smithi130 ceph-mon[105521]: pgmap v1931: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 237 op/s; 15837/104730 objects degraded (15.122%); 5339/104730 objects misplaced (5.098%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:57:36.206 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:36 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:36.206 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:36 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:36.206 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/104730 objects degraded (15.122%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:36.206 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:36 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:57:36.207 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:36 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:57:36.207 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:36 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:36.207 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:36 smithi130 ceph-mon[110068]: pgmap v1931: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 237 op/s; 15837/104730 objects degraded (15.122%); 5339/104730 objects misplaced (5.098%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:57:36.207 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:36 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:36.207 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:36 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:36.207 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/104730 objects degraded (15.122%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:36.207 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:36 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:57:36.207 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:36 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:57:36.207 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:36 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:36.259 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:36.280 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:36.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:36.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:36.315 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:57:36.316 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:57:36.382 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:36.403 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:36.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:36.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:36.423 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:36 smithi179 ceph-mon[93519]: pgmap v1931: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 237 op/s; 15837/104730 objects degraded (15.122%); 5339/104730 objects misplaced (5.098%); 921 KiB/s, 14 objects/s recovering 2024-08-17T17:57:36.423 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:36 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:36.423 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:36 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:36.423 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/104730 objects degraded (15.122%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:36.423 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:36 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:57:36.423 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:36 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:57:36.423 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:36 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:36.505 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:36.534 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:36.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:36.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:36.636 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:36.658 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:36.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:36.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:36.760 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:36.785 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:36.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:36.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:36.815 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:57:36.816 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T17:57:36.816 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T17:57:36.816 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T17:57:36.816 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.816 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:82940 45.c 45:37c40ad6:::benchmark_data_smithi130_172551_object82939:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:57:36.816 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.327053+0000", 2024-08-17T17:57:36.816 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 538.09470710000005, 2024-08-17T17:57:36.816 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.080169211000000004, 2024-08-17T17:57:36.816 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.816 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.816 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.817 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.817 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.817 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 82940 2024-08-17T17:57:36.817 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.817 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.817 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.817 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.817 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327053+0000", 2024-08-17T17:57:36.817 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.817 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.817 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.817 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.817 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327053+0000", 2024-08-17T17:57:36.818 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.818 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.818 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.818 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.818 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327051+0000", 2024-08-17T17:57:36.818 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:57:36.818 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.818 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.818 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.818 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327120+0000", 2024-08-17T17:57:36.818 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.8663000000000003e-05 2024-08-17T17:57:36.818 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.818 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.819 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.819 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327135+0000", 2024-08-17T17:57:36.819 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5429000000000001e-05 2024-08-17T17:57:36.819 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.819 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.819 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.819 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327142+0000", 2024-08-17T17:57:36.819 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4280000000000001e-06 2024-08-17T17:57:36.819 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.819 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.819 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.819 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327185+0000", 2024-08-17T17:57:36.819 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3405999999999997e-05 2024-08-17T17:57:36.819 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.820 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.820 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.820 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327319+0000", 2024-08-17T17:57:36.820 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013401299999999999 2024-08-17T17:57:36.820 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.820 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.820 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:57:36.820 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.327462+0000", 2024-08-17T17:57:36.820 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014319699999999999 2024-08-17T17:57:36.820 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.820 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.820 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.820 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.332836+0000", 2024-08-17T17:57:36.820 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0053734919999999997 2024-08-17T17:57:36.821 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.821 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.821 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.821 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.333565+0000", 2024-08-17T17:57:36.821 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00072900500000000002 2024-08-17T17:57:36.821 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.821 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.821 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.821 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.407138+0000", 2024-08-17T17:57:36.821 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.073573349999999996 2024-08-17T17:57:36.821 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.821 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.821 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.822 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.407185+0000", 2024-08-17T17:57:36.822 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7311999999999997e-05 2024-08-17T17:57:36.822 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.822 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.822 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.822 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.407222+0000", 2024-08-17T17:57:36.822 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6621000000000001e-05 2024-08-17T17:57:36.822 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.822 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.822 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.822 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.822 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.822 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:82944 45.c 45:3e028341:::benchmark_data_smithi130_172551_object82943:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:57:36.823 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:38.329686+0000", 2024-08-17T17:57:36.823 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 538.09207351099997, 2024-08-17T17:57:36.823 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.091734019, 2024-08-17T17:57:36.823 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.823 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.823 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.823 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.823 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.823 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 82944 2024-08-17T17:57:36.823 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.823 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.823 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.823 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.823 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.329686+0000", 2024-08-17T17:57:36.824 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.824 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.824 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.824 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.824 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.329686+0000", 2024-08-17T17:57:36.824 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.824 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.824 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.824 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.824 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.329686+0000", 2024-08-17T17:57:36.824 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:57:36.824 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.824 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.825 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.825 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.329753+0000", 2024-08-17T17:57:36.825 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7683999999999998e-05 2024-08-17T17:57:36.825 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.825 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.825 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.825 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.329764+0000", 2024-08-17T17:57:36.825 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0334e-05 2024-08-17T17:57:36.825 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.825 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.825 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.825 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.329769+0000", 2024-08-17T17:57:36.825 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8099999999999997e-06 2024-08-17T17:57:36.826 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.826 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.826 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.826 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.329961+0000", 2024-08-17T17:57:36.826 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019194700000000001 2024-08-17T17:57:36.826 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.826 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.826 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.826 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.330073+0000", 2024-08-17T17:57:36.826 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000112793 2024-08-17T17:57:36.826 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.826 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.826 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:57:36.827 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.330203+0000", 2024-08-17T17:57:36.827 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000129212 2024-08-17T17:57:36.827 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.827 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.827 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.827 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.332999+0000", 2024-08-17T17:57:36.827 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027968870000000001 2024-08-17T17:57:36.827 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.827 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.827 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.827 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.333696+0000", 2024-08-17T17:57:36.827 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00069637200000000001 2024-08-17T17:57:36.827 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.828 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.828 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.828 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.421298+0000", 2024-08-17T17:57:36.828 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.087602625000000003 2024-08-17T17:57:36.828 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.828 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.828 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.828 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.421379+0000", 2024-08-17T17:57:36.828 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.1094999999999995e-05 2024-08-17T17:57:36.828 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.828 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.828 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.829 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:38.421420+0000", 2024-08-17T17:57:36.829 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0859999999999998e-05 2024-08-17T17:57:36.829 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.829 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.829 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.829 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.829 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.829 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:90850 45.a 45:52267e40:::benchmark_data_smithi130_172551_object90849:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:57:36.829 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:53.711337+0000", 2024-08-17T17:57:36.829 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 522.71042262599997, 2024-08-17T17:57:36.829 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.078178569000000003, 2024-08-17T17:57:36.829 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.830 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.830 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.830 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.830 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.830 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 90850 2024-08-17T17:57:36.830 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.830 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.830 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.830 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.830 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711337+0000", 2024-08-17T17:57:36.830 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.830 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.831 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.831 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.831 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711337+0000", 2024-08-17T17:57:36.831 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.831 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.831 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.831 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.831 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711336+0000", 2024-08-17T17:57:36.831 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:57:36.831 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.831 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.831 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.831 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711402+0000", 2024-08-17T17:57:36.832 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6050000000000006e-05 2024-08-17T17:57:36.832 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.832 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.832 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.832 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711411+0000", 2024-08-17T17:57:36.832 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.5560000000000008e-06 2024-08-17T17:57:36.832 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.832 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.832 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.832 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711415+0000", 2024-08-17T17:57:36.832 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1999999999999996e-06 2024-08-17T17:57:36.832 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.833 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.833 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.833 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711456+0000", 2024-08-17T17:57:36.833 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1356000000000002e-05 2024-08-17T17:57:36.833 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.833 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.833 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.833 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711588+0000", 2024-08-17T17:57:36.833 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013130800000000001 2024-08-17T17:57:36.833 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.833 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.833 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:57:36.833 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.711732+0000", 2024-08-17T17:57:36.833 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000144392 2024-08-17T17:57:36.834 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.834 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.834 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.834 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.714345+0000", 2024-08-17T17:57:36.834 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002613074 2024-08-17T17:57:36.834 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.834 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.834 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.834 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.787326+0000", 2024-08-17T17:57:36.834 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.072981089999999998 2024-08-17T17:57:36.834 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.834 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.834 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.835 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.789423+0000", 2024-08-17T17:57:36.835 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020972270000000001 2024-08-17T17:57:36.835 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.835 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.835 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.835 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.789474+0000", 2024-08-17T17:57:36.835 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0813000000000003e-05 2024-08-17T17:57:36.835 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.835 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.835 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.835 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.789516+0000", 2024-08-17T17:57:36.835 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1517999999999998e-05 2024-08-17T17:57:36.835 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.835 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.836 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.836 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.836 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.836 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:90856 45.9 45:9cde683e:::benchmark_data_smithi130_172551_object90855:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:57:36.836 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:53.717331+0000", 2024-08-17T17:57:36.836 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 522.70442868999999, 2024-08-17T17:57:36.836 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.073021671999999996, 2024-08-17T17:57:36.836 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.836 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.836 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.836 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.836 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.837 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 90856 2024-08-17T17:57:36.837 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.837 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.837 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.837 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.837 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717331+0000", 2024-08-17T17:57:36.837 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.837 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.837 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.837 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.837 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717331+0000", 2024-08-17T17:57:36.837 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.837 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717330+0000", 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717387+0000", 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7080999999999998e-05 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717401+0000", 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3961000000000001e-05 2024-08-17T17:57:36.838 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.839 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.839 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.839 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717407+0000", 2024-08-17T17:57:36.839 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0630000000000002e-06 2024-08-17T17:57:36.839 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.839 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.839 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.839 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717455+0000", 2024-08-17T17:57:36.839 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7966000000000001e-05 2024-08-17T17:57:36.839 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.839 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.839 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.839 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717595+0000", 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014063300000000001 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717749+0000", 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000154222 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.720647+0000", 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028974209999999999 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.840 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.720929+0000", 2024-08-17T17:57:36.841 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00028230200000000001 2024-08-17T17:57:36.841 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.841 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.841 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.841 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.790271+0000", 2024-08-17T17:57:36.841 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.069341497000000002 2024-08-17T17:57:36.841 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.841 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.841 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.841 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.790320+0000", 2024-08-17T17:57:36.841 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9051999999999997e-05 2024-08-17T17:57:36.841 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.841 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.842 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.842 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.790353+0000", 2024-08-17T17:57:36.842 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3086000000000001e-05 2024-08-17T17:57:36.842 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.842 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.842 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.842 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.842 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.842 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:90857 45.a 45:574290ca:::benchmark_data_smithi130_172551_object90856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:57:36.842 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:53.717418+0000", 2024-08-17T17:57:36.842 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 522.70434210300004, 2024-08-17T17:57:36.842 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.072220353000000001, 2024-08-17T17:57:36.842 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.843 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.843 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.843 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.843 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.843 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 90857 2024-08-17T17:57:36.843 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.843 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.843 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.843 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.843 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717418+0000", 2024-08-17T17:57:36.843 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.843 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.843 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.843 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.844 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717418+0000", 2024-08-17T17:57:36.844 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.844 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.844 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.844 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.844 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717417+0000", 2024-08-17T17:57:36.844 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:57:36.844 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.844 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.844 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.844 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717454+0000", 2024-08-17T17:57:36.844 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6261999999999999e-05 2024-08-17T17:57:36.844 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.844 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717466+0000", 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1992e-05 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717469+0000", 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.839e-06 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717504+0000", 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.481e-05 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.845 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.846 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.846 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717625+0000", 2024-08-17T17:57:36.846 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000121107 2024-08-17T17:57:36.846 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.846 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.846 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:57:36.846 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.717757+0000", 2024-08-17T17:57:36.846 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013128700000000001 2024-08-17T17:57:36.846 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.846 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.846 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.846 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.720559+0000", 2024-08-17T17:57:36.846 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028020409999999999 2024-08-17T17:57:36.847 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.847 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.847 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.847 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.789228+0000", 2024-08-17T17:57:36.847 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.068669189000000005 2024-08-17T17:57:36.847 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.847 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.847 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.847 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.789555+0000", 2024-08-17T17:57:36.847 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00032744700000000002 2024-08-17T17:57:36.847 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.847 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.847 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.847 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.789602+0000", 2024-08-17T17:57:36.848 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6687999999999998e-05 2024-08-17T17:57:36.848 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.848 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.848 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.848 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:53.789638+0000", 2024-08-17T17:57:36.848 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6140000000000003e-05 2024-08-17T17:57:36.848 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.848 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.848 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.848 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.848 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.848 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:91365 45.c 45:381dbc23:::benchmark_data_smithi130_172551_object91364:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:57:36.848 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:54.621472+0000", 2024-08-17T17:57:36.849 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 521.80028787000003, 2024-08-17T17:57:36.849 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.067773287000000002, 2024-08-17T17:57:36.849 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.849 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.849 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.849 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.849 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.849 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 91365 2024-08-17T17:57:36.849 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.849 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.849 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.849 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.849 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621472+0000", 2024-08-17T17:57:36.850 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.850 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.850 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.850 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.850 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621472+0000", 2024-08-17T17:57:36.850 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.850 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.850 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.850 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.850 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621471+0000", 2024-08-17T17:57:36.850 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:57:36.850 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.850 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.851 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.851 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621555+0000", 2024-08-17T17:57:36.851 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3290999999999999e-05 2024-08-17T17:57:36.851 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.851 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.851 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.851 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621565+0000", 2024-08-17T17:57:36.851 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0910000000000001e-05 2024-08-17T17:57:36.851 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.851 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.851 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.851 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621570+0000", 2024-08-17T17:57:36.851 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2209999999999999e-06 2024-08-17T17:57:36.851 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.852 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.852 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.852 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621630+0000", 2024-08-17T17:57:36.852 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0263999999999998e-05 2024-08-17T17:57:36.852 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.852 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.852 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.852 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621709+0000", 2024-08-17T17:57:36.852 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.8641999999999997e-05 2024-08-17T17:57:36.852 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.852 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.852 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:57:36.852 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.621798+0000", 2024-08-17T17:57:36.852 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.9588999999999999e-05 2024-08-17T17:57:36.853 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.853 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.853 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.853 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.627027+0000", 2024-08-17T17:57:36.853 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0052288689999999997 2024-08-17T17:57:36.853 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.853 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.853 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.853 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.629739+0000", 2024-08-17T17:57:36.853 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027122449999999998 2024-08-17T17:57:36.853 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.853 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.853 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.853 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689175+0000", 2024-08-17T17:57:36.854 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.059435756999999999 2024-08-17T17:57:36.854 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.854 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.854 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.854 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689218+0000", 2024-08-17T17:57:36.854 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3129e-05 2024-08-17T17:57:36.854 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.854 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.854 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.854 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689245+0000", 2024-08-17T17:57:36.854 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.711e-05 2024-08-17T17:57:36.854 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.854 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.854 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:91366 45.c 45:31c17a2b:::benchmark_data_smithi130_172551_object91365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:54.622876+0000", 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 521.79888378199996, 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.066455942000000004, 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 91366 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.855 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.856 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.856 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.622876+0000", 2024-08-17T17:57:36.856 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.856 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.856 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.856 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.856 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.622876+0000", 2024-08-17T17:57:36.856 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.856 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.856 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.856 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.856 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.622875+0000", 2024-08-17T17:57:36.856 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:57:36.856 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.857 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.857 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.857 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.622924+0000", 2024-08-17T17:57:36.857 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9209999999999998e-05 2024-08-17T17:57:36.857 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.857 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.857 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.857 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.622939+0000", 2024-08-17T17:57:36.857 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4564e-05 2024-08-17T17:57:36.857 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.857 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.857 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.857 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.622954+0000", 2024-08-17T17:57:36.858 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5536000000000002e-05 2024-08-17T17:57:36.858 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.858 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.858 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.858 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.623961+0000", 2024-08-17T17:57:36.858 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0010069879999999999 2024-08-17T17:57:36.858 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.858 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.858 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.858 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624089+0000", 2024-08-17T17:57:36.858 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000127747 2024-08-17T17:57:36.858 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.858 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.858 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:57:36.859 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624337+0000", 2024-08-17T17:57:36.859 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00024801899999999998 2024-08-17T17:57:36.859 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.859 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.859 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.859 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.629686+0000", 2024-08-17T17:57:36.859 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0053493459999999996 2024-08-17T17:57:36.859 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.859 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.859 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.859 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.629849+0000", 2024-08-17T17:57:36.859 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016321900000000001 2024-08-17T17:57:36.859 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689281+0000", 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.059432064 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689311+0000", 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9754e-05 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689332+0000", 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0818999999999998e-05 2024-08-17T17:57:36.860 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:91369 45.c 45:3fbff2c1:::benchmark_data_smithi130_172551_object91368:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1181)", 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:48:54.624118+0000", 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 521.79764191200002, 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.065287511000000006, 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 91369 2024-08-17T17:57:36.861 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.862 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.862 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.862 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.862 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624118+0000", 2024-08-17T17:57:36.862 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.862 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.862 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.862 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.862 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624118+0000", 2024-08-17T17:57:36.862 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.862 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.862 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.862 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.862 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624117+0000", 2024-08-17T17:57:36.863 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:57:36.863 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.863 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.863 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.863 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624160+0000", 2024-08-17T17:57:36.863 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2967999999999999e-05 2024-08-17T17:57:36.863 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.863 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.863 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.863 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624167+0000", 2024-08-17T17:57:36.863 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.8990000000000004e-06 2024-08-17T17:57:36.863 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.863 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.864 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.864 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.624172+0000", 2024-08-17T17:57:36.864 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8319999999999996e-06 2024-08-17T17:57:36.864 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.864 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.864 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.864 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.627046+0000", 2024-08-17T17:57:36.864 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028742910000000002 2024-08-17T17:57:36.864 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.864 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.864 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.864 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.627158+0000", 2024-08-17T17:57:36.864 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00011133600000000001 2024-08-17T17:57:36.865 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.865 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.865 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:57:36.865 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.627294+0000", 2024-08-17T17:57:36.865 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000136415 2024-08-17T17:57:36.865 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.865 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.865 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.865 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.629898+0000", 2024-08-17T17:57:36.865 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0026042819999999999 2024-08-17T17:57:36.865 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.865 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.865 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.865 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.630090+0000", 2024-08-17T17:57:36.866 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019181999999999999 2024-08-17T17:57:36.866 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.866 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.866 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.866 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689364+0000", 2024-08-17T17:57:36.866 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.059274134999999999 2024-08-17T17:57:36.866 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.866 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.866 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.866 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689388+0000", 2024-08-17T17:57:36.866 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3839999999999999e-05 2024-08-17T17:57:36.866 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.866 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:48:54.689405+0000", 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.7360999999999999e-05 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:108187 45.a 45:56008bcf:::benchmark_data_smithi130_172551_object3380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:25.915418+0000", 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 490.50634154099998, 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.104042786, 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.867 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.868 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.868 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.868 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 108187 2024-08-17T17:57:36.868 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.868 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.868 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.868 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.868 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915418+0000", 2024-08-17T17:57:36.868 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.868 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.868 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.868 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.868 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915418+0000", 2024-08-17T17:57:36.868 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.869 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.869 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.869 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.869 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915417+0000", 2024-08-17T17:57:36.869 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T17:57:36.869 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.869 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.869 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.869 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915425+0000", 2024-08-17T17:57:36.869 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7860000000000002e-06 2024-08-17T17:57:36.869 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.869 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.869 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.869 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915441+0000", 2024-08-17T17:57:36.870 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5480000000000001e-05 2024-08-17T17:57:36.870 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.870 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.870 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.870 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915447+0000", 2024-08-17T17:57:36.870 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6139999999999997e-06 2024-08-17T17:57:36.870 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.870 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.870 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.870 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915496+0000", 2024-08-17T17:57:36.870 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8374999999999997e-05 2024-08-17T17:57:36.870 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.871 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.871 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.871 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915635+0000", 2024-08-17T17:57:36.871 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000138758 2024-08-17T17:57:36.871 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.871 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.871 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:57:36.871 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.915761+0000", 2024-08-17T17:57:36.871 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000126573 2024-08-17T17:57:36.871 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.871 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.871 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.871 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:25.918644+0000", 2024-08-17T17:57:36.871 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028828930000000001 2024-08-17T17:57:36.872 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.872 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.872 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.872 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.017278+0000", 2024-08-17T17:57:36.872 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.098633437000000004 2024-08-17T17:57:36.872 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.872 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.872 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.872 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.019367+0000", 2024-08-17T17:57:36.872 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020891909999999998 2024-08-17T17:57:36.872 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.872 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.872 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.873 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.019417+0000", 2024-08-17T17:57:36.873 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9982999999999999e-05 2024-08-17T17:57:36.873 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.873 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.873 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.873 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.019461+0000", 2024-08-17T17:57:36.873 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4385999999999998e-05 2024-08-17T17:57:36.873 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.873 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.873 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.873 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.873 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.873 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:108356 45.a 45:5e4e7849:::benchmark_data_smithi130_172551_object3549:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:57:36.873 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:26.371218+0000", 2024-08-17T17:57:36.874 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 490.05054198400001, 2024-08-17T17:57:36.874 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.077600958999999997, 2024-08-17T17:57:36.874 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.874 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.874 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.874 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.874 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.874 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 108356 2024-08-17T17:57:36.874 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.874 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.874 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.874 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.874 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371218+0000", 2024-08-17T17:57:36.875 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.875 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.875 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.875 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.875 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371218+0000", 2024-08-17T17:57:36.875 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.875 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.875 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.875 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.875 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371217+0000", 2024-08-17T17:57:36.875 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T17:57:36.875 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.875 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.875 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371222+0000", 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2700000000000004e-06 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371233+0000", 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1413999999999999e-05 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371238+0000", 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4100000000000001e-06 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.876 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371281+0000", 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3653999999999999e-05 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371366+0000", 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.4938999999999994e-05 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.371441+0000", 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.4386000000000002e-05 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.877 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.374160+0000", 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027197570000000002 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.375982+0000", 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001821671 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.448743+0000", 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.072761274000000001 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.878 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.879 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.448774+0000", 2024-08-17T17:57:36.879 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1004000000000003e-05 2024-08-17T17:57:36.879 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.879 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.879 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.879 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:26.448819+0000", 2024-08-17T17:57:36.879 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4406000000000001e-05 2024-08-17T17:57:36.879 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.879 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.879 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.879 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.879 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.879 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109023 45.a 45:5d1a62c7:::benchmark_data_smithi130_172551_object4216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:57:36.880 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:27.877934+0000", 2024-08-17T17:57:36.880 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 488.54382632800002, 2024-08-17T17:57:36.880 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.119736067, 2024-08-17T17:57:36.880 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.880 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.880 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.880 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.880 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.880 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109023 2024-08-17T17:57:36.880 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.880 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.880 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.880 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.880 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.877934+0000", 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.877934+0000", 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.877933+0000", 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.881 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.877938+0000", 2024-08-17T17:57:36.882 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.755e-06 2024-08-17T17:57:36.882 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.882 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.882 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.882 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.877951+0000", 2024-08-17T17:57:36.882 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2995e-05 2024-08-17T17:57:36.882 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.882 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.882 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.882 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.877957+0000", 2024-08-17T17:57:36.882 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7690000000000002e-06 2024-08-17T17:57:36.882 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.882 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.882 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.883 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.878003+0000", 2024-08-17T17:57:36.883 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5886000000000001e-05 2024-08-17T17:57:36.883 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.883 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.883 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.883 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.878117+0000", 2024-08-17T17:57:36.883 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000114449 2024-08-17T17:57:36.883 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.883 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.883 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:57:36.883 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.878221+0000", 2024-08-17T17:57:36.883 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010336399999999999 2024-08-17T17:57:36.883 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.883 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.880965+0000", 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027437289999999999 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.886359+0000", 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0053943530000000002 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.997513+0000", 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.111154077 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.884 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.885 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.885 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.997617+0000", 2024-08-17T17:57:36.885 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000103948 2024-08-17T17:57:36.885 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.885 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.885 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.885 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:27.997670+0000", 2024-08-17T17:57:36.885 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2692999999999998e-05 2024-08-17T17:57:36.885 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.885 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.885 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.885 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.885 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.885 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109151 45.a 45:566f02b3:::benchmark_data_smithi130_172551_object4344:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:57:36.886 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:28.221634+0000", 2024-08-17T17:57:36.886 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 488.20012580299999, 2024-08-17T17:57:36.886 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.079254931000000001, 2024-08-17T17:57:36.886 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.886 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.886 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.886 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.886 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.886 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109151 2024-08-17T17:57:36.886 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.886 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.886 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.887 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.887 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221634+0000", 2024-08-17T17:57:36.887 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.887 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.887 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.887 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.887 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221634+0000", 2024-08-17T17:57:36.887 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.887 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.887 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.887 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.887 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221633+0000", 2024-08-17T17:57:36.887 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:57:36.888 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.888 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.888 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.888 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221637+0000", 2024-08-17T17:57:36.888 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8539999999999999e-06 2024-08-17T17:57:36.888 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.888 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.888 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.888 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221644+0000", 2024-08-17T17:57:36.888 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.401e-06 2024-08-17T17:57:36.888 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.888 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.888 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.889 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221648+0000", 2024-08-17T17:57:36.889 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3050000000000001e-06 2024-08-17T17:57:36.889 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.889 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.889 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.889 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221680+0000", 2024-08-17T17:57:36.889 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2404999999999999e-05 2024-08-17T17:57:36.889 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.889 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.889 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.889 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221753+0000", 2024-08-17T17:57:36.889 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3312000000000006e-05 2024-08-17T17:57:36.889 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.890 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.890 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:57:36.890 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.221844+0000", 2024-08-17T17:57:36.890 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0953999999999997e-05 2024-08-17T17:57:36.890 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.890 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.890 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.890 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.224607+0000", 2024-08-17T17:57:36.890 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002762376 2024-08-17T17:57:36.890 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.890 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.890 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.891 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.237911+0000", 2024-08-17T17:57:36.891 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013304744 2024-08-17T17:57:36.891 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.891 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.891 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.891 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.300805+0000", 2024-08-17T17:57:36.891 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.062893874000000002 2024-08-17T17:57:36.891 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.891 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.891 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.891 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.300867+0000", 2024-08-17T17:57:36.891 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1592000000000003e-05 2024-08-17T17:57:36.891 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.891 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.892 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.892 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.300889+0000", 2024-08-17T17:57:36.892 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.2019e-05 2024-08-17T17:57:36.892 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.892 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.892 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.892 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.892 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.892 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109163 45.a 45:54edb7bc:::benchmark_data_smithi130_172551_object4356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:57:36.892 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:28.242813+0000", 2024-08-17T17:57:36.892 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 488.17894673400002, 2024-08-17T17:57:36.892 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.079628663000000002, 2024-08-17T17:57:36.892 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.892 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.893 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.893 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.893 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.893 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109163 2024-08-17T17:57:36.893 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.893 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.893 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.893 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.893 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242813+0000", 2024-08-17T17:57:36.893 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.893 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.893 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.893 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.893 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242813+0000", 2024-08-17T17:57:36.894 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.894 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.894 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.894 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.894 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242812+0000", 2024-08-17T17:57:36.894 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:57:36.894 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.894 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.894 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.894 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242816+0000", 2024-08-17T17:57:36.894 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3449999999999999e-06 2024-08-17T17:57:36.894 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.894 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.894 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242827+0000", 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0137e-05 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242832+0000", 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5439999999999998e-06 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242873+0000", 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0948999999999997e-05 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.895 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.242992+0000", 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000119027 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.243110+0000", 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000118292 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.246498+0000", 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0033872749999999999 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.896 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.247473+0000", 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00097564699999999995 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.322376+0000", 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.074903091000000005 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.322409+0000", 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2419000000000003e-05 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.897 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.898 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.322442+0000", 2024-08-17T17:57:36.898 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2947000000000002e-05 2024-08-17T17:57:36.898 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.898 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.898 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.898 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.898 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.898 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109168 45.a 45:51a73ff8:::benchmark_data_smithi130_172551_object4361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:57:36.898 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:28.249348+0000", 2024-08-17T17:57:36.898 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 488.17241160499998, 2024-08-17T17:57:36.898 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.073180470999999997, 2024-08-17T17:57:36.898 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.898 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109168 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249348+0000", 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249348+0000", 2024-08-17T17:57:36.899 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249347+0000", 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249352+0000", 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9239999999999998e-06 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249362+0000", 2024-08-17T17:57:36.900 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9110000000000005e-06 2024-08-17T17:57:36.901 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.901 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.901 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.901 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249366+0000", 2024-08-17T17:57:36.901 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2629999999999997e-06 2024-08-17T17:57:36.901 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.901 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.901 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.901 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249400+0000", 2024-08-17T17:57:36.901 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3858e-05 2024-08-17T17:57:36.901 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.901 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.901 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.901 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249476+0000", 2024-08-17T17:57:36.902 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.5542000000000003e-05 2024-08-17T17:57:36.902 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.902 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.902 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:57:36.902 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.249543+0000", 2024-08-17T17:57:36.902 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7414000000000002e-05 2024-08-17T17:57:36.902 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.902 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.902 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.902 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.252152+0000", 2024-08-17T17:57:36.902 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002609077 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.252895+0000", 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00074267699999999996 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.322474+0000", 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.069579616999999996 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.322505+0000", 2024-08-17T17:57:36.903 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0766000000000003e-05 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:28.322529+0000", 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3533e-05 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109535 45.a 45:50e21fca:::benchmark_data_smithi130_172551_object4728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:29.154039+0000", 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 487.26772097700001, 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.12176530300000001, 2024-08-17T17:57:36.904 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109535 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154039+0000", 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.905 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154039+0000", 2024-08-17T17:57:36.906 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.906 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.906 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.906 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.906 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154038+0000", 2024-08-17T17:57:36.906 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:57:36.906 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.906 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.906 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.906 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154043+0000", 2024-08-17T17:57:36.906 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0560000000000004e-06 2024-08-17T17:57:36.906 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.906 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.906 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.907 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154054+0000", 2024-08-17T17:57:36.907 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.057e-05 2024-08-17T17:57:36.907 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.907 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.907 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.907 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154058+0000", 2024-08-17T17:57:36.907 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.702e-06 2024-08-17T17:57:36.907 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.907 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.907 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.907 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154097+0000", 2024-08-17T17:57:36.907 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8847999999999998e-05 2024-08-17T17:57:36.907 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.908 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.908 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.908 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154205+0000", 2024-08-17T17:57:36.908 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000107408 2024-08-17T17:57:36.908 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.908 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.908 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:57:36.908 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.154309+0000", 2024-08-17T17:57:36.908 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010401 2024-08-17T17:57:36.908 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.908 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.908 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.908 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.157600+0000", 2024-08-17T17:57:36.909 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003291661 2024-08-17T17:57:36.909 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.909 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.909 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.909 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.158088+0000", 2024-08-17T17:57:36.909 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00048783099999999999 2024-08-17T17:57:36.909 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.909 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.909 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.909 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.275734+0000", 2024-08-17T17:57:36.909 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.117646024 2024-08-17T17:57:36.909 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.909 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.909 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.275771+0000", 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6519000000000001e-05 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.275804+0000", 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3569999999999999e-05 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109546 45.a 45:5a3ddbc0:::benchmark_data_smithi130_172551_object4739:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:29.173181+0000", 2024-08-17T17:57:36.910 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 487.24857915799998, 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.109487797, 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109546 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173181+0000", 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.911 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.912 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.912 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173181+0000", 2024-08-17T17:57:36.912 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.912 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.912 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.912 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.912 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173180+0000", 2024-08-17T17:57:36.912 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:57:36.912 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.912 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.912 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.912 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173186+0000", 2024-08-17T17:57:36.912 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2029999999999997e-06 2024-08-17T17:57:36.912 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173197+0000", 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1167e-05 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173203+0000", 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.241e-06 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173232+0000", 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8569999999999999e-05 2024-08-17T17:57:36.913 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.914 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.914 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.914 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173308+0000", 2024-08-17T17:57:36.914 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.6097e-05 2024-08-17T17:57:36.914 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.914 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.914 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:57:36.914 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.173373+0000", 2024-08-17T17:57:36.914 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4941e-05 2024-08-17T17:57:36.914 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.914 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.914 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.914 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.175940+0000", 2024-08-17T17:57:36.915 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025670039999999999 2024-08-17T17:57:36.915 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.915 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.915 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.915 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.182032+0000", 2024-08-17T17:57:36.915 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0060918719999999999 2024-08-17T17:57:36.915 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.915 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.915 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.915 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.282582+0000", 2024-08-17T17:57:36.915 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.10055022700000001 2024-08-17T17:57:36.915 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.915 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.915 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.916 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.282629+0000", 2024-08-17T17:57:36.916 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6981999999999998e-05 2024-08-17T17:57:36.916 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.916 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.916 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.916 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.282668+0000", 2024-08-17T17:57:36.916 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9573999999999998e-05 2024-08-17T17:57:36.916 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.916 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.916 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.916 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.916 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.916 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:109547 45.a 45:5fae21b8:::benchmark_data_smithi130_172551_object4740:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:57:36.917 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:29.176203+0000", 2024-08-17T17:57:36.917 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 487.24555645300001, 2024-08-17T17:57:36.917 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.106643664, 2024-08-17T17:57:36.917 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.917 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.917 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.917 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.917 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.917 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 109547 2024-08-17T17:57:36.917 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.917 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.917 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.918 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.918 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176203+0000", 2024-08-17T17:57:36.918 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.918 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.918 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.918 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.918 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176203+0000", 2024-08-17T17:57:36.918 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.918 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.918 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.918 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.918 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176203+0000", 2024-08-17T17:57:36.918 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T17:57:36.918 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176210+0000", 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.1670000000000002e-06 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176220+0000", 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0344e-05 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176226+0000", 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9499999999999998e-06 2024-08-17T17:57:36.919 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.920 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.920 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.920 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176253+0000", 2024-08-17T17:57:36.920 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.6786000000000001e-05 2024-08-17T17:57:36.920 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.920 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.920 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.920 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176323+0000", 2024-08-17T17:57:36.920 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0520999999999998e-05 2024-08-17T17:57:36.920 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.920 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.920 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:57:36.920 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.176387+0000", 2024-08-17T17:57:36.920 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3226999999999996e-05 2024-08-17T17:57:36.921 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.921 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.921 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.921 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.178044+0000", 2024-08-17T17:57:36.921 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0016577759999999999 2024-08-17T17:57:36.921 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.921 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.921 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.921 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.182089+0000", 2024-08-17T17:57:36.921 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0040443559999999998 2024-08-17T17:57:36.921 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.921 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.921 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.921 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.282793+0000", 2024-08-17T17:57:36.922 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.10070444200000001 2024-08-17T17:57:36.922 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.922 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.922 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.922 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.282825+0000", 2024-08-17T17:57:36.922 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1643000000000002e-05 2024-08-17T17:57:36.922 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.922 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.922 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.922 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:29.282847+0000", 2024-08-17T17:57:36.922 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.213e-05 2024-08-17T17:57:36.922 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.923 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.923 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.923 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.923 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.923 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:112660 45.a 45:593714f6:::benchmark_data_smithi130_172551_object7853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1186)", 2024-08-17T17:57:36.923 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:49:36.180053+0000", 2024-08-17T17:57:36.923 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 480.241707324, 2024-08-17T17:57:36.923 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.072488023999999998, 2024-08-17T17:57:36.923 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.923 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.923 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.923 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.923 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.923 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 112660 2024-08-17T17:57:36.924 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.924 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.924 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.924 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.924 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.180053+0000", 2024-08-17T17:57:36.924 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.924 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.924 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.924 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.924 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.180053+0000", 2024-08-17T17:57:36.924 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.924 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.924 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.924 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.180051+0000", 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.180057+0000", 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1889999999999998e-06 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.180067+0000", 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0675e-05 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.925 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.180072+0000", 2024-08-17T17:57:36.926 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1560000000000003e-06 2024-08-17T17:57:36.926 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.926 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.926 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.926 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.180115+0000", 2024-08-17T17:57:36.926 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2870000000000001e-05 2024-08-17T17:57:36.926 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.926 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.926 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.926 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.180209+0000", 2024-08-17T17:57:36.926 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4090000000000002e-05 2024-08-17T17:57:36.926 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.927 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.927 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:57:36.927 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.180300+0000", 2024-08-17T17:57:36.927 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0067000000000004e-05 2024-08-17T17:57:36.927 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.927 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.927 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.927 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.183363+0000", 2024-08-17T17:57:36.927 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003063325 2024-08-17T17:57:36.927 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.927 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.927 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.927 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.184852+0000", 2024-08-17T17:57:36.927 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0014896009999999999 2024-08-17T17:57:36.928 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.928 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.928 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.928 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.252465+0000", 2024-08-17T17:57:36.928 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.067612754999999997 2024-08-17T17:57:36.928 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.928 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.928 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.928 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.252515+0000", 2024-08-17T17:57:36.928 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.935e-05 2024-08-17T17:57:36.928 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.928 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.928 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.928 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:49:36.252541+0000", 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5987999999999999e-05 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:117533 45.c 45:39d9de8a:::benchmark_data_smithi130_172551_object12726:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1193)", 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:50:19.736989+0000", 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 436.68477093299998, 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.077276261999999998, 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.929 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.930 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 117533 2024-08-17T17:57:36.930 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.930 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.930 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.930 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.930 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:19.736989+0000", 2024-08-17T17:57:36.930 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.930 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.930 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.930 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.930 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:19.736989+0000", 2024-08-17T17:57:36.930 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.930 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.931 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.931 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.931 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:19.736986+0000", 2024-08-17T17:57:36.931 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999976 2024-08-17T17:57:36.931 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.931 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.931 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.931 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:19.736997+0000", 2024-08-17T17:57:36.931 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0845999999999999e-05 2024-08-17T17:57:36.931 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.931 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.931 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.932 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:19.737020+0000", 2024-08-17T17:57:36.932 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3184999999999999e-05 2024-08-17T17:57:36.932 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.932 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.932 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.932 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:19.737029+0000", 2024-08-17T17:57:36.932 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.8689999999999993e-06 2024-08-17T17:57:36.932 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.932 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.932 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.932 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:19.737078+0000", 2024-08-17T17:57:36.932 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8526999999999998e-05 2024-08-17T17:57:36.932 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.933 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.933 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.933 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:19.737171+0000", 2024-08-17T17:57:36.933 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.3195000000000005e-05 2024-08-17T17:57:36.933 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.933 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.933 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:57:36.933 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:19.737254+0000", 2024-08-17T17:57:36.933 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3372000000000001e-05 2024-08-17T17:57:36.933 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.933 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.933 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.933 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:19.739922+0000", 2024-08-17T17:57:36.933 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0026680100000000002 2024-08-17T17:57:36.934 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.934 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.934 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.934 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:19.740173+0000", 2024-08-17T17:57:36.934 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00025081600000000001 2024-08-17T17:57:36.934 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.934 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.934 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.934 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:19.814178+0000", 2024-08-17T17:57:36.934 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.074005214999999999 2024-08-17T17:57:36.934 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.934 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.934 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.935 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:19.814225+0000", 2024-08-17T17:57:36.935 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6026000000000003e-05 2024-08-17T17:57:36.935 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.935 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.935 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.935 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:50:19.814265+0000", 2024-08-17T17:57:36.935 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0655999999999998e-05 2024-08-17T17:57:36.935 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.935 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.935 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.935 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.935 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.935 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.45158.0:163281 45.7 45:ecfa5766:::benchmark_data_smithi130_172551_object58474:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1236)", 2024-08-17T17:57:36.935 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:56:38.483058+0000", 2024-08-17T17:57:36.936 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 57.938702034000002, 2024-08-17T17:57:36.936 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.026853259000000001, 2024-08-17T17:57:36.936 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T17:57:36.936 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:57:36.936 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T17:57:36.936 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.45158", 2024-08-17T17:57:36.936 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:36.936 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 163281 2024-08-17T17:57:36.936 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.936 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T17:57:36.936 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.936 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T17:57:36.937 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:38.483058+0000", 2024-08-17T17:57:36.937 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.937 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.937 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.937 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T17:57:36.937 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:38.483058+0000", 2024-08-17T17:57:36.937 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T17:57:36.937 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.937 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.937 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T17:57:36.937 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:38.483057+0000", 2024-08-17T17:57:36.937 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T17:57:36.937 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.937 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.938 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T17:57:36.938 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:38.483078+0000", 2024-08-17T17:57:36.938 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0985999999999999e-05 2024-08-17T17:57:36.938 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.938 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.938 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T17:57:36.938 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:38.483089+0000", 2024-08-17T17:57:36.938 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0852e-05 2024-08-17T17:57:36.938 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.938 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.938 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T17:57:36.938 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:38.483094+0000", 2024-08-17T17:57:36.938 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1719999999999998e-06 2024-08-17T17:57:36.939 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.939 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.939 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T17:57:36.939 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:38.484486+0000", 2024-08-17T17:57:36.939 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001392101 2024-08-17T17:57:36.939 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.939 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.939 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T17:57:36.939 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:38.484587+0000", 2024-08-17T17:57:36.939 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0001011 2024-08-17T17:57:36.939 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.939 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.939 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:57:36.940 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:38.484683+0000", 2024-08-17T17:57:36.940 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6167000000000003e-05 2024-08-17T17:57:36.940 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.940 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.940 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T17:57:36.940 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:38.487188+0000", 2024-08-17T17:57:36.940 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025044339999999998 2024-08-17T17:57:36.940 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.940 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.940 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.940 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:38.488737+0000", 2024-08-17T17:57:36.940 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001549501 2024-08-17T17:57:36.940 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.940 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:38.506877+0000", 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.018140041999999999 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:38.506911+0000", 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3980000000000003e-05 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:38.509911+0000", 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0029997999999999999 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.941 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.942 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.942 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T17:57:36.942 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T17:57:36.942 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:57:36.942 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:36.963 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:36.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:36.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:37.008 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1931,"stamp":"2024-08-17T17:57:35.105983+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2286688229,"num_objects":34910,"num_object_clones":0,"num_object_copies":104730,"num_objects_missing_on_primary":1613,"num_objects_missing":1613,"num_objects_degraded":15837,"num_objects_misplaced":5339,"num_objects_unfound":0,"num_objects_dirty":34910,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":175004,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37549,"num_bytes_recovered":2282232845,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108807,"ondisk_log_size":108807,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7676036,"kb_used_data":6198228,"kb_used_omap":295,"kb_used_meta":1477400,"kb_avail":554794876,"statfs":{"total":575970213888,"available":568109953024,"internally_reserved":0,"allocated":6346985472,"data_stored":6316862473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1512858063},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-155844608,"num_objects":-2378,"num_object_clones":0,"num_object_copies":-7134,"num_objects_missing_on_primary":-143,"num_objects_missing":-143,"num_objects_degraded":0,"num_objects_misplaced":-426,"num_objects_unfound":0,"num_objects_dirty":-2378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2378,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9437184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002978"},"pg_stats":[{"pgid":"45.f","version":"1241'10919","reported_seq":21482,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:31.307536+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:57:31.307536+0000","last_peered":"2024-08-17T17:57:31.307536+0000","last_clean":"2024-08-17T17:57:31.307536+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:57:31.307536+0000","last_undegraded":"2024-08-17T17:57:31.307536+0000","last_fullsized":"2024-08-17T17:57:31.307536+0000","mapping_epoch":1194,"log_start":"1175'4031","ondisk_log_start":"1175'4031","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6888,"log_dups_size":0,"ondisk_log_size":6888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147128320,"num_objects":2245,"num_object_clones":0,"num_object_copies":6735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10919,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1241'10873","reported_seq":21354,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:32.708269+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:57:32.708269+0000","last_peered":"2024-08-17T17:57:32.708269+0000","last_clean":"2024-08-17T17:57:32.708269+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:57:32.708269+0000","last_undegraded":"2024-08-17T17:57:32.708269+0000","last_fullsized":"2024-08-17T17:57:32.708269+0000","mapping_epoch":1211,"log_start":"1175'3832","ondisk_log_start":"1175'3832","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7041,"log_dups_size":0,"ondisk_log_size":7041,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139001856,"num_objects":2121,"num_object_clones":0,"num_object_copies":6363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10873,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1241'11217","reported_seq":22087,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:33.665209+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:57:33.665209+0000","last_peered":"2024-08-17T17:57:33.665209+0000","last_clean":"2024-08-17T17:57:33.665209+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:57:33.665209+0000","last_undegraded":"2024-08-17T17:57:33.665209+0000","last_fullsized":"2024-08-17T17:57:33.665209+0000","mapping_epoch":1205,"log_start":"1176'5268","ondisk_log_start":"1176'5268","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5949,"log_dups_size":0,"ondisk_log_size":5949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141557784,"num_objects":2161,"num_object_clones":0,"num_object_copies":6483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2161,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11217,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1241'10814","reported_seq":10896,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:32.668940+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:57:32.668940+0000","last_peered":"2024-08-17T17:57:32.668940+0000","last_clean":"2024-08-17T17:57:32.668940+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:57:32.668940+0000","last_undegraded":"2024-08-17T17:57:32.668940+0000","last_fullsized":"2024-08-17T17:57:32.668940+0000","mapping_epoch":1167,"log_start":"1175'4089","ondisk_log_start":"1175'4089","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6725,"log_dups_size":0,"ondisk_log_size":6725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143261696,"num_objects":2186,"num_object_clones":0,"num_object_copies":6558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10814,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1241'10915","reported_seq":16191,"reported_epoch":1241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:31.793936+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:57:31.793936+0000","last_peered":"2024-08-17T17:57:31.793936+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:57:31.793936+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1176'5026","ondisk_log_start":"1176'5026","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5889,"log_dups_size":0,"ondisk_log_size":5889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144113664,"num_objects":2199,"num_object_clones":0,"num_object_copies":6597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":2199,"num_objects_unfound":0,"num_objects_dirty":2199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10915,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":1681}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1241'10948","reported_seq":11030,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:32.908196+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:57:32.908196+0000","last_peered":"2024-08-17T17:57:32.908196+0000","last_clean":"2024-08-17T17:57:32.908196+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:57:32.908196+0000","last_undegraded":"2024-08-17T17:57:32.908196+0000","last_fullsized":"2024-08-17T17:57:32.908196+0000","mapping_epoch":1167,"log_start":"1175'4277","ondisk_log_start":"1175'4277","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6671,"log_dups_size":0,"ondisk_log_size":6671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140640256,"num_objects":2146,"num_object_clones":0,"num_object_copies":6438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10948,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1241'11021","reported_seq":11103,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:32.951973+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:57:32.951973+0000","last_peered":"2024-08-17T17:57:32.951973+0000","last_clean":"2024-08-17T17:57:32.951973+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:57:32.951973+0000","last_undegraded":"2024-08-17T17:57:32.951973+0000","last_fullsized":"2024-08-17T17:57:32.951973+0000","mapping_epoch":1167,"log_start":"1175'4269","ondisk_log_start":"1175'4269","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6752,"log_dups_size":0,"ondisk_log_size":6752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144506880,"num_objects":2205,"num_object_clones":0,"num_object_copies":6615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11021,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1241'10842","reported_seq":20335,"reported_epoch":1241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:31.789207+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:57:31.789207+0000","last_peered":"2024-08-17T17:57:31.789207+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:57:31.789207+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1176'4875","ondisk_log_start":"1176'4875","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5967,"log_dups_size":0,"ondisk_log_size":5967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140771328,"num_objects":2148,"num_object_clones":0,"num_object_copies":6444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10842,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3108,"num_bytes_recovered":203685888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":828},{"shards":"1,7","objects":81}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1241'10884","reported_seq":10976,"reported_epoch":1241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:32.725213+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:57:32.725213+0000","last_peered":"2024-08-17T17:57:32.725213+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:57:32.725213+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1175'4177","ondisk_log_start":"1175'4177","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6707,"log_dups_size":0,"ondisk_log_size":6707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143654912,"num_objects":2192,"num_object_clones":0,"num_object_copies":6576,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2192,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10884,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":2192},{"shards":"1,6","objects":-3641}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1241'10981","reported_seq":14090,"reported_epoch":1241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:33.663900+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:57:33.663900+0000","last_peered":"2024-08-17T17:57:33.663900+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:57:33.663900+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1176'5006","ondisk_log_start":"1176'5006","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5975,"log_dups_size":0,"ondisk_log_size":5975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143720448,"num_objects":2193,"num_object_clones":0,"num_object_copies":6579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10981,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":632}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1241'10857","reported_seq":11022,"reported_epoch":1241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:33.842148+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:57:33.842148+0000","last_peered":"2024-08-17T17:57:33.842148+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:57:33.842148+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1170'1700","ondisk_log_start":"1170'1700","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9157,"log_dups_size":0,"ondisk_log_size":9157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145293312,"num_objects":2217,"num_object_clones":0,"num_object_copies":6651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10857,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-2936}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1241'10904","reported_seq":17129,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:34.149643+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:57:34.149643+0000","last_peered":"2024-08-17T17:57:34.149643+0000","last_clean":"2024-08-17T17:57:34.149643+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:57:34.149643+0000","last_undegraded":"2024-08-17T17:57:34.149643+0000","last_fullsized":"2024-08-17T17:57:34.149643+0000","mapping_epoch":1183,"log_start":"1170'1800","ondisk_log_start":"1170'1800","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9104,"log_dups_size":0,"ondisk_log_size":9104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140378112,"num_objects":2142,"num_object_clones":0,"num_object_copies":6426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10904,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1241'10937","reported_seq":11100,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:33.554440+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:57:33.554440+0000","last_peered":"2024-08-17T17:57:33.554440+0000","last_clean":"2024-08-17T17:57:33.554440+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:57:33.554440+0000","last_undegraded":"2024-08-17T17:57:33.554440+0000","last_fullsized":"2024-08-17T17:57:33.554440+0000","mapping_epoch":1167,"log_start":"1176'4991","ondisk_log_start":"1176'4991","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5946,"log_dups_size":0,"ondisk_log_size":5946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145686528,"num_objects":2223,"num_object_clones":0,"num_object_copies":6669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10937,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1241'10817","reported_seq":10899,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:32.962310+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:57:32.962310+0000","last_peered":"2024-08-17T17:57:32.962310+0000","last_clean":"2024-08-17T17:57:32.962310+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:57:32.962310+0000","last_undegraded":"2024-08-17T17:57:32.962310+0000","last_fullsized":"2024-08-17T17:57:32.962310+0000","mapping_epoch":1167,"log_start":"1175'3825","ondisk_log_start":"1175'3825","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138739712,"num_objects":2117,"num_object_clones":0,"num_object_copies":6351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10817,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1241'11109","reported_seq":28321,"reported_epoch":1241,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:57:31.543010+0000","last_change":"2024-08-17T17:55:37.941989+0000","last_active":"2024-08-17T17:57:31.543010+0000","last_peered":"2024-08-17T17:57:31.543010+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:57:31.543010+0000","last_undegraded":"2024-08-17T17:57:31.543010+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1175'4191","ondisk_log_start":"1175'4191","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6918,"log_dups_size":0,"ondisk_log_size":6918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147783680,"num_objects":2255,"num_object_clones":0,"num_object_copies":6765,"num_objects_missing_on_primary":1613,"num_objects_missing":1613,"num_objects_degraded":0,"num_objects_misplaced":3140,"num_objects_unfound":0,"num_objects_dirty":2255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11109,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6534,"num_bytes_recovered":428212224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":12},{"shards":"0,4,6","objects":1356},{"shards":"0,6","objects":1},{"shards":"4,5,6","objects":886}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1241'10688","reported_seq":24037,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:33.711550+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:57:33.711550+0000","last_peered":"2024-08-17T17:57:33.711550+0000","last_clean":"2024-08-17T17:57:33.711550+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:57:33.711550+0000","last_undegraded":"2024-08-17T17:57:33.711550+0000","last_fullsized":"2024-08-17T17:57:33.711550+0000","mapping_epoch":1196,"log_start":"1176'4769","ondisk_log_start":"1176'4769","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5919,"log_dups_size":0,"ondisk_log_size":5919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139853824,"num_objects":2134,"num_object_clones":0,"num_object_copies":6402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10688,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1940,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.554709+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:57:27.554709+0000","last_peered":"2024-08-17T17:57:27.554709+0000","last_clean":"2024-08-17T17:57:27.554709+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:57:27.554709+0000","last_undegraded":"2024-08-17T17:57:27.554709+0000","last_fullsized":"2024-08-17T17:57:27.554709+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":518,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470182+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:57:16.470182+0000","last_peered":"2024-08-17T17:57:16.470182+0000","last_clean":"2024-08-17T17:57:16.470182+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:57:16.470182+0000","last_undegraded":"2024-08-17T17:57:16.470182+0000","last_fullsized":"2024-08-17T17:57:16.470182+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1859,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.491006+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:57:27.491006+0000","last_peered":"2024-08-17T17:57:27.491006+0000","last_clean":"2024-08-17T17:57:27.491006+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:57:27.491006+0000","last_undegraded":"2024-08-17T17:57:27.491006+0000","last_fullsized":"2024-08-17T17:57:27.491006+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1883,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490111+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:57:27.490111+0000","last_peered":"2024-08-17T17:57:27.490111+0000","last_clean":"2024-08-17T17:57:27.490111+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:57:27.490111+0000","last_undegraded":"2024-08-17T17:57:27.490111+0000","last_fullsized":"2024-08-17T17:57:27.490111+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":80,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470841+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:57:16.470841+0000","last_peered":"2024-08-17T17:57:16.470841+0000","last_clean":"2024-08-17T17:57:16.470841+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:57:16.470841+0000","last_undegraded":"2024-08-17T17:57:16.470841+0000","last_fullsized":"2024-08-17T17:57:16.470841+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1277,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490403+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:57:27.490403+0000","last_peered":"2024-08-17T17:57:27.490403+0000","last_clean":"2024-08-17T17:57:27.490403+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:57:27.490403+0000","last_undegraded":"2024-08-17T17:57:27.490403+0000","last_fullsized":"2024-08-17T17:57:27.490403+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":135,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.488317+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:57:27.488317+0000","last_peered":"2024-08-17T17:57:27.488317+0000","last_clean":"2024-08-17T17:57:27.488317+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:57:27.488317+0000","last_undegraded":"2024-08-17T17:57:27.488317+0000","last_fullsized":"2024-08-17T17:57:27.488317+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":655,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490713+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:57:27.490713+0000","last_peered":"2024-08-17T17:57:27.490713+0000","last_clean":"2024-08-17T17:57:27.490713+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:57:27.490713+0000","last_undegraded":"2024-08-17T17:57:27.490713+0000","last_fullsized":"2024-08-17T17:57:27.490713+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1898,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.553194+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:57:27.553194+0000","last_peered":"2024-08-17T17:57:27.553194+0000","last_clean":"2024-08-17T17:57:27.553194+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:57:27.553194+0000","last_undegraded":"2024-08-17T17:57:27.553194+0000","last_fullsized":"2024-08-17T17:57:27.553194+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1900,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.546352+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:57:27.546352+0000","last_peered":"2024-08-17T17:57:27.546352+0000","last_clean":"2024-08-17T17:57:27.546352+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:57:27.546352+0000","last_undegraded":"2024-08-17T17:57:27.546352+0000","last_fullsized":"2024-08-17T17:57:27.546352+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":120,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.487710+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:57:27.487710+0000","last_peered":"2024-08-17T17:57:27.487710+0000","last_clean":"2024-08-17T17:57:27.487710+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:57:27.487710+0000","last_undegraded":"2024-08-17T17:57:27.487710+0000","last_fullsized":"2024-08-17T17:57:27.487710+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1872,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490617+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:57:27.490617+0000","last_peered":"2024-08-17T17:57:27.490617+0000","last_clean":"2024-08-17T17:57:27.490617+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:57:27.490617+0000","last_undegraded":"2024-08-17T17:57:27.490617+0000","last_fullsized":"2024-08-17T17:57:27.490617+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1927,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.487225+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:57:27.487225+0000","last_peered":"2024-08-17T17:57:27.487225+0000","last_clean":"2024-08-17T17:57:27.487225+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:57:27.487225+0000","last_undegraded":"2024-08-17T17:57:27.487225+0000","last_fullsized":"2024-08-17T17:57:27.487225+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1921,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.554165+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:57:27.554165+0000","last_peered":"2024-08-17T17:57:27.554165+0000","last_clean":"2024-08-17T17:57:27.554165+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:57:27.554165+0000","last_undegraded":"2024-08-17T17:57:27.554165+0000","last_fullsized":"2024-08-17T17:57:27.554165+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1894,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490020+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:57:27.490020+0000","last_peered":"2024-08-17T17:57:27.490020+0000","last_clean":"2024-08-17T17:57:27.490020+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:57:27.490020+0000","last_undegraded":"2024-08-17T17:57:27.490020+0000","last_fullsized":"2024-08-17T17:57:27.490020+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1215,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470792+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:57:16.470792+0000","last_peered":"2024-08-17T17:57:16.470792+0000","last_clean":"2024-08-17T17:57:16.470792+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:57:16.470792+0000","last_undegraded":"2024-08-17T17:57:16.470792+0000","last_fullsized":"2024-08-17T17:57:16.470792+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1866,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.489922+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:57:27.489922+0000","last_peered":"2024-08-17T17:57:27.489922+0000","last_clean":"2024-08-17T17:57:27.489922+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:57:27.489922+0000","last_undegraded":"2024-08-17T17:57:27.489922+0000","last_fullsized":"2024-08-17T17:57:27.489922+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1915,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.554779+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:57:27.554779+0000","last_peered":"2024-08-17T17:57:27.554779+0000","last_clean":"2024-08-17T17:57:27.554779+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:57:27.554779+0000","last_undegraded":"2024-08-17T17:57:27.554779+0000","last_fullsized":"2024-08-17T17:57:27.554779+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1822,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490076+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:57:27.490076+0000","last_peered":"2024-08-17T17:57:27.490076+0000","last_clean":"2024-08-17T17:57:27.490076+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:57:27.490076+0000","last_undegraded":"2024-08-17T17:57:27.490076+0000","last_fullsized":"2024-08-17T17:57:27.490076+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":534,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.484637+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:57:16.484637+0000","last_peered":"2024-08-17T17:57:16.484637+0000","last_clean":"2024-08-17T17:57:16.484637+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:57:16.484637+0000","last_undegraded":"2024-08-17T17:57:16.484637+0000","last_fullsized":"2024-08-17T17:57:16.484637+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1211,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.484431+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:57:16.484431+0000","last_peered":"2024-08-17T17:57:16.484431+0000","last_clean":"2024-08-17T17:57:16.484431+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:57:16.484431+0000","last_undegraded":"2024-08-17T17:57:16.484431+0000","last_fullsized":"2024-08-17T17:57:16.484431+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1920,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.487816+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:57:27.487816+0000","last_peered":"2024-08-17T17:57:27.487816+0000","last_clean":"2024-08-17T17:57:27.487816+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:57:27.487816+0000","last_undegraded":"2024-08-17T17:57:27.487816+0000","last_fullsized":"2024-08-17T17:57:27.487816+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":372,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.486175+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:57:16.486175+0000","last_peered":"2024-08-17T17:57:16.486175+0000","last_clean":"2024-08-17T17:57:16.486175+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:57:16.486175+0000","last_undegraded":"2024-08-17T17:57:16.486175+0000","last_fullsized":"2024-08-17T17:57:16.486175+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":518,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470478+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:57:16.470478+0000","last_peered":"2024-08-17T17:57:16.470478+0000","last_clean":"2024-08-17T17:57:16.470478+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:57:16.470478+0000","last_undegraded":"2024-08-17T17:57:16.470478+0000","last_fullsized":"2024-08-17T17:57:16.470478+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1196,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470399+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:57:16.470399+0000","last_peered":"2024-08-17T17:57:16.470399+0000","last_clean":"2024-08-17T17:57:16.470399+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:57:16.470399+0000","last_undegraded":"2024-08-17T17:57:16.470399+0000","last_fullsized":"2024-08-17T17:57:16.470399+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1204,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470858+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:57:16.470858+0000","last_peered":"2024-08-17T17:57:16.470858+0000","last_clean":"2024-08-17T17:57:16.470858+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:57:16.470858+0000","last_undegraded":"2024-08-17T17:57:16.470858+0000","last_fullsized":"2024-08-17T17:57:16.470858+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1968,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.553896+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:57:27.553896+0000","last_peered":"2024-08-17T17:57:27.553896+0000","last_clean":"2024-08-17T17:57:27.553896+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:57:27.553896+0000","last_undegraded":"2024-08-17T17:57:27.553896+0000","last_fullsized":"2024-08-17T17:57:27.553896+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1851,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.489968+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:57:27.489968+0000","last_peered":"2024-08-17T17:57:27.489968+0000","last_clean":"2024-08-17T17:57:27.489968+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:57:27.489968+0000","last_undegraded":"2024-08-17T17:57:27.489968+0000","last_fullsized":"2024-08-17T17:57:27.489968+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1926,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.488183+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:57:27.488183+0000","last_peered":"2024-08-17T17:57:27.488183+0000","last_clean":"2024-08-17T17:57:27.488183+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:57:27.488183+0000","last_undegraded":"2024-08-17T17:57:27.488183+0000","last_fullsized":"2024-08-17T17:57:27.488183+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1853,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490960+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:57:27.490960+0000","last_peered":"2024-08-17T17:57:27.490960+0000","last_clean":"2024-08-17T17:57:27.490960+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:57:27.490960+0000","last_undegraded":"2024-08-17T17:57:27.490960+0000","last_fullsized":"2024-08-17T17:57:27.490960+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1886,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.553280+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:57:27.553280+0000","last_peered":"2024-08-17T17:57:27.553280+0000","last_clean":"2024-08-17T17:57:27.553280+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:57:27.553280+0000","last_undegraded":"2024-08-17T17:57:27.553280+0000","last_fullsized":"2024-08-17T17:57:27.553280+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1231,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.484552+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:57:16.484552+0000","last_peered":"2024-08-17T17:57:16.484552+0000","last_clean":"2024-08-17T17:57:16.484552+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:57:16.484552+0000","last_undegraded":"2024-08-17T17:57:16.484552+0000","last_fullsized":"2024-08-17T17:57:16.484552+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1932,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.488397+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:57:27.488397+0000","last_peered":"2024-08-17T17:57:27.488397+0000","last_clean":"2024-08-17T17:57:27.488397+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:57:27.488397+0000","last_undegraded":"2024-08-17T17:57:27.488397+0000","last_fullsized":"2024-08-17T17:57:27.488397+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":119,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.487639+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:57:27.487639+0000","last_peered":"2024-08-17T17:57:27.487639+0000","last_clean":"2024-08-17T17:57:27.487639+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:57:27.487639+0000","last_undegraded":"2024-08-17T17:57:27.487639+0000","last_fullsized":"2024-08-17T17:57:27.487639+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1951,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.488322+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:57:27.488322+0000","last_peered":"2024-08-17T17:57:27.488322+0000","last_clean":"2024-08-17T17:57:27.488322+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:57:27.488322+0000","last_undegraded":"2024-08-17T17:57:27.488322+0000","last_fullsized":"2024-08-17T17:57:27.488322+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":688,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490479+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:57:27.490479+0000","last_peered":"2024-08-17T17:57:27.490479+0000","last_clean":"2024-08-17T17:57:27.490479+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:57:27.490479+0000","last_undegraded":"2024-08-17T17:57:27.490479+0000","last_fullsized":"2024-08-17T17:57:27.490479+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1932,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.488415+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:57:27.488415+0000","last_peered":"2024-08-17T17:57:27.488415+0000","last_clean":"2024-08-17T17:57:27.488415+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:57:27.488415+0000","last_undegraded":"2024-08-17T17:57:27.488415+0000","last_fullsized":"2024-08-17T17:57:27.488415+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1261,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.553040+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:57:27.553040+0000","last_peered":"2024-08-17T17:57:27.553040+0000","last_clean":"2024-08-17T17:57:27.553040+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:57:27.553040+0000","last_undegraded":"2024-08-17T17:57:27.553040+0000","last_fullsized":"2024-08-17T17:57:27.553040+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":179,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.487790+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:57:27.487790+0000","last_peered":"2024-08-17T17:57:27.487790+0000","last_clean":"2024-08-17T17:57:27.487790+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:57:27.487790+0000","last_undegraded":"2024-08-17T17:57:27.487790+0000","last_fullsized":"2024-08-17T17:57:27.487790+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1943,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.488590+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:57:27.488590+0000","last_peered":"2024-08-17T17:57:27.488590+0000","last_clean":"2024-08-17T17:57:27.488590+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:57:27.488590+0000","last_undegraded":"2024-08-17T17:57:27.488590+0000","last_fullsized":"2024-08-17T17:57:27.488590+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1921,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.487042+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:57:27.487042+0000","last_peered":"2024-08-17T17:57:27.487042+0000","last_clean":"2024-08-17T17:57:27.487042+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:57:27.487042+0000","last_undegraded":"2024-08-17T17:57:27.487042+0000","last_fullsized":"2024-08-17T17:57:27.487042+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1941,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.542637+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:57:27.542637+0000","last_peered":"2024-08-17T17:57:27.542637+0000","last_clean":"2024-08-17T17:57:27.542637+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:57:27.542637+0000","last_undegraded":"2024-08-17T17:57:27.542637+0000","last_fullsized":"2024-08-17T17:57:27.542637+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1947,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.554237+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:57:27.554237+0000","last_peered":"2024-08-17T17:57:27.554237+0000","last_clean":"2024-08-17T17:57:27.554237+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:57:27.554237+0000","last_undegraded":"2024-08-17T17:57:27.554237+0000","last_fullsized":"2024-08-17T17:57:27.554237+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1927,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.581707+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:57:27.581707+0000","last_peered":"2024-08-17T17:57:27.581707+0000","last_clean":"2024-08-17T17:57:27.581707+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:57:27.581707+0000","last_undegraded":"2024-08-17T17:57:27.581707+0000","last_fullsized":"2024-08-17T17:57:27.581707+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1238,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.484319+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:57:16.484319+0000","last_peered":"2024-08-17T17:57:16.484319+0000","last_clean":"2024-08-17T17:57:16.484319+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:57:16.484319+0000","last_undegraded":"2024-08-17T17:57:16.484319+0000","last_fullsized":"2024-08-17T17:57:16.484319+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1184,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470921+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:57:16.470921+0000","last_peered":"2024-08-17T17:57:16.470921+0000","last_clean":"2024-08-17T17:57:16.470921+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:57:16.470921+0000","last_undegraded":"2024-08-17T17:57:16.470921+0000","last_fullsized":"2024-08-17T17:57:16.470921+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1836,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490365+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:57:27.490365+0000","last_peered":"2024-08-17T17:57:27.490365+0000","last_clean":"2024-08-17T17:57:27.490365+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:57:27.490365+0000","last_undegraded":"2024-08-17T17:57:27.490365+0000","last_fullsized":"2024-08-17T17:57:27.490365+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":167,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.542748+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:57:27.542748+0000","last_peered":"2024-08-17T17:57:27.542748+0000","last_clean":"2024-08-17T17:57:27.542748+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:57:27.542748+0000","last_undegraded":"2024-08-17T17:57:27.542748+0000","last_fullsized":"2024-08-17T17:57:27.542748+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1550,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.484204+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:57:16.484204+0000","last_peered":"2024-08-17T17:57:16.484204+0000","last_clean":"2024-08-17T17:57:16.484204+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:57:16.484204+0000","last_undegraded":"2024-08-17T17:57:16.484204+0000","last_fullsized":"2024-08-17T17:57:16.484204+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1927,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.488666+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:57:27.488666+0000","last_peered":"2024-08-17T17:57:27.488666+0000","last_clean":"2024-08-17T17:57:27.488666+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:57:27.488666+0000","last_undegraded":"2024-08-17T17:57:27.488666+0000","last_fullsized":"2024-08-17T17:57:27.488666+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1867,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.489933+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:57:27.489933+0000","last_peered":"2024-08-17T17:57:27.489933+0000","last_clean":"2024-08-17T17:57:27.489933+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:57:27.489933+0000","last_undegraded":"2024-08-17T17:57:27.489933+0000","last_fullsized":"2024-08-17T17:57:27.489933+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":518,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470776+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:57:16.470776+0000","last_peered":"2024-08-17T17:57:16.470776+0000","last_clean":"2024-08-17T17:57:16.470776+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:57:16.470776+0000","last_undegraded":"2024-08-17T17:57:16.470776+0000","last_fullsized":"2024-08-17T17:57:16.470776+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":606,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.542582+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:57:27.542582+0000","last_peered":"2024-08-17T17:57:27.542582+0000","last_clean":"2024-08-17T17:57:27.542582+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:57:27.542582+0000","last_undegraded":"2024-08-17T17:57:27.542582+0000","last_fullsized":"2024-08-17T17:57:27.542582+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":622,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.543119+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:57:27.543119+0000","last_peered":"2024-08-17T17:57:27.543119+0000","last_clean":"2024-08-17T17:57:27.543119+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:57:27.543119+0000","last_undegraded":"2024-08-17T17:57:27.543119+0000","last_fullsized":"2024-08-17T17:57:27.543119+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1202,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470616+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:57:16.470616+0000","last_peered":"2024-08-17T17:57:16.470616+0000","last_clean":"2024-08-17T17:57:16.470616+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:57:16.470616+0000","last_undegraded":"2024-08-17T17:57:16.470616+0000","last_fullsized":"2024-08-17T17:57:16.470616+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":638,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490379+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:57:27.490379+0000","last_peered":"2024-08-17T17:57:27.490379+0000","last_clean":"2024-08-17T17:57:27.490379+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:57:27.490379+0000","last_undegraded":"2024-08-17T17:57:27.490379+0000","last_fullsized":"2024-08-17T17:57:27.490379+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1864,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490807+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:57:27.490807+0000","last_peered":"2024-08-17T17:57:27.490807+0000","last_clean":"2024-08-17T17:57:27.490807+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:57:27.490807+0000","last_undegraded":"2024-08-17T17:57:27.490807+0000","last_fullsized":"2024-08-17T17:57:27.490807+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1960,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.553355+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:57:27.553355+0000","last_peered":"2024-08-17T17:57:27.553355+0000","last_clean":"2024-08-17T17:57:27.553355+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:57:27.553355+0000","last_undegraded":"2024-08-17T17:57:27.553355+0000","last_fullsized":"2024-08-17T17:57:27.553355+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":671,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490294+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:57:27.490294+0000","last_peered":"2024-08-17T17:57:27.490294+0000","last_clean":"2024-08-17T17:57:27.490294+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:57:27.490294+0000","last_undegraded":"2024-08-17T17:57:27.490294+0000","last_fullsized":"2024-08-17T17:57:27.490294+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1191,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470473+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:57:16.470473+0000","last_peered":"2024-08-17T17:57:16.470473+0000","last_clean":"2024-08-17T17:57:16.470473+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:57:16.470473+0000","last_undegraded":"2024-08-17T17:57:16.470473+0000","last_fullsized":"2024-08-17T17:57:16.470473+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1190,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.484538+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:57:16.484538+0000","last_peered":"2024-08-17T17:57:16.484538+0000","last_clean":"2024-08-17T17:57:16.484538+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:57:16.484538+0000","last_undegraded":"2024-08-17T17:57:16.484538+0000","last_fullsized":"2024-08-17T17:57:16.484538+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":655,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.487304+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:57:27.487304+0000","last_peered":"2024-08-17T17:57:27.487304+0000","last_clean":"2024-08-17T17:57:27.487304+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:57:27.487304+0000","last_undegraded":"2024-08-17T17:57:27.487304+0000","last_fullsized":"2024-08-17T17:57:27.487304+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1208,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.484837+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:57:16.484837+0000","last_peered":"2024-08-17T17:57:16.484837+0000","last_clean":"2024-08-17T17:57:16.484837+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:57:16.484837+0000","last_undegraded":"2024-08-17T17:57:16.484837+0000","last_fullsized":"2024-08-17T17:57:16.484837+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":92,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.484149+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:57:16.484149+0000","last_peered":"2024-08-17T17:57:16.484149+0000","last_clean":"2024-08-17T17:57:16.484149+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:57:16.484149+0000","last_undegraded":"2024-08-17T17:57:16.484149+0000","last_fullsized":"2024-08-17T17:57:16.484149+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1224,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.484868+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:57:16.484868+0000","last_peered":"2024-08-17T17:57:16.484868+0000","last_clean":"2024-08-17T17:57:16.484868+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:57:16.484868+0000","last_undegraded":"2024-08-17T17:57:16.484868+0000","last_fullsized":"2024-08-17T17:57:16.484868+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1241,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.470348+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:57:16.470348+0000","last_peered":"2024-08-17T17:57:16.470348+0000","last_clean":"2024-08-17T17:57:16.470348+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:57:16.470348+0000","last_undegraded":"2024-08-17T17:57:16.470348+0000","last_fullsized":"2024-08-17T17:57:16.470348+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1865,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490878+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:57:27.490878+0000","last_peered":"2024-08-17T17:57:27.490878+0000","last_clean":"2024-08-17T17:57:27.490878+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:57:27.490878+0000","last_undegraded":"2024-08-17T17:57:27.490878+0000","last_fullsized":"2024-08-17T17:57:27.490878+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1185,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.484504+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:57:16.484504+0000","last_peered":"2024-08-17T17:57:16.484504+0000","last_clean":"2024-08-17T17:57:16.484504+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:57:16.484504+0000","last_undegraded":"2024-08-17T17:57:16.484504+0000","last_fullsized":"2024-08-17T17:57:16.484504+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1918,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.553992+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:57:27.553992+0000","last_peered":"2024-08-17T17:57:27.553992+0000","last_clean":"2024-08-17T17:57:27.553992+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:57:27.553992+0000","last_undegraded":"2024-08-17T17:57:27.553992+0000","last_fullsized":"2024-08-17T17:57:27.553992+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1919,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.553360+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:57:27.553360+0000","last_peered":"2024-08-17T17:57:27.553360+0000","last_clean":"2024-08-17T17:57:27.553360+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:57:27.553360+0000","last_undegraded":"2024-08-17T17:57:27.553360+0000","last_fullsized":"2024-08-17T17:57:27.553360+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":598,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.484454+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:57:16.484454+0000","last_peered":"2024-08-17T17:57:16.484454+0000","last_clean":"2024-08-17T17:57:16.484454+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:57:16.484454+0000","last_undegraded":"2024-08-17T17:57:16.484454+0000","last_fullsized":"2024-08-17T17:57:16.484454+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1210,"reported_epoch":1240,"state":"active+clean","last_fresh":"2024-08-17T17:57:16.484747+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:57:16.484747+0000","last_peered":"2024-08-17T17:57:16.484747+0000","last_clean":"2024-08-17T17:57:16.484747+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:57:16.484747+0000","last_undegraded":"2024-08-17T17:57:16.484747+0000","last_fullsized":"2024-08-17T17:57:16.484747+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1866,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490544+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:57:27.490544+0000","last_peered":"2024-08-17T17:57:27.490544+0000","last_clean":"2024-08-17T17:57:27.490544+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:57:27.490544+0000","last_undegraded":"2024-08-17T17:57:27.490544+0000","last_fullsized":"2024-08-17T17:57:27.490544+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":2286092312,"num_objects":34884,"num_object_clones":0,"num_object_copies":104652,"num_objects_missing_on_primary":1613,"num_objects_missing":1613,"num_objects_degraded":15837,"num_objects_misplaced":5339,"num_objects_unfound":0,"num_objects_dirty":34884,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":174726,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37520,"num_bytes_recovered":2280456192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8255451136,"data_stored":8255438920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108600,"ondisk_log_size":108600,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234440,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1289920,"kb_used_data":984248,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":92455232,"statfs":{"total":95995035648,"available":94674157568,"internally_reserved":0,"allocated":1007869952,"data_stored":1002841643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496075,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1428804,"kb_used_data":1131452,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":92316348,"statfs":{"total":95995035648,"available":94531940352,"internally_reserved":0,"allocated":1158606848,"data_stored":1153574944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757713,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326327,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1502188,"kb_used_data":1267044,"kb_used_omap":57,"kb_used_meta":235014,"kb_avail":92242964,"statfs":{"total":95995035648,"available":94456795136,"internally_reserved":0,"allocated":1297453056,"data_stored":1292439927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":240655099},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405566,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1363764,"kb_used_data":1123372,"kb_used_omap":61,"kb_used_meta":240322,"kb_avail":92381388,"statfs":{"total":95995035648,"available":94598541312,"internally_reserved":0,"allocated":1150332928,"data_stored":1145316771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":246090680},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452100,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1033384,"kb_used_data":857184,"kb_used_omap":54,"kb_used_meta":176137,"kb_avail":92711768,"statfs":{"total":95995035648,"available":94936850432,"internally_reserved":0,"allocated":877756416,"data_stored":872747248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":180364686},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474791,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1057976,"kb_used_data":834928,"kb_used_omap":119,"kb_used_meta":222920,"kb_avail":92687176,"statfs":{"total":95995035648,"available":94911668224,"internally_reserved":0,"allocated":854966272,"data_stored":849941940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":228270814},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019348,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":840372224,"data_stored":840368152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":859963392,"data_stored":859963392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1135607808,"data_stored":1135607808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1279791104,"data_stored":1279787032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":915079168,"data_stored":915079168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1144000512,"data_stored":1143996440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":992739328,"data_stored":992739328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:57:37.065 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:37.086 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:37.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:37.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:37.187 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:37.209 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:37.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:37.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:37.310 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:37.331 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:37.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:37.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:37.384 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:37 smithi130 ceph-mon[105521]: from='client.36066 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:57:37.384 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:37 smithi130 ceph-mon[110068]: from='client.36066 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:57:37.432 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:37.454 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:37.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:37.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:37.457 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:37 smithi179 ceph-mon[93519]: from='client.36066 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:57:37.555 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:37.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:37.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:37.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:37.678 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:37.700 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:37.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:37.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:37.802 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:37.823 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:37.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:37.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:37.843 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:37.925 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:37.946 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:37.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:37.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:38.048 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:38.069 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:38.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:38.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:38.135 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:57:38.171 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:38.198 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:38.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:38.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:38.300 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:38.322 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:38.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:38.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:38.364 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:38 smithi130 ceph-mon[105521]: pgmap v1932: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 240 op/s; 15837/103725 objects degraded (15.268%); 5104/103725 objects misplaced (4.921%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:57:38.364 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:38 smithi130 ceph-mon[110068]: pgmap v1932: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 240 op/s; 15837/103725 objects degraded (15.268%); 5104/103725 objects misplaced (4.921%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:57:38.419 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:57:38.424 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:38.449 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:38.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:38.544 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:38 smithi179 ceph-mon[93519]: pgmap v1932: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 240 op/s; 15837/103725 objects degraded (15.268%); 5104/103725 objects misplaced (4.921%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:57:38.550 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:38.575 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:38.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:38.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:38.677 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:38.698 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:38.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:38.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:38.800 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:38.822 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:38.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:38.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:38.924 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:38.945 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:38.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:38.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:39.047 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:39.068 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:39.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:39.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:39.169 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:39.173 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T17:57:39.192 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:39.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:39.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:39.293 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:39.314 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:39.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:39.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:39.416 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:39.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:39 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2680992661' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:57:39.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:39 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:57:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:39 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2680992661' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:57:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:39 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:57:39.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:39.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:39.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:39.540 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:39.561 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:39.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:39.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:39.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:39 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2680992661' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:57:39.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:39 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:57:39.663 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:39.686 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:39.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:39.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:39.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:39.809 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:39.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:39.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:39.911 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:39.933 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:39.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:39.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:40.013 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:57:40.035 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:40.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:40.058 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:40.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:40.159 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:40.163 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:57:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:57:39] "GET /metrics HTTP/1.1" 200 36803 "" "Prometheus/2.51.0" 2024-08-17T17:57:40.190 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:40.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:40.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:40.292 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:40.317 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:40.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:40.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:40.395 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T17:57:40.395 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T17:57:40.395 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T17:57:40.395 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T17:57:40.396 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:57:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:57:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:57:40.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:40 smithi179 ceph-mon[93519]: pgmap v1933: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 266 op/s; 15837/101664 objects degraded (15.578%); 5104/101664 objects misplaced (5.020%); 810 KiB/s, 12 objects/s recovering 2024-08-17T17:57:40.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:40 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:57:40.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:40 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:57:40.397 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:40 smithi179 ceph-mon[93519]: osdmap e1242: 8 total, 8 up, 6 in 2024-08-17T17:57:40.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:40 smithi130 ceph-mon[105521]: pgmap v1933: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 266 op/s; 15837/101664 objects degraded (15.578%); 5104/101664 objects misplaced (5.020%); 810 KiB/s, 12 objects/s recovering 2024-08-17T17:57:40.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:40 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:57:40.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:40 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:57:40.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:40 smithi130 ceph-mon[105521]: osdmap e1242: 8 total, 8 up, 6 in 2024-08-17T17:57:40.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:40 smithi130 ceph-mon[110068]: pgmap v1933: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 266 op/s; 15837/101664 objects degraded (15.578%); 5104/101664 objects misplaced (5.020%); 810 KiB/s, 12 objects/s recovering 2024-08-17T17:57:40.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:40 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:57:40.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:40 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:57:40.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:40 smithi130 ceph-mon[110068]: osdmap e1242: 8 total, 8 up, 6 in 2024-08-17T17:57:40.419 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:40.442 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:40.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:40.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:40.544 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:40.566 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:40.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:40.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:40.667 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:40.692 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:40.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:40.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:40.793 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:40.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:40.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:40.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:40.923 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:40.944 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:40.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:40.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:41.046 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:41.054 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:57:41.054 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:57:41.062 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:41.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:41.068 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:41.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:41.074 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:57:41.074 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:57:41.095 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:57:41.095 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:57:41.125 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:57:41.125 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:57:41.141 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:57:41.142 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:57:41.158 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:57:41.158 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:57:41.169 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:41.180 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:57:41.180 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:57:41.198 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:57:41.198 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:57:41.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:41.199 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:41.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:41.261 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:57:41.300 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:41.324 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:41.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:41.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:41.348 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:41 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/101664 objects degraded (15.578%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:41.348 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:41 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3960780139' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:41.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:41 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/101664 objects degraded (15.578%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:41.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:41 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3960780139' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:41.426 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:41.455 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:41.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:41.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:41.538 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:57:41.556 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:41.577 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:41.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:41.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:41.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:41 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/101664 objects degraded (15.578%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:41.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:41 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3960780139' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:41.679 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:41.704 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:41.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:41.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:41.806 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:41.827 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:41.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:41.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:41.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:41.950 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:41.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:41.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:42.052 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:42.080 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:42.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:42.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:42.182 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:42.210 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:42.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:42.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:42.312 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:42.338 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:42.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:42.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:42.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:42 smithi179 ceph-mon[93519]: pgmap v1935: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 227 op/s; 15837/101238 objects degraded (15.643%); 5104/101238 objects misplaced (5.042%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:57:42.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:42 smithi130 ceph-mon[105521]: pgmap v1935: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 227 op/s; 15837/101238 objects degraded (15.643%); 5104/101238 objects misplaced (5.042%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:57:42.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:42 smithi130 ceph-mon[110068]: pgmap v1935: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 227 op/s; 15837/101238 objects degraded (15.643%); 5104/101238 objects misplaced (5.042%); 941 KiB/s, 14 objects/s recovering 2024-08-17T17:57:42.440 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:42.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:42.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:42.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:42.518 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:57:42.564 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:42.584 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:42.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:42.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:42.686 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:42.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:42.708 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:42.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:42.809 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:42.830 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:42.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:42.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:42.931 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:42.952 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:42.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:42.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:43.054 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:43.076 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:43.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:43.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:43.178 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:43.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:43.200 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:43.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:43.301 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:43.332 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:43.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:43.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:43.435 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:43.444 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:43.444 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:43.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:43.457 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:43.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:43.559 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:43.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:43.612 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:43.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:43.699 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:57:43.699 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:57:43.713 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:43.736 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:43.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:43.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:43.837 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:43.858 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:43.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:43.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:43.960 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:43.981 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:43.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:43.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:44.083 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:44.113 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:44.113 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:44.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:44.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:44.189 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:57:44.215 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:44.236 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:44.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:44.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:44.338 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:44.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:44 smithi179 ceph-mon[93519]: pgmap v1936: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 230 op/s; 15837/99129 objects degraded (15.976%); 4884/99129 objects misplaced (4.927%); 960 KiB/s, 14 objects/s recovering 2024-08-17T17:57:44.360 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:44.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:44.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:44.372 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:57:44.373 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:57:44.462 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:44.483 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:44.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:44.528 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:57:44.586 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:44.609 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:44.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:44.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:44 smithi130 ceph-mon[105521]: pgmap v1936: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 230 op/s; 15837/99129 objects degraded (15.976%); 4884/99129 objects misplaced (4.927%); 960 KiB/s, 14 objects/s recovering 2024-08-17T17:57:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:44 smithi130 ceph-mon[110068]: pgmap v1936: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 230 op/s; 15837/99129 objects degraded (15.976%); 4884/99129 objects misplaced (4.927%); 960 KiB/s, 14 objects/s recovering 2024-08-17T17:57:44.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:44.740 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:44.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:44.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:44.814 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:57:44.842 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:44.864 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:44.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:44.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:44.966 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:44.989 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:44.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:44.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:45.091 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:45.115 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:45.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:45.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:45.218 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:45.238 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:45.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:45.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:45.340 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:45.361 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:45.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:45.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:45.462 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:45.484 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:45.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:45.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:45.586 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:45.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:45 smithi179 ceph-mon[93519]: from='client.55603 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:57:45.607 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:45.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:45.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:45.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:45 smithi130 ceph-mon[105521]: from='client.55603 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:57:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:45 smithi130 ceph-mon[110068]: from='client.55603 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:57:45.709 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:45.731 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:45.740 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:45.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:45.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:45.842 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:45.864 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:45.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:45.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:45.966 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:45.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:45.988 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:45.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:46.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:46.115 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:46.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:46.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:46.198 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1936,"stamp":"2024-08-17T17:57:43.108145+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2164332517,"num_objects":33043,"num_object_clones":0,"num_object_copies":99129,"num_objects_missing_on_primary":1462,"num_objects_missing":1462,"num_objects_degraded":15837,"num_objects_misplaced":4884,"num_objects_unfound":0,"num_objects_dirty":33043,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":176871,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37699,"num_bytes_recovered":2292063245,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108674,"ondisk_log_size":108674,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7381828,"kb_used_data":5895764,"kb_used_omap":295,"kb_used_meta":1485720,"kb_avail":555089084,"statfs":{"total":575970213888,"available":568411222016,"internally_reserved":0,"allocated":6037262336,"data_stored":6007109593,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1521377743},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1,1,0,3,2,4,7],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-150929408,"num_objects":-2303,"num_object_clones":0,"num_object_copies":-6909,"num_objects_missing_on_primary":-151,"num_objects_missing":-151,"num_objects_degraded":0,"num_objects_misplaced":-455,"num_objects_unfound":0,"num_objects_dirty":-2303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2303,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":150,"num_bytes_recovered":9830400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003062"},"pg_stats":[{"pgid":"45.f","version":"1242'11085","reported_seq":21650,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:41.367168+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:57:41.367168+0000","last_peered":"2024-08-17T17:57:41.367168+0000","last_clean":"2024-08-17T17:57:41.367168+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:57:41.367168+0000","last_undegraded":"2024-08-17T17:57:41.367168+0000","last_fullsized":"2024-08-17T17:57:41.367168+0000","mapping_epoch":1194,"log_start":"1175'4231","ondisk_log_start":"1175'4231","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6854,"log_dups_size":0,"ondisk_log_size":6854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136249344,"num_objects":2079,"num_object_clones":0,"num_object_copies":6237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11085,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1242'10990","reported_seq":21472,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:42.872859+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:57:42.872859+0000","last_peered":"2024-08-17T17:57:42.872859+0000","last_clean":"2024-08-17T17:57:42.872859+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:57:42.872859+0000","last_undegraded":"2024-08-17T17:57:42.872859+0000","last_fullsized":"2024-08-17T17:57:42.872859+0000","mapping_epoch":1211,"log_start":"1175'3932","ondisk_log_start":"1175'3932","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7058,"log_dups_size":0,"ondisk_log_size":7058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131334144,"num_objects":2004,"num_object_clones":0,"num_object_copies":6012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10990,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1241'11284","reported_seq":22154,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:38.471012+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:57:38.471012+0000","last_peered":"2024-08-17T17:57:38.471012+0000","last_clean":"2024-08-17T17:57:38.471012+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:57:38.471012+0000","last_undegraded":"2024-08-17T17:57:38.471012+0000","last_fullsized":"2024-08-17T17:57:38.471012+0000","mapping_epoch":1205,"log_start":"1180'5368","ondisk_log_start":"1180'5368","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5916,"log_dups_size":0,"ondisk_log_size":5916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137166872,"num_objects":2094,"num_object_clones":0,"num_object_copies":6282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2094,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11284,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1242'10964","reported_seq":11047,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:42.944808+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:57:42.944808+0000","last_peered":"2024-08-17T17:57:42.944808+0000","last_clean":"2024-08-17T17:57:42.944808+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:57:42.944808+0000","last_undegraded":"2024-08-17T17:57:42.944808+0000","last_fullsized":"2024-08-17T17:57:42.944808+0000","mapping_epoch":1167,"log_start":"1175'4289","ondisk_log_start":"1175'4289","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6675,"log_dups_size":0,"ondisk_log_size":6675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133431296,"num_objects":2036,"num_object_clones":0,"num_object_copies":6108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10964,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1242'11069","reported_seq":16347,"reported_epoch":1242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:41.613443+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:57:41.613443+0000","last_peered":"2024-08-17T17:57:41.613443+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:57:41.613443+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1176'5126","ondisk_log_start":"1176'5126","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134021120,"num_objects":2045,"num_object_clones":0,"num_object_copies":6135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":2045,"num_objects_unfound":0,"num_objects_dirty":2045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11069,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":1527}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1242'11105","reported_seq":11188,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:42.909704+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:57:42.909704+0000","last_peered":"2024-08-17T17:57:42.909704+0000","last_clean":"2024-08-17T17:57:42.909704+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:57:42.909704+0000","last_undegraded":"2024-08-17T17:57:42.909704+0000","last_fullsized":"2024-08-17T17:57:42.909704+0000","mapping_epoch":1167,"log_start":"1175'4377","ondisk_log_start":"1175'4377","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6728,"log_dups_size":0,"ondisk_log_size":6728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130351104,"num_objects":1989,"num_object_clones":0,"num_object_copies":5967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11105,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1242'11159","reported_seq":11242,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:42.864435+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:57:42.864435+0000","last_peered":"2024-08-17T17:57:42.864435+0000","last_clean":"2024-08-17T17:57:42.864435+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:57:42.864435+0000","last_undegraded":"2024-08-17T17:57:42.864435+0000","last_fullsized":"2024-08-17T17:57:42.864435+0000","mapping_epoch":1167,"log_start":"1175'4469","ondisk_log_start":"1175'4469","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6690,"log_dups_size":0,"ondisk_log_size":6690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135462912,"num_objects":2067,"num_object_clones":0,"num_object_copies":6201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11159,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1242'10988","reported_seq":20483,"reported_epoch":1242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:41.698588+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:57:41.698588+0000","last_peered":"2024-08-17T17:57:41.698588+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:57:41.698588+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1176'5075","ondisk_log_start":"1176'5075","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5913,"log_dups_size":0,"ondisk_log_size":5913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131203072,"num_objects":2002,"num_object_clones":0,"num_object_copies":6006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10988,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3108,"num_bytes_recovered":203685888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":682},{"shards":"1,7","objects":-65}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1242'11013","reported_seq":11106,"reported_epoch":1242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:42.929283+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:57:42.929283+0000","last_peered":"2024-08-17T17:57:42.929283+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:57:42.929283+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1175'4277","ondisk_log_start":"1175'4277","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6736,"log_dups_size":0,"ondisk_log_size":6736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135200768,"num_objects":2063,"num_object_clones":0,"num_object_copies":6189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11013,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":2063},{"shards":"1,6","objects":-3770}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1241'11050","reported_seq":14159,"reported_epoch":1241,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:38.445450+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:57:38.445450+0000","last_peered":"2024-08-17T17:57:38.445450+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:57:38.445450+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1176'5106","ondisk_log_start":"1176'5106","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5944,"log_dups_size":0,"ondisk_log_size":5944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139198464,"num_objects":2124,"num_object_clones":0,"num_object_copies":6372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11050,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":563}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1241'10925","reported_seq":11092,"reported_epoch":1242,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:39.189325+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:57:39.189325+0000","last_peered":"2024-08-17T17:57:39.189325+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:57:39.189325+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1170'1800","ondisk_log_start":"1170'1800","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9125,"log_dups_size":0,"ondisk_log_size":9125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140836864,"num_objects":2149,"num_object_clones":0,"num_object_copies":6447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2149,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10925,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-3004}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1241'10978","reported_seq":17205,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.187303+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:57:39.187303+0000","last_peered":"2024-08-17T17:57:39.187303+0000","last_clean":"2024-08-17T17:57:39.187303+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:57:39.187303+0000","last_undegraded":"2024-08-17T17:57:39.187303+0000","last_fullsized":"2024-08-17T17:57:39.187303+0000","mapping_epoch":1183,"log_start":"1170'1800","ondisk_log_start":"1170'1800","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9178,"log_dups_size":0,"ondisk_log_size":9178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135528448,"num_objects":2068,"num_object_clones":0,"num_object_copies":6204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10978,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1241'11009","reported_seq":11172,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:38.507621+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:57:38.507621+0000","last_peered":"2024-08-17T17:57:38.507621+0000","last_clean":"2024-08-17T17:57:38.507621+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:57:38.507621+0000","last_undegraded":"2024-08-17T17:57:38.507621+0000","last_fullsized":"2024-08-17T17:57:38.507621+0000","mapping_epoch":1167,"log_start":"1176'5091","ondisk_log_start":"1176'5091","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140967936,"num_objects":2151,"num_object_clones":0,"num_object_copies":6453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11009,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1242'10957","reported_seq":11040,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:42.933575+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:57:42.933575+0000","last_peered":"2024-08-17T17:57:42.933575+0000","last_clean":"2024-08-17T17:57:42.933575+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:57:42.933575+0000","last_undegraded":"2024-08-17T17:57:42.933575+0000","last_fullsized":"2024-08-17T17:57:42.933575+0000","mapping_epoch":1167,"log_start":"1175'3925","ondisk_log_start":"1175'3925","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7032,"log_dups_size":0,"ondisk_log_size":7032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129564672,"num_objects":1977,"num_object_clones":0,"num_object_copies":5931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10957,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1242'11260","reported_seq":28924,"reported_epoch":1242,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:57:41.648053+0000","last_change":"2024-08-17T17:55:37.941989+0000","last_active":"2024-08-17T17:57:41.648053+0000","last_peered":"2024-08-17T17:57:41.648053+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:57:41.648053+0000","last_undegraded":"2024-08-17T17:57:41.648053+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1175'4391","ondisk_log_start":"1175'4391","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6869,"log_dups_size":0,"ondisk_log_size":6869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137887744,"num_objects":2104,"num_object_clones":0,"num_object_copies":6312,"num_objects_missing_on_primary":1462,"num_objects_missing":1462,"num_objects_degraded":0,"num_objects_misplaced":2839,"num_objects_unfound":0,"num_objects_dirty":2104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11260,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6684,"num_bytes_recovered":438042624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":13},{"shards":"0,4,6","objects":1356},{"shards":"4,5,6","objects":735}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1241'10757","reported_seq":24107,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:38.494102+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:57:38.494102+0000","last_peered":"2024-08-17T17:57:38.494102+0000","last_clean":"2024-08-17T17:57:38.494102+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:57:38.494102+0000","last_undegraded":"2024-08-17T17:57:38.494102+0000","last_fullsized":"2024-08-17T17:57:38.494102+0000","mapping_epoch":1196,"log_start":"1176'4869","ondisk_log_start":"1176'4869","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5888,"log_dups_size":0,"ondisk_log_size":5888,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135331840,"num_objects":2065,"num_object_clones":0,"num_object_copies":6195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10757,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1942,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.181854+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:57:39.181854+0000","last_peered":"2024-08-17T17:57:39.181854+0000","last_clean":"2024-08-17T17:57:39.181854+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:57:39.181854+0000","last_undegraded":"2024-08-17T17:57:39.181854+0000","last_fullsized":"2024-08-17T17:57:39.181854+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":519,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.548705+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:57:27.548705+0000","last_peered":"2024-08-17T17:57:27.548705+0000","last_clean":"2024-08-17T17:57:27.548705+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:57:27.548705+0000","last_undegraded":"2024-08-17T17:57:27.548705+0000","last_fullsized":"2024-08-17T17:57:27.548705+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1859,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.491006+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:57:27.491006+0000","last_peered":"2024-08-17T17:57:27.491006+0000","last_clean":"2024-08-17T17:57:27.491006+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:57:27.491006+0000","last_undegraded":"2024-08-17T17:57:27.491006+0000","last_fullsized":"2024-08-17T17:57:27.491006+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1883,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490111+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:57:27.490111+0000","last_peered":"2024-08-17T17:57:27.490111+0000","last_clean":"2024-08-17T17:57:27.490111+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:57:27.490111+0000","last_undegraded":"2024-08-17T17:57:27.490111+0000","last_fullsized":"2024-08-17T17:57:27.490111+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":81,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.549175+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:57:27.549175+0000","last_peered":"2024-08-17T17:57:27.549175+0000","last_clean":"2024-08-17T17:57:27.549175+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:57:27.549175+0000","last_undegraded":"2024-08-17T17:57:27.549175+0000","last_fullsized":"2024-08-17T17:57:27.549175+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1277,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490403+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:57:27.490403+0000","last_peered":"2024-08-17T17:57:27.490403+0000","last_clean":"2024-08-17T17:57:27.490403+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:57:27.490403+0000","last_undegraded":"2024-08-17T17:57:27.490403+0000","last_fullsized":"2024-08-17T17:57:27.490403+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":137,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.267749+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:57:39.267749+0000","last_peered":"2024-08-17T17:57:39.267749+0000","last_clean":"2024-08-17T17:57:39.267749+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:57:39.267749+0000","last_undegraded":"2024-08-17T17:57:39.267749+0000","last_fullsized":"2024-08-17T17:57:39.267749+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":655,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490713+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:57:27.490713+0000","last_peered":"2024-08-17T17:57:27.490713+0000","last_clean":"2024-08-17T17:57:27.490713+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:57:27.490713+0000","last_undegraded":"2024-08-17T17:57:27.490713+0000","last_fullsized":"2024-08-17T17:57:27.490713+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1900,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.180984+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:57:39.180984+0000","last_peered":"2024-08-17T17:57:39.180984+0000","last_clean":"2024-08-17T17:57:39.180984+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:57:39.180984+0000","last_undegraded":"2024-08-17T17:57:39.180984+0000","last_fullsized":"2024-08-17T17:57:39.180984+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1902,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.187392+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:57:39.187392+0000","last_peered":"2024-08-17T17:57:39.187392+0000","last_clean":"2024-08-17T17:57:39.187392+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:57:39.187392+0000","last_undegraded":"2024-08-17T17:57:39.187392+0000","last_fullsized":"2024-08-17T17:57:39.187392+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":122,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.265463+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:57:39.265463+0000","last_peered":"2024-08-17T17:57:39.265463+0000","last_clean":"2024-08-17T17:57:39.265463+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:57:39.265463+0000","last_undegraded":"2024-08-17T17:57:39.265463+0000","last_fullsized":"2024-08-17T17:57:39.265463+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1872,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490617+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:57:27.490617+0000","last_peered":"2024-08-17T17:57:27.490617+0000","last_clean":"2024-08-17T17:57:27.490617+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:57:27.490617+0000","last_undegraded":"2024-08-17T17:57:27.490617+0000","last_fullsized":"2024-08-17T17:57:27.490617+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1929,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.264931+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:57:39.264931+0000","last_peered":"2024-08-17T17:57:39.264931+0000","last_clean":"2024-08-17T17:57:39.264931+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:57:39.264931+0000","last_undegraded":"2024-08-17T17:57:39.264931+0000","last_fullsized":"2024-08-17T17:57:39.264931+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1923,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.181810+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:57:39.181810+0000","last_peered":"2024-08-17T17:57:39.181810+0000","last_clean":"2024-08-17T17:57:39.181810+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:57:39.181810+0000","last_undegraded":"2024-08-17T17:57:39.181810+0000","last_fullsized":"2024-08-17T17:57:39.181810+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1894,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490020+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:57:27.490020+0000","last_peered":"2024-08-17T17:57:27.490020+0000","last_clean":"2024-08-17T17:57:27.490020+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:57:27.490020+0000","last_undegraded":"2024-08-17T17:57:27.490020+0000","last_fullsized":"2024-08-17T17:57:27.490020+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1216,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.549258+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:57:27.549258+0000","last_peered":"2024-08-17T17:57:27.549258+0000","last_clean":"2024-08-17T17:57:27.549258+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:57:27.549258+0000","last_undegraded":"2024-08-17T17:57:27.549258+0000","last_fullsized":"2024-08-17T17:57:27.549258+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1866,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.489922+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:57:27.489922+0000","last_peered":"2024-08-17T17:57:27.489922+0000","last_clean":"2024-08-17T17:57:27.489922+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:57:27.489922+0000","last_undegraded":"2024-08-17T17:57:27.489922+0000","last_fullsized":"2024-08-17T17:57:27.489922+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1917,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.181951+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:57:39.181951+0000","last_peered":"2024-08-17T17:57:39.181951+0000","last_clean":"2024-08-17T17:57:39.181951+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:57:39.181951+0000","last_undegraded":"2024-08-17T17:57:39.181951+0000","last_fullsized":"2024-08-17T17:57:39.181951+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1822,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490076+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:57:27.490076+0000","last_peered":"2024-08-17T17:57:27.490076+0000","last_clean":"2024-08-17T17:57:27.490076+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:57:27.490076+0000","last_undegraded":"2024-08-17T17:57:27.490076+0000","last_fullsized":"2024-08-17T17:57:27.490076+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":535,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.548073+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:57:27.548073+0000","last_peered":"2024-08-17T17:57:27.548073+0000","last_clean":"2024-08-17T17:57:27.548073+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:57:27.548073+0000","last_undegraded":"2024-08-17T17:57:27.548073+0000","last_fullsized":"2024-08-17T17:57:27.548073+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1212,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.548152+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:57:27.548152+0000","last_peered":"2024-08-17T17:57:27.548152+0000","last_clean":"2024-08-17T17:57:27.548152+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:57:27.548152+0000","last_undegraded":"2024-08-17T17:57:27.548152+0000","last_fullsized":"2024-08-17T17:57:27.548152+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1922,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.265680+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:57:39.265680+0000","last_peered":"2024-08-17T17:57:39.265680+0000","last_clean":"2024-08-17T17:57:39.265680+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:57:39.265680+0000","last_undegraded":"2024-08-17T17:57:39.265680+0000","last_fullsized":"2024-08-17T17:57:39.265680+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":373,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.548063+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:57:27.548063+0000","last_peered":"2024-08-17T17:57:27.548063+0000","last_clean":"2024-08-17T17:57:27.548063+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:57:27.548063+0000","last_undegraded":"2024-08-17T17:57:27.548063+0000","last_fullsized":"2024-08-17T17:57:27.548063+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":519,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.548958+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:57:27.548958+0000","last_peered":"2024-08-17T17:57:27.548958+0000","last_clean":"2024-08-17T17:57:27.548958+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:57:27.548958+0000","last_undegraded":"2024-08-17T17:57:27.548958+0000","last_fullsized":"2024-08-17T17:57:27.548958+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1197,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.549069+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:57:27.549069+0000","last_peered":"2024-08-17T17:57:27.549069+0000","last_clean":"2024-08-17T17:57:27.549069+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:57:27.549069+0000","last_undegraded":"2024-08-17T17:57:27.549069+0000","last_fullsized":"2024-08-17T17:57:27.549069+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1205,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.549298+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:57:27.549298+0000","last_peered":"2024-08-17T17:57:27.549298+0000","last_clean":"2024-08-17T17:57:27.549298+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:57:27.549298+0000","last_undegraded":"2024-08-17T17:57:27.549298+0000","last_fullsized":"2024-08-17T17:57:27.549298+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1970,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.181551+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:57:39.181551+0000","last_peered":"2024-08-17T17:57:39.181551+0000","last_clean":"2024-08-17T17:57:39.181551+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:57:39.181551+0000","last_undegraded":"2024-08-17T17:57:39.181551+0000","last_fullsized":"2024-08-17T17:57:39.181551+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1851,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.489968+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:57:27.489968+0000","last_peered":"2024-08-17T17:57:27.489968+0000","last_clean":"2024-08-17T17:57:27.489968+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:57:27.489968+0000","last_undegraded":"2024-08-17T17:57:27.489968+0000","last_fullsized":"2024-08-17T17:57:27.489968+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1928,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.265596+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:57:39.265596+0000","last_peered":"2024-08-17T17:57:39.265596+0000","last_clean":"2024-08-17T17:57:39.265596+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:57:39.265596+0000","last_undegraded":"2024-08-17T17:57:39.265596+0000","last_fullsized":"2024-08-17T17:57:39.265596+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1853,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490960+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:57:27.490960+0000","last_peered":"2024-08-17T17:57:27.490960+0000","last_clean":"2024-08-17T17:57:27.490960+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:57:27.490960+0000","last_undegraded":"2024-08-17T17:57:27.490960+0000","last_fullsized":"2024-08-17T17:57:27.490960+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1888,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.181082+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:57:39.181082+0000","last_peered":"2024-08-17T17:57:39.181082+0000","last_clean":"2024-08-17T17:57:39.181082+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:57:39.181082+0000","last_undegraded":"2024-08-17T17:57:39.181082+0000","last_fullsized":"2024-08-17T17:57:39.181082+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1232,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.548294+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:57:27.548294+0000","last_peered":"2024-08-17T17:57:27.548294+0000","last_clean":"2024-08-17T17:57:27.548294+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:57:27.548294+0000","last_undegraded":"2024-08-17T17:57:27.548294+0000","last_fullsized":"2024-08-17T17:57:27.548294+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1934,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.267863+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:57:39.267863+0000","last_peered":"2024-08-17T17:57:39.267863+0000","last_clean":"2024-08-17T17:57:39.267863+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:57:39.267863+0000","last_undegraded":"2024-08-17T17:57:39.267863+0000","last_fullsized":"2024-08-17T17:57:39.267863+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":121,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.265752+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:57:39.265752+0000","last_peered":"2024-08-17T17:57:39.265752+0000","last_clean":"2024-08-17T17:57:39.265752+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:57:39.265752+0000","last_undegraded":"2024-08-17T17:57:39.265752+0000","last_fullsized":"2024-08-17T17:57:39.265752+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1953,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.267757+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:57:39.267757+0000","last_peered":"2024-08-17T17:57:39.267757+0000","last_clean":"2024-08-17T17:57:39.267757+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:57:39.267757+0000","last_undegraded":"2024-08-17T17:57:39.267757+0000","last_fullsized":"2024-08-17T17:57:39.267757+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":688,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490479+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:57:27.490479+0000","last_peered":"2024-08-17T17:57:27.490479+0000","last_clean":"2024-08-17T17:57:27.490479+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:57:27.490479+0000","last_undegraded":"2024-08-17T17:57:27.490479+0000","last_fullsized":"2024-08-17T17:57:27.490479+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1934,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.267870+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:57:39.267870+0000","last_peered":"2024-08-17T17:57:39.267870+0000","last_clean":"2024-08-17T17:57:39.267870+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:57:39.267870+0000","last_undegraded":"2024-08-17T17:57:39.267870+0000","last_fullsized":"2024-08-17T17:57:39.267870+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1263,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.180818+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:57:39.180818+0000","last_peered":"2024-08-17T17:57:39.180818+0000","last_clean":"2024-08-17T17:57:39.180818+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:57:39.180818+0000","last_undegraded":"2024-08-17T17:57:39.180818+0000","last_fullsized":"2024-08-17T17:57:39.180818+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":181,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.265592+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:57:39.265592+0000","last_peered":"2024-08-17T17:57:39.265592+0000","last_clean":"2024-08-17T17:57:39.265592+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:57:39.265592+0000","last_undegraded":"2024-08-17T17:57:39.265592+0000","last_fullsized":"2024-08-17T17:57:39.265592+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1945,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.265161+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:57:39.265161+0000","last_peered":"2024-08-17T17:57:39.265161+0000","last_clean":"2024-08-17T17:57:39.265161+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:57:39.265161+0000","last_undegraded":"2024-08-17T17:57:39.265161+0000","last_fullsized":"2024-08-17T17:57:39.265161+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1923,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.264828+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:57:39.264828+0000","last_peered":"2024-08-17T17:57:39.264828+0000","last_clean":"2024-08-17T17:57:39.264828+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:57:39.264828+0000","last_undegraded":"2024-08-17T17:57:39.264828+0000","last_fullsized":"2024-08-17T17:57:39.264828+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1943,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.187073+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:57:39.187073+0000","last_peered":"2024-08-17T17:57:39.187073+0000","last_clean":"2024-08-17T17:57:39.187073+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:57:39.187073+0000","last_undegraded":"2024-08-17T17:57:39.187073+0000","last_fullsized":"2024-08-17T17:57:39.187073+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1949,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.181887+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:57:39.181887+0000","last_peered":"2024-08-17T17:57:39.181887+0000","last_clean":"2024-08-17T17:57:39.181887+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:57:39.181887+0000","last_undegraded":"2024-08-17T17:57:39.181887+0000","last_fullsized":"2024-08-17T17:57:39.181887+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1929,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.211625+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:57:39.211625+0000","last_peered":"2024-08-17T17:57:39.211625+0000","last_clean":"2024-08-17T17:57:39.211625+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:57:39.211625+0000","last_undegraded":"2024-08-17T17:57:39.211625+0000","last_fullsized":"2024-08-17T17:57:39.211625+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1239,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.548030+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:57:27.548030+0000","last_peered":"2024-08-17T17:57:27.548030+0000","last_clean":"2024-08-17T17:57:27.548030+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:57:27.548030+0000","last_undegraded":"2024-08-17T17:57:27.548030+0000","last_fullsized":"2024-08-17T17:57:27.548030+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1185,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.549220+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:57:27.549220+0000","last_peered":"2024-08-17T17:57:27.549220+0000","last_clean":"2024-08-17T17:57:27.549220+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:57:27.549220+0000","last_undegraded":"2024-08-17T17:57:27.549220+0000","last_fullsized":"2024-08-17T17:57:27.549220+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1836,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490365+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:57:27.490365+0000","last_peered":"2024-08-17T17:57:27.490365+0000","last_clean":"2024-08-17T17:57:27.490365+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:57:27.490365+0000","last_undegraded":"2024-08-17T17:57:27.490365+0000","last_fullsized":"2024-08-17T17:57:27.490365+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":169,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.187122+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:57:39.187122+0000","last_peered":"2024-08-17T17:57:39.187122+0000","last_clean":"2024-08-17T17:57:39.187122+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:57:39.187122+0000","last_undegraded":"2024-08-17T17:57:39.187122+0000","last_fullsized":"2024-08-17T17:57:39.187122+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1551,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.547667+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:57:27.547667+0000","last_peered":"2024-08-17T17:57:27.547667+0000","last_clean":"2024-08-17T17:57:27.547667+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:57:27.547667+0000","last_undegraded":"2024-08-17T17:57:27.547667+0000","last_fullsized":"2024-08-17T17:57:27.547667+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1930,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.265264+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:57:39.265264+0000","last_peered":"2024-08-17T17:57:39.265264+0000","last_clean":"2024-08-17T17:57:39.265264+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:57:39.265264+0000","last_undegraded":"2024-08-17T17:57:39.265264+0000","last_fullsized":"2024-08-17T17:57:39.265264+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1867,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.489933+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:57:27.489933+0000","last_peered":"2024-08-17T17:57:27.489933+0000","last_clean":"2024-08-17T17:57:27.489933+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:57:27.489933+0000","last_undegraded":"2024-08-17T17:57:27.489933+0000","last_fullsized":"2024-08-17T17:57:27.489933+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":519,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.549134+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:57:27.549134+0000","last_peered":"2024-08-17T17:57:27.549134+0000","last_clean":"2024-08-17T17:57:27.549134+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:57:27.549134+0000","last_undegraded":"2024-08-17T17:57:27.549134+0000","last_fullsized":"2024-08-17T17:57:27.549134+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":608,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.186927+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:57:39.186927+0000","last_peered":"2024-08-17T17:57:39.186927+0000","last_clean":"2024-08-17T17:57:39.186927+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:57:39.186927+0000","last_undegraded":"2024-08-17T17:57:39.186927+0000","last_fullsized":"2024-08-17T17:57:39.186927+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":624,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.187559+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:57:39.187559+0000","last_peered":"2024-08-17T17:57:39.187559+0000","last_clean":"2024-08-17T17:57:39.187559+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:57:39.187559+0000","last_undegraded":"2024-08-17T17:57:39.187559+0000","last_fullsized":"2024-08-17T17:57:39.187559+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1203,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.550920+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:57:27.550920+0000","last_peered":"2024-08-17T17:57:27.550920+0000","last_clean":"2024-08-17T17:57:27.550920+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:57:27.550920+0000","last_undegraded":"2024-08-17T17:57:27.550920+0000","last_fullsized":"2024-08-17T17:57:27.550920+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":638,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490379+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:57:27.490379+0000","last_peered":"2024-08-17T17:57:27.490379+0000","last_clean":"2024-08-17T17:57:27.490379+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:57:27.490379+0000","last_undegraded":"2024-08-17T17:57:27.490379+0000","last_fullsized":"2024-08-17T17:57:27.490379+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1864,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490807+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:57:27.490807+0000","last_peered":"2024-08-17T17:57:27.490807+0000","last_clean":"2024-08-17T17:57:27.490807+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:57:27.490807+0000","last_undegraded":"2024-08-17T17:57:27.490807+0000","last_fullsized":"2024-08-17T17:57:27.490807+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1962,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.181159+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:57:39.181159+0000","last_peered":"2024-08-17T17:57:39.181159+0000","last_clean":"2024-08-17T17:57:39.181159+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:57:39.181159+0000","last_undegraded":"2024-08-17T17:57:39.181159+0000","last_fullsized":"2024-08-17T17:57:39.181159+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":671,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490294+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:57:27.490294+0000","last_peered":"2024-08-17T17:57:27.490294+0000","last_clean":"2024-08-17T17:57:27.490294+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:57:27.490294+0000","last_undegraded":"2024-08-17T17:57:27.490294+0000","last_fullsized":"2024-08-17T17:57:27.490294+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1192,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.550830+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:57:27.550830+0000","last_peered":"2024-08-17T17:57:27.550830+0000","last_clean":"2024-08-17T17:57:27.550830+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:57:27.550830+0000","last_undegraded":"2024-08-17T17:57:27.550830+0000","last_fullsized":"2024-08-17T17:57:27.550830+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1191,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.548231+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:57:27.548231+0000","last_peered":"2024-08-17T17:57:27.548231+0000","last_clean":"2024-08-17T17:57:27.548231+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:57:27.548231+0000","last_undegraded":"2024-08-17T17:57:27.548231+0000","last_fullsized":"2024-08-17T17:57:27.548231+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":657,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.265014+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:57:39.265014+0000","last_peered":"2024-08-17T17:57:39.265014+0000","last_clean":"2024-08-17T17:57:39.265014+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:57:39.265014+0000","last_undegraded":"2024-08-17T17:57:39.265014+0000","last_fullsized":"2024-08-17T17:57:39.265014+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1209,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.548200+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:57:27.548200+0000","last_peered":"2024-08-17T17:57:27.548200+0000","last_clean":"2024-08-17T17:57:27.548200+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:57:27.548200+0000","last_undegraded":"2024-08-17T17:57:27.548200+0000","last_fullsized":"2024-08-17T17:57:27.548200+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":93,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.547614+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:57:27.547614+0000","last_peered":"2024-08-17T17:57:27.547614+0000","last_clean":"2024-08-17T17:57:27.547614+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:57:27.547614+0000","last_undegraded":"2024-08-17T17:57:27.547614+0000","last_fullsized":"2024-08-17T17:57:27.547614+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1225,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.548411+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:57:27.548411+0000","last_peered":"2024-08-17T17:57:27.548411+0000","last_clean":"2024-08-17T17:57:27.548411+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:57:27.548411+0000","last_undegraded":"2024-08-17T17:57:27.548411+0000","last_fullsized":"2024-08-17T17:57:27.548411+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1242,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.548856+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:57:27.548856+0000","last_peered":"2024-08-17T17:57:27.548856+0000","last_clean":"2024-08-17T17:57:27.548856+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:57:27.548856+0000","last_undegraded":"2024-08-17T17:57:27.548856+0000","last_fullsized":"2024-08-17T17:57:27.548856+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1865,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490878+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:57:27.490878+0000","last_peered":"2024-08-17T17:57:27.490878+0000","last_clean":"2024-08-17T17:57:27.490878+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:57:27.490878+0000","last_undegraded":"2024-08-17T17:57:27.490878+0000","last_fullsized":"2024-08-17T17:57:27.490878+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1186,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.547865+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:57:27.547865+0000","last_peered":"2024-08-17T17:57:27.547865+0000","last_clean":"2024-08-17T17:57:27.547865+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:57:27.547865+0000","last_undegraded":"2024-08-17T17:57:27.547865+0000","last_fullsized":"2024-08-17T17:57:27.547865+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1920,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.181662+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:57:39.181662+0000","last_peered":"2024-08-17T17:57:39.181662+0000","last_clean":"2024-08-17T17:57:39.181662+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:57:39.181662+0000","last_undegraded":"2024-08-17T17:57:39.181662+0000","last_fullsized":"2024-08-17T17:57:39.181662+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1921,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.184594+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:57:39.184594+0000","last_peered":"2024-08-17T17:57:39.184594+0000","last_clean":"2024-08-17T17:57:39.184594+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:57:39.184594+0000","last_undegraded":"2024-08-17T17:57:39.184594+0000","last_fullsized":"2024-08-17T17:57:39.184594+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":599,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.547965+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:57:27.547965+0000","last_peered":"2024-08-17T17:57:27.547965+0000","last_clean":"2024-08-17T17:57:27.547965+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:57:27.547965+0000","last_undegraded":"2024-08-17T17:57:27.547965+0000","last_fullsized":"2024-08-17T17:57:27.547965+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1211,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.547920+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:57:27.547920+0000","last_peered":"2024-08-17T17:57:27.547920+0000","last_clean":"2024-08-17T17:57:27.547920+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:57:27.547920+0000","last_undegraded":"2024-08-17T17:57:27.547920+0000","last_fullsized":"2024-08-17T17:57:27.547920+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1866,"reported_epoch":1241,"state":"active+clean","last_fresh":"2024-08-17T17:57:27.490544+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:57:27.490544+0000","last_peered":"2024-08-17T17:57:27.490544+0000","last_clean":"2024-08-17T17:57:27.490544+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:57:27.490544+0000","last_undegraded":"2024-08-17T17:57:27.490544+0000","last_fullsized":"2024-08-17T17:57:27.490544+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":2163736600,"num_objects":33017,"num_object_clones":0,"num_object_copies":99051,"num_objects_missing_on_primary":1462,"num_objects_missing":1462,"num_objects_degraded":15837,"num_objects_misplaced":4884,"num_objects_unfound":0,"num_objects_dirty":33017,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":176593,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37670,"num_bytes_recovered":2290286592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7936487424,"data_stored":7936475208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108467,"ondisk_log_size":108467,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234442,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1232656,"kb_used_data":922824,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92512496,"statfs":{"total":95995035648,"available":94732795904,"internally_reserved":0,"allocated":944971776,"data_stored":939936031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496077,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1359956,"kb_used_data":1058444,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92385196,"statfs":{"total":95995035648,"available":94602440704,"internally_reserved":0,"allocated":1083846656,"data_stored":1078807316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757715,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326328,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1461612,"kb_used_data":1226532,"kb_used_omap":57,"kb_used_meta":235014,"kb_avail":92283540,"statfs":{"total":95995035648,"available":94498344960,"internally_reserved":0,"allocated":1255968768,"data_stored":1250955639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":240655099},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405567,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1332788,"kb_used_data":1088236,"kb_used_omap":61,"kb_used_meta":244482,"kb_avail":92412364,"statfs":{"total":95995035648,"available":94630260736,"internally_reserved":0,"allocated":1114353664,"data_stored":1109337507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":250350520},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452102,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":980088,"kb_used_data":799728,"kb_used_omap":54,"kb_used_meta":180297,"kb_avail":92765064,"statfs":{"total":95995035648,"available":94991425536,"internally_reserved":0,"allocated":818921472,"data_stored":813904868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":184624526},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474793,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1014728,"kb_used_data":800000,"kb_used_omap":119,"kb_used_meta":214600,"kb_avail":92730424,"statfs":{"total":95995035648,"available":94955954176,"internally_reserved":0,"allocated":819200000,"data_stored":814168232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":219751134},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1,0,3,2,4,7],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019350,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":803475456,"data_stored":803471384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":802095104,"data_stored":802095104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1100677120,"data_stored":1100677120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1238306816,"data_stored":1238302744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":905248768,"data_stored":905248768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1069355008,"data_stored":1069350936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":929431552,"data_stored":929431552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:57:46.202 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:57:46.217 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:46.238 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:46.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:46.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:46.340 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:46.362 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:46.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:46.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:46.430 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:46 smithi130 ceph-mon[105521]: pgmap v1937: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 228 op/s; 15837/97866 objects degraded (16.182%); 4884/97866 objects misplaced (4.990%); 960 KiB/s, 14 objects/s recovering 2024-08-17T17:57:46.430 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/97866 objects degraded (16.182%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:46.430 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:46 smithi130 ceph-mon[110068]: pgmap v1937: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 228 op/s; 15837/97866 objects degraded (16.182%); 4884/97866 objects misplaced (4.990%); 960 KiB/s, 14 objects/s recovering 2024-08-17T17:57:46.431 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/97866 objects degraded (16.182%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:46.464 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:46.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:46.495 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:46.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:46 smithi179 ceph-mon[93519]: pgmap v1937: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 228 op/s; 15837/97866 objects degraded (16.182%); 4884/97866 objects misplaced (4.990%); 960 KiB/s, 14 objects/s recovering 2024-08-17T17:57:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/97866 objects degraded (16.182%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:46.596 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:46.619 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:46.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:46.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:46.721 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:46.742 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:46.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:46.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:46.844 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:46.875 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:46.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:46.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:46.927 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T17:57:46.927 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:57:46.977 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:46.982 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:47.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:47.006 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:47.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:47.106 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:47.128 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:47.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:47.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:47.230 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:47.252 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:47.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:47.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:47.328 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:57:47.354 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:47.379 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:47.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:47.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:47.481 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:47.505 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:47.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:47.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:47.518 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:47 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2434288009' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:57:47.518 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:47 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:57:47.520 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:47 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2434288009' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:57:47.520 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:47 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:57:47.520 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:47 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2434288009' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:57:47.520 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:47 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:57:47.608 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:47.631 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:47.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:47.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:47.733 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:47.755 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:47.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:47.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:47.857 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:47.881 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:47.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:47.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:47.902 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:57:47.982 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:48.023 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:48.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:48.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:48.125 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:48.155 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:48.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:48.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:48.258 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:48.283 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:48.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:48.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:48.384 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:48.407 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:48.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:48.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:48.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:48 smithi130 ceph-mon[105521]: pgmap v1938: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 226 op/s; 15837/96933 objects degraded (16.338%); 4648/96933 objects misplaced (4.795%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:57:48.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:48 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:57:48.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:48 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:57:48.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:48 smithi130 ceph-mon[105521]: osdmap e1243: 8 total, 8 up, 6 in 2024-08-17T17:57:48.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:48 smithi130 ceph-mon[110068]: pgmap v1938: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 226 op/s; 15837/96933 objects degraded (16.338%); 4648/96933 objects misplaced (4.795%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:57:48.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:48 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:57:48.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:48 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:57:48.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:48 smithi130 ceph-mon[110068]: osdmap e1243: 8 total, 8 up, 6 in 2024-08-17T17:57:48.509 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:48.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:48.533 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:48.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:48 smithi179 ceph-mon[93519]: pgmap v1938: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 226 op/s; 15837/96933 objects degraded (16.338%); 4648/96933 objects misplaced (4.795%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:57:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:48 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:57:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:48 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:57:48.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:48 smithi179 ceph-mon[93519]: osdmap e1243: 8 total, 8 up, 6 in 2024-08-17T17:57:48.634 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:48.655 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:48.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:48.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:48.757 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:48.778 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:48.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:48.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:48.879 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:48.900 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:48.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:48.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:49.002 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:49.029 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:49.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:49.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:49.131 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:49.156 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:49.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:49.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:49.258 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:49.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:49.280 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:49.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:49.381 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:49.405 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:49.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:49.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:49.507 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:49.528 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:49.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:49.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:49.630 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:49.652 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:49.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:49.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:49.754 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:49.782 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:49.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:49.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:49.884 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:49.906 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:49.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:49.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:57:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:57:49] "GET /metrics HTTP/1.1" 200 36807 "" "Prometheus/2.51.0" 2024-08-17T17:57:50.008 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:50.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:50.030 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:50.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:50.131 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:50.163 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:50.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:50.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:50.265 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:50.287 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:50.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:50.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:50.389 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:50.413 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:50.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:50.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:50.514 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:50.537 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:50.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:50.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:50 smithi179 ceph-mon[93519]: pgmap v1940: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 241 op/s; 15837/94476 objects degraded (16.763%); 4648/94476 objects misplaced (4.920%); 985 KiB/s, 15 objects/s recovering 2024-08-17T17:57:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:57:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:57:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:57:50.636 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:50.639 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:50.664 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:50.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:50.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:50.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:50 smithi130 ceph-mon[110068]: pgmap v1940: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 241 op/s; 15837/94476 objects degraded (16.763%); 4648/94476 objects misplaced (4.920%); 985 KiB/s, 15 objects/s recovering 2024-08-17T17:57:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:50 smithi130 ceph-mon[105521]: pgmap v1940: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 241 op/s; 15837/94476 objects degraded (16.763%); 4648/94476 objects misplaced (4.920%); 985 KiB/s, 15 objects/s recovering 2024-08-17T17:57:50.766 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:50.792 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:50.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:50.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:50.893 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:50.922 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:50.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:50.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:51.024 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:51.049 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:51.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:51.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:51.151 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:51.180 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:51.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:51.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:51.282 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:51.303 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:51.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:51.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:51.355 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/94476 objects degraded (16.763%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:51.355 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/94476 objects degraded (16.763%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:51.405 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:51.426 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:51.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:51.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:51.528 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:51.539 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:57:51.539 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15837/93924 objects degraded (16.862%), 5 pgs degraded, 6 pgs undersized","count":11},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1792,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1243,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":31308,"data_bytes":2050627557,"bytes_used":7270395904,"bytes_avail":568699817984,"bytes_total":575970213888,"degraded_objects":15837,"degraded_total":93924,"degraded_ratio":0.16861505046633452,"misplaced_objects":4648,"misplaced_total":93924,"misplaced_ratio":0.04948681913036071,"recovering_objects_per_sec":15,"recovering_bytes_per_sec":1002420,"recovering_keys_per_sec":0,"num_objects_recovered":153,"num_bytes_recovered":10027008,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":243},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":26,"modified":"2024-08-17T17:57:13.099933+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (10m)\n [==========================..] (remaining: 44s)","progress":0.93258428573608398,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:57:51.557 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:51.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:51.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:51.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/94476 objects degraded (16.763%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:51.659 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:51.681 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:51.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:51.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:51.783 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:51.806 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:51.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:51.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:51.908 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:51.929 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:51.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:51.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:52.031 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:52.055 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:52.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:52.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:52.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:57:52.157 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:52.188 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:52.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:52.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:52.290 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:52.311 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:52.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:52.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:52.413 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:52.437 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:52.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:52.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:52.539 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:52.560 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:52.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:52.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:52.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:52 smithi179 ceph-mon[93519]: pgmap v1941: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 243 op/s; 15837/93924 objects degraded (16.862%); 4648/93924 objects misplaced (4.949%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:57:52.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:52 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/673882479' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:57:52.661 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:52.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:52 smithi130 ceph-mon[105521]: pgmap v1941: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 243 op/s; 15837/93924 objects degraded (16.862%); 4648/93924 objects misplaced (4.949%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:57:52.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:52 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/673882479' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:57:52.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:52 smithi130 ceph-mon[110068]: pgmap v1941: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 243 op/s; 15837/93924 objects degraded (16.862%); 4648/93924 objects misplaced (4.949%); 979 KiB/s, 15 objects/s recovering 2024-08-17T17:57:52.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:52 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/673882479' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:57:52.683 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:52.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:52.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:52.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:52.806 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:52.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:52.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:52.908 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:52.929 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:52.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:52.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:53.030 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:53.051 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:53.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:53.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:53.153 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:53.174 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:53.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:53.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:53.276 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:53.298 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:53.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:53.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:53.400 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:53.422 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:53.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:53.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:53.523 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:53.547 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:53.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:53.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:53.549 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:57:53.550 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:57:53.648 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:53.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:53.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:53.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:53.766 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:57:53.766 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:57:53.788 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:57:53.788 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:57:53.795 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:53.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:53.818 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:53.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:53.818 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:57:53.819 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:57:53.842 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:57:53.842 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:57:53.865 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:57:53.865 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:57:53.886 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:57:53.886 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:57:53.902 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:57:53.903 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:57:53.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:53.923 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:57:53.923 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:57:53.941 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:53.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:53.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:54.043 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:54.065 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:54.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:54.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:54.167 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:54.192 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:54.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:54.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:54.294 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:54.315 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:54.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:54.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:54.417 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:54.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:54.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:54.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:54.534 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:54.543 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:54.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:54.567 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:54.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:54.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:54 smithi179 ceph-mon[93519]: pgmap v1942: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 261 op/s; 15837/91278 objects degraded (17.350%); 4383/91278 objects misplaced (4.802%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:57:54.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/946801950' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:54.648 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:54 smithi130 ceph-mon[105521]: pgmap v1942: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 261 op/s; 15837/91278 objects degraded (17.350%); 4383/91278 objects misplaced (4.802%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:57:54.648 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/946801950' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:54.648 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:54 smithi130 ceph-mon[110068]: pgmap v1942: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 261 op/s; 15837/91278 objects degraded (17.350%); 4383/91278 objects misplaced (4.802%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:57:54.648 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/946801950' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:57:54.669 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:54.690 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:54.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:54.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:54.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:54.814 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:54.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:54.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:54.916 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:54.937 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:54.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:54.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:55.039 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:55.061 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:55.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:55.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:55.163 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:55.192 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:55.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:55.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:55.294 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:55.322 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:55.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:55.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:55.424 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:55.453 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:55.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:55.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:55.555 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:55.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:55.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:55.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:55.677 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:55.702 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:55.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:55.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:55.805 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:55.830 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:55.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:55.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:55.932 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:57:55.953 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:55.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:55.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:57:56.054 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:56.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:56.082 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:56.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:56.183 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:56.204 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:56.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:56.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:56.306 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:56.327 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:56.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:56.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:56.429 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:56.454 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:56.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:56.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:56.556 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:56.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:56.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:56.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:56.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:56 smithi179 ceph-mon[93519]: pgmap v1943: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 264 op/s; 15837/89919 objects degraded (17.613%); 4383/89919 objects misplaced (4.874%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:57:56.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/89919 objects degraded (17.613%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:56.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:56 smithi130 ceph-mon[105521]: pgmap v1943: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 264 op/s; 15837/89919 objects degraded (17.613%); 4383/89919 objects misplaced (4.874%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:57:56.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/89919 objects degraded (17.613%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:56 smithi130 ceph-mon[110068]: pgmap v1943: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 264 op/s; 15837/89919 objects degraded (17.613%); 4383/89919 objects misplaced (4.874%); 998 KiB/s, 15 objects/s recovering 2024-08-17T17:57:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/89919 objects degraded (17.613%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:57:56.678 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:56.699 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:56.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:56.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:56.801 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:56.828 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:56.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:56.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:56.930 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:56.955 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:56.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:56.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:57.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:57.082 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:57.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:57.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:57.109 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:57:57.109 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T17:57:57.109 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.109 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180089 45.1 45:8af2af77:::benchmark_data_smithi130_172551_object75282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.109 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:54.951875+0000", 2024-08-17T17:57:57.109 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.74166634600000003, 2024-08-17T17:57:57.109 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.74169512999999998, 2024-08-17T17:57:57.109 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.109 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.110 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:57:57.110 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.110 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.110 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.110 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180089 2024-08-17T17:57:57.110 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.110 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.110 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.110 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.110 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.951875+0000", 2024-08-17T17:57:57.110 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.110 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.111 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.111 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.111 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.951875+0000", 2024-08-17T17:57:57.111 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.111 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.111 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.111 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.111 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.951877+0000", 2024-08-17T17:57:57.111 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.843e-06 2024-08-17T17:57:57.111 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.111 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.111 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.111 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.951883+0000", 2024-08-17T17:57:57.111 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.511e-06 2024-08-17T17:57:57.112 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.112 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.112 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.112 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.951896+0000", 2024-08-17T17:57:57.112 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2857000000000001e-05 2024-08-17T17:57:57.112 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.112 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.112 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.112 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.951904+0000", 2024-08-17T17:57:57.112 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2670000000000006e-06 2024-08-17T17:57:57.112 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.112 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.112 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.112 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.972612+0000", 2024-08-17T17:57:57.113 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020707803 2024-08-17T17:57:57.113 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.113 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.113 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.113 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.975409+0000", 2024-08-17T17:57:57.113 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002797187 2024-08-17T17:57:57.113 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.113 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.113 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.113 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.001613+0000", 2024-08-17T17:57:57.113 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.026203474000000001 2024-08-17T17:57:57.113 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.113 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.113 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:57:57.114 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.004374+0000", 2024-08-17T17:57:57.114 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002761576 2024-08-17T17:57:57.114 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.114 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.114 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.114 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.114 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.114 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:179981 45.1 45:8b1ca178:::benchmark_data_smithi130_172551_object75174:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.114 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:54.623852+0000", 2024-08-17T17:57:57.114 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 1.069689401, 2024-08-17T17:57:57.114 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.069791503, 2024-08-17T17:57:57.114 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 179981 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.623852+0000", 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.115 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.116 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.623852+0000", 2024-08-17T17:57:57.116 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.116 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.116 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.116 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.116 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.623853+0000", 2024-08-17T17:57:57.116 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.443e-06 2024-08-17T17:57:57.116 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.116 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.116 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.116 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.623860+0000", 2024-08-17T17:57:57.116 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6209999999999998e-06 2024-08-17T17:57:57.116 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.117 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.117 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.117 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.623871+0000", 2024-08-17T17:57:57.117 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1702000000000001e-05 2024-08-17T17:57:57.117 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.117 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.117 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.117 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.623878+0000", 2024-08-17T17:57:57.117 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4280000000000001e-06 2024-08-17T17:57:57.118 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.118 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.118 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.118 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.730054+0000", 2024-08-17T17:57:57.118 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.106175666 2024-08-17T17:57:57.118 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.118 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.118 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.118 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.731285+0000", 2024-08-17T17:57:57.118 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001231036 2024-08-17T17:57:57.118 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.118 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.749597+0000", 2024-08-17T17:57:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018312214 2024-08-17T17:57:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:57:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.752788+0000", 2024-08-17T17:57:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0031908829999999998 2024-08-17T17:57:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.119 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180315 45.1 45:852257b1:::benchmark_data_smithi130_172551_object75508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.120 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.605405+0000", 2024-08-17T17:57:57.120 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.088135778999999997, 2024-08-17T17:57:57.120 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.088304199, 2024-08-17T17:57:57.120 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.120 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.120 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:57:57.120 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.120 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.120 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.120 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180315 2024-08-17T17:57:57.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.120 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.121 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.605405+0000", 2024-08-17T17:57:57.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.121 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.121 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.121 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.605405+0000", 2024-08-17T17:57:57.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.121 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.121 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.121 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.605407+0000", 2024-08-17T17:57:57.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.708e-06 2024-08-17T17:57:57.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.121 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.122 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.605415+0000", 2024-08-17T17:57:57.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8690000000000004e-06 2024-08-17T17:57:57.122 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.122 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.122 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.605430+0000", 2024-08-17T17:57:57.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5333e-05 2024-08-17T17:57:57.122 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.122 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.122 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.605440+0000", 2024-08-17T17:57:57.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.842e-06 2024-08-17T17:57:57.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.123 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.616433+0000", 2024-08-17T17:57:57.123 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.01099286 2024-08-17T17:57:57.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.123 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.619201+0000", 2024-08-17T17:57:57.123 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027677639999999998 2024-08-17T17:57:57.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.123 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.649434+0000", 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.030233346000000001 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.653228+0000", 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0037939359999999999 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180055 45.1 45:8f588c43:::benchmark_data_smithi130_172551_object75248:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:54.861667+0000", 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.83187370900000002, 2024-08-17T17:57:57.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.83210578800000001, 2024-08-17T17:57:57.125 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.125 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.125 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:57:57.125 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.125 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.125 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.125 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180055 2024-08-17T17:57:57.125 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.125 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.125 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.125 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.125 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.861667+0000", 2024-08-17T17:57:57.125 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.861667+0000", 2024-08-17T17:57:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.861669+0000", 2024-08-17T17:57:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.435e-06 2024-08-17T17:57:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.126 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.861673+0000", 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1439999999999996e-06 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.861683+0000", 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0074999999999999e-05 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.861688+0000", 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9389999999999996e-06 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.127 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.912424+0000", 2024-08-17T17:57:57.128 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.050735726000000002 2024-08-17T17:57:57.128 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.128 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.128 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.128 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.915844+0000", 2024-08-17T17:57:57.128 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0034202939999999999 2024-08-17T17:57:57.128 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.128 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.128 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.128 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.963555+0000", 2024-08-17T17:57:57.128 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.047710731999999999 2024-08-17T17:57:57.128 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.128 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.129 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:57:57.129 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.966235+0000", 2024-08-17T17:57:57.129 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026799020000000001 2024-08-17T17:57:57.129 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.129 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.129 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.129 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.677165+0000", 2024-08-17T17:57:57.129 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.71093083499999998 2024-08-17T17:57:57.129 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.129 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.129 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:57:57.129 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.679327+0000", 2024-08-17T17:57:57.129 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002161615 2024-08-17T17:57:57.130 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.130 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.130 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:57:57.130 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.683656+0000", 2024-08-17T17:57:57.130 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0043290719999999998 2024-08-17T17:57:57.130 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.130 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.130 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.130 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.130 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.130 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180128 45.1 45:85704fa0:::benchmark_data_smithi130_172551_object75321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.130 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.216542+0000", 2024-08-17T17:57:57.130 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.476999065, 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.47736467599999999, 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180128 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.216542+0000", 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.132 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.216542+0000", 2024-08-17T17:57:57.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.132 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.132 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.216543+0000", 2024-08-17T17:57:57.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2759999999999999e-06 2024-08-17T17:57:57.132 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.132 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.216549+0000", 2024-08-17T17:57:57.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7339999999999996e-06 2024-08-17T17:57:57.133 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.133 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.133 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.133 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.216564+0000", 2024-08-17T17:57:57.133 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4708999999999999e-05 2024-08-17T17:57:57.133 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.133 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.133 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.133 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.216573+0000", 2024-08-17T17:57:57.133 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9479999999999997e-06 2024-08-17T17:57:57.133 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.133 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.133 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.133 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.300316+0000", 2024-08-17T17:57:57.134 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.083743001999999997 2024-08-17T17:57:57.134 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.134 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.134 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.134 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.303925+0000", 2024-08-17T17:57:57.134 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003609499 2024-08-17T17:57:57.134 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.134 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.134 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.134 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.324861+0000", 2024-08-17T17:57:57.134 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020935648000000001 2024-08-17T17:57:57.134 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.134 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.135 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:57:57.135 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.327874+0000", 2024-08-17T17:57:57.135 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0030129010000000001 2024-08-17T17:57:57.135 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.135 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.135 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.135 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.135 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.135 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180319 45.1 45:815cf7e2:::benchmark_data_smithi130_172551_object75512:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.135 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.614752+0000", 2024-08-17T17:57:57.135 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.078789311000000001, 2024-08-17T17:57:57.135 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.079216463000000001, 2024-08-17T17:57:57.135 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.135 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for missing object", 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180319 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.614752+0000", 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.136 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.614752+0000", 2024-08-17T17:57:57.137 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.137 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.137 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.137 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.137 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.614753+0000", 2024-08-17T17:57:57.137 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.716e-06 2024-08-17T17:57:57.137 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.137 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.137 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.137 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.614760+0000", 2024-08-17T17:57:57.137 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0950000000000001e-06 2024-08-17T17:57:57.137 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.137 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.137 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.614774+0000", 2024-08-17T17:57:57.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4643e-05 2024-08-17T17:57:57.138 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.138 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.138 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.614783+0000", 2024-08-17T17:57:57.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6109999999999994e-06 2024-08-17T17:57:57.138 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.138 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.138 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.656730+0000", 2024-08-17T17:57:57.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.041947050999999999 2024-08-17T17:57:57.139 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.139 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.139 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.139 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.660489+0000", 2024-08-17T17:57:57.139 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003758763 2024-08-17T17:57:57.139 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.139 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.139 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.139 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.139 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.139 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180138 45.1 45:82f89eb9:::benchmark_data_smithi130_172551_object75331:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.139 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.237733+0000", 2024-08-17T17:57:57.139 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.45580793400000003, 2024-08-17T17:57:57.139 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.45629064499999999, 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180138 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.237733+0000", 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.140 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.237733+0000", 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.237735+0000", 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.671e-06 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.237741+0000", 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9170000000000003e-06 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.141 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.237755+0000", 2024-08-17T17:57:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4086e-05 2024-08-17T17:57:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.237764+0000", 2024-08-17T17:57:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5060000000000001e-06 2024-08-17T17:57:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.142 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.305131+0000", 2024-08-17T17:57:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.067366896999999995 2024-08-17T17:57:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.308520+0000", 2024-08-17T17:57:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0033884330000000002 2024-08-17T17:57:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.343746+0000", 2024-08-17T17:57:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.035225935 2024-08-17T17:57:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.347176+0000", 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0034302880000000001 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180243 45.1 45:8d3ed445:::benchmark_data_smithi130_172551_object75436:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.447436+0000", 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.24610459300000001, 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.24664689400000001, 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:57:57.144 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.145 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.145 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.145 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180243 2024-08-17T17:57:57.145 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.145 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.145 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.145 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.447436+0000", 2024-08-17T17:57:57.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.145 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.145 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.145 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.447436+0000", 2024-08-17T17:57:57.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.146 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.146 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.146 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.146 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.447438+0000", 2024-08-17T17:57:57.146 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.663e-06 2024-08-17T17:57:57.146 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.146 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.146 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.146 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.447445+0000", 2024-08-17T17:57:57.146 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1770000000000001e-06 2024-08-17T17:57:57.146 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.146 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.146 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.447462+0000", 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7184e-05 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.447471+0000", 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5140000000000001e-06 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.485047+0000", 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.037575756000000002 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.488034+0000", 2024-08-17T17:57:57.148 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029874110000000001 2024-08-17T17:57:57.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.593734+0000", 2024-08-17T17:57:57.148 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.10570027999999999 2024-08-17T17:57:57.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:57:57.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.596845+0000", 2024-08-17T17:57:57.148 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003110121 2024-08-17T17:57:57.148 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.149 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.149 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.149 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.149 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "MOSDPGPushReply(45.1 1243/1189 [PushReplyOp(45:8b1ca178:::benchmark_data_smithi130_172551_object75174:head)])", 2024-08-17T17:57:57.149 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.357451+0000", 2024-08-17T17:57:57.149 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.336089727, 2024-08-17T17:57:57.149 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.33669168100000002, 2024-08-17T17:57:57.149 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.149 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.149 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "queued for pg", 2024-08-17T17:57:57.149 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.149 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.150 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.357451+0000", 2024-08-17T17:57:57.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.150 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.150 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.150 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.150 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.357451+0000", 2024-08-17T17:57:57.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.150 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.150 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.150 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.150 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.357452+0000", 2024-08-17T17:57:57.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2699999999999999e-07 2024-08-17T17:57:57.150 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.150 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.357455+0000", 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.484e-06 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.357465+0000", 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5920000000000007e-06 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.357472+0000", 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2930000000000003e-06 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.151 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.152 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.152 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180332 45.1 45:87bc20d0:::benchmark_data_smithi130_172551_object75525:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.152 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.643747+0000", 2024-08-17T17:57:57.152 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.049794098000000002, 2024-08-17T17:57:57.152 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.050435079000000001, 2024-08-17T17:57:57.152 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.152 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.152 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for missing object", 2024-08-17T17:57:57.152 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.152 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.152 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180332 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.643747+0000", 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.643747+0000", 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.153 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.154 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.643749+0000", 2024-08-17T17:57:57.154 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7489999999999999e-06 2024-08-17T17:57:57.154 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.154 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.154 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.154 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.643755+0000", 2024-08-17T17:57:57.154 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1120000000000001e-06 2024-08-17T17:57:57.154 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.154 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.154 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.154 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.643768+0000", 2024-08-17T17:57:57.154 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2996000000000001e-05 2024-08-17T17:57:57.155 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.155 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.155 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.155 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.643776+0000", 2024-08-17T17:57:57.155 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1019999999999997e-06 2024-08-17T17:57:57.155 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.155 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.155 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.155 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.661874+0000", 2024-08-17T17:57:57.155 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018097756 2024-08-17T17:57:57.155 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.155 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.155 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.155 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.664576+0000", 2024-08-17T17:57:57.156 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027028009999999999 2024-08-17T17:57:57.156 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.156 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.156 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.156 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.156 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.156 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180333 45.1 45.58a95411 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.156 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.644381+0000", 2024-08-17T17:57:57.156 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.049159900999999999, 2024-08-17T17:57:57.156 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.049850841999999999, 2024-08-17T17:57:57.156 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.156 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.156 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "queued for pg", 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180333 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.644381+0000", 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.644381+0000", 2024-08-17T17:57:57.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.158 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.158 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.158 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.158 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.644382+0000", 2024-08-17T17:57:57.158 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5000000000000001e-07 2024-08-17T17:57:57.158 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.158 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.158 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.158 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.644388+0000", 2024-08-17T17:57:57.158 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9320000000000001e-06 2024-08-17T17:57:57.158 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.158 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.158 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.644400+0000", 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1798e-05 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.644407+0000", 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9310000000000002e-06 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "MOSDPGPush(45.1 1243/1189 [PushOp(45:815cf7e2:::benchmark_data_smithi130_172551_object75512:head, version: 1175'4812, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(45:815cf7e2:::benchmark_data_smithi130_172551_object75512:head@1175'4812, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.661394+0000", 2024-08-17T17:57:57.159 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.032147091000000003, 2024-08-17T17:57:57.160 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.032911017000000001, 2024-08-17T17:57:57.160 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.160 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.160 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "queued for pg", 2024-08-17T17:57:57.160 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.160 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.160 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.160 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.661394+0000", 2024-08-17T17:57:57.160 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.160 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.160 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.160 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.160 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.661394+0000", 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.661395+0000", 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7899999999999998e-07 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.661432+0000", 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6909000000000002e-05 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.661448+0000", 2024-08-17T17:57:57.162 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.658e-05 2024-08-17T17:57:57.162 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.162 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.162 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.162 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.661458+0000", 2024-08-17T17:57:57.162 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0149000000000001e-05 2024-08-17T17:57:57.162 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.162 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.162 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.162 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.162 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.162 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "MOSDPGPush(45.1 1243/1189 [PushOp(45:87bc20d0:::benchmark_data_smithi130_172551_object75525:head, version: 1175'4813, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(45:87bc20d0:::benchmark_data_smithi130_172551_object75525:head@1175'4813, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:57:57.162 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.665442+0000", 2024-08-17T17:57:57.163 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.028099382999999999, 2024-08-17T17:57:57.163 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028919331, 2024-08-17T17:57:57.163 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.163 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.163 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:57:57.163 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.163 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.163 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.163 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.665442+0000", 2024-08-17T17:57:57.163 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.163 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.163 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.163 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.164 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.665442+0000", 2024-08-17T17:57:57.164 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.164 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.164 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.164 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.665443+0000", 2024-08-17T17:57:57.164 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.23e-07 2024-08-17T17:57:57.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.165 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.165 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.165 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.665480+0000", 2024-08-17T17:57:57.165 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7583000000000003e-05 2024-08-17T17:57:57.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.165 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.165 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.165 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.665497+0000", 2024-08-17T17:57:57.165 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6779e-05 2024-08-17T17:57:57.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.165 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.165 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.665507+0000", 2024-08-17T17:57:57.166 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0131e-05 2024-08-17T17:57:57.166 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.688775+0000", 2024-08-17T17:57:57.166 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023268262000000001 2024-08-17T17:57:57.166 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:57:57.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.689035+0000", 2024-08-17T17:57:57.166 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00025935999999999998 2024-08-17T17:57:57.166 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.166 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.167 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.167 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.167 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180291 45.1 45:8dc2334d:::benchmark_data_smithi130_172551_object75484:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.167 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.541195+0000", 2024-08-17T17:57:57.167 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.15234599400000001, 2024-08-17T17:57:57.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.153216878, 2024-08-17T17:57:57.167 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.167 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.167 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:57:57.167 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.167 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.167 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.168 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180291 2024-08-17T17:57:57.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.168 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.541195+0000", 2024-08-17T17:57:57.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.541195+0000", 2024-08-17T17:57:57.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.541197+0000", 2024-08-17T17:57:57.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2759999999999999e-06 2024-08-17T17:57:57.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.541203+0000", 2024-08-17T17:57:57.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9660000000000001e-06 2024-08-17T17:57:57.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.541220+0000", 2024-08-17T17:57:57.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6269999999999998e-05 2024-08-17T17:57:57.170 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.170 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.170 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.170 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.541227+0000", 2024-08-17T17:57:57.170 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7980000000000006e-06 2024-08-17T17:57:57.170 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.170 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.170 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.170 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.612124+0000", 2024-08-17T17:57:57.170 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.070896838000000004 2024-08-17T17:57:57.170 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.170 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.170 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.615236+0000", 2024-08-17T17:57:57.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003111843 2024-08-17T17:57:57.171 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.631962+0000", 2024-08-17T17:57:57.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016726172000000001 2024-08-17T17:57:57.171 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:57:57.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.635162+0000", 2024-08-17T17:57:57.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0031994110000000001 2024-08-17T17:57:57.171 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.171 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.172 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.172 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.172 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop_reply(client.45158.0:180055 45.1 e1243/1189)", 2024-08-17T17:57:57.172 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.686688+0000", 2024-08-17T17:57:57.172 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0068529410000000004, 2024-08-17T17:57:57.172 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0077820889999999998, 2024-08-17T17:57:57.172 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.172 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.172 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "queued for pg", 2024-08-17T17:57:57.172 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.172 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.172 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.686688+0000", 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.686688+0000", 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.686690+0000", 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6080000000000001e-06 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.686702+0000", 2024-08-17T17:57:57.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2479e-05 2024-08-17T17:57:57.174 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.686720+0000", 2024-08-17T17:57:57.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8240000000000002e-05 2024-08-17T17:57:57.174 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.686733+0000", 2024-08-17T17:57:57.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2656e-05 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180109 45.1 45:8b5b5ed3:::benchmark_data_smithi130_172551_object75302:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.185733+0000", 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.50780776900000002, 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.508775898, 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.175 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.176 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180109 2024-08-17T17:57:57.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.176 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.176 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.176 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.185733+0000", 2024-08-17T17:57:57.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.176 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.176 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.185733+0000", 2024-08-17T17:57:57.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.185734+0000", 2024-08-17T17:57:57.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6499999999999998e-07 2024-08-17T17:57:57.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.185739+0000", 2024-08-17T17:57:57.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3560000000000003e-06 2024-08-17T17:57:57.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.185749+0000", 2024-08-17T17:57:57.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0698e-05 2024-08-17T17:57:57.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.178 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.185756+0000", 2024-08-17T17:57:57.178 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4849999999999997e-06 2024-08-17T17:57:57.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.178 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.197086+0000", 2024-08-17T17:57:57.178 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011330458 2024-08-17T17:57:57.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.200845+0000", 2024-08-17T17:57:57.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0037591669999999999 2024-08-17T17:57:57.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.286212+0000", 2024-08-17T17:57:57.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.085367107999999997 2024-08-17T17:57:57.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:57:57.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.289157+0000", 2024-08-17T17:57:57.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029446199999999998 2024-08-17T17:57:57.179 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.180 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.180 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.180 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180251 45.1 45:802efd63:::benchmark_data_smithi130_172551_object75444:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.180 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.462616+0000", 2024-08-17T17:57:57.180 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.23092525699999999, 2024-08-17T17:57:57.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.23195450500000001, 2024-08-17T17:57:57.180 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.180 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.180 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:57:57.180 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.181 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.181 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.181 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180251 2024-08-17T17:57:57.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.181 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.181 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.462616+0000", 2024-08-17T17:57:57.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.181 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.462616+0000", 2024-08-17T17:57:57.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.462617+0000", 2024-08-17T17:57:57.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6759999999999999e-06 2024-08-17T17:57:57.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.462624+0000", 2024-08-17T17:57:57.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5169999999999996e-06 2024-08-17T17:57:57.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.462638+0000", 2024-08-17T17:57:57.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4078e-05 2024-08-17T17:57:57.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.462646+0000", 2024-08-17T17:57:57.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3240000000000003e-06 2024-08-17T17:57:57.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.488385+0000", 2024-08-17T17:57:57.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025738977 2024-08-17T17:57:57.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.491291+0000", 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029056199999999998 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.541713+0000", 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.050421685000000001 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.545889+0000", 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0041767729999999999 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.184 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.185 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.185 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop_reply(client.45158.0:180055 45.1 e1243/1189)", 2024-08-17T17:57:57.185 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.687890+0000", 2024-08-17T17:57:57.185 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0056511799999999996, 2024-08-17T17:57:57.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0067371449999999999, 2024-08-17T17:57:57.185 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.185 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.185 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "queued for pg", 2024-08-17T17:57:57.185 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.687890+0000", 2024-08-17T17:57:57.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.687890+0000", 2024-08-17T17:57:57.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.687891+0000", 2024-08-17T17:57:57.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1900000000000001e-07 2024-08-17T17:57:57.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.687895+0000", 2024-08-17T17:57:57.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2919999999999997e-06 2024-08-17T17:57:57.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.687905+0000", 2024-08-17T17:57:57.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9550000000000004e-06 2024-08-17T17:57:57.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.687911+0000", 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3230000000000003e-06 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180082 45.1 45:8bbf5453:::benchmark_data_smithi130_172551_object75275:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:54.929981+0000", 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.76356008399999997, 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.76468826700000003, 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.188 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180082 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.929981+0000", 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.929981+0000", 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.190 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.190 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.929983+0000", 2024-08-17T17:57:57.190 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2069999999999998e-06 2024-08-17T17:57:57.190 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.190 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.190 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.190 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.929990+0000", 2024-08-17T17:57:57.190 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7220000000000002e-06 2024-08-17T17:57:57.190 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.190 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.190 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.190 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.930001+0000", 2024-08-17T17:57:57.190 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1389999999999999e-05 2024-08-17T17:57:57.191 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.191 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.191 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.191 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.930009+0000", 2024-08-17T17:57:57.191 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5900000000000002e-06 2024-08-17T17:57:57.191 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.191 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.191 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.191 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.966502+0000", 2024-08-17T17:57:57.191 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.036492687000000003 2024-08-17T17:57:57.191 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.191 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.191 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.191 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.969166+0000", 2024-08-17T17:57:57.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026639509999999999 2024-08-17T17:57:57.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.192 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.987461+0000", 2024-08-17T17:57:57.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018295345000000001 2024-08-17T17:57:57.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:57:57.192 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.990188+0000", 2024-08-17T17:57:57.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027273419999999998 2024-08-17T17:57:57.192 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.192 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180087 45.1 45:8d4f01d7:::benchmark_data_smithi130_172551_object75280:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:54.939622+0000", 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.75391903100000002, 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.75510758, 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180087 2024-08-17T17:57:57.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.194 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.939622+0000", 2024-08-17T17:57:57.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.194 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.939622+0000", 2024-08-17T17:57:57.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.194 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.939624+0000", 2024-08-17T17:57:57.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.874e-06 2024-08-17T17:57:57.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.195 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.939630+0000", 2024-08-17T17:57:57.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3740000000000003e-06 2024-08-17T17:57:57.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.195 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.939645+0000", 2024-08-17T17:57:57.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4979e-05 2024-08-17T17:57:57.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.939654+0000", 2024-08-17T17:57:57.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8300000000000002e-06 2024-08-17T17:57:57.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.969459+0000", 2024-08-17T17:57:57.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.029804803000000001 2024-08-17T17:57:57.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:54.972178+0000", 2024-08-17T17:57:57.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027192610000000002 2024-08-17T17:57:57.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.015364+0000", 2024-08-17T17:57:57.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.043185699000000001 2024-08-17T17:57:57.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:57:57.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.017968+0000", 2024-08-17T17:57:57.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002604393 2024-08-17T17:57:57.197 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.197 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.197 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.197 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:180204 45.1 45:841bcb36:::benchmark_data_smithi130_172551_object75397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1243)", 2024-08-17T17:57:57.198 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:57:55.370035+0000", 2024-08-17T17:57:57.198 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.323505975, 2024-08-17T17:57:57.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.324755923, 2024-08-17T17:57:57.198 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:57:57.198 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:57:57.198 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for rw locks", 2024-08-17T17:57:57.198 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:57:57.198 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:57:57.198 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:57:57.198 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 180204 2024-08-17T17:57:57.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.198 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:57:57.198 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.198 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.370035+0000", 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.370035+0000", 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.370036+0000", 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1999999999999999e-07 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.199 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:57:57.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.370039+0000", 2024-08-17T17:57:57.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.304e-06 2024-08-17T17:57:57.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:57:57.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.370049+0000", 2024-08-17T17:57:57.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4180000000000003e-06 2024-08-17T17:57:57.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:57:57.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.370056+0000", 2024-08-17T17:57:57.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2980000000000003e-06 2024-08-17T17:57:57.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.481914+0000", 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.111858016 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.484706+0000", 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027917630000000001 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.516990+0000", 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.032284329 2024-08-17T17:57:57.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:57:57.202 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:57:57.202 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T17:57:57.202 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:57:55.523442+0000", 2024-08-17T17:57:57.202 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.006451615 2024-08-17T17:57:57.202 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.202 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:57:57.202 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.202 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:57:57.202 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T17:57:57.202 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 21 2024-08-17T17:57:57.202 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:57:57.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:57.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:57.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:57.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:57.326 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:57.352 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:57.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:57.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:57.454 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:57.475 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:57.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:57.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:57.576 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:57.600 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:57.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:57.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:57.702 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:57.724 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:57.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:57.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:57.825 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:57.851 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:57.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:57.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:57.953 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:57.974 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:57.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:57.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:58.021 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:57:58.076 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:58.083 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:57:58.106 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:58.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:58.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:58.208 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:58.242 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:58.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:58.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:58.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:58 smithi130 ceph-mon[110068]: pgmap v1944: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 262 op/s; 15837/89070 objects degraded (17.780%); 4149/89070 objects misplaced (4.658%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:57:58.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:58.303 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:57:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:58.344 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:58.379 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:58.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:58.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:58.480 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:58.504 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:58.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:58.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:58 smithi179 ceph-mon[93519]: pgmap v1944: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 262 op/s; 15837/89070 objects degraded (17.780%); 4149/89070 objects misplaced (4.658%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:57:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:57:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:58.605 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:58.628 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:58 smithi130 ceph-mon[105521]: pgmap v1944: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 262 op/s; 15837/89070 objects degraded (17.780%); 4149/89070 objects misplaced (4.658%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:57:58.628 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:57:58.628 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:57:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:57:58.629 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:58.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:58.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:58.731 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:57:58.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:58.759 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:58.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:57:58.860 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:57:58.883 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:58.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:58.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:57:58.985 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:58.994 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:57:59.013 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:59.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:59.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:59.114 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:59.136 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:59.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:59.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:59.238 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:59.268 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:59.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:59.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:59.370 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:57:59.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:59.392 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:59.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:57:59.493 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:57:59.514 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:59.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:59.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:57:59.616 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:57:59.637 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:59.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:59.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:57:59.739 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:57:59.760 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:57:59.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:59.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:57:59.862 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:57:59.892 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:57:59.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:57:59.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:57:59.994 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:00.016 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:00.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:00.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:00.118 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:00.138 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:00.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:00.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:00.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:57:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:57:59] "GET /metrics HTTP/1.1" 200 36808 "" "Prometheus/2.51.0" 2024-08-17T17:58:00.240 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:00.263 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:00.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:00.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:00.313 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T17:58:00.365 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:00.387 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:00.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:00.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:00.489 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:00.515 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:00.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:00.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:00 smithi179 ceph-mon[93519]: pgmap v1945: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 284 op/s; 15837/86868 objects degraded (18.231%); 4149/86868 objects misplaced (4.776%); 890 KiB/s, 13 objects/s recovering 2024-08-17T17:58:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:00 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3080159314' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:00 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:58:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:58:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:58:00.617 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:00.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:00.639 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:00.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:00.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:00 smithi130 ceph-mon[105521]: pgmap v1945: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 284 op/s; 15837/86868 objects degraded (18.231%); 4149/86868 objects misplaced (4.776%); 890 KiB/s, 13 objects/s recovering 2024-08-17T17:58:00.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:00 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3080159314' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:00.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:00 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:00 smithi130 ceph-mon[110068]: pgmap v1945: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 284 op/s; 15837/86868 objects degraded (18.231%); 4149/86868 objects misplaced (4.776%); 890 KiB/s, 13 objects/s recovering 2024-08-17T17:58:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:00 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3080159314' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:00 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:00.739 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:00.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:00.761 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:00.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:00.862 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:00.884 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:00.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:00.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:00.986 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:01.008 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:01.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:01.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:01.110 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:01.131 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:01.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:01.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:01.232 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:01.255 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:01.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:01.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:01.357 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:01.392 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:01.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:01.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:01.494 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:01.523 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:01.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:01.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:01.527 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:01 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:58:01.527 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:01 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:58:01.527 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:01 smithi130 ceph-mon[105521]: osdmap e1244: 8 total, 8 up, 6 in 2024-08-17T17:58:01.527 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/86868 objects degraded (18.231%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:01.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:01 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:58:01.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:01 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:58:01.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:01 smithi130 ceph-mon[110068]: osdmap e1244: 8 total, 8 up, 6 in 2024-08-17T17:58:01.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/86868 objects degraded (18.231%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:01 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:58:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:01 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:58:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:01 smithi179 ceph-mon[93519]: osdmap e1244: 8 total, 8 up, 6 in 2024-08-17T17:58:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/86868 objects degraded (18.231%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:01.624 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:01.643 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:01.646 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:01.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:01.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:01.747 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:01.769 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:01.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:01.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:01.871 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:01.896 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:01.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:01.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:01.902 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:58:01.903 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:58:01.999 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:02.020 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:02.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:02.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:02.122 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:02.142 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:02.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:02.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:02.245 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:02.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:02.267 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:02.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:02.368 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:02.389 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:02.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:02.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:02.449 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:02 smithi130 ceph-mon[105521]: pgmap v1947: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 254 op/s; 15837/86301 objects degraded (18.351%); 4149/86301 objects misplaced (4.808%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:58:02.449 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:02 smithi130 ceph-mon[105521]: from='client.36111 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:02.449 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:02 smithi130 ceph-mon[110068]: pgmap v1947: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 254 op/s; 15837/86301 objects degraded (18.351%); 4149/86301 objects misplaced (4.808%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:58:02.449 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:02 smithi130 ceph-mon[110068]: from='client.36111 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:02.491 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:02.512 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:02.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:02.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:02 smithi179 ceph-mon[93519]: pgmap v1947: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.8 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 254 op/s; 15837/86301 objects degraded (18.351%); 4149/86301 objects misplaced (4.808%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:58:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:02 smithi179 ceph-mon[93519]: from='client.36111 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:02.614 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:02.636 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:02.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:02.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:02.662 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:02.737 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:02.759 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:02.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:02.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:02.861 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:02.882 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:02.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:02.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:02.984 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:03.008 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:03.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:03.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:03.110 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:03.131 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:03.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:03.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:03.219 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1947,"stamp":"2024-08-17T17:58:01.113128+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1884100581,"num_objects":28767,"num_object_clones":0,"num_object_copies":86301,"num_objects_missing_on_primary":1223,"num_objects_missing":1223,"num_objects_degraded":15837,"num_objects_misplaced":4149,"num_objects_unfound":0,"num_objects_dirty":28767,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":181147,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37940,"num_bytes_recovered":2307857421,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108750,"ondisk_log_size":108750,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6676484,"kb_used_data":5180692,"kb_used_omap":295,"kb_used_meta":1495448,"kb_avail":555794428,"statfs":{"total":575970213888,"available":569133494272,"internally_reserved":0,"allocated":5305028608,"data_stored":5274816377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1531339215},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,0,4,2,0,0,4,4,6],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-166526976,"num_objects":-2541,"num_object_clones":0,"num_object_copies":-7623,"num_objects_missing_on_primary":-162,"num_objects_missing":-162,"num_objects_degraded":0,"num_objects_misplaced":-499,"num_objects_unfound":0,"num_objects_dirty":-2541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2541,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":164,"num_bytes_recovered":10747904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002593"},"pg_stats":[{"pgid":"45.f","version":"1243'11329","reported_seq":21896,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:56.524744+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:57:56.524744+0000","last_peered":"2024-08-17T17:57:56.524744+0000","last_clean":"2024-08-17T17:57:56.524744+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:57:56.524744+0000","last_undegraded":"2024-08-17T17:57:56.524744+0000","last_fullsized":"2024-08-17T17:57:56.524744+0000","mapping_epoch":1194,"log_start":"1175'4431","ondisk_log_start":"1175'4431","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6898,"log_dups_size":0,"ondisk_log_size":6898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120258560,"num_objects":1835,"num_object_clones":0,"num_object_copies":5505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11329,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1243'11212","reported_seq":21695,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:57.877553+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:57:57.877553+0000","last_peered":"2024-08-17T17:57:57.877553+0000","last_clean":"2024-08-17T17:57:57.877553+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:57:57.877553+0000","last_undegraded":"2024-08-17T17:57:57.877553+0000","last_fullsized":"2024-08-17T17:57:57.877553+0000","mapping_epoch":1211,"log_start":"1175'4232","ondisk_log_start":"1175'4232","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6980,"log_dups_size":0,"ondisk_log_size":6980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116785152,"num_objects":1782,"num_object_clones":0,"num_object_copies":5346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11212,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1243'11596","reported_seq":22470,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:58.742654+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:57:58.742654+0000","last_peered":"2024-08-17T17:57:58.742654+0000","last_clean":"2024-08-17T17:57:58.742654+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:57:58.742654+0000","last_undegraded":"2024-08-17T17:57:58.742654+0000","last_fullsized":"2024-08-17T17:57:58.742654+0000","mapping_epoch":1205,"log_start":"1180'5668","ondisk_log_start":"1180'5668","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5928,"log_dups_size":0,"ondisk_log_size":5928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116719640,"num_objects":1782,"num_object_clones":0,"num_object_copies":5346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1782,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11596,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1243'11222","reported_seq":11306,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:57.932893+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:57:57.932893+0000","last_peered":"2024-08-17T17:57:57.932893+0000","last_clean":"2024-08-17T17:57:57.932893+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:57:57.932893+0000","last_undegraded":"2024-08-17T17:57:57.932893+0000","last_fullsized":"2024-08-17T17:57:57.932893+0000","mapping_epoch":1167,"log_start":"1175'4489","ondisk_log_start":"1175'4489","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6733,"log_dups_size":0,"ondisk_log_size":6733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116523008,"num_objects":1778,"num_object_clones":0,"num_object_copies":5334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11222,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1243'11324","reported_seq":16604,"reported_epoch":1243,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:56.536017+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:57:56.536017+0000","last_peered":"2024-08-17T17:57:56.536017+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:57:56.536017+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1180'5426","ondisk_log_start":"1180'5426","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5898,"log_dups_size":0,"ondisk_log_size":5898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117309440,"num_objects":1790,"num_object_clones":0,"num_object_copies":5370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":1790,"num_objects_unfound":0,"num_objects_dirty":1790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11324,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":1272}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1243'11319","reported_seq":11403,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:57.914572+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:57:57.914572+0000","last_peered":"2024-08-17T17:57:57.914572+0000","last_clean":"2024-08-17T17:57:57.914572+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:57:57.914572+0000","last_undegraded":"2024-08-17T17:57:57.914572+0000","last_fullsized":"2024-08-17T17:57:57.914572+0000","mapping_epoch":1167,"log_start":"1175'4577","ondisk_log_start":"1175'4577","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6742,"log_dups_size":0,"ondisk_log_size":6742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116326400,"num_objects":1775,"num_object_clones":0,"num_object_copies":5325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11319,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1243'11394","reported_seq":11478,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:57.738852+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:57:57.738852+0000","last_peered":"2024-08-17T17:57:57.738852+0000","last_clean":"2024-08-17T17:57:57.738852+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:57:57.738852+0000","last_undegraded":"2024-08-17T17:57:57.738852+0000","last_fullsized":"2024-08-17T17:57:57.738852+0000","mapping_epoch":1167,"log_start":"1175'4669","ondisk_log_start":"1175'4669","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6725,"log_dups_size":0,"ondisk_log_size":6725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120061952,"num_objects":1832,"num_object_clones":0,"num_object_copies":5496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11394,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1243'11214","reported_seq":20711,"reported_epoch":1243,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:56.819881+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:57:56.819881+0000","last_peered":"2024-08-17T17:57:56.819881+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:57:56.819881+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1180'5275","ondisk_log_start":"1180'5275","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5939,"log_dups_size":0,"ondisk_log_size":5939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116391936,"num_objects":1776,"num_object_clones":0,"num_object_copies":5328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11214,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3108,"num_bytes_recovered":203685888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":456},{"shards":"1,7","objects":-291}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1243'11257","reported_seq":11351,"reported_epoch":1243,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:57.731008+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:57:57.731008+0000","last_peered":"2024-08-17T17:57:57.731008+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:57:57.731008+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1175'4577","ondisk_log_start":"1175'4577","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6680,"log_dups_size":0,"ondisk_log_size":6680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119209984,"num_objects":1819,"num_object_clones":0,"num_object_copies":5457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11257,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":1819},{"shards":"1,6","objects":-4014}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1243'11354","reported_seq":14467,"reported_epoch":1243,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:58.746434+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:57:58.746434+0000","last_peered":"2024-08-17T17:57:58.746434+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:57:58.746434+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1180'5406","ondisk_log_start":"1180'5406","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5948,"log_dups_size":0,"ondisk_log_size":5948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119275520,"num_objects":1820,"num_object_clones":0,"num_object_copies":5460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11354,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":259}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1243'11248","reported_seq":11417,"reported_epoch":1243,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:57:59.131828+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:57:59.131828+0000","last_peered":"2024-08-17T17:57:59.131828+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:57:59.131828+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1170'2100","ondisk_log_start":"1170'2100","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9148,"log_dups_size":0,"ondisk_log_size":9148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119668736,"num_objects":1826,"num_object_clones":0,"num_object_copies":5478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11248,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-3327}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1243'11324","reported_seq":17553,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:59.018418+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:57:59.018418+0000","last_peered":"2024-08-17T17:57:59.018418+0000","last_clean":"2024-08-17T17:57:59.018418+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:57:59.018418+0000","last_undegraded":"2024-08-17T17:57:59.018418+0000","last_fullsized":"2024-08-17T17:57:59.018418+0000","mapping_epoch":1183,"log_start":"1170'2200","ondisk_log_start":"1170'2200","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9124,"log_dups_size":0,"ondisk_log_size":9124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112852992,"num_objects":1722,"num_object_clones":0,"num_object_copies":5166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1722,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11324,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1243'11333","reported_seq":11500,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:58.765190+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:57:58.765190+0000","last_peered":"2024-08-17T17:57:58.765190+0000","last_clean":"2024-08-17T17:57:58.765190+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:57:58.765190+0000","last_undegraded":"2024-08-17T17:57:58.765190+0000","last_fullsized":"2024-08-17T17:57:58.765190+0000","mapping_epoch":1167,"log_start":"1180'5391","ondisk_log_start":"1180'5391","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5942,"log_dups_size":0,"ondisk_log_size":5942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119734272,"num_objects":1827,"num_object_clones":0,"num_object_copies":5481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11333,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1243'11182","reported_seq":11266,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:57.798418+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:57:57.798418+0000","last_peered":"2024-08-17T17:57:57.798418+0000","last_clean":"2024-08-17T17:57:57.798418+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:57:57.798418+0000","last_undegraded":"2024-08-17T17:57:57.798418+0000","last_fullsized":"2024-08-17T17:57:57.798418+0000","mapping_epoch":1167,"log_start":"1175'4125","ondisk_log_start":"1175'4125","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7057,"log_dups_size":0,"ondisk_log_size":7057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114819072,"num_objects":1752,"num_object_clones":0,"num_object_copies":5256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11182,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1243'11501","reported_seq":29888,"reported_epoch":1243,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:57:56.783917+0000","last_change":"2024-08-17T17:55:37.941989+0000","last_active":"2024-08-17T17:57:56.783917+0000","last_peered":"2024-08-17T17:57:56.783917+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:57:56.783917+0000","last_undegraded":"2024-08-17T17:57:56.783917+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1175'4591","ondisk_log_start":"1175'4591","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6910,"log_dups_size":0,"ondisk_log_size":6910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122093568,"num_objects":1863,"num_object_clones":0,"num_object_copies":5589,"num_objects_missing_on_primary":1223,"num_objects_missing":1223,"num_objects_degraded":0,"num_objects_misplaced":2359,"num_objects_unfound":0,"num_objects_dirty":1863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11501,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6925,"num_bytes_recovered":453836800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":11},{"shards":"0,4,6","objects":1356},{"shards":"4,5,6","objects":496}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1243'11060","reported_seq":24414,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:58.759735+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:57:58.759735+0000","last_peered":"2024-08-17T17:57:58.759735+0000","last_clean":"2024-08-17T17:57:58.759735+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:57:58.759735+0000","last_undegraded":"2024-08-17T17:57:58.759735+0000","last_fullsized":"2024-08-17T17:57:58.759735+0000","mapping_epoch":1196,"log_start":"1180'5169","ondisk_log_start":"1180'5169","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5891,"log_dups_size":0,"ondisk_log_size":5891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115474432,"num_objects":1762,"num_object_clones":0,"num_object_copies":5286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11060,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1944,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.329207+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:57:47.329207+0000","last_peered":"2024-08-17T17:57:47.329207+0000","last_clean":"2024-08-17T17:57:47.329207+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:57:47.329207+0000","last_undegraded":"2024-08-17T17:57:47.329207+0000","last_fullsized":"2024-08-17T17:57:47.329207+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":520,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.235124+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:57:39.235124+0000","last_peered":"2024-08-17T17:57:39.235124+0000","last_clean":"2024-08-17T17:57:39.235124+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:57:39.235124+0000","last_undegraded":"2024-08-17T17:57:39.235124+0000","last_fullsized":"2024-08-17T17:57:39.235124+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1864,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.335767+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:57:47.335767+0000","last_peered":"2024-08-17T17:57:47.335767+0000","last_clean":"2024-08-17T17:57:47.335767+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:57:47.335767+0000","last_undegraded":"2024-08-17T17:57:47.335767+0000","last_fullsized":"2024-08-17T17:57:47.335767+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1887,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.334670+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:57:47.334670+0000","last_peered":"2024-08-17T17:57:47.334670+0000","last_clean":"2024-08-17T17:57:47.334670+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:57:47.334670+0000","last_undegraded":"2024-08-17T17:57:47.334670+0000","last_fullsized":"2024-08-17T17:57:47.334670+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":82,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.235625+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:57:39.235625+0000","last_peered":"2024-08-17T17:57:39.235625+0000","last_clean":"2024-08-17T17:57:39.235625+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:57:39.235625+0000","last_undegraded":"2024-08-17T17:57:39.235625+0000","last_fullsized":"2024-08-17T17:57:39.235625+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1281,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.335009+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:57:47.335009+0000","last_peered":"2024-08-17T17:57:47.335009+0000","last_clean":"2024-08-17T17:57:47.335009+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:57:47.335009+0000","last_undegraded":"2024-08-17T17:57:47.335009+0000","last_fullsized":"2024-08-17T17:57:47.335009+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":139,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.325367+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:57:47.325367+0000","last_peered":"2024-08-17T17:57:47.325367+0000","last_clean":"2024-08-17T17:57:47.325367+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:57:47.325367+0000","last_undegraded":"2024-08-17T17:57:47.325367+0000","last_fullsized":"2024-08-17T17:57:47.325367+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":659,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.335201+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:57:47.335201+0000","last_peered":"2024-08-17T17:57:47.335201+0000","last_clean":"2024-08-17T17:57:47.335201+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:57:47.335201+0000","last_undegraded":"2024-08-17T17:57:47.335201+0000","last_fullsized":"2024-08-17T17:57:47.335201+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1902,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.328931+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:57:47.328931+0000","last_peered":"2024-08-17T17:57:47.328931+0000","last_clean":"2024-08-17T17:57:47.328931+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:57:47.328931+0000","last_undegraded":"2024-08-17T17:57:47.328931+0000","last_fullsized":"2024-08-17T17:57:47.328931+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1904,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.339423+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:57:47.339423+0000","last_peered":"2024-08-17T17:57:47.339423+0000","last_clean":"2024-08-17T17:57:47.339423+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:57:47.339423+0000","last_undegraded":"2024-08-17T17:57:47.339423+0000","last_fullsized":"2024-08-17T17:57:47.339423+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":124,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.324246+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:57:47.324246+0000","last_peered":"2024-08-17T17:57:47.324246+0000","last_clean":"2024-08-17T17:57:47.324246+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:57:47.324246+0000","last_undegraded":"2024-08-17T17:57:47.324246+0000","last_fullsized":"2024-08-17T17:57:47.324246+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1876,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.335403+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:57:47.335403+0000","last_peered":"2024-08-17T17:57:47.335403+0000","last_clean":"2024-08-17T17:57:47.335403+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:57:47.335403+0000","last_undegraded":"2024-08-17T17:57:47.335403+0000","last_fullsized":"2024-08-17T17:57:47.335403+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1931,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.323823+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:57:47.323823+0000","last_peered":"2024-08-17T17:57:47.323823+0000","last_clean":"2024-08-17T17:57:47.323823+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:57:47.323823+0000","last_undegraded":"2024-08-17T17:57:47.323823+0000","last_fullsized":"2024-08-17T17:57:47.323823+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1925,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.329230+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:57:47.329230+0000","last_peered":"2024-08-17T17:57:47.329230+0000","last_clean":"2024-08-17T17:57:47.329230+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:57:47.329230+0000","last_undegraded":"2024-08-17T17:57:47.329230+0000","last_fullsized":"2024-08-17T17:57:47.329230+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1898,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.334848+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:57:47.334848+0000","last_peered":"2024-08-17T17:57:47.334848+0000","last_clean":"2024-08-17T17:57:47.334848+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:57:47.334848+0000","last_undegraded":"2024-08-17T17:57:47.334848+0000","last_fullsized":"2024-08-17T17:57:47.334848+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1217,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.235744+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:57:39.235744+0000","last_peered":"2024-08-17T17:57:39.235744+0000","last_clean":"2024-08-17T17:57:39.235744+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:57:39.235744+0000","last_undegraded":"2024-08-17T17:57:39.235744+0000","last_fullsized":"2024-08-17T17:57:39.235744+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1870,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.334690+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:57:47.334690+0000","last_peered":"2024-08-17T17:57:47.334690+0000","last_clean":"2024-08-17T17:57:47.334690+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:57:47.334690+0000","last_undegraded":"2024-08-17T17:57:47.334690+0000","last_fullsized":"2024-08-17T17:57:47.334690+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1919,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.329315+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:57:47.329315+0000","last_peered":"2024-08-17T17:57:47.329315+0000","last_clean":"2024-08-17T17:57:47.329315+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:57:47.329315+0000","last_undegraded":"2024-08-17T17:57:47.329315+0000","last_fullsized":"2024-08-17T17:57:47.329315+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1826,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.334597+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:57:47.334597+0000","last_peered":"2024-08-17T17:57:47.334597+0000","last_clean":"2024-08-17T17:57:47.334597+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:57:47.334597+0000","last_undegraded":"2024-08-17T17:57:47.334597+0000","last_fullsized":"2024-08-17T17:57:47.334597+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":536,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.178766+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:57:39.178766+0000","last_peered":"2024-08-17T17:57:39.178766+0000","last_clean":"2024-08-17T17:57:39.178766+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:57:39.178766+0000","last_undegraded":"2024-08-17T17:57:39.178766+0000","last_fullsized":"2024-08-17T17:57:39.178766+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1213,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.177621+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:57:39.177621+0000","last_peered":"2024-08-17T17:57:39.177621+0000","last_clean":"2024-08-17T17:57:39.177621+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:57:39.177621+0000","last_undegraded":"2024-08-17T17:57:39.177621+0000","last_fullsized":"2024-08-17T17:57:39.177621+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1924,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.324485+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:57:47.324485+0000","last_peered":"2024-08-17T17:57:47.324485+0000","last_clean":"2024-08-17T17:57:47.324485+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:57:47.324485+0000","last_undegraded":"2024-08-17T17:57:47.324485+0000","last_fullsized":"2024-08-17T17:57:47.324485+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":374,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.177840+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:57:39.177840+0000","last_peered":"2024-08-17T17:57:39.177840+0000","last_clean":"2024-08-17T17:57:39.177840+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:57:39.177840+0000","last_undegraded":"2024-08-17T17:57:39.177840+0000","last_fullsized":"2024-08-17T17:57:39.177840+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":520,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.235463+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:57:39.235463+0000","last_peered":"2024-08-17T17:57:39.235463+0000","last_clean":"2024-08-17T17:57:39.235463+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:57:39.235463+0000","last_undegraded":"2024-08-17T17:57:39.235463+0000","last_fullsized":"2024-08-17T17:57:39.235463+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1198,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.237428+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:57:39.237428+0000","last_peered":"2024-08-17T17:57:39.237428+0000","last_clean":"2024-08-17T17:57:39.237428+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:57:39.237428+0000","last_undegraded":"2024-08-17T17:57:39.237428+0000","last_fullsized":"2024-08-17T17:57:39.237428+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1206,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.235790+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:57:39.235790+0000","last_peered":"2024-08-17T17:57:39.235790+0000","last_clean":"2024-08-17T17:57:39.235790+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:57:39.235790+0000","last_undegraded":"2024-08-17T17:57:39.235790+0000","last_fullsized":"2024-08-17T17:57:39.235790+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1972,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.329049+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:57:47.329049+0000","last_peered":"2024-08-17T17:57:47.329049+0000","last_clean":"2024-08-17T17:57:47.329049+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:57:47.329049+0000","last_undegraded":"2024-08-17T17:57:47.329049+0000","last_fullsized":"2024-08-17T17:57:47.329049+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1856,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.335098+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:57:47.335098+0000","last_peered":"2024-08-17T17:57:47.335098+0000","last_clean":"2024-08-17T17:57:47.335098+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:57:47.335098+0000","last_undegraded":"2024-08-17T17:57:47.335098+0000","last_fullsized":"2024-08-17T17:57:47.335098+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1930,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.324488+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:57:47.324488+0000","last_peered":"2024-08-17T17:57:47.324488+0000","last_clean":"2024-08-17T17:57:47.324488+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:57:47.324488+0000","last_undegraded":"2024-08-17T17:57:47.324488+0000","last_fullsized":"2024-08-17T17:57:47.324488+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1858,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.335590+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:57:47.335590+0000","last_peered":"2024-08-17T17:57:47.335590+0000","last_clean":"2024-08-17T17:57:47.335590+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:57:47.335590+0000","last_undegraded":"2024-08-17T17:57:47.335590+0000","last_fullsized":"2024-08-17T17:57:47.335590+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1890,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.328972+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:57:47.328972+0000","last_peered":"2024-08-17T17:57:47.328972+0000","last_clean":"2024-08-17T17:57:47.328972+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:57:47.328972+0000","last_undegraded":"2024-08-17T17:57:47.328972+0000","last_fullsized":"2024-08-17T17:57:47.328972+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1233,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.177834+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:57:39.177834+0000","last_peered":"2024-08-17T17:57:39.177834+0000","last_clean":"2024-08-17T17:57:39.177834+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:57:39.177834+0000","last_undegraded":"2024-08-17T17:57:39.177834+0000","last_fullsized":"2024-08-17T17:57:39.177834+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1937,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.325462+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:57:47.325462+0000","last_peered":"2024-08-17T17:57:47.325462+0000","last_clean":"2024-08-17T17:57:47.325462+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:57:47.325462+0000","last_undegraded":"2024-08-17T17:57:47.325462+0000","last_fullsized":"2024-08-17T17:57:47.325462+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":123,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.324095+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:57:47.324095+0000","last_peered":"2024-08-17T17:57:47.324095+0000","last_clean":"2024-08-17T17:57:47.324095+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:57:47.324095+0000","last_undegraded":"2024-08-17T17:57:47.324095+0000","last_fullsized":"2024-08-17T17:57:47.324095+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1956,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.325595+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:57:47.325595+0000","last_peered":"2024-08-17T17:57:47.325595+0000","last_clean":"2024-08-17T17:57:47.325595+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:57:47.325595+0000","last_undegraded":"2024-08-17T17:57:47.325595+0000","last_fullsized":"2024-08-17T17:57:47.325595+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":692,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.335240+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:57:47.335240+0000","last_peered":"2024-08-17T17:57:47.335240+0000","last_clean":"2024-08-17T17:57:47.335240+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:57:47.335240+0000","last_undegraded":"2024-08-17T17:57:47.335240+0000","last_fullsized":"2024-08-17T17:57:47.335240+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1937,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.325540+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:57:47.325540+0000","last_peered":"2024-08-17T17:57:47.325540+0000","last_clean":"2024-08-17T17:57:47.325540+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:57:47.325540+0000","last_undegraded":"2024-08-17T17:57:47.325540+0000","last_fullsized":"2024-08-17T17:57:47.325540+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1265,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.328811+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:57:47.328811+0000","last_peered":"2024-08-17T17:57:47.328811+0000","last_clean":"2024-08-17T17:57:47.328811+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:57:47.328811+0000","last_undegraded":"2024-08-17T17:57:47.328811+0000","last_fullsized":"2024-08-17T17:57:47.328811+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":183,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.324399+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:57:47.324399+0000","last_peered":"2024-08-17T17:57:47.324399+0000","last_clean":"2024-08-17T17:57:47.324399+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:57:47.324399+0000","last_undegraded":"2024-08-17T17:57:47.324399+0000","last_fullsized":"2024-08-17T17:57:47.324399+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1947,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.324774+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:57:47.324774+0000","last_peered":"2024-08-17T17:57:47.324774+0000","last_clean":"2024-08-17T17:57:47.324774+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:57:47.324774+0000","last_undegraded":"2024-08-17T17:57:47.324774+0000","last_fullsized":"2024-08-17T17:57:47.324774+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1925,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.324049+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:57:47.324049+0000","last_peered":"2024-08-17T17:57:47.324049+0000","last_clean":"2024-08-17T17:57:47.324049+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:57:47.324049+0000","last_undegraded":"2024-08-17T17:57:47.324049+0000","last_fullsized":"2024-08-17T17:57:47.324049+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1945,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.339100+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:57:47.339100+0000","last_peered":"2024-08-17T17:57:47.339100+0000","last_clean":"2024-08-17T17:57:47.339100+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:57:47.339100+0000","last_undegraded":"2024-08-17T17:57:47.339100+0000","last_fullsized":"2024-08-17T17:57:47.339100+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1951,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.329247+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:57:47.329247+0000","last_peered":"2024-08-17T17:57:47.329247+0000","last_clean":"2024-08-17T17:57:47.329247+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:57:47.329247+0000","last_undegraded":"2024-08-17T17:57:47.329247+0000","last_fullsized":"2024-08-17T17:57:47.329247+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1931,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.332697+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:57:47.332697+0000","last_peered":"2024-08-17T17:57:47.332697+0000","last_clean":"2024-08-17T17:57:47.332697+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:57:47.332697+0000","last_undegraded":"2024-08-17T17:57:47.332697+0000","last_fullsized":"2024-08-17T17:57:47.332697+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1240,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.177474+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:57:39.177474+0000","last_peered":"2024-08-17T17:57:39.177474+0000","last_clean":"2024-08-17T17:57:39.177474+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:57:39.177474+0000","last_undegraded":"2024-08-17T17:57:39.177474+0000","last_fullsized":"2024-08-17T17:57:39.177474+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1186,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.235693+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:57:39.235693+0000","last_peered":"2024-08-17T17:57:39.235693+0000","last_clean":"2024-08-17T17:57:39.235693+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:57:39.235693+0000","last_undegraded":"2024-08-17T17:57:39.235693+0000","last_fullsized":"2024-08-17T17:57:39.235693+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1840,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.334919+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:57:47.334919+0000","last_peered":"2024-08-17T17:57:47.334919+0000","last_clean":"2024-08-17T17:57:47.334919+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:57:47.334919+0000","last_undegraded":"2024-08-17T17:57:47.334919+0000","last_fullsized":"2024-08-17T17:57:47.334919+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":171,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.339214+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:57:47.339214+0000","last_peered":"2024-08-17T17:57:47.339214+0000","last_clean":"2024-08-17T17:57:47.339214+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:57:47.339214+0000","last_undegraded":"2024-08-17T17:57:47.339214+0000","last_fullsized":"2024-08-17T17:57:47.339214+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1552,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.177370+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:57:39.177370+0000","last_peered":"2024-08-17T17:57:39.177370+0000","last_clean":"2024-08-17T17:57:39.177370+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:57:39.177370+0000","last_undegraded":"2024-08-17T17:57:39.177370+0000","last_fullsized":"2024-08-17T17:57:39.177370+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1932,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.324916+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:57:47.324916+0000","last_peered":"2024-08-17T17:57:47.324916+0000","last_clean":"2024-08-17T17:57:47.324916+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:57:47.324916+0000","last_undegraded":"2024-08-17T17:57:47.324916+0000","last_fullsized":"2024-08-17T17:57:47.324916+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1872,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.334789+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:57:47.334789+0000","last_peered":"2024-08-17T17:57:47.334789+0000","last_clean":"2024-08-17T17:57:47.334789+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:57:47.334789+0000","last_undegraded":"2024-08-17T17:57:47.334789+0000","last_fullsized":"2024-08-17T17:57:47.334789+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":520,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.235666+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:57:39.235666+0000","last_peered":"2024-08-17T17:57:39.235666+0000","last_clean":"2024-08-17T17:57:39.235666+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:57:39.235666+0000","last_undegraded":"2024-08-17T17:57:39.235666+0000","last_fullsized":"2024-08-17T17:57:39.235666+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":610,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.339023+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:57:47.339023+0000","last_peered":"2024-08-17T17:57:47.339023+0000","last_clean":"2024-08-17T17:57:47.339023+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:57:47.339023+0000","last_undegraded":"2024-08-17T17:57:47.339023+0000","last_fullsized":"2024-08-17T17:57:47.339023+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":626,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.339595+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:57:47.339595+0000","last_peered":"2024-08-17T17:57:47.339595+0000","last_clean":"2024-08-17T17:57:47.339595+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:57:47.339595+0000","last_undegraded":"2024-08-17T17:57:47.339595+0000","last_fullsized":"2024-08-17T17:57:47.339595+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1204,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.235446+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:57:39.235446+0000","last_peered":"2024-08-17T17:57:39.235446+0000","last_clean":"2024-08-17T17:57:39.235446+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:57:39.235446+0000","last_undegraded":"2024-08-17T17:57:39.235446+0000","last_fullsized":"2024-08-17T17:57:39.235446+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":642,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.334892+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:57:47.334892+0000","last_peered":"2024-08-17T17:57:47.334892+0000","last_clean":"2024-08-17T17:57:47.334892+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:57:47.334892+0000","last_undegraded":"2024-08-17T17:57:47.334892+0000","last_fullsized":"2024-08-17T17:57:47.334892+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1868,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.335283+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:57:47.335283+0000","last_peered":"2024-08-17T17:57:47.335283+0000","last_clean":"2024-08-17T17:57:47.335283+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:57:47.335283+0000","last_undegraded":"2024-08-17T17:57:47.335283+0000","last_fullsized":"2024-08-17T17:57:47.335283+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1964,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.329043+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:57:47.329043+0000","last_peered":"2024-08-17T17:57:47.329043+0000","last_clean":"2024-08-17T17:57:47.329043+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:57:47.329043+0000","last_undegraded":"2024-08-17T17:57:47.329043+0000","last_fullsized":"2024-08-17T17:57:47.329043+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":675,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.334835+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:57:47.334835+0000","last_peered":"2024-08-17T17:57:47.334835+0000","last_clean":"2024-08-17T17:57:47.334835+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:57:47.334835+0000","last_undegraded":"2024-08-17T17:57:47.334835+0000","last_fullsized":"2024-08-17T17:57:47.334835+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1193,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.235327+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:57:39.235327+0000","last_peered":"2024-08-17T17:57:39.235327+0000","last_clean":"2024-08-17T17:57:39.235327+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:57:39.235327+0000","last_undegraded":"2024-08-17T17:57:39.235327+0000","last_fullsized":"2024-08-17T17:57:39.235327+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1192,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.177768+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:57:39.177768+0000","last_peered":"2024-08-17T17:57:39.177768+0000","last_clean":"2024-08-17T17:57:39.177768+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:57:39.177768+0000","last_undegraded":"2024-08-17T17:57:39.177768+0000","last_fullsized":"2024-08-17T17:57:39.177768+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":659,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.323930+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:57:47.323930+0000","last_peered":"2024-08-17T17:57:47.323930+0000","last_clean":"2024-08-17T17:57:47.323930+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:57:47.323930+0000","last_undegraded":"2024-08-17T17:57:47.323930+0000","last_fullsized":"2024-08-17T17:57:47.323930+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1210,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.178846+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:57:39.178846+0000","last_peered":"2024-08-17T17:57:39.178846+0000","last_clean":"2024-08-17T17:57:39.178846+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:57:39.178846+0000","last_undegraded":"2024-08-17T17:57:39.178846+0000","last_fullsized":"2024-08-17T17:57:39.178846+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":94,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.177303+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:57:39.177303+0000","last_peered":"2024-08-17T17:57:39.177303+0000","last_clean":"2024-08-17T17:57:39.177303+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:57:39.177303+0000","last_undegraded":"2024-08-17T17:57:39.177303+0000","last_fullsized":"2024-08-17T17:57:39.177303+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1226,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.177936+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:57:39.177936+0000","last_peered":"2024-08-17T17:57:39.177936+0000","last_clean":"2024-08-17T17:57:39.177936+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:57:39.177936+0000","last_undegraded":"2024-08-17T17:57:39.177936+0000","last_fullsized":"2024-08-17T17:57:39.177936+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1243,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.235278+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:57:39.235278+0000","last_peered":"2024-08-17T17:57:39.235278+0000","last_clean":"2024-08-17T17:57:39.235278+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:57:39.235278+0000","last_undegraded":"2024-08-17T17:57:39.235278+0000","last_fullsized":"2024-08-17T17:57:39.235278+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1869,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.335403+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:57:47.335403+0000","last_peered":"2024-08-17T17:57:47.335403+0000","last_clean":"2024-08-17T17:57:47.335403+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:57:47.335403+0000","last_undegraded":"2024-08-17T17:57:47.335403+0000","last_fullsized":"2024-08-17T17:57:47.335403+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1187,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.177548+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:57:39.177548+0000","last_peered":"2024-08-17T17:57:39.177548+0000","last_clean":"2024-08-17T17:57:39.177548+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:57:39.177548+0000","last_undegraded":"2024-08-17T17:57:39.177548+0000","last_fullsized":"2024-08-17T17:57:39.177548+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1922,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.329152+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:57:47.329152+0000","last_peered":"2024-08-17T17:57:47.329152+0000","last_clean":"2024-08-17T17:57:47.329152+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:57:47.329152+0000","last_undegraded":"2024-08-17T17:57:47.329152+0000","last_fullsized":"2024-08-17T17:57:47.329152+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1923,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.328802+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:57:47.328802+0000","last_peered":"2024-08-17T17:57:47.328802+0000","last_clean":"2024-08-17T17:57:47.328802+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:57:47.328802+0000","last_undegraded":"2024-08-17T17:57:47.328802+0000","last_fullsized":"2024-08-17T17:57:47.328802+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":600,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.177537+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:57:39.177537+0000","last_peered":"2024-08-17T17:57:39.177537+0000","last_clean":"2024-08-17T17:57:39.177537+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:57:39.177537+0000","last_undegraded":"2024-08-17T17:57:39.177537+0000","last_fullsized":"2024-08-17T17:57:39.177537+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1212,"reported_epoch":1242,"state":"active+clean","last_fresh":"2024-08-17T17:57:39.177696+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:57:39.177696+0000","last_peered":"2024-08-17T17:57:39.177696+0000","last_clean":"2024-08-17T17:57:39.177696+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:57:39.177696+0000","last_undegraded":"2024-08-17T17:57:39.177696+0000","last_fullsized":"2024-08-17T17:57:39.177696+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1870,"reported_epoch":1243,"state":"active+clean","last_fresh":"2024-08-17T17:57:47.334972+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:57:47.334972+0000","last_peered":"2024-08-17T17:57:47.334972+0000","last_clean":"2024-08-17T17:57:47.334972+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:57:47.334972+0000","last_undegraded":"2024-08-17T17:57:47.334972+0000","last_fullsized":"2024-08-17T17:57:47.334972+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":1883504664,"num_objects":28741,"num_object_clones":0,"num_object_copies":86223,"num_objects_missing_on_primary":1223,"num_objects_missing":1223,"num_objects_degraded":15837,"num_objects_misplaced":4149,"num_objects_unfound":0,"num_objects_dirty":28741,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":180869,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37911,"num_bytes_recovered":2306080768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7177121792,"data_stored":7177109576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108543,"ondisk_log_size":108543,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234445,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1116320,"kb_used_data":814808,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92628832,"statfs":{"total":95995035648,"available":94851923968,"internally_reserved":0,"allocated":834363392,"data_stored":829320211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496080,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1254116,"kb_used_data":944284,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92491036,"statfs":{"total":95995035648,"available":94710820864,"internally_reserved":0,"allocated":966946816,"data_stored":961900040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757718,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326332,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1281868,"kb_used_data":1050948,"kb_used_omap":57,"kb_used_meta":230854,"kb_avail":92463284,"statfs":{"total":95995035648,"available":94682402816,"internally_reserved":0,"allocated":1076170752,"data_stored":1071142751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":236395259},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405571,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1162132,"kb_used_data":921740,"kb_used_omap":61,"kb_used_meta":240322,"kb_avail":92583020,"statfs":{"total":95995035648,"available":94805012480,"internally_reserved":0,"allocated":943861760,"data_stored":938830731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":246090680},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452105,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":881672,"kb_used_data":709632,"kb_used_omap":54,"kb_used_meta":171977,"kb_avail":92863480,"statfs":{"total":95995035648,"available":95092203520,"internally_reserved":0,"allocated":726663168,"data_stored":721639128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":176104846},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474796,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":980376,"kb_used_data":739280,"kb_used_omap":119,"kb_used_meta":240968,"kb_avail":92764776,"statfs":{"total":95995035648,"available":94991130624,"internally_reserved":0,"allocated":757022720,"data_stored":751983516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":246751966},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,4,2,0,0,4,4,6],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019353,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":740429824,"data_stored":740425752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":708837376,"data_stored":708837376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":928645120,"data_stored":928645120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1055789056,"data_stored":1055784984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":889454592,"data_stored":889454592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":946016256,"data_stored":946012184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":820051968,"data_stored":820051968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:58:03.233 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:03.256 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:03.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:03.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:03.358 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:03.381 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:03.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:03.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:03.483 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:03.509 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:03.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:03.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:03.611 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:03.634 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:03.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:03.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:03.736 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:03.761 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:03.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:03.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:03.863 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:03.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:03.885 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:03.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:03.986 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:04.006 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:04.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:04.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:04.108 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:04.130 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:04.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:04.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:04.230 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:58:04.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:04.254 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:04.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:04.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:04.356 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:04.377 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:04.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:04.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:04.479 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:04.500 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:04.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:04.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:04.524 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:04 smithi130 ceph-mon[105521]: pgmap v1948: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 256 op/s; 15837/83595 objects degraded (18.945%); 3892/83595 objects misplaced (4.656%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:58:04.524 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:04 smithi130 ceph-mon[110068]: pgmap v1948: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 256 op/s; 15837/83595 objects degraded (18.945%); 3892/83595 objects misplaced (4.656%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:58:04.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:04 smithi179 ceph-mon[93519]: pgmap v1948: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 256 op/s; 15837/83595 objects degraded (18.945%); 3892/83595 objects misplaced (4.656%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:58:04.602 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:04.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:04.628 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:04.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:04.729 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:04.750 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:04.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:04.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:04.852 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:04.873 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:04.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:04.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:04.974 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:04.996 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:04.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:04.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:05.097 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:05.122 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:05.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:05.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:05.224 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:05.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:05.248 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:05.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:05.349 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:05.370 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:05.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:05.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:05.471 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:05.500 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:05.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:05.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:05.602 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:05.629 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:05.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:05.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:05.731 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:05.752 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:05.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:05.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:05.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:05.875 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:05.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:05.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:05.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:06.002 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:06.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:06.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:06.105 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:06.137 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:06.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:06.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:06.226 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:58:06.239 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:06.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:06.262 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:06.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:06.363 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:06.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:06.385 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:06.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:06.405 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:06.427 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:58:06.427 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:58:06.427 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T17:58:06.427 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T17:58:06.427 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:58:06.486 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:06.508 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:06.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:06.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:06.534 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:06 smithi130 ceph-mon[105521]: pgmap v1949: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 261 op/s; 15837/82071 objects degraded (19.297%); 3892/82071 objects misplaced (4.742%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:58:06.534 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/82071 objects degraded (19.297%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:06.534 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:06 smithi130 ceph-mon[110068]: pgmap v1949: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 261 op/s; 15837/82071 objects degraded (19.297%); 3892/82071 objects misplaced (4.742%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:58:06.534 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/82071 objects degraded (19.297%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:06.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:06 smithi179 ceph-mon[93519]: pgmap v1949: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 261 op/s; 15837/82071 objects degraded (19.297%); 3892/82071 objects misplaced (4.742%); 1.1 MiB/s, 16 objects/s recovering 2024-08-17T17:58:06.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/82071 objects degraded (19.297%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:06.610 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:06.638 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:06.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:06.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:06.740 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:06.762 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:06.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:06.863 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:06.889 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:06.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:06.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:06.913 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:58:06.913 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:58:06.936 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:58:06.936 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:58:06.954 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:58:06.954 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:58:06.970 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:58:06.971 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:58:06.986 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:58:06.986 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:58:06.991 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:07.010 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:58:07.010 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:58:07.013 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:07.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:07.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:07.027 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:58:07.027 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:58:07.043 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:58:07.043 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:58:07.115 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:07.136 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:07.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:07.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:07.238 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:07.260 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:07.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:07.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:07.362 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:07.385 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:07.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:07.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:07.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:07.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:07.518 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:07.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:07.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:07 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2174367854' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:58:07.619 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:07.641 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:07.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:07.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:07.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:07 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2174367854' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:58:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:07 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2174367854' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:58:07.743 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:07.764 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:07.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:07.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:07.866 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:07.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:07.888 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:07.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:07.989 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:08.018 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:08.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:08.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:08.120 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:08.151 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:08.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:08.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:08.253 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:08.275 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:08.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:08.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:08.377 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:08.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:08.399 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:08.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:08.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:08 smithi130 ceph-mon[105521]: pgmap v1950: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 265 op/s; 15837/81102 objects degraded (19.527%); 3625/81102 objects misplaced (4.470%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:58:08.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:08 smithi130 ceph-mon[110068]: pgmap v1950: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 265 op/s; 15837/81102 objects degraded (19.527%); 3625/81102 objects misplaced (4.470%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:58:08.499 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:08.520 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:08.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:08.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:08 smithi179 ceph-mon[93519]: pgmap v1950: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 265 op/s; 15837/81102 objects degraded (19.527%); 3625/81102 objects misplaced (4.470%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:58:08.622 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:08.646 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:08.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:08.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:08.748 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:08.769 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:08.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:08.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:08.870 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:08.901 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:08.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:08.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:09.003 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:09.037 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:09.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:09.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:09.139 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:09.160 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:09.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:09.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:09.262 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:09.283 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:09.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:09.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:09.385 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:09.390 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:58:09.400 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:09.421 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:09.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:09.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:09.524 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:09.545 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:09.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:09.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:09.647 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:09.692 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:09.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:09.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:09.794 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:09.824 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:09.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:09.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:09.870 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:58:09.926 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:09.965 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:09.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:09.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:10.067 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:10.068 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:58:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:58:09] "GET /metrics HTTP/1.1" 200 36808 "" "Prometheus/2.51.0" 2024-08-17T17:58:10.088 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:10.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:10.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:10.190 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:10.212 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:10.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:10.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:10.313 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:10.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:10.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:10.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:10.458 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:10.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:10.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:10.559 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:10.591 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:10.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:10.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:10 smithi179 ceph-mon[93519]: pgmap v1951: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 269 op/s; 15837/78786 objects degraded (20.101%); 3625/78786 objects misplaced (4.601%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:58:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:58:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:58:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:58:10.632 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:10 smithi130 ceph-mon[105521]: pgmap v1951: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 269 op/s; 15837/78786 objects degraded (20.101%); 3625/78786 objects misplaced (4.601%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:58:10.633 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:10 smithi130 ceph-mon[110068]: pgmap v1951: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 269 op/s; 15837/78786 objects degraded (20.101%); 3625/78786 objects misplaced (4.601%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:58:10.693 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:10.720 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:10.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:10.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:10.822 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:10.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:10.844 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:10.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:10.929 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:10.944 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:10.958 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:10.966 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:10.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:10.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:11.068 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:11.089 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:11.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:11.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:11.193 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:11.215 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:11.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:11.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:11.316 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:11.336 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T17:58:11.343 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:11.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:11.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:11.445 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:11.467 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:11.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:11.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:11.568 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:11.590 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:11.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:11.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:11.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/78786 objects degraded (20.101%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:11.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:11 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3608882862' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:58:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:11 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:58:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/78786 objects degraded (20.101%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:11 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3608882862' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:58:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:11 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:58:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/78786 objects degraded (20.101%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:11 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3608882862' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:58:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:11 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:58:11.691 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:11.713 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:11.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:11.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:11.815 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:11.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:11.838 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:11.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:11.939 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:11.960 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:11.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:11.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:12.062 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:12.085 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:12.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:12.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:12.187 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:12.223 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:12.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:12.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:12.325 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:12.348 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:12.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:12.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:12.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:12 smithi130 ceph-mon[105521]: pgmap v1952: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 246 op/s; 15837/78321 objects degraded (20.221%); 3625/78321 objects misplaced (4.628%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:58:12.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:12 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:58:12.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:12 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:58:12.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:12 smithi130 ceph-mon[105521]: osdmap e1245: 8 total, 8 up, 6 in 2024-08-17T17:58:12.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:58:12.389 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:12 smithi130 ceph-mon[110068]: pgmap v1952: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 246 op/s; 15837/78321 objects degraded (20.221%); 3625/78321 objects misplaced (4.628%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:58:12.389 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:12 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:58:12.389 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:12 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:58:12.389 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:12 smithi130 ceph-mon[110068]: osdmap e1245: 8 total, 8 up, 6 in 2024-08-17T17:58:12.389 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:58:12.449 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:12.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:12.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:12.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:12.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:12.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:12 smithi179 ceph-mon[93519]: pgmap v1952: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 246 op/s; 15837/78321 objects degraded (20.221%); 3625/78321 objects misplaced (4.628%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:58:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:12 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:58:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:12 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:58:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:12 smithi179 ceph-mon[93519]: osdmap e1245: 8 total, 8 up, 6 in 2024-08-17T17:58:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:58:12.601 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:12.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:12.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:12.703 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:12.724 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:12.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:12.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:12.826 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:12.851 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:12.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:12.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:12.953 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:12.977 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:12.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:12.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:13.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:13.099 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:13.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:13.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:13.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:13.229 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:13.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:13.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:13.331 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:13.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:13.354 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:13.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:13.455 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:13.480 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:13.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:13.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:13.582 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:13.607 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:13.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:13.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:13.708 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:13.729 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:13.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:13.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:13.831 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:13.852 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:13.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:13.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:13.954 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:13.996 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:13.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:13.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:14.098 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:14.121 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:14.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:14.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:14.157 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:58:14.222 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:14.256 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:14.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:14.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:14.259 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:58:14.260 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:58:14.358 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:14.380 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:14.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:14.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:14.400 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:14 smithi130 ceph-mon[105521]: pgmap v1954: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 262 op/s; 15837/75717 objects degraded (20.916%); 3363/75717 objects misplaced (4.442%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:58:14.401 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:14 smithi130 ceph-mon[110068]: pgmap v1954: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 262 op/s; 15837/75717 objects degraded (20.916%); 3363/75717 objects misplaced (4.442%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:58:14.481 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:14.507 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:14.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:14.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:14 smithi179 ceph-mon[93519]: pgmap v1954: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 262 op/s; 15837/75717 objects degraded (20.916%); 3363/75717 objects misplaced (4.442%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:58:14.609 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:14.632 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:14.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:14.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:14.734 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:14.781 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:14.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:14.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:14.883 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:14.916 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:14.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:14.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:15.017 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:15.045 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:15.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:15.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:15.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:15.169 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:15.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:15.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:15.185 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1954,"stamp":"2024-08-17T17:58:13.117467+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1652889573,"num_objects":25239,"num_object_clones":0,"num_object_copies":75717,"num_objects_missing_on_primary":957,"num_objects_missing":957,"num_objects_degraded":15837,"num_objects_misplaced":3363,"num_objects_unfound":0,"num_objects_dirty":25239,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":184675,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38205,"num_bytes_recovered":2325224461,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108578,"ondisk_log_size":108578,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6105412,"kb_used_data":4604052,"kb_used_omap":295,"kb_used_meta":1501016,"kb_avail":556365500,"statfs":{"total":575970213888,"available":569718272000,"internally_reserved":0,"allocated":4714549248,"data_stored":4684277529,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1537040847},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-172097536,"num_objects":-2626,"num_object_clones":0,"num_object_copies":-7878,"num_objects_missing_on_primary":-180,"num_objects_missing":-180,"num_objects_degraded":0,"num_objects_misplaced":-529,"num_objects_unfound":0,"num_objects_dirty":-2626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2626,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":11862016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003871"},"pg_stats":[{"pgid":"45.f","version":"1245'11575","reported_seq":22146,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.878667+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:58:11.878667+0000","last_peered":"2024-08-17T17:58:11.878667+0000","last_clean":"2024-08-17T17:58:11.878667+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:58:11.878667+0000","last_undegraded":"2024-08-17T17:58:11.878667+0000","last_fullsized":"2024-08-17T17:58:11.878667+0000","mapping_epoch":1194,"log_start":"1175'4731","ondisk_log_start":"1175'4731","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6844,"log_dups_size":0,"ondisk_log_size":6844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104136704,"num_objects":1589,"num_object_clones":0,"num_object_copies":4767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11575,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1245'11487","reported_seq":21972,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:12.837016+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:58:12.837016+0000","last_peered":"2024-08-17T17:58:12.837016+0000","last_clean":"2024-08-17T17:58:12.837016+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:58:12.837016+0000","last_undegraded":"2024-08-17T17:58:12.837016+0000","last_fullsized":"2024-08-17T17:58:12.837016+0000","mapping_epoch":1211,"log_start":"1175'4432","ondisk_log_start":"1175'4432","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98762752,"num_objects":1507,"num_object_clones":0,"num_object_copies":4521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11487,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1244'11763","reported_seq":22639,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:08.624401+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:58:08.624401+0000","last_peered":"2024-08-17T17:58:08.624401+0000","last_clean":"2024-08-17T17:58:08.624401+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:58:08.624401+0000","last_undegraded":"2024-08-17T17:58:08.624401+0000","last_fullsized":"2024-08-17T17:58:08.624401+0000","mapping_epoch":1205,"log_start":"1181'5868","ondisk_log_start":"1181'5868","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5895,"log_dups_size":0,"ondisk_log_size":5895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105775128,"num_objects":1615,"num_object_clones":0,"num_object_copies":4845,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1615,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11763,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1245'11457","reported_seq":11543,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:12.757399+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:58:12.757399+0000","last_peered":"2024-08-17T17:58:12.757399+0000","last_clean":"2024-08-17T17:58:12.757399+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:58:12.757399+0000","last_undegraded":"2024-08-17T17:58:12.757399+0000","last_fullsized":"2024-08-17T17:58:12.757399+0000","mapping_epoch":1167,"log_start":"1175'4789","ondisk_log_start":"1175'4789","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6668,"log_dups_size":0,"ondisk_log_size":6668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101122048,"num_objects":1543,"num_object_clones":0,"num_object_copies":4629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11457,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1245'11579","reported_seq":16863,"reported_epoch":1245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:11.857037+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:58:11.857037+0000","last_peered":"2024-08-17T17:58:11.857037+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:58:11.857037+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1181'5626","ondisk_log_start":"1181'5626","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5953,"log_dups_size":0,"ondisk_log_size":5953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100597760,"num_objects":1535,"num_object_clones":0,"num_object_copies":4605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":1535,"num_objects_unfound":0,"num_objects_dirty":1535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11579,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":1017}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1245'11574","reported_seq":11660,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:12.654628+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:58:12.654628+0000","last_peered":"2024-08-17T17:58:12.654628+0000","last_clean":"2024-08-17T17:58:12.654628+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:58:12.654628+0000","last_undegraded":"2024-08-17T17:58:12.654628+0000","last_fullsized":"2024-08-17T17:58:12.654628+0000","mapping_epoch":1167,"log_start":"1175'4877","ondisk_log_start":"1175'4877","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99614720,"num_objects":1520,"num_object_clones":0,"num_object_copies":4560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11574,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1245'11638","reported_seq":11724,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:12.832688+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:58:12.832688+0000","last_peered":"2024-08-17T17:58:12.832688+0000","last_clean":"2024-08-17T17:58:12.832688+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:58:12.832688+0000","last_undegraded":"2024-08-17T17:58:12.832688+0000","last_fullsized":"2024-08-17T17:58:12.832688+0000","mapping_epoch":1167,"log_start":"1176'4969","ondisk_log_start":"1176'4969","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6669,"log_dups_size":0,"ondisk_log_size":6669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104071168,"num_objects":1588,"num_object_clones":0,"num_object_copies":4764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11638,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1245'11459","reported_seq":20960,"reported_epoch":1245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:11.820389+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:58:11.820389+0000","last_peered":"2024-08-17T17:58:11.820389+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:58:11.820389+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1181'5575","ondisk_log_start":"1181'5575","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5884,"log_dups_size":0,"ondisk_log_size":5884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100335616,"num_objects":1531,"num_object_clones":0,"num_object_copies":4593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11459,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3108,"num_bytes_recovered":203685888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":211},{"shards":"1,7","objects":-536}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1245'11514","reported_seq":11610,"reported_epoch":1245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:12.803128+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:58:12.803128+0000","last_peered":"2024-08-17T17:58:12.803128+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:58:12.803128+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1175'4777","ondisk_log_start":"1175'4777","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6737,"log_dups_size":0,"ondisk_log_size":6737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102367232,"num_objects":1562,"num_object_clones":0,"num_object_copies":4686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11514,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":1562},{"shards":"1,6","objects":-4271}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1244'11531","reported_seq":14646,"reported_epoch":1244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:08.604370+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:58:08.604370+0000","last_peered":"2024-08-17T17:58:08.604370+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:58:08.604370+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1181'5606","ondisk_log_start":"1181'5606","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5925,"log_dups_size":0,"ondisk_log_size":5925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107675648,"num_objects":1643,"num_object_clones":0,"num_object_copies":4929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11531,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":82}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1244'11426","reported_seq":11597,"reported_epoch":1244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:09.085890+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:58:09.085890+0000","last_peered":"2024-08-17T17:58:09.085890+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:58:09.085890+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1170'2300","ondisk_log_start":"1170'2300","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9126,"log_dups_size":0,"ondisk_log_size":9126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108003328,"num_objects":1648,"num_object_clones":0,"num_object_copies":4944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11426,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-3505}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1244'11467","reported_seq":17699,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:09.164077+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:58:09.164077+0000","last_peered":"2024-08-17T17:58:09.164077+0000","last_clean":"2024-08-17T17:58:09.164077+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:58:09.164077+0000","last_undegraded":"2024-08-17T17:58:09.164077+0000","last_fullsized":"2024-08-17T17:58:09.164077+0000","mapping_epoch":1183,"log_start":"1170'2300","ondisk_log_start":"1170'2300","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9167,"log_dups_size":0,"ondisk_log_size":9167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103481344,"num_objects":1579,"num_object_clones":0,"num_object_copies":4737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11467,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1244'11496","reported_seq":11666,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:08.580112+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:58:08.580112+0000","last_peered":"2024-08-17T17:58:08.580112+0000","last_clean":"2024-08-17T17:58:08.580112+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:58:08.580112+0000","last_undegraded":"2024-08-17T17:58:08.580112+0000","last_fullsized":"2024-08-17T17:58:08.580112+0000","mapping_epoch":1167,"log_start":"1181'5591","ondisk_log_start":"1181'5591","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5905,"log_dups_size":0,"ondisk_log_size":5905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109051904,"num_objects":1664,"num_object_clones":0,"num_object_copies":4992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11496,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1245'11436","reported_seq":11522,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:12.761036+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:58:12.761036+0000","last_peered":"2024-08-17T17:58:12.761036+0000","last_clean":"2024-08-17T17:58:12.761036+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:58:12.761036+0000","last_undegraded":"2024-08-17T17:58:12.761036+0000","last_fullsized":"2024-08-17T17:58:12.761036+0000","mapping_epoch":1167,"log_start":"1175'4425","ondisk_log_start":"1175'4425","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7011,"log_dups_size":0,"ondisk_log_size":7011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98172928,"num_objects":1498,"num_object_clones":0,"num_object_copies":4494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11436,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1245'11766","reported_seq":30955,"reported_epoch":1245,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:58:11.903753+0000","last_change":"2024-08-17T17:55:37.941989+0000","last_active":"2024-08-17T17:58:11.903753+0000","last_peered":"2024-08-17T17:58:11.903753+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:58:11.903753+0000","last_undegraded":"2024-08-17T17:58:11.903753+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1175'4891","ondisk_log_start":"1175'4891","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6875,"log_dups_size":0,"ondisk_log_size":6875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104726528,"num_objects":1598,"num_object_clones":0,"num_object_copies":4794,"num_objects_missing_on_primary":957,"num_objects_missing":957,"num_objects_degraded":0,"num_objects_misplaced":1828,"num_objects_unfound":0,"num_objects_dirty":1598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11766,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7190,"num_bytes_recovered":471203840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"0,4,5,6","objects":12},{"shards":"0,4,6","objects":1356},{"shards":"4,5,6","objects":230}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1244'11229","reported_seq":24585,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:08.655382+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:58:08.655382+0000","last_peered":"2024-08-17T17:58:08.655382+0000","last_clean":"2024-08-17T17:58:08.655382+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:58:08.655382+0000","last_undegraded":"2024-08-17T17:58:08.655382+0000","last_fullsized":"2024-08-17T17:58:08.655382+0000","mapping_epoch":1196,"log_start":"1180'5269","ondisk_log_start":"1180'5269","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5960,"log_dups_size":0,"ondisk_log_size":5960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104398848,"num_objects":1593,"num_object_clones":0,"num_object_copies":4779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11229,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1948,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.383301+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:58:11.383301+0000","last_peered":"2024-08-17T17:58:11.383301+0000","last_clean":"2024-08-17T17:58:11.383301+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:58:11.383301+0000","last_undegraded":"2024-08-17T17:58:11.383301+0000","last_fullsized":"2024-08-17T17:58:11.383301+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":522,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.318631+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:58:00.318631+0000","last_peered":"2024-08-17T17:58:00.318631+0000","last_clean":"2024-08-17T17:58:00.318631+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:58:00.318631+0000","last_undegraded":"2024-08-17T17:58:00.318631+0000","last_fullsized":"2024-08-17T17:58:00.318631+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1866,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.322450+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:58:00.322450+0000","last_peered":"2024-08-17T17:58:00.322450+0000","last_clean":"2024-08-17T17:58:00.322450+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:58:00.322450+0000","last_undegraded":"2024-08-17T17:58:00.322450+0000","last_fullsized":"2024-08-17T17:58:00.322450+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1889,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.320287+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:58:00.320287+0000","last_peered":"2024-08-17T17:58:00.320287+0000","last_clean":"2024-08-17T17:58:00.320287+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:58:00.320287+0000","last_undegraded":"2024-08-17T17:58:00.320287+0000","last_fullsized":"2024-08-17T17:58:00.320287+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":84,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.319183+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:58:00.319183+0000","last_peered":"2024-08-17T17:58:00.319183+0000","last_clean":"2024-08-17T17:58:00.319183+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:58:00.319183+0000","last_undegraded":"2024-08-17T17:58:00.319183+0000","last_fullsized":"2024-08-17T17:58:00.319183+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1283,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.320444+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:58:00.320444+0000","last_peered":"2024-08-17T17:58:00.320444+0000","last_clean":"2024-08-17T17:58:00.320444+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:58:00.320444+0000","last_undegraded":"2024-08-17T17:58:00.320444+0000","last_fullsized":"2024-08-17T17:58:00.320444+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":143,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.331348+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:58:11.331348+0000","last_peered":"2024-08-17T17:58:11.331348+0000","last_clean":"2024-08-17T17:58:11.331348+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:58:11.331348+0000","last_undegraded":"2024-08-17T17:58:11.331348+0000","last_fullsized":"2024-08-17T17:58:11.331348+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":661,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.320705+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:58:00.320705+0000","last_peered":"2024-08-17T17:58:00.320705+0000","last_clean":"2024-08-17T17:58:00.320705+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:58:00.320705+0000","last_undegraded":"2024-08-17T17:58:00.320705+0000","last_fullsized":"2024-08-17T17:58:00.320705+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1906,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.382867+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:58:11.382867+0000","last_peered":"2024-08-17T17:58:11.382867+0000","last_clean":"2024-08-17T17:58:11.382867+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:58:11.382867+0000","last_undegraded":"2024-08-17T17:58:11.382867+0000","last_fullsized":"2024-08-17T17:58:11.382867+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1906,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.315072+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:58:00.315072+0000","last_peered":"2024-08-17T17:58:00.315072+0000","last_clean":"2024-08-17T17:58:00.315072+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:58:00.315072+0000","last_undegraded":"2024-08-17T17:58:00.315072+0000","last_fullsized":"2024-08-17T17:58:00.315072+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":128,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.331234+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:58:11.331234+0000","last_peered":"2024-08-17T17:58:11.331234+0000","last_clean":"2024-08-17T17:58:11.331234+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:58:11.331234+0000","last_undegraded":"2024-08-17T17:58:11.331234+0000","last_fullsized":"2024-08-17T17:58:11.331234+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1878,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.322916+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:58:00.322916+0000","last_peered":"2024-08-17T17:58:00.322916+0000","last_clean":"2024-08-17T17:58:00.322916+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:58:00.322916+0000","last_undegraded":"2024-08-17T17:58:00.322916+0000","last_fullsized":"2024-08-17T17:58:00.322916+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1935,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.330836+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:58:11.330836+0000","last_peered":"2024-08-17T17:58:11.330836+0000","last_clean":"2024-08-17T17:58:11.330836+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:58:11.330836+0000","last_undegraded":"2024-08-17T17:58:11.330836+0000","last_fullsized":"2024-08-17T17:58:11.330836+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1929,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.383195+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:58:11.383195+0000","last_peered":"2024-08-17T17:58:11.383195+0000","last_clean":"2024-08-17T17:58:11.383195+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:58:11.383195+0000","last_undegraded":"2024-08-17T17:58:11.383195+0000","last_fullsized":"2024-08-17T17:58:11.383195+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1900,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.321896+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:58:00.321896+0000","last_peered":"2024-08-17T17:58:00.321896+0000","last_clean":"2024-08-17T17:58:00.321896+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:58:00.321896+0000","last_undegraded":"2024-08-17T17:58:00.321896+0000","last_fullsized":"2024-08-17T17:58:00.321896+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1219,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.319208+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:58:00.319208+0000","last_peered":"2024-08-17T17:58:00.319208+0000","last_clean":"2024-08-17T17:58:00.319208+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:58:00.319208+0000","last_undegraded":"2024-08-17T17:58:00.319208+0000","last_fullsized":"2024-08-17T17:58:00.319208+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1872,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.321700+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:58:00.321700+0000","last_peered":"2024-08-17T17:58:00.321700+0000","last_clean":"2024-08-17T17:58:00.321700+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:58:00.321700+0000","last_undegraded":"2024-08-17T17:58:00.321700+0000","last_fullsized":"2024-08-17T17:58:00.321700+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1923,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.383360+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:58:11.383360+0000","last_peered":"2024-08-17T17:58:11.383360+0000","last_clean":"2024-08-17T17:58:11.383360+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:58:11.383360+0000","last_undegraded":"2024-08-17T17:58:11.383360+0000","last_fullsized":"2024-08-17T17:58:11.383360+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1828,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.320155+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:58:00.320155+0000","last_peered":"2024-08-17T17:58:00.320155+0000","last_clean":"2024-08-17T17:58:00.320155+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:58:00.320155+0000","last_undegraded":"2024-08-17T17:58:00.320155+0000","last_fullsized":"2024-08-17T17:58:00.320155+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":538,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.327261+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:58:00.327261+0000","last_peered":"2024-08-17T17:58:00.327261+0000","last_clean":"2024-08-17T17:58:00.327261+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:58:00.327261+0000","last_undegraded":"2024-08-17T17:58:00.327261+0000","last_fullsized":"2024-08-17T17:58:00.327261+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1215,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.327165+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:58:00.327165+0000","last_peered":"2024-08-17T17:58:00.327165+0000","last_clean":"2024-08-17T17:58:00.327165+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:58:00.327165+0000","last_undegraded":"2024-08-17T17:58:00.327165+0000","last_fullsized":"2024-08-17T17:58:00.327165+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1928,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.331631+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:58:11.331631+0000","last_peered":"2024-08-17T17:58:11.331631+0000","last_clean":"2024-08-17T17:58:11.331631+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:58:11.331631+0000","last_undegraded":"2024-08-17T17:58:11.331631+0000","last_fullsized":"2024-08-17T17:58:11.331631+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":376,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.327184+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:58:00.327184+0000","last_peered":"2024-08-17T17:58:00.327184+0000","last_clean":"2024-08-17T17:58:00.327184+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:58:00.327184+0000","last_undegraded":"2024-08-17T17:58:00.327184+0000","last_fullsized":"2024-08-17T17:58:00.327184+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":522,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.318900+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:58:00.318900+0000","last_peered":"2024-08-17T17:58:00.318900+0000","last_clean":"2024-08-17T17:58:00.318900+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:58:00.318900+0000","last_undegraded":"2024-08-17T17:58:00.318900+0000","last_fullsized":"2024-08-17T17:58:00.318900+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1200,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.318802+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:58:00.318802+0000","last_peered":"2024-08-17T17:58:00.318802+0000","last_clean":"2024-08-17T17:58:00.318802+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:58:00.318802+0000","last_undegraded":"2024-08-17T17:58:00.318802+0000","last_fullsized":"2024-08-17T17:58:00.318802+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1208,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.319267+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:58:00.319267+0000","last_peered":"2024-08-17T17:58:00.319267+0000","last_clean":"2024-08-17T17:58:00.319267+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:58:00.319267+0000","last_undegraded":"2024-08-17T17:58:00.319267+0000","last_fullsized":"2024-08-17T17:58:00.319267+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1976,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.382980+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:58:11.382980+0000","last_peered":"2024-08-17T17:58:11.382980+0000","last_clean":"2024-08-17T17:58:11.382980+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:58:11.382980+0000","last_undegraded":"2024-08-17T17:58:11.382980+0000","last_fullsized":"2024-08-17T17:58:11.382980+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1858,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.321818+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:58:00.321818+0000","last_peered":"2024-08-17T17:58:00.321818+0000","last_clean":"2024-08-17T17:58:00.321818+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:58:00.321818+0000","last_undegraded":"2024-08-17T17:58:00.321818+0000","last_fullsized":"2024-08-17T17:58:00.321818+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1934,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.331890+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:58:11.331890+0000","last_peered":"2024-08-17T17:58:11.331890+0000","last_clean":"2024-08-17T17:58:11.331890+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:58:11.331890+0000","last_undegraded":"2024-08-17T17:58:11.331890+0000","last_fullsized":"2024-08-17T17:58:11.331890+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1860,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.321180+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:58:00.321180+0000","last_peered":"2024-08-17T17:58:00.321180+0000","last_clean":"2024-08-17T17:58:00.321180+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:58:00.321180+0000","last_undegraded":"2024-08-17T17:58:00.321180+0000","last_fullsized":"2024-08-17T17:58:00.321180+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1894,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.382970+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:58:11.382970+0000","last_peered":"2024-08-17T17:58:11.382970+0000","last_clean":"2024-08-17T17:58:11.382970+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:58:11.382970+0000","last_undegraded":"2024-08-17T17:58:11.382970+0000","last_fullsized":"2024-08-17T17:58:11.382970+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1235,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.327278+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:58:00.327278+0000","last_peered":"2024-08-17T17:58:00.327278+0000","last_clean":"2024-08-17T17:58:00.327278+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:58:00.327278+0000","last_undegraded":"2024-08-17T17:58:00.327278+0000","last_fullsized":"2024-08-17T17:58:00.327278+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1941,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.331526+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:58:11.331526+0000","last_peered":"2024-08-17T17:58:11.331526+0000","last_clean":"2024-08-17T17:58:11.331526+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:58:11.331526+0000","last_undegraded":"2024-08-17T17:58:11.331526+0000","last_fullsized":"2024-08-17T17:58:11.331526+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":128,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.331091+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:58:11.331091+0000","last_peered":"2024-08-17T17:58:11.331091+0000","last_clean":"2024-08-17T17:58:11.331091+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:58:11.331091+0000","last_undegraded":"2024-08-17T17:58:11.331091+0000","last_fullsized":"2024-08-17T17:58:11.331091+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1960,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.331439+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:58:11.331439+0000","last_peered":"2024-08-17T17:58:11.331439+0000","last_clean":"2024-08-17T17:58:11.331439+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:58:11.331439+0000","last_undegraded":"2024-08-17T17:58:11.331439+0000","last_fullsized":"2024-08-17T17:58:11.331439+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":694,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.322834+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:58:00.322834+0000","last_peered":"2024-08-17T17:58:00.322834+0000","last_clean":"2024-08-17T17:58:00.322834+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:58:00.322834+0000","last_undegraded":"2024-08-17T17:58:00.322834+0000","last_fullsized":"2024-08-17T17:58:00.322834+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1941,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.331640+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:58:11.331640+0000","last_peered":"2024-08-17T17:58:11.331640+0000","last_clean":"2024-08-17T17:58:11.331640+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:58:11.331640+0000","last_undegraded":"2024-08-17T17:58:11.331640+0000","last_fullsized":"2024-08-17T17:58:11.331640+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1269,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.382662+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:58:11.382662+0000","last_peered":"2024-08-17T17:58:11.382662+0000","last_clean":"2024-08-17T17:58:11.382662+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:58:11.382662+0000","last_undegraded":"2024-08-17T17:58:11.382662+0000","last_fullsized":"2024-08-17T17:58:11.382662+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":187,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.331499+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:58:11.331499+0000","last_peered":"2024-08-17T17:58:11.331499+0000","last_clean":"2024-08-17T17:58:11.331499+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:58:11.331499+0000","last_undegraded":"2024-08-17T17:58:11.331499+0000","last_fullsized":"2024-08-17T17:58:11.331499+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1951,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.331302+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:58:11.331302+0000","last_peered":"2024-08-17T17:58:11.331302+0000","last_clean":"2024-08-17T17:58:11.331302+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:58:11.331302+0000","last_undegraded":"2024-08-17T17:58:11.331302+0000","last_fullsized":"2024-08-17T17:58:11.331302+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1929,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.331791+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:58:11.331791+0000","last_peered":"2024-08-17T17:58:11.331791+0000","last_clean":"2024-08-17T17:58:11.331791+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:58:11.331791+0000","last_undegraded":"2024-08-17T17:58:11.331791+0000","last_fullsized":"2024-08-17T17:58:11.331791+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1947,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.319576+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:58:00.319576+0000","last_peered":"2024-08-17T17:58:00.319576+0000","last_clean":"2024-08-17T17:58:00.319576+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:58:00.319576+0000","last_undegraded":"2024-08-17T17:58:00.319576+0000","last_fullsized":"2024-08-17T17:58:00.319576+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1955,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.383196+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:58:11.383196+0000","last_peered":"2024-08-17T17:58:11.383196+0000","last_clean":"2024-08-17T17:58:11.383196+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:58:11.383196+0000","last_undegraded":"2024-08-17T17:58:11.383196+0000","last_fullsized":"2024-08-17T17:58:11.383196+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1935,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.396998+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:58:11.396998+0000","last_peered":"2024-08-17T17:58:11.396998+0000","last_clean":"2024-08-17T17:58:11.396998+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:58:11.396998+0000","last_undegraded":"2024-08-17T17:58:11.396998+0000","last_fullsized":"2024-08-17T17:58:11.396998+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1242,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.327002+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:58:00.327002+0000","last_peered":"2024-08-17T17:58:00.327002+0000","last_clean":"2024-08-17T17:58:00.327002+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:58:00.327002+0000","last_undegraded":"2024-08-17T17:58:00.327002+0000","last_fullsized":"2024-08-17T17:58:00.327002+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1188,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.319175+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:58:00.319175+0000","last_peered":"2024-08-17T17:58:00.319175+0000","last_clean":"2024-08-17T17:58:00.319175+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:58:00.319175+0000","last_undegraded":"2024-08-17T17:58:00.319175+0000","last_fullsized":"2024-08-17T17:58:00.319175+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1842,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.320436+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:58:00.320436+0000","last_peered":"2024-08-17T17:58:00.320436+0000","last_clean":"2024-08-17T17:58:00.320436+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:58:00.320436+0000","last_undegraded":"2024-08-17T17:58:00.320436+0000","last_fullsized":"2024-08-17T17:58:00.320436+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":173,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.314695+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:58:00.314695+0000","last_peered":"2024-08-17T17:58:00.314695+0000","last_clean":"2024-08-17T17:58:00.314695+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:58:00.314695+0000","last_undegraded":"2024-08-17T17:58:00.314695+0000","last_fullsized":"2024-08-17T17:58:00.314695+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1554,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.326882+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:58:00.326882+0000","last_peered":"2024-08-17T17:58:00.326882+0000","last_clean":"2024-08-17T17:58:00.326882+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:58:00.326882+0000","last_undegraded":"2024-08-17T17:58:00.326882+0000","last_fullsized":"2024-08-17T17:58:00.326882+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1936,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.331425+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:58:11.331425+0000","last_peered":"2024-08-17T17:58:11.331425+0000","last_clean":"2024-08-17T17:58:11.331425+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:58:11.331425+0000","last_undegraded":"2024-08-17T17:58:11.331425+0000","last_fullsized":"2024-08-17T17:58:11.331425+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1874,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.321848+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:58:00.321848+0000","last_peered":"2024-08-17T17:58:00.321848+0000","last_clean":"2024-08-17T17:58:00.321848+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:58:00.321848+0000","last_undegraded":"2024-08-17T17:58:00.321848+0000","last_fullsized":"2024-08-17T17:58:00.321848+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":522,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.319184+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:58:00.319184+0000","last_peered":"2024-08-17T17:58:00.319184+0000","last_clean":"2024-08-17T17:58:00.319184+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:58:00.319184+0000","last_undegraded":"2024-08-17T17:58:00.319184+0000","last_fullsized":"2024-08-17T17:58:00.319184+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":612,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.319520+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:58:00.319520+0000","last_peered":"2024-08-17T17:58:00.319520+0000","last_clean":"2024-08-17T17:58:00.319520+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:58:00.319520+0000","last_undegraded":"2024-08-17T17:58:00.319520+0000","last_fullsized":"2024-08-17T17:58:00.319520+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":628,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.315837+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:58:00.315837+0000","last_peered":"2024-08-17T17:58:00.315837+0000","last_clean":"2024-08-17T17:58:00.315837+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:58:00.315837+0000","last_undegraded":"2024-08-17T17:58:00.315837+0000","last_fullsized":"2024-08-17T17:58:00.315837+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1206,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.319035+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:58:00.319035+0000","last_peered":"2024-08-17T17:58:00.319035+0000","last_clean":"2024-08-17T17:58:00.319035+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:58:00.319035+0000","last_undegraded":"2024-08-17T17:58:00.319035+0000","last_fullsized":"2024-08-17T17:58:00.319035+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":644,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.320691+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:58:00.320691+0000","last_peered":"2024-08-17T17:58:00.320691+0000","last_clean":"2024-08-17T17:58:00.320691+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:58:00.320691+0000","last_undegraded":"2024-08-17T17:58:00.320691+0000","last_fullsized":"2024-08-17T17:58:00.320691+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1870,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.320793+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:58:00.320793+0000","last_peered":"2024-08-17T17:58:00.320793+0000","last_clean":"2024-08-17T17:58:00.320793+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:58:00.320793+0000","last_undegraded":"2024-08-17T17:58:00.320793+0000","last_fullsized":"2024-08-17T17:58:00.320793+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1968,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.383055+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:58:11.383055+0000","last_peered":"2024-08-17T17:58:11.383055+0000","last_clean":"2024-08-17T17:58:11.383055+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:58:11.383055+0000","last_undegraded":"2024-08-17T17:58:11.383055+0000","last_fullsized":"2024-08-17T17:58:11.383055+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":677,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.320603+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:58:00.320603+0000","last_peered":"2024-08-17T17:58:00.320603+0000","last_clean":"2024-08-17T17:58:00.320603+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:58:00.320603+0000","last_undegraded":"2024-08-17T17:58:00.320603+0000","last_fullsized":"2024-08-17T17:58:00.320603+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1195,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.318819+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:58:00.318819+0000","last_peered":"2024-08-17T17:58:00.318819+0000","last_clean":"2024-08-17T17:58:00.318819+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:58:00.318819+0000","last_undegraded":"2024-08-17T17:58:00.318819+0000","last_fullsized":"2024-08-17T17:58:00.318819+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1194,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.327251+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:58:00.327251+0000","last_peered":"2024-08-17T17:58:00.327251+0000","last_clean":"2024-08-17T17:58:00.327251+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:58:00.327251+0000","last_undegraded":"2024-08-17T17:58:00.327251+0000","last_fullsized":"2024-08-17T17:58:00.327251+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":663,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.330933+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:58:11.330933+0000","last_peered":"2024-08-17T17:58:11.330933+0000","last_clean":"2024-08-17T17:58:11.330933+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:58:11.330933+0000","last_undegraded":"2024-08-17T17:58:11.330933+0000","last_fullsized":"2024-08-17T17:58:11.330933+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1212,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.327392+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:58:00.327392+0000","last_peered":"2024-08-17T17:58:00.327392+0000","last_clean":"2024-08-17T17:58:00.327392+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:58:00.327392+0000","last_undegraded":"2024-08-17T17:58:00.327392+0000","last_fullsized":"2024-08-17T17:58:00.327392+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":96,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.326855+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:58:00.326855+0000","last_peered":"2024-08-17T17:58:00.326855+0000","last_clean":"2024-08-17T17:58:00.326855+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:58:00.326855+0000","last_undegraded":"2024-08-17T17:58:00.326855+0000","last_fullsized":"2024-08-17T17:58:00.326855+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1228,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.327406+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:58:00.327406+0000","last_peered":"2024-08-17T17:58:00.327406+0000","last_clean":"2024-08-17T17:58:00.327406+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:58:00.327406+0000","last_undegraded":"2024-08-17T17:58:00.327406+0000","last_fullsized":"2024-08-17T17:58:00.327406+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1245,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.318812+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:58:00.318812+0000","last_peered":"2024-08-17T17:58:00.318812+0000","last_clean":"2024-08-17T17:58:00.318812+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:58:00.318812+0000","last_undegraded":"2024-08-17T17:58:00.318812+0000","last_fullsized":"2024-08-17T17:58:00.318812+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1871,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.321061+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:58:00.321061+0000","last_peered":"2024-08-17T17:58:00.321061+0000","last_clean":"2024-08-17T17:58:00.321061+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:58:00.321061+0000","last_undegraded":"2024-08-17T17:58:00.321061+0000","last_fullsized":"2024-08-17T17:58:00.321061+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1189,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.327104+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:58:00.327104+0000","last_peered":"2024-08-17T17:58:00.327104+0000","last_clean":"2024-08-17T17:58:00.327104+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:58:00.327104+0000","last_undegraded":"2024-08-17T17:58:00.327104+0000","last_fullsized":"2024-08-17T17:58:00.327104+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1926,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.383092+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:58:11.383092+0000","last_peered":"2024-08-17T17:58:11.383092+0000","last_clean":"2024-08-17T17:58:11.383092+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:58:11.383092+0000","last_undegraded":"2024-08-17T17:58:11.383092+0000","last_fullsized":"2024-08-17T17:58:11.383092+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1928,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.382737+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:58:11.382737+0000","last_peered":"2024-08-17T17:58:11.382737+0000","last_clean":"2024-08-17T17:58:11.382737+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:58:11.382737+0000","last_undegraded":"2024-08-17T17:58:11.382737+0000","last_fullsized":"2024-08-17T17:58:11.382737+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":602,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.327150+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:58:00.327150+0000","last_peered":"2024-08-17T17:58:00.327150+0000","last_clean":"2024-08-17T17:58:00.327150+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:58:00.327150+0000","last_undegraded":"2024-08-17T17:58:00.327150+0000","last_fullsized":"2024-08-17T17:58:00.327150+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1214,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.328249+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:58:00.328249+0000","last_peered":"2024-08-17T17:58:00.328249+0000","last_clean":"2024-08-17T17:58:00.328249+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:58:00.328249+0000","last_undegraded":"2024-08-17T17:58:00.328249+0000","last_fullsized":"2024-08-17T17:58:00.328249+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1872,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:58:00.320804+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:58:00.320804+0000","last_peered":"2024-08-17T17:58:00.320804+0000","last_clean":"2024-08-17T17:58:00.320804+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:58:00.320804+0000","last_undegraded":"2024-08-17T17:58:00.320804+0000","last_fullsized":"2024-08-17T17:58:00.320804+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":1652293656,"num_objects":25213,"num_object_clones":0,"num_object_copies":75639,"num_objects_missing_on_primary":957,"num_objects_missing":957,"num_objects_degraded":15837,"num_objects_misplaced":3363,"num_objects_unfound":0,"num_objects_dirty":25213,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":184397,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38176,"num_bytes_recovered":2323447808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6559969280,"data_stored":6559957064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108371,"ondisk_log_size":108371,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234448,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1014464,"kb_used_data":708792,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":92730688,"statfs":{"total":95995035648,"available":94956224512,"internally_reserved":0,"allocated":725803008,"data_stored":720744955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496083,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1117316,"kb_used_data":815804,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92627836,"statfs":{"total":95995035648,"available":94850904064,"internally_reserved":0,"allocated":835383296,"data_stored":830321648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757721,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326334,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1184348,"kb_used_data":949268,"kb_used_omap":57,"kb_used_meta":235014,"kb_avail":92560804,"statfs":{"total":95995035648,"available":94782263296,"internally_reserved":0,"allocated":972050432,"data_stored":967014995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":240655099},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405573,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1074084,"kb_used_data":842012,"kb_used_omap":61,"kb_used_meta":232002,"kb_avail":92671068,"statfs":{"total":95995035648,"available":94895173632,"internally_reserved":0,"allocated":862220288,"data_stored":857181823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":237571000},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452108,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":807384,"kb_used_data":613136,"kb_used_omap":54,"kb_used_meta":194185,"kb_avail":92937768,"statfs":{"total":95995035648,"available":95168274432,"internally_reserved":0,"allocated":627851264,"data_stored":622819788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":198845838},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474799,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":907816,"kb_used_data":675040,"kb_used_omap":119,"kb_used_meta":232648,"kb_avail":92837336,"statfs":{"total":95995035648,"available":95065432064,"internally_reserved":0,"allocated":691240960,"data_stored":686194320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":238232286},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019356,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":672600064,"data_stored":672595992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":608567296,"data_stored":608567296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":847446016,"data_stored":847446016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":955125760,"data_stored":955121688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":871759872,"data_stored":871759872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":812912640,"data_stored":812908568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":703660032,"data_stored":703660032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:58:15.188 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:58:15.271 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:15.294 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:15.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:15.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:15.396 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:15.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:15 smithi130 ceph-mon[105521]: from='client.36123 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:15.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:15 smithi130 ceph-mon[110068]: from='client.36123 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:15.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:15.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:15.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:15.519 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:15.541 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:15.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:15.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:15.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:15 smithi179 ceph-mon[93519]: from='client.36123 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:15.643 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:15.664 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:15.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:15.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:15.766 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:15.796 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:15.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:15.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:15.898 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:15.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:15.920 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:15.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:16.021 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:16.042 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:16.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:16.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:16.143 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:16.165 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:16.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:16.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:16.266 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:16.293 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:16.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:16.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:16.395 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:16.415 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:16.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:16.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:16.517 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:16.539 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:16.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:16.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:16 smithi179 ceph-mon[93519]: pgmap v1955: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 267 op/s; 15837/74037 objects degraded (21.391%); 3363/74037 objects misplaced (4.542%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:58:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/74037 objects degraded (21.391%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:16.641 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:16.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:16 smithi130 ceph-mon[110068]: pgmap v1955: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 267 op/s; 15837/74037 objects degraded (21.391%); 3363/74037 objects misplaced (4.542%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:58:16.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/74037 objects degraded (21.391%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:16 smithi130 ceph-mon[105521]: pgmap v1955: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 267 op/s; 15837/74037 objects degraded (21.391%); 3363/74037 objects misplaced (4.542%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:58:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/74037 objects degraded (21.391%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:16.671 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:16.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:16.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:16.773 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:16.794 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:16.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:16.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:16.895 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:16.921 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:16.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:16.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:17.023 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:17.046 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:17.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:17.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:17.147 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:17.170 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:17.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:17.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:17.272 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:17.280 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:58:17.280 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:58:17.300 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:17.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:17.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:17.402 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:17.426 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:17.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:17.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:17.528 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:17.549 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:17.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:17.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:17.651 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:17.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:17.678 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:17.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:17.779 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:17.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:17.831 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:17.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:17.850 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:58:17.850 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:58:17.884 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:58:17.884 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:58:17.918 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:58:17.918 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:58:17.932 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:17.945 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:58:17.946 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:58:17.953 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:17.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:17.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:17.964 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:58:17.964 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:58:17.981 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:58:17.981 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:58:18.006 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:58:18.006 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:58:18.026 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:58:18.027 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:58:18.055 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:18.075 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:18.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:18.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:18.177 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:18.198 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:18.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:18.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:18.300 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:18.326 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:18.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:18.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:18.397 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:58:18.429 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:18.450 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:18.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:18.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:18.552 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:18.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:18.574 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:18.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:18.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:18 smithi130 ceph-mon[105521]: pgmap v1956: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 270 op/s; 15837/72984 objects degraded (21.699%); 3078/72984 objects misplaced (4.217%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:58:18.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3700823340' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:58:18.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:18 smithi130 ceph-mon[110068]: pgmap v1956: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 270 op/s; 15837/72984 objects degraded (21.699%); 3078/72984 objects misplaced (4.217%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:58:18.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3700823340' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:58:18.674 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:18.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:18.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:18.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:18.797 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:18.819 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:18.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:18.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:18 smithi179 ceph-mon[93519]: pgmap v1956: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 270 op/s; 15837/72984 objects degraded (21.699%); 3078/72984 objects misplaced (4.217%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:58:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3700823340' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:58:18.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:18.944 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:18.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:18.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:19.013 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:19.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:19.073 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:19.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:19.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:19.174 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:19.209 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:19.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:19.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:19.311 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:19.337 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:19.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:19.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:19.391 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:19.439 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:19.460 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:19.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:19.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:19.562 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:19.592 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:19.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:19.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:19.694 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:19.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:19.719 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:19.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:19.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:19.841 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:19.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:19.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:19.942 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:19.967 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:19.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:19.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:20.068 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:20.090 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:20.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:20.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:58:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:58:19] "GET /metrics HTTP/1.1" 200 36805 "" "Prometheus/2.51.0" 2024-08-17T17:58:20.192 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:20.214 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:20.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:20.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:20.316 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:20.337 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:20.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:20.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:20.439 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:20.465 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:20.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:20.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:20.567 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:20 smithi179 ceph-mon[93519]: pgmap v1957: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 287 op/s; 15837/70149 objects degraded (22.576%); 3078/70149 objects misplaced (4.388%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:58:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:58:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:58:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:58:20.597 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:20.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:20.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:20.621 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:20 smithi130 ceph-mon[105521]: pgmap v1957: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 287 op/s; 15837/70149 objects degraded (22.576%); 3078/70149 objects misplaced (4.388%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:58:20.621 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:20 smithi130 ceph-mon[110068]: pgmap v1957: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 287 op/s; 15837/70149 objects degraded (22.576%); 3078/70149 objects misplaced (4.388%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:58:20.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:20.740 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:20.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:20.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:20.842 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:20.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:20.864 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:20.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:20.965 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:20.990 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:20.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:20.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:21.092 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:21.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:21.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:21.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:21.214 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:21.242 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:21.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:21.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:21.343 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:21.370 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:21.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:21.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:21.471 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:21.495 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:21.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:21.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:21.597 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:21.619 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:21.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:21.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:21.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/70149 objects degraded (22.576%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/70149 objects degraded (22.576%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:21.720 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:21.744 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:21.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:21.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/70149 objects degraded (22.576%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:21.846 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:21.867 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:21.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:21.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:21.969 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:21.995 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:21.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:21.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:22.097 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:22.119 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:22.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:22.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:22.220 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:22.248 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:22.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:22.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:22.349 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:22.364 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:58:22.370 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:22.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:22.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:22.472 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:22.493 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:22.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:22.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:22.595 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:22.616 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:22.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:22.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:22.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:22 smithi130 ceph-mon[105521]: pgmap v1958: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 291 op/s; 15837/69579 objects degraded (22.761%); 3078/69579 objects misplaced (4.424%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:58:22.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:22 smithi130 ceph-mon[110068]: pgmap v1958: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 291 op/s; 15837/69579 objects degraded (22.761%); 3078/69579 objects misplaced (4.424%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:58:22.718 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:22.740 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:22.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:22.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:22.842 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:22 smithi179 ceph-mon[93519]: pgmap v1958: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 291 op/s; 15837/69579 objects degraded (22.761%); 3078/69579 objects misplaced (4.424%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:58:22.874 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:22.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:22.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:22.976 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:22.997 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:22.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:22.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:23.099 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:23.120 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:23.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:23.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:23.223 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:23.243 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:23.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:23.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:23.345 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:23.367 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:23.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:23.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:23.469 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:23.498 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:23.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:23.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:23.600 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:23.637 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:23.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:23.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:23.739 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:23.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:23.762 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:23.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:23.782 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:23.793 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:23.863 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:23.913 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:23.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:23.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:24.015 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:24.038 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:24.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:24.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:24.140 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:24.163 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:24.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:24.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:24.220 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:58:24.221 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15837/66681 objects degraded (23.750%), 5 pgs degraded, 6 pgs undersized","count":11},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1825,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1245,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":22227,"data_bytes":1455495141,"bytes_used":5774774272,"bytes_avail":570195439616,"bytes_total":575970213888,"degraded_objects":15837,"degraded_total":66681,"degraded_ratio":0.23750393665361946,"misplaced_objects":2777,"misplaced_total":66681,"misplaced_ratio":0.041646046100088478,"recovering_objects_per_sec":24,"recovering_bytes_per_sec":1617534,"recovering_keys_per_sec":0,"num_objects_recovered":291,"num_bytes_recovered":19070976,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":329},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":26,"modified":"2024-08-17T17:57:13.099933+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (10m)\n [==========================..] (remaining: 46s)","progress":0.93258428573608398,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:58:24.265 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:24.304 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:24.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:24.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:24.401 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T17:58:24.406 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:24.429 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:24.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:24.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:24.531 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:24.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:24.554 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:24.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:24.655 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:24.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:24 smithi130 ceph-mon[110068]: pgmap v1959: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 329 op/s; 15837/66681 objects degraded (23.750%); 2777/66681 objects misplaced (4.165%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:58:24.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:24 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2917992679' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:24.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:24 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:24.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:24 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2103051395' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:58:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:24 smithi130 ceph-mon[105521]: pgmap v1959: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 329 op/s; 15837/66681 objects degraded (23.750%); 2777/66681 objects misplaced (4.165%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:58:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:24 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2917992679' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:24 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:24 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2103051395' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:58:24.682 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:24.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:24.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:24.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:24.817 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:24.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:24.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:24.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:24 smithi179 ceph-mon[93519]: pgmap v1959: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 329 op/s; 15837/66681 objects degraded (23.750%); 2777/66681 objects misplaced (4.165%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:58:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:24 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2917992679' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:24 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:24 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2103051395' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:58:24.918 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:24.940 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:24.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:24.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:25.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:25.079 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:25.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:25.180 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:25.210 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:25.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:25.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:25.312 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:25.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:25.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:25.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:25 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:58:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:25 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:58:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:25 smithi130 ceph-mon[110068]: osdmap e1246: 8 total, 8 up, 6 in 2024-08-17T17:58:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:25 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:58:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:25 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:58:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:25 smithi130 ceph-mon[105521]: osdmap e1246: 8 total, 8 up, 6 in 2024-08-17T17:58:25.436 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:25.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:25.473 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:25.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:25.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:25.598 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:25.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:25.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:25.700 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:25.724 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:25.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:25.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:25.826 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:25 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:58:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:25 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:58:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:25 smithi179 ceph-mon[93519]: osdmap e1246: 8 total, 8 up, 6 in 2024-08-17T17:58:25.850 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:25.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:25.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:25.951 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:25.974 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:25.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:25.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:26.075 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:26.100 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:26.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:26.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:26.202 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:26.231 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:58:26.231 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:58:26.255 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:26.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:26.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:26.357 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:26.389 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:26.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:26.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:26.491 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:26.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:26.515 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:26.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:26.616 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:26.645 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:26 smithi130 ceph-mon[105521]: pgmap v1961: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 308 op/s; 15837/64773 objects degraded (24.450%); 2777/64773 objects misplaced (4.287%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:58:26.646 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15837/64773 objects degraded (24.450%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:26.646 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:26 smithi130 ceph-mon[110068]: pgmap v1961: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 308 op/s; 15837/64773 objects degraded (24.450%); 2777/64773 objects misplaced (4.287%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:58:26.647 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15837/64773 objects degraded (24.450%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:26.681 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:26.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:26.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:26.783 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:26.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:26.831 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:26.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:26.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:26 smithi179 ceph-mon[93519]: pgmap v1961: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 308 op/s; 15837/64773 objects degraded (24.450%); 2777/64773 objects misplaced (4.287%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:58:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15837/64773 objects degraded (24.450%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:26.932 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:26.955 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:26.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:26.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:27.057 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:27.078 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:27.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:27.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:27.180 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:27.216 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:27.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:27.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:27.318 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:27.339 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:27.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:27.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:27.441 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:27.486 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:27.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:27.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:27.588 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:27.612 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:27.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:27.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:27.713 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:27.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:27.737 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:27.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:27.838 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:27.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:27.884 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:27.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:27.985 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:28.019 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:28.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:28.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:28.121 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:28.151 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:28.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:28.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:28.253 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:28.276 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:28.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:28.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:28.361 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:58:28.361 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T17:58:28.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.361 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:191157 45.b 45:dc381bd8:::benchmark_data_smithi130_172551_object86350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1246)", 2024-08-17T17:58:28.361 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:28.355647+0000", 2024-08-17T17:58:28.361 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0042724659999999999, 2024-08-17T17:58:28.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.004301104, 2024-08-17T17:58:28.361 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:28.362 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:28.362 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:58:28.362 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:28.362 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:28.362 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:28.362 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 191157 2024-08-17T17:58:28.362 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.362 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:28.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:28.362 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.355647+0000", 2024-08-17T17:58:28.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:28.362 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:28.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.355647+0000", 2024-08-17T17:58:28.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:28.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:28.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.355648+0000", 2024-08-17T17:58:28.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3859999999999999e-06 2024-08-17T17:58:28.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:28.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.355653+0000", 2024-08-17T17:58:28.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7260000000000002e-06 2024-08-17T17:58:28.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.355669+0000", 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6274e-05 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.355677+0000", 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2179999999999999e-06 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.358697+0000", 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0030196149999999998 2024-08-17T17:58:28.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.358849+0000", 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015176499999999999 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3,4", 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.358992+0000", 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014350499999999999 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.45158.0:191159 45.a e1246/1167 45:571694f0:::benchmark_data_smithi130_172551_object86352:head v 1246'11938, mlcod=1246'11935)", 2024-08-17T17:58:28.365 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:28.359772+0000", 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.00014692500000000001, 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00024025699999999999, 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.359772+0000", 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.359772+0000", 2024-08-17T17:58:28.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.359773+0000", 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.097e-06 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.359781+0000", 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8329999999999994e-06 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.359794+0000", 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2408e-05 2024-08-17T17:58:28.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.359807+0000", 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3232e-05 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.359947+0000", 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013978499999999999 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.359966+0000", 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9701999999999999e-05 2024-08-17T17:58:28.368 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:191158 45.b 45.5509cf6b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1246)", 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:28.359854+0000", 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 6.4830000000000001e-05, 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020267500000000001, 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "queued for pg", 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:28.369 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 191158 2024-08-17T17:58:28.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.370 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:28.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:28.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.359854+0000", 2024-08-17T17:58:28.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:28.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:28.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.359854+0000", 2024-08-17T17:58:28.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:28.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.371 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:28.371 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.359856+0000", 2024-08-17T17:58:28.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.257e-06 2024-08-17T17:58:28.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.371 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.371 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:28.371 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.359860+0000", 2024-08-17T17:58:28.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4499999999999997e-06 2024-08-17T17:58:28.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.371 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.371 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:28.371 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.359870+0000", 2024-08-17T17:58:28.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9739999999999997e-06 2024-08-17T17:58:28.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:28.372 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:28.372 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:28.372 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:28.359876+0000", 2024-08-17T17:58:28.372 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0000000000000002e-06 2024-08-17T17:58:28.372 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:28.372 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:28.372 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:28.372 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:28.372 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T17:58:28.372 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 3 2024-08-17T17:58:28.372 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:58:28.378 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:28.410 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:28.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:28.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:28.512 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:28.547 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:28.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:28.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:28 smithi179 ceph-mon[93519]: pgmap v1962: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 15962/62931 objects degraded (25.364%); 2431/62931 objects misplaced (3.863%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:58:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:58:28.648 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:28.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:28 smithi130 ceph-mon[105521]: pgmap v1962: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 15962/62931 objects degraded (25.364%); 2431/62931 objects misplaced (3.863%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:58:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:58:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:28 smithi130 ceph-mon[110068]: pgmap v1962: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 15962/62931 objects degraded (25.364%); 2431/62931 objects misplaced (3.863%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:58:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:58:28.684 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:28.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:28.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:28.785 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:28.815 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:28.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:28.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:28.916 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:28.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:28.960 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:28.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:29.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:29.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:29.097 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:29.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:29.198 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:29.212 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:29.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:29.239 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:29.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:29.341 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:29.364 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:29.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:29.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:29.466 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:29.496 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:29.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:29.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:29.598 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:29.611 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:58:29.620 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:29.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:29.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:29.721 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:29.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:29.761 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:29.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:29.764 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:58:29.861 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:29.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:29.917 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:29.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:29.920 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:58:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:58:29] "GET /metrics HTTP/1.1" 200 36803 "" "Prometheus/2.51.0" 2024-08-17T17:58:30.018 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:30.039 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:30.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:30.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:30.141 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:30.205 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:30.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:30.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:30.307 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:30.329 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:30.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:30.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:30.431 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:30.465 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:30.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:30.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:30.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:30.589 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:30.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:30.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:30.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:30 smithi179 ceph-mon[93519]: pgmap v1963: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 431 op/s; 15962/57189 objects degraded (27.911%); 2431/57189 objects misplaced (4.251%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:58:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:58:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:58:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:58:30.611 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:30 smithi130 ceph-mon[105521]: pgmap v1963: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 431 op/s; 15962/57189 objects degraded (27.911%); 2431/57189 objects misplaced (4.251%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:58:30.611 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:30 smithi130 ceph-mon[110068]: pgmap v1963: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 431 op/s; 15962/57189 objects degraded (27.911%); 2431/57189 objects misplaced (4.251%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:58:30.683 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:58:30.683 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:58:30.691 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:30.715 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:58:30.715 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:58:30.750 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:30.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:30.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:30.756 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:58:30.756 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:58:30.795 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:58:30.795 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:58:30.832 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:58:30.832 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:58:30.852 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:30.867 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:58:30.867 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:58:30.875 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:30.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:30.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:30.899 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:58:30.899 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:58:30.929 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:58:30.930 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:58:30.976 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:30.997 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:30.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:30.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:31.099 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:31.137 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:31.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:31.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:31.239 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:31.263 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:31.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:31.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:31.365 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:31.390 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:31.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:31.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:31.492 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:31.524 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:31.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:31.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:31.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3186974754' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:58:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15962/57189 objects degraded (27.911%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:31.626 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:31.648 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:31.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:31.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:31.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3186974754' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:58:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15962/57189 objects degraded (27.911%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3186974754' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:58:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15962/57189 objects degraded (27.911%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:31.749 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:31.772 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:31.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:31.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:31.874 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:31.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:31.941 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:31.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:32.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:32.082 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:32.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:32.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:32.184 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:32.214 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:32.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:32.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:32.316 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:32.354 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:32.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:32.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:32.456 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:32.478 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:32.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:32.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:32.580 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:32 smithi179 ceph-mon[93519]: pgmap v1964: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 451 op/s; 15962/56040 objects degraded (28.483%); 2431/56040 objects misplaced (4.338%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:58:32.609 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:32.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:32.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:32.610 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:32 smithi130 ceph-mon[105521]: pgmap v1964: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 451 op/s; 15962/56040 objects degraded (28.483%); 2431/56040 objects misplaced (4.338%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:58:32.611 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:32 smithi130 ceph-mon[110068]: pgmap v1964: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 451 op/s; 15962/56040 objects degraded (28.483%); 2431/56040 objects misplaced (4.338%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:58:32.711 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:32.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:32.735 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:32.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:32.836 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:32.860 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:32.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:32.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:32.936 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:32.936 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:32.962 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:33.011 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:33.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:33.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:33.113 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:33.145 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:33.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:33.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:33.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:33.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:33.291 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:33.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:33.392 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:33.431 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:33.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:33.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:33.533 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:33.569 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:33.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:33.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:33.671 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:33.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:33.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:33.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:33.796 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:33.818 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:33.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:33.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:33.920 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:33.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:33.959 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:33.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:34.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:34.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:34.104 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:34.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:34.205 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:34.244 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:34.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:34.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:34.346 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:34.370 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:34.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:34.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:34.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:34 smithi130 ceph-mon[110068]: pgmap v1965: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 546 op/s; 16167/50280 objects degraded (32.154%); 2058/50280 objects misplaced (4.093%); 256 KiB/s, 4 objects/s recovering 2024-08-17T17:58:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:34 smithi130 ceph-mon[105521]: pgmap v1965: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 546 op/s; 16167/50280 objects degraded (32.154%); 2058/50280 objects misplaced (4.093%); 256 KiB/s, 4 objects/s recovering 2024-08-17T17:58:34.471 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:34.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:58:34.513 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:34.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:34.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:34.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:34 smithi179 ceph-mon[93519]: pgmap v1965: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 4.4 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 546 op/s; 16167/50280 objects degraded (32.154%); 2058/50280 objects misplaced (4.093%); 256 KiB/s, 4 objects/s recovering 2024-08-17T17:58:34.615 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:34.641 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:34.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:34.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:34.743 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:34.765 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:34.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:34.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:34.867 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:34.889 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:34.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:34.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:34.991 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:35.039 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:35.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:35.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:35.141 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:35.186 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:35.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:35.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:35.287 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:35.312 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:35.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:35.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:35.414 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:35.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:35.447 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:35.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:35.548 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:35.589 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:35.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:35.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:35.690 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:35.714 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:35.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:35.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:35.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:35.885 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:35.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:35.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:35.986 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:36.008 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:36.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:36.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:36.110 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:36.160 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:36.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:36.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:36.261 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:36.335 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:36.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:36.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:36.402 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:58:36.403 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:58:36.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:36.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:36.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:36.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:36.525 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:36 smithi130 ceph-mon[105521]: pgmap v1966: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 975 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 557 op/s; 16167/46842 objects degraded (34.514%); 2058/46842 objects misplaced (4.393%); 239 KiB/s, 4 objects/s recovering 2024-08-17T17:58:36.525 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16167/46842 objects degraded (34.514%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:36.525 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:36 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:58:36.525 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:36 smithi130 ceph-mon[110068]: pgmap v1966: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 975 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 557 op/s; 16167/46842 objects degraded (34.514%); 2058/46842 objects misplaced (4.393%); 239 KiB/s, 4 objects/s recovering 2024-08-17T17:58:36.526 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16167/46842 objects degraded (34.514%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:36.526 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:36 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:58:36.562 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:36.583 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:36.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:36 smithi179 ceph-mon[93519]: pgmap v1966: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 975 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 557 op/s; 16167/46842 objects degraded (34.514%); 2058/46842 objects misplaced (4.393%); 239 KiB/s, 4 objects/s recovering 2024-08-17T17:58:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16167/46842 objects degraded (34.514%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:36 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:58:36.684 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:36.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:36.749 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:36.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:36.850 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:36.875 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:36.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:36.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:36.977 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:36.999 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:36.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:37.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:37.100 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:37.123 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:37.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:37.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:37.225 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:37.249 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:37.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:37.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:37.351 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:37.397 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:37.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:37.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:37.499 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:37.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:37.530 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:37.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:37.631 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:37.654 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:37.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:37.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:37.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:37 smithi130 ceph-mon[110068]: from='client.36141 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:37 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2020100949' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:58:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:37 smithi130 ceph-mon[105521]: from='client.36141 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:37 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2020100949' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:58:37.750 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:37 smithi179 ceph-mon[93519]: from='client.36141 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:37.751 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:37 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2020100949' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:58:37.756 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:37.800 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:37.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:37.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:37.903 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:37.952 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:37.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:37.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:38.054 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:38.066 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:58:38.078 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:38.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:38.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:38.180 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:38.206 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:38.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:38.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:38.308 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:38.345 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:38.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:38.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:38.447 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:38.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:38.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:38.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:38.574 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:38.615 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:38.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:38.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:38.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:38 smithi130 ceph-mon[105521]: pgmap v1967: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 951 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 529 op/s; 16167/45693 objects degraded (35.382%); 1869/45693 objects misplaced (4.090%); 213 KiB/s, 4 objects/s recovering 2024-08-17T17:58:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:38 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:58:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:38 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2020100949' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:58:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:38 smithi130 ceph-mon[105521]: osdmap e1247: 8 total, 8 up, 6 in 2024-08-17T17:58:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:38 smithi130 ceph-mon[110068]: pgmap v1967: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 951 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 529 op/s; 16167/45693 objects degraded (35.382%); 1869/45693 objects misplaced (4.090%); 213 KiB/s, 4 objects/s recovering 2024-08-17T17:58:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:38 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:58:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:38 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2020100949' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:58:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:38 smithi130 ceph-mon[110068]: osdmap e1247: 8 total, 8 up, 6 in 2024-08-17T17:58:38.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:38 smithi179 ceph-mon[93519]: pgmap v1967: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 951 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 529 op/s; 16167/45693 objects degraded (35.382%); 1869/45693 objects misplaced (4.090%); 213 KiB/s, 4 objects/s recovering 2024-08-17T17:58:38.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:38 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:58:38.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:38 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2020100949' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:58:38.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:38 smithi179 ceph-mon[93519]: osdmap e1247: 8 total, 8 up, 6 in 2024-08-17T17:58:38.718 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:38.745 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:38.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:38.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:38.846 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:38.879 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:38.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:38.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:38.980 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:39.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:39.037 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:39.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:39.105 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:58:39.105 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:58:39.138 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:39.161 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:39.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:39.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:39.263 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:39.288 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:39.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:39.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:39.390 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:39.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:39.431 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:39.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:39.532 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:39.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:39.572 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:39.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:39.673 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:39.718 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:39.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:39.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:39.812 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1966,"stamp":"2024-08-17T17:58:35.125164+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1022105573,"num_objects":15614,"num_object_clones":0,"num_object_copies":46842,"num_objects_missing_on_primary":718,"num_objects_missing":718,"num_objects_degraded":16167,"num_objects_misplaced":2058,"num_objects_unfound":0,"num_objects_dirty":15614,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":194300,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38456,"num_bytes_recovered":2341084173,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108703,"ondisk_log_size":108703,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4476708,"kb_used_data":2940660,"kb_used_omap":295,"kb_used_meta":1535704,"kb_avail":557994204,"statfs":{"total":575970213888,"available":571386064896,"internally_reserved":0,"allocated":3011235840,"data_stored":2980889761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1572561359},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,4,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-391708672,"num_objects":-5977,"num_object_clones":0,"num_object_copies":-17931,"num_objects_missing_on_primary":-36,"num_objects_missing":-36,"num_objects_degraded":330,"num_objects_misplaced":-719,"num_objects_unfound":0,"num_objects_dirty":-5977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5977,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":2621440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.730317"},"pg_stats":[{"pgid":"45.f","version":"1246'12100","reported_seq":22673,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:31.978324+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:58:31.978324+0000","last_peered":"2024-08-17T17:58:31.978324+0000","last_clean":"2024-08-17T17:58:31.978324+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:58:31.978324+0000","last_undegraded":"2024-08-17T17:58:31.978324+0000","last_fullsized":"2024-08-17T17:58:31.978324+0000","mapping_epoch":1194,"log_start":"1180'5231","ondisk_log_start":"1180'5231","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6869,"log_dups_size":0,"ondisk_log_size":6869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69730304,"num_objects":1064,"num_object_clones":0,"num_object_copies":3192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12100,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1246'12044","reported_seq":22530,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:32.890085+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:58:32.890085+0000","last_peered":"2024-08-17T17:58:32.890085+0000","last_clean":"2024-08-17T17:58:32.890085+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:58:32.890085+0000","last_undegraded":"2024-08-17T17:58:32.890085+0000","last_fullsized":"2024-08-17T17:58:32.890085+0000","mapping_epoch":1211,"log_start":"1176'5032","ondisk_log_start":"1176'5032","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7012,"log_dups_size":0,"ondisk_log_size":7012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62259200,"num_objects":950,"num_object_clones":0,"num_object_copies":2850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12044,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1246'12462","reported_seq":23342,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:33.797683+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:58:33.797683+0000","last_peered":"2024-08-17T17:58:33.797683+0000","last_clean":"2024-08-17T17:58:33.797683+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:58:33.797683+0000","last_undegraded":"2024-08-17T17:58:33.797683+0000","last_fullsized":"2024-08-17T17:58:33.797683+0000","mapping_epoch":1205,"log_start":"1184'6568","ondisk_log_start":"1184'6568","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5894,"log_dups_size":0,"ondisk_log_size":5894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59965464,"num_objects":916,"num_object_clones":0,"num_object_copies":2748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":916,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12462,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1246'12017","reported_seq":12104,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:32.900427+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:58:32.900427+0000","last_peered":"2024-08-17T17:58:32.900427+0000","last_clean":"2024-08-17T17:58:32.900427+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:58:32.900427+0000","last_undegraded":"2024-08-17T17:58:32.900427+0000","last_fullsized":"2024-08-17T17:58:32.900427+0000","mapping_epoch":1167,"log_start":"1180'5289","ondisk_log_start":"1180'5289","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6728,"log_dups_size":0,"ondisk_log_size":6728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64421888,"num_objects":983,"num_object_clones":0,"num_object_copies":2949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12017,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1246'12082","reported_seq":17368,"reported_epoch":1246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:31.936956+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:58:31.936956+0000","last_peered":"2024-08-17T17:58:31.936956+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:58:31.936956+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1182'6126","ondisk_log_start":"1182'6126","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5956,"log_dups_size":0,"ondisk_log_size":5956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67633152,"num_objects":1032,"num_object_clones":0,"num_object_copies":3096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":518,"num_objects_misplaced":1032,"num_objects_unfound":0,"num_objects_dirty":1032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12082,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":514}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1246'12113","reported_seq":12200,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:32.935050+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:58:32.935050+0000","last_peered":"2024-08-17T17:58:32.935050+0000","last_clean":"2024-08-17T17:58:32.935050+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:58:32.935050+0000","last_undegraded":"2024-08-17T17:58:32.935050+0000","last_fullsized":"2024-08-17T17:58:32.935050+0000","mapping_epoch":1167,"log_start":"1180'5377","ondisk_log_start":"1180'5377","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6736,"log_dups_size":0,"ondisk_log_size":6736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64290816,"num_objects":981,"num_object_clones":0,"num_object_copies":2943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12113,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1246'12232","reported_seq":12319,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:32.923769+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:58:32.923769+0000","last_peered":"2024-08-17T17:58:32.923769+0000","last_clean":"2024-08-17T17:58:32.923769+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:58:32.923769+0000","last_undegraded":"2024-08-17T17:58:32.923769+0000","last_fullsized":"2024-08-17T17:58:32.923769+0000","mapping_epoch":1167,"log_start":"1180'5469","ondisk_log_start":"1180'5469","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6763,"log_dups_size":0,"ondisk_log_size":6763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65142784,"num_objects":994,"num_object_clones":0,"num_object_copies":2982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12232,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1246'11977","reported_seq":21481,"reported_epoch":1246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:31.949526+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:58:31.949526+0000","last_peered":"2024-08-17T17:58:31.949526+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:58:31.949526+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1182'6075","ondisk_log_start":"1182'6075","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5902,"log_dups_size":0,"ondisk_log_size":5902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66387968,"num_objects":1013,"num_object_clones":0,"num_object_copies":3039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11977,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3108,"num_bytes_recovered":203685888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,7","objects":-1054}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1246'12085","reported_seq":12182,"reported_epoch":1246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:32.907439+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:58:32.907439+0000","last_peered":"2024-08-17T17:58:32.907439+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:58:32.907439+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1180'5377","ondisk_log_start":"1180'5377","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6708,"log_dups_size":0,"ondisk_log_size":6708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64946176,"num_objects":991,"num_object_clones":0,"num_object_copies":2973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12085,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":991},{"shards":"1,6","objects":-4842}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1246'12200","reported_seq":15320,"reported_epoch":1246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:33.782070+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:58:33.782070+0000","last_peered":"2024-08-17T17:58:33.782070+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:58:33.782070+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1182'6306","ondisk_log_start":"1182'6306","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5894,"log_dups_size":0,"ondisk_log_size":5894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63832064,"num_objects":974,"num_object_clones":0,"num_object_copies":2922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12200,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":-587}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1246'12092","reported_seq":12267,"reported_epoch":1246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:34.168056+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:58:34.168056+0000","last_peered":"2024-08-17T17:58:34.168056+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:58:34.168056+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1174'3000","ondisk_log_start":"1174'3000","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9092,"log_dups_size":0,"ondisk_log_size":9092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64356352,"num_objects":982,"num_object_clones":0,"num_object_copies":2946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12092,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-4171}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1246'12148","reported_seq":18383,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:34.207043+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:58:34.207043+0000","last_peered":"2024-08-17T17:58:34.207043+0000","last_clean":"2024-08-17T17:58:34.207043+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:58:34.207043+0000","last_undegraded":"2024-08-17T17:58:34.207043+0000","last_fullsized":"2024-08-17T17:58:34.207043+0000","mapping_epoch":1183,"log_start":"1171'3000","ondisk_log_start":"1171'3000","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9148,"log_dups_size":0,"ondisk_log_size":9148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58851328,"num_objects":898,"num_object_clones":0,"num_object_copies":2694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12148,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1246'12238","reported_seq":12412,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:33.739617+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:58:33.739617+0000","last_peered":"2024-08-17T17:58:33.739617+0000","last_clean":"2024-08-17T17:58:33.739617+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:58:33.739617+0000","last_undegraded":"2024-08-17T17:58:33.739617+0000","last_fullsized":"2024-08-17T17:58:33.739617+0000","mapping_epoch":1167,"log_start":"1182'6291","ondisk_log_start":"1182'6291","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5947,"log_dups_size":0,"ondisk_log_size":5947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60424192,"num_objects":922,"num_object_clones":0,"num_object_copies":2766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12238,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1246'11980","reported_seq":12067,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:32.962483+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:58:32.962483+0000","last_peered":"2024-08-17T17:58:32.962483+0000","last_clean":"2024-08-17T17:58:32.962483+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:58:32.962483+0000","last_undegraded":"2024-08-17T17:58:32.962483+0000","last_fullsized":"2024-08-17T17:58:32.962483+0000","mapping_epoch":1167,"log_start":"1176'4925","ondisk_log_start":"1176'4925","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62521344,"num_objects":954,"num_object_clones":0,"num_object_copies":2862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11980,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1246'12338","reported_seq":32269,"reported_epoch":1246,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:32.017431+0000","last_change":"2024-08-17T17:58:23.302199+0000","last_active":"2024-08-17T17:58:32.017431+0000","last_peered":"2024-08-17T17:58:32.017431+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:58:32.017431+0000","last_undegraded":"2024-08-17T17:58:23.294905+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1180'5491","ondisk_log_start":"1180'5491","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6847,"log_dups_size":0,"ondisk_log_size":6847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67239936,"num_objects":1026,"num_object_clones":0,"num_object_copies":3078,"num_objects_missing_on_primary":718,"num_objects_missing":718,"num_objects_degraded":330,"num_objects_misplaced":1026,"num_objects_unfound":0,"num_objects_dirty":1026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12338,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7441,"num_bytes_recovered":487063552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"0,4,6","objects":1026},{"shards":"0,6","objects":-330}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1246'11914","reported_seq":25274,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:33.737352+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:58:33.737352+0000","last_peered":"2024-08-17T17:58:33.737352+0000","last_clean":"2024-08-17T17:58:33.737352+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:58:33.737352+0000","last_undegraded":"2024-08-17T17:58:33.737352+0000","last_fullsized":"2024-08-17T17:58:33.737352+0000","mapping_epoch":1196,"log_start":"1182'5969","ondisk_log_start":"1182'5969","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5945,"log_dups_size":0,"ondisk_log_size":5945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59506688,"num_objects":908,"num_object_clones":0,"num_object_copies":2724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11914,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1950,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.417235+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:58:24.417235+0000","last_peered":"2024-08-17T17:58:24.417235+0000","last_clean":"2024-08-17T17:58:24.417235+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:58:24.417235+0000","last_undegraded":"2024-08-17T17:58:24.417235+0000","last_fullsized":"2024-08-17T17:58:24.417235+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":523,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.437359+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:58:11.437359+0000","last_peered":"2024-08-17T17:58:11.437359+0000","last_clean":"2024-08-17T17:58:11.437359+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:58:11.437359+0000","last_undegraded":"2024-08-17T17:58:11.437359+0000","last_fullsized":"2024-08-17T17:58:11.437359+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1870,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.408632+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:58:24.408632+0000","last_peered":"2024-08-17T17:58:24.408632+0000","last_clean":"2024-08-17T17:58:24.408632+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:58:24.408632+0000","last_undegraded":"2024-08-17T17:58:24.408632+0000","last_fullsized":"2024-08-17T17:58:24.408632+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1894,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.403895+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:58:24.403895+0000","last_peered":"2024-08-17T17:58:24.403895+0000","last_clean":"2024-08-17T17:58:24.403895+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:58:24.403895+0000","last_undegraded":"2024-08-17T17:58:24.403895+0000","last_fullsized":"2024-08-17T17:58:24.403895+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":85,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.437836+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:58:11.437836+0000","last_peered":"2024-08-17T17:58:11.437836+0000","last_clean":"2024-08-17T17:58:11.437836+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:58:11.437836+0000","last_undegraded":"2024-08-17T17:58:11.437836+0000","last_fullsized":"2024-08-17T17:58:11.437836+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1287,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.410806+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:58:24.410806+0000","last_peered":"2024-08-17T17:58:24.410806+0000","last_clean":"2024-08-17T17:58:24.410806+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:58:24.410806+0000","last_undegraded":"2024-08-17T17:58:24.410806+0000","last_fullsized":"2024-08-17T17:58:24.410806+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":145,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.408308+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:58:24.408308+0000","last_peered":"2024-08-17T17:58:24.408308+0000","last_clean":"2024-08-17T17:58:24.408308+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:58:24.408308+0000","last_undegraded":"2024-08-17T17:58:24.408308+0000","last_fullsized":"2024-08-17T17:58:24.408308+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":665,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.404409+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:58:24.404409+0000","last_peered":"2024-08-17T17:58:24.404409+0000","last_clean":"2024-08-17T17:58:24.404409+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:58:24.404409+0000","last_undegraded":"2024-08-17T17:58:24.404409+0000","last_fullsized":"2024-08-17T17:58:24.404409+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1908,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.413281+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:58:24.413281+0000","last_peered":"2024-08-17T17:58:24.413281+0000","last_clean":"2024-08-17T17:58:24.413281+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:58:24.413281+0000","last_undegraded":"2024-08-17T17:58:24.413281+0000","last_fullsized":"2024-08-17T17:58:24.413281+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1910,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.405940+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:58:24.405940+0000","last_peered":"2024-08-17T17:58:24.405940+0000","last_clean":"2024-08-17T17:58:24.405940+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:58:24.405940+0000","last_undegraded":"2024-08-17T17:58:24.405940+0000","last_fullsized":"2024-08-17T17:58:24.405940+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":130,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.407733+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:58:24.407733+0000","last_peered":"2024-08-17T17:58:24.407733+0000","last_clean":"2024-08-17T17:58:24.407733+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:58:24.407733+0000","last_undegraded":"2024-08-17T17:58:24.407733+0000","last_fullsized":"2024-08-17T17:58:24.407733+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1882,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.411154+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:58:24.411154+0000","last_peered":"2024-08-17T17:58:24.411154+0000","last_clean":"2024-08-17T17:58:24.411154+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:58:24.411154+0000","last_undegraded":"2024-08-17T17:58:24.411154+0000","last_fullsized":"2024-08-17T17:58:24.411154+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1937,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.407492+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:58:24.407492+0000","last_peered":"2024-08-17T17:58:24.407492+0000","last_clean":"2024-08-17T17:58:24.407492+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:58:24.407492+0000","last_undegraded":"2024-08-17T17:58:24.407492+0000","last_fullsized":"2024-08-17T17:58:24.407492+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1931,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.416721+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:58:24.416721+0000","last_peered":"2024-08-17T17:58:24.416721+0000","last_clean":"2024-08-17T17:58:24.416721+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:58:24.416721+0000","last_undegraded":"2024-08-17T17:58:24.416721+0000","last_fullsized":"2024-08-17T17:58:24.416721+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1905,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.403555+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:58:24.403555+0000","last_peered":"2024-08-17T17:58:24.403555+0000","last_clean":"2024-08-17T17:58:24.403555+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:58:24.403555+0000","last_undegraded":"2024-08-17T17:58:24.403555+0000","last_fullsized":"2024-08-17T17:58:24.403555+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1220,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.437843+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:58:11.437843+0000","last_peered":"2024-08-17T17:58:11.437843+0000","last_clean":"2024-08-17T17:58:11.437843+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:58:11.437843+0000","last_undegraded":"2024-08-17T17:58:11.437843+0000","last_fullsized":"2024-08-17T17:58:11.437843+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1876,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.403296+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:58:24.403296+0000","last_peered":"2024-08-17T17:58:24.403296+0000","last_clean":"2024-08-17T17:58:24.403296+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:58:24.403296+0000","last_undegraded":"2024-08-17T17:58:24.403296+0000","last_fullsized":"2024-08-17T17:58:24.403296+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1926,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.417260+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:58:24.417260+0000","last_peered":"2024-08-17T17:58:24.417260+0000","last_clean":"2024-08-17T17:58:24.417260+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:58:24.417260+0000","last_undegraded":"2024-08-17T17:58:24.417260+0000","last_fullsized":"2024-08-17T17:58:24.417260+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1832,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.403791+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:58:24.403791+0000","last_peered":"2024-08-17T17:58:24.403791+0000","last_clean":"2024-08-17T17:58:24.403791+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:58:24.403791+0000","last_undegraded":"2024-08-17T17:58:24.403791+0000","last_fullsized":"2024-08-17T17:58:24.403791+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":539,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.462607+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:58:11.462607+0000","last_peered":"2024-08-17T17:58:11.462607+0000","last_clean":"2024-08-17T17:58:11.462607+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:58:11.462607+0000","last_undegraded":"2024-08-17T17:58:11.462607+0000","last_fullsized":"2024-08-17T17:58:11.462607+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1216,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.462671+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:58:11.462671+0000","last_peered":"2024-08-17T17:58:11.462671+0000","last_clean":"2024-08-17T17:58:11.462671+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:58:11.462671+0000","last_undegraded":"2024-08-17T17:58:11.462671+0000","last_fullsized":"2024-08-17T17:58:11.462671+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1930,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.407990+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:58:24.407990+0000","last_peered":"2024-08-17T17:58:24.407990+0000","last_clean":"2024-08-17T17:58:24.407990+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:58:24.407990+0000","last_undegraded":"2024-08-17T17:58:24.407990+0000","last_fullsized":"2024-08-17T17:58:24.407990+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":377,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.464941+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:58:11.464941+0000","last_peered":"2024-08-17T17:58:11.464941+0000","last_clean":"2024-08-17T17:58:11.464941+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:58:11.464941+0000","last_undegraded":"2024-08-17T17:58:11.464941+0000","last_fullsized":"2024-08-17T17:58:11.464941+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":523,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.437644+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:58:11.437644+0000","last_peered":"2024-08-17T17:58:11.437644+0000","last_clean":"2024-08-17T17:58:11.437644+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:58:11.437644+0000","last_undegraded":"2024-08-17T17:58:11.437644+0000","last_fullsized":"2024-08-17T17:58:11.437644+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1201,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.439613+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:58:11.439613+0000","last_peered":"2024-08-17T17:58:11.439613+0000","last_clean":"2024-08-17T17:58:11.439613+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:58:11.439613+0000","last_undegraded":"2024-08-17T17:58:11.439613+0000","last_fullsized":"2024-08-17T17:58:11.439613+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1209,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.437911+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:58:11.437911+0000","last_peered":"2024-08-17T17:58:11.437911+0000","last_clean":"2024-08-17T17:58:11.437911+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:58:11.437911+0000","last_undegraded":"2024-08-17T17:58:11.437911+0000","last_fullsized":"2024-08-17T17:58:11.437911+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1978,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.416475+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:58:24.416475+0000","last_peered":"2024-08-17T17:58:24.416475+0000","last_clean":"2024-08-17T17:58:24.416475+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:58:24.416475+0000","last_undegraded":"2024-08-17T17:58:24.416475+0000","last_fullsized":"2024-08-17T17:58:24.416475+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1862,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.403468+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:58:24.403468+0000","last_peered":"2024-08-17T17:58:24.403468+0000","last_clean":"2024-08-17T17:58:24.403468+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:58:24.403468+0000","last_undegraded":"2024-08-17T17:58:24.403468+0000","last_fullsized":"2024-08-17T17:58:24.403468+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1936,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.408181+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:58:24.408181+0000","last_peered":"2024-08-17T17:58:24.408181+0000","last_clean":"2024-08-17T17:58:24.408181+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:58:24.408181+0000","last_undegraded":"2024-08-17T17:58:24.408181+0000","last_fullsized":"2024-08-17T17:58:24.408181+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1864,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.409878+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:58:24.409878+0000","last_peered":"2024-08-17T17:58:24.409878+0000","last_clean":"2024-08-17T17:58:24.409878+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:58:24.409878+0000","last_undegraded":"2024-08-17T17:58:24.409878+0000","last_fullsized":"2024-08-17T17:58:24.409878+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1896,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.413366+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:58:24.413366+0000","last_peered":"2024-08-17T17:58:24.413366+0000","last_clean":"2024-08-17T17:58:24.413366+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:58:24.413366+0000","last_undegraded":"2024-08-17T17:58:24.413366+0000","last_fullsized":"2024-08-17T17:58:24.413366+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1236,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.462849+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:58:11.462849+0000","last_peered":"2024-08-17T17:58:11.462849+0000","last_clean":"2024-08-17T17:58:11.462849+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:58:11.462849+0000","last_undegraded":"2024-08-17T17:58:11.462849+0000","last_fullsized":"2024-08-17T17:58:11.462849+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1943,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.408440+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:58:24.408440+0000","last_peered":"2024-08-17T17:58:24.408440+0000","last_clean":"2024-08-17T17:58:24.408440+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:58:24.408440+0000","last_undegraded":"2024-08-17T17:58:24.408440+0000","last_fullsized":"2024-08-17T17:58:24.408440+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":130,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.407569+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:58:24.407569+0000","last_peered":"2024-08-17T17:58:24.407569+0000","last_clean":"2024-08-17T17:58:24.407569+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:58:24.407569+0000","last_undegraded":"2024-08-17T17:58:24.407569+0000","last_fullsized":"2024-08-17T17:58:24.407569+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1962,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.408351+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:58:24.408351+0000","last_peered":"2024-08-17T17:58:24.408351+0000","last_clean":"2024-08-17T17:58:24.408351+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:58:24.408351+0000","last_undegraded":"2024-08-17T17:58:24.408351+0000","last_fullsized":"2024-08-17T17:58:24.408351+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":699,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.411026+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:58:24.411026+0000","last_peered":"2024-08-17T17:58:24.411026+0000","last_clean":"2024-08-17T17:58:24.411026+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:58:24.411026+0000","last_undegraded":"2024-08-17T17:58:24.411026+0000","last_fullsized":"2024-08-17T17:58:24.411026+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1943,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.408527+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:58:24.408527+0000","last_peered":"2024-08-17T17:58:24.408527+0000","last_clean":"2024-08-17T17:58:24.408527+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:58:24.408527+0000","last_undegraded":"2024-08-17T17:58:24.408527+0000","last_fullsized":"2024-08-17T17:58:24.408527+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1272,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.413173+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:58:24.413173+0000","last_peered":"2024-08-17T17:58:24.413173+0000","last_clean":"2024-08-17T17:58:24.413173+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:58:24.413173+0000","last_undegraded":"2024-08-17T17:58:24.413173+0000","last_fullsized":"2024-08-17T17:58:24.413173+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":190,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.407939+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:58:24.407939+0000","last_peered":"2024-08-17T17:58:24.407939+0000","last_clean":"2024-08-17T17:58:24.407939+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:58:24.407939+0000","last_undegraded":"2024-08-17T17:58:24.407939+0000","last_fullsized":"2024-08-17T17:58:24.407939+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1953,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.408521+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:58:24.408521+0000","last_peered":"2024-08-17T17:58:24.408521+0000","last_clean":"2024-08-17T17:58:24.408521+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:58:24.408521+0000","last_undegraded":"2024-08-17T17:58:24.408521+0000","last_fullsized":"2024-08-17T17:58:24.408521+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1931,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.409600+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:58:24.409600+0000","last_peered":"2024-08-17T17:58:24.409600+0000","last_clean":"2024-08-17T17:58:24.409600+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:58:24.409600+0000","last_undegraded":"2024-08-17T17:58:24.409600+0000","last_fullsized":"2024-08-17T17:58:24.409600+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1952,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.414949+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:58:24.414949+0000","last_peered":"2024-08-17T17:58:24.414949+0000","last_clean":"2024-08-17T17:58:24.414949+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:58:24.414949+0000","last_undegraded":"2024-08-17T17:58:24.414949+0000","last_fullsized":"2024-08-17T17:58:24.414949+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1957,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.416792+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:58:24.416792+0000","last_peered":"2024-08-17T17:58:24.416792+0000","last_clean":"2024-08-17T17:58:24.416792+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:58:24.416792+0000","last_undegraded":"2024-08-17T17:58:24.416792+0000","last_fullsized":"2024-08-17T17:58:24.416792+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1937,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.442838+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:58:24.442838+0000","last_peered":"2024-08-17T17:58:24.442838+0000","last_clean":"2024-08-17T17:58:24.442838+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:58:24.442838+0000","last_undegraded":"2024-08-17T17:58:24.442838+0000","last_fullsized":"2024-08-17T17:58:24.442838+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1243,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.462403+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:58:11.462403+0000","last_peered":"2024-08-17T17:58:11.462403+0000","last_clean":"2024-08-17T17:58:11.462403+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:58:11.462403+0000","last_undegraded":"2024-08-17T17:58:11.462403+0000","last_fullsized":"2024-08-17T17:58:11.462403+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1189,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.437846+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:58:11.437846+0000","last_peered":"2024-08-17T17:58:11.437846+0000","last_clean":"2024-08-17T17:58:11.437846+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:58:11.437846+0000","last_undegraded":"2024-08-17T17:58:11.437846+0000","last_fullsized":"2024-08-17T17:58:11.437846+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1846,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.410504+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:58:24.410504+0000","last_peered":"2024-08-17T17:58:24.410504+0000","last_clean":"2024-08-17T17:58:24.410504+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:58:24.410504+0000","last_undegraded":"2024-08-17T17:58:24.410504+0000","last_fullsized":"2024-08-17T17:58:24.410504+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":178,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.409672+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:58:24.409672+0000","last_peered":"2024-08-17T17:58:24.409672+0000","last_clean":"2024-08-17T17:58:24.409672+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:58:24.409672+0000","last_undegraded":"2024-08-17T17:58:24.409672+0000","last_fullsized":"2024-08-17T17:58:24.409672+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1555,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.462287+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:58:11.462287+0000","last_peered":"2024-08-17T17:58:11.462287+0000","last_clean":"2024-08-17T17:58:11.462287+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:58:11.462287+0000","last_undegraded":"2024-08-17T17:58:11.462287+0000","last_fullsized":"2024-08-17T17:58:11.462287+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1938,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.408649+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:58:24.408649+0000","last_peered":"2024-08-17T17:58:24.408649+0000","last_clean":"2024-08-17T17:58:24.408649+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:58:24.408649+0000","last_undegraded":"2024-08-17T17:58:24.408649+0000","last_fullsized":"2024-08-17T17:58:24.408649+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1878,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.403379+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:58:24.403379+0000","last_peered":"2024-08-17T17:58:24.403379+0000","last_clean":"2024-08-17T17:58:24.403379+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:58:24.403379+0000","last_undegraded":"2024-08-17T17:58:24.403379+0000","last_fullsized":"2024-08-17T17:58:24.403379+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":523,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.437936+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:58:11.437936+0000","last_peered":"2024-08-17T17:58:11.437936+0000","last_clean":"2024-08-17T17:58:11.437936+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:58:11.437936+0000","last_undegraded":"2024-08-17T17:58:11.437936+0000","last_fullsized":"2024-08-17T17:58:11.437936+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":617,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.425408+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:58:24.425408+0000","last_peered":"2024-08-17T17:58:24.425408+0000","last_clean":"2024-08-17T17:58:24.425408+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:58:24.425408+0000","last_undegraded":"2024-08-17T17:58:24.425408+0000","last_fullsized":"2024-08-17T17:58:24.425408+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":632,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.403433+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:58:24.403433+0000","last_peered":"2024-08-17T17:58:24.403433+0000","last_clean":"2024-08-17T17:58:24.403433+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:58:24.403433+0000","last_undegraded":"2024-08-17T17:58:24.403433+0000","last_fullsized":"2024-08-17T17:58:24.403433+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1207,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.437881+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:58:11.437881+0000","last_peered":"2024-08-17T17:58:11.437881+0000","last_clean":"2024-08-17T17:58:11.437881+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:58:11.437881+0000","last_undegraded":"2024-08-17T17:58:11.437881+0000","last_fullsized":"2024-08-17T17:58:11.437881+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":648,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.404309+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:58:24.404309+0000","last_peered":"2024-08-17T17:58:24.404309+0000","last_clean":"2024-08-17T17:58:24.404309+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:58:24.404309+0000","last_undegraded":"2024-08-17T17:58:24.404309+0000","last_fullsized":"2024-08-17T17:58:24.404309+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1874,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.404486+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:58:24.404486+0000","last_peered":"2024-08-17T17:58:24.404486+0000","last_clean":"2024-08-17T17:58:24.404486+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:58:24.404486+0000","last_undegraded":"2024-08-17T17:58:24.404486+0000","last_fullsized":"2024-08-17T17:58:24.404486+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1970,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.413446+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:58:24.413446+0000","last_peered":"2024-08-17T17:58:24.413446+0000","last_clean":"2024-08-17T17:58:24.413446+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:58:24.413446+0000","last_undegraded":"2024-08-17T17:58:24.413446+0000","last_fullsized":"2024-08-17T17:58:24.413446+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":681,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.404213+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:58:24.404213+0000","last_peered":"2024-08-17T17:58:24.404213+0000","last_clean":"2024-08-17T17:58:24.404213+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:58:24.404213+0000","last_undegraded":"2024-08-17T17:58:24.404213+0000","last_fullsized":"2024-08-17T17:58:24.404213+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1196,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.437536+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:58:11.437536+0000","last_peered":"2024-08-17T17:58:11.437536+0000","last_clean":"2024-08-17T17:58:11.437536+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:58:11.437536+0000","last_undegraded":"2024-08-17T17:58:11.437536+0000","last_fullsized":"2024-08-17T17:58:11.437536+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1195,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.462807+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:58:11.462807+0000","last_peered":"2024-08-17T17:58:11.462807+0000","last_clean":"2024-08-17T17:58:11.462807+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:58:11.462807+0000","last_undegraded":"2024-08-17T17:58:11.462807+0000","last_fullsized":"2024-08-17T17:58:11.462807+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":666,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.407620+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:58:24.407620+0000","last_peered":"2024-08-17T17:58:24.407620+0000","last_clean":"2024-08-17T17:58:24.407620+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:58:24.407620+0000","last_undegraded":"2024-08-17T17:58:24.407620+0000","last_fullsized":"2024-08-17T17:58:24.407620+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1213,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.462742+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:58:11.462742+0000","last_peered":"2024-08-17T17:58:11.462742+0000","last_clean":"2024-08-17T17:58:11.462742+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:58:11.462742+0000","last_undegraded":"2024-08-17T17:58:11.462742+0000","last_fullsized":"2024-08-17T17:58:11.462742+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":97,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.462223+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:58:11.462223+0000","last_peered":"2024-08-17T17:58:11.462223+0000","last_clean":"2024-08-17T17:58:11.462223+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:58:11.462223+0000","last_undegraded":"2024-08-17T17:58:11.462223+0000","last_fullsized":"2024-08-17T17:58:11.462223+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1229,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.462752+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:58:11.462752+0000","last_peered":"2024-08-17T17:58:11.462752+0000","last_clean":"2024-08-17T17:58:11.462752+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:58:11.462752+0000","last_undegraded":"2024-08-17T17:58:11.462752+0000","last_fullsized":"2024-08-17T17:58:11.462752+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1246,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.437491+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:58:11.437491+0000","last_peered":"2024-08-17T17:58:11.437491+0000","last_clean":"2024-08-17T17:58:11.437491+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:58:11.437491+0000","last_undegraded":"2024-08-17T17:58:11.437491+0000","last_fullsized":"2024-08-17T17:58:11.437491+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1876,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.408955+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:58:24.408955+0000","last_peered":"2024-08-17T17:58:24.408955+0000","last_clean":"2024-08-17T17:58:24.408955+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:58:24.408955+0000","last_undegraded":"2024-08-17T17:58:24.408955+0000","last_fullsized":"2024-08-17T17:58:24.408955+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1190,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.462409+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:58:11.462409+0000","last_peered":"2024-08-17T17:58:11.462409+0000","last_clean":"2024-08-17T17:58:11.462409+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:58:11.462409+0000","last_undegraded":"2024-08-17T17:58:11.462409+0000","last_fullsized":"2024-08-17T17:58:11.462409+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1929,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.416590+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:58:24.416590+0000","last_peered":"2024-08-17T17:58:24.416590+0000","last_clean":"2024-08-17T17:58:24.416590+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:58:24.416590+0000","last_undegraded":"2024-08-17T17:58:24.416590+0000","last_fullsized":"2024-08-17T17:58:24.416590+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1930,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.412958+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:58:24.412958+0000","last_peered":"2024-08-17T17:58:24.412958+0000","last_clean":"2024-08-17T17:58:24.412958+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:58:24.412958+0000","last_undegraded":"2024-08-17T17:58:24.412958+0000","last_fullsized":"2024-08-17T17:58:24.412958+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":603,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.462590+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:58:11.462590+0000","last_peered":"2024-08-17T17:58:11.462590+0000","last_clean":"2024-08-17T17:58:11.462590+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:58:11.462590+0000","last_undegraded":"2024-08-17T17:58:11.462590+0000","last_fullsized":"2024-08-17T17:58:11.462590+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1215,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:58:11.462653+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:58:11.462653+0000","last_peered":"2024-08-17T17:58:11.462653+0000","last_clean":"2024-08-17T17:58:11.462653+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:58:11.462653+0000","last_undegraded":"2024-08-17T17:58:11.462653+0000","last_fullsized":"2024-08-17T17:58:11.462653+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1876,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.404427+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:58:24.404427+0000","last_peered":"2024-08-17T17:58:24.404427+0000","last_clean":"2024-08-17T17:58:24.404427+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:58:24.404427+0000","last_undegraded":"2024-08-17T17:58:24.404427+0000","last_fullsized":"2024-08-17T17:58:24.404427+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":1021509656,"num_objects":15588,"num_object_clones":0,"num_object_copies":46764,"num_objects_missing_on_primary":718,"num_objects_missing":718,"num_objects_degraded":16167,"num_objects_misplaced":2058,"num_objects_unfound":0,"num_objects_dirty":15588,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":194022,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38427,"num_bytes_recovered":2339307520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4794232832,"data_stored":4794220616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108496,"ondisk_log_size":108496,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234452,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":767568,"kb_used_data":461896,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":92977584,"statfs":{"total":95995035648,"available":95209046016,"internally_reserved":0,"allocated":472981504,"data_stored":467916015,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496087,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":819220,"kb_used_data":517708,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92925932,"statfs":{"total":95995035648,"available":95156154368,"internally_reserved":0,"allocated":530132992,"data_stored":525063908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757725,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326339,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":815740,"kb_used_data":572340,"kb_used_omap":57,"kb_used_meta":243334,"kb_avail":92929412,"statfs":{"total":95995035648,"available":95159717888,"internally_reserved":0,"allocated":586076160,"data_stored":581025851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":249174779},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405578,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":758532,"kb_used_data":495932,"kb_used_omap":61,"kb_used_meta":262530,"kb_avail":92986620,"statfs":{"total":95995035648,"available":95218298880,"internally_reserved":0,"allocated":507834368,"data_stored":502781031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":268831672},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452112,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599032,"kb_used_data":408944,"kb_used_omap":54,"kb_used_meta":190025,"kb_avail":93146120,"statfs":{"total":95995035648,"available":95381626880,"internally_reserved":0,"allocated":418758656,"data_stored":413712308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":194585998},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474803,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":716616,"kb_used_data":483840,"kb_used_omap":119,"kb_used_meta":232648,"kb_avail":93028536,"statfs":{"total":95995035648,"available":95261220864,"internally_reserved":0,"allocated":495452160,"data_stored":490390648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":238232286},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019360,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":464064512,"data_stored":464060440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":401473536,"data_stored":401473536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":487981056,"data_stored":487981056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":560009216,"data_stored":560005144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":837746688,"data_stored":837746688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":512757760,"data_stored":512753688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":442302464,"data_stored":442302464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:58:39.820 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:39.843 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:39.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:39.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:39.945 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:39.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:39.971 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:39.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:40.072 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:40.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:40.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:40.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:40.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:58:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:58:39] "GET /metrics HTTP/1.1" 200 36803 "" "Prometheus/2.51.0" 2024-08-17T17:58:40.196 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:40.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:40.234 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:40.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:40.335 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:40.358 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:40.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:40.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:40.460 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:40.504 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:40.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:40.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:40.508 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:40 smithi179 ceph-mon[93519]: pgmap v1969: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 807 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 613 op/s; 16367/38766 objects degraded (42.220%); 1669/38766 objects misplaced (4.305%); 0 B/s, 2 objects/s recovering 2024-08-17T17:58:40.508 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:58:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:58:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:58:40.603 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:40 smithi130 ceph-mon[105521]: pgmap v1969: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 807 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 613 op/s; 16367/38766 objects degraded (42.220%); 1669/38766 objects misplaced (4.305%); 0 B/s, 2 objects/s recovering 2024-08-17T17:58:40.603 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:40 smithi130 ceph-mon[110068]: pgmap v1969: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 807 MiB data, 3.8 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 613 op/s; 16367/38766 objects degraded (42.220%); 1669/38766 objects misplaced (4.305%); 0 B/s, 2 objects/s recovering 2024-08-17T17:58:40.606 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:40.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:40.641 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:40.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:40.650 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:58:40.672 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:40.742 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:40.764 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:40.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:40.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:40.836 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:58:40.866 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:40.922 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:40.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:40.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:41.024 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:41.046 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:41.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:41.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:41.148 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:41.174 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:41.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:41.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:41.276 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:41.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:41.355 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:41.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:41.456 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:41.479 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:41.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:41.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:41.581 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:41.619 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:41.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:41.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:41 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16367/38766 objects degraded (42.220%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:41 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:41 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:41 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16367/38766 objects degraded (42.220%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:41 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:41.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:41 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:41.721 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:41.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:41.775 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:41.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:41 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16367/38766 objects degraded (42.220%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:41 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:41 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:41.875 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:41.916 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:41.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:41.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:42.018 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:42.044 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:42.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:42.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:42.146 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:42.185 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:42.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:42.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:42.287 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:42.331 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:42.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:42.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:42.433 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:42.456 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:42.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:42.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:42.558 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:42.582 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:42.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:42.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:42.684 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:42.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:42.709 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:42.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:42.810 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:42.817 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:58:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:42 smithi179 ceph-mon[93519]: pgmap v1970: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 782 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 16367/37602 objects degraded (43.527%); 1669/37602 objects misplaced (4.439%); 0 B/s, 2 objects/s recovering 2024-08-17T17:58:42.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:42.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:42.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:42.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:58:42.853 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:42.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:42.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:42.954 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:42.976 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:42.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:42.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:43.078 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:43.132 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:43.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:43.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:43.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:42 smithi130 ceph-mon[110068]: pgmap v1970: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 782 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 16367/37602 objects degraded (43.527%); 1669/37602 objects misplaced (4.439%); 0 B/s, 2 objects/s recovering 2024-08-17T17:58:43.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:43.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:43.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:43.171 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:58:43.177 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:42 smithi130 ceph-mon[105521]: pgmap v1970: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 782 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 614 op/s; 16367/37602 objects degraded (43.527%); 1669/37602 objects misplaced (4.439%); 0 B/s, 2 objects/s recovering 2024-08-17T17:58:43.177 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:43.177 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:43.177 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:43.177 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:58:43.235 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:43.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:43.300 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:43.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:43.401 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:43.425 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:43.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:43.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:43.527 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:43.563 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:43.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:43.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:43.665 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:43.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:43.699 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:43.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:43.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:43.849 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:43.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:43.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:43.950 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:43.976 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:43.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:43.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:44.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:44.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:44.106 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:44.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:44.208 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:44.230 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:44.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:44.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:44.289 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:44.331 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:44.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:44.384 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:44.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:44.485 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:44.569 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:44.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:44.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:44.672 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:44.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:44.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:44.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:44.797 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:44.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:44 smithi179 ceph-mon[93519]: pgmap v1971: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 665 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 16557/31983 objects degraded (51.768%); 1297/31983 objects misplaced (4.055%); 0 B/s, 4 objects/s recovering 2024-08-17T17:58:44.846 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:44.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:44.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:44.948 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:44.974 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:44 smithi130 ceph-mon[105521]: pgmap v1971: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 665 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 16557/31983 objects degraded (51.768%); 1297/31983 objects misplaced (4.055%); 0 B/s, 4 objects/s recovering 2024-08-17T17:58:44.975 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:44 smithi130 ceph-mon[110068]: pgmap v1971: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 665 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 16557/31983 objects degraded (51.768%); 1297/31983 objects misplaced (4.055%); 0 B/s, 4 objects/s recovering 2024-08-17T17:58:45.013 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:45.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:45.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:45.114 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:45.159 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:45.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:45.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:45.261 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:45.284 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:45.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:45.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:45.386 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:45.400 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:58:45.400 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:58:45.400 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:58:45.401 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:58:45.401 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:58:45.401 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:58:45.401 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:58:45.401 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:58:45.401 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:58:45.401 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:58:45.402 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:58:45.402 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:58:45.402 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:58:45.402 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:58:45.402 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:58:45.402 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:58:45.421 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:45.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:45.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:45.526 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:45.549 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:45.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:45.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:45.607 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:45.650 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:45.672 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:45.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:45.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:45.774 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:45.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:45.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:45.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:45.923 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:45.946 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:45.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:45.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:46.048 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:46.087 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:46.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:46.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:45 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/592274817' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:58:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:45 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:46.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:45 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:46.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:45 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/592274817' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:58:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:45 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:46.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:45 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:45 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/592274817' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:58:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:45 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:46.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:45 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:58:46.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:46.211 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:46.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:46.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:46.312 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:46.336 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:46.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:46.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:46.438 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:46.478 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:46.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:46.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:46.579 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:46.619 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:46.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:46.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:46.721 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:46.768 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:46.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:46.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:46 smithi179 ceph-mon[93519]: pgmap v1972: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 597 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 16557/28716 objects degraded (57.658%); 1297/28716 objects misplaced (4.517%); 0 B/s, 4 objects/s recovering 2024-08-17T17:58:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16557/28716 objects degraded (57.658%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:46.867 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:46.871 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:46.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:46.925 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:46.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:46.989 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:46 smithi130 ceph-mon[105521]: pgmap v1972: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 597 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 16557/28716 objects degraded (57.658%); 1297/28716 objects misplaced (4.517%); 0 B/s, 4 objects/s recovering 2024-08-17T17:58:46.989 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16557/28716 objects degraded (57.658%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:46.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:46 smithi130 ceph-mon[110068]: pgmap v1972: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 597 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 16557/28716 objects degraded (57.658%); 1297/28716 objects misplaced (4.517%); 0 B/s, 4 objects/s recovering 2024-08-17T17:58:46.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16557/28716 objects degraded (57.658%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:47.027 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:47.084 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:47.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:47.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:47.185 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:47.233 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:47.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:47.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:47.338 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:47.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:47.386 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:47.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:47.487 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:47.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:47.567 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:47.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:47.668 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:47.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:47.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:47.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:47.797 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:47.819 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:47.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:47.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:47.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:47.955 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:47.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:47.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:48.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:48.093 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:48.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:48.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:48.195 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:48.228 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:48.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:48.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:48.267 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:58:48.330 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:48.377 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:48.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:48.478 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:48.501 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:48.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:48.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:48.602 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:48.641 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:48.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:48.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:48.743 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:48.789 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:48.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:48.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:48.891 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:48.939 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:48.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:48.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:49.040 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:49.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:49.081 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:49.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:48 smithi179 ceph-mon[93519]: pgmap v1973: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 573 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 604 op/s; 16497/27540 objects degraded (59.902%); 1154/27540 objects misplaced (4.190%); 0 B/s, 4 objects/s recovering 2024-08-17T17:58:49.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:48 smithi130 ceph-mon[105521]: pgmap v1973: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 573 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 604 op/s; 16497/27540 objects degraded (59.902%); 1154/27540 objects misplaced (4.190%); 0 B/s, 4 objects/s recovering 2024-08-17T17:58:49.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:48 smithi130 ceph-mon[110068]: pgmap v1973: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 573 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 604 op/s; 16497/27540 objects degraded (59.902%); 1154/27540 objects misplaced (4.190%); 0 B/s, 4 objects/s recovering 2024-08-17T17:58:49.182 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:49.205 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:49.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:49.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:49.307 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:49.330 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:49.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:49.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:49.385 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:58:49.385 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T17:58:49.385 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T17:58:49.385 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T17:58:49.385 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.385 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:106932 45.8 45:18d24a25:::benchmark_data_smithi130_172551_object2125:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.385 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:23.289066+0000", 2024-08-17T17:58:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 566.09162918899995, 2024-08-17T17:58:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22737446, 2024-08-17T17:58:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 106932 2024-08-17T17:58:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.386 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289066+0000", 2024-08-17T17:58:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289066+0000", 2024-08-17T17:58:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289068+0000", 2024-08-17T17:58:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2670000000000001e-06 2024-08-17T17:58:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.387 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289074+0000", 2024-08-17T17:58:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1750000000000002e-06 2024-08-17T17:58:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289087+0000", 2024-08-17T17:58:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2638e-05 2024-08-17T17:58:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.289094+0000", 2024-08-17T17:58:49.388 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7719999999999997e-06 2024-08-17T17:58:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.363175+0000", 2024-08-17T17:58:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.074080928000000004 2024-08-17T17:58:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.365629+0000", 2024-08-17T17:58:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002453864 2024-08-17T17:58:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.389 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.417171+0000", 2024-08-17T17:58:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.051542814999999999 2024-08-17T17:58:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.417273+0000", 2024-08-17T17:58:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000101457 2024-08-17T17:58:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.417423+0000", 2024-08-17T17:58:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000149846 2024-08-17T17:58:49.390 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.452410+0000", 2024-08-17T17:58:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.034987799 2024-08-17T17:58:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.511559+0000", 2024-08-17T17:58:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.059148575000000002 2024-08-17T17:58:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.511606+0000", 2024-08-17T17:58:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7194000000000002e-05 2024-08-17T17:58:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.511673+0000", 2024-08-17T17:58:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6995999999999993e-05 2024-08-17T17:58:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.516440+0000", 2024-08-17T17:58:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0047671340000000001 2024-08-17T17:58:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.392 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:106967 45.8 45:13e64e6c:::benchmark_data_smithi130_172551_object2160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:23.378357+0000", 2024-08-17T17:58:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 566.00233796199996, 2024-08-17T17:58:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.279186029, 2024-08-17T17:58:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.393 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 106967 2024-08-17T17:58:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378357+0000", 2024-08-17T17:58:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378357+0000", 2024-08-17T17:58:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.394 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378359+0000", 2024-08-17T17:58:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1780000000000002e-06 2024-08-17T17:58:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378367+0000", 2024-08-17T17:58:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3649999999999998e-06 2024-08-17T17:58:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378387+0000", 2024-08-17T17:58:49.395 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9945999999999998e-05 2024-08-17T17:58:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.378396+0000", 2024-08-17T17:58:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6420000000000002e-06 2024-08-17T17:58:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.534504+0000", 2024-08-17T17:58:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.15610795299999999 2024-08-17T17:58:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.396 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.537672+0000", 2024-08-17T17:58:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0031681180000000001 2024-08-17T17:58:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.580200+0000", 2024-08-17T17:58:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042527593000000002 2024-08-17T17:58:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.580441+0000", 2024-08-17T17:58:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00024059999999999999 2024-08-17T17:58:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.397 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.580882+0000", 2024-08-17T17:58:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00044118599999999999 2024-08-17T17:58:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.612775+0000", 2024-08-17T17:58:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.031893171999999997 2024-08-17T17:58:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.398 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.653587+0000", 2024-08-17T17:58:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.040812290000000001 2024-08-17T17:58:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.653642+0000", 2024-08-17T17:58:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4500999999999999e-05 2024-08-17T17:58:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.653792+0000", 2024-08-17T17:58:49.399 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000150235 2024-08-17T17:58:49.400 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.400 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.400 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.400 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.657543+0000", 2024-08-17T17:58:49.400 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0037512499999999998 2024-08-17T17:58:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:106987 45.8 45:18bd4662:::benchmark_data_smithi130_172551_object2180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:23.445589+0000", 2024-08-17T17:58:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 565.93510572499997, 2024-08-17T17:58:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.21594681099999999, 2024-08-17T17:58:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.401 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 106987 2024-08-17T17:58:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445589+0000", 2024-08-17T17:58:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.402 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445589+0000", 2024-08-17T17:58:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445592+0000", 2024-08-17T17:58:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.418e-06 2024-08-17T17:58:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445599+0000", 2024-08-17T17:58:49.403 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.588e-06 2024-08-17T17:58:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445616+0000", 2024-08-17T17:58:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6401000000000001e-05 2024-08-17T17:58:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.445626+0000", 2024-08-17T17:58:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0465000000000001e-05 2024-08-17T17:58:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.404 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.537729+0000", 2024-08-17T17:58:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.092102937999999995 2024-08-17T17:58:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.539000+0000", 2024-08-17T17:58:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0012709609999999999 2024-08-17T17:58:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.595265+0000", 2024-08-17T17:58:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.056264464 2024-08-17T17:58:49.405 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.595396+0000", 2024-08-17T17:58:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000130996 2024-08-17T17:58:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.595582+0000", 2024-08-17T17:58:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000186097 2024-08-17T17:58:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.406 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.612782+0000", 2024-08-17T17:58:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017199941 2024-08-17T17:58:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.657596+0000", 2024-08-17T17:58:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.044814639000000003 2024-08-17T17:58:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.657637+0000", 2024-08-17T17:58:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1032999999999998e-05 2024-08-17T17:58:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.407 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.657728+0000", 2024-08-17T17:58:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0958000000000006e-05 2024-08-17T17:58:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.661536+0000", 2024-08-17T17:58:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0038079120000000001 2024-08-17T17:58:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.408 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:107041 45.8 45:11e0ef02:::benchmark_data_smithi130_172551_object2234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:23.556499+0000", 2024-08-17T17:58:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 565.82419577400003, 2024-08-17T17:58:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.225671231, 2024-08-17T17:58:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 107041 2024-08-17T17:58:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.409 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556499+0000", 2024-08-17T17:58:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556499+0000", 2024-08-17T17:58:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.410 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556501+0000", 2024-08-17T17:58:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1670000000000002e-06 2024-08-17T17:58:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556508+0000", 2024-08-17T17:58:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9430000000000003e-06 2024-08-17T17:58:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556528+0000", 2024-08-17T17:58:49.411 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.999e-05 2024-08-17T17:58:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.556537+0000", 2024-08-17T17:58:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5890000000000003e-06 2024-08-17T17:58:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.674728+0000", 2024-08-17T17:58:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.118190689 2024-08-17T17:58:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.412 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.675674+0000", 2024-08-17T17:58:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00094605799999999995 2024-08-17T17:58:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.709070+0000", 2024-08-17T17:58:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.033396239000000001 2024-08-17T17:58:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.709171+0000", 2024-08-17T17:58:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010085 2024-08-17T17:58:49.413 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.709302+0000", 2024-08-17T17:58:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013068800000000001 2024-08-17T17:58:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.724080+0000", 2024-08-17T17:58:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014778964 2024-08-17T17:58:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.414 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.727981+0000", 2024-08-17T17:58:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039007310000000002 2024-08-17T17:58:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.739624+0000", 2024-08-17T17:58:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011642857 2024-08-17T17:58:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.415 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.416 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.739859+0000", 2024-08-17T17:58:49.416 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000234959 2024-08-17T17:58:49.416 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.416 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.416 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:23.782171+0000", 2024-08-17T17:58:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042311506999999998 2024-08-17T17:58:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108281 45.8 45:17388508:::benchmark_data_smithi130_172551_object3474:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:26.195227+0000", 2024-08-17T17:58:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 563.18546798, 2024-08-17T17:58:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.232226402, 2024-08-17T17:58:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.417 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108281 2024-08-17T17:58:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195227+0000", 2024-08-17T17:58:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.418 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195227+0000", 2024-08-17T17:58:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195229+0000", 2024-08-17T17:58:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4449999999999999e-06 2024-08-17T17:58:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.419 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195235+0000", 2024-08-17T17:58:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5440000000000004e-06 2024-08-17T17:58:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195247+0000", 2024-08-17T17:58:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2245999999999999e-05 2024-08-17T17:58:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.195255+0000", 2024-08-17T17:58:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6790000000000005e-06 2024-08-17T17:58:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.420 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.312742+0000", 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.11748729500000001 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.313545+0000", 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00080311800000000002 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.355977+0000", 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042431218999999999 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.421 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.356037+0000", 2024-08-17T17:58:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9966000000000002e-05 2024-08-17T17:58:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.356111+0000", 2024-08-17T17:58:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4802999999999995e-05 2024-08-17T17:58:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.422 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.377134+0000", 2024-08-17T17:58:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021022968 2024-08-17T17:58:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.425050+0000", 2024-08-17T17:58:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.047915439999999997 2024-08-17T17:58:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.425098+0000", 2024-08-17T17:58:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7867000000000001e-05 2024-08-17T17:58:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.423 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.424 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.425 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.425140+0000", 2024-08-17T17:58:49.425 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2126000000000003e-05 2024-08-17T17:58:49.425 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.425 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.425 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.425 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:26.427454+0000", 2024-08-17T17:58:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0023136860000000001 2024-08-17T17:58:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108681 45.8 45:10018309:::benchmark_data_smithi130_172551_object3874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:27.139549+0000", 2024-08-17T17:58:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 562.24114571600001, 2024-08-17T17:58:49.426 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.21731466799999999, 2024-08-17T17:58:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108681 2024-08-17T17:58:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139549+0000", 2024-08-17T17:58:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.427 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139549+0000", 2024-08-17T17:58:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139550+0000", 2024-08-17T17:58:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4600000000000005e-07 2024-08-17T17:58:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139553+0000", 2024-08-17T17:58:49.428 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.774e-06 2024-08-17T17:58:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139560+0000", 2024-08-17T17:58:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4089999999999997e-06 2024-08-17T17:58:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.139565+0000", 2024-08-17T17:58:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3059999999999997e-06 2024-08-17T17:58:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.429 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.261331+0000", 2024-08-17T17:58:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.121765262 2024-08-17T17:58:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.262881+0000", 2024-08-17T17:58:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0015505829999999999 2024-08-17T17:58:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.294557+0000", 2024-08-17T17:58:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.031675523999999997 2024-08-17T17:58:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.430 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.294741+0000", 2024-08-17T17:58:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000183833 2024-08-17T17:58:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.294884+0000", 2024-08-17T17:58:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014387499999999999 2024-08-17T17:58:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.320518+0000", 2024-08-17T17:58:49.431 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025633501 2024-08-17T17:58:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.320533+0000", 2024-08-17T17:58:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4738e-05 2024-08-17T17:58:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.352852+0000", 2024-08-17T17:58:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.032319379000000002 2024-08-17T17:58:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.432 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.352917+0000", 2024-08-17T17:58:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4584000000000004e-05 2024-08-17T17:58:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.356864+0000", 2024-08-17T17:58:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039473540000000001 2024-08-17T17:58:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.433 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108685 45.8 45:188d0d10:::benchmark_data_smithi130_172551_object3878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:27.141094+0000", 2024-08-17T17:58:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 562.23960136000005, 2024-08-17T17:58:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22079505299999999, 2024-08-17T17:58:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108685 2024-08-17T17:58:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.434 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141094+0000", 2024-08-17T17:58:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141094+0000", 2024-08-17T17:58:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141095+0000", 2024-08-17T17:58:49.435 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.469e-06 2024-08-17T17:58:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141100+0000", 2024-08-17T17:58:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2519999999999999e-06 2024-08-17T17:58:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141115+0000", 2024-08-17T17:58:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4592e-05 2024-08-17T17:58:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.436 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.141123+0000", 2024-08-17T17:58:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0369999999999995e-06 2024-08-17T17:58:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.262940+0000", 2024-08-17T17:58:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.12181718800000001 2024-08-17T17:58:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.265083+0000", 2024-08-17T17:58:49.437 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021428350000000001 2024-08-17T17:58:49.438 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.438 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.438 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.438 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.309940+0000", 2024-08-17T17:58:49.438 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.044856832999999999 2024-08-17T17:58:49.438 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.438 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.438 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.438 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.310048+0000", 2024-08-17T17:58:49.438 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000107886 2024-08-17T17:58:49.438 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.438 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.438 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.439 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.310277+0000", 2024-08-17T17:58:49.439 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00022885700000000001 2024-08-17T17:58:49.439 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.439 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.439 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.439 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.357133+0000", 2024-08-17T17:58:49.439 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.046856061999999997 2024-08-17T17:58:49.439 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.439 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.439 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.439 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.357191+0000", 2024-08-17T17:58:49.439 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8743999999999999e-05 2024-08-17T17:58:49.439 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.439 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.357325+0000", 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013331899999999999 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.357474+0000", 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000148796 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.361889+0000", 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0044151829999999996 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.440 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.441 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.441 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.441 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.441 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:108963 45.8 45:1d275979:::benchmark_data_smithi130_172551_object4156:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.441 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:27.755594+0000", 2024-08-17T17:58:49.441 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 561.62510103399995, 2024-08-17T17:58:49.441 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.27754511999999998, 2024-08-17T17:58:49.441 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.441 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.441 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.441 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.441 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.441 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.442 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 108963 2024-08-17T17:58:49.442 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.442 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.442 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.442 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.442 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755594+0000", 2024-08-17T17:58:49.442 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.442 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.442 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.442 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.442 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755594+0000", 2024-08-17T17:58:49.442 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.443 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.443 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.443 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.443 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755595+0000", 2024-08-17T17:58:49.443 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4799999999999997e-07 2024-08-17T17:58:49.443 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.443 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.443 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.443 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755598+0000", 2024-08-17T17:58:49.443 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5599999999999998e-06 2024-08-17T17:58:49.443 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.443 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.444 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.444 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755610+0000", 2024-08-17T17:58:49.444 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.184e-05 2024-08-17T17:58:49.444 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.444 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.444 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.444 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.755617+0000", 2024-08-17T17:58:49.444 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5950000000000004e-06 2024-08-17T17:58:49.444 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.444 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.444 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.444 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.841456+0000", 2024-08-17T17:58:49.444 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.085839020000000002 2024-08-17T17:58:49.444 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.445 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.445 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.445 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.846882+0000", 2024-08-17T17:58:49.445 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0054257439999999997 2024-08-17T17:58:49.445 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.445 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.445 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.445 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.875006+0000", 2024-08-17T17:58:49.445 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028124712999999999 2024-08-17T17:58:49.445 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.445 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.445 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.445 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.875105+0000", 2024-08-17T17:58:49.445 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8784999999999997e-05 2024-08-17T17:58:49.446 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.446 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.446 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.446 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.875242+0000", 2024-08-17T17:58:49.446 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013735599999999999 2024-08-17T17:58:49.446 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.446 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.446 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.446 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.028166+0000", 2024-08-17T17:58:49.446 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.15292386199999999 2024-08-17T17:58:49.446 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.446 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.446 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.447 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.028216+0000", 2024-08-17T17:58:49.447 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9851000000000001e-05 2024-08-17T17:58:49.447 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.447 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.447 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.447 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.028272+0000", 2024-08-17T17:58:49.447 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5674999999999998e-05 2024-08-17T17:58:49.447 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.447 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.447 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.447 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.029209+0000", 2024-08-17T17:58:49.447 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00093682800000000001 2024-08-17T17:58:49.448 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.448 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.448 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.448 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.033139+0000", 2024-08-17T17:58:49.448 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039305429999999999 2024-08-17T17:58:49.448 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.448 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.448 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.448 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.448 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.448 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109021 45.8 45:14c0b812:::benchmark_data_smithi130_172551_object4214:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.448 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:27.872171+0000", 2024-08-17T17:58:49.449 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 561.50852397999995, 2024-08-17T17:58:49.449 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22437405799999999, 2024-08-17T17:58:49.449 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.449 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.449 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.449 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.449 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.449 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.449 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109021 2024-08-17T17:58:49.449 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.449 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.449 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.449 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.449 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872171+0000", 2024-08-17T17:58:49.450 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.450 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.450 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.450 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.450 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872171+0000", 2024-08-17T17:58:49.450 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.450 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.450 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.450 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.450 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872173+0000", 2024-08-17T17:58:49.450 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.922e-06 2024-08-17T17:58:49.450 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.451 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.451 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.451 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872180+0000", 2024-08-17T17:58:49.451 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5899999999999996e-06 2024-08-17T17:58:49.451 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.451 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.451 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.451 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872192+0000", 2024-08-17T17:58:49.451 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2275e-05 2024-08-17T17:58:49.451 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.451 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.451 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.451 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:27.872199+0000", 2024-08-17T17:58:49.451 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7750000000000004e-06 2024-08-17T17:58:49.452 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.452 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.452 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.452 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.041304+0000", 2024-08-17T17:58:49.452 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16910571499999999 2024-08-17T17:58:49.452 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.452 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.452 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.452 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.044079+0000", 2024-08-17T17:58:49.452 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027747480000000001 2024-08-17T17:58:49.452 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.452 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.452 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.453 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.068075+0000", 2024-08-17T17:58:49.453 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023995408999999999 2024-08-17T17:58:49.453 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.453 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.453 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.453 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.068206+0000", 2024-08-17T17:58:49.453 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013108299999999999 2024-08-17T17:58:49.453 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.453 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.453 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.453 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.068334+0000", 2024-08-17T17:58:49.453 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000128773 2024-08-17T17:58:49.453 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.454 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.454 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.454 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.084130+0000", 2024-08-17T17:58:49.454 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015795113999999999 2024-08-17T17:58:49.454 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.454 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.454 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.454 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.084175+0000", 2024-08-17T17:58:49.454 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5321000000000002e-05 2024-08-17T17:58:49.454 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.454 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.454 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.454 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.092490+0000", 2024-08-17T17:58:49.455 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0083154240000000001 2024-08-17T17:58:49.455 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.455 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.455 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.455 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.092654+0000", 2024-08-17T17:58:49.455 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000163599 2024-08-17T17:58:49.455 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.455 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.455 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.455 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.096545+0000", 2024-08-17T17:58:49.455 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0038913099999999998 2024-08-17T17:58:49.455 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.455 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.456 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.456 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.456 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.456 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109270 45.8 45:15211971:::benchmark_data_smithi130_172551_object4463:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.456 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.480079+0000", 2024-08-17T17:58:49.456 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 560.90061561000005, 2024-08-17T17:58:49.456 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.29654451799999998, 2024-08-17T17:58:49.456 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.456 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.456 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.456 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.456 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.457 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.457 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109270 2024-08-17T17:58:49.457 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.457 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.457 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.457 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.457 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480079+0000", 2024-08-17T17:58:49.457 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.457 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.457 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.457 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.457 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480079+0000", 2024-08-17T17:58:49.457 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.458 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.458 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.458 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.458 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480081+0000", 2024-08-17T17:58:49.458 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3659999999999999e-06 2024-08-17T17:58:49.458 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.458 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.458 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.458 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480086+0000", 2024-08-17T17:58:49.458 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2290000000000002e-06 2024-08-17T17:58:49.458 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.458 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.459 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.459 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480102+0000", 2024-08-17T17:58:49.459 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6399e-05 2024-08-17T17:58:49.459 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.459 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.459 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.459 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.480110+0000", 2024-08-17T17:58:49.459 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8390000000000007e-06 2024-08-17T17:58:49.459 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.459 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.459 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.459 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.574022+0000", 2024-08-17T17:58:49.459 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.093911290999999994 2024-08-17T17:58:49.460 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.460 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.460 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.460 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.574696+0000", 2024-08-17T17:58:49.460 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00067411700000000001 2024-08-17T17:58:49.460 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.460 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.460 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.460 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.719056+0000", 2024-08-17T17:58:49.460 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.144360194 2024-08-17T17:58:49.460 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.460 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.460 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.461 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.719205+0000", 2024-08-17T17:58:49.461 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014905499999999999 2024-08-17T17:58:49.461 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.461 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.461 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.461 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.719356+0000", 2024-08-17T17:58:49.461 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015102599999999999 2024-08-17T17:58:49.461 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.461 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.461 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.461 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.738239+0000", 2024-08-17T17:58:49.461 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018882638 2024-08-17T17:58:49.462 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:49.463 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.463 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.463 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.463 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.772530+0000", 2024-08-17T17:58:49.464 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.034291508999999998 2024-08-17T17:58:49.464 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.464 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.464 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.464 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.772581+0000", 2024-08-17T17:58:49.464 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0745000000000003e-05 2024-08-17T17:58:49.464 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.464 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.464 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.464 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.772702+0000", 2024-08-17T17:58:49.464 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012133600000000001 2024-08-17T17:58:49.465 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.465 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.465 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.465 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.776624+0000", 2024-08-17T17:58:49.465 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039217740000000003 2024-08-17T17:58:49.465 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.465 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.465 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.465 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.465 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.466 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109286 45.8 45:1465a8ca:::benchmark_data_smithi130_172551_object4479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.466 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.517695+0000", 2024-08-17T17:58:49.466 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 560.86300015699999, 2024-08-17T17:58:49.466 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26377572999999999, 2024-08-17T17:58:49.466 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.466 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.466 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.466 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.466 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.466 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.466 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109286 2024-08-17T17:58:49.466 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.467 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.467 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.467 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.467 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517695+0000", 2024-08-17T17:58:49.467 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.467 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.467 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.467 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.467 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517695+0000", 2024-08-17T17:58:49.467 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.467 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.467 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.467 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.468 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517697+0000", 2024-08-17T17:58:49.468 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7290000000000001e-06 2024-08-17T17:58:49.468 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.468 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.468 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.468 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517702+0000", 2024-08-17T17:58:49.468 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0780000000000003e-06 2024-08-17T17:58:49.468 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.468 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.468 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.468 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517710+0000", 2024-08-17T17:58:49.468 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7439999999999996e-06 2024-08-17T17:58:49.469 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.469 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.469 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.469 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517717+0000", 2024-08-17T17:58:49.469 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0769999999999996e-06 2024-08-17T17:58:49.469 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.469 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.469 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.469 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.574731+0000", 2024-08-17T17:58:49.469 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.057014505 2024-08-17T17:58:49.469 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.469 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.469 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.470 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.575221+0000", 2024-08-17T17:58:49.470 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00049007399999999996 2024-08-17T17:58:49.470 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.470 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.470 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.470 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.738278+0000", 2024-08-17T17:58:49.470 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16305646800000001 2024-08-17T17:58:49.470 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.470 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.470 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.470 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.738388+0000", 2024-08-17T17:58:49.470 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010992700000000001 2024-08-17T17:58:49.471 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.471 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.471 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.471 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.738536+0000", 2024-08-17T17:58:49.471 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014820799999999999 2024-08-17T17:58:49.471 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.471 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.471 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.471 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.753488+0000", 2024-08-17T17:58:49.471 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014951951999999999 2024-08-17T17:58:49.471 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.471 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.471 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.471 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.776680+0000", 2024-08-17T17:58:49.472 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023192260999999999 2024-08-17T17:58:49.472 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.472 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.472 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.472 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.776719+0000", 2024-08-17T17:58:49.472 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8858e-05 2024-08-17T17:58:49.472 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.472 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.472 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.472 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.776829+0000", 2024-08-17T17:58:49.472 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011035399999999999 2024-08-17T17:58:49.472 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.472 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.473 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.473 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.781471+0000", 2024-08-17T17:58:49.473 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046414949999999998 2024-08-17T17:58:49.473 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.473 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.473 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.473 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.473 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.473 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109287 45.8 45:1ce0b8c0:::benchmark_data_smithi130_172551_object4480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.473 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.517973+0000", 2024-08-17T17:58:49.473 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 560.86272236800005, 2024-08-17T17:58:49.473 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.27116035399999999, 2024-08-17T17:58:49.473 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.474 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.474 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.474 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.474 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.474 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.474 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109287 2024-08-17T17:58:49.474 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.474 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.474 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.474 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.474 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517973+0000", 2024-08-17T17:58:49.474 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.475 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.475 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.475 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.475 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517973+0000", 2024-08-17T17:58:49.475 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.475 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.475 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.475 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.475 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517974+0000", 2024-08-17T17:58:49.475 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0780000000000001e-06 2024-08-17T17:58:49.475 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.475 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.475 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.475 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517979+0000", 2024-08-17T17:58:49.476 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.967e-06 2024-08-17T17:58:49.476 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.476 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.476 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.476 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.517995+0000", 2024-08-17T17:58:49.476 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6325e-05 2024-08-17T17:58:49.476 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.476 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.476 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.476 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.518003+0000", 2024-08-17T17:58:49.476 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9559999999999997e-06 2024-08-17T17:58:49.476 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.476 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.477 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.477 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.575256+0000", 2024-08-17T17:58:49.477 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.057252589 2024-08-17T17:58:49.477 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.477 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.477 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.477 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.704722+0000", 2024-08-17T17:58:49.477 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.12946607399999999 2024-08-17T17:58:49.477 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.477 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.477 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.477 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.753525+0000", 2024-08-17T17:58:49.477 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.048803226999999998 2024-08-17T17:58:49.478 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.478 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.478 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.478 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.753627+0000", 2024-08-17T17:58:49.478 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000102113 2024-08-17T17:58:49.478 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.478 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.478 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.478 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.753767+0000", 2024-08-17T17:58:49.478 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000140106 2024-08-17T17:58:49.478 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.478 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.478 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.479 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.768534+0000", 2024-08-17T17:58:49.479 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014766359 2024-08-17T17:58:49.479 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.479 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.479 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.479 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.781525+0000", 2024-08-17T17:58:49.479 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012991669000000001 2024-08-17T17:58:49.479 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.479 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.479 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.479 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.781568+0000", 2024-08-17T17:58:49.479 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2818000000000002e-05 2024-08-17T17:58:49.479 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.480 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.480 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.480 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.781626+0000", 2024-08-17T17:58:49.480 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8424999999999997e-05 2024-08-17T17:58:49.480 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.480 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.480 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.480 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789133+0000", 2024-08-17T17:58:49.480 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0075066480000000003 2024-08-17T17:58:49.480 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.480 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.480 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.481 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.481 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.481 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109293 45.8 45:166b0a81:::benchmark_data_smithi130_172551_object4486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.481 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.533090+0000", 2024-08-17T17:58:49.481 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 560.84760491700001, 2024-08-17T17:58:49.481 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26005087900000001, 2024-08-17T17:58:49.481 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.481 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.481 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.481 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.481 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.481 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.481 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109293 2024-08-17T17:58:49.481 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.482 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.482 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.482 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.482 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533090+0000", 2024-08-17T17:58:49.482 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.482 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.482 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.482 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.482 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533090+0000", 2024-08-17T17:58:49.482 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.482 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.482 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.483 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.483 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533092+0000", 2024-08-17T17:58:49.483 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0719999999999998e-06 2024-08-17T17:58:49.483 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.483 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.483 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.483 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533100+0000", 2024-08-17T17:58:49.483 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2699999999999999e-06 2024-08-17T17:58:49.483 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.483 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.483 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.483 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533113+0000", 2024-08-17T17:58:49.483 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3903999999999999e-05 2024-08-17T17:58:49.483 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.484 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.484 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.484 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.533121+0000", 2024-08-17T17:58:49.484 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9160000000000001e-06 2024-08-17T17:58:49.484 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.484 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.484 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.484 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.704763+0000", 2024-08-17T17:58:49.484 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.17164132700000001 2024-08-17T17:58:49.484 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.484 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.485 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.485 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.707453+0000", 2024-08-17T17:58:49.485 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026901220000000001 2024-08-17T17:58:49.485 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.485 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.485 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.485 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.768583+0000", 2024-08-17T17:58:49.485 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.061130048999999999 2024-08-17T17:58:49.485 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.485 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.485 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.485 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.768675+0000", 2024-08-17T17:58:49.485 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1698000000000002e-05 2024-08-17T17:58:49.486 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.486 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.486 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.486 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.768806+0000", 2024-08-17T17:58:49.486 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013118900000000001 2024-08-17T17:58:49.486 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.486 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.486 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.486 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789153+0000", 2024-08-17T17:58:49.486 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020347371999999999 2024-08-17T17:58:49.486 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.486 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.487 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.487 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789212+0000", 2024-08-17T17:58:49.487 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9163000000000003e-05 2024-08-17T17:58:49.487 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.487 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.487 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.487 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789274+0000", 2024-08-17T17:58:49.487 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1339000000000003e-05 2024-08-17T17:58:49.487 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.487 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.487 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.487 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.789347+0000", 2024-08-17T17:58:49.487 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3458e-05 2024-08-17T17:58:49.487 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.488 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.488 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.488 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.793141+0000", 2024-08-17T17:58:49.488 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0037940000000000001 2024-08-17T17:58:49.488 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.488 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.488 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.488 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.488 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.488 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109294 45.8 45:16984eff:::benchmark_data_smithi130_172551_object4487:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.488 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.534050+0000", 2024-08-17T17:58:49.488 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 560.84664537900005, 2024-08-17T17:58:49.489 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.33223749499999999, 2024-08-17T17:58:49.489 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.489 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.489 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.489 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.489 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.489 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.489 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109294 2024-08-17T17:58:49.489 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.489 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.489 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.489 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.489 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534050+0000", 2024-08-17T17:58:49.489 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.490 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.490 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.490 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.490 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534050+0000", 2024-08-17T17:58:49.490 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.490 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.490 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.490 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.490 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534051+0000", 2024-08-17T17:58:49.490 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4789999999999999e-06 2024-08-17T17:58:49.490 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.490 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.491 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.491 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534057+0000", 2024-08-17T17:58:49.491 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5489999999999997e-06 2024-08-17T17:58:49.491 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.491 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.491 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.491 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534075+0000", 2024-08-17T17:58:49.491 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7782999999999999e-05 2024-08-17T17:58:49.491 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.491 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.491 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.491 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.534083+0000", 2024-08-17T17:58:49.492 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1089999999999998e-06 2024-08-17T17:58:49.492 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.492 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.492 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.492 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.793348+0000", 2024-08-17T17:58:49.492 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.259265264 2024-08-17T17:58:49.492 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.492 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.492 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.492 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.794825+0000", 2024-08-17T17:58:49.492 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0014772260000000001 2024-08-17T17:58:49.492 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.493 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.493 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.493 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.820020+0000", 2024-08-17T17:58:49.493 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025194431 2024-08-17T17:58:49.493 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.493 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.493 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.493 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.820117+0000", 2024-08-17T17:58:49.493 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7385000000000004e-05 2024-08-17T17:58:49.493 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.493 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.493 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.494 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.820245+0000", 2024-08-17T17:58:49.494 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012771000000000001 2024-08-17T17:58:49.494 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.494 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.494 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.494 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.842202+0000", 2024-08-17T17:58:49.494 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021957300999999999 2024-08-17T17:58:49.494 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.494 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.494 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.494 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.842217+0000", 2024-08-17T17:58:49.494 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5438999999999999e-05 2024-08-17T17:58:49.495 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.495 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.495 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.495 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.862154+0000", 2024-08-17T17:58:49.495 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019936878000000002 2024-08-17T17:58:49.495 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.495 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.495 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.495 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.862230+0000", 2024-08-17T17:58:49.495 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6091e-05 2024-08-17T17:58:49.495 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.495 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.496 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.496 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.866287+0000", 2024-08-17T17:58:49.496 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0040568499999999999 2024-08-17T17:58:49.496 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.496 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.496 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.496 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.496 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.496 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109309 45.8 45:12b009b3:::benchmark_data_smithi130_172551_object4502:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.496 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:28.570036+0000", 2024-08-17T17:58:49.496 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 560.81065902199998, 2024-08-17T17:58:49.496 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.300907009, 2024-08-17T17:58:49.497 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.497 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.497 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.497 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.497 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.497 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.497 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109309 2024-08-17T17:58:49.497 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.497 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.497 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.497 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.497 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570036+0000", 2024-08-17T17:58:49.498 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.498 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.498 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.498 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.498 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570036+0000", 2024-08-17T17:58:49.498 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.498 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.498 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.498 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.498 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570038+0000", 2024-08-17T17:58:49.498 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.688e-06 2024-08-17T17:58:49.498 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.499 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.499 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.499 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570046+0000", 2024-08-17T17:58:49.499 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3899999999999993e-06 2024-08-17T17:58:49.499 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.499 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.499 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.499 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570056+0000", 2024-08-17T17:58:49.499 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0217999999999999e-05 2024-08-17T17:58:49.499 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.499 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.499 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.499 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.570064+0000", 2024-08-17T17:58:49.499 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.892e-06 2024-08-17T17:58:49.500 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.500 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.500 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.500 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.794964+0000", 2024-08-17T17:58:49.500 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22489930799999999 2024-08-17T17:58:49.500 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.500 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.500 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.500 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.796783+0000", 2024-08-17T17:58:49.500 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001819443 2024-08-17T17:58:49.500 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.500 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.501 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.501 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.836565+0000", 2024-08-17T17:58:49.501 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.039782114 2024-08-17T17:58:49.501 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.501 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.501 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.501 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.836800+0000", 2024-08-17T17:58:49.501 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00023482700000000001 2024-08-17T17:58:49.501 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.501 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.501 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.501 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.837157+0000", 2024-08-17T17:58:49.501 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00035710700000000001 2024-08-17T17:58:49.502 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.502 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.502 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.502 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.842249+0000", 2024-08-17T17:58:49.502 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005092315 2024-08-17T17:58:49.502 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.502 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.502 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.502 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.842261+0000", 2024-08-17T17:58:49.502 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.189e-05 2024-08-17T17:58:49.502 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.502 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.502 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.866457+0000", 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.024195761 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.866646+0000", 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018857100000000001 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:28.870943+0000", 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0042974850000000002 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.503 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109672 45.8 45:19e4da43:::benchmark_data_smithi130_172551_object4865:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:29.496933+0000", 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 559.88376160999997, 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.221123546, 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109672 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.504 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.505 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.505 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.505 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496933+0000", 2024-08-17T17:58:49.505 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.505 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.505 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.505 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.505 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496933+0000", 2024-08-17T17:58:49.505 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.505 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.505 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.505 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.506 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496934+0000", 2024-08-17T17:58:49.506 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2899999999999999e-07 2024-08-17T17:58:49.506 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.506 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.506 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.506 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496936+0000", 2024-08-17T17:58:49.506 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5320000000000001e-06 2024-08-17T17:58:49.506 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.506 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.506 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.506 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496945+0000", 2024-08-17T17:58:49.506 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6160000000000002e-06 2024-08-17T17:58:49.507 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.507 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.507 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.507 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.496950+0000", 2024-08-17T17:58:49.507 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4140000000000002e-06 2024-08-17T17:58:49.507 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.507 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.507 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.507 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.568388+0000", 2024-08-17T17:58:49.507 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.071437234000000002 2024-08-17T17:58:49.507 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.507 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.507 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.508 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.569153+0000", 2024-08-17T17:58:49.508 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00076482600000000003 2024-08-17T17:58:49.508 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.508 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.508 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.508 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.697807+0000", 2024-08-17T17:58:49.508 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.128654886 2024-08-17T17:58:49.508 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.508 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.508 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.508 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.697938+0000", 2024-08-17T17:58:49.508 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013016899999999999 2024-08-17T17:58:49.508 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.508 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.509 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.509 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.698100+0000", 2024-08-17T17:58:49.509 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016217099999999999 2024-08-17T17:58:49.509 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.509 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.509 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.509 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.713384+0000", 2024-08-17T17:58:49.509 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015284588 2024-08-17T17:58:49.509 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.509 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.509 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.509 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.713429+0000", 2024-08-17T17:58:49.509 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4895000000000003e-05 2024-08-17T17:58:49.510 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.510 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.510 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.510 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.713504+0000", 2024-08-17T17:58:49.510 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4994999999999996e-05 2024-08-17T17:58:49.510 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.510 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.510 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.510 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.713752+0000", 2024-08-17T17:58:49.510 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00024759199999999998 2024-08-17T17:58:49.510 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.510 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.511 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.511 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.718057+0000", 2024-08-17T17:58:49.511 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.004305199 2024-08-17T17:58:49.511 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.511 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.511 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.511 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.511 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.511 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109679 45.8 45:13e4c37d:::benchmark_data_smithi130_172551_object4872:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.511 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:29.510400+0000", 2024-08-17T17:58:49.511 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 559.87029525399998, 2024-08-17T17:58:49.511 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22616212299999999, 2024-08-17T17:58:49.511 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.512 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.512 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.512 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.512 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.512 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.512 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109679 2024-08-17T17:58:49.512 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.512 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.512 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.512 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.512 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510400+0000", 2024-08-17T17:58:49.512 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.512 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.512 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.513 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.513 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510400+0000", 2024-08-17T17:58:49.513 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.513 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.513 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.513 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.513 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510402+0000", 2024-08-17T17:58:49.513 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.736e-06 2024-08-17T17:58:49.513 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.513 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.513 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.513 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510407+0000", 2024-08-17T17:58:49.514 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2140000000000004e-06 2024-08-17T17:58:49.514 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.514 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.514 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.514 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510423+0000", 2024-08-17T17:58:49.514 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6066000000000002e-05 2024-08-17T17:58:49.514 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.514 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.514 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.514 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.510430+0000", 2024-08-17T17:58:49.514 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1590000000000004e-06 2024-08-17T17:58:49.514 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.515 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.515 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.515 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.569188+0000", 2024-08-17T17:58:49.515 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.058757566999999997 2024-08-17T17:58:49.515 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.515 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.515 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.515 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.570491+0000", 2024-08-17T17:58:49.515 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001303228 2024-08-17T17:58:49.515 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.515 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.515 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.515 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.708557+0000", 2024-08-17T17:58:49.515 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.13806636799999999 2024-08-17T17:58:49.516 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.516 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.516 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.516 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.708713+0000", 2024-08-17T17:58:49.516 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015618000000000001 2024-08-17T17:58:49.516 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.516 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.516 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.516 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.708867+0000", 2024-08-17T17:58:49.516 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015330699999999999 2024-08-17T17:58:49.516 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.516 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.517 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.517 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.721271+0000", 2024-08-17T17:58:49.517 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012404178 2024-08-17T17:58:49.517 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.517 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.517 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.517 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.724727+0000", 2024-08-17T17:58:49.517 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003456083 2024-08-17T17:58:49.517 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.517 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.517 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.517 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.727154+0000", 2024-08-17T17:58:49.517 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0024270379999999999 2024-08-17T17:58:49.518 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.518 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.518 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.518 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.727215+0000", 2024-08-17T17:58:49.518 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1525999999999993e-05 2024-08-17T17:58:49.518 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.518 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.518 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.518 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.736562+0000", 2024-08-17T17:58:49.518 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0093464729999999992 2024-08-17T17:58:49.518 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.518 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.518 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.518 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.519 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.519 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109712 45.8 45:1acdc34f:::benchmark_data_smithi130_172551_object4905:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.519 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:29.603003+0000", 2024-08-17T17:58:49.519 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 559.77769253899999, 2024-08-17T17:58:49.519 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.21928672099999999, 2024-08-17T17:58:49.519 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.519 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.519 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.519 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.519 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.519 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.519 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109712 2024-08-17T17:58:49.519 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.519 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.520 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.520 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.520 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603003+0000", 2024-08-17T17:58:49.520 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.520 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.520 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.520 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.520 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603003+0000", 2024-08-17T17:58:49.520 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.520 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.520 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.520 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.520 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603005+0000", 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3209999999999999e-06 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603013+0000", 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4610000000000008e-06 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603033+0000", 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9606000000000002e-05 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.521 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.603046+0000", 2024-08-17T17:58:49.522 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2634e-05 2024-08-17T17:58:49.522 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.522 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.522 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.522 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.718108+0000", 2024-08-17T17:58:49.522 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.115062581 2024-08-17T17:58:49.522 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.522 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.522 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.522 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.721208+0000", 2024-08-17T17:58:49.522 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0030999180000000001 2024-08-17T17:58:49.522 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.523 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.523 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.523 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.758271+0000", 2024-08-17T17:58:49.524 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.037063082999999997 2024-08-17T17:58:49.524 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.524 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.524 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.524 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.758375+0000", 2024-08-17T17:58:49.524 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000103467 2024-08-17T17:58:49.524 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.524 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.524 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.524 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.758517+0000", 2024-08-17T17:58:49.524 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014272699999999999 2024-08-17T17:58:49.525 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.525 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.525 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.525 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.772488+0000", 2024-08-17T17:58:49.525 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013970419 2024-08-17T17:58:49.525 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.525 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.525 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.525 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.788268+0000", 2024-08-17T17:58:49.525 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015780076000000001 2024-08-17T17:58:49.525 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.525 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.525 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.526 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.816948+0000", 2024-08-17T17:58:49.526 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028680039000000001 2024-08-17T17:58:49.526 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.526 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.526 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.526 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.817013+0000", 2024-08-17T17:58:49.526 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5081999999999998e-05 2024-08-17T17:58:49.526 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.526 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.526 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.526 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.822289+0000", 2024-08-17T17:58:49.526 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005276307 2024-08-17T17:58:49.526 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.527 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.527 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.527 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.527 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.527 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:109714 45.8 45:16437e11:::benchmark_data_smithi130_172551_object4907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1185)", 2024-08-17T17:58:49.527 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:49:29.605847+0000", 2024-08-17T17:58:49.527 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 559.77484785499996, 2024-08-17T17:58:49.527 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26878229799999998, 2024-08-17T17:58:49.527 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.527 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.527 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.527 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.527 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 109714 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605847+0000", 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605847+0000", 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.528 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.529 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.529 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605849+0000", 2024-08-17T17:58:49.529 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9209999999999999e-06 2024-08-17T17:58:49.529 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.529 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.529 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.529 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605856+0000", 2024-08-17T17:58:49.529 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6209999999999998e-06 2024-08-17T17:58:49.529 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.529 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.529 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.529 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605870+0000", 2024-08-17T17:58:49.529 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4395e-05 2024-08-17T17:58:49.529 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.530 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.530 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.530 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.605879+0000", 2024-08-17T17:58:49.530 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4239999999999993e-06 2024-08-17T17:58:49.530 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.530 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.530 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.530 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.721311+0000", 2024-08-17T17:58:49.530 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.115432828 2024-08-17T17:58:49.530 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.530 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.530 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T17:58:49.530 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.722135+0000", 2024-08-17T17:58:49.531 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00082366000000000004 2024-08-17T17:58:49.531 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.531 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.531 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.531 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.783886+0000", 2024-08-17T17:58:49.531 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.061750939999999997 2024-08-17T17:58:49.531 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.531 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.531 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.531 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.784004+0000", 2024-08-17T17:58:49.531 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011846899999999999 2024-08-17T17:58:49.531 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.531 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.531 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:58:49.532 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.784179+0000", 2024-08-17T17:58:49.532 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017467599999999999 2024-08-17T17:58:49.532 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.532 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.532 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.532 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.791605+0000", 2024-08-17T17:58:49.532 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0074256260000000003 2024-08-17T17:58:49.532 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.532 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.532 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.532 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.791767+0000", 2024-08-17T17:58:49.532 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016228 2024-08-17T17:58:49.532 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.532 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.533 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.533 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.869906+0000", 2024-08-17T17:58:49.533 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.078138847999999997 2024-08-17T17:58:49.533 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.533 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.533 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.533 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.870304+0000", 2024-08-17T17:58:49.533 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00039816299999999999 2024-08-17T17:58:49.533 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.533 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.533 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.533 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:49:29.874630+0000", 2024-08-17T17:58:49.533 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0043254469999999996 2024-08-17T17:58:49.533 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.534 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.534 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.534 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.534 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.534 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:203344 45.b 45:d602dbc9:::benchmark_data_smithi130_172551_object98537:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1247)", 2024-08-17T17:58:49.534 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:48.408052+0000", 2024-08-17T17:58:49.534 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.972643009, 2024-08-17T17:58:49.534 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.065139592999999996, 2024-08-17T17:58:49.534 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:58:49.534 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:58:49.534 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:58:49.534 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T17:58:49.534 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 203344 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.408052+0000", 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.408052+0000", 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.535 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.536 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:58:49.536 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.408054+0000", 2024-08-17T17:58:49.536 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8199999999999999e-06 2024-08-17T17:58:49.536 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.536 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.536 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:58:49.536 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.408060+0000", 2024-08-17T17:58:49.536 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2559999999999999e-06 2024-08-17T17:58:49.536 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.536 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.536 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:58:49.536 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.408074+0000", 2024-08-17T17:58:49.536 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3842999999999999e-05 2024-08-17T17:58:49.537 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.537 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.537 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:58:49.537 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.408082+0000", 2024-08-17T17:58:49.537 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.593e-06 2024-08-17T17:58:49.537 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.537 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.537 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:58:49.537 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.410409+0000", 2024-08-17T17:58:49.537 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0023273370000000001 2024-08-17T17:58:49.537 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.537 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.537 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:58:49.537 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.410539+0000", 2024-08-17T17:58:49.538 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012982799999999999 2024-08-17T17:58:49.538 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.538 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.538 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3,4", 2024-08-17T17:58:49.538 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.410671+0000", 2024-08-17T17:58:49.538 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013182100000000001 2024-08-17T17:58:49.538 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.538 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.538 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T17:58:49.538 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.413886+0000", 2024-08-17T17:58:49.538 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032149499999999998 2024-08-17T17:58:49.538 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.538 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.538 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.414192+0000", 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000306626 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.446413+0000", 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.032220578 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.472635+0000", 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.026222058999999999 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.539 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T17:58:49.540 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.472763+0000", 2024-08-17T17:58:49.540 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012845200000000001 2024-08-17T17:58:49.540 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:58:49.540 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:58:49.540 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T17:58:49.540 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:48.473192+0000", 2024-08-17T17:58:49.540 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00042842999999999999 2024-08-17T17:58:49.540 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.540 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.540 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.540 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:58:49.540 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:58:49.540 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:58:49.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:49.541 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:49.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:49.642 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:49.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:49.664 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:49.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:49.764 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:49.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:49.787 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:49.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:49.888 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:49.910 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:49.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:49.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:50.011 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:50.033 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:50.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:50.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:50.135 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:50.156 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:58:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:58:49] "GET /metrics HTTP/1.1" 200 36791 "" "Prometheus/2.51.0" 2024-08-17T17:58:50.158 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:50.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:50.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:50.260 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:50.282 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:50.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:50.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:50.384 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:50.425 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:50.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:50.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:50.527 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:50.562 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:50.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:50.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:58:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:58:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:58:50.663 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:50.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:50.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:50.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:50.787 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:50.809 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:50.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:50.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:50.911 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:50.949 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:50.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:50.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:51.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:51.074 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:51.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:51.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:51.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:50 smithi179 ceph-mon[93519]: pgmap v1974: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 429 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 16695/20667 objects degraded (80.781%); 956/20667 objects misplaced (4.626%); 0 B/s, 5 objects/s recovering 2024-08-17T17:58:51.111 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:50 smithi130 ceph-mon[110068]: pgmap v1974: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 429 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 16695/20667 objects degraded (80.781%); 956/20667 objects misplaced (4.626%); 0 B/s, 5 objects/s recovering 2024-08-17T17:58:51.111 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:50 smithi130 ceph-mon[105521]: pgmap v1974: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 429 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 16695/20667 objects degraded (80.781%); 956/20667 objects misplaced (4.626%); 0 B/s, 5 objects/s recovering 2024-08-17T17:58:51.175 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:51.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:51.214 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:51.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:51.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:51.351 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:51.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:51.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:51.453 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:51.476 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:51.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:51.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:51.577 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:51.627 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:51.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:51.728 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:51.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:51.784 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:51.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:51.885 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:51.892 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16695/20667 objects degraded (80.781%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:51.893 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16695/20667 objects degraded (80.781%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:51.906 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:51.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:51.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:52.008 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:52.031 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:52.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:52.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16695/20667 objects degraded (80.781%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:52.132 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:52.180 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:52.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:52.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:52.237 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:58:52.283 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:52.336 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:52.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:52.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:52.438 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:52.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:52.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:52.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:52.562 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:52.584 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:52.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:52.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:52.685 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:52.709 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:52.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:52.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:52.811 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:52.858 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:52.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:52.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:52 smithi130 ceph-mon[105521]: pgmap v1975: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 404 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 536 op/s; 16695/19431 objects degraded (85.919%); 956/19431 objects misplaced (4.920%); 0 B/s, 5 objects/s recovering 2024-08-17T17:58:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:52 smithi130 ceph-mon[110068]: pgmap v1975: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 404 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 536 op/s; 16695/19431 objects degraded (85.919%); 956/19431 objects misplaced (4.920%); 0 B/s, 5 objects/s recovering 2024-08-17T17:58:52.960 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:52.983 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:52.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:52.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:53.085 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:53.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:52 smithi179 ceph-mon[93519]: pgmap v1975: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 404 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 536 op/s; 16695/19431 objects degraded (85.919%); 956/19431 objects misplaced (4.920%); 0 B/s, 5 objects/s recovering 2024-08-17T17:58:53.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:53.132 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:53.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:53.231 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:58:53.233 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:58:53.235 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:53.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:53.259 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:53.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:53.360 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:53.382 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:53.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:53.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:53.484 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:53.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:53.524 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:53.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:53.625 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:53.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:53.663 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:53.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:53.704 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:58:53.764 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:53.787 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:53.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:53.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:53.889 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:53.912 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:53.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:53.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:54.014 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:54.035 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:54.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:54.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:54.137 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:54.193 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:54.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:54.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:54.294 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:54.316 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:54.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:54.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:54.418 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:54.465 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:54.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:54.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:54.567 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:54.617 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:54.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:54.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:54.719 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:54.738 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T17:58:54.742 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:54.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:54.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:54.844 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:54.868 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:54.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:54.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:54 smithi130 ceph-mon[110068]: pgmap v1976: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 290 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 16704/13989 objects degraded (119.408%); 772/13989 objects misplaced (5.519%); 0 B/s, 12 objects/s recovering 2024-08-17T17:58:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:54 smithi130 ceph-mon[110068]: from='client.55690 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1615191491' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:54 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:54 smithi130 ceph-mon[105521]: pgmap v1976: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 290 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 16704/13989 objects degraded (119.408%); 772/13989 objects misplaced (5.519%); 0 B/s, 12 objects/s recovering 2024-08-17T17:58:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:54 smithi130 ceph-mon[105521]: from='client.55690 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1615191491' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:54 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:54.970 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:54.994 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:54.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:54.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:54 smithi179 ceph-mon[93519]: pgmap v1976: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 290 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 655 op/s; 16704/13989 objects degraded (119.408%); 772/13989 objects misplaced (5.519%); 0 B/s, 12 objects/s recovering 2024-08-17T17:58:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:54 smithi179 ceph-mon[93519]: from='client.55690 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:58:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1615191491' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:54 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:58:55.096 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:55.135 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:55.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:55.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:55.237 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:55.262 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:55.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:55.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:55.364 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:55.386 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:55.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:55.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:55.488 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:55.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:55.510 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:55.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:55.611 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:55.634 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:55.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:55.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:55.736 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:55.791 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:55.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:55.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:55.893 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:55 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:58:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:55 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:58:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:55 smithi130 ceph-mon[110068]: osdmap e1248: 8 total, 8 up, 6 in 2024-08-17T17:58:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:55 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:58:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:55 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:58:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:55 smithi130 ceph-mon[105521]: osdmap e1248: 8 total, 8 up, 6 in 2024-08-17T17:58:55.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:55.918 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:55.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:56.019 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:56.043 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:56.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:56.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:55 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:58:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:55 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:58:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:55 smithi179 ceph-mon[93519]: osdmap e1248: 8 total, 8 up, 6 in 2024-08-17T17:58:56.145 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:56.186 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:56.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:56.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:56.288 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:56.312 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:56.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:56.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:56.414 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:56.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:56.458 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:56.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:56.559 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:56.582 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:56.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:56.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:56.684 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:56.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:56.718 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:56.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:56.819 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:56.841 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:56.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:56.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:56.943 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:56.985 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:56.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:56.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:56.997 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:56 smithi130 ceph-mon[110068]: pgmap v1978: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 222 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 599 op/s; 16704/10716 objects degraded (155.879%); 772/10716 objects misplaced (7.204%); 0 B/s, 12 objects/s recovering 2024-08-17T17:58:56.997 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16704/10716 objects degraded (155.879%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:56.997 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:56 smithi130 ceph-mon[105521]: pgmap v1978: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 222 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 599 op/s; 16704/10716 objects degraded (155.879%); 772/10716 objects misplaced (7.204%); 0 B/s, 12 objects/s recovering 2024-08-17T17:58:56.997 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16704/10716 objects degraded (155.879%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:57.014 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:56 smithi179 ceph-mon[93519]: pgmap v1978: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 222 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 599 op/s; 16704/10716 objects degraded (155.879%); 772/10716 objects misplaced (7.204%); 0 B/s, 12 objects/s recovering 2024-08-17T17:58:57.014 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16704/10716 objects degraded (155.879%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:58:57.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:57.145 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:57.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:57.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:57.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:57.300 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:57.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:57.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:57.402 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:57.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:57.428 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:57.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:57.530 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:57.553 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:57.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:57.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:57.655 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:57.691 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:57.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:57.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:57.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:57.814 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:57.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:57.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:57.826 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":1976,"stamp":"2024-08-17T17:58:53.132012+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":304420837,"num_objects":4663,"num_object_clones":0,"num_object_copies":13989,"num_objects_missing_on_primary":553,"num_objects_missing":553,"num_objects_degraded":16704,"num_objects_misplaced":772,"num_objects_unfound":0,"num_objects_dirty":4663,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":205251,"num_write_kb":6709904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38620,"num_bytes_recovered":2341084173,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108554,"ondisk_log_size":108554,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":2541956,"kb_used_data":966996,"kb_used_omap":295,"kb_used_meta":1574552,"kb_avail":559928956,"statfs":{"total":575970213888,"available":573367250944,"internally_reserved":0,"allocated":990203904,"data_stored":959813209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1612341711},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,4,6,5,5,3,1,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-515833856,"num_objects":-7871,"num_object_clones":0,"num_object_copies":-23613,"num_objects_missing_on_primary":-146,"num_objects_missing":-146,"num_objects_degraded":337,"num_objects_misplaced":-897,"num_objects_unfound":0,"num_objects_dirty":-7871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7871,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":145,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003986"},"pg_stats":[{"pgid":"45.f","version":"1247'12914","reported_seq":23489,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:52.222382+0000","last_change":"2024-08-17T17:50:34.772631+0000","last_active":"2024-08-17T17:58:52.222382+0000","last_peered":"2024-08-17T17:58:52.222382+0000","last_clean":"2024-08-17T17:58:52.222382+0000","last_became_active":"2024-08-17T17:50:34.750006+0000","last_became_peered":"2024-08-17T17:50:34.750006+0000","last_unstale":"2024-08-17T17:58:52.222382+0000","last_undegraded":"2024-08-17T17:58:52.222382+0000","last_fullsized":"2024-08-17T17:58:52.222382+0000","mapping_epoch":1194,"log_start":"1181'6031","ondisk_log_start":"1181'6031","created":1167,"last_epoch_clean":1195,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6883,"log_dups_size":0,"ondisk_log_size":6883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:18.031161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16384000,"num_objects":250,"num_object_clones":0,"num_object_copies":750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12914,"num_write_kb":421248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5195,"num_bytes_recovered":291438592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.e","version":"1247'12789","reported_seq":23276,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:52.974632+0000","last_change":"2024-08-17T17:52:05.298412+0000","last_active":"2024-08-17T17:58:52.974632+0000","last_peered":"2024-08-17T17:58:52.974632+0000","last_clean":"2024-08-17T17:58:52.974632+0000","last_became_active":"2024-08-17T17:52:05.285732+0000","last_became_peered":"2024-08-17T17:52:05.285732+0000","last_unstale":"2024-08-17T17:58:52.974632+0000","last_undegraded":"2024-08-17T17:58:52.974632+0000","last_fullsized":"2024-08-17T17:58:52.974632+0000","mapping_epoch":1211,"log_start":"1181'5732","ondisk_log_start":"1181'5732","created":1167,"last_epoch_clean":1212,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":7057,"log_dups_size":0,"ondisk_log_size":7057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:07.923019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13434880,"num_objects":205,"num_object_clones":0,"num_object_copies":615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12789,"num_write_kb":415808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5182,"num_bytes_recovered":268566528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.d","version":"1247'12988","reported_seq":23868,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:48.822613+0000","last_change":"2024-08-17T17:51:25.208627+0000","last_active":"2024-08-17T17:58:48.822613+0000","last_peered":"2024-08-17T17:58:48.822613+0000","last_clean":"2024-08-17T17:58:48.822613+0000","last_became_active":"2024-08-17T17:51:25.198395+0000","last_became_peered":"2024-08-17T17:51:25.198395+0000","last_unstale":"2024-08-17T17:58:48.822613+0000","last_undegraded":"2024-08-17T17:58:48.822613+0000","last_fullsized":"2024-08-17T17:58:48.822613+0000","mapping_epoch":1205,"log_start":"1186'7068","ondisk_log_start":"1186'7068","created":1167,"last_epoch_clean":1206,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5920,"log_dups_size":0,"ondisk_log_size":5920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:43.795600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25493528,"num_objects":390,"num_object_clones":0,"num_object_copies":1170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":390,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12988,"num_write_kb":428033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5349,"num_bytes_recovered":292159488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.c","version":"1247'12772","reported_seq":12860,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:52.934500+0000","last_change":"2024-08-17T17:47:36.549805+0000","last_active":"2024-08-17T17:58:52.934500+0000","last_peered":"2024-08-17T17:58:52.934500+0000","last_clean":"2024-08-17T17:58:52.934500+0000","last_became_active":"2024-08-17T17:47:36.549347+0000","last_became_peered":"2024-08-17T17:47:36.549347+0000","last_unstale":"2024-08-17T17:58:52.934500+0000","last_undegraded":"2024-08-17T17:58:52.934500+0000","last_fullsized":"2024-08-17T17:58:52.934500+0000","mapping_epoch":1167,"log_start":"1182'6089","ondisk_log_start":"1182'6089","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6683,"log_dups_size":0,"ondisk_log_size":6683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:13.664011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14942208,"num_objects":228,"num_object_clones":0,"num_object_copies":684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12772,"num_write_kb":416000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.b","version":"1247'12831","reported_seq":18119,"reported_epoch":1247,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:52.008585+0000","last_change":"2024-08-17T17:49:45.766315+0000","last_active":"2024-08-17T17:58:52.008585+0000","last_peered":"2024-08-17T17:58:52.008585+0000","last_clean":"2024-08-17T17:48:06.615748+0000","last_became_active":"2024-08-17T17:49:45.755758+0000","last_became_peered":"2024-08-17T17:49:45.755758+0000","last_unstale":"2024-08-17T17:58:52.008585+0000","last_undegraded":"2024-08-17T17:49:45.688815+0000","last_fullsized":"2024-08-17T17:49:45.644549+0000","mapping_epoch":1188,"log_start":"1185'6926","ondisk_log_start":"1185'6926","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5905,"log_dups_size":0,"ondisk_log_size":5905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18546688,"num_objects":283,"num_object_clones":0,"num_object_copies":849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":283,"num_objects_misplaced":518,"num_objects_unfound":0,"num_objects_dirty":283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12831,"num_write_kb":419648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2552,"num_bytes_recovered":167247872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,4],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4","objects":-235}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.a","version":"1247'12850","reported_seq":12938,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:52.932212+0000","last_change":"2024-08-17T17:47:36.545558+0000","last_active":"2024-08-17T17:58:52.932212+0000","last_peered":"2024-08-17T17:58:52.932212+0000","last_clean":"2024-08-17T17:58:52.932212+0000","last_became_active":"2024-08-17T17:47:36.545233+0000","last_became_peered":"2024-08-17T17:47:36.545233+0000","last_unstale":"2024-08-17T17:58:52.932212+0000","last_undegraded":"2024-08-17T17:58:52.932212+0000","last_fullsized":"2024-08-17T17:58:52.932212+0000","mapping_epoch":1167,"log_start":"1182'6177","ondisk_log_start":"1182'6177","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6673,"log_dups_size":0,"ondisk_log_size":6673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:32.625912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15990784,"num_objects":244,"num_object_clones":0,"num_object_copies":732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12850,"num_write_kb":419008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.9","version":"1247'12994","reported_seq":13082,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:52.983181+0000","last_change":"2024-08-17T17:47:36.549583+0000","last_active":"2024-08-17T17:58:52.983181+0000","last_peered":"2024-08-17T17:58:52.983181+0000","last_clean":"2024-08-17T17:58:52.983181+0000","last_became_active":"2024-08-17T17:47:36.549189+0000","last_became_peered":"2024-08-17T17:47:36.549189+0000","last_unstale":"2024-08-17T17:58:52.983181+0000","last_undegraded":"2024-08-17T17:58:52.983181+0000","last_fullsized":"2024-08-17T17:58:52.983181+0000","mapping_epoch":1167,"log_start":"1182'6269","ondisk_log_start":"1182'6269","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6725,"log_dups_size":0,"ondisk_log_size":6725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:29.667999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15204352,"num_objects":232,"num_object_clones":0,"num_object_copies":696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12994,"num_write_kb":423232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.8","version":"1247'12718","reported_seq":22224,"reported_epoch":1247,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:51.974905+0000","last_change":"2024-08-17T17:49:45.707198+0000","last_active":"2024-08-17T17:58:51.974905+0000","last_peered":"2024-08-17T17:58:51.974905+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.032587+0000","last_became_peered":"2024-08-17T17:48:41.032587+0000","last_unstale":"2024-08-17T17:58:51.974905+0000","last_undegraded":"2024-08-17T17:48:40.856891+0000","last_fullsized":"2024-08-17T17:48:40.647563+0000","mapping_epoch":1179,"log_start":"1185'6775","ondisk_log_start":"1185'6775","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17825792,"num_objects":272,"num_object_clones":0,"num_object_copies":816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12718,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3108,"num_bytes_recovered":203685888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,7],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,7","objects":-1795}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"45.7","version":"1247'12849","reported_seq":12947,"reported_epoch":1247,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:52.950686+0000","last_change":"2024-08-17T17:49:46.874727+0000","last_active":"2024-08-17T17:58:52.950686+0000","last_peered":"2024-08-17T17:58:52.950686+0000","last_clean":"2024-08-17T17:49:44.783975+0000","last_became_active":"2024-08-17T17:49:46.859535+0000","last_became_peered":"2024-08-17T17:49:46.859535+0000","last_unstale":"2024-08-17T17:58:52.950686+0000","last_undegraded":"2024-08-17T17:49:46.670511+0000","last_fullsized":"2024-08-17T17:49:46.646387+0000","mapping_epoch":1189,"log_start":"1182'6177","ondisk_log_start":"1182'6177","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6672,"log_dups_size":0,"ondisk_log_size":6672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:11.691065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14876672,"num_objects":227,"num_object_clones":0,"num_object_copies":681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12849,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":227},{"shards":"1,6","objects":-5606}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"45.6","version":"1247'12793","reported_seq":15915,"reported_epoch":1247,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:48.741263+0000","last_change":"2024-08-17T17:49:45.676277+0000","last_active":"2024-08-17T17:58:48.741263+0000","last_peered":"2024-08-17T17:58:48.741263+0000","last_clean":"2024-08-17T17:48:06.626561+0000","last_became_active":"2024-08-17T17:48:08.741340+0000","last_became_peered":"2024-08-17T17:48:08.741340+0000","last_unstale":"2024-08-17T17:58:48.741263+0000","last_undegraded":"2024-08-17T17:49:45.673903+0000","last_fullsized":"2024-08-17T17:48:08.618014+0000","mapping_epoch":1173,"log_start":"1185'6906","ondisk_log_start":"1185'6906","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5887,"log_dups_size":0,"ondisk_log_size":5887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24969216,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1561,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12793,"num_write_kb":421568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1470,"num_bytes_recovered":96337920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3","objects":-1180}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.5","version":"1247'12723","reported_seq":12900,"reported_epoch":1247,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:49.227845+0000","last_change":"2024-08-17T17:49:45.683462+0000","last_active":"2024-08-17T17:58:49.227845+0000","last_peered":"2024-08-17T17:58:49.227845+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:48:41.093471+0000","last_became_peered":"2024-08-17T17:48:41.093471+0000","last_unstale":"2024-08-17T17:58:49.227845+0000","last_undegraded":"2024-08-17T17:48:40.711671+0000","last_fullsized":"2024-08-17T17:48:40.643586+0000","mapping_epoch":1179,"log_start":"1175'3600","ondisk_log_start":"1175'3600","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9123,"log_dups_size":0,"ondisk_log_size":9123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23003136,"num_objects":351,"num_object_clones":0,"num_object_copies":1053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12723,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":-4802}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.4","version":"1247'12688","reported_seq":18926,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:49.249399+0000","last_change":"2024-08-17T17:49:12.772235+0000","last_active":"2024-08-17T17:58:49.249399+0000","last_peered":"2024-08-17T17:58:49.249399+0000","last_clean":"2024-08-17T17:58:49.249399+0000","last_became_active":"2024-08-17T17:49:12.759679+0000","last_became_peered":"2024-08-17T17:49:12.759679+0000","last_unstale":"2024-08-17T17:58:49.249399+0000","last_undegraded":"2024-08-17T17:58:49.249399+0000","last_fullsized":"2024-08-17T17:58:49.249399+0000","mapping_epoch":1183,"log_start":"1174'3500","ondisk_log_start":"1174'3500","created":1167,"last_epoch_clean":1184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":9188,"log_dups_size":0,"ondisk_log_size":9188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:58.834717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23461888,"num_objects":358,"num_object_clones":0,"num_object_copies":1074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12688,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3025,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"45.3","version":"1247'12765","reported_seq":12941,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:48.799009+0000","last_change":"2024-08-17T17:47:36.547565+0000","last_active":"2024-08-17T17:58:48.799009+0000","last_peered":"2024-08-17T17:58:48.799009+0000","last_clean":"2024-08-17T17:58:48.799009+0000","last_became_active":"2024-08-17T17:47:36.547253+0000","last_became_peered":"2024-08-17T17:47:36.547253+0000","last_unstale":"2024-08-17T17:58:48.799009+0000","last_undegraded":"2024-08-17T17:58:48.799009+0000","last_fullsized":"2024-08-17T17:58:48.799009+0000","mapping_epoch":1167,"log_start":"1185'6791","ondisk_log_start":"1185'6791","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5974,"log_dups_size":0,"ondisk_log_size":5974,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:25.362747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25886720,"num_objects":395,"num_object_clones":0,"num_object_copies":1185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12765,"num_write_kb":421120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"45.2","version":"1247'12720","reported_seq":12808,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:52.971392+0000","last_change":"2024-08-17T17:47:36.550475+0000","last_active":"2024-08-17T17:58:52.971392+0000","last_peered":"2024-08-17T17:58:52.971392+0000","last_clean":"2024-08-17T17:58:52.971392+0000","last_became_active":"2024-08-17T17:47:36.550135+0000","last_became_peered":"2024-08-17T17:47:36.550135+0000","last_unstale":"2024-08-17T17:58:52.971392+0000","last_undegraded":"2024-08-17T17:58:52.971392+0000","last_fullsized":"2024-08-17T17:58:52.971392+0000","mapping_epoch":1167,"log_start":"1181'5725","ondisk_log_start":"1181'5725","created":1167,"last_epoch_clean":1168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:32:15.906223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14024704,"num_objects":214,"num_object_clones":0,"num_object_copies":642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12720,"num_write_kb":413888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"45.1","version":"1247'13110","reported_seq":33537,"reported_epoch":1247,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:58:52.250018+0000","last_change":"2024-08-17T17:58:23.302199+0000","last_active":"2024-08-17T17:58:52.250018+0000","last_peered":"2024-08-17T17:58:52.250018+0000","last_clean":"2024-08-17T17:47:35.525121+0000","last_became_active":"2024-08-17T17:50:10.186465+0000","last_became_peered":"2024-08-17T17:50:10.186465+0000","last_unstale":"2024-08-17T17:58:52.250018+0000","last_undegraded":"2024-08-17T17:58:23.294905+0000","last_fullsized":"2024-08-17T17:50:03.452473+0000","mapping_epoch":1189,"log_start":"1182'6291","ondisk_log_start":"1182'6291","created":1167,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":6819,"log_dups_size":0,"ondisk_log_size":6819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16646144,"num_objects":254,"num_object_clones":0,"num_object_copies":762,"num_objects_missing_on_primary":553,"num_objects_missing":553,"num_objects_degraded":1102,"num_objects_misplaced":254,"num_objects_unfound":0,"num_objects_dirty":254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13110,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7605,"num_bytes_recovered":487063552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"0,4,6","objects":254},{"shards":"0,6","objects":-1102}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"45.0","version":"1247'12469","reported_seq":25831,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:48.814603+0000","last_change":"2024-08-17T17:50:36.689058+0000","last_active":"2024-08-17T17:58:48.814603+0000","last_peered":"2024-08-17T17:58:48.814603+0000","last_clean":"2024-08-17T17:58:48.814603+0000","last_became_active":"2024-08-17T17:50:36.674580+0000","last_became_peered":"2024-08-17T17:50:36.674580+0000","last_unstale":"2024-08-17T17:58:48.814603+0000","last_undegraded":"2024-08-17T17:58:48.814603+0000","last_fullsized":"2024-08-17T17:58:48.814603+0000","mapping_epoch":1196,"log_start":"1185'6569","ondisk_log_start":"1185'6569","created":1167,"last_epoch_clean":1197,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:47:35.525121+0000","last_clean_scrub_stamp":"2024-08-17T17:47:35.525121+0000","objects_scrubbed":0,"log_size":5900,"log_dups_size":0,"ondisk_log_size":5900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:55.290296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23134208,"num_objects":353,"num_object_clones":0,"num_object_copies":1059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12469,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5105,"num_bytes_recovered":334561280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1953,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.392177+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:58:37.392177+0000","last_peered":"2024-08-17T17:58:37.392177+0000","last_clean":"2024-08-17T17:58:37.392177+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:58:37.392177+0000","last_undegraded":"2024-08-17T17:58:37.392177+0000","last_fullsized":"2024-08-17T17:58:37.392177+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":524,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.394418+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:58:24.394418+0000","last_peered":"2024-08-17T17:58:24.394418+0000","last_clean":"2024-08-17T17:58:24.394418+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:58:24.394418+0000","last_undegraded":"2024-08-17T17:58:24.394418+0000","last_fullsized":"2024-08-17T17:58:24.394418+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1872,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.366620+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:58:37.366620+0000","last_peered":"2024-08-17T17:58:37.366620+0000","last_clean":"2024-08-17T17:58:37.366620+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:58:37.366620+0000","last_undegraded":"2024-08-17T17:58:37.366620+0000","last_fullsized":"2024-08-17T17:58:37.366620+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1896,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.363873+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:58:37.363873+0000","last_peered":"2024-08-17T17:58:37.363873+0000","last_clean":"2024-08-17T17:58:37.363873+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:58:37.363873+0000","last_undegraded":"2024-08-17T17:58:37.363873+0000","last_fullsized":"2024-08-17T17:58:37.363873+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":86,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.395286+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:58:24.395286+0000","last_peered":"2024-08-17T17:58:24.395286+0000","last_clean":"2024-08-17T17:58:24.395286+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:58:24.395286+0000","last_undegraded":"2024-08-17T17:58:24.395286+0000","last_fullsized":"2024-08-17T17:58:24.395286+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1289,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.364332+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:58:37.364332+0000","last_peered":"2024-08-17T17:58:37.364332+0000","last_clean":"2024-08-17T17:58:37.364332+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:58:37.364332+0000","last_undegraded":"2024-08-17T17:58:37.364332+0000","last_fullsized":"2024-08-17T17:58:37.364332+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":147,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.376572+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:58:37.376572+0000","last_peered":"2024-08-17T17:58:37.376572+0000","last_clean":"2024-08-17T17:58:37.376572+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:58:37.376572+0000","last_undegraded":"2024-08-17T17:58:37.376572+0000","last_fullsized":"2024-08-17T17:58:37.376572+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":667,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.363735+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:58:37.363735+0000","last_peered":"2024-08-17T17:58:37.363735+0000","last_clean":"2024-08-17T17:58:37.363735+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:58:37.363735+0000","last_undegraded":"2024-08-17T17:58:37.363735+0000","last_fullsized":"2024-08-17T17:58:37.363735+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1911,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.387578+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:58:37.387578+0000","last_peered":"2024-08-17T17:58:37.387578+0000","last_clean":"2024-08-17T17:58:37.387578+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:58:37.387578+0000","last_undegraded":"2024-08-17T17:58:37.387578+0000","last_fullsized":"2024-08-17T17:58:37.387578+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1913,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.369594+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:58:37.369594+0000","last_peered":"2024-08-17T17:58:37.369594+0000","last_clean":"2024-08-17T17:58:37.369594+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:58:37.369594+0000","last_undegraded":"2024-08-17T17:58:37.369594+0000","last_fullsized":"2024-08-17T17:58:37.369594+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":132,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.378217+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:58:37.378217+0000","last_peered":"2024-08-17T17:58:37.378217+0000","last_clean":"2024-08-17T17:58:37.378217+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:58:37.378217+0000","last_undegraded":"2024-08-17T17:58:37.378217+0000","last_fullsized":"2024-08-17T17:58:37.378217+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1884,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.368892+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:58:37.368892+0000","last_peered":"2024-08-17T17:58:37.368892+0000","last_clean":"2024-08-17T17:58:37.368892+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:58:37.368892+0000","last_undegraded":"2024-08-17T17:58:37.368892+0000","last_fullsized":"2024-08-17T17:58:37.368892+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1939,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.375588+0000","last_change":"2024-08-17T17:49:45.663172+0000","last_active":"2024-08-17T17:58:37.375588+0000","last_peered":"2024-08-17T17:58:37.375588+0000","last_clean":"2024-08-17T17:58:37.375588+0000","last_became_active":"2024-08-17T17:49:45.662861+0000","last_became_peered":"2024-08-17T17:49:45.662861+0000","last_unstale":"2024-08-17T17:58:37.375588+0000","last_undegraded":"2024-08-17T17:58:37.375588+0000","last_fullsized":"2024-08-17T17:58:37.375588+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:03.534098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1934,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.396070+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:58:37.396070+0000","last_peered":"2024-08-17T17:58:37.396070+0000","last_clean":"2024-08-17T17:58:37.396070+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:58:37.396070+0000","last_undegraded":"2024-08-17T17:58:37.396070+0000","last_fullsized":"2024-08-17T17:58:37.396070+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1907,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.367195+0000","last_change":"2024-08-17T17:48:39.656667+0000","last_active":"2024-08-17T17:58:37.367195+0000","last_peered":"2024-08-17T17:58:37.367195+0000","last_clean":"2024-08-17T17:58:37.367195+0000","last_became_active":"2024-08-17T17:48:39.656286+0000","last_became_peered":"2024-08-17T17:48:39.656286+0000","last_unstale":"2024-08-17T17:58:37.367195+0000","last_undegraded":"2024-08-17T17:58:37.367195+0000","last_fullsized":"2024-08-17T17:58:37.367195+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:09.151249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1221,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.396649+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:58:24.396649+0000","last_peered":"2024-08-17T17:58:24.396649+0000","last_clean":"2024-08-17T17:58:24.396649+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:58:24.396649+0000","last_undegraded":"2024-08-17T17:58:24.396649+0000","last_fullsized":"2024-08-17T17:58:24.396649+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1878,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.366915+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:58:37.366915+0000","last_peered":"2024-08-17T17:58:37.366915+0000","last_clean":"2024-08-17T17:58:37.366915+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:58:37.366915+0000","last_undegraded":"2024-08-17T17:58:37.366915+0000","last_fullsized":"2024-08-17T17:58:37.366915+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1928,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.392226+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:58:37.392226+0000","last_peered":"2024-08-17T17:58:37.392226+0000","last_clean":"2024-08-17T17:58:37.392226+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:58:37.392226+0000","last_undegraded":"2024-08-17T17:58:37.392226+0000","last_fullsized":"2024-08-17T17:58:37.392226+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1834,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.363123+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:58:37.363123+0000","last_peered":"2024-08-17T17:58:37.363123+0000","last_clean":"2024-08-17T17:58:37.363123+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:58:37.363123+0000","last_undegraded":"2024-08-17T17:58:37.363123+0000","last_fullsized":"2024-08-17T17:58:37.363123+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":540,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.405416+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:58:24.405416+0000","last_peered":"2024-08-17T17:58:24.405416+0000","last_clean":"2024-08-17T17:58:24.405416+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:58:24.405416+0000","last_undegraded":"2024-08-17T17:58:24.405416+0000","last_fullsized":"2024-08-17T17:58:24.405416+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1217,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.404054+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:58:24.404054+0000","last_peered":"2024-08-17T17:58:24.404054+0000","last_clean":"2024-08-17T17:58:24.404054+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:58:24.404054+0000","last_undegraded":"2024-08-17T17:58:24.404054+0000","last_fullsized":"2024-08-17T17:58:24.404054+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1933,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.379829+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:58:37.379829+0000","last_peered":"2024-08-17T17:58:37.379829+0000","last_clean":"2024-08-17T17:58:37.379829+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:58:37.379829+0000","last_undegraded":"2024-08-17T17:58:37.379829+0000","last_fullsized":"2024-08-17T17:58:37.379829+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":378,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.405263+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:58:24.405263+0000","last_peered":"2024-08-17T17:58:24.405263+0000","last_clean":"2024-08-17T17:58:24.405263+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:58:24.405263+0000","last_undegraded":"2024-08-17T17:58:24.405263+0000","last_fullsized":"2024-08-17T17:58:24.405263+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":524,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.394857+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:58:24.394857+0000","last_peered":"2024-08-17T17:58:24.394857+0000","last_clean":"2024-08-17T17:58:24.394857+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:58:24.394857+0000","last_undegraded":"2024-08-17T17:58:24.394857+0000","last_fullsized":"2024-08-17T17:58:24.394857+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1202,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.397227+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:58:24.397227+0000","last_peered":"2024-08-17T17:58:24.397227+0000","last_clean":"2024-08-17T17:58:24.397227+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:58:24.397227+0000","last_undegraded":"2024-08-17T17:58:24.397227+0000","last_fullsized":"2024-08-17T17:58:24.397227+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1210,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.396694+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:58:24.396694+0000","last_peered":"2024-08-17T17:58:24.396694+0000","last_clean":"2024-08-17T17:58:24.396694+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:58:24.396694+0000","last_undegraded":"2024-08-17T17:58:24.396694+0000","last_fullsized":"2024-08-17T17:58:24.396694+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1980,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.395710+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:58:37.395710+0000","last_peered":"2024-08-17T17:58:37.395710+0000","last_clean":"2024-08-17T17:58:37.395710+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:58:37.395710+0000","last_undegraded":"2024-08-17T17:58:37.395710+0000","last_fullsized":"2024-08-17T17:58:37.395710+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1864,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.367106+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:58:37.367106+0000","last_peered":"2024-08-17T17:58:37.367106+0000","last_clean":"2024-08-17T17:58:37.367106+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:58:37.367106+0000","last_undegraded":"2024-08-17T17:58:37.367106+0000","last_fullsized":"2024-08-17T17:58:37.367106+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1939,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.377460+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:58:37.377460+0000","last_peered":"2024-08-17T17:58:37.377460+0000","last_clean":"2024-08-17T17:58:37.377460+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:58:37.377460+0000","last_undegraded":"2024-08-17T17:58:37.377460+0000","last_fullsized":"2024-08-17T17:58:37.377460+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1866,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.368119+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:58:37.368119+0000","last_peered":"2024-08-17T17:58:37.368119+0000","last_clean":"2024-08-17T17:58:37.368119+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:58:37.368119+0000","last_undegraded":"2024-08-17T17:58:37.368119+0000","last_fullsized":"2024-08-17T17:58:37.368119+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1898,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.387672+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:58:37.387672+0000","last_peered":"2024-08-17T17:58:37.387672+0000","last_clean":"2024-08-17T17:58:37.387672+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:58:37.387672+0000","last_undegraded":"2024-08-17T17:58:37.387672+0000","last_fullsized":"2024-08-17T17:58:37.387672+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1237,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.404185+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:58:24.404185+0000","last_peered":"2024-08-17T17:58:24.404185+0000","last_clean":"2024-08-17T17:58:24.404185+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:58:24.404185+0000","last_undegraded":"2024-08-17T17:58:24.404185+0000","last_fullsized":"2024-08-17T17:58:24.404185+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1945,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.376769+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:58:37.376769+0000","last_peered":"2024-08-17T17:58:37.376769+0000","last_clean":"2024-08-17T17:58:37.376769+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:58:37.376769+0000","last_undegraded":"2024-08-17T17:58:37.376769+0000","last_fullsized":"2024-08-17T17:58:37.376769+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":132,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.377437+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:58:37.377437+0000","last_peered":"2024-08-17T17:58:37.377437+0000","last_clean":"2024-08-17T17:58:37.377437+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:58:37.377437+0000","last_undegraded":"2024-08-17T17:58:37.377437+0000","last_fullsized":"2024-08-17T17:58:37.377437+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1964,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.376681+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:58:37.376681+0000","last_peered":"2024-08-17T17:58:37.376681+0000","last_clean":"2024-08-17T17:58:37.376681+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:58:37.376681+0000","last_undegraded":"2024-08-17T17:58:37.376681+0000","last_fullsized":"2024-08-17T17:58:37.376681+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":701,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.368671+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:58:37.368671+0000","last_peered":"2024-08-17T17:58:37.368671+0000","last_clean":"2024-08-17T17:58:37.368671+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:58:37.368671+0000","last_undegraded":"2024-08-17T17:58:37.368671+0000","last_fullsized":"2024-08-17T17:58:37.368671+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1945,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.376862+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:58:37.376862+0000","last_peered":"2024-08-17T17:58:37.376862+0000","last_clean":"2024-08-17T17:58:37.376862+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:58:37.376862+0000","last_undegraded":"2024-08-17T17:58:37.376862+0000","last_fullsized":"2024-08-17T17:58:37.376862+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1274,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.387165+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:58:37.387165+0000","last_peered":"2024-08-17T17:58:37.387165+0000","last_clean":"2024-08-17T17:58:37.387165+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:58:37.387165+0000","last_undegraded":"2024-08-17T17:58:37.387165+0000","last_fullsized":"2024-08-17T17:58:37.387165+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":192,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.379266+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:58:37.379266+0000","last_peered":"2024-08-17T17:58:37.379266+0000","last_clean":"2024-08-17T17:58:37.379266+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:58:37.379266+0000","last_undegraded":"2024-08-17T17:58:37.379266+0000","last_fullsized":"2024-08-17T17:58:37.379266+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1956,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.376548+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:58:37.376548+0000","last_peered":"2024-08-17T17:58:37.376548+0000","last_clean":"2024-08-17T17:58:37.376548+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:58:37.376548+0000","last_undegraded":"2024-08-17T17:58:37.376548+0000","last_fullsized":"2024-08-17T17:58:37.376548+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1934,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.379699+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:58:37.379699+0000","last_peered":"2024-08-17T17:58:37.379699+0000","last_clean":"2024-08-17T17:58:37.379699+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:58:37.379699+0000","last_undegraded":"2024-08-17T17:58:37.379699+0000","last_fullsized":"2024-08-17T17:58:37.379699+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1954,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.369582+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:58:37.369582+0000","last_peered":"2024-08-17T17:58:37.369582+0000","last_clean":"2024-08-17T17:58:37.369582+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:58:37.369582+0000","last_undegraded":"2024-08-17T17:58:37.369582+0000","last_fullsized":"2024-08-17T17:58:37.369582+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1959,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.396115+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:58:37.396115+0000","last_peered":"2024-08-17T17:58:37.396115+0000","last_clean":"2024-08-17T17:58:37.396115+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:58:37.396115+0000","last_undegraded":"2024-08-17T17:58:37.396115+0000","last_fullsized":"2024-08-17T17:58:37.396115+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1940,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.397178+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:58:37.397178+0000","last_peered":"2024-08-17T17:58:37.397178+0000","last_clean":"2024-08-17T17:58:37.397178+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:58:37.397178+0000","last_undegraded":"2024-08-17T17:58:37.397178+0000","last_fullsized":"2024-08-17T17:58:37.397178+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1244,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.403697+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:58:24.403697+0000","last_peered":"2024-08-17T17:58:24.403697+0000","last_clean":"2024-08-17T17:58:24.403697+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:58:24.403697+0000","last_undegraded":"2024-08-17T17:58:24.403697+0000","last_fullsized":"2024-08-17T17:58:24.403697+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1190,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.395298+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:58:24.395298+0000","last_peered":"2024-08-17T17:58:24.395298+0000","last_clean":"2024-08-17T17:58:24.395298+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:58:24.395298+0000","last_undegraded":"2024-08-17T17:58:24.395298+0000","last_fullsized":"2024-08-17T17:58:24.395298+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1848,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.364249+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:58:37.364249+0000","last_peered":"2024-08-17T17:58:37.364249+0000","last_clean":"2024-08-17T17:58:37.364249+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:58:37.364249+0000","last_undegraded":"2024-08-17T17:58:37.364249+0000","last_fullsized":"2024-08-17T17:58:37.364249+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":180,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.369725+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:58:37.369725+0000","last_peered":"2024-08-17T17:58:37.369725+0000","last_clean":"2024-08-17T17:58:37.369725+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:58:37.369725+0000","last_undegraded":"2024-08-17T17:58:37.369725+0000","last_fullsized":"2024-08-17T17:58:37.369725+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1556,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.403547+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:58:24.403547+0000","last_peered":"2024-08-17T17:58:24.403547+0000","last_clean":"2024-08-17T17:58:24.403547+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:58:24.403547+0000","last_undegraded":"2024-08-17T17:58:24.403547+0000","last_fullsized":"2024-08-17T17:58:24.403547+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1940,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.376653+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:58:37.376653+0000","last_peered":"2024-08-17T17:58:37.376653+0000","last_clean":"2024-08-17T17:58:37.376653+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:58:37.376653+0000","last_undegraded":"2024-08-17T17:58:37.376653+0000","last_fullsized":"2024-08-17T17:58:37.376653+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1880,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.367016+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:58:37.367016+0000","last_peered":"2024-08-17T17:58:37.367016+0000","last_clean":"2024-08-17T17:58:37.367016+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:58:37.367016+0000","last_undegraded":"2024-08-17T17:58:37.367016+0000","last_fullsized":"2024-08-17T17:58:37.367016+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":524,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.395112+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:58:24.395112+0000","last_peered":"2024-08-17T17:58:24.395112+0000","last_clean":"2024-08-17T17:58:24.395112+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:58:24.395112+0000","last_undegraded":"2024-08-17T17:58:24.395112+0000","last_fullsized":"2024-08-17T17:58:24.395112+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":619,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.369435+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:58:37.369435+0000","last_peered":"2024-08-17T17:58:37.369435+0000","last_clean":"2024-08-17T17:58:37.369435+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:58:37.369435+0000","last_undegraded":"2024-08-17T17:58:37.369435+0000","last_fullsized":"2024-08-17T17:58:37.369435+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":635,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.374137+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:58:37.374137+0000","last_peered":"2024-08-17T17:58:37.374137+0000","last_clean":"2024-08-17T17:58:37.374137+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:58:37.374137+0000","last_undegraded":"2024-08-17T17:58:37.374137+0000","last_fullsized":"2024-08-17T17:58:37.374137+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1208,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.394817+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:58:24.394817+0000","last_peered":"2024-08-17T17:58:24.394817+0000","last_clean":"2024-08-17T17:58:24.394817+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:58:24.394817+0000","last_undegraded":"2024-08-17T17:58:24.394817+0000","last_fullsized":"2024-08-17T17:58:24.394817+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":651,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.363667+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:58:37.363667+0000","last_peered":"2024-08-17T17:58:37.363667+0000","last_clean":"2024-08-17T17:58:37.363667+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:58:37.363667+0000","last_undegraded":"2024-08-17T17:58:37.363667+0000","last_fullsized":"2024-08-17T17:58:37.363667+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1877,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.363873+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:58:37.363873+0000","last_peered":"2024-08-17T17:58:37.363873+0000","last_clean":"2024-08-17T17:58:37.363873+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:58:37.363873+0000","last_undegraded":"2024-08-17T17:58:37.363873+0000","last_fullsized":"2024-08-17T17:58:37.363873+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1972,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.387746+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:58:37.387746+0000","last_peered":"2024-08-17T17:58:37.387746+0000","last_clean":"2024-08-17T17:58:37.387746+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:58:37.387746+0000","last_undegraded":"2024-08-17T17:58:37.387746+0000","last_fullsized":"2024-08-17T17:58:37.387746+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":683,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.363537+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:58:37.363537+0000","last_peered":"2024-08-17T17:58:37.363537+0000","last_clean":"2024-08-17T17:58:37.363537+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:58:37.363537+0000","last_undegraded":"2024-08-17T17:58:37.363537+0000","last_fullsized":"2024-08-17T17:58:37.363537+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1197,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.394644+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:58:24.394644+0000","last_peered":"2024-08-17T17:58:24.394644+0000","last_clean":"2024-08-17T17:58:24.394644+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:58:24.394644+0000","last_undegraded":"2024-08-17T17:58:24.394644+0000","last_fullsized":"2024-08-17T17:58:24.394644+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1196,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.404170+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:58:24.404170+0000","last_peered":"2024-08-17T17:58:24.404170+0000","last_clean":"2024-08-17T17:58:24.404170+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:58:24.404170+0000","last_undegraded":"2024-08-17T17:58:24.404170+0000","last_fullsized":"2024-08-17T17:58:24.404170+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":668,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.375351+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:58:37.375351+0000","last_peered":"2024-08-17T17:58:37.375351+0000","last_clean":"2024-08-17T17:58:37.375351+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:58:37.375351+0000","last_undegraded":"2024-08-17T17:58:37.375351+0000","last_fullsized":"2024-08-17T17:58:37.375351+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1214,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.405537+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:58:24.405537+0000","last_peered":"2024-08-17T17:58:24.405537+0000","last_clean":"2024-08-17T17:58:24.405537+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:58:24.405537+0000","last_undegraded":"2024-08-17T17:58:24.405537+0000","last_fullsized":"2024-08-17T17:58:24.405537+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":98,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.403494+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:58:24.403494+0000","last_peered":"2024-08-17T17:58:24.403494+0000","last_clean":"2024-08-17T17:58:24.403494+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:58:24.403494+0000","last_undegraded":"2024-08-17T17:58:24.403494+0000","last_fullsized":"2024-08-17T17:58:24.403494+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1230,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.404252+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:58:24.404252+0000","last_peered":"2024-08-17T17:58:24.404252+0000","last_clean":"2024-08-17T17:58:24.404252+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:58:24.404252+0000","last_undegraded":"2024-08-17T17:58:24.404252+0000","last_fullsized":"2024-08-17T17:58:24.404252+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1247,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.394647+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:58:24.394647+0000","last_peered":"2024-08-17T17:58:24.394647+0000","last_clean":"2024-08-17T17:58:24.394647+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:58:24.394647+0000","last_undegraded":"2024-08-17T17:58:24.394647+0000","last_fullsized":"2024-08-17T17:58:24.394647+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1878,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.367979+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:58:37.367979+0000","last_peered":"2024-08-17T17:58:37.367979+0000","last_clean":"2024-08-17T17:58:37.367979+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:58:37.367979+0000","last_undegraded":"2024-08-17T17:58:37.367979+0000","last_fullsized":"2024-08-17T17:58:37.367979+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1191,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.403850+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:58:24.403850+0000","last_peered":"2024-08-17T17:58:24.403850+0000","last_clean":"2024-08-17T17:58:24.403850+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:58:24.403850+0000","last_undegraded":"2024-08-17T17:58:24.403850+0000","last_fullsized":"2024-08-17T17:58:24.403850+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1931,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.395886+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:58:37.395886+0000","last_peered":"2024-08-17T17:58:37.395886+0000","last_clean":"2024-08-17T17:58:37.395886+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:58:37.395886+0000","last_undegraded":"2024-08-17T17:58:37.395886+0000","last_fullsized":"2024-08-17T17:58:37.395886+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1932,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.387272+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:58:37.387272+0000","last_peered":"2024-08-17T17:58:37.387272+0000","last_clean":"2024-08-17T17:58:37.387272+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:58:37.387272+0000","last_undegraded":"2024-08-17T17:58:37.387272+0000","last_fullsized":"2024-08-17T17:58:37.387272+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":604,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.403974+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:58:24.403974+0000","last_peered":"2024-08-17T17:58:24.403974+0000","last_clean":"2024-08-17T17:58:24.403974+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:58:24.403974+0000","last_undegraded":"2024-08-17T17:58:24.403974+0000","last_fullsized":"2024-08-17T17:58:24.403974+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1216,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:58:24.404117+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:58:24.404117+0000","last_peered":"2024-08-17T17:58:24.404117+0000","last_clean":"2024-08-17T17:58:24.404117+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:58:24.404117+0000","last_undegraded":"2024-08-17T17:58:24.404117+0000","last_fullsized":"2024-08-17T17:58:24.404117+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1878,"reported_epoch":1247,"state":"active+clean","last_fresh":"2024-08-17T17:58:37.363764+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:58:37.363764+0000","last_peered":"2024-08-17T17:58:37.363764+0000","last_clean":"2024-08-17T17:58:37.363764+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:58:37.363764+0000","last_undegraded":"2024-08-17T17:58:37.363764+0000","last_fullsized":"2024-08-17T17:58:37.363764+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":45,"num_pg":16,"stat_sum":{"num_bytes":303824920,"num_objects":4637,"num_object_clones":0,"num_object_copies":13911,"num_objects_missing_on_primary":553,"num_objects_missing":553,"num_objects_degraded":16704,"num_objects_misplaced":772,"num_objects_unfound":0,"num_objects_dirty":4637,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":204973,"num_write_kb":6707457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38591,"num_bytes_recovered":2339307520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2733387776,"data_stored":2733375560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108347,"ondisk_log_size":108347,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234456,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":435744,"kb_used_data":125912,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93309408,"statfs":{"total":95995035648,"available":95548833792,"internally_reserved":0,"allocated":128933888,"data_stored":123860963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496091,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":442788,"kb_used_data":132956,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93302364,"statfs":{"total":95995035648,"available":95541620736,"internally_reserved":0,"allocated":136146944,"data_stored":131070424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757729,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326342,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":502668,"kb_used_data":241156,"kb_used_omap":57,"kb_used_meta":261382,"kb_avail":93242484,"statfs":{"total":95995035648,"available":95480303616,"internally_reserved":0,"allocated":246943744,"data_stored":241885999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":267655931},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405581,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":462996,"kb_used_data":200396,"kb_used_omap":61,"kb_used_meta":262530,"kb_avail":93282156,"statfs":{"total":95995035648,"available":95520927744,"internally_reserved":0,"allocated":205205504,"data_stored":200144731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":268831672},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,4,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452116,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311048,"kb_used_data":120960,"kb_used_omap":54,"kb_used_meta":190025,"kb_avail":93434104,"statfs":{"total":95995035648,"available":95676522496,"internally_reserved":0,"allocated":123863040,"data_stored":118809256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":194585998},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474807,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":386712,"kb_used_data":145616,"kb_used_omap":119,"kb_used_meta":240968,"kb_avail":93358440,"statfs":{"total":95995035648,"available":95599042560,"internally_reserved":0,"allocated":149110784,"data_stored":144041836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":246751966},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,1,2,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019364,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":115675136,"data_stored":115671064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":107741184,"data_stored":107741184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":185794560,"data_stored":185794560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":223023104,"data_stored":223019032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":788529152,"data_stored":788529152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1087897600,"data_stored":1087897600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":121114624,"data_stored":121110552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":45,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":103612416,"data_stored":103612416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:58:57.830 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:58:57.916 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:57.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:57.938 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:57.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:58.012 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:58:58.013 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:58:58.039 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:58.061 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:58.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:58.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:58.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:58:58.163 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:58.186 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:58.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:58.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:58.288 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:58.331 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:58.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:58.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:58.433 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:58:58.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:58.484 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:58.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:58:58.585 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:58.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:58.635 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:58.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:58.644 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Removed 104804 objects 2024-08-17T17:58:58.644 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Clean up completed and total clean up time :579.903 2024-08-17T17:58:58.736 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:58:58.758 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:58.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:58.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:58:58.859 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:58.880 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:58.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:58.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:58.982 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:58:58.993 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:58 smithi130 ceph-mon[105521]: pgmap v1979: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 196 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 16486/9438 objects degraded (174.677%); 772/9438 objects misplaced (8.180%); 0 B/s, 12 objects/s recovering 2024-08-17T17:58:58.993 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:58 smithi130 ceph-mon[110068]: pgmap v1979: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 196 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 16486/9438 objects degraded (174.677%); 772/9438 objects misplaced (8.180%); 0 B/s, 12 objects/s recovering 2024-08-17T17:58:59.003 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:59.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:59.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:58:59.093 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:58:59.093 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:58:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:58 smithi179 ceph-mon[93519]: pgmap v1979: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 196 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 16486/9438 objects degraded (174.677%); 772/9438 objects misplaced (8.180%); 0 B/s, 12 objects/s recovering 2024-08-17T17:58:59.106 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:59.114 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:58:59.115 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:58:59.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:59.130 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:59.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:59.133 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:58:59.134 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:58:59.151 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:58:59.151 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:58:59.168 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:58:59.168 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:58:59.184 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:58:59.185 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:58:59.200 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:58:59.200 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:58:59.217 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:58:59.218 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:58:59.232 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:58:59.253 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:59.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:59.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:58:59.355 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:59.379 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:59.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:59.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:59.481 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:58:59.502 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:59.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:59.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:58:59.604 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:58:59.626 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:59.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:59.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:58:59.727 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:58:59.749 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:58:59.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:59.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:58:59.799 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-08-17T17:58:59.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-08-17T17:58:59.851 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:58:59.876 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:58:59.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:58:59.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:58:59.978 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:00.003 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:00.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:00.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:00.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:58:59 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3432340667' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:59:00.106 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:00.130 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:00.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:00.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:58:59 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3432340667' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:59:00.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:58:59 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3432340667' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:59:00.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:58:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:58:59] "GET /metrics HTTP/1.1" 200 36789 "" "Prometheus/2.51.0" 2024-08-17T17:59:00.232 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:00.257 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:00.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:00.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:00.307 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:00.309 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:00.322 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:00.359 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:00.385 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:00.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:00.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:00.439 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:59:00.488 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:00.513 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:00.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:00.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:59:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:59:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:59:00.615 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:00.640 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:00.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:00.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:00.742 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:00.765 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:00.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:00.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:00.867 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:00.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:00.891 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:00.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:00.992 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:01.018 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:01.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:01.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:00 smithi179 ceph-mon[93519]: pgmap v1980: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 50 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 607 op/s; 16668/2436 objects degraded (684.236%); 590/2436 objects misplaced (24.220%); 0 B/s, 16 objects/s recovering 2024-08-17T17:59:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:00 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16668/2436 objects degraded (684.236%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:59:01.094 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:00 smithi130 ceph-mon[105521]: pgmap v1980: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 50 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 607 op/s; 16668/2436 objects degraded (684.236%); 590/2436 objects misplaced (24.220%); 0 B/s, 16 objects/s recovering 2024-08-17T17:59:01.094 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:00 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16668/2436 objects degraded (684.236%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:59:01.094 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:00 smithi130 ceph-mon[110068]: pgmap v1980: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 50 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 607 op/s; 16668/2436 objects degraded (684.236%); 590/2436 objects misplaced (24.220%); 0 B/s, 16 objects/s recovering 2024-08-17T17:59:01.094 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:00 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16668/2436 objects degraded (684.236%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:59:01.121 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:01.146 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:01.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:01.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:01.248 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:01.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:01.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:01.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:01.376 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:01.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:01.398 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:01.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:01.499 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:01.524 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:01.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:01.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:01.626 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:01.647 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:01.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:01.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:01.749 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:01.770 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:01.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:01.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:01.872 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:01.897 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:01.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:01.999 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:02.021 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:02.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:02.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:02.123 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:02.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:02.146 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:02.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:02.230 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:59:02.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:02.295 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:02.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:02.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:02.398 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:02.423 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:02.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:02.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:02.525 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:02.546 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:02.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:02.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:02.651 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:02.674 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:02.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:02.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:02.776 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:02.802 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:02.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:02.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:02.904 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:02.929 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:02.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:02.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:03.030 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:03.052 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:03.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:03.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:03.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:02 smithi179 ceph-mon[93519]: pgmap v1981: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 27 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 16668/1368 objects degraded (1218.421%); 590/1368 objects misplaced (43.129%); 0 B/s, 16 objects/s recovering 2024-08-17T17:59:03.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:02 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:03.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:02 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:03.153 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:03.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:02 smithi130 ceph-mon[110068]: pgmap v1981: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 27 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 16668/1368 objects degraded (1218.421%); 590/1368 objects misplaced (43.129%); 0 B/s, 16 objects/s recovering 2024-08-17T17:59:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:02 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:02 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:02 smithi130 ceph-mon[105521]: pgmap v1981: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 27 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 16668/1368 objects degraded (1218.421%); 590/1368 objects misplaced (43.129%); 0 B/s, 16 objects/s recovering 2024-08-17T17:59:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:02 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:02 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:03.179 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:03.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:03.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:03.281 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:03.306 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:03.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:03.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:03.407 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:03.429 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:03.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:03.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:03.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:03.554 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:03.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:03.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:03.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:03.681 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:03.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:03.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:03.783 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:03.808 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:03.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:03.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:03.910 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:03.932 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:03.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:03.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:04.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:04.056 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:04.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:04.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:04.157 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:04.183 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:04.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:04.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:04.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:04.306 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:04.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:04.408 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:04.430 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:04.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:04.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:04.532 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:04.557 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:04.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:04.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:04.659 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:04.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:04.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:04.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:04.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:04.809 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:04.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:04.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:04.911 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:04.933 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:04.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:04.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:05.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:05.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:05.057 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:05.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:05.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:04 smithi179 ceph-mon[93519]: pgmap v1982: 89 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 16599/78 objects degraded (21280.769%); 518/78 objects misplaced (664.103%); 0 B/s, 62 objects/s recovering 2024-08-17T17:59:05.157 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:05.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:04 smithi130 ceph-mon[105521]: pgmap v1982: 89 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 16599/78 objects degraded (21280.769%); 518/78 objects misplaced (664.103%); 0 B/s, 62 objects/s recovering 2024-08-17T17:59:05.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:04 smithi130 ceph-mon[110068]: pgmap v1982: 89 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 16599/78 objects degraded (21280.769%); 518/78 objects misplaced (664.103%); 0 B/s, 62 objects/s recovering 2024-08-17T17:59:05.183 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:05.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:05.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:05.285 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:05.310 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:05.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:05.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:05.412 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:05.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:05.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:05.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:05.540 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:05.561 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:05.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:05.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:05.663 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:05.689 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:05.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:05.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:05.791 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:05.813 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:05.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:05.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:05.914 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:05.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:05 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:59:05.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:05 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:59:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:05 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:05.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:05 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 16599/78 objects degraded (21280.769%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:59:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:05 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:59:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:05 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:59:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:05 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:05 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 16599/78 objects degraded (21280.769%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:59:05.940 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:05.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:05.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:06.041 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:06.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:06.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:06.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:05 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:59:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:05 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:59:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:05 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:05 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 16599/78 objects degraded (21280.769%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:59:06.105 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:06.110 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:06.168 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:06.194 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:06.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:06.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:06.296 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:06.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:06.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:06.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:06.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:06.443 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:06.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:06.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:06.545 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:06.569 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:06.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:06.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:06.671 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:06.695 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:06.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:06.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:06.796 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:06.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:06.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:06.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:06.924 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:06.945 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:06.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:06.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:07.046 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:07.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:07.068 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:07.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:07.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:06 smithi179 ceph-mon[93519]: pgmap v1983: 89 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 340 op/s; 16599/78 objects degraded (21280.769%); 518/78 objects misplaced (664.103%); 0 B/s, 60 objects/s recovering 2024-08-17T17:59:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:59:07.099 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 16599/78 objects degraded (21280.769%), 5 pgs degraded, 6 pgs undersized","count":11},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1868,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1249,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916918,"num_in_osds":6,"osd_in_since":1723916984,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+remapped","count":1}],"num_pgs":89,"num_pools":5,"num_objects":26,"data_bytes":595917,"bytes_used":1666605056,"bytes_avail":574303608832,"bytes_total":575970213888,"degraded_objects":16599,"degraded_total":78,"degraded_ratio":212.80769230769229,"misplaced_objects":518,"misplaced_total":78,"misplaced_ratio":6.6410256410256405,"recovering_objects_per_sec":60,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":630,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":340},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":28,"modified":"2024-08-17T17:58:59.134301+0000","services":{}},"progress_events":{"5d1fe27c-00cc-42f3-91d3-4d449184a99b":{"message":"Global Recovery Event (11m)\n [==========================..] (remaining: 49s)","progress":0.93258428573608398,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T17:59:07.100 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:06 smithi130 ceph-mon[105521]: pgmap v1983: 89 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 340 op/s; 16599/78 objects degraded (21280.769%); 518/78 objects misplaced (664.103%); 0 B/s, 60 objects/s recovering 2024-08-17T17:59:07.100 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:06 smithi130 ceph-mon[110068]: pgmap v1983: 89 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 340 op/s; 16599/78 objects degraded (21280.769%); 518/78 objects misplaced (664.103%); 0 B/s, 60 objects/s recovering 2024-08-17T17:59:07.170 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:07.191 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:07.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:07.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:07.293 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:07.314 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:07.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:07.416 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:07.437 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:07.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:07.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:07.539 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:07.567 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:07.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:07.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:07.669 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:07.690 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:07.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:07.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:07.792 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:07.817 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:07.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:07.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:07.918 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:07.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:07.941 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:07.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:08.042 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:08.064 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:08.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:08.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:08.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:07 smithi179 ceph-mon[93519]: osdmap e1249: 8 total, 8 up, 6 in 2024-08-17T17:59:08.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:07 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1604318508' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:59:08.165 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:08.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:07 smithi130 ceph-mon[110068]: osdmap e1249: 8 total, 8 up, 6 in 2024-08-17T17:59:08.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:07 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1604318508' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:59:08.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:07 smithi130 ceph-mon[105521]: osdmap e1249: 8 total, 8 up, 6 in 2024-08-17T17:59:08.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:07 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1604318508' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:59:08.187 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:08.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:08.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:08.289 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:08.310 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:08.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:08.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:08.412 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:08.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:08.433 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:08.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:08.534 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:08.555 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:08.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:08.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:08.657 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:08.678 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:08.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:08.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:08.780 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:08.801 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:08.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:08.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:08.903 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:08.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:08 smithi130 ceph-mon[110068]: pgmap v1985: 89 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 311 op/s; 16599/78 objects degraded (21280.769%); 518/78 objects misplaced (664.103%); 0 B/s, 62 objects/s recovering 2024-08-17T17:59:08.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:08 smithi130 ceph-mon[110068]: osdmap e1250: 8 total, 8 up, 6 in 2024-08-17T17:59:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:08 smithi130 ceph-mon[105521]: pgmap v1985: 89 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 311 op/s; 16599/78 objects degraded (21280.769%); 518/78 objects misplaced (664.103%); 0 B/s, 62 objects/s recovering 2024-08-17T17:59:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:08 smithi130 ceph-mon[105521]: osdmap e1250: 8 total, 8 up, 6 in 2024-08-17T17:59:08.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:08.925 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:08.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:09.026 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:09.047 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:09.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:09.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:08 smithi179 ceph-mon[93519]: pgmap v1985: 89 pgs: 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 311 op/s; 16599/78 objects degraded (21280.769%); 518/78 objects misplaced (664.103%); 0 B/s, 62 objects/s recovering 2024-08-17T17:59:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:08 smithi179 ceph-mon[93519]: osdmap e1250: 8 total, 8 up, 6 in 2024-08-17T17:59:09.149 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:09.170 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:09.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:09.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:09.271 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:09.293 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:09.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:09.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:09.395 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:09.416 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:09.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:09.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:09.517 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:09.539 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:09.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:09.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:09.641 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:09.662 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:09.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:09.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:09.764 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:09.784 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:09.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:09.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:09.886 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:09.909 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:09.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:09.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:10.011 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:10.033 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:10.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:10.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:10.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:09 smithi179 ceph-mon[93519]: osdmap e1251: 8 total, 8 up, 6 in 2024-08-17T17:59:10.134 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:10.155 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:10.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:10.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:10.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:09 smithi130 ceph-mon[105521]: osdmap e1251: 8 total, 8 up, 6 in 2024-08-17T17:59:10.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:09 smithi130 ceph-mon[110068]: osdmap e1251: 8 total, 8 up, 6 in 2024-08-17T17:59:10.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:59:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:59:09] "GET /metrics HTTP/1.1" 200 36789 "" "Prometheus/2.51.0" 2024-08-17T17:59:10.257 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:10.278 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:10.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:10.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:10.380 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:10.401 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:10.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:10.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:10.503 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:10.524 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:10.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:10.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:59:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:59:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:59:10.625 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:10.647 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:10.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:10.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:10.736 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:59:10.736 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:59:10.750 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:10.773 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:10.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:10.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:10.875 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:10.897 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:10.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:10.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:10.999 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:11.013 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:10 smithi130 ceph-mon[105521]: pgmap v1988: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 14412/78 objects degraded (18476.923%); 518/78 objects misplaced (664.103%); 0 B/s, 364 objects/s recovering 2024-08-17T17:59:11.013 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:10 smithi130 ceph-mon[105521]: osdmap e1252: 8 total, 8 up, 6 in 2024-08-17T17:59:11.013 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:10 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 14412/78 objects degraded (18476.923%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:59:11.013 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:10 smithi130 ceph-mon[105521]: osdmap e1253: 8 total, 8 up, 6 in 2024-08-17T17:59:11.013 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:10 smithi130 ceph-mon[110068]: pgmap v1988: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 14412/78 objects degraded (18476.923%); 518/78 objects misplaced (664.103%); 0 B/s, 364 objects/s recovering 2024-08-17T17:59:11.014 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:10 smithi130 ceph-mon[110068]: osdmap e1252: 8 total, 8 up, 6 in 2024-08-17T17:59:11.014 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:10 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 14412/78 objects degraded (18476.923%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:59:11.014 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:10 smithi130 ceph-mon[110068]: osdmap e1253: 8 total, 8 up, 6 in 2024-08-17T17:59:11.020 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:11.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:11.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:10 smithi179 ceph-mon[93519]: pgmap v1988: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 14412/78 objects degraded (18476.923%); 518/78 objects misplaced (664.103%); 0 B/s, 364 objects/s recovering 2024-08-17T17:59:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:10 smithi179 ceph-mon[93519]: osdmap e1252: 8 total, 8 up, 6 in 2024-08-17T17:59:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:10 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 14412/78 objects degraded (18476.923%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:59:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:10 smithi179 ceph-mon[93519]: osdmap e1253: 8 total, 8 up, 6 in 2024-08-17T17:59:11.123 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:11.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:11.147 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:11.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:11.248 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:11.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:11.271 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:11.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:11.373 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:11.395 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:11.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:11.496 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:11.518 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:11.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:11.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:11.620 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:11.643 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:11.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:11.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:11.745 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:11.766 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:11.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:11.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:11.776 INFO:teuthology.orchestra.run.smithi130.stderr:pool 'unique_pool_0' removed 2024-08-17T17:59:11.869 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:11.890 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:11.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:11.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:11.992 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:12.015 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:12.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:12.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:12.117 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:12.137 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:12.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:12.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:11 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/848878102' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:59:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:11 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:59:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:11 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:59:12.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:11 smithi130 ceph-mon[110068]: osdmap e1254: 8 total, 8 up, 6 in 2024-08-17T17:59:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:11 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/848878102' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:59:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:11 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:59:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:11 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:59:12.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:11 smithi130 ceph-mon[105521]: osdmap e1254: 8 total, 8 up, 6 in 2024-08-17T17:59:12.240 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:12.263 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:12.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:12.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:11 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/848878102' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:59:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:11 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:59:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:11 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:59:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:11 smithi179 ceph-mon[93519]: osdmap e1254: 8 total, 8 up, 6 in 2024-08-17T17:59:12.365 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:12.388 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:12.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:12.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:12.490 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:12.511 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:12.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:12.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:12.553 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:59:12.553 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:59:12.613 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:12.637 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:12.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:12.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:12.739 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:12.760 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:12.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:12.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:12.862 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:12.886 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:12.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:12.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:12.988 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:13.012 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:13.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:13.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:13.113 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:13.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:13.138 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:13.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:13.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:12 smithi130 ceph-mon[105521]: pgmap v1991: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 14412/78 objects degraded (18476.923%); 518/78 objects misplaced (664.103%); 0 B/s, 546 objects/s recovering 2024-08-17T17:59:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:59:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:12 smithi130 ceph-mon[110068]: pgmap v1991: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 14412/78 objects degraded (18476.923%); 518/78 objects misplaced (664.103%); 0 B/s, 546 objects/s recovering 2024-08-17T17:59:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:59:13.239 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:13.262 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:13.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:13.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:13.264 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:12 smithi179 ceph-mon[93519]: pgmap v1991: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail; 14412/78 objects degraded (18476.923%); 518/78 objects misplaced (664.103%); 0 B/s, 546 objects/s recovering 2024-08-17T17:59:13.264 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:59:13.364 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:13.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:13.387 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:13.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:13.488 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:13.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:13.514 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:13.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:13.618 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:13.643 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:13.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:13.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:13.745 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:13.767 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:13.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:13.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:13.869 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:13.894 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:13.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:13.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:13.964 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:13 smithi130 ceph-mon[105521]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14412/78 objects degraded (18476.923%), 4 pgs degraded, 5 pgs undersized) 2024-08-17T17:59:13.964 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:13 smithi130 ceph-mon[110068]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14412/78 objects degraded (18476.923%), 4 pgs degraded, 5 pgs undersized) 2024-08-17T17:59:13.987 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:59:13.988 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:59:13.988 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-08-17T17:59:13.988 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_1 16 2024-08-17T17:59:13.996 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:14.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:14.054 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:14.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:14.155 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:14.177 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:14.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:14.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:14.278 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:14.304 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:14.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:14.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:14.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:13 smithi179 ceph-mon[93519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14412/78 objects degraded (18476.923%), 4 pgs degraded, 5 pgs undersized) 2024-08-17T17:59:14.406 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:14.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:14.433 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:14.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:14.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:14.556 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:14.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:14.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:14.659 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:14.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:14.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:14.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:14.786 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:14.812 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:14.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:14.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:14.914 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:14.938 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:14.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:14.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:15.028 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:14 smithi130 ceph-mon[105521]: pgmap v1993: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:15.028 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:14 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]} : dispatch 2024-08-17T17:59:15.029 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:14 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]} : dispatch 2024-08-17T17:59:15.029 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]} : dispatch 2024-08-17T17:59:15.029 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:14 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]} : dispatch 2024-08-17T17:59:15.029 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:14 smithi130 ceph-mon[110068]: pgmap v1993: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:15.029 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:14 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]} : dispatch 2024-08-17T17:59:15.029 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:14 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]} : dispatch 2024-08-17T17:59:15.029 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]} : dispatch 2024-08-17T17:59:15.029 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:14 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]} : dispatch 2024-08-17T17:59:15.040 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:15.053 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:59:15.065 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:15.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:15.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:15.167 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:15.189 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:15.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:15.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:15.291 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:15.317 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:15.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:15.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:14 smithi179 ceph-mon[93519]: pgmap v1993: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:14 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]} : dispatch 2024-08-17T17:59:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:14 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]} : dispatch 2024-08-17T17:59:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]} : dispatch 2024-08-17T17:59:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:14 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]} : dispatch 2024-08-17T17:59:15.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:15.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:15.442 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:15.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:15.543 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:15.564 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:15.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:15.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:15.666 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:15.687 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:15.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:15.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:15.789 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:15.814 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:15.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:15.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:15.916 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:15.938 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:15.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:15.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:16.040 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:16.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:16.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:16.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:16.160 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:15 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]}]': finished 2024-08-17T17:59:16.160 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:15 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]}]': finished 2024-08-17T17:59:16.160 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:15 smithi130 ceph-mon[105521]: osdmap e1255: 8 total, 8 up, 6 in 2024-08-17T17:59:16.160 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:15 smithi130 ceph-mon[105521]: osdmap e1256: 8 total, 8 up, 6 in 2024-08-17T17:59:16.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:15 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]}]': finished 2024-08-17T17:59:16.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:15 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]}]': finished 2024-08-17T17:59:16.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:15 smithi130 ceph-mon[110068]: osdmap e1255: 8 total, 8 up, 6 in 2024-08-17T17:59:16.160 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:15 smithi130 ceph-mon[110068]: osdmap e1256: 8 total, 8 up, 6 in 2024-08-17T17:59:16.168 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:16.195 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:16.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:16.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:16.244 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:15 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.18", "id": [3, 2]}]': finished 2024-08-17T17:59:16.244 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:15 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.19", "id": [1, 2]}]': finished 2024-08-17T17:59:16.244 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:15 smithi179 ceph-mon[93519]: osdmap e1255: 8 total, 8 up, 6 in 2024-08-17T17:59:16.245 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:15 smithi179 ceph-mon[93519]: osdmap e1256: 8 total, 8 up, 6 in 2024-08-17T17:59:16.296 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:16.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:16.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:16.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:16.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:16.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:16.442 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:16.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:16.495 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:59:16.495 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:59:16.505 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:59:16.505 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:59:16.522 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:59:16.522 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:59:16.539 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:59:16.539 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:59:16.544 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:16.555 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:59:16.555 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:59:16.570 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:16.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:16.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:16.571 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:59:16.571 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:59:16.586 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:59:16.587 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:59:16.602 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:59:16.602 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:59:16.671 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:16.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:16.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:16.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:16.795 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:16.803 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:16.803 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:16.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:16.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:16.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:16.923 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:16.944 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:16.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:16.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:17.045 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:17.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:16 smithi130 ceph-mon[105521]: pgmap v1995: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:17.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:16 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2046121071' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:59:17.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:16 smithi130 ceph-mon[110068]: pgmap v1995: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:17.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:16 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2046121071' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:59:17.067 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:17.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:17.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:17.169 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:17.195 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:17.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:17.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:17.256 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:17.296 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:17.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:17.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:17.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:17.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:16 smithi179 ceph-mon[93519]: pgmap v1995: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:17.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:16 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2046121071' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:59:17.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:17.441 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:17.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:17.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:17.543 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:17.569 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:17.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:17.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:17.671 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:17.696 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:17.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:17.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:17.797 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:17.819 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:17.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:17.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:17.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:17.930 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T17:59:17.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:17.944 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:17.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:18.045 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:18.071 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:18.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:18.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:18.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:17 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/616412516' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:59:18.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:17 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:59:18.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:17 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:18.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:17 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/616412516' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:59:18.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:17 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:59:18.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:17 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:18.172 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:18.193 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:18.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:18.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:18.295 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:18.321 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:18.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:18.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:18.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:17 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/616412516' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:59:18.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:17 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:59:18.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:17 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:18.424 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:18.446 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:18.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:18.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:18.548 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:18.574 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:18.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:18.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:18.676 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:18.697 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:18.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:18.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:18.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:18.820 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:18.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:18.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:18.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:18.942 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:18.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:18.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:19.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:19.065 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:19.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:19.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:19.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:18 smithi130 ceph-mon[105521]: pgmap v1997: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:19.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:18 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:59:19.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:18 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:59:19.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:18 smithi130 ceph-mon[105521]: osdmap e1257: 8 total, 8 up, 6 in 2024-08-17T17:59:19.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:18 smithi130 ceph-mon[110068]: pgmap v1997: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:19.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:18 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:59:19.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:18 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:59:19.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:18 smithi130 ceph-mon[110068]: osdmap e1257: 8 total, 8 up, 6 in 2024-08-17T17:59:19.168 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:19.193 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:19.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:19.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:19.294 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:19.320 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:19.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:19.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:18 smithi179 ceph-mon[93519]: pgmap v1997: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:18 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:59:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:18 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:59:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:18 smithi179 ceph-mon[93519]: osdmap e1257: 8 total, 8 up, 6 in 2024-08-17T17:59:19.421 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:19.447 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:19.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:19.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:19.549 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:19.573 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:19.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:19.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:19.675 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:19.700 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:19.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:19.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:19.736 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:59:19.801 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:19.825 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:19.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:19.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:19.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:59:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:59:19] "GET /metrics HTTP/1.1" 200 34367 "" "Prometheus/2.51.0" 2024-08-17T17:59:19.927 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:19.950 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:19.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:19.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:20.052 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:20.075 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:20.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:20.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:20.177 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:20.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:20.202 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:20.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:20.303 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:20.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:20.327 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:20.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:20.428 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:20.449 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:20.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:20.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:20.551 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:20.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:20.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:20.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:59:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:59:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:59:20.677 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:20.699 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:20.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:20.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:20.801 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:20.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:20.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:20.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:20.924 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:20.946 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:20.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:20.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:21.048 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:21.074 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:21.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:21.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:21.175 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:21.197 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:21.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:21.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:21.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:20 smithi130 ceph-mon[105521]: pgmap v1999: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:21.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:20 smithi130 ceph-mon[110068]: pgmap v1999: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:21.298 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:21.324 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:21.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:21.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:21.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:20 smithi179 ceph-mon[93519]: pgmap v1999: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:21.425 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:21.451 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:21.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:21.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:21.553 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:21.577 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:21.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:21.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:21.679 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:21.703 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:21.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:21.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:21.805 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:21.827 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:21.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:21.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:21.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:21.940 INFO:teuthology.orchestra.run.smithi130.stderr:pool 'unique_pool_1' created 2024-08-17T17:59:21.954 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:21.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:21.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:21.996 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:59:21.998 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:59:22.056 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:22.079 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:22.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:22.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:22.181 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:22.204 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:22.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:22.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:22.269 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:21 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3294923282' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:59:22.270 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:21 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:59:22.270 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:21 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3294923282' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:59:22.270 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:21 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:59:22.306 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:22.328 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:22.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:22.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:22.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:21 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3294923282' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:59:22.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:21 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:59:22.429 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:22.455 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:22.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:22.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:22.557 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:22.582 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:22.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:22.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:22.683 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:22.707 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:22.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:22.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:22.786 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-08-17T17:59:22.809 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:22.832 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:22.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:22.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:22.934 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:22.961 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:22.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:22.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:23.063 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:23.084 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:23.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:23.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:23.091 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:59:23.147 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:22 smithi130 ceph-mon[105521]: pgmap v2000: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:23.147 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:22 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-17T17:59:23.147 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:22 smithi130 ceph-mon[105521]: osdmap e1258: 8 total, 8 up, 6 in 2024-08-17T17:59:23.148 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:22 smithi130 ceph-mon[105521]: from='client.36198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:59:23.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:22 smithi130 ceph-mon[110068]: pgmap v2000: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:23.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:22 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-17T17:59:23.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:22 smithi130 ceph-mon[110068]: osdmap e1258: 8 total, 8 up, 6 in 2024-08-17T17:59:23.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:22 smithi130 ceph-mon[110068]: from='client.36198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:59:23.187 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:23.211 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:23.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:23.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:23.313 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:23.337 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:23.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:23.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:23.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:22 smithi179 ceph-mon[93519]: pgmap v2000: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:22 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-17T17:59:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:22 smithi179 ceph-mon[93519]: osdmap e1258: 8 total, 8 up, 6 in 2024-08-17T17:59:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:22 smithi179 ceph-mon[93519]: from='client.36198 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:59:23.439 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:23.450 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:23.464 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:23.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:23.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:23.565 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:23.591 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:23.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:23.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:23.693 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:23.714 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:23.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:23.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:23.816 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:23.841 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:23.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:23.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:23.935 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2001,"stamp":"2024-08-17T17:59:21.934270+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":1776653,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":220,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":1650112,"kb_used_data":80184,"kb_used_omap":295,"kb_used_meta":1567640,"kb_avail":560820800,"statfs":{"total":575970213888,"available":574280499200,"internally_reserved":0,"allocated":82108416,"data_stored":52390583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1605263823},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.796205"},"pg_stats":[{"pgid":"46.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"46.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:59:21.929283+0000","last_change":"2024-08-17T17:59:21.929283+0000","last_active":"2024-08-17T17:59:21.929283+0000","last_peered":"2024-08-17T17:59:21.929283+0000","last_clean":"2024-08-17T17:59:21.929283+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:59:21.929283+0000","last_undegraded":"2024-08-17T17:59:21.929283+0000","last_fullsized":"2024-08-17T17:59:21.929283+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1971,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260533+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:59:16.260533+0000","last_peered":"2024-08-17T17:59:16.260533+0000","last_clean":"2024-08-17T17:59:16.260533+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:59:16.260533+0000","last_undegraded":"2024-08-17T17:59:16.260533+0000","last_fullsized":"2024-08-17T17:59:16.260533+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":534,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.909896+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:59:15.909896+0000","last_peered":"2024-08-17T17:59:15.909896+0000","last_clean":"2024-08-17T17:59:15.909896+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:59:15.909896+0000","last_undegraded":"2024-08-17T17:59:15.909896+0000","last_fullsized":"2024-08-17T17:59:15.909896+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1892,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337860+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:59:18.337860+0000","last_peered":"2024-08-17T17:59:18.337860+0000","last_clean":"2024-08-17T17:59:18.337860+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:59:18.337860+0000","last_undegraded":"2024-08-17T17:59:18.337860+0000","last_fullsized":"2024-08-17T17:59:18.337860+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1916,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337606+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:59:18.337606+0000","last_peered":"2024-08-17T17:59:18.337606+0000","last_clean":"2024-08-17T17:59:18.337606+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:59:18.337606+0000","last_undegraded":"2024-08-17T17:59:18.337606+0000","last_fullsized":"2024-08-17T17:59:18.337606+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":96,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.910449+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:59:15.910449+0000","last_peered":"2024-08-17T17:59:15.910449+0000","last_clean":"2024-08-17T17:59:15.910449+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:59:15.910449+0000","last_undegraded":"2024-08-17T17:59:15.910449+0000","last_fullsized":"2024-08-17T17:59:15.910449+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1309,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337295+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:59:18.337295+0000","last_peered":"2024-08-17T17:59:18.337295+0000","last_clean":"2024-08-17T17:59:18.337295+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:59:18.337295+0000","last_undegraded":"2024-08-17T17:59:18.337295+0000","last_fullsized":"2024-08-17T17:59:18.337295+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":165,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260582+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:59:16.260582+0000","last_peered":"2024-08-17T17:59:16.260582+0000","last_clean":"2024-08-17T17:59:16.260582+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:59:16.260582+0000","last_undegraded":"2024-08-17T17:59:16.260582+0000","last_fullsized":"2024-08-17T17:59:16.260582+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":687,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.338888+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:59:18.338888+0000","last_peered":"2024-08-17T17:59:18.338888+0000","last_clean":"2024-08-17T17:59:18.338888+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:59:18.338888+0000","last_undegraded":"2024-08-17T17:59:18.338888+0000","last_fullsized":"2024-08-17T17:59:18.338888+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1929,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260080+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:59:16.260080+0000","last_peered":"2024-08-17T17:59:16.260080+0000","last_clean":"2024-08-17T17:59:16.260080+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:59:16.260080+0000","last_undegraded":"2024-08-17T17:59:16.260080+0000","last_fullsized":"2024-08-17T17:59:16.260080+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1933,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337463+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:59:18.337463+0000","last_peered":"2024-08-17T17:59:18.337463+0000","last_clean":"2024-08-17T17:59:18.337463+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:59:18.337463+0000","last_undegraded":"2024-08-17T17:59:18.337463+0000","last_fullsized":"2024-08-17T17:59:18.337463+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":150,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260083+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:59:16.260083+0000","last_peered":"2024-08-17T17:59:16.260083+0000","last_clean":"2024-08-17T17:59:16.260083+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:59:16.260083+0000","last_undegraded":"2024-08-17T17:59:16.260083+0000","last_fullsized":"2024-08-17T17:59:16.260083+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1905,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337845+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:59:18.337845+0000","last_peered":"2024-08-17T17:59:18.337845+0000","last_clean":"2024-08-17T17:59:18.337845+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:59:18.337845+0000","last_undegraded":"2024-08-17T17:59:18.337845+0000","last_fullsized":"2024-08-17T17:59:18.337845+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337056+0000","last_change":"2024-08-17T17:59:15.912662+0000","last_active":"2024-08-17T17:59:18.337056+0000","last_peered":"2024-08-17T17:59:18.337056+0000","last_clean":"2024-08-17T17:59:18.337056+0000","last_became_active":"2024-08-17T17:59:15.912410+0000","last_became_peered":"2024-08-17T17:59:15.912410+0000","last_unstale":"2024-08-17T17:59:18.337056+0000","last_undegraded":"2024-08-17T17:59:18.337056+0000","last_fullsized":"2024-08-17T17:59:18.337056+0000","mapping_epoch":1255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:07:10.368942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1952,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260430+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:59:16.260430+0000","last_peered":"2024-08-17T17:59:16.260430+0000","last_clean":"2024-08-17T17:59:16.260430+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:59:16.260430+0000","last_undegraded":"2024-08-17T17:59:16.260430+0000","last_fullsized":"2024-08-17T17:59:16.260430+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.336908+0000","last_change":"2024-08-17T17:59:16.261638+0000","last_active":"2024-08-17T17:59:18.336908+0000","last_peered":"2024-08-17T17:59:18.336908+0000","last_clean":"2024-08-17T17:59:18.336908+0000","last_became_active":"2024-08-17T17:59:16.261277+0000","last_became_peered":"2024-08-17T17:59:16.261277+0000","last_unstale":"2024-08-17T17:59:18.336908+0000","last_undegraded":"2024-08-17T17:59:18.336908+0000","last_fullsized":"2024-08-17T17:59:18.336908+0000","mapping_epoch":1255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:10:50.522770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1231,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.910536+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:59:15.910536+0000","last_peered":"2024-08-17T17:59:15.910536+0000","last_clean":"2024-08-17T17:59:15.910536+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:59:15.910536+0000","last_undegraded":"2024-08-17T17:59:15.910536+0000","last_fullsized":"2024-08-17T17:59:15.910536+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1898,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337894+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:59:18.337894+0000","last_peered":"2024-08-17T17:59:18.337894+0000","last_clean":"2024-08-17T17:59:18.337894+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:59:18.337894+0000","last_undegraded":"2024-08-17T17:59:18.337894+0000","last_fullsized":"2024-08-17T17:59:18.337894+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1946,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260572+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:59:16.260572+0000","last_peered":"2024-08-17T17:59:16.260572+0000","last_clean":"2024-08-17T17:59:16.260572+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:59:16.260572+0000","last_undegraded":"2024-08-17T17:59:16.260572+0000","last_fullsized":"2024-08-17T17:59:16.260572+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1854,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337517+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:59:18.337517+0000","last_peered":"2024-08-17T17:59:18.337517+0000","last_clean":"2024-08-17T17:59:18.337517+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:59:18.337517+0000","last_undegraded":"2024-08-17T17:59:18.337517+0000","last_fullsized":"2024-08-17T17:59:18.337517+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":549,"reported_epoch":1255,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.480928+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:59:15.480928+0000","last_peered":"2024-08-17T17:59:15.480928+0000","last_clean":"2024-08-17T17:59:15.480928+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:59:15.480928+0000","last_undegraded":"2024-08-17T17:59:15.480928+0000","last_fullsized":"2024-08-17T17:59:15.480928+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1226,"reported_epoch":1255,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.480639+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:59:15.480639+0000","last_peered":"2024-08-17T17:59:15.480639+0000","last_clean":"2024-08-17T17:59:15.480639+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:59:15.480639+0000","last_undegraded":"2024-08-17T17:59:15.480639+0000","last_fullsized":"2024-08-17T17:59:15.480639+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1951,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260360+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:59:16.260360+0000","last_peered":"2024-08-17T17:59:16.260360+0000","last_clean":"2024-08-17T17:59:16.260360+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:59:16.260360+0000","last_undegraded":"2024-08-17T17:59:16.260360+0000","last_fullsized":"2024-08-17T17:59:16.260360+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":387,"reported_epoch":1255,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.480608+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:59:15.480608+0000","last_peered":"2024-08-17T17:59:15.480608+0000","last_clean":"2024-08-17T17:59:15.480608+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:59:15.480608+0000","last_undegraded":"2024-08-17T17:59:15.480608+0000","last_fullsized":"2024-08-17T17:59:15.480608+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":534,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.910236+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:59:15.910236+0000","last_peered":"2024-08-17T17:59:15.910236+0000","last_clean":"2024-08-17T17:59:15.910236+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:59:15.910236+0000","last_undegraded":"2024-08-17T17:59:15.910236+0000","last_fullsized":"2024-08-17T17:59:15.910236+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1212,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.910569+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:59:15.910569+0000","last_peered":"2024-08-17T17:59:15.910569+0000","last_clean":"2024-08-17T17:59:15.910569+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:59:15.910569+0000","last_undegraded":"2024-08-17T17:59:15.910569+0000","last_fullsized":"2024-08-17T17:59:15.910569+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1220,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.910559+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:59:15.910559+0000","last_peered":"2024-08-17T17:59:15.910559+0000","last_clean":"2024-08-17T17:59:15.910559+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:59:15.910559+0000","last_undegraded":"2024-08-17T17:59:15.910559+0000","last_fullsized":"2024-08-17T17:59:15.910559+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1999,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260261+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:59:16.260261+0000","last_peered":"2024-08-17T17:59:16.260261+0000","last_clean":"2024-08-17T17:59:16.260261+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:59:16.260261+0000","last_undegraded":"2024-08-17T17:59:16.260261+0000","last_fullsized":"2024-08-17T17:59:16.260261+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1884,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.338292+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:59:18.338292+0000","last_peered":"2024-08-17T17:59:18.338292+0000","last_clean":"2024-08-17T17:59:18.338292+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:59:18.338292+0000","last_undegraded":"2024-08-17T17:59:18.338292+0000","last_fullsized":"2024-08-17T17:59:18.338292+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1957,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260531+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:59:16.260531+0000","last_peered":"2024-08-17T17:59:16.260531+0000","last_clean":"2024-08-17T17:59:16.260531+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:59:16.260531+0000","last_undegraded":"2024-08-17T17:59:16.260531+0000","last_fullsized":"2024-08-17T17:59:16.260531+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1886,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.338189+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:59:18.338189+0000","last_peered":"2024-08-17T17:59:18.338189+0000","last_clean":"2024-08-17T17:59:18.338189+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:59:18.338189+0000","last_undegraded":"2024-08-17T17:59:18.338189+0000","last_fullsized":"2024-08-17T17:59:18.338189+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1917,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260074+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:59:16.260074+0000","last_peered":"2024-08-17T17:59:16.260074+0000","last_clean":"2024-08-17T17:59:16.260074+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:59:16.260074+0000","last_undegraded":"2024-08-17T17:59:16.260074+0000","last_fullsized":"2024-08-17T17:59:16.260074+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1246,"reported_epoch":1255,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.480865+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:59:15.480865+0000","last_peered":"2024-08-17T17:59:15.480865+0000","last_clean":"2024-08-17T17:59:15.480865+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:59:15.480865+0000","last_undegraded":"2024-08-17T17:59:15.480865+0000","last_fullsized":"2024-08-17T17:59:15.480865+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1963,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260699+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:59:16.260699+0000","last_peered":"2024-08-17T17:59:16.260699+0000","last_clean":"2024-08-17T17:59:16.260699+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:59:16.260699+0000","last_undegraded":"2024-08-17T17:59:16.260699+0000","last_fullsized":"2024-08-17T17:59:16.260699+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":150,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260082+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:59:16.260082+0000","last_peered":"2024-08-17T17:59:16.260082+0000","last_clean":"2024-08-17T17:59:16.260082+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:59:16.260082+0000","last_undegraded":"2024-08-17T17:59:16.260082+0000","last_fullsized":"2024-08-17T17:59:16.260082+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1982,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260706+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:59:16.260706+0000","last_peered":"2024-08-17T17:59:16.260706+0000","last_clean":"2024-08-17T17:59:16.260706+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:59:16.260706+0000","last_undegraded":"2024-08-17T17:59:16.260706+0000","last_fullsized":"2024-08-17T17:59:16.260706+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":721,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337657+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:59:18.337657+0000","last_peered":"2024-08-17T17:59:18.337657+0000","last_clean":"2024-08-17T17:59:18.337657+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:59:18.337657+0000","last_undegraded":"2024-08-17T17:59:18.337657+0000","last_fullsized":"2024-08-17T17:59:18.337657+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1963,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260779+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:59:16.260779+0000","last_peered":"2024-08-17T17:59:16.260779+0000","last_clean":"2024-08-17T17:59:16.260779+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:59:16.260779+0000","last_undegraded":"2024-08-17T17:59:16.260779+0000","last_fullsized":"2024-08-17T17:59:16.260779+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1292,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.259983+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:59:16.259983+0000","last_peered":"2024-08-17T17:59:16.259983+0000","last_clean":"2024-08-17T17:59:16.259983+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:59:16.259983+0000","last_undegraded":"2024-08-17T17:59:16.259983+0000","last_fullsized":"2024-08-17T17:59:16.259983+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":210,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260260+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:59:16.260260+0000","last_peered":"2024-08-17T17:59:16.260260+0000","last_clean":"2024-08-17T17:59:16.260260+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:59:16.260260+0000","last_undegraded":"2024-08-17T17:59:16.260260+0000","last_fullsized":"2024-08-17T17:59:16.260260+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1974,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260757+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:59:16.260757+0000","last_peered":"2024-08-17T17:59:16.260757+0000","last_clean":"2024-08-17T17:59:16.260757+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:59:16.260757+0000","last_undegraded":"2024-08-17T17:59:16.260757+0000","last_fullsized":"2024-08-17T17:59:16.260757+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1952,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.259584+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:59:16.259584+0000","last_peered":"2024-08-17T17:59:16.259584+0000","last_clean":"2024-08-17T17:59:16.259584+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:59:16.259584+0000","last_undegraded":"2024-08-17T17:59:16.259584+0000","last_fullsized":"2024-08-17T17:59:16.259584+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1974,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337444+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:59:18.337444+0000","last_peered":"2024-08-17T17:59:18.337444+0000","last_clean":"2024-08-17T17:59:18.337444+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:59:18.337444+0000","last_undegraded":"2024-08-17T17:59:18.337444+0000","last_fullsized":"2024-08-17T17:59:18.337444+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1978,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260435+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:59:16.260435+0000","last_peered":"2024-08-17T17:59:16.260435+0000","last_clean":"2024-08-17T17:59:16.260435+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:59:16.260435+0000","last_undegraded":"2024-08-17T17:59:16.260435+0000","last_fullsized":"2024-08-17T17:59:16.260435+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1958,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.259646+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:59:16.259646+0000","last_peered":"2024-08-17T17:59:16.259646+0000","last_clean":"2024-08-17T17:59:16.259646+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:59:16.259646+0000","last_undegraded":"2024-08-17T17:59:16.259646+0000","last_fullsized":"2024-08-17T17:59:16.259646+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1253,"reported_epoch":1255,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.480498+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:59:15.480498+0000","last_peered":"2024-08-17T17:59:15.480498+0000","last_clean":"2024-08-17T17:59:15.480498+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:59:15.480498+0000","last_undegraded":"2024-08-17T17:59:15.480498+0000","last_fullsized":"2024-08-17T17:59:15.480498+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1200,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.910455+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:59:15.910455+0000","last_peered":"2024-08-17T17:59:15.910455+0000","last_clean":"2024-08-17T17:59:15.910455+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:59:15.910455+0000","last_undegraded":"2024-08-17T17:59:15.910455+0000","last_fullsized":"2024-08-17T17:59:15.910455+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1869,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337197+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:59:18.337197+0000","last_peered":"2024-08-17T17:59:18.337197+0000","last_clean":"2024-08-17T17:59:18.337197+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:59:18.337197+0000","last_undegraded":"2024-08-17T17:59:18.337197+0000","last_fullsized":"2024-08-17T17:59:18.337197+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":200,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337305+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:59:18.337305+0000","last_peered":"2024-08-17T17:59:18.337305+0000","last_clean":"2024-08-17T17:59:18.337305+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:59:18.337305+0000","last_undegraded":"2024-08-17T17:59:18.337305+0000","last_fullsized":"2024-08-17T17:59:18.337305+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1565,"reported_epoch":1255,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.480305+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:59:15.480305+0000","last_peered":"2024-08-17T17:59:15.480305+0000","last_clean":"2024-08-17T17:59:15.480305+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:59:15.480305+0000","last_undegraded":"2024-08-17T17:59:15.480305+0000","last_fullsized":"2024-08-17T17:59:15.480305+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1958,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260875+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:59:16.260875+0000","last_peered":"2024-08-17T17:59:16.260875+0000","last_clean":"2024-08-17T17:59:16.260875+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:59:16.260875+0000","last_undegraded":"2024-08-17T17:59:16.260875+0000","last_fullsized":"2024-08-17T17:59:16.260875+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1900,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.338174+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:59:18.338174+0000","last_peered":"2024-08-17T17:59:18.338174+0000","last_clean":"2024-08-17T17:59:18.338174+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:59:18.338174+0000","last_undegraded":"2024-08-17T17:59:18.338174+0000","last_fullsized":"2024-08-17T17:59:18.338174+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":534,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.910283+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:59:15.910283+0000","last_peered":"2024-08-17T17:59:15.910283+0000","last_clean":"2024-08-17T17:59:15.910283+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:59:15.910283+0000","last_undegraded":"2024-08-17T17:59:15.910283+0000","last_fullsized":"2024-08-17T17:59:15.910283+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":639,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337051+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:59:18.337051+0000","last_peered":"2024-08-17T17:59:18.337051+0000","last_clean":"2024-08-17T17:59:18.337051+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:59:18.337051+0000","last_undegraded":"2024-08-17T17:59:18.337051+0000","last_fullsized":"2024-08-17T17:59:18.337051+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":655,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.338101+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:59:18.338101+0000","last_peered":"2024-08-17T17:59:18.338101+0000","last_clean":"2024-08-17T17:59:18.338101+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:59:18.338101+0000","last_undegraded":"2024-08-17T17:59:18.338101+0000","last_fullsized":"2024-08-17T17:59:18.338101+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1218,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.910371+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:59:15.910371+0000","last_peered":"2024-08-17T17:59:15.910371+0000","last_clean":"2024-08-17T17:59:15.910371+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:59:15.910371+0000","last_undegraded":"2024-08-17T17:59:15.910371+0000","last_fullsized":"2024-08-17T17:59:15.910371+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":671,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337226+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:59:18.337226+0000","last_peered":"2024-08-17T17:59:18.337226+0000","last_clean":"2024-08-17T17:59:18.337226+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:59:18.337226+0000","last_undegraded":"2024-08-17T17:59:18.337226+0000","last_fullsized":"2024-08-17T17:59:18.337226+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1897,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.338900+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:59:18.338900+0000","last_peered":"2024-08-17T17:59:18.338900+0000","last_clean":"2024-08-17T17:59:18.338900+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:59:18.338900+0000","last_undegraded":"2024-08-17T17:59:18.338900+0000","last_fullsized":"2024-08-17T17:59:18.338900+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1990,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260153+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:59:16.260153+0000","last_peered":"2024-08-17T17:59:16.260153+0000","last_clean":"2024-08-17T17:59:16.260153+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:59:16.260153+0000","last_undegraded":"2024-08-17T17:59:16.260153+0000","last_fullsized":"2024-08-17T17:59:16.260153+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":703,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337170+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:59:18.337170+0000","last_peered":"2024-08-17T17:59:18.337170+0000","last_clean":"2024-08-17T17:59:18.337170+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:59:18.337170+0000","last_undegraded":"2024-08-17T17:59:18.337170+0000","last_fullsized":"2024-08-17T17:59:18.337170+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1207,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.910040+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:59:15.910040+0000","last_peered":"2024-08-17T17:59:15.910040+0000","last_clean":"2024-08-17T17:59:15.910040+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:59:15.910040+0000","last_undegraded":"2024-08-17T17:59:15.910040+0000","last_fullsized":"2024-08-17T17:59:15.910040+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1205,"reported_epoch":1255,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.480863+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:59:15.480863+0000","last_peered":"2024-08-17T17:59:15.480863+0000","last_clean":"2024-08-17T17:59:15.480863+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:59:15.480863+0000","last_undegraded":"2024-08-17T17:59:15.480863+0000","last_fullsized":"2024-08-17T17:59:15.480863+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":686,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.259779+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:59:16.259779+0000","last_peered":"2024-08-17T17:59:16.259779+0000","last_clean":"2024-08-17T17:59:16.259779+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:59:16.259779+0000","last_undegraded":"2024-08-17T17:59:16.259779+0000","last_fullsized":"2024-08-17T17:59:16.259779+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1223,"reported_epoch":1255,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.481039+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:59:15.481039+0000","last_peered":"2024-08-17T17:59:15.481039+0000","last_clean":"2024-08-17T17:59:15.481039+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:59:15.481039+0000","last_undegraded":"2024-08-17T17:59:15.481039+0000","last_fullsized":"2024-08-17T17:59:15.481039+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":107,"reported_epoch":1255,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.480267+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:59:15.480267+0000","last_peered":"2024-08-17T17:59:15.480267+0000","last_clean":"2024-08-17T17:59:15.480267+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:59:15.480267+0000","last_undegraded":"2024-08-17T17:59:15.480267+0000","last_fullsized":"2024-08-17T17:59:15.480267+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1239,"reported_epoch":1255,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.480869+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:59:15.480869+0000","last_peered":"2024-08-17T17:59:15.480869+0000","last_clean":"2024-08-17T17:59:15.480869+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:59:15.480869+0000","last_undegraded":"2024-08-17T17:59:15.480869+0000","last_fullsized":"2024-08-17T17:59:15.480869+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1257,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.910032+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:59:15.910032+0000","last_peered":"2024-08-17T17:59:15.910032+0000","last_clean":"2024-08-17T17:59:15.910032+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:59:15.910032+0000","last_undegraded":"2024-08-17T17:59:15.910032+0000","last_fullsized":"2024-08-17T17:59:15.910032+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1898,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.338048+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:59:18.338048+0000","last_peered":"2024-08-17T17:59:18.338048+0000","last_clean":"2024-08-17T17:59:18.338048+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:59:18.338048+0000","last_undegraded":"2024-08-17T17:59:18.338048+0000","last_fullsized":"2024-08-17T17:59:18.338048+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1200,"reported_epoch":1255,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.480571+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:59:15.480571+0000","last_peered":"2024-08-17T17:59:15.480571+0000","last_clean":"2024-08-17T17:59:15.480571+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:59:15.480571+0000","last_undegraded":"2024-08-17T17:59:15.480571+0000","last_fullsized":"2024-08-17T17:59:15.480571+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1949,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.260350+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:59:16.260350+0000","last_peered":"2024-08-17T17:59:16.260350+0000","last_clean":"2024-08-17T17:59:16.260350+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:59:16.260350+0000","last_undegraded":"2024-08-17T17:59:16.260350+0000","last_fullsized":"2024-08-17T17:59:16.260350+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1950,"reported_epoch":1256,"state":"active+clean","last_fresh":"2024-08-17T17:59:16.259914+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:59:16.259914+0000","last_peered":"2024-08-17T17:59:16.259914+0000","last_clean":"2024-08-17T17:59:16.259914+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:59:16.259914+0000","last_undegraded":"2024-08-17T17:59:16.259914+0000","last_fullsized":"2024-08-17T17:59:16.259914+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":613,"reported_epoch":1255,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.480701+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:59:15.480701+0000","last_peered":"2024-08-17T17:59:15.480701+0000","last_clean":"2024-08-17T17:59:15.480701+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:59:15.480701+0000","last_undegraded":"2024-08-17T17:59:15.480701+0000","last_fullsized":"2024-08-17T17:59:15.480701+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1225,"reported_epoch":1255,"state":"active+clean","last_fresh":"2024-08-17T17:59:15.480580+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:59:15.480580+0000","last_peered":"2024-08-17T17:59:15.480580+0000","last_clean":"2024-08-17T17:59:15.480580+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:59:15.480580+0000","last_undegraded":"2024-08-17T17:59:15.480580+0000","last_fullsized":"2024-08-17T17:59:15.480580+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1898,"reported_epoch":1257,"state":"active+clean","last_fresh":"2024-08-17T17:59:18.337481+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:59:18.337481+0000","last_peered":"2024-08-17T17:59:18.337481+0000","last_clean":"2024-08-17T17:59:18.337481+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:59:18.337481+0000","last_undegraded":"2024-08-17T17:59:18.337481+0000","last_fullsized":"2024-08-17T17:59:18.337481+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":46,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234461,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":315576,"kb_used_data":13580,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93429576,"statfs":{"total":95995035648,"available":95671885824,"internally_reserved":0,"allocated":13905920,"data_stored":8981419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496096,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311416,"kb_used_data":13580,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93433736,"statfs":{"total":95995035648,"available":95676145664,"internally_reserved":0,"allocated":13905920,"data_stored":8981896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757734,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326348,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":270524,"kb_used_data":12996,"kb_used_omap":57,"kb_used_meta":257222,"kb_avail":93474628,"statfs":{"total":95995035648,"available":95718019072,"internally_reserved":0,"allocated":13307904,"data_stored":8394738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":263396091},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405587,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":271688,"kb_used_data":13008,"kb_used_omap":61,"kb_used_meta":258370,"kb_avail":93473464,"statfs":{"total":95995035648,"available":95716827136,"internally_reserved":0,"allocated":13320192,"data_stored":8399926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":264571832},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452121,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":211872,"kb_used_data":13216,"kb_used_omap":54,"kb_used_meta":198345,"kb_avail":93533280,"statfs":{"total":95995035648,"available":95778078720,"internally_reserved":0,"allocated":13533184,"data_stored":8520788,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":203105678},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474812,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":269036,"kb_used_data":13804,"kb_used_omap":119,"kb_used_meta":254856,"kb_avail":93476116,"statfs":{"total":95995035648,"available":95719542784,"internally_reserved":0,"allocated":14135296,"data_stored":9111816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":260973278},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019369,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:59:23.943 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:23.965 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:23.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:23.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:24.066 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:24.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:24.092 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:24.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:24.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:23 smithi130 ceph-mon[105521]: osdmap e1259: 8 total, 8 up, 6 in 2024-08-17T17:59:24.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:23 smithi130 ceph-mon[110068]: osdmap e1259: 8 total, 8 up, 6 in 2024-08-17T17:59:24.193 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:24.215 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:24.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:24.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:24.316 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:24.338 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:24.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:24.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:24.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:23 smithi179 ceph-mon[93519]: osdmap e1259: 8 total, 8 up, 6 in 2024-08-17T17:59:24.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:24.462 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:24.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:24.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:24.564 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:24.587 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:24.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:24.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:24.648 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:24.667 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:59:24.667 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T17:59:24.667 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T17:59:24.667 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T17:59:24.667 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:59:24.690 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:24.712 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:24.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:24.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:24.813 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:24.839 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:24.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:24.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:24.941 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:24.966 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:24.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:24.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:25.068 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:25.093 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:25.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:25.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:25.195 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:25.203 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:24 smithi130 ceph-mon[105521]: pgmap v2003: 89 pgs: 6 creating+peering, 4 unknown, 79 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:25.204 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:24 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:59:25.204 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:24 smithi130 ceph-mon[105521]: osdmap e1260: 8 total, 8 up, 6 in 2024-08-17T17:59:25.204 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:24 smithi130 ceph-mon[110068]: pgmap v2003: 89 pgs: 6 creating+peering, 4 unknown, 79 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:25.204 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:24 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:59:25.204 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:24 smithi130 ceph-mon[110068]: osdmap e1260: 8 total, 8 up, 6 in 2024-08-17T17:59:25.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:25.221 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:25.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:25.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:25.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:24 smithi179 ceph-mon[93519]: pgmap v2003: 89 pgs: 6 creating+peering, 4 unknown, 79 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:25.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:24 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:59:25.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:24 smithi179 ceph-mon[93519]: osdmap e1260: 8 total, 8 up, 6 in 2024-08-17T17:59:25.344 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:25.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:25.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:25.446 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:25.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:25.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:25.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:25.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:25.594 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:25.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:25.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:25.696 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:25.718 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:25.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:25.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:25.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:25.841 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:25.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:25.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:25.943 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:25.965 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:25.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:25.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:26.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:26.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:26.088 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:26.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:26.189 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:26.215 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:26.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:26.316 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:26.338 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:26.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:26.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:26.439 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:26.461 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:26.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:26.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:26.563 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:26.585 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:26.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:26.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:26.686 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:26.712 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:26.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:26.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:26.814 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:26.839 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:26.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:26.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:26.939 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:59:26.941 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:26.962 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:26.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:26.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:27.064 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:27.085 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:27.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:27.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:27.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:26 smithi130 ceph-mon[105521]: pgmap v2005: 89 pgs: 6 creating+peering, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:27.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:26 smithi130 ceph-mon[110068]: pgmap v2005: 89 pgs: 6 creating+peering, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:27.187 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:27.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:27.209 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:27.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:27.310 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:27.336 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:27.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:27.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:27.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:26 smithi179 ceph-mon[93519]: pgmap v2005: 89 pgs: 6 creating+peering, 83 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:27.361 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:27.390 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:59:27.438 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:27.464 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:27.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:27.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:27.566 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:27.587 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:27.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:27.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:27.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:27.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:27.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:27.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:27.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:27.838 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:27.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:27.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:27.940 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:27.961 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:27.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:27.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:28.063 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:28.084 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:28.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:28.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:28.186 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:28.210 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:28.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:28.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:28.312 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:28.333 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:28.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:28.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:28.435 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:28.459 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:28.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:28.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:28.475 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:28 smithi130 ceph-mon[105521]: pgmap v2006: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:28.475 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:28.475 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:59:28.475 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:28 smithi130 ceph-mon[110068]: pgmap v2006: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:28.475 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:28.475 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:59:28.560 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:28.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:28.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:28.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:28.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:28 smithi179 ceph-mon[93519]: pgmap v2006: 89 pgs: 4 creating+peering, 85 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:59:28.686 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:28.708 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:28.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:28.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:28.810 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:28.835 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:28.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:28.937 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:28.958 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:28.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:28.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:29.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:29.081 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:29.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:29.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:29.109 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:59:29.109 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:59:29.126 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:59:29.126 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:59:29.145 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:59:29.145 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:59:29.162 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:59:29.162 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:59:29.178 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:59:29.178 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:59:29.184 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:29.194 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:59:29.195 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:59:29.209 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:29.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:29.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:29.211 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:59:29.211 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:59:29.226 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:59:29.227 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:59:29.311 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:29.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:29.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:29.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1774918797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:59:29.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1774918797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:59:29.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:29.460 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:29.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:29.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:29.562 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:29.586 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:29.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:29.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:29.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1774918797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:59:29.688 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:29.710 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:29.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:29.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:29.812 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:29.834 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:29.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:29.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:29.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:59:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:59:29] "GET /metrics HTTP/1.1" 200 36715 "" "Prometheus/2.51.0" 2024-08-17T17:59:29.936 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:29.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:29.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:29.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:30.058 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:30.079 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:30.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:30.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:30.181 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:30.207 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:30.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:30.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:30.309 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:30.334 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:30.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:30.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:30.436 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:30.460 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:30.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:30.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:30.562 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:30.584 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:30.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:30.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:30 smithi179 ceph-mon[93519]: pgmap v2007: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:59:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:59:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:59:30.605 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:30 smithi130 ceph-mon[105521]: pgmap v2007: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:30.605 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:30 smithi130 ceph-mon[110068]: pgmap v2007: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:30.686 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:30.708 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:30.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:30.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:30.809 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:30.835 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:30.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:30.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:30.860 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:30.860 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:30.937 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:30.962 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:30.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:30.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:31.064 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:31.088 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:31.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:31.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:31.189 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:31.214 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:31.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:31.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:31.316 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:31.322 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T17:59:31.323 INFO:teuthology.orchestra.run.smithi130.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-08-17T17:59:31.342 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:31.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:31.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:31.444 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:31.470 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:31.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:31.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:31.568 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/944094722' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:59:31.568 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/882807701' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:59:31.568 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/944094722' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:59:31.568 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/882807701' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:59:31.571 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:31.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/944094722' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:59:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/882807701' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:59:31.597 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:31.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:31.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:31.698 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:31.722 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:31.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:31.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:31.824 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:31.845 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:31.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:31.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:31.947 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:31.968 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:31.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:31.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:32.059 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:59:32.070 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:32.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:32.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:32.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:32.198 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:32.223 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:32.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:32.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:32.324 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:32.349 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:32.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:32.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:32.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:32 smithi130 ceph-mon[105521]: pgmap v2008: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:32.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:32 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:59:32.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/944094722' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:59:32.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/882807701' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:59:32.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:32 smithi130 ceph-mon[105521]: osdmap e1261: 8 total, 8 up, 6 in 2024-08-17T17:59:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:32 smithi130 ceph-mon[110068]: pgmap v2008: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:32 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:59:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/944094722' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:59:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/882807701' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:59:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:32 smithi130 ceph-mon[110068]: osdmap e1261: 8 total, 8 up, 6 in 2024-08-17T17:59:32.451 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:32.476 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:32.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:32.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:32.577 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:32 smithi179 ceph-mon[93519]: pgmap v2008: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:32 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:59:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/944094722' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:59:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/882807701' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:59:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:32 smithi179 ceph-mon[93519]: osdmap e1261: 8 total, 8 up, 6 in 2024-08-17T17:59:32.599 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:32.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:32.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:32.701 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:32.726 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:32.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:32.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:32.828 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:32.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:32.854 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:32.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:32.955 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:32.977 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:32.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:32.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:33.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:33.100 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:33.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:33.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:33.202 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:33.228 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:33.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:33.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:33.331 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:33.356 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:33.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:33.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:33.458 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:33.484 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:33.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:33.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:33.585 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:33.607 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:33 smithi130 ceph-mon[105521]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:59:33.607 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:33 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T17:59:33.607 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:33 smithi130 ceph-mon[110068]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:59:33.607 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:33 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T17:59:33.611 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:33.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:33.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:33.712 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:33.734 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:33.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:33.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:33.837 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:33.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:33 smithi179 ceph-mon[93519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:59:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:33 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T17:59:33.858 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:33.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:33.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:33.960 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:33.986 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:33.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:33.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:34.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:34.110 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:34.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:34.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:34.212 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:34.233 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:34.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:34.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:34.335 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:34.356 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:34.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:34.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:34.369 DEBUG:teuthology.orchestra.run.smithi130:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-08-17T17:59:34.400 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:59:34.400 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:59:34.447 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:hints = 1 2024-08-17T17:59:34.447 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:59:34.447 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Object prefix: benchmark_data_smithi130_267720 2024-08-17T17:59:34.447 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:59:34.447 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:59:34.458 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:34.490 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:34.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:34.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:34.592 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:34.634 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:34.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:34.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:34.664 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:34 smithi130 ceph-mon[105521]: pgmap v2010: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:34.664 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:34 smithi130 ceph-mon[110068]: pgmap v2010: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:34.736 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:34.761 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:34.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:34.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:34.769 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:59:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:34 smithi179 ceph-mon[93519]: pgmap v2010: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:34.863 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:34.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:34.909 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:34.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:35.009 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:35.034 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:35.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:35.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:35.136 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:35.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:35.178 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:35.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:35.278 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:35.316 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:35.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:35.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:35.352 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T17:59:35.353 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T17:59:35.419 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:35.445 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:35.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:35.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:35.447 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 1 15 3000 2985 186.481 186.562 0.0101646 0.0053368 2024-08-17T17:59:35.547 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:35.588 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:35.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:35.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:35.690 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:35.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:35.719 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:35.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:35.820 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:35.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:35.846 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:35.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:35.947 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:35.973 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:35.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:35.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:36.075 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:36.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:36.114 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:36.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:36.208 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T17:59:36.208 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:59:36.216 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:36.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:36.241 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:36.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:36.342 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:36.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:36.381 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:36.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:36.448 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 2 16 6073 6057 189.231 192 0.00440208 0.00527226 2024-08-17T17:59:36.482 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:36.518 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:36.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:36.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:36.620 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:36.645 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:36 smithi130 ceph-mon[105521]: pgmap v2011: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:36.645 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:36 smithi130 ceph-mon[105521]: from='client.46106 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:59:36.646 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:36 smithi130 ceph-mon[110068]: pgmap v2011: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:36.646 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:36 smithi130 ceph-mon[110068]: from='client.46106 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:59:36.661 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:36.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:36.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:36.696 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2011,"stamp":"2024-08-17T17:59:35.142417+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":1776653,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":1641852,"kb_used_data":71448,"kb_used_omap":295,"kb_used_meta":1567640,"kb_avail":560829060,"statfs":{"total":575970213888,"available":574288957440,"internally_reserved":0,"allocated":73162752,"data_stored":46865628,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":1605263823},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002213"},"pg_stats":[{"pgid":"46.f","version":"0'0","reported_seq":11,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.950955+0000","last_change":"2024-08-17T17:59:22.956823+0000","last_active":"2024-08-17T17:59:23.950955+0000","last_peered":"2024-08-17T17:59:23.950955+0000","last_clean":"2024-08-17T17:59:23.950955+0000","last_became_active":"2024-08-17T17:59:22.956437+0000","last_became_peered":"2024-08-17T17:59:22.956437+0000","last_unstale":"2024-08-17T17:59:23.950955+0000","last_undegraded":"2024-08-17T17:59:23.950955+0000","last_fullsized":"2024-08-17T17:59:23.950955+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:17.161240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.e","version":"0'0","reported_seq":17,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601578+0000","last_change":"2024-08-17T17:59:22.954716+0000","last_active":"2024-08-17T17:59:31.601578+0000","last_peered":"2024-08-17T17:59:31.601578+0000","last_clean":"2024-08-17T17:59:31.601578+0000","last_became_active":"2024-08-17T17:59:22.954381+0000","last_became_peered":"2024-08-17T17:59:22.954381+0000","last_unstale":"2024-08-17T17:59:31.601578+0000","last_undegraded":"2024-08-17T17:59:31.601578+0000","last_fullsized":"2024-08-17T17:59:31.601578+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:59:39.256751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"46.d","version":"0'0","reported_seq":17,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.489892+0000","last_change":"2024-08-17T17:59:22.954900+0000","last_active":"2024-08-17T17:59:31.489892+0000","last_peered":"2024-08-17T17:59:31.489892+0000","last_clean":"2024-08-17T17:59:31.489892+0000","last_became_active":"2024-08-17T17:59:22.954599+0000","last_became_peered":"2024-08-17T17:59:22.954599+0000","last_unstale":"2024-08-17T17:59:31.489892+0000","last_undegraded":"2024-08-17T17:59:31.489892+0000","last_fullsized":"2024-08-17T17:59:31.489892+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:22:00.453498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.c","version":"0'0","reported_seq":11,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.950757+0000","last_change":"2024-08-17T17:59:22.951540+0000","last_active":"2024-08-17T17:59:23.950757+0000","last_peered":"2024-08-17T17:59:23.950757+0000","last_clean":"2024-08-17T17:59:23.950757+0000","last_became_active":"2024-08-17T17:59:22.951248+0000","last_became_peered":"2024-08-17T17:59:22.951248+0000","last_unstale":"2024-08-17T17:59:23.950757+0000","last_undegraded":"2024-08-17T17:59:23.950757+0000","last_fullsized":"2024-08-17T17:59:23.950757+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:55.458182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.b","version":"0'0","reported_seq":17,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.489790+0000","last_change":"2024-08-17T17:59:22.957875+0000","last_active":"2024-08-17T17:59:31.489790+0000","last_peered":"2024-08-17T17:59:31.489790+0000","last_clean":"2024-08-17T17:59:31.489790+0000","last_became_active":"2024-08-17T17:59:22.957601+0000","last_became_peered":"2024-08-17T17:59:22.957601+0000","last_unstale":"2024-08-17T17:59:31.489790+0000","last_undegraded":"2024-08-17T17:59:31.489790+0000","last_fullsized":"2024-08-17T17:59:31.489790+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:40.456894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.a","version":"0'0","reported_seq":11,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000364+0000","last_change":"2024-08-17T17:59:22.951416+0000","last_active":"2024-08-17T17:59:24.000364+0000","last_peered":"2024-08-17T17:59:24.000364+0000","last_clean":"2024-08-17T17:59:24.000364+0000","last_became_active":"2024-08-17T17:59:22.951110+0000","last_became_peered":"2024-08-17T17:59:22.951110+0000","last_unstale":"2024-08-17T17:59:24.000364+0000","last_undegraded":"2024-08-17T17:59:24.000364+0000","last_fullsized":"2024-08-17T17:59:24.000364+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:32.365091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.9","version":"0'0","reported_seq":17,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.489429+0000","last_change":"2024-08-17T17:59:22.957896+0000","last_active":"2024-08-17T17:59:31.489429+0000","last_peered":"2024-08-17T17:59:31.489429+0000","last_clean":"2024-08-17T17:59:31.489429+0000","last_became_active":"2024-08-17T17:59:22.957648+0000","last_became_peered":"2024-08-17T17:59:22.957648+0000","last_unstale":"2024-08-17T17:59:31.489429+0000","last_undegraded":"2024-08-17T17:59:31.489429+0000","last_fullsized":"2024-08-17T17:59:31.489429+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:11.940932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.8","version":"0'0","reported_seq":17,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601096+0000","last_change":"2024-08-17T17:59:22.951248+0000","last_active":"2024-08-17T17:59:31.601096+0000","last_peered":"2024-08-17T17:59:31.601096+0000","last_clean":"2024-08-17T17:59:31.601096+0000","last_became_active":"2024-08-17T17:59:22.950981+0000","last_became_peered":"2024-08-17T17:59:22.950981+0000","last_unstale":"2024-08-17T17:59:31.601096+0000","last_undegraded":"2024-08-17T17:59:31.601096+0000","last_fullsized":"2024-08-17T17:59:31.601096+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:24:07.143610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.7","version":"0'0","reported_seq":17,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601810+0000","last_change":"2024-08-17T17:59:22.959269+0000","last_active":"2024-08-17T17:59:31.601810+0000","last_peered":"2024-08-17T17:59:31.601810+0000","last_clean":"2024-08-17T17:59:31.601810+0000","last_became_active":"2024-08-17T17:59:22.958845+0000","last_became_peered":"2024-08-17T17:59:22.958845+0000","last_unstale":"2024-08-17T17:59:31.601810+0000","last_undegraded":"2024-08-17T17:59:31.601810+0000","last_fullsized":"2024-08-17T17:59:31.601810+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:03:29.978344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"46.6","version":"0'0","reported_seq":11,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.951141+0000","last_change":"2024-08-17T17:59:22.958347+0000","last_active":"2024-08-17T17:59:23.951141+0000","last_peered":"2024-08-17T17:59:23.951141+0000","last_clean":"2024-08-17T17:59:23.951141+0000","last_became_active":"2024-08-17T17:59:22.957969+0000","last_became_peered":"2024-08-17T17:59:22.957969+0000","last_unstale":"2024-08-17T17:59:23.951141+0000","last_undegraded":"2024-08-17T17:59:23.951141+0000","last_fullsized":"2024-08-17T17:59:23.951141+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:07.728594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.5","version":"0'0","reported_seq":17,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.326483+0000","last_change":"2024-08-17T17:59:22.956504+0000","last_active":"2024-08-17T17:59:31.326483+0000","last_peered":"2024-08-17T17:59:31.326483+0000","last_clean":"2024-08-17T17:59:31.326483+0000","last_became_active":"2024-08-17T17:59:22.955747+0000","last_became_peered":"2024-08-17T17:59:22.955747+0000","last_unstale":"2024-08-17T17:59:31.326483+0000","last_undegraded":"2024-08-17T17:59:31.326483+0000","last_fullsized":"2024-08-17T17:59:31.326483+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:54:51.990739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"46.4","version":"0'0","reported_seq":11,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000585+0000","last_change":"2024-08-17T17:59:22.952797+0000","last_active":"2024-08-17T17:59:24.000585+0000","last_peered":"2024-08-17T17:59:24.000585+0000","last_clean":"2024-08-17T17:59:24.000585+0000","last_became_active":"2024-08-17T17:59:22.952475+0000","last_became_peered":"2024-08-17T17:59:22.952475+0000","last_unstale":"2024-08-17T17:59:24.000585+0000","last_undegraded":"2024-08-17T17:59:24.000585+0000","last_fullsized":"2024-08-17T17:59:24.000585+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:42:19.302395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.3","version":"0'0","reported_seq":17,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.602147+0000","last_change":"2024-08-17T17:59:22.954906+0000","last_active":"2024-08-17T17:59:31.602147+0000","last_peered":"2024-08-17T17:59:31.602147+0000","last_clean":"2024-08-17T17:59:31.602147+0000","last_became_active":"2024-08-17T17:59:22.954623+0000","last_became_peered":"2024-08-17T17:59:22.954623+0000","last_unstale":"2024-08-17T17:59:31.602147+0000","last_undegraded":"2024-08-17T17:59:31.602147+0000","last_fullsized":"2024-08-17T17:59:31.602147+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:19.582387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.2","version":"0'0","reported_seq":17,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.489467+0000","last_change":"2024-08-17T17:59:22.956809+0000","last_active":"2024-08-17T17:59:31.489467+0000","last_peered":"2024-08-17T17:59:31.489467+0000","last_clean":"2024-08-17T17:59:31.489467+0000","last_became_active":"2024-08-17T17:59:22.956543+0000","last_became_peered":"2024-08-17T17:59:22.956543+0000","last_unstale":"2024-08-17T17:59:31.489467+0000","last_undegraded":"2024-08-17T17:59:31.489467+0000","last_fullsized":"2024-08-17T17:59:31.489467+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:31:37.454620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.1","version":"0'0","reported_seq":11,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.950383+0000","last_change":"2024-08-17T17:59:22.954701+0000","last_active":"2024-08-17T17:59:23.950383+0000","last_peered":"2024-08-17T17:59:23.950383+0000","last_clean":"2024-08-17T17:59:23.950383+0000","last_became_active":"2024-08-17T17:59:22.954510+0000","last_became_peered":"2024-08-17T17:59:22.954510+0000","last_unstale":"2024-08-17T17:59:23.950383+0000","last_undegraded":"2024-08-17T17:59:23.950383+0000","last_fullsized":"2024-08-17T17:59:23.950383+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:19:01.061765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.0","version":"0'0","reported_seq":17,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.600952+0000","last_change":"2024-08-17T17:59:22.958808+0000","last_active":"2024-08-17T17:59:31.600952+0000","last_peered":"2024-08-17T17:59:31.600952+0000","last_clean":"2024-08-17T17:59:31.600952+0000","last_became_active":"2024-08-17T17:59:22.958515+0000","last_became_peered":"2024-08-17T17:59:22.958515+0000","last_unstale":"2024-08-17T17:59:31.600952+0000","last_undegraded":"2024-08-17T17:59:31.600952+0000","last_fullsized":"2024-08-17T17:59:31.600952+0000","mapping_epoch":1258,"log_start":"0'0","ondisk_log_start":"0'0","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:30.812796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":1981,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.490114+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T17:59:31.490114+0000","last_peered":"2024-08-17T17:59:31.490114+0000","last_clean":"2024-08-17T17:59:31.490114+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T17:59:31.490114+0000","last_undegraded":"2024-08-17T17:59:31.490114+0000","last_fullsized":"2024-08-17T17:59:31.490114+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":538,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.950240+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T17:59:23.950240+0000","last_peered":"2024-08-17T17:59:23.950240+0000","last_clean":"2024-08-17T17:59:23.950240+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T17:59:23.950240+0000","last_undegraded":"2024-08-17T17:59:23.950240+0000","last_fullsized":"2024-08-17T17:59:23.950240+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1900,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.326969+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T17:59:31.326969+0000","last_peered":"2024-08-17T17:59:31.326969+0000","last_clean":"2024-08-17T17:59:31.326969+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T17:59:31.326969+0000","last_undegraded":"2024-08-17T17:59:31.326969+0000","last_fullsized":"2024-08-17T17:59:31.326969+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1924,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.325534+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T17:59:31.325534+0000","last_peered":"2024-08-17T17:59:31.325534+0000","last_clean":"2024-08-17T17:59:31.325534+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T17:59:31.325534+0000","last_undegraded":"2024-08-17T17:59:31.325534+0000","last_fullsized":"2024-08-17T17:59:31.325534+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":100,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.951013+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T17:59:23.951013+0000","last_peered":"2024-08-17T17:59:23.951013+0000","last_clean":"2024-08-17T17:59:23.951013+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T17:59:23.951013+0000","last_undegraded":"2024-08-17T17:59:23.951013+0000","last_fullsized":"2024-08-17T17:59:23.951013+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1317,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.325911+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T17:59:31.325911+0000","last_peered":"2024-08-17T17:59:31.325911+0000","last_clean":"2024-08-17T17:59:31.325911+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T17:59:31.325911+0000","last_undegraded":"2024-08-17T17:59:31.325911+0000","last_fullsized":"2024-08-17T17:59:31.325911+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":175,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601723+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T17:59:31.601723+0000","last_peered":"2024-08-17T17:59:31.601723+0000","last_clean":"2024-08-17T17:59:31.601723+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T17:59:31.601723+0000","last_undegraded":"2024-08-17T17:59:31.601723+0000","last_fullsized":"2024-08-17T17:59:31.601723+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":695,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.326086+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T17:59:31.326086+0000","last_peered":"2024-08-17T17:59:31.326086+0000","last_clean":"2024-08-17T17:59:31.326086+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T17:59:31.326086+0000","last_undegraded":"2024-08-17T17:59:31.326086+0000","last_fullsized":"2024-08-17T17:59:31.326086+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1939,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.489759+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T17:59:31.489759+0000","last_peered":"2024-08-17T17:59:31.489759+0000","last_clean":"2024-08-17T17:59:31.489759+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T17:59:31.489759+0000","last_undegraded":"2024-08-17T17:59:31.489759+0000","last_fullsized":"2024-08-17T17:59:31.489759+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1941,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601509+0000","last_change":"2024-08-17T17:31:49.809062+0000","last_active":"2024-08-17T17:59:31.601509+0000","last_peered":"2024-08-17T17:59:31.601509+0000","last_clean":"2024-08-17T17:59:31.601509+0000","last_became_active":"2024-08-17T17:31:49.808756+0000","last_became_peered":"2024-08-17T17:31:49.808756+0000","last_unstale":"2024-08-17T17:59:31.601509+0000","last_undegraded":"2024-08-17T17:59:31.601509+0000","last_fullsized":"2024-08-17T17:59:31.601509+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:38.878866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":160,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601558+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T17:59:31.601558+0000","last_peered":"2024-08-17T17:59:31.601558+0000","last_clean":"2024-08-17T17:59:31.601558+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T17:59:31.601558+0000","last_undegraded":"2024-08-17T17:59:31.601558+0000","last_fullsized":"2024-08-17T17:59:31.601558+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1913,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.326764+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T17:59:31.326764+0000","last_peered":"2024-08-17T17:59:31.326764+0000","last_clean":"2024-08-17T17:59:31.326764+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T17:59:31.326764+0000","last_undegraded":"2024-08-17T17:59:31.326764+0000","last_fullsized":"2024-08-17T17:59:31.326764+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":22,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.600920+0000","last_change":"2024-08-17T17:59:15.912662+0000","last_active":"2024-08-17T17:59:31.600920+0000","last_peered":"2024-08-17T17:59:31.600920+0000","last_clean":"2024-08-17T17:59:31.600920+0000","last_became_active":"2024-08-17T17:59:15.912410+0000","last_became_peered":"2024-08-17T17:59:15.912410+0000","last_unstale":"2024-08-17T17:59:31.600920+0000","last_undegraded":"2024-08-17T17:59:31.600920+0000","last_fullsized":"2024-08-17T17:59:31.600920+0000","mapping_epoch":1255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:07:10.368942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1962,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.490126+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T17:59:31.490126+0000","last_peered":"2024-08-17T17:59:31.490126+0000","last_clean":"2024-08-17T17:59:31.490126+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T17:59:31.490126+0000","last_undegraded":"2024-08-17T17:59:31.490126+0000","last_fullsized":"2024-08-17T17:59:31.490126+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":22,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.600902+0000","last_change":"2024-08-17T17:59:16.261638+0000","last_active":"2024-08-17T17:59:31.600902+0000","last_peered":"2024-08-17T17:59:31.600902+0000","last_clean":"2024-08-17T17:59:31.600902+0000","last_became_active":"2024-08-17T17:59:16.261277+0000","last_became_peered":"2024-08-17T17:59:16.261277+0000","last_unstale":"2024-08-17T17:59:31.600902+0000","last_undegraded":"2024-08-17T17:59:31.600902+0000","last_fullsized":"2024-08-17T17:59:31.600902+0000","mapping_epoch":1255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:10:50.522770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1235,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.951174+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T17:59:23.951174+0000","last_peered":"2024-08-17T17:59:23.951174+0000","last_clean":"2024-08-17T17:59:23.951174+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T17:59:23.951174+0000","last_undegraded":"2024-08-17T17:59:23.951174+0000","last_fullsized":"2024-08-17T17:59:23.951174+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1906,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.324949+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T17:59:31.324949+0000","last_peered":"2024-08-17T17:59:31.324949+0000","last_clean":"2024-08-17T17:59:31.324949+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T17:59:31.324949+0000","last_undegraded":"2024-08-17T17:59:31.324949+0000","last_fullsized":"2024-08-17T17:59:31.324949+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1956,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.490307+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T17:59:31.490307+0000","last_peered":"2024-08-17T17:59:31.490307+0000","last_clean":"2024-08-17T17:59:31.490307+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T17:59:31.490307+0000","last_undegraded":"2024-08-17T17:59:31.490307+0000","last_fullsized":"2024-08-17T17:59:31.490307+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1862,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.325452+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T17:59:31.325452+0000","last_peered":"2024-08-17T17:59:31.325452+0000","last_clean":"2024-08-17T17:59:31.325452+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T17:59:31.325452+0000","last_undegraded":"2024-08-17T17:59:31.325452+0000","last_fullsized":"2024-08-17T17:59:31.325452+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":554,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000835+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T17:59:24.000835+0000","last_peered":"2024-08-17T17:59:24.000835+0000","last_clean":"2024-08-17T17:59:24.000835+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T17:59:24.000835+0000","last_undegraded":"2024-08-17T17:59:24.000835+0000","last_fullsized":"2024-08-17T17:59:24.000835+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1231,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000734+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T17:59:24.000734+0000","last_peered":"2024-08-17T17:59:24.000734+0000","last_clean":"2024-08-17T17:59:24.000734+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T17:59:24.000734+0000","last_undegraded":"2024-08-17T17:59:24.000734+0000","last_fullsized":"2024-08-17T17:59:24.000734+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1961,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601710+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T17:59:31.601710+0000","last_peered":"2024-08-17T17:59:31.601710+0000","last_clean":"2024-08-17T17:59:31.601710+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T17:59:31.601710+0000","last_undegraded":"2024-08-17T17:59:31.601710+0000","last_fullsized":"2024-08-17T17:59:31.601710+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":392,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000696+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T17:59:24.000696+0000","last_peered":"2024-08-17T17:59:24.000696+0000","last_clean":"2024-08-17T17:59:24.000696+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T17:59:24.000696+0000","last_undegraded":"2024-08-17T17:59:24.000696+0000","last_fullsized":"2024-08-17T17:59:24.000696+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":538,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.950713+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T17:59:23.950713+0000","last_peered":"2024-08-17T17:59:23.950713+0000","last_clean":"2024-08-17T17:59:23.950713+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T17:59:23.950713+0000","last_undegraded":"2024-08-17T17:59:23.950713+0000","last_fullsized":"2024-08-17T17:59:23.950713+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1216,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.950527+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T17:59:23.950527+0000","last_peered":"2024-08-17T17:59:23.950527+0000","last_clean":"2024-08-17T17:59:23.950527+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T17:59:23.950527+0000","last_undegraded":"2024-08-17T17:59:23.950527+0000","last_fullsized":"2024-08-17T17:59:23.950527+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1224,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.951168+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T17:59:23.951168+0000","last_peered":"2024-08-17T17:59:23.951168+0000","last_clean":"2024-08-17T17:59:23.951168+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T17:59:23.951168+0000","last_undegraded":"2024-08-17T17:59:23.951168+0000","last_fullsized":"2024-08-17T17:59:23.951168+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":2009,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.489973+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T17:59:31.489973+0000","last_peered":"2024-08-17T17:59:31.489973+0000","last_clean":"2024-08-17T17:59:31.489973+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T17:59:31.489973+0000","last_undegraded":"2024-08-17T17:59:31.489973+0000","last_fullsized":"2024-08-17T17:59:31.489973+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1892,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.325132+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T17:59:31.325132+0000","last_peered":"2024-08-17T17:59:31.325132+0000","last_clean":"2024-08-17T17:59:31.325132+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T17:59:31.325132+0000","last_undegraded":"2024-08-17T17:59:31.325132+0000","last_fullsized":"2024-08-17T17:59:31.325132+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1967,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.602088+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T17:59:31.602088+0000","last_peered":"2024-08-17T17:59:31.602088+0000","last_clean":"2024-08-17T17:59:31.602088+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T17:59:31.602088+0000","last_undegraded":"2024-08-17T17:59:31.602088+0000","last_fullsized":"2024-08-17T17:59:31.602088+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1894,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.327101+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T17:59:31.327101+0000","last_peered":"2024-08-17T17:59:31.327101+0000","last_clean":"2024-08-17T17:59:31.327101+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T17:59:31.327101+0000","last_undegraded":"2024-08-17T17:59:31.327101+0000","last_fullsized":"2024-08-17T17:59:31.327101+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1927,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.489877+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T17:59:31.489877+0000","last_peered":"2024-08-17T17:59:31.489877+0000","last_clean":"2024-08-17T17:59:31.489877+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T17:59:31.489877+0000","last_undegraded":"2024-08-17T17:59:31.489877+0000","last_fullsized":"2024-08-17T17:59:31.489877+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1251,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000862+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T17:59:24.000862+0000","last_peered":"2024-08-17T17:59:24.000862+0000","last_clean":"2024-08-17T17:59:24.000862+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T17:59:24.000862+0000","last_undegraded":"2024-08-17T17:59:24.000862+0000","last_fullsized":"2024-08-17T17:59:24.000862+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1973,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.602164+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T17:59:31.602164+0000","last_peered":"2024-08-17T17:59:31.602164+0000","last_clean":"2024-08-17T17:59:31.602164+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T17:59:31.602164+0000","last_undegraded":"2024-08-17T17:59:31.602164+0000","last_fullsized":"2024-08-17T17:59:31.602164+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":160,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601551+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T17:59:31.601551+0000","last_peered":"2024-08-17T17:59:31.601551+0000","last_clean":"2024-08-17T17:59:31.601551+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T17:59:31.601551+0000","last_undegraded":"2024-08-17T17:59:31.601551+0000","last_fullsized":"2024-08-17T17:59:31.601551+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1992,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601854+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T17:59:31.601854+0000","last_peered":"2024-08-17T17:59:31.601854+0000","last_clean":"2024-08-17T17:59:31.601854+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T17:59:31.601854+0000","last_undegraded":"2024-08-17T17:59:31.601854+0000","last_fullsized":"2024-08-17T17:59:31.601854+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":729,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.326604+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T17:59:31.326604+0000","last_peered":"2024-08-17T17:59:31.326604+0000","last_clean":"2024-08-17T17:59:31.326604+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T17:59:31.326604+0000","last_undegraded":"2024-08-17T17:59:31.326604+0000","last_fullsized":"2024-08-17T17:59:31.326604+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1973,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.602070+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T17:59:31.602070+0000","last_peered":"2024-08-17T17:59:31.602070+0000","last_clean":"2024-08-17T17:59:31.602070+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T17:59:31.602070+0000","last_undegraded":"2024-08-17T17:59:31.602070+0000","last_fullsized":"2024-08-17T17:59:31.602070+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1302,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.489565+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T17:59:31.489565+0000","last_peered":"2024-08-17T17:59:31.489565+0000","last_clean":"2024-08-17T17:59:31.489565+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T17:59:31.489565+0000","last_undegraded":"2024-08-17T17:59:31.489565+0000","last_fullsized":"2024-08-17T17:59:31.489565+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":220,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.602065+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T17:59:31.602065+0000","last_peered":"2024-08-17T17:59:31.602065+0000","last_clean":"2024-08-17T17:59:31.602065+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T17:59:31.602065+0000","last_undegraded":"2024-08-17T17:59:31.602065+0000","last_fullsized":"2024-08-17T17:59:31.602065+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1984,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601870+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T17:59:31.601870+0000","last_peered":"2024-08-17T17:59:31.601870+0000","last_clean":"2024-08-17T17:59:31.601870+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T17:59:31.601870+0000","last_undegraded":"2024-08-17T17:59:31.601870+0000","last_fullsized":"2024-08-17T17:59:31.601870+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1962,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601049+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T17:59:31.601049+0000","last_peered":"2024-08-17T17:59:31.601049+0000","last_clean":"2024-08-17T17:59:31.601049+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T17:59:31.601049+0000","last_undegraded":"2024-08-17T17:59:31.601049+0000","last_fullsized":"2024-08-17T17:59:31.601049+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1982,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601084+0000","last_change":"2024-08-17T17:48:39.667601+0000","last_active":"2024-08-17T17:59:31.601084+0000","last_peered":"2024-08-17T17:59:31.601084+0000","last_clean":"2024-08-17T17:59:31.601084+0000","last_became_active":"2024-08-17T17:48:39.667190+0000","last_became_peered":"2024-08-17T17:48:39.667190+0000","last_unstale":"2024-08-17T17:59:31.601084+0000","last_undegraded":"2024-08-17T17:59:31.601084+0000","last_fullsized":"2024-08-17T17:59:31.601084+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:48.971881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1988,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.490238+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T17:59:31.490238+0000","last_peered":"2024-08-17T17:59:31.490238+0000","last_clean":"2024-08-17T17:59:31.490238+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T17:59:31.490238+0000","last_undegraded":"2024-08-17T17:59:31.490238+0000","last_fullsized":"2024-08-17T17:59:31.490238+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1968,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.489566+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T17:59:31.489566+0000","last_peered":"2024-08-17T17:59:31.489566+0000","last_clean":"2024-08-17T17:59:31.489566+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T17:59:31.489566+0000","last_undegraded":"2024-08-17T17:59:31.489566+0000","last_fullsized":"2024-08-17T17:59:31.489566+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1258,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000376+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T17:59:24.000376+0000","last_peered":"2024-08-17T17:59:24.000376+0000","last_clean":"2024-08-17T17:59:24.000376+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T17:59:24.000376+0000","last_undegraded":"2024-08-17T17:59:24.000376+0000","last_fullsized":"2024-08-17T17:59:24.000376+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1204,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.951079+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T17:59:23.951079+0000","last_peered":"2024-08-17T17:59:23.951079+0000","last_clean":"2024-08-17T17:59:23.951079+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T17:59:23.951079+0000","last_undegraded":"2024-08-17T17:59:23.951079+0000","last_fullsized":"2024-08-17T17:59:23.951079+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1877,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.325289+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T17:59:31.325289+0000","last_peered":"2024-08-17T17:59:31.325289+0000","last_clean":"2024-08-17T17:59:31.325289+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T17:59:31.325289+0000","last_undegraded":"2024-08-17T17:59:31.325289+0000","last_fullsized":"2024-08-17T17:59:31.325289+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":208,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601871+0000","last_change":"2024-08-17T17:48:07.628566+0000","last_active":"2024-08-17T17:59:31.601871+0000","last_peered":"2024-08-17T17:59:31.601871+0000","last_clean":"2024-08-17T17:59:31.601871+0000","last_became_active":"2024-08-17T17:48:07.622523+0000","last_became_peered":"2024-08-17T17:48:07.622523+0000","last_unstale":"2024-08-17T17:59:31.601871+0000","last_undegraded":"2024-08-17T17:59:31.601871+0000","last_fullsized":"2024-08-17T17:59:31.601871+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:57.125760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1570,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000272+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T17:59:24.000272+0000","last_peered":"2024-08-17T17:59:24.000272+0000","last_clean":"2024-08-17T17:59:24.000272+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T17:59:24.000272+0000","last_undegraded":"2024-08-17T17:59:24.000272+0000","last_fullsized":"2024-08-17T17:59:24.000272+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1968,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601985+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T17:59:31.601985+0000","last_peered":"2024-08-17T17:59:31.601985+0000","last_clean":"2024-08-17T17:59:31.601985+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T17:59:31.601985+0000","last_undegraded":"2024-08-17T17:59:31.601985+0000","last_fullsized":"2024-08-17T17:59:31.601985+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1908,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.325045+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T17:59:31.325045+0000","last_peered":"2024-08-17T17:59:31.325045+0000","last_clean":"2024-08-17T17:59:31.325045+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T17:59:31.325045+0000","last_undegraded":"2024-08-17T17:59:31.325045+0000","last_fullsized":"2024-08-17T17:59:31.325045+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":538,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.950927+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T17:59:23.950927+0000","last_peered":"2024-08-17T17:59:23.950927+0000","last_clean":"2024-08-17T17:59:23.950927+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T17:59:23.950927+0000","last_undegraded":"2024-08-17T17:59:23.950927+0000","last_fullsized":"2024-08-17T17:59:23.950927+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":647,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.600955+0000","last_change":"2024-08-17T17:49:45.652511+0000","last_active":"2024-08-17T17:59:31.600955+0000","last_peered":"2024-08-17T17:59:31.600955+0000","last_clean":"2024-08-17T17:59:31.600955+0000","last_became_active":"2024-08-17T17:49:45.652143+0000","last_became_peered":"2024-08-17T17:49:45.652143+0000","last_unstale":"2024-08-17T17:59:31.600955+0000","last_undegraded":"2024-08-17T17:59:31.600955+0000","last_fullsized":"2024-08-17T17:59:31.600955+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:12.794187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":663,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601431+0000","last_change":"2024-08-17T17:49:45.656390+0000","last_active":"2024-08-17T17:59:31.601431+0000","last_peered":"2024-08-17T17:59:31.601431+0000","last_clean":"2024-08-17T17:59:31.601431+0000","last_became_active":"2024-08-17T17:49:45.655745+0000","last_became_peered":"2024-08-17T17:49:45.655745+0000","last_unstale":"2024-08-17T17:59:31.601431+0000","last_undegraded":"2024-08-17T17:59:31.601431+0000","last_fullsized":"2024-08-17T17:59:31.601431+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:01:26.210554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1222,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.950635+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T17:59:23.950635+0000","last_peered":"2024-08-17T17:59:23.950635+0000","last_clean":"2024-08-17T17:59:23.950635+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T17:59:23.950635+0000","last_undegraded":"2024-08-17T17:59:23.950635+0000","last_fullsized":"2024-08-17T17:59:23.950635+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":679,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.326247+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T17:59:31.326247+0000","last_peered":"2024-08-17T17:59:31.326247+0000","last_clean":"2024-08-17T17:59:31.326247+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T17:59:31.326247+0000","last_undegraded":"2024-08-17T17:59:31.326247+0000","last_fullsized":"2024-08-17T17:59:31.326247+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1905,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.326213+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T17:59:31.326213+0000","last_peered":"2024-08-17T17:59:31.326213+0000","last_clean":"2024-08-17T17:59:31.326213+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T17:59:31.326213+0000","last_undegraded":"2024-08-17T17:59:31.326213+0000","last_fullsized":"2024-08-17T17:59:31.326213+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2000,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.489972+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T17:59:31.489972+0000","last_peered":"2024-08-17T17:59:31.489972+0000","last_clean":"2024-08-17T17:59:31.489972+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T17:59:31.489972+0000","last_undegraded":"2024-08-17T17:59:31.489972+0000","last_fullsized":"2024-08-17T17:59:31.489972+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":712,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.326073+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T17:59:31.326073+0000","last_peered":"2024-08-17T17:59:31.326073+0000","last_clean":"2024-08-17T17:59:31.326073+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T17:59:31.326073+0000","last_undegraded":"2024-08-17T17:59:31.326073+0000","last_fullsized":"2024-08-17T17:59:31.326073+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1211,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.950412+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T17:59:23.950412+0000","last_peered":"2024-08-17T17:59:23.950412+0000","last_clean":"2024-08-17T17:59:23.950412+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T17:59:23.950412+0000","last_undegraded":"2024-08-17T17:59:23.950412+0000","last_fullsized":"2024-08-17T17:59:23.950412+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1210,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000874+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T17:59:24.000874+0000","last_peered":"2024-08-17T17:59:24.000874+0000","last_clean":"2024-08-17T17:59:24.000874+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T17:59:24.000874+0000","last_undegraded":"2024-08-17T17:59:24.000874+0000","last_fullsized":"2024-08-17T17:59:24.000874+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":696,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.601435+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T17:59:31.601435+0000","last_peered":"2024-08-17T17:59:31.601435+0000","last_clean":"2024-08-17T17:59:31.601435+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T17:59:31.601435+0000","last_undegraded":"2024-08-17T17:59:31.601435+0000","last_fullsized":"2024-08-17T17:59:31.601435+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1228,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000997+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T17:59:24.000997+0000","last_peered":"2024-08-17T17:59:24.000997+0000","last_clean":"2024-08-17T17:59:24.000997+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T17:59:24.000997+0000","last_undegraded":"2024-08-17T17:59:24.000997+0000","last_fullsized":"2024-08-17T17:59:24.000997+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":112,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000189+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T17:59:24.000189+0000","last_peered":"2024-08-17T17:59:24.000189+0000","last_clean":"2024-08-17T17:59:24.000189+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T17:59:24.000189+0000","last_undegraded":"2024-08-17T17:59:24.000189+0000","last_fullsized":"2024-08-17T17:59:24.000189+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1244,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000995+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T17:59:24.000995+0000","last_peered":"2024-08-17T17:59:24.000995+0000","last_clean":"2024-08-17T17:59:24.000995+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T17:59:24.000995+0000","last_undegraded":"2024-08-17T17:59:24.000995+0000","last_fullsized":"2024-08-17T17:59:24.000995+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1261,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:23.950487+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T17:59:23.950487+0000","last_peered":"2024-08-17T17:59:23.950487+0000","last_clean":"2024-08-17T17:59:23.950487+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T17:59:23.950487+0000","last_undegraded":"2024-08-17T17:59:23.950487+0000","last_fullsized":"2024-08-17T17:59:23.950487+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1906,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.326972+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T17:59:31.326972+0000","last_peered":"2024-08-17T17:59:31.326972+0000","last_clean":"2024-08-17T17:59:31.326972+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T17:59:31.326972+0000","last_undegraded":"2024-08-17T17:59:31.326972+0000","last_fullsized":"2024-08-17T17:59:31.326972+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1205,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000495+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T17:59:24.000495+0000","last_peered":"2024-08-17T17:59:24.000495+0000","last_clean":"2024-08-17T17:59:24.000495+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T17:59:24.000495+0000","last_undegraded":"2024-08-17T17:59:24.000495+0000","last_fullsized":"2024-08-17T17:59:24.000495+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1959,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.489902+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T17:59:31.489902+0000","last_peered":"2024-08-17T17:59:31.489902+0000","last_clean":"2024-08-17T17:59:31.489902+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T17:59:31.489902+0000","last_undegraded":"2024-08-17T17:59:31.489902+0000","last_fullsized":"2024-08-17T17:59:31.489902+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1960,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.489601+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T17:59:31.489601+0000","last_peered":"2024-08-17T17:59:31.489601+0000","last_clean":"2024-08-17T17:59:31.489601+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T17:59:31.489601+0000","last_undegraded":"2024-08-17T17:59:31.489601+0000","last_fullsized":"2024-08-17T17:59:31.489601+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":618,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000667+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T17:59:24.000667+0000","last_peered":"2024-08-17T17:59:24.000667+0000","last_clean":"2024-08-17T17:59:24.000667+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T17:59:24.000667+0000","last_undegraded":"2024-08-17T17:59:24.000667+0000","last_fullsized":"2024-08-17T17:59:24.000667+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1230,"reported_epoch":1260,"state":"active+clean","last_fresh":"2024-08-17T17:59:24.000577+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T17:59:24.000577+0000","last_peered":"2024-08-17T17:59:24.000577+0000","last_clean":"2024-08-17T17:59:24.000577+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T17:59:24.000577+0000","last_undegraded":"2024-08-17T17:59:24.000577+0000","last_fullsized":"2024-08-17T17:59:24.000577+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1906,"reported_epoch":1261,"state":"active+clean","last_fresh":"2024-08-17T17:59:31.326401+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T17:59:31.326401+0000","last_peered":"2024-08-17T17:59:31.326401+0000","last_clean":"2024-08-17T17:59:31.326401+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T17:59:31.326401+0000","last_undegraded":"2024-08-17T17:59:31.326401+0000","last_fullsized":"2024-08-17T17:59:31.326401+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":46,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234464,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313212,"kb_used_data":11096,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93431940,"statfs":{"total":95995035648,"available":95674306560,"internally_reserved":0,"allocated":11362304,"data_stored":7325088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496099,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":309052,"kb_used_data":11096,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93436100,"statfs":{"total":95995035648,"available":95678566400,"internally_reserved":0,"allocated":11362304,"data_stored":7325565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757737,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326351,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":269640,"kb_used_data":12108,"kb_used_omap":57,"kb_used_meta":257222,"kb_avail":93475512,"statfs":{"total":95995035648,"available":95718924288,"internally_reserved":0,"allocated":12398592,"data_stored":7868872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":263396091},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405590,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":270804,"kb_used_data":12120,"kb_used_omap":61,"kb_used_meta":258370,"kb_avail":93474348,"statfs":{"total":95995035648,"available":95717732352,"internally_reserved":0,"allocated":12410880,"data_stored":7874060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":264571832},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452124,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":210760,"kb_used_data":12108,"kb_used_omap":54,"kb_used_meta":198345,"kb_avail":93534392,"statfs":{"total":95995035648,"available":95779217408,"internally_reserved":0,"allocated":12398592,"data_stored":7868753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":203105678},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474815,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":268384,"kb_used_data":12920,"kb_used_omap":119,"kb_used_meta":254856,"kb_avail":93476768,"statfs":{"total":95995035648,"available":95720210432,"internally_reserved":0,"allocated":13230080,"data_stored":8603290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":260973278},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019372,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:59:36.700 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:59:36.700 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:59:36.700 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:59:36.701 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.027486 2024-08-17T17:59:36.701 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 2 0.027486194265614272 2024-08-17T17:59:36.763 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:36.787 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:36.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:36.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:36 smithi179 ceph-mon[93519]: pgmap v2011: 89 pgs: 89 active+clean; 582 KiB data, 1.6 GiB used, 535 GiB / 536 GiB avail 2024-08-17T17:59:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:36 smithi179 ceph-mon[93519]: from='client.46106 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:59:36.852 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:36.888 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:36.911 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:36.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:36.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:37.012 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:37.037 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:37.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:37.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:37.139 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:37.180 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:37.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:37.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:37.282 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:37.305 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:37.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:37.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:37.407 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:37.432 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:37.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:37.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:37.448 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 3 16 8674 8658 180.338 162.562 0.0085158 0.00553874 2024-08-17T17:59:37.534 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:37.562 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:37.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:37.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:37.663 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:37.690 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:37.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:37.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:37.792 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:37.829 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:37.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:37.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:37.931 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:37.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:37.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:37.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:38.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:38.096 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:38.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:38.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:38.198 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:38.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:38.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:38.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:38.364 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:38.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:38.400 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:38.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:38.448 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 4 16 11439 11423 178.453 172.812 0.00578526 0.00559738 2024-08-17T17:59:38.467 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:38.470 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:59:38.501 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:38.526 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:38.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:38.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:38.628 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:38.654 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:38.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:38.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:38.756 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:38.782 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:38.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:38.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:38.884 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:38.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:38.926 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:38.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:38.960 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:38 smithi130 ceph-mon[105521]: pgmap v2012: 89 pgs: 89 active+clean; 62 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 6.2 MiB/s wr, 98 op/s 2024-08-17T17:59:38.960 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:38 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:38.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:38 smithi130 ceph-mon[110068]: pgmap v2012: 89 pgs: 89 active+clean; 62 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 6.2 MiB/s wr, 98 op/s 2024-08-17T17:59:38.961 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:38 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:39.027 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:39.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:39.075 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:39.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:39.175 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:39.201 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:39.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:39.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:39.303 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:39.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:38 smithi179 ceph-mon[93519]: pgmap v2012: 89 pgs: 89 active+clean; 62 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 6.2 MiB/s wr, 98 op/s 2024-08-17T17:59:39.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:38 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:39.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:39.344 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:39.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:39.445 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:39.448 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 5 16 13877 13861 173.235 152.375 0.0079456 0.00576592 2024-08-17T17:59:39.470 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:39.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:39.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:39.571 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:39.598 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:39.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:39.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:39.699 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:39.724 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:39.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:39.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:39.825 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:39.869 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:39.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:39.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:39.971 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:40.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:40.011 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:40.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:40.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:40.136 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:40.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:40.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:40.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:59:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:59:39] "GET /metrics HTTP/1.1" 200 36715 "" "Prometheus/2.51.0" 2024-08-17T17:59:40.238 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:40.264 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:40.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:40.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:40.366 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:40.392 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:40.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:40.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:40.448 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 6 16 16384 16368 170.475 156.688 0.00642341 0.00585969 2024-08-17T17:59:40.493 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:40.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:40.533 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:40.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:40.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:59:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:59:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:59:40.633 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:40.673 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:40.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:40.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:40.775 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:40.814 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:40.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:40.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:40.916 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:40.939 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:40.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:40.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:40.949 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:40.950 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:41.041 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:41.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:41.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:41.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:41.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:40 smithi130 ceph-mon[105521]: pgmap v2013: 89 pgs: 89 active+clean; 478 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 48 MiB/s wr, 763 op/s 2024-08-17T17:59:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:40 smithi130 ceph-mon[110068]: pgmap v2013: 89 pgs: 89 active+clean; 478 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 48 MiB/s wr, 763 op/s 2024-08-17T17:59:41.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:41.217 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:41.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:41.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:41.318 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:41.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:40 smithi179 ceph-mon[93519]: pgmap v2013: 89 pgs: 89 active+clean; 478 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 48 MiB/s wr, 763 op/s 2024-08-17T17:59:41.356 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:41.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:41.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:41.448 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 7 16 18581 18565 165.736 137.312 0.00650088 0.00602694 2024-08-17T17:59:41.458 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:41.498 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:41.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:41.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:41.600 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:41.621 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:41.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:41.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:41.690 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T17:59:41.690 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T17:59:41.691 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:59:41.691 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:59:41.723 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:41.726 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T17:59:41.726 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:59:41.747 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:41.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:41.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:41.761 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T17:59:41.762 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:59:41.792 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T17:59:41.792 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:59:41.824 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T17:59:41.825 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:59:41.848 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:41.870 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T17:59:41.870 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:59:41.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:41.889 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:41.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:41.904 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T17:59:41.904 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:59:41.991 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:42.033 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:42.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:42.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:42.135 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:42.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:41 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3656432597' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:59:42.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:41 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3656432597' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:59:42.173 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:42.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:42.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:42.275 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:42.300 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:42.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:42.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:42.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:41 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3656432597' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:59:42.402 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:42.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:42.427 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:42.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:42.448 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 8 16 20229 20213 157.893 103 0.0100441 0.00632743 2024-08-17T17:59:42.528 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:42.552 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:42.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:42.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:42.654 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:42.679 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:42.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:42.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:42.782 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:42.806 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:42.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:42.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:42.907 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:42.945 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:42.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:42.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:43.047 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:43.071 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:43.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:43.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:43.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:42 smithi130 ceph-mon[105521]: pgmap v2014: 89 pgs: 89 active+clean; 635 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T17:59:43.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:43.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:59:43.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:42 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1959073762' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:59:43.129 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:42 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:59:43.129 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:42 smithi130 ceph-mon[110068]: pgmap v2014: 89 pgs: 89 active+clean; 635 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T17:59:43.129 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:43.129 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:59:43.129 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:42 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1959073762' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:59:43.129 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:42 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:59:43.174 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:43.201 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:43.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:43.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:43.302 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:43.315 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T17:59:43.327 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:43.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:43.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:43.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:42 smithi179 ceph-mon[93519]: pgmap v2014: 89 pgs: 89 active+clean; 635 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T17:59:43.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:43.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:59:43.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:42 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1959073762' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:59:43.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:42 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:59:43.429 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:43.448 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 9 16 22078 22062 153.189 115.562 0.00197004 0.00646506 2024-08-17T17:59:43.450 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:43.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:43.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:43.552 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:43.577 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:43.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:43.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:43.679 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:43.715 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:43.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:43.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:43.818 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:43.843 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:43.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:43.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:43.944 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:43.970 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:43.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:43.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:44.071 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:44.098 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:44.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:44.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:44.199 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:44.220 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:44.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:44.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:44.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:44.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:43 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:59:44.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:43 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:59:44.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:43 smithi179 ceph-mon[93519]: osdmap e1262: 8 total, 8 up, 6 in 2024-08-17T17:59:44.351 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:44.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:44.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:44.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:43 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:59:44.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:43 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:59:44.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:43 smithi130 ceph-mon[105521]: osdmap e1262: 8 total, 8 up, 6 in 2024-08-17T17:59:44.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:43 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:59:44.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:43 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:59:44.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:43 smithi130 ceph-mon[110068]: osdmap e1262: 8 total, 8 up, 6 in 2024-08-17T17:59:44.448 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 10 16 23762 23746 148.394 105.25 0.00935728 0.00673307 2024-08-17T17:59:44.453 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:44.488 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:44.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:44.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:44.590 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:44.615 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:44.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:44.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:44.717 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:44.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:44.739 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:44.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:44.815 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:59:44.840 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:44.865 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:44.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:44.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:44.967 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:44.992 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:44.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:44.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:45.094 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:45.114 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:45.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:45.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:45.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:44 smithi130 ceph-mon[105521]: pgmap v2015: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:59:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:44 smithi130 ceph-mon[110068]: pgmap v2015: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:59:45.215 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:45.241 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:45.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:45.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:45.342 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:45.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:44 smithi179 ceph-mon[93519]: pgmap v2015: 89 pgs: 89 active+clean; 1.1 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:59:45.383 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:45.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:45.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:45.448 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 11 16 25372 25356 144.051 100.625 0.00306812 0.00693627 2024-08-17T17:59:45.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:45.507 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:45.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:45.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:45.609 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:45.635 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:45.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:45.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:45.737 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:45.776 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:45.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:45.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:45.878 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:45.899 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:45.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:45.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:46.001 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:46.024 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:46.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:46.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:46.126 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:46.162 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:46.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:46.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:46.263 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:46.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:46.306 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:46.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:46.407 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:46.431 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:46.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:46.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:46.448 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 12 16 26873 26857 139.864 93.8125 0.00660544 0.00714413 2024-08-17T17:59:46.533 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:46.557 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:46.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:46.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:46.659 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:46.682 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:46.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:46.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:46.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:46.819 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:46.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:46.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:46.921 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:46.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:46.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:46.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:47.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:47.082 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:47.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:47.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:47.184 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:47.222 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:47.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:47.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:47.324 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:47.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:46 smithi179 ceph-mon[93519]: pgmap v2017: 89 pgs: 89 active+clean; 1.3 GiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-08-17T17:59:47.351 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:47.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:47.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:47.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:46 smithi130 ceph-mon[105521]: pgmap v2017: 89 pgs: 89 active+clean; 1.3 GiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-08-17T17:59:47.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:46 smithi130 ceph-mon[110068]: pgmap v2017: 89 pgs: 89 active+clean; 1.3 GiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-08-17T17:59:47.448 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 13 16 28186 28170 135.417 82.0625 0.00860171 0.00737828 2024-08-17T17:59:47.453 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:47.477 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:47.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:47.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:47.579 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:47.604 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:47.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:47.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:47.706 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:47.731 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:47.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:47.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:47.833 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:47.869 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:47.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:47.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:47.970 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:47.998 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:47.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:47.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:48.100 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:48.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:48.122 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:48.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:48.223 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:48.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:48.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:48.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:48.352 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:48.389 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:48.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:48.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:48.449 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 14 16 29828 29812 133.074 102.625 0.00807574 0.00750761 2024-08-17T17:59:48.491 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:48.532 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:48.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:48.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:48.634 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:48.657 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:48.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:48.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:48.759 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:48.781 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:48.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:48.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:48.882 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:48.904 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:48.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:48.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:49.006 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:49.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:49.031 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:49.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:49.132 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:49.156 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:49.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:49.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:49.258 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:49.295 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:49.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:49.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:49.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:48 smithi179 ceph-mon[93519]: pgmap v2018: 89 pgs: 89 active+clean; 1.4 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-08-17T17:59:49.397 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:49.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:48 smithi130 ceph-mon[110068]: pgmap v2018: 89 pgs: 89 active+clean; 1.4 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-08-17T17:59:49.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:48 smithi130 ceph-mon[105521]: pgmap v2018: 89 pgs: 89 active+clean; 1.4 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-08-17T17:59:49.435 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:49.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:49.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:49.449 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 15 16 30774 30758 128.144 59.125 0.00633977 0.00779735 2024-08-17T17:59:49.537 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:49.558 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:49.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:49.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:49.660 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:49.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:49.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:49.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:49.787 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:49.809 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:49.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:49.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:49.911 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:49.936 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:49.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:49.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:50.038 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:50.064 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:50.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:50.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:50.166 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:59:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:59:49] "GET /metrics HTTP/1.1" 200 36776 "" "Prometheus/2.51.0" 2024-08-17T17:59:50.192 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:50.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:50.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:50.294 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:50.319 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:50.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:50.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:50.421 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:50.446 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:50.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:50.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:50.449 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 16 15 32238 32223 125.857 91.5625 0.00739024 0.00793962 2024-08-17T17:59:50.548 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:50.569 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:50.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:50.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:50.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 17:59:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T17:59:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T17:59:50.671 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:50.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:50.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:50.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:50.797 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:50.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:50.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:50.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:50.923 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:50.946 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:50.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:50.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:51.048 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:51.073 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:51.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:51.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:51.175 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:51.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:51.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:51.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:51.315 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:51.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:51.338 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:51.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:51.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:50 smithi179 ceph-mon[93519]: pgmap v2019: 89 pgs: 89 active+clean; 1.7 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 125 MiB/s wr, 2k op/s 2024-08-17T17:59:51.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:50 smithi130 ceph-mon[105521]: pgmap v2019: 89 pgs: 89 active+clean; 1.7 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 125 MiB/s wr, 2k op/s 2024-08-17T17:59:51.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:50 smithi130 ceph-mon[110068]: pgmap v2019: 89 pgs: 89 active+clean; 1.7 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 125 MiB/s wr, 2k op/s 2024-08-17T17:59:51.439 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:51.449 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 17 16 33753 33737 124.019 94.625 0.0114275 0.00805712 2024-08-17T17:59:51.478 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:51.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:51.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:51.580 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:51.615 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:51.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:51.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:51.717 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:51.742 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:51.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:51.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:51.843 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:51.883 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:51.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:51.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:51.986 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:52.012 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:52.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:52.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:52.114 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:52.138 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:52.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:52.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:52.240 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:52.266 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:52.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:52.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:52.368 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:52.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:52.407 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:52.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:52.449 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 18 15 35351 35336 122.681 99.9375 0.0116403 0.00814469 2024-08-17T17:59:52.509 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:52.533 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:52.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:52.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:52.635 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:52.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:52.660 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:52.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:52.761 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:52.786 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:52.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:52.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:52.888 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:52.915 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:52.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:52.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:53.016 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:53.054 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:53.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:53.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:53.156 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:53.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:52 smithi130 ceph-mon[110068]: pgmap v2020: 89 pgs: 89 active+clean; 1.8 GiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-08-17T17:59:53.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:52 smithi130 ceph-mon[105521]: pgmap v2020: 89 pgs: 89 active+clean; 1.8 GiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-08-17T17:59:53.181 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:53.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:53.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:53.283 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:53.325 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:53.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:53.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:53.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:52 smithi179 ceph-mon[93519]: pgmap v2020: 89 pgs: 89 active+clean; 1.8 GiB data, 6.9 GiB used, 529 GiB / 536 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-08-17T17:59:53.427 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:53.449 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 19 16 37108 37092 122 109.75 0.00821632 0.00819124 2024-08-17T17:59:53.451 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:53.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:53.553 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:53.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:53.610 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:53.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:53.634 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:59:53.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:53.757 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:53.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:53.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:53.859 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:53.902 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:53.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:53.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:54.004 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:54.029 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:54.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:54.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:54.131 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:54.155 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:54.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:54.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:54.257 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:54.282 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:54.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:54.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:54.384 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:54.409 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:54.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:54.449 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T17:59:54.449606+0000 min lat: 0.00128899 max lat: 0.397745 avg lat: 0.00825518 2024-08-17T17:59:54.449 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:59:54.449 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 20 16 38758 38742 121.056 103.125 0.00793707 0.00825518 2024-08-17T17:59:54.511 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:54.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:54.576 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:54.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:54.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:54.702 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:54.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:54.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:54.804 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:54.848 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:54.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:54.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:54.949 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:54.990 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:54.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:54.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:55.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:54 smithi130 ceph-mon[110068]: pgmap v2021: 89 pgs: 89 active+clean; 2.1 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-08-17T17:59:55.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:54 smithi130 ceph-mon[105521]: pgmap v2021: 89 pgs: 89 active+clean; 2.1 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-08-17T17:59:55.092 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:55.135 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:55.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:55.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:55.237 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:55.262 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:55.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:55.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:54 smithi179 ceph-mon[93519]: pgmap v2021: 89 pgs: 89 active+clean; 2.1 GiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-08-17T17:59:55.364 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:55.387 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:55.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:55.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:55.449 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 21 16 40285 40269 119.835 95.4375 0.0153683 0.00833737 2024-08-17T17:59:55.489 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:55.522 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T17:59:55.522 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T17:59:55.522 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.522 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:40261 46.c e1262/1258 46:34f0833d:::benchmark_data_smithi130_267720_object40260:head v 1262'2565, mlcod=1262'2563)", 2024-08-17T17:59:55.523 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:55.429805+0000", 2024-08-17T17:59:55.523 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.00041001399999999999, 2024-08-17T17:59:55.523 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000437121, 2024-08-17T17:59:55.523 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:59:55.523 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:59:55.523 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T17:59:55.523 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:59:55.523 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.523 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:59:55.523 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:55.429805+0000", 2024-08-17T17:59:55.523 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:59:55.523 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:59:55.524 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.524 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:59:55.524 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:55.429805+0000", 2024-08-17T17:59:55.524 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:59:55.524 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:59:55.524 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.524 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:59:55.524 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:55.429807+0000", 2024-08-17T17:59:55.524 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1960000000000001e-06 2024-08-17T17:59:55.524 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:59:55.524 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.524 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:59:55.524 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:55.429893+0000", 2024-08-17T17:59:55.524 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6302000000000001e-05 2024-08-17T17:59:55.525 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:59:55.525 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.525 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:59:55.525 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:55.429911+0000", 2024-08-17T17:59:55.525 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8054e-05 2024-08-17T17:59:55.525 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:59:55.525 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.525 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:59:55.525 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:55.430034+0000", 2024-08-17T17:59:55.525 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000123404 2024-08-17T17:59:55.525 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:59:55.525 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.525 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T17:59:55.526 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:55.430195+0000", 2024-08-17T17:59:55.526 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016091200000000001 2024-08-17T17:59:55.526 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:59:55.526 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.526 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T17:59:55.526 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:55.430218+0000", 2024-08-17T17:59:55.526 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2929999999999999e-05 2024-08-17T17:59:55.526 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:59:55.526 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:59:55.526 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:59:55.526 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:59:55.526 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.526 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:40262 46.8 e1262/1258)", 2024-08-17T17:59:55.527 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:55.430084+0000", 2024-08-17T17:59:55.527 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.00013186299999999999, 2024-08-17T17:59:55.527 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00021155999999999999, 2024-08-17T17:59:55.527 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T17:59:55.527 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T17:59:55.527 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "queued for pg", 2024-08-17T17:59:55.527 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T17:59:55.527 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.527 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T17:59:55.527 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:55.430084+0000", 2024-08-17T17:59:55.527 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:59:55.527 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:59:55.527 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.528 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T17:59:55.528 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:55.430084+0000", 2024-08-17T17:59:55.528 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T17:59:55.528 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:59:55.528 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.528 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T17:59:55.528 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:55.430084+0000", 2024-08-17T17:59:55.528 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8400000000000004e-07 2024-08-17T17:59:55.528 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:59:55.528 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.528 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T17:59:55.528 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:55.430107+0000", 2024-08-17T17:59:55.528 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3065999999999999e-05 2024-08-17T17:59:55.529 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:59:55.529 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.529 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T17:59:55.529 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:55.430117+0000", 2024-08-17T17:59:55.529 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0061000000000001e-05 2024-08-17T17:59:55.529 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T17:59:55.529 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T17:59:55.529 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T17:59:55.529 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:55.430124+0000", 2024-08-17T17:59:55.529 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2999999999999998e-06 2024-08-17T17:59:55.529 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:59:55.529 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T17:59:55.529 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:59:55.530 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T17:59:55.530 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T17:59:55.530 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 2 2024-08-17T17:59:55.530 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T17:59:55.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:55.531 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:55.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:55.632 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:55.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:55.677 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:55.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:55.778 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:55.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:55.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:55.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:55.923 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:55.946 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:55.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:55.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:56.048 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:59:56.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:59:56.085 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:56.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:56.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:59:56.187 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:56.211 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:56.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:56.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:56.312 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:56.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:56.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:56.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:56.437 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:56.452 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 22 16 41702 41686 118.413 88.5625 0.0059826 0.00843861 2024-08-17T17:59:56.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:56.489 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:56.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:56.590 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:56.616 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:56.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:56.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:56.718 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:56.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:56.757 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:56.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:56.858 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:56.884 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:56.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:56.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:56.986 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:56.995 INFO:teuthology.orchestra.run.smithi130.stderr:set osd.2 primary-affinity to 0.0274862 (818012) 2024-08-17T17:59:57.030 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:57.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:57.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:57.132 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:57.153 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:57.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:57.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:57.255 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:57.276 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:57.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:57.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:57 smithi179 ceph-mon[93519]: pgmap v2022: 89 pgs: 89 active+clean; 2.2 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:59:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:57 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/446660403' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.027486194265614272} : dispatch 2024-08-17T17:59:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:57 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.027486194265614272} : dispatch 2024-08-17T17:59:57.377 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:57.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:57.399 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:57.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:57.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:57 smithi130 ceph-mon[110068]: pgmap v2022: 89 pgs: 89 active+clean; 2.2 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:59:57.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:57 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/446660403' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.027486194265614272} : dispatch 2024-08-17T17:59:57.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:57 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.027486194265614272} : dispatch 2024-08-17T17:59:57.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:57 smithi130 ceph-mon[105521]: pgmap v2022: 89 pgs: 89 active+clean; 2.2 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:59:57.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:57 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/446660403' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.027486194265614272} : dispatch 2024-08-17T17:59:57.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:57 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.027486194265614272} : dispatch 2024-08-17T17:59:57.450 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 23 16 42604 42588 115.715 56.375 0.00226646 0.0084837 2024-08-17T17:59:57.500 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:57.521 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:57.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:57.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:57.623 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:57.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:57.645 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:57.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:57.746 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:57.768 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:57.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:57.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:57.870 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:57.892 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:57.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:57.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:57.993 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:58.017 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:58.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:58.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:58.118 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:59:58.145 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:58.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:58.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:59:58.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:58.287 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:58.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:58.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:58 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.027486194265614272}]': finished 2024-08-17T17:59:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:58 smithi179 ceph-mon[93519]: osdmap e1263: 8 total, 8 up, 6 in 2024-08-17T17:59:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:59:58.389 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:59:58.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:58.418 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:58.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:59:58.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:58 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.027486194265614272}]': finished 2024-08-17T17:59:58.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:58 smithi130 ceph-mon[105521]: osdmap e1263: 8 total, 8 up, 6 in 2024-08-17T17:59:58.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:58.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:59:58.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:58 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.027486194265614272}]': finished 2024-08-17T17:59:58.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:58 smithi130 ceph-mon[110068]: osdmap e1263: 8 total, 8 up, 6 in 2024-08-17T17:59:58.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T17:59:58.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:59:58.450 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 24 16 43285 43269 112.667 42.5625 0.0110114 0.00887011 2024-08-17T17:59:58.518 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:58.543 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:58.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:58.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:58.645 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:58.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:58.694 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:58.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:58.795 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:58.843 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:58.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:58.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:58.945 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:59:58.987 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:58.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:58.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:59:59.089 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:59.114 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:59.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:59.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:59.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:59 smithi130 ceph-mon[105521]: pgmap v2024: 89 pgs: 3 peering, 86 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-08-17T17:59:59.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:59 smithi130 ceph-mon[105521]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:59:59.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 17:59:59 smithi130 ceph-mon[105521]: osdmap e1264: 8 total, 8 up, 6 in 2024-08-17T17:59:59.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:59 smithi130 ceph-mon[110068]: pgmap v2024: 89 pgs: 3 peering, 86 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-08-17T17:59:59.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:59 smithi130 ceph-mon[110068]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:59:59.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 17:59:59 smithi130 ceph-mon[110068]: osdmap e1264: 8 total, 8 up, 6 in 2024-08-17T17:59:59.216 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:59:59.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:59.252 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:59.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:59:59.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:59 smithi179 ceph-mon[93519]: pgmap v2024: 89 pgs: 3 peering, 86 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-08-17T17:59:59.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:59 smithi179 ceph-mon[93519]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:59:59.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 17:59:59 smithi179 ceph-mon[93519]: osdmap e1264: 8 total, 8 up, 6 in 2024-08-17T17:59:59.353 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:59:59.386 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T17:59:59.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:59.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:59:59.450 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 25 16 44671 44655 111.625 86.625 0.0132818 0.00895265 2024-08-17T17:59:59.488 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:59.514 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:59.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:59.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:59.616 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:59.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:59.646 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:59.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:59.747 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:59:59.772 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:59.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:59.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:59:59.874 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:59:59.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:59:59.895 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T17:59:59.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:59:59.996 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:00.033 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:00.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:00.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:00.135 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:00.156 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:00.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:00.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:00.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 17:59:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:17:59:59] "GET /metrics HTTP/1.1" 200 36776 "" "Prometheus/2.51.0" 2024-08-17T18:00:00.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:00 smithi130 ceph-mon[110068]: Health detail: HEALTH_WARN noscrub flag(s) set; Reduced data availability: 2 pgs inactive, 2 pgs peering 2024-08-17T18:00:00.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:00 smithi130 ceph-mon[110068]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-08-17T18:00:00.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:00 smithi130 ceph-mon[110068]: [WRN] PG_AVAILABILITY: Reduced data availability: 2 pgs inactive, 2 pgs peering 2024-08-17T18:00:00.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:00 smithi130 ceph-mon[110068]: pg 4.d is stuck peering for 28m, current state peering, last acting [1,2,6] 2024-08-17T18:00:00.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:00 smithi130 ceph-mon[110068]: pg 4.1c is stuck peering for 28m, current state peering, last acting [1,2,3] 2024-08-17T18:00:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:00 smithi130 ceph-mon[105521]: Health detail: HEALTH_WARN noscrub flag(s) set; Reduced data availability: 2 pgs inactive, 2 pgs peering 2024-08-17T18:00:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:00 smithi130 ceph-mon[105521]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-08-17T18:00:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:00 smithi130 ceph-mon[105521]: [WRN] PG_AVAILABILITY: Reduced data availability: 2 pgs inactive, 2 pgs peering 2024-08-17T18:00:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:00 smithi130 ceph-mon[105521]: pg 4.d is stuck peering for 28m, current state peering, last acting [1,2,6] 2024-08-17T18:00:00.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:00 smithi130 ceph-mon[105521]: pg 4.1c is stuck peering for 28m, current state peering, last acting [1,2,3] 2024-08-17T18:00:00.258 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:00.284 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:00.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:00.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:00.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:00:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:00:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:00:00.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:00 smithi179 ceph-mon[93519]: Health detail: HEALTH_WARN noscrub flag(s) set; Reduced data availability: 2 pgs inactive, 2 pgs peering 2024-08-17T18:00:00.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:00 smithi179 ceph-mon[93519]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-08-17T18:00:00.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:00 smithi179 ceph-mon[93519]: [WRN] PG_AVAILABILITY: Reduced data availability: 2 pgs inactive, 2 pgs peering 2024-08-17T18:00:00.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:00 smithi179 ceph-mon[93519]: pg 4.d is stuck peering for 28m, current state peering, last acting [1,2,6] 2024-08-17T18:00:00.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:00 smithi179 ceph-mon[93519]: pg 4.1c is stuck peering for 28m, current state peering, last acting [1,2,3] 2024-08-17T18:00:00.386 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:00.410 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:00.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:00.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:00.450 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 26 16 46037 46021 110.615 85.375 0.00754297 0.00903473 2024-08-17T18:00:00.511 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:00.535 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:00.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:00.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:00.637 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:00.678 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:00.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:00.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:00.780 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:00.823 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:00.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:00.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:00.925 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:00.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:00.961 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:00.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:01.062 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:01.084 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:01.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:01.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:01.185 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:01.209 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:01.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:01.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:01.311 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:01.332 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:01.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:01.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:01.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:01 smithi179 ceph-mon[93519]: pgmap v2026: 89 pgs: 8 peering, 81 active+clean; 2.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-08-17T18:00:01.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:01 smithi130 ceph-mon[105521]: pgmap v2026: 89 pgs: 8 peering, 81 active+clean; 2.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-08-17T18:00:01.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:01 smithi130 ceph-mon[110068]: pgmap v2026: 89 pgs: 8 peering, 81 active+clean; 2.6 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-08-17T18:00:01.433 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:01.450 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 27 16 47123 47107 109.031 67.875 0.00873635 0.00916537 2024-08-17T18:00:01.457 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:01.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:01.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:01.558 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:01.599 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:01.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:01.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:01.701 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:01.739 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:01.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:01.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:01.840 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:01.866 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:01.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:01.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:01.968 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:01.993 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:01.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:01.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:02.095 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:02.119 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:02.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:02.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:02.221 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:02.243 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:02.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:02.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:02.256 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:00:02.256 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:00:02.297 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:00:02.298 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:00:02.330 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:00:02.330 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:00:02.345 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:02.367 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:02.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:02.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:02.371 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:00:02.371 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:00:02.391 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T18:00:02.403 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:00:02.403 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:00:02.434 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:00:02.434 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:00:02.453 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 28 16 48518 48502 108.251 87.1875 0.00877392 0.00923092 2024-08-17T18:00:02.465 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:02.469 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:02.473 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:00:02.473 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:00:02.504 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:00:02.504 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:00:02.529 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:02.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:02.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:02.632 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:02.656 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:02.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:02.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:02.757 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:02.780 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:02.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:02.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:02.882 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:02.932 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:02.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:02.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:03.034 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:03.058 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:03.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:03.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:03.160 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:03 smithi130 ceph-mon[105521]: pgmap v2027: 89 pgs: 8 peering, 81 active+clean; 2.6 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T18:00:03.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:03 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2400145985' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:03 smithi130 ceph-mon[110068]: pgmap v2027: 89 pgs: 8 peering, 81 active+clean; 2.6 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T18:00:03.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:03 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2400145985' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:03.200 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:03.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:03.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:03.302 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:03.324 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:03.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:03.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:03 smithi179 ceph-mon[93519]: pgmap v2027: 89 pgs: 8 peering, 81 active+clean; 2.6 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T18:00:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:03 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2400145985' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:03.427 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:03.450 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 29 16 49882 49866 107.457 85.25 0.00713785 0.0092991 2024-08-17T18:00:03.461 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:03.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:03.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:03.563 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:03.587 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:03.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:03.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:03.688 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:03.730 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:03.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:03.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:03.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:03.873 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:03.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:03.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:03.975 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:04.023 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:04.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:04.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:04.125 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:04.149 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:04.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:04.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:04.163 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:04.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:04 smithi130 ceph-mon[105521]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-08-17T18:00:04.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:04 smithi130 ceph-mon[110068]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-08-17T18:00:04.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:04.299 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:04.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:04.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:04 smithi179 ceph-mon[93519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-08-17T18:00:04.401 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:04.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:04.439 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:04.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:04.450 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 30 16 51281 51265 106.79 87.4375 0.0072896 0.00935848 2024-08-17T18:00:04.541 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:04.582 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:04.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:04.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:04.684 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:04.707 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:04.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:04.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:04.808 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:04.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:04.862 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:04.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:04.963 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:04.988 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:04.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:04.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:05.090 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:05.129 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:05.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:05.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:05.230 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:05.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:05.267 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:05.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:05.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:05 smithi179 ceph-mon[93519]: pgmap v2028: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 94 MiB/s wr, 1.50k op/s; 24 KiB/s, 0 objects/s recovering 2024-08-17T18:00:05.346 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:00:05.368 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:05.384 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:05 smithi130 ceph-mon[105521]: pgmap v2028: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 94 MiB/s wr, 1.50k op/s; 24 KiB/s, 0 objects/s recovering 2024-08-17T18:00:05.384 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:05 smithi130 ceph-mon[110068]: pgmap v2028: 89 pgs: 89 active+clean; 3.0 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 94 MiB/s wr, 1.50k op/s; 24 KiB/s, 0 objects/s recovering 2024-08-17T18:00:05.422 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:05.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:05.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:05.450 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 31 16 52630 52614 106.065 84.3125 0.00908368 0.00942141 2024-08-17T18:00:05.524 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:05.548 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:05.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:05.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:05.649 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:05.671 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:05.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:05.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:05.773 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:05.800 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:05.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:05.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:05.901 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:05.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:05.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:05.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:06.030 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:06.076 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:06.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:06.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:06.180 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:06.204 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:06.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:06.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:06.306 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:06.317 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:06 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:00:06.347 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:06.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:06.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:06.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:06 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:00:06.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:06 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:00:06.449 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:06.451 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 32 16 53833 53817 105.099 75.1875 0.00236535 0.00950427 2024-08-17T18:00:06.470 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:06.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:06.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:06.572 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:06.606 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:06.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:06.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:06.708 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:06.743 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:06.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:06.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:06.845 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:06.870 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:06.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:06.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:06.972 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:06.988 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7] out_osds: [5, 4] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T18:00:06.988 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:00:06.988 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T18:00:06.988 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.970250 2024-08-17T18:00:06.989 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 6 0.9702495392048367 2024-08-17T18:00:06.994 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:06.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:06.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:07.095 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:07.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:07.139 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:07.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:07.240 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:07.261 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:07.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:07.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:07.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:07 smithi179 ceph-mon[93519]: pgmap v2029: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s; 24 KiB/s, 0 objects/s recovering 2024-08-17T18:00:07.363 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:07.408 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:07.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:07.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:07.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:07 smithi130 ceph-mon[105521]: pgmap v2029: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s; 24 KiB/s, 0 objects/s recovering 2024-08-17T18:00:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:07 smithi130 ceph-mon[110068]: pgmap v2029: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s; 24 KiB/s, 0 objects/s recovering 2024-08-17T18:00:07.451 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 33 16 54850 54834 103.841 63.5625 0.0120167 0.00962472 2024-08-17T18:00:07.511 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:07.550 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:07.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:07.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:07.652 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:07.680 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:07.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:07.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:07.782 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:07.806 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:07.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:07.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:07.908 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:07.947 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:07.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:07.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:08.049 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:08.075 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:08.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:08.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:08.111 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:00:08.177 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:08.199 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:08.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:08.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:08.300 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:08.343 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:08.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:08.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:08.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:08 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2732418541' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:00:08.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:08 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:00:08.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:08.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:08 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2732418541' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:00:08.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:08 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:00:08.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:08.445 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:08.451 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 34 16 56111 56095 103.104 78.8125 0.00979658 0.00969325 2024-08-17T18:00:08.482 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:08.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:08.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:08.584 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:08 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2732418541' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:00:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:08 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:00:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:08.621 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:08.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:08.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:08.723 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:08.760 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:08.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:08.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:08.862 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:08.888 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:08.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:08.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:08.990 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:09.018 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:09.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:09.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:09.119 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:09.159 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:09.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:09.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:09.201 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:09.261 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:09.284 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:09.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:09.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:09 smithi179 ceph-mon[93519]: pgmap v2030: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 19 KiB/s, 0 objects/s recovering 2024-08-17T18:00:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:09 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:00:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:09 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:00:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:09 smithi179 ceph-mon[93519]: osdmap e1265: 8 total, 8 up, 6 in 2024-08-17T18:00:09.375 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:09 smithi130 ceph-mon[105521]: pgmap v2030: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 19 KiB/s, 0 objects/s recovering 2024-08-17T18:00:09.375 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:09 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:00:09.375 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:09 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:00:09.375 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:09 smithi130 ceph-mon[105521]: osdmap e1265: 8 total, 8 up, 6 in 2024-08-17T18:00:09.378 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:09 smithi130 ceph-mon[110068]: pgmap v2030: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 19 KiB/s, 0 objects/s recovering 2024-08-17T18:00:09.378 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:09 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:00:09.378 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:09 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:00:09.378 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:09 smithi130 ceph-mon[110068]: osdmap e1265: 8 total, 8 up, 6 in 2024-08-17T18:00:09.386 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:09.409 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:09.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:09.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:09.451 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 35 16 57382 57366 102.427 79.4375 0.0115193 0.00975737 2024-08-17T18:00:09.511 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:09.550 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:09.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:09.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:09.652 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:09.679 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:09.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:09.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:09.781 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:09.833 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:00:09.834 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:00:09.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:09.850 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:09.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:09.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:00:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:00:09] "GET /metrics HTTP/1.1" 200 36776 "" "Prometheus/2.51.0" 2024-08-17T18:00:09.951 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:09.991 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:09.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:09.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:10.093 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:10.132 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:10.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:10.233 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:10.257 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:10.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:10.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:10.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:00:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:00:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:00:10.359 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:10.399 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:10.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:10.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:10.451 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 36 16 58488 58472 101.502 69.125 0.0128569 0.00984658 2024-08-17T18:00:10.502 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:10.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:10.529 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:10.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:10.630 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:10.655 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:10.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:10.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:10.756 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:10.789 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:10.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:10.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:10.891 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:10.918 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:10.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:10.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:11.020 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:11.058 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:11.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:11.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:11.160 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:11.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:11 smithi130 ceph-mon[105521]: pgmap v2032: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 19 KiB/s, 0 objects/s recovering 2024-08-17T18:00:11.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:11 smithi130 ceph-mon[110068]: pgmap v2032: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 19 KiB/s, 0 objects/s recovering 2024-08-17T18:00:11.181 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:11.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:11.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:11.235 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:00:11.283 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:11.349 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:11.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:11.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:11.451 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:11.452 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 37 16 59419 59403 100.331 58.1875 0.0931172 0.00995529 2024-08-17T18:00:11.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:11.494 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:11.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:11.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:11 smithi179 ceph-mon[93519]: pgmap v2032: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 19 KiB/s, 0 objects/s recovering 2024-08-17T18:00:11.594 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:11.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:11.644 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:11.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:11.745 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:11.768 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:11.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:11.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:11.869 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:11.892 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:11.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:11.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:11.994 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:12.022 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:12.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:12.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:12.127 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:12.161 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:12.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:12.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:12.263 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:12.300 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:12.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:12.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:12.401 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:12.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:12.443 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:12.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:12.451 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 38 16 60528 60512 99.5145 69.3125 0.00676677 0.0100437 2024-08-17T18:00:12.544 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:12.581 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:12.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:12.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:12.683 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:12.722 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:12.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:12.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:12.824 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:12.867 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:12.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:12.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:12.969 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:13.005 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:13.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:13.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:13.107 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:13.130 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:13.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:13.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:13.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:13.254 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:13.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:13.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:13.355 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:13.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:13.382 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:13.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:13.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:13 smithi130 ceph-mon[105521]: pgmap v2033: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 19 KiB/s, 0 objects/s recovering 2024-08-17T18:00:13.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:13.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:13.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:00:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:13 smithi130 ceph-mon[110068]: pgmap v2033: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 19 KiB/s, 0 objects/s recovering 2024-08-17T18:00:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:00:13.452 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 39 16 61567 61551 98.6277 64.9375 0.0238555 0.0101337 2024-08-17T18:00:13.483 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:13.506 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:13.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:13.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:13 smithi179 ceph-mon[93519]: pgmap v2033: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 78 MiB/s wr, 1.25k op/s; 19 KiB/s, 0 objects/s recovering 2024-08-17T18:00:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:00:13.608 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:13.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:13.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:13.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:13.757 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:13.778 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T18:00:13.780 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:13.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:13.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:13.882 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:13.905 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:13.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:13.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:14.007 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:14.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:14.054 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:14.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:14.155 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:14.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:14.178 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:14.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:14.215 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:14.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:14.304 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:14.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:14.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:14.406 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:14.430 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:14.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:14.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:14.452 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:00:14.452173+0000 min lat: 0.00128899 max lat: 1.04903 avg lat: 0.0102031 2024-08-17T18:00:14.452 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:00:14.452 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 40 16 62715 62699 97.9557 71.75 0.0127967 0.0102031 2024-08-17T18:00:14.533 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:14.558 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:14.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:14.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:14 smithi179 ceph-mon[93519]: pgmap v2034: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-08-17T18:00:14.660 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:14.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:14 smithi130 ceph-mon[105521]: pgmap v2034: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-08-17T18:00:14.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:14 smithi130 ceph-mon[110068]: pgmap v2034: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-08-17T18:00:14.686 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:14.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:14.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:14.789 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:14.859 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:14.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:14.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:14.961 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:14.983 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:14.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:14.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:15.085 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:15.108 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:15.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:15.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:15.210 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:15.250 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:15.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:15.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:15.352 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:15.376 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:15.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:15.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:15.452 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 41 16 63911 63895 97.3895 74.75 0.0168261 0.0102629 2024-08-17T18:00:15.478 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:15.502 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:15.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:15.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:15.604 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:15.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:15.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:15.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:15.758 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:15.781 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:15.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:15.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:15.883 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:15.924 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:15.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:15.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:16.026 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:16.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:16.061 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:16.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:16.162 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:16.199 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:16.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:16.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:16.300 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:16.323 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:16.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:16.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:16.401 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:16 smithi130 ceph-mon[105521]: pgmap v2035: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T18:00:16.401 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:16 smithi130 ceph-mon[110068]: pgmap v2035: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T18:00:16.426 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:16.451 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:16.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:16.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:16.452 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 42 16 65102 65086 96.8429 74.4375 0.00826793 0.010321 2024-08-17T18:00:16.553 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:16.592 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:16 smithi179 ceph-mon[93519]: pgmap v2035: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T18:00:16.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:16.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:16.694 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:16.717 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:16.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:16.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:16.751 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:00:16.751 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:00:16.751 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:00:16.751 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:00:16.751 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:00:16.751 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:00:16.757 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:00:16.757 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:00:16.786 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:00:16.786 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:00:16.818 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:00:16.818 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:00:16.819 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:16.850 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:00:16.850 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:00:16.855 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:16.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:16.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:16.884 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:00:16.885 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:00:16.957 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:16.997 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:16.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:16.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:17.099 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:17.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:17.148 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:17.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:17.249 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:17.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:17.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:17.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:17.340 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:17 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/631040000' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:17.340 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:17 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/631040000' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:17.376 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:17.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:17.419 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:17.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:17.452 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 43 15 66207 66192 96.1982 69.125 0.00826002 0.0103894 2024-08-17T18:00:17.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:17 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/631040000' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:17.520 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:17.559 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:17.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:17.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:17.661 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:17.688 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:17.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:17.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:17.790 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:17.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:17.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:17.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:17.895 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:17.924 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:17.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:17.965 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:17.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:18.066 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:18.090 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:18.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:18.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:18.192 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:18.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:18.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:18.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:18.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:18.353 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:18.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:18.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:18.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:18 smithi130 ceph-mon[105521]: pgmap v2036: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T18:00:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:18 smithi130 ceph-mon[110068]: pgmap v2036: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T18:00:18.452 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 44 16 67170 67154 95.3781 60.125 0.00261752 0.0104604 2024-08-17T18:00:18.455 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:18.475 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:18.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:18.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:18.577 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:18 smithi179 ceph-mon[93519]: pgmap v2036: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T18:00:18.598 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:18.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:18.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:18.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:18.741 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:18.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:18.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:18.843 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:18.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:18.868 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:18.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:18.969 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:19.008 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:19.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:19.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:19.110 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:19.147 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:19.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:19.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:19.249 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:19.273 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:19.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:19.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:19.375 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:19.412 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:19.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:19.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:19.452 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 45 16 68092 68076 94.539 57.625 0.00764341 0.0105726 2024-08-17T18:00:19.515 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:19.537 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:19.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:19.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:19.639 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:19.663 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:19.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:19.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:19.765 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:19.808 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:19.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:19.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:19.909 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:19.936 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:19.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:19.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:20.007 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:00:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:00:19] "GET /metrics HTTP/1.1" 200 36794 "" "Prometheus/2.51.0" 2024-08-17T18:00:20.038 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:20.061 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:20.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:20.163 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:20.199 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:20.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:20.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:20.269 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:20.276 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:20 smithi130 ceph-mon[105521]: pgmap v2037: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T18:00:20.277 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:20 smithi130 ceph-mon[110068]: pgmap v2037: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T18:00:20.301 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:20.361 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:20.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:20.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:20.453 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 46 16 69213 69197 94.0067 70.0625 0.00783529 0.0106325 2024-08-17T18:00:20.463 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:20.516 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:20.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:20.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:20 smithi179 ceph-mon[93519]: pgmap v2037: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T18:00:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:00:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:00:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:00:20.617 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:20.640 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:20.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:20.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:20.741 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:20.780 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:20.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:20.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:20.882 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:20.927 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:20.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:20.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:21.029 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:21.052 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:21.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:21.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:21.154 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:21.188 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:21.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:21.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:21.291 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:21.344 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:21.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:21.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:21.446 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:21.453 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 47 16 70259 70243 93.3958 65.375 0.0256607 0.0107004 2024-08-17T18:00:21.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:21.491 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:21.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:21.592 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:21.618 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:21.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:21.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:21.720 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:21.777 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:21.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:21.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:21.878 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:21.903 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:21.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:21.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:22.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:22.007 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:00:22.029 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:22.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:22.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:22.131 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:22.212 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:22.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:22.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:22.240 INFO:teuthology.orchestra.run.smithi130.stderr:set osd.6 primary-affinity to 0.97025 (8635862) 2024-08-17T18:00:22.307 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:22 smithi130 ceph-mon[105521]: pgmap v2038: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 59 MiB/s wr, 940 op/s 2024-08-17T18:00:22.308 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:22 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/506740445' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.9702495392048367} : dispatch 2024-08-17T18:00:22.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:22 smithi130 ceph-mon[110068]: pgmap v2038: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 59 MiB/s wr, 940 op/s 2024-08-17T18:00:22.308 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:22 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/506740445' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.9702495392048367} : dispatch 2024-08-17T18:00:22.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:22.378 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:22.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:22.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:22.455 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 48 16 71223 71207 92.7033 60.25 0.0103441 0.0107819 2024-08-17T18:00:22.480 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:22.504 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:22.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:22.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:22.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:22 smithi179 ceph-mon[93519]: pgmap v2038: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 59 MiB/s wr, 940 op/s 2024-08-17T18:00:22.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:22 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/506740445' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 0.9702495392048367} : dispatch 2024-08-17T18:00:22.605 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:22.628 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:22.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:22.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:22.730 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:22.753 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:22.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:22.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:22.855 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:22.879 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:22.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:22.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:22.980 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:23.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:23.028 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:23.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:23.129 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:23.153 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:23.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:23.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:23.255 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:23.280 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:23.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:23.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:23.382 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:23.406 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:23.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:23.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:23.455 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 49 16 72312 72296 92.2002 68.0625 0.00273533 0.0108366 2024-08-17T18:00:23.508 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:23.547 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:23.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:23.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:23 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/506740445' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.9702495392048367}]': finished 2024-08-17T18:00:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:23 smithi179 ceph-mon[93519]: osdmap e1266: 8 total, 8 up, 6 in 2024-08-17T18:00:23.648 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:23.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:23 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/506740445' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.9702495392048367}]': finished 2024-08-17T18:00:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:23 smithi130 ceph-mon[110068]: osdmap e1266: 8 total, 8 up, 6 in 2024-08-17T18:00:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:23 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/506740445' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.9702495392048367}]': finished 2024-08-17T18:00:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:23 smithi130 ceph-mon[105521]: osdmap e1266: 8 total, 8 up, 6 in 2024-08-17T18:00:23.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:23.689 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:23.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:23.790 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:23.814 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:23.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:23.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:23.916 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:23.940 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:23.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:23.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:24.043 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:24.072 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:24.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:24.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:24.173 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:24.197 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:24.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:24.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:24.299 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:24.339 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:24.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:24.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:24.441 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:24.457 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 50 16 73310 73294 91.5998 62.375 0.0184227 0.0109091 2024-08-17T18:00:24.463 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:24.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:24.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:24.565 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:24.589 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:24.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:24.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:24 smithi179 ceph-mon[93519]: pgmap v2040: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T18:00:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:24 smithi130 ceph-mon[105521]: pgmap v2040: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T18:00:24.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:24 smithi130 ceph-mon[110068]: pgmap v2040: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T18:00:24.690 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:24.732 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:24.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:24.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:24.834 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:24.875 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:24.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:24.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:24.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:25.019 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:25.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:25.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:25.120 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:25.144 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:25.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:25.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:25.245 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:25.288 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:25.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:25.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:25.390 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:25.433 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:25.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:25.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:25.457 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 51 16 74328 74312 91.0512 63.625 0.00702407 0.0109772 2024-08-17T18:00:25.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:25.559 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:25.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:25.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:25.660 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:25.684 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:25.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:25.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:25.787 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:25.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:25.811 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:25.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:25.912 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:25.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:25.978 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:25.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:26.079 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:26.103 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:26.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:26.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:26.204 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:26.228 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:26.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:26.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:26.268 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:00:26.286 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:26 smithi130 ceph-mon[105521]: pgmap v2041: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-08-17T18:00:26.286 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:26 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/250443770' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:00:26.286 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:26 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:00:26.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:26 smithi130 ceph-mon[110068]: pgmap v2041: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-08-17T18:00:26.286 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:26 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/250443770' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:00:26.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:26 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:00:26.330 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:26.389 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:26.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:26.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:26.457 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 52 16 75376 75360 90.5598 65.5 0.00543115 0.0110368 2024-08-17T18:00:26.491 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:26.536 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:26.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:26.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:26 smithi179 ceph-mon[93519]: pgmap v2041: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-08-17T18:00:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:26 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/250443770' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:00:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:26 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:00:26.638 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:26.661 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:26.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:26.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:26.763 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:26.803 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:26.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:26.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:26.859 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:00:26.859 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:00:26.859 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:00:26.859 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:00:26.859 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.859 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:123156 45.d 45:b1bbc672:::benchmark_data_smithi130_172551_object18349:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1204)", 2024-08-17T18:00:26.859 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:51:24.170478+0000", 2024-08-17T18:00:26.859 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 542.63065160400004, 2024-08-17T18:00:26.859 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0378718060000001, 2024-08-17T18:00:26.859 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 123156 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.170478+0000", 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.860 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.861 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.170478+0000", 2024-08-17T18:00:26.861 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.861 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.861 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.861 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.861 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.170480+0000", 2024-08-17T18:00:26.861 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9369999999999998e-06 2024-08-17T18:00:26.861 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.861 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.861 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.861 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.170487+0000", 2024-08-17T18:00:26.861 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3969999999999997e-06 2024-08-17T18:00:26.861 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.862 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.862 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.862 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.170501+0000", 2024-08-17T18:00:26.862 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3906e-05 2024-08-17T18:00:26.862 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.862 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.862 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.862 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.170509+0000", 2024-08-17T18:00:26.862 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5979999999999999e-06 2024-08-17T18:00:26.862 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.862 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.862 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.862 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.194407+0000", 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023897966 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.194556+0000", 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014946600000000001 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.194699+0000", 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000142614 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.863 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.200425+0000", 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0057257460000000003 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.200435+0000", 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0519999999999999e-05 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.208269+0000", 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.007834267 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.208350+0000", 2024-08-17T18:00:26.864 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0389000000000006e-05 2024-08-17T18:00:26.865 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.865 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.865 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.865 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.865 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.865 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:123159 45.d 45:b2552ba5:::benchmark_data_smithi130_172551_object18352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1205)", 2024-08-17T18:00:26.865 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:51:24.616417+0000", 2024-08-17T18:00:26.865 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 542.18471276000002, 2024-08-17T18:00:26.865 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.59829702900000004, 2024-08-17T18:00:26.865 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.865 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.865 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.865 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:00:26.865 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:00:26.866 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:00:26.866 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 123159 2024-08-17T18:00:26.866 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.866 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.866 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.866 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.866 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616417+0000", 2024-08-17T18:00:26.866 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.866 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.866 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.866 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.866 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616417+0000", 2024-08-17T18:00:26.866 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.866 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.867 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.867 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.867 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616418+0000", 2024-08-17T18:00:26.867 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.561e-06 2024-08-17T18:00:26.867 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.867 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.867 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.867 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616425+0000", 2024-08-17T18:00:26.867 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5950000000000004e-06 2024-08-17T18:00:26.867 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.867 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.867 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.867 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616439+0000", 2024-08-17T18:00:26.867 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3927000000000001e-05 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616446+0000", 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6810000000000001e-06 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616515+0000", 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9352999999999998e-05 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.616527+0000", 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2452e-05 2024-08-17T18:00:26.868 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.208825+0000", 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.59229791399999998 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.208935+0000", 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010940100000000001 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.209032+0000", 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7205000000000002e-05 2024-08-17T18:00:26.869 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.870 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.870 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.870 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214543+0000", 2024-08-17T18:00:26.870 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005510923 2024-08-17T18:00:26.870 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.870 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.870 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:00:26.870 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214577+0000", 2024-08-17T18:00:26.870 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4337e-05 2024-08-17T18:00:26.870 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.870 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.870 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.870 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214620+0000", 2024-08-17T18:00:26.870 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3101e-05 2024-08-17T18:00:26.871 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.871 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.871 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.871 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214670+0000", 2024-08-17T18:00:26.871 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9857000000000001e-05 2024-08-17T18:00:26.871 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.871 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.871 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.871 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214714+0000", 2024-08-17T18:00:26.872 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3721999999999999e-05 2024-08-17T18:00:26.872 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.872 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.872 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.872 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.872 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.872 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:123165 45.d 45:b3c02af0:::benchmark_data_smithi130_172551_object18358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1205)", 2024-08-17T18:00:26.872 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:51:24.775611+0000", 2024-08-17T18:00:26.872 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 542.02551808500004, 2024-08-17T18:00:26.872 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.442593493, 2024-08-17T18:00:26.872 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.872 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.872 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.873 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:00:26.873 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:00:26.873 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:00:26.873 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 123165 2024-08-17T18:00:26.873 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.873 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.873 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.873 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.873 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775611+0000", 2024-08-17T18:00:26.873 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.873 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.873 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.873 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.874 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775611+0000", 2024-08-17T18:00:26.874 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.874 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.874 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.874 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.874 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775613+0000", 2024-08-17T18:00:26.874 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.536e-06 2024-08-17T18:00:26.874 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.874 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.874 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.874 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775619+0000", 2024-08-17T18:00:26.874 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9959999999999999e-06 2024-08-17T18:00:26.875 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.875 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.875 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.875 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775633+0000", 2024-08-17T18:00:26.875 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4225e-05 2024-08-17T18:00:26.875 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.875 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.875 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.875 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775641+0000", 2024-08-17T18:00:26.875 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.391e-06 2024-08-17T18:00:26.875 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.875 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.875 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.876 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775698+0000", 2024-08-17T18:00:26.876 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7506000000000001e-05 2024-08-17T18:00:26.876 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.876 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.876 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:00:26.876 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.775707+0000", 2024-08-17T18:00:26.876 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.687e-06 2024-08-17T18:00:26.876 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.876 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.876 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.876 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.212049+0000", 2024-08-17T18:00:26.876 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.436342444 2024-08-17T18:00:26.876 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.212137+0000", 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8209999999999997e-05 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.212229+0000", 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1496999999999995e-05 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214764+0000", 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025347389999999998 2024-08-17T18:00:26.877 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.878 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.878 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.878 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218082+0000", 2024-08-17T18:00:26.878 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0033186040000000002 2024-08-17T18:00:26.878 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.878 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.878 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:00:26.878 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218097+0000", 2024-08-17T18:00:26.878 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4748e-05 2024-08-17T18:00:26.878 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.878 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.878 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.878 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218154+0000", 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7399999999999999e-05 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218205+0000", 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0510000000000003e-05 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:123166 45.d 45:b8803e29:::benchmark_data_smithi130_172551_object18359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1205)", 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:51:24.792323+0000", 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 542.00880622299997, 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.42608478500000002, 2024-08-17T18:00:26.879 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.880 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.880 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.880 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:00:26.880 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:00:26.880 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:00:26.880 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 123166 2024-08-17T18:00:26.880 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.880 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.880 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.880 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.880 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792323+0000", 2024-08-17T18:00:26.880 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.880 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.881 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.881 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.881 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792323+0000", 2024-08-17T18:00:26.881 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.881 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.881 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.881 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.881 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792325+0000", 2024-08-17T18:00:26.881 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.903e-06 2024-08-17T18:00:26.881 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.881 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.881 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.881 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792331+0000", 2024-08-17T18:00:26.881 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0929999999999999e-06 2024-08-17T18:00:26.882 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.882 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.882 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.882 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792345+0000", 2024-08-17T18:00:26.882 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3763e-05 2024-08-17T18:00:26.882 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.882 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.882 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.882 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792352+0000", 2024-08-17T18:00:26.882 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3939999999999998e-06 2024-08-17T18:00:26.882 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.882 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.882 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792425+0000", 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2426000000000001e-05 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.792437+0000", 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1905e-05 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214813+0000", 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.42237641500000001 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.883 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.214895+0000", 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1496999999999996e-05 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.215021+0000", 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012648100000000001 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218257+0000", 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032360790000000002 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.884 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218313+0000", 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5809e-05 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218327+0000", 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3713999999999999e-05 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218371+0000", 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3788000000000002e-05 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.885 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.886 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218408+0000", 2024-08-17T18:00:26.886 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7518000000000002e-05 2024-08-17T18:00:26.886 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.886 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.886 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.886 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.886 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.886 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:123176 45.d 45:b4b9fb89:::benchmark_data_smithi130_172551_object18369:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1205)", 2024-08-17T18:00:26.886 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:51:24.872841+0000", 2024-08-17T18:00:26.886 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 541.92828883000004, 2024-08-17T18:00:26.886 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35194215600000001, 2024-08-17T18:00:26.886 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.886 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.887 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.887 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:00:26.887 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:00:26.887 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:00:26.887 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 123176 2024-08-17T18:00:26.887 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.887 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.887 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.887 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.887 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872841+0000", 2024-08-17T18:00:26.887 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.887 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.887 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.887 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.888 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872841+0000", 2024-08-17T18:00:26.888 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.888 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.888 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.888 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.888 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872843+0000", 2024-08-17T18:00:26.888 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9350000000000001e-06 2024-08-17T18:00:26.888 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.888 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.888 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.888 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872847+0000", 2024-08-17T18:00:26.888 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8339999999999998e-06 2024-08-17T18:00:26.888 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.888 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.889 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.889 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872858+0000", 2024-08-17T18:00:26.889 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0899e-05 2024-08-17T18:00:26.889 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.889 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.889 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.889 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872865+0000", 2024-08-17T18:00:26.889 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3149999999999997e-06 2024-08-17T18:00:26.889 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.889 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.889 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.889 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872917+0000", 2024-08-17T18:00:26.889 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2454999999999998e-05 2024-08-17T18:00:26.890 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.890 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.890 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:00:26.890 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:24.872926+0000", 2024-08-17T18:00:26.890 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1560000000000001e-06 2024-08-17T18:00:26.890 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.890 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.890 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.890 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218467+0000", 2024-08-17T18:00:26.890 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.34554088599999999 2024-08-17T18:00:26.890 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.890 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.890 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.890 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218550+0000", 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2532000000000002e-05 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.218662+0000", 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000112627 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.224602+0000", 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0059394620000000004 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.891 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.224661+0000", 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9670000000000003e-05 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.224677+0000", 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5271e-05 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.224733+0000", 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6258999999999998e-05 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.892 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:51:25.224783+0000", 2024-08-17T18:00:26.893 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9855000000000003e-05 2024-08-17T18:00:26.893 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.893 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.893 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.893 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.893 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.893 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30108 46.6 e1262/1258 46:61535681:::benchmark_data_smithi130_267720_object30107:head v 1262'1902, mlcod=1262'1900)", 2024-08-17T18:00:26.893 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.622400+0000", 2024-08-17T18:00:26.893 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 38.178729814999997, 2024-08-17T18:00:26.893 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.39093171999999998, 2024-08-17T18:00:26.893 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.893 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.893 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.893 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.894 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.894 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.894 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622400+0000", 2024-08-17T18:00:26.894 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.894 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.894 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.894 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.894 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622400+0000", 2024-08-17T18:00:26.894 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.894 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.894 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.894 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622400+0000", 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9100000000000003e-07 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622436+0000", 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5828999999999999e-05 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622446+0000", 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4800000000000007e-06 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.895 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622452+0000", 2024-08-17T18:00:26.896 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0700000000000003e-06 2024-08-17T18:00:26.896 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.896 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.896 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.896 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622736+0000", 2024-08-17T18:00:26.896 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00028467899999999998 2024-08-17T18:00:26.896 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.896 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.896 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.896 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622753+0000", 2024-08-17T18:00:26.896 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6099e-05 2024-08-17T18:00:26.896 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.896 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.896 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.897 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.628849+0000", 2024-08-17T18:00:26.897 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0060961069999999999 2024-08-17T18:00:26.897 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.897 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.897 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.897 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013331+0000", 2024-08-17T18:00:26.897 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.384482765 2024-08-17T18:00:26.897 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.897 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.897 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.897 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.897 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.897 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30097 46.9 e1262/1258 46:9fdfcdfb:::benchmark_data_smithi130_267720_object30096:head v 1262'1885, mlcod=1262'1880)", 2024-08-17T18:00:26.897 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.623801+0000", 2024-08-17T18:00:26.898 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 38.177328250000002, 2024-08-17T18:00:26.898 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.389256359, 2024-08-17T18:00:26.898 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.898 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.898 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.898 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.898 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.898 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.898 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623801+0000", 2024-08-17T18:00:26.898 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.898 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.898 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.898 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.898 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623801+0000", 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623802+0000", 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4000000000000002e-07 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623824+0000", 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1685e-05 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.899 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623833+0000", 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3940000000000002e-06 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623840+0000", 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0169999999999999e-06 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.624711+0000", 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00087102700000000002 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.900 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.624729+0000", 2024-08-17T18:00:26.901 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8536999999999999e-05 2024-08-17T18:00:26.901 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.901 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.901 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.901 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013015+0000", 2024-08-17T18:00:26.901 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38828545599999997 2024-08-17T18:00:26.901 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.901 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.901 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.901 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013058+0000", 2024-08-17T18:00:26.901 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2703000000000001e-05 2024-08-17T18:00:26.901 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.901 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.901 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.902 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.902 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.902 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30110 46.e e1262/1258 46:7442b1f6:::benchmark_data_smithi130_267720_object30109:head v 1262'1868, mlcod=1262'1866)", 2024-08-17T18:00:26.902 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.625942+0000", 2024-08-17T18:00:26.902 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 38.175187426000001, 2024-08-17T18:00:26.902 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38829237100000003, 2024-08-17T18:00:26.902 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.902 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.902 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.902 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.902 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.902 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.902 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625942+0000", 2024-08-17T18:00:26.902 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.903 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.903 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.903 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.903 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625942+0000", 2024-08-17T18:00:26.903 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.903 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.903 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.903 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.903 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625942+0000", 2024-08-17T18:00:26.903 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3500000000000002e-07 2024-08-17T18:00:26.903 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.903 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625969+0000", 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6052000000000001e-05 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625976+0000", 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4019999999999996e-06 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625981+0000", 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4489999999999998e-06 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.904 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.905 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.905 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.626277+0000", 2024-08-17T18:00:26.905 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00029525500000000002 2024-08-17T18:00:26.905 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.905 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.905 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.905 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.626293+0000", 2024-08-17T18:00:26.905 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6812000000000001e-05 2024-08-17T18:00:26.905 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.905 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.905 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.905 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.014152+0000", 2024-08-17T18:00:26.905 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38785813499999999 2024-08-17T18:00:26.906 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.906 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.906 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.906 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.014234+0000", 2024-08-17T18:00:26.906 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2930999999999996e-05 2024-08-17T18:00:26.906 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.906 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.906 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.906 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.906 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.906 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30111 46.8 e1262/1258 46:189f8eaf:::benchmark_data_smithi130_267720_object30110:head v 1262'1896, mlcod=1262'1894)", 2024-08-17T18:00:26.906 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.627403+0000", 2024-08-17T18:00:26.906 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 38.173726840999997, 2024-08-17T18:00:26.906 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38543158300000002, 2024-08-17T18:00:26.907 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.907 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.907 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.907 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.907 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.907 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.907 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627403+0000", 2024-08-17T18:00:26.907 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.907 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.907 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.907 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.907 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627403+0000", 2024-08-17T18:00:26.907 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.907 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.908 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.908 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.908 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627403+0000", 2024-08-17T18:00:26.908 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1900000000000002e-07 2024-08-17T18:00:26.908 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.908 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.908 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.908 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627430+0000", 2024-08-17T18:00:26.908 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6347999999999999e-05 2024-08-17T18:00:26.908 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.908 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.908 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.908 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627439+0000", 2024-08-17T18:00:26.908 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0710000000000001e-06 2024-08-17T18:00:26.909 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.909 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.909 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.909 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627445+0000", 2024-08-17T18:00:26.909 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8850000000000004e-06 2024-08-17T18:00:26.909 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.909 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.909 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.909 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627583+0000", 2024-08-17T18:00:26.909 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000138002 2024-08-17T18:00:26.909 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.909 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.909 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.909 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627602+0000", 2024-08-17T18:00:26.910 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8913e-05 2024-08-17T18:00:26.910 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.910 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.910 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.910 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012774+0000", 2024-08-17T18:00:26.910 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38517220000000002 2024-08-17T18:00:26.910 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.910 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.910 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.910 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012834+0000", 2024-08-17T18:00:26.910 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0445000000000001e-05 2024-08-17T18:00:26.910 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.910 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30106 46.9 e1262/1258 46:95064b20:::benchmark_data_smithi130_267720_object30105:head v 1262'1886, mlcod=1262'1883)", 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.627466+0000", 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 38.173663331999997, 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38562097299999998, 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627466+0000", 2024-08-17T18:00:26.911 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.912 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.912 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.912 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.912 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627466+0000", 2024-08-17T18:00:26.912 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.912 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.912 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.912 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.912 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627467+0000", 2024-08-17T18:00:26.912 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3700000000000001e-07 2024-08-17T18:00:26.912 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.912 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.912 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.912 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627489+0000", 2024-08-17T18:00:26.913 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2473999999999999e-05 2024-08-17T18:00:26.913 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.913 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.913 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.913 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627497+0000", 2024-08-17T18:00:26.913 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4699999999999996e-06 2024-08-17T18:00:26.913 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.913 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.913 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.913 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627502+0000", 2024-08-17T18:00:26.913 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0579999999999997e-06 2024-08-17T18:00:26.913 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.913 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.913 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.914 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627874+0000", 2024-08-17T18:00:26.914 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00037268400000000003 2024-08-17T18:00:26.914 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.914 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.914 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.914 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627892+0000", 2024-08-17T18:00:26.914 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7900000000000001e-05 2024-08-17T18:00:26.914 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.914 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.914 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.914 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013063+0000", 2024-08-17T18:00:26.914 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38517096099999998 2024-08-17T18:00:26.914 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013087+0000", 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3989e-05 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30116 46.e e1262/1258 46:74110d7a:::benchmark_data_smithi130_267720_object30115:head v 1262'1869, mlcod=1262'1866)", 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.627778+0000", 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 38.173351597999996, 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38918829500000002, 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.915 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.916 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.916 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.916 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.916 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.916 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627778+0000", 2024-08-17T18:00:26.916 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.916 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.916 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.916 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.916 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627778+0000", 2024-08-17T18:00:26.916 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.916 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.916 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627779+0000", 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2200000000000003e-07 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627826+0000", 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7339999999999997e-05 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627835+0000", 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6549999999999993e-06 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.917 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.918 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627840+0000", 2024-08-17T18:00:26.918 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4299999999999997e-06 2024-08-17T18:00:26.918 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.918 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.918 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.918 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.628737+0000", 2024-08-17T18:00:26.918 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00089676600000000001 2024-08-17T18:00:26.918 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.918 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.918 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.918 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.628754+0000", 2024-08-17T18:00:26.918 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7533999999999998e-05 2024-08-17T18:00:26.918 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.919 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.919 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.919 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016632+0000", 2024-08-17T18:00:26.919 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38787752599999997 2024-08-17T18:00:26.919 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.919 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.919 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.919 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016966+0000", 2024-08-17T18:00:26.919 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00033422200000000001 2024-08-17T18:00:26.919 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.919 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.919 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.919 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.920 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.920 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:30123 46.5 46:ad019766:::benchmark_data_smithi130_267720_object30122:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:00:26.920 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.629381+0000", 2024-08-17T18:00:26.920 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 38.171748264000001, 2024-08-17T18:00:26.920 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38780136100000001, 2024-08-17T18:00:26.920 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.920 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.920 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.920 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:00:26.920 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:00:26.920 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:00:26.920 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 30123 2024-08-17T18:00:26.920 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.920 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629381+0000", 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629381+0000", 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629383+0000", 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.525e-06 2024-08-17T18:00:26.921 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.922 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.922 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.922 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629433+0000", 2024-08-17T18:00:26.922 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0513999999999998e-05 2024-08-17T18:00:26.922 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.922 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.922 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.922 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012724+0000", 2024-08-17T18:00:26.922 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38329109099999997 2024-08-17T18:00:26.922 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.922 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.922 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.922 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012741+0000", 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6629999999999998e-05 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012903+0000", 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016237399999999999 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013070+0000", 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016684100000000001 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:00:26.923 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013223+0000", 2024-08-17T18:00:26.924 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015269900000000001 2024-08-17T18:00:26.924 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.924 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.924 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:00:26.924 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.014685+0000", 2024-08-17T18:00:26.924 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0014624869999999999 2024-08-17T18:00:26.924 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.924 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.924 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.924 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.015642+0000", 2024-08-17T18:00:26.924 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00095662199999999996 2024-08-17T18:00:26.924 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.924 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.924 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016944+0000", 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0013019869999999999 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.017105+0000", 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016044900000000001 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.017183+0000", 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8141999999999998e-05 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.925 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.926 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.926 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30122 46.c e1262/1258 46:3a3a1109:::benchmark_data_smithi130_267720_object30121:head v 1262'1891, mlcod=1262'1889)", 2024-08-17T18:00:26.926 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.629974+0000", 2024-08-17T18:00:26.926 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 38.171155818000003, 2024-08-17T18:00:26.926 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38593537500000002, 2024-08-17T18:00:26.926 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.926 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.926 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.926 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.926 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.926 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.926 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629974+0000", 2024-08-17T18:00:26.926 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.927 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.927 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.927 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.927 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629974+0000", 2024-08-17T18:00:26.927 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.927 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.927 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.927 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.927 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629974+0000", 2024-08-17T18:00:26.927 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2500000000000005e-07 2024-08-17T18:00:26.927 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.927 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.927 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.927 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.630007+0000", 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2543999999999999e-05 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012721+0000", 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38271419800000001 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012740+0000", 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8525e-05 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.928 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012886+0000", 2024-08-17T18:00:26.929 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014585800000000001 2024-08-17T18:00:26.929 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.929 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.929 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.929 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012909+0000", 2024-08-17T18:00:26.929 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.332e-05 2024-08-17T18:00:26.929 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.929 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.929 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.929 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.015868+0000", 2024-08-17T18:00:26.929 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029595070000000001 2024-08-17T18:00:26.929 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.929 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.929 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.930 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.015909+0000", 2024-08-17T18:00:26.930 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0698000000000002e-05 2024-08-17T18:00:26.930 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.930 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.930 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.930 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.930 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.930 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30128 46.e e1262/1258 46:7b0313e1:::benchmark_data_smithi130_267720_object30127:head v 1262'1870, mlcod=1262'1866)", 2024-08-17T18:00:26.930 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.633214+0000", 2024-08-17T18:00:26.930 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 38.167915794000002, 2024-08-17T18:00:26.930 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.384036457, 2024-08-17T18:00:26.930 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.930 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.930 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.931 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.931 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.931 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.931 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.633214+0000", 2024-08-17T18:00:26.931 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.931 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.931 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.931 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.931 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.633214+0000", 2024-08-17T18:00:26.931 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.931 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.931 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.931 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.931 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.633215+0000", 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1599999999999999e-06 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.633239+0000", 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4559e-05 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012733+0000", 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.379493731 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.932 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012744+0000", 2024-08-17T18:00:26.933 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.061e-05 2024-08-17T18:00:26.933 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.933 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.933 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.933 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.014260+0000", 2024-08-17T18:00:26.933 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001516414 2024-08-17T18:00:26.933 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.933 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.933 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.933 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.014277+0000", 2024-08-17T18:00:26.933 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6629e-05 2024-08-17T18:00:26.933 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.933 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.934 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.934 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.017168+0000", 2024-08-17T18:00:26.934 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028907759999999998 2024-08-17T18:00:26.934 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.934 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.934 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.934 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.017250+0000", 2024-08-17T18:00:26.934 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2577999999999994e-05 2024-08-17T18:00:26.934 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.934 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.934 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.934 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.934 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.934 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:31329 46.5 46:a82a9ff3:::benchmark_data_smithi130_267720_object31328:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:00:26.935 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:49.792134+0000", 2024-08-17T18:00:26.935 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 37.008995151000001, 2024-08-17T18:00:26.935 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.125990458, 2024-08-17T18:00:26.935 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.935 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.935 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.935 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:00:26.935 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:00:26.935 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:00:26.935 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 31329 2024-08-17T18:00:26.935 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.935 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.935 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.792134+0000", 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.792134+0000", 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.792136+0000", 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5799999999999999e-06 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.936 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.937 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.792158+0000", 2024-08-17T18:00:26.937 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2138999999999999e-05 2024-08-17T18:00:26.937 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.937 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.937 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.937 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.792170+0000", 2024-08-17T18:00:26.937 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1455e-05 2024-08-17T18:00:26.937 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.937 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.937 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.937 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.792177+0000", 2024-08-17T18:00:26.937 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0310000000000001e-06 2024-08-17T18:00:26.937 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.937 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.938 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.938 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.792234+0000", 2024-08-17T18:00:26.938 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7442999999999999e-05 2024-08-17T18:00:26.938 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.938 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.938 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.938 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.792364+0000", 2024-08-17T18:00:26.938 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013035400000000001 2024-08-17T18:00:26.938 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.938 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.938 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:00:26.938 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.792487+0000", 2024-08-17T18:00:26.938 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012217599999999999 2024-08-17T18:00:26.939 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.939 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.939 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:00:26.939 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.795343+0000", 2024-08-17T18:00:26.939 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028568439999999999 2024-08-17T18:00:26.939 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.939 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.939 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.939 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.795397+0000", 2024-08-17T18:00:26.939 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3260999999999997e-05 2024-08-17T18:00:26.939 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.939 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.939 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.939 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.918023+0000", 2024-08-17T18:00:26.940 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.12262649 2024-08-17T18:00:26.940 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.940 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.940 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.940 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.918080+0000", 2024-08-17T18:00:26.940 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7250000000000002e-05 2024-08-17T18:00:26.940 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.940 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.940 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.940 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.918125+0000", 2024-08-17T18:00:26.940 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4434999999999997e-05 2024-08-17T18:00:26.940 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.940 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.941 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.941 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.941 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.941 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:31335 46.5 46:a3cebb84:::benchmark_data_smithi130_267720_object31334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:00:26.941 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:49.794046+0000", 2024-08-17T18:00:26.941 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 37.007083067000003, 2024-08-17T18:00:26.941 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.133552857, 2024-08-17T18:00:26.941 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.941 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.941 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.941 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:00:26.941 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:00:26.941 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:00:26.942 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 31335 2024-08-17T18:00:26.942 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.942 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.942 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.942 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.942 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.794046+0000", 2024-08-17T18:00:26.942 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.942 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.942 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.942 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.942 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.794046+0000", 2024-08-17T18:00:26.942 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.943 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.943 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.943 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.943 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.794047+0000", 2024-08-17T18:00:26.943 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6800000000000009e-07 2024-08-17T18:00:26.943 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.943 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.943 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.943 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.794093+0000", 2024-08-17T18:00:26.943 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5407000000000001e-05 2024-08-17T18:00:26.943 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.943 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.943 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.944 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.794106+0000", 2024-08-17T18:00:26.944 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3524e-05 2024-08-17T18:00:26.944 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.944 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.944 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.944 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.794115+0000", 2024-08-17T18:00:26.944 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5900000000000008e-06 2024-08-17T18:00:26.944 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.944 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.944 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.944 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.795429+0000", 2024-08-17T18:00:26.944 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001313788 2024-08-17T18:00:26.944 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.944 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.945 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.945 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.795542+0000", 2024-08-17T18:00:26.945 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000113464 2024-08-17T18:00:26.945 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.945 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.945 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:00:26.945 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.795658+0000", 2024-08-17T18:00:26.945 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011582 2024-08-17T18:00:26.945 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.945 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.945 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:00:26.945 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.797528+0000", 2024-08-17T18:00:26.945 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0018699509999999999 2024-08-17T18:00:26.945 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.946 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.946 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.946 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.797701+0000", 2024-08-17T18:00:26.946 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017315000000000001 2024-08-17T18:00:26.946 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.946 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.946 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.946 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.918319+0000", 2024-08-17T18:00:26.946 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.120617921 2024-08-17T18:00:26.946 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.946 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.946 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.946 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.918359+0000", 2024-08-17T18:00:26.947 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9740000000000002e-05 2024-08-17T18:00:26.947 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.947 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.947 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.947 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.927599+0000", 2024-08-17T18:00:26.947 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0092405339999999999 2024-08-17T18:00:26.947 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.947 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.947 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.947 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.947 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.947 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:31343 46.5 46:a9c71dd4:::benchmark_data_smithi130_267720_object31342:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:00:26.947 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:49.801248+0000", 2024-08-17T18:00:26.948 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 36.999881166000002, 2024-08-17T18:00:26.948 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.12757489599999999, 2024-08-17T18:00:26.948 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.948 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.948 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.948 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:00:26.948 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:00:26.948 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:00:26.948 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 31343 2024-08-17T18:00:26.948 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.948 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.948 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.948 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.948 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.801248+0000", 2024-08-17T18:00:26.949 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.949 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.949 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.949 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.949 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.801248+0000", 2024-08-17T18:00:26.949 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.949 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.949 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.949 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.949 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.801250+0000", 2024-08-17T18:00:26.949 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.142e-06 2024-08-17T18:00:26.949 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.949 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.801268+0000", 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8349999999999999e-05 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.801277+0000", 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8040000000000007e-06 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.801282+0000", 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9339999999999997e-06 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.950 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.951 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.951 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.801332+0000", 2024-08-17T18:00:26.951 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0438000000000001e-05 2024-08-17T18:00:26.951 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.951 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.951 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.951 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.801423+0000", 2024-08-17T18:00:26.951 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0969000000000003e-05 2024-08-17T18:00:26.951 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.951 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.952 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:00:26.952 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.801502+0000", 2024-08-17T18:00:26.952 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8763000000000005e-05 2024-08-17T18:00:26.952 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.952 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.952 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:00:26.952 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.802491+0000", 2024-08-17T18:00:26.952 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00098964600000000006 2024-08-17T18:00:26.952 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.952 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.952 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.952 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.802742+0000", 2024-08-17T18:00:26.953 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00025081099999999998 2024-08-17T18:00:26.953 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.953 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.953 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.953 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.928715+0000", 2024-08-17T18:00:26.953 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.12597279 2024-08-17T18:00:26.953 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.953 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.953 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.953 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.928769+0000", 2024-08-17T18:00:26.953 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3698e-05 2024-08-17T18:00:26.954 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.954 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.954 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.954 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.928823+0000", 2024-08-17T18:00:26.954 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4551e-05 2024-08-17T18:00:26.954 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.954 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.954 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.954 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.954 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.954 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:67153 46.5 46:a0791987:::benchmark_data_smithi130_267720_object67152:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:00:26.954 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:18.363721+0000", 2024-08-17T18:00:26.955 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 8.4374084699999994, 2024-08-17T18:00:26.955 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.268888927, 2024-08-17T18:00:26.955 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.955 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.955 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.955 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:00:26.955 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:00:26.955 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:00:26.955 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 67153 2024-08-17T18:00:26.955 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.955 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.955 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.955 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.956 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.363721+0000", 2024-08-17T18:00:26.956 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:26.957 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.957 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.957 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.957 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.957 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.363721+0000", 2024-08-17T18:00:26.957 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.957 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.958 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.958 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.958 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.363723+0000", 2024-08-17T18:00:26.958 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0530000000000001e-06 2024-08-17T18:00:26.958 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.958 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.958 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.958 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.363748+0000", 2024-08-17T18:00:26.958 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4856000000000001e-05 2024-08-17T18:00:26.958 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.958 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.958 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.959 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.363760+0000", 2024-08-17T18:00:26.959 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2021000000000001e-05 2024-08-17T18:00:26.959 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.959 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.959 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.959 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.363768+0000", 2024-08-17T18:00:26.959 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1480000000000006e-06 2024-08-17T18:00:26.959 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.959 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.959 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.959 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.363919+0000", 2024-08-17T18:00:26.959 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015129799999999999 2024-08-17T18:00:26.959 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.959 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.960 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.960 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364109+0000", 2024-08-17T18:00:26.960 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000189648 2024-08-17T18:00:26.960 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.960 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.960 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:00:26.960 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364313+0000", 2024-08-17T18:00:26.960 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020383599999999999 2024-08-17T18:00:26.960 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.960 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.960 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:00:26.960 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.371469+0000", 2024-08-17T18:00:26.960 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0071559889999999998 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.371507+0000", 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8126000000000001e-05 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.632354+0000", 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26084694899999999 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.632505+0000", 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000150796 2024-08-17T18:00:26.961 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.962 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.962 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.962 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.632610+0000", 2024-08-17T18:00:26.962 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000105207 2024-08-17T18:00:26.962 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.962 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.962 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.962 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.962 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.962 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:67155 46.5 46:a92f1b4b:::benchmark_data_smithi130_267720_object67154:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:00:26.962 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:18.364358+0000", 2024-08-17T18:00:26.962 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 8.4367711019999998, 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26850078500000002, 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 67155 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364358+0000", 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.963 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364358+0000", 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364359+0000", 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5499999999999997e-07 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364379+0000", 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0270000000000001e-05 2024-08-17T18:00:26.964 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.965 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.965 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.965 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364388+0000", 2024-08-17T18:00:26.965 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3040000000000005e-06 2024-08-17T18:00:26.965 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.965 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.965 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.965 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364393+0000", 2024-08-17T18:00:26.965 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5509999999999999e-06 2024-08-17T18:00:26.965 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.965 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.965 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.965 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.366719+0000", 2024-08-17T18:00:26.966 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002325457 2024-08-17T18:00:26.966 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.966 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.966 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.966 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.366860+0000", 2024-08-17T18:00:26.966 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000141528 2024-08-17T18:00:26.966 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.966 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.966 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:00:26.966 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.366996+0000", 2024-08-17T18:00:26.966 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013588900000000001 2024-08-17T18:00:26.966 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.966 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.967 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:00:26.967 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.371518+0000", 2024-08-17T18:00:26.967 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0045220360000000001 2024-08-17T18:00:26.967 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.967 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.967 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.967 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.371683+0000", 2024-08-17T18:00:26.967 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016451399999999999 2024-08-17T18:00:26.967 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.967 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.967 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.967 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.632648+0000", 2024-08-17T18:00:26.967 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.260965216 2024-08-17T18:00:26.967 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.968 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.968 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.968 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.632762+0000", 2024-08-17T18:00:26.968 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011415 2024-08-17T18:00:26.968 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.968 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.968 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.968 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.632859+0000", 2024-08-17T18:00:26.968 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7114999999999994e-05 2024-08-17T18:00:26.968 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.968 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.968 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.968 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.968 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:67159 46.5 46:a582ac5b:::benchmark_data_smithi130_267720_object67158:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:18.365333+0000", 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 8.4357960970000008, 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26777571900000002, 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 67159 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.969 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:00:26.970 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.365333+0000", 2024-08-17T18:00:26.970 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.970 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.970 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.970 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:00:26.970 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.365333+0000", 2024-08-17T18:00:26.970 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:00:26.970 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.970 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.970 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:00:26.970 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.365335+0000", 2024-08-17T18:00:26.970 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.189e-06 2024-08-17T18:00:26.970 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.971 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.971 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:00:26.971 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.365353+0000", 2024-08-17T18:00:26.971 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8051999999999999e-05 2024-08-17T18:00:26.971 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.971 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.971 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:00:26.971 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.365363+0000", 2024-08-17T18:00:26.971 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0190999999999999e-05 2024-08-17T18:00:26.971 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.971 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.971 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:00:26.971 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.365369+0000", 2024-08-17T18:00:26.972 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4230000000000002e-06 2024-08-17T18:00:26.972 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.972 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.972 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:00:26.972 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.369140+0000", 2024-08-17T18:00:26.972 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0037702059999999999 2024-08-17T18:00:26.972 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.972 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.972 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:00:26.972 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.369253+0000", 2024-08-17T18:00:26.972 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011332600000000001 2024-08-17T18:00:26.972 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.972 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.973 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:00:26.973 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.369372+0000", 2024-08-17T18:00:26.973 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000119064 2024-08-17T18:00:26.973 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.973 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.973 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:00:26.973 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.371724+0000", 2024-08-17T18:00:26.973 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0023522460000000001 2024-08-17T18:00:26.973 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.973 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.973 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.973 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.372854+0000", 2024-08-17T18:00:26.973 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0011297360000000001 2024-08-17T18:00:26.973 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.974 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.974 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:26.974 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.632908+0000", 2024-08-17T18:00:26.974 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26005367400000001 2024-08-17T18:00:26.974 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.974 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.974 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:00:26.974 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.633025+0000", 2024-08-17T18:00:26.974 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011793 2024-08-17T18:00:26.974 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:00:26.974 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:00:26.974 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:00:26.974 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.633109+0000", 2024-08-17T18:00:26.975 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3682000000000003e-05 2024-08-17T18:00:26.975 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.975 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.975 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.975 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:00:26.975 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:00:26.975 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:00:26.978 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:26.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:26.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:27.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:27.122 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:27.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:27.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:27.224 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:27.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:27.263 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:27.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:27.364 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:27.403 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:27.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:27.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:27.457 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 53 16 76295 76279 89.9348 57.4375 0.0099817 0.0111128 2024-08-17T18:00:27.506 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:27.529 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:27.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:27.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:27 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:00:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:27 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:00:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:27 smithi179 ceph-mon[93519]: osdmap e1267: 8 total, 8 up, 6 in 2024-08-17T18:00:27.631 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:27.655 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:27.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:27.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:27 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:00:27.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:27 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:00:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:27 smithi130 ceph-mon[105521]: osdmap e1267: 8 total, 8 up, 6 in 2024-08-17T18:00:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:27 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:00:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:27 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:00:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:27 smithi130 ceph-mon[110068]: osdmap e1267: 8 total, 8 up, 6 in 2024-08-17T18:00:27.756 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:27.799 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:27.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:27.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:27.900 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:27.946 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:27.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:27.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:28.047 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:28.070 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:28.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:28.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:28.172 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:28.202 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:28.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:28.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:28.304 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:28.330 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:28.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:28.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:28.431 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:28.457 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 54 15 77366 77351 89.51 67 0.00921857 0.0111658 2024-08-17T18:00:28.472 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:28.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:28.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:28.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:28.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:28 smithi179 ceph-mon[93519]: pgmap v2043: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 746 op/s 2024-08-17T18:00:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:00:28.599 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:28.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:28.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:28 smithi130 ceph-mon[105521]: pgmap v2043: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 746 op/s 2024-08-17T18:00:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:00:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:28 smithi130 ceph-mon[110068]: pgmap v2043: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 47 MiB/s wr, 746 op/s 2024-08-17T18:00:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:00:28.701 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:28.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:28.727 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:28.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:28.828 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:28.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:28.852 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:28.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:28.953 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:28.982 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:28.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:28.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:29.084 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:29.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:29.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:29.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:29.210 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:29.251 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:29.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:29.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:29.353 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:29.379 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:29.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:29.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:29.457 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 55 16 78420 78404 89.0791 65.8125 0.0111991 0.0112197 2024-08-17T18:00:29.481 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:29.538 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:29.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:29.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:29.640 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:29.663 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:29.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:29.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:29.756 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:29.765 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:29.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:29.803 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:29.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:29.904 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:29.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:29.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:29.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:30.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:30.088 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:00:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:00:29] "GET /metrics HTTP/1.1" 200 36775 "" "Prometheus/2.51.0" 2024-08-17T18:00:30.097 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:30.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:30.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:30.111 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7] out_osds: [5, 4] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T18:00:30.112 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T18:00:30.199 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:30.238 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:30.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:30.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:30.340 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:30.363 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:30.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:30.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:30.377 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:30 smithi130 ceph-mon[105521]: pgmap v2044: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-17T18:00:30.377 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:30 smithi130 ceph-mon[110068]: pgmap v2044: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-17T18:00:30.457 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 56 16 79372 79356 88.5508 59.5 0.0140486 0.0112871 2024-08-17T18:00:30.465 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:30.488 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:30.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:30.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:30.590 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:30 smithi179 ceph-mon[93519]: pgmap v2044: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-17T18:00:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:00:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:00:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:00:30.615 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:30.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:30.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:30.717 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:30.738 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:30.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:30.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:30.840 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:30.861 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:30.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:30.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:30.963 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:30.985 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:30.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:30.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:31.087 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:31.112 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:31.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:31.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:31.214 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:31.239 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:31.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:31.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:31.341 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:31.378 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:31.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:31.457 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 57 16 79956 79940 87.6376 36.5 0.00307369 0.0113959 2024-08-17T18:00:31.480 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:31.502 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:00:31.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:31.529 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:31.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:31.630 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:31.672 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:31.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:31.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:31.774 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:31.833 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:31.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:31.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:31.935 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:31.975 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:31.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:32.076 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:32.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:32.117 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:32.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:32.218 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:32.242 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:32.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:32.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:32.344 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:32.365 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:32.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:32.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:32.458 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 58 16 80780 80764 87.0145 51.5 0.0201141 0.0114864 2024-08-17T18:00:32.466 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:32.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:32.488 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:32.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:32.589 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:32 smithi179 ceph-mon[93519]: pgmap v2045: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T18:00:32.613 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:32.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:32.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:32.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:32 smithi130 ceph-mon[105521]: pgmap v2045: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T18:00:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:32 smithi130 ceph-mon[110068]: pgmap v2045: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T18:00:32.715 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:32.739 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:32.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:32.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:32.841 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:32.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:32.909 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:32.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:33.010 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:33.034 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:33.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:33.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:33.136 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:33.179 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:33.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:33.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:33.281 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:33.315 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:33.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:33.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:33.417 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:33.440 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:33.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:33.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:33.458 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 59 16 81794 81778 86.6138 63.375 0.0104207 0.01154 2024-08-17T18:00:33.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:33.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:33.588 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:33.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:33.601 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T18:00:33.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:33.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:33.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:33.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:33.871 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:33.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:33.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:33.918 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:00:33.973 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:34.014 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:34.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:34.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:34.118 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:34.141 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:34.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:34.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:34.243 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:34.267 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:34.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:34.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:34.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:34 smithi179 ceph-mon[93519]: pgmap v2046: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 60 MiB/s wr, 956 op/s 2024-08-17T18:00:34.369 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:34.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:34.419 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:34.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:34.458 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:00:34.458102+0000 min lat: 0.00128899 max lat: 1.04903 avg lat: 0.0115974 2024-08-17T18:00:34.458 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:00:34.458 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 60 16 82765 82749 86.1816 60.6875 0.00906098 0.0115974 2024-08-17T18:00:34.504 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:34.521 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:34.551 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:34 smithi130 ceph-mon[110068]: pgmap v2046: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 60 MiB/s wr, 956 op/s 2024-08-17T18:00:34.551 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:00:34.551 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:00:34.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:34 smithi130 ceph-mon[105521]: pgmap v2046: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 60 MiB/s wr, 956 op/s 2024-08-17T18:00:34.592 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:34.603 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:00:34.603 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:00:34.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:34.610 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:34.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:34.636 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:00:34.636 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:00:34.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:00:34.667 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:00:34.695 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:00:34.695 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:00:34.710 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:34.745 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:00:34.746 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:00:34.777 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:00:34.778 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:00:34.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:34.779 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:34.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:34.805 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:00:34.805 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:00:34.880 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:34.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:34.919 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:34.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:35.020 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:35.047 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:35.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:35.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:35.149 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:35.176 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:35.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:35.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:35.277 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:35.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:35.324 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:35.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:35.426 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:35.448 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:35.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:35.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:35.458 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 61 16 83636 83620 85.6611 54.4375 0.0210352 0.0116682 2024-08-17T18:00:35.543 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:35 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/622167614' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:35.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:35 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/622167614' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:35.550 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:35.565 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:35 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/622167614' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:35.573 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:35.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:35.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:35.675 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:35.699 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:35.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:35.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:35.800 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:35.824 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:35.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:35.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:35.925 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:35.968 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:35.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:35.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:36.070 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:36.114 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:36.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:36.216 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:36.239 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:36.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:36.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:36.341 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:36.376 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:36.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:36.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:36.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:36 smithi130 ceph-mon[105521]: pgmap v2047: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 60 MiB/s wr, 955 op/s 2024-08-17T18:00:36.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:36 smithi130 ceph-mon[110068]: pgmap v2047: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 60 MiB/s wr, 955 op/s 2024-08-17T18:00:36.458 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 62 16 84627 84611 85.2784 61.9375 0.00716565 0.0117209 2024-08-17T18:00:36.478 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:36.516 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:36.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:36.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:36 smithi179 ceph-mon[93519]: pgmap v2047: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 60 MiB/s wr, 955 op/s 2024-08-17T18:00:36.617 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:36.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:36.664 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:36.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:36.765 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:36.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:36.820 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:36.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:36.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:36.979 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:36.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:36.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:37.081 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:37.105 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:37.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:37.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:37.206 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:37.225 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:00:37.225 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:00:37.253 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:37.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:37.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:37.355 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:37.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:37.401 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:37.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:37.458 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 63 15 85506 85491 84.7977 55 0.0241889 0.0117867 2024-08-17T18:00:37.503 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:37.525 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:37.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:37.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:37.628 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:37.653 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:37.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:37.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:37.755 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:37.798 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:37.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:37.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:37 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:37 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:37 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:00:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:37 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:00:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:37 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:37.902 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:37 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:37 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:37 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:00:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:37 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:00:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:37 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:37 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:37 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:37.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:37 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:00:37.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:37 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:00:37.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:37 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:37.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:37.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:37.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:37.956 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T18:00:38.031 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:38.080 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:38.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:38.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:38.182 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:38.195 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:00:38.205 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:38.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:38.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:38.290 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:00:38.307 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:38.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:38.346 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:38.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:38.447 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:38.459 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 64 16 86346 86330 84.2904 52.4375 0.0257697 0.0118582 2024-08-17T18:00:38.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:38.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:38.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:38.574 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:38.612 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:38.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:38.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:38.714 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:38.737 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:38.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:38.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:38.776 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:38 smithi179 ceph-mon[93519]: pgmap v2048: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 60 MiB/s wr, 956 op/s 2024-08-17T18:00:38.839 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:38.882 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:38.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:38.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:38.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:38 smithi130 ceph-mon[105521]: pgmap v2048: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 60 MiB/s wr, 956 op/s 2024-08-17T18:00:38.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:38 smithi130 ceph-mon[110068]: pgmap v2048: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 60 MiB/s wr, 956 op/s 2024-08-17T18:00:38.984 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:39.023 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:39.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:39.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:39.125 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:39.150 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:39.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:39.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:39.252 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:39.326 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:39.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:39.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:39.428 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:39.453 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:39.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:39.459 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 65 16 87278 87262 83.8896 58.25 0.0161984 0.0119138 2024-08-17T18:00:39.554 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:39.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:39.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:39.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:39.678 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:39.708 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:39.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:39.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:39.810 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:39.856 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:39.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:39.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:39.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:00:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:00:39] "GET /metrics HTTP/1.1" 200 36775 "" "Prometheus/2.51.0" 2024-08-17T18:00:39.958 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:40.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:40.010 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:40.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:40.111 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:40.136 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:40.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:40.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:40.212 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:00:40.212 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:00:40.238 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:40.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:40.263 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:40.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:40.365 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:40.408 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:40.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:40.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:40.460 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 66 16 88056 88040 83.3553 48.625 0.0152233 0.0119905 2024-08-17T18:00:40.509 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:40.533 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:40.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:40.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:40.555 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:00:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:00:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:00:40.636 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:40.673 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:40.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:40.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:40.775 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:40.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:40.815 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:40.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:40.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:40 smithi179 ceph-mon[93519]: pgmap v2049: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T18:00:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:40 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1550911238' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:00:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:40 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:00:40.915 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:40 smithi130 ceph-mon[105521]: pgmap v2049: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T18:00:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:40 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1550911238' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:00:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:40 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:00:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:40 smithi130 ceph-mon[110068]: pgmap v2049: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-08-17T18:00:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:40 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1550911238' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:00:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:40 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:00:40.922 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:00:40.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:40.952 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:40.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:40.979 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T18:00:41.054 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:41.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:41.087 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:41.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:41.153 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:41.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:41.211 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:41.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:41.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:41.313 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:41.317 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:00:41.338 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:41.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:41.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:41.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:41.460 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 67 16 88806 88790 82.81 46.875 0.0195795 0.0120708 2024-08-17T18:00:41.485 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:41.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:41.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:41.586 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T18:00:41.587 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:41.630 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:41.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:41.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:41.734 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:41.769 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:41 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:00:41.769 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:41 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T18:00:41.769 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:41 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:00:41.770 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:41 smithi179 ceph-mon[93519]: osdmap e1268: 8 total, 8 up, 6 in 2024-08-17T18:00:41.770 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:41 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2565505810' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:00:41.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:41.771 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:41.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:41.872 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:41.895 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:41.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:41.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:41 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:00:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:41 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T18:00:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:41 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:00:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:41 smithi130 ceph-mon[105521]: osdmap e1268: 8 total, 8 up, 6 in 2024-08-17T18:00:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:41 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2565505810' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:00:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:41 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:00:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:41 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T18:00:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:41 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:00:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:41 smithi130 ceph-mon[110068]: osdmap e1268: 8 total, 8 up, 6 in 2024-08-17T18:00:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:41 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2565505810' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:00:41.997 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:42.035 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:42.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:42.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:42.137 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:42.172 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:42.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:42.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:42.274 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:42.300 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:42.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:42.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:42.401 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:42.437 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:42.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:42.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:42.460 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 68 15 89739 89724 82.4506 58.375 0.0305756 0.0121228 2024-08-17T18:00:42.539 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:42.564 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:42.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:42.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:42.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:42 smithi130 ceph-mon[105521]: pgmap v2051: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 57 MiB/s wr, 912 op/s 2024-08-17T18:00:42.663 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:42 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2565505810' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:00:42.663 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:42 smithi130 ceph-mon[105521]: osdmap e1269: 8 total, 8 up, 6 in 2024-08-17T18:00:42.663 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:42.663 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:00:42.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:42 smithi130 ceph-mon[110068]: pgmap v2051: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 57 MiB/s wr, 912 op/s 2024-08-17T18:00:42.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:42 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2565505810' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:00:42.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:42 smithi130 ceph-mon[110068]: osdmap e1269: 8 total, 8 up, 6 in 2024-08-17T18:00:42.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:42.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:00:42.666 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:42.689 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:42.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:42.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:42.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:42.815 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:42.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:42.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:42.840 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:42 smithi179 ceph-mon[93519]: pgmap v2051: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 57 MiB/s wr, 912 op/s 2024-08-17T18:00:42.840 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:42 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2565505810' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:00:42.841 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:42 smithi179 ceph-mon[93519]: osdmap e1269: 8 total, 8 up, 6 in 2024-08-17T18:00:42.841 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:42.841 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:00:42.917 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:42.960 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:42.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:42.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:43.062 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:43.102 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:43.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:43.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:43.204 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:43.243 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:43.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:43.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:43.286 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:00:43.286 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:00:43.345 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:43.378 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:43.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:43.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:43.461 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 69 16 90714 90698 82.1379 60.875 0.0143251 0.0121692 2024-08-17T18:00:43.480 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:43.504 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T18:00:43.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:43.506 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:43.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:43.606 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:43.630 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:43.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:43.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:43.732 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:43.778 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:43.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:43.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:43.880 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:43.903 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:43.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:43.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:44.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:44.033 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:44.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:44.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:44.055 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T18:00:44.135 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:44.165 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:44.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:44.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:44.267 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:44.294 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:44.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:44.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:44.382 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:00:44.395 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:44.424 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:44.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:44.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:44.425 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:00:44.461 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 70 16 91614 91598 81.768 56.25 0.0146045 0.0122236 2024-08-17T18:00:44.522 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:44.526 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:44.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:44.592 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:44.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:44.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:44 smithi130 ceph-mon[105521]: pgmap v2053: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T18:00:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:44 smithi130 ceph-mon[110068]: pgmap v2053: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T18:00:44.693 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:44.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:44.759 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:44.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:44.821 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:44 smithi179 ceph-mon[93519]: pgmap v2053: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T18:00:44.860 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:44.883 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:44.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:44.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:44.985 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:45.024 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:45.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:45.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:45.126 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:45.159 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:45.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:45.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:45.206 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:00:45.206 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:00:45.242 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:00:45.242 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:00:45.261 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:45.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:45.302 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:45.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:45.303 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:00:45.303 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:00:45.358 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:00:45.358 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:00:45.394 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:00:45.395 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:00:45.403 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:45.432 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:00:45.433 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:00:45.444 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:45.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:45.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:45.461 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 71 16 92422 92406 81.3276 50.5 0.0140173 0.0122897 2024-08-17T18:00:45.468 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:00:45.468 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:00:45.500 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:00:45.500 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:00:45.547 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:45.569 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:45.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:45.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:45.671 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:45.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:45.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:45.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:45.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:45.841 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:45.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:45.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:45.885 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:45 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1894646581' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:45.886 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:45 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1894646581' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:45.944 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:45.968 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:45.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:45.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:45.981 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:45 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1894646581' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:46.070 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:46.132 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:46.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:46.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:46.234 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:46.260 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:46.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:46.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:46.261 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:00:46.261 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:00:46.262 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T18:00:46.262 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:00:46.362 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:46.413 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:46.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:46.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:46.461 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 72 16 93346 93330 81.0001 57.75 0.0224373 0.0123406 2024-08-17T18:00:46.515 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:46.538 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:46.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:46.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:46.588 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:46.640 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:46.664 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:46.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:46.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:46.765 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:46.788 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:46.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:46.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:46.872 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:46 smithi130 ceph-mon[105521]: pgmap v2054: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 62 MiB/s wr, 994 op/s 2024-08-17T18:00:46.872 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:46 smithi130 ceph-mon[110068]: pgmap v2054: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 62 MiB/s wr, 994 op/s 2024-08-17T18:00:46.890 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:46.914 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:46.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:46.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:46.944 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:46 smithi179 ceph-mon[93519]: pgmap v2054: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 62 MiB/s wr, 994 op/s 2024-08-17T18:00:46.977 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T18:00:47.016 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:47.037 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:47.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:47.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:47.139 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:47.176 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:47.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:47.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:47.277 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:47.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:47.313 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:47.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:47.342 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:00:47.414 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:47.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:47.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:47.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:47.461 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 73 16 94238 94222 80.6542 55.75 0.016248 0.0123932 2024-08-17T18:00:47.539 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:47.562 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:47.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:47.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:47.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:47.698 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:47.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:47.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:47.801 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:47.849 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:47.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:47.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:47.951 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:47.977 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:47.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:47.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:48.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:48.117 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:48.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:48.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:48.219 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:48.259 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:48.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:48.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:48.361 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:48.384 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:48.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:48.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:48.461 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 74 16 95124 95108 80.3125 55.375 0.00770626 0.0124464 2024-08-17T18:00:48.485 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:48.527 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:48.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:48.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:48.629 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:48.661 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:48.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:48.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:48.762 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:48.788 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:48.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:48.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:48.890 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:48.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:48 smithi130 ceph-mon[105521]: pgmap v2055: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 39 MiB/s wr, 622 op/s 2024-08-17T18:00:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:48 smithi130 ceph-mon[110068]: pgmap v2055: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 39 MiB/s wr, 622 op/s 2024-08-17T18:00:48.922 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:48 smithi179 ceph-mon[93519]: pgmap v2055: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 39 MiB/s wr, 622 op/s 2024-08-17T18:00:48.927 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:48.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:48.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:49.028 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:49.054 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:49.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:49.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:49.156 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:49.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:49.201 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:49.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:49.254 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:00:49.254 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:00:49.254 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:00:49.255 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:00:49.255 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:00:49.302 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:49.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:49.324 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:49.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:49.425 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:49.446 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:49.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:49.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:49.461 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 75 16 95903 95887 79.8908 48.6875 0.00356601 0.012488 2024-08-17T18:00:49.547 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:49.568 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:49.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:49.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:49.670 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:49.703 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:49.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:49.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:49.805 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:49.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:49.843 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:49.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:49.944 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:49.970 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:49.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:49.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:50.003 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T18:00:50.072 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:50.111 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:50.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:50.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:00:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:00:49] "GET /metrics HTTP/1.1" 200 36786 "" "Prometheus/2.51.0" 2024-08-17T18:00:50.213 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:50.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:50.257 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:50.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:50.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:00:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:00:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:00:50.358 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:50.361 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:00:50.386 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:50.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:50.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:50.462 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 76 16 96621 96605 79.4298 44.875 0.0267266 0.0125838 2024-08-17T18:00:50.488 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:50.512 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:50.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:50.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:50.614 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:50.660 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:50.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:50.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:50.762 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:50.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:50.786 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:50.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:50.824 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:50 smithi179 ceph-mon[93519]: pgmap v2056: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T18:00:50.888 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:50.912 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:50.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:50.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:50 smithi130 ceph-mon[105521]: pgmap v2056: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T18:00:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:50 smithi130 ceph-mon[110068]: pgmap v2056: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T18:00:51.014 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:51.037 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:51.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:51.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:51.138 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:51.183 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:51.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:51.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:51.285 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:51.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:51.336 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:51.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:51.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:51.459 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:51.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:51.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:51.461 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 77 16 97467 97451 79.0849 52.875 0.0446383 0.0126385 2024-08-17T18:00:51.561 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:51.582 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:51.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:51.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:51.644 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T18:00:51.683 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:51.706 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:51.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:51.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:51.737 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:51 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2642322764' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:00:51.737 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:51 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2642322764' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:00:51.808 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:51.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:51.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:51.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:51.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:51.979 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:51 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2642322764' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:00:51.990 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:51.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:51.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:52.092 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:52.116 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:52.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:52.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:52.218 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:52.240 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:52.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:52.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:52.246 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:00:52.342 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:52.384 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:52.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:52.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:52.462 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 78 16 98301 98285 78.7392 52.125 0.00695905 0.0126947 2024-08-17T18:00:52.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:52.529 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:52.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:52.620 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:00:52.620 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T18:00:52.620 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T18:00:52.620 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T18:00:52.620 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.620 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1209/1189 cost 1049576)", 2024-08-17T18:00:52.620 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:51:52.969675+0000", 2024-08-17T18:00:52.620 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 539.64727516300002, 2024-08-17T18:00:52.620 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025244299999999998, 2024-08-17T18:00:52.620 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.620 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.621 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.621 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.621 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.621 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.969675+0000", 2024-08-17T18:00:52.621 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.621 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.621 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.621 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.621 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.969675+0000", 2024-08-17T18:00:52.621 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.621 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.621 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.621 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.622 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.969674+0000", 2024-08-17T18:00:52.622 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:00:52.622 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.622 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.622 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.622 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.969687+0000", 2024-08-17T18:00:52.622 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3352e-05 2024-08-17T18:00:52.622 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.622 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.622 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.622 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.969702+0000", 2024-08-17T18:00:52.623 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5099000000000001e-05 2024-08-17T18:00:52.623 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.623 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.623 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.623 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.969708+0000", 2024-08-17T18:00:52.623 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5840000000000002e-06 2024-08-17T18:00:52.623 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.623 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.623 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.623 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.969753+0000", 2024-08-17T18:00:52.624 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5194000000000001e-05 2024-08-17T18:00:52.624 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.624 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.624 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.624 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:51:52.972199+0000", 2024-08-17T18:00:52.624 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024463330000000002 2024-08-17T18:00:52.624 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.624 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.624 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.625 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.625 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.625 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1213/1189 cost 1049576)", 2024-08-17T18:00:52.625 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:52:14.841426+0000", 2024-08-17T18:00:52.625 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 517.77552341399996, 2024-08-17T18:00:52.625 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002875641, 2024-08-17T18:00:52.625 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.625 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.625 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.625 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.625 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.625 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.841426+0000", 2024-08-17T18:00:52.625 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.626 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.626 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.626 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.626 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.841426+0000", 2024-08-17T18:00:52.626 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.626 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.626 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.626 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.626 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.841425+0000", 2024-08-17T18:00:52.626 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:00:52.626 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.626 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.627 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.627 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.841437+0000", 2024-08-17T18:00:52.627 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1294999999999999e-05 2024-08-17T18:00:52.627 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.627 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.627 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.627 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.841450+0000", 2024-08-17T18:00:52.627 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3532e-05 2024-08-17T18:00:52.627 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.627 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.627 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.627 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.841456+0000", 2024-08-17T18:00:52.627 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8510000000000004e-06 2024-08-17T18:00:52.628 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.628 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.628 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.628 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.841498+0000", 2024-08-17T18:00:52.628 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1644999999999999e-05 2024-08-17T18:00:52.628 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.628 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.628 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.628 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:52:14.844302+0000", 2024-08-17T18:00:52.628 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028043579999999999 2024-08-17T18:00:52.628 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.628 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.629 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.629 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.629 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.629 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1218/1189 cost 1049576)", 2024-08-17T18:00:52.629 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:53:15.866535+0000", 2024-08-17T18:00:52.629 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 456.750415062, 2024-08-17T18:00:52.629 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002412342, 2024-08-17T18:00:52.629 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.629 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.629 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.629 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.630 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.630 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:15.866535+0000", 2024-08-17T18:00:52.630 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.630 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.630 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.630 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.630 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:15.866535+0000", 2024-08-17T18:00:52.630 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.630 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.630 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.630 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.630 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:15.866534+0000", 2024-08-17T18:00:52.631 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:00:52.631 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.631 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.631 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.631 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:15.866548+0000", 2024-08-17T18:00:52.631 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4005999999999999e-05 2024-08-17T18:00:52.631 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.631 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.632 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.632 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:15.866561+0000", 2024-08-17T18:00:52.632 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3193000000000001e-05 2024-08-17T18:00:52.632 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.632 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.632 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.632 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:15.866568+0000", 2024-08-17T18:00:52.632 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5950000000000004e-06 2024-08-17T18:00:52.632 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.632 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.632 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.632 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:15.866622+0000", 2024-08-17T18:00:52.633 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.4459000000000002e-05 2024-08-17T18:00:52.633 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.633 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.633 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.633 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:15.868947+0000", 2024-08-17T18:00:52.633 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0023251919999999998 2024-08-17T18:00:52.633 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.633 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.633 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.633 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.633 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.633 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1220/1189 cost 1049576)", 2024-08-17T18:00:52.633 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:53:35.649489+0000", 2024-08-17T18:00:52.633 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 436.96746041199998, 2024-08-17T18:00:52.634 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0010817629999999999, 2024-08-17T18:00:52.634 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.634 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.634 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.634 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.634 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.634 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:35.649489+0000", 2024-08-17T18:00:52.634 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.634 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.634 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.634 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.634 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:35.649489+0000", 2024-08-17T18:00:52.634 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.634 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.635 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.635 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.635 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:35.649488+0000", 2024-08-17T18:00:52.635 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:00:52.635 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.635 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.635 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.635 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:35.649501+0000", 2024-08-17T18:00:52.635 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2798e-05 2024-08-17T18:00:52.635 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.635 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.635 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.635 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:35.649515+0000", 2024-08-17T18:00:52.636 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4148000000000001e-05 2024-08-17T18:00:52.636 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.636 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.636 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.636 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:35.649522+0000", 2024-08-17T18:00:52.636 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4930000000000003e-06 2024-08-17T18:00:52.636 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.636 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.636 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.636 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:35.649563+0000", 2024-08-17T18:00:52.636 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1050999999999997e-05 2024-08-17T18:00:52.636 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.636 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.637 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.637 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:53:35.650571+0000", 2024-08-17T18:00:52.637 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001008543 2024-08-17T18:00:52.637 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.637 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.637 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.637 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.637 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.637 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1222/1189 cost 1049576)", 2024-08-17T18:00:52.637 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:54:04.814620+0000", 2024-08-17T18:00:52.637 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 407.80232952400002, 2024-08-17T18:00:52.637 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00091970799999999998, 2024-08-17T18:00:52.637 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.638 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.638 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.638 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.638 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.638 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:04.814620+0000", 2024-08-17T18:00:52.638 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.638 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.638 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.638 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.638 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:04.814620+0000", 2024-08-17T18:00:52.638 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.638 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.639 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.639 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.639 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:04.814619+0000", 2024-08-17T18:00:52.639 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:00:52.639 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.639 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.639 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.639 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:04.814640+0000", 2024-08-17T18:00:52.639 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0852999999999998e-05 2024-08-17T18:00:52.639 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.639 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.639 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.639 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:04.814664+0000", 2024-08-17T18:00:52.640 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.44e-05 2024-08-17T18:00:52.640 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.640 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.640 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.640 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:04.814675+0000", 2024-08-17T18:00:52.640 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.084e-05 2024-08-17T18:00:52.640 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.640 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.640 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.640 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:04.814830+0000", 2024-08-17T18:00:52.640 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000155153 2024-08-17T18:00:52.640 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.640 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.641 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.641 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:04.815540+0000", 2024-08-17T18:00:52.641 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00071003899999999998 2024-08-17T18:00:52.641 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.641 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.641 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.641 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.641 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.641 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1224/1189 cost 1049576)", 2024-08-17T18:00:52.641 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:54:23.378225+0000", 2024-08-17T18:00:52.641 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 389.23872473400002, 2024-08-17T18:00:52.641 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00088701899999999996, 2024-08-17T18:00:52.641 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.642 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.642 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.642 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.642 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.642 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:23.378225+0000", 2024-08-17T18:00:52.642 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.642 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.642 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.642 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.642 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:23.378225+0000", 2024-08-17T18:00:52.642 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.642 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.642 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.643 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.643 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:23.378223+0000", 2024-08-17T18:00:52.643 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:00:52.643 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.643 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.643 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.643 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:23.378249+0000", 2024-08-17T18:00:52.643 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5466999999999999e-05 2024-08-17T18:00:52.643 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.643 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.643 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.643 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:23.378277+0000", 2024-08-17T18:00:52.643 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8878999999999999e-05 2024-08-17T18:00:52.644 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.644 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.644 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.644 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:23.378290+0000", 2024-08-17T18:00:52.644 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2296999999999999e-05 2024-08-17T18:00:52.644 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.644 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.644 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.644 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:23.378412+0000", 2024-08-17T18:00:52.644 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012243700000000001 2024-08-17T18:00:52.644 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.644 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.644 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.645 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:23.379112+0000", 2024-08-17T18:00:52.645 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00069998599999999999 2024-08-17T18:00:52.645 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.645 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.645 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.645 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.645 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.645 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1226/1189 cost 1049576)", 2024-08-17T18:00:52.645 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:54:44.997626+0000", 2024-08-17T18:00:52.645 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 367.61932411399999, 2024-08-17T18:00:52.645 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0011255869999999999, 2024-08-17T18:00:52.645 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.645 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.646 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.646 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.646 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.646 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:44.997626+0000", 2024-08-17T18:00:52.646 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.646 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.646 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.646 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.646 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:44.997626+0000", 2024-08-17T18:00:52.646 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.646 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.646 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.646 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.647 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:44.997625+0000", 2024-08-17T18:00:52.647 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:00:52.647 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.647 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.647 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.648 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:44.997639+0000", 2024-08-17T18:00:52.648 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4064e-05 2024-08-17T18:00:52.648 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.648 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.648 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.648 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:44.997652+0000", 2024-08-17T18:00:52.648 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2863e-05 2024-08-17T18:00:52.648 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.648 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.649 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.649 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:44.997657+0000", 2024-08-17T18:00:52.649 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9020000000000004e-06 2024-08-17T18:00:52.649 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.649 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.649 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.649 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:44.997736+0000", 2024-08-17T18:00:52.649 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.8498000000000006e-05 2024-08-17T18:00:52.649 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.649 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.649 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.649 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:54:44.998751+0000", 2024-08-17T18:00:52.649 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00101541 2024-08-17T18:00:52.650 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.650 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.650 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.650 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.650 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.650 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1228/1189 cost 1049576)", 2024-08-17T18:00:52.650 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:55:04.258671+0000", 2024-08-17T18:00:52.650 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 348.358278541, 2024-08-17T18:00:52.650 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002642779, 2024-08-17T18:00:52.650 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.650 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.650 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.650 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.651 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.651 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:04.258671+0000", 2024-08-17T18:00:52.651 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.651 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.651 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.651 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.651 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:04.258671+0000", 2024-08-17T18:00:52.651 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.651 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.651 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.651 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.651 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:04.258670+0000", 2024-08-17T18:00:52.651 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:00:52.652 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.652 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.652 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.652 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:04.258684+0000", 2024-08-17T18:00:52.652 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4173e-05 2024-08-17T18:00:52.652 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.652 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.652 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.652 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:04.258698+0000", 2024-08-17T18:00:52.652 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4234e-05 2024-08-17T18:00:52.652 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.652 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.652 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.652 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:04.258704+0000", 2024-08-17T18:00:52.653 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.626e-06 2024-08-17T18:00:52.653 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.653 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.653 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.653 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:04.258749+0000", 2024-08-17T18:00:52.653 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5062999999999999e-05 2024-08-17T18:00:52.653 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.653 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.653 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.653 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:04.261314+0000", 2024-08-17T18:00:52.653 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025651879999999999 2024-08-17T18:00:52.653 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.654 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.654 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.654 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.654 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.654 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:52.655 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1232/1189 cost 1049576)", 2024-08-17T18:00:52.656 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:55:47.190989+0000", 2024-08-17T18:00:52.656 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 305.42596040900003, 2024-08-17T18:00:52.656 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001531003, 2024-08-17T18:00:52.656 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.656 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.656 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.656 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.656 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.656 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:47.190989+0000", 2024-08-17T18:00:52.656 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.656 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.656 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.657 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.657 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:47.190989+0000", 2024-08-17T18:00:52.657 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.657 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.657 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.657 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.657 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:47.190989+0000", 2024-08-17T18:00:52.657 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:00:52.657 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.657 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.657 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.657 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:47.191000+0000", 2024-08-17T18:00:52.657 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1798e-05 2024-08-17T18:00:52.658 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.658 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.658 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.658 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:47.191011+0000", 2024-08-17T18:00:52.658 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.075e-05 2024-08-17T18:00:52.658 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.658 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.658 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.658 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:47.191016+0000", 2024-08-17T18:00:52.658 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7999999999999998e-06 2024-08-17T18:00:52.658 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.658 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.659 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.659 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:47.191067+0000", 2024-08-17T18:00:52.659 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1264999999999998e-05 2024-08-17T18:00:52.659 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.659 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.659 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.659 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:55:47.192520+0000", 2024-08-17T18:00:52.659 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001453235 2024-08-17T18:00:52.659 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.659 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.660 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.660 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.660 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.660 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1235/1189 cost 1049576)", 2024-08-17T18:00:52.660 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:56:25.857116+0000", 2024-08-17T18:00:52.660 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 266.75983376099998, 2024-08-17T18:00:52.661 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00086994600000000002, 2024-08-17T18:00:52.661 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.661 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.661 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.661 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.661 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.661 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:25.857116+0000", 2024-08-17T18:00:52.661 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.661 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.661 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.662 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.662 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:25.857116+0000", 2024-08-17T18:00:52.662 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.662 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.662 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.662 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.662 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:25.857114+0000", 2024-08-17T18:00:52.662 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:00:52.662 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.662 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.662 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.663 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:25.857137+0000", 2024-08-17T18:00:52.663 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.2327000000000002e-05 2024-08-17T18:00:52.663 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.663 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.663 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.663 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:25.857165+0000", 2024-08-17T18:00:52.663 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8751999999999998e-05 2024-08-17T18:00:52.663 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.663 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.663 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.663 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:25.857178+0000", 2024-08-17T18:00:52.663 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2087e-05 2024-08-17T18:00:52.664 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.664 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.664 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.664 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:25.857291+0000", 2024-08-17T18:00:52.664 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000113001 2024-08-17T18:00:52.664 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.664 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.664 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.664 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:25.857986+0000", 2024-08-17T18:00:52.664 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00069554199999999995 2024-08-17T18:00:52.664 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.664 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.664 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.665 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.665 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.665 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1235/1189 cost 1049576)", 2024-08-17T18:00:52.665 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:56:27.399078+0000", 2024-08-17T18:00:52.665 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 265.21787198200002, 2024-08-17T18:00:52.665 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00086405599999999998, 2024-08-17T18:00:52.665 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.665 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.665 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.665 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.665 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.665 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:27.399078+0000", 2024-08-17T18:00:52.665 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.666 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.666 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.666 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.666 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:27.399078+0000", 2024-08-17T18:00:52.666 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.666 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.666 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.666 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.666 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:27.399076+0000", 2024-08-17T18:00:52.666 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:00:52.666 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.667 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.667 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.667 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:27.399098+0000", 2024-08-17T18:00:52.667 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.1756e-05 2024-08-17T18:00:52.667 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.667 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.667 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.667 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:27.399124+0000", 2024-08-17T18:00:52.667 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.561e-05 2024-08-17T18:00:52.667 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.667 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.667 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.667 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:27.399135+0000", 2024-08-17T18:00:52.668 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1639e-05 2024-08-17T18:00:52.668 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.668 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.668 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.668 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:27.399231+0000", 2024-08-17T18:00:52.668 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5674999999999995e-05 2024-08-17T18:00:52.668 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.668 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.668 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.669 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:27.399942+0000", 2024-08-17T18:00:52.669 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00071091600000000002 2024-08-17T18:00:52.669 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.669 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.669 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.670 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.670 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.670 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1237/1189 cost 1049576)", 2024-08-17T18:00:52.670 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:56:50.732091+0000", 2024-08-17T18:00:52.670 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 241.884858616, 2024-08-17T18:00:52.670 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00087279199999999995, 2024-08-17T18:00:52.670 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.670 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.671 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.671 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.671 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.671 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:50.732091+0000", 2024-08-17T18:00:52.671 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.671 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.671 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.671 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.671 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:50.732091+0000", 2024-08-17T18:00:52.671 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.671 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.671 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.671 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.672 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:50.732089+0000", 2024-08-17T18:00:52.672 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:00:52.672 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.672 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.672 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.672 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:50.732115+0000", 2024-08-17T18:00:52.672 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5242999999999999e-05 2024-08-17T18:00:52.672 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.672 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.672 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.672 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:50.732141+0000", 2024-08-17T18:00:52.672 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.6579000000000001e-05 2024-08-17T18:00:52.672 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:50.732152+0000", 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.096e-05 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:50.732273+0000", 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000121134 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:50.732964+0000", 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00069083299999999996 2024-08-17T18:00:52.673 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.674 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.674 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.674 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.674 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.674 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1237/1189 cost 1049576)", 2024-08-17T18:00:52.674 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:56:52.077257+0000", 2024-08-17T18:00:52.674 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 240.53969336599999, 2024-08-17T18:00:52.674 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00086678899999999997, 2024-08-17T18:00:52.674 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.674 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.674 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.675 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.675 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.675 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:52.077257+0000", 2024-08-17T18:00:52.675 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.675 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.675 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.675 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.675 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:52.077257+0000", 2024-08-17T18:00:52.675 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.675 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.675 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.675 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.675 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:52.077255+0000", 2024-08-17T18:00:52.675 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:52.077275+0000", 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.9899999999999999e-05 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:52.077299+0000", 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4583000000000002e-05 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:52.077310+0000", 2024-08-17T18:00:52.676 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0709e-05 2024-08-17T18:00:52.677 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.677 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.677 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.677 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:52.077436+0000", 2024-08-17T18:00:52.677 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000125625 2024-08-17T18:00:52.677 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.677 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.677 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.677 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:52.078123+0000", 2024-08-17T18:00:52.677 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00068774899999999998 2024-08-17T18:00:52.677 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.677 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.677 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.678 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.678 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.678 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1238/1189 cost 1049576)", 2024-08-17T18:00:52.678 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:56:59.417300+0000", 2024-08-17T18:00:52.678 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 233.19964942499999, 2024-08-17T18:00:52.678 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0017157679999999999, 2024-08-17T18:00:52.678 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.678 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.678 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.678 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.678 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.678 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:59.417300+0000", 2024-08-17T18:00:52.678 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.678 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.679 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.679 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.679 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:59.417300+0000", 2024-08-17T18:00:52.679 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.679 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.679 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.679 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.679 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:59.417299+0000", 2024-08-17T18:00:52.679 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:00:52.679 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.679 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.679 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.679 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:59.417316+0000", 2024-08-17T18:00:52.679 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.6671999999999999e-05 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:59.417329+0000", 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3674e-05 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:59.417353+0000", 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3272999999999999e-05 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:59.417394+0000", 2024-08-17T18:00:52.680 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0864e-05 2024-08-17T18:00:52.681 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.681 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.681 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.681 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:56:59.419016+0000", 2024-08-17T18:00:52.681 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001622643 2024-08-17T18:00:52.681 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.681 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.681 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.681 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.681 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.681 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1238/1189 cost 1049576)", 2024-08-17T18:00:52.681 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:57:00.782107+0000", 2024-08-17T18:00:52.681 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 231.83484254300001, 2024-08-17T18:00:52.682 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00086047200000000004, 2024-08-17T18:00:52.682 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.682 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.682 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.682 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.682 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.682 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:00.782107+0000", 2024-08-17T18:00:52.682 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.682 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.683 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.683 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.683 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:00.782107+0000", 2024-08-17T18:00:52.683 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.683 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.683 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.683 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.683 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:00.782105+0000", 2024-08-17T18:00:52.683 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:00:52.683 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.683 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.683 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.683 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:00.782132+0000", 2024-08-17T18:00:52.684 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.7086000000000001e-05 2024-08-17T18:00:52.684 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.684 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.684 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.684 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:00.782163+0000", 2024-08-17T18:00:52.684 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1041999999999999e-05 2024-08-17T18:00:52.684 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.684 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.684 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.684 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:00.782177+0000", 2024-08-17T18:00:52.684 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3573e-05 2024-08-17T18:00:52.684 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.684 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.684 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.685 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:00.782281+0000", 2024-08-17T18:00:52.685 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000104325 2024-08-17T18:00:52.685 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.685 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.685 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.685 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:00.782968+0000", 2024-08-17T18:00:52.685 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00068653599999999996 2024-08-17T18:00:52.685 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.685 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.685 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.685 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.685 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.685 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1243/1189 cost 1049576)", 2024-08-17T18:00:52.686 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:57:58.860589+0000", 2024-08-17T18:00:52.686 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 173.756361188, 2024-08-17T18:00:52.686 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028000410000000001, 2024-08-17T18:00:52.686 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.686 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.686 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.686 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.686 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.686 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:58.860589+0000", 2024-08-17T18:00:52.686 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.686 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.686 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.686 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.687 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:58.860589+0000", 2024-08-17T18:00:52.687 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.687 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.687 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.687 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.687 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:58.860587+0000", 2024-08-17T18:00:52.687 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:00:52.687 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.687 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.687 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.687 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:58.860603+0000", 2024-08-17T18:00:52.687 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5849e-05 2024-08-17T18:00:52.688 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.688 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.688 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.688 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:58.860622+0000", 2024-08-17T18:00:52.688 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.8817999999999998e-05 2024-08-17T18:00:52.688 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.688 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.689 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.689 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:58.860631+0000", 2024-08-17T18:00:52.689 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4469999999999995e-06 2024-08-17T18:00:52.689 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.689 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.689 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.689 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:58.860691+0000", 2024-08-17T18:00:52.689 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9531999999999999e-05 2024-08-17T18:00:52.689 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.690 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.690 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.690 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:57:58.863389+0000", 2024-08-17T18:00:52.690 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0026978050000000002 2024-08-17T18:00:52.690 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.690 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.690 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.690 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.690 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.690 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1243/1189 cost 1049576)", 2024-08-17T18:00:52.690 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:58:00.141063+0000", 2024-08-17T18:00:52.690 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 172.47588717299999, 2024-08-17T18:00:52.691 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0010751510000000001, 2024-08-17T18:00:52.691 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.691 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.691 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.691 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.691 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.691 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:00.141063+0000", 2024-08-17T18:00:52.691 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.691 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.691 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.691 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.691 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:00.141063+0000", 2024-08-17T18:00:52.691 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.692 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.692 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.692 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.692 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:00.141062+0000", 2024-08-17T18:00:52.692 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:00:52.692 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.692 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.692 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.692 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:00.141075+0000", 2024-08-17T18:00:52.692 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3694e-05 2024-08-17T18:00:52.692 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.693 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.693 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.693 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:00.141089+0000", 2024-08-17T18:00:52.693 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3963e-05 2024-08-17T18:00:52.693 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.693 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.693 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.693 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:00.141095+0000", 2024-08-17T18:00:52.693 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5600000000000001e-06 2024-08-17T18:00:52.693 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.693 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.693 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.693 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:00.141136+0000", 2024-08-17T18:00:52.694 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1459999999999999e-05 2024-08-17T18:00:52.694 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.694 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.694 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.694 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:00.142138+0000", 2024-08-17T18:00:52.694 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00100158 2024-08-17T18:00:52.694 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.694 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.694 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.695 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.695 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.695 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1245/1189 cost 1049576)", 2024-08-17T18:00:52.695 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:58:20.697879+0000", 2024-08-17T18:00:52.695 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 151.91907109300001, 2024-08-17T18:00:52.695 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00080314799999999995, 2024-08-17T18:00:52.696 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.696 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.696 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.696 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.696 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.696 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:20.697879+0000", 2024-08-17T18:00:52.696 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.696 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.696 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.696 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.696 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:20.697879+0000", 2024-08-17T18:00:52.696 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.697 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.697 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.697 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.697 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:20.697878+0000", 2024-08-17T18:00:52.697 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:00:52.697 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.697 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.697 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.697 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:20.697903+0000", 2024-08-17T18:00:52.697 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5714000000000001e-05 2024-08-17T18:00:52.697 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.697 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.697 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.698 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:20.697917+0000", 2024-08-17T18:00:52.698 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3456e-05 2024-08-17T18:00:52.698 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.698 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.698 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.698 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:20.697923+0000", 2024-08-17T18:00:52.698 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1650000000000003e-06 2024-08-17T18:00:52.698 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.698 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.698 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.698 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:20.697975+0000", 2024-08-17T18:00:52.698 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1703000000000003e-05 2024-08-17T18:00:52.698 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.699 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.699 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.699 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:20.698682+0000", 2024-08-17T18:00:52.699 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00070728499999999999 2024-08-17T18:00:52.699 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.699 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.699 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.699 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.699 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.699 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1245/1189 cost 1049576)", 2024-08-17T18:00:52.699 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:58:21.877650+0000", 2024-08-17T18:00:52.699 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 150.739299405, 2024-08-17T18:00:52.699 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00076571600000000005, 2024-08-17T18:00:52.699 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.700 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.700 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.700 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.700 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.700 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:21.877650+0000", 2024-08-17T18:00:52.700 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.700 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.700 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.700 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.700 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:21.877650+0000", 2024-08-17T18:00:52.700 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.701 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.701 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.701 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.701 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:21.877649+0000", 2024-08-17T18:00:52.701 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:00:52.701 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.701 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.701 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.701 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:21.877663+0000", 2024-08-17T18:00:52.701 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3249e-05 2024-08-17T18:00:52.701 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.701 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.701 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.702 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:21.877676+0000", 2024-08-17T18:00:52.702 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3645999999999999e-05 2024-08-17T18:00:52.702 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.702 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.702 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.702 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:21.877682+0000", 2024-08-17T18:00:52.702 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9340000000000003e-06 2024-08-17T18:00:52.702 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.702 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.703 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.703 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:21.877763+0000", 2024-08-17T18:00:52.703 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.1073000000000001e-05 2024-08-17T18:00:52.703 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.703 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.703 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.703 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:21.878416+0000", 2024-08-17T18:00:52.703 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00065279499999999996 2024-08-17T18:00:52.703 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.704 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.704 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.704 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.704 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.704 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "MOSDPGPull(45.1 e1245/1189 cost 1049576)", 2024-08-17T18:00:52.704 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:58:22.339635+0000", 2024-08-17T18:00:52.704 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 150.277314604, 2024-08-17T18:00:52.704 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00074982900000000003, 2024-08-17T18:00:52.704 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:52.704 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:00:52.704 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:52.704 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.704 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:52.705 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:22.339635+0000", 2024-08-17T18:00:52.705 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.705 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.705 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.705 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:52.705 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:22.339635+0000", 2024-08-17T18:00:52.705 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:52.705 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.705 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.705 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:52.705 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:22.339633+0000", 2024-08-17T18:00:52.705 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:00:52.705 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.705 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.706 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:52.706 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:22.339657+0000", 2024-08-17T18:00:52.706 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3723e-05 2024-08-17T18:00:52.706 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.706 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.706 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:52.706 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:22.339685+0000", 2024-08-17T18:00:52.706 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8348999999999999e-05 2024-08-17T18:00:52.706 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.706 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.706 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:52.706 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:22.339697+0000", 2024-08-17T18:00:52.706 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1712999999999999e-05 2024-08-17T18:00:52.707 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.707 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.707 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:52.707 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:22.339797+0000", 2024-08-17T18:00:52.707 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9432000000000006e-05 2024-08-17T18:00:52.707 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:52.707 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:52.707 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:00:52.707 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:22.340385+0000", 2024-08-17T18:00:52.707 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000588545 2024-08-17T18:00:52.707 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.707 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.707 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.708 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:52.708 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:52.708 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:00:52.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:52.711 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:52.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:52.754 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T18:00:52.812 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:52.836 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:52.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:52.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:52 smithi130 ceph-mon[110068]: pgmap v2057: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 54 MiB/s wr, 865 op/s 2024-08-17T18:00:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:52 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2642322764' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T18:00:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:52 smithi130 ceph-mon[110068]: osdmap e1270: 8 total, 8 up, 6 in 2024-08-17T18:00:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:52 smithi130 ceph-mon[105521]: pgmap v2057: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 54 MiB/s wr, 865 op/s 2024-08-17T18:00:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:52 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2642322764' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T18:00:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:52 smithi130 ceph-mon[105521]: osdmap e1270: 8 total, 8 up, 6 in 2024-08-17T18:00:52.938 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:52.970 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:52 smithi179 ceph-mon[93519]: pgmap v2057: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 54 MiB/s wr, 865 op/s 2024-08-17T18:00:52.971 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:52 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2642322764' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T18:00:52.971 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:52 smithi179 ceph-mon[93519]: osdmap e1270: 8 total, 8 up, 6 in 2024-08-17T18:00:52.987 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:52.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:52.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:53.090 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:53.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:53.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:53.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:53.215 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:53.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:53.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:53.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:53.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:53.445 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:53.453 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:53.462 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 79 16 99011 98995 78.3036 44.375 0.0216781 0.0127636 2024-08-17T18:00:53.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:53.485 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:53.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:53.523 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T18:00:53.586 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:53.624 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:53.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:53.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:53.655 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:00:53.726 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:53.772 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:53.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:53.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:53.842 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:00:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:53 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1067311273' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:00:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:53 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:00:53.873 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:53 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1067311273' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:00:53.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:53 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:00:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:53 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1067311273' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:00:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:53 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:00:53.918 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:53.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:53.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:54.020 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:54.043 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:54.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:54.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:54.145 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:54.184 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:54.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:54.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:54.286 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:54.333 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:54.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:54.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:54.435 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:54.460 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:54.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:54.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:54.462 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:00:54.462911+0000 min lat: 0.00128899 max lat: 1.04903 avg lat: 0.0128189 2024-08-17T18:00:54.462 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:00:54.463 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 80 16 99848 99832 77.9787 52.3125 0.0128299 0.0128189 2024-08-17T18:00:54.562 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:54.600 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:54.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:54.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:54.702 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:54.742 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:54.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:54.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:54.844 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:54.869 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:54.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:54.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:54.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:54 smithi130 ceph-mon[105521]: pgmap v2059: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 53 MiB/s wr, 852 op/s 2024-08-17T18:00:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:54 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:00:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:54 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:00:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:54 smithi130 ceph-mon[105521]: osdmap e1271: 8 total, 8 up, 6 in 2024-08-17T18:00:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:54 smithi130 ceph-mon[110068]: pgmap v2059: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 53 MiB/s wr, 852 op/s 2024-08-17T18:00:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:54 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:00:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:54 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:00:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:54 smithi130 ceph-mon[110068]: osdmap e1271: 8 total, 8 up, 6 in 2024-08-17T18:00:54.971 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:55.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:55.018 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:55.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:54 smithi179 ceph-mon[93519]: pgmap v2059: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 53 MiB/s wr, 852 op/s 2024-08-17T18:00:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:54 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:00:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:54 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:00:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:54 smithi179 ceph-mon[93519]: osdmap e1271: 8 total, 8 up, 6 in 2024-08-17T18:00:55.119 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:55.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:55.160 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:55.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:55.261 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:55.286 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:55.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:55.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:55.388 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:55.426 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:55.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:55.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:55.462 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 81 16 100724 100708 77.6919 54.75 0.0186122 0.012865 2024-08-17T18:00:55.528 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:55.553 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:55.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:55.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:55.655 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:55.698 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:55.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:55.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:55.702 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:00:55.702 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T18:00:55.703 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.703 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:100917 46.c 46:3f5282e1:::benchmark_data_smithi130_267720_object100916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1271)", 2024-08-17T18:00:55.703 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:55.676399+0000", 2024-08-17T18:00:55.703 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.023693263999999999, 2024-08-17T18:00:55.703 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.023716816000000002, 2024-08-17T18:00:55.703 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:55.703 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:00:55.703 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:00:55.703 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:00:55.703 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:00:55.703 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 100917 2024-08-17T18:00:55.704 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.704 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:55.704 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.704 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:55.704 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.676399+0000", 2024-08-17T18:00:55.704 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:55.704 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.704 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.704 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:55.704 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.676399+0000", 2024-08-17T18:00:55.704 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:55.704 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.705 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.705 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:55.705 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.676398+0000", 2024-08-17T18:00:55.705 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:00:55.705 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.705 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.705 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:55.705 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.676434+0000", 2024-08-17T18:00:55.705 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.5481999999999997e-05 2024-08-17T18:00:55.705 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.705 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.705 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:55.705 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.676445+0000", 2024-08-17T18:00:55.706 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.153e-05 2024-08-17T18:00:55.706 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.706 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.706 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:55.706 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.676449+0000", 2024-08-17T18:00:55.706 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0840000000000002e-06 2024-08-17T18:00:55.706 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.706 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.706 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:55.706 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.682084+0000", 2024-08-17T18:00:55.706 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0056347960000000001 2024-08-17T18:00:55.707 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.707 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.707 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:00:55.707 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.682228+0000", 2024-08-17T18:00:55.707 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014429800000000001 2024-08-17T18:00:55.707 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.707 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.707 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:00:55.707 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.682373+0000", 2024-08-17T18:00:55.707 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014448099999999999 2024-08-17T18:00:55.707 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.707 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.707 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:00:55.708 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.684953+0000", 2024-08-17T18:00:55.708 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002580284 2024-08-17T18:00:55.708 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.708 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.708 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:00:55.708 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.685956+0000", 2024-08-17T18:00:55.708 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0010026880000000001 2024-08-17T18:00:55.708 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:55.708 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:55.708 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:55.708 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.709 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.709 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.36225.0:100938 46.9 e1271/1258 46:97c134be:::benchmark_data_smithi130_267720_object100937:head v 1271'6224, mlcod=1271'6221)", 2024-08-17T18:00:55.709 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:55.699401+0000", 2024-08-17T18:00:55.709 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.00069092999999999997, 2024-08-17T18:00:55.709 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00076661000000000003, 2024-08-17T18:00:55.709 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:00:55.709 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "started", 2024-08-17T18:00:55.709 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:00:55.709 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.709 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:00:55.709 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.699401+0000", 2024-08-17T18:00:55.709 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:55.709 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.710 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.710 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:00:55.710 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.699401+0000", 2024-08-17T18:00:55.710 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:00:55.710 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.710 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.710 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:00:55.710 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.699400+0000", 2024-08-17T18:00:55.710 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:00:55.710 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.710 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.710 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:00:55.710 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.699458+0000", 2024-08-17T18:00:55.711 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7080000000000002e-05 2024-08-17T18:00:55.711 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.711 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.711 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:00:55.711 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.699468+0000", 2024-08-17T18:00:55.711 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0281e-05 2024-08-17T18:00:55.711 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.711 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.711 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:00:55.711 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.699472+0000", 2024-08-17T18:00:55.711 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2440000000000004e-06 2024-08-17T18:00:55.711 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.711 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.712 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:00:55.712 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.699528+0000", 2024-08-17T18:00:55.712 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5494000000000001e-05 2024-08-17T18:00:55.712 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:00:55.712 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:00:55.712 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:00:55.712 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:55.699551+0000", 2024-08-17T18:00:55.712 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3068999999999999e-05 2024-08-17T18:00:55.712 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:55.712 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:00:55.712 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:55.712 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:00:55.712 INFO:teuthology.orchestra.run.smithi179.stdout: ], 2024-08-17T18:00:55.713 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 2 2024-08-17T18:00:55.713 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:00:55.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:55.828 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:55.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:55.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:55.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:55.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:55.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:55.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:56.052 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:56.074 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:56.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:56.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:56.176 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:56.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:56.208 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:56.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:56.309 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:56.350 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:56.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:56.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:56.418 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T18:00:56.453 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:56.464 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 82 16 101404 101388 77.2626 42.5 0.0158491 0.0129364 2024-08-17T18:00:56.487 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:56.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:56.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:56.590 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:56.615 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:56.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:56.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:56.717 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:56.724 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:00:56.768 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:56.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:56.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:56.869 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:56.905 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:56 smithi130 ceph-mon[105521]: pgmap v2061: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 60 MiB/s wr, 958 op/s 2024-08-17T18:00:56.905 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:56 smithi130 ceph-mon[110068]: pgmap v2061: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 60 MiB/s wr, 958 op/s 2024-08-17T18:00:56.908 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:56.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:56.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:56.993 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:56 smithi179 ceph-mon[93519]: pgmap v2061: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 60 MiB/s wr, 958 op/s 2024-08-17T18:00:57.009 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:00:57.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:57.049 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:57.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:00:57.150 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:57.200 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:57.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:57.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:57.302 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:57.327 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:57.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:57.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:57.428 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:57.465 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 83 16 102287 102271 76.9946 55.1875 0.0187833 0.0129823 2024-08-17T18:00:57.475 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:57.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:57.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:57.577 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:57.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:57.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:57.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:57.704 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:57.750 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:57.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:57.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:57.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:57.880 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:57.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:57.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:57.982 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:58.025 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:58.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:58.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:58.127 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:58.150 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:58.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:58.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:58.252 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:58.298 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:58.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:58.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:58.400 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:58.426 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:58.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:58.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:58.465 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 84 16 103144 103128 76.7156 53.5625 0.0290672 0.0130297 2024-08-17T18:00:58.498 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:58 smithi130 ceph-mon[110068]: pgmap v2062: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 37 MiB/s wr, 597 op/s 2024-08-17T18:00:58.498 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:58.498 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:00:58.498 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:58 smithi130 ceph-mon[105521]: pgmap v2062: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 37 MiB/s wr, 597 op/s 2024-08-17T18:00:58.499 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:58.499 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:00:58.528 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:58.539 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:00:58.570 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:58.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:58.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:58 smithi179 ceph-mon[93519]: pgmap v2062: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 37 MiB/s wr, 597 op/s 2024-08-17T18:00:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:00:58.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:00:58.671 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:58.693 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:58.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:58.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:58.795 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:00:58.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:58.857 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:58.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:00:58.958 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:00:59.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:59.006 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:59.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:00:59.107 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:59.130 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:59.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:59.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:59.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:00:59.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:59.288 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:00:59.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:00:59.335 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:00:59.335 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:00:59.335 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:00:59.335 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:00:59.335 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:00:59.335 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:00:59.335 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:00:59.335 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:00:59.335 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:00:59.336 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:00:59.336 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:00:59.336 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:00:59.354 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:00:59 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3054965454' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:59.354 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:00:59.355 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:00:59.355 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:00:59 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3054965454' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:59.390 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:00:59.401 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:00:59.401 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:00:59.413 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:00:59.414 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:59.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:59.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:00:59.469 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 85 16 103856 103840 76.3356 44.5 0.00903123 0.0130937 2024-08-17T18:00:59.516 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:00:59.537 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:59.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:59.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:00:59.576 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:00:59 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3054965454' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:00:59.640 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:59.663 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:59.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:59.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:00:59.765 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:00:59.792 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:59.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:59.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:00:59.861 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:00:59.861 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:00:59.894 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:00:59.921 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:00:59.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:00:59.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:00.024 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:00.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:00.049 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:00.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:00.150 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:00.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:00:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:00:59] "GET /metrics HTTP/1.1" 200 36785 "" "Prometheus/2.51.0" 2024-08-17T18:01:00.184 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:00.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:00.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:00.286 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:00.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:00.313 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:00.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:00.348 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T18:01:00.413 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:00.447 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:00.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:00.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:00.466 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 86 16 104731 104715 76.0838 54.6875 0.0164423 0.0131377 2024-08-17T18:01:00.549 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:00.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:00.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:00.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:00.577 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:00 smithi179 ceph-mon[93519]: pgmap v2063: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T18:01:00.577 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:00 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1432185879' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:01:00.577 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:01:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:01:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:01:00.609 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T18:01:00.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:00 smithi130 ceph-mon[105521]: pgmap v2063: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T18:01:00.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:00 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1432185879' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:01:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:00 smithi130 ceph-mon[110068]: pgmap v2063: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-08-17T18:01:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:00 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1432185879' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:01:00.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:00.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:00.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:00.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:00.817 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:00.861 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:00.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:00.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:00.960 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:01:00.962 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:00.991 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:00.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:00.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:01.093 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:01.117 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:01.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:01.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:01.167 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T18:01:01.220 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:01.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:01.245 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:01.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:01.346 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:01.368 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:01.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:01.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:01.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 87 16 105571 105555 75.8127 52.5 0.00879428 0.0131852 2024-08-17T18:01:01.470 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:01.520 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:01.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:01.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:01.622 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:01.650 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:01.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:01.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:01.654 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:01 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1432185879' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:01:01.654 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:01 smithi130 ceph-mon[105521]: osdmap e1272: 8 total, 8 up, 6 in 2024-08-17T18:01:01.654 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:01 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1432185879' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:01:01.654 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:01 smithi130 ceph-mon[110068]: osdmap e1272: 8 total, 8 up, 6 in 2024-08-17T18:01:01.752 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:01.776 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:01.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:01.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:01 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1432185879' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:01:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:01 smithi179 ceph-mon[93519]: osdmap e1272: 8 total, 8 up, 6 in 2024-08-17T18:01:01.879 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:01.903 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:01.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:01.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:02.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:02.029 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:02.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:02.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:02.132 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:02.156 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:02.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:02.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:02.258 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:02.294 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:02.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:02.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:02.394 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:01:02.397 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:02.404 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:02.420 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:02.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:02.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:02.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 88 16 106407 106391 75.5445 52.25 0.0295 0.0132303 2024-08-17T18:01:02.522 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:02.546 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:02.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:02.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:02.648 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:02 smithi130 ceph-mon[105521]: pgmap v2065: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 33 MiB/s wr, 534 op/s 2024-08-17T18:01:02.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:02 smithi130 ceph-mon[110068]: pgmap v2065: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 33 MiB/s wr, 534 op/s 2024-08-17T18:01:02.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:02.699 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:02.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:02.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:02 smithi179 ceph-mon[93519]: pgmap v2065: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 33 MiB/s wr, 534 op/s 2024-08-17T18:01:02.850 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:02.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:02.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:02.956 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:02.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:02.981 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:02.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:03.083 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:03.106 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:03.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:03.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:03.208 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:03.232 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:03.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:03.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:03.333 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:03.373 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:03.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:03.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:03.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 89 16 107206 107190 75.2567 49.9375 0.0210834 0.0132825 2024-08-17T18:01:03.475 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:03.516 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:03.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:03.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:03.618 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:03.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:03.661 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:03.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:03.762 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:03.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:03.799 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:03.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:03.900 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:03.924 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:03.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:03.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:04.026 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:04.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:04.048 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:04.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:04.149 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:04.173 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:04.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:04.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:04.235 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:01:04.235 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:01:04.275 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:04.297 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:04.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:04.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:04.400 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:04.439 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:04.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:04.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:04.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Total time run: 90.0075 2024-08-17T18:01:04.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Total writes made: 107980 2024-08-17T18:01:04.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Write size: 65536 2024-08-17T18:01:04.468 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Object size: 65536 2024-08-17T18:01:04.468 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Bandwidth (MB/sec): 74.9798 2024-08-17T18:01:04.468 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev Bandwidth: 32.2529 2024-08-17T18:01:04.468 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max bandwidth (MB/sec): 192 2024-08-17T18:01:04.468 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min bandwidth (MB/sec): 36.5 2024-08-17T18:01:04.468 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Average IOPS: 1199 2024-08-17T18:01:04.468 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev IOPS: 516.046 2024-08-17T18:01:04.468 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max IOPS: 3072 2024-08-17T18:01:04.468 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min IOPS: 584 2024-08-17T18:01:04.468 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Average Latency(s): 0.0133324 2024-08-17T18:01:04.468 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev Latency(s): 0.018148 2024-08-17T18:01:04.468 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max latency(s): 1.04903 2024-08-17T18:01:04.469 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min latency(s): 0.00128899 2024-08-17T18:01:04.471 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T18:01:04.542 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:04.566 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:04.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:04.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:04.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:04 smithi179 ceph-mon[93519]: pgmap v2066: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 53 MiB/s wr, 851 op/s 2024-08-17T18:01:04.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:04 smithi130 ceph-mon[105521]: pgmap v2066: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 53 MiB/s wr, 851 op/s 2024-08-17T18:01:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:04 smithi130 ceph-mon[110068]: pgmap v2066: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 53 MiB/s wr, 851 op/s 2024-08-17T18:01:04.668 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:04.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:04.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:04.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:04.762 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:04.793 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:04.849 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:04.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:04.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:04.951 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:04.993 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:04.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:04.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:05.095 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:05.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:05.116 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:05.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:05.166 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T18:01:05.217 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:05.241 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:05.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:05.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:05.342 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:05.395 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:05.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:05.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:05.452 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:01:05.467 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:01:05.497 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:05.532 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:05.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:05.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:05 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3013663899' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:01:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:05 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:01:05.634 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:05.662 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:05.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:05.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:05.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:05 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3013663899' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:01:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:05 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:01:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:05 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3013663899' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:01:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:05 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:01:05.764 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:05.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:05.793 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:05.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:05.895 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:05.918 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:05.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:05.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:06.020 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:06.055 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:06.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:06.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:06.157 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:06.181 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:06.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:06.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:06.283 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:06.307 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:06.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:06.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:06.409 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:06.451 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:06.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:06.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:06.552 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:06.575 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:06.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:06.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:06 smithi130 ceph-mon[105521]: pgmap v2067: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s 2024-08-17T18:01:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:06 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:01:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:06 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:01:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:06 smithi130 ceph-mon[105521]: osdmap e1273: 8 total, 8 up, 6 in 2024-08-17T18:01:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:06 smithi130 ceph-mon[110068]: pgmap v2067: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s 2024-08-17T18:01:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:06 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:01:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:06 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:01:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:06 smithi130 ceph-mon[110068]: osdmap e1273: 8 total, 8 up, 6 in 2024-08-17T18:01:06.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:06.698 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:06.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:06.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:06.777 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:06 smithi179 ceph-mon[93519]: pgmap v2067: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s 2024-08-17T18:01:06.777 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:06 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:01:06.777 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:06 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:01:06.777 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:06 smithi179 ceph-mon[93519]: osdmap e1273: 8 total, 8 up, 6 in 2024-08-17T18:01:06.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:06.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:06.841 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:06.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:06.942 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:06.966 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:06.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:06.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:07.068 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:07.102 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:07.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:07.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:07.204 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:07.226 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:07.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:07.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:07.329 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:07.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:07.368 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:07.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:07.409 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:01:07.410 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:01:07.411 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T18:01:07.469 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:07.493 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:07.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:07.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:07.595 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:07.618 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:07.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:07.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:07.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:07 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1893428852' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:01:07.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:07 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:01:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:07 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1893428852' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:01:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:07 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:01:07.719 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:07.766 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:07.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:07.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:07.780 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:07 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1893428852' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:01:07.781 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:07 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:01:07.867 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:07.891 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:07.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:07.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:07.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:08.045 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:08.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:08.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:08.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:08.163 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T18:01:08.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:08.174 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:08.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:08.276 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:08.300 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:08.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:08.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:08.402 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:08.441 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:08.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:08.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:08.477 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:01:08.538 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T18:01:08.543 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:08.623 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:08.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:08.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:08.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:08 smithi130 ceph-mon[105521]: pgmap v2069: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 35 MiB/s wr, 611 op/s 2024-08-17T18:01:08.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:08 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:01:08.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:08 smithi130 ceph-mon[105521]: osdmap e1274: 8 total, 8 up, 6 in 2024-08-17T18:01:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:08 smithi130 ceph-mon[110068]: pgmap v2069: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 35 MiB/s wr, 611 op/s 2024-08-17T18:01:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:08 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:01:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:08 smithi130 ceph-mon[110068]: osdmap e1274: 8 total, 8 up, 6 in 2024-08-17T18:01:08.726 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:08.748 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:08.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:08.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:08.768 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:08 smithi179 ceph-mon[93519]: pgmap v2069: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 35 MiB/s wr, 611 op/s 2024-08-17T18:01:08.768 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:08 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:01:08.768 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:08 smithi179 ceph-mon[93519]: osdmap e1274: 8 total, 8 up, 6 in 2024-08-17T18:01:08.850 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:08.873 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:08.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:08.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:08.975 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:09.027 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:09.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:09.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:09.129 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:09.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:09.152 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:09.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:09.242 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:01:09.242 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:01:09.253 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:09.270 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:09.283 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:01:09.283 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:01:09.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:09.305 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:09.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:09.319 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:01:09.320 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:01:09.321 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:01:09.369 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:01:09.370 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:01:09.400 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:01:09.400 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:01:09.406 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:09.429 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:09.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:09.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:09.432 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:01:09.432 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:01:09.462 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:01:09.462 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:01:09.498 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:01:09.498 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:01:09.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:09.585 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:09.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:09.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:09.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:09 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3238915188' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:01:09.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:09 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3238915188' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:01:09.688 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:09.710 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:09.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:09.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:09.807 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:09 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3238915188' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:01:09.812 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:09.847 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:09.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:09.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:09.949 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:09.984 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:09.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:09.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:10.086 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:10.109 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:10.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:10.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:01:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:01:09] "GET /metrics HTTP/1.1" 200 36785 "" "Prometheus/2.51.0" 2024-08-17T18:01:10.211 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:10.241 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:10.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:10.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:10.342 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:10.365 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:10.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:10.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:10.423 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:01:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:01:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:01:10.467 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:10.503 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:10.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:10.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:10.604 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:10.626 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:10.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:10.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:10.727 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:10.758 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:10.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:10.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:10.775 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:10 smithi179 ceph-mon[93519]: pgmap v2071: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 127 B/s rd, 43 MiB/s wr, 1.03k op/s 2024-08-17T18:01:10.778 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:01:10.778 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:01:10.778 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:01:10.778 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:01:10.778 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:01:10.860 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:10.883 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:10.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:10.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:10.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:10 smithi130 ceph-mon[105521]: pgmap v2071: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 127 B/s rd, 43 MiB/s wr, 1.03k op/s 2024-08-17T18:01:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:10 smithi130 ceph-mon[110068]: pgmap v2071: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 127 B/s rd, 43 MiB/s wr, 1.03k op/s 2024-08-17T18:01:10.984 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:11.009 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:11.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:11.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:11.111 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:11.134 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:11.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:11.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:11.235 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:11.259 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:11.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:11.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:11.361 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:11.384 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:11.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:11.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:11.485 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:11.511 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:11.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:11.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:11.531 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T18:01:11.613 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:11.644 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:11.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:11.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:11.746 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:11.767 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:11.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:11.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:11.842 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:01:11.868 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:11.889 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:11.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:11.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:11.991 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:12.014 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:12.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:12.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:12.116 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:12.143 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:12.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:12.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:12.245 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:12.287 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:12.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:12.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:12.388 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:12.423 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:12.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:12.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:12.524 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:12.561 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:12.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:12.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:12.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:12 smithi179 ceph-mon[93519]: pgmap v2072: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 127 B/s rd, 13 MiB/s wr, 555 op/s 2024-08-17T18:01:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:01:12.663 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:12.686 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:12.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:12.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:12.762 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:12 smithi130 ceph-mon[105521]: pgmap v2072: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 127 B/s rd, 13 MiB/s wr, 555 op/s 2024-08-17T18:01:12.762 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:01:12.763 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:12 smithi130 ceph-mon[110068]: pgmap v2072: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 127 B/s rd, 13 MiB/s wr, 555 op/s 2024-08-17T18:01:12.763 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:01:12.788 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:12.811 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:12.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:12.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:12.914 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:12.935 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:12.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:12.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:13.037 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:13.058 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:13.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:13.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:13.160 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:13.188 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:13.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:13.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:13.289 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:13.313 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:13.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:13.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:13.414 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:13.458 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:13.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:13.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:13.559 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:13.595 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:13.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:13.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:13.697 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:13.727 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:13.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:13.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:13.730 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:01:13.730 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T18:01:13.730 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T18:01:13.730 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T18:01:13.731 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.731 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:30138 46.a 46:577d6c08:::benchmark_data_smithi130_267720_object30137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:01:13.731 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:48.638681+0000", 2024-08-17T18:01:13.731 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 85.088908821000004, 2024-08-17T18:01:13.731 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37615081299999997, 2024-08-17T18:01:13.731 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.731 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.731 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.731 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.731 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.731 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 30138 2024-08-17T18:01:13.731 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.731 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.732 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.732 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.732 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638681+0000", 2024-08-17T18:01:13.732 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.732 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.732 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.732 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.732 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638681+0000", 2024-08-17T18:01:13.732 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.732 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.732 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.732 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.732 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638680+0000", 2024-08-17T18:01:13.733 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:01:13.733 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.733 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.733 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.733 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638734+0000", 2024-08-17T18:01:13.733 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3424999999999998e-05 2024-08-17T18:01:13.733 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.733 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.733 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.733 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638742+0000", 2024-08-17T18:01:13.733 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2819999999999996e-06 2024-08-17T18:01:13.733 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.733 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.734 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.734 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638746+0000", 2024-08-17T18:01:13.734 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9740000000000004e-06 2024-08-17T18:01:13.734 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.734 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.734 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.734 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638787+0000", 2024-08-17T18:01:13.734 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0868999999999998e-05 2024-08-17T18:01:13.734 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.734 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.734 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.734 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638874+0000", 2024-08-17T18:01:13.734 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6833e-05 2024-08-17T18:01:13.735 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.735 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.735 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.735 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638957+0000", 2024-08-17T18:01:13.735 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3745000000000005e-05 2024-08-17T18:01:13.735 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.735 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.735 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.735 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.639982+0000", 2024-08-17T18:01:13.735 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001024107 2024-08-17T18:01:13.735 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.735 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.735 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.736 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.640316+0000", 2024-08-17T18:01:13.736 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00033452200000000002 2024-08-17T18:01:13.736 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.736 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.736 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.736 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:49.014750+0000", 2024-08-17T18:01:13.736 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37443375699999998 2024-08-17T18:01:13.736 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.737 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.737 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.737 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:49.014807+0000", 2024-08-17T18:01:13.738 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7263999999999999e-05 2024-08-17T18:01:13.738 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.738 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.738 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.738 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:49.014832+0000", 2024-08-17T18:01:13.738 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4921000000000001e-05 2024-08-17T18:01:13.738 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.738 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.738 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.738 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.738 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.738 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42530 46.e 46:7a5bb013:::benchmark_data_smithi130_267720_object42529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:01:13.738 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:01:13.739 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 76.718153620999999, 2024-08-17T18:01:13.739 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.037353389, 2024-08-17T18:01:13.739 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.739 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:01:13.739 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.739 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.739 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.739 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42530 2024-08-17T18:01:13.739 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.739 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.739 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.739 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.739 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:01:13.739 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.740 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.740 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.740 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.740 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:01:13.740 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.740 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.740 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.740 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.740 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:01:13.740 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:01:13.740 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.740 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.740 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.740 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009488+0000", 2024-08-17T18:01:13.741 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2281999999999999e-05 2024-08-17T18:01:13.741 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.741 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.741 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.741 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009508+0000", 2024-08-17T18:01:13.741 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0250000000000001e-05 2024-08-17T18:01:13.741 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.741 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.741 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.741 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009514+0000", 2024-08-17T18:01:13.741 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.161e-06 2024-08-17T18:01:13.741 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.741 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.741 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.742 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009564+0000", 2024-08-17T18:01:13.742 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0028999999999998e-05 2024-08-17T18:01:13.742 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.742 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.742 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:01:13.742 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009573+0000", 2024-08-17T18:01:13.742 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.9369999999999993e-06 2024-08-17T18:01:13.742 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.742 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.742 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.742 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029341+0000", 2024-08-17T18:01:13.742 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0197678189999999 2024-08-17T18:01:13.742 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.743 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.743 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for missing object", 2024-08-17T18:01:13.743 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029434+0000", 2024-08-17T18:01:13.743 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.3185999999999999e-05 2024-08-17T18:01:13.743 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.743 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.743 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.743 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033201+0000", 2024-08-17T18:01:13.743 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003766934 2024-08-17T18:01:13.743 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.743 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.743 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for degraded object", 2024-08-17T18:01:13.743 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033231+0000", 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9657000000000001e-05 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.046712+0000", 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013481102 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.046790+0000", 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7853000000000002e-05 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.744 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.745 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.745 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42561 46.e 46:7b14be6d:::benchmark_data_smithi130_267720_object42560:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:01:13.745 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.039454+0000", 2024-08-17T18:01:13.745 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 76.688136092999997, 2024-08-17T18:01:13.745 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99762182099999996, 2024-08-17T18:01:13.745 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.745 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.745 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.745 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.746 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.746 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42561 2024-08-17T18:01:13.746 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.746 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.746 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.746 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.747 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039454+0000", 2024-08-17T18:01:13.747 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.747 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.747 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.747 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.747 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039454+0000", 2024-08-17T18:01:13.747 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.747 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.747 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.747 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.747 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039452+0000", 2024-08-17T18:01:13.747 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:01:13.747 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.748 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.748 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.748 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039516+0000", 2024-08-17T18:01:13.748 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3046000000000006e-05 2024-08-17T18:01:13.748 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.748 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.748 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.748 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039528+0000", 2024-08-17T18:01:13.748 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2916e-05 2024-08-17T18:01:13.748 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.748 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.748 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.748 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039534+0000", 2024-08-17T18:01:13.748 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2179999999999998e-06 2024-08-17T18:01:13.749 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.749 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.749 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.749 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039834+0000", 2024-08-17T18:01:13.749 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00030059599999999998 2024-08-17T18:01:13.749 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.749 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.749 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:01:13.749 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039844+0000", 2024-08-17T18:01:13.749 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5840000000000001e-06 2024-08-17T18:01:13.749 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.749 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.749 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.749 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029614+0000", 2024-08-17T18:01:13.750 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98977001600000003 2024-08-17T18:01:13.750 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.750 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.750 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.750 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029696+0000", 2024-08-17T18:01:13.750 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2041999999999998e-05 2024-08-17T18:01:13.750 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.750 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.750 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.750 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029809+0000", 2024-08-17T18:01:13.750 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000113503 2024-08-17T18:01:13.750 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.750 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.750 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.751 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033156+0000", 2024-08-17T18:01:13.751 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0033461559999999999 2024-08-17T18:01:13.751 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.751 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.751 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.751 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036913+0000", 2024-08-17T18:01:13.751 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037575149999999999 2024-08-17T18:01:13.751 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.751 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.751 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.751 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036976+0000", 2024-08-17T18:01:13.751 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2580000000000001e-05 2024-08-17T18:01:13.751 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.751 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.752 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.752 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037020+0000", 2024-08-17T18:01:13.752 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4678000000000002e-05 2024-08-17T18:01:13.752 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.752 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.752 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.752 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037076+0000", 2024-08-17T18:01:13.752 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5449999999999999e-05 2024-08-17T18:01:13.752 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.752 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.752 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.752 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.752 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.752 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42566 46.e 46:733e1503:::benchmark_data_smithi130_267720_object42565:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:01:13.753 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.042517+0000", 2024-08-17T18:01:13.753 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 76.685072563999995, 2024-08-17T18:01:13.753 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99468539099999997, 2024-08-17T18:01:13.753 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.753 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.753 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.753 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.753 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.753 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42566 2024-08-17T18:01:13.753 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.753 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.753 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.753 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.753 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042517+0000", 2024-08-17T18:01:13.754 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.754 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.754 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.754 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.754 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042517+0000", 2024-08-17T18:01:13.754 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.754 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.754 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.754 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.754 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042516+0000", 2024-08-17T18:01:13.754 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:01:13.754 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.754 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.754 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.755 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042575+0000", 2024-08-17T18:01:13.755 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8802999999999999e-05 2024-08-17T18:01:13.755 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.755 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.755 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.755 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042588+0000", 2024-08-17T18:01:13.755 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2812e-05 2024-08-17T18:01:13.755 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.755 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.755 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.755 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042594+0000", 2024-08-17T18:01:13.755 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.55e-06 2024-08-17T18:01:13.755 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.755 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.756 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.756 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042634+0000", 2024-08-17T18:01:13.756 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9817999999999997e-05 2024-08-17T18:01:13.756 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.756 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.756 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:01:13.756 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042640+0000", 2024-08-17T18:01:13.756 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7570000000000001e-06 2024-08-17T18:01:13.756 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.756 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.756 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.756 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.030766+0000", 2024-08-17T18:01:13.756 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98812632600000005 2024-08-17T18:01:13.756 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.757 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.757 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.757 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.030858+0000", 2024-08-17T18:01:13.757 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2035999999999998e-05 2024-08-17T18:01:13.757 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.757 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.757 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.757 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.030950+0000", 2024-08-17T18:01:13.757 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.1605000000000001e-05 2024-08-17T18:01:13.757 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.757 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.757 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.757 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033167+0000", 2024-08-17T18:01:13.757 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002216951 2024-08-17T18:01:13.758 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.758 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.758 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.758 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036943+0000", 2024-08-17T18:01:13.758 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037761840000000001 2024-08-17T18:01:13.758 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.758 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.758 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.758 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037108+0000", 2024-08-17T18:01:13.758 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016495400000000001 2024-08-17T18:01:13.758 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.758 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.758 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.758 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037137+0000", 2024-08-17T18:01:13.759 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8826000000000001e-05 2024-08-17T18:01:13.759 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.759 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.759 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.759 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037203+0000", 2024-08-17T18:01:13.759 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6042000000000002e-05 2024-08-17T18:01:13.759 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.759 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.759 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.759 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.759 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.759 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42587 46.e 46:755c4a73:::benchmark_data_smithi130_267720_object42586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:01:13.759 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.057082+0000", 2024-08-17T18:01:13.760 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 76.670508428999995, 2024-08-17T18:01:13.760 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98134466099999995, 2024-08-17T18:01:13.760 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.760 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.760 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.760 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.760 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.760 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42587 2024-08-17T18:01:13.760 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.760 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.760 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.760 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.760 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057082+0000", 2024-08-17T18:01:13.760 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.761 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.761 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.761 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.761 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057082+0000", 2024-08-17T18:01:13.761 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.761 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.761 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.761 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.761 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057080+0000", 2024-08-17T18:01:13.761 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:01:13.761 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.761 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.761 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.762 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057140+0000", 2024-08-17T18:01:13.762 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9277000000000002e-05 2024-08-17T18:01:13.762 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.762 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.762 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.762 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057152+0000", 2024-08-17T18:01:13.762 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2279000000000001e-05 2024-08-17T18:01:13.762 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.762 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.762 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.762 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057157+0000", 2024-08-17T18:01:13.762 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9389999999999996e-06 2024-08-17T18:01:13.762 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.763 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.763 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.763 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057223+0000", 2024-08-17T18:01:13.763 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5721000000000003e-05 2024-08-17T18:01:13.763 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.763 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.763 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:01:13.763 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057232+0000", 2024-08-17T18:01:13.763 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6209999999999999e-06 2024-08-17T18:01:13.763 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.763 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.763 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.763 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.031638+0000", 2024-08-17T18:01:13.763 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97440567300000003 2024-08-17T18:01:13.764 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.764 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.764 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.764 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.031716+0000", 2024-08-17T18:01:13.764 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7650000000000004e-05 2024-08-17T18:01:13.764 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.764 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.764 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.764 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.031819+0000", 2024-08-17T18:01:13.764 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010341899999999999 2024-08-17T18:01:13.764 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.764 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.764 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.764 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033981+0000", 2024-08-17T18:01:13.765 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0021622220000000001 2024-08-17T18:01:13.765 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.765 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.765 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.765 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037227+0000", 2024-08-17T18:01:13.765 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0032460060000000001 2024-08-17T18:01:13.765 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.765 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.765 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.765 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.038264+0000", 2024-08-17T18:01:13.765 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0010368949999999999 2024-08-17T18:01:13.765 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.765 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.766 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.766 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.038321+0000", 2024-08-17T18:01:13.766 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7354e-05 2024-08-17T18:01:13.766 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.766 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.766 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.766 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.038426+0000", 2024-08-17T18:01:13.766 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000104776 2024-08-17T18:01:13.766 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.766 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.766 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.766 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.766 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.766 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42593 46.e 46:77bdd57c:::benchmark_data_smithi130_267720_object42592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:01:13.767 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.061397+0000", 2024-08-17T18:01:13.767 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 76.666192577999993, 2024-08-17T18:01:13.767 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98195155000000001, 2024-08-17T18:01:13.767 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.767 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.767 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.767 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.767 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.767 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42593 2024-08-17T18:01:13.767 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.767 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.767 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.767 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.767 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061397+0000", 2024-08-17T18:01:13.768 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.768 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.768 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.768 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.768 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061397+0000", 2024-08-17T18:01:13.768 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.768 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.768 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.768 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.768 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061390+0000", 2024-08-17T18:01:13.768 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999924 2024-08-17T18:01:13.768 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.768 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.769 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.769 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061432+0000", 2024-08-17T18:01:13.769 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2111999999999999e-05 2024-08-17T18:01:13.769 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.769 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.769 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.769 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061445+0000", 2024-08-17T18:01:13.769 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3037000000000001e-05 2024-08-17T18:01:13.769 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.769 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.769 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.769 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061450+0000", 2024-08-17T18:01:13.769 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.4759999999999997e-06 2024-08-17T18:01:13.769 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.770 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.770 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.770 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061497+0000", 2024-08-17T18:01:13.770 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6696999999999997e-05 2024-08-17T18:01:13.770 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.770 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.770 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:01:13.770 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061504+0000", 2024-08-17T18:01:13.770 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7100000000000001e-06 2024-08-17T18:01:13.770 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.770 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.770 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.770 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033248+0000", 2024-08-17T18:01:13.770 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97174373999999997 2024-08-17T18:01:13.771 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.771 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.771 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.771 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033325+0000", 2024-08-17T18:01:13.771 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7785000000000001e-05 2024-08-17T18:01:13.771 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.771 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.771 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.771 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033416+0000", 2024-08-17T18:01:13.771 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0221000000000002e-05 2024-08-17T18:01:13.771 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.771 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.771 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.771 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036950+0000", 2024-08-17T18:01:13.772 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0035344 2024-08-17T18:01:13.772 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.772 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.772 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.772 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.040390+0000", 2024-08-17T18:01:13.772 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0034401890000000002 2024-08-17T18:01:13.772 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.772 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.772 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.772 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.043240+0000", 2024-08-17T18:01:13.772 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028503019999999999 2024-08-17T18:01:13.772 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.772 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.772 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.773 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.043292+0000", 2024-08-17T18:01:13.773 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1598999999999998e-05 2024-08-17T18:01:13.773 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.773 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.773 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.773 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.043349+0000", 2024-08-17T18:01:13.773 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6907999999999998e-05 2024-08-17T18:01:13.773 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.773 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.773 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.773 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.773 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.773 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:67147 46.a 46:5c4e42f4:::benchmark_data_smithi130_267720_object67146:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:01:13.773 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:18.360318+0000", 2024-08-17T18:01:13.774 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 55.367272313000001, 2024-08-17T18:01:13.774 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.26496706399999997, 2024-08-17T18:01:13.774 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.774 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.774 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.774 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.774 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.774 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 67147 2024-08-17T18:01:13.774 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.774 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.774 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.774 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.774 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360318+0000", 2024-08-17T18:01:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360318+0000", 2024-08-17T18:01:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360316+0000", 2024-08-17T18:01:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:01:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.775 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.776 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360367+0000", 2024-08-17T18:01:13.776 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1192e-05 2024-08-17T18:01:13.776 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.776 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.776 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.776 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360383+0000", 2024-08-17T18:01:13.776 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5948999999999999e-05 2024-08-17T18:01:13.776 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.776 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.776 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.776 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360390+0000", 2024-08-17T18:01:13.776 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6139999999999997e-06 2024-08-17T18:01:13.776 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.777 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.777 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.777 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360421+0000", 2024-08-17T18:01:13.777 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1208999999999999e-05 2024-08-17T18:01:13.777 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.777 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.777 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.777 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360518+0000", 2024-08-17T18:01:13.777 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7254000000000001e-05 2024-08-17T18:01:13.777 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.777 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.777 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.777 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360612+0000", 2024-08-17T18:01:13.778 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.3487999999999997e-05 2024-08-17T18:01:13.778 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.778 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.778 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.778 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.362572+0000", 2024-08-17T18:01:13.778 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019598089999999999 2024-08-17T18:01:13.778 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.778 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.778 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.778 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.370147+0000", 2024-08-17T18:01:13.778 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0075756510000000001 2024-08-17T18:01:13.778 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.778 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.625187+0000", 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.25503976 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.625243+0000", 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5581000000000002e-05 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.625285+0000", 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2268000000000003e-05 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.779 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.780 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.780 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.780 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.780 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:67152 46.a 46:567c055a:::benchmark_data_smithi130_267720_object67151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:01:13.780 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:18.363407+0000", 2024-08-17T18:01:13.780 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 55.364182640999999, 2024-08-17T18:01:13.780 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.267197095, 2024-08-17T18:01:13.780 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.780 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.780 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.780 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.780 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.780 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 67152 2024-08-17T18:01:13.780 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.781 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.781 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.781 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.781 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363407+0000", 2024-08-17T18:01:13.781 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.781 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.781 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.781 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.781 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363407+0000", 2024-08-17T18:01:13.781 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.781 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.781 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.781 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.782 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363407+0000", 2024-08-17T18:01:13.782 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:01:13.782 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.782 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.782 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.782 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363466+0000", 2024-08-17T18:01:13.782 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9027999999999998e-05 2024-08-17T18:01:13.782 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.782 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.782 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.782 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363477+0000", 2024-08-17T18:01:13.782 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.172e-05 2024-08-17T18:01:13.782 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.782 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.783 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.783 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363482+0000", 2024-08-17T18:01:13.783 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3560000000000003e-06 2024-08-17T18:01:13.783 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.783 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.783 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.783 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363534+0000", 2024-08-17T18:01:13.783 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1989e-05 2024-08-17T18:01:13.783 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.783 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.783 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.783 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363659+0000", 2024-08-17T18:01:13.783 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012534499999999999 2024-08-17T18:01:13.783 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.784 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.784 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.784 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363815+0000", 2024-08-17T18:01:13.784 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00015637100000000001 2024-08-17T18:01:13.784 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.784 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.784 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.784 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.366241+0000", 2024-08-17T18:01:13.784 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002425539 2024-08-17T18:01:13.784 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.784 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.784 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.784 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.370190+0000", 2024-08-17T18:01:13.785 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0039493870000000004 2024-08-17T18:01:13.785 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.785 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.785 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.785 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630531+0000", 2024-08-17T18:01:13.785 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.26034059700000001 2024-08-17T18:01:13.785 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.785 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.785 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.785 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630580+0000", 2024-08-17T18:01:13.785 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8797000000000001e-05 2024-08-17T18:01:13.785 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.785 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.785 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.786 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630604+0000", 2024-08-17T18:01:13.786 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4739000000000001e-05 2024-08-17T18:01:13.786 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.786 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.786 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.786 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.786 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.786 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:67154 46.e 46:7f5ee838:::benchmark_data_smithi130_267720_object67153:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:01:13.786 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:18.364048+0000", 2024-08-17T18:01:13.786 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 55.363542068000001, 2024-08-17T18:01:13.786 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.26435667299999999, 2024-08-17T18:01:13.786 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.786 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.787 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.787 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.787 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.787 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 67154 2024-08-17T18:01:13.787 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.787 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.787 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.787 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.787 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364048+0000", 2024-08-17T18:01:13.787 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.787 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.787 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.787 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.787 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364048+0000", 2024-08-17T18:01:13.788 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.788 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.788 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.788 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.788 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364047+0000", 2024-08-17T18:01:13.788 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:01:13.788 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.788 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.788 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.788 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364111+0000", 2024-08-17T18:01:13.788 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3903000000000001e-05 2024-08-17T18:01:13.788 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.788 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364118+0000", 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.1860000000000003e-06 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364122+0000", 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3620000000000002e-06 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364160+0000", 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7951999999999998e-05 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.789 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.790 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.790 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364257+0000", 2024-08-17T18:01:13.790 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7085999999999998e-05 2024-08-17T18:01:13.790 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.790 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.790 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.790 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364526+0000", 2024-08-17T18:01:13.790 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00026916100000000002 2024-08-17T18:01:13.790 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.790 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.790 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.790 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.366442+0000", 2024-08-17T18:01:13.790 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019158510000000001 2024-08-17T18:01:13.790 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.370399+0000", 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0039570430000000004 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.628298+0000", 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.257899298 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.628364+0000", 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5518000000000005e-05 2024-08-17T18:01:13.791 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.628405+0000", 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0898e-05 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:67161 46.a 46:56328e3d:::benchmark_data_smithi130_267720_object67160:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:18.367430+0000", 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 55.360160108999999, 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.26329809900000001, 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.792 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.793 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.793 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.793 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.793 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 67161 2024-08-17T18:01:13.793 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.793 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.793 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.793 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.793 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367430+0000", 2024-08-17T18:01:13.793 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.793 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.793 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.793 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.794 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367430+0000", 2024-08-17T18:01:13.794 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.794 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.794 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.794 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.794 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367429+0000", 2024-08-17T18:01:13.794 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:01:13.794 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.794 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.794 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.794 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367498+0000", 2024-08-17T18:01:13.794 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.8891000000000002e-05 2024-08-17T18:01:13.794 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.795 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.795 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.795 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367513+0000", 2024-08-17T18:01:13.795 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5858e-05 2024-08-17T18:01:13.795 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.795 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.795 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.795 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367519+0000", 2024-08-17T18:01:13.795 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5500000000000002e-06 2024-08-17T18:01:13.795 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.795 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.795 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.795 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367554+0000", 2024-08-17T18:01:13.795 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4779e-05 2024-08-17T18:01:13.796 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.796 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.796 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.796 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367677+0000", 2024-08-17T18:01:13.796 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012334800000000001 2024-08-17T18:01:13.796 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.796 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.796 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.796 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367823+0000", 2024-08-17T18:01:13.796 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000146036 2024-08-17T18:01:13.796 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.796 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.796 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.797 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.370201+0000", 2024-08-17T18:01:13.797 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0023783250000000001 2024-08-17T18:01:13.797 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.797 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.797 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.797 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.370897+0000", 2024-08-17T18:01:13.797 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00069569499999999997 2024-08-17T18:01:13.797 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.797 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.797 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.797 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630642+0000", 2024-08-17T18:01:13.797 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.25974509099999998 2024-08-17T18:01:13.797 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.798 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.798 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.798 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630692+0000", 2024-08-17T18:01:13.798 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9332000000000001e-05 2024-08-17T18:01:13.798 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.798 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.798 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.798 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630728+0000", 2024-08-17T18:01:13.798 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6446999999999998e-05 2024-08-17T18:01:13.798 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.798 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.798 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.798 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.798 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.799 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:67164 46.e 46:7b31c258:::benchmark_data_smithi130_267720_object67163:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:01:13.799 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:18.371892+0000", 2024-08-17T18:01:13.799 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 55.355698322000002, 2024-08-17T18:01:13.799 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.25660291499999999, 2024-08-17T18:01:13.799 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.799 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.799 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.799 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.799 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.799 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 67164 2024-08-17T18:01:13.799 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.799 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.799 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.799 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.800 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.371892+0000", 2024-08-17T18:01:13.800 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.800 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.800 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.800 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.800 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.371892+0000", 2024-08-17T18:01:13.800 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.800 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.800 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.800 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.800 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.371891+0000", 2024-08-17T18:01:13.800 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:01:13.800 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.800 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.801 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.802 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.371938+0000", 2024-08-17T18:01:13.802 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7608000000000002e-05 2024-08-17T18:01:13.802 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.802 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.802 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.802 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.371950+0000", 2024-08-17T18:01:13.802 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1882000000000001e-05 2024-08-17T18:01:13.802 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.802 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.802 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.802 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.371956+0000", 2024-08-17T18:01:13.803 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2090000000000002e-06 2024-08-17T18:01:13.803 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.803 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.803 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.803 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.371993+0000", 2024-08-17T18:01:13.803 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6325999999999997e-05 2024-08-17T18:01:13.803 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.803 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.803 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.803 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.372110+0000", 2024-08-17T18:01:13.803 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000117552 2024-08-17T18:01:13.803 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.803 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.803 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.804 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.372256+0000", 2024-08-17T18:01:13.804 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014551500000000001 2024-08-17T18:01:13.804 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.804 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.804 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.804 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.374568+0000", 2024-08-17T18:01:13.804 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0023123190000000002 2024-08-17T18:01:13.804 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.804 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.804 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.804 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.374855+0000", 2024-08-17T18:01:13.804 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00028710500000000001 2024-08-17T18:01:13.804 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.804 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.805 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.805 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.628441+0000", 2024-08-17T18:01:13.805 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.25358550600000002 2024-08-17T18:01:13.805 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.805 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.805 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.805 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.628473+0000", 2024-08-17T18:01:13.805 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2255000000000002e-05 2024-08-17T18:01:13.805 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.805 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.805 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.805 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.628495+0000", 2024-08-17T18:01:13.805 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.1594e-05 2024-08-17T18:01:13.805 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.806 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.806 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.806 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.806 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.806 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:67167 46.4 46:20e86b34:::benchmark_data_smithi130_267720_object67166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:01:13.806 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:18.373424+0000", 2024-08-17T18:01:13.806 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 55.354165952000002, 2024-08-17T18:01:13.806 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.25262484000000002, 2024-08-17T18:01:13.806 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.806 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.806 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.806 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.806 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.807 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 67167 2024-08-17T18:01:13.807 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.807 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.807 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.807 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.807 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.373424+0000", 2024-08-17T18:01:13.807 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.807 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.807 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.807 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.807 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.373424+0000", 2024-08-17T18:01:13.807 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.807 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.807 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.808 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.808 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.373423+0000", 2024-08-17T18:01:13.808 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:01:13.808 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.808 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.808 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.808 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.373486+0000", 2024-08-17T18:01:13.808 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2868999999999997e-05 2024-08-17T18:01:13.808 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.808 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.808 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.808 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.373499+0000", 2024-08-17T18:01:13.808 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3519999999999999e-05 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.373505+0000", 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3340000000000001e-06 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.373549+0000", 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4172000000000003e-05 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.373681+0000", 2024-08-17T18:01:13.809 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013226199999999999 2024-08-17T18:01:13.810 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.810 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.810 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:01:13.810 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.373824+0000", 2024-08-17T18:01:13.810 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014263399999999999 2024-08-17T18:01:13.810 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.810 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.810 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.810 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.376054+0000", 2024-08-17T18:01:13.810 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0022303969999999998 2024-08-17T18:01:13.810 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.810 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.810 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.811 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.376111+0000", 2024-08-17T18:01:13.811 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7136999999999999e-05 2024-08-17T18:01:13.811 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.811 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.811 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.811 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.625962+0000", 2024-08-17T18:01:13.811 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.249850552 2024-08-17T18:01:13.811 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.811 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.811 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.811 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.626013+0000", 2024-08-17T18:01:13.811 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0722e-05 2024-08-17T18:01:13.811 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.811 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.812 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.812 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.626049+0000", 2024-08-17T18:01:13.812 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6260999999999997e-05 2024-08-17T18:01:13.812 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.812 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.812 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.812 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.812 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.812 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79514 46.a 46:572757ec:::benchmark_data_smithi130_267720_object79513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:01:13.812 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.619984+0000", 2024-08-17T18:01:13.812 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 43.107606277000002, 2024-08-17T18:01:13.812 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.39803696399999999, 2024-08-17T18:01:13.812 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.813 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.813 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.813 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.813 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.813 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79514 2024-08-17T18:01:13.813 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.813 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.813 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.813 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.813 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.619984+0000", 2024-08-17T18:01:13.813 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.813 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.813 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.813 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.814 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.619984+0000", 2024-08-17T18:01:13.814 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.814 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.814 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.814 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.814 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.619983+0000", 2024-08-17T18:01:13.814 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:01:13.814 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.814 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.814 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.814 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620039+0000", 2024-08-17T18:01:13.814 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6446999999999997e-05 2024-08-17T18:01:13.814 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.815 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.815 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.815 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620050+0000", 2024-08-17T18:01:13.815 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0359e-05 2024-08-17T18:01:13.815 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.815 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.815 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.815 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620055+0000", 2024-08-17T18:01:13.815 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0479999999999998e-06 2024-08-17T18:01:13.815 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.815 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.815 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.815 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620085+0000", 2024-08-17T18:01:13.816 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9850000000000001e-05 2024-08-17T18:01:13.816 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.816 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.816 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.816 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620160+0000", 2024-08-17T18:01:13.816 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.5499000000000003e-05 2024-08-17T18:01:13.816 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.816 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.816 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.816 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620240+0000", 2024-08-17T18:01:13.816 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0418000000000002e-05 2024-08-17T18:01:13.816 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.816 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.817 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.817 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.622161+0000", 2024-08-17T18:01:13.817 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001920342 2024-08-17T18:01:13.817 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.817 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.817 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.817 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.627515+0000", 2024-08-17T18:01:13.817 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0053542399999999997 2024-08-17T18:01:13.817 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.817 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.818 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.818 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017941+0000", 2024-08-17T18:01:13.818 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.390425929 2024-08-17T18:01:13.818 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.818 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.818 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.818 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017991+0000", 2024-08-17T18:01:13.818 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9988999999999999e-05 2024-08-17T18:01:13.818 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.818 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.818 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.818 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.018021+0000", 2024-08-17T18:01:13.818 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9762999999999999e-05 2024-08-17T18:01:13.819 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.819 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.819 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.819 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.819 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.819 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79523 46.a 46:590eeb35:::benchmark_data_smithi130_267720_object79522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:01:13.819 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.631753+0000", 2024-08-17T18:01:13.819 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 43.095836951000003, 2024-08-17T18:01:13.819 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38901796, 2024-08-17T18:01:13.819 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.819 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.819 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.819 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.819 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.820 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79523 2024-08-17T18:01:13.820 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.820 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.820 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.820 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.820 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631753+0000", 2024-08-17T18:01:13.820 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.820 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.820 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.820 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.820 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631753+0000", 2024-08-17T18:01:13.820 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.820 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.821 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.821 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.821 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631752+0000", 2024-08-17T18:01:13.821 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:01:13.821 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.821 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.821 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.821 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631818+0000", 2024-08-17T18:01:13.821 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5757000000000001e-05 2024-08-17T18:01:13.821 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.821 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.821 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.821 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631828+0000", 2024-08-17T18:01:13.822 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0200000000000001e-05 2024-08-17T18:01:13.822 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.822 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.822 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.822 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631832+0000", 2024-08-17T18:01:13.822 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0430000000000001e-06 2024-08-17T18:01:13.822 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.822 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.822 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.822 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631869+0000", 2024-08-17T18:01:13.822 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6730999999999998e-05 2024-08-17T18:01:13.822 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.822 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.822 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.823 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631941+0000", 2024-08-17T18:01:13.823 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.1763999999999999e-05 2024-08-17T18:01:13.823 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.823 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.823 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.823 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632027+0000", 2024-08-17T18:01:13.823 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6599000000000002e-05 2024-08-17T18:01:13.823 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.823 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.823 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.823 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635101+0000", 2024-08-17T18:01:13.823 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0030730430000000001 2024-08-17T18:01:13.823 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.823 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.641996+0000", 2024-08-17T18:01:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0068956429999999999 2024-08-17T18:01:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020716+0000", 2024-08-17T18:01:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.378719474 2024-08-17T18:01:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020748+0000", 2024-08-17T18:01:13.824 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2026000000000001e-05 2024-08-17T18:01:13.825 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.825 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.825 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.825 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020771+0000", 2024-08-17T18:01:13.825 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3370000000000002e-05 2024-08-17T18:01:13.825 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.825 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.825 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.825 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.825 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.825 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79526 46.a 46:540ef56b:::benchmark_data_smithi130_267720_object79525:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:01:13.825 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:01:13.825 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 43.095309632999999, 2024-08-17T18:01:13.825 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38861050899999999, 2024-08-17T18:01:13.826 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.826 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.826 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.826 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.826 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.826 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79526 2024-08-17T18:01:13.826 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.826 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.826 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.826 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.826 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:01:13.826 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.826 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632360+0000", 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0008999999999999e-05 2024-08-17T18:01:13.827 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.828 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.828 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.828 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632369+0000", 2024-08-17T18:01:13.828 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6590000000000002e-06 2024-08-17T18:01:13.828 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.828 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.828 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.828 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632373+0000", 2024-08-17T18:01:13.828 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8840000000000004e-06 2024-08-17T18:01:13.828 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.828 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.828 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.828 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.633861+0000", 2024-08-17T18:01:13.828 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001487326 2024-08-17T18:01:13.829 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.829 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.829 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.829 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.633938+0000", 2024-08-17T18:01:13.829 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7625000000000003e-05 2024-08-17T18:01:13.829 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.829 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.829 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.829 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.634026+0000", 2024-08-17T18:01:13.829 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7950999999999999e-05 2024-08-17T18:01:13.829 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.829 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.829 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.830 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635234+0000", 2024-08-17T18:01:13.830 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0012074340000000001 2024-08-17T18:01:13.830 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.830 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.830 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.830 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.642019+0000", 2024-08-17T18:01:13.830 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0067856890000000001 2024-08-17T18:01:13.830 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.830 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.830 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.830 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020838+0000", 2024-08-17T18:01:13.830 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37881851700000002 2024-08-17T18:01:13.830 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.830 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.831 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.831 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020870+0000", 2024-08-17T18:01:13.831 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2261999999999997e-05 2024-08-17T18:01:13.831 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.831 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.831 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.831 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020891+0000", 2024-08-17T18:01:13.831 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0777000000000001e-05 2024-08-17T18:01:13.831 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.831 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.831 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.831 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.831 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79528 46.e 46:716b67ef:::benchmark_data_smithi130_267720_object79527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:01:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.635764+0000", 2024-08-17T18:01:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 43.091825814000003, 2024-08-17T18:01:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38226721899999999, 2024-08-17T18:01:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79528 2024-08-17T18:01:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.832 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.833 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635764+0000", 2024-08-17T18:01:13.833 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.833 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.833 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.833 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.833 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635764+0000", 2024-08-17T18:01:13.833 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.833 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.833 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.833 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.833 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635763+0000", 2024-08-17T18:01:13.833 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:01:13.833 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.834 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.834 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.834 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635828+0000", 2024-08-17T18:01:13.834 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4821999999999997e-05 2024-08-17T18:01:13.834 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.834 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.834 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.834 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635838+0000", 2024-08-17T18:01:13.834 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9599999999999995e-06 2024-08-17T18:01:13.834 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.834 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.834 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.834 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635842+0000", 2024-08-17T18:01:13.835 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3280000000000002e-06 2024-08-17T18:01:13.835 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.835 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.835 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.835 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635865+0000", 2024-08-17T18:01:13.835 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3600000000000001e-05 2024-08-17T18:01:13.835 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.835 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.835 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.835 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635943+0000", 2024-08-17T18:01:13.835 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.8098999999999999e-05 2024-08-17T18:01:13.835 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.835 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.835 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.636030+0000", 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7188000000000006e-05 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.637969+0000", 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001938148 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.639658+0000", 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0016891600000000001 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.836 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.837 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017955+0000", 2024-08-17T18:01:13.837 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.378297194 2024-08-17T18:01:13.837 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.837 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.837 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.837 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017992+0000", 2024-08-17T18:01:13.837 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6832999999999998e-05 2024-08-17T18:01:13.837 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.837 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.837 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.837 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.018031+0000", 2024-08-17T18:01:13.837 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9646e-05 2024-08-17T18:01:13.837 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.838 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.838 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.838 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.838 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.838 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79529 46.a 46:5d7561ca:::benchmark_data_smithi130_267720_object79528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:01:13.838 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.635869+0000", 2024-08-17T18:01:13.838 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 43.091721153000002, 2024-08-17T18:01:13.838 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38782661600000001, 2024-08-17T18:01:13.838 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.838 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.838 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.838 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.838 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.838 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79529 2024-08-17T18:01:13.839 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.839 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.839 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.839 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.839 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635869+0000", 2024-08-17T18:01:13.839 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.839 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.839 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.839 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.839 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635869+0000", 2024-08-17T18:01:13.839 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.839 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.839 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635868+0000", 2024-08-17T18:01:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:01:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635935+0000", 2024-08-17T18:01:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6509999999999998e-05 2024-08-17T18:01:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635942+0000", 2024-08-17T18:01:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3139999999999998e-06 2024-08-17T18:01:13.840 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.841 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.841 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.841 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635945+0000", 2024-08-17T18:01:13.841 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.7580000000000001e-06 2024-08-17T18:01:13.841 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.841 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.841 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.841 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635983+0000", 2024-08-17T18:01:13.841 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8318000000000001e-05 2024-08-17T18:01:13.841 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.841 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.841 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.841 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.636088+0000", 2024-08-17T18:01:13.841 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000104718 2024-08-17T18:01:13.842 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.842 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.842 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.842 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.636213+0000", 2024-08-17T18:01:13.842 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000124852 2024-08-17T18:01:13.842 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.842 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.842 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.842 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.637790+0000", 2024-08-17T18:01:13.842 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0015775839999999999 2024-08-17T18:01:13.842 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.842 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.842 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.843 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.642450+0000", 2024-08-17T18:01:13.843 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0046600260000000003 2024-08-17T18:01:13.843 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.843 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.843 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.843 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.023638+0000", 2024-08-17T18:01:13.843 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38118750600000001 2024-08-17T18:01:13.843 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.843 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.843 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.843 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.023669+0000", 2024-08-17T18:01:13.843 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0998999999999999e-05 2024-08-17T18:01:13.843 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.843 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.844 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.844 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.023695+0000", 2024-08-17T18:01:13.844 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.6645e-05 2024-08-17T18:01:13.844 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.844 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.844 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.844 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.844 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.844 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79534 46.a 46:5daa6ba5:::benchmark_data_smithi130_267720_object79533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:01:13.844 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.643773+0000", 2024-08-17T18:01:13.844 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 43.083817052999997, 2024-08-17T18:01:13.844 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38289725099999999, 2024-08-17T18:01:13.844 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.844 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.845 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.845 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.845 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.845 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79534 2024-08-17T18:01:13.845 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.845 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.845 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.845 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.845 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643773+0000", 2024-08-17T18:01:13.845 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.845 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.845 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.845 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.845 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643773+0000", 2024-08-17T18:01:13.846 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.846 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.846 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.846 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.846 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643772+0000", 2024-08-17T18:01:13.846 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:01:13.846 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.846 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.846 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.846 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643824+0000", 2024-08-17T18:01:13.846 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2240000000000001e-05 2024-08-17T18:01:13.846 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.846 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.846 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.847 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643833+0000", 2024-08-17T18:01:13.847 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6770000000000001e-06 2024-08-17T18:01:13.847 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.847 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.847 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.847 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643837+0000", 2024-08-17T18:01:13.847 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8089999999999999e-06 2024-08-17T18:01:13.847 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.847 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.847 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.847 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643868+0000", 2024-08-17T18:01:13.847 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1560000000000003e-05 2024-08-17T18:01:13.847 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.848 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.848 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.848 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643961+0000", 2024-08-17T18:01:13.848 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2584999999999995e-05 2024-08-17T18:01:13.848 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.848 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.848 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.848 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.644059+0000", 2024-08-17T18:01:13.848 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7851999999999998e-05 2024-08-17T18:01:13.848 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.848 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.848 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.848 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.646074+0000", 2024-08-17T18:01:13.848 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020149550000000001 2024-08-17T18:01:13.849 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.849 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.849 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.849 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647702+0000", 2024-08-17T18:01:13.849 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001628811 2024-08-17T18:01:13.849 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.849 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.849 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.849 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.026581+0000", 2024-08-17T18:01:13.849 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37887865300000001 2024-08-17T18:01:13.849 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.849 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.849 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.850 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.026633+0000", 2024-08-17T18:01:13.850 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2040000000000003e-05 2024-08-17T18:01:13.850 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.850 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.850 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.850 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.026670+0000", 2024-08-17T18:01:13.850 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7038e-05 2024-08-17T18:01:13.850 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.850 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.850 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.850 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.850 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.850 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79535 46.4 46:2496b234:::benchmark_data_smithi130_267720_object79534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:01:13.850 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:01:13.851 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 43.083724537000002, 2024-08-17T18:01:13.851 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38560203399999998, 2024-08-17T18:01:13.851 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.851 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.851 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.851 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.851 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.851 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79535 2024-08-17T18:01:13.851 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.851 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.851 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.851 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.851 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:01:13.851 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.852 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.852 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.852 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.852 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:01:13.852 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.852 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.852 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.852 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.852 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:01:13.852 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:01:13.852 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.852 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.852 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.852 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643927+0000", 2024-08-17T18:01:13.853 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1629000000000002e-05 2024-08-17T18:01:13.853 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.853 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.853 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.853 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643933+0000", 2024-08-17T18:01:13.853 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5130000000000002e-06 2024-08-17T18:01:13.853 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.853 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.853 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.853 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643937+0000", 2024-08-17T18:01:13.853 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3479999999999997e-06 2024-08-17T18:01:13.853 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.853 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.853 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.854 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643975+0000", 2024-08-17T18:01:13.854 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7771999999999997e-05 2024-08-17T18:01:13.854 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.854 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.854 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.854 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.644055+0000", 2024-08-17T18:01:13.854 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0144999999999999e-05 2024-08-17T18:01:13.854 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.854 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.854 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:01:13.854 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.644142+0000", 2024-08-17T18:01:13.854 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6539000000000006e-05 2024-08-17T18:01:13.854 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.854 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.855 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.855 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.646160+0000", 2024-08-17T18:01:13.855 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002018373 2024-08-17T18:01:13.855 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.855 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.855 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.855 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647237+0000", 2024-08-17T18:01:13.855 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001076465 2024-08-17T18:01:13.855 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.855 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.855 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.855 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.029404+0000", 2024-08-17T18:01:13.855 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38216758000000001 2024-08-17T18:01:13.855 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.856 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.856 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.856 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.029443+0000", 2024-08-17T18:01:13.856 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8846e-05 2024-08-17T18:01:13.856 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.856 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.856 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.856 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.029468+0000", 2024-08-17T18:01:13.856 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4355e-05 2024-08-17T18:01:13.856 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.856 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.856 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.856 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.856 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.857 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79538 46.e 46:7d19c96c:::benchmark_data_smithi130_267720_object79537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:01:13.857 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.647351+0000", 2024-08-17T18:01:13.857 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 43.080239139, 2024-08-17T18:01:13.857 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37361270899999999, 2024-08-17T18:01:13.857 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:01:13.857 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:01:13.857 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:01:13.857 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:01:13.857 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:13.857 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79538 2024-08-17T18:01:13.857 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.857 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:01:13.857 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.857 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:01:13.858 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647351+0000", 2024-08-17T18:01:13.858 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.858 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.858 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.858 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:01:13.858 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647351+0000", 2024-08-17T18:01:13.858 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:01:13.858 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.858 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.858 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:01:13.858 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647350+0000", 2024-08-17T18:01:13.858 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:01:13.858 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.858 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.859 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:01:13.859 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647421+0000", 2024-08-17T18:01:13.859 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0347999999999998e-05 2024-08-17T18:01:13.859 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.859 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.859 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:01:13.859 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647429+0000", 2024-08-17T18:01:13.859 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3059999999999998e-06 2024-08-17T18:01:13.859 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.859 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.859 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:01:13.859 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647433+0000", 2024-08-17T18:01:13.859 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7290000000000002e-06 2024-08-17T18:01:13.859 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647476+0000", 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3343999999999997e-05 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647600+0000", 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012389700000000001 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647740+0000", 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014001899999999999 2024-08-17T18:01:13.860 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.861 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.861 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:01:13.861 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.649745+0000", 2024-08-17T18:01:13.861 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020055450000000001 2024-08-17T18:01:13.861 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.861 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.861 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.861 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.650277+0000", 2024-08-17T18:01:13.861 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00053202999999999996 2024-08-17T18:01:13.861 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.861 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.861 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:13.861 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020872+0000", 2024-08-17T18:01:13.861 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37059452300000001 2024-08-17T18:01:13.862 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.862 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.862 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:01:13.862 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020923+0000", 2024-08-17T18:01:13.862 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1481999999999999e-05 2024-08-17T18:01:13.862 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:01:13.862 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:01:13.862 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:01:13.862 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020964+0000", 2024-08-17T18:01:13.862 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0132000000000001e-05 2024-08-17T18:01:13.862 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.862 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.862 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.862 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:01:13.863 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:01:13.863 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:01:13.863 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:13.904 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:13.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:13.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:14.006 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:14.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:14.051 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:14.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:14.152 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:14.195 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:14.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:14.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:14.297 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:14.337 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:14.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:14.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:14.439 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:14.485 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:14.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:14.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:14.587 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:14.597 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T18:01:14.613 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:14.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:14.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:14.714 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:14 smithi130 ceph-mon[105521]: pgmap v2073: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 127 B/s rd, 11 MiB/s wr, 967 op/s 2024-08-17T18:01:14.715 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:14 smithi130 ceph-mon[110068]: pgmap v2073: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 127 B/s rd, 11 MiB/s wr, 967 op/s 2024-08-17T18:01:14.715 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:14.737 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:14.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:14.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:14.839 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:14 smithi179 ceph-mon[93519]: pgmap v2073: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 127 B/s rd, 11 MiB/s wr, 967 op/s 2024-08-17T18:01:14.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:14.887 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:14.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:14.988 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:15.011 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:15.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:15.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:15.113 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:15.155 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:15.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:15.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:15.257 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:15.282 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:15.324 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:15.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:15.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:15.426 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:15.478 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:15.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:15.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:15.580 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:15.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:15.618 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:15.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:15.719 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:15.741 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:15.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:15.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:15.842 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:15.865 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:15.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:15.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:15.967 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:15.990 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:15.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:15.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:16.092 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:16.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:16.135 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:16.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:16.237 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:16.277 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:16.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:16.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:16.379 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:16.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:16.429 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:16.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:16.530 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:16.552 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:16.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:16.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:16.614 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:16.654 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:16.701 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:16.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:16.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:16.715 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:01:16.755 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:16 smithi130 ceph-mon[105521]: pgmap v2074: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 104 B/s rd, 9.0 MiB/s wr, 814 op/s 2024-08-17T18:01:16.756 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:16 smithi130 ceph-mon[110068]: pgmap v2074: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 104 B/s rd, 9.0 MiB/s wr, 814 op/s 2024-08-17T18:01:16.805 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:16 smithi179 ceph-mon[93519]: pgmap v2074: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 104 B/s rd, 9.0 MiB/s wr, 814 op/s 2024-08-17T18:01:16.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:16.852 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:16.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:16.953 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:16.993 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:16.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:16.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:17.095 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:17.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:17.137 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:17.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:17.238 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:17.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:17.275 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:17.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:17.376 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:17.399 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:17.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:17.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:17.454 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T18:01:17.501 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:17.523 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:17.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:17.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:17.533 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:17 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4204699332' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T18:01:17.534 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:17 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4204699332' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T18:01:17.626 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:17.666 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:17.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:17.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:17.744 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:17 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4204699332' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T18:01:17.768 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:17.805 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:17.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:17.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:17.907 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:17.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:17.931 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:17.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:18.033 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:18.064 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:18.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:18.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:18.166 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:18.189 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:18.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:18.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:18.291 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:18.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:18.331 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:18.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:18.431 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:18.455 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:18.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:18.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:18.557 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:18.580 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:18.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:18.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:18.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:18 smithi130 ceph-mon[105521]: pgmap v2075: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 6.2 MiB/s wr, 787 op/s 2024-08-17T18:01:18.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4204699332' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T18:01:18.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:18 smithi130 ceph-mon[105521]: osdmap e1275: 8 total, 8 up, 6 in 2024-08-17T18:01:18.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:18 smithi130 ceph-mon[110068]: pgmap v2075: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 6.2 MiB/s wr, 787 op/s 2024-08-17T18:01:18.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4204699332' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T18:01:18.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:18 smithi130 ceph-mon[110068]: osdmap e1275: 8 total, 8 up, 6 in 2024-08-17T18:01:18.682 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:18.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:18.716 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:18.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:18.817 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:18.842 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:18.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:18.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:18 smithi179 ceph-mon[93519]: pgmap v2075: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 102 B/s rd, 6.2 MiB/s wr, 787 op/s 2024-08-17T18:01:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4204699332' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T18:01:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:18 smithi179 ceph-mon[93519]: osdmap e1275: 8 total, 8 up, 6 in 2024-08-17T18:01:18.944 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:18.977 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:18.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:18.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:19.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:19.106 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:19.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:19.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:19.208 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:19.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:19.234 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:19.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:19.335 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:19.358 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:19.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:19.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:19.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:19.502 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:19.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:19.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:19.604 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:19.626 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:19.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:19.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:19.728 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:19.752 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:19.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:19.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:19.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:19.875 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:19.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:19.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:19.977 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:20.001 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:20.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:20.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:20.078 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:01:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:01:19] "GET /metrics HTTP/1.1" 200 36789 "" "Prometheus/2.51.0" 2024-08-17T18:01:20.103 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:20.125 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:20.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:20.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:20.227 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:20.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:20.283 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:20.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:20.384 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:20.406 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:20.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:20.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:20.471 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:01:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:01:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:01:20.508 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:20.553 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:20.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:20.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:20.641 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T18:01:20.655 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:20.679 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:20.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:20.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:20.735 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:20 smithi130 ceph-mon[110068]: pgmap v2077: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 766 op/s 2024-08-17T18:01:20.735 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:20 smithi130 ceph-mon[105521]: pgmap v2077: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 766 op/s 2024-08-17T18:01:20.781 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:20.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:20.818 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:20.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:20 smithi179 ceph-mon[93519]: pgmap v2077: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 766 op/s 2024-08-17T18:01:20.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:20.942 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:20.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:20.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:21.043 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:21.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:21.108 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:21.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:21.209 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:21.231 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:21.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:21.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:21.334 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:21.388 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:01:21.388 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:01:21.388 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.388 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:121537 46.d 46:b4432f7c:::benchmark_data_smithi130_267720_object13554:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1275)", 2024-08-17T18:01:21.388 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:01:21.377684+0000", 2024-08-17T18:01:21.388 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0084670600000000002, 2024-08-17T18:01:21.388 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0084965419999999993, 2024-08-17T18:01:21.388 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:01:21.389 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:01:21.389 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:01:21.389 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:01:21.389 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:01:21.389 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:21.389 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 121537 2024-08-17T18:01:21.389 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.389 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:01:21.389 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.390 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:01:21.390 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377684+0000", 2024-08-17T18:01:21.390 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:01:21.390 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.390 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.391 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:01:21.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377684+0000", 2024-08-17T18:01:21.391 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:01:21.391 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.391 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.391 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:01:21.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377685+0000", 2024-08-17T18:01:21.391 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3340000000000001e-06 2024-08-17T18:01:21.391 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.392 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.392 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:01:21.392 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377690+0000", 2024-08-17T18:01:21.392 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4360000000000004e-06 2024-08-17T18:01:21.392 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.392 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.392 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:01:21.392 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377700+0000", 2024-08-17T18:01:21.392 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0319000000000001e-05 2024-08-17T18:01:21.392 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.392 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.392 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:01:21.393 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377707+0000", 2024-08-17T18:01:21.393 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2790000000000004e-06 2024-08-17T18:01:21.393 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.393 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.393 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:01:21.393 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.379329+0000", 2024-08-17T18:01:21.393 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001622694 2024-08-17T18:01:21.393 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.393 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.393 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:01:21.394 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.379438+0000", 2024-08-17T18:01:21.394 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000109168 2024-08-17T18:01:21.394 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.394 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.394 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:01:21.394 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.379547+0000", 2024-08-17T18:01:21.394 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000109083 2024-08-17T18:01:21.394 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.394 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.394 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:01:21.394 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.383841+0000", 2024-08-17T18:01:21.394 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0042934169999999999 2024-08-17T18:01:21.395 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:21.395 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:01:21.395 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:21.395 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.395 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.395 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:121538 46.0 46:05e9b44a:::benchmark_data_smithi130_267720_object13555:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1275)", 2024-08-17T18:01:21.395 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:01:21.377717+0000", 2024-08-17T18:01:21.395 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0084346830000000001, 2024-08-17T18:01:21.395 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0085353819999999993, 2024-08-17T18:01:21.395 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:01:21.395 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:01:21.395 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:01:21.395 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:01:21.396 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:01:21.396 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:21.396 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 121538 2024-08-17T18:01:21.396 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.396 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:01:21.396 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.396 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:01:21.396 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377717+0000", 2024-08-17T18:01:21.396 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:01:21.396 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.396 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.396 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:01:21.396 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377717+0000", 2024-08-17T18:01:21.396 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:01:21.397 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.397 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.397 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:01:21.397 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377717+0000", 2024-08-17T18:01:21.397 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5999999999999999e-07 2024-08-17T18:01:21.397 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.397 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.397 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:01:21.397 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377719+0000", 2024-08-17T18:01:21.397 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.798e-06 2024-08-17T18:01:21.397 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.397 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.397 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:01:21.398 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377724+0000", 2024-08-17T18:01:21.398 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4500000000000003e-06 2024-08-17T18:01:21.398 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.398 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.398 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:01:21.398 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377729+0000", 2024-08-17T18:01:21.398 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0810000000000001e-06 2024-08-17T18:01:21.398 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.398 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.399 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:01:21.399 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377791+0000", 2024-08-17T18:01:21.399 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1348999999999998e-05 2024-08-17T18:01:21.399 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.399 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.399 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:01:21.399 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377917+0000", 2024-08-17T18:01:21.399 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012625599999999999 2024-08-17T18:01:21.399 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.399 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.399 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:01:21.399 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.378033+0000", 2024-08-17T18:01:21.400 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000116057 2024-08-17T18:01:21.400 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:21.400 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:01:21.400 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:21.400 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.400 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.400 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:121541 46.0 46:0a6165e9:::benchmark_data_smithi130_267720_object13558:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1275)", 2024-08-17T18:01:21.400 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:01:21.377743+0000", 2024-08-17T18:01:21.400 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0084084759999999998, 2024-08-17T18:01:21.400 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0085671580000000001, 2024-08-17T18:01:21.400 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:01:21.400 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:01:21.400 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:01:21.400 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:01:21.401 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:01:21.401 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:21.401 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 121541 2024-08-17T18:01:21.401 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.401 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:01:21.401 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.401 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:01:21.401 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377743+0000", 2024-08-17T18:01:21.401 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:01:21.401 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.401 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.401 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:01:21.401 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377743+0000", 2024-08-17T18:01:21.401 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:01:21.402 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.402 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.402 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:01:21.402 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377743+0000", 2024-08-17T18:01:21.402 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8200000000000001e-07 2024-08-17T18:01:21.402 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.402 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.402 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:01:21.402 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377745+0000", 2024-08-17T18:01:21.402 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.781e-06 2024-08-17T18:01:21.402 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.402 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.402 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377750+0000", 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2569999999999998e-06 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.377754+0000", 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1729999999999997e-06 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.381388+0000", 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0036335120000000002 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.403 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:01:21.404 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.381501+0000", 2024-08-17T18:01:21.404 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011288700000000001 2024-08-17T18:01:21.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:21.404 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:21.404 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:01:21.404 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:21.381612+0000", 2024-08-17T18:01:21.404 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011172599999999999 2024-08-17T18:01:21.404 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:21.404 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:01:21.404 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:21.404 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:21.404 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T18:01:21.404 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 3 2024-08-17T18:01:21.404 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:01:21.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:21.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:21.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:21.506 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:21.553 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:21.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:21.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:21.655 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:21.679 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:21.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:21.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:21.781 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:21.804 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:21.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:21.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:21.906 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:21.945 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:21.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:21.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:22.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:22.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:22.085 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:22.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:22.186 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:22.222 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:22.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:22.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:22.314 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:22.324 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:22.330 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:22.335 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T18:01:22.347 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:22.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:22.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:22.448 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:22.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:22.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:22.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:22.489 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:01:22.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:22 smithi130 ceph-mon[110068]: pgmap v2078: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 766 op/s 2024-08-17T18:01:22.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:22 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/305603882' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:01:22.542 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:22 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:01:22.543 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:22 smithi130 ceph-mon[105521]: pgmap v2078: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 766 op/s 2024-08-17T18:01:22.543 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:22 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/305603882' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:01:22.543 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:22 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:01:22.572 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:22.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:22.637 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:22.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:22.738 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:22.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:22.807 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:22.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:22 smithi179 ceph-mon[93519]: pgmap v2078: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 766 op/s 2024-08-17T18:01:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:22 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/305603882' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:01:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:22 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:01:22.908 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:22.952 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:22.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:22.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:23.053 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:23.076 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:23.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:23.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:23.178 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:23.241 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:23.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:23.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:23.343 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:23.383 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:23.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:23.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:23.485 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:23.508 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:23.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:23.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:23.610 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:23.650 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:23.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:23.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:23.752 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:23.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:23.783 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:23.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:23 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:01:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:23 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:01:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:23 smithi179 ceph-mon[93519]: osdmap e1276: 8 total, 8 up, 6 in 2024-08-17T18:01:23.884 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:23.910 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:23.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:23.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:23.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:23 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:01:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:23 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:01:23.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:23 smithi130 ceph-mon[105521]: osdmap e1276: 8 total, 8 up, 6 in 2024-08-17T18:01:23.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:23 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:01:23.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:23 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:01:23.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:23 smithi130 ceph-mon[110068]: osdmap e1276: 8 total, 8 up, 6 in 2024-08-17T18:01:24.012 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:24.045 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:24.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:24.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:24.147 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:24.186 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:24.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:24.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:24.288 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:24.326 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:24.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:24.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:24.427 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:24.466 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:24.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:24.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:24.568 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:24.592 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:24.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:24.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:24.694 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:24.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:24.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:24.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:24.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:24.837 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:24.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:24.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:24.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:24 smithi179 ceph-mon[93519]: pgmap v2080: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 975 op/s 2024-08-17T18:01:24.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:24 smithi130 ceph-mon[105521]: pgmap v2080: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 975 op/s 2024-08-17T18:01:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:24 smithi130 ceph-mon[110068]: pgmap v2080: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 975 op/s 2024-08-17T18:01:24.939 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:24.962 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:24.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:24.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:25.064 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:25.088 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:25.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:25.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:25.189 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:25.214 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:25.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:25.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:25.316 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:25.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:25.364 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:25.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:25.465 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:25.507 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:25.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:25.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:25.609 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:25.648 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:25.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:25.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:25.750 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:25.790 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:25.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:25.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:25.892 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:25.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:25.939 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:25.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:26.040 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:26.063 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:26.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:26.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:26.165 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:26.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:26.211 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:26.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:26.312 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:26.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:26.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:26.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:26.437 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:26.483 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:26.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:26.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:26.585 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:26.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:26.609 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:26.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:26.710 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:26.769 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:26.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:26.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:26.813 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:26 smithi130 ceph-mon[105521]: pgmap v2081: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 921 op/s 2024-08-17T18:01:26.813 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:26 smithi130 ceph-mon[110068]: pgmap v2081: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 921 op/s 2024-08-17T18:01:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:26 smithi179 ceph-mon[93519]: pgmap v2081: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 921 op/s 2024-08-17T18:01:26.874 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:26.915 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:26.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:26.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:27.017 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:27.064 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:27.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:27.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:27.166 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:27.189 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:27.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:27.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:27.291 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:27.337 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:27.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:27.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:27.439 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:27.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:27.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:27.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:27.564 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:27.614 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:27.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:27.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:01:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:01:27.716 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:27.739 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:27.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:27.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:27.841 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:01:27.864 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:27.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:27.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:27.880 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:27.967 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:27.989 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:27.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:27.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:28.091 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:28.114 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:28.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:28.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:28.216 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:28.228 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:01:28.228 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:01:28.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:28.260 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:28.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:28.271 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:01:28.271 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:01:28.314 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:01:28.315 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:01:28.360 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:01:28.360 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:01:28.362 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:28.385 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:28.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:28.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:28.387 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:01:28.387 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:01:28.416 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:01:28.417 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:01:28.458 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:01:28.458 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:01:28.486 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:28.502 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:01:28.502 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:01:28.509 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:28.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:28.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:28.572 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T18:01:28.610 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:28.647 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:28.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:28.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:28.725 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:01:28.749 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:28.772 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:28.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:28.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:28 smithi179 ceph-mon[93519]: pgmap v2082: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 836 op/s 2024-08-17T18:01:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:28 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3403240557' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:01:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:28 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:01:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:28 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2037139692' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:01:28.878 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:28 smithi130 ceph-mon[105521]: pgmap v2082: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 836 op/s 2024-08-17T18:01:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:28 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3403240557' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:01:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:28 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:01:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:28 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2037139692' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:01:28.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:28 smithi130 ceph-mon[110068]: pgmap v2082: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 836 op/s 2024-08-17T18:01:28.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:28 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3403240557' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:01:28.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:28 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:01:28.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:28 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2037139692' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:01:28.923 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:28.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:28.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:29.025 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:29.046 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:29.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:29.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:29.148 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:29.189 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:29.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:29.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:29.291 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:29.317 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:29.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:29.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:29.419 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:29.442 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:29.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:29.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:29.544 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:29.593 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:29.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:29.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:29.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:29 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:01:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:29 smithi130 ceph-mon[105521]: osdmap e1277: 8 total, 8 up, 6 in 2024-08-17T18:01:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:29 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:01:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:29 smithi130 ceph-mon[110068]: osdmap e1277: 8 total, 8 up, 6 in 2024-08-17T18:01:29.695 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:29.718 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:29.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:29.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:29.820 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:29.839 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T18:01:29.839 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:01:29.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:29 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:01:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:29 smithi179 ceph-mon[93519]: osdmap e1277: 8 total, 8 up, 6 in 2024-08-17T18:01:29.843 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:29.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:29.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:29.947 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:29.989 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:29.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:29.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:30.090 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:30.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:30.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:30.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:30.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:01:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:01:29] "GET /metrics HTTP/1.1" 200 36783 "" "Prometheus/2.51.0" 2024-08-17T18:01:30.215 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:30.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:30.262 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:30.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:30.363 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:30.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:30.408 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:30.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:30.509 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:30.533 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:30.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:30.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:30.578 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:01:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:01:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:01:30.619 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:30 smithi130 ceph-mon[110068]: pgmap v2084: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 985 op/s 2024-08-17T18:01:30.619 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:30 smithi130 ceph-mon[105521]: pgmap v2084: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 985 op/s 2024-08-17T18:01:30.635 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:30.657 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:30.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:30.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:30.759 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:30.797 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:30.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:30.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:30 smithi179 ceph-mon[93519]: pgmap v2084: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 985 op/s 2024-08-17T18:01:30.899 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:30.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:30.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:30.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:31.058 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:31.082 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:31.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:31.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:31.184 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:31.221 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:31.228 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:31.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:31.245 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:31.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:31.346 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:31.371 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:31.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:31.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:31.473 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:31.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:31.496 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:31.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:31.597 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:31.639 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:31.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:31.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:31.741 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:31.787 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:31.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:31.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:31.889 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:31.928 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:31.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:31.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:32.030 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:32.053 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:32.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:32.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:32.155 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:32.196 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:32.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:32.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:32.298 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:32.336 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:32.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:32.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:32.438 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:32.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:32.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:32.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:32.563 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:32.606 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:32.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:32.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:32.661 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:01:32.665 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:32 smithi130 ceph-mon[105521]: pgmap v2085: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 848 op/s 2024-08-17T18:01:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:32 smithi130 ceph-mon[110068]: pgmap v2085: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 848 op/s 2024-08-17T18:01:32.709 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:32.732 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:32.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:32.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:32.834 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:32.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:32 smithi179 ceph-mon[93519]: pgmap v2085: 89 pgs: 89 active+clean; 5.5 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 848 op/s 2024-08-17T18:01:32.859 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:32.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:32.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:32.961 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:33.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:33.014 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:33.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:33.115 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:33.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:33.143 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:33.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:33.244 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:33.281 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:33.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:33.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:33.383 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:33.417 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:33.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:33.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:33.519 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:33.544 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:33.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:33.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:33.601 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:01:33.601 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:01:33.646 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:33.683 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:33.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:33.785 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:33.808 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:33.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:33.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:33.910 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:33.952 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:33.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:33.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:34.054 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:34.078 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:34.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:34.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:34.180 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:34.203 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:34.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:34.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:34.305 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:34.344 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:34.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:34.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:34.392 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T18:01:34.446 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:34.469 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:34.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:34.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:34.571 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:34.594 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:34.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:34.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:34.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:34 smithi130 ceph-mon[105521]: pgmap v2086: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 810 op/s 2024-08-17T18:01:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:34 smithi130 ceph-mon[110068]: pgmap v2086: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 810 op/s 2024-08-17T18:01:34.696 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:34.750 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:34.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:34.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:34 smithi179 ceph-mon[93519]: pgmap v2086: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 810 op/s 2024-08-17T18:01:34.852 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:34.915 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:34.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:34.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:35.017 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:35.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:35.067 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:35.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:35.168 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:35.191 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:35.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:35.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:35.293 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:35.315 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:35.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:35.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:35.417 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:35.442 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:35.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:35.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:35.543 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:35.610 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:35.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:35.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:35.712 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:35.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:35.760 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:35.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:35.835 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:35.861 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:35.914 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:35.924 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:35.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:35.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:36.026 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:36.080 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:36.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:36.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:36.182 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:36.207 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:36.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:36.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:36.213 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:01:36.216 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:01:36.309 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:36.345 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:36.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:36.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:36.447 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:36.484 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:36.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:36.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:36.586 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:36.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:36.609 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:36.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:36.677 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:36 smithi130 ceph-mon[105521]: pgmap v2087: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 809 op/s 2024-08-17T18:01:36.677 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:36 smithi130 ceph-mon[110068]: pgmap v2087: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 809 op/s 2024-08-17T18:01:36.710 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:36.734 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:36.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:36.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:36.835 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:36.857 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:36.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:36.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:36.959 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:36.981 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:36.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:36.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:37.083 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:37.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:36 smithi179 ceph-mon[93519]: pgmap v2087: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 809 op/s 2024-08-17T18:01:37.106 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:37.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:37.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:37.208 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:37.249 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:37.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:37.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:37.351 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:37.393 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:37.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:37.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:37.495 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:37.519 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:37.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:37.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:37.622 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:37.663 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:37.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:37.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:37.764 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:37.788 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:37.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:37.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:37.890 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:37.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:37 smithi130 ceph-mon[105521]: from='client.36321 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:01:37.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:37 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:01:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:37 smithi130 ceph-mon[110068]: from='client.36321 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:01:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:37 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:01:37.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:37.930 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:37.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:38.031 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:38.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:38.083 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:38.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:37 smithi179 ceph-mon[93519]: from='client.36321 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:01:38.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:37 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:01:38.186 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:38.209 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:38.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:38.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:38.311 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:38.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:38.337 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:38.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:38.438 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:38.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:38.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:38.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:38.563 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:38.606 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:38.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:38.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:38.708 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:38.752 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:38.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:38.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:38 smithi179 ceph-mon[93519]: pgmap v2088: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 812 op/s 2024-08-17T18:01:38.854 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:38.889 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:38 smithi130 ceph-mon[105521]: pgmap v2088: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 812 op/s 2024-08-17T18:01:38.889 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:38 smithi130 ceph-mon[110068]: pgmap v2088: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 812 op/s 2024-08-17T18:01:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:38.895 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:38.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:38.996 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:39.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:39.036 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:39.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:39.137 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:39.180 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:39.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:39.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:39.282 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:39.314 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:39.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:39.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:39.416 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:39.437 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:39.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:39.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:39.539 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:39.564 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:39.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:39.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:39.665 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:39.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:39.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:39.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:39.793 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:39.820 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:39.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:39.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:39.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:39.959 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:39.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:39.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:40.061 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:40.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:40.084 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:40.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:40.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:01:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:01:39] "GET /metrics HTTP/1.1" 200 36783 "" "Prometheus/2.51.0" 2024-08-17T18:01:40.184 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:40.210 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:40.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:40.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:40.312 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:40.338 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:40.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:40.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:40.440 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:40.463 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:40.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:40.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:40.565 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:01:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:01:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:01:40.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:40.626 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:40.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:40.727 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:40.762 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:40 smithi130 ceph-mon[105521]: pgmap v2089: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 768 op/s 2024-08-17T18:01:40.762 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:40 smithi130 ceph-mon[110068]: pgmap v2089: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 768 op/s 2024-08-17T18:01:40.763 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2087,"stamp":"2024-08-17T18:01:35.187298+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5587867621,"num_objects":85282,"num_object_clones":0,"num_object_copies":255846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85282,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":130984,"num_write_kb":6913168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":1973261,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105513,"ondisk_log_size":105513,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":18558616,"kb_used_data":16465636,"kb_used_omap":295,"kb_used_meta":2092568,"kb_avail":543912296,"statfs":{"total":575970213888,"available":556966191104,"internally_reserved":0,"allocated":16860811264,"data_stored":16837677795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":2142790095},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-530907136,"num_objects":-8101,"num_object_clones":0,"num_object_copies":-24303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8101,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003166"},"pg_stats":[{"pgid":"46.f","version":"1277'8279","reported_seq":8306,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:32.982012+0000","last_change":"2024-08-17T17:59:22.956823+0000","last_active":"2024-08-17T18:01:32.982012+0000","last_peered":"2024-08-17T18:01:32.982012+0000","last_clean":"2024-08-17T18:01:32.982012+0000","last_became_active":"2024-08-17T17:59:22.956437+0000","last_became_peered":"2024-08-17T17:59:22.956437+0000","last_unstale":"2024-08-17T18:01:32.982012+0000","last_undegraded":"2024-08-17T18:01:32.982012+0000","last_fullsized":"2024-08-17T18:01:32.982012+0000","mapping_epoch":1258,"log_start":"1262'1400","ondisk_log_start":"1262'1400","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6879,"log_dups_size":0,"ondisk_log_size":6879,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:17.161240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356319232,"num_objects":5437,"num_object_clones":0,"num_object_copies":16311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8279,"num_write_kb":438912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.e","version":"1277'8160","reported_seq":8207,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:33.014041+0000","last_change":"2024-08-17T17:59:58.047028+0000","last_active":"2024-08-17T18:01:33.014041+0000","last_peered":"2024-08-17T18:01:33.014041+0000","last_clean":"2024-08-17T18:01:33.014041+0000","last_became_active":"2024-08-17T17:59:58.027451+0000","last_became_peered":"2024-08-17T17:59:58.027451+0000","last_unstale":"2024-08-17T18:01:33.014041+0000","last_undegraded":"2024-08-17T18:01:33.014041+0000","last_fullsized":"2024-08-17T18:01:33.014041+0000","mapping_epoch":1263,"log_start":"1261'1100","ondisk_log_start":"1261'1100","created":1258,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":7060,"log_dups_size":0,"ondisk_log_size":7060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:20:25.306207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345505792,"num_objects":5272,"num_object_clones":0,"num_object_copies":15816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8160,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.d","version":"1277'8304","reported_seq":8354,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:32.676968+0000","last_change":"2024-08-17T17:59:22.954900+0000","last_active":"2024-08-17T18:01:32.676968+0000","last_peered":"2024-08-17T18:01:32.676968+0000","last_clean":"2024-08-17T18:01:32.676968+0000","last_became_active":"2024-08-17T17:59:22.954599+0000","last_became_peered":"2024-08-17T17:59:22.954599+0000","last_unstale":"2024-08-17T18:01:32.676968+0000","last_undegraded":"2024-08-17T18:01:32.676968+0000","last_fullsized":"2024-08-17T18:01:32.676968+0000","mapping_epoch":1258,"log_start":"1262'1700","ondisk_log_start":"1262'1700","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6604,"log_dups_size":0,"ondisk_log_size":6604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:22:00.453498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355270680,"num_objects":5422,"num_object_clones":0,"num_object_copies":16266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5422,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8304,"num_write_kb":439169,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.c","version":"1277'8224","reported_seq":8251,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:33.009142+0000","last_change":"2024-08-17T17:59:22.951540+0000","last_active":"2024-08-17T18:01:33.009142+0000","last_peered":"2024-08-17T18:01:33.009142+0000","last_clean":"2024-08-17T18:01:33.009142+0000","last_became_active":"2024-08-17T17:59:22.951248+0000","last_became_peered":"2024-08-17T17:59:22.951248+0000","last_unstale":"2024-08-17T18:01:33.009142+0000","last_undegraded":"2024-08-17T18:01:33.009142+0000","last_fullsized":"2024-08-17T18:01:33.009142+0000","mapping_epoch":1258,"log_start":"1262'1400","ondisk_log_start":"1262'1400","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6824,"log_dups_size":0,"ondisk_log_size":6824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:55.458182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352321536,"num_objects":5376,"num_object_clones":0,"num_object_copies":16128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8224,"num_write_kb":435200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.b","version":"1277'8077","reported_seq":8126,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:32.658394+0000","last_change":"2024-08-17T17:59:22.957875+0000","last_active":"2024-08-17T18:01:32.658394+0000","last_peered":"2024-08-17T18:01:32.658394+0000","last_clean":"2024-08-17T18:01:32.658394+0000","last_became_active":"2024-08-17T17:59:22.957601+0000","last_became_peered":"2024-08-17T17:59:22.957601+0000","last_unstale":"2024-08-17T18:01:32.658394+0000","last_undegraded":"2024-08-17T18:01:32.658394+0000","last_fullsized":"2024-08-17T18:01:32.658394+0000","mapping_epoch":1258,"log_start":"1262'1500","ondisk_log_start":"1262'1500","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6577,"log_dups_size":0,"ondisk_log_size":6577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:40.456894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345964544,"num_objects":5279,"num_object_clones":0,"num_object_copies":15837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8077,"num_write_kb":427392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.a","version":"1277'8218","reported_seq":8245,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:33.010002+0000","last_change":"2024-08-17T17:59:22.951416+0000","last_active":"2024-08-17T18:01:33.010002+0000","last_peered":"2024-08-17T18:01:33.010002+0000","last_clean":"2024-08-17T18:01:33.010002+0000","last_became_active":"2024-08-17T17:59:22.951110+0000","last_became_peered":"2024-08-17T17:59:22.951110+0000","last_unstale":"2024-08-17T18:01:33.010002+0000","last_undegraded":"2024-08-17T18:01:33.010002+0000","last_fullsized":"2024-08-17T18:01:33.010002+0000","mapping_epoch":1258,"log_start":"1261'1200","ondisk_log_start":"1261'1200","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":7018,"log_dups_size":0,"ondisk_log_size":7018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:32.365091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351272960,"num_objects":5360,"num_object_clones":0,"num_object_copies":16080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8218,"num_write_kb":434496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.9","version":"1277'8032","reported_seq":8082,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:32.656381+0000","last_change":"2024-08-17T17:59:22.957896+0000","last_active":"2024-08-17T18:01:32.656381+0000","last_peered":"2024-08-17T18:01:32.656381+0000","last_clean":"2024-08-17T18:01:32.656381+0000","last_became_active":"2024-08-17T17:59:22.957648+0000","last_became_peered":"2024-08-17T17:59:22.957648+0000","last_unstale":"2024-08-17T18:01:32.656381+0000","last_undegraded":"2024-08-17T18:01:32.656381+0000","last_fullsized":"2024-08-17T18:01:32.656381+0000","mapping_epoch":1258,"log_start":"1262'1500","ondisk_log_start":"1262'1500","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6532,"log_dups_size":0,"ondisk_log_size":6532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:11.940932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344850432,"num_objects":5262,"num_object_clones":0,"num_object_copies":15786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8032,"num_write_kb":425408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.8","version":"1277'8140","reported_seq":8189,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:31.571486+0000","last_change":"2024-08-17T17:59:22.951248+0000","last_active":"2024-08-17T18:01:31.571486+0000","last_peered":"2024-08-17T18:01:31.571486+0000","last_clean":"2024-08-17T18:01:31.571486+0000","last_became_active":"2024-08-17T17:59:22.950981+0000","last_became_peered":"2024-08-17T17:59:22.950981+0000","last_unstale":"2024-08-17T18:01:31.571486+0000","last_undegraded":"2024-08-17T18:01:31.571486+0000","last_fullsized":"2024-08-17T18:01:31.571486+0000","mapping_epoch":1258,"log_start":"1262'2200","ondisk_log_start":"1262'2200","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":5940,"log_dups_size":0,"ondisk_log_size":5940,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:24:07.143610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348389376,"num_objects":5316,"num_object_clones":0,"num_object_copies":15948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8140,"num_write_kb":430592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.7","version":"1277'8162","reported_seq":8222,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:31.533848+0000","last_change":"2024-08-17T17:59:58.050355+0000","last_active":"2024-08-17T18:01:31.533848+0000","last_peered":"2024-08-17T18:01:31.533848+0000","last_clean":"2024-08-17T18:01:31.533848+0000","last_became_active":"2024-08-17T17:59:58.022280+0000","last_became_peered":"2024-08-17T17:59:58.022280+0000","last_unstale":"2024-08-17T18:01:31.533848+0000","last_undegraded":"2024-08-17T18:01:31.533848+0000","last_fullsized":"2024-08-17T18:01:31.533848+0000","mapping_epoch":1263,"log_start":"1262'2200","ondisk_log_start":"1262'2200","created":1258,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":5962,"log_dups_size":0,"ondisk_log_size":5962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:07.979130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352059392,"num_objects":5372,"num_object_clones":0,"num_object_copies":16116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8162,"num_write_kb":433088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.6","version":"1277'8169","reported_seq":8196,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:32.981922+0000","last_change":"2024-08-17T17:59:22.958347+0000","last_active":"2024-08-17T18:01:32.981922+0000","last_peered":"2024-08-17T18:01:32.981922+0000","last_clean":"2024-08-17T18:01:32.981922+0000","last_became_active":"2024-08-17T17:59:22.957969+0000","last_became_peered":"2024-08-17T17:59:22.957969+0000","last_unstale":"2024-08-17T18:01:32.981922+0000","last_undegraded":"2024-08-17T18:01:32.981922+0000","last_fullsized":"2024-08-17T18:01:32.981922+0000","mapping_epoch":1258,"log_start":"1261'1300","ondisk_log_start":"1261'1300","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6869,"log_dups_size":0,"ondisk_log_size":6869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:07.728594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343998464,"num_objects":5249,"num_object_clones":0,"num_object_copies":15747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8169,"num_write_kb":429376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.5","version":"1277'8262","reported_seq":8311,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:34.177537+0000","last_change":"2024-08-17T17:59:22.956504+0000","last_active":"2024-08-17T18:01:34.177537+0000","last_peered":"2024-08-17T18:01:34.177537+0000","last_clean":"2024-08-17T18:01:34.177537+0000","last_became_active":"2024-08-17T17:59:22.955747+0000","last_became_peered":"2024-08-17T17:59:22.955747+0000","last_unstale":"2024-08-17T18:01:34.177537+0000","last_undegraded":"2024-08-17T18:01:34.177537+0000","last_fullsized":"2024-08-17T18:01:34.177537+0000","mapping_epoch":1258,"log_start":"1262'2100","ondisk_log_start":"1262'2100","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6162,"log_dups_size":0,"ondisk_log_size":6162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:54:51.990739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343408640,"num_objects":5240,"num_object_clones":0,"num_object_copies":15720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8262,"num_write_kb":432064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"46.4","version":"1277'8002","reported_seq":8029,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:32.973439+0000","last_change":"2024-08-17T17:59:22.952797+0000","last_active":"2024-08-17T18:01:32.973439+0000","last_peered":"2024-08-17T18:01:32.973439+0000","last_clean":"2024-08-17T18:01:32.973439+0000","last_became_active":"2024-08-17T17:59:22.952475+0000","last_became_peered":"2024-08-17T17:59:22.952475+0000","last_unstale":"2024-08-17T18:01:32.973439+0000","last_undegraded":"2024-08-17T18:01:32.973439+0000","last_fullsized":"2024-08-17T18:01:32.973439+0000","mapping_epoch":1258,"log_start":"1261'1000","ondisk_log_start":"1261'1000","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":7002,"log_dups_size":0,"ondisk_log_size":7002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:42:19.302395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345505792,"num_objects":5272,"num_object_clones":0,"num_object_copies":15816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8002,"num_write_kb":424768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.3","version":"1277'8033","reported_seq":8082,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:31.579045+0000","last_change":"2024-08-17T17:59:22.954906+0000","last_active":"2024-08-17T18:01:31.579045+0000","last_peered":"2024-08-17T18:01:31.579045+0000","last_clean":"2024-08-17T18:01:31.579045+0000","last_became_active":"2024-08-17T17:59:22.954623+0000","last_became_peered":"2024-08-17T17:59:22.954623+0000","last_unstale":"2024-08-17T18:01:31.579045+0000","last_undegraded":"2024-08-17T18:01:31.579045+0000","last_fullsized":"2024-08-17T18:01:31.579045+0000","mapping_epoch":1258,"log_start":"1262'2100","ondisk_log_start":"1262'2100","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":5933,"log_dups_size":0,"ondisk_log_size":5933,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:19.582387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351207424,"num_objects":5359,"num_object_clones":0,"num_object_copies":16077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8033,"num_write_kb":428544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.2","version":"1277'8254","reported_seq":8304,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:32.671295+0000","last_change":"2024-08-17T17:59:22.956809+0000","last_active":"2024-08-17T18:01:32.671295+0000","last_peered":"2024-08-17T18:01:32.671295+0000","last_clean":"2024-08-17T18:01:32.671295+0000","last_became_active":"2024-08-17T17:59:22.956543+0000","last_became_peered":"2024-08-17T17:59:22.956543+0000","last_unstale":"2024-08-17T18:01:32.671295+0000","last_undegraded":"2024-08-17T18:01:32.671295+0000","last_fullsized":"2024-08-17T18:01:32.671295+0000","mapping_epoch":1258,"log_start":"1262'1700","ondisk_log_start":"1262'1700","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6554,"log_dups_size":0,"ondisk_log_size":6554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:31:37.454620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352714752,"num_objects":5382,"num_object_clones":0,"num_object_copies":16146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8254,"num_write_kb":436352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.1","version":"1277'8259","reported_seq":8286,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:32.867907+0000","last_change":"2024-08-17T17:59:22.954701+0000","last_active":"2024-08-17T18:01:32.867907+0000","last_peered":"2024-08-17T18:01:32.867907+0000","last_clean":"2024-08-17T18:01:32.867907+0000","last_became_active":"2024-08-17T17:59:22.954510+0000","last_became_peered":"2024-08-17T17:59:22.954510+0000","last_unstale":"2024-08-17T18:01:32.867907+0000","last_undegraded":"2024-08-17T18:01:32.867907+0000","last_fullsized":"2024-08-17T18:01:32.867907+0000","mapping_epoch":1258,"log_start":"1261'1400","ondisk_log_start":"1261'1400","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6859,"log_dups_size":0,"ondisk_log_size":6859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:19:01.061765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347799552,"num_objects":5307,"num_object_clones":0,"num_object_copies":15921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8259,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.0","version":"1277'8131","reported_seq":8191,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:32.656073+0000","last_change":"2024-08-17T17:59:58.062636+0000","last_active":"2024-08-17T18:01:32.656073+0000","last_peered":"2024-08-17T18:01:32.656073+0000","last_clean":"2024-08-17T18:01:32.656073+0000","last_became_active":"2024-08-17T17:59:58.032875+0000","last_became_peered":"2024-08-17T17:59:58.032875+0000","last_unstale":"2024-08-17T18:01:32.656073+0000","last_undegraded":"2024-08-17T18:01:32.656073+0000","last_fullsized":"2024-08-17T18:01:32.656073+0000","mapping_epoch":1263,"log_start":"1262'1600","ondisk_log_start":"1262'1600","created":1258,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6531,"log_dups_size":0,"ondisk_log_size":6531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:24.031388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350683136,"num_objects":5351,"num_object_clones":0,"num_object_copies":16053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8131,"num_write_kb":431424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":2014,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.604770+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T18:01:28.604770+0000","last_peered":"2024-08-17T18:01:28.604770+0000","last_clean":"2024-08-17T18:01:28.604770+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T18:01:28.604770+0000","last_undegraded":"2024-08-17T18:01:28.604770+0000","last_fullsized":"2024-08-17T18:01:28.604770+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":554,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.496531+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T18:01:22.496531+0000","last_peered":"2024-08-17T18:01:22.496531+0000","last_clean":"2024-08-17T18:01:22.496531+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T18:01:22.496531+0000","last_undegraded":"2024-08-17T18:01:22.496531+0000","last_fullsized":"2024-08-17T18:01:22.496531+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1933,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.600728+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T18:01:28.600728+0000","last_peered":"2024-08-17T18:01:28.600728+0000","last_clean":"2024-08-17T18:01:28.600728+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T18:01:28.600728+0000","last_undegraded":"2024-08-17T18:01:28.600728+0000","last_fullsized":"2024-08-17T18:01:28.600728+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1957,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.593188+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T18:01:28.593188+0000","last_peered":"2024-08-17T18:01:28.593188+0000","last_clean":"2024-08-17T18:01:28.593188+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T18:01:28.593188+0000","last_undegraded":"2024-08-17T18:01:28.593188+0000","last_fullsized":"2024-08-17T18:01:28.593188+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":116,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497058+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T18:01:22.497058+0000","last_peered":"2024-08-17T18:01:22.497058+0000","last_clean":"2024-08-17T18:01:22.497058+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T18:01:22.497058+0000","last_undegraded":"2024-08-17T18:01:22.497058+0000","last_fullsized":"2024-08-17T18:01:22.497058+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1350,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.592022+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T18:01:28.592022+0000","last_peered":"2024-08-17T18:01:28.592022+0000","last_clean":"2024-08-17T18:01:28.592022+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T18:01:28.592022+0000","last_undegraded":"2024-08-17T18:01:28.592022+0000","last_fullsized":"2024-08-17T18:01:28.592022+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":208,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.572687+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T18:01:28.572687+0000","last_peered":"2024-08-17T18:01:28.572687+0000","last_clean":"2024-08-17T18:01:28.572687+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T18:01:28.572687+0000","last_undegraded":"2024-08-17T18:01:28.572687+0000","last_fullsized":"2024-08-17T18:01:28.572687+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":728,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.592427+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T18:01:28.592427+0000","last_peered":"2024-08-17T18:01:28.592427+0000","last_clean":"2024-08-17T18:01:28.592427+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T18:01:28.592427+0000","last_undegraded":"2024-08-17T18:01:28.592427+0000","last_fullsized":"2024-08-17T18:01:28.592427+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1972,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.600442+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T18:01:28.600442+0000","last_peered":"2024-08-17T18:01:28.600442+0000","last_clean":"2024-08-17T18:01:28.600442+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T18:01:28.600442+0000","last_undegraded":"2024-08-17T18:01:28.600442+0000","last_fullsized":"2024-08-17T18:01:28.600442+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":582,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.573071+0000","last_change":"2024-08-17T17:59:58.027770+0000","last_active":"2024-08-17T18:01:28.573071+0000","last_peered":"2024-08-17T18:01:28.573071+0000","last_clean":"2024-08-17T18:01:28.573071+0000","last_became_active":"2024-08-17T17:59:58.027251+0000","last_became_peered":"2024-08-17T17:59:58.027251+0000","last_unstale":"2024-08-17T18:01:28.573071+0000","last_undegraded":"2024-08-17T18:01:28.573071+0000","last_fullsized":"2024-08-17T18:01:28.573071+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:26.416690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":193,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.572683+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T18:01:28.572683+0000","last_peered":"2024-08-17T18:01:28.572683+0000","last_clean":"2024-08-17T18:01:28.572683+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T18:01:28.572683+0000","last_undegraded":"2024-08-17T18:01:28.572683+0000","last_fullsized":"2024-08-17T18:01:28.572683+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1945,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.591542+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T18:01:28.591542+0000","last_peered":"2024-08-17T18:01:28.591542+0000","last_clean":"2024-08-17T18:01:28.591542+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T18:01:28.591542+0000","last_undegraded":"2024-08-17T18:01:28.591542+0000","last_fullsized":"2024-08-17T18:01:28.591542+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1848,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.593440+0000","last_change":"2024-08-17T17:59:58.044651+0000","last_active":"2024-08-17T18:01:28.593440+0000","last_peered":"2024-08-17T18:01:28.593440+0000","last_clean":"2024-08-17T18:01:28.593440+0000","last_became_active":"2024-08-17T17:59:58.043830+0000","last_became_peered":"2024-08-17T17:59:58.043830+0000","last_unstale":"2024-08-17T18:01:28.593440+0000","last_undegraded":"2024-08-17T18:01:28.593440+0000","last_fullsized":"2024-08-17T18:01:28.593440+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:56.568577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1995,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.599586+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T18:01:28.599586+0000","last_peered":"2024-08-17T18:01:28.599586+0000","last_clean":"2024-08-17T18:01:28.599586+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T18:01:28.599586+0000","last_undegraded":"2024-08-17T18:01:28.599586+0000","last_fullsized":"2024-08-17T18:01:28.599586+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1795,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.604391+0000","last_change":"2024-08-17T17:59:58.028639+0000","last_active":"2024-08-17T18:01:28.604391+0000","last_peered":"2024-08-17T18:01:28.604391+0000","last_clean":"2024-08-17T18:01:28.604391+0000","last_became_active":"2024-08-17T17:59:58.028314+0000","last_became_peered":"2024-08-17T17:59:58.028314+0000","last_unstale":"2024-08-17T18:01:28.604391+0000","last_undegraded":"2024-08-17T18:01:28.604391+0000","last_fullsized":"2024-08-17T18:01:28.604391+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:25:03.619052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1251,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497310+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T18:01:22.497310+0000","last_peered":"2024-08-17T18:01:22.497310+0000","last_clean":"2024-08-17T18:01:22.497310+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T18:01:22.497310+0000","last_undegraded":"2024-08-17T18:01:22.497310+0000","last_fullsized":"2024-08-17T18:01:22.497310+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1939,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.597983+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T18:01:28.597983+0000","last_peered":"2024-08-17T18:01:28.597983+0000","last_clean":"2024-08-17T18:01:28.597983+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T18:01:28.597983+0000","last_undegraded":"2024-08-17T18:01:28.597983+0000","last_fullsized":"2024-08-17T18:01:28.597983+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1989,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.606786+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T18:01:28.606786+0000","last_peered":"2024-08-17T18:01:28.606786+0000","last_clean":"2024-08-17T18:01:28.606786+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T18:01:28.606786+0000","last_undegraded":"2024-08-17T18:01:28.606786+0000","last_fullsized":"2024-08-17T18:01:28.606786+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1895,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.593125+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T18:01:28.593125+0000","last_peered":"2024-08-17T18:01:28.593125+0000","last_clean":"2024-08-17T18:01:28.593125+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T18:01:28.593125+0000","last_undegraded":"2024-08-17T18:01:28.593125+0000","last_fullsized":"2024-08-17T18:01:28.593125+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":570,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.498143+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T18:01:22.498143+0000","last_peered":"2024-08-17T18:01:22.498143+0000","last_clean":"2024-08-17T18:01:22.498143+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T18:01:22.498143+0000","last_undegraded":"2024-08-17T18:01:22.498143+0000","last_fullsized":"2024-08-17T18:01:22.498143+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1247,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497948+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T18:01:22.497948+0000","last_peered":"2024-08-17T18:01:22.497948+0000","last_clean":"2024-08-17T18:01:22.497948+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T18:01:22.497948+0000","last_undegraded":"2024-08-17T18:01:22.497948+0000","last_fullsized":"2024-08-17T18:01:22.497948+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1994,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.572999+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T18:01:28.572999+0000","last_peered":"2024-08-17T18:01:28.572999+0000","last_clean":"2024-08-17T18:01:28.572999+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T18:01:28.572999+0000","last_undegraded":"2024-08-17T18:01:28.572999+0000","last_fullsized":"2024-08-17T18:01:28.572999+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":408,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497908+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T18:01:22.497908+0000","last_peered":"2024-08-17T18:01:22.497908+0000","last_clean":"2024-08-17T18:01:22.497908+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T18:01:22.497908+0000","last_undegraded":"2024-08-17T18:01:22.497908+0000","last_fullsized":"2024-08-17T18:01:22.497908+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":554,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.496872+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T18:01:22.496872+0000","last_peered":"2024-08-17T18:01:22.496872+0000","last_clean":"2024-08-17T18:01:22.496872+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T18:01:22.496872+0000","last_undegraded":"2024-08-17T18:01:22.496872+0000","last_fullsized":"2024-08-17T18:01:22.496872+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1232,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.496795+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T18:01:22.496795+0000","last_peered":"2024-08-17T18:01:22.496795+0000","last_clean":"2024-08-17T18:01:22.496795+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T18:01:22.496795+0000","last_undegraded":"2024-08-17T18:01:22.496795+0000","last_fullsized":"2024-08-17T18:01:22.496795+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1240,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497312+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T18:01:22.497312+0000","last_peered":"2024-08-17T18:01:22.497312+0000","last_clean":"2024-08-17T18:01:22.497312+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T18:01:22.497312+0000","last_undegraded":"2024-08-17T18:01:22.497312+0000","last_fullsized":"2024-08-17T18:01:22.497312+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":2042,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.599342+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T18:01:28.599342+0000","last_peered":"2024-08-17T18:01:28.599342+0000","last_clean":"2024-08-17T18:01:28.599342+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T18:01:28.599342+0000","last_undegraded":"2024-08-17T18:01:28.599342+0000","last_fullsized":"2024-08-17T18:01:28.599342+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1925,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.598243+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T18:01:28.598243+0000","last_peered":"2024-08-17T18:01:28.598243+0000","last_clean":"2024-08-17T18:01:28.598243+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T18:01:28.598243+0000","last_undegraded":"2024-08-17T18:01:28.598243+0000","last_fullsized":"2024-08-17T18:01:28.598243+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":2000,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.573407+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T18:01:28.573407+0000","last_peered":"2024-08-17T18:01:28.573407+0000","last_clean":"2024-08-17T18:01:28.573407+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T18:01:28.573407+0000","last_undegraded":"2024-08-17T18:01:28.573407+0000","last_fullsized":"2024-08-17T18:01:28.573407+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1927,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.596799+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T18:01:28.596799+0000","last_peered":"2024-08-17T18:01:28.596799+0000","last_clean":"2024-08-17T18:01:28.596799+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T18:01:28.596799+0000","last_undegraded":"2024-08-17T18:01:28.596799+0000","last_fullsized":"2024-08-17T18:01:28.596799+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1960,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.600527+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T18:01:28.600527+0000","last_peered":"2024-08-17T18:01:28.600527+0000","last_clean":"2024-08-17T18:01:28.600527+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T18:01:28.600527+0000","last_undegraded":"2024-08-17T18:01:28.600527+0000","last_fullsized":"2024-08-17T18:01:28.600527+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1267,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.498036+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T18:01:22.498036+0000","last_peered":"2024-08-17T18:01:22.498036+0000","last_clean":"2024-08-17T18:01:22.498036+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T18:01:22.498036+0000","last_undegraded":"2024-08-17T18:01:22.498036+0000","last_fullsized":"2024-08-17T18:01:22.498036+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":2006,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.572873+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T18:01:28.572873+0000","last_peered":"2024-08-17T18:01:28.572873+0000","last_clean":"2024-08-17T18:01:28.572873+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T18:01:28.572873+0000","last_undegraded":"2024-08-17T18:01:28.572873+0000","last_fullsized":"2024-08-17T18:01:28.572873+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":193,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.572586+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T18:01:28.572586+0000","last_peered":"2024-08-17T18:01:28.572586+0000","last_clean":"2024-08-17T18:01:28.572586+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T18:01:28.572586+0000","last_undegraded":"2024-08-17T18:01:28.572586+0000","last_fullsized":"2024-08-17T18:01:28.572586+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2025,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.572781+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T18:01:28.572781+0000","last_peered":"2024-08-17T18:01:28.572781+0000","last_clean":"2024-08-17T18:01:28.572781+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T18:01:28.572781+0000","last_undegraded":"2024-08-17T18:01:28.572781+0000","last_fullsized":"2024-08-17T18:01:28.572781+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":762,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.591427+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T18:01:28.591427+0000","last_peered":"2024-08-17T18:01:28.591427+0000","last_clean":"2024-08-17T18:01:28.591427+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T18:01:28.591427+0000","last_undegraded":"2024-08-17T18:01:28.591427+0000","last_fullsized":"2024-08-17T18:01:28.591427+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":2006,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.572954+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T18:01:28.572954+0000","last_peered":"2024-08-17T18:01:28.572954+0000","last_clean":"2024-08-17T18:01:28.572954+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T18:01:28.572954+0000","last_undegraded":"2024-08-17T18:01:28.572954+0000","last_fullsized":"2024-08-17T18:01:28.572954+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1335,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.600226+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T18:01:28.600226+0000","last_peered":"2024-08-17T18:01:28.600226+0000","last_clean":"2024-08-17T18:01:28.600226+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T18:01:28.600226+0000","last_undegraded":"2024-08-17T18:01:28.600226+0000","last_fullsized":"2024-08-17T18:01:28.600226+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":253,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.572858+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T18:01:28.572858+0000","last_peered":"2024-08-17T18:01:28.572858+0000","last_clean":"2024-08-17T18:01:28.572858+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T18:01:28.572858+0000","last_undegraded":"2024-08-17T18:01:28.572858+0000","last_fullsized":"2024-08-17T18:01:28.572858+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":2017,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.574574+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T18:01:28.574574+0000","last_peered":"2024-08-17T18:01:28.574574+0000","last_clean":"2024-08-17T18:01:28.574574+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T18:01:28.574574+0000","last_undegraded":"2024-08-17T18:01:28.574574+0000","last_fullsized":"2024-08-17T18:01:28.574574+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1995,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.575175+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T18:01:28.575175+0000","last_peered":"2024-08-17T18:01:28.575175+0000","last_clean":"2024-08-17T18:01:28.575175+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T18:01:28.575175+0000","last_undegraded":"2024-08-17T18:01:28.575175+0000","last_fullsized":"2024-08-17T18:01:28.575175+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1842,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.593260+0000","last_change":"2024-08-17T17:59:58.030150+0000","last_active":"2024-08-17T18:01:28.593260+0000","last_peered":"2024-08-17T18:01:28.593260+0000","last_clean":"2024-08-17T18:01:28.593260+0000","last_became_active":"2024-08-17T17:59:58.029407+0000","last_became_peered":"2024-08-17T17:59:58.029407+0000","last_unstale":"2024-08-17T18:01:28.593260+0000","last_undegraded":"2024-08-17T18:01:28.593260+0000","last_fullsized":"2024-08-17T18:01:28.593260+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:31.632793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2021,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.599670+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T18:01:28.599670+0000","last_peered":"2024-08-17T18:01:28.599670+0000","last_clean":"2024-08-17T18:01:28.599670+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T18:01:28.599670+0000","last_undegraded":"2024-08-17T18:01:28.599670+0000","last_fullsized":"2024-08-17T18:01:28.599670+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2001,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.599978+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T18:01:28.599978+0000","last_peered":"2024-08-17T18:01:28.599978+0000","last_clean":"2024-08-17T18:01:28.599978+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T18:01:28.599978+0000","last_undegraded":"2024-08-17T18:01:28.599978+0000","last_fullsized":"2024-08-17T18:01:28.599978+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1274,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497706+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T18:01:22.497706+0000","last_peered":"2024-08-17T18:01:22.497706+0000","last_clean":"2024-08-17T18:01:22.497706+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T18:01:22.497706+0000","last_undegraded":"2024-08-17T18:01:22.497706+0000","last_fullsized":"2024-08-17T18:01:22.497706+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1220,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497091+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T18:01:22.497091+0000","last_peered":"2024-08-17T18:01:22.497091+0000","last_clean":"2024-08-17T18:01:22.497091+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T18:01:22.497091+0000","last_undegraded":"2024-08-17T18:01:22.497091+0000","last_fullsized":"2024-08-17T18:01:22.497091+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1910,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.591915+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T18:01:28.591915+0000","last_peered":"2024-08-17T18:01:28.591915+0000","last_clean":"2024-08-17T18:01:28.591915+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T18:01:28.591915+0000","last_undegraded":"2024-08-17T18:01:28.591915+0000","last_fullsized":"2024-08-17T18:01:28.591915+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":36,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497717+0000","last_change":"2024-08-17T17:59:58.027683+0000","last_active":"2024-08-17T18:01:22.497717+0000","last_peered":"2024-08-17T18:01:22.497717+0000","last_clean":"2024-08-17T18:01:22.497717+0000","last_became_active":"2024-08-17T17:59:58.027358+0000","last_became_peered":"2024-08-17T17:59:58.027358+0000","last_unstale":"2024-08-17T18:01:22.497717+0000","last_undegraded":"2024-08-17T18:01:22.497717+0000","last_fullsized":"2024-08-17T18:01:22.497717+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:04.758259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1586,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497457+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T18:01:22.497457+0000","last_peered":"2024-08-17T18:01:22.497457+0000","last_clean":"2024-08-17T18:01:22.497457+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T18:01:22.497457+0000","last_undegraded":"2024-08-17T18:01:22.497457+0000","last_fullsized":"2024-08-17T18:01:22.497457+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2001,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.574707+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T18:01:28.574707+0000","last_peered":"2024-08-17T18:01:28.574707+0000","last_clean":"2024-08-17T18:01:28.574707+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T18:01:28.574707+0000","last_undegraded":"2024-08-17T18:01:28.574707+0000","last_fullsized":"2024-08-17T18:01:28.574707+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1941,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.598074+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T18:01:28.598074+0000","last_peered":"2024-08-17T18:01:28.598074+0000","last_clean":"2024-08-17T18:01:28.598074+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T18:01:28.598074+0000","last_undegraded":"2024-08-17T18:01:28.598074+0000","last_fullsized":"2024-08-17T18:01:28.598074+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":554,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497052+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T18:01:22.497052+0000","last_peered":"2024-08-17T18:01:22.497052+0000","last_clean":"2024-08-17T18:01:22.497052+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T18:01:22.497052+0000","last_undegraded":"2024-08-17T18:01:22.497052+0000","last_fullsized":"2024-08-17T18:01:22.497052+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":514,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497572+0000","last_change":"2024-08-17T17:59:58.021264+0000","last_active":"2024-08-17T18:01:22.497572+0000","last_peered":"2024-08-17T18:01:22.497572+0000","last_clean":"2024-08-17T18:01:22.497572+0000","last_became_active":"2024-08-17T17:59:58.020957+0000","last_became_peered":"2024-08-17T17:59:58.020957+0000","last_unstale":"2024-08-17T18:01:22.497572+0000","last_undegraded":"2024-08-17T18:01:22.497572+0000","last_fullsized":"2024-08-17T18:01:22.497572+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:42.190270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":543,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.573373+0000","last_change":"2024-08-17T17:59:58.043761+0000","last_active":"2024-08-17T18:01:28.573373+0000","last_peered":"2024-08-17T18:01:28.573373+0000","last_clean":"2024-08-17T18:01:28.573373+0000","last_became_active":"2024-08-17T17:59:58.043206+0000","last_became_peered":"2024-08-17T17:59:58.043206+0000","last_unstale":"2024-08-17T18:01:28.573373+0000","last_undegraded":"2024-08-17T18:01:28.573373+0000","last_fullsized":"2024-08-17T18:01:28.573373+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:37.774589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1238,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.496991+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T18:01:22.496991+0000","last_peered":"2024-08-17T18:01:22.496991+0000","last_clean":"2024-08-17T18:01:22.496991+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T18:01:22.496991+0000","last_undegraded":"2024-08-17T18:01:22.496991+0000","last_fullsized":"2024-08-17T18:01:22.496991+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":712,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.598674+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T18:01:28.598674+0000","last_peered":"2024-08-17T18:01:28.598674+0000","last_clean":"2024-08-17T18:01:28.598674+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T18:01:28.598674+0000","last_undegraded":"2024-08-17T18:01:28.598674+0000","last_fullsized":"2024-08-17T18:01:28.598674+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1938,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.592514+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T18:01:28.592514+0000","last_peered":"2024-08-17T18:01:28.592514+0000","last_clean":"2024-08-17T18:01:28.592514+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T18:01:28.592514+0000","last_undegraded":"2024-08-17T18:01:28.592514+0000","last_fullsized":"2024-08-17T18:01:28.592514+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2033,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.600608+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T18:01:28.600608+0000","last_peered":"2024-08-17T18:01:28.600608+0000","last_clean":"2024-08-17T18:01:28.600608+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T18:01:28.600608+0000","last_undegraded":"2024-08-17T18:01:28.600608+0000","last_fullsized":"2024-08-17T18:01:28.600608+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":744,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.600829+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T18:01:28.600829+0000","last_peered":"2024-08-17T18:01:28.600829+0000","last_clean":"2024-08-17T18:01:28.600829+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T18:01:28.600829+0000","last_undegraded":"2024-08-17T18:01:28.600829+0000","last_fullsized":"2024-08-17T18:01:28.600829+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1227,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.496693+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T18:01:22.496693+0000","last_peered":"2024-08-17T18:01:22.496693+0000","last_clean":"2024-08-17T18:01:22.496693+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T18:01:22.496693+0000","last_undegraded":"2024-08-17T18:01:22.496693+0000","last_fullsized":"2024-08-17T18:01:22.496693+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1226,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.498021+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T18:01:22.498021+0000","last_peered":"2024-08-17T18:01:22.498021+0000","last_clean":"2024-08-17T18:01:22.498021+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T18:01:22.498021+0000","last_undegraded":"2024-08-17T18:01:22.498021+0000","last_fullsized":"2024-08-17T18:01:22.498021+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":729,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.574453+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T18:01:28.574453+0000","last_peered":"2024-08-17T18:01:28.574453+0000","last_clean":"2024-08-17T18:01:28.574453+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T18:01:28.574453+0000","last_undegraded":"2024-08-17T18:01:28.574453+0000","last_fullsized":"2024-08-17T18:01:28.574453+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1244,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.498261+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T18:01:22.498261+0000","last_peered":"2024-08-17T18:01:22.498261+0000","last_clean":"2024-08-17T18:01:22.498261+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T18:01:22.498261+0000","last_undegraded":"2024-08-17T18:01:22.498261+0000","last_fullsized":"2024-08-17T18:01:22.498261+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":128,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497438+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T18:01:22.497438+0000","last_peered":"2024-08-17T18:01:22.497438+0000","last_clean":"2024-08-17T18:01:22.497438+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T18:01:22.497438+0000","last_undegraded":"2024-08-17T18:01:22.497438+0000","last_fullsized":"2024-08-17T18:01:22.497438+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1260,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.498139+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T18:01:22.498139+0000","last_peered":"2024-08-17T18:01:22.498139+0000","last_clean":"2024-08-17T18:01:22.498139+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T18:01:22.498139+0000","last_undegraded":"2024-08-17T18:01:22.498139+0000","last_fullsized":"2024-08-17T18:01:22.498139+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1277,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.496689+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T18:01:22.496689+0000","last_peered":"2024-08-17T18:01:22.496689+0000","last_clean":"2024-08-17T18:01:22.496689+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T18:01:22.496689+0000","last_undegraded":"2024-08-17T18:01:22.496689+0000","last_fullsized":"2024-08-17T18:01:22.496689+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1939,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.595876+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T18:01:28.595876+0000","last_peered":"2024-08-17T18:01:28.595876+0000","last_clean":"2024-08-17T18:01:28.595876+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T18:01:28.595876+0000","last_undegraded":"2024-08-17T18:01:28.595876+0000","last_fullsized":"2024-08-17T18:01:28.595876+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1221,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497622+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T18:01:22.497622+0000","last_peered":"2024-08-17T18:01:22.497622+0000","last_clean":"2024-08-17T18:01:22.497622+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T18:01:22.497622+0000","last_undegraded":"2024-08-17T18:01:22.497622+0000","last_fullsized":"2024-08-17T18:01:22.497622+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1992,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.599445+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T18:01:28.599445+0000","last_peered":"2024-08-17T18:01:28.599445+0000","last_clean":"2024-08-17T18:01:28.599445+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T18:01:28.599445+0000","last_undegraded":"2024-08-17T18:01:28.599445+0000","last_fullsized":"2024-08-17T18:01:28.599445+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1993,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.599538+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T18:01:28.599538+0000","last_peered":"2024-08-17T18:01:28.599538+0000","last_clean":"2024-08-17T18:01:28.599538+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T18:01:28.599538+0000","last_undegraded":"2024-08-17T18:01:28.599538+0000","last_fullsized":"2024-08-17T18:01:28.599538+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":634,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497894+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T18:01:22.497894+0000","last_peered":"2024-08-17T18:01:22.497894+0000","last_clean":"2024-08-17T18:01:22.497894+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T18:01:22.497894+0000","last_undegraded":"2024-08-17T18:01:22.497894+0000","last_fullsized":"2024-08-17T18:01:22.497894+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1246,"reported_epoch":1276,"state":"active+clean","last_fresh":"2024-08-17T18:01:22.497693+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T18:01:22.497693+0000","last_peered":"2024-08-17T18:01:22.497693+0000","last_clean":"2024-08-17T18:01:22.497693+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T18:01:22.497693+0000","last_undegraded":"2024-08-17T18:01:22.497693+0000","last_fullsized":"2024-08-17T18:01:22.497693+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1939,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.598810+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T18:01:28.598810+0000","last_peered":"2024-08-17T18:01:28.598810+0000","last_clean":"2024-08-17T18:01:28.598810+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T18:01:28.598810+0000","last_undegraded":"2024-08-17T18:01:28.598810+0000","last_fullsized":"2024-08-17T18:01:28.598810+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":46,"num_pg":16,"stat_sum":{"num_bytes":5587271704,"num_objects":85256,"num_object_clones":0,"num_object_copies":255768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85256,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":130706,"num_write_kb":6910721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16706056192,"data_stored":16706043976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105306,"ondisk_log_size":105306,"up":48,"acting":48,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234488,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2697436,"kb_used_data":2400084,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":91047716,"statfs":{"total":95995035648,"available":93232861184,"internally_reserved":0,"allocated":2457686016,"data_stored":2453818562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496123,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2720024,"kb_used_data":2410192,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":91025128,"statfs":{"total":95995035648,"available":93209731072,"internally_reserved":0,"allocated":2468036608,"data_stored":2464173703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757761,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326375,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3083712,"kb_used_data":2718136,"kb_used_omap":57,"kb_used_meta":365510,"kb_avail":90661440,"statfs":{"total":95995035648,"available":92837314560,"internally_reserved":0,"allocated":2783371264,"data_stored":2779527126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":374283003},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405614,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3460364,"kb_used_data":3053316,"kb_used_omap":61,"kb_used_meta":406914,"kb_avail":90284788,"statfs":{"total":95995035648,"available":92451622912,"internally_reserved":0,"allocated":3126595584,"data_stored":3122744346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":416680888},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452148,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2746824,"kb_used_data":2428860,"kb_used_omap":54,"kb_used_meta":317897,"kb_avail":90998328,"statfs":{"total":95995035648,"available":93182287872,"internally_reserved":0,"allocated":2487152640,"data_stored":2483304311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":325526926},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474839,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3850256,"kb_used_data":3455048,"kb_used_omap":119,"kb_used_meta":395080,"kb_avail":89894896,"statfs":{"total":95995035648,"available":92052373504,"internally_reserved":0,"allocated":3537969152,"data_stored":3534109747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":404562654},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019396,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3497201664,"data_stored":3497197592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2467631104,"data_stored":2467627032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3112239104,"data_stored":3112239104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2748252160,"data_stored":2748252160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2443182080,"data_stored":2443182080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2437550080,"data_stored":2437546008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:01:40.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:01:40.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:40.769 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:40.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:40.870 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:40.892 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:40.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:40.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:40.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:41.039 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:40 smithi179 ceph-mon[93519]: pgmap v2089: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 768 op/s 2024-08-17T18:01:41.043 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:41.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:41.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:41.145 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:41.172 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:41.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:41.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:41.274 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:41.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:41.297 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:41.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:41.398 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:41.436 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:41.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:41.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:41.538 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:41.568 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:41.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:41.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:41.670 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:41.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:41.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:41.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:41.793 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:41.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:41.818 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:41.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:41.919 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:41.941 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:41.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:41.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:42.043 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:42.064 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:42.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:42.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:42.166 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:42.200 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:42.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:42.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:42.302 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:42.341 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:42.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:42.443 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:42.480 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:42.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:42.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:42.582 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:42.609 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:42.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:42.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:42.646 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:01:42.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:42.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:42.768 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:42.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:42.869 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:42.911 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:42.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:42.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:42.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:42 smithi130 ceph-mon[105521]: pgmap v2090: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 698 op/s 2024-08-17T18:01:42.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:42 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3473140090' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:01:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:01:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:01:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:01:42.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:42 smithi130 ceph-mon[110068]: pgmap v2090: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 698 op/s 2024-08-17T18:01:42.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:42 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3473140090' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:01:42.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:01:42.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:01:42.920 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:01:43.014 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:43.047 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:43.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:43.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:42 smithi179 ceph-mon[93519]: pgmap v2090: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 698 op/s 2024-08-17T18:01:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:42 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3473140090' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:01:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:01:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:01:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:01:43.148 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:43.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:43.183 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:43.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:43.284 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:43.310 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:43.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:43.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:43.412 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:43.436 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:43.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:43.538 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:43.560 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:43.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:43.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:43.662 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:43.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:43.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:43.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:43.786 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:43.845 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:43.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:43.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:43.894 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:43 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:01:43.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:43 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T18:01:43.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:43 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3473140090' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:01:43.895 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:43 smithi130 ceph-mon[110068]: osdmap e1278: 8 total, 8 up, 6 in 2024-08-17T18:01:43.895 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:43 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:01:43.895 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:43 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T18:01:43.895 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:43 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3473140090' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:01:43.895 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:43 smithi130 ceph-mon[105521]: osdmap e1278: 8 total, 8 up, 6 in 2024-08-17T18:01:43.948 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:43.970 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:43.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:43.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:44.043 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:01:44.043 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:01:44.072 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:43 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:01:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:43 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T18:01:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:43 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3473140090' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:01:44.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:43 smithi179 ceph-mon[93519]: osdmap e1278: 8 total, 8 up, 6 in 2024-08-17T18:01:44.094 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:44.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:44.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:44.196 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:44.242 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:44.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:44.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:44.346 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:44.392 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:44.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:44.493 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:44.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:44.531 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:44.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:44.632 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:44.674 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:44.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:44.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:44.776 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:44.799 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:44.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:44.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:44.900 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:44.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:44 smithi130 ceph-mon[105521]: pgmap v2092: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 770 op/s 2024-08-17T18:01:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:44 smithi130 ceph-mon[110068]: pgmap v2092: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 770 op/s 2024-08-17T18:01:44.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:44.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:44.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:45.051 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:45.087 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:45.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:45.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:45.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:44 smithi179 ceph-mon[93519]: pgmap v2092: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 770 op/s 2024-08-17T18:01:45.189 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:45.197 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:45.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T18:01:45.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:45.268 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:45.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:45.369 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:45.428 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:45.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:45.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:45.530 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:45.589 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:45.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:45.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:45.691 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:45.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:45.742 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:45.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:45.806 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:01:45.843 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:45.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:45.906 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:45.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:46.007 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:46.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:46.067 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:46.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:46.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:46.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:46.206 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:46.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:46.307 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:46.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:46.330 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:46.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:46.431 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:46.454 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:46.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:46.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:46.467 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:01:46.467 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:01:46.467 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:01:46.467 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:01:46.467 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:01:46.467 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:01:46.467 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:01:46.467 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:01:46.468 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:01:46.468 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:01:46.468 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:01:46.468 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:01:46.468 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:01:46.468 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:01:46.468 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:01:46.468 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:01:46.556 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:46.579 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:46.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:46.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:46.681 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:46.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:46.731 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:46.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:46.831 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:46.871 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:46.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:46.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:46 smithi130 ceph-mon[110068]: pgmap v2093: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 768 op/s 2024-08-17T18:01:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:46 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2611464987' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:01:46.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:46 smithi130 ceph-mon[105521]: pgmap v2093: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 768 op/s 2024-08-17T18:01:46.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:46 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2611464987' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:01:46.974 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:47.013 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:47.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:47.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:46 smithi179 ceph-mon[93519]: pgmap v2093: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 768 op/s 2024-08-17T18:01:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:46 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2611464987' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:01:47.115 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:47.137 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:47.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:47.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:47.239 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:47.262 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:47.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:47.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:47.364 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:47.410 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:47.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:47.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:47.513 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:47.537 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:47.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:47.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:47.638 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:47.679 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:47.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:47.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:47.781 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:47.803 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:47.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:47.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:47.821 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:47.821 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:47.905 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:47.961 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:47.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:47.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:48.062 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:48.086 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:48.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:48.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:48.188 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:48.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:48.241 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:48.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:48.342 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:48.408 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:48.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:48.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:48.510 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:48.534 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:48.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:48.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:48.636 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:48.689 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:48.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:48.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:48.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:48.815 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:48.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:48.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:48.917 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:48.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:48.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:48.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:49.052 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:49.092 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:49.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:49.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:49.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:48 smithi179 ceph-mon[93519]: pgmap v2094: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 762 op/s 2024-08-17T18:01:49.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:48 smithi130 ceph-mon[105521]: pgmap v2094: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 762 op/s 2024-08-17T18:01:49.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:48 smithi130 ceph-mon[110068]: pgmap v2094: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 762 op/s 2024-08-17T18:01:49.194 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:49.220 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:49.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:49.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:49.283 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:01:49.322 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:49.346 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:49.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:49.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:49.448 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:49.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:49.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:49.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:49.573 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:49.597 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:49.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:49.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:49.699 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:49.723 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:49.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:49.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:49.825 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:49.850 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:49.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:49.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:49.952 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:49.990 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:49.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:49.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:50.024 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:01:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:01:49] "GET /metrics HTTP/1.1" 200 36788 "" "Prometheus/2.51.0" 2024-08-17T18:01:50.093 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:50.120 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:50.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:50.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:50.222 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:50.248 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:50.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:50.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:50.350 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:50.375 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:50.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:50.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:50.477 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:50.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:50.524 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:50.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:50.590 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:01:50.596 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:01:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:01:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:01:50.598 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:01:50.625 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:50.648 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:50.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:50.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:50.750 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:50.774 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:50.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:50.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:50.876 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:50.898 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:50.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:50.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:50.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:50 smithi130 ceph-mon[105521]: pgmap v2095: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 792 op/s 2024-08-17T18:01:50.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:50 smithi130 ceph-mon[105521]: from='client.36333 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:01:50.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:50 smithi130 ceph-mon[110068]: pgmap v2095: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 792 op/s 2024-08-17T18:01:50.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:50 smithi130 ceph-mon[110068]: from='client.36333 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:01:51.000 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:51.039 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:51.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:51.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:50 smithi179 ceph-mon[93519]: pgmap v2095: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 792 op/s 2024-08-17T18:01:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:50 smithi179 ceph-mon[93519]: from='client.36333 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:01:51.141 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:51.164 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:51.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:51.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:51.266 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:51.288 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:51.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:51.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:51.390 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:51.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:51.414 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:51.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:51.515 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:51.561 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:51.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:51.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:51.663 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:51.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:51.733 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:51.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:51.834 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:51.889 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:51.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:51.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:51.991 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:52.031 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:52.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:52.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:52.133 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:52.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:52.180 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:52.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:52.230 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2095,"stamp":"2024-08-17T18:01:49.194329+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4823717861,"num_objects":73622,"num_object_clones":0,"num_object_copies":220866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73622,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":142644,"num_write_kb":6913168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":1973261,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105773,"ondisk_log_size":105773,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":16377844,"kb_used_data":14238980,"kb_used_omap":295,"kb_used_meta":2138392,"kb_avail":546093068,"statfs":{"total":575970213888,"available":559199301632,"internally_reserved":0,"allocated":14580715520,"data_stored":14557536739,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":2189713871},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,4,11,9,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-519700480,"num_objects":-7930,"num_object_clones":0,"num_object_copies":-23790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7930,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006088"},"pg_stats":[{"pgid":"46.f","version":"1278'8996","reported_seq":9024,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:47.932763+0000","last_change":"2024-08-17T17:59:22.956823+0000","last_active":"2024-08-17T18:01:47.932763+0000","last_peered":"2024-08-17T18:01:47.932763+0000","last_clean":"2024-08-17T18:01:47.932763+0000","last_became_active":"2024-08-17T17:59:22.956437+0000","last_became_peered":"2024-08-17T17:59:22.956437+0000","last_unstale":"2024-08-17T18:01:47.932763+0000","last_undegraded":"2024-08-17T18:01:47.932763+0000","last_fullsized":"2024-08-17T18:01:47.932763+0000","mapping_epoch":1258,"log_start":"1262'2100","ondisk_log_start":"1262'2100","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6896,"log_dups_size":0,"ondisk_log_size":6896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:17.161240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309329920,"num_objects":4720,"num_object_clones":0,"num_object_copies":14160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8996,"num_write_kb":438912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.e","version":"1278'8869","reported_seq":8916,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:47.999037+0000","last_change":"2024-08-17T17:59:58.047028+0000","last_active":"2024-08-17T18:01:47.999037+0000","last_peered":"2024-08-17T18:01:47.999037+0000","last_clean":"2024-08-17T18:01:47.999037+0000","last_became_active":"2024-08-17T17:59:58.027451+0000","last_became_peered":"2024-08-17T17:59:58.027451+0000","last_unstale":"2024-08-17T18:01:47.999037+0000","last_undegraded":"2024-08-17T18:01:47.999037+0000","last_fullsized":"2024-08-17T18:01:47.999037+0000","mapping_epoch":1263,"log_start":"1262'1800","ondisk_log_start":"1262'1800","created":1258,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":7069,"log_dups_size":0,"ondisk_log_size":7069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:20:25.306207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299040768,"num_objects":4563,"num_object_clones":0,"num_object_copies":13689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8869,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.d","version":"1278'9056","reported_seq":9109,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:47.702920+0000","last_change":"2024-08-17T17:59:22.954900+0000","last_active":"2024-08-17T18:01:47.702920+0000","last_peered":"2024-08-17T18:01:47.702920+0000","last_clean":"2024-08-17T18:01:47.702920+0000","last_became_active":"2024-08-17T17:59:22.954599+0000","last_became_peered":"2024-08-17T17:59:22.954599+0000","last_unstale":"2024-08-17T18:01:47.702920+0000","last_undegraded":"2024-08-17T18:01:47.702920+0000","last_fullsized":"2024-08-17T18:01:47.702920+0000","mapping_epoch":1258,"log_start":"1262'2500","ondisk_log_start":"1262'2500","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6556,"log_dups_size":0,"ondisk_log_size":6556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:22:00.453498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305987608,"num_objects":4670,"num_object_clones":0,"num_object_copies":14010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4670,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9056,"num_write_kb":439169,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.c","version":"1278'9010","reported_seq":9038,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:48.012301+0000","last_change":"2024-08-17T17:59:22.951540+0000","last_active":"2024-08-17T18:01:48.012301+0000","last_peered":"2024-08-17T18:01:48.012301+0000","last_clean":"2024-08-17T18:01:48.012301+0000","last_became_active":"2024-08-17T17:59:22.951248+0000","last_became_peered":"2024-08-17T17:59:22.951248+0000","last_unstale":"2024-08-17T18:01:48.012301+0000","last_undegraded":"2024-08-17T18:01:48.012301+0000","last_fullsized":"2024-08-17T18:01:48.012301+0000","mapping_epoch":1258,"log_start":"1262'2100","ondisk_log_start":"1262'2100","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6910,"log_dups_size":0,"ondisk_log_size":6910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:55.458182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300810240,"num_objects":4590,"num_object_clones":0,"num_object_copies":13770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9010,"num_write_kb":435200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.b","version":"1278'8840","reported_seq":8892,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:47.732729+0000","last_change":"2024-08-17T17:59:22.957875+0000","last_active":"2024-08-17T18:01:47.732729+0000","last_peered":"2024-08-17T18:01:47.732729+0000","last_clean":"2024-08-17T18:01:47.732729+0000","last_became_active":"2024-08-17T17:59:22.957601+0000","last_became_peered":"2024-08-17T17:59:22.957601+0000","last_unstale":"2024-08-17T18:01:47.732729+0000","last_undegraded":"2024-08-17T18:01:47.732729+0000","last_fullsized":"2024-08-17T18:01:47.732729+0000","mapping_epoch":1258,"log_start":"1262'2300","ondisk_log_start":"1262'2300","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6540,"log_dups_size":0,"ondisk_log_size":6540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:40.456894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295960576,"num_objects":4516,"num_object_clones":0,"num_object_copies":13548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8840,"num_write_kb":427392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.a","version":"1278'8964","reported_seq":8992,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:47.999342+0000","last_change":"2024-08-17T17:59:22.951416+0000","last_active":"2024-08-17T18:01:47.999342+0000","last_peered":"2024-08-17T18:01:47.999342+0000","last_clean":"2024-08-17T18:01:47.999342+0000","last_became_active":"2024-08-17T17:59:22.951110+0000","last_became_peered":"2024-08-17T17:59:22.951110+0000","last_unstale":"2024-08-17T18:01:47.999342+0000","last_undegraded":"2024-08-17T18:01:47.999342+0000","last_fullsized":"2024-08-17T18:01:47.999342+0000","mapping_epoch":1258,"log_start":"1262'1900","ondisk_log_start":"1262'1900","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":7064,"log_dups_size":0,"ondisk_log_size":7064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:32.365091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302383104,"num_objects":4614,"num_object_clones":0,"num_object_copies":13842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8964,"num_write_kb":434496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.9","version":"1278'8809","reported_seq":8861,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:47.776812+0000","last_change":"2024-08-17T17:59:22.957896+0000","last_active":"2024-08-17T18:01:47.776812+0000","last_peered":"2024-08-17T18:01:47.776812+0000","last_clean":"2024-08-17T18:01:47.776812+0000","last_became_active":"2024-08-17T17:59:22.957648+0000","last_became_peered":"2024-08-17T17:59:22.957648+0000","last_unstale":"2024-08-17T18:01:47.776812+0000","last_undegraded":"2024-08-17T18:01:47.776812+0000","last_fullsized":"2024-08-17T18:01:47.776812+0000","mapping_epoch":1258,"log_start":"1262'2200","ondisk_log_start":"1262'2200","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6609,"log_dups_size":0,"ondisk_log_size":6609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:11.940932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293928960,"num_objects":4485,"num_object_clones":0,"num_object_copies":13455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8809,"num_write_kb":425408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.8","version":"1278'8823","reported_seq":8874,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:46.585770+0000","last_change":"2024-08-17T17:59:22.951248+0000","last_active":"2024-08-17T18:01:46.585770+0000","last_peered":"2024-08-17T18:01:46.585770+0000","last_clean":"2024-08-17T18:01:46.585770+0000","last_became_active":"2024-08-17T17:59:22.950981+0000","last_became_peered":"2024-08-17T17:59:22.950981+0000","last_unstale":"2024-08-17T18:01:46.585770+0000","last_undegraded":"2024-08-17T18:01:46.585770+0000","last_fullsized":"2024-08-17T18:01:46.585770+0000","mapping_epoch":1258,"log_start":"1264'2900","ondisk_log_start":"1264'2900","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":5923,"log_dups_size":0,"ondisk_log_size":5923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:24:07.143610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303628288,"num_objects":4633,"num_object_clones":0,"num_object_copies":13899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8823,"num_write_kb":430592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.7","version":"1278'8910","reported_seq":8972,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:46.589460+0000","last_change":"2024-08-17T17:59:58.050355+0000","last_active":"2024-08-17T18:01:46.589460+0000","last_peered":"2024-08-17T18:01:46.589460+0000","last_clean":"2024-08-17T18:01:46.589460+0000","last_became_active":"2024-08-17T17:59:58.022280+0000","last_became_peered":"2024-08-17T17:59:58.022280+0000","last_unstale":"2024-08-17T18:01:46.589460+0000","last_undegraded":"2024-08-17T18:01:46.589460+0000","last_fullsized":"2024-08-17T18:01:46.589460+0000","mapping_epoch":1263,"log_start":"1264'3000","ondisk_log_start":"1264'3000","created":1258,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":5910,"log_dups_size":0,"ondisk_log_size":5910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:07.979130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303038464,"num_objects":4624,"num_object_clones":0,"num_object_copies":13872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8910,"num_write_kb":433088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.6","version":"1278'8892","reported_seq":8920,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:48.021893+0000","last_change":"2024-08-17T17:59:22.958347+0000","last_active":"2024-08-17T18:01:48.021893+0000","last_peered":"2024-08-17T18:01:48.021893+0000","last_clean":"2024-08-17T18:01:48.021893+0000","last_became_active":"2024-08-17T17:59:22.957969+0000","last_became_peered":"2024-08-17T17:59:22.957969+0000","last_unstale":"2024-08-17T18:01:48.021893+0000","last_undegraded":"2024-08-17T18:01:48.021893+0000","last_fullsized":"2024-08-17T18:01:48.021893+0000","mapping_epoch":1258,"log_start":"1262'2000","ondisk_log_start":"1262'2000","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6892,"log_dups_size":0,"ondisk_log_size":6892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:07.728594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296615936,"num_objects":4526,"num_object_clones":0,"num_object_copies":13578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8892,"num_write_kb":429376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.5","version":"1278'8885","reported_seq":8937,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:47.765396+0000","last_change":"2024-08-17T17:59:22.956504+0000","last_active":"2024-08-17T18:01:47.765396+0000","last_peered":"2024-08-17T18:01:47.765396+0000","last_clean":"2024-08-17T18:01:47.765396+0000","last_became_active":"2024-08-17T17:59:22.955747+0000","last_became_peered":"2024-08-17T17:59:22.955747+0000","last_unstale":"2024-08-17T18:01:47.765396+0000","last_undegraded":"2024-08-17T18:01:47.765396+0000","last_fullsized":"2024-08-17T18:01:47.765396+0000","mapping_epoch":1258,"log_start":"1264'2700","ondisk_log_start":"1264'2700","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6185,"log_dups_size":0,"ondisk_log_size":6185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:54:51.990739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302579712,"num_objects":4617,"num_object_clones":0,"num_object_copies":13851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8885,"num_write_kb":432064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"46.4","version":"1278'8757","reported_seq":8785,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:47.983722+0000","last_change":"2024-08-17T17:59:22.952797+0000","last_active":"2024-08-17T18:01:47.983722+0000","last_peered":"2024-08-17T18:01:47.983722+0000","last_clean":"2024-08-17T18:01:47.983722+0000","last_became_active":"2024-08-17T17:59:22.952475+0000","last_became_peered":"2024-08-17T17:59:22.952475+0000","last_unstale":"2024-08-17T18:01:47.983722+0000","last_undegraded":"2024-08-17T18:01:47.983722+0000","last_fullsized":"2024-08-17T18:01:47.983722+0000","mapping_epoch":1258,"log_start":"1262'1700","ondisk_log_start":"1262'1700","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":7057,"log_dups_size":0,"ondisk_log_size":7057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:42:19.302395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296026112,"num_objects":4517,"num_object_clones":0,"num_object_copies":13551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8757,"num_write_kb":424768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.3","version":"1278'8738","reported_seq":8790,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:46.512977+0000","last_change":"2024-08-17T17:59:22.954906+0000","last_active":"2024-08-17T18:01:46.512977+0000","last_peered":"2024-08-17T18:01:46.512977+0000","last_clean":"2024-08-17T18:01:46.512977+0000","last_became_active":"2024-08-17T17:59:22.954623+0000","last_became_peered":"2024-08-17T17:59:22.954623+0000","last_unstale":"2024-08-17T18:01:46.512977+0000","last_undegraded":"2024-08-17T18:01:46.512977+0000","last_fullsized":"2024-08-17T18:01:46.512977+0000","mapping_epoch":1258,"log_start":"1264'2800","ondisk_log_start":"1264'2800","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":5938,"log_dups_size":0,"ondisk_log_size":5938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:19.582387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305004544,"num_objects":4654,"num_object_clones":0,"num_object_copies":13962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8738,"num_write_kb":428544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.2","version":"1278'9009","reported_seq":9061,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:47.765844+0000","last_change":"2024-08-17T17:59:22.956809+0000","last_active":"2024-08-17T18:01:47.765844+0000","last_peered":"2024-08-17T18:01:47.765844+0000","last_clean":"2024-08-17T18:01:47.765844+0000","last_became_active":"2024-08-17T17:59:22.956543+0000","last_became_peered":"2024-08-17T17:59:22.956543+0000","last_unstale":"2024-08-17T18:01:47.765844+0000","last_undegraded":"2024-08-17T18:01:47.765844+0000","last_fullsized":"2024-08-17T18:01:47.765844+0000","mapping_epoch":1258,"log_start":"1262'2400","ondisk_log_start":"1262'2400","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6609,"log_dups_size":0,"ondisk_log_size":6609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:31:37.454620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303235072,"num_objects":4627,"num_object_clones":0,"num_object_copies":13881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9009,"num_write_kb":436352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.1","version":"1278'8973","reported_seq":8997,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:48.021891+0000","last_change":"2024-08-17T17:59:22.954701+0000","last_active":"2024-08-17T18:01:48.021891+0000","last_peered":"2024-08-17T18:01:48.021891+0000","last_clean":"2024-08-17T18:01:48.021891+0000","last_became_active":"2024-08-17T17:59:22.954510+0000","last_became_peered":"2024-08-17T17:59:22.954510+0000","last_unstale":"2024-08-17T18:01:48.021891+0000","last_undegraded":"2024-08-17T18:01:48.021891+0000","last_fullsized":"2024-08-17T18:01:48.021891+0000","mapping_epoch":1258,"log_start":"1262'2100","ondisk_log_start":"1262'2100","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6873,"log_dups_size":0,"ondisk_log_size":6873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:19:01.061765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301006848,"num_objects":4593,"num_object_clones":0,"num_object_copies":13779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8973,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.0","version":"1278'8835","reported_seq":8897,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:47.765625+0000","last_change":"2024-08-17T17:59:58.062636+0000","last_active":"2024-08-17T18:01:47.765625+0000","last_peered":"2024-08-17T18:01:47.765625+0000","last_clean":"2024-08-17T18:01:47.765625+0000","last_became_active":"2024-08-17T17:59:58.032875+0000","last_became_peered":"2024-08-17T17:59:58.032875+0000","last_unstale":"2024-08-17T18:01:47.765625+0000","last_undegraded":"2024-08-17T18:01:47.765625+0000","last_fullsized":"2024-08-17T18:01:47.765625+0000","mapping_epoch":1263,"log_start":"1262'2300","ondisk_log_start":"1262'2300","created":1258,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6535,"log_dups_size":0,"ondisk_log_size":6535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:24.031388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304545792,"num_objects":4647,"num_object_clones":0,"num_object_copies":13941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8835,"num_write_kb":431424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":2016,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.660805+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T18:01:42.660805+0000","last_peered":"2024-08-17T18:01:42.660805+0000","last_clean":"2024-08-17T18:01:42.660805+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T18:01:42.660805+0000","last_undegraded":"2024-08-17T18:01:42.660805+0000","last_fullsized":"2024-08-17T18:01:42.660805+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":555,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.578690+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T18:01:28.578690+0000","last_peered":"2024-08-17T18:01:28.578690+0000","last_clean":"2024-08-17T18:01:28.578690+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T18:01:28.578690+0000","last_undegraded":"2024-08-17T18:01:28.578690+0000","last_fullsized":"2024-08-17T18:01:28.578690+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1935,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658589+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T18:01:42.658589+0000","last_peered":"2024-08-17T18:01:42.658589+0000","last_clean":"2024-08-17T18:01:42.658589+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T18:01:42.658589+0000","last_undegraded":"2024-08-17T18:01:42.658589+0000","last_fullsized":"2024-08-17T18:01:42.658589+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1959,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658214+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T18:01:42.658214+0000","last_peered":"2024-08-17T18:01:42.658214+0000","last_clean":"2024-08-17T18:01:42.658214+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T18:01:42.658214+0000","last_undegraded":"2024-08-17T18:01:42.658214+0000","last_fullsized":"2024-08-17T18:01:42.658214+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":117,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.579548+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T18:01:28.579548+0000","last_peered":"2024-08-17T18:01:28.579548+0000","last_clean":"2024-08-17T18:01:28.579548+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T18:01:28.579548+0000","last_undegraded":"2024-08-17T18:01:28.579548+0000","last_fullsized":"2024-08-17T18:01:28.579548+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1352,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658336+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T18:01:42.658336+0000","last_peered":"2024-08-17T18:01:42.658336+0000","last_clean":"2024-08-17T18:01:42.658336+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T18:01:42.658336+0000","last_undegraded":"2024-08-17T18:01:42.658336+0000","last_fullsized":"2024-08-17T18:01:42.658336+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":210,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.656909+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T18:01:42.656909+0000","last_peered":"2024-08-17T18:01:42.656909+0000","last_clean":"2024-08-17T18:01:42.656909+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T18:01:42.656909+0000","last_undegraded":"2024-08-17T18:01:42.656909+0000","last_fullsized":"2024-08-17T18:01:42.656909+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":730,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658201+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T18:01:42.658201+0000","last_peered":"2024-08-17T18:01:42.658201+0000","last_clean":"2024-08-17T18:01:42.658201+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T18:01:42.658201+0000","last_undegraded":"2024-08-17T18:01:42.658201+0000","last_fullsized":"2024-08-17T18:01:42.658201+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1974,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659332+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T18:01:42.659332+0000","last_peered":"2024-08-17T18:01:42.659332+0000","last_clean":"2024-08-17T18:01:42.659332+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T18:01:42.659332+0000","last_undegraded":"2024-08-17T18:01:42.659332+0000","last_fullsized":"2024-08-17T18:01:42.659332+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":584,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.657453+0000","last_change":"2024-08-17T17:59:58.027770+0000","last_active":"2024-08-17T18:01:42.657453+0000","last_peered":"2024-08-17T18:01:42.657453+0000","last_clean":"2024-08-17T18:01:42.657453+0000","last_became_active":"2024-08-17T17:59:58.027251+0000","last_became_peered":"2024-08-17T17:59:58.027251+0000","last_unstale":"2024-08-17T18:01:42.657453+0000","last_undegraded":"2024-08-17T18:01:42.657453+0000","last_fullsized":"2024-08-17T18:01:42.657453+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:26.416690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":195,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.656400+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T18:01:42.656400+0000","last_peered":"2024-08-17T18:01:42.656400+0000","last_clean":"2024-08-17T18:01:42.656400+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T18:01:42.656400+0000","last_undegraded":"2024-08-17T18:01:42.656400+0000","last_fullsized":"2024-08-17T18:01:42.656400+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1947,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.663474+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T18:01:42.663474+0000","last_peered":"2024-08-17T18:01:42.663474+0000","last_clean":"2024-08-17T18:01:42.663474+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T18:01:42.663474+0000","last_undegraded":"2024-08-17T18:01:42.663474+0000","last_fullsized":"2024-08-17T18:01:42.663474+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1850,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658525+0000","last_change":"2024-08-17T17:59:58.044651+0000","last_active":"2024-08-17T18:01:42.658525+0000","last_peered":"2024-08-17T18:01:42.658525+0000","last_clean":"2024-08-17T18:01:42.658525+0000","last_became_active":"2024-08-17T17:59:58.043830+0000","last_became_peered":"2024-08-17T17:59:58.043830+0000","last_unstale":"2024-08-17T18:01:42.658525+0000","last_undegraded":"2024-08-17T18:01:42.658525+0000","last_fullsized":"2024-08-17T18:01:42.658525+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:56.568577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1997,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.656786+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T18:01:42.656786+0000","last_peered":"2024-08-17T18:01:42.656786+0000","last_clean":"2024-08-17T18:01:42.656786+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T18:01:42.656786+0000","last_undegraded":"2024-08-17T18:01:42.656786+0000","last_fullsized":"2024-08-17T18:01:42.656786+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1797,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658058+0000","last_change":"2024-08-17T17:59:58.028639+0000","last_active":"2024-08-17T18:01:42.658058+0000","last_peered":"2024-08-17T18:01:42.658058+0000","last_clean":"2024-08-17T18:01:42.658058+0000","last_became_active":"2024-08-17T17:59:58.028314+0000","last_became_peered":"2024-08-17T17:59:58.028314+0000","last_unstale":"2024-08-17T18:01:42.658058+0000","last_undegraded":"2024-08-17T18:01:42.658058+0000","last_fullsized":"2024-08-17T18:01:42.658058+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:25:03.619052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1252,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.579422+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T18:01:28.579422+0000","last_peered":"2024-08-17T18:01:28.579422+0000","last_clean":"2024-08-17T18:01:28.579422+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T18:01:28.579422+0000","last_undegraded":"2024-08-17T18:01:28.579422+0000","last_fullsized":"2024-08-17T18:01:28.579422+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1941,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.661521+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T18:01:42.661521+0000","last_peered":"2024-08-17T18:01:42.661521+0000","last_clean":"2024-08-17T18:01:42.661521+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T18:01:42.661521+0000","last_undegraded":"2024-08-17T18:01:42.661521+0000","last_fullsized":"2024-08-17T18:01:42.661521+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1991,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.660973+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T18:01:42.660973+0000","last_peered":"2024-08-17T18:01:42.660973+0000","last_clean":"2024-08-17T18:01:42.660973+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T18:01:42.660973+0000","last_undegraded":"2024-08-17T18:01:42.660973+0000","last_fullsized":"2024-08-17T18:01:42.660973+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1897,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658125+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T18:01:42.658125+0000","last_peered":"2024-08-17T18:01:42.658125+0000","last_clean":"2024-08-17T18:01:42.658125+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T18:01:42.658125+0000","last_undegraded":"2024-08-17T18:01:42.658125+0000","last_fullsized":"2024-08-17T18:01:42.658125+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":571,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.582580+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T18:01:28.582580+0000","last_peered":"2024-08-17T18:01:28.582580+0000","last_clean":"2024-08-17T18:01:28.582580+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T18:01:28.582580+0000","last_undegraded":"2024-08-17T18:01:28.582580+0000","last_fullsized":"2024-08-17T18:01:28.582580+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1248,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.582434+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T18:01:28.582434+0000","last_peered":"2024-08-17T18:01:28.582434+0000","last_clean":"2024-08-17T18:01:28.582434+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T18:01:28.582434+0000","last_undegraded":"2024-08-17T18:01:28.582434+0000","last_fullsized":"2024-08-17T18:01:28.582434+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1996,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.656652+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T18:01:42.656652+0000","last_peered":"2024-08-17T18:01:42.656652+0000","last_clean":"2024-08-17T18:01:42.656652+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T18:01:42.656652+0000","last_undegraded":"2024-08-17T18:01:42.656652+0000","last_fullsized":"2024-08-17T18:01:42.656652+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":409,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.582353+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T18:01:28.582353+0000","last_peered":"2024-08-17T18:01:28.582353+0000","last_clean":"2024-08-17T18:01:28.582353+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T18:01:28.582353+0000","last_undegraded":"2024-08-17T18:01:28.582353+0000","last_fullsized":"2024-08-17T18:01:28.582353+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":555,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.579025+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T18:01:28.579025+0000","last_peered":"2024-08-17T18:01:28.579025+0000","last_clean":"2024-08-17T18:01:28.579025+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T18:01:28.579025+0000","last_undegraded":"2024-08-17T18:01:28.579025+0000","last_fullsized":"2024-08-17T18:01:28.579025+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1233,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.578888+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T18:01:28.578888+0000","last_peered":"2024-08-17T18:01:28.578888+0000","last_clean":"2024-08-17T18:01:28.578888+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T18:01:28.578888+0000","last_undegraded":"2024-08-17T18:01:28.578888+0000","last_fullsized":"2024-08-17T18:01:28.578888+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1241,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.579457+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T18:01:28.579457+0000","last_peered":"2024-08-17T18:01:28.579457+0000","last_clean":"2024-08-17T18:01:28.579457+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T18:01:28.579457+0000","last_undegraded":"2024-08-17T18:01:28.579457+0000","last_fullsized":"2024-08-17T18:01:28.579457+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":2044,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.655987+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T18:01:42.655987+0000","last_peered":"2024-08-17T18:01:42.655987+0000","last_clean":"2024-08-17T18:01:42.655987+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T18:01:42.655987+0000","last_undegraded":"2024-08-17T18:01:42.655987+0000","last_fullsized":"2024-08-17T18:01:42.655987+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1927,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.661678+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T18:01:42.661678+0000","last_peered":"2024-08-17T18:01:42.661678+0000","last_clean":"2024-08-17T18:01:42.661678+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T18:01:42.661678+0000","last_undegraded":"2024-08-17T18:01:42.661678+0000","last_fullsized":"2024-08-17T18:01:42.661678+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":2002,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658137+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T18:01:42.658137+0000","last_peered":"2024-08-17T18:01:42.658137+0000","last_clean":"2024-08-17T18:01:42.658137+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T18:01:42.658137+0000","last_undegraded":"2024-08-17T18:01:42.658137+0000","last_fullsized":"2024-08-17T18:01:42.658137+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1929,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.665740+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T18:01:42.665740+0000","last_peered":"2024-08-17T18:01:42.665740+0000","last_clean":"2024-08-17T18:01:42.665740+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T18:01:42.665740+0000","last_undegraded":"2024-08-17T18:01:42.665740+0000","last_fullsized":"2024-08-17T18:01:42.665740+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1962,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659418+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T18:01:42.659418+0000","last_peered":"2024-08-17T18:01:42.659418+0000","last_clean":"2024-08-17T18:01:42.659418+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T18:01:42.659418+0000","last_undegraded":"2024-08-17T18:01:42.659418+0000","last_fullsized":"2024-08-17T18:01:42.659418+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1268,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.582650+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T18:01:28.582650+0000","last_peered":"2024-08-17T18:01:28.582650+0000","last_clean":"2024-08-17T18:01:28.582650+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T18:01:28.582650+0000","last_undegraded":"2024-08-17T18:01:28.582650+0000","last_fullsized":"2024-08-17T18:01:28.582650+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":2008,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.657222+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T18:01:42.657222+0000","last_peered":"2024-08-17T18:01:42.657222+0000","last_clean":"2024-08-17T18:01:42.657222+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T18:01:42.657222+0000","last_undegraded":"2024-08-17T18:01:42.657222+0000","last_fullsized":"2024-08-17T18:01:42.657222+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":195,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.656310+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T18:01:42.656310+0000","last_peered":"2024-08-17T18:01:42.656310+0000","last_clean":"2024-08-17T18:01:42.656310+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T18:01:42.656310+0000","last_undegraded":"2024-08-17T18:01:42.656310+0000","last_fullsized":"2024-08-17T18:01:42.656310+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2027,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.657000+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T18:01:42.657000+0000","last_peered":"2024-08-17T18:01:42.657000+0000","last_clean":"2024-08-17T18:01:42.657000+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T18:01:42.657000+0000","last_undegraded":"2024-08-17T18:01:42.657000+0000","last_fullsized":"2024-08-17T18:01:42.657000+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":764,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658624+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T18:01:42.658624+0000","last_peered":"2024-08-17T18:01:42.658624+0000","last_clean":"2024-08-17T18:01:42.658624+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T18:01:42.658624+0000","last_undegraded":"2024-08-17T18:01:42.658624+0000","last_fullsized":"2024-08-17T18:01:42.658624+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":2008,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.657319+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T18:01:42.657319+0000","last_peered":"2024-08-17T18:01:42.657319+0000","last_clean":"2024-08-17T18:01:42.657319+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T18:01:42.657319+0000","last_undegraded":"2024-08-17T18:01:42.657319+0000","last_fullsized":"2024-08-17T18:01:42.657319+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1337,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658996+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T18:01:42.658996+0000","last_peered":"2024-08-17T18:01:42.658996+0000","last_clean":"2024-08-17T18:01:42.658996+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T18:01:42.658996+0000","last_undegraded":"2024-08-17T18:01:42.658996+0000","last_fullsized":"2024-08-17T18:01:42.658996+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":255,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.656557+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T18:01:42.656557+0000","last_peered":"2024-08-17T18:01:42.656557+0000","last_clean":"2024-08-17T18:01:42.656557+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T18:01:42.656557+0000","last_undegraded":"2024-08-17T18:01:42.656557+0000","last_fullsized":"2024-08-17T18:01:42.656557+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":2019,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.656446+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T18:01:42.656446+0000","last_peered":"2024-08-17T18:01:42.656446+0000","last_clean":"2024-08-17T18:01:42.656446+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T18:01:42.656446+0000","last_undegraded":"2024-08-17T18:01:42.656446+0000","last_fullsized":"2024-08-17T18:01:42.656446+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1997,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.672402+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T18:01:42.672402+0000","last_peered":"2024-08-17T18:01:42.672402+0000","last_clean":"2024-08-17T18:01:42.672402+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T18:01:42.672402+0000","last_undegraded":"2024-08-17T18:01:42.672402+0000","last_fullsized":"2024-08-17T18:01:42.672402+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1844,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658433+0000","last_change":"2024-08-17T17:59:58.030150+0000","last_active":"2024-08-17T18:01:42.658433+0000","last_peered":"2024-08-17T18:01:42.658433+0000","last_clean":"2024-08-17T18:01:42.658433+0000","last_became_active":"2024-08-17T17:59:58.029407+0000","last_became_peered":"2024-08-17T17:59:58.029407+0000","last_unstale":"2024-08-17T18:01:42.658433+0000","last_undegraded":"2024-08-17T18:01:42.658433+0000","last_fullsized":"2024-08-17T18:01:42.658433+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:31.632793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2023,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.656877+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T18:01:42.656877+0000","last_peered":"2024-08-17T18:01:42.656877+0000","last_clean":"2024-08-17T18:01:42.656877+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T18:01:42.656877+0000","last_undegraded":"2024-08-17T18:01:42.656877+0000","last_fullsized":"2024-08-17T18:01:42.656877+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2003,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659451+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T18:01:42.659451+0000","last_peered":"2024-08-17T18:01:42.659451+0000","last_clean":"2024-08-17T18:01:42.659451+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T18:01:42.659451+0000","last_undegraded":"2024-08-17T18:01:42.659451+0000","last_fullsized":"2024-08-17T18:01:42.659451+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1275,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.584061+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T18:01:28.584061+0000","last_peered":"2024-08-17T18:01:28.584061+0000","last_clean":"2024-08-17T18:01:28.584061+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T18:01:28.584061+0000","last_undegraded":"2024-08-17T18:01:28.584061+0000","last_fullsized":"2024-08-17T18:01:28.584061+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1221,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.579546+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T18:01:28.579546+0000","last_peered":"2024-08-17T18:01:28.579546+0000","last_clean":"2024-08-17T18:01:28.579546+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T18:01:28.579546+0000","last_undegraded":"2024-08-17T18:01:28.579546+0000","last_fullsized":"2024-08-17T18:01:28.579546+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1912,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658233+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T18:01:42.658233+0000","last_peered":"2024-08-17T18:01:42.658233+0000","last_clean":"2024-08-17T18:01:42.658233+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T18:01:42.658233+0000","last_undegraded":"2024-08-17T18:01:42.658233+0000","last_fullsized":"2024-08-17T18:01:42.658233+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":37,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.582340+0000","last_change":"2024-08-17T17:59:58.027683+0000","last_active":"2024-08-17T18:01:28.582340+0000","last_peered":"2024-08-17T18:01:28.582340+0000","last_clean":"2024-08-17T18:01:28.582340+0000","last_became_active":"2024-08-17T17:59:58.027358+0000","last_became_peered":"2024-08-17T17:59:58.027358+0000","last_unstale":"2024-08-17T18:01:28.582340+0000","last_undegraded":"2024-08-17T18:01:28.582340+0000","last_fullsized":"2024-08-17T18:01:28.582340+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:04.758259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1587,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.584017+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T18:01:28.584017+0000","last_peered":"2024-08-17T18:01:28.584017+0000","last_clean":"2024-08-17T18:01:28.584017+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T18:01:28.584017+0000","last_undegraded":"2024-08-17T18:01:28.584017+0000","last_fullsized":"2024-08-17T18:01:28.584017+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2003,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.656570+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T18:01:42.656570+0000","last_peered":"2024-08-17T18:01:42.656570+0000","last_clean":"2024-08-17T18:01:42.656570+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T18:01:42.656570+0000","last_undegraded":"2024-08-17T18:01:42.656570+0000","last_fullsized":"2024-08-17T18:01:42.656570+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1943,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.661601+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T18:01:42.661601+0000","last_peered":"2024-08-17T18:01:42.661601+0000","last_clean":"2024-08-17T18:01:42.661601+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T18:01:42.661601+0000","last_undegraded":"2024-08-17T18:01:42.661601+0000","last_fullsized":"2024-08-17T18:01:42.661601+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":555,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.579406+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T18:01:28.579406+0000","last_peered":"2024-08-17T18:01:28.579406+0000","last_clean":"2024-08-17T18:01:28.579406+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T18:01:28.579406+0000","last_undegraded":"2024-08-17T18:01:28.579406+0000","last_fullsized":"2024-08-17T18:01:28.579406+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":515,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.581878+0000","last_change":"2024-08-17T17:59:58.021264+0000","last_active":"2024-08-17T18:01:28.581878+0000","last_peered":"2024-08-17T18:01:28.581878+0000","last_clean":"2024-08-17T18:01:28.581878+0000","last_became_active":"2024-08-17T17:59:58.020957+0000","last_became_peered":"2024-08-17T17:59:58.020957+0000","last_unstale":"2024-08-17T18:01:28.581878+0000","last_undegraded":"2024-08-17T18:01:28.581878+0000","last_fullsized":"2024-08-17T18:01:28.581878+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:42.190270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":545,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.657715+0000","last_change":"2024-08-17T17:59:58.043761+0000","last_active":"2024-08-17T18:01:42.657715+0000","last_peered":"2024-08-17T18:01:42.657715+0000","last_clean":"2024-08-17T18:01:42.657715+0000","last_became_active":"2024-08-17T17:59:58.043206+0000","last_became_peered":"2024-08-17T17:59:58.043206+0000","last_unstale":"2024-08-17T18:01:42.657715+0000","last_undegraded":"2024-08-17T18:01:42.657715+0000","last_fullsized":"2024-08-17T18:01:42.657715+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:37.774589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1239,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.579066+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T18:01:28.579066+0000","last_peered":"2024-08-17T18:01:28.579066+0000","last_clean":"2024-08-17T18:01:28.579066+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T18:01:28.579066+0000","last_undegraded":"2024-08-17T18:01:28.579066+0000","last_fullsized":"2024-08-17T18:01:28.579066+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":714,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.664131+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T18:01:42.664131+0000","last_peered":"2024-08-17T18:01:42.664131+0000","last_clean":"2024-08-17T18:01:42.664131+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T18:01:42.664131+0000","last_undegraded":"2024-08-17T18:01:42.664131+0000","last_fullsized":"2024-08-17T18:01:42.664131+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1940,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658314+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T18:01:42.658314+0000","last_peered":"2024-08-17T18:01:42.658314+0000","last_clean":"2024-08-17T18:01:42.658314+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T18:01:42.658314+0000","last_undegraded":"2024-08-17T18:01:42.658314+0000","last_fullsized":"2024-08-17T18:01:42.658314+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2035,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659495+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T18:01:42.659495+0000","last_peered":"2024-08-17T18:01:42.659495+0000","last_clean":"2024-08-17T18:01:42.659495+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T18:01:42.659495+0000","last_undegraded":"2024-08-17T18:01:42.659495+0000","last_fullsized":"2024-08-17T18:01:42.659495+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":747,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.663820+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T18:01:42.663820+0000","last_peered":"2024-08-17T18:01:42.663820+0000","last_clean":"2024-08-17T18:01:42.663820+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T18:01:42.663820+0000","last_undegraded":"2024-08-17T18:01:42.663820+0000","last_fullsized":"2024-08-17T18:01:42.663820+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1228,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.578938+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T18:01:28.578938+0000","last_peered":"2024-08-17T18:01:28.578938+0000","last_clean":"2024-08-17T18:01:28.578938+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T18:01:28.578938+0000","last_undegraded":"2024-08-17T18:01:28.578938+0000","last_fullsized":"2024-08-17T18:01:28.578938+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1227,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.582573+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T18:01:28.582573+0000","last_peered":"2024-08-17T18:01:28.582573+0000","last_clean":"2024-08-17T18:01:28.582573+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T18:01:28.582573+0000","last_undegraded":"2024-08-17T18:01:28.582573+0000","last_fullsized":"2024-08-17T18:01:28.582573+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":731,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.666227+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T18:01:42.666227+0000","last_peered":"2024-08-17T18:01:42.666227+0000","last_clean":"2024-08-17T18:01:42.666227+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T18:01:42.666227+0000","last_undegraded":"2024-08-17T18:01:42.666227+0000","last_fullsized":"2024-08-17T18:01:42.666227+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1245,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.582735+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T18:01:28.582735+0000","last_peered":"2024-08-17T18:01:28.582735+0000","last_clean":"2024-08-17T18:01:28.582735+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T18:01:28.582735+0000","last_undegraded":"2024-08-17T18:01:28.582735+0000","last_fullsized":"2024-08-17T18:01:28.582735+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":129,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.583967+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T18:01:28.583967+0000","last_peered":"2024-08-17T18:01:28.583967+0000","last_clean":"2024-08-17T18:01:28.583967+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T18:01:28.583967+0000","last_undegraded":"2024-08-17T18:01:28.583967+0000","last_fullsized":"2024-08-17T18:01:28.583967+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1261,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.582528+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T18:01:28.582528+0000","last_peered":"2024-08-17T18:01:28.582528+0000","last_clean":"2024-08-17T18:01:28.582528+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T18:01:28.582528+0000","last_undegraded":"2024-08-17T18:01:28.582528+0000","last_fullsized":"2024-08-17T18:01:28.582528+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1278,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.579040+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T18:01:28.579040+0000","last_peered":"2024-08-17T18:01:28.579040+0000","last_clean":"2024-08-17T18:01:28.579040+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T18:01:28.579040+0000","last_undegraded":"2024-08-17T18:01:28.579040+0000","last_fullsized":"2024-08-17T18:01:28.579040+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1941,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.665612+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T18:01:42.665612+0000","last_peered":"2024-08-17T18:01:42.665612+0000","last_clean":"2024-08-17T18:01:42.665612+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T18:01:42.665612+0000","last_undegraded":"2024-08-17T18:01:42.665612+0000","last_fullsized":"2024-08-17T18:01:42.665612+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1222,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.582014+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T18:01:28.582014+0000","last_peered":"2024-08-17T18:01:28.582014+0000","last_clean":"2024-08-17T18:01:28.582014+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T18:01:28.582014+0000","last_undegraded":"2024-08-17T18:01:28.582014+0000","last_fullsized":"2024-08-17T18:01:28.582014+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1994,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.656102+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T18:01:42.656102+0000","last_peered":"2024-08-17T18:01:42.656102+0000","last_clean":"2024-08-17T18:01:42.656102+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T18:01:42.656102+0000","last_undegraded":"2024-08-17T18:01:42.656102+0000","last_fullsized":"2024-08-17T18:01:42.656102+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1995,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.661069+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T18:01:42.661069+0000","last_peered":"2024-08-17T18:01:42.661069+0000","last_clean":"2024-08-17T18:01:42.661069+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T18:01:42.661069+0000","last_undegraded":"2024-08-17T18:01:42.661069+0000","last_fullsized":"2024-08-17T18:01:42.661069+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":635,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.582344+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T18:01:28.582344+0000","last_peered":"2024-08-17T18:01:28.582344+0000","last_clean":"2024-08-17T18:01:28.582344+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T18:01:28.582344+0000","last_undegraded":"2024-08-17T18:01:28.582344+0000","last_fullsized":"2024-08-17T18:01:28.582344+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1247,"reported_epoch":1277,"state":"active+clean","last_fresh":"2024-08-17T18:01:28.582281+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T18:01:28.582281+0000","last_peered":"2024-08-17T18:01:28.582281+0000","last_clean":"2024-08-17T18:01:28.582281+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T18:01:28.582281+0000","last_undegraded":"2024-08-17T18:01:28.582281+0000","last_fullsized":"2024-08-17T18:01:28.582281+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1941,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.665345+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T18:01:42.665345+0000","last_peered":"2024-08-17T18:01:42.665345+0000","last_clean":"2024-08-17T18:01:42.665345+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T18:01:42.665345+0000","last_undegraded":"2024-08-17T18:01:42.665345+0000","last_fullsized":"2024-08-17T18:01:42.665345+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":46,"num_pg":16,"stat_sum":{"num_bytes":4823121944,"num_objects":73596,"num_object_clones":0,"num_object_copies":220788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73596,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142366,"num_write_kb":6910721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14436347904,"data_stored":14436335688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105566,"ondisk_log_size":105566,"up":48,"acting":48,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234491,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2367660,"kb_used_data":2066148,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91377492,"statfs":{"total":95995035648,"available":93570551808,"internally_reserved":0,"allocated":2115735552,"data_stored":2111860546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496126,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2377512,"kb_used_data":2080160,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":91367640,"statfs":{"total":95995035648,"available":93560463360,"internally_reserved":0,"allocated":2130083840,"data_stored":2126213383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757764,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326378,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2759504,"kb_used_data":2371720,"kb_used_omap":57,"kb_used_meta":387718,"kb_avail":90985648,"statfs":{"total":95995035648,"available":93169303552,"internally_reserved":0,"allocated":2428641280,"data_stored":2424789590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":397023995},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405617,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3029148,"kb_used_data":2634644,"kb_used_omap":61,"kb_used_meta":394434,"kb_avail":90716004,"statfs":{"total":95995035648,"available":92893188096,"internally_reserved":0,"allocated":2697875456,"data_stored":2694016666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":403901368},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,4,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452151,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2442068,"kb_used_data":2105932,"kb_used_omap":54,"kb_used_meta":335945,"kb_avail":91303084,"statfs":{"total":95995035648,"available":93494358016,"internally_reserved":0,"allocated":2156474368,"data_stored":2152618487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":344008078},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474842,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3401952,"kb_used_data":2980376,"kb_used_omap":119,"kb_used_meta":421448,"kb_avail":90343200,"statfs":{"total":95995035648,"available":92511436800,"internally_reserved":0,"allocated":3051905024,"data_stored":3048038067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":431563486},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,3,6,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019399,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3011121152,"data_stored":3011117080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2137133056,"data_stored":2137128984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2680094720,"data_stored":2680094720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2406481920,"data_stored":2406481920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2105016320,"data_stored":2105016320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2096500736,"data_stored":2096496664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:01:52.234 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T18:01:52.234 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:01:52.234 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-08-17T18:01:52.234 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T18:01:52.234 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T18:01:52.235 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-08-17T18:01:52.245 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:52.281 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:52.305 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:52.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:52.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:52.407 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:52.453 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:52.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:52.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:52.552 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:01:52.555 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:52.581 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:52.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:52.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:52.683 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:52.706 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:52.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:52.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:52.808 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:52.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:52.854 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:52.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:52.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:52 smithi130 ceph-mon[105521]: pgmap v2096: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 770 op/s 2024-08-17T18:01:52.900 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:52 smithi130 ceph-mon[110068]: pgmap v2096: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 770 op/s 2024-08-17T18:01:52.956 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:52.963 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:52 smithi179 ceph-mon[93519]: pgmap v2096: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 770 op/s 2024-08-17T18:01:52.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:52.984 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:52.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:53.086 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:53.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:53.133 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:53.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:53.235 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:53.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:53.277 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:53.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:53.333 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:01:53.333 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:01:53.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.334 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:146876 46.3 46:c90eaf0f:::benchmark_data_smithi130_267720_object38893:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1278)", 2024-08-17T18:01:53.334 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:01:53.033492+0000", 2024-08-17T18:01:53.334 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.02537271, 2024-08-17T18:01:53.334 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.025401374000000001, 2024-08-17T18:01:53.334 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:01:53.334 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:01:53.335 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:01:53.335 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:01:53.335 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:01:53.335 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:53.336 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 146876 2024-08-17T18:01:53.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.336 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:01:53.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:01:53.337 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.033492+0000", 2024-08-17T18:01:53.337 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:01:53.337 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.337 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.338 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:01:53.338 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.033492+0000", 2024-08-17T18:01:53.338 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:01:53.338 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.338 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.339 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:01:53.339 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.033493+0000", 2024-08-17T18:01:53.340 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.353e-06 2024-08-17T18:01:53.340 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.340 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.340 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:01:53.340 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.033498+0000", 2024-08-17T18:01:53.341 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0930000000000002e-06 2024-08-17T18:01:53.341 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.341 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.341 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:01:53.341 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.033515+0000", 2024-08-17T18:01:53.342 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7082e-05 2024-08-17T18:01:53.342 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.342 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.342 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:01:53.343 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.033522+0000", 2024-08-17T18:01:53.343 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8680000000000001e-06 2024-08-17T18:01:53.343 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.343 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.343 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:01:53.344 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.033584+0000", 2024-08-17T18:01:53.344 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1828000000000005e-05 2024-08-17T18:01:53.344 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.344 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.345 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:01:53.345 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.033705+0000", 2024-08-17T18:01:53.345 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000120877 2024-08-17T18:01:53.345 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.346 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.346 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:01:53.346 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.033828+0000", 2024-08-17T18:01:53.346 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012328099999999999 2024-08-17T18:01:53.346 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.347 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.347 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:01:53.347 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.038732+0000", 2024-08-17T18:01:53.347 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0049035980000000003 2024-08-17T18:01:53.348 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.348 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.348 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:53.348 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.040426+0000", 2024-08-17T18:01:53.348 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0016941980000000001 2024-08-17T18:01:53.349 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:53.349 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:01:53.349 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:53.349 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.349 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.350 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:146890 46.7 46:e28dcc9f:::benchmark_data_smithi130_267720_object38907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1278)", 2024-08-17T18:01:53.350 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:01:53.046639+0000", 2024-08-17T18:01:53.350 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.012225222000000001, 2024-08-17T18:01:53.350 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012329368, 2024-08-17T18:01:53.351 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:01:53.351 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:01:53.351 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:01:53.351 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:01:53.351 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:01:53.352 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:53.352 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 146890 2024-08-17T18:01:53.352 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.352 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:01:53.353 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.353 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:01:53.353 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.046639+0000", 2024-08-17T18:01:53.353 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:01:53.353 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.354 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.354 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:01:53.354 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.046639+0000", 2024-08-17T18:01:53.354 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:01:53.354 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:01:53.355 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.046641+0000", 2024-08-17T18:01:53.355 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1960000000000001e-06 2024-08-17T18:01:53.355 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.356 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:01:53.356 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.046646+0000", 2024-08-17T18:01:53.356 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3669999999999996e-06 2024-08-17T18:01:53.356 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.356 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.356 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:01:53.357 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.046656+0000", 2024-08-17T18:01:53.357 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5629999999999998e-06 2024-08-17T18:01:53.357 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.357 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.357 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:01:53.358 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.046662+0000", 2024-08-17T18:01:53.358 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5150000000000003e-06 2024-08-17T18:01:53.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.358 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:01:53.359 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.046705+0000", 2024-08-17T18:01:53.359 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2676000000000003e-05 2024-08-17T18:01:53.359 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.360 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.360 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:01:53.360 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.046827+0000", 2024-08-17T18:01:53.360 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012231300000000001 2024-08-17T18:01:53.360 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.361 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:01:53.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.046964+0000", 2024-08-17T18:01:53.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000136704 2024-08-17T18:01:53.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.361 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:01:53.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050045+0000", 2024-08-17T18:01:53.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00308144 2024-08-17T18:01:53.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.361 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:53.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.054649+0000", 2024-08-17T18:01:53.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.004604141 2024-08-17T18:01:53.362 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:53.362 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:01:53.362 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:53.362 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.362 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:146892 46.3 46:cf560e90:::benchmark_data_smithi130_267720_object38909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1278)", 2024-08-17T18:01:53.362 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:01:53.050642+0000", 2024-08-17T18:01:53.362 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.008222699, 2024-08-17T18:01:53.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0083934300000000003, 2024-08-17T18:01:53.362 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:01:53.362 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:01:53.362 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:01:53.362 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:01:53.363 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:01:53.363 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:01:53.363 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 146892 2024-08-17T18:01:53.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.363 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:01:53.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:01:53.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050642+0000", 2024-08-17T18:01:53.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:01:53.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:01:53.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050642+0000", 2024-08-17T18:01:53.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:01:53.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:01:53.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050644+0000", 2024-08-17T18:01:53.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.623e-06 2024-08-17T18:01:53.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:01:53.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050649+0000", 2024-08-17T18:01:53.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2499999999999997e-06 2024-08-17T18:01:53.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:01:53.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050663+0000", 2024-08-17T18:01:53.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4679e-05 2024-08-17T18:01:53.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:01:53.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050672+0000", 2024-08-17T18:01:53.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1829999999999994e-06 2024-08-17T18:01:53.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:01:53.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050738+0000", 2024-08-17T18:01:53.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6166000000000003e-05 2024-08-17T18:01:53.366 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.366 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:01:53.366 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050934+0000", 2024-08-17T18:01:53.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00019589999999999999 2024-08-17T18:01:53.366 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.366 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:01:53.366 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.051049+0000", 2024-08-17T18:01:53.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000114954 2024-08-17T18:01:53.366 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.367 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:01:53.367 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.058861+0000", 2024-08-17T18:01:53.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0078121299999999996 2024-08-17T18:01:53.367 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:53.367 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:01:53.367 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:53.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.367 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.367 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:146893 46.6 e1278/1258 46:6fd42724:::benchmark_data_smithi130_267720_object38910:head v 1278'9177, mlcod=1278'9174)", 2024-08-17T18:01:53.367 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:01:53.050868+0000", 2024-08-17T18:01:53.367 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0079965629999999999, 2024-08-17T18:01:53.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0082289919999999992, 2024-08-17T18:01:53.367 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:01:53.367 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:01:53.368 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:01:53.368 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:01:53.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:01:53.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050868+0000", 2024-08-17T18:01:53.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:01:53.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:01:53.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050868+0000", 2024-08-17T18:01:53.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:01:53.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:01:53.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050869+0000", 2024-08-17T18:01:53.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1370000000000001e-06 2024-08-17T18:01:53.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.369 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:01:53.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050874+0000", 2024-08-17T18:01:53.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1839999999999998e-06 2024-08-17T18:01:53.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.369 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:01:53.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050886+0000", 2024-08-17T18:01:53.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1905e-05 2024-08-17T18:01:53.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:01:53.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.050893+0000", 2024-08-17T18:01:53.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5370000000000003e-06 2024-08-17T18:01:53.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:01:53.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.051430+0000", 2024-08-17T18:01:53.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000537369 2024-08-17T18:01:53.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:01:53.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:01:53.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:01:53.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:53.051448+0000", 2024-08-17T18:01:53.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7934000000000001e-05 2024-08-17T18:01:53.371 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:53.371 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:01:53.371 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:53.371 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:01:53.371 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T18:01:53.371 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 4 2024-08-17T18:01:53.371 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:01:53.377 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:53.401 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:53.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:53.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:53.502 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:53.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:53.525 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:53.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:53.626 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:53.661 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:53.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:53.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:53.763 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:53.787 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:53.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:53.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:53.889 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:53.931 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:53.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:53.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:54.033 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:54.036 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T18:01:54.056 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:54.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:54.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:54.158 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:54.216 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:54.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:54.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:54.318 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:54.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:54.379 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:54.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:54.480 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:54.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:54.505 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:54.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:54.606 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:54.643 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:54.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:54.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:54.745 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:54.804 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:54.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:54.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:54.838 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:54 smithi179 ceph-mon[93519]: pgmap v2097: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T18:01:54.853 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:54 smithi130 ceph-mon[105521]: pgmap v2097: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T18:01:54.854 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:54 smithi130 ceph-mon[110068]: pgmap v2097: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T18:01:54.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:54.934 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:54.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:54.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:55.036 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:55.059 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:55.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:55.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:55.161 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:55.186 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:55.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:55.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:55.288 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:55.311 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:55.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:55.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:55.413 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:55.420 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:01:55.459 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:01:55.459 INFO:teuthology.orchestra.run.smithi179.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T18:01:55.459 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:01:55.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:55.470 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:55.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:55.571 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:55.621 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:55.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:55.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:55.726 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:55.730 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:01:55.749 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:55.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:55.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:55.850 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:55.896 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:55.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:55.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:55.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:55 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1206194902' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:01:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:55 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:01:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:55 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1206194902' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:01:55.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:55 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:01:55.999 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:56.021 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:56.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:56.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:55 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1206194902' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:01:56.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:55 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:01:56.123 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:56.159 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:56.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:56.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:56.261 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:56.305 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:56.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:56.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:56.407 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:56.429 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:56.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:56.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:56.530 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:56.573 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:01:56.573 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:01:56.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:56.581 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:56.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:56.587 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:01:56.587 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:01:56.622 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:01:56.622 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:01:56.653 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:01:56.653 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:01:56.682 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:56.685 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:01:56.685 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:01:56.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:56.705 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:56.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:56.716 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:01:56.716 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:01:56.746 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:01:56.747 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:01:56.774 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:01:56.774 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:01:56.775 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:56 smithi130 ceph-mon[105521]: pgmap v2098: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 702 op/s 2024-08-17T18:01:56.775 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:56 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:01:56.775 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:56 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:01:56.775 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:56 smithi130 ceph-mon[105521]: osdmap e1279: 8 total, 8 up, 6 in 2024-08-17T18:01:56.775 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:56 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3505182442' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:01:56.775 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:56 smithi130 ceph-mon[110068]: pgmap v2098: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 702 op/s 2024-08-17T18:01:56.775 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:56 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:01:56.775 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:56 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:01:56.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:56 smithi130 ceph-mon[110068]: osdmap e1279: 8 total, 8 up, 6 in 2024-08-17T18:01:56.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:56 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3505182442' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:01:56.806 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:56.830 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:56.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:56.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:56.932 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:56.954 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:56.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:56.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:57.056 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:01:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:56 smithi179 ceph-mon[93519]: pgmap v2098: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 702 op/s 2024-08-17T18:01:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:56 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:01:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:56 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:01:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:56 smithi179 ceph-mon[93519]: osdmap e1279: 8 total, 8 up, 6 in 2024-08-17T18:01:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:56 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3505182442' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:01:57.102 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:57.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:57.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:01:57.204 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:57.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:57.230 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:57.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:57.331 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:57.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:57.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:57.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:57.459 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:57.482 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:57.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:57.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:57.584 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:57.621 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:57.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:57.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:57.723 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:57.770 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:57.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:57.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:57.872 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:57.904 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:57.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:57.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:58.006 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:58.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:58.051 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:58.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:58.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:01:58.152 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:01:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:01:58.177 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:58.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:58.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:58.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:01:58.301 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:58.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:58.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:01:58.403 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:58.427 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:58.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:58.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:58.528 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:58.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:58.568 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:58.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:58.669 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:01:58.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:58.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:58.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:01:58.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:01:58.815 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:58.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:58.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:01:58.917 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:58.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:58.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:58.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:59.059 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:59.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:01:58 smithi179 ceph-mon[93519]: pgmap v2100: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 825 op/s 2024-08-17T18:01:59.096 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:59.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:59.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:59.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:01:58 smithi130 ceph-mon[105521]: pgmap v2100: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 825 op/s 2024-08-17T18:01:59.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:01:58 smithi130 ceph-mon[110068]: pgmap v2100: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 825 op/s 2024-08-17T18:01:59.198 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:59.231 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:59.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:59.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:59.333 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:59.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:59.404 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:59.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:01:59.505 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:01:59.546 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:59.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:59.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:01:59.648 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:01:59.690 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:59.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:59.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:01:59.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:01:59.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:59.816 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:01:59.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:01:59.916 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:01:59.956 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:01:59.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:01:59.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:00.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:00.093 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:00.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:00.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:00.150 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:01:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:01:59] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-08-17T18:02:00.195 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:00.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:00.244 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:00.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:00.345 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:00.386 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:00.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:00.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:00.471 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:02:00.488 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:00.510 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:00.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:00.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:02:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:02:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:02:00.612 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:00.631 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:02:00.631 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:02:00.673 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:00.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:00.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:00.775 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:00.800 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:00.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:00.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:00.902 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:00.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:00.965 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:00.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:01.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:01.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:00 smithi179 ceph-mon[93519]: pgmap v2101: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 841 op/s 2024-08-17T18:02:01.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:01.109 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:01.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:00 smithi130 ceph-mon[105521]: pgmap v2101: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 841 op/s 2024-08-17T18:02:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:00 smithi130 ceph-mon[110068]: pgmap v2101: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 841 op/s 2024-08-17T18:02:01.193 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7] out_osds: [5, 4] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T18:02:01.193 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:02:01.194 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-08-17T18:02:01.194 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:02:01.210 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:01.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:01.288 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:01.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:01.389 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:01.412 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:01.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:01.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:01.513 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:01.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:01.580 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:01.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:01.681 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:01.704 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:01.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:01.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:01.733 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T18:02:01.806 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:01.823 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:02:01.828 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:01.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:01.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:01.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:02.467 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:02.468 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:02.493 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:02.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:02.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:02.595 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:02.619 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:02.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:02.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:02.721 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:02.743 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:02.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:02.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:02.845 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:02.891 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:02.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:02.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:02.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:02 smithi130 ceph-mon[105521]: pgmap v2102: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 841 op/s 2024-08-17T18:02:02.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:02 smithi130 ceph-mon[110068]: pgmap v2102: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 841 op/s 2024-08-17T18:02:02.995 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:03.029 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:03.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:03.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:03.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:02 smithi179 ceph-mon[93519]: pgmap v2102: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 841 op/s 2024-08-17T18:02:03.130 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:03.153 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:03.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:03.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:03.255 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:03.279 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:03.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:03.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:03.383 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:03.405 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:03.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:03.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:03.507 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:03.531 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:03.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:03.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:03.633 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:03.655 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:03.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:03.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:03.757 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:03.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:03.791 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:03.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:03.893 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:03.933 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:03.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:03.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:04.035 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:04.058 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:04.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:04.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:04.160 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:04.182 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:04.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:04.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:04.284 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:04.306 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:04.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:04.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:04.408 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:04.451 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:04.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:04.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:04.553 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:04.574 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:04.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:04.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:04.676 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:04.711 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:04.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:04.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:04.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:04.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:04.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:04.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:04.937 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:04.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:04.982 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:04.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:05.083 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:05.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:04 smithi179 ceph-mon[93519]: pgmap v2103: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 831 op/s 2024-08-17T18:02:05.122 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:05.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:05.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:05.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:04 smithi130 ceph-mon[105521]: pgmap v2103: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 831 op/s 2024-08-17T18:02:05.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:04 smithi130 ceph-mon[110068]: pgmap v2103: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 831 op/s 2024-08-17T18:02:05.224 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:05.247 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:05.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:05.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:05.349 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:05.371 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:05.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:05.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:05.473 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:05.494 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:05.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:05.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:05.595 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:05.620 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:05.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:05.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:05.721 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:05.742 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:05.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:05.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:05.844 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:05.866 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:05.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:05.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:05.969 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:05.990 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:05.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:05.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:06.092 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:06.114 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:06.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:06.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:06.216 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:06.239 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:06.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:06.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:06.341 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:06.365 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:06.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:06.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:06.467 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:06.490 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:06.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:06.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:06.592 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:06.616 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:06.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:06.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:06.718 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:06.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:06.741 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:06.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:06.842 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:06.885 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:06.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:06.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:06.987 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:07.024 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:07.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:07.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:07.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:06 smithi179 ceph-mon[93519]: pgmap v2104: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 831 op/s 2024-08-17T18:02:07.126 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:07.147 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:07.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:07.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:07.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:06 smithi130 ceph-mon[105521]: pgmap v2104: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 831 op/s 2024-08-17T18:02:07.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:06 smithi130 ceph-mon[110068]: pgmap v2104: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 831 op/s 2024-08-17T18:02:07.248 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:07.272 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:07.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:07.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:07.375 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:07.400 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:07.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:07.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:07.505 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:07.528 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:07.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:07.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:07.629 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:07.652 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:07.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:07.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:07.754 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:07.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:07.790 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:07.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:07.891 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:07.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:07.969 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:07.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:08.070 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:08.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:08.130 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:08.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:08.231 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:08.280 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:08.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:08.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:08.382 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:08.412 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:08.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:08.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:08.514 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:08.563 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:08.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:08.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:08.665 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:08.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:08.717 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:08.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:08.818 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:08.858 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:08.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:08.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:08 smithi130 ceph-mon[105521]: pgmap v2105: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 793 op/s 2024-08-17T18:02:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:08.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:08.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:08 smithi130 ceph-mon[110068]: pgmap v2105: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 793 op/s 2024-08-17T18:02:08.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:08.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:08.960 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:08.984 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:08.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:08.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:09.086 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:09.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:08 smithi179 ceph-mon[93519]: pgmap v2105: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 793 op/s 2024-08-17T18:02:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:09.110 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:09.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:09.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:09.214 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:09.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:09.279 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:09.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:09.380 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:09.404 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:09.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:09.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:09.506 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:09.528 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:09.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:09.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:09.630 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:09.654 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:09.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:09.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:09.756 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:09.805 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:09.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:09.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:09.907 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:09.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:09.965 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:09.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:10.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:10.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:10.102 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:10.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:02:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:02:09] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-08-17T18:02:10.199 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:10.202 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:10.248 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:10.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:10.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:10.350 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:10.373 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:10.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:10.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:10.475 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:10.498 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:10.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:10.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:10 smithi179 ceph-mon[93519]: pgmap v2106: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 963 op/s 2024-08-17T18:02:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:10 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:10 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:02:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:02:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:02:10.600 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:10.623 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:10.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:10.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:10.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:10 smithi130 ceph-mon[105521]: pgmap v2106: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 963 op/s 2024-08-17T18:02:10.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:10 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:10.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:10 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:10 smithi130 ceph-mon[110068]: pgmap v2106: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 963 op/s 2024-08-17T18:02:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:10 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:10 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:10.725 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:10.763 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:10.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:10.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:10.865 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:10.888 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:10.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:10.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:10.990 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:11.029 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:11.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:11.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:11.131 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:11.155 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:11.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:11.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:11.257 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:11.278 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:11.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:11.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:11.380 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:11.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:11.402 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:11.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:11.503 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:11.525 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:11.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:11.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:11.627 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:11.674 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:11.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:11.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:11.776 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:11.816 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:11.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:11.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:11.918 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:11.949 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:11.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:11.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:12.051 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:12.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:12.119 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:12.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:12.220 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:12.280 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:12.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:12.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:12.354 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:12.382 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:12.420 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:02:12.420 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:02:12.433 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:12.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:12.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:12.469 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:02:12.469 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:02:12.504 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:02:12.505 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:02:12.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:12.535 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:02:12.535 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:02:12.569 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:02:12.570 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:02:12.573 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:12.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:12.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:12.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:12 smithi130 ceph-mon[110068]: pgmap v2107: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 678 op/s 2024-08-17T18:02:12.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:02:12.589 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:12 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3507945797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:02:12.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:12 smithi130 ceph-mon[105521]: pgmap v2107: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 678 op/s 2024-08-17T18:02:12.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:02:12.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:12 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3507945797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:02:12.600 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:02:12.600 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:02:12.629 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:02:12.630 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:02:12.665 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:02:12.665 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:02:12.675 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:12.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:12.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:12.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:12.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:12.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:12.843 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:12.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:12 smithi179 ceph-mon[93519]: pgmap v2107: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 678 op/s 2024-08-17T18:02:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:02:12.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:12 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3507945797' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:02:12.944 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:12.980 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:12.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:12.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:13.082 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:13.118 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:13.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:13.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:13.220 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:13.243 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:13.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:13.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:13.345 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:13.368 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:13.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:13.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:13.470 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:13.504 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:13.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:13.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:13.606 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:13.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:13.631 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:13.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:13.732 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:13.753 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:13.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:13.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:13.855 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:13.890 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:13.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:13.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:13.991 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:14.037 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:14.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:14.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:14.139 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:14.161 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:14.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:14.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:14.263 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:14.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:14.305 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:14.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:14.406 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:14.430 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:14.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:14.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:14.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:14.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:14.575 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:14.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:14.676 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:14.698 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:14.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:14.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:14.705 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:14 smithi130 ceph-mon[110068]: pgmap v2108: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 993 op/s 2024-08-17T18:02:14.705 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:14 smithi130 ceph-mon[105521]: pgmap v2108: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 993 op/s 2024-08-17T18:02:14.800 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:14.824 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:14.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:14.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:14 smithi179 ceph-mon[93519]: pgmap v2108: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 993 op/s 2024-08-17T18:02:14.926 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:14.951 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:14.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:14.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:14.972 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:02:14.973 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:02:15.052 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:15.076 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:15.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:15.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:15.177 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:15.199 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:15.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:15.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:15.301 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:15.344 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:15.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:15.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:15.446 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:15.470 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:15.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:15.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:15.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:15.596 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:15.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:15.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:15.698 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:15.732 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:15.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:15.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:15.834 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:15.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:15 smithi179 ceph-mon[93519]: from='client.46235 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:02:15.883 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:15.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:15.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:15.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:15 smithi130 ceph-mon[105521]: from='client.46235 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:02:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:15 smithi130 ceph-mon[110068]: from='client.46235 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:02:15.985 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:16.008 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:16.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:16.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:16.110 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:16.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:16.131 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:16.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:16.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:16.253 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:16.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:16.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:16.355 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:16.379 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:16.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:16.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:16.481 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:16.503 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:16.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:16.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:16.605 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:16.626 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:16.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:16.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:16.728 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:16.751 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:16.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:16.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:16 smithi179 ceph-mon[93519]: pgmap v2109: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 649 op/s 2024-08-17T18:02:16.853 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:16.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:16.877 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:16.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:16.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:16 smithi130 ceph-mon[105521]: pgmap v2109: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 649 op/s 2024-08-17T18:02:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:16 smithi130 ceph-mon[110068]: pgmap v2109: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 649 op/s 2024-08-17T18:02:16.978 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:17.000 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:17.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:17.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:17.102 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:17.148 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:17.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:17.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:17.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:17.285 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:17.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:17.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:17.387 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:17.410 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:17.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:17.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:17.512 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:17.549 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:17.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:17.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:17.651 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:17.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:17.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:17.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:17.795 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:17.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:17.828 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:17.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:17.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:17.999 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:18.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:18.101 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:18.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:18.149 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:18.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:18.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:18.294 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:18.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:18.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:18.396 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:18.434 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:18.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:18.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:18.535 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:18.578 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2108,"stamp":"2024-08-17T18:02:13.202526+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3490125797,"num_objects":53273,"num_object_clones":0,"num_object_copies":159819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53273,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":162993,"num_write_kb":6913168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":1973261,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105622,"ondisk_log_size":105622,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12648724,"kb_used_data":10461412,"kb_used_omap":295,"kb_used_meta":2186968,"kb_avail":549822188,"statfs":{"total":575970213888,"available":563017920512,"internally_reserved":0,"allocated":10712485888,"data_stored":10689261795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":302641,"internal_metadata":2239455695},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-781778944,"num_objects":-11929,"num_object_clones":0,"num_object_copies":-35787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-11929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11929,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004999"},"pg_stats":[{"pgid":"46.f","version":"1279'10293","reported_seq":10322,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:13.024774+0000","last_change":"2024-08-17T17:59:22.956823+0000","last_active":"2024-08-17T18:02:13.024774+0000","last_peered":"2024-08-17T18:02:13.024774+0000","last_clean":"2024-08-17T18:02:13.024774+0000","last_became_active":"2024-08-17T17:59:22.956437+0000","last_became_peered":"2024-08-17T17:59:22.956437+0000","last_unstale":"2024-08-17T18:02:13.024774+0000","last_undegraded":"2024-08-17T18:02:13.024774+0000","last_fullsized":"2024-08-17T18:02:13.024774+0000","mapping_epoch":1258,"log_start":"1264'3400","ondisk_log_start":"1264'3400","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6893,"log_dups_size":0,"ondisk_log_size":6893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:17.161240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224329728,"num_objects":3423,"num_object_clones":0,"num_object_copies":10269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10293,"num_write_kb":438912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.e","version":"1279'10122","reported_seq":10171,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:12.984673+0000","last_change":"2024-08-17T17:59:58.047028+0000","last_active":"2024-08-17T18:02:12.984673+0000","last_peered":"2024-08-17T18:02:12.984673+0000","last_clean":"2024-08-17T18:02:12.984673+0000","last_became_active":"2024-08-17T17:59:58.027451+0000","last_became_peered":"2024-08-17T17:59:58.027451+0000","last_unstale":"2024-08-17T18:02:12.984673+0000","last_undegraded":"2024-08-17T18:02:12.984673+0000","last_fullsized":"2024-08-17T18:02:12.984673+0000","mapping_epoch":1263,"log_start":"1264'3100","ondisk_log_start":"1264'3100","created":1258,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:20:25.306207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216924160,"num_objects":3310,"num_object_clones":0,"num_object_copies":9930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10122,"num_write_kb":429824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.d","version":"1279'10304","reported_seq":10357,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:12.854042+0000","last_change":"2024-08-17T17:59:22.954900+0000","last_active":"2024-08-17T18:02:12.854042+0000","last_peered":"2024-08-17T18:02:12.854042+0000","last_clean":"2024-08-17T18:02:12.854042+0000","last_became_active":"2024-08-17T17:59:22.954599+0000","last_became_peered":"2024-08-17T17:59:22.954599+0000","last_unstale":"2024-08-17T18:02:12.854042+0000","last_undegraded":"2024-08-17T18:02:12.854042+0000","last_fullsized":"2024-08-17T18:02:12.854042+0000","mapping_epoch":1258,"log_start":"1265'3700","ondisk_log_start":"1265'3700","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6604,"log_dups_size":0,"ondisk_log_size":6604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:22:00.453498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224198680,"num_objects":3422,"num_object_clones":0,"num_object_copies":10266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3422,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10304,"num_write_kb":439169,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.c","version":"1279'10256","reported_seq":10285,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:12.903443+0000","last_change":"2024-08-17T17:59:22.951540+0000","last_active":"2024-08-17T18:02:12.903443+0000","last_peered":"2024-08-17T18:02:12.903443+0000","last_clean":"2024-08-17T18:02:12.903443+0000","last_became_active":"2024-08-17T17:59:22.951248+0000","last_became_peered":"2024-08-17T17:59:22.951248+0000","last_unstale":"2024-08-17T18:02:12.903443+0000","last_undegraded":"2024-08-17T18:02:12.903443+0000","last_fullsized":"2024-08-17T18:02:12.903443+0000","mapping_epoch":1258,"log_start":"1264'3400","ondisk_log_start":"1264'3400","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6856,"log_dups_size":0,"ondisk_log_size":6856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:55.458182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219152384,"num_objects":3344,"num_object_clones":0,"num_object_copies":10032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10256,"num_write_kb":435200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.b","version":"1279'10115","reported_seq":10168,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:12.866671+0000","last_change":"2024-08-17T17:59:22.957875+0000","last_active":"2024-08-17T18:02:12.866671+0000","last_peered":"2024-08-17T18:02:12.866671+0000","last_clean":"2024-08-17T18:02:12.866671+0000","last_became_active":"2024-08-17T17:59:22.957601+0000","last_became_peered":"2024-08-17T17:59:22.957601+0000","last_unstale":"2024-08-17T18:02:12.866671+0000","last_undegraded":"2024-08-17T18:02:12.866671+0000","last_fullsized":"2024-08-17T18:02:12.866671+0000","mapping_epoch":1258,"log_start":"1265'3500","ondisk_log_start":"1265'3500","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6615,"log_dups_size":0,"ondisk_log_size":6615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:40.456894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212402176,"num_objects":3241,"num_object_clones":0,"num_object_copies":9723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10115,"num_write_kb":427392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.a","version":"1279'10255","reported_seq":10284,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:12.997907+0000","last_change":"2024-08-17T17:59:22.951416+0000","last_active":"2024-08-17T18:02:12.997907+0000","last_peered":"2024-08-17T18:02:12.997907+0000","last_clean":"2024-08-17T18:02:12.997907+0000","last_became_active":"2024-08-17T17:59:22.951110+0000","last_became_peered":"2024-08-17T17:59:22.951110+0000","last_unstale":"2024-08-17T18:02:12.997907+0000","last_undegraded":"2024-08-17T18:02:12.997907+0000","last_fullsized":"2024-08-17T18:02:12.997907+0000","mapping_epoch":1258,"log_start":"1264'3200","ondisk_log_start":"1264'3200","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:32.365091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217776128,"num_objects":3323,"num_object_clones":0,"num_object_copies":9969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10255,"num_write_kb":434496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.9","version":"1279'10089","reported_seq":10143,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:12.870279+0000","last_change":"2024-08-17T17:59:22.957896+0000","last_active":"2024-08-17T18:02:12.870279+0000","last_peered":"2024-08-17T18:02:12.870279+0000","last_clean":"2024-08-17T18:02:12.870279+0000","last_became_active":"2024-08-17T17:59:22.957648+0000","last_became_peered":"2024-08-17T17:59:22.957648+0000","last_unstale":"2024-08-17T18:02:12.870279+0000","last_undegraded":"2024-08-17T18:02:12.870279+0000","last_fullsized":"2024-08-17T18:02:12.870279+0000","mapping_epoch":1258,"log_start":"1265'3500","ondisk_log_start":"1265'3500","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6589,"log_dups_size":0,"ondisk_log_size":6589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:11.940932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210042880,"num_objects":3205,"num_object_clones":0,"num_object_copies":9615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10089,"num_write_kb":425408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.8","version":"1279'10133","reported_seq":10187,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:11.611967+0000","last_change":"2024-08-17T17:59:22.951248+0000","last_active":"2024-08-17T18:02:11.611967+0000","last_peered":"2024-08-17T18:02:11.611967+0000","last_clean":"2024-08-17T18:02:11.611967+0000","last_became_active":"2024-08-17T17:59:22.950981+0000","last_became_peered":"2024-08-17T17:59:22.950981+0000","last_unstale":"2024-08-17T18:02:11.611967+0000","last_undegraded":"2024-08-17T18:02:11.611967+0000","last_fullsized":"2024-08-17T18:02:11.611967+0000","mapping_epoch":1258,"log_start":"1265'4200","ondisk_log_start":"1265'4200","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":5933,"log_dups_size":0,"ondisk_log_size":5933,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:24:07.143610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217776128,"num_objects":3323,"num_object_clones":0,"num_object_copies":9969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10133,"num_write_kb":430592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.7","version":"1279'10229","reported_seq":10294,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:11.647453+0000","last_change":"2024-08-17T17:59:58.050355+0000","last_active":"2024-08-17T18:02:11.647453+0000","last_peered":"2024-08-17T18:02:11.647453+0000","last_clean":"2024-08-17T18:02:11.647453+0000","last_became_active":"2024-08-17T17:59:58.022280+0000","last_became_peered":"2024-08-17T17:59:58.022280+0000","last_unstale":"2024-08-17T18:02:11.647453+0000","last_undegraded":"2024-08-17T18:02:11.647453+0000","last_fullsized":"2024-08-17T18:02:11.647453+0000","mapping_epoch":1263,"log_start":"1265'4300","ondisk_log_start":"1265'4300","created":1258,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":5929,"log_dups_size":0,"ondisk_log_size":5929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:07.979130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216596480,"num_objects":3305,"num_object_clones":0,"num_object_copies":9915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10229,"num_write_kb":433088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.6","version":"1279'10153","reported_seq":10182,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:13.028116+0000","last_change":"2024-08-17T17:59:22.958347+0000","last_active":"2024-08-17T18:02:13.028116+0000","last_peered":"2024-08-17T18:02:13.028116+0000","last_clean":"2024-08-17T18:02:13.028116+0000","last_became_active":"2024-08-17T17:59:22.957969+0000","last_became_peered":"2024-08-17T17:59:22.957969+0000","last_unstale":"2024-08-17T18:02:13.028116+0000","last_undegraded":"2024-08-17T18:02:13.028116+0000","last_fullsized":"2024-08-17T18:02:13.028116+0000","mapping_epoch":1258,"log_start":"1264'3300","ondisk_log_start":"1264'3300","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6853,"log_dups_size":0,"ondisk_log_size":6853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:07.728594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213975040,"num_objects":3265,"num_object_clones":0,"num_object_copies":9795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10153,"num_write_kb":429376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.5","version":"1279'10170","reported_seq":10222,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:12.841612+0000","last_change":"2024-08-17T17:59:22.956504+0000","last_active":"2024-08-17T18:02:12.841612+0000","last_peered":"2024-08-17T18:02:12.841612+0000","last_clean":"2024-08-17T18:02:12.841612+0000","last_became_active":"2024-08-17T17:59:22.955747+0000","last_became_peered":"2024-08-17T17:59:22.955747+0000","last_unstale":"2024-08-17T18:02:12.841612+0000","last_undegraded":"2024-08-17T18:02:12.841612+0000","last_fullsized":"2024-08-17T18:02:12.841612+0000","mapping_epoch":1258,"log_start":"1265'4000","ondisk_log_start":"1265'4000","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6170,"log_dups_size":0,"ondisk_log_size":6170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:54:51.990739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218365952,"num_objects":3332,"num_object_clones":0,"num_object_copies":9996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10170,"num_write_kb":432064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"46.4","version":"1279'10004","reported_seq":10033,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:13.001407+0000","last_change":"2024-08-17T17:59:22.952797+0000","last_active":"2024-08-17T18:02:13.001407+0000","last_peered":"2024-08-17T18:02:13.001407+0000","last_clean":"2024-08-17T18:02:13.001407+0000","last_became_active":"2024-08-17T17:59:22.952475+0000","last_became_peered":"2024-08-17T17:59:22.952475+0000","last_unstale":"2024-08-17T18:02:13.001407+0000","last_undegraded":"2024-08-17T18:02:13.001407+0000","last_fullsized":"2024-08-17T18:02:13.001407+0000","mapping_epoch":1258,"log_start":"1264'3000","ondisk_log_start":"1264'3000","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":7004,"log_dups_size":0,"ondisk_log_size":7004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:42:19.302395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214302720,"num_objects":3270,"num_object_clones":0,"num_object_copies":9810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10004,"num_write_kb":424768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"46.3","version":"1279'9999","reported_seq":10053,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:11.632525+0000","last_change":"2024-08-17T17:59:22.954906+0000","last_active":"2024-08-17T18:02:11.632525+0000","last_peered":"2024-08-17T18:02:11.632525+0000","last_clean":"2024-08-17T18:02:11.632525+0000","last_became_active":"2024-08-17T17:59:22.954623+0000","last_became_peered":"2024-08-17T17:59:22.954623+0000","last_unstale":"2024-08-17T18:02:11.632525+0000","last_undegraded":"2024-08-17T18:02:11.632525+0000","last_fullsized":"2024-08-17T18:02:11.632525+0000","mapping_epoch":1258,"log_start":"1265'4100","ondisk_log_start":"1265'4100","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":5899,"log_dups_size":0,"ondisk_log_size":5899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:19.582387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222363648,"num_objects":3393,"num_object_clones":0,"num_object_copies":10179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9999,"num_write_kb":428544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"46.2","version":"1279'10280","reported_seq":10334,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:12.835395+0000","last_change":"2024-08-17T17:59:22.956809+0000","last_active":"2024-08-17T18:02:12.835395+0000","last_peered":"2024-08-17T18:02:12.835395+0000","last_clean":"2024-08-17T18:02:12.835395+0000","last_became_active":"2024-08-17T17:59:22.956543+0000","last_became_peered":"2024-08-17T17:59:22.956543+0000","last_unstale":"2024-08-17T18:02:12.835395+0000","last_undegraded":"2024-08-17T18:02:12.835395+0000","last_fullsized":"2024-08-17T18:02:12.835395+0000","mapping_epoch":1258,"log_start":"1265'3700","ondisk_log_start":"1265'3700","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6580,"log_dups_size":0,"ondisk_log_size":6580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:31:37.454620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219938816,"num_objects":3356,"num_object_clones":0,"num_object_copies":10068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10280,"num_write_kb":436352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"46.1","version":"1279'10233","reported_seq":10262,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:13.024576+0000","last_change":"2024-08-17T17:59:22.954701+0000","last_active":"2024-08-17T18:02:13.024576+0000","last_peered":"2024-08-17T18:02:13.024576+0000","last_clean":"2024-08-17T18:02:13.024576+0000","last_became_active":"2024-08-17T17:59:22.954510+0000","last_became_peered":"2024-08-17T17:59:22.954510+0000","last_unstale":"2024-08-17T18:02:13.024576+0000","last_undegraded":"2024-08-17T18:02:13.024576+0000","last_fullsized":"2024-08-17T18:02:13.024576+0000","mapping_epoch":1258,"log_start":"1264'3400","ondisk_log_start":"1264'3400","created":1258,"last_epoch_clean":1259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6833,"log_dups_size":0,"ondisk_log_size":6833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:19:01.061765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218431488,"num_objects":3333,"num_object_clones":0,"num_object_copies":9999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10233,"num_write_kb":434112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"46.0","version":"1279'10080","reported_seq":10145,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:02:12.821208+0000","last_change":"2024-08-17T17:59:58.062636+0000","last_active":"2024-08-17T18:02:12.821208+0000","last_peered":"2024-08-17T18:02:12.821208+0000","last_clean":"2024-08-17T18:02:12.821208+0000","last_became_active":"2024-08-17T17:59:58.032875+0000","last_became_peered":"2024-08-17T17:59:58.032875+0000","last_unstale":"2024-08-17T18:02:12.821208+0000","last_undegraded":"2024-08-17T18:02:12.821208+0000","last_fullsized":"2024-08-17T18:02:12.821208+0000","mapping_epoch":1263,"log_start":"1265'3500","ondisk_log_start":"1265'3500","created":1258,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:59:21.929283+0000","last_clean_scrub_stamp":"2024-08-17T17:59:21.929283+0000","objects_scrubbed":0,"log_size":6580,"log_dups_size":0,"ondisk_log_size":6580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:24.031388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222953472,"num_objects":3402,"num_object_clones":0,"num_object_copies":10206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10080,"num_write_kb":431424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":2018,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.750987+0000","last_change":"2024-08-17T17:48:40.741047+0000","last_active":"2024-08-17T18:01:55.750987+0000","last_peered":"2024-08-17T18:01:55.750987+0000","last_clean":"2024-08-17T18:01:55.750987+0000","last_became_active":"2024-08-17T17:48:40.740665+0000","last_became_peered":"2024-08-17T17:48:40.740665+0000","last_unstale":"2024-08-17T18:01:55.750987+0000","last_undegraded":"2024-08-17T18:01:55.750987+0000","last_fullsized":"2024-08-17T18:01:55.750987+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:02:30.389606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":570,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":556,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.647836+0000","last_change":"2024-08-17T17:49:45.674305+0000","last_active":"2024-08-17T18:01:42.647836+0000","last_peered":"2024-08-17T18:01:42.647836+0000","last_clean":"2024-08-17T18:01:42.647836+0000","last_became_active":"2024-08-17T17:49:45.674002+0000","last_became_peered":"2024-08-17T17:49:45.674002+0000","last_unstale":"2024-08-17T18:01:42.647836+0000","last_undegraded":"2024-08-17T18:01:42.647836+0000","last_fullsized":"2024-08-17T18:01:42.647836+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:08.013386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1938,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.750548+0000","last_change":"2024-08-17T17:31:49.807571+0000","last_active":"2024-08-17T18:01:55.750548+0000","last_peered":"2024-08-17T18:01:55.750548+0000","last_clean":"2024-08-17T18:01:55.750548+0000","last_became_active":"2024-08-17T17:31:49.804851+0000","last_became_peered":"2024-08-17T17:31:49.804851+0000","last_unstale":"2024-08-17T18:01:55.750548+0000","last_undegraded":"2024-08-17T18:01:55.750548+0000","last_fullsized":"2024-08-17T18:01:55.750548+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:41.710234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1961,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.740295+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T18:01:55.740295+0000","last_peered":"2024-08-17T18:01:55.740295+0000","last_clean":"2024-08-17T18:01:55.740295+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T18:01:55.740295+0000","last_undegraded":"2024-08-17T18:01:55.740295+0000","last_fullsized":"2024-08-17T18:01:55.740295+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":118,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.648771+0000","last_change":"2024-08-17T17:49:45.678851+0000","last_active":"2024-08-17T18:01:42.648771+0000","last_peered":"2024-08-17T18:01:42.648771+0000","last_clean":"2024-08-17T18:01:42.648771+0000","last_became_active":"2024-08-17T17:49:45.674253+0000","last_became_peered":"2024-08-17T17:49:45.674253+0000","last_unstale":"2024-08-17T18:01:42.648771+0000","last_undegraded":"2024-08-17T18:01:42.648771+0000","last_fullsized":"2024-08-17T18:01:42.648771+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:13.032622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1354,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.740500+0000","last_change":"2024-08-17T17:49:45.666484+0000","last_active":"2024-08-17T18:01:55.740500+0000","last_peered":"2024-08-17T18:01:55.740500+0000","last_clean":"2024-08-17T18:01:55.740500+0000","last_became_active":"2024-08-17T17:49:45.666209+0000","last_became_peered":"2024-08-17T17:49:45.666209+0000","last_unstale":"2024-08-17T18:01:55.740500+0000","last_undegraded":"2024-08-17T18:01:55.740500+0000","last_fullsized":"2024-08-17T18:01:55.740500+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:19.631420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":212,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.743052+0000","last_change":"2024-08-17T17:49:45.670256+0000","last_active":"2024-08-17T18:01:55.743052+0000","last_peered":"2024-08-17T18:01:55.743052+0000","last_clean":"2024-08-17T18:01:55.743052+0000","last_became_active":"2024-08-17T17:49:45.658766+0000","last_became_peered":"2024-08-17T17:49:45.658766+0000","last_unstale":"2024-08-17T18:01:55.743052+0000","last_undegraded":"2024-08-17T18:01:55.743052+0000","last_fullsized":"2024-08-17T18:01:55.743052+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:29.387737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":732,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.740092+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T18:01:55.740092+0000","last_peered":"2024-08-17T18:01:55.740092+0000","last_clean":"2024-08-17T18:01:55.740092+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T18:01:55.740092+0000","last_undegraded":"2024-08-17T18:01:55.740092+0000","last_fullsized":"2024-08-17T18:01:55.740092+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":1976,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.751060+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T18:01:55.751060+0000","last_peered":"2024-08-17T18:01:55.751060+0000","last_clean":"2024-08-17T18:01:55.751060+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T18:01:55.751060+0000","last_undegraded":"2024-08-17T18:01:55.751060+0000","last_fullsized":"2024-08-17T18:01:55.751060+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":586,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.743409+0000","last_change":"2024-08-17T17:59:58.027770+0000","last_active":"2024-08-17T18:01:55.743409+0000","last_peered":"2024-08-17T18:01:55.743409+0000","last_clean":"2024-08-17T18:01:55.743409+0000","last_became_active":"2024-08-17T17:59:58.027251+0000","last_became_peered":"2024-08-17T17:59:58.027251+0000","last_unstale":"2024-08-17T18:01:55.743409+0000","last_undegraded":"2024-08-17T18:01:55.743409+0000","last_fullsized":"2024-08-17T18:01:55.743409+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:26.416690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":197,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.742563+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T18:01:55.742563+0000","last_peered":"2024-08-17T18:01:55.742563+0000","last_clean":"2024-08-17T18:01:55.742563+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T18:01:55.742563+0000","last_undegraded":"2024-08-17T18:01:55.742563+0000","last_fullsized":"2024-08-17T18:01:55.742563+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1950,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.750225+0000","last_change":"2024-08-17T17:49:45.673223+0000","last_active":"2024-08-17T18:01:55.750225+0000","last_peered":"2024-08-17T18:01:55.750225+0000","last_clean":"2024-08-17T18:01:55.750225+0000","last_became_active":"2024-08-17T17:49:45.672917+0000","last_became_peered":"2024-08-17T17:49:45.672917+0000","last_unstale":"2024-08-17T18:01:55.750225+0000","last_undegraded":"2024-08-17T18:01:55.750225+0000","last_fullsized":"2024-08-17T18:01:55.750225+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:49:34.339334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1852,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.740383+0000","last_change":"2024-08-17T17:59:58.044651+0000","last_active":"2024-08-17T18:01:55.740383+0000","last_peered":"2024-08-17T18:01:55.740383+0000","last_clean":"2024-08-17T18:01:55.740383+0000","last_became_active":"2024-08-17T17:59:58.043830+0000","last_became_peered":"2024-08-17T17:59:58.043830+0000","last_unstale":"2024-08-17T18:01:55.740383+0000","last_undegraded":"2024-08-17T18:01:55.740383+0000","last_fullsized":"2024-08-17T18:01:55.740383+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:56.568577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1999,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.750037+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T18:01:55.750037+0000","last_peered":"2024-08-17T18:01:55.750037+0000","last_clean":"2024-08-17T18:01:55.750037+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T18:01:55.750037+0000","last_undegraded":"2024-08-17T18:01:55.750037+0000","last_fullsized":"2024-08-17T18:01:55.750037+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1799,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.747549+0000","last_change":"2024-08-17T17:59:58.028639+0000","last_active":"2024-08-17T18:01:55.747549+0000","last_peered":"2024-08-17T18:01:55.747549+0000","last_clean":"2024-08-17T18:01:55.747549+0000","last_became_active":"2024-08-17T17:59:58.028314+0000","last_became_peered":"2024-08-17T17:59:58.028314+0000","last_unstale":"2024-08-17T18:01:55.747549+0000","last_undegraded":"2024-08-17T18:01:55.747549+0000","last_fullsized":"2024-08-17T18:01:55.747549+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:25:03.619052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1253,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.648527+0000","last_change":"2024-08-17T17:48:40.651231+0000","last_active":"2024-08-17T18:01:42.648527+0000","last_peered":"2024-08-17T18:01:42.648527+0000","last_clean":"2024-08-17T18:01:42.648527+0000","last_became_active":"2024-08-17T17:48:40.650750+0000","last_became_peered":"2024-08-17T17:48:40.650750+0000","last_unstale":"2024-08-17T18:01:42.648527+0000","last_undegraded":"2024-08-17T18:01:42.648527+0000","last_fullsized":"2024-08-17T18:01:42.648527+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.439636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1944,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.741153+0000","last_change":"2024-08-17T17:31:49.809867+0000","last_active":"2024-08-17T18:01:55.741153+0000","last_peered":"2024-08-17T18:01:55.741153+0000","last_clean":"2024-08-17T18:01:55.741153+0000","last_became_active":"2024-08-17T17:31:49.807966+0000","last_became_peered":"2024-08-17T17:31:49.807966+0000","last_unstale":"2024-08-17T18:01:55.741153+0000","last_undegraded":"2024-08-17T18:01:55.741153+0000","last_fullsized":"2024-08-17T18:01:55.741153+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:47.530456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1993,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.751146+0000","last_change":"2024-08-17T17:48:40.649289+0000","last_active":"2024-08-17T18:01:55.751146+0000","last_peered":"2024-08-17T18:01:55.751146+0000","last_clean":"2024-08-17T18:01:55.751146+0000","last_became_active":"2024-08-17T17:48:40.648767+0000","last_became_peered":"2024-08-17T17:48:40.648767+0000","last_unstale":"2024-08-17T18:01:55.751146+0000","last_undegraded":"2024-08-17T18:01:55.751146+0000","last_fullsized":"2024-08-17T18:01:55.751146+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:12.797288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1900,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.740228+0000","last_change":"2024-08-17T17:31:49.805636+0000","last_active":"2024-08-17T18:01:55.740228+0000","last_peered":"2024-08-17T18:01:55.740228+0000","last_clean":"2024-08-17T18:01:55.740228+0000","last_became_active":"2024-08-17T17:31:49.803842+0000","last_became_peered":"2024-08-17T17:31:49.803842+0000","last_unstale":"2024-08-17T18:01:55.740228+0000","last_undegraded":"2024-08-17T18:01:55.740228+0000","last_fullsized":"2024-08-17T18:01:55.740228+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:22:23.623207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":572,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659310+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T18:01:42.659310+0000","last_peered":"2024-08-17T18:01:42.659310+0000","last_clean":"2024-08-17T18:01:42.659310+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T18:01:42.659310+0000","last_undegraded":"2024-08-17T18:01:42.659310+0000","last_fullsized":"2024-08-17T18:01:42.659310+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1249,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659380+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T18:01:42.659380+0000","last_peered":"2024-08-17T18:01:42.659380+0000","last_clean":"2024-08-17T18:01:42.659380+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T18:01:42.659380+0000","last_undegraded":"2024-08-17T18:01:42.659380+0000","last_fullsized":"2024-08-17T18:01:42.659380+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1998,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.742788+0000","last_change":"2024-08-17T17:31:15.550175+0000","last_active":"2024-08-17T18:01:55.742788+0000","last_peered":"2024-08-17T18:01:55.742788+0000","last_clean":"2024-08-17T18:01:55.742788+0000","last_became_active":"2024-08-17T17:31:15.549932+0000","last_became_peered":"2024-08-17T17:31:15.549932+0000","last_unstale":"2024-08-17T18:01:55.742788+0000","last_undegraded":"2024-08-17T18:01:55.742788+0000","last_fullsized":"2024-08-17T18:01:55.742788+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:34.679091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":410,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659338+0000","last_change":"2024-08-17T17:49:45.685584+0000","last_active":"2024-08-17T18:01:42.659338+0000","last_peered":"2024-08-17T18:01:42.659338+0000","last_clean":"2024-08-17T18:01:42.659338+0000","last_became_active":"2024-08-17T17:49:45.650514+0000","last_became_peered":"2024-08-17T17:49:45.650514+0000","last_unstale":"2024-08-17T18:01:42.659338+0000","last_undegraded":"2024-08-17T18:01:42.659338+0000","last_fullsized":"2024-08-17T18:01:42.659338+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:55.219841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4744,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":556,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.648261+0000","last_change":"2024-08-17T17:49:45.697664+0000","last_active":"2024-08-17T18:01:42.648261+0000","last_peered":"2024-08-17T18:01:42.648261+0000","last_clean":"2024-08-17T18:01:42.648261+0000","last_became_active":"2024-08-17T17:49:45.697366+0000","last_became_peered":"2024-08-17T17:49:45.697366+0000","last_unstale":"2024-08-17T18:01:42.648261+0000","last_undegraded":"2024-08-17T18:01:42.648261+0000","last_fullsized":"2024-08-17T18:01:42.648261+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:45.747199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1234,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.648143+0000","last_change":"2024-08-17T17:49:45.661308+0000","last_active":"2024-08-17T18:01:42.648143+0000","last_peered":"2024-08-17T18:01:42.648143+0000","last_clean":"2024-08-17T18:01:42.648143+0000","last_became_active":"2024-08-17T17:49:45.661035+0000","last_became_peered":"2024-08-17T17:49:45.661035+0000","last_unstale":"2024-08-17T18:01:42.648143+0000","last_undegraded":"2024-08-17T18:01:42.648143+0000","last_fullsized":"2024-08-17T18:01:42.648143+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:57.613380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1242,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.648598+0000","last_change":"2024-08-17T17:31:15.548562+0000","last_active":"2024-08-17T18:01:42.648598+0000","last_peered":"2024-08-17T18:01:42.648598+0000","last_clean":"2024-08-17T18:01:42.648598+0000","last_became_active":"2024-08-17T17:31:15.548237+0000","last_became_peered":"2024-08-17T17:31:15.548237+0000","last_unstale":"2024-08-17T18:01:42.648598+0000","last_undegraded":"2024-08-17T18:01:42.648598+0000","last_fullsized":"2024-08-17T18:01:42.648598+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:38:41.277442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":2046,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.749781+0000","last_change":"2024-08-17T17:49:45.681206+0000","last_active":"2024-08-17T18:01:55.749781+0000","last_peered":"2024-08-17T18:01:55.749781+0000","last_clean":"2024-08-17T18:01:55.749781+0000","last_became_active":"2024-08-17T17:49:45.671924+0000","last_became_peered":"2024-08-17T17:49:45.671924+0000","last_unstale":"2024-08-17T18:01:55.749781+0000","last_undegraded":"2024-08-17T18:01:55.749781+0000","last_fullsized":"2024-08-17T18:01:55.749781+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:44:38.452521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1930,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.741287+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T18:01:55.741287+0000","last_peered":"2024-08-17T18:01:55.741287+0000","last_clean":"2024-08-17T18:01:55.741287+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T18:01:55.741287+0000","last_undegraded":"2024-08-17T18:01:55.741287+0000","last_fullsized":"2024-08-17T18:01:55.741287+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":2004,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.742715+0000","last_change":"2024-08-17T17:49:45.676916+0000","last_active":"2024-08-17T18:01:55.742715+0000","last_peered":"2024-08-17T18:01:55.742715+0000","last_clean":"2024-08-17T18:01:55.742715+0000","last_became_active":"2024-08-17T17:49:45.676554+0000","last_became_peered":"2024-08-17T17:49:45.676554+0000","last_unstale":"2024-08-17T18:01:55.742715+0000","last_undegraded":"2024-08-17T18:01:55.742715+0000","last_fullsized":"2024-08-17T18:01:55.742715+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:50.424824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1932,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.740616+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T18:01:55.740616+0000","last_peered":"2024-08-17T18:01:55.740616+0000","last_clean":"2024-08-17T18:01:55.740616+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T18:01:55.740616+0000","last_undegraded":"2024-08-17T18:01:55.740616+0000","last_fullsized":"2024-08-17T18:01:55.740616+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1964,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.752461+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T18:01:55.752461+0000","last_peered":"2024-08-17T18:01:55.752461+0000","last_clean":"2024-08-17T18:01:55.752461+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T18:01:55.752461+0000","last_undegraded":"2024-08-17T18:01:55.752461+0000","last_fullsized":"2024-08-17T18:01:55.752461+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1269,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659290+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T18:01:42.659290+0000","last_peered":"2024-08-17T18:01:42.659290+0000","last_clean":"2024-08-17T18:01:42.659290+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T18:01:42.659290+0000","last_undegraded":"2024-08-17T18:01:42.659290+0000","last_fullsized":"2024-08-17T18:01:42.659290+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":2011,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.743250+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T18:01:55.743250+0000","last_peered":"2024-08-17T18:01:55.743250+0000","last_clean":"2024-08-17T18:01:55.743250+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T18:01:55.743250+0000","last_undegraded":"2024-08-17T18:01:55.743250+0000","last_fullsized":"2024-08-17T18:01:55.743250+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":197,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.742468+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T18:01:55.742468+0000","last_peered":"2024-08-17T18:01:55.742468+0000","last_clean":"2024-08-17T18:01:55.742468+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T18:01:55.742468+0000","last_undegraded":"2024-08-17T18:01:55.742468+0000","last_fullsized":"2024-08-17T18:01:55.742468+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2030,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.743172+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T18:01:55.743172+0000","last_peered":"2024-08-17T18:01:55.743172+0000","last_clean":"2024-08-17T18:01:55.743172+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T18:01:55.743172+0000","last_undegraded":"2024-08-17T18:01:55.743172+0000","last_fullsized":"2024-08-17T18:01:55.743172+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":766,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.750030+0000","last_change":"2024-08-17T17:48:07.629992+0000","last_active":"2024-08-17T18:01:55.750030+0000","last_peered":"2024-08-17T18:01:55.750030+0000","last_clean":"2024-08-17T18:01:55.750030+0000","last_became_active":"2024-08-17T17:48:07.629530+0000","last_became_peered":"2024-08-17T17:48:07.629530+0000","last_unstale":"2024-08-17T18:01:55.750030+0000","last_undegraded":"2024-08-17T18:01:55.750030+0000","last_fullsized":"2024-08-17T18:01:55.750030+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:26.103410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":2011,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.743321+0000","last_change":"2024-08-17T17:31:49.812746+0000","last_active":"2024-08-17T18:01:55.743321+0000","last_peered":"2024-08-17T18:01:55.743321+0000","last_clean":"2024-08-17T18:01:55.743321+0000","last_became_active":"2024-08-17T17:31:49.810895+0000","last_became_peered":"2024-08-17T17:31:49.810895+0000","last_unstale":"2024-08-17T18:01:55.743321+0000","last_undegraded":"2024-08-17T18:01:55.743321+0000","last_fullsized":"2024-08-17T18:01:55.743321+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:47.894199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1339,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.752374+0000","last_change":"2024-08-17T17:49:45.695083+0000","last_active":"2024-08-17T18:01:55.752374+0000","last_peered":"2024-08-17T18:01:55.752374+0000","last_clean":"2024-08-17T18:01:55.752374+0000","last_became_active":"2024-08-17T17:49:45.694600+0000","last_became_peered":"2024-08-17T17:49:45.694600+0000","last_unstale":"2024-08-17T18:01:55.752374+0000","last_undegraded":"2024-08-17T18:01:55.752374+0000","last_fullsized":"2024-08-17T18:01:55.752374+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:17.710807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":257,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.742703+0000","last_change":"2024-08-17T17:50:33.497749+0000","last_active":"2024-08-17T18:01:55.742703+0000","last_peered":"2024-08-17T18:01:55.742703+0000","last_clean":"2024-08-17T18:01:55.742703+0000","last_became_active":"2024-08-17T17:48:40.656971+0000","last_became_peered":"2024-08-17T17:48:40.656971+0000","last_unstale":"2024-08-17T18:01:55.742703+0000","last_undegraded":"2024-08-17T18:01:55.742703+0000","last_fullsized":"2024-08-17T18:01:55.742703+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1193,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:26.726136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":2021,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.747022+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T18:01:55.747022+0000","last_peered":"2024-08-17T18:01:55.747022+0000","last_clean":"2024-08-17T18:01:55.747022+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T18:01:55.747022+0000","last_undegraded":"2024-08-17T18:01:55.747022+0000","last_fullsized":"2024-08-17T18:01:55.747022+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1999,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.743622+0000","last_change":"2024-08-17T17:31:15.540994+0000","last_active":"2024-08-17T18:01:55.743622+0000","last_peered":"2024-08-17T18:01:55.743622+0000","last_clean":"2024-08-17T18:01:55.743622+0000","last_became_active":"2024-08-17T17:31:15.540678+0000","last_became_peered":"2024-08-17T17:31:15.540678+0000","last_unstale":"2024-08-17T18:01:55.743622+0000","last_undegraded":"2024-08-17T18:01:55.743622+0000","last_fullsized":"2024-08-17T18:01:55.743622+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:47.583922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1847,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.740344+0000","last_change":"2024-08-17T17:59:58.030150+0000","last_active":"2024-08-17T18:01:55.740344+0000","last_peered":"2024-08-17T18:01:55.740344+0000","last_clean":"2024-08-17T18:01:55.740344+0000","last_became_active":"2024-08-17T17:59:58.029407+0000","last_became_peered":"2024-08-17T17:59:58.029407+0000","last_unstale":"2024-08-17T18:01:55.740344+0000","last_undegraded":"2024-08-17T18:01:55.740344+0000","last_fullsized":"2024-08-17T18:01:55.740344+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:31.632793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2025,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.750115+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T18:01:55.750115+0000","last_peered":"2024-08-17T18:01:55.750115+0000","last_clean":"2024-08-17T18:01:55.750115+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T18:01:55.750115+0000","last_undegraded":"2024-08-17T18:01:55.750115+0000","last_fullsized":"2024-08-17T18:01:55.750115+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2005,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.751940+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T18:01:55.751940+0000","last_peered":"2024-08-17T18:01:55.751940+0000","last_clean":"2024-08-17T18:01:55.751940+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T18:01:55.751940+0000","last_undegraded":"2024-08-17T18:01:55.751940+0000","last_fullsized":"2024-08-17T18:01:55.751940+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1276,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658955+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T18:01:42.658955+0000","last_peered":"2024-08-17T18:01:42.658955+0000","last_clean":"2024-08-17T18:01:42.658955+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T18:01:42.658955+0000","last_undegraded":"2024-08-17T18:01:42.658955+0000","last_fullsized":"2024-08-17T18:01:42.658955+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1222,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.648755+0000","last_change":"2024-08-17T17:49:45.661773+0000","last_active":"2024-08-17T18:01:42.648755+0000","last_peered":"2024-08-17T18:01:42.648755+0000","last_clean":"2024-08-17T18:01:42.648755+0000","last_became_active":"2024-08-17T17:49:45.661473+0000","last_became_peered":"2024-08-17T17:49:45.661473+0000","last_unstale":"2024-08-17T18:01:42.648755+0000","last_undegraded":"2024-08-17T18:01:42.648755+0000","last_fullsized":"2024-08-17T18:01:42.648755+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.712822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1914,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.740583+0000","last_change":"2024-08-17T17:49:45.669117+0000","last_active":"2024-08-17T18:01:55.740583+0000","last_peered":"2024-08-17T18:01:55.740583+0000","last_clean":"2024-08-17T18:01:55.740583+0000","last_became_active":"2024-08-17T17:49:45.668842+0000","last_became_peered":"2024-08-17T17:49:45.668842+0000","last_unstale":"2024-08-17T18:01:55.740583+0000","last_undegraded":"2024-08-17T18:01:55.740583+0000","last_fullsized":"2024-08-17T18:01:55.740583+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:03.962921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":38,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659195+0000","last_change":"2024-08-17T17:59:58.027683+0000","last_active":"2024-08-17T18:01:42.659195+0000","last_peered":"2024-08-17T18:01:42.659195+0000","last_clean":"2024-08-17T18:01:42.659195+0000","last_became_active":"2024-08-17T17:59:58.027358+0000","last_became_peered":"2024-08-17T17:59:58.027358+0000","last_unstale":"2024-08-17T18:01:42.659195+0000","last_undegraded":"2024-08-17T18:01:42.659195+0000","last_fullsized":"2024-08-17T18:01:42.659195+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:04.758259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1588,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658840+0000","last_change":"2024-08-17T17:48:39.645592+0000","last_active":"2024-08-17T18:01:42.658840+0000","last_peered":"2024-08-17T18:01:42.658840+0000","last_clean":"2024-08-17T18:01:42.658840+0000","last_became_active":"2024-08-17T17:48:39.645163+0000","last_became_peered":"2024-08-17T17:48:39.645163+0000","last_unstale":"2024-08-17T18:01:42.658840+0000","last_undegraded":"2024-08-17T18:01:42.658840+0000","last_fullsized":"2024-08-17T18:01:42.658840+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:26.081197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2005,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.747138+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T18:01:55.747138+0000","last_peered":"2024-08-17T18:01:55.747138+0000","last_clean":"2024-08-17T18:01:55.747138+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T18:01:55.747138+0000","last_undegraded":"2024-08-17T18:01:55.747138+0000","last_fullsized":"2024-08-17T18:01:55.747138+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1946,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.741239+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T18:01:55.741239+0000","last_peered":"2024-08-17T18:01:55.741239+0000","last_clean":"2024-08-17T18:01:55.741239+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T18:01:55.741239+0000","last_undegraded":"2024-08-17T18:01:55.741239+0000","last_fullsized":"2024-08-17T18:01:55.741239+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":556,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.648459+0000","last_change":"2024-08-17T17:49:45.669478+0000","last_active":"2024-08-17T18:01:42.648459+0000","last_peered":"2024-08-17T18:01:42.648459+0000","last_clean":"2024-08-17T18:01:42.648459+0000","last_became_active":"2024-08-17T17:49:45.669123+0000","last_became_peered":"2024-08-17T17:49:45.669123+0000","last_unstale":"2024-08-17T18:01:42.648459+0000","last_undegraded":"2024-08-17T18:01:42.648459+0000","last_fullsized":"2024-08-17T18:01:42.648459+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:39.082033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":516,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659008+0000","last_change":"2024-08-17T17:59:58.021264+0000","last_active":"2024-08-17T18:01:42.659008+0000","last_peered":"2024-08-17T18:01:42.659008+0000","last_clean":"2024-08-17T18:01:42.659008+0000","last_became_active":"2024-08-17T17:59:58.020957+0000","last_became_peered":"2024-08-17T17:59:58.020957+0000","last_unstale":"2024-08-17T18:01:42.659008+0000","last_undegraded":"2024-08-17T18:01:42.659008+0000","last_fullsized":"2024-08-17T18:01:42.659008+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:42.190270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":548,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.743919+0000","last_change":"2024-08-17T17:59:58.043761+0000","last_active":"2024-08-17T18:01:55.743919+0000","last_peered":"2024-08-17T18:01:55.743919+0000","last_clean":"2024-08-17T18:01:55.743919+0000","last_became_active":"2024-08-17T17:59:58.043206+0000","last_became_peered":"2024-08-17T17:59:58.043206+0000","last_unstale":"2024-08-17T18:01:55.743919+0000","last_undegraded":"2024-08-17T18:01:55.743919+0000","last_fullsized":"2024-08-17T18:01:55.743919+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:37.774589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1240,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.648380+0000","last_change":"2024-08-17T17:49:45.662348+0000","last_active":"2024-08-17T18:01:42.648380+0000","last_peered":"2024-08-17T18:01:42.648380+0000","last_clean":"2024-08-17T18:01:42.648380+0000","last_became_active":"2024-08-17T17:49:45.662076+0000","last_became_peered":"2024-08-17T17:49:45.662076+0000","last_unstale":"2024-08-17T18:01:42.648380+0000","last_undegraded":"2024-08-17T18:01:42.648380+0000","last_fullsized":"2024-08-17T18:01:42.648380+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:59.686266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":716,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.741104+0000","last_change":"2024-08-17T17:49:45.668867+0000","last_active":"2024-08-17T18:01:55.741104+0000","last_peered":"2024-08-17T18:01:55.741104+0000","last_clean":"2024-08-17T18:01:55.741104+0000","last_became_active":"2024-08-17T17:49:45.668541+0000","last_became_peered":"2024-08-17T17:49:45.668541+0000","last_unstale":"2024-08-17T18:01:55.741104+0000","last_undegraded":"2024-08-17T18:01:55.741104+0000","last_fullsized":"2024-08-17T18:01:55.741104+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:11.808055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":1942,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.740187+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T18:01:55.740187+0000","last_peered":"2024-08-17T18:01:55.740187+0000","last_clean":"2024-08-17T18:01:55.740187+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T18:01:55.740187+0000","last_undegraded":"2024-08-17T18:01:55.740187+0000","last_fullsized":"2024-08-17T18:01:55.740187+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2038,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.753490+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T18:01:55.753490+0000","last_peered":"2024-08-17T18:01:55.753490+0000","last_clean":"2024-08-17T18:01:55.753490+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T18:01:55.753490+0000","last_undegraded":"2024-08-17T18:01:55.753490+0000","last_fullsized":"2024-08-17T18:01:55.753490+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":749,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.741018+0000","last_change":"2024-08-17T17:48:07.623488+0000","last_active":"2024-08-17T18:01:55.741018+0000","last_peered":"2024-08-17T18:01:55.741018+0000","last_clean":"2024-08-17T18:01:55.741018+0000","last_became_active":"2024-08-17T17:48:07.623053+0000","last_became_peered":"2024-08-17T17:48:07.623053+0000","last_unstale":"2024-08-17T18:01:55.741018+0000","last_undegraded":"2024-08-17T18:01:55.741018+0000","last_fullsized":"2024-08-17T18:01:55.741018+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:55.847353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1229,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.648763+0000","last_change":"2024-08-17T17:30:43.433522+0000","last_active":"2024-08-17T18:01:42.648763+0000","last_peered":"2024-08-17T18:01:42.648763+0000","last_clean":"2024-08-17T18:01:42.648763+0000","last_became_active":"2024-08-17T17:30:43.432133+0000","last_became_peered":"2024-08-17T17:30:43.432133+0000","last_unstale":"2024-08-17T18:01:42.648763+0000","last_undegraded":"2024-08-17T18:01:42.648763+0000","last_fullsized":"2024-08-17T18:01:42.648763+0000","mapping_epoch":506,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":507,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:33.724706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1228,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659214+0000","last_change":"2024-08-17T17:48:07.632502+0000","last_active":"2024-08-17T18:01:42.659214+0000","last_peered":"2024-08-17T18:01:42.659214+0000","last_clean":"2024-08-17T18:01:42.659214+0000","last_became_active":"2024-08-17T17:48:07.629726+0000","last_became_peered":"2024-08-17T17:48:07.629726+0000","last_unstale":"2024-08-17T18:01:42.659214+0000","last_undegraded":"2024-08-17T18:01:42.659214+0000","last_fullsized":"2024-08-17T18:01:42.659214+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:33.182997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":733,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.743250+0000","last_change":"2024-08-17T17:49:45.662424+0000","last_active":"2024-08-17T18:01:55.743250+0000","last_peered":"2024-08-17T18:01:55.743250+0000","last_clean":"2024-08-17T18:01:55.743250+0000","last_became_active":"2024-08-17T17:49:45.661977+0000","last_became_peered":"2024-08-17T17:49:45.661977+0000","last_unstale":"2024-08-17T18:01:55.743250+0000","last_undegraded":"2024-08-17T18:01:55.743250+0000","last_fullsized":"2024-08-17T18:01:55.743250+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:46.099134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1246,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659519+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T18:01:42.659519+0000","last_peered":"2024-08-17T18:01:42.659519+0000","last_clean":"2024-08-17T18:01:42.659519+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T18:01:42.659519+0000","last_undegraded":"2024-08-17T18:01:42.659519+0000","last_fullsized":"2024-08-17T18:01:42.659519+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":130,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.658775+0000","last_change":"2024-08-17T17:49:45.676923+0000","last_active":"2024-08-17T18:01:42.658775+0000","last_peered":"2024-08-17T18:01:42.658775+0000","last_clean":"2024-08-17T18:01:42.658775+0000","last_became_active":"2024-08-17T17:49:45.676423+0000","last_became_peered":"2024-08-17T17:49:45.676423+0000","last_unstale":"2024-08-17T18:01:42.658775+0000","last_undegraded":"2024-08-17T18:01:42.658775+0000","last_fullsized":"2024-08-17T18:01:42.658775+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.061990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1262,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659460+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T18:01:42.659460+0000","last_peered":"2024-08-17T18:01:42.659460+0000","last_clean":"2024-08-17T18:01:42.659460+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T18:01:42.659460+0000","last_undegraded":"2024-08-17T18:01:42.659460+0000","last_fullsized":"2024-08-17T18:01:42.659460+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":1279,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.647986+0000","last_change":"2024-08-17T17:48:40.648915+0000","last_active":"2024-08-17T18:01:42.647986+0000","last_peered":"2024-08-17T18:01:42.647986+0000","last_clean":"2024-08-17T18:01:42.647986+0000","last_became_active":"2024-08-17T17:48:40.648543+0000","last_became_peered":"2024-08-17T17:48:40.648543+0000","last_unstale":"2024-08-17T18:01:42.647986+0000","last_undegraded":"2024-08-17T18:01:42.647986+0000","last_fullsized":"2024-08-17T18:01:42.647986+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:46.073841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1943,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.740472+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T18:01:55.740472+0000","last_peered":"2024-08-17T18:01:55.740472+0000","last_clean":"2024-08-17T18:01:55.740472+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T18:01:55.740472+0000","last_undegraded":"2024-08-17T18:01:55.740472+0000","last_fullsized":"2024-08-17T18:01:55.740472+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1223,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659087+0000","last_change":"2024-08-17T17:49:45.661097+0000","last_active":"2024-08-17T18:01:42.659087+0000","last_peered":"2024-08-17T18:01:42.659087+0000","last_clean":"2024-08-17T18:01:42.659087+0000","last_became_active":"2024-08-17T17:49:45.660829+0000","last_became_peered":"2024-08-17T17:49:45.660829+0000","last_unstale":"2024-08-17T18:01:42.659087+0000","last_undegraded":"2024-08-17T18:01:42.659087+0000","last_fullsized":"2024-08-17T18:01:42.659087+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:31:00.439609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1996,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.749902+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T18:01:55.749902+0000","last_peered":"2024-08-17T18:01:55.749902+0000","last_clean":"2024-08-17T18:01:55.749902+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T18:01:55.749902+0000","last_undegraded":"2024-08-17T18:01:55.749902+0000","last_fullsized":"2024-08-17T18:01:55.749902+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1997,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.752499+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T18:01:55.752499+0000","last_peered":"2024-08-17T18:01:55.752499+0000","last_clean":"2024-08-17T18:01:55.752499+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T18:01:55.752499+0000","last_undegraded":"2024-08-17T18:01:55.752499+0000","last_fullsized":"2024-08-17T18:01:55.752499+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":636,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659052+0000","last_change":"2024-08-17T17:48:07.625241+0000","last_active":"2024-08-17T18:01:42.659052+0000","last_peered":"2024-08-17T18:01:42.659052+0000","last_clean":"2024-08-17T18:01:42.659052+0000","last_became_active":"2024-08-17T17:48:07.624926+0000","last_became_peered":"2024-08-17T17:48:07.624926+0000","last_unstale":"2024-08-17T18:01:42.659052+0000","last_undegraded":"2024-08-17T18:01:42.659052+0000","last_fullsized":"2024-08-17T18:01:42.659052+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:48:19.391417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1248,"reported_epoch":1278,"state":"active+clean","last_fresh":"2024-08-17T18:01:42.659108+0000","last_change":"2024-08-17T17:49:45.688478+0000","last_active":"2024-08-17T18:01:42.659108+0000","last_peered":"2024-08-17T18:01:42.659108+0000","last_clean":"2024-08-17T18:01:42.659108+0000","last_became_active":"2024-08-17T17:49:45.652101+0000","last_became_peered":"2024-08-17T17:49:45.652101+0000","last_unstale":"2024-08-17T18:01:42.659108+0000","last_undegraded":"2024-08-17T18:01:42.659108+0000","last_fullsized":"2024-08-17T18:01:42.659108+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:05.523867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1944,"reported_epoch":1279,"state":"active+clean","last_fresh":"2024-08-17T18:01:55.741260+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T18:01:55.741260+0000","last_peered":"2024-08-17T18:01:55.741260+0000","last_clean":"2024-08-17T18:01:55.741260+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T18:01:55.741260+0000","last_undegraded":"2024-08-17T18:01:55.741260+0000","last_fullsized":"2024-08-17T18:01:55.741260+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":46,"num_pg":16,"stat_sum":{"num_bytes":3489529880,"num_objects":53247,"num_object_clones":0,"num_object_copies":159741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53247,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":162715,"num_write_kb":6910721,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10580602880,"data_stored":10580590664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105415,"ondisk_log_size":105415,"up":48,"acting":48,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":5530,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":22050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234496,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1803708,"kb_used_data":1498036,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":91941444,"statfs":{"total":95995035648,"available":94148038656,"internally_reserved":0,"allocated":1533988864,"data_stored":1530106306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496131,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1809656,"kb_used_data":1508144,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":91935496,"statfs":{"total":95995035648,"available":94141947904,"internally_reserved":0,"allocated":1544339456,"data_stored":1540461447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757769,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326383,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2123040,"kb_used_data":1717208,"kb_used_omap":57,"kb_used_meta":405766,"kb_avail":91622112,"statfs":{"total":95995035648,"available":93821042688,"internally_reserved":0,"allocated":1758420992,"data_stored":1754561750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":415505147},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405621,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2432620,"kb_used_data":2038116,"kb_used_omap":61,"kb_used_meta":394434,"kb_avail":91312532,"statfs":{"total":95995035648,"available":93504032768,"internally_reserved":0,"allocated":2087030784,"data_stored":2083164442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":403901368},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452156,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1888932,"kb_used_data":1534876,"kb_used_omap":54,"kb_used_meta":353993,"kb_avail":91856220,"statfs":{"total":95995035648,"available":94060769280,"internally_reserved":0,"allocated":1571713024,"data_stored":1567849591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":362489230},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474847,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2590768,"kb_used_data":2165032,"kb_used_omap":119,"kb_used_meta":425608,"kb_avail":91154384,"statfs":{"total":95995035648,"available":93342089216,"internally_reserved":0,"allocated":2216992768,"data_stored":2213118259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":435823326},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019404,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2178158592,"data_stored":2178154520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1553272832,"data_stored":1553268760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2071396352,"data_stored":2071396352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1737359360,"data_stored":1737359360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1522532352,"data_stored":1522532352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":46,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1517883392,"data_stored":1517879320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:02:18.583 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool set .mgr pgp_num 1 2024-08-17T18:02:18.597 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:18.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:18.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:18.699 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:18.728 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:18 smithi130 ceph-mon[105521]: pgmap v2110: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T18:02:18.729 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:18 smithi130 ceph-mon[110068]: pgmap v2110: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T18:02:18.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:18.749 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:18.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:18 smithi179 ceph-mon[93519]: pgmap v2110: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 709 op/s 2024-08-17T18:02:18.849 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:18.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:18.908 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:18.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:19.009 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:19.033 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:19.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:19.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:19.135 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:19.141 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:02:19.158 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:19.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:19.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:19.260 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:19.303 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:19.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:19.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:19.405 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:19.427 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:19.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:19.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:19.529 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:19.555 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:02:19.555 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:02:19.555 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:02:19.555 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:02:19.556 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.556 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:204054 45.b 45:df630a61:::benchmark_data_smithi130_172551_object99247:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1247)", 2024-08-17T18:02:19.556 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.563571+0000", 2024-08-17T18:02:19.556 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 209.92317100400001, 2024-08-17T18:02:19.556 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.36158967800000003, 2024-08-17T18:02:19.556 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.556 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.556 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.556 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.556 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:02:19.556 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:02:19.556 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 204054 2024-08-17T18:02:19.556 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.557 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.557 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.557 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.557 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.563571+0000", 2024-08-17T18:02:19.557 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.557 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.557 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.557 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.557 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.563571+0000", 2024-08-17T18:02:19.557 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.557 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.557 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.557 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.558 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.563573+0000", 2024-08-17T18:02:19.558 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7489999999999999e-06 2024-08-17T18:02:19.558 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.558 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.558 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.558 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.563580+0000", 2024-08-17T18:02:19.558 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5490000000000003e-06 2024-08-17T18:02:19.558 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.558 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.558 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.558 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.563593+0000", 2024-08-17T18:02:19.559 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3472000000000001e-05 2024-08-17T18:02:19.559 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.559 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.559 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.559 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.563602+0000", 2024-08-17T18:02:19.559 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0380000000000005e-06 2024-08-17T18:02:19.560 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.560 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.560 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.560 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.564522+0000", 2024-08-17T18:02:19.560 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00091952600000000005 2024-08-17T18:02:19.560 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.560 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.560 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.560 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.564707+0000", 2024-08-17T18:02:19.560 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018508399999999999 2024-08-17T18:02:19.560 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.561 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.561 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3,4", 2024-08-17T18:02:19.561 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.564863+0000", 2024-08-17T18:02:19.561 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015653100000000001 2024-08-17T18:02:19.561 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.561 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.561 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.561 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.570672+0000", 2024-08-17T18:02:19.561 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0058088530000000001 2024-08-17T18:02:19.561 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.561 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.561 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.561 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.573176+0000", 2024-08-17T18:02:19.562 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025036699999999999 2024-08-17T18:02:19.562 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.562 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.562 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.562 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.573921+0000", 2024-08-17T18:02:19.562 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00074494899999999996 2024-08-17T18:02:19.562 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.562 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.562 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.562 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579362+0000", 2024-08-17T18:02:19.562 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0054412870000000004 2024-08-17T18:02:19.562 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.562 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.563 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.563 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579529+0000", 2024-08-17T18:02:19.563 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016677400000000001 2024-08-17T18:02:19.563 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.563 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.563 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.563 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925161+0000", 2024-08-17T18:02:19.563 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.345632196 2024-08-17T18:02:19.563 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.563 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.563 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.563 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.563 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.564 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.45158.0:204051 45.4 e1247/1183 45:2bc8134d:::benchmark_data_smithi130_172551_object99244:head v 1247'12702, mlcod=1247'12699)", 2024-08-17T18:02:19.564 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.566198+0000", 2024-08-17T18:02:19.564 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 209.92054429500001, 2024-08-17T18:02:19.564 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35852299300000001, 2024-08-17T18:02:19.564 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.564 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.564 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.564 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.564 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.564 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.564 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566198+0000", 2024-08-17T18:02:19.564 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.564 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.564 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566198+0000", 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566199+0000", 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1149999999999999e-06 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566205+0000", 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.339e-06 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.565 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.566 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.566 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566214+0000", 2024-08-17T18:02:19.566 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6709999999999994e-06 2024-08-17T18:02:19.566 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.566 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.566 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.566 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566223+0000", 2024-08-17T18:02:19.566 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8869999999999998e-06 2024-08-17T18:02:19.566 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.566 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.566 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.566 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566275+0000", 2024-08-17T18:02:19.566 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1496000000000003e-05 2024-08-17T18:02:19.567 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.567 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.567 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.567 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566325+0000", 2024-08-17T18:02:19.567 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0773999999999999e-05 2024-08-17T18:02:19.567 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.567 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.567 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.567 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.924682+0000", 2024-08-17T18:02:19.567 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35835662600000001 2024-08-17T18:02:19.567 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.567 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.567 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.568 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.924721+0000", 2024-08-17T18:02:19.568 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9085000000000003e-05 2024-08-17T18:02:19.568 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.568 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.568 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.568 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.568 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.568 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:204058 45.8 45:1d08798f:::benchmark_data_smithi130_172551_object99251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1247)", 2024-08-17T18:02:19.568 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.568520+0000", 2024-08-17T18:02:19.568 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 209.91822283799999, 2024-08-17T18:02:19.568 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.41966336300000001, 2024-08-17T18:02:19.568 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.568 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.568 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.569 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.569 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:02:19.569 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:02:19.569 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 204058 2024-08-17T18:02:19.569 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.569 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.569 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.569 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.569 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.568520+0000", 2024-08-17T18:02:19.569 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.569 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.569 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.569 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.569 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.568520+0000", 2024-08-17T18:02:19.570 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.570 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.570 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.570 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.570 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.568521+0000", 2024-08-17T18:02:19.570 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4920000000000001e-06 2024-08-17T18:02:19.570 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.570 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.570 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.570 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.568526+0000", 2024-08-17T18:02:19.570 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5920000000000002e-06 2024-08-17T18:02:19.570 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.571 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.571 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.571 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.568537+0000", 2024-08-17T18:02:19.571 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1644999999999999e-05 2024-08-17T18:02:19.571 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.571 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.571 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.571 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.568544+0000", 2024-08-17T18:02:19.571 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9040000000000003e-06 2024-08-17T18:02:19.571 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.571 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.571 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.571 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.569664+0000", 2024-08-17T18:02:19.571 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001119943 2024-08-17T18:02:19.572 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.572 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.572 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.572 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.569779+0000", 2024-08-17T18:02:19.572 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011475 2024-08-17T18:02:19.572 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.572 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.572 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T18:02:19.572 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.569904+0000", 2024-08-17T18:02:19.572 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012463400000000001 2024-08-17T18:02:19.572 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.572 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.572 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.573 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.573913+0000", 2024-08-17T18:02:19.573 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0040094070000000004 2024-08-17T18:02:19.573 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.573 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.573 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.573 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.573989+0000", 2024-08-17T18:02:19.573 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5797999999999995e-05 2024-08-17T18:02:19.573 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.573 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.573 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.573 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.931060+0000", 2024-08-17T18:02:19.573 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35707166600000001 2024-08-17T18:02:19.573 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.574 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.574 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.574 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.931114+0000", 2024-08-17T18:02:19.574 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3903999999999998e-05 2024-08-17T18:02:19.574 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.574 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.574 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.574 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988183+0000", 2024-08-17T18:02:19.574 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.057068628000000003 2024-08-17T18:02:19.574 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.574 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.574 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.574 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.575 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.575 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.45158.0:204062 45.4 e1247/1183 45:26797482:::benchmark_data_smithi130_172551_object99255:head v 1247'12703, mlcod=1247'12700)", 2024-08-17T18:02:19.575 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.576909+0000", 2024-08-17T18:02:19.575 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 209.90983337899999, 2024-08-17T18:02:19.575 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.34812056600000002, 2024-08-17T18:02:19.575 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.575 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.575 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.575 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.575 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.575 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.575 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576909+0000", 2024-08-17T18:02:19.575 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.575 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.576 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.576 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.576 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576909+0000", 2024-08-17T18:02:19.576 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.576 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.576 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.576 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.576 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576910+0000", 2024-08-17T18:02:19.576 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.037e-06 2024-08-17T18:02:19.576 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.576 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.576 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.576 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576918+0000", 2024-08-17T18:02:19.576 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7700000000000001e-06 2024-08-17T18:02:19.577 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.577 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.577 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.577 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576929+0000", 2024-08-17T18:02:19.577 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1073000000000001e-05 2024-08-17T18:02:19.577 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.577 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.577 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.577 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576936+0000", 2024-08-17T18:02:19.577 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1189999999999999e-06 2024-08-17T18:02:19.577 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.577 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.577 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.577 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.577009+0000", 2024-08-17T18:02:19.578 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2737999999999994e-05 2024-08-17T18:02:19.578 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.578 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.578 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.578 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.577028+0000", 2024-08-17T18:02:19.578 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9354000000000001e-05 2024-08-17T18:02:19.578 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.578 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.578 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.578 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925003+0000", 2024-08-17T18:02:19.579 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.34797440499999999 2024-08-17T18:02:19.579 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.579 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.579 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.579 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925030+0000", 2024-08-17T18:02:19.579 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.707e-05 2024-08-17T18:02:19.579 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.579 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.579 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.579 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.579 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.579 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:204067 45.8 45:10b899da:::benchmark_data_smithi130_172551_object99260:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1247)", 2024-08-17T18:02:19.579 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.579603+0000", 2024-08-17T18:02:19.580 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 209.907138903, 2024-08-17T18:02:19.580 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.40960074899999999, 2024-08-17T18:02:19.580 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.580 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.580 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.580 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.580 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:02:19.580 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:02:19.580 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 204067 2024-08-17T18:02:19.580 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.580 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.580 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.580 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.581 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579603+0000", 2024-08-17T18:02:19.581 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.581 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.581 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.581 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.581 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579603+0000", 2024-08-17T18:02:19.581 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.581 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.581 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.581 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.581 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579605+0000", 2024-08-17T18:02:19.581 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.652e-06 2024-08-17T18:02:19.581 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.581 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.582 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.582 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579611+0000", 2024-08-17T18:02:19.582 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9200000000000001e-06 2024-08-17T18:02:19.582 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.582 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.582 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.582 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579621+0000", 2024-08-17T18:02:19.582 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.006e-05 2024-08-17T18:02:19.582 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.582 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.582 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.582 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579628+0000", 2024-08-17T18:02:19.582 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4450000000000001e-06 2024-08-17T18:02:19.583 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.583 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.583 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.583 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579683+0000", 2024-08-17T18:02:19.583 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5606000000000003e-05 2024-08-17T18:02:19.583 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.583 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.583 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.583 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579828+0000", 2024-08-17T18:02:19.583 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014495600000000001 2024-08-17T18:02:19.583 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.583 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.583 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T18:02:19.583 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579957+0000", 2024-08-17T18:02:19.584 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012935299999999999 2024-08-17T18:02:19.584 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.584 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.584 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.584 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.931008+0000", 2024-08-17T18:02:19.584 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35105088200000001 2024-08-17T18:02:19.584 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.584 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.584 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.584 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988197+0000", 2024-08-17T18:02:19.584 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.057188956999999999 2024-08-17T18:02:19.584 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.584 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.584 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988244+0000", 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6322000000000001e-05 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988295+0000", 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1306e-05 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.989204+0000", 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00090928999999999999 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.585 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.586 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.586 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.586 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:204069 45.b 45:d53ba6ff:::benchmark_data_smithi130_172551_object99262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1247)", 2024-08-17T18:02:19.586 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.584843+0000", 2024-08-17T18:02:19.586 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 209.90189959700001, 2024-08-17T18:02:19.586 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.40363016699999998, 2024-08-17T18:02:19.586 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.586 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.586 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.586 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.586 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:02:19.586 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:02:19.586 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 204069 2024-08-17T18:02:19.586 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.587 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.587 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.587 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.587 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.584843+0000", 2024-08-17T18:02:19.587 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.587 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.587 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.587 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.587 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.584843+0000", 2024-08-17T18:02:19.587 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.587 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.587 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.587 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.587 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.584845+0000", 2024-08-17T18:02:19.588 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0640000000000001e-06 2024-08-17T18:02:19.588 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.588 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.588 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.588 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.584853+0000", 2024-08-17T18:02:19.588 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8690000000000004e-06 2024-08-17T18:02:19.588 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.588 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.588 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.588 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.924578+0000", 2024-08-17T18:02:19.588 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.33972480399999999 2024-08-17T18:02:19.588 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.588 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.588 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.589 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.924594+0000", 2024-08-17T18:02:19.589 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6133999999999998e-05 2024-08-17T18:02:19.589 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.589 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.589 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.589 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925279+0000", 2024-08-17T18:02:19.589 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00068548999999999997 2024-08-17T18:02:19.589 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.589 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.589 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.589 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925916+0000", 2024-08-17T18:02:19.589 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00063705599999999999 2024-08-17T18:02:19.589 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.590 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.590 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3,4", 2024-08-17T18:02:19.590 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.926085+0000", 2024-08-17T18:02:19.590 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000168982 2024-08-17T18:02:19.590 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.590 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.590 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.590 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.987761+0000", 2024-08-17T18:02:19.590 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.061675727999999999 2024-08-17T18:02:19.590 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.590 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.590 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.590 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.987906+0000", 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014464400000000001 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988003+0000", 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7380999999999995e-05 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988069+0000", 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6128000000000001e-05 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988131+0000", 2024-08-17T18:02:19.591 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1888000000000001e-05 2024-08-17T18:02:19.592 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.592 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.592 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.592 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988473+0000", 2024-08-17T18:02:19.592 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000341999 2024-08-17T18:02:19.592 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.592 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.592 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.592 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.592 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.592 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.45158.0:204093 45.7 e1247/1189 45:ee2752c9:::benchmark_data_smithi130_172551_object99286:head v 1247'12739, mlcod=1170'705)", 2024-08-17T18:02:19.592 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.613500+0000", 2024-08-17T18:02:19.592 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 209.873242541, 2024-08-17T18:02:19.593 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.374334255, 2024-08-17T18:02:19.593 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.593 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.593 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.593 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.593 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.593 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.593 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.613500+0000", 2024-08-17T18:02:19.593 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.593 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.593 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.593 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.593 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.613500+0000", 2024-08-17T18:02:19.593 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.594 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.594 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.594 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.594 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.613501+0000", 2024-08-17T18:02:19.594 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4389999999999999e-06 2024-08-17T18:02:19.594 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.594 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.594 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.594 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.613514+0000", 2024-08-17T18:02:19.594 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2313000000000001e-05 2024-08-17T18:02:19.594 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.594 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.594 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925150+0000", 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.31163655000000001 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925157+0000", 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5350000000000001e-06 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.926087+0000", 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00093048299999999996 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.595 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.926114+0000", 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6763000000000001e-05 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.987738+0000", 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.061623566999999997 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.987834+0000", 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6605000000000001e-05 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.596 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.597 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.597 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.597 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:30111 46.8 46:189f8eaf:::benchmark_data_smithi130_267720_object30110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:02:19.597 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.623717+0000", 2024-08-17T18:02:19.597 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 150.86302576599999, 2024-08-17T18:02:19.597 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38956026100000002, 2024-08-17T18:02:19.597 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.597 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.597 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.597 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.597 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:19.597 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:19.597 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 30111 2024-08-17T18:02:19.597 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.598 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.598 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.598 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.598 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623717+0000", 2024-08-17T18:02:19.598 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.598 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.598 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.598 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.598 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623717+0000", 2024-08-17T18:02:19.598 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.598 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.598 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.598 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623726+0000", 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1579999999999994e-06 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623751+0000", 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5194999999999998e-05 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623764+0000", 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3210999999999999e-05 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.599 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.600 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623771+0000", 2024-08-17T18:02:19.600 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2490000000000004e-06 2024-08-17T18:02:19.600 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.600 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.600 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.600 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.624248+0000", 2024-08-17T18:02:19.600 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000476828 2024-08-17T18:02:19.600 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.600 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.600 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.600 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.624387+0000", 2024-08-17T18:02:19.600 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013830900000000001 2024-08-17T18:02:19.600 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.600 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.601 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:02:19.601 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.624521+0000", 2024-08-17T18:02:19.601 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013415700000000001 2024-08-17T18:02:19.601 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.601 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.601 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.601 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627377+0000", 2024-08-17T18:02:19.601 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028564889999999998 2024-08-17T18:02:19.601 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.601 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.601 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.601 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627536+0000", 2024-08-17T18:02:19.601 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015840400000000001 2024-08-17T18:02:19.601 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.602 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.602 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.602 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013165+0000", 2024-08-17T18:02:19.602 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38562930099999998 2024-08-17T18:02:19.602 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.602 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.602 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.602 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013228+0000", 2024-08-17T18:02:19.602 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3109999999999997e-05 2024-08-17T18:02:19.602 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.602 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.602 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.602 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013277+0000", 2024-08-17T18:02:19.602 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8850000000000002e-05 2024-08-17T18:02:19.603 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.603 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.603 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.603 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.603 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.603 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:30127 46.8 46:16873e59:::benchmark_data_smithi130_267720_object30126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:02:19.603 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.632099+0000", 2024-08-17T18:02:19.603 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 150.854643237, 2024-08-17T18:02:19.603 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38487052399999999, 2024-08-17T18:02:19.603 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.603 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.603 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.603 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.603 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:19.604 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:19.604 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 30127 2024-08-17T18:02:19.604 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.604 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.604 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.604 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.604 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632099+0000", 2024-08-17T18:02:19.604 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.604 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.604 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.604 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.604 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632099+0000", 2024-08-17T18:02:19.604 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.604 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.605 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.605 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.605 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632101+0000", 2024-08-17T18:02:19.605 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4160000000000001e-06 2024-08-17T18:02:19.605 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.605 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.605 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.605 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632121+0000", 2024-08-17T18:02:19.605 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0633e-05 2024-08-17T18:02:19.605 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.605 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.605 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.605 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632136+0000", 2024-08-17T18:02:19.605 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4622e-05 2024-08-17T18:02:19.606 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.606 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.606 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.606 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632144+0000", 2024-08-17T18:02:19.606 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8450000000000003e-06 2024-08-17T18:02:19.606 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.606 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.606 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.606 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632184+0000", 2024-08-17T18:02:19.606 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0441999999999997e-05 2024-08-17T18:02:19.606 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.606 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.606 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.606 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632304+0000", 2024-08-17T18:02:19.607 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000119548 2024-08-17T18:02:19.607 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.607 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.607 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:02:19.607 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632459+0000", 2024-08-17T18:02:19.607 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000155165 2024-08-17T18:02:19.607 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.607 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.607 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.607 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.633815+0000", 2024-08-17T18:02:19.607 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001356554 2024-08-17T18:02:19.607 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.607 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.607 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.608 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.633999+0000", 2024-08-17T18:02:19.608 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018367200000000001 2024-08-17T18:02:19.608 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.608 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.608 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.608 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016770+0000", 2024-08-17T18:02:19.608 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38277128399999999 2024-08-17T18:02:19.608 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.608 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.608 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.608 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016927+0000", 2024-08-17T18:02:19.608 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015688599999999999 2024-08-17T18:02:19.608 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.608 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.609 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.609 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016970+0000", 2024-08-17T18:02:19.609 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2456999999999997e-05 2024-08-17T18:02:19.609 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.609 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.609 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.609 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.609 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.609 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:42538 46.7 46:e888a4cb:::benchmark_data_smithi130_267720_object42537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:02:19.609 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:57.009390+0000", 2024-08-17T18:02:19.609 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 142.477351992, 2024-08-17T18:02:19.609 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0254410919999999, 2024-08-17T18:02:19.609 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.610 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.610 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:02:19.610 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.610 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:19.610 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:19.610 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 42538 2024-08-17T18:02:19.610 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.610 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.610 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.610 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.610 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009390+0000", 2024-08-17T18:02:19.610 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.611 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.611 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.611 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.611 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009390+0000", 2024-08-17T18:02:19.611 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.611 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.611 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.611 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.611 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009392+0000", 2024-08-17T18:02:19.611 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.978e-06 2024-08-17T18:02:19.611 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.611 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.611 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.611 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009418+0000", 2024-08-17T18:02:19.612 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5666000000000001e-05 2024-08-17T18:02:19.612 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.612 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.612 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.612 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009441+0000", 2024-08-17T18:02:19.612 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3311000000000001e-05 2024-08-17T18:02:19.612 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.612 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.612 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.612 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009450+0000", 2024-08-17T18:02:19.612 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6680000000000007e-06 2024-08-17T18:02:19.612 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.612 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.612 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.613 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009513+0000", 2024-08-17T18:02:19.613 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2739999999999999e-05 2024-08-17T18:02:19.613 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.613 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.613 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op must wait for map", 2024-08-17T18:02:19.613 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009519+0000", 2024-08-17T18:02:19.613 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6599999999999998e-06 2024-08-17T18:02:19.613 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.613 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.613 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.613 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.013068+0000", 2024-08-17T18:02:19.613 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003548257 2024-08-17T18:02:19.613 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.614 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.614 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:02:19.614 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.013078+0000", 2024-08-17T18:02:19.614 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0146000000000001e-05 2024-08-17T18:02:19.614 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.614 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.614 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.614 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.024786+0000", 2024-08-17T18:02:19.614 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.011707755 2024-08-17T18:02:19.614 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.614 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.614 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:02:19.614 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.024983+0000", 2024-08-17T18:02:19.614 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000197479 2024-08-17T18:02:19.615 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.615 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.615 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.615 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.034686+0000", 2024-08-17T18:02:19.615 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0097024959999999997 2024-08-17T18:02:19.615 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.615 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.615 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.615 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.034832+0000", 2024-08-17T18:02:19.615 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014593599999999999 2024-08-17T18:02:19.615 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.615 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.615 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.615 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.616 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.616 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:42545 46.7 46:eec73fd4:::benchmark_data_smithi130_267720_object42544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:02:19.616 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:57.016850+0000", 2024-08-17T18:02:19.616 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 142.46989232499999, 2024-08-17T18:02:19.616 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0183753090000001, 2024-08-17T18:02:19.616 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.616 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.616 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.616 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.616 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:19.616 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:19.616 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 42545 2024-08-17T18:02:19.616 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.616 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.617 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.617 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.617 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.016850+0000", 2024-08-17T18:02:19.617 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.617 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.617 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.617 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.617 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.016850+0000", 2024-08-17T18:02:19.617 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.617 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.617 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.617 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.617 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.016852+0000", 2024-08-17T18:02:19.617 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5489999999999999e-06 2024-08-17T18:02:19.618 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.618 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.618 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.618 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.016879+0000", 2024-08-17T18:02:19.618 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.7605000000000001e-05 2024-08-17T18:02:19.618 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.618 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.618 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.618 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.016891+0000", 2024-08-17T18:02:19.618 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2201000000000001e-05 2024-08-17T18:02:19.618 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.618 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.618 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.619 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.016899+0000", 2024-08-17T18:02:19.619 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5259999999999996e-06 2024-08-17T18:02:19.619 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.619 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.619 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.619 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.017059+0000", 2024-08-17T18:02:19.619 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000160471 2024-08-17T18:02:19.619 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.619 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.619 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:02:19.619 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.017070+0000", 2024-08-17T18:02:19.619 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0297e-05 2024-08-17T18:02:19.619 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.619 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.620 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.620 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.025286+0000", 2024-08-17T18:02:19.620 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0082163319999999 2024-08-17T18:02:19.620 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.620 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.620 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.620 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.025449+0000", 2024-08-17T18:02:19.620 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016304700000000001 2024-08-17T18:02:19.620 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.620 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.620 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:02:19.620 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.025644+0000", 2024-08-17T18:02:19.620 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00019469499999999999 2024-08-17T18:02:19.620 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.034826+0000", 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.009182183 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.034876+0000", 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0396999999999999e-05 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035011+0000", 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013465999999999999 2024-08-17T18:02:19.621 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.622 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.622 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.622 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035141+0000", 2024-08-17T18:02:19.622 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013020999999999999 2024-08-17T18:02:19.622 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.622 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.622 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.622 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035225+0000", 2024-08-17T18:02:19.622 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4135999999999995e-05 2024-08-17T18:02:19.622 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.622 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.622 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.622 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.622 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:42560 46.7 46:ea5a980c:::benchmark_data_smithi130_267720_object42559:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:02:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:57.039166+0000", 2024-08-17T18:02:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 142.44757649799999, 2024-08-17T18:02:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.99687259800000005, 2024-08-17T18:02:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 42560 2024-08-17T18:02:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.623 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.624 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.624 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039166+0000", 2024-08-17T18:02:19.624 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.624 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.624 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.624 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.624 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039166+0000", 2024-08-17T18:02:19.624 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.624 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.624 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.624 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.624 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039168+0000", 2024-08-17T18:02:19.624 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8199999999999999e-06 2024-08-17T18:02:19.624 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039196+0000", 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8391e-05 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039210+0000", 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3878e-05 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039218+0000", 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6199999999999999e-06 2024-08-17T18:02:19.625 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.626 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.626 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.626 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039534+0000", 2024-08-17T18:02:19.626 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00031600399999999999 2024-08-17T18:02:19.626 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.626 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.626 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:02:19.626 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039546+0000", 2024-08-17T18:02:19.626 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2279999999999999e-05 2024-08-17T18:02:19.626 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.626 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.626 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.626 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.027308+0000", 2024-08-17T18:02:19.627 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98776202300000004 2024-08-17T18:02:19.627 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.627 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.627 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.627 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.027482+0000", 2024-08-17T18:02:19.627 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000173759 2024-08-17T18:02:19.627 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.627 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.627 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:02:19.627 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.027655+0000", 2024-08-17T18:02:19.627 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017299899999999999 2024-08-17T18:02:19.627 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.627 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.627 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.628 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035288+0000", 2024-08-17T18:02:19.628 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0076332570000000001 2024-08-17T18:02:19.628 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.628 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.628 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.628 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035303+0000", 2024-08-17T18:02:19.628 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5491000000000001e-05 2024-08-17T18:02:19.628 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.628 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.628 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.628 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035373+0000", 2024-08-17T18:02:19.628 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9369999999999995e-05 2024-08-17T18:02:19.628 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.628 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.629 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.629 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035932+0000", 2024-08-17T18:02:19.629 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00055875599999999999 2024-08-17T18:02:19.629 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.629 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.629 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.629 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036039+0000", 2024-08-17T18:02:19.629 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010695 2024-08-17T18:02:19.629 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.629 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.629 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.629 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.629 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.629 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:42564 46.7 46:e9df56bd:::benchmark_data_smithi130_267720_object42563:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:02:19.630 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:57.042569+0000", 2024-08-17T18:02:19.630 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 142.444173541, 2024-08-17T18:02:19.630 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.99393814300000005, 2024-08-17T18:02:19.630 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.630 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.630 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.630 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.630 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:19.630 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:19.630 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 42564 2024-08-17T18:02:19.630 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.630 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.630 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.630 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042569+0000", 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042569+0000", 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042570+0000", 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6020000000000001e-06 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.631 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.632 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042595+0000", 2024-08-17T18:02:19.632 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4083999999999998e-05 2024-08-17T18:02:19.632 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.632 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.632 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.632 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042607+0000", 2024-08-17T18:02:19.632 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2653999999999999e-05 2024-08-17T18:02:19.632 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.632 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.632 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.632 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042615+0000", 2024-08-17T18:02:19.632 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1019999999999997e-06 2024-08-17T18:02:19.632 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.632 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042720+0000", 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000104873 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042731+0000", 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0614e-05 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.029280+0000", 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98654872400000004 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.633 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.634 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.634 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.029430+0000", 2024-08-17T18:02:19.634 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015073800000000001 2024-08-17T18:02:19.634 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.634 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.634 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:02:19.634 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.029604+0000", 2024-08-17T18:02:19.634 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000173828 2024-08-17T18:02:19.634 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.634 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.634 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.634 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035306+0000", 2024-08-17T18:02:19.634 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005702403 2024-08-17T18:02:19.634 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.635 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.635 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.635 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036092+0000", 2024-08-17T18:02:19.635 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00078518000000000001 2024-08-17T18:02:19.635 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.635 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.635 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.635 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036337+0000", 2024-08-17T18:02:19.635 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00024485899999999999 2024-08-17T18:02:19.635 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.635 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.635 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.635 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036411+0000", 2024-08-17T18:02:19.635 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4957999999999996e-05 2024-08-17T18:02:19.636 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.636 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.636 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.636 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036507+0000", 2024-08-17T18:02:19.636 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5524000000000002e-05 2024-08-17T18:02:19.636 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.636 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.636 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.636 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.636 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.636 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:42598 46.7 46:e04df74a:::benchmark_data_smithi130_267720_object42597:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:02:19.636 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:57.065153+0000", 2024-08-17T18:02:19.636 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 142.42158898400001, 2024-08-17T18:02:19.636 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.97388112800000004, 2024-08-17T18:02:19.637 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.637 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.637 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.637 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.637 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:19.637 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:19.637 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 42598 2024-08-17T18:02:19.637 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.637 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.637 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.637 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.637 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065153+0000", 2024-08-17T18:02:19.638 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.638 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.638 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.638 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.638 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065153+0000", 2024-08-17T18:02:19.638 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.638 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.638 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.638 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.638 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065154+0000", 2024-08-17T18:02:19.638 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3200000000000004e-07 2024-08-17T18:02:19.639 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.639 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.639 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.639 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065172+0000", 2024-08-17T18:02:19.639 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7638999999999998e-05 2024-08-17T18:02:19.639 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.639 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.639 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.639 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065181+0000", 2024-08-17T18:02:19.639 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7099999999999996e-06 2024-08-17T18:02:19.640 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.640 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.640 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.640 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065185+0000", 2024-08-17T18:02:19.640 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.814e-06 2024-08-17T18:02:19.640 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.640 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.640 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.640 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065240+0000", 2024-08-17T18:02:19.640 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4125000000000001e-05 2024-08-17T18:02:19.640 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.640 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.640 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:02:19.640 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065250+0000", 2024-08-17T18:02:19.641 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0869999999999999e-05 2024-08-17T18:02:19.641 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.641 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.641 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.641 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.031172+0000", 2024-08-17T18:02:19.641 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.965921213 2024-08-17T18:02:19.641 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.641 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.641 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.641 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.031355+0000", 2024-08-17T18:02:19.641 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018302600000000001 2024-08-17T18:02:19.641 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.641 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.641 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:02:19.642 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.031461+0000", 2024-08-17T18:02:19.642 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010650999999999999 2024-08-17T18:02:19.642 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.642 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.642 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.642 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035310+0000", 2024-08-17T18:02:19.642 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0038484470000000001 2024-08-17T18:02:19.642 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.642 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.642 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.642 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036223+0000", 2024-08-17T18:02:19.642 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000913264 2024-08-17T18:02:19.642 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.642 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.643 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.643 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.038875+0000", 2024-08-17T18:02:19.643 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026517440000000001 2024-08-17T18:02:19.643 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.643 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.643 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.643 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.038948+0000", 2024-08-17T18:02:19.643 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3411999999999994e-05 2024-08-17T18:02:19.643 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.643 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.643 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.643 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.039035+0000", 2024-08-17T18:02:19.643 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6521999999999996e-05 2024-08-17T18:02:19.643 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:42601 46.7 46:e04c9391:::benchmark_data_smithi130_267720_object42600:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:57.067752+0000", 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 142.41899050999999, 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98219549399999995, 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:19.644 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 42601 2024-08-17T18:02:19.645 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.645 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.645 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.645 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.645 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067752+0000", 2024-08-17T18:02:19.645 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.645 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.645 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.645 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.645 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067752+0000", 2024-08-17T18:02:19.645 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.645 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.645 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.646 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.646 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067753+0000", 2024-08-17T18:02:19.646 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3799999999999996e-07 2024-08-17T18:02:19.646 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.646 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.646 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.646 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067771+0000", 2024-08-17T18:02:19.646 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8167000000000001e-05 2024-08-17T18:02:19.646 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.646 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.646 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.646 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067780+0000", 2024-08-17T18:02:19.646 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9220000000000003e-06 2024-08-17T18:02:19.646 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.647 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.647 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.647 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067785+0000", 2024-08-17T18:02:19.647 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2639999999999999e-06 2024-08-17T18:02:19.647 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.647 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.647 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.647 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067851+0000", 2024-08-17T18:02:19.647 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6042000000000002e-05 2024-08-17T18:02:19.647 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.647 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.647 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:02:19.647 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067859+0000", 2024-08-17T18:02:19.647 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3450000000000006e-06 2024-08-17T18:02:19.648 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.648 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.648 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.648 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036577+0000", 2024-08-17T18:02:19.648 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.968717418 2024-08-17T18:02:19.648 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.648 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.648 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.648 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036706+0000", 2024-08-17T18:02:19.648 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000129116 2024-08-17T18:02:19.648 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.648 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.648 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:02:19.649 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036865+0000", 2024-08-17T18:02:19.649 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000159059 2024-08-17T18:02:19.649 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.649 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.649 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.649 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.043100+0000", 2024-08-17T18:02:19.649 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0062351480000000003 2024-08-17T18:02:19.649 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.649 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.649 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.649 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.045320+0000", 2024-08-17T18:02:19.649 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002219702 2024-08-17T18:02:19.649 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.649 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.650 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.650 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.049699+0000", 2024-08-17T18:02:19.650 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0043796570000000003 2024-08-17T18:02:19.650 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.650 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.650 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.650 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.049804+0000", 2024-08-17T18:02:19.650 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010448800000000001 2024-08-17T18:02:19.650 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.650 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.650 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.650 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.049947+0000", 2024-08-17T18:02:19.650 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000143428 2024-08-17T18:02:19.650 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.651 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.651 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.651 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.651 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.651 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79527 46.7 46:eed33f3f:::benchmark_data_smithi130_267720_object79526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:02:19.651 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.632282+0000", 2024-08-17T18:02:19.651 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 108.85446028200001, 2024-08-17T18:02:19.651 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38843913299999999, 2024-08-17T18:02:19.651 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.651 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.651 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.651 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.651 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:19.651 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:19.652 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79527 2024-08-17T18:02:19.652 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.652 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.652 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.652 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.652 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632282+0000", 2024-08-17T18:02:19.652 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.652 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.652 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.652 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.652 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632282+0000", 2024-08-17T18:02:19.652 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.652 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.652 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.653 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.653 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632283+0000", 2024-08-17T18:02:19.653 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.78e-07 2024-08-17T18:02:19.653 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.653 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.653 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.653 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632304+0000", 2024-08-17T18:02:19.653 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1094000000000002e-05 2024-08-17T18:02:19.653 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.653 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.653 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.653 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632314+0000", 2024-08-17T18:02:19.653 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9289999999999993e-06 2024-08-17T18:02:19.653 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.654 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.654 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.654 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632321+0000", 2024-08-17T18:02:19.654 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7399999999999998e-06 2024-08-17T18:02:19.654 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.654 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.654 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.654 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.635991+0000", 2024-08-17T18:02:19.654 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0036704569999999998 2024-08-17T18:02:19.654 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.654 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.654 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.654 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636300+0000", 2024-08-17T18:02:19.655 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00030820700000000001 2024-08-17T18:02:19.655 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.655 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.655 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:02:19.655 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636833+0000", 2024-08-17T18:02:19.655 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00053374499999999997 2024-08-17T18:02:19.655 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.655 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.655 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.655 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.641888+0000", 2024-08-17T18:02:19.655 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0050543869999999996 2024-08-17T18:02:19.655 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.655 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.655 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.656 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.643224+0000", 2024-08-17T18:02:19.656 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0013359750000000001 2024-08-17T18:02:19.656 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.656 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.656 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.656 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020614+0000", 2024-08-17T18:02:19.656 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37739075999999999 2024-08-17T18:02:19.656 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.656 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.656 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.656 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020677+0000", 2024-08-17T18:02:19.656 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2242999999999993e-05 2024-08-17T18:02:19.656 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.656 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.657 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.657 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020721+0000", 2024-08-17T18:02:19.657 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4617999999999999e-05 2024-08-17T18:02:19.657 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.657 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.657 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.657 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.657 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.657 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:95864 46.7 46:e0257e6e:::benchmark_data_smithi130_267720_object95863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-08-17T18:02:19.657 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:49.266079+0000", 2024-08-17T18:02:19.657 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 90.220663200999994, 2024-08-17T18:02:19.657 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35426025, 2024-08-17T18:02:19.657 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.657 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.658 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.658 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.658 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:19.658 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:19.658 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 95864 2024-08-17T18:02:19.658 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.658 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.658 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.658 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.658 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266079+0000", 2024-08-17T18:02:19.658 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.658 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.658 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.659 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.659 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266079+0000", 2024-08-17T18:02:19.659 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.659 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.659 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.659 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.659 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266081+0000", 2024-08-17T18:02:19.659 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7290000000000001e-06 2024-08-17T18:02:19.659 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.659 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.659 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.659 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266104+0000", 2024-08-17T18:02:19.659 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3125999999999998e-05 2024-08-17T18:02:19.659 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.660 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.660 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.660 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266117+0000", 2024-08-17T18:02:19.660 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2931999999999999e-05 2024-08-17T18:02:19.660 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.660 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.660 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.660 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266124+0000", 2024-08-17T18:02:19.660 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4309999999999997e-06 2024-08-17T18:02:19.660 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.660 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.660 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.660 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266184+0000", 2024-08-17T18:02:19.661 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9741999999999998e-05 2024-08-17T18:02:19.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.661 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.661 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.661 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266324+0000", 2024-08-17T18:02:19.661 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013939399999999999 2024-08-17T18:02:19.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.661 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.661 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:02:19.661 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266467+0000", 2024-08-17T18:02:19.661 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000143657 2024-08-17T18:02:19.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.661 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.661 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.662 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.273981+0000", 2024-08-17T18:02:19.662 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0075134829999999996 2024-08-17T18:02:19.662 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.662 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.662 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.662 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.277410+0000", 2024-08-17T18:02:19.662 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0034292860000000001 2024-08-17T18:02:19.662 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.662 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.662 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.662 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.620234+0000", 2024-08-17T18:02:19.662 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.34282429800000003 2024-08-17T18:02:19.662 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.662 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.663 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.663 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.620293+0000", 2024-08-17T18:02:19.663 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8974000000000002e-05 2024-08-17T18:02:19.663 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.663 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.663 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.663 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.620339+0000", 2024-08-17T18:02:19.663 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6198000000000001e-05 2024-08-17T18:02:19.663 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.663 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.663 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.663 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.663 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.663 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:95870 46.7 46:ea577ff7:::benchmark_data_smithi130_267720_object95869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-08-17T18:02:19.666 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:49.269591+0000", 2024-08-17T18:02:19.666 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 90.217151556999994, 2024-08-17T18:02:19.666 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35086666100000002, 2024-08-17T18:02:19.666 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.666 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.666 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.666 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.666 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:19.666 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:19.667 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 95870 2024-08-17T18:02:19.667 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.667 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.667 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.667 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.667 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269591+0000", 2024-08-17T18:02:19.667 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.667 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.667 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.667 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.667 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269591+0000", 2024-08-17T18:02:19.667 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.667 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.668 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.668 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.668 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269592+0000", 2024-08-17T18:02:19.668 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3170000000000001e-06 2024-08-17T18:02:19.668 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.668 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.668 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.668 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269614+0000", 2024-08-17T18:02:19.668 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2010000000000001e-05 2024-08-17T18:02:19.668 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.668 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.668 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.668 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269626+0000", 2024-08-17T18:02:19.668 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2104e-05 2024-08-17T18:02:19.669 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.669 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.669 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.669 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269634+0000", 2024-08-17T18:02:19.669 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3459999999999996e-06 2024-08-17T18:02:19.670 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.670 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.670 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.670 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269743+0000", 2024-08-17T18:02:19.670 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010924799999999999 2024-08-17T18:02:19.670 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.670 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.670 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.670 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269878+0000", 2024-08-17T18:02:19.670 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013560699999999999 2024-08-17T18:02:19.670 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.670 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.670 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:02:19.671 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.270018+0000", 2024-08-17T18:02:19.671 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013919299999999999 2024-08-17T18:02:19.671 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.671 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.671 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.671 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.274481+0000", 2024-08-17T18:02:19.671 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0044628890000000003 2024-08-17T18:02:19.671 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.671 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.671 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.671 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.277929+0000", 2024-08-17T18:02:19.671 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003448156 2024-08-17T18:02:19.671 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.671 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.672 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.672 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.620384+0000", 2024-08-17T18:02:19.672 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.34245572699999999 2024-08-17T18:02:19.672 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.672 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.672 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.672 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.620426+0000", 2024-08-17T18:02:19.672 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1900999999999997e-05 2024-08-17T18:02:19.672 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.672 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.672 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.672 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.620458+0000", 2024-08-17T18:02:19.672 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1163e-05 2024-08-17T18:02:19.672 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.673 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.673 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.673 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.673 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.673 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:95883 46.7 46:ed78bf3e:::benchmark_data_smithi130_267720_object95882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-08-17T18:02:19.673 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:49.282472+0000", 2024-08-17T18:02:19.673 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 90.204270061000003, 2024-08-17T18:02:19.673 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.34845915799999999, 2024-08-17T18:02:19.673 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.673 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.673 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.673 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.673 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:19.674 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:19.674 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 95883 2024-08-17T18:02:19.674 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.674 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.674 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.674 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.674 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.282472+0000", 2024-08-17T18:02:19.674 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.674 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.674 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.674 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.674 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.282472+0000", 2024-08-17T18:02:19.674 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.674 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.675 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.675 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.675 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.282473+0000", 2024-08-17T18:02:19.675 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4600000000000004e-07 2024-08-17T18:02:19.675 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.675 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.675 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.675 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.282495+0000", 2024-08-17T18:02:19.675 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1571000000000001e-05 2024-08-17T18:02:19.675 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.675 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.675 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.675 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.282503+0000", 2024-08-17T18:02:19.675 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7069999999999998e-06 2024-08-17T18:02:19.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.676 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.676 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.676 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.282509+0000", 2024-08-17T18:02:19.676 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.015e-06 2024-08-17T18:02:19.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.676 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.676 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.676 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.282558+0000", 2024-08-17T18:02:19.676 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8318000000000001e-05 2024-08-17T18:02:19.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.676 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.676 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.677 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.616414+0000", 2024-08-17T18:02:19.677 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.33385609300000002 2024-08-17T18:02:19.677 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.677 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.677 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:02:19.677 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.616566+0000", 2024-08-17T18:02:19.677 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015180099999999999 2024-08-17T18:02:19.677 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.677 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.677 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.677 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630615+0000", 2024-08-17T18:02:19.677 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014049216999999999 2024-08-17T18:02:19.677 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.677 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.678 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.678 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630649+0000", 2024-08-17T18:02:19.678 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3905000000000001e-05 2024-08-17T18:02:19.678 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.678 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.678 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.678 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630840+0000", 2024-08-17T18:02:19.678 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000191106 2024-08-17T18:02:19.678 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.678 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.678 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.678 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630891+0000", 2024-08-17T18:02:19.679 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1144999999999999e-05 2024-08-17T18:02:19.679 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.679 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.679 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.679 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630931+0000", 2024-08-17T18:02:19.679 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0534000000000002e-05 2024-08-17T18:02:19.679 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.679 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.679 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.679 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.680 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.680 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:95888 46.3 46:c3756994:::benchmark_data_smithi130_267720_object95887:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-08-17T18:02:19.680 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:49.287821+0000", 2024-08-17T18:02:19.680 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 90.198921447000004, 2024-08-17T18:02:19.680 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35515286499999998, 2024-08-17T18:02:19.680 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:19.680 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:19.680 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:19.680 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:19.680 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:19.680 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:19.680 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 95888 2024-08-17T18:02:19.680 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.680 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:19.681 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.681 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:19.681 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.287821+0000", 2024-08-17T18:02:19.681 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.681 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.681 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:19.681 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.287821+0000", 2024-08-17T18:02:19.681 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:19.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.681 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.681 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:19.681 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.287822+0000", 2024-08-17T18:02:19.681 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3e-06 2024-08-17T18:02:19.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.682 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:19.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.287843+0000", 2024-08-17T18:02:19.682 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0489999999999999e-05 2024-08-17T18:02:19.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.682 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:19.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.616279+0000", 2024-08-17T18:02:19.682 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.32843580300000003 2024-08-17T18:02:19.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.682 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:19.683 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.616296+0000", 2024-08-17T18:02:19.683 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7003999999999998e-05 2024-08-17T18:02:19.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.683 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.683 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:19.683 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.616375+0000", 2024-08-17T18:02:19.683 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9618999999999998e-05 2024-08-17T18:02:19.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.683 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.683 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:19.683 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.616529+0000", 2024-08-17T18:02:19.683 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015364099999999999 2024-08-17T18:02:19.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.683 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.684 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:02:19.684 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.616671+0000", 2024-08-17T18:02:19.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000141707 2024-08-17T18:02:19.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.684 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:19.684 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630669+0000", 2024-08-17T18:02:19.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013998745999999999 2024-08-17T18:02:19.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.684 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.684 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.631980+0000", 2024-08-17T18:02:19.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0013102579999999999 2024-08-17T18:02:19.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:19.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.642830+0000", 2024-08-17T18:02:19.685 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.01085066 2024-08-17T18:02:19.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:19.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.642926+0000", 2024-08-17T18:02:19.685 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5964999999999993e-05 2024-08-17T18:02:19.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:19.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:19.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:19.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.642974+0000", 2024-08-17T18:02:19.686 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7672e-05 2024-08-17T18:02:19.686 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.686 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.686 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.686 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:19.686 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:19.686 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:02:19.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:19.687 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:19.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:19.788 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:19.811 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:19.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:19.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:19.913 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:19.937 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:19.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:19.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:20.039 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:20.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:20.076 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:20.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:20.147 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:02:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:02:19] "GET /metrics HTTP/1.1" 200 36788 "" "Prometheus/2.51.0" 2024-08-17T18:02:20.177 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:20.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:20.203 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:20.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:20.304 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:20.328 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:20.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:20.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:20.430 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:20.465 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:20.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:20.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:20.475 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:02:20.568 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:20.591 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:20.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:20.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:20 smithi179 ceph-mon[93519]: pgmap v2111: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 914 op/s 2024-08-17T18:02:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1196902634' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:02:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:02:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:02:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:02:20.630 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T18:02:20.694 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:20.708 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:20 smithi130 ceph-mon[105521]: pgmap v2111: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 914 op/s 2024-08-17T18:02:20.708 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1196902634' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:02:20.708 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:20 smithi130 ceph-mon[110068]: pgmap v2111: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 914 op/s 2024-08-17T18:02:20.708 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1196902634' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:02:20.749 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:20.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:20.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:20.851 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:20.903 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:20.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:20.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:21.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:21.027 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:21.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:21.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:21.129 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:21.151 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:21.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:21.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:21.253 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:21.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:21.297 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:21.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:21.398 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:21.443 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:21.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:21.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:21.546 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:21.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:21.598 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:21.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:21.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:21 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:02:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:21 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1196902634' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:02:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:21 smithi130 ceph-mon[105521]: osdmap e1280: 8 total, 8 up, 6 in 2024-08-17T18:02:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:21 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:02:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:21 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1196902634' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:02:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:21 smithi130 ceph-mon[110068]: osdmap e1280: 8 total, 8 up, 6 in 2024-08-17T18:02:21.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:21.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:21.754 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:21.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:21 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:02:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:21 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1196902634' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:02:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:21 smithi179 ceph-mon[93519]: osdmap e1280: 8 total, 8 up, 6 in 2024-08-17T18:02:21.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:21.877 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:21.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:21.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:21.979 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:22.002 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:22.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:22.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:22.104 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:22.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:22.149 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:22.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:22.235 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:22.235 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:22.249 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:22.311 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:22.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:22.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:22.413 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:22.468 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:22.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:22.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:22.570 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:22.593 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:22.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:22.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:22.659 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:22 smithi130 ceph-mon[105521]: pgmap v2113: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 774 op/s 2024-08-17T18:02:22.659 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:22 smithi130 ceph-mon[110068]: pgmap v2113: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 774 op/s 2024-08-17T18:02:22.695 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:22.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:22.734 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:22.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:22.835 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:22.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:22 smithi179 ceph-mon[93519]: pgmap v2113: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 774 op/s 2024-08-17T18:02:22.860 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:22.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:22.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:22.962 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:22.998 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:22.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:22.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:23.100 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:23.124 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:23.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:23.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:23.226 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:23.264 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:23.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:23.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:23.366 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:23.413 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:23.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:23.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:23.456 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:02:23.515 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:23.536 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:23.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:23.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:23.638 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:23.661 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:23.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:23.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:23.763 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:23.787 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:23.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:23.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:23.889 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:23.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:23.914 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:23.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:24.015 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:24.038 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:24.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:24.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:24.140 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:24.186 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:24.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:24.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:24.288 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:24.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:24.331 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:24.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:24.431 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:24.472 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:24.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:24.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:24.574 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:24.611 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:24.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:24.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:24.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:24 smithi130 ceph-mon[105521]: pgmap v2114: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T18:02:24.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:24 smithi130 ceph-mon[110068]: pgmap v2114: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T18:02:24.713 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:24.731 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:24.735 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:24.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:24.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:24.836 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:24 smithi179 ceph-mon[93519]: pgmap v2114: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T18:02:24.909 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:24.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:24.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:25.010 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:25.056 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:25.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:25.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:25.158 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:25.182 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:25.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:25.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:25.283 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:25.313 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:25.340 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:25.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:25.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:25.442 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:25.467 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:25.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:25.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:25.569 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:25.630 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:25.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:25.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:25.732 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:25.755 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:25.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:25.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:25.857 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:25.881 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:25.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:25.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:25.983 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:26.006 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:26.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:26.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:26.108 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:26.151 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:26.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:26.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:26.252 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:26.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:26.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:26.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:26.376 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:26.397 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:26.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:26.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:26.489 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:02:26.489 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:02:26.499 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:26.521 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:26.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:26.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:26.526 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:02:26.527 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:02:26.564 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:02:26.564 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:02:26.605 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:02:26.606 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:02:26.623 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:26.636 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:02:26.637 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:02:26.645 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:26.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:26.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:26.667 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:02:26.668 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:02:26.701 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:02:26.701 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:02:26.730 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:02:26.731 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:02:26.731 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:26 smithi130 ceph-mon[105521]: pgmap v2115: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T18:02:26.731 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:26 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1727544912' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:02:26.731 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:26 smithi130 ceph-mon[110068]: pgmap v2115: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T18:02:26.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:26 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1727544912' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:02:26.747 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:26.789 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:26.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:26.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:26 smithi179 ceph-mon[93519]: pgmap v2115: 89 pgs: 89 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 802 op/s 2024-08-17T18:02:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:26 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1727544912' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:02:26.891 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:26.932 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:26.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:26.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:27.034 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:27.056 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:27.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:27.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:27.158 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:27.181 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:27.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:27.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:27.283 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:27.306 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:27.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:27.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:27.408 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:27.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:27.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:27.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:27.535 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:27.558 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:27.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:27.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:27.660 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:27.682 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:27.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:27.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:27.784 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:27.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:27.808 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:27.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:27.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:02:27.870 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:02:27.871 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:02:27.909 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:27.934 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:27.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:27.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:28.036 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:28.060 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:28.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:28.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:28.162 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:28.187 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:28.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:28.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:28.289 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:28.313 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:28.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:28.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:28.414 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:28.439 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:28.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:28.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:28.541 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:28.563 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:28.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:28.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:28.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:28.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:28 smithi130 ceph-mon[105521]: pgmap v2116: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 807 op/s 2024-08-17T18:02:28.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:28 smithi130 ceph-mon[110068]: pgmap v2116: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 807 op/s 2024-08-17T18:02:28.706 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:28.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:28.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:28.808 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:28.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:28.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:28.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:28.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:28 smithi179 ceph-mon[93519]: pgmap v2116: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 807 op/s 2024-08-17T18:02:28.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:28.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:28.973 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:28.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:29.074 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:29.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:29.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:29.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:29.198 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:29.235 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:29.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:29.337 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:29.374 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:29.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:29.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:29.476 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:29.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:29.522 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:29.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:29.623 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:29.646 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:29.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:29.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:29.748 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:29.772 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:29.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:29.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:29.874 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:29.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:29.913 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:29.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:29.914 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:02:30.014 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:30.064 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:30.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:30.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:30.079 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:02:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:02:29] "GET /metrics HTTP/1.1" 200 36787 "" "Prometheus/2.51.0" 2024-08-17T18:02:30.167 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:30.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:30.221 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:30.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:30.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:30.389 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:30.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:30.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:30.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:30.502 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:02:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:02:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:02:30.512 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:02:30.512 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:02:30.514 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:30.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:30.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:30.616 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:30.637 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:30.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:30.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:30 smithi130 ceph-mon[105521]: pgmap v2117: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 826 op/s 2024-08-17T18:02:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:30 smithi130 ceph-mon[110068]: pgmap v2117: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 826 op/s 2024-08-17T18:02:30.739 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:30.780 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:30.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:30.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:30 smithi179 ceph-mon[93519]: pgmap v2117: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 826 op/s 2024-08-17T18:02:30.882 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:30.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:30.928 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:30.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:31.029 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:31.052 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:31.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:31.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:31.154 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:31.177 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:31.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:31.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:31.273 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T18:02:31.278 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:31.381 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:31.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:31.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:31.483 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:31.523 INFO:teuthology.orchestra.run.smithi130.stderr:set pool 1 pgp_num to 1 2024-08-17T18:02:31.524 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:31.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:31.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:31.626 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:31.675 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:31.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:31.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:31.777 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:31.800 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:31.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:31.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1963298006' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T18:02:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:31 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T18:02:31.901 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1963298006' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T18:02:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:31 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T18:02:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1963298006' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T18:02:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:31 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T18:02:31.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:31.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:31.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:32.052 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:32.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:32.087 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:32.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:32.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:32.212 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:32.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:32.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:32.314 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:32.337 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:32.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:32.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:32.439 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:32.463 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:32.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:32.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:32.565 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:32.587 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:32 smithi130 ceph-mon[105521]: pgmap v2118: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 769 op/s 2024-08-17T18:02:32.587 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:32 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T18:02:32.587 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:32 smithi130 ceph-mon[105521]: osdmap e1281: 8 total, 8 up, 6 in 2024-08-17T18:02:32.587 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:32 smithi130 ceph-mon[110068]: pgmap v2118: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 769 op/s 2024-08-17T18:02:32.588 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:32 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T18:02:32.588 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:32 smithi130 ceph-mon[110068]: osdmap e1281: 8 total, 8 up, 6 in 2024-08-17T18:02:32.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:32.589 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:32.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:32.690 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:32.713 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:32.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:32.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:32.815 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:32.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:32.842 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:32.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:32 smithi179 ceph-mon[93519]: pgmap v2118: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 769 op/s 2024-08-17T18:02:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:32 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T18:02:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:32 smithi179 ceph-mon[93519]: osdmap e1281: 8 total, 8 up, 6 in 2024-08-17T18:02:32.944 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:32.980 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:32.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:33.082 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:33.104 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:33.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:33.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:33.206 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:33.245 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:33.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:33.347 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:33.446 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:33.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:33.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:33.546 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:02:33.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:33 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3382357170' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:02:33.546 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:33 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:02:33.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:33 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3382357170' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:02:33.547 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:33 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:02:33.548 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:33.571 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:33.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:33.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:33.673 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:33.712 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:33.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:33.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:33.814 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:33.836 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:33.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:33.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:33.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:33 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3382357170' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:02:33.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:33 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:02:33.938 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:33.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:33.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:33.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:34.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:34.103 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:34.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:34.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:34.205 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:34.233 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:34.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:34.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:34.335 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:34.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:34.375 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:34.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:34.476 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:34.498 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:34.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:34.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:34.600 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:34.641 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:34.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:34.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:34.743 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:34.786 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:34.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:34.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:34 smithi179 ceph-mon[93519]: pgmap v2120: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 827 op/s 2024-08-17T18:02:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:34 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:02:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:34 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:02:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:34 smithi179 ceph-mon[93519]: osdmap e1282: 8 total, 8 up, 6 in 2024-08-17T18:02:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:34 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:34 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:34 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:02:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:34 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:02:34.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:34 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:34.888 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:34.912 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:34.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:34.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[105521]: pgmap v2120: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 827 op/s 2024-08-17T18:02:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:02:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:02:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[105521]: osdmap e1282: 8 total, 8 up, 6 in 2024-08-17T18:02:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:34.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:02:34.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:02:34.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:34.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[110068]: pgmap v2120: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 827 op/s 2024-08-17T18:02:34.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:02:34.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:02:34.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[110068]: osdmap e1282: 8 total, 8 up, 6 in 2024-08-17T18:02:34.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:34.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:34.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:02:34.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:02:34.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:34 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:35.013 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:35.036 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:35.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:35.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:35.138 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:35.160 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:35.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:35.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:35.184 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:35.208 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:35.262 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:35.314 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:35.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:35.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:35.416 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:35.463 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:35.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:35.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:35.564 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:35.610 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:35.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:35.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:35.712 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:35.735 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:35.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:35.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:35.836 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:35.859 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:35.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:35.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:35.961 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:35.985 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:35.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:35.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:36.086 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:36.124 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:36.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:36.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:36.226 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:36.260 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:36.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:36.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:36.362 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:36.400 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:36.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:36.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:36.502 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:36.525 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:36.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:36.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:36.626 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:36.635 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:02:36.648 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:36.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:36.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:36 smithi130 ceph-mon[105521]: pgmap v2122: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 938 op/s 2024-08-17T18:02:36.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:36 smithi130 ceph-mon[110068]: pgmap v2122: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 938 op/s 2024-08-17T18:02:36.750 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:36.772 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:36.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:36.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:36 smithi179 ceph-mon[93519]: pgmap v2122: 89 pgs: 89 active+clean; 2.3 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 938 op/s 2024-08-17T18:02:36.875 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:36.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:36.899 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:36.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:37.000 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:37.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:37.041 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:37.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:37.142 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:37.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:37.166 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:37.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:37.267 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:37.291 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:37.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:37.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:37.392 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:37.416 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:37.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:37.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:37.517 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:37.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:37.568 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:37.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:37.669 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:37.693 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:37.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:37.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:37.795 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:37.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:37.858 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:37.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:37.959 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:37.983 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:37.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:37.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:38.085 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:38.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:38.135 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:38.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:38.236 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:38.258 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:38.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:38.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:38.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:38.406 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:38.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:38.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:38.508 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:38.547 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:38.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:38.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:38.649 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:38.671 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:38.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:38.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:38.773 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:38.798 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:38.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:38.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:38 smithi179 ceph-mon[93519]: pgmap v2123: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 608 op/s 2024-08-17T18:02:38.900 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:38.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:38 smithi130 ceph-mon[105521]: pgmap v2123: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 608 op/s 2024-08-17T18:02:38.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:38 smithi130 ceph-mon[110068]: pgmap v2123: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 608 op/s 2024-08-17T18:02:38.924 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:38.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:38.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:39.026 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:39.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:39.051 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:39.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:39.152 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:39.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:39.190 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:39.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:39.291 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:39.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:39.315 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:39.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:39.416 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:39.439 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:39.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:39.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:39.541 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:39.576 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:39.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:39.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:39.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:39.703 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:39.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:39.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:39.728 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:02:39.728 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:02:39.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:39.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:39.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:39.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:39.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:39.972 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:39.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:39.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:40.074 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:40.097 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:40.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:40.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:40.142 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7] out_osds: [5, 4] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T18:02:40.142 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:02:40.142 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 3, 6, 7] 2024-08-17T18:02:40.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd out 6 2024-08-17T18:02:40.158 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:02:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:02:39] "GET /metrics HTTP/1.1" 200 36787 "" "Prometheus/2.51.0" 2024-08-17T18:02:40.199 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:40.236 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:40.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:40.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:40.338 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:40.360 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:40.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:40.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:40.462 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:40.484 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:40.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:40.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:40.552 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:02:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:02:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:02:40.587 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:40.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:40.638 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:40.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:40.739 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:40.762 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:40.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:40.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:40 smithi179 ceph-mon[93519]: pgmap v2124: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T18:02:40.864 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:40.905 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:40.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:40.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:40.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:40 smithi130 ceph-mon[105521]: pgmap v2124: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T18:02:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:40 smithi130 ceph-mon[110068]: pgmap v2124: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T18:02:41.007 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:41.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:41.029 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:41.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:41.130 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:41.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:41.177 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:41.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:41.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:41.299 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:41.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:41.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:41.401 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:41.452 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:41.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:41.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:41.554 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:41.575 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:41.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:41.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:41.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:41.700 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:41.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:41.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:41.802 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:41.826 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:41.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:41.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:41.928 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:41.949 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:41.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:41.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:42.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:42.073 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:42.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:42.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:42.175 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:42.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:42.229 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:42.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:42.330 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:42.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:42.373 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:42.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:42.474 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:42.496 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:42.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:42.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:42.598 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:42.636 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:42.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:42.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:42.738 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:42.768 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:42.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:42.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:42 smithi179 ceph-mon[93519]: pgmap v2125: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 857 op/s 2024-08-17T18:02:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:02:42.870 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:42.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:42.894 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:42.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:42 smithi130 ceph-mon[105521]: pgmap v2125: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 857 op/s 2024-08-17T18:02:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:02:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:42 smithi130 ceph-mon[110068]: pgmap v2125: 89 pgs: 89 active+clean; 2.0 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 857 op/s 2024-08-17T18:02:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:02:42.996 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:43.017 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:43.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:43.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:43.119 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:43.158 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:43.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:43.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:43.260 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:43.281 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:43.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:43.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:43.383 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:43.403 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:43.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:43.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:43.505 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:43.526 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:43.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:43.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:43.628 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:43.669 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:43.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:43.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:43.772 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:43.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:43.819 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:43.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:43.921 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:43.949 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:43.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:43.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:44.050 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:44.089 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:44.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:44.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:44.190 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:44.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:44.230 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:44.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:44.331 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:44.358 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:02:44.359 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:02:44.359 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:02:44.359 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:02:44.359 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:02:44.360 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:02:44.360 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:02:44.360 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:02:44.360 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:02:44.360 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:02:44.360 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:02:44.360 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:02:44.360 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:02:44.361 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:02:44.361 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:02:44.361 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:02:44.371 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:44.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:44.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:44.475 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:44.501 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:44.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:44.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:44.603 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:44.618 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:44 smithi130 ceph-mon[105521]: pgmap v2126: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 826 op/s 2024-08-17T18:02:44.619 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:44 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1505781625' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:02:44.619 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:44 smithi130 ceph-mon[110068]: pgmap v2126: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 826 op/s 2024-08-17T18:02:44.619 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:44 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1505781625' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:02:44.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:44.629 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:44.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:44.730 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:44.756 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:44.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:44.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:44 smithi179 ceph-mon[93519]: pgmap v2126: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 826 op/s 2024-08-17T18:02:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:44 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1505781625' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:02:44.858 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:44.883 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:44.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:44.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:44.985 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:45.025 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:45.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:45.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:45.128 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:45.151 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:45.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:45.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:45.253 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:45.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:45.290 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:45.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:45.391 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:45.414 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:45.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:45.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:45.516 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:45.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:45.557 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:45.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:45.658 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:45.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:45.704 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:45.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:45.748 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T18:02:45.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:45.826 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:45.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:45.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:45.928 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:45.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:45.965 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:45.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:46.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:46.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:46.129 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:46.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:46.229 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:46.252 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:46.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:46.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:46.354 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:46.376 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:46.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:46.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:46.477 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:46.520 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:46.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:46.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:46.622 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:46.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:46.662 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:46.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:46.763 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:46.766 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:46.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:46.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:46.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:46 smithi179 ceph-mon[93519]: pgmap v2127: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 707 op/s 2024-08-17T18:02:46.911 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:46 smithi130 ceph-mon[110068]: pgmap v2127: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 707 op/s 2024-08-17T18:02:46.911 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:46 smithi130 ceph-mon[105521]: pgmap v2127: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 707 op/s 2024-08-17T18:02:46.924 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:46.979 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:46.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:46.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:47.081 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:47.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:47.119 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:47.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:47.220 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:47.257 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:47.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:47.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:47.360 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:47.381 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:47.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:47.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:47.483 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:47.507 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:47.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:47.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:47.609 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:47.632 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:47.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:47.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:47.724 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:47.728 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:47.734 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:47.756 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:47.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:47.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:47.858 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:47.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:47.906 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:47.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:48.007 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:48.044 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:48.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:48.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:48.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:48.169 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:48.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:48.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:48.271 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:48.317 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:48.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:48.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:48.419 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:48.441 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:48.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:48.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:48.543 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:48.565 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:48.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:48.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:48.667 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:48.709 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:48.710 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:02:48.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:48.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:48.729 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:48 smithi130 ceph-mon[105521]: pgmap v2128: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 744 op/s 2024-08-17T18:02:48.729 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:48 smithi130 ceph-mon[110068]: pgmap v2128: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 744 op/s 2024-08-17T18:02:48.812 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:48.835 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:48.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:48.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:48.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:48 smithi179 ceph-mon[93519]: pgmap v2128: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 744 op/s 2024-08-17T18:02:48.937 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:48.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:48.996 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:48.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:49.097 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:49.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:49.146 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:49.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:49.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:49.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:49.293 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:49.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:49.394 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:49.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:49.418 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:49.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:49.519 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:49.541 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:49.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:49.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:49.643 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:49.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:49.690 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:49.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:49.792 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:49.829 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:49.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:49.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:49.931 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:49.955 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:49.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:49.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:50.056 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:50.079 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:50.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:50.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:02:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:02:49] "GET /metrics HTTP/1.1" 200 36782 "" "Prometheus/2.51.0" 2024-08-17T18:02:50.181 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:50.226 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:50.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:50.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:50.328 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:50.365 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:50.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:50.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:50.466 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:50.498 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:50.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:50.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:50.568 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:02:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:02:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:02:50.600 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:50.621 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:50.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:50.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:50.723 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:50.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:50.770 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:50.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:50 smithi179 ceph-mon[93519]: pgmap v2129: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 930 op/s 2024-08-17T18:02:50.870 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:50.893 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:50.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:50.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:50.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:50 smithi130 ceph-mon[110068]: pgmap v2129: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 930 op/s 2024-08-17T18:02:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:50 smithi130 ceph-mon[105521]: pgmap v2129: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 930 op/s 2024-08-17T18:02:50.995 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:51.035 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:51.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:51.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:51.137 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:51.158 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:51.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:51.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:51.260 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:51.300 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:51.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:51.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:51.402 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:51.442 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:51.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:51.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:51.544 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:51.566 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:51.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:51.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:51.668 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:51.716 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:51.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:51.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:51.818 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:51.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:51.843 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:51.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:51.944 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:51.968 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:51.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:51.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:52.069 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:52.118 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:52.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:52.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:52.220 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:52.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:52.273 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:52.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:52.375 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:52.398 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:52.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:52.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:52.500 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:52.522 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:52.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:52.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:52.545 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:52.624 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:52.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:52.677 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:52.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:52.778 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:52.834 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:52.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:52.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:52.860 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:02:52.861 INFO:teuthology.orchestra.run.smithi130.stderr:marked out osd.6. 2024-08-17T18:02:52.936 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:52.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:52.963 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:52.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:53.065 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:53.086 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:53.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:53.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:52 smithi179 ceph-mon[93519]: pgmap v2130: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 643 op/s 2024-08-17T18:02:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:52 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/231441244' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:02:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:52 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:02:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:52 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2421659327' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-17T18:02:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:52 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-17T18:02:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:52 smithi179 ceph-mon[93519]: Client client.admin marked osd.6 out, while it was still marked up 2024-08-17T18:02:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:52 smithi130 ceph-mon[110068]: pgmap v2130: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 643 op/s 2024-08-17T18:02:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:52 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/231441244' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:02:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:52 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:02:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:52 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2421659327' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-17T18:02:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:52 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-17T18:02:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:52 smithi130 ceph-mon[110068]: Client client.admin marked osd.6 out, while it was still marked up 2024-08-17T18:02:53.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:52 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T18:02:52.775+0000 7fa856f19640 -1 mon.a@0(leader).osd e1282 definitely_dead 0 2024-08-17T18:02:53.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:52 smithi130 ceph-mon[105521]: pgmap v2130: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 643 op/s 2024-08-17T18:02:53.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:52 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/231441244' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:02:53.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:52 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:02:53.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:52 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2421659327' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-17T18:02:53.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:52 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["6"]} : dispatch 2024-08-17T18:02:53.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:52 smithi130 ceph-mon[105521]: Client client.admin marked osd.6 out, while it was still marked up 2024-08-17T18:02:53.188 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:53.210 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:53.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:53.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:53.312 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:53.333 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:53.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:53.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:53.435 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:53.456 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:53.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:53.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:53.559 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:53.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:53.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:53.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:53.687 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:53.713 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:53.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:53.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:53.815 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:53.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:53.838 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:53.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:53.939 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:53.961 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:53.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:53.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:54.000 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:53 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:02:54.000 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:53 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T18:02:54.000 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:53 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:02:54.000 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:53 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-08-17T18:02:54.000 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:53 smithi130 ceph-mon[105521]: osdmap e1283: 8 total, 8 up, 5 in 2024-08-17T18:02:54.000 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:53 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:02:54.000 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:53 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T18:02:54.000 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:53 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:02:54.001 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:53 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-08-17T18:02:54.001 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:53 smithi130 ceph-mon[110068]: osdmap e1283: 8 total, 8 up, 5 in 2024-08-17T18:02:54.005 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:02:54.005 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:02:54.005 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:02:54.005 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:02:54.005 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.006 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:204051 45.4 45:2bc8134d:::benchmark_data_smithi130_172551_object99244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1247)", 2024-08-17T18:02:54.006 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.559113+0000", 2024-08-17T18:02:54.006 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 244.44472004299999, 2024-08-17T18:02:54.006 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37097546799999997, 2024-08-17T18:02:54.006 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.006 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.006 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.006 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:54.006 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:02:54.006 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:02:54.006 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 204051 2024-08-17T18:02:54.006 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.007 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.007 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.007 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.007 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.559113+0000", 2024-08-17T18:02:54.007 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.007 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.007 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.007 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.007 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.559113+0000", 2024-08-17T18:02:54.007 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.007 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.008 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.008 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.008 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.559115+0000", 2024-08-17T18:02:54.008 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.996e-06 2024-08-17T18:02:54.008 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.008 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.008 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.008 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.559121+0000", 2024-08-17T18:02:54.008 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0850000000000002e-06 2024-08-17T18:02:54.008 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.008 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.008 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.009 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.559138+0000", 2024-08-17T18:02:54.009 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7411e-05 2024-08-17T18:02:54.009 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.009 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.009 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.009 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.559146+0000", 2024-08-17T18:02:54.009 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2199999999999992e-06 2024-08-17T18:02:54.009 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.009 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.009 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.009 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.565056+0000", 2024-08-17T18:02:54.009 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005909521 2024-08-17T18:02:54.009 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.010 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.010 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.010 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.565165+0000", 2024-08-17T18:02:54.010 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000109702 2024-08-17T18:02:54.010 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.010 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.010 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:02:54.010 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.565272+0000", 2024-08-17T18:02:54.010 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010613800000000001 2024-08-17T18:02:54.010 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.010 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.010 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:54.011 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.574966+0000", 2024-08-17T18:02:54.011 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0096942039999999997 2024-08-17T18:02:54.011 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.011 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.011 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:54.011 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.581843+0000", 2024-08-17T18:02:54.011 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0068772740000000001 2024-08-17T18:02:54.011 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.011 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.011 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:54.011 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925499+0000", 2024-08-17T18:02:54.011 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.34365609000000003 2024-08-17T18:02:54.011 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.012 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.012 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.012 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925552+0000", 2024-08-17T18:02:54.012 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2321999999999998e-05 2024-08-17T18:02:54.012 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.012 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.012 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.012 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.930088+0000", 2024-08-17T18:02:54.012 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0045365049999999997 2024-08-17T18:02:54.012 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.012 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.012 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.012 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.013 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.013 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:204062 45.4 45:26797482:::benchmark_data_smithi130_172551_object99255:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1247)", 2024-08-17T18:02:54.013 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.571762+0000", 2024-08-17T18:02:54.013 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 244.432070654, 2024-08-17T18:02:54.013 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35852473800000001, 2024-08-17T18:02:54.013 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.013 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.013 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.013 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:54.013 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:02:54.013 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:02:54.013 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 204062 2024-08-17T18:02:54.013 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.014 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.014 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.014 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.014 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.571762+0000", 2024-08-17T18:02:54.014 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.014 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.014 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.014 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.014 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.571762+0000", 2024-08-17T18:02:54.014 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.571763+0000", 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4720000000000001e-06 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.571768+0000", 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2930000000000002e-06 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.571778+0000", 2024-08-17T18:02:54.015 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9709999999999999e-06 2024-08-17T18:02:54.016 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.016 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.016 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.016 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.571784+0000", 2024-08-17T18:02:54.016 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0610000000000001e-06 2024-08-17T18:02:54.016 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.016 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.016 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.016 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576577+0000", 2024-08-17T18:02:54.016 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0047934750000000002 2024-08-17T18:02:54.016 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.016 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.016 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.016 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576688+0000", 2024-08-17T18:02:54.017 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011034799999999999 2024-08-17T18:02:54.017 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.017 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.017 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:02:54.017 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576805+0000", 2024-08-17T18:02:54.017 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011749200000000001 2024-08-17T18:02:54.017 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.017 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.017 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:54.017 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.582713+0000", 2024-08-17T18:02:54.017 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0059080510000000001 2024-08-17T18:02:54.018 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.018 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.018 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:54.018 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.584277+0000", 2024-08-17T18:02:54.018 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0015638189999999999 2024-08-17T18:02:54.018 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.018 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.018 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:54.018 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.930142+0000", 2024-08-17T18:02:54.018 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.34586505000000001 2024-08-17T18:02:54.018 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.018 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.018 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.019 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.930241+0000", 2024-08-17T18:02:54.019 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8806000000000002e-05 2024-08-17T18:02:54.019 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.019 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.019 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.019 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.930287+0000", 2024-08-17T18:02:54.019 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5899999999999998e-05 2024-08-17T18:02:54.019 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.019 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.019 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.019 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.019 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.019 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:30110 46.e 46:7442b1f6:::benchmark_data_smithi130_267720_object30109:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:02:54.019 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.623656+0000", 2024-08-17T18:02:54.020 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 185.38017685299999, 2024-08-17T18:02:54.020 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.39323034000000001, 2024-08-17T18:02:54.020 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.020 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.020 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.020 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:54.020 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:54.021 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:54.021 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 30110 2024-08-17T18:02:54.021 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.021 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.021 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.021 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.021 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623656+0000", 2024-08-17T18:02:54.022 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.022 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.022 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.022 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.022 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623656+0000", 2024-08-17T18:02:54.022 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.022 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.022 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.022 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.022 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623657+0000", 2024-08-17T18:02:54.022 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4589999999999999e-06 2024-08-17T18:02:54.022 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.022 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.023 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.023 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623683+0000", 2024-08-17T18:02:54.023 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6276000000000001e-05 2024-08-17T18:02:54.023 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.023 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.023 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.023 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623695+0000", 2024-08-17T18:02:54.023 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.185e-05 2024-08-17T18:02:54.023 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.023 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.023 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.023 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623703+0000", 2024-08-17T18:02:54.023 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7870000000000002e-06 2024-08-17T18:02:54.024 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.024 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.024 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.024 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623904+0000", 2024-08-17T18:02:54.024 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020092299999999999 2024-08-17T18:02:54.024 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.024 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.024 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.024 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.624071+0000", 2024-08-17T18:02:54.024 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016722199999999999 2024-08-17T18:02:54.024 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.024 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.024 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:02:54.025 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.624212+0000", 2024-08-17T18:02:54.025 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014109699999999999 2024-08-17T18:02:54.025 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.025 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.025 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:54.025 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625748+0000", 2024-08-17T18:02:54.025 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001535766 2024-08-17T18:02:54.025 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.025 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.025 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:54.025 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.626167+0000", 2024-08-17T18:02:54.025 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000418739 2024-08-17T18:02:54.025 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.026 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.026 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:54.026 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016776+0000", 2024-08-17T18:02:54.026 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.390609015 2024-08-17T18:02:54.026 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.026 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.026 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.026 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016843+0000", 2024-08-17T18:02:54.026 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6847999999999994e-05 2024-08-17T18:02:54.026 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.026 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.027 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.027 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016886+0000", 2024-08-17T18:02:54.027 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3358000000000001e-05 2024-08-17T18:02:54.027 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.027 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.027 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.027 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.027 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.027 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:30116 46.e 46:74110d7a:::benchmark_data_smithi130_267720_object30115:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:02:54.027 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.625972+0000", 2024-08-17T18:02:54.027 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 185.377860341, 2024-08-17T18:02:54.027 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.39189922500000002, 2024-08-17T18:02:54.027 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.028 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.028 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.028 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:54.028 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:54.028 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:54.028 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 30116 2024-08-17T18:02:54.028 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.028 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.028 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.028 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.028 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625972+0000", 2024-08-17T18:02:54.028 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.028 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.029 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.029 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.029 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625972+0000", 2024-08-17T18:02:54.029 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.029 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.029 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.029 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.029 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625974+0000", 2024-08-17T18:02:54.029 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3549999999999999e-06 2024-08-17T18:02:54.029 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.029 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.029 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.029 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625997+0000", 2024-08-17T18:02:54.030 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3280000000000001e-05 2024-08-17T18:02:54.030 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.030 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.030 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.030 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.626009+0000", 2024-08-17T18:02:54.030 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.216e-05 2024-08-17T18:02:54.030 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.030 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.030 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.030 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.626017+0000", 2024-08-17T18:02:54.030 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5100000000000001e-06 2024-08-17T18:02:54.030 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.030 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.031 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.031 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.626392+0000", 2024-08-17T18:02:54.031 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00037497099999999998 2024-08-17T18:02:54.031 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.031 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.031 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.031 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.626503+0000", 2024-08-17T18:02:54.031 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011165199999999999 2024-08-17T18:02:54.031 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.031 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.031 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:02:54.031 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.626942+0000", 2024-08-17T18:02:54.031 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00043872300000000002 2024-08-17T18:02:54.031 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.032 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.032 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:54.032 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.628850+0000", 2024-08-17T18:02:54.032 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0019078159999999999 2024-08-17T18:02:54.032 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.032 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.032 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:54.032 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629148+0000", 2024-08-17T18:02:54.032 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00029787500000000002 2024-08-17T18:02:54.032 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.033 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.033 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:54.033 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.017771+0000", 2024-08-17T18:02:54.033 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38862370899999998 2024-08-17T18:02:54.033 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.033 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.033 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.033 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.017833+0000", 2024-08-17T18:02:54.033 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1492e-05 2024-08-17T18:02:54.033 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.034 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.034 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.034 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.017871+0000", 2024-08-17T18:02:54.034 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8682e-05 2024-08-17T18:02:54.034 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.034 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.034 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.034 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.034 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.034 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:30128 46.e 46:7b0313e1:::benchmark_data_smithi130_267720_object30127:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:02:54.034 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.632335+0000", 2024-08-17T18:02:54.034 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 185.371497869, 2024-08-17T18:02:54.034 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38563995499999998, 2024-08-17T18:02:54.035 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.035 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.035 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.035 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:54.035 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:54.035 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:54.035 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 30128 2024-08-17T18:02:54.035 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.035 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.035 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.035 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.036 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632335+0000", 2024-08-17T18:02:54.036 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.036 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.036 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.036 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.036 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632335+0000", 2024-08-17T18:02:54.036 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.036 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.036 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.036 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.036 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632336+0000", 2024-08-17T18:02:54.036 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.688e-06 2024-08-17T18:02:54.036 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.036 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.037 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.037 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632371+0000", 2024-08-17T18:02:54.037 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4100999999999998e-05 2024-08-17T18:02:54.037 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.037 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.037 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.037 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632386+0000", 2024-08-17T18:02:54.037 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5548000000000001e-05 2024-08-17T18:02:54.037 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.037 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.037 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.037 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632394+0000", 2024-08-17T18:02:54.037 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1189999999999997e-06 2024-08-17T18:02:54.038 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.038 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.038 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.038 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632459+0000", 2024-08-17T18:02:54.038 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4819000000000004e-05 2024-08-17T18:02:54.038 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.038 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.038 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.038 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632597+0000", 2024-08-17T18:02:54.038 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013788400000000001 2024-08-17T18:02:54.038 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.038 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.038 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:02:54.039 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632738+0000", 2024-08-17T18:02:54.039 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014124600000000001 2024-08-17T18:02:54.039 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.039 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.039 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:54.039 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.634000+0000", 2024-08-17T18:02:54.039 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001261947 2024-08-17T18:02:54.039 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.039 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.039 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:54.039 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.635856+0000", 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0018556499999999999 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.017906+0000", 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38204977400000001 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.017945+0000", 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9536999999999998e-05 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.040 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.017975+0000", 2024-08-17T18:02:54.041 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9641999999999998e-05 2024-08-17T18:02:54.041 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.041 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.041 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.041 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.041 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.041 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:30133 46.e 46:708f0a37:::benchmark_data_smithi130_267720_object30132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:02:54.041 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.634356+0000", 2024-08-17T18:02:54.041 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 185.36947666899999, 2024-08-17T18:02:54.041 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38373346400000002, 2024-08-17T18:02:54.041 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.041 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.041 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.041 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:02:54.042 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:02:54.042 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:02:54.042 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 30133 2024-08-17T18:02:54.042 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.042 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.042 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.042 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.042 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.634356+0000", 2024-08-17T18:02:54.043 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.043 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.043 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.043 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.043 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.634356+0000", 2024-08-17T18:02:54.043 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.043 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.043 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.043 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.043 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.634357+0000", 2024-08-17T18:02:54.043 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6000000000000002e-07 2024-08-17T18:02:54.043 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.043 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.044 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.044 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.634380+0000", 2024-08-17T18:02:54.044 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3348000000000001e-05 2024-08-17T18:02:54.044 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.044 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.044 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.044 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.634392+0000", 2024-08-17T18:02:54.044 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1909000000000001e-05 2024-08-17T18:02:54.044 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.044 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.044 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.044 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.634400+0000", 2024-08-17T18:02:54.044 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6179999999999997e-06 2024-08-17T18:02:54.044 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.045 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.045 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.045 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.634478+0000", 2024-08-17T18:02:54.045 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8617999999999998e-05 2024-08-17T18:02:54.045 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.045 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.045 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.045 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.634595+0000", 2024-08-17T18:02:54.045 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011684999999999999 2024-08-17T18:02:54.045 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.045 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.045 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:02:54.045 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.634734+0000", 2024-08-17T18:02:54.045 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000139152 2024-08-17T18:02:54.046 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.046 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.046 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:54.046 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.635917+0000", 2024-08-17T18:02:54.046 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0011829880000000001 2024-08-17T18:02:54.046 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.046 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.046 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:02:54.046 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.635972+0000", 2024-08-17T18:02:54.046 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4493000000000002e-05 2024-08-17T18:02:54.046 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.046 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.046 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:02:54.046 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.018023+0000", 2024-08-17T18:02:54.047 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38205109500000001 2024-08-17T18:02:54.047 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.047 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.047 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.047 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.018062+0000", 2024-08-17T18:02:54.047 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8899000000000001e-05 2024-08-17T18:02:54.047 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.047 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.047 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.047 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.018089+0000", 2024-08-17T18:02:54.047 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.7634e-05 2024-08-17T18:02:54.047 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.047 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.048 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.048 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.048 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.048 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:67152 46.a e1265/1258 46:567c055a:::benchmark_data_smithi130_267720_object67151:head v 1265'4213, mlcod=1265'4208)", 2024-08-17T18:02:54.048 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:18.363920+0000", 2024-08-17T18:02:54.048 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 155.63991283999999, 2024-08-17T18:02:54.048 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26646195, 2024-08-17T18:02:54.048 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.048 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.048 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.049 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.049 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.049 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.049 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.363920+0000", 2024-08-17T18:02:54.049 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.049 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.049 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.049 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.049 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.363920+0000", 2024-08-17T18:02:54.049 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.049 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.049 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.049 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.050 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.363921+0000", 2024-08-17T18:02:54.050 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4700000000000001e-07 2024-08-17T18:02:54.050 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.050 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.050 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.050 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364233+0000", 2024-08-17T18:02:54.050 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00031249899999999999 2024-08-17T18:02:54.050 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.050 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.050 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.050 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364243+0000", 2024-08-17T18:02:54.050 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.978e-06 2024-08-17T18:02:54.050 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.050 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.051 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.051 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364251+0000", 2024-08-17T18:02:54.051 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5950000000000001e-06 2024-08-17T18:02:54.051 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.051 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.051 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.051 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364319+0000", 2024-08-17T18:02:54.051 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8659999999999997e-05 2024-08-17T18:02:54.051 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.051 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.051 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.051 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364337+0000", 2024-08-17T18:02:54.051 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8213e-05 2024-08-17T18:02:54.052 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.052 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.052 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.052 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.630335+0000", 2024-08-17T18:02:54.052 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.265997223 2024-08-17T18:02:54.052 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.052 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.052 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.052 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.630382+0000", 2024-08-17T18:02:54.052 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7034999999999999e-05 2024-08-17T18:02:54.052 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.052 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.052 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.053 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.053 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.053 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:67151 46.0 e1265/1263 46:0c84d484:::benchmark_data_smithi130_267720_object67150:head v 1265'4140, mlcod=1265'4138)", 2024-08-17T18:02:54.053 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:18.364875+0000", 2024-08-17T18:02:54.053 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 155.63895744800001, 2024-08-17T18:02:54.053 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.266023552, 2024-08-17T18:02:54.053 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.053 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.053 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.053 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.053 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.053 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.053 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364875+0000", 2024-08-17T18:02:54.053 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.054 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.054 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.054 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.054 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364875+0000", 2024-08-17T18:02:54.054 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.054 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.054 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.054 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.054 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364876+0000", 2024-08-17T18:02:54.054 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7199999999999998e-07 2024-08-17T18:02:54.054 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.054 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.055 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.055 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.364897+0000", 2024-08-17T18:02:54.055 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0987000000000001e-05 2024-08-17T18:02:54.055 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.055 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.055 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.055 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.622025+0000", 2024-08-17T18:02:54.055 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.25712805399999999 2024-08-17T18:02:54.055 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.055 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.055 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.055 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.622045+0000", 2024-08-17T18:02:54.055 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0163e-05 2024-08-17T18:02:54.055 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.056 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.056 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.056 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.622119+0000", 2024-08-17T18:02:54.056 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3719000000000003e-05 2024-08-17T18:02:54.056 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.056 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.056 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.056 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.622145+0000", 2024-08-17T18:02:54.056 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5814999999999999e-05 2024-08-17T18:02:54.056 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.056 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.056 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.056 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.630290+0000", 2024-08-17T18:02:54.057 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0081449699999999996 2024-08-17T18:02:54.057 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.057 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.057 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.057 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:18.630899+0000", 2024-08-17T18:02:54.057 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00060917199999999995 2024-08-17T18:02:54.057 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.057 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.057 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.057 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.057 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.057 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:79514 46.a e1267/1258 46:572757ec:::benchmark_data_smithi130_267720_object79513:head v 1267'4980, mlcod=1267'4976)", 2024-08-17T18:02:54.057 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.620383+0000", 2024-08-17T18:02:54.058 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 143.383449955, 2024-08-17T18:02:54.058 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.39699816799999998, 2024-08-17T18:02:54.058 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.058 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.058 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.058 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.058 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.058 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.058 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.620383+0000", 2024-08-17T18:02:54.058 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.059 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.059 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.059 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.059 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.620383+0000", 2024-08-17T18:02:54.059 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.059 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.059 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.059 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.059 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.620383+0000", 2024-08-17T18:02:54.059 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8199999999999999e-07 2024-08-17T18:02:54.059 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.059 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.059 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.060 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.620413+0000", 2024-08-17T18:02:54.060 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9492000000000002e-05 2024-08-17T18:02:54.060 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.060 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.060 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.060 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.620423+0000", 2024-08-17T18:02:54.060 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0241000000000001e-05 2024-08-17T18:02:54.060 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.060 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.060 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.060 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.620430+0000", 2024-08-17T18:02:54.060 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9949999999999999e-06 2024-08-17T18:02:54.060 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.061 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.061 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.061 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.620506+0000", 2024-08-17T18:02:54.061 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6260999999999994e-05 2024-08-17T18:02:54.061 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.061 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.061 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.061 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.620524+0000", 2024-08-17T18:02:54.061 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7241e-05 2024-08-17T18:02:54.061 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.061 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.061 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.061 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.642416+0000", 2024-08-17T18:02:54.061 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021892028000000001 2024-08-17T18:02:54.062 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.062 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.062 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.063 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.017381+0000", 2024-08-17T18:02:54.063 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37496522799999998 2024-08-17T18:02:54.063 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.063 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.063 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.064 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.064 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.064 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:79518 46.2 e1267/1258 46:404c947b:::benchmark_data_smithi130_267720_object79517:head v 1267'5023, mlcod=1267'5021)", 2024-08-17T18:02:54.064 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.627482+0000", 2024-08-17T18:02:54.064 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 143.376350954, 2024-08-17T18:02:54.065 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.39025481699999998, 2024-08-17T18:02:54.065 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.065 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.065 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.065 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.065 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.066 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.066 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627482+0000", 2024-08-17T18:02:54.066 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.066 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.066 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.066 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.066 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627482+0000", 2024-08-17T18:02:54.066 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.067 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.067 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.067 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.067 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627482+0000", 2024-08-17T18:02:54.067 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5999999999999999e-07 2024-08-17T18:02:54.067 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.067 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.067 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.067 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627507+0000", 2024-08-17T18:02:54.067 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4508999999999999e-05 2024-08-17T18:02:54.067 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.067 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.068 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.068 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627514+0000", 2024-08-17T18:02:54.068 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1790000000000002e-06 2024-08-17T18:02:54.068 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.068 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.068 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.068 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627518+0000", 2024-08-17T18:02:54.068 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5619999999999997e-06 2024-08-17T18:02:54.068 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.068 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.068 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.068 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627871+0000", 2024-08-17T18:02:54.068 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00035256300000000002 2024-08-17T18:02:54.068 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.069 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.069 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.069 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627896+0000", 2024-08-17T18:02:54.069 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5113000000000002e-05 2024-08-17T18:02:54.069 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.069 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.069 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.070 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.017698+0000", 2024-08-17T18:02:54.070 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38980219500000002 2024-08-17T18:02:54.070 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.070 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.070 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.070 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.017736+0000", 2024-08-17T18:02:54.070 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8336e-05 2024-08-17T18:02:54.070 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.070 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.070 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.070 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.070 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.070 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:79519 46.2 e1267/1258 46:4d9ed74c:::benchmark_data_smithi130_267720_object79518:head v 1267'5024, mlcod=1267'5021)", 2024-08-17T18:02:54.071 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.627533+0000", 2024-08-17T18:02:54.071 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 143.37629938399999, 2024-08-17T18:02:54.071 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.39581428699999999, 2024-08-17T18:02:54.071 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.071 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.071 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.071 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.071 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.071 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.071 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627533+0000", 2024-08-17T18:02:54.071 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.071 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.071 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.072 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.072 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627533+0000", 2024-08-17T18:02:54.072 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.072 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.072 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.072 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.072 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627534+0000", 2024-08-17T18:02:54.072 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1e-07 2024-08-17T18:02:54.072 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.072 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.072 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.072 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627552+0000", 2024-08-17T18:02:54.072 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8811e-05 2024-08-17T18:02:54.072 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.073 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.073 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.073 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627558+0000", 2024-08-17T18:02:54.073 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8869999999999997e-06 2024-08-17T18:02:54.073 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.073 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.073 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.073 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627562+0000", 2024-08-17T18:02:54.073 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9589999999999997e-06 2024-08-17T18:02:54.073 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.073 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.073 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.073 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.635989+0000", 2024-08-17T18:02:54.073 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0084265590000000001 2024-08-17T18:02:54.074 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.074 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.074 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.074 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636007+0000", 2024-08-17T18:02:54.074 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7915999999999999e-05 2024-08-17T18:02:54.074 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.074 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.074 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.074 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023166+0000", 2024-08-17T18:02:54.074 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38715940199999999 2024-08-17T18:02:54.074 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.074 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.074 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.075 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023348+0000", 2024-08-17T18:02:54.075 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018144299999999999 2024-08-17T18:02:54.075 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.075 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.075 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.075 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.075 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.075 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:79523 46.a e1267/1258 46:590eeb35:::benchmark_data_smithi130_267720_object79522:head v 1267'4981, mlcod=1267'4978)", 2024-08-17T18:02:54.075 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.632210+0000", 2024-08-17T18:02:54.075 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 143.37162306900001, 2024-08-17T18:02:54.075 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38836737100000002, 2024-08-17T18:02:54.075 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.075 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.076 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.076 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.076 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.076 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.076 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632210+0000", 2024-08-17T18:02:54.076 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.076 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.076 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.076 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.076 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632210+0000", 2024-08-17T18:02:54.076 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.076 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.076 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.077 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.077 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632210+0000", 2024-08-17T18:02:54.077 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7300000000000005e-07 2024-08-17T18:02:54.077 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.077 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.077 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.077 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632242+0000", 2024-08-17T18:02:54.077 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2095999999999999e-05 2024-08-17T18:02:54.077 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.077 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.077 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.077 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632266+0000", 2024-08-17T18:02:54.077 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3901999999999999e-05 2024-08-17T18:02:54.077 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.078 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.078 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.078 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632273+0000", 2024-08-17T18:02:54.078 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0269999999999998e-06 2024-08-17T18:02:54.078 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.078 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.078 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.078 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.633777+0000", 2024-08-17T18:02:54.078 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0015034390000000001 2024-08-17T18:02:54.078 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.078 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.078 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.079 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.633794+0000", 2024-08-17T18:02:54.079 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6818000000000001e-05 2024-08-17T18:02:54.079 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.079 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.079 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.079 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020530+0000", 2024-08-17T18:02:54.079 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38673647300000003 2024-08-17T18:02:54.079 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.079 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.079 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.079 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020577+0000", 2024-08-17T18:02:54.079 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6842999999999999e-05 2024-08-17T18:02:54.079 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.080 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.080 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.080 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.080 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.080 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:79526 46.a e1267/1258 46:540ef56b:::benchmark_data_smithi130_267720_object79525:head v 1267'4982, mlcod=1267'4978)", 2024-08-17T18:02:54.080 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.634106+0000", 2024-08-17T18:02:54.080 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 143.369726821, 2024-08-17T18:02:54.080 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38650425900000002, 2024-08-17T18:02:54.080 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.080 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.081 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.081 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.081 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.081 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.081 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.634106+0000", 2024-08-17T18:02:54.081 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.081 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.081 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.081 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.081 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.634106+0000", 2024-08-17T18:02:54.081 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.081 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.081 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.082 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.082 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.634106+0000", 2024-08-17T18:02:54.082 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2099999999999996e-07 2024-08-17T18:02:54.082 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.082 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.082 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.082 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.634176+0000", 2024-08-17T18:02:54.082 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9218e-05 2024-08-17T18:02:54.082 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.082 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.083 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.083 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.634188+0000", 2024-08-17T18:02:54.083 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1891e-05 2024-08-17T18:02:54.083 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.083 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.083 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.083 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.634194+0000", 2024-08-17T18:02:54.083 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0469999999999999e-06 2024-08-17T18:02:54.083 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.083 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.083 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.083 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636429+0000", 2024-08-17T18:02:54.083 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0022350759999999999 2024-08-17T18:02:54.083 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.084 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.084 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.084 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636448+0000", 2024-08-17T18:02:54.084 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9673999999999998e-05 2024-08-17T18:02:54.084 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.084 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.084 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.084 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020581+0000", 2024-08-17T18:02:54.084 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38413211600000002 2024-08-17T18:02:54.084 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.084 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.084 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.085 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:54.086 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020610+0000", 2024-08-17T18:02:54.086 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9516e-05 2024-08-17T18:02:54.086 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.086 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.086 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.086 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.086 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.086 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:79528 46.e e1267/1263 46:716b67ef:::benchmark_data_smithi130_267720_object79527:head v 1267'4936, mlcod=1267'4934)", 2024-08-17T18:02:54.087 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.636073+0000", 2024-08-17T18:02:54.087 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 143.36776002799999, 2024-08-17T18:02:54.087 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38137969599999999, 2024-08-17T18:02:54.087 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.087 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.087 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.087 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.087 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.087 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.087 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636073+0000", 2024-08-17T18:02:54.087 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.087 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.087 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.087 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.088 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636073+0000", 2024-08-17T18:02:54.088 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.088 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.088 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.088 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.088 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636074+0000", 2024-08-17T18:02:54.088 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8400000000000002e-07 2024-08-17T18:02:54.088 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.088 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.088 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.088 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636187+0000", 2024-08-17T18:02:54.088 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000113349 2024-08-17T18:02:54.088 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.089 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.089 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.089 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636206+0000", 2024-08-17T18:02:54.089 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8889000000000002e-05 2024-08-17T18:02:54.089 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.089 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.089 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.089 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636214+0000", 2024-08-17T18:02:54.089 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2989999999999997e-06 2024-08-17T18:02:54.089 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.089 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.089 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.089 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636854+0000", 2024-08-17T18:02:54.090 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00063973100000000005 2024-08-17T18:02:54.090 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.090 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.090 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.090 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636871+0000", 2024-08-17T18:02:54.090 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.668e-05 2024-08-17T18:02:54.090 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.090 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.090 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.090 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.643780+0000", 2024-08-17T18:02:54.090 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0069096089999999997 2024-08-17T18:02:54.090 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.091 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.091 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.091 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.017452+0000", 2024-08-17T18:02:54.091 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37367215500000001 2024-08-17T18:02:54.091 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.091 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.091 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.091 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.091 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.091 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:79529 46.a e1267/1258 46:5d7561ca:::benchmark_data_smithi130_267720_object79528:head v 1267'4983, mlcod=1267'4978)", 2024-08-17T18:02:54.091 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.636750+0000", 2024-08-17T18:02:54.091 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 143.36708258100001, 2024-08-17T18:02:54.091 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38668297699999998, 2024-08-17T18:02:54.091 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.092 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.092 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.092 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.092 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.092 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.092 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636750+0000", 2024-08-17T18:02:54.092 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.092 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.092 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.092 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.092 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636750+0000", 2024-08-17T18:02:54.092 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.092 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.093 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.093 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.093 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636750+0000", 2024-08-17T18:02:54.093 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1199999999999998e-07 2024-08-17T18:02:54.093 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.093 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.093 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.093 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636778+0000", 2024-08-17T18:02:54.093 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.7987000000000002e-05 2024-08-17T18:02:54.093 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.093 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.094 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.094 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636787+0000", 2024-08-17T18:02:54.094 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5629999999999992e-06 2024-08-17T18:02:54.094 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.094 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.094 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.094 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636793+0000", 2024-08-17T18:02:54.094 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.879e-06 2024-08-17T18:02:54.095 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.095 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.095 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.095 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639058+0000", 2024-08-17T18:02:54.095 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0022647520000000001 2024-08-17T18:02:54.095 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.095 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.095 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.095 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639076+0000", 2024-08-17T18:02:54.095 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8022000000000001e-05 2024-08-17T18:02:54.095 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.095 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023352+0000", 2024-08-17T18:02:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38427597000000002 2024-08-17T18:02:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023433+0000", 2024-08-17T18:02:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1391999999999996e-05 2024-08-17T18:02:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.096 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:79507 46.b e1267/1258 46:d8bdbef4:::benchmark_data_smithi130_267720_object79506:head v 1267'4907, mlcod=1267'4903)", 2024-08-17T18:02:54.097 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.639439+0000", 2024-08-17T18:02:54.097 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 143.364393105, 2024-08-17T18:02:54.097 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37803722299999998, 2024-08-17T18:02:54.097 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.097 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.097 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.097 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.097 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.097 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.097 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639439+0000", 2024-08-17T18:02:54.097 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.097 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.097 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.098 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.098 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639439+0000", 2024-08-17T18:02:54.098 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.098 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.098 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.098 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.098 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639440+0000", 2024-08-17T18:02:54.098 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7100000000000011e-07 2024-08-17T18:02:54.098 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.098 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.098 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.098 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639467+0000", 2024-08-17T18:02:54.099 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6650000000000001e-05 2024-08-17T18:02:54.099 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.099 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.099 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.099 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639478+0000", 2024-08-17T18:02:54.099 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1071e-05 2024-08-17T18:02:54.099 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.099 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.099 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.099 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639485+0000", 2024-08-17T18:02:54.099 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5270000000000004e-06 2024-08-17T18:02:54.099 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.099 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.100 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.100 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639567+0000", 2024-08-17T18:02:54.100 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1939000000000003e-05 2024-08-17T18:02:54.100 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.100 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.100 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.100 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639587+0000", 2024-08-17T18:02:54.100 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0078999999999999e-05 2024-08-17T18:02:54.100 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.100 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.100 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.100 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.017430+0000", 2024-08-17T18:02:54.100 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37784301999999997 2024-08-17T18:02:54.100 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.101 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.101 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.101 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.017477+0000", 2024-08-17T18:02:54.101 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6965999999999997e-05 2024-08-17T18:02:54.101 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.101 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.101 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.101 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.101 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.101 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:79527 46.7 e1267/1263 46:eed33f3f:::benchmark_data_smithi130_267720_object79526:head v 1267'5033, mlcod=1267'5030)", 2024-08-17T18:02:54.101 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.639652+0000", 2024-08-17T18:02:54.101 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 143.36418106900001, 2024-08-17T18:02:54.102 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38078736800000001, 2024-08-17T18:02:54.102 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.102 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.102 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.102 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.102 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.102 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.102 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639652+0000", 2024-08-17T18:02:54.102 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.102 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.102 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.102 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.102 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639652+0000", 2024-08-17T18:02:54.103 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.103 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.103 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.103 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.103 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639652+0000", 2024-08-17T18:02:54.103 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5799999999999998e-07 2024-08-17T18:02:54.103 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.103 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.103 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.104 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639678+0000", 2024-08-17T18:02:54.104 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5213999999999999e-05 2024-08-17T18:02:54.104 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.104 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.104 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.104 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639688+0000", 2024-08-17T18:02:54.104 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0736e-05 2024-08-17T18:02:54.104 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.104 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.104 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.105 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.639694+0000", 2024-08-17T18:02:54.105 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.049e-06 2024-08-17T18:02:54.105 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.105 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.105 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.105 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.642322+0000", 2024-08-17T18:02:54.105 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026275130000000002 2024-08-17T18:02:54.106 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.106 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.106 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.106 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.642340+0000", 2024-08-17T18:02:54.106 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8207999999999999e-05 2024-08-17T18:02:54.106 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.106 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.106 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.106 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020389+0000", 2024-08-17T18:02:54.106 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37804870000000002 2024-08-17T18:02:54.106 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.106 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.106 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.107 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020439+0000", 2024-08-17T18:02:54.107 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0189999999999999e-05 2024-08-17T18:02:54.107 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.107 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.107 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.107 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.107 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.107 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:79531 46.2 e1267/1258 46:4c0c978d:::benchmark_data_smithi130_267720_object79530:head v 1267'5025, mlcod=1267'5021)", 2024-08-17T18:02:54.107 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.641964+0000", 2024-08-17T18:02:54.107 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 143.36186831099999, 2024-08-17T18:02:54.107 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38155197200000002, 2024-08-17T18:02:54.107 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.107 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.108 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.108 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.108 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.108 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.108 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.641964+0000", 2024-08-17T18:02:54.108 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.108 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.108 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.108 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.108 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.641964+0000", 2024-08-17T18:02:54.109 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.109 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.109 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.109 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.109 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.641965+0000", 2024-08-17T18:02:54.109 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0599999999999999e-07 2024-08-17T18:02:54.109 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.109 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.109 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.109 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.641988+0000", 2024-08-17T18:02:54.109 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2603999999999999e-05 2024-08-17T18:02:54.109 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.109 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.109 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.110 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.641999+0000", 2024-08-17T18:02:54.110 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1106e-05 2024-08-17T18:02:54.110 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.110 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.110 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.110 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.642006+0000", 2024-08-17T18:02:54.110 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.221e-06 2024-08-17T18:02:54.110 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.110 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.110 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.110 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.642072+0000", 2024-08-17T18:02:54.110 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6194999999999999e-05 2024-08-17T18:02:54.111 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.111 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.111 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.111 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.642090+0000", 2024-08-17T18:02:54.111 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8071999999999999e-05 2024-08-17T18:02:54.111 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.111 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.111 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.111 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023436+0000", 2024-08-17T18:02:54.111 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.381345991 2024-08-17T18:02:54.111 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.111 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.111 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.112 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023516+0000", 2024-08-17T18:02:54.112 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9876999999999994e-05 2024-08-17T18:02:54.112 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.112 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.112 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.112 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.112 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.112 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:79534 46.a e1267/1258 46:5daa6ba5:::benchmark_data_smithi130_267720_object79533:head v 1267'4984, mlcod=1267'4978)", 2024-08-17T18:02:54.112 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.644123+0000", 2024-08-17T18:02:54.112 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 143.35970957800001, 2024-08-17T18:02:54.112 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37952068700000002, 2024-08-17T18:02:54.112 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.112 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.112 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.113 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.113 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.113 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.113 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644123+0000", 2024-08-17T18:02:54.113 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.113 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.113 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.113 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.113 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644123+0000", 2024-08-17T18:02:54.113 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.113 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.113 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.113 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.114 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644124+0000", 2024-08-17T18:02:54.114 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0100000000000003e-07 2024-08-17T18:02:54.114 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.114 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.114 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.114 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644214+0000", 2024-08-17T18:02:54.114 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0373999999999999e-05 2024-08-17T18:02:54.114 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.114 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.114 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.114 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.017470+0000", 2024-08-17T18:02:54.114 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.373255638 2024-08-17T18:02:54.115 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.115 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.115 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.115 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.017482+0000", 2024-08-17T18:02:54.115 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1697e-05 2024-08-17T18:02:54.115 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.115 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.115 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.115 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.017720+0000", 2024-08-17T18:02:54.115 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00023869200000000001 2024-08-17T18:02:54.115 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.115 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.017739+0000", 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8343999999999999e-05 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023521+0000", 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0057820320000000003 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023644+0000", 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012300899999999999 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.116 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.117 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.117 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.117 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.117 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:79536 46.0 e1267/1263 46:078355fd:::benchmark_data_smithi130_267720_object79535:head v 1267'4946, mlcod=1267'4944)", 2024-08-17T18:02:54.117 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.644805+0000", 2024-08-17T18:02:54.117 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 143.35902770800001, 2024-08-17T18:02:54.117 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37603124199999999, 2024-08-17T18:02:54.117 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:02:54.117 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:02:54.117 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:02:54.117 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:02:54.117 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.117 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:02:54.118 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644805+0000", 2024-08-17T18:02:54.118 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.118 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.118 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.118 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:02:54.118 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644805+0000", 2024-08-17T18:02:54.118 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:02:54.118 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.118 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.118 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:02:54.118 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644806+0000", 2024-08-17T18:02:54.118 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3200000000000004e-07 2024-08-17T18:02:54.118 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:02:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644856+0000", 2024-08-17T18:02:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0102000000000002e-05 2024-08-17T18:02:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:02:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.017697+0000", 2024-08-17T18:02:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37284087199999999 2024-08-17T18:02:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:02:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.017705+0000", 2024-08-17T18:02:54.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1799999999999996e-06 2024-08-17T18:02:54.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:02:54.120 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.017782+0000", 2024-08-17T18:02:54.120 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7305000000000006e-05 2024-08-17T18:02:54.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:02:54.120 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.017811+0000", 2024-08-17T18:02:54.120 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8742999999999999e-05 2024-08-17T18:02:54.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:02:54.120 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020784+0000", 2024-08-17T18:02:54.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029732209999999999 2024-08-17T18:02:54.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:02:54.121 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:02:54.121 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:02:54.121 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020836+0000", 2024-08-17T18:02:54.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2086999999999998e-05 2024-08-17T18:02:54.121 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.121 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.121 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.121 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:02:54.121 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:02:54.121 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:02:54.122 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:53 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:02:54.122 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:53 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T18:02:54.122 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:53 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:02:54.122 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:53 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2024-08-17T18:02:54.122 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:53 smithi179 ceph-mon[93519]: osdmap e1283: 8 total, 8 up, 5 in 2024-08-17T18:02:54.146 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:54.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:54.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:54.248 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:54.273 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:54.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:54.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:54.375 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:54.400 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:54.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:54.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:54.502 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:54.528 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:54.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:54.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:54.629 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:54.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:54.652 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:54.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:54.753 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:54.779 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:54.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:54.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:54.881 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:54.912 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:54.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:54.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:55.014 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:55.040 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:55.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:55.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:54 smithi179 ceph-mon[93519]: pgmap v2132: 89 pgs: 4 unknown, 26 peering, 59 active+clean; 1.3 GiB data, 5.3 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 764 op/s 2024-08-17T18:02:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:54 smithi179 ceph-mon[93519]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T18:02:55.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:54 smithi179 ceph-mon[93519]: osdmap e1284: 8 total, 8 up, 5 in 2024-08-17T18:02:55.143 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:55.165 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:55.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:55.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:55.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:54 smithi130 ceph-mon[105521]: pgmap v2132: 89 pgs: 4 unknown, 26 peering, 59 active+clean; 1.3 GiB data, 5.3 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 764 op/s 2024-08-17T18:02:55.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:54 smithi130 ceph-mon[105521]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T18:02:55.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:54 smithi130 ceph-mon[105521]: osdmap e1284: 8 total, 8 up, 5 in 2024-08-17T18:02:55.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:54 smithi130 ceph-mon[110068]: pgmap v2132: 89 pgs: 4 unknown, 26 peering, 59 active+clean; 1.3 GiB data, 5.3 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 764 op/s 2024-08-17T18:02:55.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:54 smithi130 ceph-mon[110068]: Health check failed: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T18:02:55.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:54 smithi130 ceph-mon[110068]: osdmap e1284: 8 total, 8 up, 5 in 2024-08-17T18:02:55.266 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:55.287 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:55.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:55.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:55.389 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:55.429 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:55.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:55.531 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:55.554 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:55.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:55.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:55.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:55.680 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:55.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:55.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:55.782 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:55.806 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:55.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:55.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:55.908 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:02:55.944 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:55.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:55.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:02:56.047 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:56.073 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:56.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:56.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:56.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[105521]: osdmap e1285: 8 total, 8 up, 5 in 2024-08-17T18:02:56.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[105521]: 46.6 starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12155 2024-08-17T18:02:56.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[105521]: 46.c starting backfill to osd.1 from (0'0,0'0] MAX to 1282'12259 2024-08-17T18:02:56.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[105521]: 46.1 starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12268 2024-08-17T18:02:56.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[105521]: 46.2 starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12309 2024-08-17T18:02:56.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[105521]: 46.f starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12395 2024-08-17T18:02:56.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[105521]: 46.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1282'12028 2024-08-17T18:02:56.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[105521]: 46.5 starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12237 2024-08-17T18:02:56.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[105521]: osdmap e1286: 8 total, 8 up, 5 in 2024-08-17T18:02:56.145 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[110068]: osdmap e1285: 8 total, 8 up, 5 in 2024-08-17T18:02:56.145 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[110068]: 46.6 starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12155 2024-08-17T18:02:56.145 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[110068]: 46.c starting backfill to osd.1 from (0'0,0'0] MAX to 1282'12259 2024-08-17T18:02:56.145 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[110068]: 46.1 starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12268 2024-08-17T18:02:56.145 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[110068]: 46.2 starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12309 2024-08-17T18:02:56.145 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[110068]: 46.f starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12395 2024-08-17T18:02:56.145 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[110068]: 46.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1282'12028 2024-08-17T18:02:56.145 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[110068]: 46.5 starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12237 2024-08-17T18:02:56.145 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:55 smithi130 ceph-mon[110068]: osdmap e1286: 8 total, 8 up, 5 in 2024-08-17T18:02:56.175 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:56.211 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:56.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:56.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:56.313 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:56.338 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:56.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:56.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:56.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:55 smithi179 ceph-mon[93519]: osdmap e1285: 8 total, 8 up, 5 in 2024-08-17T18:02:56.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:55 smithi179 ceph-mon[93519]: 46.6 starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12155 2024-08-17T18:02:56.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:55 smithi179 ceph-mon[93519]: 46.c starting backfill to osd.1 from (0'0,0'0] MAX to 1282'12259 2024-08-17T18:02:56.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:55 smithi179 ceph-mon[93519]: 46.1 starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12268 2024-08-17T18:02:56.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:55 smithi179 ceph-mon[93519]: 46.2 starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12309 2024-08-17T18:02:56.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:55 smithi179 ceph-mon[93519]: 46.f starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12395 2024-08-17T18:02:56.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:55 smithi179 ceph-mon[93519]: 46.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1282'12028 2024-08-17T18:02:56.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:55 smithi179 ceph-mon[93519]: 46.5 starting backfill to osd.7 from (0'0,0'0] MAX to 1282'12237 2024-08-17T18:02:56.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:55 smithi179 ceph-mon[93519]: osdmap e1286: 8 total, 8 up, 5 in 2024-08-17T18:02:56.440 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:56.465 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:56.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:56.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:56.566 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:56.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:56.612 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:56.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:56.714 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:56.738 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:56.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:56.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:56.840 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:56.883 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:56.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:56.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:56.985 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:57.024 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:57.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:57.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:57.126 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:57.157 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:57.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:57.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:02:57.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:56 smithi130 ceph-mon[105521]: pgmap v2135: 89 pgs: 4 unknown, 26 peering, 59 active+clean; 1.3 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 660 op/s 2024-08-17T18:02:57.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:56 smithi130 ceph-mon[105521]: osdmap e1287: 8 total, 8 up, 5 in 2024-08-17T18:02:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:56 smithi130 ceph-mon[110068]: pgmap v2135: 89 pgs: 4 unknown, 26 peering, 59 active+clean; 1.3 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 660 op/s 2024-08-17T18:02:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:56 smithi130 ceph-mon[110068]: osdmap e1287: 8 total, 8 up, 5 in 2024-08-17T18:02:57.259 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:57.294 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:57.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:57.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:56 smithi179 ceph-mon[93519]: pgmap v2135: 89 pgs: 4 unknown, 26 peering, 59 active+clean; 1.3 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 660 op/s 2024-08-17T18:02:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:56 smithi179 ceph-mon[93519]: osdmap e1287: 8 total, 8 up, 5 in 2024-08-17T18:02:57.396 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:02:57.420 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:57.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:57.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:02:57.522 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:57.565 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:57.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:57.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:57.667 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:57.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:57.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:57.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:57.712 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T18:02:57.793 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:57.854 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:57.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:57.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:57.956 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:57.979 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:57.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:57.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:58.081 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:58.149 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:58.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:58.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:58.250 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:58.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:58.276 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:58.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:58.377 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:58.420 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:58.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:58.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:58.522 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:58.540 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:58 smithi130 ceph-mon[105521]: pgmap v2138: 89 pgs: 4 unknown, 26 peering, 59 active+clean; 1.3 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 100 op/s; 4 B/s, 0 objects/s recovering 2024-08-17T18:02:58.540 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:58.540 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:02:58.540 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:58 smithi130 ceph-mon[110068]: pgmap v2138: 89 pgs: 4 unknown, 26 peering, 59 active+clean; 1.3 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 100 op/s; 4 B/s, 0 objects/s recovering 2024-08-17T18:02:58.540 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:58.540 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:02:58.545 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:58.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:58.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:58 smithi179 ceph-mon[93519]: pgmap v2138: 89 pgs: 4 unknown, 26 peering, 59 active+clean; 1.3 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 100 op/s; 4 B/s, 0 objects/s recovering 2024-08-17T18:02:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:02:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:02:58.632 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:02:58.647 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:58.669 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:58.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:58.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:58.714 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:02:58.771 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:02:58.793 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:02:58.793 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:02:58.836 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:58.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:58.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:02:58.838 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:02:58.838 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:02:58.885 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:02:58.885 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:02:58.939 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:58.946 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:02:58.946 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:02:58.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:58.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:58.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:58.978 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:02:58.978 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:02:59.007 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:02:59.008 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:02:59.039 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:02:59.039 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:02:59.075 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:02:59.075 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:02:59.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:02:59.101 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:59.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:59.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:02:59.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:02:59.227 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:59.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:59.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:02:59.329 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:59.370 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:59.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:59.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:59.472 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:02:59.496 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:02:59.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:59.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:02:59.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:02:59 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4008237313' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:02:59.598 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:59.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:59.637 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:59.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:59.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:02:59 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4008237313' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:02:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:02:59 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4008237313' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:02:59.738 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:02:59.802 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:59.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:59.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:02:59.904 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:02:59.944 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:02:59.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:02:59.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:00.046 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:00.070 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:00.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:00.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:00.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:02:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:02:59] "GET /metrics HTTP/1.1" 200 36780 "" "Prometheus/2.51.0" 2024-08-17T18:03:00.172 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:00.206 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:00.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:00.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:00.308 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:00.332 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:00.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:00.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:00.433 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:00.451 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:00.456 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:00.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:00.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:00.557 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:03:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:03:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:03:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:00 smithi179 ceph-mon[93519]: pgmap v2139: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 1.1 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 454 op/s; 7612/55833 objects misplaced (13.634%); 5.1 MiB/s, 1 keys/s, 83 objects/s recovering 2024-08-17T18:03:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:00 smithi179 ceph-mon[93519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-08-17T18:03:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:00 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T18:03:00.602 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:00.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:00.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:00.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:00 smithi130 ceph-mon[105521]: pgmap v2139: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 1.1 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 454 op/s; 7612/55833 objects misplaced (13.634%); 5.1 MiB/s, 1 keys/s, 83 objects/s recovering 2024-08-17T18:03:00.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:00 smithi130 ceph-mon[105521]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-08-17T18:03:00.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:00 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T18:03:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:00 smithi130 ceph-mon[110068]: pgmap v2139: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 1.1 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 454 op/s; 7612/55833 objects misplaced (13.634%); 5.1 MiB/s, 1 keys/s, 83 objects/s recovering 2024-08-17T18:03:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:00 smithi130 ceph-mon[110068]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2024-08-17T18:03:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:00 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T18:03:00.704 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:00.739 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:00.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:00.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:00.840 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:00.874 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:00.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:00.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:00.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:01.031 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:01.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:01.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:01.133 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:01.158 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:01.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:01.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:01.260 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:01.303 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:01.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:01.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:01.405 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:01.445 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:01.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:01.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:01.547 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:01.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:01.590 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:01.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:01.691 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:01.714 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:01.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:01.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:01.816 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:01.841 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:01.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:01.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:01.943 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:01.967 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:01.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:01.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:02.069 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:02.120 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:02.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:02.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:02.222 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:02.261 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:02.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:02.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:02.363 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:02.386 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:02.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:02.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:02.408 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:03:02.412 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 7] out_osds: [5, 4, 6] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T18:03:02.412 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T18:03:02.488 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:02.512 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:02.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:02.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:02.613 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:02.657 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:02 smithi130 ceph-mon[105521]: pgmap v2140: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 1.1 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 382 op/s; 7612/55833 objects misplaced (13.634%); 4.3 MiB/s, 1 keys/s, 70 objects/s recovering 2024-08-17T18:03:02.657 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:02 smithi130 ceph-mon[110068]: pgmap v2140: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 1.1 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 382 op/s; 7612/55833 objects misplaced (13.634%); 4.3 MiB/s, 1 keys/s, 70 objects/s recovering 2024-08-17T18:03:02.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:02.671 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:02.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:02.772 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:02.794 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:02.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:02.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:02.812 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:03:02.812 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:03:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:02 smithi179 ceph-mon[93519]: pgmap v2140: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 82 active+clean; 1.1 GiB data, 4.9 GiB used, 442 GiB / 447 GiB avail; 0 B/s wr, 382 op/s; 7612/55833 objects misplaced (13.634%); 4.3 MiB/s, 1 keys/s, 70 objects/s recovering 2024-08-17T18:03:02.896 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:02.921 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:02.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:02.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:03.022 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:03.048 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:03.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:03.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:03.150 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:03.176 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:03.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:03.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:03.278 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:03.302 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:03.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:03.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:03.404 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:03.427 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:03.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:03.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:03.529 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:03.551 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:03.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:03.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:03.653 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:03.674 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:03.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:03.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:03.776 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:03.797 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:03.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:03.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:03.884 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T18:03:03.898 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:03.920 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:03.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:03.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:04.021 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:04.042 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:04.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:04.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:04.144 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:04.167 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:04.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:04.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:04.199 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:04.200 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:04.202 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:04.268 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:04.290 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:04.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:04.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:04.572 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:03:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:04 smithi130 ceph-mon[105521]: pgmap v2141: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 888 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 851 op/s; 5192/42699 objects misplaced (12.160%); 8.5 MiB/s, 1 keys/s, 137 objects/s recovering 2024-08-17T18:03:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:04 smithi130 ceph-mon[105521]: osdmap e1288: 8 total, 8 up, 5 in 2024-08-17T18:03:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:04 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/17078343' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:03:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:04 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:03:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:04 smithi130 ceph-mon[110068]: pgmap v2141: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 888 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 851 op/s; 5192/42699 objects misplaced (12.160%); 8.5 MiB/s, 1 keys/s, 137 objects/s recovering 2024-08-17T18:03:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:04 smithi130 ceph-mon[110068]: osdmap e1288: 8 total, 8 up, 5 in 2024-08-17T18:03:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:04 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/17078343' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:03:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:04 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:03:04.673 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:04.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:04.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:04.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:04.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:04.841 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:04.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:04.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:04 smithi179 ceph-mon[93519]: pgmap v2141: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 888 MiB data, 4.4 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 851 op/s; 5192/42699 objects misplaced (12.160%); 8.5 MiB/s, 1 keys/s, 137 objects/s recovering 2024-08-17T18:03:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:04 smithi179 ceph-mon[93519]: osdmap e1288: 8 total, 8 up, 5 in 2024-08-17T18:03:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:04 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/17078343' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:03:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:04 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:03:04.944 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:04.967 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:04.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:04.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:05.069 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:05.107 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:05.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:05.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:05.209 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:05.231 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:05.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:05.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:05.333 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:05.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:05.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:05.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:05.457 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:05.482 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:05.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:05.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:05.584 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:05.619 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:05.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:05.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:05 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:03:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:05 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:03:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:05 smithi130 ceph-mon[105521]: osdmap e1289: 8 total, 8 up, 5 in 2024-08-17T18:03:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:05 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:03:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:05 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:03:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:05 smithi130 ceph-mon[110068]: osdmap e1289: 8 total, 8 up, 5 in 2024-08-17T18:03:05.721 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:05.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:05.762 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:05.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:05 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:03:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:05 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:03:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:05 smithi179 ceph-mon[93519]: osdmap e1289: 8 total, 8 up, 5 in 2024-08-17T18:03:05.862 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:05.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:05.903 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:05.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:06.004 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:06.026 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:06.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:06.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:06.128 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:06.149 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:06.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:06.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:06.251 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:06.272 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:06.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:06.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:06.374 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:06.395 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:06.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:06.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:06.497 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:06.518 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:06.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:06.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:06.620 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:06.641 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:06.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:06.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:06.743 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:06.764 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:06.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:06.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:06.866 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:06.887 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:06.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:06.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:06.989 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:07.012 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:07.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:07.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:07.114 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:07.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:07.149 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:07.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:07.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:06 smithi130 ceph-mon[105521]: pgmap v2144: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 888 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 800 op/s; 5192/42699 objects misplaced (12.160%); 8.5 MiB/s, 1 keys/s, 137 objects/s recovering 2024-08-17T18:03:07.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:06 smithi130 ceph-mon[105521]: osdmap e1290: 8 total, 8 up, 5 in 2024-08-17T18:03:07.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:06 smithi130 ceph-mon[110068]: pgmap v2144: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 888 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 800 op/s; 5192/42699 objects misplaced (12.160%); 8.5 MiB/s, 1 keys/s, 137 objects/s recovering 2024-08-17T18:03:07.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:06 smithi130 ceph-mon[110068]: osdmap e1290: 8 total, 8 up, 5 in 2024-08-17T18:03:07.251 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:07.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:07.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:07.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:06 smithi179 ceph-mon[93519]: pgmap v2144: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 888 MiB data, 4.2 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 800 op/s; 5192/42699 objects misplaced (12.160%); 8.5 MiB/s, 1 keys/s, 137 objects/s recovering 2024-08-17T18:03:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:06 smithi179 ceph-mon[93519]: osdmap e1290: 8 total, 8 up, 5 in 2024-08-17T18:03:07.376 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:07.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:07.406 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:07.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:07.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:07.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:07.555 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:07.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:07.656 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:07.695 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:07.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:07.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:07.798 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:07.819 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:07.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:07.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:07.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:07 smithi130 ceph-mon[105521]: osdmap e1291: 8 total, 8 up, 5 in 2024-08-17T18:03:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:07 smithi130 ceph-mon[110068]: osdmap e1291: 8 total, 8 up, 5 in 2024-08-17T18:03:07.921 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:07.945 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:07.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:07.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:08.047 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:08.072 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:08.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:08.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:08.174 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:08.207 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:08.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:08.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:08.309 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:08.330 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:08.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:08.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:08.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:07 smithi179 ceph-mon[93519]: osdmap e1291: 8 total, 8 up, 5 in 2024-08-17T18:03:08.432 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:08.454 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:08.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:08.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:08.556 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:08.579 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:08.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:08.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:08.681 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:08.705 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:08.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:08.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:08.807 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:08.828 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:08.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:08.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:08.930 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:08.956 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:08.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:08.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:09.058 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:09.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:09.082 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:09.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:09.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:09.205 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:09.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:09.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:09.307 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:09.329 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:09.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:09.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:08 smithi179 ceph-mon[93519]: pgmap v2147: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 856 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 131 op/s; 5192/41124 objects misplaced (12.625%) 2024-08-17T18:03:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:09.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:08 smithi130 ceph-mon[105521]: pgmap v2147: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 856 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 131 op/s; 5192/41124 objects misplaced (12.625%) 2024-08-17T18:03:09.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:08 smithi130 ceph-mon[110068]: pgmap v2147: 89 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 856 MiB data, 4.1 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 131 op/s; 5192/41124 objects misplaced (12.625%) 2024-08-17T18:03:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:09.434 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:09.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:09.467 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:09.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:09.569 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:09.591 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:09.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:09.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:09.693 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:09.716 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:09.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:09.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:09.817 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:09.845 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:09.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:09.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:09.948 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:09.969 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:09.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:09.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:10.073 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:10.098 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:10.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:10.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:10.125 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:10 smithi130 ceph-mon[110068]: osdmap e1292: 8 total, 8 up, 5 in 2024-08-17T18:03:10.125 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:03:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:03:09] "GET /metrics HTTP/1.1" 200 36780 "" "Prometheus/2.51.0" 2024-08-17T18:03:10.127 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:10 smithi130 ceph-mon[105521]: osdmap e1292: 8 total, 8 up, 5 in 2024-08-17T18:03:10.200 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:10.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:10.241 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:10.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:10.342 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:10.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:10 smithi179 ceph-mon[93519]: osdmap e1292: 8 total, 8 up, 5 in 2024-08-17T18:03:10.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:03:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:03:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:03:10.365 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:10.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:10.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:10.467 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:10.488 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:10.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:10.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:10.590 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:10.631 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:10.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:10.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:10.732 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:10.778 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:10.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:10.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:10.880 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:10.903 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:10.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:10.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:11.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:11.027 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:11.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:11.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:11.129 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:11.151 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:11.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:11.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:11.253 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:11.271 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:11 smithi130 ceph-mon[105521]: pgmap v2149: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 713 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 2830/34257 objects misplaced (8.261%); 19 MiB/s, 311 objects/s recovering 2024-08-17T18:03:11.272 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:11 smithi130 ceph-mon[105521]: osdmap e1293: 8 total, 8 up, 5 in 2024-08-17T18:03:11.272 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:11 smithi130 ceph-mon[110068]: pgmap v2149: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 713 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 2830/34257 objects misplaced (8.261%); 19 MiB/s, 311 objects/s recovering 2024-08-17T18:03:11.273 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:11 smithi130 ceph-mon[110068]: osdmap e1293: 8 total, 8 up, 5 in 2024-08-17T18:03:11.292 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:11.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:11.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:11 smithi179 ceph-mon[93519]: pgmap v2149: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 713 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 578 op/s; 2830/34257 objects misplaced (8.261%); 19 MiB/s, 311 objects/s recovering 2024-08-17T18:03:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:11 smithi179 ceph-mon[93519]: osdmap e1293: 8 total, 8 up, 5 in 2024-08-17T18:03:11.394 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:11.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:11.433 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:11.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:11.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:11.576 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:11.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:11.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:11.678 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:11.711 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:11.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:11.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:11.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:11.834 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:11.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:11.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:11.935 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:11.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:11.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:11.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:12.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:12.100 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:12.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:12.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:12.167 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:03:12.202 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:12.261 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:12.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:12.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:12.363 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:12.385 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:12.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:12.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:12.487 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:12.509 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:12.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:12.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:12.610 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:12.632 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:12.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:12.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:12.734 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:12.756 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:12.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:12.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:12.857 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:12.882 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:12.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:12.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:12.984 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:13.006 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:13.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:13.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:13.108 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:13.130 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:13.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:13.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:13.232 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:13.253 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:13.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:13.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:13.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:13 smithi130 ceph-mon[105521]: pgmap v2151: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 713 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 522 op/s; 2830/34257 objects misplaced (8.261%); 18 MiB/s, 281 objects/s recovering 2024-08-17T18:03:13.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:13 smithi130 ceph-mon[105521]: osdmap e1294: 8 total, 8 up, 5 in 2024-08-17T18:03:13.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:13.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:03:13.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:13 smithi130 ceph-mon[110068]: pgmap v2151: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 713 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 522 op/s; 2830/34257 objects misplaced (8.261%); 18 MiB/s, 281 objects/s recovering 2024-08-17T18:03:13.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:13 smithi130 ceph-mon[110068]: osdmap e1294: 8 total, 8 up, 5 in 2024-08-17T18:03:13.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:13.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:03:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:13 smithi179 ceph-mon[93519]: pgmap v2151: 89 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 84 active+clean; 713 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 522 op/s; 2830/34257 objects misplaced (8.261%); 18 MiB/s, 281 objects/s recovering 2024-08-17T18:03:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:13 smithi179 ceph-mon[93519]: osdmap e1294: 8 total, 8 up, 5 in 2024-08-17T18:03:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:03:13.356 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:13.377 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:13.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:13.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:13.479 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:13.500 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:13.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:13.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:13.602 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:13.624 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:13.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:13.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:13.725 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:13.748 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:13.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:13.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:13.850 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:13.874 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:13.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:13.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:13.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:14.000 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:14.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:14.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:14.102 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:14.136 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:14.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:14.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:14.238 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:14.266 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:14.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:14.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:14.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:14 smithi179 ceph-mon[93519]: osdmap e1295: 8 total, 8 up, 5 in 2024-08-17T18:03:14.367 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:14.388 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:14.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:14.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:14.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:14 smithi130 ceph-mon[105521]: osdmap e1295: 8 total, 8 up, 5 in 2024-08-17T18:03:14.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:14 smithi130 ceph-mon[110068]: osdmap e1295: 8 total, 8 up, 5 in 2024-08-17T18:03:14.491 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:14.512 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:14.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:14.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:14.613 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:14.635 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:14.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:14.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:14.737 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:14.760 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:14.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:14.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:14.861 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:14.883 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:14.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:14.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:14.985 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:15.008 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:15.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:15.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:15.110 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:15.135 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:15.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:15.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:15.237 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:15.282 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:15.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:15.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:15 smithi179 ceph-mon[93519]: pgmap v2154: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 529 MiB data, 3.5 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 695 op/s; 487/25434 objects misplaced (1.915%); 27 MiB/s, 428 objects/s recovering 2024-08-17T18:03:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:15 smithi179 ceph-mon[93519]: osdmap e1296: 8 total, 8 up, 5 in 2024-08-17T18:03:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:15 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]} : dispatch 2024-08-17T18:03:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:15 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]} : dispatch 2024-08-17T18:03:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:15 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 7]} : dispatch 2024-08-17T18:03:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:15 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]} : dispatch 2024-08-17T18:03:15.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:15 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]} : dispatch 2024-08-17T18:03:15.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:15 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]} : dispatch 2024-08-17T18:03:15.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:15 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]} : dispatch 2024-08-17T18:03:15.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:15 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 7]} : dispatch 2024-08-17T18:03:15.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:15 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 7]} : dispatch 2024-08-17T18:03:15.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:15 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]} : dispatch 2024-08-17T18:03:15.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:15 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]} : dispatch 2024-08-17T18:03:15.345 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:15 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 7]} : dispatch 2024-08-17T18:03:15.366 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[110068]: pgmap v2154: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 529 MiB data, 3.5 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 695 op/s; 487/25434 objects misplaced (1.915%); 27 MiB/s, 428 objects/s recovering 2024-08-17T18:03:15.366 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[110068]: osdmap e1296: 8 total, 8 up, 5 in 2024-08-17T18:03:15.366 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]} : dispatch 2024-08-17T18:03:15.367 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]} : dispatch 2024-08-17T18:03:15.367 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 7]} : dispatch 2024-08-17T18:03:15.367 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]} : dispatch 2024-08-17T18:03:15.367 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]} : dispatch 2024-08-17T18:03:15.367 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]} : dispatch 2024-08-17T18:03:15.367 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]} : dispatch 2024-08-17T18:03:15.367 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 7]} : dispatch 2024-08-17T18:03:15.368 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 7]} : dispatch 2024-08-17T18:03:15.368 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]} : dispatch 2024-08-17T18:03:15.368 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]} : dispatch 2024-08-17T18:03:15.368 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 7]} : dispatch 2024-08-17T18:03:15.368 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[105521]: pgmap v2154: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 529 MiB data, 3.5 GiB used, 443 GiB / 447 GiB avail; 0 B/s wr, 695 op/s; 487/25434 objects misplaced (1.915%); 27 MiB/s, 428 objects/s recovering 2024-08-17T18:03:15.368 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[105521]: osdmap e1296: 8 total, 8 up, 5 in 2024-08-17T18:03:15.368 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]} : dispatch 2024-08-17T18:03:15.369 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]} : dispatch 2024-08-17T18:03:15.369 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 7]} : dispatch 2024-08-17T18:03:15.369 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]} : dispatch 2024-08-17T18:03:15.369 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]} : dispatch 2024-08-17T18:03:15.369 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]} : dispatch 2024-08-17T18:03:15.369 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]} : dispatch 2024-08-17T18:03:15.369 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 7]} : dispatch 2024-08-17T18:03:15.369 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 7]} : dispatch 2024-08-17T18:03:15.369 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]} : dispatch 2024-08-17T18:03:15.369 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]} : dispatch 2024-08-17T18:03:15.369 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:15 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 7]} : dispatch 2024-08-17T18:03:15.385 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:15.417 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:15.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:15.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:15.519 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:15.540 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:15.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:15.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:15.641 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:15.662 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:15.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:15.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:15.764 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:15.785 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:15.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:15.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:15.887 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:15.910 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:15.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:15.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:16.012 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:16.034 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:16.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:16.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:16.136 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:16.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:16.179 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:16.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:16.280 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:16.320 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]': finished 2024-08-17T18:03:16.320 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]}]': finished 2024-08-17T18:03:16.320 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 7]}]': finished 2024-08-17T18:03:16.320 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2024-08-17T18:03:16.320 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2024-08-17T18:03:16.321 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 7]}]': finished 2024-08-17T18:03:16.321 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[105521]: osdmap e1297: 8 total, 8 up, 5 in 2024-08-17T18:03:16.321 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[105521]: osdmap e1298: 8 total, 8 up, 5 in 2024-08-17T18:03:16.322 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]': finished 2024-08-17T18:03:16.322 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]}]': finished 2024-08-17T18:03:16.322 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 7]}]': finished 2024-08-17T18:03:16.322 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2024-08-17T18:03:16.322 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2024-08-17T18:03:16.322 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 7]}]': finished 2024-08-17T18:03:16.322 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[110068]: osdmap e1297: 8 total, 8 up, 5 in 2024-08-17T18:03:16.322 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:16 smithi130 ceph-mon[110068]: osdmap e1298: 8 total, 8 up, 5 in 2024-08-17T18:03:16.334 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:16.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:16.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:16.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [0, 2]}]': finished 2024-08-17T18:03:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.c", "id": [3, 2]}]': finished 2024-08-17T18:03:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 7]}]': finished 2024-08-17T18:03:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2024-08-17T18:03:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2024-08-17T18:03:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 7]}]': finished 2024-08-17T18:03:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:16 smithi179 ceph-mon[93519]: osdmap e1297: 8 total, 8 up, 5 in 2024-08-17T18:03:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:16 smithi179 ceph-mon[93519]: osdmap e1298: 8 total, 8 up, 5 in 2024-08-17T18:03:16.437 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:16.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:16.477 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:16.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:16.578 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:16.612 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:16.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:16.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:16.715 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:16.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:16.737 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:16.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:16.838 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:16.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:16.906 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:16.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:17.007 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:17.029 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:17.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:17.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:17.131 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:17.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:17.175 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:17.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:17.276 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:17.316 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:17.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:17.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:17.323 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:17 smithi130 ceph-mon[105521]: pgmap v2157: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 529 MiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 734 op/s; 487/25434 objects misplaced (1.915%); 28 MiB/s, 452 objects/s recovering 2024-08-17T18:03:17.324 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:03:17.324 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:03:17.324 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:03:17.324 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:03:17.324 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:03:17.324 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:03:17.325 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:03:17.325 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:03:17.325 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:17 smithi130 ceph-mon[110068]: pgmap v2157: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 529 MiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 734 op/s; 487/25434 objects misplaced (1.915%); 28 MiB/s, 452 objects/s recovering 2024-08-17T18:03:17.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:17 smithi179 ceph-mon[93519]: pgmap v2157: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 529 MiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 734 op/s; 487/25434 objects misplaced (1.915%); 28 MiB/s, 452 objects/s recovering 2024-08-17T18:03:17.358 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:03:17.358 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:03:17.385 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:03:17.385 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:03:17.415 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:03:17.415 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:03:17.418 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:17.440 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:17.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:17.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:17.451 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:03:17.451 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:03:17.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:17.582 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:17.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:17.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:17.684 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:17.720 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:17.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:17.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:17.821 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:17.861 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:17.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:17.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:17.963 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:17.985 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:17.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:17.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:18.087 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:18.124 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:18.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:18.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:18.225 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:18.244 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T18:03:18.269 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:18.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:18.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:18.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2701694427' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:03:18.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4225957114' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:03:18.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:18 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:03:18.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:18 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:18.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:18.392 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:18.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:18.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2701694427' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:03:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4225957114' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:03:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:18 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:03:18.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:18 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2701694427' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:03:18.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4225957114' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:03:18.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:18 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:03:18.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:18 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:18.494 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:18.517 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:18.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:18.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:18.618 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:18.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:18.644 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:18.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:18.745 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:18.767 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:18.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:18.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:18.869 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:18.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:18.892 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:18.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:18.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:19.033 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:19.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:19.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:19.135 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:19.200 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:19.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:19.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:19.302 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:19.325 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:19.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:19.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:19 smithi179 ceph-mon[93519]: pgmap v2159: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 505 MiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 512 op/s; 487/24312 objects misplaced (2.003%); 17 MiB/s, 271 objects/s recovering 2024-08-17T18:03:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:19 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:03:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:19 smithi179 ceph-mon[93519]: osdmap e1299: 8 total, 8 up, 5 in 2024-08-17T18:03:19.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:19 smithi130 ceph-mon[105521]: pgmap v2159: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 505 MiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 512 op/s; 487/24312 objects misplaced (2.003%); 17 MiB/s, 271 objects/s recovering 2024-08-17T18:03:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:19 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:03:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:19 smithi130 ceph-mon[105521]: osdmap e1299: 8 total, 8 up, 5 in 2024-08-17T18:03:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:19 smithi130 ceph-mon[110068]: pgmap v2159: 89 pgs: 1 active+remapped+backfilling, 2 active+clean+remapped, 86 active+clean; 505 MiB data, 3.4 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 512 op/s; 487/24312 objects misplaced (2.003%); 17 MiB/s, 271 objects/s recovering 2024-08-17T18:03:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:19 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:03:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:19 smithi130 ceph-mon[110068]: osdmap e1299: 8 total, 8 up, 5 in 2024-08-17T18:03:19.426 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:19.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:19.451 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:19.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:19.552 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:19.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:19.589 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:19.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:19.690 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:19.736 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:19.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:19.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:19.748 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:03:19.748 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T18:03:19.748 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T18:03:19.748 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T18:03:19.748 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:03:19.838 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:19.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:19.863 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:19.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:19.964 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:20.007 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:20.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:20.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:20.109 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:20.131 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:20.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:20.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:03:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:03:19] "GET /metrics HTTP/1.1" 200 36801 "" "Prometheus/2.51.0" 2024-08-17T18:03:20.183 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:20.225 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T18:03:20.232 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:20.254 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:20.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:20.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:20.356 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:20.378 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:20.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:20.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:20.480 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:20.503 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:20.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:20.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:03:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:03:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:03:20.605 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:20.655 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:20.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:20.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:20.757 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:20.779 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:20.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:20.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:20.881 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:20.920 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:20.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:20.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:21.022 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:21.058 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:21.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:21.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:21.139 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T18:03:21.160 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:21.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:21.226 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:21.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:21.327 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:21 smithi179 ceph-mon[93519]: pgmap v2161: 89 pgs: 89 active+clean; 388 MiB data, 3.1 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 433 op/s; 5.7 MiB/s, 91 objects/s recovering 2024-08-17T18:03:21.349 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:21.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:21.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:21 smithi130 ceph-mon[105521]: pgmap v2161: 89 pgs: 89 active+clean; 388 MiB data, 3.1 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 433 op/s; 5.7 MiB/s, 91 objects/s recovering 2024-08-17T18:03:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:21 smithi130 ceph-mon[110068]: pgmap v2161: 89 pgs: 89 active+clean; 388 MiB data, 3.1 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 433 op/s; 5.7 MiB/s, 91 objects/s recovering 2024-08-17T18:03:21.451 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:21.522 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:03:21.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:21.531 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:21.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:21.632 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:21.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:21.655 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:21.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:21.756 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:21.790 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:21.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:21.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:21.891 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:21.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:21.919 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:21.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:22.020 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:22.041 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:22.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:22.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:22.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:22.175 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:22.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:22.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:22.277 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:22.316 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:22.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:22.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:22.419 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:22.440 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:22.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:22.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:22.542 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:22.564 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:22.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:22.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:22.666 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:22.688 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:22.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:22.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:22.790 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:22.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:22.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:22.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:22.914 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:22.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:22.956 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:22.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:23.057 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:23.081 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:23.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:23.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:23.183 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:23.205 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:23.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:23.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:23.307 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:23.337 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:23.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:23.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:23.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:23 smithi179 ceph-mon[93519]: pgmap v2162: 89 pgs: 89 active+clean; 388 MiB data, 3.1 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 363 op/s; 4.8 MiB/s, 77 objects/s recovering 2024-08-17T18:03:23.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:23 smithi130 ceph-mon[105521]: pgmap v2162: 89 pgs: 89 active+clean; 388 MiB data, 3.1 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 363 op/s; 4.8 MiB/s, 77 objects/s recovering 2024-08-17T18:03:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:23 smithi130 ceph-mon[110068]: pgmap v2162: 89 pgs: 89 active+clean; 388 MiB data, 3.1 GiB used, 444 GiB / 447 GiB avail; 0 B/s wr, 363 op/s; 4.8 MiB/s, 77 objects/s recovering 2024-08-17T18:03:23.439 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:23.482 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:23.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:23.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:23.584 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:23.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:23.607 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:23.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:23.708 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:23.746 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:23.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:23.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:23.848 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:23.870 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:23.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:23.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:23.972 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:24.005 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:24.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:24.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:24.091 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:24.107 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:24.152 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:24.159 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:24.200 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:24.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:24.301 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:24.324 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:24.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:24.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:24.426 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:24.447 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:24.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:24.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:24.549 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:24.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:24.596 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:24.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:24.697 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:24.741 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:24.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:24.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:24.842 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:24.861 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Removed 107980 objects 2024-08-17T18:03:24.861 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Clean up completed and total clean up time :140.39 2024-08-17T18:03:24.871 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:24.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:24.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:24.973 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:24.995 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:24.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:24.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:25.097 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:25.119 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:25.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:25.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:25.221 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:25.242 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:25.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:25.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:25.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:25 smithi179 ceph-mon[93519]: pgmap v2163: 89 pgs: 89 active+clean; 108 MiB data, 2.5 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 841 op/s; 3.7 MiB/s, 59 objects/s recovering 2024-08-17T18:03:25.343 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:25.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:25.365 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:25.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:25.368 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:25 smithi130 ceph-mon[105521]: pgmap v2163: 89 pgs: 89 active+clean; 108 MiB data, 2.5 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 841 op/s; 3.7 MiB/s, 59 objects/s recovering 2024-08-17T18:03:25.368 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:25 smithi130 ceph-mon[110068]: pgmap v2163: 89 pgs: 89 active+clean; 108 MiB data, 2.5 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 841 op/s; 3.7 MiB/s, 59 objects/s recovering 2024-08-17T18:03:25.466 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:25.488 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:25.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:25.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:25.590 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:25.611 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:25.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:25.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:25.713 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:25.736 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:25.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:25.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:25.838 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:25.862 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:25.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:25.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:25.964 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:25.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:25.986 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:25.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:26.087 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:26.108 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:26.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:26.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:26.210 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:26.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:26.233 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:26.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:26.247 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-08-17T18:03:26.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-08-17T18:03:26.334 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:26.360 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:26.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:26.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:26.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:26.484 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:26.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:26.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:26.586 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:26.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:26.610 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:26.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:26.711 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:26.736 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:26.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:26.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:26.838 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:26.861 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:26.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:26.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:26.962 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:26.988 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:26.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:26.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:27.090 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:27.113 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:27.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:27.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:27.214 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:27.235 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:27.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:27.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:27.337 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:27.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:27 smithi179 ceph-mon[93519]: pgmap v2164: 89 pgs: 89 active+clean; 108 MiB data, 2.3 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 732 op/s; 3.3 MiB/s, 52 objects/s recovering 2024-08-17T18:03:27.363 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:27.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:27.364 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:27.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:27 smithi130 ceph-mon[105521]: pgmap v2164: 89 pgs: 89 active+clean; 108 MiB data, 2.3 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 732 op/s; 3.3 MiB/s, 52 objects/s recovering 2024-08-17T18:03:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:27 smithi130 ceph-mon[110068]: pgmap v2164: 89 pgs: 89 active+clean; 108 MiB data, 2.3 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 732 op/s; 3.3 MiB/s, 52 objects/s recovering 2024-08-17T18:03:27.464 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:27.488 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:27.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:27.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:27.590 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:27.614 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:27.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:27.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:27.716 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:27.741 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:27.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:27.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:27.843 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:27.869 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:27.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:27.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:27.970 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:27.993 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:27.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:27.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:28.095 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:28.118 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:28.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:28.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:28.219 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:28.242 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:28.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:28.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:03:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:28.343 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:28.364 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:28.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:28.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:28.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:03:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:28.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:03:28.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:28.466 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:28.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:28.492 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:28.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:28.593 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:28.619 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:28.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:28.720 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:28.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:28.742 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:28.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:28.843 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:28.869 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:28.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:28.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:28.970 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:28.993 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:28.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:28.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:29.095 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:29.100 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:03:29.119 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:29.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:29.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:29.221 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:29.225 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:03:29.225 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:03:29.225 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:03:29.225 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:03:29.225 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.225 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30108 46.6 e1262/1258 46:61535681:::benchmark_data_smithi130_267720_object30107:head v 1262'1902, mlcod=1262'1900)", 2024-08-17T18:03:29.225 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.622400+0000", 2024-08-17T18:03:29.225 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 220.04646491099999, 2024-08-17T18:03:29.225 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.39093171999999998, 2024-08-17T18:03:29.225 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.226 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.226 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.226 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.226 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.226 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.226 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622400+0000", 2024-08-17T18:03:29.226 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.226 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.226 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.226 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.226 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622400+0000", 2024-08-17T18:03:29.226 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.226 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.227 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.227 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622400+0000", 2024-08-17T18:03:29.227 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9100000000000003e-07 2024-08-17T18:03:29.227 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.227 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.227 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622436+0000", 2024-08-17T18:03:29.227 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5828999999999999e-05 2024-08-17T18:03:29.227 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.227 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.227 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622446+0000", 2024-08-17T18:03:29.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4800000000000007e-06 2024-08-17T18:03:29.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.228 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.228 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.228 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622452+0000", 2024-08-17T18:03:29.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0700000000000003e-06 2024-08-17T18:03:29.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.228 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.228 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.228 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622736+0000", 2024-08-17T18:03:29.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00028467899999999998 2024-08-17T18:03:29.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.622753+0000", 2024-08-17T18:03:29.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6099e-05 2024-08-17T18:03:29.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.628849+0000", 2024-08-17T18:03:29.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0060961069999999999 2024-08-17T18:03:29.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.230 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013331+0000", 2024-08-17T18:03:29.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.384482765 2024-08-17T18:03:29.230 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.230 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.230 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.230 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30097 46.9 e1262/1258 46:9fdfcdfb:::benchmark_data_smithi130_267720_object30096:head v 1262'1885, mlcod=1262'1880)", 2024-08-17T18:03:29.230 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.623801+0000", 2024-08-17T18:03:29.230 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 220.04506334600001, 2024-08-17T18:03:29.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.389256359, 2024-08-17T18:03:29.230 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.231 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.231 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.231 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623801+0000", 2024-08-17T18:03:29.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623801+0000", 2024-08-17T18:03:29.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.232 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.232 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623802+0000", 2024-08-17T18:03:29.232 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4000000000000002e-07 2024-08-17T18:03:29.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.232 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.232 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623824+0000", 2024-08-17T18:03:29.232 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1685e-05 2024-08-17T18:03:29.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.232 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.232 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623833+0000", 2024-08-17T18:03:29.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3940000000000002e-06 2024-08-17T18:03:29.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.233 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.233 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623840+0000", 2024-08-17T18:03:29.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0169999999999999e-06 2024-08-17T18:03:29.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.233 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.233 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.624711+0000", 2024-08-17T18:03:29.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00087102700000000002 2024-08-17T18:03:29.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.233 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.624729+0000", 2024-08-17T18:03:29.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8536999999999999e-05 2024-08-17T18:03:29.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013015+0000", 2024-08-17T18:03:29.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38828545599999997 2024-08-17T18:03:29.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013058+0000", 2024-08-17T18:03:29.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2703000000000001e-05 2024-08-17T18:03:29.234 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.234 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.235 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.235 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.235 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30110 46.e e1262/1258 46:7442b1f6:::benchmark_data_smithi130_267720_object30109:head v 1262'1868, mlcod=1262'1866)", 2024-08-17T18:03:29.235 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.625942+0000", 2024-08-17T18:03:29.235 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 220.042922522, 2024-08-17T18:03:29.235 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38829237100000003, 2024-08-17T18:03:29.235 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.235 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.235 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.235 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.235 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.235 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625942+0000", 2024-08-17T18:03:29.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625942+0000", 2024-08-17T18:03:29.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625942+0000", 2024-08-17T18:03:29.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3500000000000002e-07 2024-08-17T18:03:29.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625969+0000", 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6052000000000001e-05 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625976+0000", 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4019999999999996e-06 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.625981+0000", 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4489999999999998e-06 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.626277+0000", 2024-08-17T18:03:29.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00029525500000000002 2024-08-17T18:03:29.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.626293+0000", 2024-08-17T18:03:29.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6812000000000001e-05 2024-08-17T18:03:29.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.014152+0000", 2024-08-17T18:03:29.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38785813499999999 2024-08-17T18:03:29.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.014234+0000", 2024-08-17T18:03:29.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2930999999999996e-05 2024-08-17T18:03:29.239 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.239 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.239 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.239 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30111 46.8 e1262/1258 46:189f8eaf:::benchmark_data_smithi130_267720_object30110:head v 1262'1896, mlcod=1262'1894)", 2024-08-17T18:03:29.239 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.627403+0000", 2024-08-17T18:03:29.239 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 220.04146193700001, 2024-08-17T18:03:29.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38543158300000002, 2024-08-17T18:03:29.240 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.240 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.240 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.240 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.240 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.240 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.240 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627403+0000", 2024-08-17T18:03:29.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.240 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.240 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.240 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.241 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627403+0000", 2024-08-17T18:03:29.241 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.241 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.241 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.241 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.241 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627403+0000", 2024-08-17T18:03:29.241 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1900000000000002e-07 2024-08-17T18:03:29.241 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.241 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.241 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.241 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627430+0000", 2024-08-17T18:03:29.241 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6347999999999999e-05 2024-08-17T18:03:29.241 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.241 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627439+0000", 2024-08-17T18:03:29.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0710000000000001e-06 2024-08-17T18:03:29.242 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.242 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627445+0000", 2024-08-17T18:03:29.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8850000000000004e-06 2024-08-17T18:03:29.242 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.242 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627583+0000", 2024-08-17T18:03:29.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000138002 2024-08-17T18:03:29.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627602+0000", 2024-08-17T18:03:29.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8913e-05 2024-08-17T18:03:29.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012774+0000", 2024-08-17T18:03:29.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38517220000000002 2024-08-17T18:03:29.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012834+0000", 2024-08-17T18:03:29.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0445000000000001e-05 2024-08-17T18:03:29.244 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.244 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.244 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.244 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30106 46.9 e1262/1258 46:95064b20:::benchmark_data_smithi130_267720_object30105:head v 1262'1886, mlcod=1262'1883)", 2024-08-17T18:03:29.244 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.627466+0000", 2024-08-17T18:03:29.244 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 220.04139842800001, 2024-08-17T18:03:29.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38562097299999998, 2024-08-17T18:03:29.244 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.244 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.244 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627466+0000", 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627466+0000", 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627467+0000", 2024-08-17T18:03:29.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3700000000000001e-07 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627489+0000", 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2473999999999999e-05 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627497+0000", 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4699999999999996e-06 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627502+0000", 2024-08-17T18:03:29.246 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0579999999999997e-06 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627874+0000", 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00037268400000000003 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627892+0000", 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7900000000000001e-05 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013063+0000", 2024-08-17T18:03:29.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38517096099999998 2024-08-17T18:03:29.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.248 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.248 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013087+0000", 2024-08-17T18:03:29.248 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3989e-05 2024-08-17T18:03:29.248 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.248 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.248 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.248 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.248 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30116 46.e e1262/1258 46:74110d7a:::benchmark_data_smithi130_267720_object30115:head v 1262'1869, mlcod=1262'1866)", 2024-08-17T18:03:29.248 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.627778+0000", 2024-08-17T18:03:29.248 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 220.041086694, 2024-08-17T18:03:29.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38918829500000002, 2024-08-17T18:03:29.249 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.249 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.249 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.249 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.249 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.249 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.249 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627778+0000", 2024-08-17T18:03:29.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.249 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.249 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.249 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.249 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627778+0000", 2024-08-17T18:03:29.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.250 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.250 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.250 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627779+0000", 2024-08-17T18:03:29.250 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2200000000000003e-07 2024-08-17T18:03:29.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.250 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.250 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.250 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627826+0000", 2024-08-17T18:03:29.250 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7339999999999997e-05 2024-08-17T18:03:29.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.250 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.250 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627835+0000", 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6549999999999993e-06 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627840+0000", 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4299999999999997e-06 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.628737+0000", 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00089676600000000001 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.628754+0000", 2024-08-17T18:03:29.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7533999999999998e-05 2024-08-17T18:03:29.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016632+0000", 2024-08-17T18:03:29.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38787752599999997 2024-08-17T18:03:29.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016966+0000", 2024-08-17T18:03:29.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00033422200000000001 2024-08-17T18:03:29.252 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.252 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:30123 46.5 46:ad019766:::benchmark_data_smithi130_267720_object30122:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.629381+0000", 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 220.03948335999999, 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38780136100000001, 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 30123 2024-08-17T18:03:29.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.254 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629381+0000", 2024-08-17T18:03:29.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629381+0000", 2024-08-17T18:03:29.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629383+0000", 2024-08-17T18:03:29.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.525e-06 2024-08-17T18:03:29.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629433+0000", 2024-08-17T18:03:29.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0513999999999998e-05 2024-08-17T18:03:29.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012724+0000", 2024-08-17T18:03:29.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38329109099999997 2024-08-17T18:03:29.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012741+0000", 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6629999999999998e-05 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012903+0000", 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016237399999999999 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013070+0000", 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016684100000000001 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.257 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:03:29.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013223+0000", 2024-08-17T18:03:29.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015269900000000001 2024-08-17T18:03:29.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.257 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:03:29.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.014685+0000", 2024-08-17T18:03:29.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0014624869999999999 2024-08-17T18:03:29.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.257 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:03:29.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.015642+0000", 2024-08-17T18:03:29.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00095662199999999996 2024-08-17T18:03:29.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:03:29.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016944+0000", 2024-08-17T18:03:29.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0013019869999999999 2024-08-17T18:03:29.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.017105+0000", 2024-08-17T18:03:29.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016044900000000001 2024-08-17T18:03:29.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.017183+0000", 2024-08-17T18:03:29.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8141999999999998e-05 2024-08-17T18:03:29.259 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.259 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.259 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.259 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30122 46.c e1262/1258 46:3a3a1109:::benchmark_data_smithi130_267720_object30121:head v 1262'1891, mlcod=1262'1889)", 2024-08-17T18:03:29.259 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.629974+0000", 2024-08-17T18:03:29.259 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 220.03889091400001, 2024-08-17T18:03:29.259 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38593537500000002, 2024-08-17T18:03:29.259 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.259 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.260 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.260 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629974+0000", 2024-08-17T18:03:29.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629974+0000", 2024-08-17T18:03:29.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.629974+0000", 2024-08-17T18:03:29.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2500000000000005e-07 2024-08-17T18:03:29.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.630007+0000", 2024-08-17T18:03:29.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2543999999999999e-05 2024-08-17T18:03:29.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012721+0000", 2024-08-17T18:03:29.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38271419800000001 2024-08-17T18:03:29.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012740+0000", 2024-08-17T18:03:29.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8525e-05 2024-08-17T18:03:29.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012886+0000", 2024-08-17T18:03:29.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014585800000000001 2024-08-17T18:03:29.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012909+0000", 2024-08-17T18:03:29.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.332e-05 2024-08-17T18:03:29.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.015868+0000", 2024-08-17T18:03:29.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029595070000000001 2024-08-17T18:03:29.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.015909+0000", 2024-08-17T18:03:29.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0698000000000002e-05 2024-08-17T18:03:29.263 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.264 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.264 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.264 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:30128 46.e e1262/1258 46:7b0313e1:::benchmark_data_smithi130_267720_object30127:head v 1262'1870, mlcod=1262'1866)", 2024-08-17T18:03:29.264 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.633214+0000", 2024-08-17T18:03:29.264 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 220.03565089, 2024-08-17T18:03:29.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.384036457, 2024-08-17T18:03:29.264 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.264 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.264 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.264 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.633214+0000", 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.633214+0000", 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.633215+0000", 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1599999999999999e-06 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.633239+0000", 2024-08-17T18:03:29.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4559e-05 2024-08-17T18:03:29.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012733+0000", 2024-08-17T18:03:29.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.379493731 2024-08-17T18:03:29.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.012744+0000", 2024-08-17T18:03:29.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.061e-05 2024-08-17T18:03:29.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.014260+0000", 2024-08-17T18:03:29.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001516414 2024-08-17T18:03:29.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.014277+0000", 2024-08-17T18:03:29.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6629e-05 2024-08-17T18:03:29.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.017168+0000", 2024-08-17T18:03:29.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028907759999999998 2024-08-17T18:03:29.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.268 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.017250+0000", 2024-08-17T18:03:29.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2577999999999994e-05 2024-08-17T18:03:29.268 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.268 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.268 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.268 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79537 46.5 46:a7b66b89:::benchmark_data_smithi130_267720_object79536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:03:29.268 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.645143+0000", 2024-08-17T18:03:29.268 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 178.02372164499999, 2024-08-17T18:03:29.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37611263900000003, 2024-08-17T18:03:29.269 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.269 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.269 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.269 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:03:29.269 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:03:29.269 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:03:29.269 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79537 2024-08-17T18:03:29.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.269 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.645143+0000", 2024-08-17T18:03:29.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.645143+0000", 2024-08-17T18:03:29.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.645144+0000", 2024-08-17T18:03:29.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3170000000000001e-06 2024-08-17T18:03:29.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.645162+0000", 2024-08-17T18:03:29.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8176999999999999e-05 2024-08-17T18:03:29.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.645172+0000", 2024-08-17T18:03:29.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8169999999999994e-06 2024-08-17T18:03:29.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.645177+0000", 2024-08-17T18:03:29.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.091e-06 2024-08-17T18:03:29.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.645226+0000", 2024-08-17T18:03:29.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8819999999999997e-05 2024-08-17T18:03:29.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.645324+0000", 2024-08-17T18:03:29.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7811000000000003e-05 2024-08-17T18:03:29.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:03:29.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.645404+0000", 2024-08-17T18:03:29.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9946999999999999e-05 2024-08-17T18:03:29.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:03:29.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648033+0000", 2024-08-17T18:03:29.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026289109999999998 2024-08-17T18:03:29.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:03:29.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648180+0000", 2024-08-17T18:03:29.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014696199999999999 2024-08-17T18:03:29.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.274 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:03:29.274 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.021186+0000", 2024-08-17T18:03:29.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37300632500000003 2024-08-17T18:03:29.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.274 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.274 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.021225+0000", 2024-08-17T18:03:29.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9267000000000002e-05 2024-08-17T18:03:29.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.274 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.275 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.021256+0000", 2024-08-17T18:03:29.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0193999999999999e-05 2024-08-17T18:03:29.275 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.275 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.275 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.275 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:165390 46.f e1279/1258 46:f0a001e2:::benchmark_data_smithi130_267720_object57407:head v 1279'10457, mlcod=1279'10454)", 2024-08-17T18:03:29.275 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:16.001989+0000", 2024-08-17T18:03:29.275 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 72.666875505999997, 2024-08-17T18:03:29.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.27958274399999999, 2024-08-17T18:03:29.275 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.275 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.275 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.001989+0000", 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.001989+0000", 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.001990+0000", 2024-08-17T18:03:29.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8199999999999999e-07 2024-08-17T18:03:29.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.001995+0000", 2024-08-17T18:03:29.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5029999999999997e-06 2024-08-17T18:03:29.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.002005+0000", 2024-08-17T18:03:29.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5300000000000002e-06 2024-08-17T18:03:29.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.002012+0000", 2024-08-17T18:03:29.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8399999999999997e-06 2024-08-17T18:03:29.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.002211+0000", 2024-08-17T18:03:29.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000199166 2024-08-17T18:03:29.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.002231+0000", 2024-08-17T18:03:29.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9746999999999999e-05 2024-08-17T18:03:29.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.281533+0000", 2024-08-17T18:03:29.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.27930255799999998 2024-08-17T18:03:29.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.281572+0000", 2024-08-17T18:03:29.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8618000000000002e-05 2024-08-17T18:03:29.279 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.280 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.280 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.280 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.280 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.280 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:165392 46.6 e1279/1258 46:627080c9:::benchmark_data_smithi130_267720_object57409:head v 1279'10302, mlcod=1279'10300)", 2024-08-17T18:03:29.280 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:16.002913+0000", 2024-08-17T18:03:29.280 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 72.665951438999997, 2024-08-17T18:03:29.280 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.294466385, 2024-08-17T18:03:29.280 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.280 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.280 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.280 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.002913+0000", 2024-08-17T18:03:29.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.002913+0000", 2024-08-17T18:03:29.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.002914+0000", 2024-08-17T18:03:29.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7299999999999996e-07 2024-08-17T18:03:29.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.282 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.282 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.002918+0000", 2024-08-17T18:03:29.282 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6639999999999997e-06 2024-08-17T18:03:29.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.282 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.282 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.002927+0000", 2024-08-17T18:03:29.282 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4540000000000007e-06 2024-08-17T18:03:29.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.002933+0000", 2024-08-17T18:03:29.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2020000000000001e-06 2024-08-17T18:03:29.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.006888+0000", 2024-08-17T18:03:29.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039553009999999996 2024-08-17T18:03:29.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.006907+0000", 2024-08-17T18:03:29.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8495000000000002e-05 2024-08-17T18:03:29.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.297315+0000", 2024-08-17T18:03:29.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.29040839200000002 2024-08-17T18:03:29.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.297380+0000", 2024-08-17T18:03:29.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4304e-05 2024-08-17T18:03:29.284 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.284 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.284 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.285 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:165395 46.6 e1279/1258 46:629be8d4:::benchmark_data_smithi130_267720_object57412:head v 1279'10303, mlcod=1279'10300)", 2024-08-17T18:03:29.285 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:16.005757+0000", 2024-08-17T18:03:29.285 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 72.663107346000004, 2024-08-17T18:03:29.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.29174139199999999, 2024-08-17T18:03:29.285 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.285 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.285 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.285 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.005757+0000", 2024-08-17T18:03:29.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.005757+0000", 2024-08-17T18:03:29.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.005758+0000", 2024-08-17T18:03:29.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6599999999999994e-07 2024-08-17T18:03:29.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.005764+0000", 2024-08-17T18:03:29.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4890000000000003e-06 2024-08-17T18:03:29.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.005775+0000", 2024-08-17T18:03:29.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1625e-05 2024-08-17T18:03:29.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.005782+0000", 2024-08-17T18:03:29.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0099999999999998e-06 2024-08-17T18:03:29.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.288 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.288 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.281551+0000", 2024-08-17T18:03:29.288 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.27576825500000002 2024-08-17T18:03:29.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.288 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.288 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.281577+0000", 2024-08-17T18:03:29.288 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5891e-05 2024-08-17T18:03:29.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.288 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.288 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.297423+0000", 2024-08-17T18:03:29.288 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015846729 2024-08-17T18:03:29.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.297499+0000", 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5426999999999995e-05 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36225.0:165400 46.c e1279/1258 46:3f1f1835:::benchmark_data_smithi130_267720_object57417:head v 1279'10394, mlcod=1279'10392)", 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:16.007848+0000", 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 72.661016736999997, 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.28014117199999999, 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.289 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.290 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.290 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.007848+0000", 2024-08-17T18:03:29.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.007848+0000", 2024-08-17T18:03:29.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.007849+0000", 2024-08-17T18:03:29.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2699999999999998e-07 2024-08-17T18:03:29.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.007856+0000", 2024-08-17T18:03:29.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7270000000000001e-06 2024-08-17T18:03:29.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.281516+0000", 2024-08-17T18:03:29.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.273660342 2024-08-17T18:03:29.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.281526+0000", 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0198e-05 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.281597+0000", 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0983999999999997e-05 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.281619+0000", 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1708999999999998e-05 2024-08-17T18:03:29.292 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.293 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.293 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.287955+0000", 2024-08-17T18:03:29.293 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0063361240000000003 2024-08-17T18:03:29.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.293 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.293 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:16.287989+0000", 2024-08-17T18:03:29.293 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4161e-05 2024-08-17T18:03:29.293 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.293 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.293 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.294 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195103 46.5 46:ae5a8f19:::benchmark_data_smithi130_267720_object87120:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-08-17T18:03:29.294 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.858385+0000", 2024-08-17T18:03:29.294 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 35.810479411999999, 2024-08-17T18:03:29.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2669090239999998, 2024-08-17T18:03:29.294 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.294 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.294 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:03:29.294 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:03:29.294 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:03:29.294 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:03:29.294 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195103 2024-08-17T18:03:29.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.294 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.858385+0000", 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.858385+0000", 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.858387+0000", 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9410000000000001e-06 2024-08-17T18:03:29.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.858393+0000", 2024-08-17T18:03:29.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2140000000000001e-06 2024-08-17T18:03:29.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.858409+0000", 2024-08-17T18:03:29.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5943e-05 2024-08-17T18:03:29.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.858417+0000", 2024-08-17T18:03:29.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4209999999999998e-06 2024-08-17T18:03:29.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.297 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.297 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.858635+0000", 2024-08-17T18:03:29.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000218488 2024-08-17T18:03:29.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.297 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.297 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.858754+0000", 2024-08-17T18:03:29.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000118943 2024-08-17T18:03:29.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.297 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:03:29.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.858886+0000", 2024-08-17T18:03:29.298 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013200899999999999 2024-08-17T18:03:29.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:03:29.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.869163+0000", 2024-08-17T18:03:29.298 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010276931 2024-08-17T18:03:29.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.936476+0000", 2024-08-17T18:03:29.298 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.067312417999999999 2024-08-17T18:03:29.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.299 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.299 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:03:29.299 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.936486+0000", 2024-08-17T18:03:29.299 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0531e-05 2024-08-17T18:03:29.299 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.299 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.299 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.299 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.873529+0000", 2024-08-17T18:03:29.299 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.937043336 2024-08-17T18:03:29.299 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.299 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.299 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:03:29.299 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.873542+0000", 2024-08-17T18:03:29.300 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2764e-05 2024-08-17T18:03:29.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.300 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.300 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.300 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.025193+0000", 2024-08-17T18:03:29.300 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1516504830000001 2024-08-17T18:03:29.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.300 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.300 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:03:29.300 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.025287+0000", 2024-08-17T18:03:29.300 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4792999999999999e-05 2024-08-17T18:03:29.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.301 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.301 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.125214+0000", 2024-08-17T18:03:29.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.099926197999999994 2024-08-17T18:03:29.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.301 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.301 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.125294+0000", 2024-08-17T18:03:29.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0611000000000005e-05 2024-08-17T18:03:29.301 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.301 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.301 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.302 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195098 46.6 46:6706ca8f:::benchmark_data_smithi130_267720_object87115:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:03:29.302 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.876946+0000", 2024-08-17T18:03:29.302 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 35.791918641999999, 2024-08-17T18:03:29.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2508661999999999, 2024-08-17T18:03:29.302 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.302 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.302 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:03:29.302 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:03:29.302 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:03:29.302 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:03:29.302 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195098 2024-08-17T18:03:29.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.303 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.876946+0000", 2024-08-17T18:03:29.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.876946+0000", 2024-08-17T18:03:29.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.876947+0000", 2024-08-17T18:03:29.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.254e-06 2024-08-17T18:03:29.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.876950+0000", 2024-08-17T18:03:29.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.7819999999999998e-06 2024-08-17T18:03:29.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.876958+0000", 2024-08-17T18:03:29.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8539999999999997e-06 2024-08-17T18:03:29.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.876964+0000", 2024-08-17T18:03:29.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7799999999999997e-06 2024-08-17T18:03:29.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.928194+0000", 2024-08-17T18:03:29.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.051230541999999997 2024-08-17T18:03:29.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:03:29.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.928209+0000", 2024-08-17T18:03:29.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4578e-05 2024-08-17T18:03:29.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.869019+0000", 2024-08-17T18:03:29.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94080989599999998 2024-08-17T18:03:29.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:03:29.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.869030+0000", 2024-08-17T18:03:29.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1634000000000001e-05 2024-08-17T18:03:29.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.307 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.307 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.307 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.124971+0000", 2024-08-17T18:03:29.307 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.255940976 2024-08-17T18:03:29.307 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.307 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.307 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.307 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.127812+0000", 2024-08-17T18:03:29.307 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002840904 2024-08-17T18:03:29.307 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.307 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.307 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.307 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.307 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.308 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195100 46.c 46:3febd739:::benchmark_data_smithi130_267720_object87117:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:03:29.308 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.876974+0000", 2024-08-17T18:03:29.308 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 35.791890616000003, 2024-08-17T18:03:29.308 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1358371580000002, 2024-08-17T18:03:29.308 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.308 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.308 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:03:29.308 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:03:29.308 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:03:29.308 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:03:29.308 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195100 2024-08-17T18:03:29.308 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.308 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.876974+0000", 2024-08-17T18:03:29.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.876974+0000", 2024-08-17T18:03:29.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.310 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.310 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.876974+0000", 2024-08-17T18:03:29.310 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.46e-07 2024-08-17T18:03:29.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.310 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.310 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.876976+0000", 2024-08-17T18:03:29.310 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9580000000000001e-06 2024-08-17T18:03:29.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.876983+0000", 2024-08-17T18:03:29.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5130000000000002e-06 2024-08-17T18:03:29.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.311 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.876989+0000", 2024-08-17T18:03:29.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2530000000000001e-06 2024-08-17T18:03:29.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.311 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.931724+0000", 2024-08-17T18:03:29.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.054734575000000001 2024-08-17T18:03:29.312 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.312 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:03:29.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.931736+0000", 2024-08-17T18:03:29.312 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2339e-05 2024-08-17T18:03:29.312 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.312 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.874563+0000", 2024-08-17T18:03:29.312 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94282643700000002 2024-08-17T18:03:29.312 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.312 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:03:29.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.874573+0000", 2024-08-17T18:03:29.313 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0016e-05 2024-08-17T18:03:29.313 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.313 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.313 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.313 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.010624+0000", 2024-08-17T18:03:29.313 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1360511760000001 2024-08-17T18:03:29.313 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.313 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.313 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.313 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.012811+0000", 2024-08-17T18:03:29.313 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021874450000000001 2024-08-17T18:03:29.313 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.313 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.313 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.314 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.314 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.314 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195112 46.c 46:3a1cf4d6:::benchmark_data_smithi130_267720_object87129:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:03:29.314 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.888498+0000", 2024-08-17T18:03:29.314 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 35.780366571999998, 2024-08-17T18:03:29.314 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2291281509999998, 2024-08-17T18:03:29.314 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.314 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.314 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.314 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:03:29.314 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:03:29.314 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:03:29.314 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195112 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.888498+0000", 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.888498+0000", 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.888500+0000", 2024-08-17T18:03:29.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4479999999999999e-06 2024-08-17T18:03:29.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.888503+0000", 2024-08-17T18:03:29.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8029999999999998e-06 2024-08-17T18:03:29.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.888514+0000", 2024-08-17T18:03:29.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0352e-05 2024-08-17T18:03:29.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.888520+0000", 2024-08-17T18:03:29.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3160000000000002e-06 2024-08-17T18:03:29.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.931764+0000", 2024-08-17T18:03:29.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.043243722999999998 2024-08-17T18:03:29.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:03:29.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.931771+0000", 2024-08-17T18:03:29.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8530000000000003e-06 2024-08-17T18:03:29.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.874597+0000", 2024-08-17T18:03:29.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94282607299999999 2024-08-17T18:03:29.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:03:29.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.874604+0000", 2024-08-17T18:03:29.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.678e-06 2024-08-17T18:03:29.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.013040+0000", 2024-08-17T18:03:29.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1384357089999999 2024-08-17T18:03:29.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.013413+0000", 2024-08-17T18:03:29.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00037309600000000001 2024-08-17T18:03:29.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,6,7", 2024-08-17T18:03:29.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.013669+0000", 2024-08-17T18:03:29.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000256035 2024-08-17T18:03:29.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:03:29.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.116807+0000", 2024-08-17T18:03:29.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.10313798 2024-08-17T18:03:29.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.320 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:03:29.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.116874+0000", 2024-08-17T18:03:29.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7003999999999997e-05 2024-08-17T18:03:29.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.320 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:03:29.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.117191+0000", 2024-08-17T18:03:29.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000316834 2024-08-17T18:03:29.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.321 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:03:29.321 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.117312+0000", 2024-08-17T18:03:29.321 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000120844 2024-08-17T18:03:29.321 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.321 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.321 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.321 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.117376+0000", 2024-08-17T18:03:29.321 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3924000000000007e-05 2024-08-17T18:03:29.321 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.321 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.321 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.321 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.117626+0000", 2024-08-17T18:03:29.321 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00025047900000000001 2024-08-17T18:03:29.321 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.322 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.322 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.322 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.322 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.322 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195116 46.c 46:3a849f5a:::benchmark_data_smithi130_267720_object87133:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:03:29.322 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.889572+0000", 2024-08-17T18:03:29.322 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 35.779292235, 2024-08-17T18:03:29.322 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2282117619999999, 2024-08-17T18:03:29.322 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.322 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.322 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.322 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:03:29.322 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:03:29.322 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:03:29.323 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195116 2024-08-17T18:03:29.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.323 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.323 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.889572+0000", 2024-08-17T18:03:29.323 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.323 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.889572+0000", 2024-08-17T18:03:29.323 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.889574+0000", 2024-08-17T18:03:29.324 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1880000000000001e-06 2024-08-17T18:03:29.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.889580+0000", 2024-08-17T18:03:29.324 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.981e-06 2024-08-17T18:03:29.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.889591+0000", 2024-08-17T18:03:29.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1739000000000001e-05 2024-08-17T18:03:29.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.325 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.325 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.325 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.889599+0000", 2024-08-17T18:03:29.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7649999999999993e-06 2024-08-17T18:03:29.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.325 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.325 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.325 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.931805+0000", 2024-08-17T18:03:29.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042205815000000001 2024-08-17T18:03:29.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.325 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.325 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:03:29.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.931812+0000", 2024-08-17T18:03:29.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5750000000000003e-06 2024-08-17T18:03:29.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.326 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.874626+0000", 2024-08-17T18:03:29.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94281379300000001 2024-08-17T18:03:29.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.326 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:03:29.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.874634+0000", 2024-08-17T18:03:29.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6299999999999998e-06 2024-08-17T18:03:29.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.327 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.025768+0000", 2024-08-17T18:03:29.327 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1511338499999999 2024-08-17T18:03:29.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.327 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.025843+0000", 2024-08-17T18:03:29.327 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5694999999999999e-05 2024-08-17T18:03:29.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.327 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,6,7", 2024-08-17T18:03:29.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.025915+0000", 2024-08-17T18:03:29.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1923999999999998e-05 2024-08-17T18:03:29.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.328 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:03:29.328 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.116818+0000", 2024-08-17T18:03:29.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.090902539000000004 2024-08-17T18:03:29.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.328 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:03:29.328 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.116947+0000", 2024-08-17T18:03:29.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012871200000000001 2024-08-17T18:03:29.329 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.329 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.329 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:03:29.329 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.117253+0000", 2024-08-17T18:03:29.329 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00030613999999999999 2024-08-17T18:03:29.329 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.329 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.329 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:03:29.329 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.117676+0000", 2024-08-17T18:03:29.329 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00042320699999999999 2024-08-17T18:03:29.329 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.330 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.330 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.330 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.117710+0000", 2024-08-17T18:03:29.330 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3896000000000002e-05 2024-08-17T18:03:29.330 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.330 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.330 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.330 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.117784+0000", 2024-08-17T18:03:29.330 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4313000000000005e-05 2024-08-17T18:03:29.330 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.330 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.330 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.331 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.331 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195120 46.5 46:ae104962:::benchmark_data_smithi130_267720_object87137:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:03:29.331 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.945622+0000", 2024-08-17T18:03:29.331 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 35.723242528999997, 2024-08-17T18:03:29.331 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.196117987, 2024-08-17T18:03:29.331 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:03:29.331 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:03:29.331 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:03:29.331 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:03:29.331 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:03:29.331 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:03:29.331 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195120 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.945622+0000", 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.945622+0000", 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:03:29.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.945624+0000", 2024-08-17T18:03:29.333 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2390000000000001e-06 2024-08-17T18:03:29.333 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.333 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:03:29.333 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.945632+0000", 2024-08-17T18:03:29.333 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5020000000000003e-06 2024-08-17T18:03:29.333 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.333 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:03:29.333 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.945653+0000", 2024-08-17T18:03:29.333 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0645999999999999e-05 2024-08-17T18:03:29.333 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.333 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:03:29.334 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.945662+0000", 2024-08-17T18:03:29.334 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6050000000000004e-06 2024-08-17T18:03:29.334 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.334 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.334 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.334 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.945821+0000", 2024-08-17T18:03:29.334 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000159178 2024-08-17T18:03:29.334 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.334 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.334 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:03:29.334 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.945835+0000", 2024-08-17T18:03:29.334 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3284e-05 2024-08-17T18:03:29.334 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.334 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.335 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.335 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.873569+0000", 2024-08-17T18:03:29.335 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.92773477699999995 2024-08-17T18:03:29.335 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.335 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.335 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:03:29.335 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.873578+0000", 2024-08-17T18:03:29.335 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6840000000000002e-06 2024-08-17T18:03:29.335 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.335 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.335 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:03:29.335 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.025531+0000", 2024-08-17T18:03:29.335 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.151952901 2024-08-17T18:03:29.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:03:29.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.025619+0000", 2024-08-17T18:03:29.336 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8090000000000005e-05 2024-08-17T18:03:29.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6,7", 2024-08-17T18:03:29.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.026014+0000", 2024-08-17T18:03:29.336 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00039487000000000002 2024-08-17T18:03:29.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:03:29.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.125347+0000", 2024-08-17T18:03:29.337 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.099333437999999996 2024-08-17T18:03:29.337 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.337 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.337 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:03:29.337 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.125389+0000", 2024-08-17T18:03:29.337 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1467999999999997e-05 2024-08-17T18:03:29.337 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.337 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.337 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:03:29.337 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.134115+0000", 2024-08-17T18:03:29.337 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0087264260000000007 2024-08-17T18:03:29.337 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.337 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.337 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:03:29.338 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.141549+0000", 2024-08-17T18:03:29.338 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0074339560000000002 2024-08-17T18:03:29.338 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.338 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.338 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:03:29.338 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.141623+0000", 2024-08-17T18:03:29.338 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3392000000000005e-05 2024-08-17T18:03:29.338 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:03:29.338 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:03:29.338 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:03:29.338 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.141740+0000", 2024-08-17T18:03:29.338 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000117531 2024-08-17T18:03:29.338 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.338 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.339 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.339 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:03:29.339 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:03:29.339 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:03:29.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:29.340 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:29.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:29.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:29 smithi179 ceph-mon[93519]: pgmap v2165: 89 pgs: 89 active+clean; 79 MiB data, 2.2 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 681 op/s; 3.0 MiB/s, 47 objects/s recovering 2024-08-17T18:03:29.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2338274279' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:03:29.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:29 smithi130 ceph-mon[105521]: pgmap v2165: 89 pgs: 89 active+clean; 79 MiB data, 2.2 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 681 op/s; 3.0 MiB/s, 47 objects/s recovering 2024-08-17T18:03:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2338274279' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:03:29.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:29 smithi130 ceph-mon[110068]: pgmap v2165: 89 pgs: 89 active+clean; 79 MiB data, 2.2 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 681 op/s; 3.0 MiB/s, 47 objects/s recovering 2024-08-17T18:03:29.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2338274279' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:03:29.441 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:29.467 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:29.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:29.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:29.569 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:29.594 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:29.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:29.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:29.696 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:29.717 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:29.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:29.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:29.819 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:29.844 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:29.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:29.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:29.946 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:29.968 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:29.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:29.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:30.069 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:30.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:30.091 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:30.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:30.092 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:03:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:03:29] "GET /metrics HTTP/1.1" 200 36793 "" "Prometheus/2.51.0" 2024-08-17T18:03:30.192 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:30.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:30.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:30.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:30.314 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T18:03:30.319 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:30.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:30 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:03:30.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:30 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2338274279' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:03:30.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:30 smithi179 ceph-mon[93519]: osdmap e1300: 8 total, 8 up, 5 in 2024-08-17T18:03:30.345 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:30.345 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:03:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:03:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:03:30.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:30.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:30.401 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:30 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:03:30.401 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:30 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2338274279' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:03:30.401 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:30 smithi130 ceph-mon[105521]: osdmap e1300: 8 total, 8 up, 5 in 2024-08-17T18:03:30.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:30 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:03:30.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:30 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2338274279' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:03:30.402 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:30 smithi130 ceph-mon[110068]: osdmap e1300: 8 total, 8 up, 5 in 2024-08-17T18:03:30.447 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:30.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:30.469 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:30.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:30.570 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:30.585 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:03:30.592 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:30.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:30.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:30.694 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:30.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:30.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:30.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:30.817 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:30.838 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:30.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:30.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:30.909 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:03:30.910 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:03:30.925 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:03:30.925 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:03:30.939 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:30.951 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:03:30.951 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:03:30.961 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:30.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:30.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:30.967 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:03:30.967 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:03:30.983 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:03:30.983 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:03:30.998 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:03:30.999 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:03:31.014 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:03:31.014 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:03:31.030 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:03:31.030 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:03:31.063 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:31.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:31.086 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:31.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:31.187 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:31.208 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:31.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:31.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:31.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:31 smithi179 ceph-mon[93519]: pgmap v2167: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 619 op/s 2024-08-17T18:03:31.285 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/489508847' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:03:31.287 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:31 smithi130 ceph-mon[105521]: pgmap v2167: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 619 op/s 2024-08-17T18:03:31.287 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/489508847' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:03:31.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:31 smithi130 ceph-mon[110068]: pgmap v2167: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 619 op/s 2024-08-17T18:03:31.287 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/489508847' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:03:31.311 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:31.334 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:31.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:31.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:31.440 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:31.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:31.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:31.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:31.563 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:31.585 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:31.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:31.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:31.687 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:31.709 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:31.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:31.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:31.811 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:31.832 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:31.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:31.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:31.870 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:31.934 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:31.955 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:31.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:31.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:32.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:32.078 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:32.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:32.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:32.109 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T18:03:32.180 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:32.201 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:32.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:32.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:32.303 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:32.325 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:32.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:32.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:32.336 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/926122652' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:03:32.336 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/926122652' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:03:32.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/926122652' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:03:32.427 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:32.449 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:32.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:32.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:32.551 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:32.568 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:03:32.568 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:03:32.572 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:32.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:32.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:32.674 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:32.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:32.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:32.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:32.797 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:32.818 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:32.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:32.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:32.920 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:32.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:32.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:32.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:33.044 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:33.065 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:33.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:33.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:33.167 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:33.188 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:33.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:33.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:33.272 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:33 smithi179 ceph-mon[93519]: pgmap v2168: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 619 op/s 2024-08-17T18:03:33.273 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:33 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/926122652' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T18:03:33.273 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:33 smithi179 ceph-mon[93519]: osdmap e1301: 8 total, 8 up, 5 in 2024-08-17T18:03:33.290 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:33.296 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T18:03:33.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:33.312 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:33.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:33.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:33 smithi130 ceph-mon[110068]: pgmap v2168: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 619 op/s 2024-08-17T18:03:33.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:33 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/926122652' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T18:03:33.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:33 smithi130 ceph-mon[110068]: osdmap e1301: 8 total, 8 up, 5 in 2024-08-17T18:03:33.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:33 smithi130 ceph-mon[105521]: pgmap v2168: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 619 op/s 2024-08-17T18:03:33.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:33 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/926122652' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T18:03:33.388 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:33 smithi130 ceph-mon[105521]: osdmap e1301: 8 total, 8 up, 5 in 2024-08-17T18:03:33.414 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:33.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:33.436 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:33.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:33.537 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:33.558 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:33.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:33.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:33.562 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:03:33.660 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:33.681 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:33.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:33.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:33.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:33.804 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:33.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:33.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:33.906 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:33.927 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:33.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:33.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:34.028 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:34.050 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:34.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:34.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:34.151 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:34.173 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:34.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:34.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:34.275 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:34.295 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:34.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:34.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:34.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:03:34.397 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:34.418 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:34.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:34.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:34.520 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:34.541 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:34.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:34.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:34.643 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:34.664 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:34.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:34.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:34.762 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:03:34.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:34.831 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:34.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:34.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:34.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:34.955 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:34.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:34.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:35.057 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:35.079 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:35.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:35.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:35.125 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T18:03:35.181 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:35.203 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:35.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:35.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:35.305 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:35.328 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:35.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:35.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:35.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:35 smithi179 ceph-mon[93519]: pgmap v2170: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 214 op/s 2024-08-17T18:03:35.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:35 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:03:35.350 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:03:35.350 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:03:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:35 smithi130 ceph-mon[105521]: pgmap v2170: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 214 op/s 2024-08-17T18:03:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:35 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:03:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:35 smithi130 ceph-mon[110068]: pgmap v2170: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 214 op/s 2024-08-17T18:03:35.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:35 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:03:35.430 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:35.441 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:35.441 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:35.442 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:35.452 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:35.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:35.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:35.554 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:35.575 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:35.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:35.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:35.677 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:35.699 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:35.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:35.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:35.801 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:35.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:35.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:35.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:35.924 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:35.946 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:35.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:35.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:36.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T18:03:36.048 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:36.070 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:36.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:36.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:36.136 INFO:teuthology.orchestra.run.smithi130.stderr:pool 'unique_pool_1' removed 2024-08-17T18:03:36.171 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:36.194 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:36.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:36.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:36.205 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:36 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1500768103' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:03:36.205 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:36 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:03:36.205 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:36 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1500768103' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:03:36.205 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:36 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:03:36.296 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:36.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:36.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:36.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:36.330 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:36 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1500768103' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:03:36.331 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:36 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:03:36.419 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:36.440 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:36.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:36.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:36.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:36.563 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:36.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:36.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:36.665 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:36.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:36.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:36.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:36.788 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:36.789 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:03:36.811 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:36.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:36.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:36.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:36.934 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:36.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:36.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:37.037 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:37.059 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:37.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:37.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:37.161 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:37.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:37.183 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:37.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:37.227 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:37 smithi179 ceph-mon[93519]: pgmap v2171: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 157 op/s 2024-08-17T18:03:37.227 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:37 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:03:37.227 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:37 smithi179 ceph-mon[93519]: osdmap e1302: 8 total, 8 up, 5 in 2024-08-17T18:03:37.284 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:37.305 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:37.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:37.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:37.407 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:37.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:37 smithi130 ceph-mon[105521]: pgmap v2171: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 157 op/s 2024-08-17T18:03:37.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:37 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:03:37.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:37 smithi130 ceph-mon[105521]: osdmap e1302: 8 total, 8 up, 5 in 2024-08-17T18:03:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:37 smithi130 ceph-mon[110068]: pgmap v2171: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail; 0 B/s wr, 157 op/s 2024-08-17T18:03:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:37 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:03:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:37 smithi130 ceph-mon[110068]: osdmap e1302: 8 total, 8 up, 5 in 2024-08-17T18:03:37.427 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:37.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:37.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:37.529 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:37.551 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:37.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:37.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:37.653 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:37.674 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:37.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:37.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:37.776 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:37.797 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:37.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:37.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:37.899 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:37.920 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:37.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:37.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:38.022 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:38.044 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:38.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:38.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:38.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:38.167 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:38.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:38.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:38.269 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:38.290 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:38.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:38.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:38.392 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:38.413 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:38.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:38.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:38.515 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:38.535 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:38.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:38.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:38.637 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:38.659 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:38.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:38.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:38.761 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:38.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:38.785 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:38.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:38.803 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:03:38.804 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:03:38.885 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:38.906 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:38.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:38.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:39.008 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:39.030 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:39.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:39.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:39.132 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:39.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:39.154 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:39.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:39.255 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:39.277 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:39.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:39.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:39.378 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:39.400 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:39.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:39.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:39.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:39 smithi130 ceph-mon[105521]: pgmap v2173: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:39 smithi130 ceph-mon[110068]: pgmap v2173: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:39.466 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:39 smithi179 ceph-mon[93519]: pgmap v2173: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:39.491 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T18:03:39.501 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:39.555 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:39.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:39.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:39.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:39.678 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:39.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:39.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:39.765 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T18:03:39.766 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T18:03:39.766 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-08-17T18:03:39.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_2 16 2024-08-17T18:03:39.779 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:39.803 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:39.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:39.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:39.838 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:03:39.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:39.927 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:39.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:39.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:40.029 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:40.050 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:40.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:40.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:40.128 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:03:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:03:39] "GET /metrics HTTP/1.1" 200 36793 "" "Prometheus/2.51.0" 2024-08-17T18:03:40.152 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:40.179 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:40.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:40.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:40.280 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:40.450 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:03:40.450 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:03:40.450 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:03:40.451 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:03:40.451 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:03:40.451 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:03:40.452 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:03:40.452 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:03:40.452 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:03:40.452 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:03:40.453 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:03:40.453 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:03:40.469 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:03:40.470 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:03:40.477 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:40.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:40.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:40.485 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:03:40.486 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:03:40.579 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:03:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:03:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:03:40.604 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:40.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:40.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:40.706 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:40.728 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:40.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:40.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:40.829 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:40.851 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:40.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:40.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:40.953 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:40.980 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:40.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:40.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:41.082 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:41.104 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:41.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:41.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:41.206 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:41.229 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:41.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:41.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:41.330 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:41.352 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:41.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:41.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:41 smithi130 ceph-mon[105521]: pgmap v2174: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:41 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/934342890' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:03:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:41 smithi130 ceph-mon[110068]: pgmap v2174: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:41 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/934342890' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:03:41.454 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:41.476 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:41.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:41.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:41.579 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:41.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:41 smithi179 ceph-mon[93519]: pgmap v2174: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:41.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:41 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/934342890' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:03:41.600 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:41.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:41.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:41.702 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:41.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:41.725 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:41.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:41.826 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:41.847 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:41.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:41.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:41.949 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:41.971 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:41.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:41.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:42.073 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:42.095 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:42.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:42.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:42.196 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:42.218 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:42.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:42.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:42.334 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:42.356 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:42.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:42.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:42.382 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:42.458 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:42.478 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:42.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:42.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:42.580 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:42.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:42.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:42.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:42.704 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:42.726 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:42.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:42.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:42.815 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:03:42.815 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:03:42.815 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T18:03:42.815 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:03:42.828 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:42.849 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:42.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:42.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:42.951 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:42.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:43.098 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:43.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:43.127 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:03:43.199 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:43.220 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:43.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:43.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:43.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:43.343 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:43.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:43.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:43.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:43 smithi130 ceph-mon[105521]: pgmap v2175: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:43.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:03:43.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:43 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3767745405' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:03:43.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:43 smithi130 ceph-mon[110068]: pgmap v2175: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:43.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:03:43.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:43 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3767745405' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:03:43.472 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:43.493 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:43.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:43.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:43.578 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:43 smithi179 ceph-mon[93519]: pgmap v2175: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:43.578 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:03:43.579 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:43 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3767745405' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:03:43.595 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:43.603 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T18:03:43.616 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:43.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:43.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:43.673 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:03:43.718 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:43.818 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:43.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:43.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:43.920 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:43.946 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:43.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:43.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:43.995 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:03:44.048 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:44.086 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:44.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:44.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:44.143 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T18:03:44.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:44.209 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:44.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:44.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:44.311 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:44.333 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:44.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:44.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:44.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:44 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:03:44.442 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:44 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3767745405' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:03:44.443 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:44 smithi130 ceph-mon[105521]: osdmap e1303: 8 total, 8 up, 5 in 2024-08-17T18:03:44.443 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:44 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2521657737' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:03:44.443 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:03:44.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:44 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:03:44.443 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:44 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3767745405' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:03:44.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:44 smithi130 ceph-mon[110068]: osdmap e1303: 8 total, 8 up, 5 in 2024-08-17T18:03:44.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:44 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2521657737' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:03:44.444 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:03:44.444 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:44.466 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:44.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:44.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:44.481 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:44 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:03:44.481 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:44 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3767745405' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:03:44.481 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:44 smithi179 ceph-mon[93519]: osdmap e1303: 8 total, 8 up, 5 in 2024-08-17T18:03:44.481 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:44 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2521657737' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:03:44.481 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:03:44.568 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:44.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:44.590 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:44.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:44.691 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:44.714 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:44.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:44.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:44.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:44.872 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T18:03:44.933 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:44.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:44.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:45.034 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:45.057 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:45.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:45.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:45.159 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:45.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:45.181 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:45.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:45.282 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:45.304 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:45.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:45.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:45.408 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:45.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:45 smithi130 ceph-mon[105521]: pgmap v2177: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:45.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:45 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:03:45.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:45 smithi130 ceph-mon[105521]: osdmap e1304: 8 total, 8 up, 5 in 2024-08-17T18:03:45.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:45 smithi130 ceph-mon[110068]: pgmap v2177: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:45.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:45 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:03:45.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:45 smithi130 ceph-mon[110068]: osdmap e1304: 8 total, 8 up, 5 in 2024-08-17T18:03:45.429 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:45.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:45.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:45.439 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:45 smithi179 ceph-mon[93519]: pgmap v2177: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:45.440 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:45 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:03:45.440 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:45 smithi179 ceph-mon[93519]: osdmap e1304: 8 total, 8 up, 5 in 2024-08-17T18:03:45.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:45.553 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:45.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:45.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:45.655 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:45.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:45.682 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:45.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:45.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:45.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:45.807 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:45.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:45.882 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:45.909 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:45.930 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:45.932 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:45.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:45.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:46.034 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:46.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:46.056 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:46.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:46.144 INFO:teuthology.orchestra.run.smithi130.stderr:pool 'unique_pool_2' created 2024-08-17T18:03:46.158 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:46.166 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:03:46.181 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:46.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:46.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:46.283 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:46.305 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:46.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:46.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:46.408 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:46.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:46 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1222169379' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T18:03:46.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:46 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1222169379' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T18:03:46.429 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:46.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:46.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:46.445 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:46 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1222169379' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T18:03:46.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:46.554 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:46.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:46.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:46.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:46.678 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:46.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:46.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:46.780 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:46.801 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:46.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:46.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:46.903 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:46.925 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:46.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:46.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:47.027 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:47.048 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:47.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:47.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:47.149 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:47.171 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:03:47.171 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:03:47.171 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:03:47.171 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:03:47.171 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:03:47.177 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:47.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:47.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:47.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:47.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:47.302 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:47.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:47 smithi179 ceph-mon[93519]: pgmap v2179: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:47 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1222169379' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-17T18:03:47.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:47 smithi179 ceph-mon[93519]: osdmap e1305: 8 total, 8 up, 5 in 2024-08-17T18:03:47.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:47 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:47.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:47 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:47.404 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:47.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:47 smithi130 ceph-mon[110068]: pgmap v2179: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:47.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:47 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1222169379' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-17T18:03:47.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:47 smithi130 ceph-mon[110068]: osdmap e1305: 8 total, 8 up, 5 in 2024-08-17T18:03:47.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:47 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:47.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:47 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:47.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:47 smithi130 ceph-mon[105521]: pgmap v2179: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:47.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:47 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1222169379' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-17T18:03:47.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:47 smithi130 ceph-mon[105521]: osdmap e1305: 8 total, 8 up, 5 in 2024-08-17T18:03:47.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:47 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:47.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:47 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:47.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:47.427 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:47.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:47.529 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:47.554 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:47.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:47.655 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:47.679 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:47.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:47.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:47.688 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-08-17T18:03:47.781 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:47.802 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:47.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:47.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:47.882 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T18:03:47.904 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:47.948 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:47.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:47.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:48.050 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:48.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:48.073 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:48.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:48.137 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:03:48.174 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:48.195 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:48.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:48.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:48.297 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:48.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:48.322 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:48.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:48.414 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:48 smithi179 ceph-mon[93519]: osdmap e1306: 8 total, 8 up, 5 in 2024-08-17T18:03:48.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:48 smithi130 ceph-mon[105521]: osdmap e1306: 8 total, 8 up, 5 in 2024-08-17T18:03:48.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:48 smithi130 ceph-mon[110068]: osdmap e1306: 8 total, 8 up, 5 in 2024-08-17T18:03:48.423 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:48.445 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:48.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:48.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:48.547 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:48.569 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:48.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:48.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:48.590 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:48.671 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:48.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:48.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:48.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:48.795 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:48.816 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:48.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:48.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:48.918 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:48.939 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:48.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:48.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:49.041 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:49.063 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:49.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:49.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:49.165 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:49.185 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:49.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:49.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:49.287 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:49.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:49 smithi179 ceph-mon[93519]: pgmap v2182: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:49.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:49 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:03:49.294 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:49 smithi179 ceph-mon[93519]: osdmap e1307: 8 total, 8 up, 5 in 2024-08-17T18:03:49.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:49.310 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:49.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:49.360 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:03:49.360 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:03:49.360 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:49 smithi130 ceph-mon[105521]: pgmap v2182: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:49.361 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:49 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:03:49.361 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:49 smithi130 ceph-mon[105521]: osdmap e1307: 8 total, 8 up, 5 in 2024-08-17T18:03:49.361 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:49 smithi130 ceph-mon[110068]: pgmap v2182: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:49.361 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:49 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:03:49.361 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:49 smithi130 ceph-mon[110068]: osdmap e1307: 8 total, 8 up, 5 in 2024-08-17T18:03:49.378 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:03:49.379 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:03:49.395 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:03:49.395 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:03:49.410 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:49.411 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:03:49.412 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:03:49.427 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:03:49.427 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:03:49.432 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:49.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:49.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:49.442 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:03:49.443 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:03:49.458 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:03:49.459 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:03:49.474 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:03:49.474 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:03:49.535 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:49.556 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:49.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:49.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:49.658 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:49.680 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:49.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:49.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:49.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:49.797 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:03:49.797 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:03:49.803 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:49.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:49.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:49.905 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:49.926 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:49.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:49.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:50.028 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:50.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:50.052 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:50.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:50.153 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:03:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:03:49] "GET /metrics HTTP/1.1" 200 34371 "" "Prometheus/2.51.0" 2024-08-17T18:03:50.175 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:50.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:50.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:50.277 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:50.299 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:50.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:50.400 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:50.422 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:50.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:50.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:50.461 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:50 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3391388313' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:03:50.461 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:03:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:03:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:03:50.467 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:50 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3391388313' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:03:50.467 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:50 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3391388313' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:03:50.492 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T18:03:50.524 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:50.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:50.547 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:50.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:50.648 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:50.669 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:50.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:50.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:50.771 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:50.794 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:50.794 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:03:50.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:50.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:50.896 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:50.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:50.919 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:50.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:51.020 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:51.041 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:51.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:51.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:51.061 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:51.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:51.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:51.167 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:51.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:51.174 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T18:03:51.268 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:51.291 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:51.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:51.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:51.330 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:51 smithi179 ceph-mon[93519]: pgmap v2184: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:51.330 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:51 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3350362778' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:03:51.330 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:51 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:03:51.393 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:51.414 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:51.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:51.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:51.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:51 smithi130 ceph-mon[110068]: pgmap v2184: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:51.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:51 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3350362778' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:03:51.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:51 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:03:51.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:51 smithi130 ceph-mon[105521]: pgmap v2184: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:51.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:51 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3350362778' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:03:51.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:51 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:03:51.516 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:51.538 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:51.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:51.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:51.640 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:51.661 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:51.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:51.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:51.762 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:51.783 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:51.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:51.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:51.885 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:51.906 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:51.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:51.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:52.008 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:52.034 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:52.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:52.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:52.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T18:03:52.136 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:52.158 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:52.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:52.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:52.234 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:03:52.253 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:52 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:03:52.253 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:52 smithi179 ceph-mon[93519]: osdmap e1308: 8 total, 8 up, 5 in 2024-08-17T18:03:52.257 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:52 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:03:52.257 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:52 smithi130 ceph-mon[105521]: osdmap e1308: 8 total, 8 up, 5 in 2024-08-17T18:03:52.257 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:52 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:03:52.257 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:52 smithi130 ceph-mon[110068]: osdmap e1308: 8 total, 8 up, 5 in 2024-08-17T18:03:52.259 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:52.284 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:52.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:52.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:52.385 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:52.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:52.410 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:52.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:52.511 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:52.513 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:03:52.513 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:03:52.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:52.535 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:52.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:52.636 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:52.657 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:52.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:52.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:52.759 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:52.781 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:52.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:52.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:52.883 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:52.904 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:52.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:52.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:53.006 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:53.027 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:53.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:53.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:53.129 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:53.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:53.154 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:53.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:53.183 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T18:03:53.255 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:53.276 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:53.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:53.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:53.378 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:53.400 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:53.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:53.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:53.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:53 smithi130 ceph-mon[105521]: pgmap v2186: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:53.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:53 smithi130 ceph-mon[110068]: pgmap v2186: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:53.476 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:03:53.502 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:53.523 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:53.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:53.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:53.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:53 smithi179 ceph-mon[93519]: pgmap v2186: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:53.625 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:53.647 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:53.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:53.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:53.749 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:53.770 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:53.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:53.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:53.872 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:53.893 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:53.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:53.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:53.995 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:54.016 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:54.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:54.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:54.118 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:54.139 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:54.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:54.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:54.241 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:54.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:54.263 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:54.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:54.364 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:54.385 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:54.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:54.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:54.486 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:54.507 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:54.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:54.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:54.609 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:54.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:54.631 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:54.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:54.732 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:54.754 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:54.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:54.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:54.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:54.878 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:54.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:54.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:54.980 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:55.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:55.003 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:55.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:55.104 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:55.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:55.127 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:55.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:55.228 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:55.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:55.251 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:55.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:55.333 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:55 smithi130 ceph-mon[105521]: pgmap v2187: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:55.333 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:55 smithi130 ceph-mon[110068]: pgmap v2187: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:55.353 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:55.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:55.376 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:55.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:55.427 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:03:55.427 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:03:55.478 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:55.502 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:55.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:55.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:55.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:55 smithi179 ceph-mon[93519]: pgmap v2187: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:55.603 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:55.625 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:55.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:55.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:55.727 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:55.748 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:55.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:55.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:55.850 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:55.872 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:55.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:55.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:55.974 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:55.995 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:55.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:55.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:56.097 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:56.119 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:56.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:56.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:56.164 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T18:03:56.220 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:56.241 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:56.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:56.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:56.343 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:56.365 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:56.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:56.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:56.418 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:03:56.467 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:56.488 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:56.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:56.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:56.590 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:56.611 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:56.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:56.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:56.692 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:56.694 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:03:56.712 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:56.736 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:56.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:56.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:56.839 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:56.860 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:56.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:56.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:56.962 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:56.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:56.986 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:56.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:57.087 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:57.108 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:57.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:57.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:57.195 INFO:teuthology.orchestra.run.smithi130.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-08-17T18:03:57.202 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:03:57.210 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:57.233 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:57.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:57.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:57.334 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:57.356 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:57.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:57.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:57.458 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:57.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:57.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:57.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:57.581 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:57 smithi179 ceph-mon[93519]: pgmap v2188: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:57 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2712250855' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:03:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:57 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:03:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:57 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3636010239' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:03:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:57 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:57 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:03:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:03:57.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:57.603 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:57.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:57.646 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[105521]: pgmap v2188: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:57.647 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2712250855' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:03:57.647 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:03:57.647 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3636010239' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:03:57.647 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:57.647 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:57.647 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:03:57.647 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:03:57.647 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[110068]: pgmap v2188: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:57.647 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2712250855' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:03:57.648 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:03:57.648 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3636010239' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:03:57.648 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:57.648 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:57.648 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:03:57.648 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:03:57.703 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:57.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:57.726 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:57.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:57.827 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:57.848 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:57.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:57.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:57.950 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:57.972 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:57.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:57.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:58.074 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:58.095 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:58.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:58.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:58.197 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:58.218 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:58.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:58.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:58.297 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:03:58.297 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:03:58.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:58 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:03:58.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:58 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:03:58.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:58 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3636010239' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T18:03:58.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:58 smithi179 ceph-mon[93519]: osdmap e1309: 8 total, 8 up, 5 in 2024-08-17T18:03:58.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:58.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:58.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:03:58.298 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:58.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:58.341 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:58.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:58.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:03:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:03:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3636010239' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T18:03:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[105521]: osdmap e1309: 8 total, 8 up, 5 in 2024-08-17T18:03:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:03:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:58.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:03:58.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:03:58.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3636010239' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T18:03:58.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[110068]: osdmap e1309: 8 total, 8 up, 5 in 2024-08-17T18:03:58.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:58.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:58.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:03:58.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:03:58.443 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:03:58.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:58.466 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:58.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:03:58.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:58.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:58.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:58.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:58.690 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:03:58.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:58.713 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:58.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:03:58.814 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:58.836 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:58.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:58.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:58.938 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:03:58.960 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:58.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:58.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:03:58.964 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T18:03:59.062 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:59.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:59.085 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:59.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:59.187 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:03:59.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:59.211 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:59.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:03:59.218 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:03:59.312 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:03:59.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:59.336 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:59.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:03:59.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:59 smithi130 ceph-mon[110068]: pgmap v2190: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:59.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:59 smithi130 ceph-mon[110068]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:03:59.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:03:59 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T18:03:59.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:59 smithi130 ceph-mon[105521]: pgmap v2190: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:59.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:59 smithi130 ceph-mon[105521]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:03:59.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:03:59 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T18:03:59.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:59.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:59.459 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:59.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:59.463 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:59 smithi179 ceph-mon[93519]: pgmap v2190: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:03:59.463 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:59 smithi179 ceph-mon[93519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:03:59.463 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:03:59 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T18:03:59.561 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:59.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:59.584 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:59.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:03:59.686 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:03:59.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:59.710 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:03:59.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:03:59.811 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:03:59.833 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:59.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:59.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:03:59.935 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:03:59.955 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:03:59.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:03:59.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:00.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:00.079 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:00.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:00.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:00.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:03:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:03:59] "GET /metrics HTTP/1.1" 200 36713 "" "Prometheus/2.51.0" 2024-08-17T18:04:00.181 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:00.203 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:00.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:00.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:00.304 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:00.326 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:00.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:00.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:00.429 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:00.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:00.452 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:00.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:00.496 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:04:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:04:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:04:00.541 DEBUG:teuthology.orchestra.run.smithi130:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-08-17T18:04:00.545 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T18:04:00.545 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T18:04:00.553 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:00.574 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:00.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:00.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:00.655 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:hints = 1 2024-08-17T18:04:00.656 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T18:04:00.656 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Object prefix: benchmark_data_smithi130_302447 2024-08-17T18:04:00.656 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:04:00.656 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 0 0 0 0 0 0 - 0 2024-08-17T18:04:00.677 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:00.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:00.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:00.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:00.823 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:00.848 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:00.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:00.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:00.950 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:00.972 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:00.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:00.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:01.057 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:04:01.057 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:04:01.074 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:01.113 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:01.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:01.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:01.215 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:01.255 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:01.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:01.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:01 smithi179 ceph-mon[93519]: pgmap v2191: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:04:01.357 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:01.378 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:01.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:01.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:01.392 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:01 smithi130 ceph-mon[105521]: pgmap v2191: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:04:01.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:01 smithi130 ceph-mon[110068]: pgmap v2191: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:04:01.475 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:04:01.475 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:04:01.479 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:01.501 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:01.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:01.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:01.515 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:04:01.515 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:04:01.550 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:04:01.550 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:04:01.584 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:04:01.585 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:04:01.603 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:01.615 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:04:01.615 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:04:01.644 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:01.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:01.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:01.645 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:04:01.646 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:04:01.655 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 1 16 2604 2588 161.74 161.75 0.00779259 0.00615757 2024-08-17T18:04:01.676 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:04:01.677 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:04:01.708 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:04:01.708 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:04:01.746 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:01.768 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:01.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:01.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:01.825 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T18:04:01.870 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:01.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:01.893 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:01.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:01.948 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:04:01.994 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:02.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:02.044 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:02.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:02.115 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:04:02.145 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:02.192 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:02.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:02.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:02.213 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T18:04:02.294 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:02.317 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:02.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:02.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:02.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:02 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/885269984' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:04:02.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:02 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:04:02.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:02 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1996946932' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:04:02.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:02 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/885269984' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:04:02.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:02 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:04:02.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:02 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1996946932' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:04:02.419 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:02.457 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:02.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:02.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:02.558 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:02.565 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:02 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/885269984' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:04:02.566 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:02 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:04:02.566 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:02 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1996946932' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:04:02.582 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:02.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:02.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:02.656 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 2 16 5152 5136 160.488 159.25 0.00298459 0.00621767 2024-08-17T18:04:02.683 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:02.724 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:02.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:02.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:02.826 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:02.872 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:02.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:02.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:02.974 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:03.013 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:03.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:03.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:03.115 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:03.152 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:03.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:03.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:03.255 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:03.289 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:03.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:03.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:03.332 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T18:04:03.332 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:04:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:03 smithi179 ceph-mon[93519]: pgmap v2192: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:04:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:03 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:04:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:03 smithi179 ceph-mon[93519]: osdmap e1310: 8 total, 8 up, 5 in 2024-08-17T18:04:03.391 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:03.413 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:03.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:03.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:03.479 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:04:03.514 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:03.578 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:03.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:03.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:03.628 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:03 smithi130 ceph-mon[105521]: pgmap v2192: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:04:03.628 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:03 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:04:03.628 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:03 smithi130 ceph-mon[105521]: osdmap e1310: 8 total, 8 up, 5 in 2024-08-17T18:04:03.629 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:03 smithi130 ceph-mon[110068]: pgmap v2192: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 445 GiB / 447 GiB avail 2024-08-17T18:04:03.629 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:03 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:04:03.629 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:03 smithi130 ceph-mon[110068]: osdmap e1310: 8 total, 8 up, 5 in 2024-08-17T18:04:03.657 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 3 16 7524 7508 156.392 148.25 0.00961561 0.00638287 2024-08-17T18:04:03.680 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:03.703 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:03.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:03.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:03.804 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:03.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:03.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:03.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:03.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:03.952 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:03.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:03.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:04.033 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:04:04.033 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:04:04.033 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:04:04.033 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:04:04.033 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:04:04.054 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:04.075 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:04.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:04.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:04.177 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:04.200 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:04.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:04.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:04.302 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:04.325 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:04.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:04.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:04.375 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:04:04.427 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:04.448 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:04.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:04.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:04.550 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:04.572 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:04.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:04.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:04.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:04:04.656 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 4 16 9896 9880 154.354 148.25 0.00369696 0.00646919 2024-08-17T18:04:04.674 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:04.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:04.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:04.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:04.761 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T18:04:04.798 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:04.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:04.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:04.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:04.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:04.970 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:04.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:04.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:05.042 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:04:05.073 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:05.095 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:05.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:05.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:05.197 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:05.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:05.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:05.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:05.320 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:05.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:05.344 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:05.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:05.445 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:05.466 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:05.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:05.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:05.502 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:05 smithi179 ceph-mon[93519]: pgmap v2194: 89 pgs: 89 active+clean; 301 MiB data, 2.5 GiB used, 444 GiB / 447 GiB avail; 38 MiB/s wr, 600 op/s 2024-08-17T18:04:05.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:05.590 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:05.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:05.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:05.656 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 5 15 11840 11825 147.793 121.562 0.00419494 0.00675649 2024-08-17T18:04:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:05 smithi130 ceph-mon[105521]: pgmap v2194: 89 pgs: 89 active+clean; 301 MiB data, 2.5 GiB used, 444 GiB / 447 GiB avail; 38 MiB/s wr, 600 op/s 2024-08-17T18:04:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:05 smithi130 ceph-mon[110068]: pgmap v2194: 89 pgs: 89 active+clean; 301 MiB data, 2.5 GiB used, 444 GiB / 447 GiB avail; 38 MiB/s wr, 600 op/s 2024-08-17T18:04:05.691 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:05.712 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:05.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:05.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:05.814 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:05.852 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:05.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:05.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:05.953 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:05.975 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:05.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:05.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:06.077 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:06.114 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:06.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:06.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:06.216 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:06.238 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:06.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:06.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:06.340 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:06.365 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:06.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:06.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:06.467 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:06.503 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:06.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:06.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:06.532 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:06 smithi179 ceph-mon[93519]: pgmap v2195: 89 pgs: 89 active+clean; 301 MiB data, 2.9 GiB used, 444 GiB / 447 GiB avail; 37 MiB/s wr, 597 op/s 2024-08-17T18:04:06.605 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:06.627 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:06.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:06.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:06.656 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 6 16 13764 13748 143.191 120.188 0.00851503 0.00697615 2024-08-17T18:04:06.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:06 smithi130 ceph-mon[105521]: pgmap v2195: 89 pgs: 89 active+clean; 301 MiB data, 2.9 GiB used, 444 GiB / 447 GiB avail; 37 MiB/s wr, 597 op/s 2024-08-17T18:04:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:06 smithi130 ceph-mon[110068]: pgmap v2195: 89 pgs: 89 active+clean; 301 MiB data, 2.9 GiB used, 444 GiB / 447 GiB avail; 37 MiB/s wr, 597 op/s 2024-08-17T18:04:06.728 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:06.752 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:06.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:06.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:06.854 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:06.875 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:06.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:06.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:06.977 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:07.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:07.018 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:07.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:07.119 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:07.156 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:07.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:07.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:07.257 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:07.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:07.283 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:07.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:07.319 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:04:07.320 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T18:04:07.320 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T18:04:07.320 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T18:04:07.320 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.320 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:30138 46.a 46:577d6c08:::benchmark_data_smithi130_267720_object30137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:04:07.320 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:48.638681+0000", 2024-08-17T18:04:07.320 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 258.66312774099998, 2024-08-17T18:04:07.320 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37615081299999997, 2024-08-17T18:04:07.320 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.320 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.320 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.321 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.321 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.321 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 30138 2024-08-17T18:04:07.321 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.321 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.321 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.321 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.321 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638681+0000", 2024-08-17T18:04:07.321 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.321 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.321 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.321 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.321 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638681+0000", 2024-08-17T18:04:07.322 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.322 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.322 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.322 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.322 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638680+0000", 2024-08-17T18:04:07.322 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:04:07.322 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.322 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.322 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.322 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638734+0000", 2024-08-17T18:04:07.322 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3424999999999998e-05 2024-08-17T18:04:07.322 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.322 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.323 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.323 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638742+0000", 2024-08-17T18:04:07.323 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2819999999999996e-06 2024-08-17T18:04:07.323 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.323 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.323 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.323 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638746+0000", 2024-08-17T18:04:07.323 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9740000000000004e-06 2024-08-17T18:04:07.323 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.323 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.323 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.323 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638787+0000", 2024-08-17T18:04:07.323 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0868999999999998e-05 2024-08-17T18:04:07.324 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.324 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.324 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.324 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638874+0000", 2024-08-17T18:04:07.324 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6833e-05 2024-08-17T18:04:07.324 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.324 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.324 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.324 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638957+0000", 2024-08-17T18:04:07.324 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3745000000000005e-05 2024-08-17T18:04:07.324 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.324 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.324 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.325 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.639982+0000", 2024-08-17T18:04:07.325 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001024107 2024-08-17T18:04:07.325 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.325 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.325 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.325 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.640316+0000", 2024-08-17T18:04:07.325 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00033452200000000002 2024-08-17T18:04:07.325 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.325 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.325 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.325 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:49.014750+0000", 2024-08-17T18:04:07.325 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37443375699999998 2024-08-17T18:04:07.325 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.325 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.326 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.326 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:49.014807+0000", 2024-08-17T18:04:07.326 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7263999999999999e-05 2024-08-17T18:04:07.326 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.326 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.326 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.326 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:49.014832+0000", 2024-08-17T18:04:07.326 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4921000000000001e-05 2024-08-17T18:04:07.326 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.326 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.326 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.326 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.327 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.327 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42530 46.e 46:7a5bb013:::benchmark_data_smithi130_267720_object42529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:04:07.327 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:04:07.327 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 250.29237254099999, 2024-08-17T18:04:07.327 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.037353389, 2024-08-17T18:04:07.327 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.327 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:04:07.327 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.327 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.327 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.327 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42530 2024-08-17T18:04:07.327 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.327 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.327 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:04:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:04:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:04:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:04:07.328 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.329 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.329 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.329 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009488+0000", 2024-08-17T18:04:07.329 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2281999999999999e-05 2024-08-17T18:04:07.329 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.329 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.329 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.329 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009508+0000", 2024-08-17T18:04:07.329 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0250000000000001e-05 2024-08-17T18:04:07.329 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.329 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.329 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.329 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009514+0000", 2024-08-17T18:04:07.330 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.161e-06 2024-08-17T18:04:07.330 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.330 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.330 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.330 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009564+0000", 2024-08-17T18:04:07.330 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0028999999999998e-05 2024-08-17T18:04:07.330 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.330 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.330 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:04:07.330 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009573+0000", 2024-08-17T18:04:07.330 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.9369999999999993e-06 2024-08-17T18:04:07.330 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.330 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.331 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.331 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029341+0000", 2024-08-17T18:04:07.331 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0197678189999999 2024-08-17T18:04:07.331 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.331 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.331 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for missing object", 2024-08-17T18:04:07.331 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029434+0000", 2024-08-17T18:04:07.331 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.3185999999999999e-05 2024-08-17T18:04:07.331 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.331 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.331 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.331 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033201+0000", 2024-08-17T18:04:07.331 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003766934 2024-08-17T18:04:07.332 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.332 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.332 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for degraded object", 2024-08-17T18:04:07.332 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033231+0000", 2024-08-17T18:04:07.332 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9657000000000001e-05 2024-08-17T18:04:07.332 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.332 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.332 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.332 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.046712+0000", 2024-08-17T18:04:07.332 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013481102 2024-08-17T18:04:07.332 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.332 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.332 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.333 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.046790+0000", 2024-08-17T18:04:07.333 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7853000000000002e-05 2024-08-17T18:04:07.333 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.333 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.333 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.333 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.333 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.333 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42561 46.e 46:7b14be6d:::benchmark_data_smithi130_267720_object42560:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:04:07.333 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.039454+0000", 2024-08-17T18:04:07.333 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 250.26235501299999, 2024-08-17T18:04:07.333 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99762182099999996, 2024-08-17T18:04:07.333 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.333 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.333 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.334 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.334 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.334 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42561 2024-08-17T18:04:07.334 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.334 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.334 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.334 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.334 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039454+0000", 2024-08-17T18:04:07.334 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.334 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.334 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.334 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.334 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039454+0000", 2024-08-17T18:04:07.335 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.335 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.335 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.335 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.335 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039452+0000", 2024-08-17T18:04:07.335 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:04:07.335 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.335 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.335 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.335 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039516+0000", 2024-08-17T18:04:07.335 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3046000000000006e-05 2024-08-17T18:04:07.335 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.336 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.336 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.336 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039528+0000", 2024-08-17T18:04:07.336 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2916e-05 2024-08-17T18:04:07.336 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.336 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.336 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.336 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039534+0000", 2024-08-17T18:04:07.336 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2179999999999998e-06 2024-08-17T18:04:07.336 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.336 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.336 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.336 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039834+0000", 2024-08-17T18:04:07.336 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00030059599999999998 2024-08-17T18:04:07.337 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.337 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.337 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:04:07.337 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039844+0000", 2024-08-17T18:04:07.337 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5840000000000001e-06 2024-08-17T18:04:07.337 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.337 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.337 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.337 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029614+0000", 2024-08-17T18:04:07.337 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98977001600000003 2024-08-17T18:04:07.337 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.337 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.338 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.338 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029696+0000", 2024-08-17T18:04:07.338 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2041999999999998e-05 2024-08-17T18:04:07.338 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.338 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.338 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.338 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029809+0000", 2024-08-17T18:04:07.338 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000113503 2024-08-17T18:04:07.338 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.338 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.338 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.338 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033156+0000", 2024-08-17T18:04:07.338 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0033461559999999999 2024-08-17T18:04:07.338 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.339 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.339 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.339 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036913+0000", 2024-08-17T18:04:07.339 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037575149999999999 2024-08-17T18:04:07.339 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.339 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.339 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.339 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036976+0000", 2024-08-17T18:04:07.339 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2580000000000001e-05 2024-08-17T18:04:07.339 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.339 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.339 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.339 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037020+0000", 2024-08-17T18:04:07.340 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4678000000000002e-05 2024-08-17T18:04:07.340 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.340 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.340 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.340 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037076+0000", 2024-08-17T18:04:07.340 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5449999999999999e-05 2024-08-17T18:04:07.340 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.340 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.340 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.340 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.340 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.340 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42566 46.e 46:733e1503:::benchmark_data_smithi130_267720_object42565:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:04:07.340 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.042517+0000", 2024-08-17T18:04:07.340 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 250.25929148399999, 2024-08-17T18:04:07.341 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99468539099999997, 2024-08-17T18:04:07.341 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.341 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.341 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.341 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.341 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.341 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42566 2024-08-17T18:04:07.341 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.341 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.341 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.341 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.341 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042517+0000", 2024-08-17T18:04:07.341 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.342 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.342 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.342 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.342 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042517+0000", 2024-08-17T18:04:07.342 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.342 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.342 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.342 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.342 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042516+0000", 2024-08-17T18:04:07.342 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:04:07.342 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.342 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.342 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.343 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042575+0000", 2024-08-17T18:04:07.343 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8802999999999999e-05 2024-08-17T18:04:07.343 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.343 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.343 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.343 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042588+0000", 2024-08-17T18:04:07.343 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2812e-05 2024-08-17T18:04:07.343 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.343 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.343 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.343 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042594+0000", 2024-08-17T18:04:07.343 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.55e-06 2024-08-17T18:04:07.343 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.343 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.344 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.344 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042634+0000", 2024-08-17T18:04:07.344 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9817999999999997e-05 2024-08-17T18:04:07.344 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.344 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.344 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:04:07.344 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042640+0000", 2024-08-17T18:04:07.344 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7570000000000001e-06 2024-08-17T18:04:07.344 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.344 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.344 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.344 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.030766+0000", 2024-08-17T18:04:07.344 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98812632600000005 2024-08-17T18:04:07.344 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.345 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.345 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.345 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.030858+0000", 2024-08-17T18:04:07.345 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2035999999999998e-05 2024-08-17T18:04:07.345 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.345 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.345 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.345 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.030950+0000", 2024-08-17T18:04:07.345 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.1605000000000001e-05 2024-08-17T18:04:07.345 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.345 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.345 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.345 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033167+0000", 2024-08-17T18:04:07.346 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002216951 2024-08-17T18:04:07.346 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.346 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.346 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.346 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036943+0000", 2024-08-17T18:04:07.346 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037761840000000001 2024-08-17T18:04:07.346 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.346 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.346 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.346 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037108+0000", 2024-08-17T18:04:07.347 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016495400000000001 2024-08-17T18:04:07.347 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.347 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.347 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.347 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037137+0000", 2024-08-17T18:04:07.347 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8826000000000001e-05 2024-08-17T18:04:07.347 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.347 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.347 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.347 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037203+0000", 2024-08-17T18:04:07.347 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6042000000000002e-05 2024-08-17T18:04:07.347 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.347 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.348 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.348 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.348 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.348 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42587 46.e 46:755c4a73:::benchmark_data_smithi130_267720_object42586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:04:07.348 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.057082+0000", 2024-08-17T18:04:07.348 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 250.24472734899999, 2024-08-17T18:04:07.348 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98134466099999995, 2024-08-17T18:04:07.348 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.348 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.348 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.348 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.348 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.349 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42587 2024-08-17T18:04:07.349 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.349 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.349 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.349 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.349 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057082+0000", 2024-08-17T18:04:07.349 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.349 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.349 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.349 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.349 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057082+0000", 2024-08-17T18:04:07.349 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.349 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.349 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.350 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.350 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057080+0000", 2024-08-17T18:04:07.350 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:04:07.350 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.350 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.350 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.350 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057140+0000", 2024-08-17T18:04:07.350 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9277000000000002e-05 2024-08-17T18:04:07.350 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.350 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.350 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.350 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057152+0000", 2024-08-17T18:04:07.350 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2279000000000001e-05 2024-08-17T18:04:07.351 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.351 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.351 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.351 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057157+0000", 2024-08-17T18:04:07.351 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9389999999999996e-06 2024-08-17T18:04:07.351 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.351 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.351 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.351 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057223+0000", 2024-08-17T18:04:07.351 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5721000000000003e-05 2024-08-17T18:04:07.351 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.351 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.351 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:04:07.351 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057232+0000", 2024-08-17T18:04:07.352 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6209999999999999e-06 2024-08-17T18:04:07.352 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.352 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.352 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.352 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.031638+0000", 2024-08-17T18:04:07.352 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97440567300000003 2024-08-17T18:04:07.352 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.352 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.352 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.352 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.031716+0000", 2024-08-17T18:04:07.352 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7650000000000004e-05 2024-08-17T18:04:07.352 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.352 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.353 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.353 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.031819+0000", 2024-08-17T18:04:07.353 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010341899999999999 2024-08-17T18:04:07.353 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.353 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.353 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.353 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033981+0000", 2024-08-17T18:04:07.353 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0021622220000000001 2024-08-17T18:04:07.353 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.353 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.353 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.353 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037227+0000", 2024-08-17T18:04:07.353 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0032460060000000001 2024-08-17T18:04:07.354 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.354 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.354 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.354 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.038264+0000", 2024-08-17T18:04:07.354 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0010368949999999999 2024-08-17T18:04:07.354 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.354 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.354 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.354 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.038321+0000", 2024-08-17T18:04:07.354 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7354e-05 2024-08-17T18:04:07.354 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.354 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.354 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.354 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.038426+0000", 2024-08-17T18:04:07.355 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000104776 2024-08-17T18:04:07.355 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.355 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.355 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.355 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.355 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.355 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42593 46.e 46:77bdd57c:::benchmark_data_smithi130_267720_object42592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:04:07.355 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.061397+0000", 2024-08-17T18:04:07.355 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 250.24041149799999, 2024-08-17T18:04:07.355 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98195155000000001, 2024-08-17T18:04:07.355 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.355 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.355 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.356 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.356 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.356 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42593 2024-08-17T18:04:07.356 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.356 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.356 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.356 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.356 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061397+0000", 2024-08-17T18:04:07.356 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.356 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.356 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.356 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.356 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061397+0000", 2024-08-17T18:04:07.356 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.357 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.357 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.357 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.357 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061390+0000", 2024-08-17T18:04:07.357 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999924 2024-08-17T18:04:07.357 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.357 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.357 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.357 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061432+0000", 2024-08-17T18:04:07.357 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2111999999999999e-05 2024-08-17T18:04:07.357 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.357 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.357 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.357 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061445+0000", 2024-08-17T18:04:07.358 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3037000000000001e-05 2024-08-17T18:04:07.358 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.358 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.358 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.358 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061450+0000", 2024-08-17T18:04:07.358 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.4759999999999997e-06 2024-08-17T18:04:07.358 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.358 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.358 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.358 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061497+0000", 2024-08-17T18:04:07.358 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6696999999999997e-05 2024-08-17T18:04:07.358 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.358 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.359 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:04:07.359 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061504+0000", 2024-08-17T18:04:07.359 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7100000000000001e-06 2024-08-17T18:04:07.359 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.359 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.359 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.359 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033248+0000", 2024-08-17T18:04:07.359 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97174373999999997 2024-08-17T18:04:07.359 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.359 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.359 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.359 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033325+0000", 2024-08-17T18:04:07.359 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7785000000000001e-05 2024-08-17T18:04:07.359 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.360 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.360 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.360 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033416+0000", 2024-08-17T18:04:07.360 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0221000000000002e-05 2024-08-17T18:04:07.360 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.360 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.360 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.360 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036950+0000", 2024-08-17T18:04:07.360 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0035344 2024-08-17T18:04:07.360 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.360 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.360 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.360 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.040390+0000", 2024-08-17T18:04:07.360 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0034401890000000002 2024-08-17T18:04:07.361 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.361 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.361 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.361 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.043240+0000", 2024-08-17T18:04:07.361 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028503019999999999 2024-08-17T18:04:07.361 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.361 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.361 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.361 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.043292+0000", 2024-08-17T18:04:07.361 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1598999999999998e-05 2024-08-17T18:04:07.361 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.361 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.361 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.361 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.043349+0000", 2024-08-17T18:04:07.362 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6907999999999998e-05 2024-08-17T18:04:07.362 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.362 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.362 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.362 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.362 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.362 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:67147 46.a 46:5c4e42f4:::benchmark_data_smithi130_267720_object67146:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:04:07.362 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:18.360318+0000", 2024-08-17T18:04:07.362 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 228.94149123299999, 2024-08-17T18:04:07.362 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.26496706399999997, 2024-08-17T18:04:07.362 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.362 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.363 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.363 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.363 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.363 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 67147 2024-08-17T18:04:07.363 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.363 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.363 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.363 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.363 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360318+0000", 2024-08-17T18:04:07.363 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.363 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.363 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.363 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.363 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360318+0000", 2024-08-17T18:04:07.364 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.364 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.364 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.364 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.364 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360316+0000", 2024-08-17T18:04:07.364 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:04:07.364 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.364 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.364 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.364 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360367+0000", 2024-08-17T18:04:07.364 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1192e-05 2024-08-17T18:04:07.364 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.364 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.365 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.365 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360383+0000", 2024-08-17T18:04:07.365 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5948999999999999e-05 2024-08-17T18:04:07.365 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.365 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.365 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.365 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360390+0000", 2024-08-17T18:04:07.365 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6139999999999997e-06 2024-08-17T18:04:07.365 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.365 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.365 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.365 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360421+0000", 2024-08-17T18:04:07.365 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1208999999999999e-05 2024-08-17T18:04:07.365 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.366 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.366 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.366 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360518+0000", 2024-08-17T18:04:07.366 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7254000000000001e-05 2024-08-17T18:04:07.366 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.366 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.366 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.366 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360612+0000", 2024-08-17T18:04:07.366 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.3487999999999997e-05 2024-08-17T18:04:07.366 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.366 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.366 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.366 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.362572+0000", 2024-08-17T18:04:07.366 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019598089999999999 2024-08-17T18:04:07.367 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.367 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.367 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.367 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.370147+0000", 2024-08-17T18:04:07.367 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0075756510000000001 2024-08-17T18:04:07.367 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.367 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.367 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.367 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.625187+0000", 2024-08-17T18:04:07.367 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.25503976 2024-08-17T18:04:07.367 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.367 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.367 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.367 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.625243+0000", 2024-08-17T18:04:07.368 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5581000000000002e-05 2024-08-17T18:04:07.368 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.368 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.368 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.368 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.625285+0000", 2024-08-17T18:04:07.368 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2268000000000003e-05 2024-08-17T18:04:07.368 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.368 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.368 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.369 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.369 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.369 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:67152 46.a 46:567c055a:::benchmark_data_smithi130_267720_object67151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:04:07.369 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:18.363407+0000", 2024-08-17T18:04:07.369 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 228.93840156100001, 2024-08-17T18:04:07.369 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.267197095, 2024-08-17T18:04:07.369 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.369 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.369 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.369 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.369 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.369 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 67152 2024-08-17T18:04:07.369 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.369 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.370 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.370 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.370 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363407+0000", 2024-08-17T18:04:07.370 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.370 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.370 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.370 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.370 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363407+0000", 2024-08-17T18:04:07.370 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.370 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.370 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.370 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.370 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363407+0000", 2024-08-17T18:04:07.371 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:04:07.371 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.371 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.371 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.371 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363466+0000", 2024-08-17T18:04:07.371 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9027999999999998e-05 2024-08-17T18:04:07.371 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.371 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.371 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.371 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363477+0000", 2024-08-17T18:04:07.371 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.172e-05 2024-08-17T18:04:07.371 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.371 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.371 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.372 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363482+0000", 2024-08-17T18:04:07.372 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3560000000000003e-06 2024-08-17T18:04:07.372 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.372 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.372 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.372 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363534+0000", 2024-08-17T18:04:07.372 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1989e-05 2024-08-17T18:04:07.372 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.372 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.372 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.372 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363659+0000", 2024-08-17T18:04:07.372 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012534499999999999 2024-08-17T18:04:07.372 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.372 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.373 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.373 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363815+0000", 2024-08-17T18:04:07.373 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00015637100000000001 2024-08-17T18:04:07.373 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.373 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.373 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.373 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.366241+0000", 2024-08-17T18:04:07.373 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002425539 2024-08-17T18:04:07.373 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.373 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.373 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.373 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.370190+0000", 2024-08-17T18:04:07.373 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0039493870000000004 2024-08-17T18:04:07.373 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.374 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.374 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.374 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630531+0000", 2024-08-17T18:04:07.374 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.26034059700000001 2024-08-17T18:04:07.374 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.374 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.374 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.374 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630580+0000", 2024-08-17T18:04:07.374 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8797000000000001e-05 2024-08-17T18:04:07.374 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.374 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.374 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.374 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630604+0000", 2024-08-17T18:04:07.374 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4739000000000001e-05 2024-08-17T18:04:07.375 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.375 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.375 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.375 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.375 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.375 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:67154 46.e 46:7f5ee838:::benchmark_data_smithi130_267720_object67153:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:04:07.375 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:18.364048+0000", 2024-08-17T18:04:07.375 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 228.93776098800001, 2024-08-17T18:04:07.375 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.26435667299999999, 2024-08-17T18:04:07.375 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.375 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.375 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.375 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.376 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.376 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 67154 2024-08-17T18:04:07.376 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.376 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.376 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.376 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.376 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364048+0000", 2024-08-17T18:04:07.376 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.376 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.376 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.376 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.376 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364048+0000", 2024-08-17T18:04:07.376 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.377 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.377 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.377 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.377 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364047+0000", 2024-08-17T18:04:07.377 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:04:07.377 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.377 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.377 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.377 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364111+0000", 2024-08-17T18:04:07.377 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3903000000000001e-05 2024-08-17T18:04:07.377 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.377 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.377 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.377 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364118+0000", 2024-08-17T18:04:07.378 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.1860000000000003e-06 2024-08-17T18:04:07.378 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.378 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.378 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.378 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364122+0000", 2024-08-17T18:04:07.378 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3620000000000002e-06 2024-08-17T18:04:07.378 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.378 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.378 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.378 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364160+0000", 2024-08-17T18:04:07.378 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7951999999999998e-05 2024-08-17T18:04:07.378 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.378 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.379 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.379 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364257+0000", 2024-08-17T18:04:07.379 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7085999999999998e-05 2024-08-17T18:04:07.379 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.379 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.379 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.379 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.364526+0000", 2024-08-17T18:04:07.379 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00026916100000000002 2024-08-17T18:04:07.379 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.379 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.380 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.380 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.366442+0000", 2024-08-17T18:04:07.380 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019158510000000001 2024-08-17T18:04:07.380 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.380 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.380 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.380 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.370399+0000", 2024-08-17T18:04:07.380 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0039570430000000004 2024-08-17T18:04:07.380 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.380 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.380 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.380 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.628298+0000", 2024-08-17T18:04:07.380 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.257899298 2024-08-17T18:04:07.381 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.381 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.381 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.381 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.628364+0000", 2024-08-17T18:04:07.381 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5518000000000005e-05 2024-08-17T18:04:07.381 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.381 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.381 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.381 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.628405+0000", 2024-08-17T18:04:07.381 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0898e-05 2024-08-17T18:04:07.381 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.381 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.381 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.381 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.382 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.382 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:67161 46.a 46:56328e3d:::benchmark_data_smithi130_267720_object67160:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:04:07.382 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:18.367430+0000", 2024-08-17T18:04:07.382 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 228.93437902900001, 2024-08-17T18:04:07.382 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.26329809900000001, 2024-08-17T18:04:07.382 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.382 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.382 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.382 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.382 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.382 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 67161 2024-08-17T18:04:07.382 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.382 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.383 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.383 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.383 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367430+0000", 2024-08-17T18:04:07.383 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.383 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.383 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.383 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.383 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367430+0000", 2024-08-17T18:04:07.383 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.383 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.383 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.383 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.383 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367429+0000", 2024-08-17T18:04:07.383 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:04:07.384 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.384 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.384 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.384 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367498+0000", 2024-08-17T18:04:07.384 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.8891000000000002e-05 2024-08-17T18:04:07.384 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.384 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.384 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.384 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367513+0000", 2024-08-17T18:04:07.384 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5858e-05 2024-08-17T18:04:07.384 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.384 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.384 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.384 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367519+0000", 2024-08-17T18:04:07.385 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5500000000000002e-06 2024-08-17T18:04:07.385 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.385 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.385 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.385 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367554+0000", 2024-08-17T18:04:07.385 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4779e-05 2024-08-17T18:04:07.385 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.385 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.385 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.385 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367677+0000", 2024-08-17T18:04:07.385 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012334800000000001 2024-08-17T18:04:07.385 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.385 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.386 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.386 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.367823+0000", 2024-08-17T18:04:07.386 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000146036 2024-08-17T18:04:07.386 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.386 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.386 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.386 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.370201+0000", 2024-08-17T18:04:07.386 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0023783250000000001 2024-08-17T18:04:07.386 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.386 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.386 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.386 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.370897+0000", 2024-08-17T18:04:07.386 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00069569499999999997 2024-08-17T18:04:07.386 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.387 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.387 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.387 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630642+0000", 2024-08-17T18:04:07.387 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.25974509099999998 2024-08-17T18:04:07.387 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.387 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.387 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.387 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630692+0000", 2024-08-17T18:04:07.387 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9332000000000001e-05 2024-08-17T18:04:07.387 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.387 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.387 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.387 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630728+0000", 2024-08-17T18:04:07.388 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6446999999999998e-05 2024-08-17T18:04:07.388 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.388 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.388 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.388 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.388 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.388 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:67164 46.e 46:7b31c258:::benchmark_data_smithi130_267720_object67163:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:04:07.388 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:18.371892+0000", 2024-08-17T18:04:07.388 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 228.92991724199999, 2024-08-17T18:04:07.388 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.25660291499999999, 2024-08-17T18:04:07.388 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.388 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.388 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.388 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.389 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.389 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 67164 2024-08-17T18:04:07.389 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.389 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.389 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.389 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.389 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.371892+0000", 2024-08-17T18:04:07.389 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.389 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.389 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.389 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.389 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.371892+0000", 2024-08-17T18:04:07.389 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.389 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.390 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.390 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.390 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.371891+0000", 2024-08-17T18:04:07.390 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:04:07.390 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.390 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.390 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.390 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:07.392 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.371938+0000", 2024-08-17T18:04:07.392 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7608000000000002e-05 2024-08-17T18:04:07.392 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.392 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.392 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.392 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.371950+0000", 2024-08-17T18:04:07.392 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1882000000000001e-05 2024-08-17T18:04:07.393 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.393 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.393 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.393 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.371956+0000", 2024-08-17T18:04:07.393 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2090000000000002e-06 2024-08-17T18:04:07.393 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.393 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.393 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.393 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.371993+0000", 2024-08-17T18:04:07.394 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6325999999999997e-05 2024-08-17T18:04:07.394 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.394 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.394 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.394 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.372110+0000", 2024-08-17T18:04:07.394 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000117552 2024-08-17T18:04:07.394 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.394 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.394 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.394 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.372256+0000", 2024-08-17T18:04:07.394 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014551500000000001 2024-08-17T18:04:07.394 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.394 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.395 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.395 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.374568+0000", 2024-08-17T18:04:07.395 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0023123190000000002 2024-08-17T18:04:07.395 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.395 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.395 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.395 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.374855+0000", 2024-08-17T18:04:07.395 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00028710500000000001 2024-08-17T18:04:07.395 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.395 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.395 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.395 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.628441+0000", 2024-08-17T18:04:07.395 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.25358550600000002 2024-08-17T18:04:07.395 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.396 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.396 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.396 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.628473+0000", 2024-08-17T18:04:07.396 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2255000000000002e-05 2024-08-17T18:04:07.396 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.396 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.396 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.396 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.628495+0000", 2024-08-17T18:04:07.396 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.1594e-05 2024-08-17T18:04:07.396 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.396 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.396 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.396 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.397 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.397 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79514 46.a 46:572757ec:::benchmark_data_smithi130_267720_object79513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:04:07.397 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.619984+0000", 2024-08-17T18:04:07.397 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 216.68182519699999, 2024-08-17T18:04:07.397 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.39803696399999999, 2024-08-17T18:04:07.397 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.397 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.397 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.397 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.397 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.397 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79514 2024-08-17T18:04:07.397 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.397 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.397 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.398 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.398 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.619984+0000", 2024-08-17T18:04:07.398 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.398 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.398 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.398 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.398 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.619984+0000", 2024-08-17T18:04:07.398 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.398 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.398 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.398 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.398 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.619983+0000", 2024-08-17T18:04:07.398 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:04:07.399 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.399 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.399 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.399 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620039+0000", 2024-08-17T18:04:07.399 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6446999999999997e-05 2024-08-17T18:04:07.399 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.399 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.399 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.399 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620050+0000", 2024-08-17T18:04:07.399 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0359e-05 2024-08-17T18:04:07.399 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.399 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.399 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.399 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620055+0000", 2024-08-17T18:04:07.400 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0479999999999998e-06 2024-08-17T18:04:07.400 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.400 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.400 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.400 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620085+0000", 2024-08-17T18:04:07.400 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9850000000000001e-05 2024-08-17T18:04:07.400 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.400 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.400 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.400 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620160+0000", 2024-08-17T18:04:07.400 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.5499000000000003e-05 2024-08-17T18:04:07.400 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.400 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.401 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.401 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620240+0000", 2024-08-17T18:04:07.401 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0418000000000002e-05 2024-08-17T18:04:07.401 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.401 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.401 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.401 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.622161+0000", 2024-08-17T18:04:07.401 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001920342 2024-08-17T18:04:07.401 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.401 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.401 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.401 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.627515+0000", 2024-08-17T18:04:07.401 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0053542399999999997 2024-08-17T18:04:07.401 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.402 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.402 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.402 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017941+0000", 2024-08-17T18:04:07.402 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.390425929 2024-08-17T18:04:07.402 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.402 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.402 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.402 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017991+0000", 2024-08-17T18:04:07.402 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9988999999999999e-05 2024-08-17T18:04:07.402 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.402 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.402 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.402 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.018021+0000", 2024-08-17T18:04:07.402 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9762999999999999e-05 2024-08-17T18:04:07.403 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.403 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.403 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.403 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.403 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.403 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79523 46.a 46:590eeb35:::benchmark_data_smithi130_267720_object79522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:04:07.403 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.631753+0000", 2024-08-17T18:04:07.403 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 216.67005587099999, 2024-08-17T18:04:07.403 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38901796, 2024-08-17T18:04:07.403 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.403 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.403 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.403 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.404 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.404 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79523 2024-08-17T18:04:07.404 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.404 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.404 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.404 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.404 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631753+0000", 2024-08-17T18:04:07.404 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.404 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.404 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.404 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.404 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631753+0000", 2024-08-17T18:04:07.404 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.404 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.405 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.405 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.405 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631752+0000", 2024-08-17T18:04:07.405 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:04:07.405 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.405 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.405 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.405 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631818+0000", 2024-08-17T18:04:07.405 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5757000000000001e-05 2024-08-17T18:04:07.405 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.405 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.405 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.405 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631828+0000", 2024-08-17T18:04:07.405 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0200000000000001e-05 2024-08-17T18:04:07.406 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.406 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.406 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.406 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631832+0000", 2024-08-17T18:04:07.406 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0430000000000001e-06 2024-08-17T18:04:07.406 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.406 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.406 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.406 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631869+0000", 2024-08-17T18:04:07.406 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6730999999999998e-05 2024-08-17T18:04:07.406 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.406 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.406 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.407 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631941+0000", 2024-08-17T18:04:07.407 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.1763999999999999e-05 2024-08-17T18:04:07.407 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.407 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.407 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.407 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632027+0000", 2024-08-17T18:04:07.407 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6599000000000002e-05 2024-08-17T18:04:07.407 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.407 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.407 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.407 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635101+0000", 2024-08-17T18:04:07.407 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0030730430000000001 2024-08-17T18:04:07.407 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.407 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.408 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.408 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.641996+0000", 2024-08-17T18:04:07.408 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0068956429999999999 2024-08-17T18:04:07.408 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.408 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.408 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.408 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020716+0000", 2024-08-17T18:04:07.408 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.378719474 2024-08-17T18:04:07.408 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.408 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.408 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.409 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020748+0000", 2024-08-17T18:04:07.409 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2026000000000001e-05 2024-08-17T18:04:07.409 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.409 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.409 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.409 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020771+0000", 2024-08-17T18:04:07.409 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3370000000000002e-05 2024-08-17T18:04:07.409 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.409 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.409 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.409 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.409 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.409 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79526 46.a 46:540ef56b:::benchmark_data_smithi130_267720_object79525:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:04:07.409 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:04:07.410 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 216.66952855299999, 2024-08-17T18:04:07.410 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38861050899999999, 2024-08-17T18:04:07.410 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.410 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.410 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.410 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.410 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.410 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79526 2024-08-17T18:04:07.410 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.410 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.410 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.410 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.410 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:04:07.411 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.411 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.411 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.411 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.411 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:04:07.411 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.411 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.411 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.411 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.411 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:04:07.411 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:04:07.411 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.411 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.412 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.412 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632360+0000", 2024-08-17T18:04:07.412 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0008999999999999e-05 2024-08-17T18:04:07.412 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.412 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.412 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.412 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632369+0000", 2024-08-17T18:04:07.412 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6590000000000002e-06 2024-08-17T18:04:07.412 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.412 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.412 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.412 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632373+0000", 2024-08-17T18:04:07.412 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8840000000000004e-06 2024-08-17T18:04:07.412 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.413 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.413 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.413 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.633861+0000", 2024-08-17T18:04:07.413 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001487326 2024-08-17T18:04:07.413 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.413 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.413 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.413 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.633938+0000", 2024-08-17T18:04:07.413 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7625000000000003e-05 2024-08-17T18:04:07.413 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.413 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.413 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.413 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.634026+0000", 2024-08-17T18:04:07.413 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7950999999999999e-05 2024-08-17T18:04:07.414 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.414 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.414 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.414 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635234+0000", 2024-08-17T18:04:07.414 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0012074340000000001 2024-08-17T18:04:07.414 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.414 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.414 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.414 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.642019+0000", 2024-08-17T18:04:07.414 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0067856890000000001 2024-08-17T18:04:07.414 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.414 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.414 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.415 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020838+0000", 2024-08-17T18:04:07.415 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37881851700000002 2024-08-17T18:04:07.415 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.415 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.415 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.415 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020870+0000", 2024-08-17T18:04:07.415 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2261999999999997e-05 2024-08-17T18:04:07.415 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.415 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.415 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.415 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020891+0000", 2024-08-17T18:04:07.415 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0777000000000001e-05 2024-08-17T18:04:07.415 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.415 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.416 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.416 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.416 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.416 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79528 46.e 46:716b67ef:::benchmark_data_smithi130_267720_object79527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:04:07.416 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.635764+0000", 2024-08-17T18:04:07.416 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 216.666044734, 2024-08-17T18:04:07.416 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38226721899999999, 2024-08-17T18:04:07.416 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.416 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.416 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.416 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.416 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.416 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79528 2024-08-17T18:04:07.416 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.417 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.417 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.417 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.417 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635764+0000", 2024-08-17T18:04:07.417 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.417 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.417 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.417 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.417 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635764+0000", 2024-08-17T18:04:07.417 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.417 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.417 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.418 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.418 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635763+0000", 2024-08-17T18:04:07.418 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:04:07.418 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.418 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.418 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.418 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635828+0000", 2024-08-17T18:04:07.418 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4821999999999997e-05 2024-08-17T18:04:07.418 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.418 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.418 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.418 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635838+0000", 2024-08-17T18:04:07.418 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9599999999999995e-06 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635842+0000", 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3280000000000002e-06 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635865+0000", 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3600000000000001e-05 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635943+0000", 2024-08-17T18:04:07.419 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.8098999999999999e-05 2024-08-17T18:04:07.420 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.420 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.420 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.420 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.636030+0000", 2024-08-17T18:04:07.420 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7188000000000006e-05 2024-08-17T18:04:07.420 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.420 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.420 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.420 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.637969+0000", 2024-08-17T18:04:07.420 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001938148 2024-08-17T18:04:07.420 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.420 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.421 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.421 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.639658+0000", 2024-08-17T18:04:07.421 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0016891600000000001 2024-08-17T18:04:07.421 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.421 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.421 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.421 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017955+0000", 2024-08-17T18:04:07.421 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.378297194 2024-08-17T18:04:07.421 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.421 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.421 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.421 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017992+0000", 2024-08-17T18:04:07.421 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6832999999999998e-05 2024-08-17T18:04:07.421 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.422 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.422 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.422 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.018031+0000", 2024-08-17T18:04:07.422 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9646e-05 2024-08-17T18:04:07.422 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.422 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.422 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.422 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.422 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.422 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79529 46.a 46:5d7561ca:::benchmark_data_smithi130_267720_object79528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:04:07.422 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.635869+0000", 2024-08-17T18:04:07.422 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 216.665940073, 2024-08-17T18:04:07.422 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38782661600000001, 2024-08-17T18:04:07.422 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.423 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.423 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.423 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.423 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.423 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79529 2024-08-17T18:04:07.423 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.423 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.423 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.423 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.423 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635869+0000", 2024-08-17T18:04:07.423 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.423 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.423 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.424 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.424 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635869+0000", 2024-08-17T18:04:07.424 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.424 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.424 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.424 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.424 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635868+0000", 2024-08-17T18:04:07.424 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:04:07.424 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.424 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.424 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.424 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635935+0000", 2024-08-17T18:04:07.424 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6509999999999998e-05 2024-08-17T18:04:07.424 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.425 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.425 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.425 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635942+0000", 2024-08-17T18:04:07.425 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3139999999999998e-06 2024-08-17T18:04:07.425 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.425 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.425 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.425 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635945+0000", 2024-08-17T18:04:07.425 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.7580000000000001e-06 2024-08-17T18:04:07.425 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.425 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.425 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.425 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635983+0000", 2024-08-17T18:04:07.426 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8318000000000001e-05 2024-08-17T18:04:07.426 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.426 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.426 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.426 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.636088+0000", 2024-08-17T18:04:07.426 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000104718 2024-08-17T18:04:07.426 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.426 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.426 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.426 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.636213+0000", 2024-08-17T18:04:07.426 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000124852 2024-08-17T18:04:07.426 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.426 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.427 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.427 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.637790+0000", 2024-08-17T18:04:07.427 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0015775839999999999 2024-08-17T18:04:07.427 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.427 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.427 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.427 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.642450+0000", 2024-08-17T18:04:07.427 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0046600260000000003 2024-08-17T18:04:07.427 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.427 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.427 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.427 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.023638+0000", 2024-08-17T18:04:07.427 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38118750600000001 2024-08-17T18:04:07.427 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.428 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.428 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.428 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.023669+0000", 2024-08-17T18:04:07.428 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0998999999999999e-05 2024-08-17T18:04:07.428 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.428 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.428 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.428 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.023695+0000", 2024-08-17T18:04:07.428 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.6645e-05 2024-08-17T18:04:07.428 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.428 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.428 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.428 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.429 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.429 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79534 46.a 46:5daa6ba5:::benchmark_data_smithi130_267720_object79533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:04:07.429 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.643773+0000", 2024-08-17T18:04:07.429 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 216.65803597300001, 2024-08-17T18:04:07.429 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38289725099999999, 2024-08-17T18:04:07.429 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.429 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.429 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.429 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.429 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.429 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79534 2024-08-17T18:04:07.429 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.429 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.429 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.430 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.430 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643773+0000", 2024-08-17T18:04:07.430 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.430 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.430 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.430 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.430 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643773+0000", 2024-08-17T18:04:07.430 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.430 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.430 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.430 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.430 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643772+0000", 2024-08-17T18:04:07.430 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:04:07.431 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.431 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.431 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.431 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643824+0000", 2024-08-17T18:04:07.431 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2240000000000001e-05 2024-08-17T18:04:07.431 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.431 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.431 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.431 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643833+0000", 2024-08-17T18:04:07.431 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6770000000000001e-06 2024-08-17T18:04:07.431 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.431 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.431 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.432 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643837+0000", 2024-08-17T18:04:07.432 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8089999999999999e-06 2024-08-17T18:04:07.432 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.432 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.432 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.432 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643868+0000", 2024-08-17T18:04:07.432 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1560000000000003e-05 2024-08-17T18:04:07.432 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.432 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.432 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.432 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643961+0000", 2024-08-17T18:04:07.432 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2584999999999995e-05 2024-08-17T18:04:07.432 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.433 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.433 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.433 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.644059+0000", 2024-08-17T18:04:07.433 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7851999999999998e-05 2024-08-17T18:04:07.433 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.433 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.433 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.433 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.646074+0000", 2024-08-17T18:04:07.433 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020149550000000001 2024-08-17T18:04:07.433 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.433 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.433 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.433 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647702+0000", 2024-08-17T18:04:07.433 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001628811 2024-08-17T18:04:07.434 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.434 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.434 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.434 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.026581+0000", 2024-08-17T18:04:07.434 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37887865300000001 2024-08-17T18:04:07.434 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.434 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.434 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.434 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.026633+0000", 2024-08-17T18:04:07.434 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2040000000000003e-05 2024-08-17T18:04:07.434 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.434 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.434 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.435 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.026670+0000", 2024-08-17T18:04:07.435 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7038e-05 2024-08-17T18:04:07.435 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.435 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.435 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.435 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.435 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.435 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79535 46.4 46:2496b234:::benchmark_data_smithi130_267720_object79534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:04:07.435 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:04:07.435 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 216.65794345699999, 2024-08-17T18:04:07.435 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38560203399999998, 2024-08-17T18:04:07.435 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.435 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.435 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.436 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.436 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.436 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79535 2024-08-17T18:04:07.436 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.436 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.436 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.436 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.436 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:04:07.436 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.436 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.436 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.436 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.436 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:04:07.436 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.437 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.437 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.437 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.437 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:04:07.437 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:04:07.437 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.437 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.437 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.437 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643927+0000", 2024-08-17T18:04:07.437 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1629000000000002e-05 2024-08-17T18:04:07.437 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.437 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.437 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.438 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643933+0000", 2024-08-17T18:04:07.438 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5130000000000002e-06 2024-08-17T18:04:07.438 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.438 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.438 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.438 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643937+0000", 2024-08-17T18:04:07.438 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3479999999999997e-06 2024-08-17T18:04:07.438 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.438 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.438 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.438 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643975+0000", 2024-08-17T18:04:07.438 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7771999999999997e-05 2024-08-17T18:04:07.438 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.438 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.439 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.439 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.644055+0000", 2024-08-17T18:04:07.439 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0144999999999999e-05 2024-08-17T18:04:07.439 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.439 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.439 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:04:07.439 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.644142+0000", 2024-08-17T18:04:07.439 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6539000000000006e-05 2024-08-17T18:04:07.439 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.439 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.439 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.439 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.646160+0000", 2024-08-17T18:04:07.439 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002018373 2024-08-17T18:04:07.440 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.440 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.440 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.440 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647237+0000", 2024-08-17T18:04:07.440 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001076465 2024-08-17T18:04:07.440 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.440 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.440 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.440 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.029404+0000", 2024-08-17T18:04:07.440 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38216758000000001 2024-08-17T18:04:07.440 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.440 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.440 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.440 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.029443+0000", 2024-08-17T18:04:07.441 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8846e-05 2024-08-17T18:04:07.441 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.441 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.441 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.441 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.029468+0000", 2024-08-17T18:04:07.441 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4355e-05 2024-08-17T18:04:07.441 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.441 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.441 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.441 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.441 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.441 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79538 46.e 46:7d19c96c:::benchmark_data_smithi130_267720_object79537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:04:07.441 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.647351+0000", 2024-08-17T18:04:07.441 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 216.65445805900001, 2024-08-17T18:04:07.442 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37361270899999999, 2024-08-17T18:04:07.442 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.442 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.442 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.442 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.442 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.442 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79538 2024-08-17T18:04:07.442 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.442 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.442 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.442 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.442 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647351+0000", 2024-08-17T18:04:07.442 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.443 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.443 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.443 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.443 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647351+0000", 2024-08-17T18:04:07.443 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.443 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.443 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.443 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.443 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647350+0000", 2024-08-17T18:04:07.443 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:04:07.443 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.443 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.443 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.443 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647421+0000", 2024-08-17T18:04:07.444 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0347999999999998e-05 2024-08-17T18:04:07.444 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.444 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.444 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.444 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647429+0000", 2024-08-17T18:04:07.444 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3059999999999998e-06 2024-08-17T18:04:07.444 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.444 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.444 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.444 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647433+0000", 2024-08-17T18:04:07.444 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7290000000000002e-06 2024-08-17T18:04:07.444 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.444 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.444 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.445 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647476+0000", 2024-08-17T18:04:07.445 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3343999999999997e-05 2024-08-17T18:04:07.445 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.445 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.445 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.445 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647600+0000", 2024-08-17T18:04:07.445 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012389700000000001 2024-08-17T18:04:07.445 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.445 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.445 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.445 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647740+0000", 2024-08-17T18:04:07.445 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014001899999999999 2024-08-17T18:04:07.445 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.446 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.446 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.446 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.649745+0000", 2024-08-17T18:04:07.446 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020055450000000001 2024-08-17T18:04:07.446 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.446 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.446 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.446 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.650277+0000", 2024-08-17T18:04:07.446 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00053202999999999996 2024-08-17T18:04:07.446 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.446 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.446 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.446 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020872+0000", 2024-08-17T18:04:07.447 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37059452300000001 2024-08-17T18:04:07.447 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.447 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.447 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.447 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020923+0000", 2024-08-17T18:04:07.447 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1481999999999999e-05 2024-08-17T18:04:07.447 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.447 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.447 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.447 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020964+0000", 2024-08-17T18:04:07.447 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0132000000000001e-05 2024-08-17T18:04:07.447 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.447 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.448 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.448 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.448 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.448 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:165418 46.e 46:7099396e:::benchmark_data_smithi130_267720_object57435:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1279)", 2024-08-17T18:04:07.448 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:02:16.031407+0000", 2024-08-17T18:04:07.448 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 111.270402156, 2024-08-17T18:04:07.448 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.26760932100000001, 2024-08-17T18:04:07.448 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:04:07.448 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:07.448 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:04:07.448 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:04:07.448 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:07.448 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 165418 2024-08-17T18:04:07.448 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.449 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:04:07.449 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.449 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:04:07.449 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031407+0000", 2024-08-17T18:04:07.449 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.449 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.449 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.449 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:04:07.449 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031407+0000", 2024-08-17T18:04:07.449 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:04:07.449 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.449 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.449 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:04:07.449 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031406+0000", 2024-08-17T18:04:07.450 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:04:07.450 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.450 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.450 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:04:07.450 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031410+0000", 2024-08-17T18:04:07.450 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8340000000000001e-06 2024-08-17T18:04:07.450 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.450 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.450 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:04:07.450 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031423+0000", 2024-08-17T18:04:07.450 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2577e-05 2024-08-17T18:04:07.450 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.450 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.451 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:04:07.451 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031427+0000", 2024-08-17T18:04:07.451 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5170000000000001e-06 2024-08-17T18:04:07.451 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.451 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.451 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:04:07.451 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031460+0000", 2024-08-17T18:04:07.451 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2641999999999997e-05 2024-08-17T18:04:07.451 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.451 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.451 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:04:07.451 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031534+0000", 2024-08-17T18:04:07.451 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.4727999999999993e-05 2024-08-17T18:04:07.451 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.452 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.452 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:04:07.452 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031614+0000", 2024-08-17T18:04:07.452 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9921999999999998e-05 2024-08-17T18:04:07.452 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.452 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.452 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:04:07.452 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.034323+0000", 2024-08-17T18:04:07.452 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027090270000000001 2024-08-17T18:04:07.452 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.452 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.452 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.452 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.034810+0000", 2024-08-17T18:04:07.452 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00048701899999999999 2024-08-17T18:04:07.453 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.453 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.453 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:07.453 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.298931+0000", 2024-08-17T18:04:07.453 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.26412039300000001 2024-08-17T18:04:07.453 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.453 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.453 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:04:07.453 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.298980+0000", 2024-08-17T18:04:07.453 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9416999999999998e-05 2024-08-17T18:04:07.453 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:04:07.453 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:04:07.453 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:04:07.454 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.299016+0000", 2024-08-17T18:04:07.454 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.5902999999999998e-05 2024-08-17T18:04:07.454 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.454 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.454 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.454 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:04:07.454 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:04:07.454 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:04:07.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:07.456 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:07.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:07.557 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:07.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:07.590 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:07.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:07.657 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 7 16 15386 15370 137.216 101.375 0.00777264 0.00728068 2024-08-17T18:04:07.691 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:07.731 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:07.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:07.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:07.832 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:07.856 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:07.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:07.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:07.958 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:07.979 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:07.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:07.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:08.081 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:08.103 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:08.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:08.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:08.205 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:08.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:08.229 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:08.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:08.279 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:04:08.331 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:08.349 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T18:04:08.354 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:08.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:08.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:08.456 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:08.510 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:08.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:08.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:08.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:08 smithi179 ceph-mon[93519]: pgmap v2196: 89 pgs: 89 active+clean; 602 MiB data, 3.3 GiB used, 444 GiB / 447 GiB avail; 60 MiB/s wr, 962 op/s 2024-08-17T18:04:08.611 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:08.632 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:08.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:08.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:08.656 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 8 16 17028 17012 132.891 102.625 0.00351073 0.00748226 2024-08-17T18:04:08.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:08 smithi130 ceph-mon[105521]: pgmap v2196: 89 pgs: 89 active+clean; 602 MiB data, 3.3 GiB used, 444 GiB / 447 GiB avail; 60 MiB/s wr, 962 op/s 2024-08-17T18:04:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:08 smithi130 ceph-mon[110068]: pgmap v2196: 89 pgs: 89 active+clean; 602 MiB data, 3.3 GiB used, 444 GiB / 447 GiB avail; 60 MiB/s wr, 962 op/s 2024-08-17T18:04:08.734 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:08.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:08.778 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:08.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:08.879 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:08.902 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:08.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:08.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:09.004 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:09.041 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:09.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:09.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:09.143 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:09.165 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:09.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:09.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:09.266 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:09.304 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:09.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:09.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:09.406 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:09.445 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:09.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:09.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:09.547 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:09.568 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:09.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:09.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:09.657 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 9 16 18677 18661 129.576 103.062 0.0102387 0.0077109 2024-08-17T18:04:09.670 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:09.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:09.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:09.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:09.794 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:09.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:09.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:09.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:09.936 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:09.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:09.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:09.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:10.060 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:10.081 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:10.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:10.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:10.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:04:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:04:09] "GET /metrics HTTP/1.1" 200 36713 "" "Prometheus/2.51.0" 2024-08-17T18:04:10.182 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:10.204 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:10.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:10.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:10.306 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:10.346 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:10.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:10.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:10.448 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:10.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:10.479 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:10.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:10.580 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:10 smithi179 ceph-mon[93519]: pgmap v2197: 89 pgs: 89 active+clean; 951 MiB data, 4.8 GiB used, 442 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T18:04:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:04:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:04:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:04:10.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:10.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:10.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:10.657 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 10 16 20170 20154 125.948 93.3125 0.0118192 0.00793235 2024-08-17T18:04:10.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:10 smithi130 ceph-mon[105521]: pgmap v2197: 89 pgs: 89 active+clean; 951 MiB data, 4.8 GiB used, 442 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T18:04:10.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:10 smithi130 ceph-mon[110068]: pgmap v2197: 89 pgs: 89 active+clean; 951 MiB data, 4.8 GiB used, 442 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T18:04:10.703 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:10.724 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:10.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:10.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:10.826 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:10.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:10.864 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:10.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:10.965 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:11.064 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:11.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:11.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:11.165 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:11.205 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:11.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:11.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:11.306 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:11.328 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:11.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:11.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:11.430 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:11.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:11.462 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:11.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:11.563 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:11.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:11.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:11.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:11.657 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 11 16 21768 21752 123.577 99.875 0.0076918 0.00808457 2024-08-17T18:04:11.686 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:11.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:11.736 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:11.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:11.837 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:11.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:11.872 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:11.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:11.973 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:11.994 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:11.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:11.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:12.096 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:12.119 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:12.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:12.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:12.220 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:12.242 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:12.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:12.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:12.344 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:12.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:12.374 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:12.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:12.475 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:12.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:12.526 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:12.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:12.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:12 smithi179 ceph-mon[93519]: pgmap v2198: 89 pgs: 89 active+clean; 951 MiB data, 4.8 GiB used, 442 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T18:04:12.627 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:12.649 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:12.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:12.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:12.657 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 12 16 23250 23234 120.997 92.625 0.00907085 0.00825809 2024-08-17T18:04:12.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:12 smithi130 ceph-mon[105521]: pgmap v2198: 89 pgs: 89 active+clean; 951 MiB data, 4.8 GiB used, 442 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T18:04:12.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:12 smithi130 ceph-mon[110068]: pgmap v2198: 89 pgs: 89 active+clean; 951 MiB data, 4.8 GiB used, 442 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T18:04:12.751 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:12.772 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:12.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:12.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:12.874 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:12.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:12.916 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:12.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:13.017 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:13.056 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:13.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:13.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:13.158 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:13.182 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:13.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:13.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:13.284 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:13.305 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:13.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:13.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:13.407 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:13.429 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:13.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:13.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:13.464 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:04:13.464 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:04:13.464 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:04:13.464 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:04:13.531 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:13.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:13.553 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:13.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:04:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:04:13.654 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:13.657 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 13 16 24761 24745 118.954 94.4375 0.0058307 0.00839977 2024-08-17T18:04:13.676 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:13.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:13.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:13.778 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:13.799 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:13.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:13.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:13.901 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:13.922 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:13.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:13.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:14.024 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:14.045 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:14.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:14.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:14.147 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:14.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:14.184 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:14.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:14.285 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:14.307 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:14.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:14.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:14.408 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:14.430 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:14.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:14.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:14.532 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:14.553 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:14.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:14.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:14.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:14 smithi179 ceph-mon[93519]: pgmap v2199: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T18:04:14.655 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:14.657 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 14 16 26195 26179 116.858 89.625 0.00548772 0.00855133 2024-08-17T18:04:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:14 smithi130 ceph-mon[105521]: pgmap v2199: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T18:04:14.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:14 smithi130 ceph-mon[110068]: pgmap v2199: 89 pgs: 89 active+clean; 1.4 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T18:04:14.695 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:14.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:14.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:14.797 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:14.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:14.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:14.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:14.963 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:14.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:14.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:15.065 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:15.110 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:15.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:15.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:15.212 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:15.233 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:15.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:15.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:15.335 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:15.357 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:15.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:15.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:15.459 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:15.498 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:15.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:15.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:15.600 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:15.622 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:15.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:15.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:15.657 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 15 16 27532 27516 114.638 83.5625 0.00923288 0.0087179 2024-08-17T18:04:15.723 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:15.761 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:15.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:15.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:15.863 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:15.885 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:15.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:15.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:15.987 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:16.008 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:16.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:16.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:16.110 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:16.132 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:16.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:16.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:16.234 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:16.255 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:16.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:16.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:16.357 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:16.378 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:16.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:16.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:16.480 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:16.524 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:16.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:16.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:16.626 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:16.634 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:16 smithi130 ceph-mon[105521]: pgmap v2200: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T18:04:16.635 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:16 smithi130 ceph-mon[110068]: pgmap v2200: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T18:04:16.657 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 16 16 29016 29000 113.269 92.75 0.0169756 0.00882124 2024-08-17T18:04:16.664 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:16.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:16.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:16.766 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:16.806 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:16.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:16.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:16.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:16 smithi179 ceph-mon[93519]: pgmap v2200: 89 pgs: 89 active+clean; 1.4 GiB data, 6.3 GiB used, 441 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-08-17T18:04:16.908 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:16.949 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:16.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:16.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:17.051 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:17.090 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:17.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:17.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:17.192 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:17.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:17.215 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:17.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:17.316 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:17.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:17.339 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:17.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:17.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:17.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:17.489 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:17.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:17.590 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:17.637 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:17.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:17.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:17.657 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 17 16 30530 30514 112.172 94.625 0.0103796 0.00890741 2024-08-17T18:04:17.740 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:17.761 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:17.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:17.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:17.863 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:17.885 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:17.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:17.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:17.986 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:18.008 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:18.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:18.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:18.110 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:18.148 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:18.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:18.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:18.250 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:18.271 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:18.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:18.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:18.373 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:18.394 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:18.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:18.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:18.496 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:18.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:18.535 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:18.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:18.636 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:18.645 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:18 smithi179 ceph-mon[93519]: pgmap v2201: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-08-17T18:04:18.657 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 18 16 31761 31745 110.214 76.9375 0.0085911 0.00906558 2024-08-17T18:04:18.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:18 smithi130 ceph-mon[105521]: pgmap v2201: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-08-17T18:04:18.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:18 smithi130 ceph-mon[110068]: pgmap v2201: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 440 GiB / 447 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-08-17T18:04:18.673 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:18.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:18.775 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:18.819 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:18.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:18.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:18.922 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:18.946 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:18.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:18.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:19.048 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:19.069 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:19.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:19.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:19.172 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:19.209 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:19.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:19.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:19.312 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:19.356 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:19.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:19.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:19.458 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:19.497 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:19.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:19.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2686326337' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:04:19.597 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2686326337' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:04:19.598 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2686326337' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:04:19.599 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:19.622 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:19.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:19.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:19.657 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 19 16 33043 33027 108.63 80.125 0.00935015 0.00919861 2024-08-17T18:04:19.723 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:04:19.723 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:19.745 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:19.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:19.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:19.783 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:04:19.784 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:04:19.846 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:19.868 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:19.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:19.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:19.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:04:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:04:19] "GET /metrics HTTP/1.1" 200 36780 "" "Prometheus/2.51.0" 2024-08-17T18:04:19.969 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:19.991 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:19.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:19.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:20.093 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:20.135 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:20.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:20.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:20.237 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:20.258 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:20.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:20.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:20.360 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:20.382 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:20.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:20.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:20.484 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:20.507 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:20.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:20.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:04:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:04:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:04:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:20 smithi179 ceph-mon[93519]: pgmap v2202: 89 pgs: 89 active+clean; 1.9 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T18:04:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:20 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:04:20.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2686326337' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:04:20.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:20 smithi179 ceph-mon[93519]: osdmap e1311: 8 total, 8 up, 5 in 2024-08-17T18:04:20.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:20 smithi179 ceph-mon[93519]: from='client.46394 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:04:20.608 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:20.645 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:20.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:20.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:20.658 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:04:20.657983+0000 min lat: 0.00111317 max lat: 0.181404 avg lat: 0.00926989 2024-08-17T18:04:20.658 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:04:20.658 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 20 16 34515 34499 107.798 92 0.0095012 0.00926989 2024-08-17T18:04:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:20 smithi130 ceph-mon[105521]: pgmap v2202: 89 pgs: 89 active+clean; 1.9 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T18:04:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:20 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:04:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2686326337' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:04:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:20 smithi130 ceph-mon[105521]: osdmap e1311: 8 total, 8 up, 5 in 2024-08-17T18:04:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:20 smithi130 ceph-mon[105521]: from='client.46394 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:04:20.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:20 smithi130 ceph-mon[110068]: pgmap v2202: 89 pgs: 89 active+clean; 1.9 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T18:04:20.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:20 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:04:20.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2686326337' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:04:20.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:20 smithi130 ceph-mon[110068]: osdmap e1311: 8 total, 8 up, 5 in 2024-08-17T18:04:20.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:20 smithi130 ceph-mon[110068]: from='client.46394 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:04:20.747 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:20.783 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2203,"stamp":"2024-08-17T18:04:19.398317+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1988892621,"num_objects":30365,"num_object_clones":0,"num_object_copies":91095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30365,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":30617,"num_write_kb":1944143,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":2372458,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":30546,"ondisk_log_size":30546,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":8019508,"kb_used_data":5879628,"kb_used_omap":293,"kb_used_meta":2139482,"kb_avail":460706252,"statfs":{"total":479975178240,"available":471763202048,"internally_reserved":0,"allocated":6020739072,"data_stored":6000222822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":300937,"internal_metadata":2190829687},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,8,4,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":992215040,"num_objects":15140,"num_object_clones":0,"num_object_copies":45420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15140,"num_write_kb":968960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.154588"},"pg_stats":[{"pgid":"47.f","version":"1310'1896","reported_seq":1909,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:18.041729+0000","last_change":"2024-08-17T18:03:47.164644+0000","last_active":"2024-08-17T18:04:18.041729+0000","last_peered":"2024-08-17T18:04:18.041729+0000","last_clean":"2024-08-17T18:04:18.041729+0000","last_became_active":"2024-08-17T18:03:47.164347+0000","last_became_peered":"2024-08-17T18:03:47.164347+0000","last_unstale":"2024-08-17T18:04:18.041729+0000","last_undegraded":"2024-08-17T18:04:18.041729+0000","last_fullsized":"2024-08-17T18:04:18.041729+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":1896,"log_dups_size":0,"ondisk_log_size":1896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:45.053022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124256256,"num_objects":1896,"num_object_clones":0,"num_object_copies":5688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1896,"num_write_kb":121344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"47.e","version":"1310'1897","reported_seq":1910,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:17.988545+0000","last_change":"2024-08-17T18:03:47.552274+0000","last_active":"2024-08-17T18:04:17.988545+0000","last_peered":"2024-08-17T18:04:17.988545+0000","last_clean":"2024-08-17T18:04:17.988545+0000","last_became_active":"2024-08-17T18:03:47.552091+0000","last_became_peered":"2024-08-17T18:03:47.552091+0000","last_unstale":"2024-08-17T18:04:17.988545+0000","last_undegraded":"2024-08-17T18:04:17.988545+0000","last_fullsized":"2024-08-17T18:04:17.988545+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":1897,"log_dups_size":0,"ondisk_log_size":1897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:12.712828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124321792,"num_objects":1897,"num_object_clones":0,"num_object_copies":5691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1897,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1897,"num_write_kb":121408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"47.d","version":"1310'1943","reported_seq":1964,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:18.185100+0000","last_change":"2024-08-17T18:03:47.554603+0000","last_active":"2024-08-17T18:04:18.185100+0000","last_peered":"2024-08-17T18:04:18.185100+0000","last_clean":"2024-08-17T18:04:18.185100+0000","last_became_active":"2024-08-17T18:03:47.554333+0000","last_became_peered":"2024-08-17T18:03:47.554333+0000","last_unstale":"2024-08-17T18:04:18.185100+0000","last_undegraded":"2024-08-17T18:04:18.185100+0000","last_fullsized":"2024-08-17T18:04:18.185100+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":1943,"log_dups_size":0,"ondisk_log_size":1943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:18:40.715146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127336448,"num_objects":1943,"num_object_clones":0,"num_object_copies":5829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1943,"num_write_kb":124352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"47.c","version":"1310'1780","reported_seq":1800,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:16.845457+0000","last_change":"2024-08-17T18:03:47.552428+0000","last_active":"2024-08-17T18:04:16.845457+0000","last_peered":"2024-08-17T18:04:16.845457+0000","last_clean":"2024-08-17T18:04:16.845457+0000","last_became_active":"2024-08-17T18:03:47.552150+0000","last_became_peered":"2024-08-17T18:03:47.552150+0000","last_unstale":"2024-08-17T18:04:16.845457+0000","last_undegraded":"2024-08-17T18:04:16.845457+0000","last_fullsized":"2024-08-17T18:04:16.845457+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":1780,"log_dups_size":0,"ondisk_log_size":1780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:17.973865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116654080,"num_objects":1780,"num_object_clones":0,"num_object_copies":5340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1780,"num_write_kb":113920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"47.b","version":"1310'1906","reported_seq":1919,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:17.995002+0000","last_change":"2024-08-17T18:03:47.552119+0000","last_active":"2024-08-17T18:04:17.995002+0000","last_peered":"2024-08-17T18:04:17.995002+0000","last_clean":"2024-08-17T18:04:17.995002+0000","last_became_active":"2024-08-17T18:03:47.551896+0000","last_became_peered":"2024-08-17T18:03:47.551896+0000","last_unstale":"2024-08-17T18:04:17.995002+0000","last_undegraded":"2024-08-17T18:04:17.995002+0000","last_fullsized":"2024-08-17T18:04:17.995002+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":1906,"log_dups_size":0,"ondisk_log_size":1906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:35.488974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124911616,"num_objects":1906,"num_object_clones":0,"num_object_copies":5718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1906,"num_write_kb":121984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"47.a","version":"1310'1877","reported_seq":1898,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:16.827668+0000","last_change":"2024-08-17T18:03:47.555308+0000","last_active":"2024-08-17T18:04:16.827668+0000","last_peered":"2024-08-17T18:04:16.827668+0000","last_clean":"2024-08-17T18:04:16.827668+0000","last_became_active":"2024-08-17T18:03:47.555021+0000","last_became_peered":"2024-08-17T18:03:47.555021+0000","last_unstale":"2024-08-17T18:04:16.827668+0000","last_undegraded":"2024-08-17T18:04:16.827668+0000","last_fullsized":"2024-08-17T18:04:16.827668+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":1877,"log_dups_size":0,"ondisk_log_size":1877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:14:58.052101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123011072,"num_objects":1877,"num_object_clones":0,"num_object_copies":5631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1877,"num_write_kb":120128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"47.9","version":"1310'1837","reported_seq":1858,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:16.843699+0000","last_change":"2024-08-17T18:03:47.163441+0000","last_active":"2024-08-17T18:04:16.843699+0000","last_peered":"2024-08-17T18:04:16.843699+0000","last_clean":"2024-08-17T18:04:16.843699+0000","last_became_active":"2024-08-17T18:03:47.163183+0000","last_became_peered":"2024-08-17T18:03:47.163183+0000","last_unstale":"2024-08-17T18:04:16.843699+0000","last_undegraded":"2024-08-17T18:04:16.843699+0000","last_fullsized":"2024-08-17T18:04:16.843699+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":1837,"log_dups_size":0,"ondisk_log_size":1837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:16:22.094167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120389632,"num_objects":1837,"num_object_clones":0,"num_object_copies":5511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1837,"num_write_kb":117568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"47.8","version":"1310'1838","reported_seq":1859,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:16.827387+0000","last_change":"2024-08-17T18:03:47.552328+0000","last_active":"2024-08-17T18:04:16.827387+0000","last_peered":"2024-08-17T18:04:16.827387+0000","last_clean":"2024-08-17T18:04:16.827387+0000","last_became_active":"2024-08-17T18:03:47.552038+0000","last_became_peered":"2024-08-17T18:03:47.552038+0000","last_unstale":"2024-08-17T18:04:16.827387+0000","last_undegraded":"2024-08-17T18:04:16.827387+0000","last_fullsized":"2024-08-17T18:04:16.827387+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":1838,"log_dups_size":0,"ondisk_log_size":1838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:19:48.727551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120455168,"num_objects":1838,"num_object_clones":0,"num_object_copies":5514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1838,"num_write_kb":117632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"47.7","version":"1310'1849","reported_seq":1870,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:16.845548+0000","last_change":"2024-08-17T18:03:47.554701+0000","last_active":"2024-08-17T18:04:16.845548+0000","last_peered":"2024-08-17T18:04:16.845548+0000","last_clean":"2024-08-17T18:04:16.845548+0000","last_became_active":"2024-08-17T18:03:47.554447+0000","last_became_peered":"2024-08-17T18:03:47.554447+0000","last_unstale":"2024-08-17T18:04:16.845548+0000","last_undegraded":"2024-08-17T18:04:16.845548+0000","last_fullsized":"2024-08-17T18:04:16.845548+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":1849,"log_dups_size":0,"ondisk_log_size":1849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:48:00.754957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121176064,"num_objects":1849,"num_object_clones":0,"num_object_copies":5547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1849,"num_write_kb":118336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"47.6","version":"1310'1962","reported_seq":1981,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:18.190823+0000","last_change":"2024-08-17T18:03:47.554302+0000","last_active":"2024-08-17T18:04:18.190823+0000","last_peered":"2024-08-17T18:04:18.190823+0000","last_clean":"2024-08-17T18:04:18.190823+0000","last_became_active":"2024-08-17T18:03:47.554007+0000","last_became_peered":"2024-08-17T18:03:47.554007+0000","last_unstale":"2024-08-17T18:04:18.190823+0000","last_undegraded":"2024-08-17T18:04:18.190823+0000","last_fullsized":"2024-08-17T18:04:18.190823+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":1962,"log_dups_size":0,"ondisk_log_size":1962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:33.388008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128581632,"num_objects":1962,"num_object_clones":0,"num_object_copies":5886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1962,"num_write_kb":125568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"47.5","version":"1310'2009","reported_seq":2030,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:17.975054+0000","last_change":"2024-08-17T18:03:47.165707+0000","last_active":"2024-08-17T18:04:17.975054+0000","last_peered":"2024-08-17T18:04:17.975054+0000","last_clean":"2024-08-17T18:04:17.975054+0000","last_became_active":"2024-08-17T18:03:47.164680+0000","last_became_peered":"2024-08-17T18:03:47.164680+0000","last_unstale":"2024-08-17T18:04:17.975054+0000","last_undegraded":"2024-08-17T18:04:17.975054+0000","last_fullsized":"2024-08-17T18:04:17.975054+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2009,"log_dups_size":0,"ondisk_log_size":2009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:23.646652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131661824,"num_objects":2009,"num_object_clones":0,"num_object_copies":6027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2009,"num_write_kb":128576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"47.4","version":"1310'2023","reported_seq":2044,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:17.985364+0000","last_change":"2024-08-17T18:03:47.552175+0000","last_active":"2024-08-17T18:04:17.985364+0000","last_peered":"2024-08-17T18:04:17.985364+0000","last_clean":"2024-08-17T18:04:17.985364+0000","last_became_active":"2024-08-17T18:03:47.551849+0000","last_became_peered":"2024-08-17T18:03:47.551849+0000","last_unstale":"2024-08-17T18:04:17.985364+0000","last_undegraded":"2024-08-17T18:04:17.985364+0000","last_fullsized":"2024-08-17T18:04:17.985364+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2023,"log_dups_size":0,"ondisk_log_size":2023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:53:18.186746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132579328,"num_objects":2023,"num_object_clones":0,"num_object_copies":6069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2023,"num_write_kb":129472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"47.3","version":"1310'1907","reported_seq":1928,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:18.162184+0000","last_change":"2024-08-17T18:03:47.554231+0000","last_active":"2024-08-17T18:04:18.162184+0000","last_peered":"2024-08-17T18:04:18.162184+0000","last_clean":"2024-08-17T18:04:18.162184+0000","last_became_active":"2024-08-17T18:03:47.553930+0000","last_became_peered":"2024-08-17T18:03:47.553930+0000","last_unstale":"2024-08-17T18:04:18.162184+0000","last_undegraded":"2024-08-17T18:04:18.162184+0000","last_fullsized":"2024-08-17T18:04:18.162184+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":1907,"log_dups_size":0,"ondisk_log_size":1907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:03.166631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124977152,"num_objects":1907,"num_object_clones":0,"num_object_copies":5721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1907,"num_write_kb":122048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"47.2","version":"1310'1812","reported_seq":1833,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:16.837508+0000","last_change":"2024-08-17T18:03:47.555345+0000","last_active":"2024-08-17T18:04:16.837508+0000","last_peered":"2024-08-17T18:04:16.837508+0000","last_clean":"2024-08-17T18:04:16.837508+0000","last_became_active":"2024-08-17T18:03:47.555078+0000","last_became_peered":"2024-08-17T18:03:47.555078+0000","last_unstale":"2024-08-17T18:04:16.837508+0000","last_undegraded":"2024-08-17T18:04:16.837508+0000","last_fullsized":"2024-08-17T18:04:16.837508+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":1812,"log_dups_size":0,"ondisk_log_size":1812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:01.368247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118751232,"num_objects":1812,"num_object_clones":0,"num_object_copies":5436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1812,"num_write_kb":115968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"47.1","version":"1310'1927","reported_seq":1948,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:18.140901+0000","last_change":"2024-08-17T18:03:47.556582+0000","last_active":"2024-08-17T18:04:18.140901+0000","last_peered":"2024-08-17T18:04:18.140901+0000","last_clean":"2024-08-17T18:04:18.140901+0000","last_became_active":"2024-08-17T18:03:47.555341+0000","last_became_peered":"2024-08-17T18:03:47.555341+0000","last_unstale":"2024-08-17T18:04:18.140901+0000","last_undegraded":"2024-08-17T18:04:18.140901+0000","last_fullsized":"2024-08-17T18:04:18.140901+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":1927,"log_dups_size":0,"ondisk_log_size":1927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:59.827708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126287872,"num_objects":1927,"num_object_clones":0,"num_object_copies":5781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1927,"num_write_kb":123328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"47.0","version":"1310'1876","reported_seq":1897,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:16.793000+0000","last_change":"2024-08-17T18:03:47.160129+0000","last_active":"2024-08-17T18:04:16.793000+0000","last_peered":"2024-08-17T18:04:16.793000+0000","last_clean":"2024-08-17T18:04:16.793000+0000","last_became_active":"2024-08-17T18:03:47.159815+0000","last_became_peered":"2024-08-17T18:03:47.159815+0000","last_unstale":"2024-08-17T18:04:16.793000+0000","last_undegraded":"2024-08-17T18:04:16.793000+0000","last_fullsized":"2024-08-17T18:04:16.793000+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":1876,"log_dups_size":0,"ondisk_log_size":1876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:16:58.124301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122945536,"num_objects":1876,"num_object_clones":0,"num_object_copies":5628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1876,"num_write_kb":120064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":2094,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208140+0000","last_change":"2024-08-17T18:02:53.922692+0000","last_active":"2024-08-17T18:04:02.208140+0000","last_peered":"2024-08-17T18:04:02.208140+0000","last_clean":"2024-08-17T18:04:02.208140+0000","last_became_active":"2024-08-17T18:02:53.897657+0000","last_became_peered":"2024-08-17T18:02:53.897657+0000","last_unstale":"2024-08-17T18:04:02.208140+0000","last_undegraded":"2024-08-17T18:04:02.208140+0000","last_fullsized":"2024-08-17T18:04:02.208140+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:05.276140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":522,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.211664+0000","last_change":"2024-08-17T18:02:53.898464+0000","last_active":"2024-08-17T18:04:02.211664+0000","last_peered":"2024-08-17T18:04:02.211664+0000","last_clean":"2024-08-17T18:04:02.211664+0000","last_became_active":"2024-08-17T18:02:53.896827+0000","last_became_peered":"2024-08-17T18:02:53.896827+0000","last_unstale":"2024-08-17T18:04:02.211664+0000","last_undegraded":"2024-08-17T18:04:02.211664+0000","last_fullsized":"2024-08-17T18:04:02.211664+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:59.352028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":2008,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.213956+0000","last_change":"2024-08-17T18:02:53.884107+0000","last_active":"2024-08-17T18:04:02.213956+0000","last_peered":"2024-08-17T18:04:02.213956+0000","last_clean":"2024-08-17T18:04:02.213956+0000","last_became_active":"2024-08-17T18:02:53.883708+0000","last_became_peered":"2024-08-17T18:02:53.883708+0000","last_unstale":"2024-08-17T18:04:02.213956+0000","last_undegraded":"2024-08-17T18:04:02.213956+0000","last_fullsized":"2024-08-17T18:04:02.213956+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:54:44.873165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2024,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.210036+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T18:04:02.210036+0000","last_peered":"2024-08-17T18:04:02.210036+0000","last_clean":"2024-08-17T18:04:02.210036+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T18:04:02.210036+0000","last_undegraded":"2024-08-17T18:04:02.210036+0000","last_fullsized":"2024-08-17T18:04:02.210036+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":84,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208592+0000","last_change":"2024-08-17T18:02:53.901529+0000","last_active":"2024-08-17T18:04:02.208592+0000","last_peered":"2024-08-17T18:04:02.208592+0000","last_clean":"2024-08-17T18:04:02.208592+0000","last_became_active":"2024-08-17T18:02:53.895916+0000","last_became_peered":"2024-08-17T18:02:53.895916+0000","last_unstale":"2024-08-17T18:04:02.208592+0000","last_undegraded":"2024-08-17T18:04:02.208592+0000","last_fullsized":"2024-08-17T18:04:02.208592+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:10.751360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":22,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.524944+0000","last_change":"2024-08-17T18:03:16.089811+0000","last_active":"2024-08-17T18:03:57.524944+0000","last_peered":"2024-08-17T18:03:57.524944+0000","last_clean":"2024-08-17T18:03:57.524944+0000","last_became_active":"2024-08-17T18:03:16.089571+0000","last_became_peered":"2024-08-17T18:03:16.089571+0000","last_unstale":"2024-08-17T18:03:57.524944+0000","last_undegraded":"2024-08-17T18:03:57.524944+0000","last_fullsized":"2024-08-17T18:03:57.524944+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:04.630361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":286,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208496+0000","last_change":"2024-08-17T18:02:53.888880+0000","last_active":"2024-08-17T18:04:02.208496+0000","last_peered":"2024-08-17T18:04:02.208496+0000","last_clean":"2024-08-17T18:04:02.208496+0000","last_became_active":"2024-08-17T18:02:53.880008+0000","last_became_peered":"2024-08-17T18:02:53.880008+0000","last_unstale":"2024-08-17T18:04:02.208496+0000","last_undegraded":"2024-08-17T18:04:02.208496+0000","last_fullsized":"2024-08-17T18:04:02.208496+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:20.923751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":795,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.214337+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T18:04:02.214337+0000","last_peered":"2024-08-17T18:04:02.214337+0000","last_clean":"2024-08-17T18:04:02.214337+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T18:04:02.214337+0000","last_undegraded":"2024-08-17T18:04:02.214337+0000","last_fullsized":"2024-08-17T18:04:02.214337+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":2039,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.210401+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T18:04:02.210401+0000","last_peered":"2024-08-17T18:04:02.210401+0000","last_clean":"2024-08-17T18:04:02.210401+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T18:04:02.210401+0000","last_undegraded":"2024-08-17T18:04:02.210401+0000","last_fullsized":"2024-08-17T18:04:02.210401+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":649,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.212483+0000","last_change":"2024-08-17T17:59:58.027770+0000","last_active":"2024-08-17T18:04:02.212483+0000","last_peered":"2024-08-17T18:04:02.212483+0000","last_clean":"2024-08-17T18:04:02.212483+0000","last_became_active":"2024-08-17T17:59:58.027251+0000","last_became_peered":"2024-08-17T17:59:58.027251+0000","last_unstale":"2024-08-17T18:04:02.212483+0000","last_undegraded":"2024-08-17T18:04:02.212483+0000","last_fullsized":"2024-08-17T18:04:02.212483+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:26.416690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":260,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.211068+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T18:04:02.211068+0000","last_peered":"2024-08-17T18:04:02.211068+0000","last_clean":"2024-08-17T18:04:02.211068+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T18:04:02.211068+0000","last_undegraded":"2024-08-17T18:04:02.211068+0000","last_fullsized":"2024-08-17T18:04:02.211068+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2019,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.215334+0000","last_change":"2024-08-17T18:02:53.884656+0000","last_active":"2024-08-17T18:04:02.215334+0000","last_peered":"2024-08-17T18:04:02.215334+0000","last_clean":"2024-08-17T18:04:02.215334+0000","last_became_active":"2024-08-17T18:02:53.884367+0000","last_became_peered":"2024-08-17T18:02:53.884367+0000","last_unstale":"2024-08-17T18:04:02.215334+0000","last_undegraded":"2024-08-17T18:04:02.215334+0000","last_fullsized":"2024-08-17T18:04:02.215334+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:56.231275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1922,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.210165+0000","last_change":"2024-08-17T18:02:53.873055+0000","last_active":"2024-08-17T18:04:02.210165+0000","last_peered":"2024-08-17T18:04:02.210165+0000","last_clean":"2024-08-17T18:04:02.210165+0000","last_became_active":"2024-08-17T18:02:53.870830+0000","last_became_peered":"2024-08-17T18:02:53.870830+0000","last_unstale":"2024-08-17T18:04:02.210165+0000","last_undegraded":"2024-08-17T18:04:02.210165+0000","last_fullsized":"2024-08-17T18:04:02.210165+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:59:06.504086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2062,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208990+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T18:04:02.208990+0000","last_peered":"2024-08-17T18:04:02.208990+0000","last_clean":"2024-08-17T18:04:02.208990+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T18:04:02.208990+0000","last_undegraded":"2024-08-17T18:04:02.208990+0000","last_fullsized":"2024-08-17T18:04:02.208990+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1862,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208555+0000","last_change":"2024-08-17T17:59:58.028639+0000","last_active":"2024-08-17T18:04:02.208555+0000","last_peered":"2024-08-17T18:04:02.208555+0000","last_clean":"2024-08-17T18:04:02.208555+0000","last_became_active":"2024-08-17T17:59:58.028314+0000","last_became_peered":"2024-08-17T17:59:58.028314+0000","last_unstale":"2024-08-17T18:04:02.208555+0000","last_undegraded":"2024-08-17T18:04:02.208555+0000","last_fullsized":"2024-08-17T18:04:02.208555+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:25:03.619052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1210,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.212778+0000","last_change":"2024-08-17T18:02:53.896420+0000","last_active":"2024-08-17T18:04:02.212778+0000","last_peered":"2024-08-17T18:04:02.212778+0000","last_clean":"2024-08-17T18:04:02.212778+0000","last_became_active":"2024-08-17T18:02:53.896100+0000","last_became_peered":"2024-08-17T18:02:53.896100+0000","last_unstale":"2024-08-17T18:04:02.212778+0000","last_undegraded":"2024-08-17T18:04:02.212778+0000","last_fullsized":"2024-08-17T18:04:02.212778+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:42:03.801967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2013,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.210036+0000","last_change":"2024-08-17T18:02:53.887620+0000","last_active":"2024-08-17T18:04:02.210036+0000","last_peered":"2024-08-17T18:04:02.210036+0000","last_clean":"2024-08-17T18:04:02.210036+0000","last_became_active":"2024-08-17T18:02:53.886848+0000","last_became_peered":"2024-08-17T18:02:53.886848+0000","last_unstale":"2024-08-17T18:04:02.210036+0000","last_undegraded":"2024-08-17T18:04:02.210036+0000","last_fullsized":"2024-08-17T18:04:02.210036+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:52.227176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2063,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208231+0000","last_change":"2024-08-17T18:02:53.898305+0000","last_active":"2024-08-17T18:04:02.208231+0000","last_peered":"2024-08-17T18:04:02.208231+0000","last_clean":"2024-08-17T18:04:02.208231+0000","last_became_active":"2024-08-17T18:02:53.897963+0000","last_became_peered":"2024-08-17T18:02:53.897963+0000","last_unstale":"2024-08-17T18:04:02.208231+0000","last_undegraded":"2024-08-17T18:04:02.208231+0000","last_fullsized":"2024-08-17T18:04:02.208231+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:32.466527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1969,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.209968+0000","last_change":"2024-08-17T18:02:53.896767+0000","last_active":"2024-08-17T18:04:02.209968+0000","last_peered":"2024-08-17T18:04:02.209968+0000","last_clean":"2024-08-17T18:04:02.209968+0000","last_became_active":"2024-08-17T18:02:53.896306+0000","last_became_peered":"2024-08-17T18:02:53.896306+0000","last_unstale":"2024-08-17T18:04:02.209968+0000","last_undegraded":"2024-08-17T18:04:02.209968+0000","last_fullsized":"2024-08-17T18:04:02.209968+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:59:14.985211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":603,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.525221+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T18:03:57.525221+0000","last_peered":"2024-08-17T18:03:57.525221+0000","last_clean":"2024-08-17T18:03:57.525221+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T18:03:57.525221+0000","last_undegraded":"2024-08-17T18:03:57.525221+0000","last_fullsized":"2024-08-17T18:03:57.525221+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1280,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.525056+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T18:03:57.525056+0000","last_peered":"2024-08-17T18:03:57.525056+0000","last_clean":"2024-08-17T18:03:57.525056+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T18:03:57.525056+0000","last_undegraded":"2024-08-17T18:03:57.525056+0000","last_fullsized":"2024-08-17T18:03:57.525056+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2068,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.211368+0000","last_change":"2024-08-17T18:02:53.882334+0000","last_active":"2024-08-17T18:04:02.211368+0000","last_peered":"2024-08-17T18:04:02.211368+0000","last_clean":"2024-08-17T18:04:02.211368+0000","last_became_active":"2024-08-17T18:02:53.881815+0000","last_became_peered":"2024-08-17T18:02:53.881815+0000","last_unstale":"2024-08-17T18:04:02.211368+0000","last_undegraded":"2024-08-17T18:04:02.211368+0000","last_fullsized":"2024-08-17T18:04:02.211368+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:39.671800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":461,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.525162+0000","last_change":"2024-08-17T18:02:53.912837+0000","last_active":"2024-08-17T18:03:57.525162+0000","last_peered":"2024-08-17T18:03:57.525162+0000","last_clean":"2024-08-17T18:03:57.525162+0000","last_became_active":"2024-08-17T18:02:53.883152+0000","last_became_peered":"2024-08-17T18:02:53.883152+0000","last_unstale":"2024-08-17T18:03:57.525162+0000","last_undegraded":"2024-08-17T18:03:57.525162+0000","last_fullsized":"2024-08-17T18:03:57.525162+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:46.261612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":36,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.210084+0000","last_change":"2024-08-17T18:03:16.091944+0000","last_active":"2024-08-17T18:04:02.210084+0000","last_peered":"2024-08-17T18:04:02.210084+0000","last_clean":"2024-08-17T18:04:02.210084+0000","last_became_active":"2024-08-17T18:03:16.091314+0000","last_became_peered":"2024-08-17T18:03:16.091314+0000","last_unstale":"2024-08-17T18:04:02.210084+0000","last_undegraded":"2024-08-17T18:04:02.210084+0000","last_fullsized":"2024-08-17T18:04:02.210084+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:39.248867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1200,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.207979+0000","last_change":"2024-08-17T18:02:53.890198+0000","last_active":"2024-08-17T18:04:02.207979+0000","last_peered":"2024-08-17T18:04:02.207979+0000","last_clean":"2024-08-17T18:04:02.207979+0000","last_became_active":"2024-08-17T18:02:53.887007+0000","last_became_peered":"2024-08-17T18:02:53.887007+0000","last_unstale":"2024-08-17T18:04:02.207979+0000","last_undegraded":"2024-08-17T18:04:02.207979+0000","last_fullsized":"2024-08-17T18:04:02.207979+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:25.555909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":534,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.525216+0000","last_change":"2024-08-17T18:02:53.873945+0000","last_active":"2024-08-17T18:03:57.525216+0000","last_peered":"2024-08-17T18:03:57.525216+0000","last_clean":"2024-08-17T18:03:57.525216+0000","last_became_active":"2024-08-17T18:02:53.873642+0000","last_became_peered":"2024-08-17T18:02:53.873642+0000","last_unstale":"2024-08-17T18:03:57.525216+0000","last_undegraded":"2024-08-17T18:03:57.525216+0000","last_fullsized":"2024-08-17T18:03:57.525216+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:41.375927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":106,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208600+0000","last_change":"2024-08-17T18:03:16.104851+0000","last_active":"2024-08-17T18:04:02.208600+0000","last_peered":"2024-08-17T18:04:02.208600+0000","last_clean":"2024-08-17T18:04:02.208600+0000","last_became_active":"2024-08-17T18:03:16.091641+0000","last_became_peered":"2024-08-17T18:03:16.091641+0000","last_unstale":"2024-08-17T18:04:02.208600+0000","last_undegraded":"2024-08-17T18:04:02.208600+0000","last_fullsized":"2024-08-17T18:04:02.208600+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.222166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1993,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.210133+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T18:04:02.210133+0000","last_peered":"2024-08-17T18:04:02.210133+0000","last_clean":"2024-08-17T18:04:02.210133+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T18:04:02.210133+0000","last_undegraded":"2024-08-17T18:04:02.210133+0000","last_fullsized":"2024-08-17T18:04:02.210133+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":2074,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208704+0000","last_change":"2024-08-17T18:02:53.896607+0000","last_active":"2024-08-17T18:04:02.208704+0000","last_peered":"2024-08-17T18:04:02.208704+0000","last_clean":"2024-08-17T18:04:02.208704+0000","last_became_active":"2024-08-17T18:02:53.896092+0000","last_became_peered":"2024-08-17T18:02:53.896092+0000","last_unstale":"2024-08-17T18:04:02.208704+0000","last_undegraded":"2024-08-17T18:04:02.208704+0000","last_fullsized":"2024-08-17T18:04:02.208704+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:38.309846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1995,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.214952+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T18:04:02.214952+0000","last_peered":"2024-08-17T18:04:02.214952+0000","last_clean":"2024-08-17T18:04:02.214952+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T18:04:02.214952+0000","last_undegraded":"2024-08-17T18:04:02.214952+0000","last_fullsized":"2024-08-17T18:04:02.214952+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2027,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.210499+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T18:04:02.210499+0000","last_peered":"2024-08-17T18:04:02.210499+0000","last_clean":"2024-08-17T18:04:02.210499+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T18:04:02.210499+0000","last_undegraded":"2024-08-17T18:04:02.210499+0000","last_fullsized":"2024-08-17T18:04:02.210499+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1300,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.525217+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T18:03:57.525217+0000","last_peered":"2024-08-17T18:03:57.525217+0000","last_clean":"2024-08-17T18:03:57.525217+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T18:03:57.525217+0000","last_undegraded":"2024-08-17T18:03:57.525217+0000","last_fullsized":"2024-08-17T18:03:57.525217+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":2074,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208634+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T18:04:02.208634+0000","last_peered":"2024-08-17T18:04:02.208634+0000","last_clean":"2024-08-17T18:04:02.208634+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T18:04:02.208634+0000","last_undegraded":"2024-08-17T18:04:02.208634+0000","last_fullsized":"2024-08-17T18:04:02.208634+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":260,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.210965+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T18:04:02.210965+0000","last_peered":"2024-08-17T18:04:02.210965+0000","last_clean":"2024-08-17T18:04:02.210965+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T18:04:02.210965+0000","last_undegraded":"2024-08-17T18:04:02.210965+0000","last_fullsized":"2024-08-17T18:04:02.210965+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2093,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208544+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T18:04:02.208544+0000","last_peered":"2024-08-17T18:04:02.208544+0000","last_clean":"2024-08-17T18:04:02.208544+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T18:04:02.208544+0000","last_undegraded":"2024-08-17T18:04:02.208544+0000","last_fullsized":"2024-08-17T18:04:02.208544+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":836,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.214686+0000","last_change":"2024-08-17T18:02:53.881500+0000","last_active":"2024-08-17T18:04:02.214686+0000","last_peered":"2024-08-17T18:04:02.214686+0000","last_clean":"2024-08-17T18:04:02.214686+0000","last_became_active":"2024-08-17T18:02:53.880651+0000","last_became_peered":"2024-08-17T18:02:53.880651+0000","last_unstale":"2024-08-17T18:04:02.214686+0000","last_undegraded":"2024-08-17T18:04:02.214686+0000","last_fullsized":"2024-08-17T18:04:02.214686+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:03.044635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":22,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.524885+0000","last_change":"2024-08-17T18:03:16.090238+0000","last_active":"2024-08-17T18:03:57.524885+0000","last_peered":"2024-08-17T18:03:57.524885+0000","last_clean":"2024-08-17T18:03:57.524885+0000","last_became_active":"2024-08-17T18:03:16.090003+0000","last_became_peered":"2024-08-17T18:03:16.090003+0000","last_unstale":"2024-08-17T18:03:57.524885+0000","last_undegraded":"2024-08-17T18:03:57.524885+0000","last_fullsized":"2024-08-17T18:03:57.524885+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:49.706675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1409,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.210250+0000","last_change":"2024-08-17T18:02:53.895969+0000","last_active":"2024-08-17T18:04:02.210250+0000","last_peered":"2024-08-17T18:04:02.210250+0000","last_clean":"2024-08-17T18:04:02.210250+0000","last_became_active":"2024-08-17T18:02:53.895681+0000","last_became_peered":"2024-08-17T18:02:53.895681+0000","last_unstale":"2024-08-17T18:04:02.210250+0000","last_undegraded":"2024-08-17T18:04:02.210250+0000","last_fullsized":"2024-08-17T18:04:02.210250+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:55.508072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":334,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.211266+0000","last_change":"2024-08-17T18:02:53.923285+0000","last_active":"2024-08-17T18:04:02.211266+0000","last_peered":"2024-08-17T18:04:02.211266+0000","last_clean":"2024-08-17T18:04:02.211266+0000","last_became_active":"2024-08-17T18:02:53.895787+0000","last_became_peered":"2024-08-17T18:02:53.895787+0000","last_unstale":"2024-08-17T18:04:02.211266+0000","last_undegraded":"2024-08-17T18:04:02.211266+0000","last_fullsized":"2024-08-17T18:04:02.211266+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:11.411637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":2084,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.211857+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T18:04:02.211857+0000","last_peered":"2024-08-17T18:04:02.211857+0000","last_clean":"2024-08-17T18:04:02.211857+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T18:04:02.211857+0000","last_undegraded":"2024-08-17T18:04:02.211857+0000","last_fullsized":"2024-08-17T18:04:02.211857+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2069,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.211361+0000","last_change":"2024-08-17T18:02:53.890366+0000","last_active":"2024-08-17T18:04:02.211361+0000","last_peered":"2024-08-17T18:04:02.211361+0000","last_clean":"2024-08-17T18:04:02.211361+0000","last_became_active":"2024-08-17T18:02:53.889898+0000","last_became_peered":"2024-08-17T18:02:53.889898+0000","last_unstale":"2024-08-17T18:04:02.211361+0000","last_undegraded":"2024-08-17T18:04:02.211361+0000","last_fullsized":"2024-08-17T18:04:02.211361+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:53.298525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1910,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.210114+0000","last_change":"2024-08-17T17:59:58.030150+0000","last_active":"2024-08-17T18:04:02.210114+0000","last_peered":"2024-08-17T18:04:02.210114+0000","last_clean":"2024-08-17T18:04:02.210114+0000","last_became_active":"2024-08-17T17:59:58.029407+0000","last_became_peered":"2024-08-17T17:59:58.029407+0000","last_unstale":"2024-08-17T18:04:02.210114+0000","last_undegraded":"2024-08-17T18:04:02.210114+0000","last_fullsized":"2024-08-17T18:04:02.210114+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:31.632793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2088,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.209230+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T18:04:02.209230+0000","last_peered":"2024-08-17T18:04:02.209230+0000","last_clean":"2024-08-17T18:04:02.209230+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T18:04:02.209230+0000","last_undegraded":"2024-08-17T18:04:02.209230+0000","last_fullsized":"2024-08-17T18:04:02.209230+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2068,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208069+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T18:04:02.208069+0000","last_peered":"2024-08-17T18:04:02.208069+0000","last_clean":"2024-08-17T18:04:02.208069+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T18:04:02.208069+0000","last_undegraded":"2024-08-17T18:04:02.208069+0000","last_fullsized":"2024-08-17T18:04:02.208069+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1307,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.525160+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T18:03:57.525160+0000","last_peered":"2024-08-17T18:03:57.525160+0000","last_clean":"2024-08-17T18:03:57.525160+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T18:03:57.525160+0000","last_undegraded":"2024-08-17T18:03:57.525160+0000","last_fullsized":"2024-08-17T18:03:57.525160+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1162,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.525113+0000","last_change":"2024-08-17T18:02:53.879600+0000","last_active":"2024-08-17T18:03:57.525113+0000","last_peered":"2024-08-17T18:03:57.525113+0000","last_clean":"2024-08-17T18:03:57.525113+0000","last_became_active":"2024-08-17T18:02:53.879236+0000","last_became_peered":"2024-08-17T18:02:53.879236+0000","last_unstale":"2024-08-17T18:03:57.525113+0000","last_undegraded":"2024-08-17T18:03:57.525113+0000","last_fullsized":"2024-08-17T18:03:57.525113+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:09:08.995893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1984,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.214000+0000","last_change":"2024-08-17T18:02:53.883991+0000","last_active":"2024-08-17T18:04:02.214000+0000","last_peered":"2024-08-17T18:04:02.214000+0000","last_clean":"2024-08-17T18:04:02.214000+0000","last_became_active":"2024-08-17T18:02:53.883668+0000","last_became_peered":"2024-08-17T18:02:53.883668+0000","last_unstale":"2024-08-17T18:04:02.214000+0000","last_undegraded":"2024-08-17T18:04:02.214000+0000","last_fullsized":"2024-08-17T18:04:02.214000+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:49.144723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":69,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.525131+0000","last_change":"2024-08-17T17:59:58.027683+0000","last_active":"2024-08-17T18:03:57.525131+0000","last_peered":"2024-08-17T18:03:57.525131+0000","last_clean":"2024-08-17T18:03:57.525131+0000","last_became_active":"2024-08-17T17:59:58.027358+0000","last_became_peered":"2024-08-17T17:59:58.027358+0000","last_unstale":"2024-08-17T18:03:57.525131+0000","last_undegraded":"2024-08-17T18:03:57.525131+0000","last_fullsized":"2024-08-17T18:03:57.525131+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:04.758259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1645,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.524639+0000","last_change":"2024-08-17T18:02:56.855080+0000","last_active":"2024-08-17T18:03:57.524639+0000","last_peered":"2024-08-17T18:03:57.524639+0000","last_clean":"2024-08-17T18:03:57.524639+0000","last_became_active":"2024-08-17T18:02:56.854831+0000","last_became_peered":"2024-08-17T18:02:56.854831+0000","last_unstale":"2024-08-17T18:03:57.524639+0000","last_undegraded":"2024-08-17T18:03:57.524639+0000","last_fullsized":"2024-08-17T18:03:57.524639+0000","mapping_epoch":1286,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:25:24.111782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2068,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.212910+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T18:04:02.212910+0000","last_peered":"2024-08-17T18:04:02.212910+0000","last_clean":"2024-08-17T18:04:02.212910+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T18:04:02.212910+0000","last_undegraded":"2024-08-17T18:04:02.212910+0000","last_fullsized":"2024-08-17T18:04:02.212910+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2009,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.210158+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T18:04:02.210158+0000","last_peered":"2024-08-17T18:04:02.210158+0000","last_clean":"2024-08-17T18:04:02.210158+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T18:04:02.210158+0000","last_undegraded":"2024-08-17T18:04:02.210158+0000","last_fullsized":"2024-08-17T18:04:02.210158+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":522,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208495+0000","last_change":"2024-08-17T18:02:53.897151+0000","last_active":"2024-08-17T18:04:02.208495+0000","last_peered":"2024-08-17T18:04:02.208495+0000","last_clean":"2024-08-17T18:04:02.208495+0000","last_became_active":"2024-08-17T18:02:53.895480+0000","last_became_peered":"2024-08-17T18:02:53.895480+0000","last_unstale":"2024-08-17T18:04:02.208495+0000","last_undegraded":"2024-08-17T18:04:02.208495+0000","last_fullsized":"2024-08-17T18:04:02.208495+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:17.316423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":547,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.524699+0000","last_change":"2024-08-17T17:59:58.021264+0000","last_active":"2024-08-17T18:03:57.524699+0000","last_peered":"2024-08-17T18:03:57.524699+0000","last_clean":"2024-08-17T18:03:57.524699+0000","last_became_active":"2024-08-17T17:59:58.020957+0000","last_became_peered":"2024-08-17T17:59:58.020957+0000","last_unstale":"2024-08-17T18:03:57.524699+0000","last_undegraded":"2024-08-17T18:03:57.524699+0000","last_fullsized":"2024-08-17T18:03:57.524699+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:42.190270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":617,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208706+0000","last_change":"2024-08-17T18:02:53.883783+0000","last_active":"2024-08-17T18:04:02.208706+0000","last_peered":"2024-08-17T18:04:02.208706+0000","last_clean":"2024-08-17T18:04:02.208706+0000","last_became_active":"2024-08-17T18:02:53.883489+0000","last_became_peered":"2024-08-17T18:02:53.883489+0000","last_unstale":"2024-08-17T18:04:02.208706+0000","last_undegraded":"2024-08-17T18:04:02.208706+0000","last_fullsized":"2024-08-17T18:04:02.208706+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:08.849092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1206,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.211465+0000","last_change":"2024-08-17T18:02:53.896125+0000","last_active":"2024-08-17T18:04:02.211465+0000","last_peered":"2024-08-17T18:04:02.211465+0000","last_clean":"2024-08-17T18:04:02.211465+0000","last_became_active":"2024-08-17T18:02:53.895789+0000","last_became_peered":"2024-08-17T18:02:53.895789+0000","last_unstale":"2024-08-17T18:04:02.211465+0000","last_undegraded":"2024-08-17T18:04:02.211465+0000","last_fullsized":"2024-08-17T18:04:02.211465+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:25:25.798463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":786,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.214378+0000","last_change":"2024-08-17T18:02:53.883815+0000","last_active":"2024-08-17T18:04:02.214378+0000","last_peered":"2024-08-17T18:04:02.214378+0000","last_clean":"2024-08-17T18:04:02.214378+0000","last_became_active":"2024-08-17T18:02:53.883374+0000","last_became_peered":"2024-08-17T18:02:53.883374+0000","last_unstale":"2024-08-17T18:04:02.214378+0000","last_undegraded":"2024-08-17T18:04:02.214378+0000","last_fullsized":"2024-08-17T18:04:02.214378+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:37:52.718473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2005,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.214430+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T18:04:02.214430+0000","last_peered":"2024-08-17T18:04:02.214430+0000","last_clean":"2024-08-17T18:04:02.214430+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T18:04:02.214430+0000","last_undegraded":"2024-08-17T18:04:02.214430+0000","last_fullsized":"2024-08-17T18:04:02.214430+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2101,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.210626+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T18:04:02.210626+0000","last_peered":"2024-08-17T18:04:02.210626+0000","last_clean":"2024-08-17T18:04:02.210626+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T18:04:02.210626+0000","last_undegraded":"2024-08-17T18:04:02.210626+0000","last_fullsized":"2024-08-17T18:04:02.210626+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":793,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208397+0000","last_change":"2024-08-17T18:03:16.090929+0000","last_active":"2024-08-17T18:04:02.208397+0000","last_peered":"2024-08-17T18:04:02.208397+0000","last_clean":"2024-08-17T18:04:02.208397+0000","last_became_active":"2024-08-17T18:03:16.090554+0000","last_became_peered":"2024-08-17T18:03:16.090554+0000","last_unstale":"2024-08-17T18:04:02.208397+0000","last_undegraded":"2024-08-17T18:04:02.208397+0000","last_fullsized":"2024-08-17T18:04:02.208397+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:01.783375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":515,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.211173+0000","last_change":"2024-08-17T18:02:53.881999+0000","last_active":"2024-08-17T18:04:02.211173+0000","last_peered":"2024-08-17T18:04:02.211173+0000","last_clean":"2024-08-17T18:04:02.211173+0000","last_became_active":"2024-08-17T18:02:53.881681+0000","last_became_peered":"2024-08-17T18:02:53.881681+0000","last_unstale":"2024-08-17T18:04:02.211173+0000","last_undegraded":"2024-08-17T18:04:02.211173+0000","last_fullsized":"2024-08-17T18:04:02.211173+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:39:51.334407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1273,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.525217+0000","last_change":"2024-08-17T18:02:53.890396+0000","last_active":"2024-08-17T18:03:57.525217+0000","last_peered":"2024-08-17T18:03:57.525217+0000","last_clean":"2024-08-17T18:03:57.525217+0000","last_became_active":"2024-08-17T18:02:53.879118+0000","last_became_peered":"2024-08-17T18:02:53.879118+0000","last_unstale":"2024-08-17T18:03:57.525217+0000","last_undegraded":"2024-08-17T18:03:57.525217+0000","last_fullsized":"2024-08-17T18:03:57.525217+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:14.074623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":803,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.209008+0000","last_change":"2024-08-17T18:02:53.891213+0000","last_active":"2024-08-17T18:04:02.209008+0000","last_peered":"2024-08-17T18:04:02.209008+0000","last_clean":"2024-08-17T18:04:02.209008+0000","last_became_active":"2024-08-17T18:02:53.890853+0000","last_became_peered":"2024-08-17T18:02:53.890853+0000","last_unstale":"2024-08-17T18:04:02.209008+0000","last_undegraded":"2024-08-17T18:04:02.209008+0000","last_fullsized":"2024-08-17T18:04:02.209008+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:19.916697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1277,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.525300+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T18:03:57.525300+0000","last_peered":"2024-08-17T18:03:57.525300+0000","last_clean":"2024-08-17T18:03:57.525300+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T18:03:57.525300+0000","last_undegraded":"2024-08-17T18:03:57.525300+0000","last_fullsized":"2024-08-17T18:03:57.525300+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":171,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.524588+0000","last_change":"2024-08-17T18:02:53.887617+0000","last_active":"2024-08-17T18:03:57.524588+0000","last_peered":"2024-08-17T18:03:57.524588+0000","last_clean":"2024-08-17T18:03:57.524588+0000","last_became_active":"2024-08-17T18:02:53.887266+0000","last_became_peered":"2024-08-17T18:02:53.887266+0000","last_unstale":"2024-08-17T18:03:57.524588+0000","last_undegraded":"2024-08-17T18:03:57.524588+0000","last_fullsized":"2024-08-17T18:03:57.524588+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:57:02.775040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1293,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.525104+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T18:03:57.525104+0000","last_peered":"2024-08-17T18:03:57.525104+0000","last_clean":"2024-08-17T18:03:57.525104+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T18:03:57.525104+0000","last_undegraded":"2024-08-17T18:03:57.525104+0000","last_fullsized":"2024-08-17T18:03:57.525104+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":85,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.207979+0000","last_change":"2024-08-17T18:02:53.919476+0000","last_active":"2024-08-17T18:04:02.207979+0000","last_peered":"2024-08-17T18:04:02.207979+0000","last_clean":"2024-08-17T18:04:02.207979+0000","last_became_active":"2024-08-17T18:02:53.896368+0000","last_became_peered":"2024-08-17T18:02:53.896368+0000","last_unstale":"2024-08-17T18:04:02.207979+0000","last_undegraded":"2024-08-17T18:04:02.207979+0000","last_fullsized":"2024-08-17T18:04:02.207979+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:10:30.848886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2006,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.214472+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T18:04:02.214472+0000","last_peered":"2024-08-17T18:04:02.214472+0000","last_clean":"2024-08-17T18:04:02.214472+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T18:04:02.214472+0000","last_undegraded":"2024-08-17T18:04:02.214472+0000","last_fullsized":"2024-08-17T18:04:02.214472+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1264,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.524807+0000","last_change":"2024-08-17T18:02:53.879170+0000","last_active":"2024-08-17T18:03:57.524807+0000","last_peered":"2024-08-17T18:03:57.524807+0000","last_clean":"2024-08-17T18:03:57.524807+0000","last_became_active":"2024-08-17T18:02:53.878913+0000","last_became_peered":"2024-08-17T18:02:53.878913+0000","last_unstale":"2024-08-17T18:03:57.524807+0000","last_undegraded":"2024-08-17T18:03:57.524807+0000","last_fullsized":"2024-08-17T18:03:57.524807+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:55:37.600470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2059,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.208845+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T18:04:02.208845+0000","last_peered":"2024-08-17T18:04:02.208845+0000","last_clean":"2024-08-17T18:04:02.208845+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T18:04:02.208845+0000","last_undegraded":"2024-08-17T18:04:02.208845+0000","last_fullsized":"2024-08-17T18:04:02.208845+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2060,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.210985+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T18:04:02.210985+0000","last_peered":"2024-08-17T18:04:02.210985+0000","last_clean":"2024-08-17T18:04:02.210985+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T18:04:02.210985+0000","last_undegraded":"2024-08-17T18:04:02.210985+0000","last_fullsized":"2024-08-17T18:04:02.210985+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":687,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.524887+0000","last_change":"2024-08-17T18:03:16.091420+0000","last_active":"2024-08-17T18:03:57.524887+0000","last_peered":"2024-08-17T18:03:57.524887+0000","last_clean":"2024-08-17T18:03:57.524887+0000","last_became_active":"2024-08-17T18:03:16.091113+0000","last_became_peered":"2024-08-17T18:03:16.091113+0000","last_unstale":"2024-08-17T18:03:57.524887+0000","last_undegraded":"2024-08-17T18:03:57.524887+0000","last_fullsized":"2024-08-17T18:03:57.524887+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:47.770911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1295,"reported_epoch":1309,"state":"active+clean","last_fresh":"2024-08-17T18:03:57.525063+0000","last_change":"2024-08-17T18:02:53.893927+0000","last_active":"2024-08-17T18:03:57.525063+0000","last_peered":"2024-08-17T18:03:57.525063+0000","last_clean":"2024-08-17T18:03:57.525063+0000","last_became_active":"2024-08-17T18:02:53.883188+0000","last_became_peered":"2024-08-17T18:02:53.883188+0000","last_unstale":"2024-08-17T18:03:57.525063+0000","last_undegraded":"2024-08-17T18:03:57.525063+0000","last_fullsized":"2024-08-17T18:03:57.525063+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:19.296965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2006,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.214522+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T18:04:02.214522+0000","last_peered":"2024-08-17T18:04:02.214522+0000","last_clean":"2024-08-17T18:04:02.214522+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T18:04:02.214522+0000","last_undegraded":"2024-08-17T18:04:02.214522+0000","last_fullsized":"2024-08-17T18:04:02.214522+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":47,"num_pg":16,"stat_sum":{"num_bytes":1988296704,"num_objects":30339,"num_object_clones":0,"num_object_copies":91017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30339,"num_write_kb":1941696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6093340672,"data_stored":6093340672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":30339,"ondisk_log_size":30339,"up":48,"acting":48,"num_store_stats":5},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":95,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":10948,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234521,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1016044,"kb_used_data":714532,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":92729108,"statfs":{"total":95995035648,"available":94954606592,"internally_reserved":0,"allocated":731680768,"data_stored":727571163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568496156,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757794,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326408,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1582364,"kb_used_data":1120340,"kb_used_omap":57,"kb_used_meta":461894,"kb_avail":92162788,"statfs":{"total":95995035648,"available":94374694912,"internally_reserved":0,"allocated":1147228160,"data_stored":1143130196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":472980219},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405647,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2006560,"kb_used_data":1531480,"kb_used_omap":61,"kb_used_meta":475010,"kb_avail":91738592,"statfs":{"total":95995035648,"available":93940318208,"internally_reserved":0,"allocated":1568235520,"data_stored":1564138059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":486411192},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452181,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1533912,"kb_used_data":1139600,"kb_used_omap":54,"kb_used_meta":394249,"kb_avail":92211240,"statfs":{"total":95995035648,"available":94424309760,"internally_reserved":0,"allocated":1166950400,"data_stored":1162856368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":403711374},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474872,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1880628,"kb_used_data":1373676,"kb_used_omap":119,"kb_used_meta":506824,"kb_avail":91864524,"statfs":{"total":95995035648,"available":94069272576,"internally_reserved":0,"allocated":1406644224,"data_stored":1402527036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":518988510},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019429,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":47,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1410400256,"data_stored":1410400256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":47,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1207304192,"data_stored":1207304192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":47,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1572143104,"data_stored":1572143104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":47,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1150222336,"data_stored":1150222336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":47,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":753270784,"data_stored":753270784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:04:20.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:04:20.790 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:20.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:20.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:20.892 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:20.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:20.915 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:20.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:21.016 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:21.038 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:21.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:21.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:21.140 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:21.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:21.162 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:21.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:21.263 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:21.319 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:21.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:21.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:21.421 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:21.457 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:21.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:21.559 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:21.580 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:21.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:21.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:21.658 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 21 16 35872 35856 106.703 84.8125 0.0105687 0.00936512 2024-08-17T18:04:21.681 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:21.719 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:21.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:21.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:21.821 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:21.883 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:04:21.889 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:04:21.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:21.903 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:21.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:22.004 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:22.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:22.057 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:22.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:22.158 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:22.180 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:22.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:22.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:22.282 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:22.303 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:22.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:22.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:22.405 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:22.441 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:22.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:22.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:22.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:22.582 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:22.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:22.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:22.658 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 22 16 37179 37163 105.565 81.6875 0.0109677 0.00946557 2024-08-17T18:04:22.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:22 smithi130 ceph-mon[105521]: pgmap v2204: 89 pgs: 89 active+clean; 1.9 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 95 MiB/s wr, 1.51k op/s 2024-08-17T18:04:22.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:22 smithi130 ceph-mon[110068]: pgmap v2204: 89 pgs: 89 active+clean; 1.9 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 95 MiB/s wr, 1.51k op/s 2024-08-17T18:04:22.684 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:22.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:22.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:22.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:22.822 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:22.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:22 smithi179 ceph-mon[93519]: pgmap v2204: 89 pgs: 89 active+clean; 1.9 GiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 95 MiB/s wr, 1.51k op/s 2024-08-17T18:04:22.867 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:22.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:22.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:22.969 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:22.990 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:22.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:22.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:23.092 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:23.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:23.131 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:23.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:23.232 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:23.248 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:04:23.253 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:23.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:23.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:23.355 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:23.377 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:23.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:23.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:23.479 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:23.524 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:23.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:23.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:23.626 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:23.649 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:23.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:23.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:23.658 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 23 16 38394 38378 104.275 75.9375 0.00754184 0.00958334 2024-08-17T18:04:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:23 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2563582813' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:04:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:23 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2563582813' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:04:23.751 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:23.757 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:04:23.757 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:04:23.757 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:04:23.757 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:04:23.757 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:04:23.757 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:04:23.758 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:04:23.758 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:04:23.759 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:04:23.759 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:04:23.759 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:04:23.759 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:04:23.759 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:04:23.759 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:04:23.760 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:04:23.760 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:04:23.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:23.773 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:23.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:23 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2563582813' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:04:23.874 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:23.915 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:23.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:23.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:24.017 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:24.056 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:24.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:24.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:24.158 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:24.181 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:24.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:24.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:24.282 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:24.304 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:24.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:24.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:24.406 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:24.447 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:24.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:24.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:24.549 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:24.590 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:24.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:24.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:24.658 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 24 16 39683 39667 103.287 80.5625 0.00691226 0.0096754 2024-08-17T18:04:24.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:24 smithi130 ceph-mon[105521]: pgmap v2205: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T18:04:24.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:24 smithi130 ceph-mon[110068]: pgmap v2205: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T18:04:24.692 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:24.713 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:24.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:24.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:24.815 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:24.837 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:24.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:24.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:24.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:24 smithi179 ceph-mon[93519]: pgmap v2205: 89 pgs: 89 active+clean; 2.3 GiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T18:04:24.939 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:24.978 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:24.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:24.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:25.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:25.104 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:25.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:25.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:25.205 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:25.243 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:25.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:25.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:25.345 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:25.369 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:25.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:25.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:25.471 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:25.503 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:25.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:25.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:25.604 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:25.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:25.627 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:25.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:25.659 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 25 16 40633 40617 101.53 59.375 0.0203915 0.00984212 2024-08-17T18:04:25.728 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:25.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:25.751 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:25.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:25.852 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:25.889 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:25.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:25.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:25.991 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:26.026 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:26.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:26.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:26.127 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:26.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:26.149 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:26.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:26.250 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:26.272 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:26.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:26.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:26.374 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:26.413 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:26.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:26.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:26.515 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:26.536 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:26.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:26.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:26.638 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:26.659 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 26 16 41738 41722 100.281 69.0625 0.0101432 0.00996527 2024-08-17T18:04:26.660 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:26.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:26.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:26.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:26 smithi130 ceph-mon[105521]: pgmap v2206: 89 pgs: 89 active+clean; 2.3 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T18:04:26.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:26 smithi130 ceph-mon[110068]: pgmap v2206: 89 pgs: 89 active+clean; 2.3 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T18:04:26.762 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:26.783 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:26.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:26.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:26.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:26 smithi179 ceph-mon[93519]: pgmap v2206: 89 pgs: 89 active+clean; 2.3 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T18:04:26.885 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:26.906 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:26.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:26.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:27.008 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:27.030 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:27.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:27.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:27.132 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:27.154 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:27.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:27.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:27.255 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:27.293 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:27.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:27.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:27.395 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:27.416 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:27.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:27.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:27.518 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:27.540 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:27.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:27.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:27.642 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:27.659 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 27 16 42942 42926 99.3539 75.25 0.0212407 0.0100583 2024-08-17T18:04:27.663 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:27.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:27.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:27.765 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:27.805 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:27.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:27.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:27.906 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:27.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:27.956 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:27.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:28.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:28.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:28.109 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:28.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:28.210 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:28.244 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:28.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:28.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:28.346 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:28.395 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:28.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:28.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:28.497 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:28.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:28.519 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:28.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:28.582 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:28 smithi130 ceph-mon[105521]: pgmap v2207: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-17T18:04:28.582 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:04:28.582 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:04:28.582 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:28 smithi130 ceph-mon[110068]: pgmap v2207: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-17T18:04:28.583 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:04:28.583 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:04:28.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:28 smithi179 ceph-mon[93519]: pgmap v2207: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 438 GiB / 447 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-08-17T18:04:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:04:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:04:28.620 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:28.659 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 28 16 44119 44103 98.4326 73.5625 0.0316438 0.0101519 2024-08-17T18:04:28.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:28.660 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:28.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:28.671 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:04:28.671 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:04:28.761 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:28.804 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:28.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:28.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:28.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:28.927 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:28.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:28.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:29.029 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:29.050 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:29.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:29.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:29.152 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:29.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:29.173 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:29.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:29.274 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:29.313 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:29.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:29.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:29.415 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:29.455 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:29.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:29.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:29.557 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:29.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:29.579 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:29.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:29.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:29 smithi179 ceph-mon[93519]: from='client.55990 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:04:29.659 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 29 16 45320 45304 97.6265 75.0625 0.00482307 0.01023 2024-08-17T18:04:29.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:29 smithi130 ceph-mon[105521]: from='client.55990 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:04:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:29 smithi130 ceph-mon[110068]: from='client.55990 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:04:29.680 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:29.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:29.702 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:29.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:29.803 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:29.824 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:29.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:29.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:29.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T18:04:29.925 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:29.947 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:29.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:29.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:30.049 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:30.100 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:04:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:04:29] "GET /metrics HTTP/1.1" 200 36776 "" "Prometheus/2.51.0" 2024-08-17T18:04:30.120 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:30.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:30.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:04:30.165 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:04:30.222 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:30.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:30.244 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:30.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:30.345 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:30.386 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:30.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:30.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:30.449 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:04:30.489 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:30.510 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:30.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:30.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:30 smithi179 ceph-mon[93519]: pgmap v2208: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T18:04:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:04:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:04:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:04:30.613 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:30.634 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:30.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:30.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:30.659 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 30 16 46410 46394 96.643 68.125 0.0140296 0.010341 2024-08-17T18:04:30.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:30 smithi130 ceph-mon[105521]: pgmap v2208: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T18:04:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:30 smithi130 ceph-mon[110068]: pgmap v2208: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-08-17T18:04:30.736 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:30.778 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:30.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:30.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:30.880 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:30.901 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:30.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:30.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:30.932 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2207,"stamp":"2024-08-17T18:04:27.251169+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2594183117,"num_objects":39601,"num_object_clones":0,"num_object_copies":118803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39601,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":39853,"num_write_kb":2535247,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":2372458,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39782,"ondisk_log_size":39782,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":9595396,"kb_used_data":7429212,"kb_used_omap":293,"kb_used_meta":2165850,"kb_avail":459130364,"statfs":{"total":479975178240,"available":470149492736,"internally_reserved":0,"allocated":7607513088,"data_stored":7586959948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":300937,"internal_metadata":2217830519},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,7,3,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":869335040,"num_objects":13265,"num_object_clones":0,"num_object_copies":39795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13265,"num_write_kb":848960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003683"},"pg_stats":[{"pgid":"47.f","version":"1311'2311","reported_seq":2325,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:23.012892+0000","last_change":"2024-08-17T18:03:47.164644+0000","last_active":"2024-08-17T18:04:23.012892+0000","last_peered":"2024-08-17T18:04:23.012892+0000","last_clean":"2024-08-17T18:04:23.012892+0000","last_became_active":"2024-08-17T18:03:47.164347+0000","last_became_peered":"2024-08-17T18:03:47.164347+0000","last_unstale":"2024-08-17T18:04:23.012892+0000","last_undegraded":"2024-08-17T18:04:23.012892+0000","last_fullsized":"2024-08-17T18:04:23.012892+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2311,"log_dups_size":0,"ondisk_log_size":2311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:45.053022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151453696,"num_objects":2311,"num_object_clones":0,"num_object_copies":6933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2311,"num_write_kb":147904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"47.e","version":"1311'2335","reported_seq":2349,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:23.013304+0000","last_change":"2024-08-17T18:03:47.552274+0000","last_active":"2024-08-17T18:04:23.013304+0000","last_peered":"2024-08-17T18:04:23.013304+0000","last_clean":"2024-08-17T18:04:23.013304+0000","last_became_active":"2024-08-17T18:03:47.552091+0000","last_became_peered":"2024-08-17T18:03:47.552091+0000","last_unstale":"2024-08-17T18:04:23.013304+0000","last_undegraded":"2024-08-17T18:04:23.013304+0000","last_fullsized":"2024-08-17T18:04:23.013304+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2335,"log_dups_size":0,"ondisk_log_size":2335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:12.712828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153026560,"num_objects":2335,"num_object_clones":0,"num_object_copies":7005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2335,"num_write_kb":149440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"47.d","version":"1311'2360","reported_seq":2383,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:23.139818+0000","last_change":"2024-08-17T18:03:47.554603+0000","last_active":"2024-08-17T18:04:23.139818+0000","last_peered":"2024-08-17T18:04:23.139818+0000","last_clean":"2024-08-17T18:04:23.139818+0000","last_became_active":"2024-08-17T18:03:47.554333+0000","last_became_peered":"2024-08-17T18:03:47.554333+0000","last_unstale":"2024-08-17T18:04:23.139818+0000","last_undegraded":"2024-08-17T18:04:23.139818+0000","last_fullsized":"2024-08-17T18:04:23.139818+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2360,"log_dups_size":0,"ondisk_log_size":2360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:18:40.715146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154664960,"num_objects":2360,"num_object_clones":0,"num_object_copies":7080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2360,"num_write_kb":151040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"47.c","version":"1311'2541","reported_seq":2564,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:26.872213+0000","last_change":"2024-08-17T18:03:47.552428+0000","last_active":"2024-08-17T18:04:26.872213+0000","last_peered":"2024-08-17T18:04:26.872213+0000","last_clean":"2024-08-17T18:04:26.872213+0000","last_became_active":"2024-08-17T18:03:47.552150+0000","last_became_peered":"2024-08-17T18:03:47.552150+0000","last_unstale":"2024-08-17T18:04:26.872213+0000","last_undegraded":"2024-08-17T18:04:26.872213+0000","last_fullsized":"2024-08-17T18:04:26.872213+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2541,"log_dups_size":0,"ondisk_log_size":2541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:06:17.973865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166526976,"num_objects":2541,"num_object_clones":0,"num_object_copies":7623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2541,"num_write_kb":162624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"47.b","version":"1311'2253","reported_seq":2267,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:23.022076+0000","last_change":"2024-08-17T18:03:47.552119+0000","last_active":"2024-08-17T18:04:23.022076+0000","last_peered":"2024-08-17T18:04:23.022076+0000","last_clean":"2024-08-17T18:04:23.022076+0000","last_became_active":"2024-08-17T18:03:47.551896+0000","last_became_peered":"2024-08-17T18:03:47.551896+0000","last_unstale":"2024-08-17T18:04:23.022076+0000","last_undegraded":"2024-08-17T18:04:23.022076+0000","last_fullsized":"2024-08-17T18:04:23.022076+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2253,"log_dups_size":0,"ondisk_log_size":2253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:35.488974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147652608,"num_objects":2253,"num_object_clones":0,"num_object_copies":6759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2253,"num_write_kb":144192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"47.a","version":"1311'2644","reported_seq":2667,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:26.835604+0000","last_change":"2024-08-17T18:03:47.555308+0000","last_active":"2024-08-17T18:04:26.835604+0000","last_peered":"2024-08-17T18:04:26.835604+0000","last_clean":"2024-08-17T18:04:26.835604+0000","last_became_active":"2024-08-17T18:03:47.555021+0000","last_became_peered":"2024-08-17T18:03:47.555021+0000","last_unstale":"2024-08-17T18:04:26.835604+0000","last_undegraded":"2024-08-17T18:04:26.835604+0000","last_fullsized":"2024-08-17T18:04:26.835604+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2644,"log_dups_size":0,"ondisk_log_size":2644,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:14:58.052101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173277184,"num_objects":2644,"num_object_clones":0,"num_object_copies":7932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2644,"num_write_kb":169216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"47.9","version":"1311'2595","reported_seq":2618,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:26.873846+0000","last_change":"2024-08-17T18:03:47.163441+0000","last_active":"2024-08-17T18:04:26.873846+0000","last_peered":"2024-08-17T18:04:26.873846+0000","last_clean":"2024-08-17T18:04:26.873846+0000","last_became_active":"2024-08-17T18:03:47.163183+0000","last_became_peered":"2024-08-17T18:03:47.163183+0000","last_unstale":"2024-08-17T18:04:26.873846+0000","last_undegraded":"2024-08-17T18:04:26.873846+0000","last_fullsized":"2024-08-17T18:04:26.873846+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2595,"log_dups_size":0,"ondisk_log_size":2595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:16:22.094167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170065920,"num_objects":2595,"num_object_clones":0,"num_object_copies":7785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2595,"num_write_kb":166080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"47.8","version":"1311'2612","reported_seq":2635,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:26.867847+0000","last_change":"2024-08-17T18:03:47.552328+0000","last_active":"2024-08-17T18:04:26.867847+0000","last_peered":"2024-08-17T18:04:26.867847+0000","last_clean":"2024-08-17T18:04:26.867847+0000","last_became_active":"2024-08-17T18:03:47.552038+0000","last_became_peered":"2024-08-17T18:03:47.552038+0000","last_unstale":"2024-08-17T18:04:26.867847+0000","last_undegraded":"2024-08-17T18:04:26.867847+0000","last_fullsized":"2024-08-17T18:04:26.867847+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2612,"log_dups_size":0,"ondisk_log_size":2612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:19:48.727551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171180032,"num_objects":2612,"num_object_clones":0,"num_object_copies":7836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2612,"num_write_kb":167168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"47.7","version":"1311'2681","reported_seq":2704,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:26.870696+0000","last_change":"2024-08-17T18:03:47.554701+0000","last_active":"2024-08-17T18:04:26.870696+0000","last_peered":"2024-08-17T18:04:26.870696+0000","last_clean":"2024-08-17T18:04:26.870696+0000","last_became_active":"2024-08-17T18:03:47.554447+0000","last_became_peered":"2024-08-17T18:03:47.554447+0000","last_unstale":"2024-08-17T18:04:26.870696+0000","last_undegraded":"2024-08-17T18:04:26.870696+0000","last_fullsized":"2024-08-17T18:04:26.870696+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2681,"log_dups_size":0,"ondisk_log_size":2681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:48:00.754957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175702016,"num_objects":2681,"num_object_clones":0,"num_object_copies":8043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2681,"num_write_kb":171584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"47.6","version":"1311'2379","reported_seq":2402,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:23.168328+0000","last_change":"2024-08-17T18:03:47.554302+0000","last_active":"2024-08-17T18:04:23.168328+0000","last_peered":"2024-08-17T18:04:23.168328+0000","last_clean":"2024-08-17T18:04:23.168328+0000","last_became_active":"2024-08-17T18:03:47.554007+0000","last_became_peered":"2024-08-17T18:03:47.554007+0000","last_unstale":"2024-08-17T18:04:23.168328+0000","last_undegraded":"2024-08-17T18:04:23.168328+0000","last_fullsized":"2024-08-17T18:04:23.168328+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2379,"log_dups_size":0,"ondisk_log_size":2379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:33.388008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155910144,"num_objects":2379,"num_object_clones":0,"num_object_copies":7137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2379,"num_write_kb":152256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"47.5","version":"1311'2434","reported_seq":2457,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:22.987539+0000","last_change":"2024-08-17T18:03:47.165707+0000","last_active":"2024-08-17T18:04:22.987539+0000","last_peered":"2024-08-17T18:04:22.987539+0000","last_clean":"2024-08-17T18:04:22.987539+0000","last_became_active":"2024-08-17T18:03:47.164680+0000","last_became_peered":"2024-08-17T18:03:47.164680+0000","last_unstale":"2024-08-17T18:04:22.987539+0000","last_undegraded":"2024-08-17T18:04:22.987539+0000","last_fullsized":"2024-08-17T18:04:22.987539+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2434,"log_dups_size":0,"ondisk_log_size":2434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:23.646652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159514624,"num_objects":2434,"num_object_clones":0,"num_object_copies":7302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2434,"num_write_kb":155776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"47.4","version":"1311'2450","reported_seq":2473,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:22.997574+0000","last_change":"2024-08-17T18:03:47.552175+0000","last_active":"2024-08-17T18:04:22.997574+0000","last_peered":"2024-08-17T18:04:22.997574+0000","last_clean":"2024-08-17T18:04:22.997574+0000","last_became_active":"2024-08-17T18:03:47.551849+0000","last_became_peered":"2024-08-17T18:03:47.551849+0000","last_unstale":"2024-08-17T18:04:22.997574+0000","last_undegraded":"2024-08-17T18:04:22.997574+0000","last_fullsized":"2024-08-17T18:04:22.997574+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2450,"log_dups_size":0,"ondisk_log_size":2450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:53:18.186746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160563200,"num_objects":2450,"num_object_clones":0,"num_object_copies":7350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2450,"num_write_kb":156800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"47.3","version":"1311'2332","reported_seq":2355,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:23.191625+0000","last_change":"2024-08-17T18:03:47.554231+0000","last_active":"2024-08-17T18:04:23.191625+0000","last_peered":"2024-08-17T18:04:23.191625+0000","last_clean":"2024-08-17T18:04:23.191625+0000","last_became_active":"2024-08-17T18:03:47.553930+0000","last_became_peered":"2024-08-17T18:03:47.553930+0000","last_unstale":"2024-08-17T18:04:23.191625+0000","last_undegraded":"2024-08-17T18:04:23.191625+0000","last_fullsized":"2024-08-17T18:04:23.191625+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2332,"log_dups_size":0,"ondisk_log_size":2332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:03.166631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152829952,"num_objects":2332,"num_object_clones":0,"num_object_copies":6996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2332,"num_write_kb":149248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"47.2","version":"1311'2635","reported_seq":2658,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:26.871346+0000","last_change":"2024-08-17T18:03:47.555345+0000","last_active":"2024-08-17T18:04:26.871346+0000","last_peered":"2024-08-17T18:04:26.871346+0000","last_clean":"2024-08-17T18:04:26.871346+0000","last_became_active":"2024-08-17T18:03:47.555078+0000","last_became_peered":"2024-08-17T18:03:47.555078+0000","last_unstale":"2024-08-17T18:04:26.871346+0000","last_undegraded":"2024-08-17T18:04:26.871346+0000","last_fullsized":"2024-08-17T18:04:26.871346+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2635,"log_dups_size":0,"ondisk_log_size":2635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:01.368247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172687360,"num_objects":2635,"num_object_clones":0,"num_object_copies":7905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2635,"num_write_kb":168640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"47.1","version":"1311'2374","reported_seq":2397,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:23.191815+0000","last_change":"2024-08-17T18:03:47.556582+0000","last_active":"2024-08-17T18:04:23.191815+0000","last_peered":"2024-08-17T18:04:23.191815+0000","last_clean":"2024-08-17T18:04:23.191815+0000","last_became_active":"2024-08-17T18:03:47.555341+0000","last_became_peered":"2024-08-17T18:03:47.555341+0000","last_unstale":"2024-08-17T18:04:23.191815+0000","last_undegraded":"2024-08-17T18:04:23.191815+0000","last_fullsized":"2024-08-17T18:04:23.191815+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2374,"log_dups_size":0,"ondisk_log_size":2374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:59.827708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155582464,"num_objects":2374,"num_object_clones":0,"num_object_copies":7122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2374,"num_write_kb":151936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"47.0","version":"1311'2639","reported_seq":2662,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:26.875811+0000","last_change":"2024-08-17T18:03:47.160129+0000","last_active":"2024-08-17T18:04:26.875811+0000","last_peered":"2024-08-17T18:04:26.875811+0000","last_clean":"2024-08-17T18:04:26.875811+0000","last_became_active":"2024-08-17T18:03:47.159815+0000","last_became_peered":"2024-08-17T18:03:47.159815+0000","last_unstale":"2024-08-17T18:04:26.875811+0000","last_undegraded":"2024-08-17T18:04:26.875811+0000","last_fullsized":"2024-08-17T18:04:26.875811+0000","mapping_epoch":1305,"log_start":"0'0","ondisk_log_start":"0'0","created":1305,"last_epoch_clean":1306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:03:46.139590+0000","last_clean_scrub_stamp":"2024-08-17T18:03:46.139590+0000","objects_scrubbed":0,"log_size":2639,"log_dups_size":0,"ondisk_log_size":2639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:16:58.124301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172949504,"num_objects":2639,"num_object_clones":0,"num_object_copies":7917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2639,"num_write_kb":168896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":2096,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.416489+0000","last_change":"2024-08-17T18:02:53.922692+0000","last_active":"2024-08-17T18:04:19.416489+0000","last_peered":"2024-08-17T18:04:19.416489+0000","last_clean":"2024-08-17T18:04:19.416489+0000","last_became_active":"2024-08-17T18:02:53.897657+0000","last_became_peered":"2024-08-17T18:02:53.897657+0000","last_unstale":"2024-08-17T18:04:19.416489+0000","last_undegraded":"2024-08-17T18:04:19.416489+0000","last_fullsized":"2024-08-17T18:04:19.416489+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:05.276140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":524,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.408535+0000","last_change":"2024-08-17T18:02:53.898464+0000","last_active":"2024-08-17T18:04:19.408535+0000","last_peered":"2024-08-17T18:04:19.408535+0000","last_clean":"2024-08-17T18:04:19.408535+0000","last_became_active":"2024-08-17T18:02:53.896827+0000","last_became_peered":"2024-08-17T18:02:53.896827+0000","last_unstale":"2024-08-17T18:04:19.408535+0000","last_undegraded":"2024-08-17T18:04:19.408535+0000","last_fullsized":"2024-08-17T18:04:19.408535+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:59.352028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":2010,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.426869+0000","last_change":"2024-08-17T18:02:53.884107+0000","last_active":"2024-08-17T18:04:19.426869+0000","last_peered":"2024-08-17T18:04:19.426869+0000","last_clean":"2024-08-17T18:04:19.426869+0000","last_became_active":"2024-08-17T18:02:53.883708+0000","last_became_peered":"2024-08-17T18:02:53.883708+0000","last_unstale":"2024-08-17T18:04:19.426869+0000","last_undegraded":"2024-08-17T18:04:19.426869+0000","last_fullsized":"2024-08-17T18:04:19.426869+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:54:44.873165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2026,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.428037+0000","last_change":"2024-08-17T17:48:39.656191+0000","last_active":"2024-08-17T18:04:19.428037+0000","last_peered":"2024-08-17T18:04:19.428037+0000","last_clean":"2024-08-17T18:04:19.428037+0000","last_became_active":"2024-08-17T17:48:39.655310+0000","last_became_peered":"2024-08-17T17:48:39.655310+0000","last_unstale":"2024-08-17T18:04:19.428037+0000","last_undegraded":"2024-08-17T18:04:19.428037+0000","last_fullsized":"2024-08-17T18:04:19.428037+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:29:12.996401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":86,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.407419+0000","last_change":"2024-08-17T18:02:53.901529+0000","last_active":"2024-08-17T18:04:19.407419+0000","last_peered":"2024-08-17T18:04:19.407419+0000","last_clean":"2024-08-17T18:04:19.407419+0000","last_became_active":"2024-08-17T18:02:53.895916+0000","last_became_peered":"2024-08-17T18:02:53.895916+0000","last_unstale":"2024-08-17T18:04:19.407419+0000","last_undegraded":"2024-08-17T18:04:19.407419+0000","last_fullsized":"2024-08-17T18:04:19.407419+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:10.751360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":23,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.201886+0000","last_change":"2024-08-17T18:03:16.089811+0000","last_active":"2024-08-17T18:04:02.201886+0000","last_peered":"2024-08-17T18:04:02.201886+0000","last_clean":"2024-08-17T18:04:02.201886+0000","last_became_active":"2024-08-17T18:03:16.089571+0000","last_became_peered":"2024-08-17T18:03:16.089571+0000","last_unstale":"2024-08-17T18:04:02.201886+0000","last_undegraded":"2024-08-17T18:04:02.201886+0000","last_fullsized":"2024-08-17T18:04:02.201886+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:04.630361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":288,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.410139+0000","last_change":"2024-08-17T18:02:53.888880+0000","last_active":"2024-08-17T18:04:19.410139+0000","last_peered":"2024-08-17T18:04:19.410139+0000","last_clean":"2024-08-17T18:04:19.410139+0000","last_became_active":"2024-08-17T18:02:53.880008+0000","last_became_peered":"2024-08-17T18:02:53.880008+0000","last_unstale":"2024-08-17T18:04:19.410139+0000","last_undegraded":"2024-08-17T18:04:19.410139+0000","last_fullsized":"2024-08-17T18:04:19.410139+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:20.923751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":797,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.423558+0000","last_change":"2024-08-17T17:49:45.693081+0000","last_active":"2024-08-17T18:04:19.423558+0000","last_peered":"2024-08-17T18:04:19.423558+0000","last_clean":"2024-08-17T18:04:19.423558+0000","last_became_active":"2024-08-17T17:49:45.692731+0000","last_became_peered":"2024-08-17T17:49:45.692731+0000","last_unstale":"2024-08-17T18:04:19.423558+0000","last_undegraded":"2024-08-17T18:04:19.423558+0000","last_fullsized":"2024-08-17T18:04:19.423558+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:04.557350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":2041,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.414091+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T18:04:19.414091+0000","last_peered":"2024-08-17T18:04:19.414091+0000","last_clean":"2024-08-17T18:04:19.414091+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T18:04:19.414091+0000","last_undegraded":"2024-08-17T18:04:19.414091+0000","last_fullsized":"2024-08-17T18:04:19.414091+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":651,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.410316+0000","last_change":"2024-08-17T17:59:58.027770+0000","last_active":"2024-08-17T18:04:19.410316+0000","last_peered":"2024-08-17T18:04:19.410316+0000","last_clean":"2024-08-17T18:04:19.410316+0000","last_became_active":"2024-08-17T17:59:58.027251+0000","last_became_peered":"2024-08-17T17:59:58.027251+0000","last_unstale":"2024-08-17T18:04:19.410316+0000","last_undegraded":"2024-08-17T18:04:19.410316+0000","last_fullsized":"2024-08-17T18:04:19.410316+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:26.416690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":262,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.407179+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T18:04:19.407179+0000","last_peered":"2024-08-17T18:04:19.407179+0000","last_clean":"2024-08-17T18:04:19.407179+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T18:04:19.407179+0000","last_undegraded":"2024-08-17T18:04:19.407179+0000","last_fullsized":"2024-08-17T18:04:19.407179+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2022,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.426044+0000","last_change":"2024-08-17T18:02:53.884656+0000","last_active":"2024-08-17T18:04:19.426044+0000","last_peered":"2024-08-17T18:04:19.426044+0000","last_clean":"2024-08-17T18:04:19.426044+0000","last_became_active":"2024-08-17T18:02:53.884367+0000","last_became_peered":"2024-08-17T18:02:53.884367+0000","last_unstale":"2024-08-17T18:04:19.426044+0000","last_undegraded":"2024-08-17T18:04:19.426044+0000","last_fullsized":"2024-08-17T18:04:19.426044+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:56.231275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1924,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.428209+0000","last_change":"2024-08-17T18:02:53.873055+0000","last_active":"2024-08-17T18:04:19.428209+0000","last_peered":"2024-08-17T18:04:19.428209+0000","last_clean":"2024-08-17T18:04:19.428209+0000","last_became_active":"2024-08-17T18:02:53.870830+0000","last_became_peered":"2024-08-17T18:02:53.870830+0000","last_unstale":"2024-08-17T18:04:19.428209+0000","last_undegraded":"2024-08-17T18:04:19.428209+0000","last_fullsized":"2024-08-17T18:04:19.428209+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:59:06.504086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2064,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.413171+0000","last_change":"2024-08-17T17:48:39.659125+0000","last_active":"2024-08-17T18:04:19.413171+0000","last_peered":"2024-08-17T18:04:19.413171+0000","last_clean":"2024-08-17T18:04:19.413171+0000","last_became_active":"2024-08-17T17:48:39.658278+0000","last_became_peered":"2024-08-17T17:48:39.658278+0000","last_unstale":"2024-08-17T18:04:19.413171+0000","last_undegraded":"2024-08-17T18:04:19.413171+0000","last_fullsized":"2024-08-17T18:04:19.413171+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:51:21.254262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1864,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.412940+0000","last_change":"2024-08-17T17:59:58.028639+0000","last_active":"2024-08-17T18:04:19.412940+0000","last_peered":"2024-08-17T18:04:19.412940+0000","last_clean":"2024-08-17T18:04:19.412940+0000","last_became_active":"2024-08-17T17:59:58.028314+0000","last_became_peered":"2024-08-17T17:59:58.028314+0000","last_unstale":"2024-08-17T18:04:19.412940+0000","last_undegraded":"2024-08-17T18:04:19.412940+0000","last_fullsized":"2024-08-17T18:04:19.412940+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:25:03.619052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1212,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.409312+0000","last_change":"2024-08-17T18:02:53.896420+0000","last_active":"2024-08-17T18:04:19.409312+0000","last_peered":"2024-08-17T18:04:19.409312+0000","last_clean":"2024-08-17T18:04:19.409312+0000","last_became_active":"2024-08-17T18:02:53.896100+0000","last_became_peered":"2024-08-17T18:02:53.896100+0000","last_unstale":"2024-08-17T18:04:19.409312+0000","last_undegraded":"2024-08-17T18:04:19.409312+0000","last_fullsized":"2024-08-17T18:04:19.409312+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:42:03.801967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2015,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.424557+0000","last_change":"2024-08-17T18:02:53.887620+0000","last_active":"2024-08-17T18:04:19.424557+0000","last_peered":"2024-08-17T18:04:19.424557+0000","last_clean":"2024-08-17T18:04:19.424557+0000","last_became_active":"2024-08-17T18:02:53.886848+0000","last_became_peered":"2024-08-17T18:02:53.886848+0000","last_unstale":"2024-08-17T18:04:19.424557+0000","last_undegraded":"2024-08-17T18:04:19.424557+0000","last_fullsized":"2024-08-17T18:04:19.424557+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:52.227176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2065,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.416590+0000","last_change":"2024-08-17T18:02:53.898305+0000","last_active":"2024-08-17T18:04:19.416590+0000","last_peered":"2024-08-17T18:04:19.416590+0000","last_clean":"2024-08-17T18:04:19.416590+0000","last_became_active":"2024-08-17T18:02:53.897963+0000","last_became_peered":"2024-08-17T18:02:53.897963+0000","last_unstale":"2024-08-17T18:04:19.416590+0000","last_undegraded":"2024-08-17T18:04:19.416590+0000","last_fullsized":"2024-08-17T18:04:19.416590+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:32.466527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1971,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.427865+0000","last_change":"2024-08-17T18:02:53.896767+0000","last_active":"2024-08-17T18:04:19.427865+0000","last_peered":"2024-08-17T18:04:19.427865+0000","last_clean":"2024-08-17T18:04:19.427865+0000","last_became_active":"2024-08-17T18:02:53.896306+0000","last_became_peered":"2024-08-17T18:02:53.896306+0000","last_unstale":"2024-08-17T18:04:19.427865+0000","last_undegraded":"2024-08-17T18:04:19.427865+0000","last_fullsized":"2024-08-17T18:04:19.427865+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:59:14.985211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":604,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.202289+0000","last_change":"2024-08-17T17:48:07.630158+0000","last_active":"2024-08-17T18:04:02.202289+0000","last_peered":"2024-08-17T18:04:02.202289+0000","last_clean":"2024-08-17T18:04:02.202289+0000","last_became_active":"2024-08-17T17:48:07.629787+0000","last_became_peered":"2024-08-17T17:48:07.629787+0000","last_unstale":"2024-08-17T18:04:02.202289+0000","last_undegraded":"2024-08-17T18:04:02.202289+0000","last_fullsized":"2024-08-17T18:04:02.202289+0000","mapping_epoch":1172,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:26.075585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1281,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.202282+0000","last_change":"2024-08-17T17:49:45.714051+0000","last_active":"2024-08-17T18:04:02.202282+0000","last_peered":"2024-08-17T18:04:02.202282+0000","last_clean":"2024-08-17T18:04:02.202282+0000","last_became_active":"2024-08-17T17:49:45.676397+0000","last_became_peered":"2024-08-17T17:49:45.676397+0000","last_unstale":"2024-08-17T18:04:02.202282+0000","last_undegraded":"2024-08-17T18:04:02.202282+0000","last_fullsized":"2024-08-17T18:04:02.202282+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:41.099971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2070,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.407448+0000","last_change":"2024-08-17T18:02:53.882334+0000","last_active":"2024-08-17T18:04:19.407448+0000","last_peered":"2024-08-17T18:04:19.407448+0000","last_clean":"2024-08-17T18:04:19.407448+0000","last_became_active":"2024-08-17T18:02:53.881815+0000","last_became_peered":"2024-08-17T18:02:53.881815+0000","last_unstale":"2024-08-17T18:04:19.407448+0000","last_undegraded":"2024-08-17T18:04:19.407448+0000","last_fullsized":"2024-08-17T18:04:19.407448+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:39.671800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":462,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.202092+0000","last_change":"2024-08-17T18:02:53.912837+0000","last_active":"2024-08-17T18:04:02.202092+0000","last_peered":"2024-08-17T18:04:02.202092+0000","last_clean":"2024-08-17T18:04:02.202092+0000","last_became_active":"2024-08-17T18:02:53.883152+0000","last_became_peered":"2024-08-17T18:02:53.883152+0000","last_unstale":"2024-08-17T18:04:02.202092+0000","last_undegraded":"2024-08-17T18:04:02.202092+0000","last_fullsized":"2024-08-17T18:04:02.202092+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:46.261612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":38,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.413762+0000","last_change":"2024-08-17T18:03:16.091944+0000","last_active":"2024-08-17T18:04:19.413762+0000","last_peered":"2024-08-17T18:04:19.413762+0000","last_clean":"2024-08-17T18:04:19.413762+0000","last_became_active":"2024-08-17T18:03:16.091314+0000","last_became_peered":"2024-08-17T18:03:16.091314+0000","last_unstale":"2024-08-17T18:04:19.413762+0000","last_undegraded":"2024-08-17T18:04:19.413762+0000","last_fullsized":"2024-08-17T18:04:19.413762+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:39.248867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1202,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.413348+0000","last_change":"2024-08-17T18:02:53.890198+0000","last_active":"2024-08-17T18:04:19.413348+0000","last_peered":"2024-08-17T18:04:19.413348+0000","last_clean":"2024-08-17T18:04:19.413348+0000","last_became_active":"2024-08-17T18:02:53.887007+0000","last_became_peered":"2024-08-17T18:02:53.887007+0000","last_unstale":"2024-08-17T18:04:19.413348+0000","last_undegraded":"2024-08-17T18:04:19.413348+0000","last_fullsized":"2024-08-17T18:04:19.413348+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:25.555909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":535,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.202549+0000","last_change":"2024-08-17T18:02:53.873945+0000","last_active":"2024-08-17T18:04:02.202549+0000","last_peered":"2024-08-17T18:04:02.202549+0000","last_clean":"2024-08-17T18:04:02.202549+0000","last_became_active":"2024-08-17T18:02:53.873642+0000","last_became_peered":"2024-08-17T18:02:53.873642+0000","last_unstale":"2024-08-17T18:04:02.202549+0000","last_undegraded":"2024-08-17T18:04:02.202549+0000","last_fullsized":"2024-08-17T18:04:02.202549+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:41.375927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":108,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.408211+0000","last_change":"2024-08-17T18:03:16.104851+0000","last_active":"2024-08-17T18:04:19.408211+0000","last_peered":"2024-08-17T18:04:19.408211+0000","last_clean":"2024-08-17T18:04:19.408211+0000","last_became_active":"2024-08-17T18:03:16.091641+0000","last_became_peered":"2024-08-17T18:03:16.091641+0000","last_unstale":"2024-08-17T18:04:19.408211+0000","last_undegraded":"2024-08-17T18:04:19.408211+0000","last_fullsized":"2024-08-17T18:04:19.408211+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.222166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1995,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.424741+0000","last_change":"2024-08-17T17:48:39.654697+0000","last_active":"2024-08-17T18:04:19.424741+0000","last_peered":"2024-08-17T18:04:19.424741+0000","last_clean":"2024-08-17T18:04:19.424741+0000","last_became_active":"2024-08-17T17:48:39.654374+0000","last_became_peered":"2024-08-17T17:48:39.654374+0000","last_unstale":"2024-08-17T18:04:19.424741+0000","last_undegraded":"2024-08-17T18:04:19.424741+0000","last_fullsized":"2024-08-17T18:04:19.424741+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:27.926793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":2076,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.408312+0000","last_change":"2024-08-17T18:02:53.896607+0000","last_active":"2024-08-17T18:04:19.408312+0000","last_peered":"2024-08-17T18:04:19.408312+0000","last_clean":"2024-08-17T18:04:19.408312+0000","last_became_active":"2024-08-17T18:02:53.896092+0000","last_became_peered":"2024-08-17T18:02:53.896092+0000","last_unstale":"2024-08-17T18:04:19.408312+0000","last_undegraded":"2024-08-17T18:04:19.408312+0000","last_fullsized":"2024-08-17T18:04:19.408312+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:50:38.309846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":1997,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.424476+0000","last_change":"2024-08-17T17:48:39.664642+0000","last_active":"2024-08-17T18:04:19.424476+0000","last_peered":"2024-08-17T18:04:19.424476+0000","last_clean":"2024-08-17T18:04:19.424476+0000","last_became_active":"2024-08-17T17:48:39.664300+0000","last_became_peered":"2024-08-17T17:48:39.664300+0000","last_unstale":"2024-08-17T18:04:19.424476+0000","last_undegraded":"2024-08-17T18:04:19.424476+0000","last_fullsized":"2024-08-17T18:04:19.424476+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:48.878152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2029,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.414183+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T18:04:19.414183+0000","last_peered":"2024-08-17T18:04:19.414183+0000","last_clean":"2024-08-17T18:04:19.414183+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T18:04:19.414183+0000","last_undegraded":"2024-08-17T18:04:19.414183+0000","last_fullsized":"2024-08-17T18:04:19.414183+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1301,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.202508+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T18:04:02.202508+0000","last_peered":"2024-08-17T18:04:02.202508+0000","last_clean":"2024-08-17T18:04:02.202508+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T18:04:02.202508+0000","last_undegraded":"2024-08-17T18:04:02.202508+0000","last_fullsized":"2024-08-17T18:04:02.202508+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":2076,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.410226+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T18:04:19.410226+0000","last_peered":"2024-08-17T18:04:19.410226+0000","last_clean":"2024-08-17T18:04:19.410226+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T18:04:19.410226+0000","last_undegraded":"2024-08-17T18:04:19.410226+0000","last_fullsized":"2024-08-17T18:04:19.410226+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":263,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.407128+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T18:04:19.407128+0000","last_peered":"2024-08-17T18:04:19.407128+0000","last_clean":"2024-08-17T18:04:19.407128+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T18:04:19.407128+0000","last_undegraded":"2024-08-17T18:04:19.407128+0000","last_fullsized":"2024-08-17T18:04:19.407128+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2095,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.410138+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T18:04:19.410138+0000","last_peered":"2024-08-17T18:04:19.410138+0000","last_clean":"2024-08-17T18:04:19.410138+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T18:04:19.410138+0000","last_undegraded":"2024-08-17T18:04:19.410138+0000","last_fullsized":"2024-08-17T18:04:19.410138+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":838,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.425877+0000","last_change":"2024-08-17T18:02:53.881500+0000","last_active":"2024-08-17T18:04:19.425877+0000","last_peered":"2024-08-17T18:04:19.425877+0000","last_clean":"2024-08-17T18:04:19.425877+0000","last_became_active":"2024-08-17T18:02:53.880651+0000","last_became_peered":"2024-08-17T18:02:53.880651+0000","last_unstale":"2024-08-17T18:04:19.425877+0000","last_undegraded":"2024-08-17T18:04:19.425877+0000","last_fullsized":"2024-08-17T18:04:19.425877+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:03.044635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":23,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.201984+0000","last_change":"2024-08-17T18:03:16.090238+0000","last_active":"2024-08-17T18:04:02.201984+0000","last_peered":"2024-08-17T18:04:02.201984+0000","last_clean":"2024-08-17T18:04:02.201984+0000","last_became_active":"2024-08-17T18:03:16.090003+0000","last_became_peered":"2024-08-17T18:03:16.090003+0000","last_unstale":"2024-08-17T18:04:02.201984+0000","last_undegraded":"2024-08-17T18:04:02.201984+0000","last_fullsized":"2024-08-17T18:04:02.201984+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:49.706675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1411,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.413943+0000","last_change":"2024-08-17T18:02:53.895969+0000","last_active":"2024-08-17T18:04:19.413943+0000","last_peered":"2024-08-17T18:04:19.413943+0000","last_clean":"2024-08-17T18:04:19.413943+0000","last_became_active":"2024-08-17T18:02:53.895681+0000","last_became_peered":"2024-08-17T18:02:53.895681+0000","last_unstale":"2024-08-17T18:04:19.413943+0000","last_undegraded":"2024-08-17T18:04:19.413943+0000","last_fullsized":"2024-08-17T18:04:19.413943+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:55.508072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":336,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.407356+0000","last_change":"2024-08-17T18:02:53.923285+0000","last_active":"2024-08-17T18:04:19.407356+0000","last_peered":"2024-08-17T18:04:19.407356+0000","last_clean":"2024-08-17T18:04:19.407356+0000","last_became_active":"2024-08-17T18:02:53.895787+0000","last_became_peered":"2024-08-17T18:02:53.895787+0000","last_unstale":"2024-08-17T18:04:19.407356+0000","last_undegraded":"2024-08-17T18:04:19.407356+0000","last_fullsized":"2024-08-17T18:04:19.407356+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:11.411637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":2086,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.408811+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T18:04:19.408811+0000","last_peered":"2024-08-17T18:04:19.408811+0000","last_clean":"2024-08-17T18:04:19.408811+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T18:04:19.408811+0000","last_undegraded":"2024-08-17T18:04:19.408811+0000","last_fullsized":"2024-08-17T18:04:19.408811+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2071,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.408296+0000","last_change":"2024-08-17T18:02:53.890366+0000","last_active":"2024-08-17T18:04:19.408296+0000","last_peered":"2024-08-17T18:04:19.408296+0000","last_clean":"2024-08-17T18:04:19.408296+0000","last_became_active":"2024-08-17T18:02:53.889898+0000","last_became_peered":"2024-08-17T18:02:53.889898+0000","last_unstale":"2024-08-17T18:04:19.408296+0000","last_undegraded":"2024-08-17T18:04:19.408296+0000","last_fullsized":"2024-08-17T18:04:19.408296+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:53.298525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1912,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.428125+0000","last_change":"2024-08-17T17:59:58.030150+0000","last_active":"2024-08-17T18:04:19.428125+0000","last_peered":"2024-08-17T18:04:19.428125+0000","last_clean":"2024-08-17T18:04:19.428125+0000","last_became_active":"2024-08-17T17:59:58.029407+0000","last_became_peered":"2024-08-17T17:59:58.029407+0000","last_unstale":"2024-08-17T18:04:19.428125+0000","last_undegraded":"2024-08-17T18:04:19.428125+0000","last_fullsized":"2024-08-17T18:04:19.428125+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:31.632793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2090,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.413351+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T18:04:19.413351+0000","last_peered":"2024-08-17T18:04:19.413351+0000","last_clean":"2024-08-17T18:04:19.413351+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T18:04:19.413351+0000","last_undegraded":"2024-08-17T18:04:19.413351+0000","last_fullsized":"2024-08-17T18:04:19.413351+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2070,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.413432+0000","last_change":"2024-08-17T17:49:45.671858+0000","last_active":"2024-08-17T18:04:19.413432+0000","last_peered":"2024-08-17T18:04:19.413432+0000","last_clean":"2024-08-17T18:04:19.413432+0000","last_became_active":"2024-08-17T17:49:45.670392+0000","last_became_peered":"2024-08-17T17:49:45.670392+0000","last_unstale":"2024-08-17T18:04:19.413432+0000","last_undegraded":"2024-08-17T18:04:19.413432+0000","last_fullsized":"2024-08-17T18:04:19.413432+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:08.803649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1308,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.201942+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T18:04:02.201942+0000","last_peered":"2024-08-17T18:04:02.201942+0000","last_clean":"2024-08-17T18:04:02.201942+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T18:04:02.201942+0000","last_undegraded":"2024-08-17T18:04:02.201942+0000","last_fullsized":"2024-08-17T18:04:02.201942+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1163,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.202408+0000","last_change":"2024-08-17T18:02:53.879600+0000","last_active":"2024-08-17T18:04:02.202408+0000","last_peered":"2024-08-17T18:04:02.202408+0000","last_clean":"2024-08-17T18:04:02.202408+0000","last_became_active":"2024-08-17T18:02:53.879236+0000","last_became_peered":"2024-08-17T18:02:53.879236+0000","last_unstale":"2024-08-17T18:04:02.202408+0000","last_undegraded":"2024-08-17T18:04:02.202408+0000","last_fullsized":"2024-08-17T18:04:02.202408+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:09:08.995893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1986,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.423947+0000","last_change":"2024-08-17T18:02:53.883991+0000","last_active":"2024-08-17T18:04:19.423947+0000","last_peered":"2024-08-17T18:04:19.423947+0000","last_clean":"2024-08-17T18:04:19.423947+0000","last_became_active":"2024-08-17T18:02:53.883668+0000","last_became_peered":"2024-08-17T18:02:53.883668+0000","last_unstale":"2024-08-17T18:04:19.423947+0000","last_undegraded":"2024-08-17T18:04:19.423947+0000","last_fullsized":"2024-08-17T18:04:19.423947+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:49.144723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":70,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.202408+0000","last_change":"2024-08-17T17:59:58.027683+0000","last_active":"2024-08-17T18:04:02.202408+0000","last_peered":"2024-08-17T18:04:02.202408+0000","last_clean":"2024-08-17T18:04:02.202408+0000","last_became_active":"2024-08-17T17:59:58.027358+0000","last_became_peered":"2024-08-17T17:59:58.027358+0000","last_unstale":"2024-08-17T18:04:02.202408+0000","last_undegraded":"2024-08-17T18:04:02.202408+0000","last_fullsized":"2024-08-17T18:04:02.202408+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:04.758259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1646,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.201729+0000","last_change":"2024-08-17T18:02:56.855080+0000","last_active":"2024-08-17T18:04:02.201729+0000","last_peered":"2024-08-17T18:04:02.201729+0000","last_clean":"2024-08-17T18:04:02.201729+0000","last_became_active":"2024-08-17T18:02:56.854831+0000","last_became_peered":"2024-08-17T18:02:56.854831+0000","last_unstale":"2024-08-17T18:04:02.201729+0000","last_undegraded":"2024-08-17T18:04:02.201729+0000","last_fullsized":"2024-08-17T18:04:02.201729+0000","mapping_epoch":1286,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:25:24.111782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2070,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.409031+0000","last_change":"2024-08-17T17:49:45.662305+0000","last_active":"2024-08-17T18:04:19.409031+0000","last_peered":"2024-08-17T18:04:19.409031+0000","last_clean":"2024-08-17T18:04:19.409031+0000","last_became_active":"2024-08-17T17:49:45.661910+0000","last_became_peered":"2024-08-17T17:49:45.661910+0000","last_unstale":"2024-08-17T18:04:19.409031+0000","last_undegraded":"2024-08-17T18:04:19.409031+0000","last_fullsized":"2024-08-17T18:04:19.409031+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:35.247535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2011,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.424651+0000","last_change":"2024-08-17T17:31:49.809578+0000","last_active":"2024-08-17T18:04:19.424651+0000","last_peered":"2024-08-17T18:04:19.424651+0000","last_clean":"2024-08-17T18:04:19.424651+0000","last_became_active":"2024-08-17T17:31:49.804332+0000","last_became_peered":"2024-08-17T17:31:49.804332+0000","last_unstale":"2024-08-17T18:04:19.424651+0000","last_undegraded":"2024-08-17T18:04:19.424651+0000","last_fullsized":"2024-08-17T18:04:19.424651+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:13.859959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":524,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.408128+0000","last_change":"2024-08-17T18:02:53.897151+0000","last_active":"2024-08-17T18:04:19.408128+0000","last_peered":"2024-08-17T18:04:19.408128+0000","last_clean":"2024-08-17T18:04:19.408128+0000","last_became_active":"2024-08-17T18:02:53.895480+0000","last_became_peered":"2024-08-17T18:02:53.895480+0000","last_unstale":"2024-08-17T18:04:19.408128+0000","last_undegraded":"2024-08-17T18:04:19.408128+0000","last_fullsized":"2024-08-17T18:04:19.408128+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:17.316423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":548,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.201913+0000","last_change":"2024-08-17T17:59:58.021264+0000","last_active":"2024-08-17T18:04:02.201913+0000","last_peered":"2024-08-17T18:04:02.201913+0000","last_clean":"2024-08-17T18:04:02.201913+0000","last_became_active":"2024-08-17T17:59:58.020957+0000","last_became_peered":"2024-08-17T17:59:58.020957+0000","last_unstale":"2024-08-17T18:04:02.201913+0000","last_undegraded":"2024-08-17T18:04:02.201913+0000","last_fullsized":"2024-08-17T18:04:02.201913+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:42.190270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":620,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.407791+0000","last_change":"2024-08-17T18:02:53.883783+0000","last_active":"2024-08-17T18:04:19.407791+0000","last_peered":"2024-08-17T18:04:19.407791+0000","last_clean":"2024-08-17T18:04:19.407791+0000","last_became_active":"2024-08-17T18:02:53.883489+0000","last_became_peered":"2024-08-17T18:02:53.883489+0000","last_unstale":"2024-08-17T18:04:19.407791+0000","last_undegraded":"2024-08-17T18:04:19.407791+0000","last_fullsized":"2024-08-17T18:04:19.407791+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:08.849092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1208,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.407552+0000","last_change":"2024-08-17T18:02:53.896125+0000","last_active":"2024-08-17T18:04:19.407552+0000","last_peered":"2024-08-17T18:04:19.407552+0000","last_clean":"2024-08-17T18:04:19.407552+0000","last_became_active":"2024-08-17T18:02:53.895789+0000","last_became_peered":"2024-08-17T18:02:53.895789+0000","last_unstale":"2024-08-17T18:04:19.407552+0000","last_undegraded":"2024-08-17T18:04:19.407552+0000","last_fullsized":"2024-08-17T18:04:19.407552+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:25:25.798463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":788,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.424729+0000","last_change":"2024-08-17T18:02:53.883815+0000","last_active":"2024-08-17T18:04:19.424729+0000","last_peered":"2024-08-17T18:04:19.424729+0000","last_clean":"2024-08-17T18:04:19.424729+0000","last_became_active":"2024-08-17T18:02:53.883374+0000","last_became_peered":"2024-08-17T18:02:53.883374+0000","last_unstale":"2024-08-17T18:04:19.424729+0000","last_undegraded":"2024-08-17T18:04:19.424729+0000","last_fullsized":"2024-08-17T18:04:19.424729+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:37:52.718473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2007,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.423646+0000","last_change":"2024-08-17T17:49:45.709197+0000","last_active":"2024-08-17T18:04:19.423646+0000","last_peered":"2024-08-17T18:04:19.423646+0000","last_clean":"2024-08-17T18:04:19.423646+0000","last_became_active":"2024-08-17T17:49:45.693768+0000","last_became_peered":"2024-08-17T17:49:45.693768+0000","last_unstale":"2024-08-17T18:04:19.423646+0000","last_undegraded":"2024-08-17T18:04:19.423646+0000","last_fullsized":"2024-08-17T18:04:19.423646+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:28.752020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2103,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.414263+0000","last_change":"2024-08-17T17:49:45.695043+0000","last_active":"2024-08-17T18:04:19.414263+0000","last_peered":"2024-08-17T18:04:19.414263+0000","last_clean":"2024-08-17T18:04:19.414263+0000","last_became_active":"2024-08-17T17:49:45.693740+0000","last_became_peered":"2024-08-17T17:49:45.693740+0000","last_unstale":"2024-08-17T18:04:19.414263+0000","last_undegraded":"2024-08-17T18:04:19.414263+0000","last_fullsized":"2024-08-17T18:04:19.414263+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:00.394558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":795,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.410041+0000","last_change":"2024-08-17T18:03:16.090929+0000","last_active":"2024-08-17T18:04:19.410041+0000","last_peered":"2024-08-17T18:04:19.410041+0000","last_clean":"2024-08-17T18:04:19.410041+0000","last_became_active":"2024-08-17T18:03:16.090554+0000","last_became_peered":"2024-08-17T18:03:16.090554+0000","last_unstale":"2024-08-17T18:04:19.410041+0000","last_undegraded":"2024-08-17T18:04:19.410041+0000","last_fullsized":"2024-08-17T18:04:19.410041+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:01.783375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":517,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.407270+0000","last_change":"2024-08-17T18:02:53.881999+0000","last_active":"2024-08-17T18:04:19.407270+0000","last_peered":"2024-08-17T18:04:19.407270+0000","last_clean":"2024-08-17T18:04:19.407270+0000","last_became_active":"2024-08-17T18:02:53.881681+0000","last_became_peered":"2024-08-17T18:02:53.881681+0000","last_unstale":"2024-08-17T18:04:19.407270+0000","last_undegraded":"2024-08-17T18:04:19.407270+0000","last_fullsized":"2024-08-17T18:04:19.407270+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:39:51.334407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1274,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.202506+0000","last_change":"2024-08-17T18:02:53.890396+0000","last_active":"2024-08-17T18:04:02.202506+0000","last_peered":"2024-08-17T18:04:02.202506+0000","last_clean":"2024-08-17T18:04:02.202506+0000","last_became_active":"2024-08-17T18:02:53.879118+0000","last_became_peered":"2024-08-17T18:02:53.879118+0000","last_unstale":"2024-08-17T18:04:02.202506+0000","last_undegraded":"2024-08-17T18:04:02.202506+0000","last_fullsized":"2024-08-17T18:04:02.202506+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:14.074623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":805,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.406771+0000","last_change":"2024-08-17T18:02:53.891213+0000","last_active":"2024-08-17T18:04:19.406771+0000","last_peered":"2024-08-17T18:04:19.406771+0000","last_clean":"2024-08-17T18:04:19.406771+0000","last_became_active":"2024-08-17T18:02:53.890853+0000","last_became_peered":"2024-08-17T18:02:53.890853+0000","last_unstale":"2024-08-17T18:04:19.406771+0000","last_undegraded":"2024-08-17T18:04:19.406771+0000","last_fullsized":"2024-08-17T18:04:19.406771+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:19.916697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1278,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.202582+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T18:04:02.202582+0000","last_peered":"2024-08-17T18:04:02.202582+0000","last_clean":"2024-08-17T18:04:02.202582+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T18:04:02.202582+0000","last_undegraded":"2024-08-17T18:04:02.202582+0000","last_fullsized":"2024-08-17T18:04:02.202582+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":172,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.201667+0000","last_change":"2024-08-17T18:02:53.887617+0000","last_active":"2024-08-17T18:04:02.201667+0000","last_peered":"2024-08-17T18:04:02.201667+0000","last_clean":"2024-08-17T18:04:02.201667+0000","last_became_active":"2024-08-17T18:02:53.887266+0000","last_became_peered":"2024-08-17T18:02:53.887266+0000","last_unstale":"2024-08-17T18:04:02.201667+0000","last_undegraded":"2024-08-17T18:04:02.201667+0000","last_fullsized":"2024-08-17T18:04:02.201667+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:57:02.775040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1294,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.202483+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T18:04:02.202483+0000","last_peered":"2024-08-17T18:04:02.202483+0000","last_clean":"2024-08-17T18:04:02.202483+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T18:04:02.202483+0000","last_undegraded":"2024-08-17T18:04:02.202483+0000","last_fullsized":"2024-08-17T18:04:02.202483+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":87,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.412780+0000","last_change":"2024-08-17T18:02:53.919476+0000","last_active":"2024-08-17T18:04:19.412780+0000","last_peered":"2024-08-17T18:04:19.412780+0000","last_clean":"2024-08-17T18:04:19.412780+0000","last_became_active":"2024-08-17T18:02:53.896368+0000","last_became_peered":"2024-08-17T18:02:53.896368+0000","last_unstale":"2024-08-17T18:04:19.412780+0000","last_undegraded":"2024-08-17T18:04:19.412780+0000","last_fullsized":"2024-08-17T18:04:19.412780+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:10:30.848886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2008,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.424351+0000","last_change":"2024-08-17T17:31:49.807841+0000","last_active":"2024-08-17T18:04:19.424351+0000","last_peered":"2024-08-17T18:04:19.424351+0000","last_clean":"2024-08-17T18:04:19.424351+0000","last_became_active":"2024-08-17T17:31:49.807534+0000","last_became_peered":"2024-08-17T17:31:49.807534+0000","last_unstale":"2024-08-17T18:04:19.424351+0000","last_undegraded":"2024-08-17T18:04:19.424351+0000","last_fullsized":"2024-08-17T18:04:19.424351+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:59.374762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1265,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.202037+0000","last_change":"2024-08-17T18:02:53.879170+0000","last_active":"2024-08-17T18:04:02.202037+0000","last_peered":"2024-08-17T18:04:02.202037+0000","last_clean":"2024-08-17T18:04:02.202037+0000","last_became_active":"2024-08-17T18:02:53.878913+0000","last_became_peered":"2024-08-17T18:02:53.878913+0000","last_unstale":"2024-08-17T18:04:02.202037+0000","last_undegraded":"2024-08-17T18:04:02.202037+0000","last_fullsized":"2024-08-17T18:04:02.202037+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:55:37.600470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2061,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.413012+0000","last_change":"2024-08-17T17:48:39.659708+0000","last_active":"2024-08-17T18:04:19.413012+0000","last_peered":"2024-08-17T18:04:19.413012+0000","last_clean":"2024-08-17T18:04:19.413012+0000","last_became_active":"2024-08-17T17:48:39.658329+0000","last_became_peered":"2024-08-17T17:48:39.658329+0000","last_unstale":"2024-08-17T18:04:19.413012+0000","last_undegraded":"2024-08-17T18:04:19.413012+0000","last_fullsized":"2024-08-17T18:04:19.413012+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:45:28.350224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2062,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.416165+0000","last_change":"2024-08-17T17:48:39.659959+0000","last_active":"2024-08-17T18:04:19.416165+0000","last_peered":"2024-08-17T18:04:19.416165+0000","last_clean":"2024-08-17T18:04:19.416165+0000","last_became_active":"2024-08-17T17:48:39.658002+0000","last_became_peered":"2024-08-17T17:48:39.658002+0000","last_unstale":"2024-08-17T18:04:19.416165+0000","last_undegraded":"2024-08-17T18:04:19.416165+0000","last_fullsized":"2024-08-17T18:04:19.416165+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:38.959561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":688,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.202407+0000","last_change":"2024-08-17T18:03:16.091420+0000","last_active":"2024-08-17T18:04:02.202407+0000","last_peered":"2024-08-17T18:04:02.202407+0000","last_clean":"2024-08-17T18:04:02.202407+0000","last_became_active":"2024-08-17T18:03:16.091113+0000","last_became_peered":"2024-08-17T18:03:16.091113+0000","last_unstale":"2024-08-17T18:04:02.202407+0000","last_undegraded":"2024-08-17T18:04:02.202407+0000","last_fullsized":"2024-08-17T18:04:02.202407+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:47.770911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1296,"reported_epoch":1310,"state":"active+clean","last_fresh":"2024-08-17T18:04:02.202492+0000","last_change":"2024-08-17T18:02:53.893927+0000","last_active":"2024-08-17T18:04:02.202492+0000","last_peered":"2024-08-17T18:04:02.202492+0000","last_clean":"2024-08-17T18:04:02.202492+0000","last_became_active":"2024-08-17T18:02:53.883188+0000","last_became_peered":"2024-08-17T18:02:53.883188+0000","last_unstale":"2024-08-17T18:04:02.202492+0000","last_undegraded":"2024-08-17T18:04:02.202492+0000","last_fullsized":"2024-08-17T18:04:02.202492+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:19.296965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2008,"reported_epoch":1311,"state":"active+clean","last_fresh":"2024-08-17T18:04:19.424856+0000","last_change":"2024-08-17T17:31:49.806457+0000","last_active":"2024-08-17T18:04:19.424856+0000","last_peered":"2024-08-17T18:04:19.424856+0000","last_clean":"2024-08-17T18:04:19.424856+0000","last_became_active":"2024-08-17T17:31:49.805442+0000","last_became_peered":"2024-08-17T17:31:49.805442+0000","last_unstale":"2024-08-17T18:04:19.424856+0000","last_undegraded":"2024-08-17T18:04:19.424856+0000","last_fullsized":"2024-08-17T18:04:19.424856+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:59.169054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":47,"num_pg":16,"stat_sum":{"num_bytes":2593587200,"num_objects":39575,"num_object_clones":0,"num_object_copies":118725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":39575,"num_write_kb":2532800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7637565440,"data_stored":7637565440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39575,"ondisk_log_size":39575,"up":48,"acting":48,"num_store_stats":5},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":95,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":10948,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234522,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1181436,"kb_used_data":871604,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":92563716,"statfs":{"total":95995035648,"available":94785245184,"internally_reserved":0,"allocated":892522496,"data_stored":888405513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568496157,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757795,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326409,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1815596,"kb_used_data":1357796,"kb_used_omap":57,"kb_used_meta":457734,"kb_avail":91929556,"statfs":{"total":95995035648,"available":94135865344,"internally_reserved":0,"allocated":1390383104,"data_stored":1386277762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":468720379},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405648,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2323184,"kb_used_data":1848104,"kb_used_omap":61,"kb_used_meta":475010,"kb_avail":91421968,"statfs":{"total":95995035648,"available":93616095232,"internally_reserved":0,"allocated":1892458496,"data_stored":1888353657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":486411192},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452183,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2109544,"kb_used_data":1688864,"kb_used_omap":54,"kb_used_meta":420617,"kb_avail":91635608,"statfs":{"total":95995035648,"available":93834862592,"internally_reserved":0,"allocated":1729396736,"data_stored":1725295326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":430712206},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474873,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2165636,"kb_used_data":1662844,"kb_used_omap":119,"kb_used_meta":502664,"kb_avail":91579516,"statfs":{"total":95995035648,"available":93777424384,"internally_reserved":0,"allocated":1702752256,"data_stored":1698627690,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":514728670},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,4,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019430,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":47,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1709309952,"data_stored":1709309952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":47,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1720516608,"data_stored":1720516608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":47,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1899036672,"data_stored":1899036672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":47,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1396178944,"data_stored":1396178944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":47,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":912523264,"data_stored":912523264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:04:30.935 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T18:04:30.936 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:04:30.936 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-08-17T18:04:30.936 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.0 'trim stale osdmaps' 2024-08-17T18:04:31.003 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:31.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:31.063 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:31.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:31.164 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:31.216 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:31.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:31.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:31.318 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:31.339 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:31.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:31.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:31.441 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:31.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:31.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:31.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:31.564 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:31.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:31.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:31.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:31.659 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 31 16 47524 47508 95.7712 69.625 0.0029198 0.0104272 2024-08-17T18:04:31.687 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:31.718 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:31.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:31.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:31.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:31.857 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:31.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:31.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:31.959 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:31.982 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:31.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:31.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:32.083 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:32.101 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:04:32.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:32.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:32.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:32.208 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:32.230 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:32.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:32.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:32.263 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:04:32.332 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:32.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:32.377 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:32.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:32.399 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:32 smithi130 ceph-mon[105521]: pgmap v2209: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T18:04:32.399 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:32 smithi130 ceph-mon[110068]: pgmap v2209: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T18:04:32.479 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:32.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:32.502 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:32.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:32.603 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:32.640 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:32.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:32.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:32.659 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 32 16 48564 48548 94.8094 65 0.0115028 0.010541 2024-08-17T18:04:32.743 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:32.780 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:32.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:32.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:32 smithi179 ceph-mon[93519]: pgmap v2209: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-08-17T18:04:32.882 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:32.951 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:32.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:32.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:33.053 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:33.074 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:33.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:33.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:33.176 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:33.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:33.222 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:33.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:33.323 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:33.365 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:33.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:33.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:33.466 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:33.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:33.505 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:33.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:33.606 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:33.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:33.628 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:33.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:33.659 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 33 16 49663 49647 94.0177 68.6875 0.0122372 0.0106296 2024-08-17T18:04:33.729 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:33.751 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:33.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:33.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:33.853 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:33.893 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:33.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:33.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:33.981 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:04:33.995 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:34.017 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:34.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:34.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:34.119 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:34.140 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:34.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:34.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:34.242 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:34.264 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:34.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:34.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:34.365 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:34.388 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:34.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:34.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:34.490 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:34.530 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:34.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:34.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:34.632 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:34.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:34.654 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:34.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:34.659 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 34 16 50719 50703 93.1935 66 0.01599 0.0107227 2024-08-17T18:04:34.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:34 smithi130 ceph-mon[105521]: pgmap v2210: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-08-17T18:04:34.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:34 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1871014105' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:04:34.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:34 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:04:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:34 smithi130 ceph-mon[110068]: pgmap v2210: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-08-17T18:04:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:34 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1871014105' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:04:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:34 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:04:34.755 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:34.801 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:34.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:34.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:34 smithi179 ceph-mon[93519]: pgmap v2210: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-08-17T18:04:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:34 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1871014105' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:04:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:34 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:04:34.903 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:34.941 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:34.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:34.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:35.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:35.082 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:35.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:35.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:35.184 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:35.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:35.207 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:35.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:35.308 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:35.330 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:35.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:35.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:35.432 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:35.468 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:35.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:35.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:35.570 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:35.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:35.593 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:35.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:35.660 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 35 16 51657 51641 92.2057 58.625 0.00858459 0.0108398 2024-08-17T18:04:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:35 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:04:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:35 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:04:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:35 smithi130 ceph-mon[105521]: osdmap e1312: 8 total, 8 up, 5 in 2024-08-17T18:04:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:35 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:04:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:35 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:04:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:35 smithi130 ceph-mon[110068]: osdmap e1312: 8 total, 8 up, 5 in 2024-08-17T18:04:35.694 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:35.716 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:35.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:35.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:35.818 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:35.839 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:35.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:35.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:35.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:35 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:04:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:35 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:04:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:35 smithi179 ceph-mon[93519]: osdmap e1312: 8 total, 8 up, 5 in 2024-08-17T18:04:35.941 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:35.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:35.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:35.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:36.025 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:04:36.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:36.099 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:36.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:36.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:36.201 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:36.223 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:36.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:36.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:36.325 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:36.346 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:36.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:36.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:36.447 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:36.469 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:36.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:36.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:36.571 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:36.593 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:36.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:36.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:36.660 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 36 16 52560 52544 91.212 56.4375 0.0248809 0.0109566 2024-08-17T18:04:36.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:36 smithi130 ceph-mon[105521]: pgmap v2212: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T18:04:36.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:36 smithi130 ceph-mon[110068]: pgmap v2212: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T18:04:36.694 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:36.717 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:36.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:36.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:36.818 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:36.840 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:36.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:36.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:36.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:36 smithi179 ceph-mon[93519]: pgmap v2212: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T18:04:36.942 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:36.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:36.965 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:36.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:37.066 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:37.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:37.089 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:37.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:37.190 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:37.232 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:37.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:37.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:37.333 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:37.372 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:37.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:37.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:37.473 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:37.516 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:37.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:37.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:37.618 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:37.641 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:37.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:37.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:37.660 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 37 16 53679 53663 90.6368 69.9375 0.0135413 0.0110266 2024-08-17T18:04:37.742 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:37.762 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:37.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:37.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:37.864 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:37.886 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:37.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:37.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:37.988 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:38.009 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:38.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:38.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:38.111 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:38.154 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:38.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:38.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:38.256 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:38.295 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:38.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:38.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:38.397 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:38.432 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:38.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:38.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:38.534 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:38.556 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:38.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:38.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:38.658 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:38.660 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 38 16 54467 54451 89.5476 49.25 0.00462661 0.0111617 2024-08-17T18:04:38.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:38 smithi130 ceph-mon[105521]: pgmap v2213: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T18:04:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:38 smithi130 ceph-mon[110068]: pgmap v2213: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T18:04:38.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:38.681 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:38.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:38.783 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:38.804 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:38.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:38.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:38 smithi179 ceph-mon[93519]: pgmap v2213: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-08-17T18:04:38.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:38.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:38.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:38.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:39.030 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:39.053 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:39.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:39.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:39.155 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:39.176 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:39.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:39.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:39.277 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:39.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:39.324 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:39.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:39.425 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:39.465 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:39.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:39.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:39.566 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:39.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:39.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:39.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:39.660 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 39 16 55457 55441 88.8379 61.875 0.00889941 0.0112483 2024-08-17T18:04:39.691 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:39.712 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:39.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:39.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:39.814 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:39.835 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:39.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:39.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:39.937 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:39.959 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:39.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:39.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:40.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:40.096 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:40.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:40.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:40.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:04:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:04:39] "GET /metrics HTTP/1.1" 200 36776 "" "Prometheus/2.51.0" 2024-08-17T18:04:40.198 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:40.220 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:40.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:40.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:40.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:40.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:40.363 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:40.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:40.464 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:40.486 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:40.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:40.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:40.591 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:40 smithi179 ceph-mon[93519]: pgmap v2214: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T18:04:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:04:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:04:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:04:40.632 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:40.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:40.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:40.660 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:04:40.660352+0000 min lat: 0.00111317 max lat: 0.233627 avg lat: 0.0113225 2024-08-17T18:04:40.660 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:04:40.660 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 40 16 56512 56496 88.2653 65.9375 0.00996799 0.0113225 2024-08-17T18:04:40.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:40 smithi130 ceph-mon[105521]: pgmap v2214: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T18:04:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:40 smithi130 ceph-mon[110068]: pgmap v2214: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T18:04:40.734 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:40.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:40.786 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:40.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:40.887 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:40.929 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:40.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:40.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:41.031 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:41.075 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:41.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:41.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:41.177 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:41.198 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:41.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:41.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:41.300 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:41.339 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:41.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:41.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:41.441 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:41.464 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:41.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:41.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:41.566 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:41.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:04:41.589 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:41.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:41.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:41.660 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 41 16 57272 57256 87.2709 47.5 0.00339431 0.0113931 2024-08-17T18:04:41.691 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:41.712 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:41.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:41.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:41.814 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:41.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:41.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:41.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:41.987 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:42.008 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:42.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:42.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:42.110 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:42.142 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:42.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:42.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:42.244 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:42.280 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:42.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:42.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:42.382 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:42.404 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:42.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:42.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:42.506 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:42.527 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:42.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:42.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:42.629 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:42.650 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:42.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:42.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:42.660 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 42 16 57834 57818 86.0293 35.125 0.00690998 0.0116159 2024-08-17T18:04:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:42 smithi130 ceph-mon[105521]: pgmap v2215: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T18:04:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:04:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:42 smithi130 ceph-mon[110068]: pgmap v2215: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T18:04:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:04:42.700 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:04:42.700 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:04:42.752 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:42.774 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:42.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:42.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:42 smithi179 ceph-mon[93519]: pgmap v2215: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T18:04:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:04:42.876 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:42.911 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:42.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:42.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:43.012 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:43.050 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:43.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:43.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:43.152 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:43.197 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:43.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:43.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:43.299 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:43.313 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:04:43.358 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:43.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:43.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:43.460 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:43.481 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:43.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:43.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:43.583 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:43.604 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:43.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:43.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:43.660 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 43 16 58779 58763 85.402 59.0625 0.0325046 0.0117022 2024-08-17T18:04:43.706 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:43.748 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:43.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:43.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:43.850 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:43.871 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:43.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:43.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:43.973 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:43.994 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:43.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:43.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:44.095 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:44.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:44.148 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:44.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:44.249 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:44.270 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:44.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:44.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:44.372 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:44.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:44.394 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:44.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:44.495 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:44.516 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:44.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:44.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:44.618 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:44.639 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:44.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:44.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:44.660 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 44 16 59729 59713 84.8104 59.375 0.0147664 0.0117846 2024-08-17T18:04:44.741 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:44.782 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:44.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:44.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:44.796 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:44 smithi130 ceph-mon[105521]: pgmap v2216: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 59 MiB/s wr, 938 op/s 2024-08-17T18:04:44.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:44 smithi130 ceph-mon[110068]: pgmap v2216: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 59 MiB/s wr, 938 op/s 2024-08-17T18:04:44.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:44 smithi179 ceph-mon[93519]: pgmap v2216: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 59 MiB/s wr, 938 op/s 2024-08-17T18:04:44.885 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:44.932 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:44.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:44.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:45.034 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:45.056 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:45.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:45.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:45.158 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:45.180 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:45.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:45.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:45.278 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T18:04:45.282 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:45.303 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:45.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:45.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:45.405 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:45.467 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:45.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:45.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:45.570 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:45.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:45.621 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:45.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:45.661 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 45 16 60703 60687 84.2765 60.875 0.0117172 0.0118602 2024-08-17T18:04:45.723 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:45.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:45.794 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:45.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:45.894 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:45.917 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:45.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:45.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:45.991 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:04:46.018 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:46.039 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:04:46.040 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:04:46.040 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:04:46.040 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:04:46.040 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:04:46.040 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:04:46.040 INFO:teuthology.orchestra.run.smithi130.stderr:bench [] [] [] [] 2024-08-17T18:04:46.040 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:04:46.040 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:04:46.040 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:04:46.040 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator dump block 2024-08-17T18:04:46.040 INFO:teuthology.orchestra.run.smithi130.stderr:Error EINVAL: invalid command 2024-08-17T18:04:46.068 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:46.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:46.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:46.170 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:46.193 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:46.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:46.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:46.294 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:46.316 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:46.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:46.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:46.418 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:46.440 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:46.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:46.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:46.542 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:46.563 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:46.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:46.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:46.662 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 46 16 61597 61581 83.659 55.875 0.0028967 0.011932 2024-08-17T18:04:46.665 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:46.686 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:46.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:46.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:46.788 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:46.810 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:46.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:46.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:46.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:46 smithi179 ceph-mon[93519]: pgmap v2217: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 54 MiB/s wr, 860 op/s 2024-08-17T18:04:46.912 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:46.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:46 smithi130 ceph-mon[105521]: pgmap v2217: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 54 MiB/s wr, 860 op/s 2024-08-17T18:04:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:46 smithi130 ceph-mon[110068]: pgmap v2217: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 54 MiB/s wr, 860 op/s 2024-08-17T18:04:46.933 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:46.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:46.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:47.035 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:47.057 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:47.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:47.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:47.159 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:47.199 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:47.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:47.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:47.301 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:47.329 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:47.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:47.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:47.431 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:47.474 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:47.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:47.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:47.575 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:47.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:47.598 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:47.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:47.662 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 47 16 62554 62538 83.1515 59.8125 0.00935481 0.0120199 2024-08-17T18:04:47.699 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:47.720 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:47.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:47.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:47.822 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:47.842 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:47.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:47.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:47.944 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:47.973 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:47.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:47.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:48.075 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:48.097 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:48.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:48.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:48.199 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:48.221 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:48.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:48.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:48.323 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:48.370 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:48.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:48.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:48.473 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:48.496 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:48.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:48.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:48.598 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:48.619 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:48.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:48.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:48.662 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 48 16 63209 63193 82.272 40.9375 0.0227397 0.0121485 2024-08-17T18:04:48.721 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:48.743 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:48.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:48.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:48 smithi179 ceph-mon[93519]: pgmap v2218: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 59 MiB/s wr, 948 op/s 2024-08-17T18:04:48.845 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:48.866 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:48.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:48.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:48.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:48 smithi130 ceph-mon[105521]: pgmap v2218: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 59 MiB/s wr, 948 op/s 2024-08-17T18:04:48.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:48 smithi130 ceph-mon[110068]: pgmap v2218: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 59 MiB/s wr, 948 op/s 2024-08-17T18:04:48.968 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:48.990 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:48.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:48.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:49.092 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:49.127 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:04:49.127 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T18:04:49.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.1 'trim stale osdmaps' 2024-08-17T18:04:49.142 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:49.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:49.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:49.245 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:49.308 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:49.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:49.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:49.374 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:04:49.397 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:04:49.397 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:04:49.411 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:49.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:49.433 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:49.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:49.435 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:04:49.435 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:04:49.475 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:04:49.475 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:04:49.511 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:04:49.511 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:04:49.534 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:49.542 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:04:49.542 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:04:49.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:49.556 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:49.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:49.572 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:04:49.573 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:04:49.602 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:04:49.602 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:04:49.603 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:49 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1483098601' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:04:49.603 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:49 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1483098601' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:04:49.630 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:04:49.630 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:04:49.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:49.662 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 49 16 64082 64066 81.7065 54.5625 0.0159699 0.0122325 2024-08-17T18:04:49.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:49.678 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:49.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:49.779 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:49.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:49.801 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:49.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:49.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:49 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1483098601' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:04:49.902 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:49.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:04:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:04:49] "GET /metrics HTTP/1.1" 200 36781 "" "Prometheus/2.51.0" 2024-08-17T18:04:49.923 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:49.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:49.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:50.025 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:50.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:50.065 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:50.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:50.166 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:50.187 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:50.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:50.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:50.288 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:50.331 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:50.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:50.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:50.432 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:50.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:50.495 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:50.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:50 smithi179 ceph-mon[93519]: pgmap v2219: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 60 MiB/s wr, 966 op/s 2024-08-17T18:04:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:04:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:04:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:04:50.595 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:50.616 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:50.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:50.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:50.662 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 50 16 65031 65015 81.2585 59.3125 0.0111669 0.0123008 2024-08-17T18:04:50.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:50 smithi130 ceph-mon[105521]: pgmap v2219: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 60 MiB/s wr, 966 op/s 2024-08-17T18:04:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:50 smithi130 ceph-mon[110068]: pgmap v2219: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 60 MiB/s wr, 966 op/s 2024-08-17T18:04:50.718 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:50.739 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:50.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:50.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:50.841 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:50.883 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:50.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:50.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:50.985 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:51.022 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:51.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:51.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:51.124 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:51.145 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:51.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:51.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:51.246 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:51.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:51.288 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:51.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:51.388 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:51.410 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:51.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:51.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:51.512 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:51.533 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:51.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:51.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:51.634 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:51.662 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 51 16 66015 65999 80.8701 61.5 0.0206678 0.0123592 2024-08-17T18:04:51.674 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:51.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:51.775 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:51.797 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:51.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:51.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:51.898 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:51.934 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:51.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:51.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:52.036 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:52.058 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:52.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:52.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:52.159 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:52.182 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:52.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:52.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:52.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:52.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:52.331 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:52.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:52.432 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:52.474 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:52.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:52.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:52.576 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:52.615 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:52.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:52.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:52.663 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 52 16 66984 66968 80.4795 60.5625 0.0129737 0.0124194 2024-08-17T18:04:52.717 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:52.738 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:52.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:52.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:52.840 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:52 smithi179 ceph-mon[93519]: pgmap v2220: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 46 MiB/s wr, 742 op/s 2024-08-17T18:04:52.843 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:52 smithi130 ceph-mon[105521]: pgmap v2220: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 46 MiB/s wr, 742 op/s 2024-08-17T18:04:52.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:52 smithi130 ceph-mon[110068]: pgmap v2220: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 46 MiB/s wr, 742 op/s 2024-08-17T18:04:52.903 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:52.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:52.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:53.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:53.027 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:53.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:53.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:53.128 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:53.151 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:53.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:53.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:53.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:04:53.252 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:53.332 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:53.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:53.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:53.434 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:53.455 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:53.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:53.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:53.557 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:53.610 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:53.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:53.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:53.664 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 53 16 67813 67797 79.937 51.8125 0.0304615 0.0125035 2024-08-17T18:04:53.712 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:53.760 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:53.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:53.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:53.862 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:53.901 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:53.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:53.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:54.003 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:54.046 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:54.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:54.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:54.147 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:54.169 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:54.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:54.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:54.272 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:54.308 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:54.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:54.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:54.411 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:54.432 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:54.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:54.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:54.490 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:04:54.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:54.561 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:54 smithi130 ceph-mon[105521]: pgmap v2221: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T18:04:54.561 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/769715263' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:04:54.561 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:54 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:04:54.561 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:54 smithi130 ceph-mon[110068]: pgmap v2221: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T18:04:54.561 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/769715263' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:04:54.561 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:54 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:04:54.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:54.579 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:54.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:54.664 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 54 16 68578 68562 79.342 47.8125 0.0116988 0.0125978 2024-08-17T18:04:54.680 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:54.701 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:54.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:54.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:54.803 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:54.824 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:54.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:54.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:54.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:54 smithi179 ceph-mon[93519]: pgmap v2221: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T18:04:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/769715263' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:04:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:54 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:04:54.926 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:54.970 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:54.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:54.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:55.071 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:55.092 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:55.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:55.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:55.194 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:55.229 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:55.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:55.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:55.300 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:04:55.301 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:04:55.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:04:55.301 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:04:55.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.301 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79507 46.b 46:d8bdbef4:::benchmark_data_smithi130_267720_object79506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:04:55.301 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.615201+0000", 2024-08-17T18:04:55.301 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 263.84557365199998, 2024-08-17T18:04:55.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.40324494700000002, 2024-08-17T18:04:55.301 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.301 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.301 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:55.301 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.301 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.302 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.302 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79507 2024-08-17T18:04:55.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.302 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.615201+0000", 2024-08-17T18:04:55.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.615201+0000", 2024-08-17T18:04:55.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.615201+0000", 2024-08-17T18:04:55.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4600000000000005e-07 2024-08-17T18:04:55.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.615223+0000", 2024-08-17T18:04:55.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1747e-05 2024-08-17T18:04:55.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.615572+0000", 2024-08-17T18:04:55.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00034875599999999998 2024-08-17T18:04:55.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.615579+0000", 2024-08-17T18:04:55.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9909999999999996e-06 2024-08-17T18:04:55.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.638229+0000", 2024-08-17T18:04:55.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022650623000000002 2024-08-17T18:04:55.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.638369+0000", 2024-08-17T18:04:55.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013929099999999999 2024-08-17T18:04:55.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T18:04:55.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.638530+0000", 2024-08-17T18:04:55.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016180900000000001 2024-08-17T18:04:55.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644666+0000", 2024-08-17T18:04:55.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0061359630000000004 2024-08-17T18:04:55.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644732+0000", 2024-08-17T18:04:55.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5506999999999995e-05 2024-08-17T18:04:55.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.018333+0000", 2024-08-17T18:04:55.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37360103 2024-08-17T18:04:55.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:04:55.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.018411+0000", 2024-08-17T18:04:55.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8343999999999994e-05 2024-08-17T18:04:55.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.307 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.307 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.307 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.018446+0000", 2024-08-17T18:04:55.307 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.434e-05 2024-08-17T18:04:55.307 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.307 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.307 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.307 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.307 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.307 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79518 46.2 46:404c947b:::benchmark_data_smithi130_267720_object79517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:04:55.307 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.623412+0000", 2024-08-17T18:04:55.307 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 263.83736245699998, 2024-08-17T18:04:55.307 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.39487024500000001, 2024-08-17T18:04:55.308 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.308 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.308 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:55.308 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.308 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.308 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.308 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79518 2024-08-17T18:04:55.308 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.308 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.308 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.308 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.308 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623412+0000", 2024-08-17T18:04:55.308 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623412+0000", 2024-08-17T18:04:55.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623413+0000", 2024-08-17T18:04:55.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3900000000000004e-07 2024-08-17T18:04:55.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623437+0000", 2024-08-17T18:04:55.310 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3998999999999998e-05 2024-08-17T18:04:55.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.310 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.310 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623449+0000", 2024-08-17T18:04:55.310 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2065000000000001e-05 2024-08-17T18:04:55.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.310 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.310 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623460+0000", 2024-08-17T18:04:55.310 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1534e-05 2024-08-17T18:04:55.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623559+0000", 2024-08-17T18:04:55.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8368000000000004e-05 2024-08-17T18:04:55.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.311 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623674+0000", 2024-08-17T18:04:55.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000115561 2024-08-17T18:04:55.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.311 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:04:55.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623803+0000", 2024-08-17T18:04:55.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012843300000000001 2024-08-17T18:04:55.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.312 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.630231+0000", 2024-08-17T18:04:55.312 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0064287770000000001 2024-08-17T18:04:55.312 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.312 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.630245+0000", 2024-08-17T18:04:55.312 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4042e-05 2024-08-17T18:04:55.312 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.312 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.018111+0000", 2024-08-17T18:04:55.312 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.387865919 2024-08-17T18:04:55.313 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.313 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.313 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:04:55.313 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.018228+0000", 2024-08-17T18:04:55.313 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011705300000000001 2024-08-17T18:04:55.313 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.313 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.313 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.313 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.018282+0000", 2024-08-17T18:04:55.313 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3655e-05 2024-08-17T18:04:55.313 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.313 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.313 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.313 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.314 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.314 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79519 46.2 46:4d9ed74c:::benchmark_data_smithi130_267720_object79518:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:04:55.314 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.623580+0000", 2024-08-17T18:04:55.314 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 263.837194415, 2024-08-17T18:04:55.314 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.40023455299999999, 2024-08-17T18:04:55.314 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.314 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.314 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:55.314 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.314 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.314 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.314 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79519 2024-08-17T18:04:55.314 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.315 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623580+0000", 2024-08-17T18:04:55.315 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623580+0000", 2024-08-17T18:04:55.315 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623581+0000", 2024-08-17T18:04:55.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5300000000000003e-07 2024-08-17T18:04:55.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623603+0000", 2024-08-17T18:04:55.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1936000000000001e-05 2024-08-17T18:04:55.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623614+0000", 2024-08-17T18:04:55.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1164e-05 2024-08-17T18:04:55.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623621+0000", 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7780000000000002e-06 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.626874+0000", 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032538810000000001 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627002+0000", 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000127176 2024-08-17T18:04:55.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:04:55.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627163+0000", 2024-08-17T18:04:55.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000160889 2024-08-17T18:04:55.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.630833+0000", 2024-08-17T18:04:55.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0036703339999999999 2024-08-17T18:04:55.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.630951+0000", 2024-08-17T18:04:55.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000117704 2024-08-17T18:04:55.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023677+0000", 2024-08-17T18:04:55.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.39272642699999999 2024-08-17T18:04:55.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:04:55.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023759+0000", 2024-08-17T18:04:55.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2211000000000003e-05 2024-08-17T18:04:55.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.320 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023814+0000", 2024-08-17T18:04:55.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.52e-05 2024-08-17T18:04:55.320 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.320 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.320 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.320 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79531 46.2 46:4c0c978d:::benchmark_data_smithi130_267720_object79530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:04:55.320 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.640037+0000", 2024-08-17T18:04:55.321 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 263.82073707299998, 2024-08-17T18:04:55.321 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38398499600000002, 2024-08-17T18:04:55.321 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.321 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.321 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:55.321 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.321 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.321 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.321 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79531 2024-08-17T18:04:55.321 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.321 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.321 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.321 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.321 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640037+0000", 2024-08-17T18:04:55.322 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.322 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.322 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.322 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.322 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640037+0000", 2024-08-17T18:04:55.322 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.322 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.322 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.322 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.322 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640038+0000", 2024-08-17T18:04:55.322 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9999999999999995e-07 2024-08-17T18:04:55.322 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.323 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640059+0000", 2024-08-17T18:04:55.323 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0310999999999999e-05 2024-08-17T18:04:55.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.323 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640071+0000", 2024-08-17T18:04:55.323 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2128e-05 2024-08-17T18:04:55.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.323 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640077+0000", 2024-08-17T18:04:55.323 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6379999999999998e-06 2024-08-17T18:04:55.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640129+0000", 2024-08-17T18:04:55.324 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2122e-05 2024-08-17T18:04:55.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640254+0000", 2024-08-17T18:04:55.324 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000124189 2024-08-17T18:04:55.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:04:55.325 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640382+0000", 2024-08-17T18:04:55.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012849700000000001 2024-08-17T18:04:55.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.325 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.325 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.325 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.643360+0000", 2024-08-17T18:04:55.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029779799999999999 2024-08-17T18:04:55.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.325 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.325 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.325 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.643375+0000", 2024-08-17T18:04:55.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5029e-05 2024-08-17T18:04:55.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.326 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023855+0000", 2024-08-17T18:04:55.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38047954299999998 2024-08-17T18:04:55.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.326 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:04:55.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023979+0000", 2024-08-17T18:04:55.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012451199999999999 2024-08-17T18:04:55.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.326 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.024022+0000", 2024-08-17T18:04:55.327 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3047000000000003e-05 2024-08-17T18:04:55.327 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.327 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.327 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.327 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79536 46.0 46:078355fd:::benchmark_data_smithi130_267720_object79535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:04:55.327 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.644048+0000", 2024-08-17T18:04:55.327 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 263.81672608100001, 2024-08-17T18:04:55.327 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37702704300000001, 2024-08-17T18:04:55.327 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.327 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.328 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:55.328 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.328 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.328 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.328 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79536 2024-08-17T18:04:55.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.328 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.328 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.328 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644048+0000", 2024-08-17T18:04:55.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.329 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.329 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644048+0000", 2024-08-17T18:04:55.329 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.329 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.329 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.329 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.329 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644049+0000", 2024-08-17T18:04:55.329 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2699999999999998e-07 2024-08-17T18:04:55.329 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.329 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.329 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.329 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644067+0000", 2024-08-17T18:04:55.329 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7757999999999998e-05 2024-08-17T18:04:55.330 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.330 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.330 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.330 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644076+0000", 2024-08-17T18:04:55.330 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.115e-06 2024-08-17T18:04:55.330 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.330 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.330 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.330 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644082+0000", 2024-08-17T18:04:55.330 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1140000000000002e-06 2024-08-17T18:04:55.330 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.330 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.330 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.331 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644134+0000", 2024-08-17T18:04:55.331 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1487999999999999e-05 2024-08-17T18:04:55.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.331 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.331 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.331 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644269+0000", 2024-08-17T18:04:55.331 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013505599999999999 2024-08-17T18:04:55.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.331 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.331 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:04:55.331 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644397+0000", 2024-08-17T18:04:55.331 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000127952 2024-08-17T18:04:55.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.331 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.646717+0000", 2024-08-17T18:04:55.332 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002320749 2024-08-17T18:04:55.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.332 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.647727+0000", 2024-08-17T18:04:55.332 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001009472 2024-08-17T18:04:55.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.332 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020974+0000", 2024-08-17T18:04:55.332 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37324734599999998 2024-08-17T18:04:55.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.333 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:04:55.333 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.021032+0000", 2024-08-17T18:04:55.333 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8298000000000003e-05 2024-08-17T18:04:55.333 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.333 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.333 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.021075+0000", 2024-08-17T18:04:55.333 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2867999999999997e-05 2024-08-17T18:04:55.333 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.333 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.333 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.333 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.334 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79539 46.2 46:487571cc:::benchmark_data_smithi130_267720_object79538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:04:55.334 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.648144+0000", 2024-08-17T18:04:55.334 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 263.81263052899999, 2024-08-17T18:04:55.334 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38286041199999998, 2024-08-17T18:04:55.334 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.334 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.334 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:55.334 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.334 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.334 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.334 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79539 2024-08-17T18:04:55.334 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.334 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.335 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.335 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.335 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648144+0000", 2024-08-17T18:04:55.335 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.335 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.335 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.335 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.335 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648144+0000", 2024-08-17T18:04:55.335 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.335 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.335 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.335 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.335 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648145+0000", 2024-08-17T18:04:55.336 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2359999999999999e-06 2024-08-17T18:04:55.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648168+0000", 2024-08-17T18:04:55.336 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2847e-05 2024-08-17T18:04:55.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648181+0000", 2024-08-17T18:04:55.336 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2819999999999999e-05 2024-08-17T18:04:55.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648188+0000", 2024-08-17T18:04:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1520000000000003e-06 2024-08-17T18:04:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648241+0000", 2024-08-17T18:04:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3285999999999998e-05 2024-08-17T18:04:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648387+0000", 2024-08-17T18:04:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014551500000000001 2024-08-17T18:04:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.337 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.338 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:04:55.338 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648543+0000", 2024-08-17T18:04:55.338 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000155957 2024-08-17T18:04:55.338 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.338 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.338 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.338 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.650656+0000", 2024-08-17T18:04:55.338 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021130250000000001 2024-08-17T18:04:55.338 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.338 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.338 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.338 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.650923+0000", 2024-08-17T18:04:55.338 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00026742200000000002 2024-08-17T18:04:55.338 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.339 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.339 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.339 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.030833+0000", 2024-08-17T18:04:55.339 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37991011499999999 2024-08-17T18:04:55.339 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.339 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.339 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:04:55.339 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.030953+0000", 2024-08-17T18:04:55.339 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000120186 2024-08-17T18:04:55.339 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.339 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.339 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.339 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.031004+0000", 2024-08-17T18:04:55.340 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0850999999999998e-05 2024-08-17T18:04:55.340 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.340 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.340 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.340 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.340 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.340 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:95869 46.d 46:bf83775d:::benchmark_data_smithi130_267720_object95868:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-08-17T18:04:55.340 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:49.269515+0000", 2024-08-17T18:04:55.340 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 245.19125955499999, 2024-08-17T18:04:55.340 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.36132123999999999, 2024-08-17T18:04:55.340 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.340 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.340 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:55.341 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.341 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.341 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.341 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 95869 2024-08-17T18:04:55.341 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.341 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.341 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.341 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.341 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269515+0000", 2024-08-17T18:04:55.341 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.341 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.341 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.341 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.342 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269515+0000", 2024-08-17T18:04:55.342 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.342 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.342 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.342 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.342 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269515+0000", 2024-08-17T18:04:55.342 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.68e-07 2024-08-17T18:04:55.342 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.342 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.342 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.342 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269531+0000", 2024-08-17T18:04:55.342 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6021000000000001e-05 2024-08-17T18:04:55.343 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.343 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.343 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.343 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269538+0000", 2024-08-17T18:04:55.343 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0419999999999997e-06 2024-08-17T18:04:55.343 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.343 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.343 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.343 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269544+0000", 2024-08-17T18:04:55.343 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2700000000000004e-06 2024-08-17T18:04:55.343 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.343 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.343 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.343 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.334777+0000", 2024-08-17T18:04:55.344 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.065232993000000003 2024-08-17T18:04:55.344 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.344 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.344 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.344 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.334940+0000", 2024-08-17T18:04:55.344 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000162898 2024-08-17T18:04:55.344 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.344 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.344 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:04:55.344 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.335062+0000", 2024-08-17T18:04:55.344 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012245800000000001 2024-08-17T18:04:55.344 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.344 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.345 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.345 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.344169+0000", 2024-08-17T18:04:55.345 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0091074559999999999 2024-08-17T18:04:55.345 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.345 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.345 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.345 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.344213+0000", 2024-08-17T18:04:55.345 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3868000000000001e-05 2024-08-17T18:04:55.345 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.345 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.345 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.345 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630486+0000", 2024-08-17T18:04:55.345 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.286272472 2024-08-17T18:04:55.346 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.346 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.346 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:04:55.346 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630682+0000", 2024-08-17T18:04:55.346 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000196422 2024-08-17T18:04:55.346 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.346 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.346 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.346 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630836+0000", 2024-08-17T18:04:55.346 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015377199999999999 2024-08-17T18:04:55.346 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.346 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.346 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.347 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.347 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.347 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:95871 46.d 46:bc19484e:::benchmark_data_smithi130_267720_object95870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-08-17T18:04:55.347 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:49.269561+0000", 2024-08-17T18:04:55.347 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 245.191213054, 2024-08-17T18:04:55.347 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.36284446199999998, 2024-08-17T18:04:55.347 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.347 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.347 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:55.347 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.347 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.347 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.347 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 95871 2024-08-17T18:04:55.348 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.348 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.348 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.348 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.348 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269561+0000", 2024-08-17T18:04:55.348 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.348 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.348 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.348 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.348 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269561+0000", 2024-08-17T18:04:55.348 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.348 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.348 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.349 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.349 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269562+0000", 2024-08-17T18:04:55.349 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0800000000000005e-07 2024-08-17T18:04:55.349 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.349 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.349 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.349 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269580+0000", 2024-08-17T18:04:55.349 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8471e-05 2024-08-17T18:04:55.349 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.349 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.349 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.349 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269587+0000", 2024-08-17T18:04:55.349 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.832e-06 2024-08-17T18:04:55.349 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.350 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.350 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.350 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269592+0000", 2024-08-17T18:04:55.350 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8189999999999999e-06 2024-08-17T18:04:55.350 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.350 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.350 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.350 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.338075+0000", 2024-08-17T18:04:55.350 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.068482687 2024-08-17T18:04:55.350 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.350 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.350 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.351 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.338231+0000", 2024-08-17T18:04:55.351 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000156169 2024-08-17T18:04:55.351 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.351 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.351 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:04:55.351 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.338381+0000", 2024-08-17T18:04:55.351 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000150748 2024-08-17T18:04:55.351 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.351 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.351 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.351 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.344225+0000", 2024-08-17T18:04:55.351 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0058432800000000002 2024-08-17T18:04:55.351 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.351 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.352 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.352 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.344270+0000", 2024-08-17T18:04:55.352 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4957999999999999e-05 2024-08-17T18:04:55.352 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.352 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.352 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.352 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.632288+0000", 2024-08-17T18:04:55.352 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.288017884 2024-08-17T18:04:55.352 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.352 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.352 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:04:55.352 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.632368+0000", 2024-08-17T18:04:55.352 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0476000000000007e-05 2024-08-17T18:04:55.352 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.353 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.353 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.353 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.632406+0000", 2024-08-17T18:04:55.353 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7629999999999997e-05 2024-08-17T18:04:55.353 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.353 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.353 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.353 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.353 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.353 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:95880 46.d 46:bbf69d49:::benchmark_data_smithi130_267720_object95879:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-08-17T18:04:55.353 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:49.334811+0000", 2024-08-17T18:04:55.353 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 245.125962904, 2024-08-17T18:04:55.353 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.29770117099999999, 2024-08-17T18:04:55.354 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.354 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.354 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:55.354 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.354 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.354 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.354 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 95880 2024-08-17T18:04:55.354 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.354 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.354 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.354 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.354 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.334811+0000", 2024-08-17T18:04:55.354 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.354 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.355 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.334811+0000", 2024-08-17T18:04:55.355 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.355 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.355 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.334813+0000", 2024-08-17T18:04:55.355 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.291e-06 2024-08-17T18:04:55.355 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.356 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.334838+0000", 2024-08-17T18:04:55.356 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5347e-05 2024-08-17T18:04:55.356 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.356 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.356 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.356 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.334847+0000", 2024-08-17T18:04:55.356 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9220000000000003e-06 2024-08-17T18:04:55.356 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.356 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.356 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.356 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.334853+0000", 2024-08-17T18:04:55.356 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7069999999999997e-06 2024-08-17T18:04:55.356 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.356 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.341151+0000", 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0062988039999999999 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.341291+0000", 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000139674 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.341439+0000", 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014737100000000001 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.357 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.358 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.348070+0000", 2024-08-17T18:04:55.358 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0066316719999999999 2024-08-17T18:04:55.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.358 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.358 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.348086+0000", 2024-08-17T18:04:55.358 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6101999999999999e-05 2024-08-17T18:04:55.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.358 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.358 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.632440+0000", 2024-08-17T18:04:55.358 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.28435371500000001 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.632483+0000", 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2852999999999998e-05 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.632513+0000", 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9713000000000002e-05 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.359 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.360 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:114685 46.0 46:062d3a5f:::benchmark_data_smithi130_267720_object6702:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1274)", 2024-08-17T18:04:55.360 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:01:12.800694+0000", 2024-08-17T18:04:55.360 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 221.66008059399999, 2024-08-17T18:04:55.360 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.292550949, 2024-08-17T18:04:55.360 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.360 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.360 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:55.360 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.360 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.360 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.360 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 114685 2024-08-17T18:04:55.360 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.360 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.360 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.361 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:12.800694+0000", 2024-08-17T18:04:55.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.361 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:12.800694+0000", 2024-08-17T18:04:55.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.361 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:12.800695+0000", 2024-08-17T18:04:55.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.435e-06 2024-08-17T18:04:55.362 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.362 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:12.800700+0000", 2024-08-17T18:04:55.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4909999999999999e-06 2024-08-17T18:04:55.362 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.362 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:12.800711+0000", 2024-08-17T18:04:55.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1613999999999999e-05 2024-08-17T18:04:55.362 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:12.800719+0000", 2024-08-17T18:04:55.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7579999999999992e-06 2024-08-17T18:04:55.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:12.802192+0000", 2024-08-17T18:04:55.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001473193 2024-08-17T18:04:55.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:12.802294+0000", 2024-08-17T18:04:55.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000101445 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:12.802395+0000", 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010128699999999999 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:12.810266+0000", 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0078712220000000006 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:12.832054+0000", 2024-08-17T18:04:55.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021787502 2024-08-17T18:04:55.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:13.093102+0000", 2024-08-17T18:04:55.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.26104845399999999 2024-08-17T18:04:55.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:04:55.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:13.093203+0000", 2024-08-17T18:04:55.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000100568 2024-08-17T18:04:55.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.366 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:01:13.093245+0000", 2024-08-17T18:04:55.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1980000000000001e-05 2024-08-17T18:04:55.366 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.366 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.366 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.366 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.366 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195096 46.9 46:9e985471:::benchmark_data_smithi130_267720_object87113:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-08-17T18:04:55.366 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.852302+0000", 2024-08-17T18:04:55.366 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 121.608472568, 2024-08-17T18:04:55.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.309431236, 2024-08-17T18:04:55.366 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.366 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.367 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:04:55.367 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.367 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.367 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.367 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195096 2024-08-17T18:04:55.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.367 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.367 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.367 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.367 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852302+0000", 2024-08-17T18:04:55.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.367 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.367 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852302+0000", 2024-08-17T18:04:55.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852303+0000", 2024-08-17T18:04:55.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.33e-06 2024-08-17T18:04:55.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852308+0000", 2024-08-17T18:04:55.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4079999999999997e-06 2024-08-17T18:04:55.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.369 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852319+0000", 2024-08-17T18:04:55.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0382e-05 2024-08-17T18:04:55.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.369 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852325+0000", 2024-08-17T18:04:55.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4649999999999999e-06 2024-08-17T18:04:55.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.369 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852449+0000", 2024-08-17T18:04:55.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012352000000000001 2024-08-17T18:04:55.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852579+0000", 2024-08-17T18:04:55.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013025800000000001 2024-08-17T18:04:55.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,6", 2024-08-17T18:04:55.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852699+0000", 2024-08-17T18:04:55.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011966799999999999 2024-08-17T18:04:55.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.856209+0000", 2024-08-17T18:04:55.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0035104649999999999 2024-08-17T18:04:55.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.371 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.371 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.371 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873340+0000", 2024-08-17T18:04:55.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017130537000000001 2024-08-17T18:04:55.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.371 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.371 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.371 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934040+0000", 2024-08-17T18:04:55.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.060700089999999998 2024-08-17T18:04:55.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.371 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.371 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.372 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934049+0000", 2024-08-17T18:04:55.372 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1239999999999994e-06 2024-08-17T18:04:55.372 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.372 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.372 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.372 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877962+0000", 2024-08-17T18:04:55.372 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94391303400000004 2024-08-17T18:04:55.372 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.372 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.372 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.372 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877971+0000", 2024-08-17T18:04:55.372 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0100000000000001e-06 2024-08-17T18:04:55.372 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.372 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.373 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.373 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.161671+0000", 2024-08-17T18:04:55.373 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.283700372 2024-08-17T18:04:55.373 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.373 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.373 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.373 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.161733+0000", 2024-08-17T18:04:55.373 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1573000000000002e-05 2024-08-17T18:04:55.373 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.373 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.373 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.373 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.373 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.374 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195105 46.9 46:94a33cb3:::benchmark_data_smithi130_267720_object87122:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-08-17T18:04:55.374 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.862150+0000", 2024-08-17T18:04:55.374 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 121.598624598, 2024-08-17T18:04:55.374 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3538093619999998, 2024-08-17T18:04:55.374 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.374 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.374 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:04:55.374 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.374 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.374 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.374 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195105 2024-08-17T18:04:55.374 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.374 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.375 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.375 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.375 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862150+0000", 2024-08-17T18:04:55.375 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.375 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.375 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.375 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.375 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862150+0000", 2024-08-17T18:04:55.375 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.375 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.375 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.375 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.375 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862151+0000", 2024-08-17T18:04:55.375 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3170000000000001e-06 2024-08-17T18:04:55.376 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.376 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.376 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.376 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862157+0000", 2024-08-17T18:04:55.376 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5040000000000002e-06 2024-08-17T18:04:55.376 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.376 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.376 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.376 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862169+0000", 2024-08-17T18:04:55.376 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2520999999999999e-05 2024-08-17T18:04:55.376 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.376 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.376 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.376 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862176+0000", 2024-08-17T18:04:55.377 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1269999999999997e-06 2024-08-17T18:04:55.377 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.377 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.377 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.377 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.868137+0000", 2024-08-17T18:04:55.377 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0059607250000000001 2024-08-17T18:04:55.377 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.377 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.377 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.377 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.868274+0000", 2024-08-17T18:04:55.377 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013746200000000001 2024-08-17T18:04:55.377 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.377 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.377 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,6", 2024-08-17T18:04:55.378 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.868404+0000", 2024-08-17T18:04:55.378 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012964699999999999 2024-08-17T18:04:55.378 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.378 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.378 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.378 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.879188+0000", 2024-08-17T18:04:55.378 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010783915 2024-08-17T18:04:55.378 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.378 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.378 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.378 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934076+0000", 2024-08-17T18:04:55.378 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.054887752999999997 2024-08-17T18:04:55.378 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.379 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.379 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.379 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934083+0000", 2024-08-17T18:04:55.379 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9859999999999997e-06 2024-08-17T18:04:55.379 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.379 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.379 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.379 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878000+0000", 2024-08-17T18:04:55.379 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94391712299999997 2024-08-17T18:04:55.379 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.379 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.379 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.379 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878009+0000", 2024-08-17T18:04:55.380 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.884e-06 2024-08-17T18:04:55.380 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.380 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.380 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.380 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.161985+0000", 2024-08-17T18:04:55.380 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2839767369999999 2024-08-17T18:04:55.380 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.380 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.380 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:04:55.380 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.162438+0000", 2024-08-17T18:04:55.380 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00045227700000000001 2024-08-17T18:04:55.380 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.380 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.380 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.381 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215917+0000", 2024-08-17T18:04:55.381 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.053479086000000002 2024-08-17T18:04:55.381 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.381 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.381 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.381 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215959+0000", 2024-08-17T18:04:55.381 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2298000000000001e-05 2024-08-17T18:04:55.381 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.381 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.381 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.381 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.381 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.381 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195107 46.9 46:9aacae76:::benchmark_data_smithi130_267720_object87124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-08-17T18:04:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.862322+0000", 2024-08-17T18:04:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 121.598452311, 2024-08-17T18:04:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3730563999999998, 2024-08-17T18:04:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:04:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195107 2024-08-17T18:04:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.382 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.383 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862322+0000", 2024-08-17T18:04:55.383 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.383 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.383 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.383 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.383 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862322+0000", 2024-08-17T18:04:55.383 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.383 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.383 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.383 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.383 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862323+0000", 2024-08-17T18:04:55.383 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3200000000000005e-07 2024-08-17T18:04:55.383 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.384 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.384 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.384 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862326+0000", 2024-08-17T18:04:55.384 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3249999999999999e-06 2024-08-17T18:04:55.384 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.384 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.384 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.384 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862335+0000", 2024-08-17T18:04:55.384 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0149999999999992e-06 2024-08-17T18:04:55.384 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.384 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.384 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.384 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862341+0000", 2024-08-17T18:04:55.385 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9909999999999999e-06 2024-08-17T18:04:55.385 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.385 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.385 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.385 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873392+0000", 2024-08-17T18:04:55.385 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011050658 2024-08-17T18:04:55.385 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.385 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.385 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.385 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873518+0000", 2024-08-17T18:04:55.385 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012659100000000001 2024-08-17T18:04:55.385 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.386 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.386 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,6", 2024-08-17T18:04:55.386 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873640+0000", 2024-08-17T18:04:55.386 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012183199999999999 2024-08-17T18:04:55.386 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.386 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.386 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.386 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934107+0000", 2024-08-17T18:04:55.386 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.060466796000000003 2024-08-17T18:04:55.386 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.386 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.386 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.387 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934114+0000", 2024-08-17T18:04:55.387 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0690000000000004e-06 2024-08-17T18:04:55.387 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.387 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.387 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.387 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878038+0000", 2024-08-17T18:04:55.387 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94392365499999997 2024-08-17T18:04:55.387 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.387 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.387 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.387 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878047+0000", 2024-08-17T18:04:55.387 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9600000000000006e-06 2024-08-17T18:04:55.387 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.388 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.388 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.388 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215977+0000", 2024-08-17T18:04:55.388 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3379302630000001 2024-08-17T18:04:55.388 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.388 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.388 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:04:55.388 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216122+0000", 2024-08-17T18:04:55.388 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014515900000000001 2024-08-17T18:04:55.388 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.388 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.388 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.388 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.235324+0000", 2024-08-17T18:04:55.389 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019201732999999999 2024-08-17T18:04:55.389 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.389 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.389 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.389 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.235378+0000", 2024-08-17T18:04:55.389 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4721e-05 2024-08-17T18:04:55.389 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.389 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.389 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.389 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.389 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.389 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195109 46.9 46:941627c3:::benchmark_data_smithi130_267720_object87126:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-08-17T18:04:55.389 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.870700+0000", 2024-08-17T18:04:55.390 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 121.59007453700001, 2024-08-17T18:04:55.390 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3651783769999999, 2024-08-17T18:04:55.390 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.390 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.390 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:04:55.390 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.390 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.390 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.390 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195109 2024-08-17T18:04:55.390 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.390 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.390 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.390 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870700+0000", 2024-08-17T18:04:55.391 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.391 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.391 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.391 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870700+0000", 2024-08-17T18:04:55.391 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.391 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.391 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.391 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870701+0000", 2024-08-17T18:04:55.391 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.415e-06 2024-08-17T18:04:55.391 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.391 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.392 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.392 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870704+0000", 2024-08-17T18:04:55.392 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8899999999999999e-06 2024-08-17T18:04:55.392 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.392 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.392 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.392 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870712+0000", 2024-08-17T18:04:55.392 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2579999999999995e-06 2024-08-17T18:04:55.392 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.392 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.392 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.392 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870718+0000", 2024-08-17T18:04:55.392 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.327e-06 2024-08-17T18:04:55.392 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.879223+0000", 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0085048620000000002 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.879355+0000", 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013201599999999999 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,6", 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.879499+0000", 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014413800000000001 2024-08-17T18:04:55.393 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.394 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.394 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.394 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934138+0000", 2024-08-17T18:04:55.394 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.05463925 2024-08-17T18:04:55.394 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.394 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.394 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.394 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934145+0000", 2024-08-17T18:04:55.394 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7399999999999998e-06 2024-08-17T18:04:55.394 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.394 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.394 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.394 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878105+0000", 2024-08-17T18:04:55.395 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94396076500000003 2024-08-17T18:04:55.395 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.395 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.395 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.395 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878115+0000", 2024-08-17T18:04:55.395 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3489999999999998e-06 2024-08-17T18:04:55.395 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.395 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.395 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.395 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216148+0000", 2024-08-17T18:04:55.395 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3380333520000001 2024-08-17T18:04:55.395 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.396 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.396 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:04:55.396 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216284+0000", 2024-08-17T18:04:55.396 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000135513 2024-08-17T18:04:55.396 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.396 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.396 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.396 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.235773+0000", 2024-08-17T18:04:55.396 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019489216 2024-08-17T18:04:55.396 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.396 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.396 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.396 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.235878+0000", 2024-08-17T18:04:55.397 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010528599999999999 2024-08-17T18:04:55.397 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.397 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.397 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.397 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.397 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.397 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195099 46.f 46:fad5c3c4:::benchmark_data_smithi130_267720_object87116:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:04:55.397 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.873814+0000", 2024-08-17T18:04:55.397 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 121.586960742, 2024-08-17T18:04:55.397 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.301145886, 2024-08-17T18:04:55.397 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.397 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.397 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:04:55.398 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.398 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.398 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.398 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195099 2024-08-17T18:04:55.398 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.398 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.398 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.398 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.398 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873814+0000", 2024-08-17T18:04:55.398 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.398 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.398 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.398 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.399 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873814+0000", 2024-08-17T18:04:55.399 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.399 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.399 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.399 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.399 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873816+0000", 2024-08-17T18:04:55.399 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9769999999999999e-06 2024-08-17T18:04:55.399 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.399 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.399 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.399 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873823+0000", 2024-08-17T18:04:55.399 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2590000000000003e-06 2024-08-17T18:04:55.399 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.399 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.400 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.400 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873834+0000", 2024-08-17T18:04:55.400 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1243000000000001e-05 2024-08-17T18:04:55.400 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.400 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.400 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.400 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873843+0000", 2024-08-17T18:04:55.400 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6079999999999996e-06 2024-08-17T18:04:55.400 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.400 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.400 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.400 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878460+0000", 2024-08-17T18:04:55.400 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046169899999999996 2024-08-17T18:04:55.400 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op must wait for map", 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878467+0000", 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9399999999999996e-06 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.932773+0000", 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.054306074000000003 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.932783+0000", 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0214000000000001e-05 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.401 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.402 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.402 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877857+0000", 2024-08-17T18:04:55.402 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94507391299999999 2024-08-17T18:04:55.402 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.402 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.402 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.402 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877867+0000", 2024-08-17T18:04:55.402 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0132000000000001e-05 2024-08-17T18:04:55.402 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.402 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.402 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.402 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.172642+0000", 2024-08-17T18:04:55.402 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2947751140000001 2024-08-17T18:04:55.402 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.174959+0000", 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002317422 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195108 46.f 46:f8e03b9e:::benchmark_data_smithi130_267720_object87125:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.873857+0000", 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 121.586917723, 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3014579089999998, 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.403 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.404 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:04:55.404 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.404 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.404 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.404 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195108 2024-08-17T18:04:55.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.404 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.404 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.404 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.404 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873857+0000", 2024-08-17T18:04:55.404 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.404 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.404 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.404 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.405 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873857+0000", 2024-08-17T18:04:55.405 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.405 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.405 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.405 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.405 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873857+0000", 2024-08-17T18:04:55.405 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8599999999999998e-07 2024-08-17T18:04:55.405 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.405 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.405 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.405 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873859+0000", 2024-08-17T18:04:55.405 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2819999999999999e-06 2024-08-17T18:04:55.405 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.405 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.406 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.406 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873865+0000", 2024-08-17T18:04:55.406 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1310000000000003e-06 2024-08-17T18:04:55.406 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.406 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.406 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.406 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873870+0000", 2024-08-17T18:04:55.406 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5770000000000004e-06 2024-08-17T18:04:55.406 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.406 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.406 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.406 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878516+0000", 2024-08-17T18:04:55.406 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046464200000000001 2024-08-17T18:04:55.407 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.407 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.407 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting_for_map not empty", 2024-08-17T18:04:55.407 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878522+0000", 2024-08-17T18:04:55.407 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.259e-06 2024-08-17T18:04:55.407 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.407 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.407 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.407 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.932812+0000", 2024-08-17T18:04:55.407 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.054289865 2024-08-17T18:04:55.407 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.407 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.407 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.407 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.932818+0000", 2024-08-17T18:04:55.408 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7259999999999996e-06 2024-08-17T18:04:55.408 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.408 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.408 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.408 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877927+0000", 2024-08-17T18:04:55.408 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94510819099999999 2024-08-17T18:04:55.408 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.408 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.408 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.408 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877935+0000", 2024-08-17T18:04:55.408 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5280000000000003e-06 2024-08-17T18:04:55.408 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.408 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.409 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.409 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.175240+0000", 2024-08-17T18:04:55.409 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.297304512 2024-08-17T18:04:55.409 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.409 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.409 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.409 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.175314+0000", 2024-08-17T18:04:55.409 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4931999999999993e-05 2024-08-17T18:04:55.409 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.409 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.409 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.409 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.409 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.409 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195110 46.1 46:81ba4812:::benchmark_data_smithi130_267720_object87127:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:04:55.410 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.873876+0000", 2024-08-17T18:04:55.410 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 121.58689824, 2024-08-17T18:04:55.410 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.503486423, 2024-08-17T18:04:55.410 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.410 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.410 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:55.410 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.410 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.410 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.410 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195110 2024-08-17T18:04:55.410 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.410 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.410 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.411 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.411 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873876+0000", 2024-08-17T18:04:55.411 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.411 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.411 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.411 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.411 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873876+0000", 2024-08-17T18:04:55.411 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.411 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.411 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.411 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.411 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873876+0000", 2024-08-17T18:04:55.411 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.84e-07 2024-08-17T18:04:55.412 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.412 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.412 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.412 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873878+0000", 2024-08-17T18:04:55.412 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9410000000000001e-06 2024-08-17T18:04:55.412 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.412 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.412 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.412 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.874329+0000", 2024-08-17T18:04:55.412 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00045083200000000002 2024-08-17T18:04:55.412 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.412 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.412 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.412 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.874337+0000", 2024-08-17T18:04:55.413 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2009999999999999e-06 2024-08-17T18:04:55.413 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.413 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.413 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.413 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934172+0000", 2024-08-17T18:04:55.413 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.059834948999999998 2024-08-17T18:04:55.413 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.413 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.413 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.413 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934179+0000", 2024-08-17T18:04:55.413 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0949999999999998e-06 2024-08-17T18:04:55.413 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.413 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.413 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.414 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877887+0000", 2024-08-17T18:04:55.414 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94370757699999996 2024-08-17T18:04:55.414 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.414 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.414 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.414 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877897+0000", 2024-08-17T18:04:55.414 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6600000000000007e-06 2024-08-17T18:04:55.414 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.414 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.414 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.414 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.204539+0000", 2024-08-17T18:04:55.414 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.32664223 2024-08-17T18:04:55.415 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.415 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.415 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.415 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.206374+0000", 2024-08-17T18:04:55.415 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001834638 2024-08-17T18:04:55.415 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.415 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.415 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,6,7", 2024-08-17T18:04:55.415 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.206574+0000", 2024-08-17T18:04:55.415 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020093300000000001 2024-08-17T18:04:55.415 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.415 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.415 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.415 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.218471+0000", 2024-08-17T18:04:55.416 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011896487000000001 2024-08-17T18:04:55.416 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.416 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.416 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.416 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.218487+0000", 2024-08-17T18:04:55.416 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6118999999999999e-05 2024-08-17T18:04:55.416 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.416 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.416 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.416 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.232569+0000", 2024-08-17T18:04:55.416 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014081425 2024-08-17T18:04:55.416 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.416 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.416 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.417 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.376529+0000", 2024-08-17T18:04:55.417 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14396075699999999 2024-08-17T18:04:55.417 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.417 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.417 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:04:55.417 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.376663+0000", 2024-08-17T18:04:55.417 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013359900000000001 2024-08-17T18:04:55.417 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.417 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.417 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.417 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377362+0000", 2024-08-17T18:04:55.417 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00069959600000000003 2024-08-17T18:04:55.417 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.418 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.418 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.418 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.418 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.418 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195111 46.9 46:9346673d:::benchmark_data_smithi130_267720_object87128:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:04:55.418 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.878090+0000", 2024-08-17T18:04:55.418 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 121.58268430699999, 2024-08-17T18:04:55.418 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4994862809999998, 2024-08-17T18:04:55.418 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.418 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.418 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:55.419 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.419 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.419 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.419 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195111 2024-08-17T18:04:55.419 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.419 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.419 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.419 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.419 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878090+0000", 2024-08-17T18:04:55.419 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.419 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.419 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.419 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.419 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878090+0000", 2024-08-17T18:04:55.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.420 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.420 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.420 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.420 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878092+0000", 2024-08-17T18:04:55.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.548e-06 2024-08-17T18:04:55.420 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.420 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.420 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.420 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878097+0000", 2024-08-17T18:04:55.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.665e-06 2024-08-17T18:04:55.420 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.420 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.421 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.421 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878115+0000", 2024-08-17T18:04:55.421 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7829000000000002e-05 2024-08-17T18:04:55.421 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.421 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.421 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.421 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878123+0000", 2024-08-17T18:04:55.421 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8029999999999997e-06 2024-08-17T18:04:55.421 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.421 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.421 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.421 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934199+0000", 2024-08-17T18:04:55.421 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.056076150999999998 2024-08-17T18:04:55.422 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.422 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.422 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.422 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934205+0000", 2024-08-17T18:04:55.422 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3559999999999998e-06 2024-08-17T18:04:55.422 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.422 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.422 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.422 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878140+0000", 2024-08-17T18:04:55.422 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.943934355 2024-08-17T18:04:55.422 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.422 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.422 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.423 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878148+0000", 2024-08-17T18:04:55.423 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8399999999999995e-06 2024-08-17T18:04:55.423 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.423 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.423 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.423 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216331+0000", 2024-08-17T18:04:55.423 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3381836009999999 2024-08-17T18:04:55.423 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.423 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.423 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.423 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216449+0000", 2024-08-17T18:04:55.423 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000117873 2024-08-17T18:04:55.423 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.424 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.424 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,3,6", 2024-08-17T18:04:55.424 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216861+0000", 2024-08-17T18:04:55.424 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00041161799999999999 2024-08-17T18:04:55.424 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.424 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.424 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.424 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.229816+0000", 2024-08-17T18:04:55.424 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012955112 2024-08-17T18:04:55.424 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.424 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.424 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.424 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.232815+0000", 2024-08-17T18:04:55.424 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029990630000000002 2024-08-17T18:04:55.425 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.425 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.425 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.425 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.234938+0000", 2024-08-17T18:04:55.425 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002122779 2024-08-17T18:04:55.425 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.425 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.425 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.425 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377412+0000", 2024-08-17T18:04:55.425 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14247422800000001 2024-08-17T18:04:55.425 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.425 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.426 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:04:55.426 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377471+0000", 2024-08-17T18:04:55.426 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9032e-05 2024-08-17T18:04:55.426 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.426 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.426 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.426 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377576+0000", 2024-08-17T18:04:55.426 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010542799999999999 2024-08-17T18:04:55.426 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.426 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.426 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.426 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.426 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.427 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195121 46.1 46:8cbb5caf:::benchmark_data_smithi130_267720_object87138:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:04:55.427 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.946248+0000", 2024-08-17T18:04:55.427 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 121.51452609099999, 2024-08-17T18:04:55.427 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4315148899999999, 2024-08-17T18:04:55.427 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.427 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.427 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:55.427 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.427 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.427 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.427 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195121 2024-08-17T18:04:55.427 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.427 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.428 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.428 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.428 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946248+0000", 2024-08-17T18:04:55.428 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.428 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.428 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.428 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.428 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946248+0000", 2024-08-17T18:04:55.428 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.428 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.428 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.428 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.428 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946250+0000", 2024-08-17T18:04:55.428 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4649999999999999e-06 2024-08-17T18:04:55.429 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.429 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.429 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.429 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946256+0000", 2024-08-17T18:04:55.429 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8839999999999999e-06 2024-08-17T18:04:55.429 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.429 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.429 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.429 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946270+0000", 2024-08-17T18:04:55.429 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3993e-05 2024-08-17T18:04:55.429 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.429 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.429 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.430 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946276+0000", 2024-08-17T18:04:55.430 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2400000000000004e-06 2024-08-17T18:04:55.430 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.430 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.430 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.430 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946334+0000", 2024-08-17T18:04:55.430 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8402000000000001e-05 2024-08-17T18:04:55.430 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.430 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.430 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.430 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946345+0000", 2024-08-17T18:04:55.430 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0645999999999999e-05 2024-08-17T18:04:55.430 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.430 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.431 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.431 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877922+0000", 2024-08-17T18:04:55.431 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.93157699800000004 2024-08-17T18:04:55.431 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.431 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.431 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.431 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877930+0000", 2024-08-17T18:04:55.431 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7079999999999997e-06 2024-08-17T18:04:55.431 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.431 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.431 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.431 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215774+0000", 2024-08-17T18:04:55.431 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.337844566 2024-08-17T18:04:55.431 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.432 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.432 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.432 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215906+0000", 2024-08-17T18:04:55.432 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013154400000000001 2024-08-17T18:04:55.432 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.432 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.432 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,6,7", 2024-08-17T18:04:55.432 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216043+0000", 2024-08-17T18:04:55.432 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000137288 2024-08-17T18:04:55.432 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.432 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.432 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.432 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.218520+0000", 2024-08-17T18:04:55.432 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002477492 2024-08-17T18:04:55.433 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.433 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.433 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.433 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.229810+0000", 2024-08-17T18:04:55.433 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011289123999999999 2024-08-17T18:04:55.433 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.433 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.433 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.433 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.234501+0000", 2024-08-17T18:04:55.433 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046915749999999999 2024-08-17T18:04:55.433 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.433 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.433 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.433 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377616+0000", 2024-08-17T18:04:55.434 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14311495099999999 2024-08-17T18:04:55.434 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.434 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.434 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:04:55.434 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377664+0000", 2024-08-17T18:04:55.434 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8047999999999998e-05 2024-08-17T18:04:55.434 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.434 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.434 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.434 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377763+0000", 2024-08-17T18:04:55.434 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8966000000000001e-05 2024-08-17T18:04:55.434 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.434 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.434 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.435 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.435 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.435 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195124 46.2 46:497fbb11:::benchmark_data_smithi130_267720_object87141:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:04:55.435 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.956154+0000", 2024-08-17T18:04:55.435 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 121.504620241, 2024-08-17T18:04:55.435 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2343982279999999, 2024-08-17T18:04:55.435 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:04:55.435 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:04:55.435 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:04:55.435 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:04:55.435 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:04:55.435 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:04:55.435 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195124 2024-08-17T18:04:55.435 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956154+0000", 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956154+0000", 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956155+0000", 2024-08-17T18:04:55.436 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1119999999999999e-06 2024-08-17T18:04:55.437 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.437 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.437 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:04:55.437 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956159+0000", 2024-08-17T18:04:55.437 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2679999999999996e-06 2024-08-17T18:04:55.437 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.437 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.437 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:04:55.437 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956170+0000", 2024-08-17T18:04:55.437 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0334e-05 2024-08-17T18:04:55.437 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.437 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.437 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:04:55.437 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956177+0000", 2024-08-17T18:04:55.438 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2849999999999997e-06 2024-08-17T18:04:55.438 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.438 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.438 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.438 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956244+0000", 2024-08-17T18:04:55.438 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6994000000000003e-05 2024-08-17T18:04:55.438 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.438 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.438 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.438 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956257+0000", 2024-08-17T18:04:55.438 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3196e-05 2024-08-17T18:04:55.438 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.438 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.438 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.439 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.876745+0000", 2024-08-17T18:04:55.439 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.92048745399999998 2024-08-17T18:04:55.439 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.439 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.439 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:04:55.439 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.876755+0000", 2024-08-17T18:04:55.439 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0001e-05 2024-08-17T18:04:55.439 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.439 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.439 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:04:55.439 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.169422+0000", 2024-08-17T18:04:55.439 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.292667488 2024-08-17T18:04:55.439 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.439 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.169523+0000", 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000101116 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6,7", 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.169908+0000", 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000384936 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.182867+0000", 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012958374 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.440 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.441 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:04:55.441 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.189357+0000", 2024-08-17T18:04:55.441 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0064902980000000002 2024-08-17T18:04:55.441 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.441 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.441 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.441 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.189592+0000", 2024-08-17T18:04:55.441 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00023549000000000001 2024-08-17T18:04:55.441 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.441 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.441 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:04:55.441 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.190415+0000", 2024-08-17T18:04:55.441 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00082231100000000005 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.190452+0000", 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7054999999999997e-05 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.190552+0000", 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000100516 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:04:55.442 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:04:55.443 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:04:55.443 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:55.464 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:55.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:55.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:55.566 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:55.589 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:55.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:55.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:55.664 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 55 16 69434 69418 78.8721 53.5 0.0113932 0.0126728 2024-08-17T18:04:55.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:55 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:04:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:55 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:04:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:55 smithi130 ceph-mon[105521]: osdmap e1313: 8 total, 8 up, 5 in 2024-08-17T18:04:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:55 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:04:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:55 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:04:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:55 smithi130 ceph-mon[110068]: osdmap e1313: 8 total, 8 up, 5 in 2024-08-17T18:04:55.690 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:55.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:55.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:55.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:55.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:55 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:04:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:55 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:04:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:55 smithi179 ceph-mon[93519]: osdmap e1313: 8 total, 8 up, 5 in 2024-08-17T18:04:55.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:55.854 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:55.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:55.955 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:55.976 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:55.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:55.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:56.078 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:56.120 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:04:56.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:56.121 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:56.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:56.222 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:56.263 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:56.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:56.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:56.365 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:56.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:56.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:56.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:56.507 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:56.528 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:56.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:56.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:56.630 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:56.651 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:56.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:56.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:56.664 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 56 16 70326 70310 78.4591 55.75 0.0171318 0.0127403 2024-08-17T18:04:56.752 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:56.774 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:56.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:56.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:56 smithi179 ceph-mon[93519]: pgmap v2223: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 56 MiB/s wr, 902 op/s 2024-08-17T18:04:56.876 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:56.914 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:56.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:56.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:56.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:56 smithi130 ceph-mon[105521]: pgmap v2223: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 56 MiB/s wr, 902 op/s 2024-08-17T18:04:56.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:56 smithi130 ceph-mon[110068]: pgmap v2223: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 56 MiB/s wr, 902 op/s 2024-08-17T18:04:57.016 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:57.038 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:57.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:57.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:57.140 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:57.161 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:57.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:57.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:57.263 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:57.300 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:57.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:57.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:57.402 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:04:57.423 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:57.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:57.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:04:57.525 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:04:57.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:57.579 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:57.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:04:57.621 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T18:04:57.667 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 57 16 71113 71097 77.941 49.1875 0.0232863 0.0128238 2024-08-17T18:04:57.681 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:57.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:57.776 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:57.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:57.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:04:57.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:04:57.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:04:57.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:04:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:04:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:04:57.878 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:57.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:57.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:57.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:58.052 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:58.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:58.104 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:58.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:58.205 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:58.239 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:58.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:58.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:58.341 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:04:58.363 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:58.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:58.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:04:58.465 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:58.500 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:58.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:58.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:04:58 smithi179 ceph-mon[93519]: pgmap v2224: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 55 MiB/s wr, 884 op/s 2024-08-17T18:04:58.604 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:58.667 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 58 16 71940 71924 77.4883 51.6875 0.0243349 0.0128976 2024-08-17T18:04:58.700 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:04:58.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:58.701 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:58.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:58.708 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:04:58.802 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:58.823 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:58.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:58.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:04:58.872 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:04:58 smithi130 ceph-mon[110068]: pgmap v2224: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 55 MiB/s wr, 884 op/s 2024-08-17T18:04:58.872 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:04:58 smithi130 ceph-mon[105521]: pgmap v2224: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 55 MiB/s wr, 884 op/s 2024-08-17T18:04:58.926 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:58.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:58.974 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:58.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:59.075 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:59.131 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:59.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:59.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:59.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:04:59.277 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:59.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:59.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:04:59.379 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:59.402 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:59.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:59.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:59.504 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:04:59.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:59.556 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:59.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:04:59.657 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:04:59.669 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 59 16 72780 72764 77.0625 52.5 0.0165704 0.0129704 2024-08-17T18:04:59.678 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:59.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:59.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:04:59.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:04:59.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:59.851 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:04:59.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:04:59.952 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:04:59.974 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:04:59.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:59.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:00.076 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:00.097 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:00.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:00.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:00.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:04:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:04:59] "GET /metrics HTTP/1.1" 200 36779 "" "Prometheus/2.51.0" 2024-08-17T18:05:00.199 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:00.220 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:00.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:00.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:00.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:00.367 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:00.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:00.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:00.469 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:00.515 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:00.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:00.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:00.537 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:05:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:05:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:05:00.618 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:00.629 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:05:00.639 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:00.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:00.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:00.669 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:05:00.669806+0000 min lat: 0.00111317 max lat: 0.450837 avg lat: 0.013039 2024-08-17T18:05:00.669 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:05:00.669 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 60 16 73571 73555 76.6021 49.4375 0.0937741 0.013039 2024-08-17T18:05:00.741 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:00.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:00.786 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:00.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:00 smithi179 ceph-mon[93519]: pgmap v2225: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 54 MiB/s wr, 871 op/s 2024-08-17T18:05:00.886 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:00.908 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:00.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:00.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:00 smithi130 ceph-mon[105521]: pgmap v2225: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 54 MiB/s wr, 871 op/s 2024-08-17T18:05:00.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:00 smithi130 ceph-mon[110068]: pgmap v2225: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 54 MiB/s wr, 871 op/s 2024-08-17T18:05:01.010 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:01.031 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:01.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:01.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:01.133 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:01.156 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:01.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:01.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:01.258 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:01.294 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:01.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:01.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:01.397 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:01.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:01.419 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:01.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:01.520 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:01.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:01.557 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:01.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:01.658 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:01.669 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 61 16 74336 74320 76.1301 47.8125 0.0116084 0.0131299 2024-08-17T18:05:01.679 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:01.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:01.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:01.781 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:01.803 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:01.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:01.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:01.905 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:01.927 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:01.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:01.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:02.029 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:02.057 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:02.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:02.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:02.159 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:02.199 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:02.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:02.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:02.300 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:02.333 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:02.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:02.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:02.434 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:02.457 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:02.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:02.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:02.559 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:02.580 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:02.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:02.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:02.670 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 62 16 75001 74985 75.5726 41.5625 0.0173682 0.0132261 2024-08-17T18:05:02.681 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:02.702 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:02.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:02.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:02.804 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:02.830 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:02.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:02.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:02 smithi179 ceph-mon[93519]: pgmap v2226: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 54 MiB/s wr, 871 op/s 2024-08-17T18:05:02.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:02 smithi130 ceph-mon[105521]: pgmap v2226: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 54 MiB/s wr, 871 op/s 2024-08-17T18:05:02.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:02 smithi130 ceph-mon[110068]: pgmap v2226: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 54 MiB/s wr, 871 op/s 2024-08-17T18:05:02.932 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:02.966 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:02.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:02.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:03.068 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:03.116 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:03.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:03.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:03.218 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:03.240 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:03.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:03.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:03.342 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:03.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:03.386 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:03.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:03.487 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:03.526 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:03.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:03.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:03.628 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:03.668 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:03.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:03.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:03.670 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 63 16 75643 75627 75.01 40.125 0.0238194 0.0133242 2024-08-17T18:05:03.736 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:05:03.736 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:05:03.736 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:05:03.736 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:05:03.736 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:05:03.736 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:05:03.736 INFO:teuthology.orchestra.run.smithi130.stderr:bench [] [] [] [] 2024-08-17T18:05:03.736 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:05:03.736 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:05:03.736 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:05:03.736 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator dump block 2024-08-17T18:05:03.737 INFO:teuthology.orchestra.run.smithi130.stderr:Error EINVAL: invalid command 2024-08-17T18:05:03.770 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:03.812 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:03.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:03.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:03.915 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:03.937 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:03.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:03.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:04.038 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:04.061 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:04.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:04.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:04.162 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:04.183 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:04.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:04.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:04.285 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:04.307 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:04.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:04.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:04 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:04 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:04.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:04 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:04.392 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:04 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:04.392 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:04 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:04.392 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:04 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:04.409 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:04.465 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:04.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:04.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:04.568 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:04.589 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:04.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:04.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:04.671 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 64 16 76514 76498 74.6873 54.4375 0.0170354 0.0133829 2024-08-17T18:05:04.691 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:04.713 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:04.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:04.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:04.815 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:04.836 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:04.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:04.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:04.937 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:04.959 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:04.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:04.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:05.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:05.102 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:05.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:05.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:05.204 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:05.226 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:05.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:05.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:05.328 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:05.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:05 smithi179 ceph-mon[93519]: pgmap v2227: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 52 MiB/s wr, 826 op/s 2024-08-17T18:05:05.349 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:05 smithi130 ceph-mon[105521]: pgmap v2227: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 52 MiB/s wr, 826 op/s 2024-08-17T18:05:05.349 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:05 smithi130 ceph-mon[110068]: pgmap v2227: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 52 MiB/s wr, 826 op/s 2024-08-17T18:05:05.349 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:05.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:05.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:05.451 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:05.491 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:05.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:05.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:05.593 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:05.614 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:05.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:05.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:05.671 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 65 16 77365 77349 74.3561 53.1875 0.018163 0.0134432 2024-08-17T18:05:05.716 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:05.737 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:05.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:05.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:05.839 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:05.860 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:05.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:05.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:05.962 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:06.010 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:06.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:06.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:06.091 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:05:06.091 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T18:05:06.091 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-08-17T18:05:06.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:06.115 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:06.133 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:06.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:06.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:06.235 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:06.314 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:06.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:06.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:06.416 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:06.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:06.505 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:06.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:06.606 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:06.628 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:06.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:06.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:06.671 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 66 16 78138 78122 73.9614 48.3125 0.00855266 0.0135145 2024-08-17T18:05:06.730 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:06.767 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:06.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:06.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:06.869 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:06.916 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:06.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:06.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:07.019 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:07.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:07.040 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:07.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:07.141 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:07.177 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:07.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:07.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:07.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:07.300 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:07.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:07.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:07 smithi179 ceph-mon[93519]: pgmap v2228: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 47 MiB/s wr, 745 op/s 2024-08-17T18:05:07.401 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:07.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:07 smithi130 ceph-mon[105521]: pgmap v2228: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 47 MiB/s wr, 745 op/s 2024-08-17T18:05:07.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:07 smithi130 ceph-mon[110068]: pgmap v2228: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 47 MiB/s wr, 745 op/s 2024-08-17T18:05:07.422 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:07.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:07.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:07.524 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:07.562 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:07.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:07.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:07.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:07.671 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 67 16 78824 78808 73.4974 42.875 0.00367255 0.0135891 2024-08-17T18:05:07.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:07.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:07.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:07.788 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:07.809 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:07.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:07.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:07.911 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:07.932 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:07.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:07.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:08.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:08.055 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:08.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:08.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:08.156 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:08.179 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:08.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:08.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:08.252 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:05:08.253 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:05:08.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.253 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:78252 47.9 47:9828af59:::benchmark_data_smithi130_302447_object78251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1313)", 2024-08-17T18:05:08.253 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:06.837868+0000", 2024-08-17T18:05:08.253 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.020717342999999999, 2024-08-17T18:05:08.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020749712, 2024-08-17T18:05:08.253 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:08.253 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:08.253 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:05:08.253 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:08.253 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:05:08.254 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:05:08.254 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 78252 2024-08-17T18:05:08.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.254 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:08.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:08.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.837868+0000", 2024-08-17T18:05:08.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:08.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.837868+0000", 2024-08-17T18:05:08.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:08.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.837869+0000", 2024-08-17T18:05:08.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.237e-06 2024-08-17T18:05:08.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:08.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.837919+0000", 2024-08-17T18:05:08.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9832e-05 2024-08-17T18:05:08.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:08.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.837928+0000", 2024-08-17T18:05:08.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2e-06 2024-08-17T18:05:08.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:08.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.837934+0000", 2024-08-17T18:05:08.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6039999999999998e-06 2024-08-17T18:05:08.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:08.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.837981+0000", 2024-08-17T18:05:08.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6854999999999998e-05 2024-08-17T18:05:08.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.257 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:08.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.838108+0000", 2024-08-17T18:05:08.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000126487 2024-08-17T18:05:08.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.257 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3", 2024-08-17T18:05:08.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.838250+0000", 2024-08-17T18:05:08.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000142607 2024-08-17T18:05:08.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.257 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:08.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.846224+0000", 2024-08-17T18:05:08.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0079740720000000005 2024-08-17T18:05:08.257 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.258 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:08.258 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.258 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:78255 47.c 47:38206949:::benchmark_data_smithi130_302447_object78254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1313)", 2024-08-17T18:05:08.258 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:06.843393+0000", 2024-08-17T18:05:08.258 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.015192344, 2024-08-17T18:05:08.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015298463999999999, 2024-08-17T18:05:08.258 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:08.258 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:08.258 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:05:08.258 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:08.258 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:05:08.259 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:05:08.259 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 78255 2024-08-17T18:05:08.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.259 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:08.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.259 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:08.259 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.843393+0000", 2024-08-17T18:05:08.259 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:08.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.843393+0000", 2024-08-17T18:05:08.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:08.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.843394+0000", 2024-08-17T18:05:08.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.392e-06 2024-08-17T18:05:08.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:08.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.843442+0000", 2024-08-17T18:05:08.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7757999999999999e-05 2024-08-17T18:05:08.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:08.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.843455+0000", 2024-08-17T18:05:08.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2853e-05 2024-08-17T18:05:08.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:08.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.843461+0000", 2024-08-17T18:05:08.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7499999999999997e-06 2024-08-17T18:05:08.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:08.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.843515+0000", 2024-08-17T18:05:08.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3943000000000002e-05 2024-08-17T18:05:08.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:08.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.843648+0000", 2024-08-17T18:05:08.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000132583 2024-08-17T18:05:08.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:05:08.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.843774+0000", 2024-08-17T18:05:08.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012579400000000001 2024-08-17T18:05:08.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:08.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.846352+0000", 2024-08-17T18:05:08.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025780690000000001 2024-08-17T18:05:08.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:08.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853719+0000", 2024-08-17T18:05:08.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00736778 2024-08-17T18:05:08.263 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.263 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:08.263 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.263 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.46373.0:78256 47.f e1313/1305 47:f3f70fb1:::benchmark_data_smithi130_302447_object78255:head v 1313'4823, mlcod=1313'4818)", 2024-08-17T18:05:08.264 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:06.849637+0000", 2024-08-17T18:05:08.264 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0089474180000000004, 2024-08-17T18:05:08.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0091200159999999999, 2024-08-17T18:05:08.264 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:08.264 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:08.264 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:05:08.264 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:08.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:08.264 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.849637+0000", 2024-08-17T18:05:08.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:08.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.849637+0000", 2024-08-17T18:05:08.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:08.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.849638+0000", 2024-08-17T18:05:08.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9999999999999998e-07 2024-08-17T18:05:08.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:08.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.849671+0000", 2024-08-17T18:05:08.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2784999999999999e-05 2024-08-17T18:05:08.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:08.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.849677+0000", 2024-08-17T18:05:08.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8410000000000002e-06 2024-08-17T18:05:08.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:08.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.849682+0000", 2024-08-17T18:05:08.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4649999999999996e-06 2024-08-17T18:05:08.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:08.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.851809+0000", 2024-08-17T18:05:08.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021273860000000002 2024-08-17T18:05:08.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:08.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.851825+0000", 2024-08-17T18:05:08.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5729000000000001e-05 2024-08-17T18:05:08.267 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.267 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:08.267 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.267 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:78257 47.c 47:35662a12:::benchmark_data_smithi130_302447_object78256:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1313)", 2024-08-17T18:05:08.267 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:06.849998+0000", 2024-08-17T18:05:08.267 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.008586715, 2024-08-17T18:05:08.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0088148600000000008, 2024-08-17T18:05:08.268 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:08.268 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:08.268 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:05:08.268 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:08.268 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:05:08.268 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:05:08.268 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 78257 2024-08-17T18:05:08.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.268 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:08.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.268 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:08.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.849998+0000", 2024-08-17T18:05:08.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:08.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.849998+0000", 2024-08-17T18:05:08.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:08.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850000+0000", 2024-08-17T18:05:08.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4929999999999999e-06 2024-08-17T18:05:08.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:08.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850032+0000", 2024-08-17T18:05:08.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2496999999999997e-05 2024-08-17T18:05:08.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:08.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850044+0000", 2024-08-17T18:05:08.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1600000000000001e-05 2024-08-17T18:05:08.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:08.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850050+0000", 2024-08-17T18:05:08.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5649999999999998e-06 2024-08-17T18:05:08.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:08.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850152+0000", 2024-08-17T18:05:08.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000101514 2024-08-17T18:05:08.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:08.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850272+0000", 2024-08-17T18:05:08.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012058299999999999 2024-08-17T18:05:08.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:05:08.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850396+0000", 2024-08-17T18:05:08.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012346199999999999 2024-08-17T18:05:08.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:08.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853082+0000", 2024-08-17T18:05:08.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026858699999999999 2024-08-17T18:05:08.272 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.272 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:08.272 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.272 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:78258 47.0 47:057dc494:::benchmark_data_smithi130_302447_object78257:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1313)", 2024-08-17T18:05:08.272 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:06.850071+0000", 2024-08-17T18:05:08.272 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0085142489999999998, 2024-08-17T18:05:08.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0088091230000000003, 2024-08-17T18:05:08.273 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:08.273 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:08.273 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:05:08.273 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:08.273 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:05:08.273 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:05:08.273 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 78258 2024-08-17T18:05:08.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.273 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:08.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:08.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850071+0000", 2024-08-17T18:05:08.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.274 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:08.274 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850071+0000", 2024-08-17T18:05:08.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.274 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:08.274 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850071+0000", 2024-08-17T18:05:08.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1200000000000003e-07 2024-08-17T18:05:08.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.275 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:08.275 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850091+0000", 2024-08-17T18:05:08.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9938000000000001e-05 2024-08-17T18:05:08.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.275 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:08.275 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850102+0000", 2024-08-17T18:05:08.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0552e-05 2024-08-17T18:05:08.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.275 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:08.275 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850106+0000", 2024-08-17T18:05:08.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8180000000000003e-06 2024-08-17T18:05:08.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:08.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850240+0000", 2024-08-17T18:05:08.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000133869 2024-08-17T18:05:08.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:08.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850351+0000", 2024-08-17T18:05:08.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000111075 2024-08-17T18:05:08.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T18:05:08.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.850471+0000", 2024-08-17T18:05:08.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011956900000000001 2024-08-17T18:05:08.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:08.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.855910+0000", 2024-08-17T18:05:08.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0054390369999999999 2024-08-17T18:05:08.277 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.277 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:08.277 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.277 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:78242 47.7 47:e092ccef:::benchmark_data_smithi130_302447_object78241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1313)", 2024-08-17T18:05:08.277 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:06.824629+0000", 2024-08-17T18:05:08.277 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.033955908999999999, 2024-08-17T18:05:08.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.034315173999999997, 2024-08-17T18:05:08.278 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:08.278 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:08.278 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:05:08.278 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:08.278 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:05:08.278 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:05:08.278 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 78242 2024-08-17T18:05:08.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.278 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:08.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:08.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.824629+0000", 2024-08-17T18:05:08.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:08.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.824629+0000", 2024-08-17T18:05:08.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:08.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.824629+0000", 2024-08-17T18:05:08.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0200000000000002e-07 2024-08-17T18:05:08.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:08.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.824674+0000", 2024-08-17T18:05:08.280 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4326999999999998e-05 2024-08-17T18:05:08.280 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.280 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.280 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:08.280 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.824681+0000", 2024-08-17T18:05:08.280 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.661e-06 2024-08-17T18:05:08.280 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.280 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.280 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:08.280 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.824687+0000", 2024-08-17T18:05:08.280 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7490000000000004e-06 2024-08-17T18:05:08.280 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.280 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.280 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:08.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.836944+0000", 2024-08-17T18:05:08.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012256680000000001 2024-08-17T18:05:08.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:08.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.839726+0000", 2024-08-17T18:05:08.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027820459999999998 2024-08-17T18:05:08.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:05:08.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.839867+0000", 2024-08-17T18:05:08.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014090500000000001 2024-08-17T18:05:08.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.282 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:08.282 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853076+0000", 2024-08-17T18:05:08.282 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013209334 2024-08-17T18:05:08.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.282 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:08.282 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.855788+0000", 2024-08-17T18:05:08.282 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027119499999999999 2024-08-17T18:05:08.282 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.282 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:08.282 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.283 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop_reply(client.46373.0:78258 47.0 e1313/1305)", 2024-08-17T18:05:08.283 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:06.853083+0000", 2024-08-17T18:05:08.283 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.005501784, 2024-08-17T18:05:08.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0059250129999999998, 2024-08-17T18:05:08.283 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:08.283 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:08.283 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "queued for pg", 2024-08-17T18:05:08.283 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:08.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:08.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853083+0000", 2024-08-17T18:05:08.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853083+0000", 2024-08-17T18:05:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853087+0000", 2024-08-17T18:05:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4460000000000001e-06 2024-08-17T18:05:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853092+0000", 2024-08-17T18:05:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.434e-06 2024-08-17T18:05:08.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853104+0000", 2024-08-17T18:05:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2328e-05 2024-08-17T18:05:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853113+0000", 2024-08-17T18:05:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8640000000000002e-06 2024-08-17T18:05:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.286 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.46373.0:78259 47.f e1313/1305 47:fb606e5b:::benchmark_data_smithi130_302447_object78258:head v 1313'4824, mlcod=1313'4818)", 2024-08-17T18:05:08.286 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:06.853572+0000", 2024-08-17T18:05:08.286 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0050130579999999999, 2024-08-17T18:05:08.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0054761469999999998, 2024-08-17T18:05:08.286 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:08.286 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:08.286 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:05:08.286 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:08.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:08.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853572+0000", 2024-08-17T18:05:08.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:08.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853572+0000", 2024-08-17T18:05:08.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:08.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853572+0000", 2024-08-17T18:05:08.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5900000000000002e-07 2024-08-17T18:05:08.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:08.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853604+0000", 2024-08-17T18:05:08.288 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2193999999999997e-05 2024-08-17T18:05:08.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.288 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:08.288 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853612+0000", 2024-08-17T18:05:08.288 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9959999999999993e-06 2024-08-17T18:05:08.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.288 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:08.288 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853619+0000", 2024-08-17T18:05:08.288 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2840000000000003e-06 2024-08-17T18:05:08.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.289 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:08.289 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.855818+0000", 2024-08-17T18:05:08.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021992100000000001 2024-08-17T18:05:08.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.289 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.289 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:08.289 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.855834+0000", 2024-08-17T18:05:08.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5964000000000002e-05 2024-08-17T18:05:08.289 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.289 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:08.289 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.290 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop_reply(client.46373.0:78257 47.c e1313/1305)", 2024-08-17T18:05:08.290 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:06.853860+0000", 2024-08-17T18:05:08.290 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0047244139999999997, 2024-08-17T18:05:08.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0052314140000000002, 2024-08-17T18:05:08.290 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:08.290 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:08.290 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "queued for pg", 2024-08-17T18:05:08.290 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:08.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:08.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853860+0000", 2024-08-17T18:05:08.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:08.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853860+0000", 2024-08-17T18:05:08.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:08.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853861+0000", 2024-08-17T18:05:08.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0800000000000004e-07 2024-08-17T18:05:08.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:08.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853865+0000", 2024-08-17T18:05:08.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7419999999999999e-06 2024-08-17T18:05:08.292 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.292 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.292 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:08.292 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853876+0000", 2024-08-17T18:05:08.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.118e-05 2024-08-17T18:05:08.292 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.292 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.292 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:08.292 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.853882+0000", 2024-08-17T18:05:08.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3609999999999997e-06 2024-08-17T18:05:08.292 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.292 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:08.292 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.293 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:78263 47.7 47:e578095c:::benchmark_data_smithi130_302447_object78262:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1313)", 2024-08-17T18:05:08.293 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:06.856573+0000", 2024-08-17T18:05:08.293 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0020117659999999999, 2024-08-17T18:05:08.293 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025605749999999998, 2024-08-17T18:05:08.293 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:08.293 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:08.293 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:05:08.293 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:08.293 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:05:08.293 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:05:08.293 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 78263 2024-08-17T18:05:08.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.294 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:08.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.294 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:08.294 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.856573+0000", 2024-08-17T18:05:08.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.294 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:08.294 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.856573+0000", 2024-08-17T18:05:08.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:08.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.294 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.856575+0000", 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.539e-06 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.856596+0000", 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1569e-05 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.856612+0000", 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6110999999999999e-05 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:08.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.856619+0000", 2024-08-17T18:05:08.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9859999999999997e-06 2024-08-17T18:05:08.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:08.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.858438+0000", 2024-08-17T18:05:08.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001818589 2024-08-17T18:05:08.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:08.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.858560+0000", 2024-08-17T18:05:08.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012165199999999999 2024-08-17T18:05:08.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:08.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:08.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:05:08.297 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:06.858680+0000", 2024-08-17T18:05:08.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000120457 2024-08-17T18:05:08.297 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.297 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:08.297 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.297 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:08.297 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T18:05:08.297 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 10 2024-08-17T18:05:08.297 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:05:08.297 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:08.335 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:08.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:08.436 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:08.477 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:08.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:08.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:08.579 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:08.601 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:08.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:08.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:08.671 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 68 15 79559 79544 73.0931 46 0.0100878 0.0136749 2024-08-17T18:05:08.704 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:08.725 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:08.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:08.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:08.827 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:08.866 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:08.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:08.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:08.968 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:09.006 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:09.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:09.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:09.108 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:09.129 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:09.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:09.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:09.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:09 smithi130 ceph-mon[105521]: pgmap v2229: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 51 MiB/s wr, 809 op/s 2024-08-17T18:05:09.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:09 smithi130 ceph-mon[110068]: pgmap v2229: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 51 MiB/s wr, 809 op/s 2024-08-17T18:05:09.231 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:09.252 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:09.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:09.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:09 smithi179 ceph-mon[93519]: pgmap v2229: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 51 MiB/s wr, 809 op/s 2024-08-17T18:05:09.354 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:09.376 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:09.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:09.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:09.477 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:09.498 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:09.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:09.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:09.600 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:09.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:09.648 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:09.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:09.671 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 69 16 80364 80348 72.762 50.25 0.023291 0.0137377 2024-08-17T18:05:09.749 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:09.789 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:09.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:09.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:09.843 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T18:05:09.891 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:09.943 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:09.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:09.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:10.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:10.097 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:05:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:05:09] "GET /metrics HTTP/1.1" 200 36779 "" "Prometheus/2.51.0" 2024-08-17T18:05:10.141 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:10.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:10.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:10.163 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:05:10.163 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:05:10.210 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:05:10.210 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:05:10.243 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:10.251 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:05:10.251 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:05:10.265 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:10.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:10.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:10.305 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:05:10.306 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:05:10.342 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:05:10.342 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:05:10.367 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:10.375 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:05:10.376 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:05:10.400 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:05:10.400 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:05:10.416 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:10.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:10.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:10.432 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:05:10.432 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:05:10.519 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:10.541 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:10.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:10.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:05:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:05:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:05:10.643 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:10.672 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 70 16 81039 81023 72.3252 42.1875 0.0170215 0.0138196 2024-08-17T18:05:10.681 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:10.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:10.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:10.783 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:10.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:10.824 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:10.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:10.925 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:10.972 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:10.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:10.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:11.074 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:11.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:11.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:11.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:11 smithi130 ceph-mon[105521]: pgmap v2230: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 52 MiB/s wr, 838 op/s 2024-08-17T18:05:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:11 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/826880757' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:05:11.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:11 smithi130 ceph-mon[110068]: pgmap v2230: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 52 MiB/s wr, 838 op/s 2024-08-17T18:05:11.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:11 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/826880757' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:05:11.197 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:11.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:11.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:11.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:11.321 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:11.342 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:11.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:11.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:11 smithi179 ceph-mon[93519]: pgmap v2230: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 52 MiB/s wr, 838 op/s 2024-08-17T18:05:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:11 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/826880757' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:05:11.444 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:11.465 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:11.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:11.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:11.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:11.589 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:11.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:11.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:11.672 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 71 16 81771 81755 71.9509 45.75 0.0226353 0.013892 2024-08-17T18:05:11.690 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:11.728 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:11.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:11.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:11.830 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:11.880 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:11.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:11.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:11.982 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:12.021 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:12.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:12.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:12.123 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:12.169 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:12.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:12.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:12.271 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:12.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:12.322 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:12.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:12.423 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:12.443 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:12.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:12.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:12.487 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:12.545 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:12.607 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:12.675 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:12.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:12.676 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 72 16 82403 82387 71.4965 39.5 0.0281856 0.013979 2024-08-17T18:05:12.777 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:12.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:12.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:12.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:12.923 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:12.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:12.969 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:12.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:13.070 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:13.082 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:05:13.124 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:13.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:13.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:13.226 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:13.246 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:13 smithi130 ceph-mon[105521]: pgmap v2231: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 40 MiB/s wr, 644 op/s 2024-08-17T18:05:13.246 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:05:13.246 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:13 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/704141038' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:05:13.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:13 smithi130 ceph-mon[110068]: pgmap v2231: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 40 MiB/s wr, 644 op/s 2024-08-17T18:05:13.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:05:13.246 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:13 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/704141038' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:05:13.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:13.251 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:13.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:13 smithi179 ceph-mon[93519]: pgmap v2231: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 40 MiB/s wr, 644 op/s 2024-08-17T18:05:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:05:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:13 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/704141038' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:05:13.343 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 18:05:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=cleanup t=2024-08-17T18:05:13.134322546Z level=info msg="Completed cleanup jobs" duration=12.452289ms 2024-08-17T18:05:13.352 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:13.374 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:13.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:13.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:13.476 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:13.521 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:13.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:13.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:13.623 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:13.669 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:13.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:13.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:13.676 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 73 16 83162 83146 71.1669 47.4375 0.0126422 0.0140444 2024-08-17T18:05:13.769 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:05:13.771 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:13.807 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:13.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:13.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:13.909 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:13.954 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:13.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:13.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:14.056 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:14.104 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:14.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:14.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:14.206 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:14.227 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:14.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:14.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:14.329 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:14.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:14 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:05:14.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:14 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T18:05:14.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:14 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/704141038' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:05:14.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:14 smithi179 ceph-mon[93519]: osdmap e1314: 8 total, 8 up, 5 in 2024-08-17T18:05:14.371 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:14.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:14.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:14 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:05:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:14 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T18:05:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:14 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/704141038' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:05:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:14 smithi130 ceph-mon[105521]: osdmap e1314: 8 total, 8 up, 5 in 2024-08-17T18:05:14.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:14 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:05:14.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:14 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T18:05:14.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:14 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/704141038' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:05:14.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:14 smithi130 ceph-mon[110068]: osdmap e1314: 8 total, 8 up, 5 in 2024-08-17T18:05:14.473 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:14.514 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:14.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:14.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:14.617 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:14.639 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:14.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:14.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:14.677 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 74 16 83837 83821 70.775 42.1875 0.0159713 0.0141235 2024-08-17T18:05:14.740 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:14.782 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:14.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:14.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:14.884 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:14.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:14.956 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:14.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:15.057 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:15.078 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:15.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:15.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:15.180 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:15.200 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:15.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:15.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:15.302 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:15.323 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:15.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:15.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:15.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:15 smithi179 ceph-mon[93519]: pgmap v2233: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 47 MiB/s wr, 754 op/s 2024-08-17T18:05:15.362 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:15 smithi130 ceph-mon[105521]: pgmap v2233: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 47 MiB/s wr, 754 op/s 2024-08-17T18:05:15.362 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:15 smithi130 ceph-mon[110068]: pgmap v2233: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 47 MiB/s wr, 754 op/s 2024-08-17T18:05:15.425 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:15.457 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:15.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:15.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:15.559 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:15.580 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:15.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:15.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:15.676 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 75 16 84636 84620 70.4972 49.9375 0.00904446 0.0141789 2024-08-17T18:05:15.682 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:15.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:15.704 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:15.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:15.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:15.826 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:15.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:15.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:15.928 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:15.985 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:15.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:15.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:16.087 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:16.140 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:16.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:16.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:16.242 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:16.282 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:16.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:16.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:16.384 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:16.405 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:16.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:16.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:16.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:16.550 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:16.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:16.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:16.651 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:16.673 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:16.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:16.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:16.676 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 76 16 85433 85417 70.225 49.8125 0.00878569 0.0142341 2024-08-17T18:05:16.775 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:16.796 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:16.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:16.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:16.897 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:16.919 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:16.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:16.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:17.020 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:17.073 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:17.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:17.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:17.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:17 smithi130 ceph-mon[105521]: pgmap v2234: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 47 MiB/s wr, 750 op/s 2024-08-17T18:05:17.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:17 smithi130 ceph-mon[110068]: pgmap v2234: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 47 MiB/s wr, 750 op/s 2024-08-17T18:05:17.175 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:17.197 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:17.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:17.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:17.299 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:17.351 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:17.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:17.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:17.453 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:17.519 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:17.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:17.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:17 smithi179 ceph-mon[93519]: pgmap v2234: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 47 MiB/s wr, 750 op/s 2024-08-17T18:05:17.621 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:17.667 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:17.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:17.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:17.676 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 77 16 86193 86177 69.9299 47.5 0.0267338 0.0142938 2024-08-17T18:05:17.769 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:17.790 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:17.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:17.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:17.892 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:17.942 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:17.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:17.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:18.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:18.065 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:05:18.066 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:05:18.066 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:05:18.066 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:05:18.066 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:05:18.066 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:05:18.066 INFO:teuthology.orchestra.run.smithi130.stderr:bench [] [] [] [] 2024-08-17T18:05:18.066 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:05:18.066 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:05:18.066 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:05:18.066 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator dump block 2024-08-17T18:05:18.066 INFO:teuthology.orchestra.run.smithi130.stderr:Error EINVAL: invalid command 2024-08-17T18:05:18.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:18.107 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:18.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:18.208 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:18.230 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:18.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:18.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:18.331 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:18.351 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:05:18.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:18.356 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:18.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:18.456 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:18.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:18.497 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:18.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:18.598 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:18.649 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:18.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:18.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:18.677 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 78 16 86945 86929 69.6354 47 0.0287604 0.0143535 2024-08-17T18:05:18.751 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:18.802 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:18.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:18.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:18.904 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:18.926 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:18.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:18.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:19.028 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:19.049 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:19.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:19.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:19.151 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:19.172 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:19.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:19.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:19.274 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:19.296 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:19.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:19.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:19.395 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:19 smithi179 ceph-mon[93519]: pgmap v2235: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 737 op/s 2024-08-17T18:05:19.398 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:19.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:19 smithi130 ceph-mon[105521]: pgmap v2235: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 737 op/s 2024-08-17T18:05:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:19 smithi130 ceph-mon[110068]: pgmap v2235: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 737 op/s 2024-08-17T18:05:19.419 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:19.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:19.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:19.521 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:19.568 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:19.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:19.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:19.670 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:19.677 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 79 16 87665 87649 69.3235 45 0.0124307 0.0144165 2024-08-17T18:05:19.713 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:19.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:19.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:19.815 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:19.861 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:19.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:19.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:19.963 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:19.985 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:19.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:19.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:20.087 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:20.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:20.131 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:20.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:20.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:05:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:05:19] "GET /metrics HTTP/1.1" 200 36779 "" "Prometheus/2.51.0" 2024-08-17T18:05:20.231 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:20.254 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:20.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:20.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:20.356 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:20.398 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:20.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:20.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:20.500 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:20.521 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:20.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:20.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:05:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:05:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:05:20.622 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:20.643 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:20.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:20.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:20.677 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:05:20.677783+0000 min lat: 0.00111317 max lat: 0.450837 avg lat: 0.0144955 2024-08-17T18:05:20.677 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:05:20.678 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 80 15 88306 88291 68.9585 40.125 0.0263123 0.0144955 2024-08-17T18:05:20.745 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:20.767 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:20.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:20.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:20.869 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:20.903 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:20.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:20.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:21.004 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:21.041 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:21.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:21.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:21.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:21.164 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:21.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:21.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:21.266 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:21.288 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:21.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:21.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:21.390 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:21.413 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:21.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:21.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:21 smithi130 ceph-mon[105521]: pgmap v2236: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 737 op/s 2024-08-17T18:05:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:21 smithi130 ceph-mon[110068]: pgmap v2236: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 737 op/s 2024-08-17T18:05:21.515 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:21.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:21.539 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:21.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:21 smithi179 ceph-mon[93519]: pgmap v2236: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 737 op/s 2024-08-17T18:05:21.640 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:21.664 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:21.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:21.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:21.677 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 81 16 88954 88938 68.6064 40.4375 0.0168655 0.0145691 2024-08-17T18:05:21.766 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:21.787 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:21.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:21.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:21.889 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:21.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:21.941 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:21.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:22.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:22.079 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:22.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:22.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:22.181 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:22.218 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:22.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:22.278 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:22.320 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:22.341 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:22.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:22.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:22.443 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:22.465 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:22.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:22.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:22.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:22.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:22.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:22.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:22.678 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 82 16 89725 89709 68.3574 48.1875 0.0151451 0.0146227 2024-08-17T18:05:22.690 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:22.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:22.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:22.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:22.813 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:22.820 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:05:22.821 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T18:05:22.821 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-08-17T18:05:22.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:22.837 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:22.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:22.938 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:23.000 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:23.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:23.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:23.018 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:05:23.018 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:05:23.102 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:23.124 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:23.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:23.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:23.225 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:23.247 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:23.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:23.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:23.349 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:23.363 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:23 smithi130 ceph-mon[105521]: pgmap v2237: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 737 op/s 2024-08-17T18:05:23.364 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:23 smithi130 ceph-mon[110068]: pgmap v2237: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 737 op/s 2024-08-17T18:05:23.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:23.409 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:23.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:23.510 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:23.532 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:23.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:23.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:23 smithi179 ceph-mon[93519]: pgmap v2237: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 737 op/s 2024-08-17T18:05:23.633 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:23.679 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 83 16 90432 90416 68.0648 44.1875 0.0194695 0.0146845 2024-08-17T18:05:23.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:23.687 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:23.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:23.788 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:23.829 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:23.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:23.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:23.931 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:23.953 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:23.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:23.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:24.030 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T18:05:24.054 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:24.075 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:24.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:24.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:24.177 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:24.199 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:24.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:24.301 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:24.322 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:24.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:24.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:24.424 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:24.446 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:24.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:24.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:24.547 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:24.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:24.599 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:24.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:24.680 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 84 16 91191 91175 67.8192 47.4375 0.0202751 0.0147389 2024-08-17T18:05:24.699 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:24.748 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:24.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:24.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:24.849 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:24.871 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:24.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:24.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:24.973 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:24.994 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:24.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:24.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:25.096 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:25.118 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:25.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:25.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:25.219 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:25.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:25.266 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:25.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:25.367 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:25.386 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:25 smithi130 ceph-mon[105521]: pgmap v2238: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 48 MiB/s wr, 762 op/s 2024-08-17T18:05:25.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:25 smithi130 ceph-mon[110068]: pgmap v2238: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 48 MiB/s wr, 762 op/s 2024-08-17T18:05:25.419 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:25.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:25.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:25.521 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:25.559 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:25.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:25.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:25.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:25 smithi179 ceph-mon[93519]: pgmap v2238: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 48 MiB/s wr, 762 op/s 2024-08-17T18:05:25.661 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:25.680 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 85 16 91903 91887 67.5449 44.5 0.072354 0.0147889 2024-08-17T18:05:25.688 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:25.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:25.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:25.790 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:25.837 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:25.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:25.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:25.940 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:25.961 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:25.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:25.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:26.063 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:26.084 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:26.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:26.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:26.186 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:26.207 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:26.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:26.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:26.309 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:26.330 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:26.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:26.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:26.432 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:26.453 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:26.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:26.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:26.555 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:26.579 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:26.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:26.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:26.680 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 86 16 92403 92387 67.1229 31.25 0.0127219 0.0148931 2024-08-17T18:05:26.682 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:26.704 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:26.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:26.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:26.806 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:26.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:26.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:26.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:26.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:26.952 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:26.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:26.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:27.054 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:27.076 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:27.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:27.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:27.100 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:27.178 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:27.239 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:27.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:27.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:27.342 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:27.378 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:27.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:27.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:27.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:27 smithi130 ceph-mon[105521]: pgmap v2239: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 42 MiB/s wr, 679 op/s 2024-08-17T18:05:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:27 smithi130 ceph-mon[110068]: pgmap v2239: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 42 MiB/s wr, 679 op/s 2024-08-17T18:05:27.480 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:27.538 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:27.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:27.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:27 smithi179 ceph-mon[93519]: pgmap v2239: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 42 MiB/s wr, 679 op/s 2024-08-17T18:05:27.640 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:27.680 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 87 15 93083 93068 66.8406 42.5625 0.0402705 0.0149539 2024-08-17T18:05:27.701 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:27.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:27.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:27.803 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:27.821 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:27.821 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:27.824 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:27.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:27.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:27.826 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:05:27.827 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:05:27.868 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:05:27.868 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:05:27.901 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:05:27.901 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:05:27.926 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:27.933 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:05:27.933 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:05:27.965 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:05:27.966 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:05:27.982 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:27.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:27.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:27.995 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:05:27.996 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:05:28.030 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:05:28.031 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:05:28.058 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:05:28.059 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:05:28.084 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:28.106 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:28.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:28.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:28.208 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:28.260 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:28.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:28.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:28.362 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:28.401 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:28.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:28.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:28 smithi130 ceph-mon[105521]: pgmap v2240: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 46 MiB/s wr, 728 op/s 2024-08-17T18:05:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:05:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:28 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1300818761' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:05:28.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:28 smithi130 ceph-mon[110068]: pgmap v2240: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 46 MiB/s wr, 728 op/s 2024-08-17T18:05:28.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:28.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:05:28.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:28 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1300818761' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:05:28.503 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:28.526 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:28.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:28.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:28 smithi179 ceph-mon[93519]: pgmap v2240: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 46 MiB/s wr, 728 op/s 2024-08-17T18:05:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:05:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:28 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1300818761' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:05:28.628 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:28.668 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:28.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:28.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:28.680 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 88 16 93711 93695 66.5264 39.1875 0.0276668 0.0150254 2024-08-17T18:05:28.771 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:28.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:28.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:28.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:28.911 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:28.965 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:28.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:28.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:29.067 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:29.105 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:29.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:29.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:29.207 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:29.229 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:29.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:29.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:29.330 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:29.371 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:29.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:29.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:29.473 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:29.494 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:29.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:29.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:29 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:05:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:29 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:05:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:29 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:29.597 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:29.634 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:29.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:29.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:29 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:05:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:29 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:05:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:29 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:29 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:05:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:29 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:05:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:29 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:29.680 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 89 16 94506 94490 66.3371 49.6875 0.02574 0.0150682 2024-08-17T18:05:29.736 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:29.777 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:29.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:29.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:29.878 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:29.902 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:29.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:29.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:30.004 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:30.026 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:30.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:30.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:30.128 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:30.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:05:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:05:29] "GET /metrics HTTP/1.1" 200 36781 "" "Prometheus/2.51.0" 2024-08-17T18:05:30.179 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:30.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:30.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:30.281 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:30.303 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:30.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:30.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:30.405 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:30.418 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:05:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:05:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:05:30.447 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:30.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:30.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:30.549 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:30.570 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:30.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:30.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:30.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:30 smithi130 ceph-mon[105521]: pgmap v2241: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 47 MiB/s wr, 750 op/s 2024-08-17T18:05:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:30 smithi130 ceph-mon[110068]: pgmap v2241: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 47 MiB/s wr, 750 op/s 2024-08-17T18:05:30.671 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:30.680 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Total time run: 90.0061 2024-08-17T18:05:30.680 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Total writes made: 95232 2024-08-17T18:05:30.681 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Write size: 65536 2024-08-17T18:05:30.681 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Object size: 65536 2024-08-17T18:05:30.681 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Bandwidth (MB/sec): 66.1288 2024-08-17T18:05:30.681 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev Bandwidth: 27.6567 2024-08-17T18:05:30.681 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max bandwidth (MB/sec): 161.75 2024-08-17T18:05:30.681 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min bandwidth (MB/sec): 31.25 2024-08-17T18:05:30.682 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Average IOPS: 1058 2024-08-17T18:05:30.682 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev IOPS: 442.507 2024-08-17T18:05:30.682 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max IOPS: 2588 2024-08-17T18:05:30.682 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min IOPS: 500 2024-08-17T18:05:30.682 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Average Latency(s): 0.0151175 2024-08-17T18:05:30.682 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev Latency(s): 0.0142119 2024-08-17T18:05:30.683 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max latency(s): 0.450837 2024-08-17T18:05:30.683 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min latency(s): 0.00111317 2024-08-17T18:05:30.685 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T18:05:30.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:30.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:30.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:30.746 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:30 smithi179 ceph-mon[93519]: pgmap v2241: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 47 MiB/s wr, 750 op/s 2024-08-17T18:05:30.794 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:30.838 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:30.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:30.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:30.940 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:30.962 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:30.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:30.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:31.063 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:31.084 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:31.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:31.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:31.186 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:31.225 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:31.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:31.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:31.326 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:31.348 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:31.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:31.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:31.374 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:05:31.449 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:31.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:31.507 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:31.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:31.608 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:31.630 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:31.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:31.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:31.731 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:31.753 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:31.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:31.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:31.855 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:31.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:31.911 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:31.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:32.012 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:32.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:32.061 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:32.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:32.162 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:32.183 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:32.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:32.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:32.285 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:32.306 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:32.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:32.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:32.408 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:32.455 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:32.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:32.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:32.557 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:32.579 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:32.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:32.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:32.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:32 smithi130 ceph-mon[105521]: pgmap v2242: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 36 MiB/s wr, 569 op/s 2024-08-17T18:05:32.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:32 smithi130 ceph-mon[110068]: pgmap v2242: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 36 MiB/s wr, 569 op/s 2024-08-17T18:05:32.680 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:32.712 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:32.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:32.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:32.814 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:32.835 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:32.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:32.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:32 smithi179 ceph-mon[93519]: pgmap v2242: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 36 MiB/s wr, 569 op/s 2024-08-17T18:05:32.937 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:32.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:32.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:32.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:33.060 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:33.082 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:33.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:33.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:33.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:33.225 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:33.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:33.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:33.327 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:33.373 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:33.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:33.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:33.475 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:33.496 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:33.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:33.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:33.598 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:33.631 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:33.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:33.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:33.733 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:33.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:33.766 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:33.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:33.868 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:33.892 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:33.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:33.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:33.994 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:34.015 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:34.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:34.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:34.117 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:34.152 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:34.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:34.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:34.254 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:34.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:34.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:34.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:34.376 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:34.414 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:34.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:34.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:34.516 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:34.537 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:34.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:34.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:34.639 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:34.681 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:34.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:34.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:34.766 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:34 smithi130 ceph-mon[105521]: pgmap v2243: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 45 MiB/s wr, 809 op/s 2024-08-17T18:05:34.766 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:34 smithi130 ceph-mon[110068]: pgmap v2243: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 45 MiB/s wr, 809 op/s 2024-08-17T18:05:34.783 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:34.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:34.820 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:34.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:34 smithi179 ceph-mon[93519]: pgmap v2243: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 45 MiB/s wr, 809 op/s 2024-08-17T18:05:34.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:34.956 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:34.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:34.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:35.058 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:35.092 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:35.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:35.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:35.193 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:35.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:35.240 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:35.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:35.341 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:35.362 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:35.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:35.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:35.463 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:35.485 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:35.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:35.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:35.525 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:05:35.525 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:05:35.525 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:05:35.526 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:05:35.526 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.526 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:204054 45.b 45:df630a61:::benchmark_data_smithi130_172551_object99247:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1247)", 2024-08-17T18:05:35.526 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.563571+0000", 2024-08-17T18:05:35.526 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 405.95896656600002, 2024-08-17T18:05:35.526 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.36158967800000003, 2024-08-17T18:05:35.526 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.526 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.526 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.526 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.526 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 204054 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.563571+0000", 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.563571+0000", 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.527 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.563573+0000", 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7489999999999999e-06 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.563580+0000", 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5490000000000003e-06 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.563593+0000", 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3472000000000001e-05 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.528 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.529 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.529 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.563602+0000", 2024-08-17T18:05:35.529 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0380000000000005e-06 2024-08-17T18:05:35.529 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.529 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.529 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.529 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.564522+0000", 2024-08-17T18:05:35.529 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00091952600000000005 2024-08-17T18:05:35.529 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.529 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.529 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.529 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.564707+0000", 2024-08-17T18:05:35.529 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018508399999999999 2024-08-17T18:05:35.529 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.530 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.530 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3,4", 2024-08-17T18:05:35.530 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.564863+0000", 2024-08-17T18:05:35.530 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015653100000000001 2024-08-17T18:05:35.530 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.530 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.530 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.530 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.570672+0000", 2024-08-17T18:05:35.530 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0058088530000000001 2024-08-17T18:05:35.530 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.530 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.534 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.534 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.573176+0000", 2024-08-17T18:05:35.534 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025036699999999999 2024-08-17T18:05:35.534 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.534 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.534 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.534 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.573921+0000", 2024-08-17T18:05:35.534 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00074494899999999996 2024-08-17T18:05:35.535 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.535 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.535 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.535 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579362+0000", 2024-08-17T18:05:35.535 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0054412870000000004 2024-08-17T18:05:35.535 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.535 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.535 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.535 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579529+0000", 2024-08-17T18:05:35.535 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016677400000000001 2024-08-17T18:05:35.535 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.535 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.536 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.536 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925161+0000", 2024-08-17T18:05:35.536 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.345632196 2024-08-17T18:05:35.536 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.536 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.536 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.536 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.536 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.536 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.45158.0:204051 45.4 e1247/1183 45:2bc8134d:::benchmark_data_smithi130_172551_object99244:head v 1247'12702, mlcod=1247'12699)", 2024-08-17T18:05:35.536 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.566198+0000", 2024-08-17T18:05:35.536 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 405.95633985699999, 2024-08-17T18:05:35.536 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35852299300000001, 2024-08-17T18:05:35.536 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.537 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.537 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.537 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.537 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.537 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.537 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566198+0000", 2024-08-17T18:05:35.537 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.537 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.537 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.537 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.537 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566198+0000", 2024-08-17T18:05:35.537 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.537 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.537 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.538 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.538 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566199+0000", 2024-08-17T18:05:35.538 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1149999999999999e-06 2024-08-17T18:05:35.538 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.538 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.538 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.538 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566205+0000", 2024-08-17T18:05:35.538 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.339e-06 2024-08-17T18:05:35.538 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.538 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.538 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.538 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566214+0000", 2024-08-17T18:05:35.538 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6709999999999994e-06 2024-08-17T18:05:35.539 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.539 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.539 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.539 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566223+0000", 2024-08-17T18:05:35.539 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8869999999999998e-06 2024-08-17T18:05:35.539 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.539 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.539 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.539 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566275+0000", 2024-08-17T18:05:35.539 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1496000000000003e-05 2024-08-17T18:05:35.539 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.539 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.539 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.540 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.566325+0000", 2024-08-17T18:05:35.540 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0773999999999999e-05 2024-08-17T18:05:35.540 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.540 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.540 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.540 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.924682+0000", 2024-08-17T18:05:35.540 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35835662600000001 2024-08-17T18:05:35.540 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.540 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.540 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.540 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.924721+0000", 2024-08-17T18:05:35.540 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9085000000000003e-05 2024-08-17T18:05:35.540 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.540 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.541 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.541 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.541 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.541 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:204058 45.8 45:1d08798f:::benchmark_data_smithi130_172551_object99251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1247)", 2024-08-17T18:05:35.541 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.568520+0000", 2024-08-17T18:05:35.541 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 405.9540184, 2024-08-17T18:05:35.541 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.41966336300000001, 2024-08-17T18:05:35.541 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.541 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.541 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.541 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.541 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:05:35.541 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:05:35.541 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 204058 2024-08-17T18:05:35.542 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.542 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.542 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.542 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.542 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.568520+0000", 2024-08-17T18:05:35.542 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.542 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.542 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.542 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.542 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.568520+0000", 2024-08-17T18:05:35.542 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.542 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.542 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.543 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.543 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.568521+0000", 2024-08-17T18:05:35.543 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4920000000000001e-06 2024-08-17T18:05:35.543 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.543 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.543 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.543 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.568526+0000", 2024-08-17T18:05:35.544 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5920000000000002e-06 2024-08-17T18:05:35.544 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.544 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.544 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.544 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.568537+0000", 2024-08-17T18:05:35.544 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1644999999999999e-05 2024-08-17T18:05:35.544 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.544 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.545 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.545 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.568544+0000", 2024-08-17T18:05:35.545 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9040000000000003e-06 2024-08-17T18:05:35.545 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.545 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.545 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.545 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.569664+0000", 2024-08-17T18:05:35.545 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001119943 2024-08-17T18:05:35.545 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.545 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.545 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.545 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.569779+0000", 2024-08-17T18:05:35.545 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011475 2024-08-17T18:05:35.545 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.546 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.546 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T18:05:35.546 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.569904+0000", 2024-08-17T18:05:35.546 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012463400000000001 2024-08-17T18:05:35.546 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.546 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.546 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.546 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.573913+0000", 2024-08-17T18:05:35.546 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0040094070000000004 2024-08-17T18:05:35.546 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.546 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.546 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.546 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.573989+0000", 2024-08-17T18:05:35.547 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5797999999999995e-05 2024-08-17T18:05:35.547 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.547 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.547 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.547 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.931060+0000", 2024-08-17T18:05:35.547 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35707166600000001 2024-08-17T18:05:35.547 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.547 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.547 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.547 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.931114+0000", 2024-08-17T18:05:35.547 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3903999999999998e-05 2024-08-17T18:05:35.547 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.547 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.548 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.548 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988183+0000", 2024-08-17T18:05:35.548 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.057068628000000003 2024-08-17T18:05:35.548 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.548 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.548 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.548 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.548 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.548 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.45158.0:204062 45.4 e1247/1183 45:26797482:::benchmark_data_smithi130_172551_object99255:head v 1247'12703, mlcod=1247'12700)", 2024-08-17T18:05:35.548 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.576909+0000", 2024-08-17T18:05:35.548 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 405.945628941, 2024-08-17T18:05:35.548 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.34812056600000002, 2024-08-17T18:05:35.548 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.549 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.549 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.549 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.549 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.549 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.549 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576909+0000", 2024-08-17T18:05:35.549 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.549 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.549 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.549 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.549 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576909+0000", 2024-08-17T18:05:35.549 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.549 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.549 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.550 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.550 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576910+0000", 2024-08-17T18:05:35.550 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.037e-06 2024-08-17T18:05:35.550 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.550 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.550 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.550 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576918+0000", 2024-08-17T18:05:35.550 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7700000000000001e-06 2024-08-17T18:05:35.550 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.550 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.550 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.550 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576929+0000", 2024-08-17T18:05:35.551 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1073000000000001e-05 2024-08-17T18:05:35.551 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.551 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.551 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.551 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.576936+0000", 2024-08-17T18:05:35.551 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1189999999999999e-06 2024-08-17T18:05:35.551 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.551 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.551 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.551 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.577009+0000", 2024-08-17T18:05:35.551 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2737999999999994e-05 2024-08-17T18:05:35.551 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.551 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.552 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.552 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.577028+0000", 2024-08-17T18:05:35.552 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9354000000000001e-05 2024-08-17T18:05:35.552 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.552 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.552 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.552 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925003+0000", 2024-08-17T18:05:35.552 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.34797440499999999 2024-08-17T18:05:35.552 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.552 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.552 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.552 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925030+0000", 2024-08-17T18:05:35.552 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.707e-05 2024-08-17T18:05:35.552 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.553 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.553 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.553 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.553 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.553 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:204067 45.8 45:10b899da:::benchmark_data_smithi130_172551_object99260:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1247)", 2024-08-17T18:05:35.553 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.579603+0000", 2024-08-17T18:05:35.553 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 405.94293446500001, 2024-08-17T18:05:35.553 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.40960074899999999, 2024-08-17T18:05:35.553 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.553 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.553 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.553 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.553 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:05:35.554 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:05:35.554 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 204067 2024-08-17T18:05:35.554 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.554 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.554 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.554 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.554 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579603+0000", 2024-08-17T18:05:35.554 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.554 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.554 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.554 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.554 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579603+0000", 2024-08-17T18:05:35.554 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.554 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.555 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.555 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.555 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579605+0000", 2024-08-17T18:05:35.555 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.652e-06 2024-08-17T18:05:35.555 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.555 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.555 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.555 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579611+0000", 2024-08-17T18:05:35.555 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9200000000000001e-06 2024-08-17T18:05:35.555 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.555 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.555 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.555 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579621+0000", 2024-08-17T18:05:35.556 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.006e-05 2024-08-17T18:05:35.556 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.556 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.556 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.556 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579628+0000", 2024-08-17T18:05:35.556 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4450000000000001e-06 2024-08-17T18:05:35.556 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.556 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.556 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.556 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579683+0000", 2024-08-17T18:05:35.556 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5606000000000003e-05 2024-08-17T18:05:35.556 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.556 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.556 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.557 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579828+0000", 2024-08-17T18:05:35.557 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014495600000000001 2024-08-17T18:05:35.557 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.557 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.557 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T18:05:35.557 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.579957+0000", 2024-08-17T18:05:35.557 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012935299999999999 2024-08-17T18:05:35.557 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.557 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.557 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.557 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.931008+0000", 2024-08-17T18:05:35.557 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35105088200000001 2024-08-17T18:05:35.557 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.557 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.558 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.558 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988197+0000", 2024-08-17T18:05:35.558 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.057188956999999999 2024-08-17T18:05:35.558 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.558 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.558 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.558 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988244+0000", 2024-08-17T18:05:35.558 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6322000000000001e-05 2024-08-17T18:05:35.558 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.558 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.558 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.558 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988295+0000", 2024-08-17T18:05:35.558 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1306e-05 2024-08-17T18:05:35.559 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.559 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.559 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.559 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.989204+0000", 2024-08-17T18:05:35.559 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00090928999999999999 2024-08-17T18:05:35.559 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.559 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.559 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.559 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.559 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.559 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.45158.0:204069 45.b 45:d53ba6ff:::benchmark_data_smithi130_172551_object99262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1247)", 2024-08-17T18:05:35.559 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.584843+0000", 2024-08-17T18:05:35.560 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 405.93769515899999, 2024-08-17T18:05:35.560 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.40363016699999998, 2024-08-17T18:05:35.560 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.560 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.560 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.560 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.560 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.45158", 2024-08-17T18:05:35.560 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1075313984", 2024-08-17T18:05:35.560 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 204069 2024-08-17T18:05:35.560 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.560 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.560 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.560 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.560 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.584843+0000", 2024-08-17T18:05:35.561 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.561 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.561 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.561 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.561 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.584843+0000", 2024-08-17T18:05:35.561 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.561 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.561 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.561 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.561 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.584845+0000", 2024-08-17T18:05:35.561 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0640000000000001e-06 2024-08-17T18:05:35.561 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.562 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.562 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.562 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.584853+0000", 2024-08-17T18:05:35.562 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8690000000000004e-06 2024-08-17T18:05:35.562 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.562 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.562 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.562 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.924578+0000", 2024-08-17T18:05:35.562 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.33972480399999999 2024-08-17T18:05:35.562 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.562 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.562 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.562 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.924594+0000", 2024-08-17T18:05:35.562 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6133999999999998e-05 2024-08-17T18:05:35.563 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.563 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.563 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.563 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925279+0000", 2024-08-17T18:05:35.563 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00068548999999999997 2024-08-17T18:05:35.563 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.563 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.563 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.563 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925916+0000", 2024-08-17T18:05:35.563 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00063705599999999999 2024-08-17T18:05:35.563 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.563 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.563 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3,4", 2024-08-17T18:05:35.564 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.926085+0000", 2024-08-17T18:05:35.564 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000168982 2024-08-17T18:05:35.564 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.564 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.564 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.564 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.987761+0000", 2024-08-17T18:05:35.564 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.061675727999999999 2024-08-17T18:05:35.564 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.564 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.564 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.564 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.987906+0000", 2024-08-17T18:05:35.564 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014464400000000001 2024-08-17T18:05:35.564 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.565 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.565 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.565 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988003+0000", 2024-08-17T18:05:35.565 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7380999999999995e-05 2024-08-17T18:05:35.565 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.565 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.565 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.565 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988069+0000", 2024-08-17T18:05:35.565 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6128000000000001e-05 2024-08-17T18:05:35.565 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.565 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.565 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.565 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988131+0000", 2024-08-17T18:05:35.566 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1888000000000001e-05 2024-08-17T18:05:35.566 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.566 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.566 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.566 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.988473+0000", 2024-08-17T18:05:35.566 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000341999 2024-08-17T18:05:35.566 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.566 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.566 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.566 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.566 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.566 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.45158.0:204093 45.7 e1247/1189 45:ee2752c9:::benchmark_data_smithi130_172551_object99286:head v 1247'12739, mlcod=1170'705)", 2024-08-17T18:05:35.567 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:58:49.613500+0000", 2024-08-17T18:05:35.567 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 405.909038103, 2024-08-17T18:05:35.567 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.374334255, 2024-08-17T18:05:35.567 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.567 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.567 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.567 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.567 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.567 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.567 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.613500+0000", 2024-08-17T18:05:35.567 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.567 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.567 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.568 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.568 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.613500+0000", 2024-08-17T18:05:35.568 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.568 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.568 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.568 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.568 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.613501+0000", 2024-08-17T18:05:35.568 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4389999999999999e-06 2024-08-17T18:05:35.568 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.568 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.568 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.568 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.613514+0000", 2024-08-17T18:05:35.568 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2313000000000001e-05 2024-08-17T18:05:35.568 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.569 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.569 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.569 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925150+0000", 2024-08-17T18:05:35.569 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.31163655000000001 2024-08-17T18:05:35.569 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.569 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.569 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.569 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.925157+0000", 2024-08-17T18:05:35.569 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5350000000000001e-06 2024-08-17T18:05:35.569 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.569 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.569 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.569 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.926087+0000", 2024-08-17T18:05:35.570 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00093048299999999996 2024-08-17T18:05:35.570 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.570 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.570 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.570 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.926114+0000", 2024-08-17T18:05:35.570 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6763000000000001e-05 2024-08-17T18:05:35.570 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.570 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.570 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.570 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.987738+0000", 2024-08-17T18:05:35.570 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.061623566999999997 2024-08-17T18:05:35.570 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.570 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.570 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.571 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:58:49.987834+0000", 2024-08-17T18:05:35.571 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6605000000000001e-05 2024-08-17T18:05:35.571 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.571 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.571 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.571 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.571 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.571 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:30111 46.8 46:189f8eaf:::benchmark_data_smithi130_267720_object30110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:05:35.571 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.623717+0000", 2024-08-17T18:05:35.571 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 346.898821328, 2024-08-17T18:05:35.571 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38956026100000002, 2024-08-17T18:05:35.571 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.571 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.571 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.572 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.572 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:05:35.572 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:05:35.572 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 30111 2024-08-17T18:05:35.572 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.572 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.572 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.572 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.572 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623717+0000", 2024-08-17T18:05:35.572 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.572 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.572 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.572 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.573 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623717+0000", 2024-08-17T18:05:35.573 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.573 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.573 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.573 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.573 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623726+0000", 2024-08-17T18:05:35.573 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1579999999999994e-06 2024-08-17T18:05:35.573 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.573 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.573 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.573 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623751+0000", 2024-08-17T18:05:35.573 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5194999999999998e-05 2024-08-17T18:05:35.574 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.574 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.574 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.574 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623764+0000", 2024-08-17T18:05:35.574 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3210999999999999e-05 2024-08-17T18:05:35.574 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.575 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.575 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.575 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.623771+0000", 2024-08-17T18:05:35.575 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2490000000000004e-06 2024-08-17T18:05:35.575 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.575 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.575 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.575 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.624248+0000", 2024-08-17T18:05:35.575 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000476828 2024-08-17T18:05:35.575 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.575 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.575 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.576 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.624387+0000", 2024-08-17T18:05:35.576 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013830900000000001 2024-08-17T18:05:35.576 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.576 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.576 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:05:35.576 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.624521+0000", 2024-08-17T18:05:35.576 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013415700000000001 2024-08-17T18:05:35.576 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.576 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.576 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.576 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627377+0000", 2024-08-17T18:05:35.576 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028564889999999998 2024-08-17T18:05:35.576 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.577 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.577 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.577 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.627536+0000", 2024-08-17T18:05:35.577 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015840400000000001 2024-08-17T18:05:35.577 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.577 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.577 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.577 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013165+0000", 2024-08-17T18:05:35.577 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38562930099999998 2024-08-17T18:05:35.577 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.577 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.577 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.577 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013228+0000", 2024-08-17T18:05:35.578 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3109999999999997e-05 2024-08-17T18:05:35.578 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.578 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.578 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.578 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.013277+0000", 2024-08-17T18:05:35.578 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8850000000000002e-05 2024-08-17T18:05:35.578 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.578 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.578 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.578 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.578 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.578 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:30127 46.8 46:16873e59:::benchmark_data_smithi130_267720_object30126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:05:35.579 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:48.632099+0000", 2024-08-17T18:05:35.579 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 346.89043879899998, 2024-08-17T18:05:35.579 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38487052399999999, 2024-08-17T18:05:35.579 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.579 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.579 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.579 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.579 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:05:35.579 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:05:35.579 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 30127 2024-08-17T18:05:35.579 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.579 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.579 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.580 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.580 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632099+0000", 2024-08-17T18:05:35.580 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.580 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.580 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.580 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.580 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632099+0000", 2024-08-17T18:05:35.580 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.580 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.580 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.580 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.580 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632101+0000", 2024-08-17T18:05:35.580 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4160000000000001e-06 2024-08-17T18:05:35.581 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.581 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.581 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.581 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632121+0000", 2024-08-17T18:05:35.581 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0633e-05 2024-08-17T18:05:35.581 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.581 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.581 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.581 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632136+0000", 2024-08-17T18:05:35.581 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4622e-05 2024-08-17T18:05:35.581 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.581 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.581 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.582 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632144+0000", 2024-08-17T18:05:35.582 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8450000000000003e-06 2024-08-17T18:05:35.582 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.582 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.582 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.582 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632184+0000", 2024-08-17T18:05:35.582 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0441999999999997e-05 2024-08-17T18:05:35.582 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.582 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.582 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.582 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632304+0000", 2024-08-17T18:05:35.582 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000119548 2024-08-17T18:05:35.582 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.583 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.583 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:05:35.583 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.632459+0000", 2024-08-17T18:05:35.583 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000155165 2024-08-17T18:05:35.583 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.583 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.583 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.583 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.633815+0000", 2024-08-17T18:05:35.583 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001356554 2024-08-17T18:05:35.583 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.583 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.583 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.583 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:48.633999+0000", 2024-08-17T18:05:35.584 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018367200000000001 2024-08-17T18:05:35.584 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.584 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.584 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.584 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016770+0000", 2024-08-17T18:05:35.584 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38277128399999999 2024-08-17T18:05:35.584 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.584 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.584 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.584 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016927+0000", 2024-08-17T18:05:35.584 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015688599999999999 2024-08-17T18:05:35.584 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.584 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.585 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.585 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:49.016970+0000", 2024-08-17T18:05:35.585 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2456999999999997e-05 2024-08-17T18:05:35.585 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.585 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.585 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.585 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.585 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.585 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:42538 46.7 46:e888a4cb:::benchmark_data_smithi130_267720_object42537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:05:35.585 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:57.009390+0000", 2024-08-17T18:05:35.585 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 338.513147554, 2024-08-17T18:05:35.585 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0254410919999999, 2024-08-17T18:05:35.585 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.586 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.586 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:05:35.586 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.586 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:05:35.586 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:05:35.586 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 42538 2024-08-17T18:05:35.586 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.586 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.586 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.586 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.586 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009390+0000", 2024-08-17T18:05:35.586 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.586 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.587 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.587 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.587 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009390+0000", 2024-08-17T18:05:35.587 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.587 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.587 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.587 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.587 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009392+0000", 2024-08-17T18:05:35.587 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.978e-06 2024-08-17T18:05:35.587 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.587 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.587 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.587 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009418+0000", 2024-08-17T18:05:35.588 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5666000000000001e-05 2024-08-17T18:05:35.588 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.588 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.588 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.588 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009441+0000", 2024-08-17T18:05:35.588 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:35.590 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3311000000000001e-05 2024-08-17T18:05:35.590 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.590 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.590 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.590 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009450+0000", 2024-08-17T18:05:35.590 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6680000000000007e-06 2024-08-17T18:05:35.590 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.590 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.591 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.591 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009513+0000", 2024-08-17T18:05:35.591 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2739999999999999e-05 2024-08-17T18:05:35.591 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.591 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.591 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op must wait for map", 2024-08-17T18:05:35.591 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.009519+0000", 2024-08-17T18:05:35.591 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6599999999999998e-06 2024-08-17T18:05:35.591 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.591 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.591 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.591 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.013068+0000", 2024-08-17T18:05:35.591 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003548257 2024-08-17T18:05:35.591 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.592 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.592 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:05:35.592 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.013078+0000", 2024-08-17T18:05:35.592 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0146000000000001e-05 2024-08-17T18:05:35.592 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.592 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.592 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.592 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.024786+0000", 2024-08-17T18:05:35.592 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.011707755 2024-08-17T18:05:35.592 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.592 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.592 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:05:35.592 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.024983+0000", 2024-08-17T18:05:35.592 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000197479 2024-08-17T18:05:35.593 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.593 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.593 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.593 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.034686+0000", 2024-08-17T18:05:35.593 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0097024959999999997 2024-08-17T18:05:35.593 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.593 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.593 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.593 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.034832+0000", 2024-08-17T18:05:35.593 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014593599999999999 2024-08-17T18:05:35.593 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.593 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.593 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.593 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.594 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.594 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:42545 46.7 46:eec73fd4:::benchmark_data_smithi130_267720_object42544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:05:35.594 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:57.016850+0000", 2024-08-17T18:05:35.594 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 338.50568788700002, 2024-08-17T18:05:35.594 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0183753090000001, 2024-08-17T18:05:35.594 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.594 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.594 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.594 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.594 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:05:35.594 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:05:35.594 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 42545 2024-08-17T18:05:35.594 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.594 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.595 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.595 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.595 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.016850+0000", 2024-08-17T18:05:35.595 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.595 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.595 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.595 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.595 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.016850+0000", 2024-08-17T18:05:35.595 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.595 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.595 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.595 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.595 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.016852+0000", 2024-08-17T18:05:35.596 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5489999999999999e-06 2024-08-17T18:05:35.596 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.596 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.596 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.596 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.016879+0000", 2024-08-17T18:05:35.596 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.7605000000000001e-05 2024-08-17T18:05:35.596 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.596 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.596 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.596 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.016891+0000", 2024-08-17T18:05:35.596 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2201000000000001e-05 2024-08-17T18:05:35.596 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.596 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.596 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.597 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.016899+0000", 2024-08-17T18:05:35.597 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5259999999999996e-06 2024-08-17T18:05:35.597 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.597 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.597 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.597 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.017059+0000", 2024-08-17T18:05:35.597 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000160471 2024-08-17T18:05:35.597 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.597 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.597 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:05:35.597 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.017070+0000", 2024-08-17T18:05:35.597 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0297e-05 2024-08-17T18:05:35.598 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.598 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.598 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.598 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.025286+0000", 2024-08-17T18:05:35.598 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0082163319999999 2024-08-17T18:05:35.598 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.598 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.598 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.598 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.025449+0000", 2024-08-17T18:05:35.598 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016304700000000001 2024-08-17T18:05:35.598 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.598 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.599 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:05:35.599 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.025644+0000", 2024-08-17T18:05:35.599 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00019469499999999999 2024-08-17T18:05:35.599 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.599 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.599 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.599 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.034826+0000", 2024-08-17T18:05:35.599 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.009182183 2024-08-17T18:05:35.599 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.599 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.599 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.599 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.034876+0000", 2024-08-17T18:05:35.599 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0396999999999999e-05 2024-08-17T18:05:35.600 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.600 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.600 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.600 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035011+0000", 2024-08-17T18:05:35.600 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013465999999999999 2024-08-17T18:05:35.600 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.600 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.600 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.600 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035141+0000", 2024-08-17T18:05:35.600 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013020999999999999 2024-08-17T18:05:35.600 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.600 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.600 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.600 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035225+0000", 2024-08-17T18:05:35.601 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4135999999999995e-05 2024-08-17T18:05:35.601 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.601 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.601 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.601 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.601 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.601 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:42560 46.7 46:ea5a980c:::benchmark_data_smithi130_267720_object42559:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:05:35.601 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:57.039166+0000", 2024-08-17T18:05:35.601 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 338.48337206000002, 2024-08-17T18:05:35.601 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.99687259800000005, 2024-08-17T18:05:35.601 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.601 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.601 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.602 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.602 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:05:35.602 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:05:35.602 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 42560 2024-08-17T18:05:35.602 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.602 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.602 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.602 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.602 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039166+0000", 2024-08-17T18:05:35.602 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.602 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.602 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.602 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.603 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039166+0000", 2024-08-17T18:05:35.603 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.603 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.603 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.603 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.603 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039168+0000", 2024-08-17T18:05:35.603 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8199999999999999e-06 2024-08-17T18:05:35.603 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.603 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.603 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.603 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039196+0000", 2024-08-17T18:05:35.603 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8391e-05 2024-08-17T18:05:35.603 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.603 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.604 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.604 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039210+0000", 2024-08-17T18:05:35.604 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3878e-05 2024-08-17T18:05:35.604 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.604 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.604 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.604 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039218+0000", 2024-08-17T18:05:35.604 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6199999999999999e-06 2024-08-17T18:05:35.604 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.604 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.605 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.605 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039534+0000", 2024-08-17T18:05:35.605 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00031600399999999999 2024-08-17T18:05:35.605 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.605 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.605 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:05:35.605 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.039546+0000", 2024-08-17T18:05:35.605 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2279999999999999e-05 2024-08-17T18:05:35.605 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.605 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.605 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.605 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.027308+0000", 2024-08-17T18:05:35.606 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98776202300000004 2024-08-17T18:05:35.606 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.606 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.606 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.606 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.027482+0000", 2024-08-17T18:05:35.606 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000173759 2024-08-17T18:05:35.606 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.606 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.606 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:05:35.606 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.027655+0000", 2024-08-17T18:05:35.606 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017299899999999999 2024-08-17T18:05:35.606 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.606 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.607 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.607 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035288+0000", 2024-08-17T18:05:35.607 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0076332570000000001 2024-08-17T18:05:35.607 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.607 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.607 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.607 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035303+0000", 2024-08-17T18:05:35.607 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5491000000000001e-05 2024-08-17T18:05:35.607 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.607 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.607 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.608 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035373+0000", 2024-08-17T18:05:35.608 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9369999999999995e-05 2024-08-17T18:05:35.608 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.608 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.608 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.608 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035932+0000", 2024-08-17T18:05:35.608 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00055875599999999999 2024-08-17T18:05:35.608 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.608 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.608 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.608 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036039+0000", 2024-08-17T18:05:35.608 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010695 2024-08-17T18:05:35.608 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.609 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.609 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.609 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.609 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.609 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:42564 46.7 46:e9df56bd:::benchmark_data_smithi130_267720_object42563:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:05:35.609 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:57.042569+0000", 2024-08-17T18:05:35.609 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 338.47996910299997, 2024-08-17T18:05:35.609 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.99393814300000005, 2024-08-17T18:05:35.609 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.609 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.609 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.609 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.610 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:05:35.610 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:05:35.610 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 42564 2024-08-17T18:05:35.610 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.610 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.610 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.610 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.610 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042569+0000", 2024-08-17T18:05:35.610 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.610 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.610 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.610 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.610 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042569+0000", 2024-08-17T18:05:35.611 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.611 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.611 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.611 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.611 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042570+0000", 2024-08-17T18:05:35.611 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6020000000000001e-06 2024-08-17T18:05:35.611 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.611 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.611 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.611 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042595+0000", 2024-08-17T18:05:35.611 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4083999999999998e-05 2024-08-17T18:05:35.611 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.611 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.612 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.612 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042607+0000", 2024-08-17T18:05:35.612 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2653999999999999e-05 2024-08-17T18:05:35.612 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.612 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.612 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.612 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042615+0000", 2024-08-17T18:05:35.612 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1019999999999997e-06 2024-08-17T18:05:35.612 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.612 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.612 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.613 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042720+0000", 2024-08-17T18:05:35.613 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000104873 2024-08-17T18:05:35.613 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.613 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.613 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:05:35.613 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.042731+0000", 2024-08-17T18:05:35.613 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0614e-05 2024-08-17T18:05:35.613 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.613 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.614 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.614 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.029280+0000", 2024-08-17T18:05:35.614 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98654872400000004 2024-08-17T18:05:35.614 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.614 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.614 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.614 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.029430+0000", 2024-08-17T18:05:35.614 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015073800000000001 2024-08-17T18:05:35.614 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.614 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.614 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:05:35.614 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.029604+0000", 2024-08-17T18:05:35.614 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000173828 2024-08-17T18:05:35.615 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.615 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.615 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.615 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035306+0000", 2024-08-17T18:05:35.615 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005702403 2024-08-17T18:05:35.615 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.615 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.615 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.615 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036092+0000", 2024-08-17T18:05:35.615 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00078518000000000001 2024-08-17T18:05:35.615 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.615 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.615 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.615 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036337+0000", 2024-08-17T18:05:35.616 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00024485899999999999 2024-08-17T18:05:35.616 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.616 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.616 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.616 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036411+0000", 2024-08-17T18:05:35.616 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4957999999999996e-05 2024-08-17T18:05:35.616 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.616 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.616 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.616 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036507+0000", 2024-08-17T18:05:35.616 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5524000000000002e-05 2024-08-17T18:05:35.616 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.616 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.617 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.617 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.617 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.617 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:42598 46.7 46:e04df74a:::benchmark_data_smithi130_267720_object42597:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:05:35.617 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:57.065153+0000", 2024-08-17T18:05:35.617 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 338.45738454600001, 2024-08-17T18:05:35.617 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.97388112800000004, 2024-08-17T18:05:35.617 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.617 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.617 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.617 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.617 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:05:35.617 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:05:35.617 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 42598 2024-08-17T18:05:35.618 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.618 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.618 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.618 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.618 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065153+0000", 2024-08-17T18:05:35.618 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.618 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.618 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.618 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.618 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065153+0000", 2024-08-17T18:05:35.618 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.618 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.619 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.619 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.619 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065154+0000", 2024-08-17T18:05:35.619 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3200000000000004e-07 2024-08-17T18:05:35.619 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.619 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.619 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.619 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065172+0000", 2024-08-17T18:05:35.619 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7638999999999998e-05 2024-08-17T18:05:35.619 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.619 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.619 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.619 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065181+0000", 2024-08-17T18:05:35.619 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7099999999999996e-06 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065185+0000", 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.814e-06 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065240+0000", 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4125000000000001e-05 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.065250+0000", 2024-08-17T18:05:35.620 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0869999999999999e-05 2024-08-17T18:05:35.621 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.621 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.621 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.621 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.031172+0000", 2024-08-17T18:05:35.621 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.965921213 2024-08-17T18:05:35.621 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.621 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.621 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.621 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.031355+0000", 2024-08-17T18:05:35.621 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018302600000000001 2024-08-17T18:05:35.621 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.621 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.621 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:05:35.622 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.031461+0000", 2024-08-17T18:05:35.622 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010650999999999999 2024-08-17T18:05:35.622 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.622 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.622 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.622 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.035310+0000", 2024-08-17T18:05:35.622 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0038484470000000001 2024-08-17T18:05:35.622 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.622 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.622 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.622 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036223+0000", 2024-08-17T18:05:35.622 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000913264 2024-08-17T18:05:35.622 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.623 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.623 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.623 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.038875+0000", 2024-08-17T18:05:35.623 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026517440000000001 2024-08-17T18:05:35.623 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.623 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.623 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.623 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.038948+0000", 2024-08-17T18:05:35.623 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3411999999999994e-05 2024-08-17T18:05:35.623 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.623 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.623 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.623 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.039035+0000", 2024-08-17T18:05:35.624 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6521999999999996e-05 2024-08-17T18:05:35.624 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.624 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.624 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.624 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.624 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.624 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:42601 46.7 46:e04c9391:::benchmark_data_smithi130_267720_object42600:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:05:35.624 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T17:59:57.067752+0000", 2024-08-17T18:05:35.624 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 338.45478607199999, 2024-08-17T18:05:35.624 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98219549399999995, 2024-08-17T18:05:35.624 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.624 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.625 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.625 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.625 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:05:35.625 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:05:35.625 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 42601 2024-08-17T18:05:35.625 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.625 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.625 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.625 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.625 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067752+0000", 2024-08-17T18:05:35.625 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.625 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.625 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.625 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.626 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067752+0000", 2024-08-17T18:05:35.626 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.626 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.626 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.626 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.626 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067753+0000", 2024-08-17T18:05:35.626 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3799999999999996e-07 2024-08-17T18:05:35.626 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.626 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.626 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.626 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067771+0000", 2024-08-17T18:05:35.626 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8167000000000001e-05 2024-08-17T18:05:35.626 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.627 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.627 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.627 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067780+0000", 2024-08-17T18:05:35.627 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9220000000000003e-06 2024-08-17T18:05:35.627 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.627 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.627 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.627 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067785+0000", 2024-08-17T18:05:35.627 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2639999999999999e-06 2024-08-17T18:05:35.627 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.627 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.627 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.627 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067851+0000", 2024-08-17T18:05:35.628 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6042000000000002e-05 2024-08-17T18:05:35.628 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.628 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.628 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:05:35.628 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:57.067859+0000", 2024-08-17T18:05:35.628 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3450000000000006e-06 2024-08-17T18:05:35.628 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.628 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.628 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.628 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036577+0000", 2024-08-17T18:05:35.628 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.968717418 2024-08-17T18:05:35.628 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.628 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.629 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.629 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036706+0000", 2024-08-17T18:05:35.629 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000129116 2024-08-17T18:05:35.629 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.629 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.629 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:05:35.629 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.036865+0000", 2024-08-17T18:05:35.629 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000159059 2024-08-17T18:05:35.629 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.629 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.629 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.629 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.043100+0000", 2024-08-17T18:05:35.629 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0062351480000000003 2024-08-17T18:05:35.629 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.630 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.630 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.630 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.045320+0000", 2024-08-17T18:05:35.630 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002219702 2024-08-17T18:05:35.630 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.630 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.630 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.630 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.049699+0000", 2024-08-17T18:05:35.630 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0043796570000000003 2024-08-17T18:05:35.630 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.630 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.630 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.630 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.049804+0000", 2024-08-17T18:05:35.630 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010448800000000001 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T17:59:58.049947+0000", 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000143428 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79527 46.7 46:eed33f3f:::benchmark_data_smithi130_267720_object79526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.632282+0000", 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 304.89025584400002, 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38843913299999999, 2024-08-17T18:05:35.631 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.632 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.632 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.632 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.632 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:05:35.632 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:05:35.632 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79527 2024-08-17T18:05:35.632 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.632 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.632 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.632 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.632 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632282+0000", 2024-08-17T18:05:35.632 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.632 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.632 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.633 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.633 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632282+0000", 2024-08-17T18:05:35.633 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.633 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.633 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.633 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.633 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632283+0000", 2024-08-17T18:05:35.633 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.78e-07 2024-08-17T18:05:35.633 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.633 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.633 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.633 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632304+0000", 2024-08-17T18:05:35.633 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1094000000000002e-05 2024-08-17T18:05:35.633 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.634 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.634 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.634 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632314+0000", 2024-08-17T18:05:35.634 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9289999999999993e-06 2024-08-17T18:05:35.634 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.634 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.634 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.634 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.632321+0000", 2024-08-17T18:05:35.634 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7399999999999998e-06 2024-08-17T18:05:35.634 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.634 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.634 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.634 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.635991+0000", 2024-08-17T18:05:35.635 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0036704569999999998 2024-08-17T18:05:35.635 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.635 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.635 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.635 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636300+0000", 2024-08-17T18:05:35.635 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00030820700000000001 2024-08-17T18:05:35.635 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.635 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.635 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:05:35.635 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.636833+0000", 2024-08-17T18:05:35.635 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00053374499999999997 2024-08-17T18:05:35.635 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.636 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.636 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.636 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.641888+0000", 2024-08-17T18:05:35.636 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0050543869999999996 2024-08-17T18:05:35.636 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.636 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.636 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.636 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.643224+0000", 2024-08-17T18:05:35.636 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0013359750000000001 2024-08-17T18:05:35.636 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.636 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.636 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.637 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020614+0000", 2024-08-17T18:05:35.637 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37739075999999999 2024-08-17T18:05:35.637 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.637 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.637 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.637 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020677+0000", 2024-08-17T18:05:35.637 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2242999999999993e-05 2024-08-17T18:05:35.637 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.637 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.637 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.637 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020721+0000", 2024-08-17T18:05:35.637 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4617999999999999e-05 2024-08-17T18:05:35.637 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.637 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.638 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.638 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.638 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.638 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:95864 46.7 46:e0257e6e:::benchmark_data_smithi130_267720_object95863:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-08-17T18:05:35.638 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:49.266079+0000", 2024-08-17T18:05:35.638 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 286.25645876300001, 2024-08-17T18:05:35.638 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35426025, 2024-08-17T18:05:35.638 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.638 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.638 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.638 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.638 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:05:35.639 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:05:35.639 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 95864 2024-08-17T18:05:35.639 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.639 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.639 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.639 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.639 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266079+0000", 2024-08-17T18:05:35.639 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.639 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.639 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.639 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.639 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266079+0000", 2024-08-17T18:05:35.639 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.639 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.640 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.640 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.640 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266081+0000", 2024-08-17T18:05:35.640 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7290000000000001e-06 2024-08-17T18:05:35.640 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.640 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.640 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.640 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266104+0000", 2024-08-17T18:05:35.640 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3125999999999998e-05 2024-08-17T18:05:35.640 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.640 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.640 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.640 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266117+0000", 2024-08-17T18:05:35.640 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2931999999999999e-05 2024-08-17T18:05:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266124+0000", 2024-08-17T18:05:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4309999999999997e-06 2024-08-17T18:05:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266184+0000", 2024-08-17T18:05:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9741999999999998e-05 2024-08-17T18:05:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.641 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266324+0000", 2024-08-17T18:05:35.642 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013939399999999999 2024-08-17T18:05:35.642 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.642 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.642 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:05:35.642 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.266467+0000", 2024-08-17T18:05:35.642 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000143657 2024-08-17T18:05:35.642 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.642 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.642 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.642 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.273981+0000", 2024-08-17T18:05:35.642 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0075134829999999996 2024-08-17T18:05:35.642 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.643 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.643 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.643 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.277410+0000", 2024-08-17T18:05:35.643 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0034292860000000001 2024-08-17T18:05:35.643 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.643 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.643 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.643 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.620234+0000", 2024-08-17T18:05:35.643 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.34282429800000003 2024-08-17T18:05:35.643 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.643 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.643 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.643 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.620293+0000", 2024-08-17T18:05:35.644 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8974000000000002e-05 2024-08-17T18:05:35.644 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.644 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.644 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.644 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.620339+0000", 2024-08-17T18:05:35.644 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6198000000000001e-05 2024-08-17T18:05:35.644 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.644 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.644 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.644 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.644 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.644 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:95870 46.7 46:ea577ff7:::benchmark_data_smithi130_267720_object95869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-08-17T18:05:35.644 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:49.269591+0000", 2024-08-17T18:05:35.645 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 286.252947119, 2024-08-17T18:05:35.645 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35086666100000002, 2024-08-17T18:05:35.645 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.645 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.645 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.645 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.645 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:05:35.645 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:05:35.645 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 95870 2024-08-17T18:05:35.645 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.645 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.646 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.646 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.646 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269591+0000", 2024-08-17T18:05:35.646 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.646 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.646 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.646 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.646 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269591+0000", 2024-08-17T18:05:35.646 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.646 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.646 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.646 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.646 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269592+0000", 2024-08-17T18:05:35.647 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3170000000000001e-06 2024-08-17T18:05:35.647 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.647 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.647 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.647 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269614+0000", 2024-08-17T18:05:35.647 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2010000000000001e-05 2024-08-17T18:05:35.647 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.647 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.647 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.647 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269626+0000", 2024-08-17T18:05:35.647 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2104e-05 2024-08-17T18:05:35.647 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.647 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.647 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.649 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269634+0000", 2024-08-17T18:05:35.649 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3459999999999996e-06 2024-08-17T18:05:35.649 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.649 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.649 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.649 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269743+0000", 2024-08-17T18:05:35.649 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010924799999999999 2024-08-17T18:05:35.649 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.649 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.649 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.650 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269878+0000", 2024-08-17T18:05:35.650 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013560699999999999 2024-08-17T18:05:35.650 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.650 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.650 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:05:35.650 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.270018+0000", 2024-08-17T18:05:35.650 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013919299999999999 2024-08-17T18:05:35.650 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.650 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.650 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.650 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.274481+0000", 2024-08-17T18:05:35.650 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0044628890000000003 2024-08-17T18:05:35.650 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.277929+0000", 2024-08-17T18:05:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003448156 2024-08-17T18:05:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.620384+0000", 2024-08-17T18:05:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.34245572699999999 2024-08-17T18:05:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.620426+0000", 2024-08-17T18:05:35.651 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1900999999999997e-05 2024-08-17T18:05:35.652 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.652 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.652 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.652 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.620458+0000", 2024-08-17T18:05:35.652 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1163e-05 2024-08-17T18:05:35.652 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.652 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.652 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.652 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.652 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.652 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:95883 46.7 46:ed78bf3e:::benchmark_data_smithi130_267720_object95882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-08-17T18:05:35.652 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:49.282472+0000", 2024-08-17T18:05:35.652 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 286.24006562300002, 2024-08-17T18:05:35.653 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.34845915799999999, 2024-08-17T18:05:35.653 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.653 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.653 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.653 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.653 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:05:35.653 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:05:35.653 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 95883 2024-08-17T18:05:35.653 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.653 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.653 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.653 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.653 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.282472+0000", 2024-08-17T18:05:35.653 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.654 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.654 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.654 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.654 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.282472+0000", 2024-08-17T18:05:35.654 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.654 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.654 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.654 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.654 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.282473+0000", 2024-08-17T18:05:35.654 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4600000000000004e-07 2024-08-17T18:05:35.654 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.654 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.654 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.655 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.282495+0000", 2024-08-17T18:05:35.655 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1571000000000001e-05 2024-08-17T18:05:35.655 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.655 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.655 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.655 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.282503+0000", 2024-08-17T18:05:35.655 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7069999999999998e-06 2024-08-17T18:05:35.655 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.655 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.655 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.655 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.282509+0000", 2024-08-17T18:05:35.655 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.015e-06 2024-08-17T18:05:35.655 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.655 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.656 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.656 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.282558+0000", 2024-08-17T18:05:35.656 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8318000000000001e-05 2024-08-17T18:05:35.656 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.656 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.656 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.656 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.616414+0000", 2024-08-17T18:05:35.656 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.33385609300000002 2024-08-17T18:05:35.656 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.656 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.656 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:05:35.656 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.616566+0000", 2024-08-17T18:05:35.656 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015180099999999999 2024-08-17T18:05:35.656 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.657 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.657 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.657 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630615+0000", 2024-08-17T18:05:35.657 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014049216999999999 2024-08-17T18:05:35.657 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.657 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.657 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.657 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630649+0000", 2024-08-17T18:05:35.657 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3905000000000001e-05 2024-08-17T18:05:35.657 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.657 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.657 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.657 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630840+0000", 2024-08-17T18:05:35.658 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000191106 2024-08-17T18:05:35.658 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.658 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.658 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.658 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630891+0000", 2024-08-17T18:05:35.658 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1144999999999999e-05 2024-08-17T18:05:35.658 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.658 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.658 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.658 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630931+0000", 2024-08-17T18:05:35.658 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0534000000000002e-05 2024-08-17T18:05:35.658 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.658 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.658 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.659 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.659 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.659 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:95888 46.3 46:c3756994:::benchmark_data_smithi130_267720_object95887:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-08-17T18:05:35.659 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:49.287821+0000", 2024-08-17T18:05:35.659 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 286.23471700900001, 2024-08-17T18:05:35.659 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.35515286499999998, 2024-08-17T18:05:35.659 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:05:35.659 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:05:35.659 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:05:35.659 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:05:35.659 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:05:35.659 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:05:35.659 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 95888 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.287821+0000", 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.287821+0000", 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:05:35.660 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.287822+0000", 2024-08-17T18:05:35.661 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3e-06 2024-08-17T18:05:35.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.661 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.661 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:05:35.661 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.287843+0000", 2024-08-17T18:05:35.661 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0489999999999999e-05 2024-08-17T18:05:35.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.661 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.661 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:05:35.661 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.616279+0000", 2024-08-17T18:05:35.661 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.32843580300000003 2024-08-17T18:05:35.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.661 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.661 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:05:35.662 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.616296+0000", 2024-08-17T18:05:35.662 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7003999999999998e-05 2024-08-17T18:05:35.662 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.662 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.662 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:05:35.662 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.616375+0000", 2024-08-17T18:05:35.662 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9618999999999998e-05 2024-08-17T18:05:35.662 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.662 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.662 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:05:35.662 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.616529+0000", 2024-08-17T18:05:35.662 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015364099999999999 2024-08-17T18:05:35.662 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.663 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.663 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:05:35.663 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.616671+0000", 2024-08-17T18:05:35.663 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000141707 2024-08-17T18:05:35.663 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.663 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.663 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:05:35.663 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630669+0000", 2024-08-17T18:05:35.663 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013998745999999999 2024-08-17T18:05:35.663 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.663 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.663 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.663 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.631980+0000", 2024-08-17T18:05:35.663 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0013102579999999999 2024-08-17T18:05:35.664 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.664 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.664 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:05:35.664 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.642830+0000", 2024-08-17T18:05:35.664 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.01085066 2024-08-17T18:05:35.664 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.664 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.664 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:05:35.664 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.642926+0000", 2024-08-17T18:05:35.664 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5964999999999993e-05 2024-08-17T18:05:35.664 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:05:35.664 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:05:35.664 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:05:35.665 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.642974+0000", 2024-08-17T18:05:35.665 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7672e-05 2024-08-17T18:05:35.665 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.665 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.665 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.665 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:05:35.665 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:05:35.665 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:05:35.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:35.666 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:35.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:35.767 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:35.789 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:35.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:35.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:35.890 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:35.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:35.913 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:35.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:36.015 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:36.036 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:36.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:36.138 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:36.159 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:36.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:36.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:36.260 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:36.297 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:36.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:36.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:36.400 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:36.421 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:36.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:36.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:36.523 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:36.555 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:36.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:36.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:36.657 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:36.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:36 smithi130 ceph-mon[105521]: pgmap v2244: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 85 B/s rd, 33 MiB/s wr, 666 op/s 2024-08-17T18:05:36.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:36 smithi130 ceph-mon[110068]: pgmap v2244: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 85 B/s rd, 33 MiB/s wr, 666 op/s 2024-08-17T18:05:36.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:36.680 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:36.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:36.781 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:36.820 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:36.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:36.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:36.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:36 smithi179 ceph-mon[93519]: pgmap v2244: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 85 B/s rd, 33 MiB/s wr, 666 op/s 2024-08-17T18:05:36.922 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:36.944 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:36.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:36.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:37.046 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:37.067 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:37.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:37.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:37.170 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:37.206 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:37.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:37.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:37.217 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:05:37.218 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:05:37.218 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:05:37.218 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:05:37.218 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:05:37.218 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:05:37.218 INFO:teuthology.orchestra.run.smithi130.stderr:bench [] [] [] [] 2024-08-17T18:05:37.218 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:05:37.218 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:05:37.218 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:05:37.219 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator dump block 2024-08-17T18:05:37.219 INFO:teuthology.orchestra.run.smithi130.stderr:Error EINVAL: invalid command 2024-08-17T18:05:37.308 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:37.353 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:37.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:37.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:37.455 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:37.476 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:37.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:37.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:37.578 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:37.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:37.600 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:37.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:37.702 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:37.723 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:37.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:37.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:37.825 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:37.846 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:37.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:37.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:37.883 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:37.948 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:38.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:38.007 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:38.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:38.108 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:38.129 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:38.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:38.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:38.231 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:38.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:38.267 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:38.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:38.368 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:38.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:38.391 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:38.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:38.450 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:05:38.492 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:38.514 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:38.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:38.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:38.616 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:38.637 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:38.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:38.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:38.739 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:38.760 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:38.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:38.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:38 smithi179 ceph-mon[93519]: pgmap v2245: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 85 B/s rd, 29 MiB/s wr, 733 op/s 2024-08-17T18:05:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:38 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3443080372' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:05:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:38 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:05:38.860 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:38 smithi130 ceph-mon[110068]: pgmap v2245: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 85 B/s rd, 29 MiB/s wr, 733 op/s 2024-08-17T18:05:38.860 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:38 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3443080372' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:05:38.860 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:38 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:05:38.860 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:38 smithi130 ceph-mon[105521]: pgmap v2245: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 85 B/s rd, 29 MiB/s wr, 733 op/s 2024-08-17T18:05:38.860 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:38 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3443080372' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:05:38.860 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:38 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:05:38.863 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:38.912 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T18:05:38.918 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:38.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:38.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:38.934 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:05:38.935 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T18:05:38.936 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-08-17T18:05:39.020 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:39.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:39.043 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:39.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:39.145 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:39.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:39.167 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:39.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:39.269 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:39.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:39.333 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:39.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:39.435 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:39.456 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:39.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:39.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:39.558 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:39.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:39.580 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:39.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:39.681 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:39.702 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:39.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:39.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:39.804 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:39.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:39 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:05:39.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:39 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:05:39.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:39 smithi130 ceph-mon[110068]: osdmap e1315: 8 total, 8 up, 5 in 2024-08-17T18:05:39.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:39 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:05:39.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:39 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:05:39.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:39 smithi130 ceph-mon[105521]: osdmap e1315: 8 total, 8 up, 5 in 2024-08-17T18:05:39.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:39.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:39.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:39 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:05:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:39 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:05:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:39 smithi179 ceph-mon[93519]: osdmap e1315: 8 total, 8 up, 5 in 2024-08-17T18:05:39.928 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:39.950 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:39.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:39.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:40.052 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:40.073 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:40.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:40.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:40.123 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:05:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:05:39] "GET /metrics HTTP/1.1" 200 36781 "" "Prometheus/2.51.0" 2024-08-17T18:05:40.175 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:40.197 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:40.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:40.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:40.299 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:40.320 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:40.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:40.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:40.422 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:40.444 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:40.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:40.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:40.473 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:05:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:05:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:05:40.546 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:40.588 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:40.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:40.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:40.690 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:40.728 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:40.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:40.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:40.830 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:40.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:40 smithi179 ceph-mon[93519]: pgmap v2247: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 14 MiB/s wr, 765 op/s 2024-08-17T18:05:40.871 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:40.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:40.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:40.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:40 smithi130 ceph-mon[110068]: pgmap v2247: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 14 MiB/s wr, 765 op/s 2024-08-17T18:05:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:40 smithi130 ceph-mon[105521]: pgmap v2247: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 14 MiB/s wr, 765 op/s 2024-08-17T18:05:40.972 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:40.993 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:40.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:40.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:41.077 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:41.095 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:41.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:41.150 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:41.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:41.251 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:41.272 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:41.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:41.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:41.373 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:41.427 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:41.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:41.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:41.529 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:41.550 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:41.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:41.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:41.652 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:41.694 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:41.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:41.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:41.796 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:41.817 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:41.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:41.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:41.918 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:41.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:05:41.939 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:41.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:41.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:42.041 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:42.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:42.064 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:42.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:42.165 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:42.187 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:42.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:42.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:42.289 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:42.310 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:42.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:42.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:42.412 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:42.455 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:42.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:42.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:42.557 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:42.578 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:42.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:42.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:42.656 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:42 smithi130 ceph-mon[105521]: pgmap v2248: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 14 MiB/s wr, 765 op/s 2024-08-17T18:05:42.656 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:42.656 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:05:42.656 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:42 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3148167663' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:05:42.656 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:42 smithi130 ceph-mon[110068]: pgmap v2248: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 14 MiB/s wr, 765 op/s 2024-08-17T18:05:42.656 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:42.657 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:05:42.657 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:42 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3148167663' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:05:42.680 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:42.703 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:42.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:42.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:42.805 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:42.841 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:42.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:42.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:42 smithi179 ceph-mon[93519]: pgmap v2248: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 14 MiB/s wr, 765 op/s 2024-08-17T18:05:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:05:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:42 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3148167663' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:05:42.944 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:42.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:42.965 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:42.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:43.067 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:43.089 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:43.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:43.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:43.196 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:43.218 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:43.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:43.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:43.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:43.341 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:43.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:43.442 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:43.464 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:43.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:43.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:43.566 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:43.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:05:43.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:05:43.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:05:43.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:05:43.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:05:43.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:05:43.666 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:05:43.667 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:05:43.667 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:05:43.667 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:05:43.667 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:05:43.667 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:05:43.667 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:05:43.667 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:05:43.667 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:05:43.668 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:05:43.715 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:43.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:43.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:43.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:43.851 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:43.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:43.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:43.953 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:43.993 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:43.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:43.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:44.095 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:44.096 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:44.116 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:44.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:44.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:44.218 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:44.267 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:44.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:44.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:44.370 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:44.392 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:44.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:44.493 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:44.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:44.522 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:44.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:44.623 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:44.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:44.664 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:44.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:44.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:44 smithi130 ceph-mon[105521]: pgmap v2249: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 2.7 MiB/s wr, 756 op/s 2024-08-17T18:05:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:44 smithi130 ceph-mon[110068]: pgmap v2249: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 2.7 MiB/s wr, 756 op/s 2024-08-17T18:05:44.765 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:44.806 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:44.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:44.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:44 smithi179 ceph-mon[93519]: pgmap v2249: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 102 B/s rd, 2.7 MiB/s wr, 756 op/s 2024-08-17T18:05:44.907 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:44.940 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:44.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:44.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:45.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:45.081 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:45.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:45.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:45.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:45.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:45.222 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:45.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:45.323 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:45.344 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:45.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:45.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:45.446 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:45.486 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:45.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:45.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:45.587 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:45.609 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:45.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:45.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:45.711 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:45.732 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:45.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:45.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:45.834 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:45.873 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:45.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:45.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:45.974 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:45.996 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:45.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:45.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:46.098 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:46.119 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:46.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:46.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:46.221 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:46.270 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:46.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:46.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:46.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:46.394 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:46.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:46.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:46.495 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:46.516 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:46.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:46.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:46.543 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:46.618 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:46.639 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:46.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:46.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:46.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:46 smithi130 ceph-mon[105521]: pgmap v2250: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 748 op/s 2024-08-17T18:05:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:46 smithi130 ceph-mon[110068]: pgmap v2250: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 748 op/s 2024-08-17T18:05:46.741 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:46.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:46.777 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:46.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:46 smithi179 ceph-mon[93519]: pgmap v2250: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 748 op/s 2024-08-17T18:05:46.878 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:46.899 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:46.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:46.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:47.001 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:47.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:47.023 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:47.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:47.124 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:47.146 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:47.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:47.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:47.247 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:47.253 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:05:47.269 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:47.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:47.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:47.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:47.393 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:47.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:47.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:47.494 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:47.517 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:47.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:47.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:47.618 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:47.665 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:47.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:47.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:47.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:47.808 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:47.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:47.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:47.910 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:47.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:47.933 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:47.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:48.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:48.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:48.072 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:48.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:48.173 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:48.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:48.201 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:48.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:48.302 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:48.339 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:48.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:48.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:48.392 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:05:48.392 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:05:48.392 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:05:48.392 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:05:48.392 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:05:48.392 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:05:48.392 INFO:teuthology.orchestra.run.smithi130.stderr:bench [] [] [] [] 2024-08-17T18:05:48.392 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:05:48.392 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:05:48.393 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:05:48.393 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator dump block 2024-08-17T18:05:48.393 INFO:teuthology.orchestra.run.smithi130.stderr:Error EINVAL: invalid command 2024-08-17T18:05:48.441 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:48.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:48.490 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:48.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:48.591 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:48.640 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:48.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:48.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:48.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:48 smithi130 ceph-mon[105521]: pgmap v2251: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 741 op/s 2024-08-17T18:05:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:48 smithi130 ceph-mon[110068]: pgmap v2251: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 741 op/s 2024-08-17T18:05:48.742 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:48.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:48.783 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:48.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:48 smithi179 ceph-mon[93519]: pgmap v2251: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 741 op/s 2024-08-17T18:05:48.884 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:48.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:48.922 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:48.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:49.023 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:49.044 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:49.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:49.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:49.125 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:05:49.126 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T18:05:49.147 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:49.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:49.170 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:49.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:49.271 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:49.294 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:49.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:49.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:49.396 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:49.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:49.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:49.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:49.519 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:49.556 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:49.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:49.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:49.659 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:49.680 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:49.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:49.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:49.782 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:49.796 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:05:49.796 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:05:49.803 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:49.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:49.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:49.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:49.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:05:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:05:49] "GET /metrics HTTP/1.1" 200 36786 "" "Prometheus/2.51.0" 2024-08-17T18:05:49.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:49.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:49.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:50.030 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:50.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:50.052 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:50.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:50.153 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:50.175 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:50.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:50.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:50.277 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:50.299 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:50.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:50.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:50.401 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:50.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:50.424 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:50.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:50.492 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:05:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:05:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:05:50.525 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:50.568 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:50.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:50.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:50.600 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T18:05:50.670 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:50.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:50.733 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:50.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:50.761 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:50 smithi179 ceph-mon[93519]: pgmap v2252: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 693 op/s 2024-08-17T18:05:50.834 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:50.843 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:50 smithi130 ceph-mon[105521]: pgmap v2252: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 693 op/s 2024-08-17T18:05:50.843 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:50 smithi130 ceph-mon[110068]: pgmap v2252: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 693 op/s 2024-08-17T18:05:50.915 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:50.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:50.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:51.017 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:51.038 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:51.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:51.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:51.140 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:51.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:51.199 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:51.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:51.211 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:51.282 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:51.301 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:51.368 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:51.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:51.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:51.470 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:51.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:51.517 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:51.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:51.618 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:51.640 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:51.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:51.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:51.741 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:51.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:51.764 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:51.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:51.865 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:51.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:51.907 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:51.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:52.008 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:52.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:52.052 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:52.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:52.153 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:52.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:52.176 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:52.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:52.277 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:52.316 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:52.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:52.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:52.418 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:52.457 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:52.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:52.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:52.508 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:05:52.559 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:52.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:52.597 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:52.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:52.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:52 smithi130 ceph-mon[105521]: pgmap v2253: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 626 op/s 2024-08-17T18:05:52.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:52 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1761782417' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:05:52.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:52 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:05:52.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:52 smithi130 ceph-mon[110068]: pgmap v2253: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 626 op/s 2024-08-17T18:05:52.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:52 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1761782417' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:05:52.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:52 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:05:52.698 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:52.719 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:52.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:52.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:52.821 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:52 smithi179 ceph-mon[93519]: pgmap v2253: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 626 op/s 2024-08-17T18:05:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:52 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1761782417' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:05:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:52 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:05:52.863 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:52.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:52.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:52.965 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:52.987 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:52.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:52.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:53.089 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:53.134 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:53.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:53.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:53.236 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:53.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:53.278 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:53.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:53.380 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:53.400 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:53.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:53.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:53.502 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:53.524 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:53.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:53.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:53.625 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:53.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:53.663 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:53.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:53.764 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:53.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:53.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:53.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:53 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:05:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:53 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:05:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:53 smithi179 ceph-mon[93519]: osdmap e1316: 8 total, 8 up, 5 in 2024-08-17T18:05:53.855 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:53 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:05:53.855 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:53 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:05:53.855 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:53 smithi130 ceph-mon[105521]: osdmap e1316: 8 total, 8 up, 5 in 2024-08-17T18:05:53.855 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:53 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:05:53.855 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:53 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:05:53.855 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:53 smithi130 ceph-mon[110068]: osdmap e1316: 8 total, 8 up, 5 in 2024-08-17T18:05:53.912 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:53.933 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:53.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:53.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:54.035 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:05:54.076 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:54.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:54.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:05:54.127 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 7] out_osds: [5, 4, 6] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T18:05:54.127 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:05:54.127 INFO:tasks.thrashosds.thrasher:Killing osd 3, live_osds are [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T18:05:54.127 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-08-17T18:05:54.128 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.3 2024-08-17T18:05:54.178 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:54.198 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:54.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:54.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:54.300 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:54.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:54.322 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:54.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:54.422 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:54.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:54.444 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:54.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:54.527 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:05:54 smithi130 systemd[1]: Stopping Ceph osd.3 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:05:54.545 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:54.593 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:54.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:54.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:54.695 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:54.717 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:54.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:54.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:54.778 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:05:54.779 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:05:54.779 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:54 smithi130 ceph-mon[105521]: pgmap v2255: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 764 op/s 2024-08-17T18:05:54.779 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:54 smithi130 ceph-mon[110068]: pgmap v2255: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 764 op/s 2024-08-17T18:05:54.815 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:05:54.816 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:05:54.819 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:54.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:54.840 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:54.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:54.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:54 smithi179 ceph-mon[93519]: pgmap v2255: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 764 op/s 2024-08-17T18:05:54.857 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:05:54.857 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:05:54.887 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:05:54.887 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:05:54.916 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:05:54.916 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:05:54.942 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:54.949 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:05:54.949 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:05:54.977 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:05:54.977 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:05:54.981 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:54.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:54.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:55.007 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:05:55.007 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:05:55.083 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:55.104 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:55.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:55.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:55.206 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:55.260 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:55.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:55.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:55.362 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:55.407 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:55.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:55.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:55.508 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:55.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:55.531 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:55.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:55.633 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:55.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:55.655 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:55.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:55.756 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:55.777 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:55.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:55.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:55.781 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:55 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1346530921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:05:55.782 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:55 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1346530921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:05:55.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:55 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1346530921' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:05:55.879 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:55.922 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:55.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:55.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:56.024 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:56.045 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:56.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:56.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:56.147 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:56.192 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:56.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:56.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:56.294 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:56.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:56.317 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:56.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:56.402 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:05:56.418 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:56.422 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:05:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[136110]: 2024-08-17T18:05:56.191+0000 7f1c805b9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T18:05:56.422 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:05:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[136110]: 2024-08-17T18:05:56.191+0000 7f1c805b9640 -1 osd.3 1316 *** Got signal Terminated *** 2024-08-17T18:05:56.422 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:05:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[136110]: 2024-08-17T18:05:56.191+0000 7f1c805b9640 -1 osd.3 1316 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T18:05:56.464 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:05:56.465 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T18:05:56.465 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T18:05:56.465 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T18:05:56.465 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:05:56.478 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:56.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:56.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:56.580 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:05:56.607 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:56.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:56.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:05:56.709 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:56.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:56.731 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:56.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:56.822 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:56 smithi130 ceph-mon[105521]: pgmap v2256: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 779 op/s 2024-08-17T18:05:56.822 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:56 smithi130 ceph-mon[105521]: osd.3 marked itself down and dead 2024-08-17T18:05:56.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:56 smithi130 ceph-mon[110068]: pgmap v2256: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 779 op/s 2024-08-17T18:05:56.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:56 smithi130 ceph-mon[110068]: osd.3 marked itself down and dead 2024-08-17T18:05:56.832 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:56.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:56 smithi179 ceph-mon[93519]: pgmap v2256: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 779 op/s 2024-08-17T18:05:56.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:56 smithi179 ceph-mon[93519]: osd.3 marked itself down and dead 2024-08-17T18:05:56.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:56.854 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:56.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:56.955 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:56.978 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:56.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:56.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:57.080 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:57.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:57.102 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:57.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:57.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:57.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:57.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:57.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:57.326 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:57.347 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:57.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:57.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:57.417 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:05:57 smithi130 podman[315124]: 2024-08-17 18:05:57.156757406 +0000 UTC m=+1.311146753 container died a4c158c50e3528fb0b38cfc7326954183eac2fd6dff97dc1801d76cde28d99e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T18:05:57.449 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:05:57.471 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:57.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:57.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:05:57.572 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:57.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:57.609 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:57.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:57.710 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:57.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:57.733 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:57.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:57.821 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:05:57.834 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:57.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:57 smithi179 ceph-mon[93519]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T18:05:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:57 smithi179 ceph-mon[93519]: osdmap e1317: 8 total, 7 up, 5 in 2024-08-17T18:05:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:57 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:05:57.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:57 smithi130 ceph-mon[110068]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T18:05:57.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:57 smithi130 ceph-mon[110068]: osdmap e1317: 8 total, 7 up, 5 in 2024-08-17T18:05:57.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:57 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:57.861 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:05:57.861 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:05:57 smithi130 podman[315124]: 2024-08-17 18:05:57.600789501 +0000 UTC m=+1.755178843 container cleanup a4c158c50e3528fb0b38cfc7326954183eac2fd6dff97dc1801d76cde28d99e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-08-17T18:05:57.861 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:05:57 smithi130 bash[315124]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3 2024-08-17T18:05:57.862 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:57 smithi130 ceph-mon[105521]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T18:05:57.862 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:57 smithi130 ceph-mon[105521]: osdmap e1317: 8 total, 7 up, 5 in 2024-08-17T18:05:57.862 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:57 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:05:57.862 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:05:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:57.910 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:57.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:58.011 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:58.032 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:58.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:58.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:58.108 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:58.134 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:58.146 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:05:57 smithi130 podman[315331]: 2024-08-17 18:05:57.919470471 +0000 UTC m=+0.825737397 container remove a4c158c50e3528fb0b38cfc7326954183eac2fd6dff97dc1801d76cde28d99e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe) 2024-08-17T18:05:58.148 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T18:05:58.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:58.156 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:58.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:58.257 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:58.278 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:58.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:58.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:58.379 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:58.435 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:58.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:58.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:58.537 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:58.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:58.577 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:58.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:58 smithi130 ceph-mon[105521]: pgmap v2258: 89 pgs: 19 stale+active+clean, 70 active+clean; 4.7 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 694 op/s 2024-08-17T18:05:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:58 smithi130 ceph-mon[105521]: osdmap e1318: 8 total, 7 up, 5 in 2024-08-17T18:05:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:58 smithi130 ceph-mon[110068]: pgmap v2258: 89 pgs: 19 stale+active+clean, 70 active+clean; 4.7 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 694 op/s 2024-08-17T18:05:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:58 smithi130 ceph-mon[110068]: osdmap e1318: 8 total, 7 up, 5 in 2024-08-17T18:05:58.677 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:58.705 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:58.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:58.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:58.807 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:58.828 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:58.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:58.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:58.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:58 smithi179 ceph-mon[93519]: pgmap v2258: 89 pgs: 19 stale+active+clean, 70 active+clean; 4.7 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 694 op/s 2024-08-17T18:05:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:58 smithi179 ceph-mon[93519]: osdmap e1318: 8 total, 7 up, 5 in 2024-08-17T18:05:58.930 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:58.952 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:58.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:58.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:59.053 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:59.075 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:59.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:59.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:59.177 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:05:59.198 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:59.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:59.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:05:59.299 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:05:59.320 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:59.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:59.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:05:59.417 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:05:59 smithi130 podman[315441]: 2024-08-17 18:05:59.044554563 +0000 UTC m=+0.920656332 container died 1bbcfe4f016fdbf4a6e19271062c52096fd508171d4528d816fb46717e687843 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=musing_black, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4) 2024-08-17T18:05:59.422 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:05:59.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:59.481 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:05:59.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:05:59.482 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:05:59.582 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:59.603 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:59.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:59.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:59.704 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:05:59.726 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:59.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:59.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:05:59.828 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:59.850 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:59.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:59.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:05:59.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:05:59 smithi130 ceph-mon[105521]: Health check failed: Degraded data redundancy: 23303/225444 objects degraded (10.336%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T18:05:59.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:05:59 smithi130 ceph-mon[110068]: Health check failed: Degraded data redundancy: 23303/225444 objects degraded (10.336%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T18:05:59.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:05:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:05:59] "GET /metrics HTTP/1.1" 200 36782 "" "Prometheus/2.51.0" 2024-08-17T18:05:59.951 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:05:59.972 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:05:59.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:05:59.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:00.074 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:00.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:05:59 smithi179 ceph-mon[93519]: Health check failed: Degraded data redundancy: 23303/225444 objects degraded (10.336%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:00.095 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:00.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:00.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:00.197 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:00.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:00.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:00.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:00.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:00.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:00.366 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:00.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:00.467 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:00.488 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:00.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:00.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:00.590 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:06:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:06:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:06:00.611 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:00.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:00.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:00.713 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:00.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:00.760 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:00.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:00.861 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:00.876 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:00 smithi130 ceph-mon[105521]: pgmap v2260: 89 pgs: 20 active+undersized, 9 stale+active+clean, 12 active+undersized+degraded, 48 active+clean; 4.6 GiB data, 17 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 799 op/s; 23303/225444 objects degraded (10.336%) 2024-08-17T18:06:00.876 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:00 smithi130 ceph-mon[110068]: pgmap v2260: 89 pgs: 20 active+undersized, 9 stale+active+clean, 12 active+undersized+degraded, 48 active+clean; 4.6 GiB data, 17 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 799 op/s; 23303/225444 objects degraded (10.336%) 2024-08-17T18:06:00.876 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:06:00 smithi130 podman[315725]: 2024-08-17 18:06:00.632604661 +0000 UTC m=+0.278984159 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T18:06:00.876 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:06:00 smithi130 podman[315725]: 2024-08-17 18:06:00.765288722 +0000 UTC m=+0.411668208 container create 3c8e46e0649d25f41fe3ff31d4572d05e9800e5654da50109586ebae49ffe6e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, RELEASE=main-cd8bdbe, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T18:06:00.882 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:00.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:00.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:00.984 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:01.023 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:01.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:01.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:00 smithi179 ceph-mon[93519]: pgmap v2260: 89 pgs: 20 active+undersized, 9 stale+active+clean, 12 active+undersized+degraded, 48 active+clean; 4.6 GiB data, 17 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 799 op/s; 23303/225444 objects degraded (10.336%) 2024-08-17T18:06:01.125 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:01.164 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:01.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:01.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:01.266 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:01.313 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:01.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:01.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:01.415 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:01.454 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:01.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:01.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:01.539 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:06:01.556 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:01.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:01.578 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:01.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:01.647 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:06:01.679 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:01.701 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:01.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:01.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:01.803 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:01.825 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:01.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:01.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:01.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:01 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/437659387' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:06:01.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:01 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:06:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:01 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/437659387' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:06:01.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:01 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:06:01.927 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:01.966 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:01.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:01.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:02.067 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:02.089 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:02.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:02.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:02.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:01 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/437659387' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:06:02.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:01 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:06:02.190 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:02.213 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:02.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:02.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:02.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:02.361 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:02.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:02.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:02.463 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:02.505 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:02.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:02.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:02.607 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:02.649 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:02.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:02.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:02 smithi130 ceph-mon[105521]: pgmap v2261: 89 pgs: 20 active+undersized, 9 stale+active+clean, 12 active+undersized+degraded, 48 active+clean; 4.6 GiB data, 17 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 530 op/s; 23303/225444 objects degraded (10.336%) 2024-08-17T18:06:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:02 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:06:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:02 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:06:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:02 smithi130 ceph-mon[105521]: osdmap e1319: 8 total, 7 up, 5 in 2024-08-17T18:06:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:02 smithi130 ceph-mon[110068]: pgmap v2261: 89 pgs: 20 active+undersized, 9 stale+active+clean, 12 active+undersized+degraded, 48 active+clean; 4.6 GiB data, 17 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 530 op/s; 23303/225444 objects degraded (10.336%) 2024-08-17T18:06:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:02 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:06:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:02 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:06:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:02 smithi130 ceph-mon[110068]: osdmap e1319: 8 total, 7 up, 5 in 2024-08-17T18:06:02.751 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:02.788 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:02.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:02.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:02.890 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:02.911 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:02.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:02.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:03.013 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:03.034 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:03.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:03.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:03.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:02 smithi179 ceph-mon[93519]: pgmap v2261: 89 pgs: 20 active+undersized, 9 stale+active+clean, 12 active+undersized+degraded, 48 active+clean; 4.6 GiB data, 17 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 530 op/s; 23303/225444 objects degraded (10.336%) 2024-08-17T18:06:03.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:02 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:06:03.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:02 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:06:03.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:02 smithi179 ceph-mon[93519]: osdmap e1319: 8 total, 7 up, 5 in 2024-08-17T18:06:03.136 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:03.178 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:03.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:03.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:03.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:03.300 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:03.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:03.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:03.402 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:03.423 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:03.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:03.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:03.525 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:03.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:03.562 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:03.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:03.663 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:03.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:03.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:03.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:03.788 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:03.810 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:03.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:03.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:03.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:03.953 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:03.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:03.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:04.054 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:04.078 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:04.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:04.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:04.180 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:04.200 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:04.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:04.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:04.302 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:04.351 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:04.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:04.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:04.454 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:04.489 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:04.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:04.590 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:04.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:04.631 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:04.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:04.732 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:04.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:04.755 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:04.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:04.856 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:04.878 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:04.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:04.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:04.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:04 smithi130 ceph-mon[105521]: pgmap v2263: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 978 op/s; 40264/215340 objects degraded (18.698%) 2024-08-17T18:06:04.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:04 smithi130 ceph-mon[110068]: pgmap v2263: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 978 op/s; 40264/215340 objects degraded (18.698%) 2024-08-17T18:06:04.980 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:05.017 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:05.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:05.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:04 smithi179 ceph-mon[93519]: pgmap v2263: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 978 op/s; 40264/215340 objects degraded (18.698%) 2024-08-17T18:06:05.119 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:05.140 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:05.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:05.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:05.242 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:05.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:05.265 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:05.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:05.366 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:05.387 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:05.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:05.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:05.489 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:05.511 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:05.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:05.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:05.613 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:05.634 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:05.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:05.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:05.736 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:05.757 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:05.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:05.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:05.859 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:05.880 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:05.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:05.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:05.982 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:06.019 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:06.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:06.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:06.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:05 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 39300/210540 objects degraded (18.666%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:06.121 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:06.157 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:06.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:06.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:06.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:05 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 39300/210540 objects degraded (18.666%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:05 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 39300/210540 objects degraded (18.666%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:06.259 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:06.299 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:06.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:06.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:06.401 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:06.424 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:06.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:06.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:06.525 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:06.546 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:06.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:06.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:06.648 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:06.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:06.671 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:06.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:06.772 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:06.793 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:06.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:06.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:06.895 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:06.916 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:06.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:06.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:07.017 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:07.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:07.039 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:07.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:07.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:06 smithi179 ceph-mon[93519]: pgmap v2264: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.3 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 822 op/s; 39300/210540 objects degraded (18.666%) 2024-08-17T18:06:07.140 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:07.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:06 smithi130 ceph-mon[105521]: pgmap v2264: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.3 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 822 op/s; 39300/210540 objects degraded (18.666%) 2024-08-17T18:06:07.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:06 smithi130 ceph-mon[110068]: pgmap v2264: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.3 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 822 op/s; 39300/210540 objects degraded (18.666%) 2024-08-17T18:06:07.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:07.181 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:07.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:07.282 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:07.303 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:07.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:07.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:07.405 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:07.428 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:07.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:07.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:07.529 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:07.553 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:07.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:07.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:07.654 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:07.676 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:07.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:07.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:07.778 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:07.799 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:07.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:07.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:07.901 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:07.922 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:07.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:07.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:08.024 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:08.045 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:08.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:08.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:08.147 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:08.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:08.184 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:08.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:08.285 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:08.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:08.321 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:08.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:08.422 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:08.457 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:08.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:08.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:08.559 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:08.599 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:08.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:08.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:08.701 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:08.723 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:08.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:08.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:08.825 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:08.865 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:08.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:08.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:08.967 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:08.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:08.990 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:08.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:09.091 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:09.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:08 smithi179 ceph-mon[93519]: pgmap v2265: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 920 op/s; 38148/203409 objects degraded (18.754%) 2024-08-17T18:06:09.115 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:09.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:09.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:09.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:08 smithi130 ceph-mon[105521]: pgmap v2265: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 920 op/s; 38148/203409 objects degraded (18.754%) 2024-08-17T18:06:09.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:08 smithi130 ceph-mon[110068]: pgmap v2265: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.1 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 920 op/s; 38148/203409 objects degraded (18.754%) 2024-08-17T18:06:09.217 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:09.238 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:09.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:09.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:09.339 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:09.360 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:09.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:09.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:09.462 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:09.484 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:09.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:09.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:09.586 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:09.621 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:09.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:09.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:09.723 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:09.759 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:09.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:09.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:09.861 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:09.882 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:09.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:09.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:09.984 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:10.005 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:10.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:10.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:10.107 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:10.130 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:10.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:10.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:06:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:06:09] "GET /metrics HTTP/1.1" 200 36782 "" "Prometheus/2.51.0" 2024-08-17T18:06:10.231 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:10.267 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:10.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:10.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:10.368 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:10.389 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:10.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:10.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:10.491 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:10.526 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:10.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:10.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:06:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:06:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:06:10.627 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:10.665 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:10.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:10.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:10.767 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:10.789 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:10.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:10.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:10.890 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:10.911 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:10.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:10.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:11.013 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:11.036 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:11.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:11.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:10 smithi179 ceph-mon[93519]: pgmap v2266: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 970 op/s; 36672/196323 objects degraded (18.679%) 2024-08-17T18:06:11.138 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:11.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:10 smithi130 ceph-mon[105521]: pgmap v2266: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 970 op/s; 36672/196323 objects degraded (18.679%) 2024-08-17T18:06:11.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:10 smithi130 ceph-mon[110068]: pgmap v2266: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 970 op/s; 36672/196323 objects degraded (18.679%) 2024-08-17T18:06:11.173 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:11.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:11.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:11.275 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:11.297 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:11.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:11.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:11.399 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:11.439 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:11.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:11.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:11.541 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:11.563 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:11.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:11.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:11.665 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:11.705 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:11.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:11.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:11.780 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 36672/196323 objects degraded (18.679%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:11.780 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 36672/196323 objects degraded (18.679%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:11.808 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:11.832 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:11.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:11.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:11.867 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:06:11.867 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:06:11.906 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:06:11.907 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:06:11.934 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:11.936 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:06:11.937 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:06:11.942 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:06:11.955 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:11.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:11.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:11.970 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:06:11.970 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:06:11.997 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:06:11.998 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:06:12.027 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:06:12.027 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:06:12.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:12.062 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:06:12.063 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:06:12.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 36672/196323 objects degraded (18.679%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:12.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:12.111 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:12.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:12.212 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:12.233 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:12.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:12.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:12.335 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:12.357 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:12.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:12.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:12.417 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:06:12 smithi130 podman[315725]: 2024-08-17 18:06:12.176878376 +0000 UTC m=+11.823257869 container init 3c8e46e0649d25f41fe3ff31d4572d05e9800e5654da50109586ebae49ffe6e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T18:06:12.417 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:06:12 smithi130 podman[315725]: 2024-08-17 18:06:12.194196447 +0000 UTC m=+11.840575937 container start 3c8e46e0649d25f41fe3ff31d4572d05e9800e5654da50109586ebae49ffe6e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-08-17T18:06:12.442 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:06:12.442 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:06:12.458 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:12.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:12.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:12.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:12.582 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:12.603 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:12.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:12.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:12.705 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:12.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:12.745 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:12.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:12.847 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:12.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:12.869 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:12.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:12.970 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:13.009 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:13.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:13.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:12 smithi179 ceph-mon[93519]: pgmap v2267: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 970 op/s; 36672/196323 objects degraded (18.679%) 2024-08-17T18:06:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:12 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2717596696' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:06:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:06:13.111 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:13.133 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:13.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:13.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:12 smithi130 ceph-mon[105521]: pgmap v2267: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 970 op/s; 36672/196323 objects degraded (18.679%) 2024-08-17T18:06:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:12 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2717596696' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:06:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:06:13.167 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:06:12 smithi130 conmon[316559]: conmon 3c8e46e0649d25f41fe3 : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-3c8e46e0649d25f41fe3ff31d4572d05e9800e5654da50109586ebae49ffe6e1.scope/memory.events 2024-08-17T18:06:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:12 smithi130 ceph-mon[110068]: pgmap v2267: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 4.0 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 970 op/s; 36672/196323 objects degraded (18.679%) 2024-08-17T18:06:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:12 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2717596696' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:06:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:13.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:06:13.234 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:13.270 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:13.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:13.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:13.394 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:13.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:13.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:13.496 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:13.517 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:13.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:13.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:13.619 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:13.636 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:06:13.640 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:13.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:13.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:13.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:06:13 smithi130 podman[315725]: 2024-08-17 18:06:13.359681435 +0000 UTC m=+13.006060918 container attach 3c8e46e0649d25f41fe3ff31d4572d05e9800e5654da50109586ebae49ffe6e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=) 2024-08-17T18:06:13.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:06:13 smithi130 podman[315725]: 2024-08-17 18:06:13.360010624 +0000 UTC m=+13.006390115 container died 3c8e46e0649d25f41fe3ff31d4572d05e9800e5654da50109586ebae49ffe6e1 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0) 2024-08-17T18:06:13.741 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:13.764 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:13.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:13.865 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:13.886 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:13.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:13.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:13.988 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:14.011 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:14.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:14.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:14.113 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:14.153 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:14.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:14.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:14.255 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:14.276 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:14.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:14.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:14.359 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:06:14.359 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:06:14.380 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:14.401 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:14.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:14.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:14.503 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:14.738 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:14.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:14.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:14.809 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.3.service' 2024-08-17T18:06:14.840 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:14.880 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:06:14.880 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-08-17T18:06:14.898 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:14.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:14.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:15.000 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:15.007 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:14 smithi130 ceph-mon[105521]: pgmap v2268: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.12k op/s; 34854/186222 objects degraded (18.716%) 2024-08-17T18:06:15.008 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:14 smithi130 ceph-mon[110068]: pgmap v2268: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.12k op/s; 34854/186222 objects degraded (18.716%) 2024-08-17T18:06:15.021 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:15.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:15.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:15.048 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T18:06:15.085 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:14 smithi179 ceph-mon[93519]: pgmap v2268: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.12k op/s; 34854/186222 objects degraded (18.716%) 2024-08-17T18:06:15.122 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:15.144 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:15.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:15.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:15.222 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:06:15.246 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:15.270 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:15.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:15.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:15.344 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:06:15.372 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:15.394 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:15.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:15.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:15.495 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:15.518 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:15.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:15.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:15.620 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:15.642 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:15.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:15.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:15.744 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:15.790 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:15.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:15.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:15.891 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:15.924 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:15.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:15.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:16.026 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:16.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:16.048 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:16.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:16.149 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:16.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:16.172 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:16.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:16.273 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:16.296 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:16.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:16.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:16.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:06:16.398 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:16.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:16.448 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:16.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:16.549 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:16.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:16.596 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:16.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:16.697 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:16.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:16.779 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:16.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:16.845 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:06:16.880 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:16.903 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:16.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:16.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:16.944 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:16 smithi130 ceph-mon[105521]: pgmap v2269: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 942 op/s; 33846/181380 objects degraded (18.660%) 2024-08-17T18:06:16.944 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 33846/181380 objects degraded (18.660%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:16.944 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:16 smithi130 ceph-mon[110068]: pgmap v2269: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 942 op/s; 33846/181380 objects degraded (18.660%) 2024-08-17T18:06:16.944 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 33846/181380 objects degraded (18.660%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:17.002 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:16 smithi179 ceph-mon[93519]: pgmap v2269: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 942 op/s; 33846/181380 objects degraded (18.660%) 2024-08-17T18:06:17.002 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 33846/181380 objects degraded (18.660%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:17.004 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:17.026 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:17.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:17.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:17.128 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:17.151 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:17.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:17.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:17.252 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:17.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:17.299 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:17.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:17.400 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:17.404 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:06:17.404 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:06:17.424 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:17.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:17.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:17.526 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:17.547 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:17.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:17.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:17.649 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:17.671 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:17.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:17.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:17.772 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:17.812 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:17.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:17.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:17.914 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:17.937 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:17.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:17.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:18.039 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:18.060 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:18.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:18.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:18.162 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:18.183 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:18.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:18.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:18.284 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:18.305 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:18.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:18.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:18.376 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T18:06:18.406 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:18.440 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:18.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:18.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:18.542 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:18.564 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:18.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:18.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:18.649 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:06:18.666 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:18.707 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:18.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:18.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:18.745 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:06:18.809 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:18.855 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:18.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:18.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:18.957 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:18.979 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:18.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:18.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:19.071 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:18 smithi179 ceph-mon[93519]: pgmap v2270: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 32569/173637 objects degraded (18.757%) 2024-08-17T18:06:19.071 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/567093565' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:06:19.072 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:18 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:06:19.082 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:19.122 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:19.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:19.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:19.128 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:18 smithi130 ceph-mon[110068]: pgmap v2270: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 32569/173637 objects degraded (18.757%) 2024-08-17T18:06:19.128 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/567093565' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:06:19.128 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:18 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:06:19.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:18 smithi130 ceph-mon[105521]: pgmap v2270: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.5 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 32569/173637 objects degraded (18.757%) 2024-08-17T18:06:19.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/567093565' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:06:19.129 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:18 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:06:19.223 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:19.245 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:19.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:19.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:19.347 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:19.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:19.386 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:19.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:19.487 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:19.520 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:19.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:19.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:19.622 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:19.657 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:19.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:19.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:19.759 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:19.794 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:19.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:19.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:19 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:06:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:19 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:06:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:19 smithi179 ceph-mon[93519]: osdmap e1320: 8 total, 7 up, 5 in 2024-08-17T18:06:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4100447463' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:06:19.880 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 7] out_osds: [5, 4, 6] dead_osds: [3] live_osds: [1, 2, 4, 5, 6, 7, 0] 2024-08-17T18:06:19.880 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:06:19.880 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-08-17T18:06:19.881 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.0 'trim stale osdmaps' 2024-08-17T18:06:19.896 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:19.979 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:06:19.979 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:06:19.979 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:06:19.979 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:06:19.979 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:06:19.979 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:06:19.979 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:06:19.980 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:06:19.980 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:06:19.980 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:06:19.980 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:06:19.980 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:06:19.980 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:06:19.980 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:06:19.980 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:06:19.980 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:06:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:06:19] "GET /metrics HTTP/1.1" 200 36789 "" "Prometheus/2.51.0" 2024-08-17T18:06:19.981 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:19 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:06:19.981 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:19 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:06:19.981 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:19 smithi130 ceph-mon[105521]: osdmap e1320: 8 total, 7 up, 5 in 2024-08-17T18:06:19.981 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4100447463' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:06:19.981 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:19 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:06:19.981 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:19 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:06:19.981 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:19 smithi130 ceph-mon[110068]: osdmap e1320: 8 total, 7 up, 5 in 2024-08-17T18:06:19.981 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4100447463' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:06:19.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:19.986 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:19.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:20.087 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:20.109 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:20.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:20.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:20.211 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:20.233 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:20.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:20.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:20.335 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:20.357 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:20.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:20.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:20.459 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:20.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:20.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:20.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:20.487 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:06:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:06:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:06:20.517 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:06:20.517 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:06:20.517 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:06:20.517 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:06:20.517 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:06:20.582 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:20.615 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:20.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:20.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:20.672 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:06:20.673 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:06:20.716 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:20.738 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:20.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:20.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:20.782 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:06:20.840 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:20.862 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:20.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:20 smithi130 ceph-mon[105521]: pgmap v2272: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 30871/165693 objects degraded (18.631%) 2024-08-17T18:06:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:20 smithi130 ceph-mon[110068]: pgmap v2272: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 30871/165693 objects degraded (18.631%) 2024-08-17T18:06:20.964 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:21.006 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:21.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:21.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:20 smithi179 ceph-mon[93519]: pgmap v2272: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 30871/165693 objects degraded (18.631%) 2024-08-17T18:06:21.108 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:21.148 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:21.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:21.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:21.250 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:21.272 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:21.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:21.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:21.345 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T18:06:21.373 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:21.411 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:21.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:21.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:21.513 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:21.535 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:21.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:21.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:21.636 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:06:21.637 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:21.685 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:21.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:21.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:21.787 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:21.802 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 30871/165693 objects degraded (18.631%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:21.811 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:21.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:21.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:21.910 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:06:21.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:22.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:22.019 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:22.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:22.120 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:22.142 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:22.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:22.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:22.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 30871/165693 objects degraded (18.631%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:22.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 30871/165693 objects degraded (18.631%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:22.244 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:22.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:22.304 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:22.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:22.405 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:22.443 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:22.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:22.545 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:22.587 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:22.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:22.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:22.674 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:06:22.689 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:22.762 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:22.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:22.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:22 smithi179 ceph-mon[93519]: pgmap v2273: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 30871/165693 objects degraded (18.631%) 2024-08-17T18:06:22.864 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:22.900 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:22.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:22.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:23.002 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:23.025 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:23.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:23.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:23.041 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:22 smithi130 ceph-mon[105521]: pgmap v2273: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 30871/165693 objects degraded (18.631%) 2024-08-17T18:06:23.042 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:22 smithi130 ceph-mon[110068]: pgmap v2273: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.4 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 30871/165693 objects degraded (18.631%) 2024-08-17T18:06:23.050 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:06:23.127 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:23.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:23.149 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:23.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:23.150 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:06:23.150 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:06:23.150 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:06:23.150 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:06:23.151 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:06:23.151 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:06:23.151 INFO:teuthology.orchestra.run.smithi130.stderr:bench [] [] [] [] 2024-08-17T18:06:23.151 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:06:23.151 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:06:23.151 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:06:23.151 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator dump block 2024-08-17T18:06:23.151 INFO:teuthology.orchestra.run.smithi130.stderr:Error EINVAL: invalid command 2024-08-17T18:06:23.251 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:23.288 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:23.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:23.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:23.390 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:23.424 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:23.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:23.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:23.505 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:06:23.506 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T18:06:23.506 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T18:06:23.506 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T18:06:23.506 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.506 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.45158.0:201692 45.b e1247/1188 45:dcb20113:::benchmark_data_smithi130_172551_object96885:head v 1247'12602, mlcod=1170'2553)", 2024-08-17T18:06:23.506 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:58:45.833937+0000", 2024-08-17T18:06:23.506 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 457.65736224900002, 2024-08-17T18:06:23.506 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0060070829999999999, 2024-08-17T18:06:23.506 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:23.506 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:23.506 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:23.507 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.507 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:23.507 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:45.833937+0000", 2024-08-17T18:06:23.507 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:23.507 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.507 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.507 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:23.507 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:45.833937+0000", 2024-08-17T18:06:23.507 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:23.507 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.507 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.507 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:23.507 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:45.833936+0000", 2024-08-17T18:06:23.508 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:06:23.508 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.508 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.508 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:23.508 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:45.833942+0000", 2024-08-17T18:06:23.508 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0619999999999997e-06 2024-08-17T18:06:23.508 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.508 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.508 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:23.508 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:45.833955+0000", 2024-08-17T18:06:23.508 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3097e-05 2024-08-17T18:06:23.508 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.508 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.509 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:23.509 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:45.833961+0000", 2024-08-17T18:06:23.509 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7169999999999996e-06 2024-08-17T18:06:23.509 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.509 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.509 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:23.509 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:45.835811+0000", 2024-08-17T18:06:23.509 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0018503320000000001 2024-08-17T18:06:23.509 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.509 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.509 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:23.509 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:45.835835+0000", 2024-08-17T18:06:23.509 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3536e-05 2024-08-17T18:06:23.509 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.510 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.510 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:23.510 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:45.839905+0000", 2024-08-17T18:06:23.510 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0040697779999999996 2024-08-17T18:06:23.510 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.510 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.510 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:23.510 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:45.839944+0000", 2024-08-17T18:06:23.510 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9218000000000003e-05 2024-08-17T18:06:23.510 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.510 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:23.510 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.510 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.511 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.511 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.45158.0:201940 45.b e1247/1188 45:defd796d:::benchmark_data_smithi130_172551_object97133:head v 1247'12621, mlcod=1170'2553)", 2024-08-17T18:06:23.511 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:58:46.207541+0000", 2024-08-17T18:06:23.511 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 457.28375758099997, 2024-08-17T18:06:23.511 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0095952429999999998, 2024-08-17T18:06:23.511 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:23.511 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:23.511 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:23.511 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.511 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:23.511 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.207541+0000", 2024-08-17T18:06:23.511 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:23.511 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.511 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.207541+0000", 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.207540+0000", 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.207552+0000", 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1817999999999999e-05 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.512 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.207573+0000", 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.1675000000000002e-05 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.207581+0000", 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9670000000000001e-06 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.207627+0000", 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5955999999999998e-05 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.513 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.514 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:23.514 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.207654+0000", 2024-08-17T18:06:23.514 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.73e-05 2024-08-17T18:06:23.514 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.514 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.514 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:23.514 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.217108+0000", 2024-08-17T18:06:23.514 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0094533889999999995 2024-08-17T18:06:23.514 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.514 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.514 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:23.514 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.217137+0000", 2024-08-17T18:06:23.514 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8721000000000002e-05 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.45158.0:201941 45.b e1247/1188 45:db14583b:::benchmark_data_smithi130_172551_object97134:head v 1247'12622, mlcod=1170'2553)", 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:58:46.207594+0000", 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 457.28370477499999, 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0095661740000000002, 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:23.515 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.207594+0000", 2024-08-17T18:06:23.516 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:23.516 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.516 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.516 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:23.516 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.207594+0000", 2024-08-17T18:06:23.516 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:23.516 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.516 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.516 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:23.516 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.207594+0000", 2024-08-17T18:06:23.516 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:06:23.516 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.516 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.516 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:23.517 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.207599+0000", 2024-08-17T18:06:23.517 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9910000000000002e-06 2024-08-17T18:06:23.517 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.517 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.517 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:23.517 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.207606+0000", 2024-08-17T18:06:23.517 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.126e-06 2024-08-17T18:06:23.517 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.517 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.517 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:23.517 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.207610+0000", 2024-08-17T18:06:23.517 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.42e-06 2024-08-17T18:06:23.517 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.517 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.209989+0000", 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002379206 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.210012+0000", 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.2736999999999999e-05 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.217141+0000", 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0071284920000000002 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.518 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.519 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:23.519 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:46.217160+0000", 2024-08-17T18:06:23.519 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.9570999999999999e-05 2024-08-17T18:06:23.519 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.519 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:23.519 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.519 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.519 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.519 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.45158.0:203377 45.b e1247/1188 45:d2f3e8c7:::benchmark_data_smithi130_172551_object98570:head v 1247'12714, mlcod=1170'2553)", 2024-08-17T18:06:23.519 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:58:48.486378+0000", 2024-08-17T18:06:23.519 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 455.00492053400001, 2024-08-17T18:06:23.519 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0053568030000000003, 2024-08-17T18:06:23.519 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:23.520 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:23.520 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:23.520 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.520 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:23.520 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:48.486378+0000", 2024-08-17T18:06:23.520 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:23.520 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.520 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.520 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:23.520 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:48.486378+0000", 2024-08-17T18:06:23.520 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:23.520 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.520 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.521 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:23.521 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:48.486377+0000", 2024-08-17T18:06:23.521 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:06:23.521 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.521 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.521 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:23.521 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:48.486387+0000", 2024-08-17T18:06:23.521 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.8840000000000006e-06 2024-08-17T18:06:23.521 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.521 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.521 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:23.521 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:48.486403+0000", 2024-08-17T18:06:23.521 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5860999999999999e-05 2024-08-17T18:06:23.521 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.522 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.522 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:23.522 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:48.486410+0000", 2024-08-17T18:06:23.522 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.1579999999999999e-06 2024-08-17T18:06:23.522 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.522 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.522 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:23.522 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:48.486463+0000", 2024-08-17T18:06:23.522 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2927000000000003e-05 2024-08-17T18:06:23.522 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.522 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.522 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:23.522 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:48.486496+0000", 2024-08-17T18:06:23.523 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3311e-05 2024-08-17T18:06:23.523 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.523 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.523 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:23.523 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:48.491691+0000", 2024-08-17T18:06:23.523 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0051950319999999996 2024-08-17T18:06:23.523 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.523 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.523 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:23.523 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:48.491735+0000", 2024-08-17T18:06:23.523 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3961000000000001e-05 2024-08-17T18:06:23.523 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.523 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:23.523 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.45158.0:205773 45.b e1247/1188 45:d3eb395b:::benchmark_data_smithi130_172551_object100966:head v 1247'12863, mlcod=1170'2553)", 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:58:52.736931+0000", 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 450.75436792800002, 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0041263749999999998, 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:52.736931+0000", 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.524 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.525 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:23.525 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:52.736931+0000", 2024-08-17T18:06:23.525 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:23.525 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.525 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.525 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:23.525 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:52.736930+0000", 2024-08-17T18:06:23.525 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:06:23.525 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.525 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.525 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:23.525 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:52.736940+0000", 2024-08-17T18:06:23.525 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.8179999999999999e-06 2024-08-17T18:06:23.525 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.526 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.526 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:23.526 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:52.736958+0000", 2024-08-17T18:06:23.526 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.8233999999999999e-05 2024-08-17T18:06:23.526 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.526 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.526 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:23.526 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:52.736965+0000", 2024-08-17T18:06:23.526 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.525e-06 2024-08-17T18:06:23.526 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.526 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.526 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:23.526 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:52.737003+0000", 2024-08-17T18:06:23.527 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7895000000000002e-05 2024-08-17T18:06:23.527 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.527 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.527 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:23.527 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:52.737022+0000", 2024-08-17T18:06:23.527 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.876e-05 2024-08-17T18:06:23.527 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.527 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.527 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:23.527 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:52.741035+0000", 2024-08-17T18:06:23.527 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0040126770000000001 2024-08-17T18:06:23.527 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:52.741057+0000", 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.2688e-05 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.45158.0:207687 45.b e1248/1188 45:df5822ef:::benchmark_data_smithi130_172551_object102880:head v 1248'12988, mlcod=1170'2553)", 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:58:55.679510+0000", 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 447.81178924400001, 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0041219159999999998, 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:23.528 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:23.529 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:23.529 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.529 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:23.529 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:55.679510+0000", 2024-08-17T18:06:23.529 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:23.529 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.529 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.529 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:23.529 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:55.679510+0000", 2024-08-17T18:06:23.529 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:23.529 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.529 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.529 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:23.529 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:55.679508+0000", 2024-08-17T18:06:23.530 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:06:23.530 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.530 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.530 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:23.530 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:55.679517+0000", 2024-08-17T18:06:23.530 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.5930000000000006e-06 2024-08-17T18:06:23.530 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.530 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.530 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:23.530 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:55.679532+0000", 2024-08-17T18:06:23.530 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5045000000000001e-05 2024-08-17T18:06:23.530 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.530 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.530 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:23.531 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:55.679538+0000", 2024-08-17T18:06:23.531 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1149999999999999e-06 2024-08-17T18:06:23.531 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.531 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.531 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:23.531 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:55.679573+0000", 2024-08-17T18:06:23.531 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.5380999999999999e-05 2024-08-17T18:06:23.531 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.531 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.531 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:23.531 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:55.679590+0000", 2024-08-17T18:06:23.531 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.7027000000000001e-05 2024-08-17T18:06:23.531 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.531 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:55.683597+0000", 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0040070640000000003 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:55.683632+0000", 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4153000000000003e-05 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.45158.0:208333 45.b e1248/1188 45:da1f24d8:::benchmark_data_smithi130_172551_object103526:head v 1248'13037, mlcod=1170'2553)", 2024-08-17T18:06:23.532 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:58:56.630862+0000", 2024-08-17T18:06:23.533 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 446.86043729900001, 2024-08-17T18:06:23.533 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0039514169999999996, 2024-08-17T18:06:23.533 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:23.533 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:23.533 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:23.533 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.533 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:23.533 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:56.630862+0000", 2024-08-17T18:06:23.533 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:23.533 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.533 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.533 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:23.533 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:56.630862+0000", 2024-08-17T18:06:23.533 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:23.534 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.534 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.534 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:23.534 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:56.630860+0000", 2024-08-17T18:06:23.534 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:06:23.534 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.534 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.534 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:23.534 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:56.630871+0000", 2024-08-17T18:06:23.534 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0237999999999999e-05 2024-08-17T18:06:23.534 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.534 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.535 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:23.535 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:56.630891+0000", 2024-08-17T18:06:23.535 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0310999999999999e-05 2024-08-17T18:06:23.535 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.535 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.535 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:23.535 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:56.630897+0000", 2024-08-17T18:06:23.535 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6320000000000002e-06 2024-08-17T18:06:23.535 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.535 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.535 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:23.535 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:56.630943+0000", 2024-08-17T18:06:23.535 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5661000000000002e-05 2024-08-17T18:06:23.535 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.536 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.536 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:23.536 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:56.630969+0000", 2024-08-17T18:06:23.536 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5795e-05 2024-08-17T18:06:23.536 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.536 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.536 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:23.536 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:56.634769+0000", 2024-08-17T18:06:23.536 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003800005 2024-08-17T18:06:23.536 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:23.536 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:23.536 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:23.536 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:58:56.634813+0000", 2024-08-17T18:06:23.537 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4051999999999998e-05 2024-08-17T18:06:23.537 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.537 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:23.537 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.537 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:23.537 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:23.537 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:06:23.537 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:23.561 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:23.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:23.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:23.663 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:23.706 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:23.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:23.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:23.808 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:23.845 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:23.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:23.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:23.946 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:23.980 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:23.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:23.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:24.082 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:24.125 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:24.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:24.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:24.227 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:24.230 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:06:24.231 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T18:06:24.231 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.1 'trim stale osdmaps' 2024-08-17T18:06:24.265 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:24.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:24.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:24.336 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T18:06:24.368 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:24.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:24.402 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:24.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:24.504 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:24.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:24.555 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:24.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:24.618 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:06:24.656 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:24.676 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:06:24.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:24.695 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:24.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:24.796 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:24.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:24.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:24.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:24.936 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:24.977 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:24.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:24.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:25.056 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:24 smithi179 ceph-mon[93519]: pgmap v2274: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 29004/155331 objects degraded (18.672%) 2024-08-17T18:06:25.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:25.101 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:24 smithi130 ceph-mon[105521]: pgmap v2274: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 29004/155331 objects degraded (18.672%) 2024-08-17T18:06:25.101 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:24 smithi130 ceph-mon[110068]: pgmap v2274: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.2 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 29004/155331 objects degraded (18.672%) 2024-08-17T18:06:25.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:25.102 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:25.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:25.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:25.225 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:25.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:25.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:25.326 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:25.348 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:25.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:25.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:25.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:25.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:25.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:25.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:25.572 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:25.593 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:25.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:25.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:25.695 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:25.716 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:25.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:25.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:25.818 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:25.839 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:25.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:25.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:25.941 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:25.974 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:25.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:25.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:26.076 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:26.099 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:26.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:26.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:26.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:26.241 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:26.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:26.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:26.343 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:26.365 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:26.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:26.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:26.407 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:06:26.467 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:26.480 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:06:26.480 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:06:26.480 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T18:06:26.481 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:06:26.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:26.510 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:26.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:26.612 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:26.633 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:26.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:26.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:26.735 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:26.757 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:26.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:26.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:26.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:26 smithi130 ceph-mon[105521]: pgmap v2275: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 28055/150510 objects degraded (18.640%) 2024-08-17T18:06:26.806 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 28055/150510 objects degraded (18.640%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:26.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:26 smithi130 ceph-mon[110068]: pgmap v2275: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 28055/150510 objects degraded (18.640%) 2024-08-17T18:06:26.806 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 28055/150510 objects degraded (18.640%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:26.859 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:26.880 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:26.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:26.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:26.982 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:27.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:27.022 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:27.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:27.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:26 smithi179 ceph-mon[93519]: pgmap v2275: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 3.1 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 28055/150510 objects degraded (18.640%) 2024-08-17T18:06:27.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 28055/150510 objects degraded (18.640%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:27.122 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:27.144 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:27.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:27.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:27.228 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T18:06:27.246 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:27.292 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:27.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:27.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:27.394 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:27.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:27.464 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:27.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:27.502 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:06:27.565 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:27.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:27.602 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:27.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:27.703 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:27.726 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:27.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:27.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:27.800 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:06:27.827 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:27.849 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:27.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:27.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:27.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:27 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1463354350' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:06:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:27.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:06:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:27 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1463354350' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:06:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:27.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:06:27.948 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:27 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1463354350' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:06:27.948 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:27.948 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:06:27.950 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:27.984 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:27.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:27.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:28.086 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:28.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:28.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:28.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:28.209 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:28.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:28.246 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:28.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:28.347 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:28.369 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:28.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:28.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:28.470 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:28.508 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:28.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:28.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:28.610 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:28.632 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:28.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:28.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:28.733 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:28.756 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:28.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:28.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:28.858 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:28.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:28.892 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:28.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:28.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:29.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:29.043 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:29.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:29.073 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:28 smithi179 ceph-mon[93519]: pgmap v2276: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 26821/143022 objects degraded (18.753%) 2024-08-17T18:06:29.073 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:28 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:06:29.073 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:28 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1463354350' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:06:29.073 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:28 smithi179 ceph-mon[93519]: osdmap e1321: 8 total, 7 up, 5 in 2024-08-17T18:06:29.073 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:06:29.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:28 smithi130 ceph-mon[105521]: pgmap v2276: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 26821/143022 objects degraded (18.753%) 2024-08-17T18:06:29.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:28 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:06:29.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:28 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1463354350' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:06:29.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:28 smithi130 ceph-mon[105521]: osdmap e1321: 8 total, 7 up, 5 in 2024-08-17T18:06:29.137 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:06:29.137 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:28 smithi130 ceph-mon[110068]: pgmap v2276: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 26821/143022 objects degraded (18.753%) 2024-08-17T18:06:29.137 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:28 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:06:29.138 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:28 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1463354350' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:06:29.138 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:28 smithi130 ceph-mon[110068]: osdmap e1321: 8 total, 7 up, 5 in 2024-08-17T18:06:29.138 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:06:29.144 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:29.190 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:29.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:29.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:29.292 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:29.315 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:29.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:29.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:29.339 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:06:29.339 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:06:29.339 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:06:29.339 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:06:29.339 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:06:29.342 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:06:29.342 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:06:29.384 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:06:29.384 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:06:29.417 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:29.428 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:06:29.429 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:06:29.430 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:06:29.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:29.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:29.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:29.457 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:06:29.458 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:06:29.485 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:06:29.485 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:06:29.528 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:06:29.528 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:06:29.540 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:29.561 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:29.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:29.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:29.569 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:06:29.569 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:06:29.663 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:29.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:29.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:29.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:29.786 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:29.807 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:29.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:29.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:29.909 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:29.930 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:29.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:29.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:30.032 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:30.044 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1959297612' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:06:30.069 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T18:06:30.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:30.070 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:30.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:30.111 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1959297612' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:06:30.111 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:06:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:06:29] "GET /metrics HTTP/1.1" 200 36787 "" "Prometheus/2.51.0" 2024-08-17T18:06:30.111 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1959297612' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:06:30.172 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:30.196 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:30.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:30.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:30.297 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:30.319 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:30.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:30.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:30.342 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:06:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:06:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:06:30.421 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:30.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:30.443 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:30.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:30.544 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:30.566 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:30.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:30.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:30.668 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:30.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:30.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:30.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:30.794 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:30.817 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:30.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:30.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:30.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:30.942 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:30.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:30.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:31.025 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:06:31.043 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:31.070 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:31.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:31.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:31.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:30 smithi179 ceph-mon[93519]: pgmap v2278: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 25271/135465 objects degraded (18.655%) 2024-08-17T18:06:31.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:30 smithi130 ceph-mon[105521]: pgmap v2278: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 25271/135465 objects degraded (18.655%) 2024-08-17T18:06:31.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:30 smithi130 ceph-mon[110068]: pgmap v2278: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 25271/135465 objects degraded (18.655%) 2024-08-17T18:06:31.171 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:31.198 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:31.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:31.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:31.299 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:31.323 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:31.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:31.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:31.425 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:31.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:31.448 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:31.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:31.549 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:31.585 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:31.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:31.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:31.687 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:31.722 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:31.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:31.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:31.824 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:31.846 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:31.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:31.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:31.948 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:31.970 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:31.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:31.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:32.072 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:32.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 25271/135465 objects degraded (18.655%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:32.107 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:32.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:32.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:32.147 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:06:32.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 25271/135465 objects degraded (18.655%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:32.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 25271/135465 objects degraded (18.655%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:32.209 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:32.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:32.264 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:32.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:32.365 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:32.386 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:32.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:32.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:32.488 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:32.533 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:32.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:32.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:32.635 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:32.668 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:32.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:32.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:32.770 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:32.806 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:32.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:32.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:32.907 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:32.929 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:32.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:32.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:33.030 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:33.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:33.053 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:33.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:33.075 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:32 smithi179 ceph-mon[93519]: pgmap v2279: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 25271/135465 objects degraded (18.655%) 2024-08-17T18:06:33.154 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:33.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:32 smithi130 ceph-mon[105521]: pgmap v2279: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 25271/135465 objects degraded (18.655%) 2024-08-17T18:06:33.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:32 smithi130 ceph-mon[110068]: pgmap v2279: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.01k op/s; 25271/135465 objects degraded (18.655%) 2024-08-17T18:06:33.185 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:33.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:33.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:33.287 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:33.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:33.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:33.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:33.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:33.440 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:33.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:33.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:33.525 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:06:33.525 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:06:33.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:33.583 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:33.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:33.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:33.685 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:33.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:33.708 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:33.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:33.809 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:33.849 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:33.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:33.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:33.951 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:33.972 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:33.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:34.074 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:34.113 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:34.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:34.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:34.215 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:34.237 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:34.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:34.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:34.256 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T18:06:34.338 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:34.360 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:34.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:34.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:34.462 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:34.483 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:34.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:34.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:34.585 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:34.607 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:34.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:34.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:34.709 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:34.731 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:34.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:34.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:34.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:34.870 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:34.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:34.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:34.971 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:35.005 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:35.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:35.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:35.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:34 smithi179 ceph-mon[93519]: pgmap v2280: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.00k op/s; 23370/125184 objects degraded (18.669%) 2024-08-17T18:06:35.097 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:06:35.108 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:35.113 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:34 smithi130 ceph-mon[105521]: pgmap v2280: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.00k op/s; 23370/125184 objects degraded (18.669%) 2024-08-17T18:06:35.113 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:34 smithi130 ceph-mon[110068]: pgmap v2280: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.5 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.00k op/s; 23370/125184 objects degraded (18.669%) 2024-08-17T18:06:35.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:35.148 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:35.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:35.249 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:35.285 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:35.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:35.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:35.387 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:35.410 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:35.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:35.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:35.512 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:35.553 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:35.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:35.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:35.654 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:35.676 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:35.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:35.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:35.778 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:35.802 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:35.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:35.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:35.904 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:35.926 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:35.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:35.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:36.028 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:36.050 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:36.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:36.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:36.152 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:36.174 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:36.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:36.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:36.276 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:36.309 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:36.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:36.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:36.410 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:36.432 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:36.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:36.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:36.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:36.567 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:36.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:36.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:36.668 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:36.689 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:36.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:36.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:36.791 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:36.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:36.830 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:36.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:36.931 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:36.956 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:06:36.956 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:06:37.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:37.024 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:37.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:36 smithi179 ceph-mon[93519]: pgmap v2281: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 997 op/s; 22477/120567 objects degraded (18.643%) 2024-08-17T18:06:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 22477/120567 objects degraded (18.643%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:37.124 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:37.126 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:36 smithi130 ceph-mon[110068]: pgmap v2281: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 997 op/s; 22477/120567 objects degraded (18.643%) 2024-08-17T18:06:37.126 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 22477/120567 objects degraded (18.643%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:37.127 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:36 smithi130 ceph-mon[105521]: pgmap v2281: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.5 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 997 op/s; 22477/120567 objects degraded (18.643%) 2024-08-17T18:06:37.127 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 22477/120567 objects degraded (18.643%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:37.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:37.171 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:37.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:37.272 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:37.293 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:37.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:37.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:37.395 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:37.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:37.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:37.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:37.518 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:37.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:37.543 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:37.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:37.587 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:06:37.644 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:37.665 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:37.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:37.671 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:06:37.671 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:06:37.671 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:06:37.671 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:06:37.671 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:06:37.671 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:06:37.671 INFO:teuthology.orchestra.run.smithi130.stderr:bench [] [] [] [] 2024-08-17T18:06:37.671 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:06:37.671 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:06:37.671 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:06:37.671 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator dump block 2024-08-17T18:06:37.672 INFO:teuthology.orchestra.run.smithi130.stderr:Error EINVAL: invalid command 2024-08-17T18:06:37.767 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:37.791 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:37.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:37.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:37.893 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:37.899 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:06:37.900 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:06:37.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:37.940 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:37.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:38.041 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:38.089 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:38.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:38.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:38.190 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:38.212 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:38.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:38.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:38.314 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:38.336 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:38.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:38.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:38.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:38.458 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:38.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:38.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:38.560 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:38.600 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:38.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:38.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:38.702 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:38.737 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:38.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:38.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:38.839 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:38.843 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T18:06:38.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:38.862 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:38.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:38.957 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:06:38.963 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:39.044 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:39.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:39.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:38 smithi179 ceph-mon[93519]: pgmap v2282: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 999 op/s; 21255/113019 objects degraded (18.807%) 2024-08-17T18:06:39.146 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:39.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:38 smithi130 ceph-mon[105521]: pgmap v2282: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 999 op/s; 21255/113019 objects degraded (18.807%) 2024-08-17T18:06:39.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:38 smithi130 ceph-mon[110068]: pgmap v2282: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 999 op/s; 21255/113019 objects degraded (18.807%) 2024-08-17T18:06:39.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:39.169 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:39.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:39.270 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:39.304 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:39.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:39.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:39.374 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:06:39.406 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:39.439 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:39.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:39.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:39.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:39.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:39.587 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:39.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:39.688 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:39.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:39.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:39.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:39.823 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:39.865 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:39.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:39.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:39.967 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:39.989 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:39.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:39.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:40.028 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:06:40.029 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T18:06:40.029 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-08-17T18:06:40.091 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:40.095 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:06:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:06:39] "GET /metrics HTTP/1.1" 200 36787 "" "Prometheus/2.51.0" 2024-08-17T18:06:40.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:40.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:40.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:40.215 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:40.236 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:40.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:40.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:40.338 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:40.392 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:40.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:40.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:40.494 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:40.530 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:40.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:40.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:40.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:40 smithi179 ceph-mon[93519]: pgmap v2283: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.09k op/s; 19630/105366 objects degraded (18.630%) 2024-08-17T18:06:40.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:40 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:40.595 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:40 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:40.595 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:06:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:06:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:06:40.631 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:40 smithi130 ceph-mon[105521]: pgmap v2283: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.09k op/s; 19630/105366 objects degraded (18.630%) 2024-08-17T18:06:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:40 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:40 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:40.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:40 smithi130 ceph-mon[110068]: pgmap v2283: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.09k op/s; 19630/105366 objects degraded (18.630%) 2024-08-17T18:06:40.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:40 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:40.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:40 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:40.682 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:40.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:40.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:40.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:40.824 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:40.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:40.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:40.926 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:40.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:40.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:40.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:41.078 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:41.118 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:41.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:41.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:41.220 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:41.255 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:41.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:41.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:41.293 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:06:41.294 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:06:41.356 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:41.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:41.379 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:41.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:41.480 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:41.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:41.521 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:41.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:41.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:41 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 19630/105366 objects degraded (18.630%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:41.622 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:41.644 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:41.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:41.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:41.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:41 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 19630/105366 objects degraded (18.630%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:41 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 19630/105366 objects degraded (18.630%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:41.746 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:41.767 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:41.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:41.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:41.869 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:41.891 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:41.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:41.993 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:42.015 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:42.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:42.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:42.019 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T18:06:42.117 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:42.138 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:42.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:42.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:42.240 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:42.263 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:42.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:42.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:42.334 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:06:42.365 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:42.387 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:42.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:42.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:42.489 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:42.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:42.523 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:42.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:42.624 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:42.630 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:42 smithi130 ceph-mon[105521]: pgmap v2284: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 835 op/s; 19630/105366 objects degraded (18.630%) 2024-08-17T18:06:42.630 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:42.630 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:06:42.630 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:42 smithi130 ceph-mon[110068]: pgmap v2284: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 835 op/s; 19630/105366 objects degraded (18.630%) 2024-08-17T18:06:42.630 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:42.630 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:06:42.635 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:42 smithi179 ceph-mon[93519]: pgmap v2284: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 835 op/s; 19630/105366 objects degraded (18.630%) 2024-08-17T18:06:42.635 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:06:42.635 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:06:42.666 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:42.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:42.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:42.768 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:42.790 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:42.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:42.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:42.891 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:42.913 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:42.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:42.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:42.977 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:06:43.015 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:43.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:43.069 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:43.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:43.098 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:06:43.170 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:43.196 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:43.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:43.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:43.298 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:43.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:43.320 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:43.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:43.402 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:06:43.421 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:43.442 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:43.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:43.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:43.544 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:43.566 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:43.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:43.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:43.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:43 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2460956985' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:06:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:43 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:06:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:43 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2460956985' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:06:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:43 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:06:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:43 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2460956985' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:06:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:43 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:06:43.668 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:43.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:43.707 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:43.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:43.808 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:43.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:43.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:43.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:43.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:43.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:43.970 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:43.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:44.071 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:44.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:44.113 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:44.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:44.214 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:44.252 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:44.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:44.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:44.354 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:44.392 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:44.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:44.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:44.494 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:44.497 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:06:44.497 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:06:44.497 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:44 smithi179 ceph-mon[93519]: pgmap v2285: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 1.11k op/s; 17862/95490 objects degraded (18.706%) 2024-08-17T18:06:44.497 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:44 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:06:44.498 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:06:44.498 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:44 smithi179 ceph-mon[93519]: osdmap e1322: 8 total, 7 up, 5 in 2024-08-17T18:06:44.498 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:44 smithi130 ceph-mon[105521]: pgmap v2285: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 1.11k op/s; 17862/95490 objects degraded (18.706%) 2024-08-17T18:06:44.498 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:44 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:06:44.498 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:06:44.498 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:44 smithi130 ceph-mon[105521]: osdmap e1322: 8 total, 7 up, 5 in 2024-08-17T18:06:44.499 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:44 smithi130 ceph-mon[110068]: pgmap v2285: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 1.11k op/s; 17862/95490 objects degraded (18.706%) 2024-08-17T18:06:44.499 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:44 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:06:44.499 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:06:44.499 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:44 smithi130 ceph-mon[110068]: osdmap e1322: 8 total, 7 up, 5 in 2024-08-17T18:06:44.516 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:44.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:44.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:44.618 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:44.639 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:44.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:44.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:44.741 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:44.763 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:44.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:44.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:44.864 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:44.887 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:44.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:44.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:44.988 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:45.010 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:45.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:45.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:45.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:45.134 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:45.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:45.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:45.235 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:45.260 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:45.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:45.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:45.362 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:45.370 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T18:06:45.384 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:45.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:45.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:45.486 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:45.508 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:45.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:45.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:45.610 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:45.651 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:45.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:45.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:45.660 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:06:45.753 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:45.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:45.776 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:45.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:45.876 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:45.898 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:45.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:45.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:46.000 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:46.046 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:46.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:46.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:46.147 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:46.169 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:46.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:46.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:46.270 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:46.325 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:46.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:46.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:46.426 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:46.447 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:46.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:46.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:46.533 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:46 smithi130 ceph-mon[105521]: pgmap v2287: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 980 op/s; 17003/91152 objects degraded (18.653%) 2024-08-17T18:06:46.533 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17003/91152 objects degraded (18.653%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:46.533 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:46 smithi130 ceph-mon[110068]: pgmap v2287: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 980 op/s; 17003/91152 objects degraded (18.653%) 2024-08-17T18:06:46.533 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17003/91152 objects degraded (18.653%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:46.549 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:46.570 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:46.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:46.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:46.671 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:46.709 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:46 smithi179 ceph-mon[93519]: pgmap v2287: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.9 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 980 op/s; 17003/91152 objects degraded (18.653%) 2024-08-17T18:06:46.709 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17003/91152 objects degraded (18.653%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:46.714 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:46.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:46.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:46.816 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:46.837 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:46.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:46.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:46.938 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:46.984 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:46.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:46.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:46.999 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:06:47.086 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:47.109 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:47.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:47.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:47.211 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:47.276 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:06:47.277 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:06:47.277 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:06:47.277 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:06:47.277 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:06:47.277 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:06:47.277 INFO:teuthology.orchestra.run.smithi130.stderr:bench [] [] [] [] 2024-08-17T18:06:47.277 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:06:47.277 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:06:47.277 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:06:47.277 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator dump block 2024-08-17T18:06:47.277 INFO:teuthology.orchestra.run.smithi130.stderr:Error EINVAL: invalid command 2024-08-17T18:06:47.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:47.282 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:47.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:47.383 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:47.407 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:47.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:47.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:47.508 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:47.540 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:47.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:47.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:47.626 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:47 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1557836890' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:06:47.642 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:47.652 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:06:47.652 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T18:06:47.652 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:47.652 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.46373.0:164885 47.f 47:fa538fff:::benchmark_data_smithi130_302447_object69650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T18:06:47.652 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:06:47.574283+0000", 2024-08-17T18:06:47.652 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.051379142000000003, 2024-08-17T18:06:47.652 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.051406034000000003, 2024-08-17T18:06:47.652 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:47.652 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:06:47.652 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:47.652 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.46373", 2024-08-17T18:06:47.653 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:06:47.653 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 164885 2024-08-17T18:06:47.653 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:47.653 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:47.653 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:47.653 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:47.653 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:06:47.574283+0000", 2024-08-17T18:06:47.653 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:47.653 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:47.653 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:47.653 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:47.653 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:06:47.574283+0000", 2024-08-17T18:06:47.653 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:47.653 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:47.654 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:47.654 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:47.654 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:06:47.574282+0000", 2024-08-17T18:06:47.654 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:06:47.654 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:47.654 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:47.654 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:47.654 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:06:47.574288+0000", 2024-08-17T18:06:47.654 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1840000000000004e-06 2024-08-17T18:06:47.654 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:47.654 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:47.654 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:47.654 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:06:47.574302+0000", 2024-08-17T18:06:47.655 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4073999999999999e-05 2024-08-17T18:06:47.655 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:47.655 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:47.655 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:47.655 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:06:47.574308+0000", 2024-08-17T18:06:47.655 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1580000000000002e-06 2024-08-17T18:06:47.655 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:47.655 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:47.655 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:47.655 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:06:47.574381+0000", 2024-08-17T18:06:47.655 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.2098e-05 2024-08-17T18:06:47.655 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:47.655 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:47.655 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:06:47.574462+0000", 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.1528999999999998e-05 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1", 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:06:47.574539+0000", 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.6846000000000003e-05 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:06:47.576876+0000", 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0023370230000000001 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:47.656 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:47.657 INFO:teuthology.orchestra.run.smithi179.stdout: ], 2024-08-17T18:06:47.657 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 1 2024-08-17T18:06:47.657 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:06:47.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:47.663 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:47.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:47.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:47 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1557836890' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:06:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:47 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1557836890' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:06:47.764 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:47.787 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:47.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:47.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:47.888 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:47.909 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:47.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:47.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:48.010 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:48.032 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:48.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:48.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:48.134 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:48.156 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:48.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:48.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:48.258 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:48.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:48.291 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:48.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:48.367 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T18:06:48.392 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:48.429 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:48.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:48.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:48.530 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:48.565 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:48.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:48.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:48.619 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:06:48.619 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:06:48.619 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:06:48.619 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:06:48.620 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:06:48.620 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:06:48.620 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:06:48.620 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:06:48.620 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:06:48.620 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:06:48.620 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:06:48.620 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:06:48.620 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:06:48.620 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:06:48.620 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:06:48.642 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:06:48.667 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:48.687 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:48.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:48.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:48.765 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:48 smithi179 ceph-mon[93519]: pgmap v2288: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.7 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 977 op/s; 15778/83691 objects degraded (18.853%) 2024-08-17T18:06:48.790 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:48.812 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:48.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:48.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:48.913 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:48.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:48 smithi130 ceph-mon[105521]: pgmap v2288: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.7 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 977 op/s; 15778/83691 objects degraded (18.853%) 2024-08-17T18:06:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:48 smithi130 ceph-mon[110068]: pgmap v2288: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.7 GiB data, 9.8 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 977 op/s; 15778/83691 objects degraded (18.853%) 2024-08-17T18:06:48.935 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:48.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:48.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:49.037 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:49.058 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:49.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:49.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:49.160 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:49.199 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:49.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:49.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:49.301 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:49.327 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:49.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:49.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:49.429 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:49.451 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:49.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:49.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:49.553 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:49.575 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:49.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:49.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:49.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:49.698 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:49.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:49.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:49.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:49.839 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:49.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:49.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:49.940 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:49.962 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:49.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:49.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:50.063 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:50.092 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:50.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:50.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:06:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:06:49] "GET /metrics HTTP/1.1" 200 36787 "" "Prometheus/2.51.0" 2024-08-17T18:06:50.194 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:50.216 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:50.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:50.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:50.318 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:50.340 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:50.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:50.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:50.442 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:50.464 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:50.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:50.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:50.565 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:50 smithi179 ceph-mon[93519]: pgmap v2289: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 966 op/s; 14265/76359 objects degraded (18.681%) 2024-08-17T18:06:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:06:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:06:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:06:50.594 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:50.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:50.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:50.696 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:50.717 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:50.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:50.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:50.819 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:50.840 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:50.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:50.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:50 smithi130 ceph-mon[105521]: pgmap v2289: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 966 op/s; 14265/76359 objects degraded (18.681%) 2024-08-17T18:06:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:50 smithi130 ceph-mon[110068]: pgmap v2289: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 966 op/s; 14265/76359 objects degraded (18.681%) 2024-08-17T18:06:50.942 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:50.963 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:50.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:50.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:51.065 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:51.109 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:51.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:51.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:51.211 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:51.232 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:51.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:51.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:51.334 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:51.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:51.357 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:51.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:51.458 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:51.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:51.498 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:51.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:51.506 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 14265/76359 objects degraded (18.681%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:51.562 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:06:51.562 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:06:51.599 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:51.623 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:51.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:51.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:51.724 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:51.760 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:51.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:51.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:51.846 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 14265/76359 objects degraded (18.681%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:51.847 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 14265/76359 objects degraded (18.681%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:51.862 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:51.899 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:51.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:51.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:51.909 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:06:51.910 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:06:51.910 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:06:51.910 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:06:51.910 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:06:52.000 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:52.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:52.035 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:52.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:52.136 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:52.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:52.194 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:52.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:52.202 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:06:52.202 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T18:06:52.203 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-08-17T18:06:52.241 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:06:52.295 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:52.364 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:52.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:52.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:52.466 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:52.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:52.517 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:52.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:52.559 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:52 smithi179 ceph-mon[93519]: pgmap v2290: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 966 op/s; 14265/76359 objects degraded (18.681%) 2024-08-17T18:06:52.618 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:52.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:52.642 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:52.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:52.685 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T18:06:52.743 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:52.750 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:52 smithi130 ceph-mon[105521]: pgmap v2290: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 966 op/s; 14265/76359 objects degraded (18.681%) 2024-08-17T18:06:52.750 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:52 smithi130 ceph-mon[110068]: pgmap v2290: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 966 op/s; 14265/76359 objects degraded (18.681%) 2024-08-17T18:06:52.765 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:52.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:52.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:52.867 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:52.904 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:52.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:52.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:52.960 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:06:53.005 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:53.043 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:53.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:53.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:53.145 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:53.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:53.167 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:53.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:53.268 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:53.290 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:53.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:53.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:53.392 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:53.414 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:53.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:53.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:53.516 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:53.537 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:53.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:53.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:53.563 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:06:53.640 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:53.661 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:53.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:53.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:53.722 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:06:53.763 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:53.784 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:53.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:53.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:53.886 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:53.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:53.908 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:53.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:54.009 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:54.031 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:54.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:54.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:54.132 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:54.153 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:54.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:54.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:54.255 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:54.279 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:54.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:54.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:54.380 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:54.426 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:54.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:54.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:54.528 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:54.551 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:54.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:54.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:54.652 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:54.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:54.674 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:54.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:54.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:54 smithi130 ceph-mon[105521]: pgmap v2291: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.4 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 926 op/s; 12686/67671 objects degraded (18.747%) 2024-08-17T18:06:54.702 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:54 smithi130 ceph-mon[110068]: pgmap v2291: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.4 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 926 op/s; 12686/67671 objects degraded (18.747%) 2024-08-17T18:06:54.775 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:54.784 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:54 smithi179 ceph-mon[93519]: pgmap v2291: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.4 GiB data, 9.1 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 926 op/s; 12686/67671 objects degraded (18.747%) 2024-08-17T18:06:54.797 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:06:54.797 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T18:06:54.797 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T18:06:54.797 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T18:06:54.797 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.797 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:30138 46.a 46:577d6c08:::benchmark_data_smithi130_267720_object30137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:06:54.797 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:48.638681+0000", 2024-08-17T18:06:54.797 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 426.14349376400003, 2024-08-17T18:06:54.797 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37615081299999997, 2024-08-17T18:06:54.797 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.797 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.798 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.798 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.798 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.798 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 30138 2024-08-17T18:06:54.798 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.798 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.798 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.798 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.798 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638681+0000", 2024-08-17T18:06:54.798 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.798 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.798 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.799 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.799 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638681+0000", 2024-08-17T18:06:54.799 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.799 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.799 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.799 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.799 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638680+0000", 2024-08-17T18:06:54.799 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:06:54.799 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.799 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.799 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.799 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638734+0000", 2024-08-17T18:06:54.799 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3424999999999998e-05 2024-08-17T18:06:54.800 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.800 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.800 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.800 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638742+0000", 2024-08-17T18:06:54.800 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2819999999999996e-06 2024-08-17T18:06:54.800 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.800 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.800 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.800 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638746+0000", 2024-08-17T18:06:54.800 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9740000000000004e-06 2024-08-17T18:06:54.800 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.800 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.800 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.800 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638787+0000", 2024-08-17T18:06:54.801 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0868999999999998e-05 2024-08-17T18:06:54.801 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.801 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.801 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.801 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638874+0000", 2024-08-17T18:06:54.801 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6833e-05 2024-08-17T18:06:54.801 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.801 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.801 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.801 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638957+0000", 2024-08-17T18:06:54.801 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3745000000000005e-05 2024-08-17T18:06:54.801 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.801 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.801 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.802 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.639982+0000", 2024-08-17T18:06:54.802 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001024107 2024-08-17T18:06:54.802 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.802 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.802 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.802 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.640316+0000", 2024-08-17T18:06:54.802 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00033452200000000002 2024-08-17T18:06:54.802 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.802 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.802 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.802 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:49.014750+0000", 2024-08-17T18:06:54.802 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37443375699999998 2024-08-17T18:06:54.802 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.803 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.803 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.803 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:49.014807+0000", 2024-08-17T18:06:54.803 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7263999999999999e-05 2024-08-17T18:06:54.803 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.803 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.803 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.803 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:49.014832+0000", 2024-08-17T18:06:54.803 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4921000000000001e-05 2024-08-17T18:06:54.803 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.803 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.803 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.803 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.804 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.804 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42530 46.e 46:7a5bb013:::benchmark_data_smithi130_267720_object42529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:06:54.804 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:06:54.804 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 417.77273856400001, 2024-08-17T18:06:54.804 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.037353389, 2024-08-17T18:06:54.804 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.804 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:06:54.804 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.804 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.804 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.804 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42530 2024-08-17T18:06:54.804 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.804 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.805 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.805 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.805 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:06:54.805 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.805 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.805 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.805 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.805 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:06:54.805 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.805 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.805 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.805 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.805 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:06:54.805 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009488+0000", 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2281999999999999e-05 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009508+0000", 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0250000000000001e-05 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009514+0000", 2024-08-17T18:06:54.806 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.161e-06 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009564+0000", 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0028999999999998e-05 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009573+0000", 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.9369999999999993e-06 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029341+0000", 2024-08-17T18:06:54.807 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0197678189999999 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for missing object", 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029434+0000", 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.3185999999999999e-05 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033201+0000", 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003766934 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for degraded object", 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033231+0000", 2024-08-17T18:06:54.808 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9657000000000001e-05 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.046712+0000", 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013481102 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.046790+0000", 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7853000000000002e-05 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.809 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.810 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42561 46.e 46:7b14be6d:::benchmark_data_smithi130_267720_object42560:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:06:54.810 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.039454+0000", 2024-08-17T18:06:54.810 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 417.74272103599998, 2024-08-17T18:06:54.810 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99762182099999996, 2024-08-17T18:06:54.810 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.810 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.810 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.810 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.810 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.810 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42561 2024-08-17T18:06:54.810 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.810 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.810 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.810 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.811 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039454+0000", 2024-08-17T18:06:54.811 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.811 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.811 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.811 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.811 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039454+0000", 2024-08-17T18:06:54.811 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.811 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.811 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.811 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.811 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039452+0000", 2024-08-17T18:06:54.811 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:06:54.811 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.811 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039516+0000", 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3046000000000006e-05 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039528+0000", 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2916e-05 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039534+0000", 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2179999999999998e-06 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.812 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039834+0000", 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00030059599999999998 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039844+0000", 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5840000000000001e-06 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029614+0000", 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98977001600000003 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.813 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.814 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.814 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029696+0000", 2024-08-17T18:06:54.814 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2041999999999998e-05 2024-08-17T18:06:54.814 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.814 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.814 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.814 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029809+0000", 2024-08-17T18:06:54.814 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000113503 2024-08-17T18:06:54.814 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.814 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.814 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.814 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033156+0000", 2024-08-17T18:06:54.814 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0033461559999999999 2024-08-17T18:06:54.814 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036913+0000", 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037575149999999999 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036976+0000", 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2580000000000001e-05 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037020+0000", 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4678000000000002e-05 2024-08-17T18:06:54.815 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.816 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.816 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.816 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037076+0000", 2024-08-17T18:06:54.816 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5449999999999999e-05 2024-08-17T18:06:54.816 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.816 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.816 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.816 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.816 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.816 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42566 46.e 46:733e1503:::benchmark_data_smithi130_267720_object42565:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:06:54.816 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.042517+0000", 2024-08-17T18:06:54.816 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 417.739657507, 2024-08-17T18:06:54.816 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99468539099999997, 2024-08-17T18:06:54.816 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.817 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.817 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.817 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.817 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.817 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42566 2024-08-17T18:06:54.817 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.817 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.817 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.817 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.817 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042517+0000", 2024-08-17T18:06:54.817 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.817 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.817 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.817 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.818 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042517+0000", 2024-08-17T18:06:54.818 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.818 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.818 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.818 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.818 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042516+0000", 2024-08-17T18:06:54.818 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:06:54.818 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.818 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.818 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.818 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042575+0000", 2024-08-17T18:06:54.818 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8802999999999999e-05 2024-08-17T18:06:54.818 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.819 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.819 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.819 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042588+0000", 2024-08-17T18:06:54.819 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2812e-05 2024-08-17T18:06:54.819 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.819 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.819 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.819 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042594+0000", 2024-08-17T18:06:54.819 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.55e-06 2024-08-17T18:06:54.819 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.819 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.819 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.819 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042634+0000", 2024-08-17T18:06:54.820 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9817999999999997e-05 2024-08-17T18:06:54.820 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.820 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.820 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:06:54.820 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042640+0000", 2024-08-17T18:06:54.820 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7570000000000001e-06 2024-08-17T18:06:54.820 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.820 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.820 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.820 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.030766+0000", 2024-08-17T18:06:54.820 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98812632600000005 2024-08-17T18:06:54.820 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.820 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.820 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.030858+0000", 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2035999999999998e-05 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.030950+0000", 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.1605000000000001e-05 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033167+0000", 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002216951 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.821 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.822 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036943+0000", 2024-08-17T18:06:54.822 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037761840000000001 2024-08-17T18:06:54.822 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.822 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.822 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.822 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037108+0000", 2024-08-17T18:06:54.822 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016495400000000001 2024-08-17T18:06:54.822 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.822 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.822 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.822 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037137+0000", 2024-08-17T18:06:54.822 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8826000000000001e-05 2024-08-17T18:06:54.822 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037203+0000", 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6042000000000002e-05 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42587 46.e 46:755c4a73:::benchmark_data_smithi130_267720_object42586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.057082+0000", 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 417.725093372, 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98134466099999995, 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.823 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.824 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.824 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.824 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.824 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42587 2024-08-17T18:06:54.824 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.824 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.824 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.824 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.824 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057082+0000", 2024-08-17T18:06:54.824 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.824 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.824 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.824 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.824 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057082+0000", 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057080+0000", 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057140+0000", 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9277000000000002e-05 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057152+0000", 2024-08-17T18:06:54.825 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2279000000000001e-05 2024-08-17T18:06:54.826 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.826 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.826 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.826 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057157+0000", 2024-08-17T18:06:54.826 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9389999999999996e-06 2024-08-17T18:06:54.826 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.826 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.826 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.826 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057223+0000", 2024-08-17T18:06:54.826 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5721000000000003e-05 2024-08-17T18:06:54.826 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.826 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.826 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:06:54.826 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057232+0000", 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6209999999999999e-06 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.031638+0000", 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97440567300000003 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.031716+0000", 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7650000000000004e-05 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.827 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.031819+0000", 2024-08-17T18:06:54.828 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010341899999999999 2024-08-17T18:06:54.828 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.828 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.828 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.828 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033981+0000", 2024-08-17T18:06:54.828 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0021622220000000001 2024-08-17T18:06:54.828 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.828 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.828 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.828 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037227+0000", 2024-08-17T18:06:54.828 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0032460060000000001 2024-08-17T18:06:54.828 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.829 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.829 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.829 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.038264+0000", 2024-08-17T18:06:54.829 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0010368949999999999 2024-08-17T18:06:54.829 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.829 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.829 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.829 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.038321+0000", 2024-08-17T18:06:54.829 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7354e-05 2024-08-17T18:06:54.829 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.829 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.829 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.829 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.038426+0000", 2024-08-17T18:06:54.829 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000104776 2024-08-17T18:06:54.830 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.830 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.830 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.830 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.830 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.830 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42593 46.e 46:77bdd57c:::benchmark_data_smithi130_267720_object42592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:06:54.830 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.061397+0000", 2024-08-17T18:06:54.830 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 417.720777521, 2024-08-17T18:06:54.830 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98195155000000001, 2024-08-17T18:06:54.830 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.830 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.830 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.830 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.830 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.831 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42593 2024-08-17T18:06:54.831 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.831 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.831 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.831 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.831 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061397+0000", 2024-08-17T18:06:54.831 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.831 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.831 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.831 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.831 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061397+0000", 2024-08-17T18:06:54.831 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.831 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.831 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061390+0000", 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999924 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061432+0000", 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2111999999999999e-05 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061445+0000", 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3037000000000001e-05 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.832 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061450+0000", 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.4759999999999997e-06 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061497+0000", 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6696999999999997e-05 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061504+0000", 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7100000000000001e-06 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.833 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033248+0000", 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97174373999999997 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033325+0000", 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7785000000000001e-05 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033416+0000", 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0221000000000002e-05 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.834 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036950+0000", 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0035344 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.040390+0000", 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0034401890000000002 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.043240+0000", 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028503019999999999 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.835 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.043292+0000", 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1598999999999998e-05 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.043349+0000", 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6907999999999998e-05 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:67147 46.a 46:5c4e42f4:::benchmark_data_smithi130_267720_object67146:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:06:54.836 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:18.360318+0000", 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 396.42185725600001, 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.26496706399999997, 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 67147 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360318+0000", 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.837 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.838 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.838 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360318+0000", 2024-08-17T18:06:54.838 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.838 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.838 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.838 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.838 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360316+0000", 2024-08-17T18:06:54.838 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:06:54.838 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.838 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.838 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.838 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360367+0000", 2024-08-17T18:06:54.838 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1192e-05 2024-08-17T18:06:54.838 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.839 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.839 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.839 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360383+0000", 2024-08-17T18:06:54.839 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5948999999999999e-05 2024-08-17T18:06:54.839 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.839 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.839 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.839 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360390+0000", 2024-08-17T18:06:54.839 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6139999999999997e-06 2024-08-17T18:06:54.839 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.839 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.839 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.839 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360421+0000", 2024-08-17T18:06:54.839 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1208999999999999e-05 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360518+0000", 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7254000000000001e-05 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.360612+0000", 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.3487999999999997e-05 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.362572+0000", 2024-08-17T18:06:54.840 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019598089999999999 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.370147+0000", 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0075756510000000001 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.625187+0000", 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.25503976 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.625243+0000", 2024-08-17T18:06:54.841 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5581000000000002e-05 2024-08-17T18:06:54.842 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.842 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.842 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.842 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.625285+0000", 2024-08-17T18:06:54.842 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2268000000000003e-05 2024-08-17T18:06:54.842 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.842 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.842 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.842 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.842 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.842 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:67152 46.a 46:567c055a:::benchmark_data_smithi130_267720_object67151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1265)", 2024-08-17T18:06:54.842 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:18.363407+0000", 2024-08-17T18:06:54.842 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 396.41876758400002, 2024-08-17T18:06:54.842 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.267197095, 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 67152 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363407+0000", 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.843 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363407+0000", 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363407+0000", 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363466+0000", 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9027999999999998e-05 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.844 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.845 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363477+0000", 2024-08-17T18:06:54.845 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.172e-05 2024-08-17T18:06:54.845 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.845 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.845 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.845 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363482+0000", 2024-08-17T18:06:54.845 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3560000000000003e-06 2024-08-17T18:06:54.845 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.845 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.845 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.845 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363534+0000", 2024-08-17T18:06:54.845 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1989e-05 2024-08-17T18:06:54.845 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.846 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.846 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.846 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363659+0000", 2024-08-17T18:06:54.846 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012534499999999999 2024-08-17T18:06:54.846 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.846 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.846 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.846 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.363815+0000", 2024-08-17T18:06:54.846 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00015637100000000001 2024-08-17T18:06:54.846 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.846 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.846 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.366241+0000", 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002425539 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.370190+0000", 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0039493870000000004 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630531+0000", 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.26034059700000001 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.847 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630580+0000", 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8797000000000001e-05 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:18.630604+0000", 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4739000000000001e-05 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79514 46.a 46:572757ec:::benchmark_data_smithi130_267720_object79513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.619984+0000", 2024-08-17T18:06:54.848 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 384.16219122000001, 2024-08-17T18:06:54.849 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.39803696399999999, 2024-08-17T18:06:54.849 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.849 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.849 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.849 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.849 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.849 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79514 2024-08-17T18:06:54.849 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.849 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.849 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:54.850 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:54.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:54.851 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.852 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.619984+0000", 2024-08-17T18:06:54.852 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.852 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.852 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.852 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.852 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.619984+0000", 2024-08-17T18:06:54.852 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.852 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.852 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.852 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.852 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.619983+0000", 2024-08-17T18:06:54.852 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:06:54.852 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.852 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.853 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.853 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620039+0000", 2024-08-17T18:06:54.853 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6446999999999997e-05 2024-08-17T18:06:54.853 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.853 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.853 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.853 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620050+0000", 2024-08-17T18:06:54.853 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0359e-05 2024-08-17T18:06:54.853 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.853 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.853 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.853 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620055+0000", 2024-08-17T18:06:54.853 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0479999999999998e-06 2024-08-17T18:06:54.853 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.854 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.854 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.854 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620085+0000", 2024-08-17T18:06:54.854 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9850000000000001e-05 2024-08-17T18:06:54.854 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.854 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.854 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.854 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620160+0000", 2024-08-17T18:06:54.854 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.5499000000000003e-05 2024-08-17T18:06:54.854 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.854 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.854 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.854 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620240+0000", 2024-08-17T18:06:54.854 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0418000000000002e-05 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.622161+0000", 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001920342 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.627515+0000", 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0053542399999999997 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017941+0000", 2024-08-17T18:06:54.855 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.390425929 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017991+0000", 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9988999999999999e-05 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.018021+0000", 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9762999999999999e-05 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.856 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79523 46.a 46:590eeb35:::benchmark_data_smithi130_267720_object79522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.631753+0000", 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 384.15042189399998, 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38901796, 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79523 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.857 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631753+0000", 2024-08-17T18:06:54.858 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.858 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.858 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.858 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.858 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631753+0000", 2024-08-17T18:06:54.858 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.858 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.858 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.858 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.858 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631752+0000", 2024-08-17T18:06:54.858 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:06:54.858 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.858 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631818+0000", 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5757000000000001e-05 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631828+0000", 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0200000000000001e-05 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631832+0000", 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0430000000000001e-06 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.859 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.860 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.860 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631869+0000", 2024-08-17T18:06:54.860 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6730999999999998e-05 2024-08-17T18:06:54.860 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.860 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.860 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.860 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631941+0000", 2024-08-17T18:06:54.860 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.1763999999999999e-05 2024-08-17T18:06:54.860 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.860 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.860 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.860 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632027+0000", 2024-08-17T18:06:54.860 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6599000000000002e-05 2024-08-17T18:06:54.861 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.861 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.861 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.861 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635101+0000", 2024-08-17T18:06:54.861 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0030730430000000001 2024-08-17T18:06:54.861 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.861 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.861 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.861 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.641996+0000", 2024-08-17T18:06:54.861 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0068956429999999999 2024-08-17T18:06:54.861 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.861 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.861 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.862 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020716+0000", 2024-08-17T18:06:54.862 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.378719474 2024-08-17T18:06:54.862 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.862 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.862 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.862 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020748+0000", 2024-08-17T18:06:54.862 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2026000000000001e-05 2024-08-17T18:06:54.862 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.862 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.862 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.862 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020771+0000", 2024-08-17T18:06:54.862 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3370000000000002e-05 2024-08-17T18:06:54.862 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.863 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.863 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.863 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.863 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.863 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79526 46.a 46:540ef56b:::benchmark_data_smithi130_267720_object79525:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:06:54.863 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:06:54.863 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 384.14989457600001, 2024-08-17T18:06:54.863 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38861050899999999, 2024-08-17T18:06:54.863 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.863 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.863 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.863 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.863 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.864 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79526 2024-08-17T18:06:54.864 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.864 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.864 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.864 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.864 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:06:54.864 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.864 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.864 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.864 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.864 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:06:54.864 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.864 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.865 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.865 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.865 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:06:54.865 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:06:54.865 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.865 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.865 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.865 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632360+0000", 2024-08-17T18:06:54.865 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0008999999999999e-05 2024-08-17T18:06:54.865 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.865 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.865 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.866 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632369+0000", 2024-08-17T18:06:54.866 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6590000000000002e-06 2024-08-17T18:06:54.866 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.866 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.866 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.866 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632373+0000", 2024-08-17T18:06:54.866 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8840000000000004e-06 2024-08-17T18:06:54.866 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.866 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.866 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.866 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.633861+0000", 2024-08-17T18:06:54.866 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001487326 2024-08-17T18:06:54.867 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.867 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.867 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.867 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.633938+0000", 2024-08-17T18:06:54.867 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7625000000000003e-05 2024-08-17T18:06:54.867 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.867 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.867 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.867 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.634026+0000", 2024-08-17T18:06:54.867 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7950999999999999e-05 2024-08-17T18:06:54.867 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.867 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.867 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.868 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635234+0000", 2024-08-17T18:06:54.868 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0012074340000000001 2024-08-17T18:06:54.868 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.868 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.868 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.868 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.642019+0000", 2024-08-17T18:06:54.868 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0067856890000000001 2024-08-17T18:06:54.868 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.868 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.868 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.868 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020838+0000", 2024-08-17T18:06:54.868 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37881851700000002 2024-08-17T18:06:54.868 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020870+0000", 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2261999999999997e-05 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020891+0000", 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0777000000000001e-05 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.869 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79528 46.e 46:716b67ef:::benchmark_data_smithi130_267720_object79527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:06:54.870 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.635764+0000", 2024-08-17T18:06:54.870 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 384.14641075700001, 2024-08-17T18:06:54.870 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38226721899999999, 2024-08-17T18:06:54.870 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.870 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.870 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.870 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.870 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.870 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79528 2024-08-17T18:06:54.870 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.870 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.870 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.870 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.870 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635764+0000", 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635764+0000", 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635763+0000", 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.871 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635828+0000", 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4821999999999997e-05 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635838+0000", 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9599999999999995e-06 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635842+0000", 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3280000000000002e-06 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.872 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635865+0000", 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3600000000000001e-05 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635943+0000", 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.8098999999999999e-05 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.636030+0000", 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7188000000000006e-05 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.637969+0000", 2024-08-17T18:06:54.873 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001938148 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.639658+0000", 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0016891600000000001 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017955+0000", 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.378297194 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017992+0000", 2024-08-17T18:06:54.874 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6832999999999998e-05 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.018031+0000", 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9646e-05 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79529 46.a 46:5d7561ca:::benchmark_data_smithi130_267720_object79528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.635869+0000", 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 384.14630609599999, 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38782661600000001, 2024-08-17T18:06:54.875 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79529 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635869+0000", 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635869+0000", 2024-08-17T18:06:54.876 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635868+0000", 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635935+0000", 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6509999999999998e-05 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635942+0000", 2024-08-17T18:06:54.877 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3139999999999998e-06 2024-08-17T18:06:54.878 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.878 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.878 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.878 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635945+0000", 2024-08-17T18:06:54.878 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.7580000000000001e-06 2024-08-17T18:06:54.878 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.878 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.878 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.878 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635983+0000", 2024-08-17T18:06:54.878 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8318000000000001e-05 2024-08-17T18:06:54.878 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.878 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.878 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.879 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.636088+0000", 2024-08-17T18:06:54.879 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000104718 2024-08-17T18:06:54.879 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.879 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.879 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.879 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.636213+0000", 2024-08-17T18:06:54.879 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000124852 2024-08-17T18:06:54.879 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.879 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.879 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.879 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.637790+0000", 2024-08-17T18:06:54.879 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0015775839999999999 2024-08-17T18:06:54.879 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.879 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.642450+0000", 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0046600260000000003 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.023638+0000", 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38118750600000001 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.023669+0000", 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0998999999999999e-05 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.880 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.881 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.881 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.023695+0000", 2024-08-17T18:06:54.881 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.6645e-05 2024-08-17T18:06:54.881 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.881 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.881 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.881 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.881 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.881 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79534 46.a 46:5daa6ba5:::benchmark_data_smithi130_267720_object79533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:06:54.881 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.643773+0000", 2024-08-17T18:06:54.881 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 384.13840199600003, 2024-08-17T18:06:54.881 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38289725099999999, 2024-08-17T18:06:54.881 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.881 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.882 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.882 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.882 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.882 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79534 2024-08-17T18:06:54.882 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.882 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.882 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.882 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.882 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643773+0000", 2024-08-17T18:06:54.882 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.882 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.882 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.882 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.882 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643773+0000", 2024-08-17T18:06:54.883 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.883 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.883 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.883 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.883 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643772+0000", 2024-08-17T18:06:54.883 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:06:54.883 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.883 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.883 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.883 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643824+0000", 2024-08-17T18:06:54.883 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2240000000000001e-05 2024-08-17T18:06:54.883 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.883 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.883 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.884 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643833+0000", 2024-08-17T18:06:54.884 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6770000000000001e-06 2024-08-17T18:06:54.884 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.884 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.884 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.884 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643837+0000", 2024-08-17T18:06:54.884 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8089999999999999e-06 2024-08-17T18:06:54.884 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.884 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.884 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.884 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643868+0000", 2024-08-17T18:06:54.884 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1560000000000003e-05 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643961+0000", 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2584999999999995e-05 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.644059+0000", 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7851999999999998e-05 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.646074+0000", 2024-08-17T18:06:54.885 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020149550000000001 2024-08-17T18:06:54.886 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.886 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.886 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.886 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647702+0000", 2024-08-17T18:06:54.886 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001628811 2024-08-17T18:06:54.886 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.886 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.886 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.886 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.026581+0000", 2024-08-17T18:06:54.886 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37887865300000001 2024-08-17T18:06:54.886 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.886 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.886 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.026633+0000", 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2040000000000003e-05 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.026670+0000", 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7038e-05 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79535 46.4 46:2496b234:::benchmark_data_smithi130_267720_object79534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 384.13830947999998, 2024-08-17T18:06:54.887 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38560203399999998, 2024-08-17T18:06:54.888 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.888 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.888 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.888 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.888 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.888 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79535 2024-08-17T18:06:54.888 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.888 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.888 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.888 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.888 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:06:54.888 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.888 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.888 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643927+0000", 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1629000000000002e-05 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.889 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.890 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.890 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643933+0000", 2024-08-17T18:06:54.890 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5130000000000002e-06 2024-08-17T18:06:54.890 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.890 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.890 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.890 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643937+0000", 2024-08-17T18:06:54.890 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3479999999999997e-06 2024-08-17T18:06:54.890 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.890 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.890 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.890 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643975+0000", 2024-08-17T18:06:54.891 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7771999999999997e-05 2024-08-17T18:06:54.891 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.891 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.891 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.891 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.644055+0000", 2024-08-17T18:06:54.891 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0144999999999999e-05 2024-08-17T18:06:54.891 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.891 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.891 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:06:54.891 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.644142+0000", 2024-08-17T18:06:54.891 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6539000000000006e-05 2024-08-17T18:06:54.891 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.891 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.891 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.892 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.646160+0000", 2024-08-17T18:06:54.892 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002018373 2024-08-17T18:06:54.892 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.892 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.892 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.892 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647237+0000", 2024-08-17T18:06:54.892 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001076465 2024-08-17T18:06:54.892 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.892 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.892 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.892 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.029404+0000", 2024-08-17T18:06:54.892 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38216758000000001 2024-08-17T18:06:54.892 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.892 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.029443+0000", 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8846e-05 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.029468+0000", 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4355e-05 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79538 46.e 46:7d19c96c:::benchmark_data_smithi130_267720_object79537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:06:54.893 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.647351+0000", 2024-08-17T18:06:54.894 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 384.13482408200002, 2024-08-17T18:06:54.894 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37361270899999999, 2024-08-17T18:06:54.894 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.894 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.894 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.894 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.894 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.894 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79538 2024-08-17T18:06:54.894 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.894 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.894 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.894 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.894 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647351+0000", 2024-08-17T18:06:54.894 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647351+0000", 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647350+0000", 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647421+0000", 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0347999999999998e-05 2024-08-17T18:06:54.895 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647429+0000", 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3059999999999998e-06 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647433+0000", 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7290000000000002e-06 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647476+0000", 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3343999999999997e-05 2024-08-17T18:06:54.896 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647600+0000", 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012389700000000001 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647740+0000", 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014001899999999999 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.649745+0000", 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020055450000000001 2024-08-17T18:06:54.897 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.898 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.898 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.898 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.650277+0000", 2024-08-17T18:06:54.898 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00053202999999999996 2024-08-17T18:06:54.898 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.898 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.898 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.898 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020872+0000", 2024-08-17T18:06:54.898 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37059452300000001 2024-08-17T18:06:54.898 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.898 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.898 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.898 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020923+0000", 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1481999999999999e-05 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020964+0000", 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0132000000000001e-05 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:165418 46.e 46:7099396e:::benchmark_data_smithi130_267720_object57435:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1279)", 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:02:16.031407+0000", 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 278.75076817899998, 2024-08-17T18:06:54.899 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.26760932100000001, 2024-08-17T18:06:54.900 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.900 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:06:54.900 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.900 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:06:54.900 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:06:54.900 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 165418 2024-08-17T18:06:54.900 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.900 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.900 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.900 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.900 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031407+0000", 2024-08-17T18:06:54.900 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.900 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.900 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031407+0000", 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031406+0000", 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031410+0000", 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8340000000000001e-06 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.901 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031423+0000", 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2577e-05 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031427+0000", 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5170000000000001e-06 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031460+0000", 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2641999999999997e-05 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.902 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031534+0000", 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.4727999999999993e-05 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.031614+0000", 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9921999999999998e-05 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.034323+0000", 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027090270000000001 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.903 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.034810+0000", 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00048701899999999999 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.298931+0000", 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.26412039300000001 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.298980+0000", 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9416999999999998e-05 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.904 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:02:16.299016+0000", 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.5902999999999998e-05 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.46373.0:115177 47.e 47:7d674894:::benchmark_data_smithi130_302447_object19942:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:05:56.557182+0000", 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 58.224992708999999, 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0269707910000001, 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.905 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.46373", 2024-08-17T18:06:54.906 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:06:54.906 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 115177 2024-08-17T18:06:54.906 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.906 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.906 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.906 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.906 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557182+0000", 2024-08-17T18:06:54.906 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.906 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.906 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.906 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.906 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557182+0000", 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557182+0000", 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557184+0000", 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.623e-06 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.907 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557188+0000", 2024-08-17T18:06:54.908 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3599999999999998e-06 2024-08-17T18:06:54.908 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.908 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.908 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.908 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557191+0000", 2024-08-17T18:06:54.908 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0070000000000002e-06 2024-08-17T18:06:54.908 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.908 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.908 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.908 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557236+0000", 2024-08-17T18:06:54.908 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4824999999999999e-05 2024-08-17T18:06:54.908 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.908 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.908 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.909 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557348+0000", 2024-08-17T18:06:54.909 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000112503 2024-08-17T18:06:54.909 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.909 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.909 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:06:54.909 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557457+0000", 2024-08-17T18:06:54.909 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000108707 2024-08-17T18:06:54.909 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.909 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.909 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.909 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.559820+0000", 2024-08-17T18:06:54.909 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0023626350000000001 2024-08-17T18:06:54.909 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.909 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.568430+0000", 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0086101579999999997 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.077053+0000", 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.50862317199999996 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.077061+0000", 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9440000000000005e-06 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.910 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.584088+0000", 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.507027487 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.584153+0000", 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4629999999999996e-05 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.46373.0:115179 47.f 47:fa81ee3a:::benchmark_data_smithi130_302447_object19944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:05:56.564940+0000", 2024-08-17T18:06:54.911 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 58.217235436000003, 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.024401452, 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.46373", 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 115179 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.564940+0000", 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.912 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.564940+0000", 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.564939+0000", 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.564943+0000", 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7029999999999997e-06 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.913 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.564954+0000", 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1124999999999999e-05 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.564959+0000", 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2019999999999997e-06 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.564985+0000", 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.6480000000000001e-05 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.914 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.565063+0000", 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.8300999999999995e-05 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.565134+0000", 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0947999999999999e-05 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.569055+0000", 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003920649 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.915 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.916 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.576387+0000", 2024-08-17T18:06:54.916 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.007332046 2024-08-17T18:06:54.916 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.916 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.916 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.916 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.077838+0000", 2024-08-17T18:06:54.916 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.50145124600000002 2024-08-17T18:06:54.916 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.916 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.916 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:06:54.916 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.077843+0000", 2024-08-17T18:06:54.916 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6630000000000001e-06 2024-08-17T18:06:54.916 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.916 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.917 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.917 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.589299+0000", 2024-08-17T18:06:54.917 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.51145641900000005 2024-08-17T18:06:54.917 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.917 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.917 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.917 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.589341+0000", 2024-08-17T18:06:54.917 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1548999999999998e-05 2024-08-17T18:06:54.917 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.917 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.917 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.917 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.917 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.917 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.46373.0:115181 47.f 47:ff784e5d:::benchmark_data_smithi130_302447_object19946:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:06:54.918 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:05:56.566481+0000", 2024-08-17T18:06:54.918 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 58.215694407000001, 2024-08-17T18:06:54.918 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.023022541, 2024-08-17T18:06:54.918 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:06:54.918 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:06:54.918 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:06:54.918 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.46373", 2024-08-17T18:06:54.918 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:06:54.918 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 115181 2024-08-17T18:06:54.918 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.918 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:06:54.918 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.918 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:06:54.918 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.566481+0000", 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.566481+0000", 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.566480+0000", 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:06:54.919 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.566484+0000", 2024-08-17T18:06:54.920 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3810000000000001e-06 2024-08-17T18:06:54.920 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.920 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.920 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:06:54.920 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.566495+0000", 2024-08-17T18:06:54.920 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.079e-05 2024-08-17T18:06:54.920 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.920 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.920 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:06:54.920 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.566499+0000", 2024-08-17T18:06:54.920 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3470000000000001e-06 2024-08-17T18:06:54.920 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.920 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.920 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.921 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.567383+0000", 2024-08-17T18:06:54.921 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000884116 2024-08-17T18:06:54.921 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.921 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.921 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:06:54.921 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.567471+0000", 2024-08-17T18:06:54.921 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7812000000000006e-05 2024-08-17T18:06:54.921 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.921 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.921 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T18:06:54.921 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.567542+0000", 2024-08-17T18:06:54.921 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0338000000000003e-05 2024-08-17T18:06:54.921 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.922 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.922 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:06:54.922 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.569181+0000", 2024-08-17T18:06:54.922 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0016393460000000001 2024-08-17T18:06:54.922 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.922 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.922 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:06:54.922 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.584045+0000", 2024-08-17T18:06:54.922 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.014863955999999999 2024-08-17T18:06:54.922 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.922 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.922 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.922 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.077869+0000", 2024-08-17T18:06:54.923 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.493823868 2024-08-17T18:06:54.923 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.923 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.923 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:06:54.923 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.077875+0000", 2024-08-17T18:06:54.923 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4019999999999999e-06 2024-08-17T18:06:54.923 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.923 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.923 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:06:54.923 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.589472+0000", 2024-08-17T18:06:54.923 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.51159689799999997 2024-08-17T18:06:54.923 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:06:54.923 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:06:54.923 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:06:54.924 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.589503+0000", 2024-08-17T18:06:54.924 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1056000000000002e-05 2024-08-17T18:06:54.924 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.924 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.924 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.924 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:06:54.924 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:06:54.924 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:06:54.951 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:54.973 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:54.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:54.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:55.074 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:55.098 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:55.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:55.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:55.200 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:55.236 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:55.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:55.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:55.338 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:55.358 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:55.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:55.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:55.460 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:55.472 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:06:55.505 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:55.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:55.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:55.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:55 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/999190161' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:06:55.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:55 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:06:55.607 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:55.628 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:55.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:55.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:55.730 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:55.751 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:55.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:55.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:55.852 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:55.890 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:55.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:55.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:55.902 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:55 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/999190161' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:06:55.902 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:55 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:06:55.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:55 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/999190161' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:06:55.902 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:55 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:06:55.984 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T18:06:55.992 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:56.014 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:56.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:56.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:56.115 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:56.172 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:56.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:56.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:56.274 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:56.295 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:56.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:56.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:56.397 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:06:56.418 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:56.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:56.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:06:56.519 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:56.558 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:56.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:56.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:56.659 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:56 smithi130 ceph-mon[105521]: pgmap v2292: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.3 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 895 op/s; 11839/63459 objects degraded (18.656%) 2024-08-17T18:06:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:56 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:06:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:56 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:06:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:56 smithi130 ceph-mon[105521]: osdmap e1323: 8 total, 7 up, 5 in 2024-08-17T18:06:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 11839/63459 objects degraded (18.656%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:56 smithi130 ceph-mon[110068]: pgmap v2292: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.3 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 895 op/s; 11839/63459 objects degraded (18.656%) 2024-08-17T18:06:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:56 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:06:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:56 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:06:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:56 smithi130 ceph-mon[110068]: osdmap e1323: 8 total, 7 up, 5 in 2024-08-17T18:06:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 11839/63459 objects degraded (18.656%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:56.696 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:56.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:56.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:56.798 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:56.820 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:56.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:56.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:56.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:56 smithi179 ceph-mon[93519]: pgmap v2292: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.3 GiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 895 op/s; 11839/63459 objects degraded (18.656%) 2024-08-17T18:06:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:56 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:06:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:56 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:06:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:56 smithi179 ceph-mon[93519]: osdmap e1323: 8 total, 7 up, 5 in 2024-08-17T18:06:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 11839/63459 objects degraded (18.656%), 18 pgs degraded (PG_DEGRADED) 2024-08-17T18:06:56.922 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:56.945 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:56.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:57.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:06:57.083 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:57.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:57.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:06:57.185 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:57.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:57.236 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:57.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:57.337 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:57.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:57.359 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:57.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:57.459 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:57.504 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:57.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:57.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:57.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:06:57.590 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:06:57.606 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:57.645 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:57.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:57.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:57.747 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:06:57.782 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:57.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:57.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:06:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:06:57.883 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:57.925 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:57.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:57.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:58.026 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:58.048 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:58.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:58.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:58.150 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:58.172 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:58.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:58.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:58.274 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:58.295 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:58.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:58.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:58.301 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:06:58.341 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:06:58.397 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:06:58.418 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:58.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:58.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:06:58.520 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:58.541 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:58.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:58.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:58.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:06:58 smithi130 ceph-mon[110068]: pgmap v2294: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.1 GiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 915 op/s; 10654/56214 objects degraded (18.953%) 2024-08-17T18:06:58.544 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:06:58 smithi130 ceph-mon[105521]: pgmap v2294: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.1 GiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 915 op/s; 10654/56214 objects degraded (18.953%) 2024-08-17T18:06:58.642 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:58.692 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:58.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:58.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:58.794 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:58.815 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:58.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:58.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:06:58 smithi179 ceph-mon[93519]: pgmap v2294: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1.1 GiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 915 op/s; 10654/56214 objects degraded (18.953%) 2024-08-17T18:06:58.917 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:58.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:58.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:58.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:59.059 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:59.080 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:59.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:59.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:59.182 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:59.227 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:59.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:59.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:59.329 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:59.366 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:59.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:59.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:59.468 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:06:59.489 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:59.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:59.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:06:59.591 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:06:59.628 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:06:59.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:59.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:06:59.729 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:59.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:59.751 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:59.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:59.852 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:06:59.872 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:06:59.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:06:59.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:06:59.896 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:06:59.896 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T18:06:59.896 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-08-17T18:06:59.974 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:00.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:00.023 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:00.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:00.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:07:00.101 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:06:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:06:59] "GET /metrics HTTP/1.1" 200 36785 "" "Prometheus/2.51.0" 2024-08-17T18:07:00.124 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:00.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:00.146 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:00.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:00.247 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:00.268 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:00.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:00.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:00.370 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:00.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:00.432 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:00.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:00.487 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:07:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:07:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:07:00.533 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:00.555 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:00.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:00.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:00.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:00.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:00 smithi130 ceph-mon[105521]: pgmap v2295: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1018 MiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 914 op/s; 9141/48918 objects degraded (18.686%) 2024-08-17T18:07:00.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:00 smithi130 ceph-mon[110068]: pgmap v2295: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1018 MiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 914 op/s; 9141/48918 objects degraded (18.686%) 2024-08-17T18:07:00.678 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:00.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:00.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:00.694 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:00.780 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:00.824 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:00.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:00.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:00.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:00 smithi179 ceph-mon[93519]: pgmap v2295: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1018 MiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 914 op/s; 9141/48918 objects degraded (18.686%) 2024-08-17T18:07:00.926 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:00.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:00.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:00.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:01.059 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:01.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:01.081 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:01.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:01.182 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:01.203 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:01.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:01.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:01.305 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:01.327 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:01.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:01.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:01.428 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:01.450 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:01.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:01.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:01.551 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:01.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:01.573 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:01.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:01.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 9141/48918 objects degraded (18.686%), 18 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 9141/48918 objects degraded (18.686%), 18 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:01.674 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:01.710 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:01.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:01.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:01.812 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 9141/48918 objects degraded (18.686%), 18 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:01.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:01.847 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:01.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:01.948 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:01.969 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:01.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:01.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:02.072 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:02.093 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:02.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:02.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:02.195 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:02.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:02.217 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:02.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:02.317 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:02.339 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:02.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:02.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:02.441 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:02.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:02.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:02.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:02.563 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:02.587 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:02.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:02.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:02.689 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:02.716 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:02.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:02.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:02.817 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:02.839 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:02.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:02.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:02.941 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:02.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:02.964 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:02.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:03.065 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:03.087 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:03.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:03.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:03.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:02 smithi130 ceph-mon[105521]: pgmap v2296: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1018 MiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 914 op/s; 9141/48918 objects degraded (18.686%) 2024-08-17T18:07:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:02 smithi130 ceph-mon[110068]: pgmap v2296: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1018 MiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 914 op/s; 9141/48918 objects degraded (18.686%) 2024-08-17T18:07:03.189 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:03.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:03.211 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:03.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:03.312 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:03.334 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:03.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:03.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:03.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:02 smithi179 ceph-mon[93519]: pgmap v2296: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 1018 MiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 914 op/s; 9141/48918 objects degraded (18.686%) 2024-08-17T18:07:03.435 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:03.456 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:03.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:03.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:03.558 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:03.580 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:03.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:03.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:03.682 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:03.718 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:03.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:03.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:03.820 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:03.841 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:03.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:03.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:03.944 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:03.992 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:03.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:03.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:04.094 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:04.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:04.117 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:04.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:04.218 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:04.240 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:04.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:04.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:04.342 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:04.363 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:04.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:04.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:04.465 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:04.486 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:04.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:04.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:04.589 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:04.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:04.639 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:04.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:04.740 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:04.778 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:04.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:04.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:04.880 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:04.901 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:04.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:04.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:05.003 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:05.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:05.026 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:05.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:05.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:05.163 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:05.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:05.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:05.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:04 smithi130 ceph-mon[105521]: pgmap v2297: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 812 MiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 954 op/s; 7332/39027 objects degraded (18.787%) 2024-08-17T18:07:05.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:04 smithi130 ceph-mon[110068]: pgmap v2297: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 812 MiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 954 op/s; 7332/39027 objects degraded (18.787%) 2024-08-17T18:07:05.265 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:05.287 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:05.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:05.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:05.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:04 smithi179 ceph-mon[93519]: pgmap v2297: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 812 MiB data, 7.6 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 954 op/s; 7332/39027 objects degraded (18.787%) 2024-08-17T18:07:05.389 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:05.411 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:05.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:05.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:05.513 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:05.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:05.535 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:05.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:05.636 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:05.658 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:05.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:05.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:05.760 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:05.781 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:05.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:05.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:05.882 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:05.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:05.930 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:05.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:06.031 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:06.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:06.053 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:06.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:06.126 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:05 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 7332/39027 objects degraded (18.787%), 18 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:06.126 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:05 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 7332/39027 objects degraded (18.787%), 18 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:06.154 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:06.175 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:06.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:06.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:06.277 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:06.298 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:06.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:06.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:06.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:05 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 7332/39027 objects degraded (18.787%), 18 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:06.400 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:06.439 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:06.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:06.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:06.471 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:06.541 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:06.594 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:06.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:06.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:06.696 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:06.759 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:06.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:06.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:06.861 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:06.882 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:06.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:06.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:06.984 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:07.005 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:07.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:07.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:07.106 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:07.128 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:07.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:07.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:07.164 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:06 smithi130 ceph-mon[105521]: pgmap v2298: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 717 MiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 965 op/s; 6429/34470 objects degraded (18.651%) 2024-08-17T18:07:07.164 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:06 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:07.164 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:06 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:07.165 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:06 smithi130 ceph-mon[110068]: pgmap v2298: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 717 MiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 965 op/s; 6429/34470 objects degraded (18.651%) 2024-08-17T18:07:07.165 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:06 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:07.165 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:06 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:07.230 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:07.252 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:07.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:07.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:06 smithi179 ceph-mon[93519]: pgmap v2298: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 717 MiB data, 7.2 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 965 op/s; 6429/34470 objects degraded (18.651%) 2024-08-17T18:07:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:06 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:06 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:07.354 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:07.399 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:07.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:07.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:07.501 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:07.522 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:07.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:07.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:07.624 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:07.646 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:07.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:07.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:07.748 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:07.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:07.785 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:07.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:07.886 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:07.906 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:07.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:07.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:07.969 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:07 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/383595169' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:07.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:07 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/383595169' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:08.008 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:08.029 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:08.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:08.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:08.131 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:08.173 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:08.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:08.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:08.275 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:08.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:08.321 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:08.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:08.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:07 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/383595169' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:08.422 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:08.443 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:08.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:08.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:08.545 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:08.576 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:08.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:08.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:08.678 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:08.699 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:08.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:08.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:08.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:08.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:08.843 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:08.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:08.944 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:08.966 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:08.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:08.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:09.068 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:09.089 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:09.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:09.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:09.191 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:09.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:09.231 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:09.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:09.332 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:09.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:08 smithi179 ceph-mon[93519]: pgmap v2299: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 568 MiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 5229/27318 objects degraded (19.141%) 2024-08-17T18:07:09.372 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:09.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:09.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:09.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:08 smithi130 ceph-mon[105521]: pgmap v2299: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 568 MiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 5229/27318 objects degraded (19.141%) 2024-08-17T18:07:09.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:08 smithi130 ceph-mon[110068]: pgmap v2299: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 568 MiB data, 7.0 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 1.02k op/s; 5229/27318 objects degraded (19.141%) 2024-08-17T18:07:09.474 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:09.495 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:09.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:09.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:09.597 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:09.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:09.619 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:09.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:09.720 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:09.741 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:09.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:09.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:09.843 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:09.864 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:09.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:09.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:09.966 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:09.987 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:09.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:09.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:10.053 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:07:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:07:09] "GET /metrics HTTP/1.1" 200 36785 "" "Prometheus/2.51.0" 2024-08-17T18:07:10.089 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:10.126 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:10.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:10.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:10.228 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:10.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:10.251 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:10.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:10.352 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:10.373 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:10.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:10.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:10.454 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:07:10.454 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:07:10.455 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:07:10.455 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:07:10.455 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:07:10.455 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:07:10.455 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:07:10.455 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:07:10.455 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:07:10.455 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:07:10.455 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:07:10.455 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:07:10.456 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:07:10.456 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:07:10.456 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:07:10.475 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:10.514 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:10.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:10.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:07:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:07:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:07:10.616 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:10.638 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:10.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:10.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:10.657 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:10.741 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:10.788 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:10.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:10.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:10.890 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:10.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:10.931 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:10.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:11.032 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:11.071 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:11.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:11.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:11.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:10 smithi130 ceph-mon[110068]: pgmap v2300: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 421 MiB data, 6.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 998 op/s; 3748/20253 objects degraded (18.506%) 2024-08-17T18:07:11.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:10 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:07:11.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:10 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:07:11.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:10 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:11.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:10 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 3748/20253 objects degraded (18.506%), 18 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:10 smithi130 ceph-mon[105521]: pgmap v2300: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 421 MiB data, 6.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 998 op/s; 3748/20253 objects degraded (18.506%) 2024-08-17T18:07:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:10 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:07:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:10 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:07:11.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:10 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:11.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:10 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 3748/20253 objects degraded (18.506%), 18 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:11.173 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:11.195 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:11.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:11.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:11.244 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:07:11.244 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:07:11.297 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:11.331 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:11.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:11.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:11.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:10 smithi179 ceph-mon[93519]: pgmap v2300: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 421 MiB data, 6.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 998 op/s; 3748/20253 objects degraded (18.506%) 2024-08-17T18:07:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:10 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:07:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:10 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:07:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:10 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:10 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 3748/20253 objects degraded (18.506%), 18 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:11.435 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:11.478 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:11.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:11.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:11.524 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:07:11.524 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:07:11.580 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:11.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:11.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:11.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:11.704 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:11.745 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:11.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:11.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:11.846 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:11.889 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:11.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:11.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:11.991 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:12.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:12.014 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:12.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:12.022 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:07:12.022 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:07:12.022 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:07:12.022 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:07:12.022 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:07:12.022 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:07:12.022 INFO:teuthology.orchestra.run.smithi130.stderr:bench [] [] [] [] 2024-08-17T18:07:12.022 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:07:12.022 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:07:12.023 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:07:12.023 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator dump block 2024-08-17T18:07:12.023 INFO:teuthology.orchestra.run.smithi130.stderr:Error EINVAL: invalid command 2024-08-17T18:07:12.115 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:12.136 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:12.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:12.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:12.238 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:12.260 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:12.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:12.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:12.362 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:12.383 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:12.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:12.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:12.485 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:12.507 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:12.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:12.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:12.608 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:12.630 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:12.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:12.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:12.732 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:12.776 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:12.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:12.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:12.878 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:12.900 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:12.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:12.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:13.002 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:13.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:13.039 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:13.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:13.140 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:13.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:12 smithi130 ceph-mon[105521]: pgmap v2301: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 421 MiB data, 6.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 795 op/s; 3748/20253 objects degraded (18.506%) 2024-08-17T18:07:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:07:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:12 smithi130 ceph-mon[110068]: pgmap v2301: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 421 MiB data, 6.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 795 op/s; 3748/20253 objects degraded (18.506%) 2024-08-17T18:07:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:07:13.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:13.180 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:13.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:13.245 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:07:13.281 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:13.302 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:13.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:13.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:12 smithi179 ceph-mon[93519]: pgmap v2301: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 421 MiB data, 6.5 GiB used, 440 GiB / 447 GiB avail; 0 B/s wr, 795 op/s; 3748/20253 objects degraded (18.506%) 2024-08-17T18:07:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:07:13.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T18:07:13.404 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:13.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:13.426 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:13.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:13.527 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:13.547 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:13.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:13.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:13.649 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:13.672 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:13.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:13.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:13.773 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:13.796 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:13.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:13.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:13.898 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:13.919 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:13.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:13.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:14.020 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:14.041 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:14.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:14.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:14.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:14.187 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:14.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:14.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:14.289 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:14.309 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:14.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:14.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:14.411 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:14.452 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:14.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:14.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:14.554 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:14.575 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:14.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:14.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:14.677 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:14.698 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:14.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:14.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:14.800 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:14.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:14.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:14.922 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:14.944 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:14.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:14.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:15.045 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:15.067 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:15.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:15.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:15.137 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:07:15.138 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T18:07:15.169 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:15.183 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Removed 95232 objects 2024-08-17T18:07:15.183 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Clean up completed and total clean up time :104.498 2024-08-17T18:07:15.195 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:15.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:15.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:15.297 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:15.319 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:15.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:15.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:15.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:14 smithi179 ceph-mon[93519]: pgmap v2302: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 222 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.06k op/s; 2035/10698 objects degraded (19.022%) 2024-08-17T18:07:15.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:14 smithi130 ceph-mon[105521]: pgmap v2302: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 222 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.06k op/s; 2035/10698 objects degraded (19.022%) 2024-08-17T18:07:15.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:14 smithi130 ceph-mon[110068]: pgmap v2302: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 222 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 1.06k op/s; 2035/10698 objects degraded (19.022%) 2024-08-17T18:07:15.421 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:15.443 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:15.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:15.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:15.545 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:15.567 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:15.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:15.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:15.669 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:15.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:15.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:15.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:15.786 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-08-17T18:07:15.786 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-08-17T18:07:15.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:15.813 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:15.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:15.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:15.915 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:15.936 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:15.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:15.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:16.038 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:16.059 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:16.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:16.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:16.161 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:16.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:15 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 2035/10698 objects degraded (19.022%), 18 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:16.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:15 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 2035/10698 objects degraded (19.022%), 18 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:16.184 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:16.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:16.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:16.260 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:16.265 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:16.275 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:16.286 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:16.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:16.309 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:16.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:16.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:15 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 2035/10698 objects degraded (19.022%), 18 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:16.410 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:16.433 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:16.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:16.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:16.535 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:16.557 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:16.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:16.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:16.658 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:16.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:16.682 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:16.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:16.783 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:16.805 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:16.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:16.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:16.907 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:16.930 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:16.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:16.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:17.032 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:17.056 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:17.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:17.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:17.141 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:16 smithi130 ceph-mon[105521]: pgmap v2303: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 128 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 911 op/s; 1138/6189 objects degraded (18.387%) 2024-08-17T18:07:17.141 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:16 smithi130 ceph-mon[110068]: pgmap v2303: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 128 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 911 op/s; 1138/6189 objects degraded (18.387%) 2024-08-17T18:07:17.157 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:17.180 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:17.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:17.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:17.281 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:17.302 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:17.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:17.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:17.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:16 smithi179 ceph-mon[93519]: pgmap v2303: 89 pgs: 36 active+undersized, 18 active+undersized+degraded, 35 active+clean; 128 MiB data, 5.8 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 911 op/s; 1138/6189 objects degraded (18.387%) 2024-08-17T18:07:17.404 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:17.428 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:17.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:17.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:17.530 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:17.551 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:17.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:17.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:17.653 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:17.679 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:17.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:17.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:17.780 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:17.802 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:17.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:17.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:17.904 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:17.929 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:17.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:17.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:17.966 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:07:18.021 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:17 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/396805322' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:07:18.022 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:17 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:07:18.022 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:17 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/396805322' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:07:18.022 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:17 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:07:18.030 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:18.056 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:18.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:18.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:18.158 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:18.180 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:18.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:18.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:18.282 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:18.308 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:18.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:18.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:18.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:17 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/396805322' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:07:18.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:17 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:07:18.410 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:18.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:18.433 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:18.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:18.534 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:18.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:18.561 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:18.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:18.662 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:18.688 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:18.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:18.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:18.789 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:18.815 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:18.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:18.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:18.917 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:18.944 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:18.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:18.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:19.046 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:19.071 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:19.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:19.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:19.173 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:19.199 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:19.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:19.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:19.301 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:19.325 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:19.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:19.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:18 smithi179 ceph-mon[93519]: pgmap v2304: 89 pgs: 37 active+undersized, 17 active+undersized+degraded, 35 active+clean; 47 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 892 op/s; 487/2301 objects degraded (21.165%) 2024-08-17T18:07:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:18 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:07:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:18 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:07:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:18 smithi179 ceph-mon[93519]: osdmap e1324: 8 total, 7 up, 5 in 2024-08-17T18:07:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:18 smithi130 ceph-mon[105521]: pgmap v2304: 89 pgs: 37 active+undersized, 17 active+undersized+degraded, 35 active+clean; 47 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 892 op/s; 487/2301 objects degraded (21.165%) 2024-08-17T18:07:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:18 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:07:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:18 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:07:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:18 smithi130 ceph-mon[105521]: osdmap e1324: 8 total, 7 up, 5 in 2024-08-17T18:07:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:18 smithi130 ceph-mon[110068]: pgmap v2304: 89 pgs: 37 active+undersized, 17 active+undersized+degraded, 35 active+clean; 47 MiB data, 5.7 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 892 op/s; 487/2301 objects degraded (21.165%) 2024-08-17T18:07:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:18 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:07:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:18 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:07:19.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:18 smithi130 ceph-mon[110068]: osdmap e1324: 8 total, 7 up, 5 in 2024-08-17T18:07:19.427 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:19.451 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:19.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:19.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:19.553 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:19.576 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:19.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:19.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:19.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:19.703 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:19.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:19.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:19.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:19.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:19.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:19.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:19.932 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:19.956 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:19.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:19.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:20.058 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:20.082 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:20.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:20.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:20.139 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 7] out_osds: [5, 4, 6] dead_osds: [3] live_osds: [1, 2, 4, 5, 6, 7, 0] 2024-08-17T18:07:20.139 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:07:20.143 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:07:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:07:19] "GET /metrics HTTP/1.1" 200 36778 "" "Prometheus/2.51.0" 2024-08-17T18:07:20.184 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:20.208 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:20.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:20.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:20.310 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:20.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:20.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:20.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:20.437 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:20.462 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:20.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:20.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:20.564 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:20.587 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:20.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:20.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:20.589 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:07:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:07:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:07:20.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:20.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:20.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:20.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:20.816 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:20.838 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:20.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:20.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:20.940 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:20.962 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:20.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:20.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:21.064 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:21.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:21.088 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:21.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:21.118 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:20 smithi130 ceph-mon[110068]: pgmap v2306: 89 pgs: 40 active+undersized, 14 active+undersized+degraded, 35 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 672 op/s; 14/78 objects degraded (17.949%) 2024-08-17T18:07:21.118 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:20 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 14/78 objects degraded (17.949%), 14 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:21.118 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:20 smithi130 ceph-mon[105521]: pgmap v2306: 89 pgs: 40 active+undersized, 14 active+undersized+degraded, 35 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 672 op/s; 14/78 objects degraded (17.949%) 2024-08-17T18:07:21.119 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:20 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 14/78 objects degraded (17.949%), 14 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:21.189 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:21.211 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:21.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:21.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:21.313 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:21.335 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:21.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:21.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:21.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:20 smithi179 ceph-mon[93519]: pgmap v2306: 89 pgs: 40 active+undersized, 14 active+undersized+degraded, 35 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 672 op/s; 14/78 objects degraded (17.949%) 2024-08-17T18:07:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:20 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 14/78 objects degraded (17.949%), 14 pgs degraded, 54 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:21.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:21.458 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:21.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:21.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:21.560 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:21.580 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:21.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:21.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:21.682 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:21.704 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:21.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:21.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:21.806 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:21.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:21.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:21.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:21.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:21.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:21.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:21.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:21.966 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:07:21.966 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:07:21.972 INFO:teuthology.orchestra.run.smithi130.stderr:pool 'unique_pool_2' removed 2024-08-17T18:07:22.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:22.072 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:22.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:22.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:22.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:21 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2667751772' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:07:22.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:21 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2667751772' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:07:22.174 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:22.196 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:22.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:22.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:22.298 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:22.319 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:22.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:22.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:22.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:21 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2667751772' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:07:22.420 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:22.443 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:22.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:22.544 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:22.570 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:22.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:22.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:22.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:07:22.646 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:07:22.647 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:07:22.665 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:07:22.665 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:07:22.672 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:22.681 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:07:22.681 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:07:22.682 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:07:22.693 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:22.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:22.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:22.697 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:07:22.697 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:07:22.713 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:07:22.713 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:07:22.729 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:07:22.729 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:07:22.744 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:07:22.744 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:07:22.795 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:22.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:22.818 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:22.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:22.919 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:22.941 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:22.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:22.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:23.043 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:23.064 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:23.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:23.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:23.166 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:23.188 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:23.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:23.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:23.290 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:23.311 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:23.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:23.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:22 smithi179 ceph-mon[93519]: pgmap v2307: 89 pgs: 40 active+undersized, 14 active+undersized+degraded, 35 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 672 op/s; 14/78 objects degraded (17.949%) 2024-08-17T18:07:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:22 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2667751772' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:07:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:22 smithi179 ceph-mon[93519]: osdmap e1325: 8 total, 7 up, 5 in 2024-08-17T18:07:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:22 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2461169790' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:23.413 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:23.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:22 smithi130 ceph-mon[105521]: pgmap v2307: 89 pgs: 40 active+undersized, 14 active+undersized+degraded, 35 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 672 op/s; 14/78 objects degraded (17.949%) 2024-08-17T18:07:23.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:22 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2667751772' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:07:23.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:22 smithi130 ceph-mon[105521]: osdmap e1325: 8 total, 7 up, 5 in 2024-08-17T18:07:23.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:22 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2461169790' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:22 smithi130 ceph-mon[110068]: pgmap v2307: 89 pgs: 40 active+undersized, 14 active+undersized+degraded, 35 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 0 B/s wr, 672 op/s; 14/78 objects degraded (17.949%) 2024-08-17T18:07:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:22 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2667751772' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:07:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:22 smithi130 ceph-mon[110068]: osdmap e1325: 8 total, 7 up, 5 in 2024-08-17T18:07:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:22 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2461169790' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:23.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:23.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:23.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:23.474 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:07:23.475 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:07:23.535 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:23.557 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:23.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:23.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:23.659 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:23.680 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:23.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:23.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:23.780 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T18:07:23.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:23.802 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:23.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:23.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:23.904 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:23.926 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:23.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:23.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:24.028 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:24.050 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:24.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:24.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:24.152 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:24.174 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T18:07:24.175 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T18:07:24.175 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-08-17T18:07:24.175 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_3 16 2024-08-17T18:07:24.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:24.177 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:24.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:24.275 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:24.277 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:24.301 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:24.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:24.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:24.403 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:24.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:24.426 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:24.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:24.527 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:24.552 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:24.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:24.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:24.654 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:24.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:24.677 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:24.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:24.779 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:24.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:24.801 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:24.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:24.902 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:24.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:24.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:24.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:25.010 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:25.029 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:25.042 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:25.054 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:25.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:25.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:25.156 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:25.177 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:25.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:25.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:25.251 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:24 smithi130 ceph-mon[105521]: pgmap v2309: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:25.251 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:24 smithi130 ceph-mon[110068]: pgmap v2309: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:25.279 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:25.300 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:25.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:25.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:25.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:24 smithi179 ceph-mon[93519]: pgmap v2309: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:25.401 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:25.423 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:25.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:25.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:25.476 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:07:25.525 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:25.546 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:25.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:25.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:25.648 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:25.674 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:25.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:25.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:25.775 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:25.797 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:25.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:25.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:25.899 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:25.925 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:25.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:25.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:26.027 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:26.051 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:26.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:26.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:26.153 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:26.175 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:26.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:26.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:26.245 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:25 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 14/78 objects degraded (17.949%), 9 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:26.245 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:25 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 14/78 objects degraded (17.949%), 9 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:26.326 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:26.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:25 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 14/78 objects degraded (17.949%), 9 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:26.352 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:26.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:26.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:26.454 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:26.479 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:26.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:26.580 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:26.606 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:26.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:26.707 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:26.729 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:26.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:26.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:26.831 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:26.853 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:26.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:26.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:26.955 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:26.980 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:26.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:26.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:27.082 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:27.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:27.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:27.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:27.208 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:27.231 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:27.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:27.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:27.332 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:27.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:26 smithi179 ceph-mon[93519]: pgmap v2310: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:27.357 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:27.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:27.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:27.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:26 smithi130 ceph-mon[105521]: pgmap v2310: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:27.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:26 smithi130 ceph-mon[110068]: pgmap v2310: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:27.460 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:27.484 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:27.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:27.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:27.586 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:27.611 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:27.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:27.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:27.713 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:27.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:27.736 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:27.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:27.837 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:27.860 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:27.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:27.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:27.962 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:27.987 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:27.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:27.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:28.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:28.110 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:28.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:28.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:28.212 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:28.238 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:28.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:28.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:28.340 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:28.363 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:28.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:28.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:28.465 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:28.491 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:28.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:28.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:28.551 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:28.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:07:28.552 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:28 smithi130 ceph-mon[105521]: pgmap v2311: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:28.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:28.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:07:28.552 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:28 smithi130 ceph-mon[110068]: pgmap v2311: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:28.593 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:28.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:07:28.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:28 smithi179 ceph-mon[93519]: pgmap v2311: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:28.618 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:28.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:28.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:28.649 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:07:28.650 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:07:28.650 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:07:28.650 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:07:28.650 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.650 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79507 46.b 46:d8bdbef4:::benchmark_data_smithi130_267720_object79506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:07:28.650 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.615201+0000", 2024-08-17T18:07:28.650 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 418.032196258, 2024-08-17T18:07:28.650 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.40324494700000002, 2024-08-17T18:07:28.650 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.650 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.650 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:07:28.651 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.651 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.651 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.651 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79507 2024-08-17T18:07:28.651 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.651 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.651 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.651 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.651 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.615201+0000", 2024-08-17T18:07:28.651 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.651 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.651 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.651 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.652 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.615201+0000", 2024-08-17T18:07:28.652 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.652 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.652 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.652 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.652 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.615201+0000", 2024-08-17T18:07:28.652 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4600000000000005e-07 2024-08-17T18:07:28.652 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.652 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.652 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.652 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.615223+0000", 2024-08-17T18:07:28.652 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1747e-05 2024-08-17T18:07:28.652 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.652 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.653 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.653 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.615572+0000", 2024-08-17T18:07:28.653 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00034875599999999998 2024-08-17T18:07:28.653 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.653 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.653 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.653 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.615579+0000", 2024-08-17T18:07:28.653 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9909999999999996e-06 2024-08-17T18:07:28.653 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.653 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.653 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.653 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.638229+0000", 2024-08-17T18:07:28.653 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022650623000000002 2024-08-17T18:07:28.653 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.654 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.654 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.654 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.638369+0000", 2024-08-17T18:07:28.654 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013929099999999999 2024-08-17T18:07:28.654 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.654 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.654 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T18:07:28.654 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.638530+0000", 2024-08-17T18:07:28.654 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016180900000000001 2024-08-17T18:07:28.654 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.654 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.654 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.654 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644666+0000", 2024-08-17T18:07:28.654 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0061359630000000004 2024-08-17T18:07:28.655 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.655 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.655 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.655 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644732+0000", 2024-08-17T18:07:28.655 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5506999999999995e-05 2024-08-17T18:07:28.655 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.655 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.655 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.655 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.018333+0000", 2024-08-17T18:07:28.655 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37360103 2024-08-17T18:07:28.655 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.655 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.655 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:07:28.655 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.018411+0000", 2024-08-17T18:07:28.656 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8343999999999994e-05 2024-08-17T18:07:28.656 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.656 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.656 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.656 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.018446+0000", 2024-08-17T18:07:28.656 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.434e-05 2024-08-17T18:07:28.656 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.656 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.656 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.656 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.656 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.657 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79518 46.2 46:404c947b:::benchmark_data_smithi130_267720_object79517:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:07:28.657 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.623412+0000", 2024-08-17T18:07:28.657 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 418.023985063, 2024-08-17T18:07:28.657 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.39487024500000001, 2024-08-17T18:07:28.657 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.657 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.657 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:07:28.657 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.657 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.657 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.657 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79518 2024-08-17T18:07:28.657 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.658 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.658 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.658 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.658 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623412+0000", 2024-08-17T18:07:28.658 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.658 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.658 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.658 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.658 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623412+0000", 2024-08-17T18:07:28.658 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.658 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.658 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.659 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.659 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623413+0000", 2024-08-17T18:07:28.659 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3900000000000004e-07 2024-08-17T18:07:28.659 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.659 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.659 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.659 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623437+0000", 2024-08-17T18:07:28.659 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3998999999999998e-05 2024-08-17T18:07:28.659 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.659 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.659 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.659 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623449+0000", 2024-08-17T18:07:28.659 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2065000000000001e-05 2024-08-17T18:07:28.659 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.660 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.660 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.660 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623460+0000", 2024-08-17T18:07:28.660 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1534e-05 2024-08-17T18:07:28.660 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.660 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.660 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.660 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623559+0000", 2024-08-17T18:07:28.660 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8368000000000004e-05 2024-08-17T18:07:28.660 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.660 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.660 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.661 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623674+0000", 2024-08-17T18:07:28.661 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000115561 2024-08-17T18:07:28.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.661 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.661 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:07:28.661 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623803+0000", 2024-08-17T18:07:28.661 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012843300000000001 2024-08-17T18:07:28.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.661 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.661 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.661 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.630231+0000", 2024-08-17T18:07:28.661 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0064287770000000001 2024-08-17T18:07:28.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.662 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.662 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.662 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.630245+0000", 2024-08-17T18:07:28.662 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4042e-05 2024-08-17T18:07:28.662 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.662 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.662 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.662 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.018111+0000", 2024-08-17T18:07:28.662 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.387865919 2024-08-17T18:07:28.662 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.662 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.662 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:07:28.663 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.018228+0000", 2024-08-17T18:07:28.663 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011705300000000001 2024-08-17T18:07:28.663 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.663 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.663 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.663 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.018282+0000", 2024-08-17T18:07:28.663 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3655e-05 2024-08-17T18:07:28.663 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.663 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.663 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.663 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.664 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.664 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79519 46.2 46:4d9ed74c:::benchmark_data_smithi130_267720_object79518:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:07:28.664 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.623580+0000", 2024-08-17T18:07:28.664 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 418.02381702100001, 2024-08-17T18:07:28.664 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.40023455299999999, 2024-08-17T18:07:28.664 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.664 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.664 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:07:28.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.665 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.665 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79519 2024-08-17T18:07:28.665 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.665 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.665 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.665 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.665 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623580+0000", 2024-08-17T18:07:28.666 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.666 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.666 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.666 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.666 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623580+0000", 2024-08-17T18:07:28.666 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.666 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.666 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.666 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.667 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623581+0000", 2024-08-17T18:07:28.667 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5300000000000003e-07 2024-08-17T18:07:28.667 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.667 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.667 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.667 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623603+0000", 2024-08-17T18:07:28.667 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1936000000000001e-05 2024-08-17T18:07:28.667 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.668 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.668 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.668 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623614+0000", 2024-08-17T18:07:28.668 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1164e-05 2024-08-17T18:07:28.668 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.668 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.668 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.668 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.623621+0000", 2024-08-17T18:07:28.669 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7780000000000002e-06 2024-08-17T18:07:28.669 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.669 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.669 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.669 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.626874+0000", 2024-08-17T18:07:28.669 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032538810000000001 2024-08-17T18:07:28.669 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.669 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.669 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.669 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627002+0000", 2024-08-17T18:07:28.670 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000127176 2024-08-17T18:07:28.670 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.670 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.670 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:07:28.670 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.627163+0000", 2024-08-17T18:07:28.670 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000160889 2024-08-17T18:07:28.670 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.670 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.671 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.671 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.630833+0000", 2024-08-17T18:07:28.671 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0036703339999999999 2024-08-17T18:07:28.671 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.671 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.671 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.671 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.630951+0000", 2024-08-17T18:07:28.671 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000117704 2024-08-17T18:07:28.671 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.671 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.671 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.671 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023677+0000", 2024-08-17T18:07:28.671 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.39272642699999999 2024-08-17T18:07:28.671 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023759+0000", 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2211000000000003e-05 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023814+0000", 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.52e-05 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.672 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79531 46.2 46:4c0c978d:::benchmark_data_smithi130_267720_object79530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:07:28.673 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.640037+0000", 2024-08-17T18:07:28.673 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 418.00735967899999, 2024-08-17T18:07:28.673 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38398499600000002, 2024-08-17T18:07:28.673 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.673 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.673 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:07:28.673 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.673 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.673 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.673 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79531 2024-08-17T18:07:28.673 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.673 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.673 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.673 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.674 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640037+0000", 2024-08-17T18:07:28.674 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.674 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.674 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.674 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.674 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640037+0000", 2024-08-17T18:07:28.674 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.674 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.674 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.674 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.674 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640038+0000", 2024-08-17T18:07:28.674 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9999999999999995e-07 2024-08-17T18:07:28.674 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.675 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.675 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.675 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640059+0000", 2024-08-17T18:07:28.675 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0310999999999999e-05 2024-08-17T18:07:28.675 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.675 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.675 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.675 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640071+0000", 2024-08-17T18:07:28.675 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2128e-05 2024-08-17T18:07:28.675 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.675 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.675 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.675 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640077+0000", 2024-08-17T18:07:28.675 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6379999999999998e-06 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640129+0000", 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2122e-05 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640254+0000", 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000124189 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.640382+0000", 2024-08-17T18:07:28.676 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012849700000000001 2024-08-17T18:07:28.677 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.677 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.677 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.677 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.643360+0000", 2024-08-17T18:07:28.677 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029779799999999999 2024-08-17T18:07:28.677 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.677 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.677 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.677 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.643375+0000", 2024-08-17T18:07:28.677 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5029e-05 2024-08-17T18:07:28.677 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.677 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.677 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.678 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023855+0000", 2024-08-17T18:07:28.678 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38047954299999998 2024-08-17T18:07:28.678 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.678 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.678 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:07:28.678 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.023979+0000", 2024-08-17T18:07:28.678 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012451199999999999 2024-08-17T18:07:28.678 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.678 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.678 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.678 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.024022+0000", 2024-08-17T18:07:28.678 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3047000000000003e-05 2024-08-17T18:07:28.679 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.679 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.679 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.679 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.679 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.679 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79536 46.0 46:078355fd:::benchmark_data_smithi130_267720_object79535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:07:28.679 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.644048+0000", 2024-08-17T18:07:28.679 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 418.00334868700003, 2024-08-17T18:07:28.679 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37702704300000001, 2024-08-17T18:07:28.679 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.679 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.679 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:07:28.680 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.680 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.680 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.680 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79536 2024-08-17T18:07:28.680 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.680 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.680 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.680 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.680 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644048+0000", 2024-08-17T18:07:28.680 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.680 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.680 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.680 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.681 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644048+0000", 2024-08-17T18:07:28.681 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.681 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.681 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.681 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644049+0000", 2024-08-17T18:07:28.681 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2699999999999998e-07 2024-08-17T18:07:28.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.681 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.681 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.681 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644067+0000", 2024-08-17T18:07:28.681 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7757999999999998e-05 2024-08-17T18:07:28.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.681 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644076+0000", 2024-08-17T18:07:28.682 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.115e-06 2024-08-17T18:07:28.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.682 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644082+0000", 2024-08-17T18:07:28.682 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1140000000000002e-06 2024-08-17T18:07:28.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.682 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644134+0000", 2024-08-17T18:07:28.682 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1487999999999999e-05 2024-08-17T18:07:28.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.683 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.683 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.683 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644269+0000", 2024-08-17T18:07:28.683 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013505599999999999 2024-08-17T18:07:28.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.683 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.683 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:07:28.683 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.644397+0000", 2024-08-17T18:07:28.683 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000127952 2024-08-17T18:07:28.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.683 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.683 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.684 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.646717+0000", 2024-08-17T18:07:28.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002320749 2024-08-17T18:07:28.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.684 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.684 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.647727+0000", 2024-08-17T18:07:28.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001009472 2024-08-17T18:07:28.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.684 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.684 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.020974+0000", 2024-08-17T18:07:28.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37324734599999998 2024-08-17T18:07:28.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.021032+0000", 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8298000000000003e-05 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.021075+0000", 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2867999999999997e-05 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:79539 46.2 46:487571cc:::benchmark_data_smithi130_267720_object79538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:07:28.685 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:30.648144+0000", 2024-08-17T18:07:28.686 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 417.999253135, 2024-08-17T18:07:28.686 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.38286041199999998, 2024-08-17T18:07:28.686 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.686 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.686 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:07:28.686 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.686 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.686 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.686 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 79539 2024-08-17T18:07:28.686 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.686 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.686 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.686 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648144+0000", 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648144+0000", 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648145+0000", 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2359999999999999e-06 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.687 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648168+0000", 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2847e-05 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648181+0000", 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2819999999999999e-05 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648188+0000", 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1520000000000003e-06 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.688 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648241+0000", 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3285999999999998e-05 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648387+0000", 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014551500000000001 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.648543+0000", 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000155957 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.690 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.650656+0000", 2024-08-17T18:07:28.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021130250000000001 2024-08-17T18:07:28.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.690 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.690 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.690 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:30.650923+0000", 2024-08-17T18:07:28.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00026742200000000002 2024-08-17T18:07:28.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.690 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.690 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.690 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.030833+0000", 2024-08-17T18:07:28.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.37991011499999999 2024-08-17T18:07:28.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.691 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.691 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:07:28.691 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.030953+0000", 2024-08-17T18:07:28.691 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000120186 2024-08-17T18:07:28.691 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.691 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.691 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.691 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:31.031004+0000", 2024-08-17T18:07:28.691 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0850999999999998e-05 2024-08-17T18:07:28.691 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.691 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.691 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.692 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.692 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.692 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:95869 46.d 46:bf83775d:::benchmark_data_smithi130_267720_object95868:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-08-17T18:07:28.692 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:49.269515+0000", 2024-08-17T18:07:28.692 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 399.377882161, 2024-08-17T18:07:28.692 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.36132123999999999, 2024-08-17T18:07:28.692 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.692 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.692 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:07:28.692 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.692 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.692 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.692 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 95869 2024-08-17T18:07:28.693 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.693 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.693 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.693 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.693 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269515+0000", 2024-08-17T18:07:28.693 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.693 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.693 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.693 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.693 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269515+0000", 2024-08-17T18:07:28.693 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.693 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.694 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.694 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.694 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269515+0000", 2024-08-17T18:07:28.694 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.68e-07 2024-08-17T18:07:28.694 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.694 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.694 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.694 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269531+0000", 2024-08-17T18:07:28.694 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6021000000000001e-05 2024-08-17T18:07:28.694 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.694 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.694 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.694 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269538+0000", 2024-08-17T18:07:28.694 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0419999999999997e-06 2024-08-17T18:07:28.695 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.695 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.695 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.695 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269544+0000", 2024-08-17T18:07:28.695 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2700000000000004e-06 2024-08-17T18:07:28.695 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.695 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.695 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.695 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.334777+0000", 2024-08-17T18:07:28.695 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.065232993000000003 2024-08-17T18:07:28.695 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.695 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.695 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.695 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.334940+0000", 2024-08-17T18:07:28.696 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000162898 2024-08-17T18:07:28.696 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.696 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:07:28.696 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.335062+0000", 2024-08-17T18:07:28.696 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012245800000000001 2024-08-17T18:07:28.696 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.696 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.696 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.344169+0000", 2024-08-17T18:07:28.696 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0091074559999999999 2024-08-17T18:07:28.696 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.344213+0000", 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3868000000000001e-05 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630486+0000", 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.286272472 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630682+0000", 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000196422 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.697 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.698 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.698 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.630836+0000", 2024-08-17T18:07:28.698 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015377199999999999 2024-08-17T18:07:28.698 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.698 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.698 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.698 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.698 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.698 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:95871 46.d 46:bc19484e:::benchmark_data_smithi130_267720_object95870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-08-17T18:07:28.698 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:00:49.269561+0000", 2024-08-17T18:07:28.698 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 399.37783566000002, 2024-08-17T18:07:28.698 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.36284446199999998, 2024-08-17T18:07:28.698 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.698 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.699 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:07:28.699 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.699 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.699 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.699 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 95871 2024-08-17T18:07:28.699 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.699 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.699 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.699 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.699 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269561+0000", 2024-08-17T18:07:28.699 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.700 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.700 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.700 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.700 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269561+0000", 2024-08-17T18:07:28.700 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.700 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.700 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.700 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.700 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269562+0000", 2024-08-17T18:07:28.700 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0800000000000005e-07 2024-08-17T18:07:28.700 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.700 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.700 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.701 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269580+0000", 2024-08-17T18:07:28.701 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8471e-05 2024-08-17T18:07:28.701 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.701 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.701 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.701 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269587+0000", 2024-08-17T18:07:28.701 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.832e-06 2024-08-17T18:07:28.701 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.701 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.701 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.702 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.269592+0000", 2024-08-17T18:07:28.702 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8189999999999999e-06 2024-08-17T18:07:28.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.702 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.702 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.702 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.338075+0000", 2024-08-17T18:07:28.702 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.068482687 2024-08-17T18:07:28.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.702 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.702 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.702 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.338231+0000", 2024-08-17T18:07:28.702 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000156169 2024-08-17T18:07:28.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:07:28.703 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.338381+0000", 2024-08-17T18:07:28.703 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000150748 2024-08-17T18:07:28.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.703 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.344225+0000", 2024-08-17T18:07:28.703 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0058432800000000002 2024-08-17T18:07:28.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.704 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.344270+0000", 2024-08-17T18:07:28.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4957999999999999e-05 2024-08-17T18:07:28.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.704 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.632288+0000", 2024-08-17T18:07:28.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.288017884 2024-08-17T18:07:28.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.704 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:07:28.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.632368+0000", 2024-08-17T18:07:28.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0476000000000007e-05 2024-08-17T18:07:28.705 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.705 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.705 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.705 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:00:49.632406+0000", 2024-08-17T18:07:28.705 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7629999999999997e-05 2024-08-17T18:07:28.705 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.705 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.705 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.705 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.705 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.705 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195096 46.9 46:9e985471:::benchmark_data_smithi130_267720_object87113:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-08-17T18:07:28.705 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.852302+0000", 2024-08-17T18:07:28.705 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 275.79509517399998, 2024-08-17T18:07:28.705 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.309431236, 2024-08-17T18:07:28.706 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.706 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.706 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:07:28.706 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.706 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.706 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.706 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195096 2024-08-17T18:07:28.707 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.707 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.707 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.707 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.707 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852302+0000", 2024-08-17T18:07:28.707 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.707 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.707 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.707 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.707 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852302+0000", 2024-08-17T18:07:28.707 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.707 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.707 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.707 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.708 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852303+0000", 2024-08-17T18:07:28.708 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.33e-06 2024-08-17T18:07:28.708 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.708 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.708 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.708 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852308+0000", 2024-08-17T18:07:28.708 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4079999999999997e-06 2024-08-17T18:07:28.708 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.708 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.708 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.708 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852319+0000", 2024-08-17T18:07:28.708 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0382e-05 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852325+0000", 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4649999999999999e-06 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852449+0000", 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012352000000000001 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852579+0000", 2024-08-17T18:07:28.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013025800000000001 2024-08-17T18:07:28.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.710 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,6", 2024-08-17T18:07:28.710 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852699+0000", 2024-08-17T18:07:28.710 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011966799999999999 2024-08-17T18:07:28.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.710 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.710 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.856209+0000", 2024-08-17T18:07:28.710 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0035104649999999999 2024-08-17T18:07:28.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873340+0000", 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017130537000000001 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934040+0000", 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.060700089999999998 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934049+0000", 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1239999999999994e-06 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877962+0000", 2024-08-17T18:07:28.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94391303400000004 2024-08-17T18:07:28.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.712 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877971+0000", 2024-08-17T18:07:28.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0100000000000001e-06 2024-08-17T18:07:28.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.712 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.713 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.161671+0000", 2024-08-17T18:07:28.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.283700372 2024-08-17T18:07:28.713 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.713 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.713 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.161733+0000", 2024-08-17T18:07:28.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1573000000000002e-05 2024-08-17T18:07:28.713 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.713 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.713 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.713 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.713 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195105 46.9 46:94a33cb3:::benchmark_data_smithi130_267720_object87122:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-08-17T18:07:28.713 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.862150+0000", 2024-08-17T18:07:28.714 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 275.78524720399997, 2024-08-17T18:07:28.714 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3538093619999998, 2024-08-17T18:07:28.714 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.714 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.714 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:07:28.714 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.714 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.714 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.714 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195105 2024-08-17T18:07:28.714 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.714 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.714 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.714 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.715 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862150+0000", 2024-08-17T18:07:28.715 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.715 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.715 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.715 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.715 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862150+0000", 2024-08-17T18:07:28.715 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.715 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.715 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.715 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.715 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862151+0000", 2024-08-17T18:07:28.715 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3170000000000001e-06 2024-08-17T18:07:28.715 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.716 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.716 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.716 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862157+0000", 2024-08-17T18:07:28.716 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5040000000000002e-06 2024-08-17T18:07:28.716 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.716 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.716 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.716 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862169+0000", 2024-08-17T18:07:28.716 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2520999999999999e-05 2024-08-17T18:07:28.716 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.716 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.716 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.716 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862176+0000", 2024-08-17T18:07:28.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1269999999999997e-06 2024-08-17T18:07:28.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.717 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.868137+0000", 2024-08-17T18:07:28.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0059607250000000001 2024-08-17T18:07:28.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.717 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.868274+0000", 2024-08-17T18:07:28.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013746200000000001 2024-08-17T18:07:28.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.717 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,6", 2024-08-17T18:07:28.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.868404+0000", 2024-08-17T18:07:28.718 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012964699999999999 2024-08-17T18:07:28.718 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.718 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.718 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.879188+0000", 2024-08-17T18:07:28.718 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010783915 2024-08-17T18:07:28.718 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.718 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.718 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934076+0000", 2024-08-17T18:07:28.718 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.054887752999999997 2024-08-17T18:07:28.718 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934083+0000", 2024-08-17T18:07:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9859999999999997e-06 2024-08-17T18:07:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878000+0000", 2024-08-17T18:07:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94391712299999997 2024-08-17T18:07:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878009+0000", 2024-08-17T18:07:28.719 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.884e-06 2024-08-17T18:07:28.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.720 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.720 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.720 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.161985+0000", 2024-08-17T18:07:28.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2839767369999999 2024-08-17T18:07:28.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.720 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.720 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:07:28.720 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.162438+0000", 2024-08-17T18:07:28.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00045227700000000001 2024-08-17T18:07:28.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.720 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.720 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.720 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215917+0000", 2024-08-17T18:07:28.721 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.053479086000000002 2024-08-17T18:07:28.721 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215959+0000", 2024-08-17T18:07:28.721 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2298000000000001e-05 2024-08-17T18:07:28.721 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.721 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.721 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.721 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.721 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195107 46.9 46:9aacae76:::benchmark_data_smithi130_267720_object87124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-08-17T18:07:28.722 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.862322+0000", 2024-08-17T18:07:28.722 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 275.78507491699997, 2024-08-17T18:07:28.722 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3730563999999998, 2024-08-17T18:07:28.722 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.722 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.722 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:07:28.722 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.722 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.722 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.722 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195107 2024-08-17T18:07:28.722 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.722 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.722 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.723 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.723 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862322+0000", 2024-08-17T18:07:28.723 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.723 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.723 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.723 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.723 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862322+0000", 2024-08-17T18:07:28.723 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.723 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.723 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.723 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.723 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862323+0000", 2024-08-17T18:07:28.723 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3200000000000005e-07 2024-08-17T18:07:28.723 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.724 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.724 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.724 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862326+0000", 2024-08-17T18:07:28.724 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3249999999999999e-06 2024-08-17T18:07:28.724 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.724 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.724 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.724 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:28.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862335+0000", 2024-08-17T18:07:28.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0149999999999992e-06 2024-08-17T18:07:28.726 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.726 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.726 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862341+0000", 2024-08-17T18:07:28.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9909999999999999e-06 2024-08-17T18:07:28.726 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.726 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873392+0000", 2024-08-17T18:07:28.727 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011050658 2024-08-17T18:07:28.727 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873518+0000", 2024-08-17T18:07:28.727 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012659100000000001 2024-08-17T18:07:28.727 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,6", 2024-08-17T18:07:28.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873640+0000", 2024-08-17T18:07:28.727 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012183199999999999 2024-08-17T18:07:28.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.728 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934107+0000", 2024-08-17T18:07:28.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.060466796000000003 2024-08-17T18:07:28.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.728 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934114+0000", 2024-08-17T18:07:28.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0690000000000004e-06 2024-08-17T18:07:28.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.728 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878038+0000", 2024-08-17T18:07:28.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94392365499999997 2024-08-17T18:07:28.729 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.729 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.729 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.729 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878047+0000", 2024-08-17T18:07:28.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9600000000000006e-06 2024-08-17T18:07:28.729 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.729 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.729 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.729 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215977+0000", 2024-08-17T18:07:28.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3379302630000001 2024-08-17T18:07:28.729 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.729 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.729 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:07:28.730 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216122+0000", 2024-08-17T18:07:28.730 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014515900000000001 2024-08-17T18:07:28.730 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.730 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.730 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.730 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.235324+0000", 2024-08-17T18:07:28.730 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019201732999999999 2024-08-17T18:07:28.730 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.730 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.730 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.730 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.235378+0000", 2024-08-17T18:07:28.730 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4721e-05 2024-08-17T18:07:28.730 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.731 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.731 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.731 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.731 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.731 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195109 46.9 46:941627c3:::benchmark_data_smithi130_267720_object87126:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-08-17T18:07:28.731 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.870700+0000", 2024-08-17T18:07:28.731 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 275.77669714299998, 2024-08-17T18:07:28.731 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3651783769999999, 2024-08-17T18:07:28.731 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.731 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.731 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:07:28.731 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.731 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.731 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.732 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195109 2024-08-17T18:07:28.732 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.732 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.732 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.732 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.732 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870700+0000", 2024-08-17T18:07:28.732 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.732 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.732 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.732 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.732 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870700+0000", 2024-08-17T18:07:28.732 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.732 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.733 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870701+0000", 2024-08-17T18:07:28.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.415e-06 2024-08-17T18:07:28.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.733 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870704+0000", 2024-08-17T18:07:28.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8899999999999999e-06 2024-08-17T18:07:28.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.733 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.734 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870712+0000", 2024-08-17T18:07:28.734 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2579999999999995e-06 2024-08-17T18:07:28.734 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.734 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.734 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.734 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870718+0000", 2024-08-17T18:07:28.734 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.327e-06 2024-08-17T18:07:28.734 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.734 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.734 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.734 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.879223+0000", 2024-08-17T18:07:28.735 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0085048620000000002 2024-08-17T18:07:28.735 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.735 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.735 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.735 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.879355+0000", 2024-08-17T18:07:28.735 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013201599999999999 2024-08-17T18:07:28.735 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.735 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.735 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,6", 2024-08-17T18:07:28.735 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.879499+0000", 2024-08-17T18:07:28.735 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014413800000000001 2024-08-17T18:07:28.735 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.735 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.736 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.736 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934138+0000", 2024-08-17T18:07:28.736 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.05463925 2024-08-17T18:07:28.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.736 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.736 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.736 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934145+0000", 2024-08-17T18:07:28.736 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7399999999999998e-06 2024-08-17T18:07:28.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.736 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.736 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.736 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878105+0000", 2024-08-17T18:07:28.736 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94396076500000003 2024-08-17T18:07:28.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.737 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.737 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.737 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878115+0000", 2024-08-17T18:07:28.737 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3489999999999998e-06 2024-08-17T18:07:28.737 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.737 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.737 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.737 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216148+0000", 2024-08-17T18:07:28.737 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3380333520000001 2024-08-17T18:07:28.737 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.737 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.737 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:07:28.737 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216284+0000", 2024-08-17T18:07:28.738 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000135513 2024-08-17T18:07:28.738 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.738 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.738 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.738 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.235773+0000", 2024-08-17T18:07:28.738 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019489216 2024-08-17T18:07:28.738 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.738 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.738 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.738 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.235878+0000", 2024-08-17T18:07:28.738 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010528599999999999 2024-08-17T18:07:28.738 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.738 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.738 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.739 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.739 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.739 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195099 46.f 46:fad5c3c4:::benchmark_data_smithi130_267720_object87116:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:07:28.739 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.873814+0000", 2024-08-17T18:07:28.739 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 275.77358334799999, 2024-08-17T18:07:28.739 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.301145886, 2024-08-17T18:07:28.739 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.739 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.739 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:07:28.739 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.739 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.739 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.739 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195099 2024-08-17T18:07:28.740 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.740 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.740 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.740 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.740 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873814+0000", 2024-08-17T18:07:28.740 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.740 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.740 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.740 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.740 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873814+0000", 2024-08-17T18:07:28.740 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.740 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.740 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.740 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873816+0000", 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9769999999999999e-06 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873823+0000", 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2590000000000003e-06 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873834+0000", 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1243000000000001e-05 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.741 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.742 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873843+0000", 2024-08-17T18:07:28.742 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6079999999999996e-06 2024-08-17T18:07:28.742 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.742 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.742 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.742 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878460+0000", 2024-08-17T18:07:28.742 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046169899999999996 2024-08-17T18:07:28.742 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.742 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.742 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op must wait for map", 2024-08-17T18:07:28.742 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878467+0000", 2024-08-17T18:07:28.742 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9399999999999996e-06 2024-08-17T18:07:28.742 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.743 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.743 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.932773+0000", 2024-08-17T18:07:28.743 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.054306074000000003 2024-08-17T18:07:28.743 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.743 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.743 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.932783+0000", 2024-08-17T18:07:28.743 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0214000000000001e-05 2024-08-17T18:07:28.743 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.743 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877857+0000", 2024-08-17T18:07:28.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94507391299999999 2024-08-17T18:07:28.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.744 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877867+0000", 2024-08-17T18:07:28.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0132000000000001e-05 2024-08-17T18:07:28.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.744 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.172642+0000", 2024-08-17T18:07:28.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2947751140000001 2024-08-17T18:07:28.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.174959+0000", 2024-08-17T18:07:28.745 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002317422 2024-08-17T18:07:28.745 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.745 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.745 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.745 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.745 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.745 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195108 46.f 46:f8e03b9e:::benchmark_data_smithi130_267720_object87125:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:07:28.745 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.873857+0000", 2024-08-17T18:07:28.745 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 275.77354032900001, 2024-08-17T18:07:28.745 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3014579089999998, 2024-08-17T18:07:28.745 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.745 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.746 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:07:28.746 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.746 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.746 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.746 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195108 2024-08-17T18:07:28.746 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.746 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.746 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.746 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.746 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873857+0000", 2024-08-17T18:07:28.746 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.746 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.746 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.747 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.747 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873857+0000", 2024-08-17T18:07:28.747 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.747 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.747 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.747 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.747 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873857+0000", 2024-08-17T18:07:28.747 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8599999999999998e-07 2024-08-17T18:07:28.747 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.747 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.747 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.747 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873859+0000", 2024-08-17T18:07:28.747 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2819999999999999e-06 2024-08-17T18:07:28.748 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.748 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.748 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.748 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873865+0000", 2024-08-17T18:07:28.748 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1310000000000003e-06 2024-08-17T18:07:28.748 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.748 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.748 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.748 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873870+0000", 2024-08-17T18:07:28.748 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5770000000000004e-06 2024-08-17T18:07:28.748 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.748 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.748 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.749 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878516+0000", 2024-08-17T18:07:28.749 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046464200000000001 2024-08-17T18:07:28.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.749 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting_for_map not empty", 2024-08-17T18:07:28.749 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878522+0000", 2024-08-17T18:07:28.749 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.259e-06 2024-08-17T18:07:28.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.749 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.749 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.932812+0000", 2024-08-17T18:07:28.749 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.054289865 2024-08-17T18:07:28.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.750 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.932818+0000", 2024-08-17T18:07:28.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7259999999999996e-06 2024-08-17T18:07:28.750 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.750 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877927+0000", 2024-08-17T18:07:28.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94510819099999999 2024-08-17T18:07:28.750 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.750 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877935+0000", 2024-08-17T18:07:28.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5280000000000003e-06 2024-08-17T18:07:28.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.175240+0000", 2024-08-17T18:07:28.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.297304512 2024-08-17T18:07:28.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.752 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.175314+0000", 2024-08-17T18:07:28.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4931999999999993e-05 2024-08-17T18:07:28.752 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.752 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.752 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.752 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.752 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.752 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195110 46.1 46:81ba4812:::benchmark_data_smithi130_267720_object87127:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:07:28.752 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.873876+0000", 2024-08-17T18:07:28.752 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 275.773520846, 2024-08-17T18:07:28.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.503486423, 2024-08-17T18:07:28.752 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.752 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.752 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:07:28.753 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.753 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.753 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.753 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195110 2024-08-17T18:07:28.753 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.753 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.753 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.753 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.753 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873876+0000", 2024-08-17T18:07:28.753 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.753 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.753 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.753 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.753 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873876+0000", 2024-08-17T18:07:28.754 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.754 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.754 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873876+0000", 2024-08-17T18:07:28.754 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.84e-07 2024-08-17T18:07:28.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.754 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.754 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873878+0000", 2024-08-17T18:07:28.754 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9410000000000001e-06 2024-08-17T18:07:28.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.755 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.755 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.874329+0000", 2024-08-17T18:07:28.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00045083200000000002 2024-08-17T18:07:28.755 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.755 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.755 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.755 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.874337+0000", 2024-08-17T18:07:28.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2009999999999999e-06 2024-08-17T18:07:28.755 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.755 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.755 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.755 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934172+0000", 2024-08-17T18:07:28.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.059834948999999998 2024-08-17T18:07:28.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.756 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.756 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934179+0000", 2024-08-17T18:07:28.756 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0949999999999998e-06 2024-08-17T18:07:28.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.756 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.756 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877887+0000", 2024-08-17T18:07:28.756 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94370757699999996 2024-08-17T18:07:28.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.757 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.757 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877897+0000", 2024-08-17T18:07:28.757 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6600000000000007e-06 2024-08-17T18:07:28.757 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.757 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.757 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.757 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.204539+0000", 2024-08-17T18:07:28.757 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.32664223 2024-08-17T18:07:28.757 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.757 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.757 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.758 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.206374+0000", 2024-08-17T18:07:28.758 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001834638 2024-08-17T18:07:28.758 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.758 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.758 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,6,7", 2024-08-17T18:07:28.758 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.206574+0000", 2024-08-17T18:07:28.758 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020093300000000001 2024-08-17T18:07:28.758 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.758 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.758 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.758 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.218471+0000", 2024-08-17T18:07:28.758 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011896487000000001 2024-08-17T18:07:28.758 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.759 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.218487+0000", 2024-08-17T18:07:28.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6118999999999999e-05 2024-08-17T18:07:28.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.759 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.232569+0000", 2024-08-17T18:07:28.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014081425 2024-08-17T18:07:28.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.759 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.376529+0000", 2024-08-17T18:07:28.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14396075699999999 2024-08-17T18:07:28.760 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.760 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.760 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:07:28.760 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.376663+0000", 2024-08-17T18:07:28.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013359900000000001 2024-08-17T18:07:28.760 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.760 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.760 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.760 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377362+0000", 2024-08-17T18:07:28.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00069959600000000003 2024-08-17T18:07:28.760 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.760 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.761 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.761 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.761 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.761 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195111 46.9 46:9346673d:::benchmark_data_smithi130_267720_object87128:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:07:28.761 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.878090+0000", 2024-08-17T18:07:28.761 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 275.76930691299998, 2024-08-17T18:07:28.761 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4994862809999998, 2024-08-17T18:07:28.761 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.761 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.761 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:07:28.761 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.762 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.762 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.762 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195111 2024-08-17T18:07:28.762 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.762 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.762 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.762 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.762 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878090+0000", 2024-08-17T18:07:28.762 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.762 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.763 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.763 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.763 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878090+0000", 2024-08-17T18:07:28.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.763 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.763 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.763 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.763 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878092+0000", 2024-08-17T18:07:28.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.548e-06 2024-08-17T18:07:28.763 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.763 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.763 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.763 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878097+0000", 2024-08-17T18:07:28.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.665e-06 2024-08-17T18:07:28.764 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.764 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.764 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.764 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878115+0000", 2024-08-17T18:07:28.764 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7829000000000002e-05 2024-08-17T18:07:28.764 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.764 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.764 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.764 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878123+0000", 2024-08-17T18:07:28.764 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8029999999999997e-06 2024-08-17T18:07:28.764 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.764 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.764 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934199+0000", 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.056076150999999998 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934205+0000", 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3559999999999998e-06 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878140+0000", 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.943934355 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.766 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878148+0000", 2024-08-17T18:07:28.766 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8399999999999995e-06 2024-08-17T18:07:28.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.766 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.766 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.766 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216331+0000", 2024-08-17T18:07:28.766 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3381836009999999 2024-08-17T18:07:28.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.766 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.766 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.766 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216449+0000", 2024-08-17T18:07:28.766 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000117873 2024-08-17T18:07:28.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.767 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,3,6", 2024-08-17T18:07:28.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216861+0000", 2024-08-17T18:07:28.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00041161799999999999 2024-08-17T18:07:28.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.767 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.229816+0000", 2024-08-17T18:07:28.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012955112 2024-08-17T18:07:28.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.767 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.232815+0000", 2024-08-17T18:07:28.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029990630000000002 2024-08-17T18:07:28.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.768 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.768 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.768 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.234938+0000", 2024-08-17T18:07:28.768 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002122779 2024-08-17T18:07:28.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.768 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.768 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.768 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377412+0000", 2024-08-17T18:07:28.768 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14247422800000001 2024-08-17T18:07:28.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.768 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.768 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:07:28.769 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377471+0000", 2024-08-17T18:07:28.769 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9032e-05 2024-08-17T18:07:28.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.769 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.769 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.769 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377576+0000", 2024-08-17T18:07:28.769 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010542799999999999 2024-08-17T18:07:28.769 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.769 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.769 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.769 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.769 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195121 46.1 46:8cbb5caf:::benchmark_data_smithi130_267720_object87138:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:07:28.770 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.946248+0000", 2024-08-17T18:07:28.770 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 275.70114869700001, 2024-08-17T18:07:28.770 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4315148899999999, 2024-08-17T18:07:28.770 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.770 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.770 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:07:28.770 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.770 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.770 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.770 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195121 2024-08-17T18:07:28.770 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.770 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946248+0000", 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946248+0000", 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946250+0000", 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4649999999999999e-06 2024-08-17T18:07:28.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.772 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.772 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946256+0000", 2024-08-17T18:07:28.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8839999999999999e-06 2024-08-17T18:07:28.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.772 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.772 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946270+0000", 2024-08-17T18:07:28.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3993e-05 2024-08-17T18:07:28.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.772 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.772 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946276+0000", 2024-08-17T18:07:28.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2400000000000004e-06 2024-08-17T18:07:28.773 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946334+0000", 2024-08-17T18:07:28.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8402000000000001e-05 2024-08-17T18:07:28.773 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946345+0000", 2024-08-17T18:07:28.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0645999999999999e-05 2024-08-17T18:07:28.773 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.774 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877922+0000", 2024-08-17T18:07:28.774 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.93157699800000004 2024-08-17T18:07:28.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.774 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.774 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877930+0000", 2024-08-17T18:07:28.774 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7079999999999997e-06 2024-08-17T18:07:28.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.774 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.774 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215774+0000", 2024-08-17T18:07:28.774 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.337844566 2024-08-17T18:07:28.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.775 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215906+0000", 2024-08-17T18:07:28.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013154400000000001 2024-08-17T18:07:28.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.775 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,6,7", 2024-08-17T18:07:28.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216043+0000", 2024-08-17T18:07:28.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000137288 2024-08-17T18:07:28.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.775 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.218520+0000", 2024-08-17T18:07:28.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002477492 2024-08-17T18:07:28.776 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.776 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.776 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.776 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.229810+0000", 2024-08-17T18:07:28.776 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011289123999999999 2024-08-17T18:07:28.776 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.776 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.776 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.776 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.234501+0000", 2024-08-17T18:07:28.776 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046915749999999999 2024-08-17T18:07:28.776 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.776 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.776 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377616+0000", 2024-08-17T18:07:28.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14311495099999999 2024-08-17T18:07:28.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.777 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:07:28.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377664+0000", 2024-08-17T18:07:28.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8047999999999998e-05 2024-08-17T18:07:28.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.777 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377763+0000", 2024-08-17T18:07:28.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8966000000000001e-05 2024-08-17T18:07:28.777 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.777 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.778 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.778 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.778 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.778 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195124 46.2 46:497fbb11:::benchmark_data_smithi130_267720_object87141:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:07:28.778 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.956154+0000", 2024-08-17T18:07:28.778 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 275.69124284700001, 2024-08-17T18:07:28.778 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2343982279999999, 2024-08-17T18:07:28.778 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.778 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.778 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:07:28.778 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.778 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:07:28.778 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:07:28.779 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195124 2024-08-17T18:07:28.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.779 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.779 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.779 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.779 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956154+0000", 2024-08-17T18:07:28.779 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.779 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.779 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.779 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956154+0000", 2024-08-17T18:07:28.779 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.779 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.780 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.780 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956155+0000", 2024-08-17T18:07:28.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1119999999999999e-06 2024-08-17T18:07:28.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.780 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.780 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.780 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956159+0000", 2024-08-17T18:07:28.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2679999999999996e-06 2024-08-17T18:07:28.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.780 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.780 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.780 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956170+0000", 2024-08-17T18:07:28.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0334e-05 2024-08-17T18:07:28.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.781 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.781 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956177+0000", 2024-08-17T18:07:28.781 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2849999999999997e-06 2024-08-17T18:07:28.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.781 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.781 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956244+0000", 2024-08-17T18:07:28.781 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6994000000000003e-05 2024-08-17T18:07:28.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.781 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.782 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956257+0000", 2024-08-17T18:07:28.782 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3196e-05 2024-08-17T18:07:28.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.782 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.782 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.782 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.876745+0000", 2024-08-17T18:07:28.782 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.92048745399999998 2024-08-17T18:07:28.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.782 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.782 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.782 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.876755+0000", 2024-08-17T18:07:28.782 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0001e-05 2024-08-17T18:07:28.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.783 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.783 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.783 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.169422+0000", 2024-08-17T18:07:28.783 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.292667488 2024-08-17T18:07:28.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.783 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.783 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.783 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.169523+0000", 2024-08-17T18:07:28.783 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000101116 2024-08-17T18:07:28.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.783 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.784 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6,7", 2024-08-17T18:07:28.784 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.169908+0000", 2024-08-17T18:07:28.784 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000384936 2024-08-17T18:07:28.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.784 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.784 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.784 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.182867+0000", 2024-08-17T18:07:28.784 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012958374 2024-08-17T18:07:28.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.784 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.784 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.784 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.189357+0000", 2024-08-17T18:07:28.784 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0064902980000000002 2024-08-17T18:07:28.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.785 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.785 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.785 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.189592+0000", 2024-08-17T18:07:28.785 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00023549000000000001 2024-08-17T18:07:28.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.785 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.785 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.785 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.190415+0000", 2024-08-17T18:07:28.785 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00082231100000000005 2024-08-17T18:07:28.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.785 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.785 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:07:28.786 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.190452+0000", 2024-08-17T18:07:28.786 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7054999999999997e-05 2024-08-17T18:07:28.786 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.786 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.786 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.786 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.190552+0000", 2024-08-17T18:07:28.786 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000100516 2024-08-17T18:07:28.786 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.786 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.786 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.786 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.786 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.786 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:115193 47.1 47:8a1837dc:::benchmark_data_smithi130_302447_object19958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:07:28.787 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:56.586429+0000", 2024-08-17T18:07:28.787 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 92.060968056999997, 2024-08-17T18:07:28.787 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0156209679999999, 2024-08-17T18:07:28.787 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.787 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.787 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:07:28.787 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.787 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:07:28.787 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:07:28.787 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115193 2024-08-17T18:07:28.787 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.787 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.787 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.787 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.788 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586429+0000", 2024-08-17T18:07:28.788 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.788 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.788 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.788 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.788 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586429+0000", 2024-08-17T18:07:28.788 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.788 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.788 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.788 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.788 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586430+0000", 2024-08-17T18:07:28.788 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.364e-06 2024-08-17T18:07:28.788 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.788 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.789 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.789 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586435+0000", 2024-08-17T18:07:28.789 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3900000000000003e-06 2024-08-17T18:07:28.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.789 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.789 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.789 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586450+0000", 2024-08-17T18:07:28.789 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5209e-05 2024-08-17T18:07:28.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.789 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.789 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.789 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586457+0000", 2024-08-17T18:07:28.789 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0099999999999998e-06 2024-08-17T18:07:28.790 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.790 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.790 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.790 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586575+0000", 2024-08-17T18:07:28.790 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000118305 2024-08-17T18:07:28.790 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.790 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.790 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:07:28.790 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586714+0000", 2024-08-17T18:07:28.790 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013882799999999999 2024-08-17T18:07:28.790 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.791 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:07:28.791 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586850+0000", 2024-08-17T18:07:28.791 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000136198 2024-08-17T18:07:28.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.791 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:07:28.791 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.594476+0000", 2024-08-17T18:07:28.791 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0076255070000000001 2024-08-17T18:07:28.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.792 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:07:28.792 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.594483+0000", 2024-08-17T18:07:28.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5549999999999997e-06 2024-08-17T18:07:28.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.792 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.792 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.792 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.081391+0000", 2024-08-17T18:07:28.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.48690757499999998 2024-08-17T18:07:28.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.792 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.792 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.793 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.081400+0000", 2024-08-17T18:07:28.793 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4089999999999992e-06 2024-08-17T18:07:28.793 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.793 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.793 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.793 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.601973+0000", 2024-08-17T18:07:28.793 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.52057316499999995 2024-08-17T18:07:28.793 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.793 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.793 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.793 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.602050+0000", 2024-08-17T18:07:28.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6452999999999995e-05 2024-08-17T18:07:28.794 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.794 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.794 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.794 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.794 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:115174 47.4 47:2d651f61:::benchmark_data_smithi130_302447_object19939:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1317)", 2024-08-17T18:07:28.794 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:57.071033+0000", 2024-08-17T18:07:28.794 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 91.576363896999993, 2024-08-17T18:07:28.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.540335236, 2024-08-17T18:07:28.794 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:07:28.794 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:07:28.794 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:07:28.795 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:07:28.795 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:07:28.795 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:07:28.795 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115174 2024-08-17T18:07:28.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.795 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:07:28.795 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.795 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:07:28.795 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071033+0000", 2024-08-17T18:07:28.795 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.795 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:07:28.796 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071033+0000", 2024-08-17T18:07:28.796 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:07:28.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.796 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:07:28.796 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071035+0000", 2024-08-17T18:07:28.796 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.666e-06 2024-08-17T18:07:28.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.796 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:07:28.796 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071041+0000", 2024-08-17T18:07:28.796 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3199999999999996e-06 2024-08-17T18:07:28.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.797 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.797 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:07:28.797 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071076+0000", 2024-08-17T18:07:28.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5286000000000001e-05 2024-08-17T18:07:28.797 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.797 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.797 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:07:28.797 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071087+0000", 2024-08-17T18:07:28.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0327e-05 2024-08-17T18:07:28.797 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.797 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.797 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.797 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071159+0000", 2024-08-17T18:07:28.798 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2201999999999998e-05 2024-08-17T18:07:28.798 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.798 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.798 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op must wait for map", 2024-08-17T18:07:28.798 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071167+0000", 2024-08-17T18:07:28.798 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1329999999999999e-06 2024-08-17T18:07:28.798 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.798 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.798 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.798 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.075440+0000", 2024-08-17T18:07:28.798 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0042733340000000002 2024-08-17T18:07:28.798 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.798 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.799 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:07:28.799 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.075449+0000", 2024-08-17T18:07:28.799 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3699999999999995e-06 2024-08-17T18:07:28.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.799 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.799 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:07:28.799 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.605615+0000", 2024-08-17T18:07:28.799 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.53016654399999996 2024-08-17T18:07:28.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:07:28.799 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:07:28.799 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:07:28.799 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.611368+0000", 2024-08-17T18:07:28.799 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0057530539999999996 2024-08-17T18:07:28.800 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.800 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.800 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.800 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:07:28.800 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:07:28.800 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:07:28.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:28.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:28.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:28.923 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:28.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:28.945 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:28.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:29.046 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:29.071 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:29.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:29.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:29.173 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:29.199 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:29.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:29.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:29.301 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:29.326 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:29.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:29.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:29.340 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:07:29.427 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:29.448 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:29.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:29.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:29.550 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:29.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:29.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:29.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:29.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/331945252' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:07:29.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/331945252' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:07:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/331945252' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:07:29.678 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:29.699 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:29.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:29.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:29.801 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:29.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:29.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:29.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:29.836 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T18:07:29.924 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:29.950 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:29.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:29.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:30.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:30.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:30.078 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:30.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:30.085 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:07:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:07:29] "GET /metrics HTTP/1.1" 200 34372 "" "Prometheus/2.51.0" 2024-08-17T18:07:30.179 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:30.201 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:30.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:30.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:30.303 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:30.324 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:30.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:30.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:30.426 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:30.449 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:30.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:30.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:30.550 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:30.553 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:30.574 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:30.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:30.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:07:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:07:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:07:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:30 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:07:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:30 smithi179 ceph-mon[93519]: pgmap v2312: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:30.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:30 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/331945252' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:07:30.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:30 smithi179 ceph-mon[93519]: osdmap e1326: 8 total, 7 up, 5 in 2024-08-17T18:07:30.664 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:30 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:07:30.664 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:30 smithi130 ceph-mon[105521]: pgmap v2312: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:30.664 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:30 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/331945252' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:07:30.665 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:30 smithi130 ceph-mon[105521]: osdmap e1326: 8 total, 7 up, 5 in 2024-08-17T18:07:30.665 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:30 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:07:30.665 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:30 smithi130 ceph-mon[110068]: pgmap v2312: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:30.665 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:30 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/331945252' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:07:30.665 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:30 smithi130 ceph-mon[110068]: osdmap e1326: 8 total, 7 up, 5 in 2024-08-17T18:07:30.676 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:30.697 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:30.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:30.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:30.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:30.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:30.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:30.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:30.923 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:30.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:30.945 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:30.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:31.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:31.067 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:31.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:31.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:31.169 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:31.195 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:31.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:31.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:31.296 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:31.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:31.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:31.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:31.345 INFO:teuthology.orchestra.run.smithi130.stderr:pool 'unique_pool_3' created 2024-08-17T18:07:31.419 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:31.441 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:31.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:31.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:31.543 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:31.570 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:31.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:31.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:31.590 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1707217015' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T18:07:31.590 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:31 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T18:07:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1707217015' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T18:07:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:31 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T18:07:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1707217015' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T18:07:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:31 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T18:07:31.671 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:31.694 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:31.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:31.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:31.796 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:31.819 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:31.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:31.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:31.884 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:31.920 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:31.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:31.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:31.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:32.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:32.066 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:32.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:32.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:32.168 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:32.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:32.191 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:32.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:32.292 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:32.314 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:32.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:32.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:32.416 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:32.437 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:32.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:32.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:32.539 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:32.561 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:32.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:32.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:32.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:32 smithi130 ceph-mon[110068]: pgmap v2314: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:32.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:32 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-17T18:07:32.643 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:32 smithi130 ceph-mon[110068]: osdmap e1327: 8 total, 7 up, 5 in 2024-08-17T18:07:32.643 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:32 smithi130 ceph-mon[105521]: pgmap v2314: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:32.644 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:32 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-17T18:07:32.644 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:32 smithi130 ceph-mon[105521]: osdmap e1327: 8 total, 7 up, 5 in 2024-08-17T18:07:32.663 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:32.667 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-08-17T18:07:32.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:32.685 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:32.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:32.787 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:32.812 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:32.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:32.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:32.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:32 smithi179 ceph-mon[93519]: pgmap v2314: 73 pgs: 36 active+undersized, 9 active+undersized+degraded, 28 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:32 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-17T18:07:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:32 smithi179 ceph-mon[93519]: osdmap e1327: 8 total, 7 up, 5 in 2024-08-17T18:07:32.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:32.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:32.936 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:32.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:33.037 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:33.058 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:33.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:33.160 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:33.186 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:33.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:33.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:33.259 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:07:33.288 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:33.314 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:33.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:33.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:33.416 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:33.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:33 smithi130 ceph-mon[105521]: osdmap e1328: 8 total, 7 up, 5 in 2024-08-17T18:07:33.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:33 smithi130 ceph-mon[110068]: osdmap e1328: 8 total, 7 up, 5 in 2024-08-17T18:07:33.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:33.442 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:33.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:33.543 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:33.568 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:33.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:33.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:33.669 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:33.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:33.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:33.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:33.723 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:07:33.723 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:07:33.729 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:07:33.729 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:07:33.745 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:07:33.746 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:07:33.746 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:07:33.761 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:07:33.762 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:07:33.777 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:07:33.777 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:07:33.793 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:07:33.793 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:07:33.794 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:33.809 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:07:33.809 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:07:33.816 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:33.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:33.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:33.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:33 smithi179 ceph-mon[93519]: osdmap e1328: 8 total, 7 up, 5 in 2024-08-17T18:07:33.917 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:33.938 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:33.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:33.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:34.040 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:34.061 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:34.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:34.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:34.163 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:34.190 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:34.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:34.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:34.292 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:34.313 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:34.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:34.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:34.415 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:34 smithi130 ceph-mon[105521]: pgmap v2317: 89 pgs: 7 creating+peering, 3 unknown, 41 active+undersized, 9 active+undersized+degraded, 29 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:34 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:07:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:34 smithi130 ceph-mon[105521]: osdmap e1329: 8 total, 7 up, 5 in 2024-08-17T18:07:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:34 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3343555558' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:34 smithi130 ceph-mon[110068]: pgmap v2317: 89 pgs: 7 creating+peering, 3 unknown, 41 active+undersized, 9 active+undersized+degraded, 29 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:34.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:34 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:07:34.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:34 smithi130 ceph-mon[110068]: osdmap e1329: 8 total, 7 up, 5 in 2024-08-17T18:07:34.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:34 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3343555558' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:34.441 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:34.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:34.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:34.543 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:34.567 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:34.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:34.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:34.644 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:07:34.644 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:07:34.669 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:34.690 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:34.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:34.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:34.792 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:34.813 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:34.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:34.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:34.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:34 smithi179 ceph-mon[93519]: pgmap v2317: 89 pgs: 7 creating+peering, 3 unknown, 41 active+undersized, 9 active+undersized+degraded, 29 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:34 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:07:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:34 smithi179 ceph-mon[93519]: osdmap e1329: 8 total, 7 up, 5 in 2024-08-17T18:07:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:34 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3343555558' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:34.915 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:34.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:34.937 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:34.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:35.038 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:35.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:35.065 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:35.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:35.166 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:35.190 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:35.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:35.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:35.292 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:35.316 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:35.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:35.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:35.413 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:35.415 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:35.417 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:35.441 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:35.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:35.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:35.543 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:35.564 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:35.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:35.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:35.666 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:35.688 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:35.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:35.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:35.789 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:35.815 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:35.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:35.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:35.917 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:35.942 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:35.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:35.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:36.043 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:36.069 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:36.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:36.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:36.171 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:36.193 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:36.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:36.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:36.295 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:36.317 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:36.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:36.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:36.347 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:07:36.348 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:07:36.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:36.442 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:36.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:36.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:36.544 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:36.569 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:36.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:36.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:36.645 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:07:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:36 smithi130 ceph-mon[105521]: pgmap v2319: 89 pgs: 7 creating+peering, 43 active+undersized, 9 active+undersized+degraded, 30 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:36.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:36 smithi130 ceph-mon[110068]: pgmap v2319: 89 pgs: 7 creating+peering, 43 active+undersized, 9 active+undersized+degraded, 30 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:36.672 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:36.698 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:36.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:36.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:36.800 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:36.825 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:36.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:36.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:36.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:36 smithi179 ceph-mon[93519]: pgmap v2319: 89 pgs: 7 creating+peering, 43 active+undersized, 9 active+undersized+degraded, 30 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:36.927 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:36.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:36.956 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:36.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:37.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:37.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:37.081 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:37.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:37.182 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:37.208 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:37.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:37.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:37.310 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:37.331 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:37.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:37.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:37.433 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:37.455 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:37.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:37.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:37.557 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:37.579 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:37.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:37.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:37.680 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:37.702 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:37.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:37.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:37.804 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:37.830 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:37.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:37.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:37.932 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:37.954 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:37.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:37.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:38.056 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:38.082 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:38.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:38.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:38.184 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:38.209 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:38.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:38.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:38.311 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:38.332 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:38.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:38.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:38.416 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T18:07:38.434 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:38.481 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:38.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:38.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:38.583 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:38.606 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:38.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:38.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:38.635 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:38.647 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:38 smithi130 ceph-mon[105521]: pgmap v2320: 89 pgs: 47 active+undersized, 9 active+undersized+degraded, 33 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:38.647 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:38 smithi130 ceph-mon[110068]: pgmap v2320: 89 pgs: 47 active+undersized, 9 active+undersized+degraded, 33 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:38.708 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:38.734 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:38.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:38.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:38.836 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:38.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:38 smithi179 ceph-mon[93519]: pgmap v2320: 89 pgs: 47 active+undersized, 9 active+undersized+degraded, 33 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:38.861 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:38.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:38.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:38.963 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:38.987 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:38.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:38.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:39.089 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:39.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:39.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:39.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:39.215 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:39.239 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:39.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:39.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:39.341 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:39.365 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:39.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:39.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:39.467 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:39.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:39.490 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:39.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:39.591 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:39.613 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:39.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:39.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:39.715 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:39.736 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:39.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:39.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:39.838 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:39.864 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:39.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:39.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:39.934 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:07:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:07:39] "GET /metrics HTTP/1.1" 200 34372 "" "Prometheus/2.51.0" 2024-08-17T18:07:39.966 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:39.987 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:39.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:39.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:40.088 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:40.114 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:40.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:40.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:40.216 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:40.240 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:40.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:40.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:40.342 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:40.367 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:40.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:40.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:40.371 INFO:teuthology.orchestra.run.smithi130.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-08-17T18:07:40.371 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:07:40.469 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:40.487 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:40 smithi130 ceph-mon[110068]: pgmap v2321: 89 pgs: 47 active+undersized, 9 active+undersized+degraded, 33 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:40.487 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:40 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4049115159' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:07:40.487 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:40 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:07:40.487 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:40 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2334401581' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:07:40.487 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:40 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:07:40.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:40 smithi130 ceph-mon[105521]: pgmap v2321: 89 pgs: 47 active+undersized, 9 active+undersized+degraded, 33 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:40.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:40 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4049115159' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:07:40.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:40 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:07:40.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:40 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2334401581' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:07:40.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:40 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:07:40.494 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:40.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:40.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:07:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:07:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:07:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:40 smithi179 ceph-mon[93519]: pgmap v2321: 89 pgs: 47 active+undersized, 9 active+undersized+degraded, 33 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:40 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4049115159' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:07:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:40 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:07:40.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:40 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2334401581' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:07:40.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:40 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:07:40.596 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:40.620 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:40.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:40.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:40.722 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:40.746 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:40.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:40.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:40.848 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:40.870 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:40.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:40.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:40.972 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:40.993 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:40.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:40.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:41.095 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:41.121 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:41.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:41.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:41.223 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:41.245 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:41.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:41.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:41.347 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:41.372 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:41.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:41.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:41 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:07:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:41 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:07:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:41 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T18:07:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:41 smithi130 ceph-mon[105521]: osdmap e1330: 8 total, 7 up, 5 in 2024-08-17T18:07:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:41 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:07:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:41 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:07:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:41 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T18:07:41.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:41 smithi130 ceph-mon[110068]: osdmap e1330: 8 total, 7 up, 5 in 2024-08-17T18:07:41.474 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:41.498 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:41.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:41.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:41.600 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:41.623 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:41.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:41.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:41.694 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:07:41.695 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:07:41.709 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:07:41.709 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:07:41.725 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:41.726 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:07:41.726 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:07:41.726 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:07:41.742 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:07:41.742 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:07:41.749 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:41.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:41.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:41.758 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:07:41.758 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:07:41.774 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:07:41.774 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:07:41.789 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:07:41.789 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:07:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:41 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:07:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:41 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:07:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:41 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T18:07:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:41 smithi179 ceph-mon[93519]: osdmap e1330: 8 total, 7 up, 5 in 2024-08-17T18:07:41.851 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:41.873 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:41.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:41.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:41.975 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:41.997 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:41.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:41.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:42.099 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:42.122 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:42.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:42.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:42.223 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:42.246 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:42.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:42.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:42.347 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:42.369 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:42.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:42.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:42.471 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:42.493 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:42.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:42.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:42.595 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:42.617 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:42.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:42.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:42 smithi130 ceph-mon[105521]: pgmap v2323: 89 pgs: 47 active+undersized, 9 active+undersized+degraded, 33 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:42 smithi130 ceph-mon[105521]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:07:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:42 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3235662754' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:07:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:42 smithi130 ceph-mon[110068]: pgmap v2323: 89 pgs: 47 active+undersized, 9 active+undersized+degraded, 33 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:42 smithi130 ceph-mon[110068]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:07:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:42 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3235662754' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:42.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:07:42.718 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:42.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:42.742 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:42.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:42.842 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:42 smithi179 ceph-mon[93519]: pgmap v2323: 89 pgs: 47 active+undersized, 9 active+undersized+degraded, 33 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:42.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:42 smithi179 ceph-mon[93519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:07:42.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:42 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3235662754' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:42.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:07:42.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:07:42.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:42.866 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:42.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:42.967 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:42.989 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:42.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:42.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:43.091 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:43.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:43.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:43.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:43.131 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:07:43.131 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:07:43.214 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:43.236 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:43.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:43.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:43.338 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:43.360 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:43.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:43.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:43.410 DEBUG:teuthology.orchestra.run.smithi130:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-08-17T18:07:43.411 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_3 2024-08-17T18:07:43.411 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:07:43.414 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T18:07:43.414 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T18:07:43.462 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:hints = 1 2024-08-17T18:07:43.462 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T18:07:43.462 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Object prefix: benchmark_data_smithi130_327597 2024-08-17T18:07:43.462 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:07:43.462 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 0 0 0 0 0 0 - 0 2024-08-17T18:07:43.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:43.512 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:43.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:43.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:43.614 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:43.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:43.662 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:43.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:43.744 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:43.763 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:43.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:43.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:43.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:43.911 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:43.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:43.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:43.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:44.058 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:44.082 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:44.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:44.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:44.184 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:44.208 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:44.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:44.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:44.310 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:44.351 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:44.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:44.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:44.453 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:44.462 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 1 16 3510 3494 218.364 218.375 0.00513392 0.00456363 2024-08-17T18:07:44.491 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:44.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:44.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:44.593 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:44.618 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:44.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:44.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:44.625 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:44.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:44 smithi130 ceph-mon[105521]: pgmap v2324: 89 pgs: 47 active+undersized, 9 active+undersized+degraded, 33 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:44 smithi130 ceph-mon[110068]: pgmap v2324: 89 pgs: 47 active+undersized, 9 active+undersized+degraded, 33 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:44.719 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:44.743 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:44.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:44.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:44.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:44 smithi179 ceph-mon[93519]: pgmap v2324: 89 pgs: 47 active+undersized, 9 active+undersized+degraded, 33 active+clean; 582 KiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 14/78 objects degraded (17.949%) 2024-08-17T18:07:44.845 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:44.882 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:44.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:44.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:44.984 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:45.009 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:45.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:45.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:45.111 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:45.133 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:07:45.136 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:45.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:45.238 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:45.262 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:45.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:45.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:45.364 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:45.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:45.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:45.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:45.463 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 2 16 6970 6954 217.196 216.25 0.00254048 0.00459561 2024-08-17T18:07:45.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:45.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:45.545 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:45.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:45.646 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:45.672 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:45.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:45.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:45.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:07:45.737 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:45 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 114/783 objects degraded (14.559%), 11 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:45.737 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:45 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 114/783 objects degraded (14.559%), 11 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:45.773 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:45.828 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:45.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:45 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 114/783 objects degraded (14.559%), 11 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:45.930 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:45.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:45.969 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:45.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:46.070 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:46.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:46.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:46.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:46.198 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:46.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:46.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:46.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:46.321 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:46.360 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:46.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:46.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:46.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:46.463 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 3 16 9998 9982 207.878 189.25 0.00574694 0.00480263 2024-08-17T18:07:46.485 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:46.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:46.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:46.587 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:46.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:46.614 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:46.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:46.715 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:46.741 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:46.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:46.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:46.842 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:46 smithi179 ceph-mon[93519]: pgmap v2325: 89 pgs: 45 active+undersized, 11 active+undersized+degraded, 33 active+clean; 15 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 1.5 MiB/s wr, 23 op/s; 114/783 objects degraded (14.559%) 2024-08-17T18:07:46.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:46.867 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:46.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:46.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:46 smithi130 ceph-mon[105521]: pgmap v2325: 89 pgs: 45 active+undersized, 11 active+undersized+degraded, 33 active+clean; 15 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 1.5 MiB/s wr, 23 op/s; 114/783 objects degraded (14.559%) 2024-08-17T18:07:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:46 smithi130 ceph-mon[110068]: pgmap v2325: 89 pgs: 45 active+undersized, 11 active+undersized+degraded, 33 active+clean; 15 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 1.5 MiB/s wr, 23 op/s; 114/783 objects degraded (14.559%) 2024-08-17T18:07:46.969 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:47.006 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:47.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:47.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:47.108 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:47.130 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:47.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:47.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:47.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:47.244 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:47.290 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:47.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:47.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:47.394 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:47.431 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:47.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:47.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:47.463 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 4 15 12756 12741 199.015 172.438 0.00673801 0.00501901 2024-08-17T18:07:47.533 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:47.561 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:47.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:47.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:47.662 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:47.695 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:47.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:47.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:47.797 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:47.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:47.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:47.923 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:47.944 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:47.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:47.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:48.045 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:48.072 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:48.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:48.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:48.174 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:48.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:48.197 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:48.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:48.299 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:48.324 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:48.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:48.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:48.426 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:48.450 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:48.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:48.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:48.463 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 5 16 15027 15011 187.587 141.875 0.00579417 0.00532508 2024-08-17T18:07:48.553 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:48.578 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:48.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:48.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:48.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:48 smithi130 ceph-mon[105521]: pgmap v2326: 89 pgs: 45 active+undersized, 11 active+undersized+degraded, 33 active+clean; 15 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 1.5 MiB/s wr, 23 op/s; 114/783 objects degraded (14.559%) 2024-08-17T18:07:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:48 smithi130 ceph-mon[110068]: pgmap v2326: 89 pgs: 45 active+undersized, 11 active+undersized+degraded, 33 active+clean; 15 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 1.5 MiB/s wr, 23 op/s; 114/783 objects degraded (14.559%) 2024-08-17T18:07:48.680 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:48.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:48.703 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:48.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:48.804 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:48.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:48.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:48.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:48 smithi179 ceph-mon[93519]: pgmap v2326: 89 pgs: 45 active+undersized, 11 active+undersized+degraded, 33 active+clean; 15 MiB data, 5.6 GiB used, 441 GiB / 447 GiB avail; 1.5 MiB/s wr, 23 op/s; 114/783 objects degraded (14.559%) 2024-08-17T18:07:48.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:48.969 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:48.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:48.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:49.071 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:49.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:49.095 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:49.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:49.196 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:49.233 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:49.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:49.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:49.335 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:49.370 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:49.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:49.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:49.463 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 6 16 17301 17285 180.007 142.125 0.00151739 0.00547624 2024-08-17T18:07:49.472 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:49.492 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:49.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:49.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:49.594 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:49.620 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:49.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:49.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:49.722 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:49.746 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:49.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:49.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:49.848 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:49.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:49.881 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:49.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:49.982 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:50.004 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:50.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:50.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:50.106 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:50.131 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:50.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:50.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:07:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:07:49] "GET /metrics HTTP/1.1" 200 36715 "" "Prometheus/2.51.0" 2024-08-17T18:07:50.233 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:50.256 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:50.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:50.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:50.358 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:50.383 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:50.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:50.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:50.463 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 7 16 19483 19467 173.773 136.375 0.00276896 0.00574876 2024-08-17T18:07:50.485 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:50.521 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:50.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:50.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:50 smithi179 ceph-mon[93519]: pgmap v2327: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 865 MiB data, 7.5 GiB used, 439 GiB / 447 GiB avail; 86 MiB/s wr, 1.38k op/s; 9629/41553 objects degraded (23.173%) 2024-08-17T18:07:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:07:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:07:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:07:50.623 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:50.647 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:50.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:50.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:50.749 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:50.776 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:50.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:50.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:50.878 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:50.902 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:50.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:50.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:50 smithi130 ceph-mon[105521]: pgmap v2327: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 865 MiB data, 7.5 GiB used, 439 GiB / 447 GiB avail; 86 MiB/s wr, 1.38k op/s; 9629/41553 objects degraded (23.173%) 2024-08-17T18:07:50.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:50 smithi130 ceph-mon[110068]: pgmap v2327: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 865 MiB data, 7.5 GiB used, 439 GiB / 447 GiB avail; 86 MiB/s wr, 1.38k op/s; 9629/41553 objects degraded (23.173%) 2024-08-17T18:07:51.004 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:51.025 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:51.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:51.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:51.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:51.162 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:51.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:51.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:51.264 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:51.288 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:51.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:51.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:51.390 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:51.424 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:51.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:51.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:51.464 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 8 15 22133 22118 172.761 165.688 0.00357245 0.00578141 2024-08-17T18:07:51.526 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:51.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:51.553 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:51.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:51.654 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:51.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:51.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:51.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:51.716 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 9629/41553 objects degraded (23.173%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:51.717 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 9629/41553 objects degraded (23.173%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:51.794 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:51.820 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:51.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:51.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:51.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 9629/41553 objects degraded (23.173%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:51.922 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:51.947 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:51.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:51.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:52.049 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:52.085 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:52.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:52.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:52.187 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:52.210 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:52.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:52.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:52.312 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:52.350 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:52.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:52.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:52.452 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:52.464 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 9 16 24798 24782 172.061 166.5 0.0123143 0.00580542 2024-08-17T18:07:52.488 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:52.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:52.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:52.590 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:52.616 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:52.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:52.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:52.718 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:52.739 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:52.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:52.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:52.841 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:52.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:52 smithi179 ceph-mon[93519]: pgmap v2328: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 865 MiB data, 7.5 GiB used, 439 GiB / 447 GiB avail; 79 MiB/s wr, 1.26k op/s; 9629/41553 objects degraded (23.173%) 2024-08-17T18:07:52.866 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:52.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:52.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:52.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:52 smithi130 ceph-mon[105521]: pgmap v2328: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 865 MiB data, 7.5 GiB used, 439 GiB / 447 GiB avail; 79 MiB/s wr, 1.26k op/s; 9629/41553 objects degraded (23.173%) 2024-08-17T18:07:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:52 smithi130 ceph-mon[110068]: pgmap v2328: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 865 MiB data, 7.5 GiB used, 439 GiB / 447 GiB avail; 79 MiB/s wr, 1.26k op/s; 9629/41553 objects degraded (23.173%) 2024-08-17T18:07:52.968 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:53.004 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:53.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:53.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:53.106 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:53.130 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:53.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:53.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:53.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:53.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:53.268 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:53.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:53.369 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:53.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:53.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:53.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:53.464 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 10 16 26944 26928 168.267 134.125 0.00581326 0.00590767 2024-08-17T18:07:53.507 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:53.532 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:53.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:53.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:53.634 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:53.660 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:53.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:53.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:53.762 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:53.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:53.801 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:53.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:53.902 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:53.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:53.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:53.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:54.030 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:54.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:54.060 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:54.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:54.161 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:54.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:54.193 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:54.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:54.294 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:54.319 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:54.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:54.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:54.421 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:54.445 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:54.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:54.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:54.464 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 11 16 29282 29266 166.253 146.125 0.0046337 0.00599201 2024-08-17T18:07:54.547 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:54.574 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:54.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:54.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:54.675 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:54.700 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:54.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:54.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:54.802 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:54.839 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:54.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:54.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:54.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:54 smithi179 ceph-mon[93519]: pgmap v2329: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 1.4 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 123 MiB/s wr, 1.96k op/s; 16322/70680 objects degraded (23.093%) 2024-08-17T18:07:54.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:54 smithi130 ceph-mon[105521]: pgmap v2329: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 1.4 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 123 MiB/s wr, 1.96k op/s; 16322/70680 objects degraded (23.093%) 2024-08-17T18:07:54.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:54 smithi130 ceph-mon[110068]: pgmap v2329: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 1.4 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 123 MiB/s wr, 1.96k op/s; 16322/70680 objects degraded (23.093%) 2024-08-17T18:07:54.941 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:54.962 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:54.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:54.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:55.064 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:55.088 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:55.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:55.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:55.190 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:55.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:55.215 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:55.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:55.316 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:55.341 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:55.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:55.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:55.443 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:55.466 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 12 16 31153 31137 162.143 116.938 0.00865049 0.00616173 2024-08-17T18:07:55.469 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:55.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:55.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:55.570 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:55.605 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:55.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:55.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:55.707 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:55.744 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:55.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:55.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:55.847 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:55.868 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:55.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:55.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:55.970 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:56.009 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:56.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:56.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:56.111 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:56.137 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:56.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:56.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:56.239 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:56.261 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:56.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:56.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:56.363 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:56.387 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:56.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:56.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:56.464 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 13 16 33355 33339 160.256 137.625 0.00591022 0.00623512 2024-08-17T18:07:56.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:56.514 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:56.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:56.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:56.616 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:56.640 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:56.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:56.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:56.659 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:07:56.659 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T18:07:56.659 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T18:07:56.660 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T18:07:56.660 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:07:56.742 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:56.766 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:56.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:56.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:56 smithi179 ceph-mon[93519]: pgmap v2330: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 1.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 134 MiB/s wr, 2.15k op/s; 17783/77487 objects degraded (22.950%) 2024-08-17T18:07:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17783/77487 objects degraded (22.950%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:56.868 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:56.893 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:56.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:56.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:56.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:56 smithi130 ceph-mon[105521]: pgmap v2330: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 1.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 134 MiB/s wr, 2.15k op/s; 17783/77487 objects degraded (22.950%) 2024-08-17T18:07:56.929 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17783/77487 objects degraded (22.950%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:56.930 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:56 smithi130 ceph-mon[110068]: pgmap v2330: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 1.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 134 MiB/s wr, 2.15k op/s; 17783/77487 objects degraded (22.950%) 2024-08-17T18:07:56.930 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17783/77487 objects degraded (22.950%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:07:56.994 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:57.021 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:57.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:57.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:57.123 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:57.146 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:57.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:57.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:57.248 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:57.272 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:57.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:57.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:57.314 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:07:57.354 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T18:07:57.374 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:57.397 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:57.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:57.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:57.466 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 14 16 35664 35648 159.092 144.312 0.0065933 0.00628062 2024-08-17T18:07:57.499 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:57.522 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:57.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:57.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:57.624 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:57.632 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:07:57.632 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:07:57.647 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:57.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:57.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:57.749 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:57.774 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:57.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:57.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:57.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:07:57.876 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:07:57.916 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:57.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:57.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:07:58.018 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:58.044 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:58.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:58.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:58.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:58.170 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:58.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:58.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:58.272 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:58.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:58.296 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:58.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:58.397 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:58.439 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:58.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:58.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:07:58.466 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 15 16 37700 37684 156.969 127.25 0.00972295 0.00636605 2024-08-17T18:07:58.515 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:58 smithi130 ceph-mon[105521]: pgmap v2331: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 1.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 133 MiB/s wr, 2.13k op/s; 17783/77487 objects degraded (22.950%) 2024-08-17T18:07:58.515 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:07:58 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2951773874' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:58.515 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:58 smithi130 ceph-mon[110068]: pgmap v2331: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 1.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 133 MiB/s wr, 2.13k op/s; 17783/77487 objects degraded (22.950%) 2024-08-17T18:07:58.515 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:07:58 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2951773874' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:58.541 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:58.564 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:58.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:58.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:58.665 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:07:58.690 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:58.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:58.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:07:58.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:58.818 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:58.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:58.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:58.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:58 smithi179 ceph-mon[93519]: pgmap v2331: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 1.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 133 MiB/s wr, 2.13k op/s; 17783/77487 objects degraded (22.950%) 2024-08-17T18:07:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:07:58 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2951773874' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:07:58.920 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:58.944 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:58.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:58.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:59.046 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:59.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:59.068 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:59.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:59.169 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:59.194 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:59.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:59.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:59.296 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:07:59.320 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:59.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:59.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:07:59.422 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:07:59.444 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:59.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:59.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:07:59.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 16 16 39333 39317 153.537 102.062 0.139375 0.00646407 2024-08-17T18:07:59.545 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:07:59.570 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:07:59.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:59.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:07:59.672 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:59.707 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:59.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:59.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:59.712 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:07:59.712 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:07:59.712 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:07:59.713 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:07:59.713 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:07:59.713 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:07:59.713 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:07:59.713 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:07:59.713 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:07:59.713 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:07:59.713 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:07:59.713 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:07:59.714 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:07:59.714 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:07:59.714 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:07:59.808 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:07:59.830 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:59.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:59.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:07:59.932 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:07:59.972 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:07:59.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:07:59.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:00.074 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:00.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:00.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:00.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:00.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:07:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:07:59] "GET /metrics HTTP/1.1" 200 36782 "" "Prometheus/2.51.0" 2024-08-17T18:08:00.197 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:00.222 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:00.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:00.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:00.324 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:00.345 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:00.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:00.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:00.447 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:00.466 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 17 16 40961 40945 150.491 101.75 0.00675433 0.00664055 2024-08-17T18:08:00.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:00.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:00.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:00.479 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:08:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:08:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:08:00.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:00.598 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:00.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:00.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:00.700 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:00.724 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:00.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:00.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:00.825 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:00.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:00 smithi179 ceph-mon[93519]: pgmap v2332: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 189 MiB/s wr, 3.02k op/s; 25238/109770 objects degraded (22.992%) 2024-08-17T18:08:00.860 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:00.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:00.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:00 smithi130 ceph-mon[105521]: pgmap v2332: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 189 MiB/s wr, 3.02k op/s; 25238/109770 objects degraded (22.992%) 2024-08-17T18:08:00.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:00 smithi130 ceph-mon[110068]: pgmap v2332: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 189 MiB/s wr, 3.02k op/s; 25238/109770 objects degraded (22.992%) 2024-08-17T18:08:00.962 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:00.986 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:00.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:00.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:01.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:01.128 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:01.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:01.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:01.230 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:01.264 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:01.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:01.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:01.366 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:01.391 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:01.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:01.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:01.466 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 18 16 42971 42955 149.109 125.625 0.0131577 0.00670112 2024-08-17T18:08:01.493 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:01.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:01.529 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:01.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:01.630 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:01.656 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:01.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:01.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:01.758 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:01.798 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:01.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:01.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:01.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 25238/109770 objects degraded (22.992%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:01.899 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:01.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 25238/109770 objects degraded (22.992%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 25238/109770 objects degraded (22.992%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:01.932 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:01.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:01.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:02.013 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:08:02.014 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:08:02.034 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:02.057 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:02.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:02.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:02.159 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:02.183 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:02.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:02.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:02.285 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:02.310 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:02.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:02.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:02.412 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:02.447 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:02.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:02.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:02.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 19 16 45014 44998 147.981 127.688 0.00666215 0.00675226 2024-08-17T18:08:02.549 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:02.588 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:02.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:02.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:02.690 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:02.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:02.735 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:02.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:02.785 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:02 smithi130 ceph-mon[105521]: pgmap v2333: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 118 MiB/s wr, 1.89k op/s; 25238/109770 objects degraded (22.992%) 2024-08-17T18:08:02.787 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:02 smithi130 ceph-mon[110068]: pgmap v2333: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 118 MiB/s wr, 1.89k op/s; 25238/109770 objects degraded (22.992%) 2024-08-17T18:08:02.836 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:02.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:02 smithi179 ceph-mon[93519]: pgmap v2333: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.2 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 118 MiB/s wr, 1.89k op/s; 25238/109770 objects degraded (22.992%) 2024-08-17T18:08:02.877 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:02.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:02.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:02.979 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:03.023 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:03.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:03.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:03.125 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:03.140 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:08:03.144 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:08:03.147 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:03.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:03.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:03.249 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:03.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:03.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:03.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:03.376 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:03.400 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:03.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:03.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:03.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:08:03.467180+0000 min lat: 0.00101746 max lat: 0.188221 avg lat: 0.00684134 2024-08-17T18:08:03.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:08:03.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 20 16 46767 46751 146.06 109.562 0.00268526 0.00684134 2024-08-17T18:08:03.501 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:03.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:03.538 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:03.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:03.639 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:03.664 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:03.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:03.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:03.765 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:03.803 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:03.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:03.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:03.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:03 smithi179 ceph-mon[93519]: from='client.46595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:08:03.904 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:03.915 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:03 smithi130 ceph-mon[105521]: from='client.46595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:08:03.915 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:03 smithi130 ceph-mon[110068]: from='client.46595 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:08:03.927 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:03.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:03.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:04.014 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:08:04.029 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:04.030 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:08:04.051 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:04.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:04.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:04.152 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:04.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:04.210 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:04.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:04.311 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:04.362 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:04.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:04.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:04.465 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:04.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 21 16 48476 48460 144.191 106.812 0.00955885 0.00693024 2024-08-17T18:08:04.489 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:04.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:04.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:04.514 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:08:04.591 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:04.614 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2333,"stamp":"2024-08-17T18:08:01.351060+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2396854221,"num_objects":36590,"num_object_clones":0,"num_object_copies":109770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25238,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36590,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":36842,"num_write_kb":2342543,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":2372458,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":36771,"ondisk_log_size":36771,"up":211,"acting":211,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":265,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":11336172,"kb_used_data":8233984,"kb_used_omap":293,"kb_used_meta":3101594,"kb_avail":457389588,"statfs":{"total":479975178240,"available":468366938112,"internally_reserved":0,"allocated":8431599616,"data_stored":8410435966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":300937,"internal_metadata":3176032375},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,6,2,12,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1490223104,"num_objects":22739,"num_object_clones":0,"num_object_copies":68217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15609,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22739,"num_write_kb":1455296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.009728"},"pg_stats":[{"pgid":"48.f","version":"1330'2172","reported_seq":2186,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:57.364480+0000","last_change":"2024-08-17T18:07:43.485683+0000","last_active":"2024-08-17T18:07:57.364480+0000","last_peered":"2024-08-17T18:07:57.364480+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:07:32.358403+0000","last_became_peered":"2024-08-17T18:07:32.358403+0000","last_unstale":"2024-08-17T18:07:57.364480+0000","last_undegraded":"2024-08-17T18:07:43.479855+0000","last_fullsized":"2024-08-17T18:07:32.353729+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2172,"log_dups_size":0,"ondisk_log_size":2172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:37.057238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142344192,"num_objects":2172,"num_object_clones":0,"num_object_copies":6516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2172,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2172,"num_write_kb":139008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":2172}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"48.e","version":"1330'2323","reported_seq":2331,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:58.075990+0000","last_change":"2024-08-17T18:07:43.467120+0000","last_active":"2024-08-17T18:07:58.075990+0000","last_peered":"2024-08-17T18:07:58.075990+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:07:32.358205+0000","last_became_peered":"2024-08-17T18:07:32.358205+0000","last_unstale":"2024-08-17T18:07:58.075990+0000","last_undegraded":"2024-08-17T18:07:43.466649+0000","last_fullsized":"2024-08-17T18:07:32.356325+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2323,"log_dups_size":0,"ondisk_log_size":2323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:06:07.665830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152240128,"num_objects":2323,"num_object_clones":0,"num_object_copies":6969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2323,"num_write_kb":148672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2323}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.d","version":"1330'2307","reported_seq":2315,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:58.078550+0000","last_change":"2024-08-17T18:07:43.474677+0000","last_active":"2024-08-17T18:07:58.078550+0000","last_peered":"2024-08-17T18:07:58.078550+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:07:32.360299+0000","last_became_peered":"2024-08-17T18:07:32.360299+0000","last_unstale":"2024-08-17T18:07:58.078550+0000","last_undegraded":"2024-08-17T18:07:43.474531+0000","last_fullsized":"2024-08-17T18:07:32.357145+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2307,"log_dups_size":0,"ondisk_log_size":2307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:34:04.597603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151191552,"num_objects":2307,"num_object_clones":0,"num_object_copies":6921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2307,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2307,"num_write_kb":147648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2307}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.c","version":"1330'2222","reported_seq":2236,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:57.367638+0000","last_change":"2024-08-17T18:07:43.480649+0000","last_active":"2024-08-17T18:07:57.367638+0000","last_peered":"2024-08-17T18:07:57.367638+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:07:32.357966+0000","last_became_peered":"2024-08-17T18:07:32.357966+0000","last_unstale":"2024-08-17T18:07:57.367638+0000","last_undegraded":"2024-08-17T18:07:43.465843+0000","last_fullsized":"2024-08-17T18:07:32.353203+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2222,"log_dups_size":0,"ondisk_log_size":2222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:09:34.960276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145620992,"num_objects":2222,"num_object_clones":0,"num_object_copies":6666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2222,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2222,"num_write_kb":142208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":2222}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"48.b","version":"1330'2303","reported_seq":2311,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:58.083078+0000","last_change":"2024-08-17T18:07:43.467212+0000","last_active":"2024-08-17T18:07:58.083078+0000","last_peered":"2024-08-17T18:07:58.083078+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:07:32.359722+0000","last_became_peered":"2024-08-17T18:07:32.359722+0000","last_unstale":"2024-08-17T18:07:58.083078+0000","last_undegraded":"2024-08-17T18:07:43.467077+0000","last_fullsized":"2024-08-17T18:07:32.356967+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2303,"log_dups_size":0,"ondisk_log_size":2303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:39:00.125513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150929408,"num_objects":2303,"num_object_clones":0,"num_object_copies":6909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2303,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2303,"num_write_kb":147392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":2303}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.a","version":"1330'2478","reported_seq":2495,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:59.186184+0000","last_change":"2024-08-17T18:07:32.359061+0000","last_active":"2024-08-17T18:07:59.186184+0000","last_peered":"2024-08-17T18:07:59.186184+0000","last_clean":"2024-08-17T18:07:59.186184+0000","last_became_active":"2024-08-17T18:07:32.357107+0000","last_became_peered":"2024-08-17T18:07:32.357107+0000","last_unstale":"2024-08-17T18:07:59.186184+0000","last_undegraded":"2024-08-17T18:07:59.186184+0000","last_fullsized":"2024-08-17T18:07:59.186184+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2478,"log_dups_size":0,"ondisk_log_size":2478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:21:12.828725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162398208,"num_objects":2478,"num_object_clones":0,"num_object_copies":7434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2478,"num_write_kb":158592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.9","version":"1330'2342","reported_seq":2349,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:58.084425+0000","last_change":"2024-08-17T18:07:43.467035+0000","last_active":"2024-08-17T18:07:58.084425+0000","last_peered":"2024-08-17T18:07:58.084425+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:07:32.357593+0000","last_became_peered":"2024-08-17T18:07:32.357593+0000","last_unstale":"2024-08-17T18:07:58.084425+0000","last_undegraded":"2024-08-17T18:07:43.466636+0000","last_fullsized":"2024-08-17T18:07:32.355928+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2342,"log_dups_size":0,"ondisk_log_size":2342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:25.372534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153485312,"num_objects":2342,"num_object_clones":0,"num_object_copies":7026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2342,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2342,"num_write_kb":149888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2342}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.8","version":"1330'2230","reported_seq":2247,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:57.366480+0000","last_change":"2024-08-17T18:07:32.361066+0000","last_active":"2024-08-17T18:07:57.366480+0000","last_peered":"2024-08-17T18:07:57.366480+0000","last_clean":"2024-08-17T18:07:57.366480+0000","last_became_active":"2024-08-17T18:07:32.360742+0000","last_became_peered":"2024-08-17T18:07:32.360742+0000","last_unstale":"2024-08-17T18:07:57.366480+0000","last_undegraded":"2024-08-17T18:07:57.366480+0000","last_fullsized":"2024-08-17T18:07:57.366480+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2230,"log_dups_size":0,"ondisk_log_size":2230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:56:22.709911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146145280,"num_objects":2230,"num_object_clones":0,"num_object_copies":6690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2230,"num_write_kb":142720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"48.7","version":"1330'2105","reported_seq":2122,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:57.362711+0000","last_change":"2024-08-17T18:07:32.358470+0000","last_active":"2024-08-17T18:07:57.362711+0000","last_peered":"2024-08-17T18:07:57.362711+0000","last_clean":"2024-08-17T18:07:57.362711+0000","last_became_active":"2024-08-17T18:07:32.358061+0000","last_became_peered":"2024-08-17T18:07:32.358061+0000","last_unstale":"2024-08-17T18:07:57.362711+0000","last_undegraded":"2024-08-17T18:07:57.362711+0000","last_fullsized":"2024-08-17T18:07:57.362711+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2105,"log_dups_size":0,"ondisk_log_size":2105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:04.200426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137953280,"num_objects":2105,"num_object_clones":0,"num_object_copies":6315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2105,"num_write_kb":134720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"48.6","version":"1330'2164","reported_seq":2178,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:57.368787+0000","last_change":"2024-08-17T18:07:43.473621+0000","last_active":"2024-08-17T18:07:57.368787+0000","last_peered":"2024-08-17T18:07:57.368787+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:07:32.356385+0000","last_became_peered":"2024-08-17T18:07:32.356385+0000","last_unstale":"2024-08-17T18:07:57.368787+0000","last_undegraded":"2024-08-17T18:07:43.466166+0000","last_fullsized":"2024-08-17T18:07:32.353200+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2164,"log_dups_size":0,"ondisk_log_size":2164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:54:10.290761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141819904,"num_objects":2164,"num_object_clones":0,"num_object_copies":6492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2164,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2164,"num_write_kb":138496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":2164}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"48.5","version":"1330'2330","reported_seq":2338,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:58.084559+0000","last_change":"2024-08-17T18:07:43.466893+0000","last_active":"2024-08-17T18:07:58.084559+0000","last_peered":"2024-08-17T18:07:58.084559+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:07:32.361165+0000","last_became_peered":"2024-08-17T18:07:32.361165+0000","last_unstale":"2024-08-17T18:07:58.084559+0000","last_undegraded":"2024-08-17T18:07:43.466631+0000","last_fullsized":"2024-08-17T18:07:32.357197+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2330,"log_dups_size":0,"ondisk_log_size":2330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:41.234391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152698880,"num_objects":2330,"num_object_clones":0,"num_object_copies":6990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2330,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2330,"num_write_kb":149120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2330}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.4","version":"1330'2378","reported_seq":2392,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:58.644441+0000","last_change":"2024-08-17T18:07:43.468966+0000","last_active":"2024-08-17T18:07:58.644441+0000","last_peered":"2024-08-17T18:07:58.644441+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:07:32.360229+0000","last_became_peered":"2024-08-17T18:07:32.360229+0000","last_unstale":"2024-08-17T18:07:58.644441+0000","last_undegraded":"2024-08-17T18:07:43.465073+0000","last_fullsized":"2024-08-17T18:07:32.353194+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2378,"log_dups_size":0,"ondisk_log_size":2378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:20:40.567207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155844608,"num_objects":2378,"num_object_clones":0,"num_object_copies":7134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2378,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2378,"num_write_kb":152192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2378}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.3","version":"1330'2420","reported_seq":2434,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:58.673867+0000","last_change":"2024-08-17T18:07:43.473742+0000","last_active":"2024-08-17T18:07:58.673867+0000","last_peered":"2024-08-17T18:07:58.673867+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:07:32.358084+0000","last_became_peered":"2024-08-17T18:07:32.358084+0000","last_unstale":"2024-08-17T18:07:58.673867+0000","last_undegraded":"2024-08-17T18:07:43.468755+0000","last_fullsized":"2024-08-17T18:07:32.353168+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2420,"log_dups_size":0,"ondisk_log_size":2420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:13:07.078138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158597120,"num_objects":2420,"num_object_clones":0,"num_object_copies":7260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2420,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2420,"num_write_kb":154880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":2420}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.2","version":"1330'2263","reported_seq":2277,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:57.355571+0000","last_change":"2024-08-17T18:07:43.472984+0000","last_active":"2024-08-17T18:07:57.355571+0000","last_peered":"2024-08-17T18:07:57.355571+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:07:32.361149+0000","last_became_peered":"2024-08-17T18:07:32.361149+0000","last_unstale":"2024-08-17T18:07:57.355571+0000","last_undegraded":"2024-08-17T18:07:43.467161+0000","last_fullsized":"2024-08-17T18:07:32.353198+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2263,"log_dups_size":0,"ondisk_log_size":2263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:39.533977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148307968,"num_objects":2263,"num_object_clones":0,"num_object_copies":6789,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2263,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2263,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2263,"num_write_kb":144832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":2263}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"48.1","version":"1330'2328","reported_seq":2339,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:58.075463+0000","last_change":"2024-08-17T18:07:32.360312+0000","last_active":"2024-08-17T18:07:58.075463+0000","last_peered":"2024-08-17T18:07:58.075463+0000","last_clean":"2024-08-17T18:07:58.075463+0000","last_became_active":"2024-08-17T18:07:32.360092+0000","last_became_peered":"2024-08-17T18:07:32.360092+0000","last_unstale":"2024-08-17T18:07:58.075463+0000","last_undegraded":"2024-08-17T18:07:58.075463+0000","last_fullsized":"2024-08-17T18:07:58.075463+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2328,"log_dups_size":0,"ondisk_log_size":2328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:41.580676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152567808,"num_objects":2328,"num_object_clones":0,"num_object_copies":6984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2328,"num_write_kb":148992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.0","version":"1330'2199","reported_seq":2214,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:57.358905+0000","last_change":"2024-08-17T18:07:32.358823+0000","last_active":"2024-08-17T18:07:57.358905+0000","last_peered":"2024-08-17T18:07:57.358905+0000","last_clean":"2024-08-17T18:07:57.358905+0000","last_became_active":"2024-08-17T18:07:32.358556+0000","last_became_peered":"2024-08-17T18:07:32.358556+0000","last_unstale":"2024-08-17T18:07:57.358905+0000","last_undegraded":"2024-08-17T18:07:57.358905+0000","last_fullsized":"2024-08-17T18:07:57.358905+0000","mapping_epoch":1327,"log_start":"0'0","ondisk_log_start":"0'0","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":2199,"log_dups_size":0,"ondisk_log_size":2199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:54:35.762483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144113664,"num_objects":2199,"num_object_clones":0,"num_object_copies":6597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2199,"num_write_kb":140736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":2140,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:40.768808+0000","last_change":"2024-08-17T18:05:57.576924+0000","last_active":"2024-08-17T18:07:40.768808+0000","last_peered":"2024-08-17T18:07:40.768808+0000","last_clean":"2024-08-17T18:05:52.513997+0000","last_became_active":"2024-08-17T18:05:57.576924+0000","last_became_peered":"2024-08-17T18:05:57.576924+0000","last_unstale":"2024-08-17T18:07:40.768808+0000","last_undegraded":"2024-08-17T18:05:57.573273+0000","last_fullsized":"2024-08-17T18:05:57.573167+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:54:28.857797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":568,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.768380+0000","last_change":"2024-08-17T18:05:57.583081+0000","last_active":"2024-08-17T18:07:40.768380+0000","last_peered":"2024-08-17T18:07:40.768380+0000","last_clean":"2024-08-17T18:05:52.508116+0000","last_became_active":"2024-08-17T18:05:57.583081+0000","last_became_peered":"2024-08-17T18:05:57.583081+0000","last_unstale":"2024-08-17T18:07:40.768380+0000","last_undegraded":"2024-08-17T18:07:40.768380+0000","last_fullsized":"2024-08-17T18:05:57.568316+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:01:37.412468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1979,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.768562+0000","last_change":"2024-08-17T18:05:57.577802+0000","last_active":"2024-08-17T18:07:40.768562+0000","last_peered":"2024-08-17T18:07:40.768562+0000","last_clean":"2024-08-17T18:02:33.566169+0000","last_became_active":"2024-08-17T18:05:57.577802+0000","last_became_peered":"2024-08-17T18:05:57.577802+0000","last_unstale":"2024-08-17T18:07:40.768562+0000","last_undegraded":"2024-08-17T18:07:40.768562+0000","last_fullsized":"2024-08-17T18:05:57.568140+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:58.502435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1779,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.768609+0000","last_change":"2024-08-17T18:05:57.576974+0000","last_active":"2024-08-17T18:07:40.768609+0000","last_peered":"2024-08-17T18:07:40.768609+0000","last_clean":"2024-08-17T17:46:20.911869+0000","last_became_active":"2024-08-17T18:05:57.576974+0000","last_became_peered":"2024-08-17T18:05:57.576974+0000","last_unstale":"2024-08-17T18:07:40.768609+0000","last_undegraded":"2024-08-17T18:07:40.768609+0000","last_fullsized":"2024-08-17T18:05:57.574507+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:12:09.245998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":130,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:40.768954+0000","last_change":"2024-08-17T18:05:57.576774+0000","last_active":"2024-08-17T18:07:40.768954+0000","last_peered":"2024-08-17T18:07:40.768954+0000","last_clean":"2024-08-17T18:05:52.504868+0000","last_became_active":"2024-08-17T18:05:57.576774+0000","last_became_peered":"2024-08-17T18:05:57.576774+0000","last_unstale":"2024-08-17T18:07:40.768954+0000","last_undegraded":"2024-08-17T18:05:57.567775+0000","last_fullsized":"2024-08-17T18:05:57.567673+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:21:08.012525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":42,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T18:07:34.221126+0000","last_change":"2024-08-17T18:03:16.089811+0000","last_active":"2024-08-17T18:07:34.221126+0000","last_peered":"2024-08-17T18:07:34.221126+0000","last_clean":"2024-08-17T18:07:34.221126+0000","last_became_active":"2024-08-17T18:03:16.089571+0000","last_became_peered":"2024-08-17T18:03:16.089571+0000","last_unstale":"2024-08-17T18:07:34.221126+0000","last_undegraded":"2024-08-17T18:07:34.221126+0000","last_fullsized":"2024-08-17T18:07:34.221126+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:04.630361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":328,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.768117+0000","last_change":"2024-08-17T18:02:53.888880+0000","last_active":"2024-08-17T18:07:40.768117+0000","last_peered":"2024-08-17T18:07:40.768117+0000","last_clean":"2024-08-17T18:07:40.768117+0000","last_became_active":"2024-08-17T18:02:53.880008+0000","last_became_peered":"2024-08-17T18:02:53.880008+0000","last_unstale":"2024-08-17T18:07:40.768117+0000","last_undegraded":"2024-08-17T18:07:40.768117+0000","last_fullsized":"2024-08-17T18:07:40.768117+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:20.923751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":571,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.767501+0000","last_change":"2024-08-17T18:05:57.576735+0000","last_active":"2024-08-17T18:07:40.767501+0000","last_peered":"2024-08-17T18:07:40.767501+0000","last_clean":"2024-08-17T17:31:35.495707+0000","last_became_active":"2024-08-17T18:05:57.576735+0000","last_became_peered":"2024-08-17T18:05:57.576735+0000","last_unstale":"2024-08-17T18:07:40.767501+0000","last_undegraded":"2024-08-17T18:07:40.767501+0000","last_fullsized":"2024-08-17T18:05:57.566720+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:37.067194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":2080,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.770145+0000","last_change":"2024-08-17T17:49:45.694736+0000","last_active":"2024-08-17T18:07:40.770145+0000","last_peered":"2024-08-17T18:07:40.770145+0000","last_clean":"2024-08-17T18:07:40.770145+0000","last_became_active":"2024-08-17T17:49:45.694080+0000","last_became_peered":"2024-08-17T17:49:45.694080+0000","last_unstale":"2024-08-17T18:07:40.770145+0000","last_undegraded":"2024-08-17T18:07:40.770145+0000","last_fullsized":"2024-08-17T18:07:40.770145+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:39.014644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":695,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.768317+0000","last_change":"2024-08-17T18:05:57.579377+0000","last_active":"2024-08-17T18:07:40.768317+0000","last_peered":"2024-08-17T18:07:40.768317+0000","last_clean":"2024-08-17T18:05:52.516056+0000","last_became_active":"2024-08-17T18:05:57.579377+0000","last_became_peered":"2024-08-17T18:05:57.579377+0000","last_unstale":"2024-08-17T18:07:40.768317+0000","last_undegraded":"2024-08-17T18:07:40.768317+0000","last_fullsized":"2024-08-17T18:05:57.568496+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:56:03.836038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":302,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.767381+0000","last_change":"2024-08-17T17:49:45.665613+0000","last_active":"2024-08-17T18:07:40.767381+0000","last_peered":"2024-08-17T18:07:40.767381+0000","last_clean":"2024-08-17T18:07:40.767381+0000","last_became_active":"2024-08-17T17:49:45.665212+0000","last_became_peered":"2024-08-17T17:49:45.665212+0000","last_unstale":"2024-08-17T18:07:40.767381+0000","last_undegraded":"2024-08-17T18:07:40.767381+0000","last_fullsized":"2024-08-17T18:07:40.767381+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:08.672157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1991,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.768646+0000","last_change":"2024-08-17T18:05:57.640487+0000","last_active":"2024-08-17T18:07:40.768646+0000","last_peered":"2024-08-17T18:07:40.768646+0000","last_clean":"2024-08-17T18:02:33.567568+0000","last_became_active":"2024-08-17T18:05:57.640487+0000","last_became_peered":"2024-08-17T18:05:57.640487+0000","last_unstale":"2024-08-17T18:07:40.768646+0000","last_undegraded":"2024-08-17T18:07:40.768646+0000","last_fullsized":"2024-08-17T18:05:57.567989+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:10:42.421766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1882,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T18:07:34.221041+0000","last_change":"2024-08-17T18:05:57.575613+0000","last_active":"2024-08-17T18:07:34.221041+0000","last_peered":"2024-08-17T18:07:34.221041+0000","last_clean":"2024-08-17T18:02:33.562857+0000","last_became_active":"2024-08-17T18:05:57.575613+0000","last_became_peered":"2024-08-17T18:05:57.575613+0000","last_unstale":"2024-08-17T18:07:34.221041+0000","last_undegraded":"2024-08-17T18:07:34.221041+0000","last_fullsized":"2024-08-17T18:05:57.569997+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:52.951059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2108,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.769410+0000","last_change":"2024-08-17T18:05:57.585337+0000","last_active":"2024-08-17T18:07:40.769410+0000","last_peered":"2024-08-17T18:07:40.769410+0000","last_clean":"2024-08-17T18:05:52.513619+0000","last_became_active":"2024-08-17T18:05:57.585337+0000","last_became_peered":"2024-08-17T18:05:57.585337+0000","last_unstale":"2024-08-17T18:07:40.769410+0000","last_undegraded":"2024-08-17T18:07:40.769410+0000","last_fullsized":"2024-08-17T18:05:57.573021+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:43:09.059614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1904,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.768000+0000","last_change":"2024-08-17T17:59:58.028639+0000","last_active":"2024-08-17T18:07:40.768000+0000","last_peered":"2024-08-17T18:07:40.768000+0000","last_clean":"2024-08-17T18:07:40.768000+0000","last_became_active":"2024-08-17T17:59:58.028314+0000","last_became_peered":"2024-08-17T17:59:58.028314+0000","last_unstale":"2024-08-17T18:07:40.768000+0000","last_undegraded":"2024-08-17T18:07:40.768000+0000","last_fullsized":"2024-08-17T18:07:40.768000+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:25:03.619052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1256,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.769756+0000","last_change":"2024-08-17T18:05:57.577121+0000","last_active":"2024-08-17T18:07:40.769756+0000","last_peered":"2024-08-17T18:07:40.769756+0000","last_clean":"2024-08-17T18:05:52.504970+0000","last_became_active":"2024-08-17T18:05:57.577121+0000","last_became_peered":"2024-08-17T18:05:57.577121+0000","last_unstale":"2024-08-17T18:07:40.769756+0000","last_undegraded":"2024-08-17T18:07:40.769756+0000","last_fullsized":"2024-08-17T18:05:57.571280+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:36.618191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1985,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.767215+0000","last_change":"2024-08-17T18:05:57.574222+0000","last_active":"2024-08-17T18:07:40.767215+0000","last_peered":"2024-08-17T18:07:40.767215+0000","last_clean":"2024-08-17T18:02:33.563359+0000","last_became_active":"2024-08-17T18:05:57.574222+0000","last_became_peered":"2024-08-17T18:05:57.574222+0000","last_unstale":"2024-08-17T18:07:40.767215+0000","last_undegraded":"2024-08-17T18:07:40.767215+0000","last_fullsized":"2024-08-17T18:05:57.566285+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:53:19.645572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2108,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.769928+0000","last_change":"2024-08-17T18:05:57.580809+0000","last_active":"2024-08-17T18:07:40.769928+0000","last_peered":"2024-08-17T18:07:40.769928+0000","last_clean":"2024-08-17T18:05:52.513782+0000","last_became_active":"2024-08-17T18:05:57.580809+0000","last_became_peered":"2024-08-17T18:05:57.580809+0000","last_unstale":"2024-08-17T18:07:40.769928+0000","last_undegraded":"2024-08-17T18:07:40.769928+0000","last_fullsized":"2024-08-17T18:05:57.568935+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:55.215259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1929,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T18:07:34.221617+0000","last_change":"2024-08-17T18:05:57.577642+0000","last_active":"2024-08-17T18:07:34.221617+0000","last_peered":"2024-08-17T18:07:34.221617+0000","last_clean":"2024-08-17T18:02:33.562619+0000","last_became_active":"2024-08-17T18:05:57.577642+0000","last_became_peered":"2024-08-17T18:05:57.577642+0000","last_unstale":"2024-08-17T18:07:34.221617+0000","last_undegraded":"2024-08-17T18:07:34.221617+0000","last_fullsized":"2024-08-17T18:05:57.570516+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:01.223666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":630,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T18:07:34.221693+0000","last_change":"2024-08-17T18:05:57.584374+0000","last_active":"2024-08-17T18:07:34.221693+0000","last_peered":"2024-08-17T18:07:34.221693+0000","last_clean":"2024-08-17T18:05:52.519977+0000","last_became_active":"2024-08-17T18:05:57.584374+0000","last_became_peered":"2024-08-17T18:05:57.584374+0000","last_unstale":"2024-08-17T18:07:34.221693+0000","last_undegraded":"2024-08-17T18:07:34.221693+0000","last_fullsized":"2024-08-17T18:05:57.570001+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:00:36.495682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1307,"reported_epoch":1329,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:34.221922+0000","last_change":"2024-08-17T18:05:57.600140+0000","last_active":"2024-08-17T18:07:34.221922+0000","last_peered":"2024-08-17T18:07:34.221922+0000","last_clean":"2024-08-17T18:05:52.519820+0000","last_became_active":"2024-08-17T18:05:57.600140+0000","last_became_peered":"2024-08-17T18:05:57.600140+0000","last_unstale":"2024-08-17T18:07:34.221922+0000","last_undegraded":"2024-08-17T18:05:57.570950+0000","last_fullsized":"2024-08-17T18:05:57.570850+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:14.513477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2109,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.767666+0000","last_change":"2024-08-17T18:02:53.882334+0000","last_active":"2024-08-17T18:07:40.767666+0000","last_peered":"2024-08-17T18:07:40.767666+0000","last_clean":"2024-08-17T18:07:40.767666+0000","last_became_active":"2024-08-17T18:02:53.881815+0000","last_became_peered":"2024-08-17T18:02:53.881815+0000","last_unstale":"2024-08-17T18:07:40.767666+0000","last_undegraded":"2024-08-17T18:07:40.767666+0000","last_fullsized":"2024-08-17T18:07:40.767666+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:39.671800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":481,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T18:07:34.221664+0000","last_change":"2024-08-17T18:02:53.912837+0000","last_active":"2024-08-17T18:07:34.221664+0000","last_peered":"2024-08-17T18:07:34.221664+0000","last_clean":"2024-08-17T18:07:34.221664+0000","last_became_active":"2024-08-17T18:02:53.883152+0000","last_became_peered":"2024-08-17T18:02:53.883152+0000","last_unstale":"2024-08-17T18:07:34.221664+0000","last_undegraded":"2024-08-17T18:07:34.221664+0000","last_fullsized":"2024-08-17T18:07:34.221664+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:46.261612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":78,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.769809+0000","last_change":"2024-08-17T18:03:16.091944+0000","last_active":"2024-08-17T18:07:40.769809+0000","last_peered":"2024-08-17T18:07:40.769809+0000","last_clean":"2024-08-17T18:07:40.769809+0000","last_became_active":"2024-08-17T18:03:16.091314+0000","last_became_peered":"2024-08-17T18:03:16.091314+0000","last_unstale":"2024-08-17T18:07:40.769809+0000","last_undegraded":"2024-08-17T18:07:40.769809+0000","last_fullsized":"2024-08-17T18:07:40.769809+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:39.248867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1242,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.768332+0000","last_change":"2024-08-17T18:02:53.890198+0000","last_active":"2024-08-17T18:07:40.768332+0000","last_peered":"2024-08-17T18:07:40.768332+0000","last_clean":"2024-08-17T18:07:40.768332+0000","last_became_active":"2024-08-17T18:02:53.887007+0000","last_became_peered":"2024-08-17T18:02:53.887007+0000","last_unstale":"2024-08-17T18:07:40.768332+0000","last_undegraded":"2024-08-17T18:07:40.768332+0000","last_fullsized":"2024-08-17T18:07:40.768332+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:25.555909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":561,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T18:07:34.222055+0000","last_change":"2024-08-17T18:05:57.574166+0000","last_active":"2024-08-17T18:07:34.222055+0000","last_peered":"2024-08-17T18:07:34.222055+0000","last_clean":"2024-08-17T18:05:52.520047+0000","last_became_active":"2024-08-17T18:05:57.574166+0000","last_became_peered":"2024-08-17T18:05:57.574166+0000","last_unstale":"2024-08-17T18:07:34.222055+0000","last_undegraded":"2024-08-17T18:07:34.222055+0000","last_fullsized":"2024-08-17T18:05:57.571604+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:29.626084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":148,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.768941+0000","last_change":"2024-08-17T18:03:16.104851+0000","last_active":"2024-08-17T18:07:40.768941+0000","last_peered":"2024-08-17T18:07:40.768941+0000","last_clean":"2024-08-17T18:07:40.768941+0000","last_became_active":"2024-08-17T18:03:16.091641+0000","last_became_peered":"2024-08-17T18:03:16.091641+0000","last_unstale":"2024-08-17T18:07:40.768941+0000","last_undegraded":"2024-08-17T18:07:40.768941+0000","last_fullsized":"2024-08-17T18:07:40.768941+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.222166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1735,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T18:07:34.221026+0000","last_change":"2024-08-17T18:05:57.571389+0000","last_active":"2024-08-17T18:07:34.221026+0000","last_peered":"2024-08-17T18:07:34.221026+0000","last_clean":"2024-08-17T17:46:20.908980+0000","last_became_active":"2024-08-17T18:05:57.571389+0000","last_became_peered":"2024-08-17T18:05:57.571389+0000","last_unstale":"2024-08-17T18:07:34.221026+0000","last_undegraded":"2024-08-17T18:07:34.221026+0000","last_fullsized":"2024-08-17T18:05:57.569629+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:02.256460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":2120,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.769043+0000","last_change":"2024-08-17T18:05:57.578371+0000","last_active":"2024-08-17T18:07:40.769043+0000","last_peered":"2024-08-17T18:07:40.769043+0000","last_clean":"2024-08-17T18:05:52.503513+0000","last_became_active":"2024-08-17T18:05:57.578371+0000","last_became_peered":"2024-08-17T18:05:57.578371+0000","last_unstale":"2024-08-17T18:07:40.769043+0000","last_undegraded":"2024-08-17T18:07:40.769043+0000","last_fullsized":"2024-08-17T18:05:57.570029+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:38.370852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":52,"reported_epoch":1329,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:34.222174+0000","last_change":"2024-08-17T18:05:57.574881+0000","last_active":"2024-08-17T18:07:34.222174+0000","last_peered":"2024-08-17T18:07:34.222174+0000","last_clean":"2024-08-17T17:21:54.225304+0000","last_became_active":"2024-08-17T18:05:57.574881+0000","last_became_peered":"2024-08-17T18:05:57.574881+0000","last_unstale":"2024-08-17T18:07:34.222174+0000","last_undegraded":"2024-08-17T18:05:57.571794+0000","last_fullsized":"2024-08-17T18:05:57.571692+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:02.046020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2069,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.770189+0000","last_change":"2024-08-17T17:48:40.654350+0000","last_active":"2024-08-17T18:07:40.770189+0000","last_peered":"2024-08-17T18:07:40.770189+0000","last_clean":"2024-08-17T18:07:40.770189+0000","last_became_active":"2024-08-17T17:48:40.654070+0000","last_became_peered":"2024-08-17T17:48:40.654070+0000","last_unstale":"2024-08-17T18:07:40.770189+0000","last_undegraded":"2024-08-17T18:07:40.770189+0000","last_fullsized":"2024-08-17T18:07:40.770189+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:06.217389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1320,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T18:07:34.222129+0000","last_change":"2024-08-17T17:49:45.661564+0000","last_active":"2024-08-17T18:07:34.222129+0000","last_peered":"2024-08-17T18:07:34.222129+0000","last_clean":"2024-08-17T18:07:34.222129+0000","last_became_active":"2024-08-17T17:49:45.660591+0000","last_became_peered":"2024-08-17T17:49:45.660591+0000","last_unstale":"2024-08-17T18:07:34.222129+0000","last_undegraded":"2024-08-17T18:07:34.222129+0000","last_fullsized":"2024-08-17T18:07:34.222129+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:44:18.772098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":2115,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.768213+0000","last_change":"2024-08-17T17:48:40.707999+0000","last_active":"2024-08-17T18:07:40.768213+0000","last_peered":"2024-08-17T18:07:40.768213+0000","last_clean":"2024-08-17T18:07:40.768213+0000","last_became_active":"2024-08-17T17:48:40.707637+0000","last_became_peered":"2024-08-17T17:48:40.707637+0000","last_unstale":"2024-08-17T18:07:40.768213+0000","last_undegraded":"2024-08-17T18:07:40.768213+0000","last_fullsized":"2024-08-17T18:07:40.768213+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:41.090852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":302,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.767325+0000","last_change":"2024-08-17T17:49:45.664578+0000","last_active":"2024-08-17T18:07:40.767325+0000","last_peered":"2024-08-17T18:07:40.767325+0000","last_clean":"2024-08-17T18:07:40.767325+0000","last_became_active":"2024-08-17T17:49:45.664248+0000","last_became_peered":"2024-08-17T17:49:45.664248+0000","last_unstale":"2024-08-17T18:07:40.767325+0000","last_undegraded":"2024-08-17T18:07:40.767325+0000","last_fullsized":"2024-08-17T18:07:40.767325+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:51.771828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2134,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.768147+0000","last_change":"2024-08-17T17:48:39.657745+0000","last_active":"2024-08-17T18:07:40.768147+0000","last_peered":"2024-08-17T18:07:40.768147+0000","last_clean":"2024-08-17T18:07:40.768147+0000","last_became_active":"2024-08-17T17:48:39.656785+0000","last_became_peered":"2024-08-17T17:48:39.656785+0000","last_unstale":"2024-08-17T18:07:40.768147+0000","last_undegraded":"2024-08-17T18:07:40.768147+0000","last_fullsized":"2024-08-17T18:07:40.768147+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:12.969757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":796,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T18:07:34.221774+0000","last_change":"2024-08-17T18:05:57.574241+0000","last_active":"2024-08-17T18:07:34.221774+0000","last_peered":"2024-08-17T18:07:34.221774+0000","last_clean":"2024-08-17T18:02:47.965435+0000","last_became_active":"2024-08-17T18:05:57.574241+0000","last_became_peered":"2024-08-17T18:05:57.574241+0000","last_unstale":"2024-08-17T18:07:34.221774+0000","last_undegraded":"2024-08-17T18:07:34.221774+0000","last_fullsized":"2024-08-17T18:05:57.570743+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:47.614827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":49,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T18:07:34.221283+0000","last_change":"2024-08-17T18:05:57.575049+0000","last_active":"2024-08-17T18:07:34.221283+0000","last_peered":"2024-08-17T18:07:34.221283+0000","last_clean":"2024-08-17T18:05:52.519631+0000","last_became_active":"2024-08-17T18:05:57.575049+0000","last_became_peered":"2024-08-17T18:05:57.575049+0000","last_unstale":"2024-08-17T18:07:34.221283+0000","last_undegraded":"2024-08-17T18:07:34.221283+0000","last_fullsized":"2024-08-17T18:05:57.570356+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:52.912431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1455,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.769946+0000","last_change":"2024-08-17T18:05:57.580125+0000","last_active":"2024-08-17T18:07:40.769946+0000","last_peered":"2024-08-17T18:07:40.769946+0000","last_clean":"2024-08-17T18:05:52.518611+0000","last_became_active":"2024-08-17T18:05:57.580125+0000","last_became_peered":"2024-08-17T18:05:57.580125+0000","last_unstale":"2024-08-17T18:07:40.769946+0000","last_undegraded":"2024-08-17T18:07:40.769946+0000","last_fullsized":"2024-08-17T18:05:57.570108+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:23.693698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":380,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:40.767538+0000","last_change":"2024-08-17T18:05:57.579732+0000","last_active":"2024-08-17T18:07:40.767538+0000","last_peered":"2024-08-17T18:07:40.767538+0000","last_clean":"2024-08-17T18:05:52.504074+0000","last_became_active":"2024-08-17T18:05:57.579732+0000","last_became_peered":"2024-08-17T18:05:57.579732+0000","last_unstale":"2024-08-17T18:07:40.767538+0000","last_undegraded":"2024-08-17T18:05:57.567447+0000","last_fullsized":"2024-08-17T18:05:57.567342+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:43:50.099066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":2126,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.769514+0000","last_change":"2024-08-17T17:48:40.659012+0000","last_active":"2024-08-17T18:07:40.769514+0000","last_peered":"2024-08-17T18:07:40.769514+0000","last_clean":"2024-08-17T18:07:40.769514+0000","last_became_active":"2024-08-17T17:48:40.658649+0000","last_became_peered":"2024-08-17T17:48:40.658649+0000","last_unstale":"2024-08-17T18:07:40.769514+0000","last_undegraded":"2024-08-17T18:07:40.769514+0000","last_fullsized":"2024-08-17T18:07:40.769514+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:56.466274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2111,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.768090+0000","last_change":"2024-08-17T18:02:53.890366+0000","last_active":"2024-08-17T18:07:40.768090+0000","last_peered":"2024-08-17T18:07:40.768090+0000","last_clean":"2024-08-17T18:07:40.768090+0000","last_became_active":"2024-08-17T18:02:53.889898+0000","last_became_peered":"2024-08-17T18:02:53.889898+0000","last_unstale":"2024-08-17T18:07:40.768090+0000","last_undegraded":"2024-08-17T18:07:40.768090+0000","last_fullsized":"2024-08-17T18:07:40.768090+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:28:53.298525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1839,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.768783+0000","last_change":"2024-08-17T18:05:57.602329+0000","last_active":"2024-08-17T18:07:40.768783+0000","last_peered":"2024-08-17T18:07:40.768783+0000","last_clean":"2024-08-17T17:46:20.908480+0000","last_became_active":"2024-08-17T18:05:57.602329+0000","last_became_peered":"2024-08-17T18:05:57.602329+0000","last_unstale":"2024-08-17T18:07:40.768783+0000","last_undegraded":"2024-08-17T18:07:40.768783+0000","last_fullsized":"2024-08-17T18:05:57.575415+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:09.248092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2130,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.769621+0000","last_change":"2024-08-17T17:48:40.656181+0000","last_active":"2024-08-17T18:07:40.769621+0000","last_peered":"2024-08-17T18:07:40.769621+0000","last_clean":"2024-08-17T18:07:40.769621+0000","last_became_active":"2024-08-17T17:48:40.655676+0000","last_became_peered":"2024-08-17T17:48:40.655676+0000","last_unstale":"2024-08-17T18:07:40.769621+0000","last_undegraded":"2024-08-17T18:07:40.769621+0000","last_fullsized":"2024-08-17T18:07:40.769621+0000","mapping_epoch":1179,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:42.722230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2114,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.768425+0000","last_change":"2024-08-17T18:05:57.602894+0000","last_active":"2024-08-17T18:07:40.768425+0000","last_peered":"2024-08-17T18:07:40.768425+0000","last_clean":"2024-08-17T18:05:52.512919+0000","last_became_active":"2024-08-17T18:05:57.602894+0000","last_became_peered":"2024-08-17T18:05:57.602894+0000","last_unstale":"2024-08-17T18:07:40.768425+0000","last_undegraded":"2024-08-17T18:07:40.768425+0000","last_fullsized":"2024-08-17T18:05:57.571813+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:23.044869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1327,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T18:07:34.221325+0000","last_change":"2024-08-17T17:48:39.656453+0000","last_active":"2024-08-17T18:07:34.221325+0000","last_peered":"2024-08-17T18:07:34.221325+0000","last_clean":"2024-08-17T18:07:34.221325+0000","last_became_active":"2024-08-17T17:48:39.656162+0000","last_became_peered":"2024-08-17T17:48:39.656162+0000","last_unstale":"2024-08-17T18:07:34.221325+0000","last_undegraded":"2024-08-17T18:07:34.221325+0000","last_fullsized":"2024-08-17T18:07:34.221325+0000","mapping_epoch":1178,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.746878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1189,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T18:07:34.221995+0000","last_change":"2024-08-17T18:05:57.600170+0000","last_active":"2024-08-17T18:07:34.221995+0000","last_peered":"2024-08-17T18:07:34.221995+0000","last_clean":"2024-08-17T18:05:52.519879+0000","last_became_active":"2024-08-17T18:05:57.600170+0000","last_became_peered":"2024-08-17T18:05:57.600170+0000","last_unstale":"2024-08-17T18:07:34.221995+0000","last_undegraded":"2024-08-17T18:07:34.221995+0000","last_fullsized":"2024-08-17T18:05:57.571031+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:41:47.355114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1955,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.767115+0000","last_change":"2024-08-17T18:05:57.610066+0000","last_active":"2024-08-17T18:07:40.767115+0000","last_peered":"2024-08-17T18:07:40.767115+0000","last_clean":"2024-08-17T18:02:33.566105+0000","last_became_active":"2024-08-17T18:05:57.610066+0000","last_became_peered":"2024-08-17T18:05:57.610066+0000","last_unstale":"2024-08-17T18:07:40.767115+0000","last_undegraded":"2024-08-17T18:07:40.767115+0000","last_fullsized":"2024-08-17T18:05:57.566726+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:20:13.741021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":96,"reported_epoch":1329,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:34.221706+0000","last_change":"2024-08-17T18:05:57.574223+0000","last_active":"2024-08-17T18:07:34.221706+0000","last_peered":"2024-08-17T18:07:34.221706+0000","last_clean":"2024-08-17T18:05:52.519792+0000","last_became_active":"2024-08-17T18:05:57.574223+0000","last_became_peered":"2024-08-17T18:05:57.574223+0000","last_unstale":"2024-08-17T18:07:34.221706+0000","last_undegraded":"2024-08-17T18:05:57.570823+0000","last_fullsized":"2024-08-17T18:05:57.570707+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:18.741714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1665,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T18:07:34.220879+0000","last_change":"2024-08-17T18:02:56.855080+0000","last_active":"2024-08-17T18:07:34.220879+0000","last_peered":"2024-08-17T18:07:34.220879+0000","last_clean":"2024-08-17T18:07:34.220879+0000","last_became_active":"2024-08-17T18:02:56.854831+0000","last_became_peered":"2024-08-17T18:02:56.854831+0000","last_unstale":"2024-08-17T18:07:34.220879+0000","last_undegraded":"2024-08-17T18:07:34.220879+0000","last_fullsized":"2024-08-17T18:07:34.220879+0000","mapping_epoch":1286,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:25:24.111782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2114,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.769645+0000","last_change":"2024-08-17T18:05:57.576854+0000","last_active":"2024-08-17T18:07:40.769645+0000","last_peered":"2024-08-17T18:07:40.769645+0000","last_clean":"2024-08-17T18:05:52.504890+0000","last_became_active":"2024-08-17T18:05:57.576854+0000","last_became_peered":"2024-08-17T18:05:57.576854+0000","last_unstale":"2024-08-17T18:07:40.769645+0000","last_undegraded":"2024-08-17T18:07:40.769645+0000","last_fullsized":"2024-08-17T18:05:57.569579+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:51.683078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":543,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.768113+0000","last_change":"2024-08-17T18:05:57.577449+0000","last_active":"2024-08-17T18:07:40.768113+0000","last_peered":"2024-08-17T18:07:40.768113+0000","last_clean":"2024-08-17T17:30:29.464147+0000","last_became_active":"2024-08-17T18:05:57.577449+0000","last_became_peered":"2024-08-17T18:05:57.577449+0000","last_unstale":"2024-08-17T18:07:40.768113+0000","last_undegraded":"2024-08-17T18:07:40.768113+0000","last_fullsized":"2024-08-17T18:05:57.567821+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:53:02.049012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":568,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.768762+0000","last_change":"2024-08-17T18:05:57.578081+0000","last_active":"2024-08-17T18:07:40.768762+0000","last_peered":"2024-08-17T18:07:40.768762+0000","last_clean":"2024-08-17T18:05:52.503345+0000","last_became_active":"2024-08-17T18:05:57.578081+0000","last_became_peered":"2024-08-17T18:05:57.578081+0000","last_unstale":"2024-08-17T18:07:40.768762+0000","last_undegraded":"2024-08-17T18:07:40.768762+0000","last_fullsized":"2024-08-17T18:05:57.569569+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:04:09.273389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":567,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T18:07:34.220741+0000","last_change":"2024-08-17T17:59:58.021264+0000","last_active":"2024-08-17T18:07:34.220741+0000","last_peered":"2024-08-17T18:07:34.220741+0000","last_clean":"2024-08-17T18:07:34.220741+0000","last_became_active":"2024-08-17T17:59:58.020957+0000","last_became_peered":"2024-08-17T17:59:58.020957+0000","last_unstale":"2024-08-17T18:07:34.220741+0000","last_undegraded":"2024-08-17T18:07:34.220741+0000","last_fullsized":"2024-08-17T18:07:34.220741+0000","mapping_epoch":1263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:42.190270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":659,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.768555+0000","last_change":"2024-08-17T18:02:53.883783+0000","last_active":"2024-08-17T18:07:40.768555+0000","last_peered":"2024-08-17T18:07:40.768555+0000","last_clean":"2024-08-17T18:07:40.768555+0000","last_became_active":"2024-08-17T18:02:53.883489+0000","last_became_peered":"2024-08-17T18:02:53.883489+0000","last_unstale":"2024-08-17T18:07:40.768555+0000","last_undegraded":"2024-08-17T18:07:40.768555+0000","last_fullsized":"2024-08-17T18:07:40.768555+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:08.849092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1252,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.767751+0000","last_change":"2024-08-17T18:05:57.582579+0000","last_active":"2024-08-17T18:07:40.767751+0000","last_peered":"2024-08-17T18:07:40.767751+0000","last_clean":"2024-08-17T18:05:52.504161+0000","last_became_active":"2024-08-17T18:05:57.582579+0000","last_became_peered":"2024-08-17T18:05:57.582579+0000","last_unstale":"2024-08-17T18:07:40.767751+0000","last_undegraded":"2024-08-17T18:07:40.767751+0000","last_fullsized":"2024-08-17T18:05:57.567726+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:16.570962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":757,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.767845+0000","last_change":"2024-08-17T18:05:57.577380+0000","last_active":"2024-08-17T18:07:40.767845+0000","last_peered":"2024-08-17T18:07:40.767845+0000","last_clean":"2024-08-17T18:02:33.567910+0000","last_became_active":"2024-08-17T18:05:57.577380+0000","last_became_peered":"2024-08-17T18:05:57.577380+0000","last_unstale":"2024-08-17T18:07:40.767845+0000","last_undegraded":"2024-08-17T18:07:40.767845+0000","last_fullsized":"2024-08-17T18:05:57.567022+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:01.958192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":74,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:40.769970+0000","last_change":"2024-08-17T18:05:57.579860+0000","last_active":"2024-08-17T18:07:40.769970+0000","last_peered":"2024-08-17T18:07:40.769970+0000","last_clean":"2024-08-17T17:21:54.224963+0000","last_became_active":"2024-08-17T18:05:57.579860+0000","last_became_peered":"2024-08-17T18:05:57.579860+0000","last_unstale":"2024-08-17T18:07:40.769970+0000","last_undegraded":"2024-08-17T18:05:57.569888+0000","last_fullsized":"2024-08-17T18:05:57.569770+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:28.990384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2146,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.770250+0000","last_change":"2024-08-17T18:05:57.580398+0000","last_active":"2024-08-17T18:07:40.770250+0000","last_peered":"2024-08-17T18:07:40.770250+0000","last_clean":"2024-08-17T18:05:52.519507+0000","last_became_active":"2024-08-17T18:05:57.580398+0000","last_became_peered":"2024-08-17T18:05:57.580398+0000","last_unstale":"2024-08-17T18:07:40.770250+0000","last_undegraded":"2024-08-17T18:07:40.770250+0000","last_fullsized":"2024-08-17T18:05:57.573723+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:30.357088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":835,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.768064+0000","last_change":"2024-08-17T18:03:16.090929+0000","last_active":"2024-08-17T18:07:40.768064+0000","last_peered":"2024-08-17T18:07:40.768064+0000","last_clean":"2024-08-17T18:07:40.768064+0000","last_became_active":"2024-08-17T18:03:16.090554+0000","last_became_peered":"2024-08-17T18:03:16.090554+0000","last_unstale":"2024-08-17T18:07:40.768064+0000","last_undegraded":"2024-08-17T18:07:40.768064+0000","last_fullsized":"2024-08-17T18:07:40.768064+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:01.783375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":557,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T18:07:40.767423+0000","last_change":"2024-08-17T18:02:53.881999+0000","last_active":"2024-08-17T18:07:40.767423+0000","last_peered":"2024-08-17T18:07:40.767423+0000","last_clean":"2024-08-17T18:07:40.767423+0000","last_became_active":"2024-08-17T18:02:53.881681+0000","last_became_peered":"2024-08-17T18:02:53.881681+0000","last_unstale":"2024-08-17T18:07:40.767423+0000","last_undegraded":"2024-08-17T18:07:40.767423+0000","last_fullsized":"2024-08-17T18:07:40.767423+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:39:51.334407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1300,"reported_epoch":1329,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:34.222062+0000","last_change":"2024-08-17T18:05:57.610266+0000","last_active":"2024-08-17T18:07:34.222062+0000","last_peered":"2024-08-17T18:07:34.222062+0000","last_clean":"2024-08-17T18:05:52.519913+0000","last_became_active":"2024-08-17T18:05:57.610266+0000","last_became_peered":"2024-08-17T18:05:57.610266+0000","last_unstale":"2024-08-17T18:07:34.222062+0000","last_undegraded":"2024-08-17T18:05:57.571269+0000","last_fullsized":"2024-08-17T18:05:57.571145+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:08:18.066784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":849,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.767113+0000","last_change":"2024-08-17T18:05:57.579896+0000","last_active":"2024-08-17T18:07:40.767113+0000","last_peered":"2024-08-17T18:07:40.767113+0000","last_clean":"2024-08-17T18:05:52.503294+0000","last_became_active":"2024-08-17T18:05:57.579896+0000","last_became_peered":"2024-08-17T18:05:57.579896+0000","last_unstale":"2024-08-17T18:07:40.767113+0000","last_undegraded":"2024-08-17T18:07:40.767113+0000","last_fullsized":"2024-08-17T18:05:57.566294+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:03:31.237283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1297,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T18:07:34.221935+0000","last_change":"2024-08-17T17:49:45.711216+0000","last_active":"2024-08-17T18:07:34.221935+0000","last_peered":"2024-08-17T18:07:34.221935+0000","last_clean":"2024-08-17T18:07:34.221935+0000","last_became_active":"2024-08-17T17:49:45.668241+0000","last_became_peered":"2024-08-17T17:49:45.668241+0000","last_unstale":"2024-08-17T18:07:34.221935+0000","last_undegraded":"2024-08-17T18:07:34.221935+0000","last_fullsized":"2024-08-17T18:07:34.221935+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:09:49.353723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":198,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T18:07:34.220866+0000","last_change":"2024-08-17T18:05:57.592009+0000","last_active":"2024-08-17T18:07:34.220866+0000","last_peered":"2024-08-17T18:07:34.220866+0000","last_clean":"2024-08-17T18:05:52.519349+0000","last_became_active":"2024-08-17T18:05:57.592009+0000","last_became_peered":"2024-08-17T18:05:57.592009+0000","last_unstale":"2024-08-17T18:07:34.220866+0000","last_undegraded":"2024-08-17T18:07:34.220866+0000","last_fullsized":"2024-08-17T18:05:57.569992+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:36.538532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1313,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T18:07:34.221969+0000","last_change":"2024-08-17T17:49:45.696572+0000","last_active":"2024-08-17T18:07:34.221969+0000","last_peered":"2024-08-17T18:07:34.221969+0000","last_clean":"2024-08-17T18:07:34.221969+0000","last_became_active":"2024-08-17T17:49:45.660189+0000","last_became_peered":"2024-08-17T17:49:45.660189+0000","last_unstale":"2024-08-17T18:07:34.221969+0000","last_undegraded":"2024-08-17T18:07:34.221969+0000","last_fullsized":"2024-08-17T18:07:34.221969+0000","mapping_epoch":1188,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1189,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:32:16.558020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":131,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:07:40.767627+0000","last_change":"2024-08-17T18:05:57.575920+0000","last_active":"2024-08-17T18:07:40.767627+0000","last_peered":"2024-08-17T18:07:40.767627+0000","last_clean":"2024-08-17T18:05:52.512678+0000","last_became_active":"2024-08-17T18:05:57.575920+0000","last_became_peered":"2024-08-17T18:05:57.575920+0000","last_unstale":"2024-08-17T18:07:40.767627+0000","last_undegraded":"2024-08-17T18:05:57.569027+0000","last_fullsized":"2024-08-17T18:05:57.568902+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:15.539634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":530,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T18:07:34.221971+0000","last_change":"2024-08-17T18:05:57.580595+0000","last_active":"2024-08-17T18:07:34.221971+0000","last_peered":"2024-08-17T18:07:34.221971+0000","last_clean":"2024-08-17T17:30:29.464678+0000","last_became_active":"2024-08-17T18:05:57.580595+0000","last_became_peered":"2024-08-17T18:05:57.580595+0000","last_unstale":"2024-08-17T18:07:34.221971+0000","last_undegraded":"2024-08-17T18:07:34.221971+0000","last_fullsized":"2024-08-17T18:05:57.571128+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:26:15.011550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1291,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T18:07:34.221359+0000","last_change":"2024-08-17T18:05:57.577751+0000","last_active":"2024-08-17T18:07:34.221359+0000","last_peered":"2024-08-17T18:07:34.221359+0000","last_clean":"2024-08-17T18:05:52.519529+0000","last_became_active":"2024-08-17T18:05:57.577751+0000","last_became_peered":"2024-08-17T18:05:57.577751+0000","last_unstale":"2024-08-17T18:07:34.221359+0000","last_undegraded":"2024-08-17T18:07:34.221359+0000","last_fullsized":"2024-08-17T18:05:57.570607+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:23.418524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2105,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.769253+0000","last_change":"2024-08-17T18:05:57.584843+0000","last_active":"2024-08-17T18:07:40.769253+0000","last_peered":"2024-08-17T18:07:40.769253+0000","last_clean":"2024-08-17T18:05:52.513487+0000","last_became_active":"2024-08-17T18:05:57.584843+0000","last_became_peered":"2024-08-17T18:05:57.584843+0000","last_unstale":"2024-08-17T18:07:40.769253+0000","last_undegraded":"2024-08-17T18:07:40.769253+0000","last_fullsized":"2024-08-17T18:05:57.572307+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:40.133256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2106,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.769337+0000","last_change":"2024-08-17T18:05:57.573848+0000","last_active":"2024-08-17T18:07:40.769337+0000","last_peered":"2024-08-17T18:07:40.769337+0000","last_clean":"2024-08-17T18:05:52.513455+0000","last_became_active":"2024-08-17T18:05:57.573848+0000","last_became_peered":"2024-08-17T18:05:57.573848+0000","last_unstale":"2024-08-17T18:07:40.769337+0000","last_undegraded":"2024-08-17T18:07:40.769337+0000","last_fullsized":"2024-08-17T18:05:57.570342+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1179,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:37:33.457167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":707,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T18:07:34.221811+0000","last_change":"2024-08-17T18:03:16.091420+0000","last_active":"2024-08-17T18:07:34.221811+0000","last_peered":"2024-08-17T18:07:34.221811+0000","last_clean":"2024-08-17T18:07:34.221811+0000","last_became_active":"2024-08-17T18:03:16.091113+0000","last_became_peered":"2024-08-17T18:03:16.091113+0000","last_unstale":"2024-08-17T18:07:34.221811+0000","last_undegraded":"2024-08-17T18:07:34.221811+0000","last_fullsized":"2024-08-17T18:07:34.221811+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:47.770911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1315,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T18:07:34.221622+0000","last_change":"2024-08-17T18:02:53.893927+0000","last_active":"2024-08-17T18:07:34.221622+0000","last_peered":"2024-08-17T18:07:34.221622+0000","last_clean":"2024-08-17T18:07:34.221622+0000","last_became_active":"2024-08-17T18:02:53.883188+0000","last_became_peered":"2024-08-17T18:02:53.883188+0000","last_unstale":"2024-08-17T18:07:34.221622+0000","last_undegraded":"2024-08-17T18:07:34.221622+0000","last_fullsized":"2024-08-17T18:07:34.221622+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:19.296965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":542,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T18:07:40.768243+0000","last_change":"2024-08-17T18:05:57.578347+0000","last_active":"2024-08-17T18:07:40.768243+0000","last_peered":"2024-08-17T18:07:40.768243+0000","last_clean":"2024-08-17T17:30:29.464349+0000","last_became_active":"2024-08-17T18:05:57.578347+0000","last_became_peered":"2024-08-17T18:05:57.578347+0000","last_unstale":"2024-08-17T18:07:40.768243+0000","last_undegraded":"2024-08-17T18:07:40.768243+0000","last_fullsized":"2024-08-17T18:05:57.568253+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:03:53.349835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":48,"num_pg":16,"stat_sum":{"num_bytes":2396258304,"num_objects":36564,"num_object_clones":0,"num_object_copies":109692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25224,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":36564,"num_write_kb":2340096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5686558720,"data_stored":5686558720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":36564,"ondisk_log_size":36564,"up":37,"acting":37,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":95,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":19,"acting":19,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":10948,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":151552,"data_stored":27468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":77,"acting":77,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":75,"acting":75,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234565,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2302896,"kb_used_data":1717160,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":91442256,"statfs":{"total":95995035648,"available":93636870144,"internally_reserved":0,"allocated":1758371840,"data_stored":1754112470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568496200,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757838,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":571,"seq":2452426326427,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3373252,"kb_used_data":2820792,"kb_used_omap":57,"kb_used_meta":552134,"kb_avail":90371900,"statfs":{"total":95995035648,"available":92540825600,"internally_reserved":0,"allocated":2888491008,"data_stored":2884344704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":58629,"internal_metadata":565385979},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405691,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2498788,"kb_used_data":1803996,"kb_used_omap":61,"kb_used_meta":694722,"kb_avail":91246364,"statfs":{"total":95995035648,"available":93436276736,"internally_reserved":0,"allocated":1847291904,"data_stored":1843044678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":711396280},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452225,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1559644,"kb_used_data":972884,"kb_used_omap":54,"kb_used_meta":586697,"kb_avail":92185508,"statfs":{"total":95995035648,"available":94397960192,"internally_reserved":0,"allocated":996233216,"data_stored":991989419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":600778126},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474916,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1601592,"kb_used_data":919152,"kb_used_omap":119,"kb_used_meta":682312,"kb_avail":92143560,"statfs":{"total":95995035648,"available":94355005440,"internally_reserved":0,"allocated":941211648,"data_stored":936944695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":698688222},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,0,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019473,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":944046080,"data_stored":944046080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1006632960,"data_stored":1006632960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1919352832,"data_stored":1919352832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1816526848,"data_stored":1816526848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:08:04.619 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_3 pgp_num 16 2024-08-17T18:08:04.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:04.620 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:04.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:04.721 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:04.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:04.745 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:04.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:04 smithi179 ceph-mon[93519]: pgmap v2334: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.7 GiB data, 12 GiB used, 436 GiB / 447 GiB avail; 158 MiB/s wr, 2.52k op/s; 30402/132354 objects degraded (22.970%) 2024-08-17T18:08:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:04 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4026861132' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:08:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:04 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:08:04.846 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:04.888 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:04.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:04.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:04.907 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:04 smithi130 ceph-mon[105521]: pgmap v2334: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.7 GiB data, 12 GiB used, 436 GiB / 447 GiB avail; 158 MiB/s wr, 2.52k op/s; 30402/132354 objects degraded (22.970%) 2024-08-17T18:08:04.907 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:04 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4026861132' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:08:04.908 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:04 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:08:04.908 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:04 smithi130 ceph-mon[110068]: pgmap v2334: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.7 GiB data, 12 GiB used, 436 GiB / 447 GiB avail; 158 MiB/s wr, 2.52k op/s; 30402/132354 objects degraded (22.970%) 2024-08-17T18:08:04.908 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:04 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4026861132' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:08:04.908 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:04 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:08:04.990 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:05.038 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:05.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:05.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:05.140 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:05.164 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:05.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:05.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:05.266 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:05.292 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:05.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:05.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:05.394 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:05.419 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:05.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:05.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:05.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 22 16 50237 50221 142.639 110.062 0.0136047 0.00700547 2024-08-17T18:08:05.520 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:05.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:05.547 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:05.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:05.648 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:05.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:05.688 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:05.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:05.788 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:05.812 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:05.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:05.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:05.822 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:05 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:08:05.822 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:05 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:08:05.822 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:05 smithi130 ceph-mon[105521]: osdmap e1331: 8 total, 7 up, 5 in 2024-08-17T18:08:05.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:05 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:08:05.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:05 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:08:05.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:05 smithi130 ceph-mon[110068]: osdmap e1331: 8 total, 7 up, 5 in 2024-08-17T18:08:05.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:05 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:08:05.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:05 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:08:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:05 smithi179 ceph-mon[93519]: osdmap e1331: 8 total, 7 up, 5 in 2024-08-17T18:08:05.914 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:05.939 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:05.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:05.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:06.041 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:06.065 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:06.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:06.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:06.092 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:08:06.114 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:08:06.166 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:06.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:06.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:06.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:06.318 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:06.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:06.343 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:06.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:06.444 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:06.466 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:06.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:06.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:06.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 23 16 51993 51977 141.209 109.75 0.00241665 0.00707596 2024-08-17T18:08:06.568 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:06.590 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:06.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:06.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:06.692 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:06.731 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:06.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:06.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:06.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:06 smithi130 ceph-mon[105521]: pgmap v2336: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 125 MiB/s wr, 2.00k op/s; 31538/137427 objects degraded (22.949%) 2024-08-17T18:08:06.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 31538/137427 objects degraded (22.949%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:06.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:06 smithi130 ceph-mon[110068]: pgmap v2336: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 125 MiB/s wr, 2.00k op/s; 31538/137427 objects degraded (22.949%) 2024-08-17T18:08:06.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 31538/137427 objects degraded (22.949%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:06.833 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:06.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:06 smithi179 ceph-mon[93519]: pgmap v2336: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 125 MiB/s wr, 2.00k op/s; 31538/137427 objects degraded (22.949%) 2024-08-17T18:08:06.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 31538/137427 objects degraded (22.949%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:06.868 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:06.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:06.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:06.970 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:06.996 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:06.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:06.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:07.097 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:07.121 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:07.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:07.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:07.223 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:07.249 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:07.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:07.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:07.350 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:07.374 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:07.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:07.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:07.467 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 24 16 53702 53686 139.776 106.812 0.00812377 0.00715016 2024-08-17T18:08:07.476 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:07.511 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:07.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:07.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:07.594 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:08:07.613 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:07.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:07.666 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:07.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:07.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:07.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:07.807 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:07.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:07.908 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:07.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:07.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:07.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:08.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:08.074 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:08.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:08.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:08.176 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:08.201 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:08.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:08.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:08.303 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:08.338 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:08.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:08.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:08.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:08.470 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 25 16 55289 55273 138.135 99.1875 0.0140158 0.00723446 2024-08-17T18:08:08.475 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:08.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:08.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:08.577 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:08.613 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:08.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:08.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:08.715 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:08.753 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:08.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:08.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:08.786 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:08 smithi130 ceph-mon[105521]: pgmap v2337: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 125 MiB/s wr, 2.00k op/s; 31538/137427 objects degraded (22.949%) 2024-08-17T18:08:08.786 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:08 smithi130 ceph-mon[110068]: pgmap v2337: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 125 MiB/s wr, 2.00k op/s; 31538/137427 objects degraded (22.949%) 2024-08-17T18:08:08.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:08 smithi179 ceph-mon[93519]: pgmap v2337: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 2.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 125 MiB/s wr, 2.00k op/s; 31538/137427 objects degraded (22.949%) 2024-08-17T18:08:08.856 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:08.879 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:08.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:08.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:08.981 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:09.023 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:09.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:09.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:09.125 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:09.150 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:09.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:09.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:09.252 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:09.287 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:09.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:09.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:09.390 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:09.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:09.422 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:09.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:09.470 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 26 16 56955 56939 136.827 104.125 0.008871 0.00730325 2024-08-17T18:08:09.523 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:09.561 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:09.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:09.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:09.662 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:09.688 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:09.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:09.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:09.790 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:09.827 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:09.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:09.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:09.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:09.963 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:09.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:09.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:10.065 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:10.078 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:08:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:08:09] "GET /metrics HTTP/1.1" 200 36782 "" "Prometheus/2.51.0" 2024-08-17T18:08:10.087 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:10.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:10.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:10.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:10.212 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:10.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:10.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:10.314 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:10.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:10.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:10.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:10.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:10.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:10.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:10.470 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 27 15 58471 58456 135.271 94.8125 0.00972834 0.00738804 2024-08-17T18:08:10.540 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:08:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:08:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:08:10.563 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:10.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:10.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:10.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:10.690 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:10.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:10.748 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:10.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:10 smithi179 ceph-mon[93519]: pgmap v2338: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 111 MiB/s wr, 1.78k op/s; 37403/163257 objects degraded (22.911%) 2024-08-17T18:08:10.849 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:10.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:10.871 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:10.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:10.872 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:10 smithi130 ceph-mon[105521]: pgmap v2338: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 111 MiB/s wr, 1.78k op/s; 37403/163257 objects degraded (22.911%) 2024-08-17T18:08:10.872 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:10 smithi130 ceph-mon[110068]: pgmap v2338: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 111 MiB/s wr, 1.78k op/s; 37403/163257 objects degraded (22.911%) 2024-08-17T18:08:10.973 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:11.016 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:11.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:11.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:11.118 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:11.139 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:11.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:11.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:11.241 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:11.267 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:11.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:11.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:11.369 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:11.391 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:11.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:11.471 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 28 16 59417 59401 132.55 59.0625 0.20162 0.00753008 2024-08-17T18:08:11.491 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:11.514 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:11.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:11.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:11.617 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:11.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:11.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:11.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:11.757 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:11.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:11.796 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:11.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:11 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:08:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 37403/163257 objects degraded (22.911%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:11 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:11 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:11.897 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:11 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:08:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 37403/163257 objects degraded (22.911%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:11 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:11.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:11 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:11 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:08:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 37403/163257 objects degraded (22.911%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:11 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:11 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:11.921 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:11.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:11.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:12.023 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:12.059 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:12.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:12.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:12.161 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:12.189 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:12.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:12.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:12.291 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:12.361 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:12.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:12.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:12.463 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:12.471 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 29 16 61058 61042 131.512 102.562 0.00413087 0.00759892 2024-08-17T18:08:12.496 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:12.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:12.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:12.598 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:12.624 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:12.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:12.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:12.726 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:12.751 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:12.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:12.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:12.852 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:12.876 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:12.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:12.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:12.978 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:12.998 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:12.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:12.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:13.100 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:13.121 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:13.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:13.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:12 smithi130 ceph-mon[105521]: pgmap v2339: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 111 MiB/s wr, 1.78k op/s; 37403/163257 objects degraded (22.911%) 2024-08-17T18:08:13.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:08:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:12 smithi130 ceph-mon[110068]: pgmap v2339: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 111 MiB/s wr, 1.78k op/s; 37403/163257 objects degraded (22.911%) 2024-08-17T18:08:13.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:08:13.223 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:13.253 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:13.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:13.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:12 smithi179 ceph-mon[93519]: pgmap v2339: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.3 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 111 MiB/s wr, 1.78k op/s; 37403/163257 objects degraded (22.911%) 2024-08-17T18:08:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:08:13.356 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:13.377 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:13.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:13.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:13.471 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 30 16 62244 62228 129.599 74.125 0.00533629 0.007702 2024-08-17T18:08:13.478 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:13.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:13.501 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:13.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:13.602 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:13.628 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:13.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:13.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:13.729 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:13.766 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:13.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:13.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:13.868 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:13.910 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:13.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:13.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:14.012 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:14.055 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:14.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:14.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:14.157 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:14.198 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:14.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:14.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:14.300 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:14.323 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:14.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:14.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:14.425 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:14.453 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:14.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:14.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:14.472 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 31 16 63539 63523 128.029 80.9375 0.0251971 0.00780512 2024-08-17T18:08:14.554 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:14.614 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:14.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:14.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:14.715 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:14.739 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:14.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:14.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:14.841 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:14.885 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:14.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:14.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:14.987 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:15.035 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:15.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:15.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:15.137 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:15.161 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:15.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:15.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:15.262 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:15.287 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:15.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:15.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:15.389 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:15.413 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:15.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:15.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:15.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:15 smithi130 ceph-mon[105521]: pgmap v2340: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 101 MiB/s wr, 1.62k op/s; 41496/180957 objects degraded (22.931%) 2024-08-17T18:08:15.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:15 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:15.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:15 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:15.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:15 smithi130 ceph-mon[110068]: pgmap v2340: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 101 MiB/s wr, 1.62k op/s; 41496/180957 objects degraded (22.931%) 2024-08-17T18:08:15.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:15 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:15.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:15 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:15.423 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:15 smithi179 ceph-mon[93519]: pgmap v2340: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.7 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 101 MiB/s wr, 1.62k op/s; 41496/180957 objects degraded (22.931%) 2024-08-17T18:08:15.423 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:15 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:15.423 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:15 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:15.472 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 32 16 64932 64916 126.749 87.0625 0.00497565 0.00788526 2024-08-17T18:08:15.515 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:15.539 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:15.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:15.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:15.641 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:15.689 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:08:15.689 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:08:15.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:15.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:15.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:15.795 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:15.829 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:15.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:15.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:15.931 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:15.969 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:15.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:15.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:16.071 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:16.094 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:16.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:16.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:16.196 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:16.220 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:16.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:16.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:16.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:16.357 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:16.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:16.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:16.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 41496/180957 objects degraded (22.931%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:16.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 41496/180957 objects degraded (22.931%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:16.459 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:16.472 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 33 16 66311 66295 125.52 86.1875 0.00992532 0.0079626 2024-08-17T18:08:16.484 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:16.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:16.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:16.586 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:16.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 41496/180957 objects degraded (22.931%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:16.611 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:16.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:16.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:16.713 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:16.735 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:16.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:16.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:16.739 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T18:08:16.838 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:16.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:16.897 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:16.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:16.989 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:08:16.997 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:17.022 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:17.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:17.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:17.057 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:08:17.057 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:08:17.108 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:08:17.108 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:08:17.124 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:17.154 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:08:17.155 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:08:17.159 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:08:17.181 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:17.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:17.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:17.191 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:08:17.191 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:08:17.230 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:08:17.230 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:08:17.230 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:17 smithi130 ceph-mon[105521]: pgmap v2341: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 91 MiB/s wr, 1.46k op/s; 42382/184914 objects degraded (22.920%) 2024-08-17T18:08:17.230 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:17 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:17.230 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:17 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:17.230 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:17 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/728235450' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:08:17.231 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:17 smithi130 ceph-mon[110068]: pgmap v2341: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 91 MiB/s wr, 1.46k op/s; 42382/184914 objects degraded (22.920%) 2024-08-17T18:08:17.231 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:17 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:17.231 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:17 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:17.231 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:17 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/728235450' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:08:17.266 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:08:17.266 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:08:17.282 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:17.300 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:08:17.300 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:08:17.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:17.340 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:17.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:17.441 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:17.466 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:17.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:17.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:17.472 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 34 16 67490 67474 123.996 73.6875 0.00722882 0.00805986 2024-08-17T18:08:17.568 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:17.591 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:17.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:17.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:17.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:17 smithi179 ceph-mon[93519]: pgmap v2341: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 91 MiB/s wr, 1.46k op/s; 42382/184914 objects degraded (22.920%) 2024-08-17T18:08:17.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:17 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:17 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:17.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:17 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/728235450' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:08:17.693 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:17.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:17.737 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:17.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:17.838 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:17.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:17.875 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:17.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:17.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:18.014 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:18.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:18.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:18.116 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:18.126 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:08:18.126 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:08:18.141 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:18.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:18.155 INFO:teuthology.orchestra.run.smithi130.stderr:set pool 48 pgp_num to 16 2024-08-17T18:08:18.243 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:18.268 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:18.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:18.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:18.347 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/727283271' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T18:08:18.347 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:18 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T18:08:18.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/727283271' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T18:08:18.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:18 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T18:08:18.370 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:18.393 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:18.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:18.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:18.472 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 35 16 68980 68964 123.114 93.125 0.00870285 0.00811787 2024-08-17T18:08:18.495 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:18.518 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:18.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:18.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/727283271' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T18:08:18.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:18 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T18:08:18.620 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:18.646 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:18.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:18.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:18.748 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:18.784 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:18.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:18.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:18.886 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:18.928 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:18.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:18.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:19.029 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:19.069 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:19.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:19.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:19.171 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:19.196 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:19.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:19.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:19.298 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:19.321 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:19.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:19.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:19 smithi130 ceph-mon[105521]: pgmap v2342: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 82 MiB/s wr, 1.32k op/s; 42382/184914 objects degraded (22.920%) 2024-08-17T18:08:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:19 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T18:08:19.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:19 smithi130 ceph-mon[105521]: osdmap e1332: 8 total, 7 up, 5 in 2024-08-17T18:08:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:19 smithi130 ceph-mon[110068]: pgmap v2342: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 82 MiB/s wr, 1.32k op/s; 42382/184914 objects degraded (22.920%) 2024-08-17T18:08:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:19 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T18:08:19.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:19 smithi130 ceph-mon[110068]: osdmap e1332: 8 total, 7 up, 5 in 2024-08-17T18:08:19.423 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:19.447 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:19.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:19.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:19.472 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 36 16 70540 70524 122.402 97.5 0.00390063 0.00816582 2024-08-17T18:08:19.549 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:19.587 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:19.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:19.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:19.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:19 smithi179 ceph-mon[93519]: pgmap v2342: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 3.8 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 82 MiB/s wr, 1.32k op/s; 42382/184914 objects degraded (22.920%) 2024-08-17T18:08:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:19 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T18:08:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:19 smithi179 ceph-mon[93519]: osdmap e1332: 8 total, 7 up, 5 in 2024-08-17T18:08:19.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:19.712 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:19.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:19.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:19.814 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:19.839 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:19.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:19.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:19.941 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:19.967 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:19.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:19.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:19.999 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:08:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:08:19] "GET /metrics HTTP/1.1" 200 36789 "" "Prometheus/2.51.0" 2024-08-17T18:08:20.069 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:20.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:08:20.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:20.138 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:20.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:20.239 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:20.263 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:20.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:20.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:20.365 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:20.387 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:20.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:20.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:20.473 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 37 16 71887 71871 121.367 84.1875 0.0207069 0.00823437 2024-08-17T18:08:20.489 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:20.494 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:08:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:08:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:08:20.531 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:20.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:20.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:20.634 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:20.655 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:20.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:20.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:20.757 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:20.789 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:20.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:20.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:20.890 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:20.927 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:20.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:20.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:21.030 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:21.055 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:21.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:21.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:21.157 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:21.184 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:21.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:21.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:21.286 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:21.309 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:21.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:21.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:21.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:21 smithi179 ceph-mon[93519]: pgmap v2344: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 86 MiB/s wr, 1.38k op/s; 46849/204645 objects degraded (22.893%) 2024-08-17T18:08:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 46849/204645 objects degraded (22.893%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:21.411 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:21 smithi130 ceph-mon[105521]: pgmap v2344: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 86 MiB/s wr, 1.38k op/s; 46849/204645 objects degraded (22.893%) 2024-08-17T18:08:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 46849/204645 objects degraded (22.893%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:21 smithi130 ceph-mon[110068]: pgmap v2344: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 86 MiB/s wr, 1.38k op/s; 46849/204645 objects degraded (22.893%) 2024-08-17T18:08:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 46849/204645 objects degraded (22.893%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:21.452 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:21.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:21.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:21.473 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 38 16 73076 73060 120.129 74.3125 0.0103287 0.00831968 2024-08-17T18:08:21.554 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:21.592 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:21.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:21.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:21.694 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:21.720 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:21.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:21.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:21.821 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:21.849 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:21.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:21.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:21.951 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:21.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:21.991 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:21.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:22.092 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:22.116 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:22.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:22.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:22.218 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:22.241 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:22.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:22.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:22.343 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:22.366 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:22.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:22.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:22.440 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:08:22.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:22 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/229267525' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:08:22.441 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:22 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:08:22.441 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:22 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/229267525' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:08:22.441 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:22 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:08:22.468 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:22.473 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 39 16 74328 74312 119.055 78.25 0.00586972 0.0083943 2024-08-17T18:08:22.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:22.503 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:22.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:22.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:22 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/229267525' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:08:22.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:22 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:08:22.604 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:22.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:22.626 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:22.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:22.727 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:22.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:22.751 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:22.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:22.852 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:22.876 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:22.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:22.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:22.978 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:23.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:23.002 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:23.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:23.103 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:23.127 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:23.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:23.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:23.228 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:23.262 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:23.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:23.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:23.364 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:23.401 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:23.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:23.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:23.473 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:08:23.473493+0000 min lat: 0.00101746 max lat: 0.209803 avg lat: 0.00847202 2024-08-17T18:08:23.473 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:08:23.473 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 40 16 75536 75520 117.966 75.5 0.0126351 0.00847202 2024-08-17T18:08:23.503 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:23.527 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:23.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:23.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:23 smithi179 ceph-mon[93519]: pgmap v2345: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 86 MiB/s wr, 1.38k op/s; 46849/204645 objects degraded (22.893%) 2024-08-17T18:08:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:23 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:08:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:23 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:08:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:23 smithi179 ceph-mon[93519]: osdmap e1333: 8 total, 7 up, 5 in 2024-08-17T18:08:23.628 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:23.653 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:23.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:23.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:23.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:23 smithi130 ceph-mon[110068]: pgmap v2345: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 86 MiB/s wr, 1.38k op/s; 46849/204645 objects degraded (22.893%) 2024-08-17T18:08:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:23 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:08:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:23 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:08:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:23 smithi130 ceph-mon[110068]: osdmap e1333: 8 total, 7 up, 5 in 2024-08-17T18:08:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:23 smithi130 ceph-mon[105521]: pgmap v2345: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.2 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 86 MiB/s wr, 1.38k op/s; 46849/204645 objects degraded (22.893%) 2024-08-17T18:08:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:23 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:08:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:23 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:08:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:23 smithi130 ceph-mon[105521]: osdmap e1333: 8 total, 7 up, 5 in 2024-08-17T18:08:23.755 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:23.768 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:08:23.776 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:23.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:23.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:23.878 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:23.903 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:23.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:23.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:24.004 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:24.052 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:24.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:24.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:24.154 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:24.198 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 7] out_osds: [5, 4, 6] dead_osds: [3] live_osds: [1, 2, 4, 5, 6, 7, 0] 2024-08-17T18:08:24.198 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:08:24.199 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-08-17T18:08:24.199 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T18:08:24.199 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T18:08:24.199 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-08-17T18:08:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:24.201 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:24.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:24.302 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:24.348 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:24.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:24.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:24.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:24.473 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 41 16 76757 76741 116.951 76.3125 0.0194214 0.0085463 2024-08-17T18:08:24.474 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:24.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:24.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:24.576 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:24.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:24.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:24.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:24.704 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:24.746 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:24.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:24.848 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:24.886 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:24.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:24.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:24.988 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:25.011 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:25.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:25.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:25.113 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:25.138 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:25.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:25.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:25.240 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:25.264 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:25.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:25.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:25.365 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:25.388 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:25.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:25.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:25.473 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 42 16 78081 78065 116.136 82.75 0.00706751 0.00860555 2024-08-17T18:08:25.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:25.514 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:25.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:25.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:25.532 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:25 smithi130 ceph-mon[105521]: pgmap v2347: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 94 MiB/s wr, 1.51k op/s; 50647/221115 objects degraded (22.905%) 2024-08-17T18:08:25.532 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:25 smithi130 ceph-mon[110068]: pgmap v2347: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 94 MiB/s wr, 1.51k op/s; 50647/221115 objects degraded (22.905%) 2024-08-17T18:08:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:25 smithi179 ceph-mon[93519]: pgmap v2347: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.5 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 94 MiB/s wr, 1.51k op/s; 50647/221115 objects degraded (22.905%) 2024-08-17T18:08:25.616 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:25.641 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:25.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:25.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:25.743 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:25.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:25.779 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:25.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:25.880 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:25.932 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:08:25.942 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:25.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:25.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:25.958 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:08:26.045 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:26.067 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:26.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:26.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:26.169 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:26.214 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:26.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:26.316 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:26.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:26.342 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:26.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:26.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 50647/221115 objects degraded (22.905%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:26.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 50647/221115 objects degraded (22.905%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:26.443 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:26.467 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:26.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:26.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:26.473 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 43 16 79409 79393 115.366 83 0.00771833 0.00866273 2024-08-17T18:08:26.569 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:26.592 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 50647/221115 objects degraded (22.905%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:26.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:26.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:26.694 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:26.735 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:26.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:26.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:26.769 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:08:26.837 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:26.860 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:26.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:26.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:26.961 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:27.013 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:27.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:27.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:27.115 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:27.139 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:27.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:27.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:27.241 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:27.265 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:27.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:27.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:27.366 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:27.402 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:27.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:27.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:27.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:27 smithi130 ceph-mon[105521]: pgmap v2348: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 103 MiB/s wr, 1.65k op/s; 51418/224547 objects degraded (22.899%) 2024-08-17T18:08:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:27 smithi130 ceph-mon[110068]: pgmap v2348: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 103 MiB/s wr, 1.65k op/s; 51418/224547 objects degraded (22.899%) 2024-08-17T18:08:27.473 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 44 16 80634 80618 114.484 76.5625 0.0131417 0.00872979 2024-08-17T18:08:27.504 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:27.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:27.525 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:27.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:27 smithi179 ceph-mon[93519]: pgmap v2348: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 103 MiB/s wr, 1.65k op/s; 51418/224547 objects degraded (22.899%) 2024-08-17T18:08:27.627 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:27.652 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:27.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:27.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:27.753 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:27.774 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:27.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:27.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:27.876 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:27.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:27.900 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:27.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:28.001 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:28.025 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:28.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:28.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:28.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:28.165 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:28.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:28.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:28.266 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:28.290 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:28.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:28.392 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:28.419 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:28.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:28.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:28.474 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 45 16 81886 81870 113.678 78.25 0.00460731 0.00879181 2024-08-17T18:08:28.520 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:28.542 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:28.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:28.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:28.644 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:08:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:28 smithi130 ceph-mon[105521]: pgmap v2349: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 856 op/s; 51418/224547 objects degraded (22.899%) 2024-08-17T18:08:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:08:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:28 smithi130 ceph-mon[110068]: pgmap v2349: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 856 op/s; 51418/224547 objects degraded (22.899%) 2024-08-17T18:08:28.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:28.668 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:28.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:28.770 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:28.807 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:28.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:28.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:28.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:28.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:08:28.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:28 smithi179 ceph-mon[93519]: pgmap v2349: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 4.6 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 856 op/s; 51418/224547 objects degraded (22.899%) 2024-08-17T18:08:28.909 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:28.942 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:28.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:28.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:29.044 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:29.071 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:29.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:29.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:29.173 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:29.222 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:29.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:29.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:29.324 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:29.346 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:29.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:29.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:29.352 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:08:29.352 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:08:29.352 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.352 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:83001 48.8 e1333/1327 48:19decf80:::benchmark_data_smithi130_327597_object83000:head v 1333'5229, mlcod=1333'5227)", 2024-08-17T18:08:29.352 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:29.341384+0000", 2024-08-17T18:08:29.352 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0091438160000000008, 2024-08-17T18:08:29.352 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0091712459999999992, 2024-08-17T18:08:29.352 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:08:29.352 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:08:29.352 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:08:29.352 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:08:29.353 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.353 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:08:29.353 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.341384+0000", 2024-08-17T18:08:29.353 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:08:29.353 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.353 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.353 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:08:29.353 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.341384+0000", 2024-08-17T18:08:29.353 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:08:29.353 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.353 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.353 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:08:29.353 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.341385+0000", 2024-08-17T18:08:29.354 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.65e-07 2024-08-17T18:08:29.354 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.354 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.354 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:08:29.354 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.341403+0000", 2024-08-17T18:08:29.354 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8808e-05 2024-08-17T18:08:29.354 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.354 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.354 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:08:29.354 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.341410+0000", 2024-08-17T18:08:29.354 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1210000000000003e-06 2024-08-17T18:08:29.354 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:08:29.355 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.341414+0000", 2024-08-17T18:08:29.355 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1010000000000002e-06 2024-08-17T18:08:29.355 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:08:29.355 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.344407+0000", 2024-08-17T18:08:29.355 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029932370000000002 2024-08-17T18:08:29.355 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:08:29.355 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.344433+0000", 2024-08-17T18:08:29.356 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5775999999999999e-05 2024-08-17T18:08:29.356 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:08:29.356 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:08:29.356 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:08:29.356 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.356 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.356 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:83002 48.8 e1333/1327 48:1adff238:::benchmark_data_smithi130_327597_object83001:head v 1333'5230, mlcod=1333'5227)", 2024-08-17T18:08:29.356 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:29.345327+0000", 2024-08-17T18:08:29.356 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0052014310000000003, 2024-08-17T18:08:29.356 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0052830660000000003, 2024-08-17T18:08:29.356 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:08:29.356 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:08:29.356 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:08:29.356 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:08:29.357 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.357 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:08:29.357 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.345327+0000", 2024-08-17T18:08:29.357 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:08:29.357 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.357 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.357 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:08:29.357 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.345327+0000", 2024-08-17T18:08:29.357 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:08:29.357 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.357 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.357 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:08:29.357 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.345328+0000", 2024-08-17T18:08:29.357 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5899999999999995e-07 2024-08-17T18:08:29.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.358 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:08:29.358 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.345348+0000", 2024-08-17T18:08:29.358 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0154000000000001e-05 2024-08-17T18:08:29.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.358 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:08:29.358 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.345360+0000", 2024-08-17T18:08:29.358 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1797000000000001e-05 2024-08-17T18:08:29.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.358 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:08:29.359 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.345366+0000", 2024-08-17T18:08:29.359 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.477e-06 2024-08-17T18:08:29.359 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.359 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.359 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:08:29.359 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.348269+0000", 2024-08-17T18:08:29.359 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029030179999999998 2024-08-17T18:08:29.359 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.359 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.359 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:08:29.359 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.348288+0000", 2024-08-17T18:08:29.359 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8872000000000002e-05 2024-08-17T18:08:29.359 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:08:29.359 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:08:29.360 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:08:29.360 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.360 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.360 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:83007 48.5 e1333/1327 48:a22eb876:::benchmark_data_smithi130_327597_object83006:head v 1333'5151, mlcod=1333'5149)", 2024-08-17T18:08:29.360 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:29.346481+0000", 2024-08-17T18:08:29.360 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.004047297, 2024-08-17T18:08:29.360 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0041747329999999999, 2024-08-17T18:08:29.360 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:08:29.360 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:08:29.360 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:08:29.360 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:08:29.360 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.360 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:08:29.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346481+0000", 2024-08-17T18:08:29.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:08:29.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.361 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:08:29.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346481+0000", 2024-08-17T18:08:29.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:08:29.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.361 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:08:29.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346482+0000", 2024-08-17T18:08:29.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6599999999999995e-07 2024-08-17T18:08:29.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:08:29.362 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346540+0000", 2024-08-17T18:08:29.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8443000000000003e-05 2024-08-17T18:08:29.362 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:08:29.362 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346569+0000", 2024-08-17T18:08:29.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8841000000000001e-05 2024-08-17T18:08:29.362 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:08:29.362 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346577+0000", 2024-08-17T18:08:29.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3059999999999998e-06 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346728+0000", 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015116899999999999 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346748+0000", 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9371000000000001e-05 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.364 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:83008 48.e e1333/1327 48:72746b0b:::benchmark_data_smithi130_327597_object83007:head v 1333'5243, mlcod=1333'5241)", 2024-08-17T18:08:29.364 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:29.346596+0000", 2024-08-17T18:08:29.364 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0039327240000000003, 2024-08-17T18:08:29.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0041053720000000004, 2024-08-17T18:08:29.364 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:08:29.364 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:08:29.364 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:08:29.364 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:08:29.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:08:29.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346596+0000", 2024-08-17T18:08:29.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:08:29.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:08:29.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346596+0000", 2024-08-17T18:08:29.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:08:29.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:08:29.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346596+0000", 2024-08-17T18:08:29.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5499999999999999e-07 2024-08-17T18:08:29.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:08:29.365 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346622+0000", 2024-08-17T18:08:29.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6616999999999999e-05 2024-08-17T18:08:29.366 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.366 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:08:29.366 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346630+0000", 2024-08-17T18:08:29.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7309999999999993e-06 2024-08-17T18:08:29.366 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.366 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:08:29.366 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346635+0000", 2024-08-17T18:08:29.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1540000000000001e-06 2024-08-17T18:08:29.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.367 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.367 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:08:29.367 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346856+0000", 2024-08-17T18:08:29.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00022015000000000001 2024-08-17T18:08:29.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.367 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.367 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:08:29.367 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.346876+0000", 2024-08-17T18:08:29.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0939999999999999e-05 2024-08-17T18:08:29.367 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:08:29.367 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:08:29.368 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:08:29.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.368 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:83012 48.1 e1333/1327 48:842cd761:::benchmark_data_smithi130_327597_object83011:head v 1333'5269, mlcod=1333'5267)", 2024-08-17T18:08:29.368 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:29.348937+0000", 2024-08-17T18:08:29.368 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.001590899, 2024-08-17T18:08:29.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0018098330000000001, 2024-08-17T18:08:29.368 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:08:29.368 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:08:29.368 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:08:29.368 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:08:29.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:08:29.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.348937+0000", 2024-08-17T18:08:29.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:08:29.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.369 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:08:29.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.348937+0000", 2024-08-17T18:08:29.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:08:29.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.369 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:08:29.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.348938+0000", 2024-08-17T18:08:29.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3600000000000002e-07 2024-08-17T18:08:29.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:08:29.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.349002+0000", 2024-08-17T18:08:29.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4185999999999998e-05 2024-08-17T18:08:29.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:08:29.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.349013+0000", 2024-08-17T18:08:29.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1002000000000001e-05 2024-08-17T18:08:29.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:08:29.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.349020+0000", 2024-08-17T18:08:29.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6429999999999998e-06 2024-08-17T18:08:29.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.371 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.371 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:08:29.371 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.349084+0000", 2024-08-17T18:08:29.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4059000000000005e-05 2024-08-17T18:08:29.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.371 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.371 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:08:29.371 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.349103+0000", 2024-08-17T18:08:29.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8496999999999999e-05 2024-08-17T18:08:29.371 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:08:29.371 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:08:29.371 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:08:29.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.372 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.372 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:83010 48.5 e1333/1327 48:a90035fb:::benchmark_data_smithi130_327597_object83009:head v 1333'5152, mlcod=1333'5149)", 2024-08-17T18:08:29.372 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:29.349376+0000", 2024-08-17T18:08:29.372 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.001152391, 2024-08-17T18:08:29.372 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001414523, 2024-08-17T18:08:29.372 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:08:29.372 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:08:29.372 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:08:29.372 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:08:29.372 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.372 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:08:29.372 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.349376+0000", 2024-08-17T18:08:29.372 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:08:29.373 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.373 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.373 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:08:29.373 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.349376+0000", 2024-08-17T18:08:29.373 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:08:29.373 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.373 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.373 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:08:29.373 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.349376+0000", 2024-08-17T18:08:29.373 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3600000000000003e-07 2024-08-17T18:08:29.373 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.373 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.373 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:08:29.374 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.349440+0000", 2024-08-17T18:08:29.374 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3904000000000003e-05 2024-08-17T18:08:29.374 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.374 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.374 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:08:29.374 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.349453+0000", 2024-08-17T18:08:29.374 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2794e-05 2024-08-17T18:08:29.374 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.374 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.374 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:08:29.374 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.349459+0000", 2024-08-17T18:08:29.374 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8390000000000003e-06 2024-08-17T18:08:29.374 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.375 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.375 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:08:29.375 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.350451+0000", 2024-08-17T18:08:29.375 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00099154899999999999 2024-08-17T18:08:29.375 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:08:29.375 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:08:29.375 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:08:29.375 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:29.350469+0000", 2024-08-17T18:08:29.375 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8351000000000001e-05 2024-08-17T18:08:29.375 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:08:29.375 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:08:29.375 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:08:29.375 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:08:29.375 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T18:08:29.376 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 6 2024-08-17T18:08:29.376 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:08:29.448 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:29.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:29.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:29.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:29.474 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 46 16 83161 83145 112.94 79.6875 0.00952841 0.00884957 2024-08-17T18:08:29.573 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:29.595 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:29.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:29.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:29.697 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:29.721 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:29.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:29.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:29.823 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:29.866 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:29.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:29.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:29.971 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:29.994 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:29.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:29.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:30.096 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:30.132 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:30.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:30.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:30.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:08:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:08:29] "GET /metrics HTTP/1.1" 200 36786 "" "Prometheus/2.51.0" 2024-08-17T18:08:30.233 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:30.256 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:30.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:30.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:30.358 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:30.383 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:30.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:30.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:30.474 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 47 16 84434 84418 112.229 79.5625 0.0149296 0.00890552 2024-08-17T18:08:30.485 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:30.509 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:30.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:30.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:30 smithi179 ceph-mon[93519]: pgmap v2350: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 55799/243774 objects degraded (22.890%) 2024-08-17T18:08:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:08:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:08:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:08:30.610 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:30.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:30.648 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:30.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:30.749 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:30 smithi130 ceph-mon[105521]: pgmap v2350: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 55799/243774 objects degraded (22.890%) 2024-08-17T18:08:30.749 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:30 smithi130 ceph-mon[110068]: pgmap v2350: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 55799/243774 objects degraded (22.890%) 2024-08-17T18:08:30.749 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:30.785 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:30.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:30.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:30.887 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:30.913 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:30.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:30.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:31.015 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:31.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:31.062 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:31.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:31.163 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:31.187 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:31.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:31.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:31.289 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:31.324 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:31.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:31.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:31.426 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:31.449 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:31.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:31.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:31.474 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 48 16 85710 85694 111.553 79.75 0.00830031 0.0089598 2024-08-17T18:08:31.551 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:31.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T18:08:31.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:31.577 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:31.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:31.678 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:31.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:31.723 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:31.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:31.824 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:31.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 55799/243774 objects degraded (22.890%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:31.848 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:31.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:31.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 55799/243774 objects degraded (22.890%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:31.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 55799/243774 objects degraded (22.890%), 20 pgs degraded, 45 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:31.950 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:31.974 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:31.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:32.076 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:32.099 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:32.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:32.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:32.201 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:32.223 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:32.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:32.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:32.325 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:32.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:32.366 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:32.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:32.468 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:32.474 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 49 16 86928 86912 110.83 76.125 0.016543 0.00901778 2024-08-17T18:08:32.491 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:32.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:32.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:32.592 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:32 smithi179 ceph-mon[93519]: pgmap v2351: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 55799/243774 objects degraded (22.890%) 2024-08-17T18:08:32.684 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:32.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:32.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:32.725 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:32 smithi130 ceph-mon[105521]: pgmap v2351: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 55799/243774 objects degraded (22.890%) 2024-08-17T18:08:32.725 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:32 smithi130 ceph-mon[110068]: pgmap v2351: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.0 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 81 MiB/s wr, 1.30k op/s; 55799/243774 objects degraded (22.890%) 2024-08-17T18:08:32.786 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:32.808 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:32.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:32.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:32.909 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:32.933 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:32.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:32.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:32.937 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:08:33.035 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:33.057 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:33.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:33.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:33.159 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:33.181 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:33.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:33.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:33.283 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:33.307 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:33.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:33.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:33.312 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:08:33.329 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:08:33.329 INFO:teuthology.orchestra.run.smithi130.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T18:08:33.329 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:08:33.408 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:33.433 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:33.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:33.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:33.474 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 50 16 88067 88051 110.036 71.1875 0.010685 0.00908342 2024-08-17T18:08:33.535 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:33.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:33.574 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:33.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:33.675 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:33.699 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:33.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:33.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:33.801 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:33.824 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:33.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:33.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:33.925 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:33.947 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:33.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:33.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:34.049 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:34.072 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:34.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:34.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:34.174 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:34.201 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:34.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:34.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:34.303 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:34.327 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:34.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:34.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:34.429 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:34.454 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:34.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:34.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:34.474 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 51 16 89274 89258 109.358 75.4375 0.0041692 0.00913895 2024-08-17T18:08:34.556 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:34.591 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:34.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:34.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:34.693 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:34.716 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:34.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:34.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:34.818 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:34.839 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:34.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:34.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:34.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:34 smithi179 ceph-mon[93519]: pgmap v2352: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 101 MiB/s wr, 1.62k op/s; 59287/259026 objects degraded (22.888%) 2024-08-17T18:08:34.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:34 smithi130 ceph-mon[105521]: pgmap v2352: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 101 MiB/s wr, 1.62k op/s; 59287/259026 objects degraded (22.888%) 2024-08-17T18:08:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:34 smithi130 ceph-mon[110068]: pgmap v2352: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 101 MiB/s wr, 1.62k op/s; 59287/259026 objects degraded (22.888%) 2024-08-17T18:08:34.941 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:34.964 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:34.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:34.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:35.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:35.097 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:35.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:35.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:35.200 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:35.234 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:35.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:35.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:35.336 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:35.373 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:35.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:35.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:35.474 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 52 16 90201 90185 108.369 57.9375 0.0681802 0.00921697 2024-08-17T18:08:35.475 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:35.498 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:35.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:35.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:35.600 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:35.641 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:35.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:35.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:35.743 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:35.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:35.782 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:35.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:35.883 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:35.907 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:35.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:35.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:36.008 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:36.032 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:36.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:36.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:36.036 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:08:36.036 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:08:36.134 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:36.147 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:08:36.147 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:08:36.155 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:36.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:36.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:36.187 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:08:36.187 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:08:36.188 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:08:36.216 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:08:36.216 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:08:36.244 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:08:36.244 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:08:36.256 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:36.275 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:08:36.276 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:08:36.293 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:36.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:36.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:36.306 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:08:36.306 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:08:36.395 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:36.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:36.434 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:36.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:36.474 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 53 16 91251 91235 107.563 65.625 0.0123727 0.00929181 2024-08-17T18:08:36.488 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:36 smithi130 ceph-mon[110068]: pgmap v2353: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 72 MiB/s wr, 1.15k op/s; 60025/262437 objects degraded (22.872%) 2024-08-17T18:08:36.488 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 60025/262437 objects degraded (22.872%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:36.488 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:36 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1027084792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:08:36.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:36 smithi130 ceph-mon[105521]: pgmap v2353: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 72 MiB/s wr, 1.15k op/s; 60025/262437 objects degraded (22.872%) 2024-08-17T18:08:36.488 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 60025/262437 objects degraded (22.872%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:36.489 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:36 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1027084792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:08:36.535 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:36.558 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:36.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:36.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:36.660 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:36.683 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:36.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:36.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:36.785 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:36.836 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:36.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:36.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:36.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:36 smithi179 ceph-mon[93519]: pgmap v2353: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 72 MiB/s wr, 1.15k op/s; 60025/262437 objects degraded (22.872%) 2024-08-17T18:08:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 60025/262437 objects degraded (22.872%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:36 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1027084792' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:08:36.938 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:36.977 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:36.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:36.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:37.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:37.103 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:37.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:37.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:37.204 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:37.242 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:37.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:37.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:37.344 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:37.379 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:37.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:37.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:37.475 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 54 16 92501 92485 107.017 78.125 0.00650616 0.00933921 2024-08-17T18:08:37.480 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:37.515 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:37.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:37.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:37.617 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:37.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:37.657 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:37.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:37.758 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:37.783 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:37.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:37.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:37.885 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:37.907 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:37.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:37.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:38.008 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:38.033 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:38.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:38.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:38.135 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:38.172 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:38.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:38.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:38.273 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:38.306 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:38.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:38.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:38.408 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:38.444 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:08:38.445 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:08:38.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:38.470 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:38.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:38.475 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 55 16 93739 93723 106.477 77.375 0.0107735 0.00938647 2024-08-17T18:08:38.571 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:38.609 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:38.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:38.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:38.654 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:38 smithi130 ceph-mon[105521]: pgmap v2354: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 66 MiB/s wr, 1.05k op/s; 60025/262437 objects degraded (22.872%) 2024-08-17T18:08:38.654 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:38 smithi130 ceph-mon[110068]: pgmap v2354: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 66 MiB/s wr, 1.05k op/s; 60025/262437 objects degraded (22.872%) 2024-08-17T18:08:38.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:38.753 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:38.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:38.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:38 smithi179 ceph-mon[93519]: pgmap v2354: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.3 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 66 MiB/s wr, 1.05k op/s; 60025/262437 objects degraded (22.872%) 2024-08-17T18:08:38.855 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:38.897 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:38.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:38.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:38.999 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:39.047 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:39.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:39.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:39.149 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:39.186 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:39.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:39.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:39.288 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:39.312 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:39.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:39.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:39.414 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:39.439 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:39.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:39.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:39.476 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 56 16 94981 94965 105.961 77.625 0.0316942 0.0094324 2024-08-17T18:08:39.541 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:39.565 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:39.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:39.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:39.667 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:39.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:39.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:39.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:39.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:39.815 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:39.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:39.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:39.820 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:08:39.820 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:08:39.916 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:39.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:39.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:39.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:40.045 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:40.070 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:40.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:40.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:40.141 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:08:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:08:39] "GET /metrics HTTP/1.1" 200 36786 "" "Prometheus/2.51.0" 2024-08-17T18:08:40.171 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:40.198 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:40.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:40.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:40.300 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:40.324 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:40.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:40.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:40.426 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:40.446 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:08:40.451 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:40.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:40.476 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 57 16 96227 96211 105.469 77.875 0.00927453 0.0094767 2024-08-17T18:08:40.553 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:40.579 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T18:08:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:40 smithi179 ceph-mon[93519]: pgmap v2355: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s; 63896/279276 objects degraded (22.879%) 2024-08-17T18:08:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:08:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:08:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:08:40.599 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:40.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:40.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:40.702 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:40.759 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:40.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:40.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:40.860 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:40.883 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:40.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:40.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:40 smithi130 ceph-mon[105521]: pgmap v2355: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s; 63896/279276 objects degraded (22.879%) 2024-08-17T18:08:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:40 smithi130 ceph-mon[110068]: pgmap v2355: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 95 MiB/s wr, 1.52k op/s; 63896/279276 objects degraded (22.879%) 2024-08-17T18:08:40.984 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:41.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:41.035 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:41.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:41.136 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:41.176 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:41.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:41.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:41.278 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:41.302 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:41.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:41.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:41.403 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:41.426 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:41.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:41.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:41.476 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 58 16 97349 97333 104.858 70.125 0.0128498 0.0095317 2024-08-17T18:08:41.528 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:41.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:41.568 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:41.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:41 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 63896/279276 objects degraded (22.879%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:41 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 63896/279276 objects degraded (22.879%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:41.669 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:41.693 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:41.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:41.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:41.794 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:41.818 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:41.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:41.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:41 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 63896/279276 objects degraded (22.879%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:41.920 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:41.955 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:41.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:41.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:42.059 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:42.091 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:42.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:42.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:42.193 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:42.221 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:42.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:42.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:42.323 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:42.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:42.361 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:42.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:42.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:42.476 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 59 16 98595 98579 104.401 77.875 0.0150986 0.00957301 2024-08-17T18:08:42.501 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:42.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:42.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:42.602 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:42.635 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:42.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:42.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:42.737 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:42.763 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:42.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:42.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:42 smithi179 ceph-mon[93519]: pgmap v2356: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 62 MiB/s wr, 985 op/s; 63896/279276 objects degraded (22.879%) 2024-08-17T18:08:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:08:42.864 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:42.888 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:42.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:42.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:42.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:42 smithi130 ceph-mon[105521]: pgmap v2356: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 62 MiB/s wr, 985 op/s; 63896/279276 objects degraded (22.879%) 2024-08-17T18:08:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:08:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:42 smithi130 ceph-mon[110068]: pgmap v2356: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 62 MiB/s wr, 985 op/s; 63896/279276 objects degraded (22.879%) 2024-08-17T18:08:42.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:08:42.989 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:43.012 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:43.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:43.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:43.114 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:43.165 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:43.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:43.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:43.267 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:43.290 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:43.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:43.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:43.392 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:43.415 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:43.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:43.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:43.476 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:08:43.476918+0000 min lat: 0.00101746 max lat: 0.237689 avg lat: 0.00961853 2024-08-17T18:08:43.477 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:08:43.477 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 60 16 99797 99781 103.913 75.125 0.0156334 0.00961853 2024-08-17T18:08:43.517 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:43.540 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:43.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:43.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:43.642 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:43.681 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:43.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:43.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:43.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:43.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:43.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:43.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:43.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:43.935 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 7] out_osds: [5, 4, 6] dead_osds: [3] live_osds: [1, 2, 4, 5, 6, 7, 0] 2024-08-17T18:08:43.935 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:08:43.935 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 3, 7] 2024-08-17T18:08:43.935 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd out 1 2024-08-17T18:08:43.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:43.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:43.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:44.058 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:44.081 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:44.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:44.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:44.183 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:44.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:44.206 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:44.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:44.307 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:44.325 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:08:44.329 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:44.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:44.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:44.357 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:08:44.431 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:44.453 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:44.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:44.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:44.478 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 61 16 100706 100690 103.139 56.8125 0.014758 0.00969086 2024-08-17T18:08:44.555 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:44.601 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:44.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:44.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:44.703 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:44.737 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:44.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:44.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:44.764 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:44 smithi130 ceph-mon[105521]: pgmap v2357: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 88 MiB/s wr, 1.41k op/s; 67371/294525 objects degraded (22.874%) 2024-08-17T18:08:44.764 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:44 smithi130 ceph-mon[110068]: pgmap v2357: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 88 MiB/s wr, 1.41k op/s; 67371/294525 objects degraded (22.874%) 2024-08-17T18:08:44.839 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:44.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:44 smithi179 ceph-mon[93519]: pgmap v2357: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 88 MiB/s wr, 1.41k op/s; 67371/294525 objects degraded (22.874%) 2024-08-17T18:08:44.874 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:44.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:44.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:44.976 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:44.998 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:44.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:44.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:45.100 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:45.122 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:45.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:45.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:45.224 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:45.247 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:45.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:45.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:45.349 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:45.372 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:45.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:45.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:45.474 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:45.477 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 62 16 101822 101806 102.601 69.75 0.0152831 0.00974192 2024-08-17T18:08:45.497 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:08:45.498 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:45.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:45.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:45.600 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:45.641 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:45.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:45.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:45.742 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:45.772 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:45 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4202182216' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:08:45.772 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:45 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4202182216' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:08:45.782 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:45.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:45.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:45 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4202182216' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:08:45.884 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:45.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:45.918 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:45.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:46.019 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:46.041 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:46.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:46.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:46.143 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:46.167 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:46.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:46.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:46.269 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:46.292 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:46.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:46.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:46.394 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:46.419 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:46.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:46.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:46.477 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 63 16 102963 102947 102.104 71.3125 0.00946989 0.00978922 2024-08-17T18:08:46.521 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:46.557 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:46.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:46.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:46.658 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:46.681 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:46.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:46.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:46.772 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:46 smithi130 ceph-mon[105521]: pgmap v2358: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.1 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s; 68070/297666 objects degraded (22.868%) 2024-08-17T18:08:46.772 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:46 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:08:46.773 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:46 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4202182216' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:08:46.773 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:46 smithi130 ceph-mon[105521]: osdmap e1334: 8 total, 7 up, 5 in 2024-08-17T18:08:46.773 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 68070/297666 objects degraded (22.868%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:46.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:46 smithi130 ceph-mon[110068]: pgmap v2358: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.1 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s; 68070/297666 objects degraded (22.868%) 2024-08-17T18:08:46.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:46 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:08:46.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:46 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4202182216' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:08:46.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:46 smithi130 ceph-mon[110068]: osdmap e1334: 8 total, 7 up, 5 in 2024-08-17T18:08:46.773 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 68070/297666 objects degraded (22.868%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:46.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:46.825 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:46.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:46.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:46 smithi179 ceph-mon[93519]: pgmap v2358: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.1 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 67 MiB/s wr, 1.07k op/s; 68070/297666 objects degraded (22.868%) 2024-08-17T18:08:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:46 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:08:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:46 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4202182216' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:08:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:46 smithi179 ceph-mon[93519]: osdmap e1334: 8 total, 7 up, 5 in 2024-08-17T18:08:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 68070/297666 objects degraded (22.868%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:46.926 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:46.951 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:46.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:46.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:47.052 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:47.093 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:47.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:47.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:47.195 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:47.217 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:47.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:47.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:47.319 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:47.368 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:47.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:47.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:47.469 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:47.478 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 64 16 104204 104188 101.72 77.5625 0.0114958 0.00982559 2024-08-17T18:08:47.507 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:47.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:47.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:47.609 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:47.631 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:47.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:47.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:47.733 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:47.758 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:47.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:47.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:47.860 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:47.892 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:47.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:47.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:47.994 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:48.017 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:48.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:48.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:48.119 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:48.162 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:48.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:48.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:48.264 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:48.287 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:48.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:48.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:48.389 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:48.411 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:48.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:48.478 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 65 16 105442 105426 101.345 77.375 0.00720245 0.00986268 2024-08-17T18:08:48.513 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:48.549 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:48.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:48.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:48.651 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:48.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:48 smithi130 ceph-mon[105521]: pgmap v2360: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.1 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 73 MiB/s wr, 1.17k op/s; 68070/297666 objects degraded (22.868%) 2024-08-17T18:08:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:48 smithi130 ceph-mon[110068]: pgmap v2360: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.1 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 73 MiB/s wr, 1.17k op/s; 68070/297666 objects degraded (22.868%) 2024-08-17T18:08:48.675 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:48.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:48.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:48.777 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:48.799 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:48.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:48.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:48.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:48 smithi179 ceph-mon[93519]: pgmap v2360: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.1 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 73 MiB/s wr, 1.17k op/s; 68070/297666 objects degraded (22.868%) 2024-08-17T18:08:48.901 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:48.947 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:48.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:48.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:49.049 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:49.072 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:49.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:49.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:49.174 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:49.219 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:49.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:49.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:49.321 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:49.376 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:49.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:49.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:49.478 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:49.479 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 66 16 106609 106593 100.915 72.9375 0.00940594 0.00990468 2024-08-17T18:08:49.500 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:49.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:49.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:49.602 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:49.625 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:49.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:49.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:49.637 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:08:49.637 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:08:49.637 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:08:49.637 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:08:49.638 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:08:49.638 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:08:49.638 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:08:49.638 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:08:49.638 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:08:49.638 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:08:49.638 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:08:49.638 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:08:49.638 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:08:49.638 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:08:49.638 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:08:49.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:08:49.727 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:49.737 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:49 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/651559823' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:08:49.737 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:49 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/651559823' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:08:49.750 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:49.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:49.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:49.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:49 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/651559823' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:08:49.852 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:49.874 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:49.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:49.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:49.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:50.031 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:50.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:50.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:50.132 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:08:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:08:49] "GET /metrics HTTP/1.1" 200 36788 "" "Prometheus/2.51.0" 2024-08-17T18:08:50.168 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:50.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:50.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:50.270 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:50.294 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:50.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:50.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:50.395 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:50.428 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:50.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:50.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:50.478 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 67 16 107667 107651 100.396 66.125 0.0116222 0.00995561 2024-08-17T18:08:50.512 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:08:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:08:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:08:50.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:50.565 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:50.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:50.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:50.666 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:50.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:50.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:50.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:50.794 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:50.833 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:50.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:50.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:50 smithi179 ceph-mon[93519]: pgmap v2361: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 73 MiB/s wr, 1.17k op/s; 71957/314556 objects degraded (22.876%) 2024-08-17T18:08:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:50 smithi130 ceph-mon[105521]: pgmap v2361: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 73 MiB/s wr, 1.17k op/s; 71957/314556 objects degraded (22.876%) 2024-08-17T18:08:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:50 smithi130 ceph-mon[110068]: pgmap v2361: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 73 MiB/s wr, 1.17k op/s; 71957/314556 objects degraded (22.876%) 2024-08-17T18:08:50.935 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:50.974 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:50.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:50.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:51.076 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:51.113 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:51.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:51.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:51.215 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:51.251 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:51.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:51.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:51.353 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:51.363 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:08:51.363 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:08:51.375 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:51.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:51.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:51.477 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:51.478 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 68 16 108902 108886 100.054 77.1875 0.0289352 0.00999001 2024-08-17T18:08:51.500 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:51.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:51.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:51.602 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:51.606 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:08:51.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 71957/314556 objects degraded (22.876%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:51 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:51 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 71957/314556 objects degraded (22.876%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:51 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:51 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:51.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:51.679 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:51.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:51.781 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:51.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:51.825 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:51.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:51.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 71957/314556 objects degraded (22.876%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:51.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:51 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:51 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:51.926 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:51.950 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:51.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:51.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:52.052 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:52.089 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:52.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:52.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:52.191 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:52.213 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:52.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:52.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:52.246 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:08:52.246 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:08:52.315 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:52.338 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:52.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:52.440 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:52.464 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:52.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:52.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:52.478 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 69 16 110027 110011 99.6232 70.3125 0.00423563 0.0100331 2024-08-17T18:08:52.566 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:52.590 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:52.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:52.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:52.692 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:52.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:52.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:52.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:52.816 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:52 smithi130 ceph-mon[105521]: pgmap v2362: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 73 MiB/s wr, 1.18k op/s; 71957/314556 objects degraded (22.876%) 2024-08-17T18:08:52.816 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:52 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:08:52.816 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:52 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:08:52.816 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:52 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:52.816 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:52 smithi130 ceph-mon[110068]: pgmap v2362: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 73 MiB/s wr, 1.18k op/s; 71957/314556 objects degraded (22.876%) 2024-08-17T18:08:52.817 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:52 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:08:52.817 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:52 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:08:52.817 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:52 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:52.817 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:52.838 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:52.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:52.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:52.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:52 smithi179 ceph-mon[93519]: pgmap v2362: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.4 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 73 MiB/s wr, 1.18k op/s; 71957/314556 objects degraded (22.876%) 2024-08-17T18:08:52.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:52 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:08:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:52 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:08:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:52 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:52.858 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T18:08:52.940 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:52.965 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:52.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:52.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:53.067 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:53.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:53.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:53.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:53.181 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:08:53.198 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:53.235 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:53.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:53.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:53.336 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:53.373 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:53.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:53.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:53.475 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:53.478 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 70 16 111282 111266 99.3205 78.4375 0.00674202 0.0100639 2024-08-17T18:08:53.514 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:53.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:53.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:53.616 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:53.639 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:53.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:53.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:53.741 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:53.764 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:53.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:53.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:53.865 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:53.888 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:53.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:53.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:53.990 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:54.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:54.030 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:54.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:54.131 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:54.156 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:54.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:54.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:54.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:08:54.248 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:08:54.258 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:54.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:54.347 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:54.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:54.448 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:54.479 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 71 16 112470 112454 98.9672 74.25 0.0268652 0.0101 2024-08-17T18:08:54.506 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:54.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:54.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:54.608 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:54.669 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:54.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:54.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:54.771 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:54.795 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:54.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:54.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:54.826 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:54 smithi179 ceph-mon[93519]: pgmap v2363: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.7 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s; 75300/328776 objects degraded (22.903%) 2024-08-17T18:08:54.897 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:54.911 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:54 smithi130 ceph-mon[105521]: pgmap v2363: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.7 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s; 75300/328776 objects degraded (22.903%) 2024-08-17T18:08:54.911 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:54 smithi130 ceph-mon[110068]: pgmap v2363: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.7 GiB data, 21 GiB used, 426 GiB / 447 GiB avail; 71 MiB/s wr, 1.14k op/s; 75300/328776 objects degraded (22.903%) 2024-08-17T18:08:54.921 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:54.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:54.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:55.023 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:55.048 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:55.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:55.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:55.150 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:55.204 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:55.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:55.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:55.264 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:08:55.264 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:08:55.264 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T18:08:55.264 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:08:55.306 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:55.345 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:55.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:55.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:55.446 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:55.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:55.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:55.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:55.479 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 72 16 113602 113586 98.5752 70.75 0.0231157 0.0101398 2024-08-17T18:08:55.572 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:55.596 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:55.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:55.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:55.698 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:55.721 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:55.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:55.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:55.823 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:55.847 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:55.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:55.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:55.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:55 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T18:08:55.660+0000 7fa856f19640 -1 mon.a@0(leader).osd e1334 definitely_dead 0 2024-08-17T18:08:55.948 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:55.971 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:55.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:55.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:56.065 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T18:08:56.073 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:56.134 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:56.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:56.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:56.233 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:08:56.236 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:56.259 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:56.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:56.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:56.361 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:56.390 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:08:56.419 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:56.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:56.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:56.479 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 73 16 114815 114799 98.2634 75.8125 0.008815 0.0101718 2024-08-17T18:08:56.521 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:56.544 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:56.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:56.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:56.547 INFO:teuthology.orchestra.run.smithi130.stderr:marked out osd.1. 2024-08-17T18:08:56.646 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:56.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:56 smithi130 ceph-mon[105521]: pgmap v2364: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.8 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 72 MiB/s wr, 1.15k op/s; 76057/332076 objects degraded (22.903%) 2024-08-17T18:08:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:56 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/577759455' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T18:08:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:56 smithi130 ceph-mon[105521]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T18:08:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 76057/332076 objects degraded (22.903%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:56 smithi130 ceph-mon[110068]: pgmap v2364: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.8 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 72 MiB/s wr, 1.15k op/s; 76057/332076 objects degraded (22.903%) 2024-08-17T18:08:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:56 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/577759455' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T18:08:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:56 smithi130 ceph-mon[110068]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T18:08:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 76057/332076 objects degraded (22.903%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:56.671 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:56.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:56.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:56.773 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:56.805 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:56.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:56.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:56 smithi179 ceph-mon[93519]: pgmap v2364: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.8 GiB data, 22 GiB used, 425 GiB / 447 GiB avail; 72 MiB/s wr, 1.15k op/s; 76057/332076 objects degraded (22.903%) 2024-08-17T18:08:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:56 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/577759455' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T18:08:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:56 smithi179 ceph-mon[93519]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T18:08:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 76057/332076 objects degraded (22.903%), 20 pgs degraded, 56 pgs undersized (PG_DEGRADED) 2024-08-17T18:08:56.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:56.933 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:56.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:56.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:57.035 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:57.058 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:57.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:57.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:57.159 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:08:57.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:57.183 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:57.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:08:57.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:57.305 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:57.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:57.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:57.406 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:57.427 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:57.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:57.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:57.479 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 74 16 115470 115454 97.4887 40.9375 0.00218938 0.0102007 2024-08-17T18:08:57.529 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:57.551 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:08:57.552 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:57.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:57.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:57.655 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:57.676 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:57.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:57.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:57.778 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:57.798 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:57.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:57.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:57 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/577759455' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T18:08:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:57 smithi179 ceph-mon[93519]: osdmap e1335: 8 total, 7 up, 4 in 2024-08-17T18:08:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:57 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:08:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:57 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1412345730' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:08:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:57 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:08:57.900 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:57 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/577759455' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T18:08:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:57 smithi130 ceph-mon[105521]: osdmap e1335: 8 total, 7 up, 4 in 2024-08-17T18:08:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:57 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:08:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:57 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1412345730' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:08:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:57 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:08:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:57 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/577759455' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T18:08:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:57 smithi130 ceph-mon[110068]: osdmap e1335: 8 total, 7 up, 4 in 2024-08-17T18:08:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:57 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:08:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:08:57.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:57 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1412345730' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:08:57.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:57 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:08:57.923 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:57.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:57.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:58.025 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:08:58.046 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:58.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:58.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:08:58.148 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:58.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:58.170 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:58.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:58.271 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:58.292 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:58.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:58.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:58.394 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:58.414 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:58.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:58.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:58.479 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 75 16 115470 115454 96.189 0 - 0.0102007 2024-08-17T18:08:58.516 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:08:58.537 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:58.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:58.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:08:58.638 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:08:58.660 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:58.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:58.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:08:58.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:58 smithi130 ceph-mon[105521]: pgmap v2366: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.8 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 72 MiB/s wr, 1.15k op/s; 76057/332076 objects degraded (22.903%) 2024-08-17T18:08:58.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:58 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:08:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:58 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:08:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:58 smithi130 ceph-mon[105521]: osdmap e1336: 8 total, 7 up, 4 in 2024-08-17T18:08:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:58 smithi130 ceph-mon[110068]: pgmap v2366: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.8 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 72 MiB/s wr, 1.15k op/s; 76057/332076 objects degraded (22.903%) 2024-08-17T18:08:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:58 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:08:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:58 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:08:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:58 smithi130 ceph-mon[110068]: osdmap e1336: 8 total, 7 up, 4 in 2024-08-17T18:08:58.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:58 smithi179 ceph-mon[93519]: pgmap v2366: 89 pgs: 36 active+undersized, 20 active+undersized+degraded, 33 active+clean; 6.8 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 72 MiB/s wr, 1.15k op/s; 76057/332076 objects degraded (22.903%) 2024-08-17T18:08:58.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:58 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:08:58.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:58 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:08:58.706 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:58 smithi179 ceph-mon[93519]: osdmap e1336: 8 total, 7 up, 4 in 2024-08-17T18:08:58.762 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:58.788 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:58.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:58.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:58.890 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:58.916 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:58.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:58.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:58.972 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:08:58.972 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:08:58.972 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:08:58.972 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:08:58.972 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:08:59.017 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:59.042 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:59.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:59.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:59.144 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:08:59.167 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:59.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:59.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:08:59.269 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:59.293 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:59.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:59.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:59.395 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:08:59.419 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:08:59.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:59.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:08:59.480 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 76 16 116457 116441 95.735 30.8438 0.00703647 0.0104414 2024-08-17T18:08:59.521 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:59.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:59.565 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:59.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:59.665 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:59.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:59.712 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:59.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:59.806 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:59 smithi179 ceph-mon[93519]: osdmap e1337: 8 total, 7 up, 4 in 2024-08-17T18:08:59.806 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:59 smithi179 ceph-mon[93519]: 48.2 starting backfill to osd.0 from (0'0,0'0] MAX to 1334'7242 2024-08-17T18:08:59.806 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:59 smithi179 ceph-mon[93519]: 48.6 starting backfill to osd.0 from (0'0,0'0] MAX to 1334'7217 2024-08-17T18:08:59.806 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:59 smithi179 ceph-mon[93519]: 48.c starting backfill to osd.0 from (0'0,0'0] MAX to 1334'7155 2024-08-17T18:08:59.807 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:08:59 smithi179 ceph-mon[93519]: 48.f starting backfill to osd.0 from (0'0,0'0] MAX to 1334'7166 2024-08-17T18:08:59.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:59.838 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T18:08:59.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:08:59.856 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:08:59.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:08:59.881 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:59 smithi130 ceph-mon[105521]: osdmap e1337: 8 total, 7 up, 4 in 2024-08-17T18:08:59.881 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:59 smithi130 ceph-mon[105521]: 48.2 starting backfill to osd.0 from (0'0,0'0] MAX to 1334'7242 2024-08-17T18:08:59.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:59 smithi130 ceph-mon[105521]: 48.6 starting backfill to osd.0 from (0'0,0'0] MAX to 1334'7217 2024-08-17T18:08:59.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:59 smithi130 ceph-mon[105521]: 48.c starting backfill to osd.0 from (0'0,0'0] MAX to 1334'7155 2024-08-17T18:08:59.882 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:08:59 smithi130 ceph-mon[105521]: 48.f starting backfill to osd.0 from (0'0,0'0] MAX to 1334'7166 2024-08-17T18:08:59.882 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:59 smithi130 ceph-mon[110068]: osdmap e1337: 8 total, 7 up, 4 in 2024-08-17T18:08:59.882 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:59 smithi130 ceph-mon[110068]: 48.2 starting backfill to osd.0 from (0'0,0'0] MAX to 1334'7242 2024-08-17T18:08:59.882 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:59 smithi130 ceph-mon[110068]: 48.6 starting backfill to osd.0 from (0'0,0'0] MAX to 1334'7217 2024-08-17T18:08:59.882 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:59 smithi130 ceph-mon[110068]: 48.c starting backfill to osd.0 from (0'0,0'0] MAX to 1334'7155 2024-08-17T18:08:59.882 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:08:59 smithi130 ceph-mon[110068]: 48.f starting backfill to osd.0 from (0'0,0'0] MAX to 1334'7166 2024-08-17T18:08:59.882 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:08:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:08:59] "GET /metrics HTTP/1.1" 200 36792 "" "Prometheus/2.51.0" 2024-08-17T18:08:59.957 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:08:59.999 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:08:59.999 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:09:00.000 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:00.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:00.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:00.037 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:09:00.038 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:09:00.072 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:09:00.073 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:09:00.073 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:09:00.102 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:00.105 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:09:00.105 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:09:00.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:00.133 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:00.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:00.135 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:09:00.135 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:09:00.164 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:09:00.164 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:09:00.164 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:09:00.194 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:09:00.194 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:09:00.234 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:00.258 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:00.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:00.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:00.360 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:00.385 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:00.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:00.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:00.433 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:09:00.480 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 77 16 117676 117660 95.4811 76.1875 0.0170624 0.0104687 2024-08-17T18:09:00.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:00.487 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:09:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:09:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:09:00.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:00.558 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:00.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:00.659 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:00.686 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:00.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:00.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:00.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:00.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:00 smithi179 ceph-mon[93519]: pgmap v2369: 89 pgs: 16 active+clean+remapped, 27 remapped+peering, 14 active+undersized, 15 active+undersized+degraded, 17 active+clean; 7.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 55 MiB/s wr, 877 op/s; 64373/344577 objects degraded (18.682%); 7230/344577 objects misplaced (2.098%); 11 KiB/s, 0 objects/s recovering 2024-08-17T18:09:00.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:00 smithi179 ceph-mon[93519]: Health check failed: Reduced data availability: 11 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-08-17T18:09:00.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:00 smithi179 ceph-mon[93519]: osdmap e1338: 8 total, 7 up, 4 in 2024-08-17T18:09:00.790 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:00 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3390493211' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:00.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:00.836 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:00.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:00 smithi130 ceph-mon[105521]: pgmap v2369: 89 pgs: 16 active+clean+remapped, 27 remapped+peering, 14 active+undersized, 15 active+undersized+degraded, 17 active+clean; 7.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 55 MiB/s wr, 877 op/s; 64373/344577 objects degraded (18.682%); 7230/344577 objects misplaced (2.098%); 11 KiB/s, 0 objects/s recovering 2024-08-17T18:09:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:00 smithi130 ceph-mon[105521]: Health check failed: Reduced data availability: 11 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-08-17T18:09:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:00 smithi130 ceph-mon[105521]: osdmap e1338: 8 total, 7 up, 4 in 2024-08-17T18:09:00.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:00 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3390493211' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:00.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:00 smithi130 ceph-mon[110068]: pgmap v2369: 89 pgs: 16 active+clean+remapped, 27 remapped+peering, 14 active+undersized, 15 active+undersized+degraded, 17 active+clean; 7.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 55 MiB/s wr, 877 op/s; 64373/344577 objects degraded (18.682%); 7230/344577 objects misplaced (2.098%); 11 KiB/s, 0 objects/s recovering 2024-08-17T18:09:00.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:00 smithi130 ceph-mon[110068]: Health check failed: Reduced data availability: 11 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2024-08-17T18:09:00.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:00 smithi130 ceph-mon[110068]: osdmap e1338: 8 total, 7 up, 4 in 2024-08-17T18:09:00.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:00 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3390493211' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:00.937 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:00.964 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:00.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:00.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:01.041 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:09:01.042 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:09:01.065 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:01.090 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:01.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:01.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:01.192 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:01.215 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:01.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:01.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:01.246 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:09:01.317 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:01.341 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:01.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:01.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:01.442 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:01.468 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:01.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:01.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:01.479 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 78 16 118896 118880 95.2345 76.25 0.0150807 0.0104957 2024-08-17T18:09:01.570 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:01.594 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:01.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:01.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:01 smithi130 ceph-mon[105521]: osdmap e1339: 8 total, 7 up, 4 in 2024-08-17T18:09:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 64373/344577 objects degraded (18.682%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:01 smithi130 ceph-mon[110068]: osdmap e1339: 8 total, 7 up, 4 in 2024-08-17T18:09:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 64373/344577 objects degraded (18.682%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:01.696 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:01.739 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:01.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:01.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:01.841 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:01.867 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:01.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:01.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:01.955 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:01 smithi179 ceph-mon[93519]: osdmap e1339: 8 total, 7 up, 4 in 2024-08-17T18:09:01.955 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 64373/344577 objects degraded (18.682%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:01.968 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:01.995 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:01.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:01.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:02.096 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:02.125 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:02.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:02.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:02.227 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:02.231 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:09:02.231 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:09:02.256 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:02.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:02.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:02.358 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:02.392 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:02.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:02.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:02.480 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 79 16 120205 120189 95.0645 81.8125 0.00763572 0.0105148 2024-08-17T18:09:02.494 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:02.537 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:02.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:02.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:02.639 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:02.680 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:02.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:02.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:02.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:02.830 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:02.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:02.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:02.883 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:02 smithi130 ceph-mon[105521]: pgmap v2372: 89 pgs: 16 active+clean+remapped, 27 remapped+peering, 14 active+undersized, 15 active+undersized+degraded, 17 active+clean; 7.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 65 MiB/s wr, 1.04k op/s; 64373/344577 objects degraded (18.682%); 7230/344577 objects misplaced (2.098%); 16 KiB/s, 0 objects/s recovering 2024-08-17T18:09:02.884 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:02 smithi130 ceph-mon[110068]: pgmap v2372: 89 pgs: 16 active+clean+remapped, 27 remapped+peering, 14 active+undersized, 15 active+undersized+degraded, 17 active+clean; 7.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 65 MiB/s wr, 1.04k op/s; 64373/344577 objects degraded (18.682%); 7230/344577 objects misplaced (2.098%); 16 KiB/s, 0 objects/s recovering 2024-08-17T18:09:02.932 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:02.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:02.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:02.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:03.043 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:09:03.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:03.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:02 smithi179 ceph-mon[93519]: pgmap v2372: 89 pgs: 16 active+clean+remapped, 27 remapped+peering, 14 active+undersized, 15 active+undersized+degraded, 17 active+clean; 7.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 65 MiB/s wr, 1.04k op/s; 64373/344577 objects degraded (18.682%); 7230/344577 objects misplaced (2.098%); 16 KiB/s, 0 objects/s recovering 2024-08-17T18:09:03.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:03.138 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:03.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:03.239 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:03.281 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:03.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:03.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:03.339 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T18:09:03.385 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:03.412 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:03.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:03.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:03.480 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:09:03.480347+0000 min lat: 0.00101746 max lat: 1.68932 avg lat: 0.0105442 2024-08-17T18:09:03.480 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:09:03.480 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 80 16 121379 121363 94.7933 73.375 0.00863738 0.0105442 2024-08-17T18:09:03.514 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:03.540 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:03.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:03.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:03.546 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:09:03.641 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:03.653 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:09:03.669 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:03.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:03.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:03.771 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:03.797 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:03.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:03.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:03.899 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:03.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:03 smithi130 ceph-mon[110068]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 11 pgs peering) 2024-08-17T18:09:03.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:03 smithi130 ceph-mon[105521]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 11 pgs peering) 2024-08-17T18:09:03.923 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:03 smithi179 ceph-mon[93519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 11 pgs peering) 2024-08-17T18:09:03.940 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:03.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:03.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:04.042 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:04.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:04.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:04.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:04.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:04.204 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:04.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:04.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:04.306 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:04.347 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:04.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:04.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:04.375 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 7] out_osds: [5, 4, 6, 1] dead_osds: [3] live_osds: [1, 2, 4, 5, 6, 7, 0] 2024-08-17T18:09:04.375 INFO:tasks.thrashosds.thrasher:Reviving osd 3 2024-08-17T18:09:04.375 INFO:tasks.cephadm.osd.3:Restarting osd.3 (starting--it wasn't running)... 2024-08-17T18:09:04.376 DEBUG:teuthology.orchestra.run.smithi130:osd.3> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.3.service 2024-08-17T18:09:04.392 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl start ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.3 2024-08-17T18:09:04.453 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:04.480 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 81 16 122575 122559 94.5457 74.75 0.00698785 0.0105723 2024-08-17T18:09:04.481 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:04.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:04.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:04.582 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:04.607 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:04.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:04.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:04.669 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:09:04.677 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:04 smithi130 systemd[1]: Starting Ceph osd.3 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:09:04.710 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:04.749 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:04.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:04.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:04.851 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:04.888 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:04.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:04.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:04.963 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:04 smithi130 ceph-mon[105521]: pgmap v2373: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.3 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 96 MiB/s wr, 1.53k op/s; 81607/358848 objects degraded (22.741%); 52392/358848 objects misplaced (14.600%); 8.5 MiB/s, 134 objects/s recovering 2024-08-17T18:09:04.964 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:04 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/353791967' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:09:04.964 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:04 smithi130 ceph-mon[110068]: pgmap v2373: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.3 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 96 MiB/s wr, 1.53k op/s; 81607/358848 objects degraded (22.741%); 52392/358848 objects misplaced (14.600%); 8.5 MiB/s, 134 objects/s recovering 2024-08-17T18:09:04.964 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:04 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/353791967' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:09:04.990 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:05.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:05.044 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:05.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:05.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:04 smithi179 ceph-mon[93519]: pgmap v2373: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.3 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 96 MiB/s wr, 1.53k op/s; 81607/358848 objects degraded (22.741%); 52392/358848 objects misplaced (14.600%); 8.5 MiB/s, 134 objects/s recovering 2024-08-17T18:09:05.045 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:04 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/353791967' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:09:05.145 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:05.186 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:05.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:05.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:05.287 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:05.315 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:05.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:05.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:05.417 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:05.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:05.439 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:05.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:05.480 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 82 16 123790 123774 94.3188 75.9375 0.00818117 0.0105971 2024-08-17T18:09:05.540 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:05.576 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:05.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:05.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:05.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:05.694 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:09:05.694 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:09:05.703 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:05.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:05.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:05.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:05.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:05.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:05.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:05.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:05 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:09:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:05 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/353791967' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:09:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:05 smithi179 ceph-mon[93519]: osdmap e1340: 8 total, 7 up, 4 in 2024-08-17T18:09:05.934 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:05.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:05.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:05.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:06.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:06.103 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:06.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:06.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:05 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:09:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:05 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/353791967' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:09:06.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:05 smithi130 ceph-mon[110068]: osdmap e1340: 8 total, 7 up, 4 in 2024-08-17T18:09:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:05 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:09:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:05 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/353791967' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:09:06.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:05 smithi130 ceph-mon[105521]: osdmap e1340: 8 total, 7 up, 4 in 2024-08-17T18:09:06.205 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:06.238 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:06.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:06.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:06.339 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:06.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:06.380 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:06.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:06.480 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 83 16 124989 124973 94.0852 74.9375 0.0106525 0.0106245 2024-08-17T18:09:06.482 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:06.517 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:06.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:06.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:06.582 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T18:09:06.619 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:06.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:06.661 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:06.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:06.762 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:06.788 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:06.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:06.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:06 smithi179 ceph-mon[93519]: pgmap v2375: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s; 82381/363480 objects degraded (22.665%); 52786/363480 objects misplaced (14.522%); 8.3 MiB/s, 131 objects/s recovering 2024-08-17T18:09:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 82381/363480 objects degraded (22.665%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:06.890 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:06.909 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:09:06.930 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:06.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:06.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:07.033 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:07.055 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:07.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:07.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:07.156 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:07.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:06 smithi130 ceph-mon[105521]: pgmap v2375: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s; 82381/363480 objects degraded (22.665%); 52786/363480 objects misplaced (14.522%); 8.3 MiB/s, 131 objects/s recovering 2024-08-17T18:09:07.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 82381/363480 objects degraded (22.665%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:07.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:06 smithi130 ceph-mon[110068]: pgmap v2375: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 66 MiB/s wr, 1.05k op/s; 82381/363480 objects degraded (22.665%); 52786/363480 objects misplaced (14.522%); 8.3 MiB/s, 131 objects/s recovering 2024-08-17T18:09:07.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 82381/363480 objects degraded (22.665%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:07.182 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:07.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:07.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:07.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:07.307 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:07.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:07.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:07.409 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:07.445 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:07.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:07.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:07.480 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 84 16 126162 126146 93.8378 73.3125 0.0110594 0.0106521 2024-08-17T18:09:07.548 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:07.571 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:07.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:07.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:07.673 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:07.699 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:07.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:07.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:07.801 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:07.824 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:07.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:07.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:07.926 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:07.952 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:07.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:07.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:08.053 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:08.077 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:08.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:08.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:08.179 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:08.221 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:08.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:08.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:08.323 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:08.367 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:08.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:08.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:08.469 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:08.480 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 85 16 127352 127336 93.6086 74.375 0.0290831 0.0106785 2024-08-17T18:09:08.507 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:08.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:08.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:08.609 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:08.632 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:08.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:08.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:08.734 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:08.776 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:08.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:08.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:08.878 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:08.902 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:08.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:08.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:09.004 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:09.012 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:09:09.012 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:09:09.012 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:08 smithi179 ceph-mon[93519]: pgmap v2376: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 50 MiB/s wr, 807 op/s; 82381/363480 objects degraded (22.665%); 52786/363480 objects misplaced (14.522%); 6.4 MiB/s, 100 objects/s recovering 2024-08-17T18:09:09.033 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:09.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:09.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:09.135 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:09.160 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:09.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:09.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:09.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:08 smithi130 ceph-mon[105521]: pgmap v2376: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 50 MiB/s wr, 807 op/s; 82381/363480 objects degraded (22.665%); 52786/363480 objects misplaced (14.522%); 6.4 MiB/s, 100 objects/s recovering 2024-08-17T18:09:09.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:08 smithi130 ceph-mon[110068]: pgmap v2376: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 50 MiB/s wr, 807 op/s; 82381/363480 objects degraded (22.665%); 52786/363480 objects misplaced (14.522%); 6.4 MiB/s, 100 objects/s recovering 2024-08-17T18:09:09.261 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:09.284 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:09.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:09.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:09.386 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:09.425 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:09.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:09.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:09.480 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 86 16 128475 128459 93.3363 70.1875 0.0152901 0.0107097 2024-08-17T18:09:09.527 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:09.566 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:09.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:09.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:09.668 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:09.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:09.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:09.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:09.728 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T18:09:09.793 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:09.833 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:09.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:09.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:09.935 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:09.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:09.978 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:09.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:10.079 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:10.096 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:09:10.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:10.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:10.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:09:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:09:09] "GET /metrics HTTP/1.1" 200 36792 "" "Prometheus/2.51.0" 2024-08-17T18:09:10.209 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:10.234 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:10.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:10.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:10.336 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:10.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:10.384 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:10.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:10.481 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 87 16 129622 129606 93.0874 71.6875 0.0344792 0.010738 2024-08-17T18:09:10.485 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:10.508 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:10.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:10.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:10.555 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:09:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:09:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:09:10.610 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:10.652 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:10.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:10.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:10.754 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:10.795 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:10.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:10.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:10.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:10 smithi179 ceph-mon[93519]: pgmap v2377: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.8 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 87 MiB/s wr, 1.39k op/s; 85555/381336 objects degraded (22.436%); 55332/381336 objects misplaced (14.510%); 12 MiB/s, 187 objects/s recovering 2024-08-17T18:09:10.897 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:10.939 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:10.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:10.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:10.948 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:10 smithi130 podman[336682]: 2024-08-17 18:09:10.594131943 +0000 UTC m=+0.054349410 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T18:09:10.948 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:10 smithi130 ceph-mon[105521]: pgmap v2377: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.8 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 87 MiB/s wr, 1.39k op/s; 85555/381336 objects degraded (22.436%); 55332/381336 objects misplaced (14.510%); 12 MiB/s, 187 objects/s recovering 2024-08-17T18:09:10.948 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:10 smithi130 ceph-mon[110068]: pgmap v2377: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.8 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 87 MiB/s wr, 1.39k op/s; 85555/381336 objects degraded (22.436%); 55332/381336 objects misplaced (14.510%); 12 MiB/s, 187 objects/s recovering 2024-08-17T18:09:11.041 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:11.065 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:11.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:11.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:11.167 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:11.191 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:11.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:11.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:11.222 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:11 smithi130 podman[336682]: 2024-08-17 18:09:11.064610659 +0000 UTC m=+0.524828115 container create 416ba17c538b57bd48b35fc458e9e0e19cae12b499dee71737043e0d1345b93f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-08-17T18:09:11.292 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:11.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:11.317 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:11.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:11.418 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:11.458 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:11.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:11.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:11.481 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 88 16 130673 130657 92.776 65.6875 0.0104613 0.0107733 2024-08-17T18:09:11.560 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:11.568 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:11 smithi130 podman[336682]: 2024-08-17 18:09:11.453030239 +0000 UTC m=+0.913247703 container init 416ba17c538b57bd48b35fc458e9e0e19cae12b499dee71737043e0d1345b93f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-08-17T18:09:11.568 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:11 smithi130 podman[336682]: 2024-08-17 18:09:11.465322687 +0000 UTC m=+0.925540147 container start 416ba17c538b57bd48b35fc458e9e0e19cae12b499dee71737043e0d1345b93f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-08-17T18:09:11.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:09:11.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:09:11.609 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:11.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:11.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:11.622 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:09:11.622 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:09:11.651 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:09:11.651 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:09:11.652 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:09:11.668 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:09:11.668 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:09:11.694 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:09:11.695 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:09:11.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:11.732 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:09:11.732 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:09:11.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:11.755 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:11.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:11.759 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:09:11.760 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:09:11.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:11.878 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:11.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:11.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:11.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 85555/381336 objects degraded (22.436%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:11.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:11 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2477873041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 85555/381336 objects degraded (22.436%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:11.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:11 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2477873041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:11.917 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:11 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[336764]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:09:11.980 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:11.997 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 85555/381336 objects degraded (22.436%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:11.997 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:11 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2477873041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:12.001 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:12.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:12.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:12.006 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:09:12.006 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T18:09:12.006 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T18:09:12.006 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [] 2024-08-17T18:09:12.006 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:09:12.103 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:12.129 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:12.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:12.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:12.231 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:12.253 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:12.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:12.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:12.354 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:12.380 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:12.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:12.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:12.416 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[336764]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:09:12.416 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[336764]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T18:09:12.417 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[336764]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:09:12.417 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[336764]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:09:12.481 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 89 16 131287 131271 92.1648 38.375 0.0275082 0.0108455 2024-08-17T18:09:12.481 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:12.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:12.507 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:12.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:12.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:09:12.608 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:12.631 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:12.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:12.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:12.732 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:12.755 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:12.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:12.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:12.810 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T18:09:12.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:12 smithi130 ceph-mon[110068]: pgmap v2378: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.8 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 77 MiB/s wr, 1.22k op/s; 85555/381336 objects degraded (22.436%); 55332/381336 objects misplaced (14.510%); 10 MiB/s, 165 objects/s recovering 2024-08-17T18:09:12.845 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:09:12.845 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:12 smithi130 ceph-mon[105521]: pgmap v2378: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.8 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 77 MiB/s wr, 1.22k op/s; 85555/381336 objects degraded (22.436%); 55332/381336 objects misplaced (14.510%); 10 MiB/s, 165 objects/s recovering 2024-08-17T18:09:12.845 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:09:12.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:12.896 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:12.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:12.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:12.997 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:13.020 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:13.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:13.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:13.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:12 smithi179 ceph-mon[93519]: pgmap v2378: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.8 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 77 MiB/s wr, 1.22k op/s; 85555/381336 objects degraded (22.436%); 55332/381336 objects misplaced (14.510%); 10 MiB/s, 165 objects/s recovering 2024-08-17T18:09:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:09:13.106 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:09:13.122 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:13.147 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:13.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:13.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:13.167 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[336764]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T18:09:13.167 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:12 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[336764]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-08-17T18:09:13.167 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:12 smithi130 podman[336682]: 2024-08-17 18:09:12.880299825 +0000 UTC m=+2.340517287 container attach 416ba17c538b57bd48b35fc458e9e0e19cae12b499dee71737043e0d1345b93f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T18:09:13.167 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:12 smithi130 bash[336682]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:09:13.167 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:12 smithi130 bash[336682]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:09:13.167 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:12 smithi130 bash[336682]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T18:09:13.167 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:12 smithi130 bash[336682]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:09:13.167 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:12 smithi130 bash[336682]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:09:13.167 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:12 smithi130 bash[336682]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T18:09:13.167 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:12 smithi130 bash[336682]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-08-17T18:09:13.248 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:13.270 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:13.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:13.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:13.396 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:13.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:13.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:13.481 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Total time run: 90.0049 2024-08-17T18:09:13.481 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Total writes made: 131962 2024-08-17T18:09:13.481 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Write size: 65536 2024-08-17T18:09:13.481 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Object size: 65536 2024-08-17T18:09:13.481 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Bandwidth (MB/sec): 91.6352 2024-08-17T18:09:13.481 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev Bandwidth: 36.4463 2024-08-17T18:09:13.481 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max bandwidth (MB/sec): 218.375 2024-08-17T18:09:13.482 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min bandwidth (MB/sec): 0 2024-08-17T18:09:13.482 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Average IOPS: 1466 2024-08-17T18:09:13.482 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev IOPS: 583.151 2024-08-17T18:09:13.482 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max IOPS: 3494 2024-08-17T18:09:13.482 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min IOPS: 0 2024-08-17T18:09:13.482 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Average Latency(s): 0.0109091 2024-08-17T18:09:13.482 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev Latency(s): 0.0207294 2024-08-17T18:09:13.482 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max latency(s): 1.68932 2024-08-17T18:09:13.482 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min latency(s): 0.00101746 2024-08-17T18:09:13.487 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T18:09:13.498 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:13.522 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:13.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:13.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:13.624 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:13.648 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:13.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:13.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:13.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[336764]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-08-17T18:09:13.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:13 smithi130 bash[336682]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-08-17T18:09:13.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[336764]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-08-17T18:09:13.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:13 smithi130 bash[336682]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-08-17T18:09:13.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[336764]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T18:09:13.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:13 smithi130 bash[336682]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T18:09:13.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[336764]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T18:09:13.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:13 smithi130 bash[336682]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T18:09:13.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:13 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate[336764]: --> ceph-volume lvm activate successful for osd ID: 3 2024-08-17T18:09:13.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:13 smithi130 bash[336682]: --> ceph-volume lvm activate successful for osd ID: 3 2024-08-17T18:09:13.750 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:13.775 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:13.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:13.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:13.877 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:13.915 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:13.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:13.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:14.016 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:14.044 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:14.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:14.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:14.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:14.171 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:14.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:14.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:14.273 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:14.297 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:14.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:14.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:14.399 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:14.422 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:14.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:14.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:14.523 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:14.549 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:14.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:14.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:14.650 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:14.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:14.685 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:14.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:14.786 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:14.809 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:14.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:14.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:14.840 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:14 smithi179 ceph-mon[93519]: pgmap v2379: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 8.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 69 MiB/s wr, 1.11k op/s; 87204/392160 objects degraded (22.237%); 57170/392160 objects misplaced (14.578%); 12 MiB/s, 186 objects/s recovering 2024-08-17T18:09:14.911 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:14.934 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:14.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:14.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:15.036 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:15.071 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:15.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:15.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:15.114 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:14 smithi130 ceph-mon[105521]: pgmap v2379: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 8.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 69 MiB/s wr, 1.11k op/s; 87204/392160 objects degraded (22.237%); 57170/392160 objects misplaced (14.578%); 12 MiB/s, 186 objects/s recovering 2024-08-17T18:09:15.114 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:14 smithi130 ceph-mon[110068]: pgmap v2379: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 8.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 69 MiB/s wr, 1.11k op/s; 87204/392160 objects degraded (22.237%); 57170/392160 objects misplaced (14.578%); 12 MiB/s, 186 objects/s recovering 2024-08-17T18:09:15.142 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:09:15.142 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:09:15.142 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T18:09:15.142 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:09:15.153 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:09:15.154 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:09:15.173 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:15.227 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:15.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:15.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:15.328 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:15.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:15.368 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:15.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:15.416 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:15 smithi130 podman[336682]: 2024-08-17 18:09:15.146983966 +0000 UTC m=+4.607201435 container died 416ba17c538b57bd48b35fc458e9e0e19cae12b499dee71737043e0d1345b93f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T18:09:15.469 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:15.511 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:15.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:15.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:15.613 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:15.651 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:15.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:15.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:15.753 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:15.790 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:15.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:15.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:15.891 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:15.907 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T18:09:15.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:15.918 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:15.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:16.019 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:16.037 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:09:16.044 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:16.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:16.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:16.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:16.176 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:16.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:16.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:16.187 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:15 smithi130 podman[337125]: 2024-08-17 18:09:15.935744649 +0000 UTC m=+2.617634228 container remove 416ba17c538b57bd48b35fc458e9e0e19cae12b499dee71737043e0d1345b93f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-08-17T18:09:16.228 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:09:16.278 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:16.301 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:16.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:16.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:16.403 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:16.441 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:16.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:16.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:16.543 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:16.571 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:16.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:16.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:16.672 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:16.711 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:16.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:16.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:16.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:16.852 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:16.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:16.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:16 smithi130 ceph-mon[105521]: pgmap v2380: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 8.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 62 MiB/s wr, 999 op/s; 87691/394890 objects degraded (22.206%); 57435/394890 objects misplaced (14.545%); 11 MiB/s, 173 objects/s recovering 2024-08-17T18:09:16.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 87691/394890 objects degraded (22.206%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:16 smithi130 ceph-mon[110068]: pgmap v2380: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 8.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 62 MiB/s wr, 999 op/s; 87691/394890 objects degraded (22.206%); 57435/394890 objects misplaced (14.545%); 11 MiB/s, 173 objects/s recovering 2024-08-17T18:09:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 87691/394890 objects degraded (22.206%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:16.953 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:16.979 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:16.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:16.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:17.081 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:17.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:16 smithi179 ceph-mon[93519]: pgmap v2380: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 8.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 62 MiB/s wr, 999 op/s; 87691/394890 objects degraded (22.206%); 57435/394890 objects misplaced (14.545%); 11 MiB/s, 173 objects/s recovering 2024-08-17T18:09:17.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 87691/394890 objects degraded (22.206%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:17.120 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:17.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:17.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:17.155 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:09:17.222 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:17.291 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:17.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:17.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:17.301 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:16 smithi130 podman[337388]: 2024-08-17 18:09:16.896349648 +0000 UTC m=+0.462483472 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T18:09:17.302 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:17 smithi130 podman[337388]: 2024-08-17 18:09:17.039166685 +0000 UTC m=+0.605300499 container create bafb5a2dcf5301fcbd2bdfc34d145a50270c8e08197f3314a2a300f8781c8199 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-08-17T18:09:17.393 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:17.442 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:17.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:17.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:17.461 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T18:09:17.544 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:17.556 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:17 smithi130 podman[337388]: 2024-08-17 18:09:17.30126916 +0000 UTC m=+0.867402987 container init bafb5a2dcf5301fcbd2bdfc34d145a50270c8e08197f3314a2a300f8781c8199 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-08-17T18:09:17.556 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:17 smithi130 podman[337388]: 2024-08-17 18:09:17.317665934 +0000 UTC m=+0.883799754 container start bafb5a2dcf5301fcbd2bdfc34d145a50270c8e08197f3314a2a300f8781c8199 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-08-17T18:09:17.556 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:17 smithi130 bash[337388]: bafb5a2dcf5301fcbd2bdfc34d145a50270c8e08197f3314a2a300f8781c8199 2024-08-17T18:09:17.556 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:17 smithi130 systemd[1]: Started Ceph osd.3 for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T18:09:17.568 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:17.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:17.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:17.669 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:17.758 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:17.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:17.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:17.859 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:17.887 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:17.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:17.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:17.988 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:18.011 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:18.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:18.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:18.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:18.137 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:18.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:18.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:18.194 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:09:18.194 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:09:18.194 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:09:18.194 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:09:18.195 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:09:18.240 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:18.277 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:18.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:18.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:18.378 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:18.399 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:18.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:18.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:18.416 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:18 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[337468]: 2024-08-17T18:09:18.049+0000 7f4e56bf2640 -1 Falling back to public interface 2024-08-17T18:09:18.501 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:18.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:18.534 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:18.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:18.588 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:09:18.635 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:18.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:18.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:18.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:18.758 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:18.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:18.801 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:18.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:18.902 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:18.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:18 smithi130 ceph-mon[105521]: pgmap v2381: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 8.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 55 MiB/s wr, 892 op/s; 87691/394890 objects degraded (22.206%); 57435/394890 objects misplaced (14.545%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T18:09:18.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:18 smithi130 ceph-mon[110068]: pgmap v2381: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 8.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 55 MiB/s wr, 892 op/s; 87691/394890 objects degraded (22.206%); 57435/394890 objects misplaced (14.545%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T18:09:18.923 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:18.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:18.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:19.025 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:19.064 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:19.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:19.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:19.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:18 smithi179 ceph-mon[93519]: pgmap v2381: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 8.0 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 55 MiB/s wr, 892 op/s; 87691/394890 objects degraded (22.206%); 57435/394890 objects misplaced (14.545%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T18:09:19.166 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:19.187 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:19.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:19.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:19.289 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:19.312 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:19.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:19.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:19.413 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:19.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:19.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:19.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:19.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:19.560 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:19.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:19.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:19.662 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:19.689 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:19.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:19.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:19.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:19.812 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:19.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:19.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:19.914 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:19.936 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:19.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:19.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:20.038 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:20.064 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:20.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:20.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:20.125 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:09:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:09:19] "GET /metrics HTTP/1.1" 200 36824 "" "Prometheus/2.51.0" 2024-08-17T18:09:20.165 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:20.192 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:20.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:20.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:20.293 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:20.315 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:20.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:20.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:20.417 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:20.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:20.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:20.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:20.539 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:20.564 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:20.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:20.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:20.577 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:09:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:09:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:09:20.666 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:20.689 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:20.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:20.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:20.715 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:09:20.745 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:09:20.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:20.815 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:20.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:20.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:20.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:20 smithi179 ceph-mon[93519]: pgmap v2382: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 85 B/s rd, 56 MiB/s wr, 1.35k op/s; 83299/379743 objects degraded (21.936%); 55160/379743 objects misplaced (14.526%); 14 MiB/s, 227 objects/s recovering 2024-08-17T18:09:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2232236818' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:09:20.867 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:20 smithi130 ceph-mon[105521]: pgmap v2382: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 85 B/s rd, 56 MiB/s wr, 1.35k op/s; 83299/379743 objects degraded (21.936%); 55160/379743 objects misplaced (14.526%); 14 MiB/s, 227 objects/s recovering 2024-08-17T18:09:20.867 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2232236818' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:09:20.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:20 smithi130 ceph-mon[110068]: pgmap v2382: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.7 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 85 B/s rd, 56 MiB/s wr, 1.35k op/s; 83299/379743 objects degraded (21.936%); 55160/379743 objects misplaced (14.526%); 14 MiB/s, 227 objects/s recovering 2024-08-17T18:09:20.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2232236818' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:09:20.917 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:20.939 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:20.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:20.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:21.041 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:21.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:21.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:21.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:21.167 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:21.191 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:21.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:21.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:21.292 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:21.319 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:21.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:21.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:21.421 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:21.443 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:21.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:21.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:21.545 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:21.571 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:21.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:21.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:21.673 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:21.708 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:21.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:21.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:21.810 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:21.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:21.835 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:21.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:21.936 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:21.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:21.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:21.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:22.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:22.081 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:21 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:09:22.081 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:21 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2232236818' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:09:22.081 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:21 smithi179 ceph-mon[93519]: osdmap e1341: 8 total, 7 up, 4 in 2024-08-17T18:09:22.081 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 83299/379743 objects degraded (21.936%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:22.098 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:22.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:22.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:22.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:21 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:09:22.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:21 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2232236818' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:09:22.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:21 smithi130 ceph-mon[105521]: osdmap e1341: 8 total, 7 up, 4 in 2024-08-17T18:09:22.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 83299/379743 objects degraded (21.936%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:22.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:21 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:09:22.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:21 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2232236818' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:09:22.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:21 smithi130 ceph-mon[110068]: osdmap e1341: 8 total, 7 up, 4 in 2024-08-17T18:09:22.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 83299/379743 objects degraded (21.936%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:22.200 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:22.225 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:22.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:22.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:22.327 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:22.352 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:22.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:22.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:22.453 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:22.477 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:22.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:22.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:22.579 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:22.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:22.619 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:22.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:22.720 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:22.745 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:22.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:22.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:22.846 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:22.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:22.869 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:22.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:22.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:22 smithi130 ceph-mon[105521]: pgmap v2384: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 30 MiB/s wr, 1.06k op/s; 83299/378633 objects degraded (22.000%); 55160/378633 objects misplaced (14.568%); 12 MiB/s, 186 objects/s recovering 2024-08-17T18:09:22.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:22 smithi130 ceph-mon[110068]: pgmap v2384: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 30 MiB/s wr, 1.06k op/s; 83299/378633 objects degraded (22.000%); 55160/378633 objects misplaced (14.568%); 12 MiB/s, 186 objects/s recovering 2024-08-17T18:09:22.970 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:22.999 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:23.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:23.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:23.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:22 smithi179 ceph-mon[93519]: pgmap v2384: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 30 MiB/s wr, 1.06k op/s; 83299/378633 objects degraded (22.000%); 55160/378633 objects misplaced (14.568%); 12 MiB/s, 186 objects/s recovering 2024-08-17T18:09:23.101 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:23.125 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:23.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:23.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:23.227 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:23.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:23.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:23.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:23.352 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:23.376 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:23.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:23.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:23.477 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:23.500 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:23.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:23.602 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:23.626 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:23.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:23.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:23.728 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:23.772 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:23.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:23.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:23.874 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:23.897 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:23.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:23.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:23.999 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:24.023 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:24.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:24.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:24.040 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T18:09:24.125 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:24.172 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:24.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:24.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:24.274 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:24.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:24.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:24.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:24.384 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:09:24.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:24.459 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:24.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:24.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:24.560 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:24.586 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:24.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:24.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:24.687 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:24.712 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:24.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:24.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:24.814 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:24.836 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:24.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:24.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:24.938 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:24.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:24.962 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:24.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:24.982 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:09:24.983 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:09:24.983 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:24 smithi130 ceph-mon[105521]: pgmap v2385: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.5 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 7.9 MiB/s wr, 1.05k op/s; 80003/368376 objects degraded (21.718%); 53473/368376 objects misplaced (14.516%); 11 MiB/s, 172 objects/s recovering 2024-08-17T18:09:24.983 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:24 smithi130 ceph-mon[110068]: pgmap v2385: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.5 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 7.9 MiB/s wr, 1.05k op/s; 80003/368376 objects degraded (21.718%); 53473/368376 objects misplaced (14.516%); 11 MiB/s, 172 objects/s recovering 2024-08-17T18:09:25.001 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:09:25.001 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:09:25.018 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:09:25.018 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:09:25.018 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.3 2024-08-17T18:09:25.036 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:09:25.036 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:09:25.063 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:25.063 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:09:25.064 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:09:25.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:24 smithi179 ceph-mon[93519]: pgmap v2385: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.5 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 102 B/s rd, 7.9 MiB/s wr, 1.05k op/s; 80003/368376 objects degraded (21.718%); 53473/368376 objects misplaced (14.516%); 11 MiB/s, 172 objects/s recovering 2024-08-17T18:09:25.095 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:09:25.095 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:09:25.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:25.099 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:25.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:25.125 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:09:25.125 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:09:25.200 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:25.240 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:25.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:25.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:25.342 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:25.386 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:25.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:25.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:25.488 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:25.511 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:25.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:25.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:25.613 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:25.636 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:25.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:25.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:25.737 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:25.785 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:25.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:25.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:25.787 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:09:25.787 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:09:25.788 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:09:25.788 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:09:25.788 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:09:25.788 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:09:25.788 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:09:25.788 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:09:25.788 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:09:25.788 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator dump block 2024-08-17T18:09:25.788 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator fragmentation block 2024-08-17T18:09:25.788 INFO:teuthology.orchestra.run.smithi130.stderr:admin_socket: invalid command 2024-08-17T18:09:25.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:25 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4275139268' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:25.887 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:25.909 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:25.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:25.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:25 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4275139268' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:25 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4275139268' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:25.973 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:09:26.012 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:26.034 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:26.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:26.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:26.136 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:26.158 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:26.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:26.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:26.195 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:09:26.195 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:09:26.260 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:26.283 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:26.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:26.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:26.385 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:26.408 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:26.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:26.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:26.510 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:26.536 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:26.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:26.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:26.638 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:26.662 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:26.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:26.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:26.764 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:26.793 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:26.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:26.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:26.841 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:26 smithi130 ceph-mon[105521]: pgmap v2386: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 102 B/s rd, 1.5 MiB/s wr, 1.04k op/s; 79437/364989 objects degraded (21.764%); 53180/364989 objects misplaced (14.570%); 11 MiB/s, 172 objects/s recovering 2024-08-17T18:09:26.841 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 79437/364989 objects degraded (21.764%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:26.842 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:26 smithi130 ceph-mon[105521]: from='osd.3 [v2:172.21.15.130:6826/3141916061,v1:172.21.15.130:6827/3141916061]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T18:09:26.842 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:26 smithi130 ceph-mon[105521]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T18:09:26.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:26 smithi130 ceph-mon[110068]: pgmap v2386: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 102 B/s rd, 1.5 MiB/s wr, 1.04k op/s; 79437/364989 objects degraded (21.764%); 53180/364989 objects misplaced (14.570%); 11 MiB/s, 172 objects/s recovering 2024-08-17T18:09:26.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 79437/364989 objects degraded (21.764%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:26.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:26 smithi130 ceph-mon[110068]: from='osd.3 [v2:172.21.15.130:6826/3141916061,v1:172.21.15.130:6827/3141916061]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T18:09:26.842 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:26 smithi130 ceph-mon[110068]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T18:09:26.842 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:26 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[337468]: 2024-08-17T18:09:26.622+0000 7f4e56bf2640 -1 osd.3 1316 log_to_monitors true 2024-08-17T18:09:26.895 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:26.903 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:09:26.904 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-3, ['dump_ops_in_flight'] 2024-08-17T18:09:26.919 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:26.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:27.014 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:09:27.020 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:27.047 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:27.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:27.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:27.060 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:26 smithi179 ceph-mon[93519]: pgmap v2386: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.4 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 102 B/s rd, 1.5 MiB/s wr, 1.04k op/s; 79437/364989 objects degraded (21.764%); 53180/364989 objects misplaced (14.570%); 11 MiB/s, 172 objects/s recovering 2024-08-17T18:09:27.060 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 79437/364989 objects degraded (21.764%), 17 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:27.061 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:26 smithi179 ceph-mon[93519]: from='osd.3 [v2:172.21.15.130:6826/3141916061,v1:172.21.15.130:6827/3141916061]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T18:09:27.061 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:26 smithi179 ceph-mon[93519]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T18:09:27.061 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:09:27.061 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:09:27.149 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:27.150 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:09:26 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[337468]: 2024-08-17T18:09:26.838+0000 7f4e4a994640 -1 osd.3 1316 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T18:09:27.172 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:27.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:27.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:27.274 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:27.311 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:27.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:27.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:27.413 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:27.437 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:27.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:27.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:27.538 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:27.578 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:27.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:27.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:27.680 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:27.703 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:27.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:27.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:27.785 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T18:09:27.804 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:27.858 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:27.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:27.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:27.959 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:27.980 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:27.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:27.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:28.055 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:09:28.083 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:28.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:27 smithi179 ceph-mon[93519]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T18:09:28.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:27 smithi179 ceph-mon[93519]: from='osd.3 [v2:172.21.15.130:6826/3141916061,v1:172.21.15.130:6827/3141916061]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T18:09:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:27 smithi179 ceph-mon[93519]: osdmap e1342: 8 total, 7 up, 4 in 2024-08-17T18:09:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:27 smithi179 ceph-mon[93519]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T18:09:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:09:28.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:09:28.100 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:27 smithi130 ceph-mon[105521]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T18:09:28.100 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:27 smithi130 ceph-mon[105521]: from='osd.3 [v2:172.21.15.130:6826/3141916061,v1:172.21.15.130:6827/3141916061]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T18:09:28.100 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:27 smithi130 ceph-mon[105521]: osdmap e1342: 8 total, 7 up, 4 in 2024-08-17T18:09:28.100 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:27 smithi130 ceph-mon[105521]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T18:09:28.100 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:09:28.100 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:09:28.100 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:27 smithi130 ceph-mon[110068]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T18:09:28.100 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:27 smithi130 ceph-mon[110068]: from='osd.3 [v2:172.21.15.130:6826/3141916061,v1:172.21.15.130:6827/3141916061]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T18:09:28.100 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:27 smithi130 ceph-mon[110068]: osdmap e1342: 8 total, 7 up, 4 in 2024-08-17T18:09:28.101 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:27 smithi130 ceph-mon[110068]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T18:09:28.101 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:09:28.101 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:09:28.104 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:28.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:28.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:28.196 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:09:28.205 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:28.227 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:28.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:28.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:28.328 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:28.349 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:28.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:28.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:28.451 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:28.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:28.474 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:28.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:28.575 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:28.597 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:28.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:28.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:28.699 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:28.720 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:28.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:28.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:28.772 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:09:28.822 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:28.850 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:28.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:28.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:28.952 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:28.974 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:28.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:28.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:29.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:28 smithi130 ceph-mon[105521]: pgmap v2388: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.4 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 614 op/s; 79437/364989 objects degraded (21.764%); 53180/364989 objects misplaced (14.570%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T18:09:29.048 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:28 smithi130 ceph-mon[105521]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T18:09:29.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:28 smithi130 ceph-mon[105521]: osd.3 [v2:172.21.15.130:6826/3141916061,v1:172.21.15.130:6827/3141916061] boot 2024-08-17T18:09:29.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:28 smithi130 ceph-mon[105521]: osdmap e1343: 8 total, 8 up, 4 in 2024-08-17T18:09:29.049 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T18:09:29.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:28 smithi130 ceph-mon[110068]: pgmap v2388: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.4 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 614 op/s; 79437/364989 objects degraded (21.764%); 53180/364989 objects misplaced (14.570%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T18:09:29.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:28 smithi130 ceph-mon[110068]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T18:09:29.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:28 smithi130 ceph-mon[110068]: osd.3 [v2:172.21.15.130:6826/3141916061,v1:172.21.15.130:6827/3141916061] boot 2024-08-17T18:09:29.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:28 smithi130 ceph-mon[110068]: osdmap e1343: 8 total, 8 up, 4 in 2024-08-17T18:09:29.049 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T18:09:29.076 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:29.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:28 smithi179 ceph-mon[93519]: pgmap v2388: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 40 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 14 active+undersized, 13 active+undersized+degraded, 18 active+clean; 7.4 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 614 op/s; 79437/364989 objects degraded (21.764%); 53180/364989 objects misplaced (14.570%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T18:09:29.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:28 smithi179 ceph-mon[93519]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T18:09:29.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:28 smithi179 ceph-mon[93519]: osd.3 [v2:172.21.15.130:6826/3141916061,v1:172.21.15.130:6827/3141916061] boot 2024-08-17T18:09:29.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:28 smithi179 ceph-mon[93519]: osdmap e1343: 8 total, 8 up, 4 in 2024-08-17T18:09:29.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T18:09:29.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:29.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:29.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:29.198 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:29.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:29.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:29.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:29.321 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:29.341 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:29.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:29.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:29.443 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:29.469 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:29.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:29.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:29.570 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:29.591 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:29.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:29.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:29.693 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:29.713 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:29.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:29.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:29.804 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:09:29.804 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:09:29.814 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:09:29.816 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:29.839 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:29.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:29.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:29.941 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:29.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:29.979 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:29.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:30.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:29 smithi179 ceph-mon[93519]: osdmap e1344: 8 total, 8 up, 4 in 2024-08-17T18:09:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:29 smithi179 ceph-mon[93519]: 48.2 continuing backfill to osd.0 from (1330'2200,1341'9217] MIN to 1341'9217 2024-08-17T18:09:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:29 smithi179 ceph-mon[93519]: 48.c continuing backfill to osd.0 from (1333'4946,1341'9131] MIN to 1341'9131 2024-08-17T18:09:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:29 smithi179 ceph-mon[93519]: 48.c starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9131 2024-08-17T18:09:30.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:29 smithi179 ceph-mon[93519]: 48.6 continuing backfill to osd.0 from (1330'2100,1341'9113] MIN to 1341'9113 2024-08-17T18:09:30.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:29 smithi179 ceph-mon[93519]: 48.6 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9113 2024-08-17T18:09:30.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:29 smithi179 ceph-mon[93519]: 48.7 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9148 2024-08-17T18:09:30.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:29 smithi179 ceph-mon[93519]: 48.0 starting backfill to osd.3 from (0'0,0'0] MAX to 1342'9123 2024-08-17T18:09:30.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:29 smithi179 ceph-mon[93519]: 48.2 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9217 2024-08-17T18:09:30.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:29 smithi179 ceph-mon[93519]: 48.f continuing backfill to osd.0 from (1333'5057,1341'9214] 48:faadb0f6:::benchmark_data_smithi130_327597_object79819:head to 1341'9214 2024-08-17T18:09:30.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:29 smithi179 ceph-mon[93519]: 48.f starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9214 2024-08-17T18:09:30.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:29 smithi179 ceph-mon[93519]: 48.8 starting backfill to osd.3 from (0'0,0'0] MAX to 1342'9197 2024-08-17T18:09:30.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/902524513' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:09:30.118 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:30.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:30.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:30.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[110068]: osdmap e1344: 8 total, 8 up, 4 in 2024-08-17T18:09:30.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[110068]: 48.2 continuing backfill to osd.0 from (1330'2200,1341'9217] MIN to 1341'9217 2024-08-17T18:09:30.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[110068]: 48.c continuing backfill to osd.0 from (1333'4946,1341'9131] MIN to 1341'9131 2024-08-17T18:09:30.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[110068]: 48.c starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9131 2024-08-17T18:09:30.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[110068]: 48.6 continuing backfill to osd.0 from (1330'2100,1341'9113] MIN to 1341'9113 2024-08-17T18:09:30.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[110068]: 48.6 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9113 2024-08-17T18:09:30.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[110068]: 48.7 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9148 2024-08-17T18:09:30.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[110068]: 48.0 starting backfill to osd.3 from (0'0,0'0] MAX to 1342'9123 2024-08-17T18:09:30.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[110068]: 48.2 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9217 2024-08-17T18:09:30.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[110068]: 48.f continuing backfill to osd.0 from (1333'5057,1341'9214] 48:faadb0f6:::benchmark_data_smithi130_327597_object79819:head to 1341'9214 2024-08-17T18:09:30.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[110068]: 48.f starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9214 2024-08-17T18:09:30.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[110068]: 48.8 starting backfill to osd.3 from (0'0,0'0] MAX to 1342'9197 2024-08-17T18:09:30.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/902524513' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:09:30.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[105521]: osdmap e1344: 8 total, 8 up, 4 in 2024-08-17T18:09:30.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[105521]: 48.2 continuing backfill to osd.0 from (1330'2200,1341'9217] MIN to 1341'9217 2024-08-17T18:09:30.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[105521]: 48.c continuing backfill to osd.0 from (1333'4946,1341'9131] MIN to 1341'9131 2024-08-17T18:09:30.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[105521]: 48.c starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9131 2024-08-17T18:09:30.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[105521]: 48.6 continuing backfill to osd.0 from (1330'2100,1341'9113] MIN to 1341'9113 2024-08-17T18:09:30.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[105521]: 48.6 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9113 2024-08-17T18:09:30.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[105521]: 48.7 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9148 2024-08-17T18:09:30.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[105521]: 48.0 starting backfill to osd.3 from (0'0,0'0] MAX to 1342'9123 2024-08-17T18:09:30.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[105521]: 48.2 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9217 2024-08-17T18:09:30.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[105521]: 48.f continuing backfill to osd.0 from (1333'5057,1341'9214] 48:faadb0f6:::benchmark_data_smithi130_327597_object79819:head to 1341'9214 2024-08-17T18:09:30.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[105521]: 48.f starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9214 2024-08-17T18:09:30.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[105521]: 48.8 starting backfill to osd.3 from (0'0,0'0] MAX to 1342'9197 2024-08-17T18:09:30.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/902524513' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:09:30.169 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:09:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:09:29] "GET /metrics HTTP/1.1" 200 36673 "" "Prometheus/2.51.0" 2024-08-17T18:09:30.220 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:30.243 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:30.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:30.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:30.345 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:30.366 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:30.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:30.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:30.468 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:30.505 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:30.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:30.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:09:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:09:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:09:30.606 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:30.628 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:30.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:30.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:30.730 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:30.770 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:30.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:30.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:30.843 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T18:09:30.872 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:30.908 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:30.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:30.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:31.010 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:31.035 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:31.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:31.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:31.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:30 smithi179 ceph-mon[93519]: pgmap v2391: 89 pgs: 15 peering, 42 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 5 active+undersized+degraded, 18 active+clean; 7.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 744 op/s; 45037/354978 objects degraded (12.687%); 14541/354978 objects misplaced (4.096%); 7.9 MiB/s, 126 objects/s recovering 2024-08-17T18:09:31.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:30 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:09:31.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:30 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/902524513' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:09:31.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:30 smithi179 ceph-mon[93519]: 48.b starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9238 2024-08-17T18:09:31.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:30 smithi179 ceph-mon[93519]: 48.d starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9164 2024-08-17T18:09:31.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:30 smithi179 ceph-mon[93519]: 48.5 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9171 2024-08-17T18:09:31.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:30 smithi179 ceph-mon[93519]: 48.9 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9446 2024-08-17T18:09:31.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:30 smithi179 ceph-mon[93519]: osdmap e1345: 8 total, 8 up, 4 in 2024-08-17T18:09:31.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:30 smithi179 ceph-mon[93519]: 48.e starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9190 2024-08-17T18:09:31.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:30 smithi179 ceph-mon[93519]: 48.4 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9180 2024-08-17T18:09:31.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:30 smithi179 ceph-mon[93519]: 48.3 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9187 2024-08-17T18:09:31.137 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:31.140 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:09:31.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[105521]: pgmap v2391: 89 pgs: 15 peering, 42 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 5 active+undersized+degraded, 18 active+clean; 7.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 744 op/s; 45037/354978 objects degraded (12.687%); 14541/354978 objects misplaced (4.096%); 7.9 MiB/s, 126 objects/s recovering 2024-08-17T18:09:31.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:09:31.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/902524513' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:09:31.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[105521]: 48.b starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9238 2024-08-17T18:09:31.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[105521]: 48.d starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9164 2024-08-17T18:09:31.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[105521]: 48.5 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9171 2024-08-17T18:09:31.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[105521]: 48.9 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9446 2024-08-17T18:09:31.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[105521]: osdmap e1345: 8 total, 8 up, 4 in 2024-08-17T18:09:31.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[105521]: 48.e starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9190 2024-08-17T18:09:31.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[105521]: 48.4 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9180 2024-08-17T18:09:31.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[105521]: 48.3 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9187 2024-08-17T18:09:31.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[110068]: pgmap v2391: 89 pgs: 15 peering, 42 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 5 active+undersized+degraded, 18 active+clean; 7.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 744 op/s; 45037/354978 objects degraded (12.687%); 14541/354978 objects misplaced (4.096%); 7.9 MiB/s, 126 objects/s recovering 2024-08-17T18:09:31.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:09:31.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/902524513' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:09:31.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[110068]: 48.b starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9238 2024-08-17T18:09:31.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[110068]: 48.d starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9164 2024-08-17T18:09:31.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[110068]: 48.5 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9171 2024-08-17T18:09:31.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[110068]: 48.9 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9446 2024-08-17T18:09:31.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[110068]: osdmap e1345: 8 total, 8 up, 4 in 2024-08-17T18:09:31.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[110068]: 48.e starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9190 2024-08-17T18:09:31.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[110068]: 48.4 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9180 2024-08-17T18:09:31.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:30 smithi130 ceph-mon[110068]: 48.3 starting backfill to osd.3 from (0'0,0'0] MAX to 1341'9187 2024-08-17T18:09:31.178 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:31.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:31.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:31.280 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:31.302 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:31.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:31.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:31.404 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:31.425 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:31.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:31.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:31.527 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:31.549 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:31.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:31.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:31.651 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:31.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:31.674 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:31.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:31.775 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:31.799 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:31.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:31.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:31.900 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:31.905 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T18:09:31.905 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 45037/354978 objects degraded (12.687%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:31.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 45037/354978 objects degraded (12.687%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:31.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 45037/354978 objects degraded (12.687%), 7 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2024-08-17T18:09:31.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:31.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:31.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:32.030 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:32.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:32.088 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:32.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:32.189 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:32.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:32.244 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:32.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:32.345 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:32.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:32.369 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:32.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:32.470 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:32.509 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:32.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:32.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:32.611 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:32.655 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:32.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:32.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:32.755 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:09:32.757 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:32.763 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:09:32.763 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:09:32.779 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:32.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:32.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:32.796 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:09:32.797 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:09:32.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:32 smithi130 ceph-mon[110068]: pgmap v2393: 89 pgs: 15 peering, 45 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+undersized+degraded, 18 active+clean; 7.1 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 980 op/s; 22071/351420 objects degraded (6.281%); 14541/351420 objects misplaced (4.138%); 10 MiB/s, 164 objects/s recovering 2024-08-17T18:09:32.836 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1291038009' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:32.837 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:09:32.837 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:09:32.876 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:09:32.876 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-08-17T18:09:32.881 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:32.907 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:09:32.907 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:09:32.938 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:09:32.938 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:32.938 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:09:32.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:32.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:32.965 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:09:32.966 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:09:32.999 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:09:32.999 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:09:33.040 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:33.062 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:32 smithi179 ceph-mon[93519]: pgmap v2393: 89 pgs: 15 peering, 45 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+undersized+degraded, 18 active+clean; 7.1 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 980 op/s; 22071/351420 objects degraded (6.281%); 14541/351420 objects misplaced (4.138%); 10 MiB/s, 164 objects/s recovering 2024-08-17T18:09:33.062 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1291038009' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:33.064 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:33.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:33.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:33.111 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:32 smithi130 ceph-mon[105521]: pgmap v2393: 89 pgs: 15 peering, 45 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 7 active+undersized, 2 active+undersized+degraded, 18 active+clean; 7.1 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 980 op/s; 22071/351420 objects degraded (6.281%); 14541/351420 objects misplaced (4.138%); 10 MiB/s, 164 objects/s recovering 2024-08-17T18:09:33.112 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1291038009' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:33.165 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:33.187 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:33.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:33.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:33.289 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:33.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:33.315 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:33.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:33.329 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:09:33.329 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:09:33.329 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:09:33.329 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:09:33.329 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:09:33.416 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:33.463 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:33.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:33.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:33.565 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:33.606 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:33.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:33.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:33.707 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:33.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:33.731 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:33.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:33.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:33.876 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:33.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:33.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:33.977 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:34.017 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:34.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:34.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:34.102 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T18:09:34.119 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:34.122 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:09:34.152 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:34.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:34.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:34.254 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:34.277 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:34.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:34.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:34.379 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:34.387 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:09:34.403 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:34.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:34.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:34.505 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:34.528 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:34.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:34.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:34.630 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:34.675 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:34.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:34.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:34.777 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:34.833 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:34.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:34.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:34.935 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:34.957 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:34.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:34.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:34.969 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:34 smithi130 ceph-mon[105521]: pgmap v2394: 89 pgs: 1 active+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 7 peering, 4 active+undersized+degraded+remapped+backfill_wait, 12 remapped+peering, 1 active+undersized+degraded, 61 active+clean; 7.1 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 998 op/s; 45033/347010 objects degraded (12.977%); 42164/347010 objects misplaced (12.151%); 14 MiB/s, 224 objects/s recovering 2024-08-17T18:09:34.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:34 smithi130 ceph-mon[110068]: pgmap v2394: 89 pgs: 1 active+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 7 peering, 4 active+undersized+degraded+remapped+backfill_wait, 12 remapped+peering, 1 active+undersized+degraded, 61 active+clean; 7.1 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 998 op/s; 45033/347010 objects degraded (12.977%); 42164/347010 objects misplaced (12.151%); 14 MiB/s, 224 objects/s recovering 2024-08-17T18:09:35.059 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:35.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:34 smithi179 ceph-mon[93519]: pgmap v2394: 89 pgs: 1 active+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 7 peering, 4 active+undersized+degraded+remapped+backfill_wait, 12 remapped+peering, 1 active+undersized+degraded, 61 active+clean; 7.1 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 998 op/s; 45033/347010 objects degraded (12.977%); 42164/347010 objects misplaced (12.151%); 14 MiB/s, 224 objects/s recovering 2024-08-17T18:09:35.111 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:35.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:35.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:35.212 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:35.234 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:35.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:35.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:35.336 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:35.357 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:35.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:35.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:35.436 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:09:35.459 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:35.469 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:09:35.470 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:09:35.470 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:09:35.470 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:151887 48.c e1345/1343 48:3edae7db:::benchmark_data_smithi130_327597_object19922:head v 1345'9422, mlcod=1345'9419)", 2024-08-17T18:09:35.470 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:35.466057+0000", 2024-08-17T18:09:35.470 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.001704473, 2024-08-17T18:09:35.470 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001731483, 2024-08-17T18:09:35.470 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:09:35.470 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:09:35.470 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:09:35.470 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:09:35.470 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:09:35.470 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:09:35.470 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:35.466057+0000", 2024-08-17T18:09:35.471 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:09:35.471 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:09:35.471 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:09:35.471 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:09:35.471 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:35.466057+0000", 2024-08-17T18:09:35.471 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:09:35.471 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:09:35.471 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:09:35.471 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:09:35.471 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:35.466058+0000", 2024-08-17T18:09:35.471 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.159e-06 2024-08-17T18:09:35.471 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:09:35.471 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:35.466066+0000", 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0369999999999995e-06 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:35.466079+0000", 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3193000000000001e-05 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:35.466087+0000", 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.038e-06 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:09:35.472 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:09:35.473 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:35.466162+0000", 2024-08-17T18:09:35.473 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4444999999999996e-05 2024-08-17T18:09:35.473 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:09:35.473 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:09:35.473 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:09:35.473 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:35.466181+0000", 2024-08-17T18:09:35.473 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9103999999999999e-05 2024-08-17T18:09:35.473 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:09:35.473 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:09:35.473 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:09:35.473 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:09:35.473 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T18:09:35.473 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 1 2024-08-17T18:09:35.473 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:09:35.481 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:35.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:35.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:35.583 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:35.605 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:35.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:35.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:35.707 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:35.731 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:35.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:35.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:35.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:35.871 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:35.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:35.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:35.973 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:35.996 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:35.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:35.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:36.097 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:36.130 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:36.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:36.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:36.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:36.258 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:09:36.277 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:36.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:36.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:36.313 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:09:36.313 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T18:09:36.313 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T18:09:36.313 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T18:09:36.313 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.313 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:30138 46.a 46:577d6c08:::benchmark_data_smithi130_267720_object30137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1262)", 2024-08-17T18:09:36.313 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:48.638681+0000", 2024-08-17T18:09:36.313 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 587.66289368699995, 2024-08-17T18:09:36.313 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37615081299999997, 2024-08-17T18:09:36.313 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.314 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.314 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.314 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:09:36.314 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:09:36.314 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 30138 2024-08-17T18:09:36.314 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.314 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.314 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.314 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.314 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638681+0000", 2024-08-17T18:09:36.314 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.314 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.314 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.315 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.315 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638681+0000", 2024-08-17T18:09:36.315 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.315 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.315 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.315 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.315 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638680+0000", 2024-08-17T18:09:36.315 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:09:36.315 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.315 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.315 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.315 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638734+0000", 2024-08-17T18:09:36.316 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3424999999999998e-05 2024-08-17T18:09:36.316 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.316 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.316 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.316 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638742+0000", 2024-08-17T18:09:36.316 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2819999999999996e-06 2024-08-17T18:09:36.316 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.316 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.316 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.316 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638746+0000", 2024-08-17T18:09:36.316 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9740000000000004e-06 2024-08-17T18:09:36.316 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.316 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.317 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.317 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638787+0000", 2024-08-17T18:09:36.317 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0868999999999998e-05 2024-08-17T18:09:36.317 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.317 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.317 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.317 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638874+0000", 2024-08-17T18:09:36.317 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6833e-05 2024-08-17T18:09:36.317 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.317 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.317 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:09:36.317 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.638957+0000", 2024-08-17T18:09:36.318 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3745000000000005e-05 2024-08-17T18:09:36.318 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.318 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.318 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.318 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.639982+0000", 2024-08-17T18:09:36.318 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001024107 2024-08-17T18:09:36.318 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.318 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.318 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.318 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:48.640316+0000", 2024-08-17T18:09:36.318 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00033452200000000002 2024-08-17T18:09:36.318 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.319 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.319 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.319 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:49.014750+0000", 2024-08-17T18:09:36.319 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37443375699999998 2024-08-17T18:09:36.319 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.319 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.319 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.319 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:49.014807+0000", 2024-08-17T18:09:36.319 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7263999999999999e-05 2024-08-17T18:09:36.319 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.319 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.319 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.319 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:49.014832+0000", 2024-08-17T18:09:36.319 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4921000000000001e-05 2024-08-17T18:09:36.320 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.320 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.320 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.320 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.320 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.320 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42530 46.e 46:7a5bb013:::benchmark_data_smithi130_267720_object42529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:09:36.320 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:09:36.320 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 579.29213848699999, 2024-08-17T18:09:36.320 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.037353389, 2024-08-17T18:09:36.320 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.320 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:09:36.320 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.320 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:09:36.321 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:09:36.321 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42530 2024-08-17T18:09:36.321 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.321 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.321 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.321 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.321 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:09:36.321 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.321 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.321 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.321 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.321 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:09:36.321 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.322 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.322 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.322 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.322 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009436+0000", 2024-08-17T18:09:36.322 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:09:36.322 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.322 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.322 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.322 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009488+0000", 2024-08-17T18:09:36.322 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2281999999999999e-05 2024-08-17T18:09:36.322 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.322 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.322 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.323 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009508+0000", 2024-08-17T18:09:36.323 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0250000000000001e-05 2024-08-17T18:09:36.323 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.323 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.323 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.323 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009514+0000", 2024-08-17T18:09:36.323 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.161e-06 2024-08-17T18:09:36.323 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.323 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.323 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.323 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009564+0000", 2024-08-17T18:09:36.323 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0028999999999998e-05 2024-08-17T18:09:36.324 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.324 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.324 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.324 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.009573+0000", 2024-08-17T18:09:36.324 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.9369999999999993e-06 2024-08-17T18:09:36.324 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.324 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.324 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.324 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029341+0000", 2024-08-17T18:09:36.324 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0197678189999999 2024-08-17T18:09:36.324 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.324 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.324 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for missing object", 2024-08-17T18:09:36.324 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029434+0000", 2024-08-17T18:09:36.325 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.3185999999999999e-05 2024-08-17T18:09:36.325 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.325 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.325 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.325 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033201+0000", 2024-08-17T18:09:36.325 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003766934 2024-08-17T18:09:36.325 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.325 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.325 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for degraded object", 2024-08-17T18:09:36.325 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033231+0000", 2024-08-17T18:09:36.325 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9657000000000001e-05 2024-08-17T18:09:36.325 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.325 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.326 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.326 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.046712+0000", 2024-08-17T18:09:36.326 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013481102 2024-08-17T18:09:36.326 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.326 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.326 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.326 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.046790+0000", 2024-08-17T18:09:36.326 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7853000000000002e-05 2024-08-17T18:09:36.326 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.326 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.326 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.326 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.326 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.327 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42561 46.e 46:7b14be6d:::benchmark_data_smithi130_267720_object42560:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:09:36.327 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.039454+0000", 2024-08-17T18:09:36.327 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 579.26212095899996, 2024-08-17T18:09:36.327 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99762182099999996, 2024-08-17T18:09:36.327 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.327 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.327 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.327 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:09:36.327 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:09:36.327 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42561 2024-08-17T18:09:36.327 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.327 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.327 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.327 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.328 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039454+0000", 2024-08-17T18:09:36.328 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.328 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.328 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.328 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.328 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039454+0000", 2024-08-17T18:09:36.328 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.328 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.328 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.328 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.328 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039452+0000", 2024-08-17T18:09:36.328 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:09:36.328 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.329 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.329 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.329 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039516+0000", 2024-08-17T18:09:36.329 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3046000000000006e-05 2024-08-17T18:09:36.329 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.329 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.329 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.329 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039528+0000", 2024-08-17T18:09:36.329 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2916e-05 2024-08-17T18:09:36.329 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.329 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.329 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.329 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039534+0000", 2024-08-17T18:09:36.330 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2179999999999998e-06 2024-08-17T18:09:36.330 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.330 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.330 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.330 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039834+0000", 2024-08-17T18:09:36.330 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00030059599999999998 2024-08-17T18:09:36.330 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.330 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.330 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.330 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.039844+0000", 2024-08-17T18:09:36.330 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5840000000000001e-06 2024-08-17T18:09:36.330 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.330 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.331 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.331 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029614+0000", 2024-08-17T18:09:36.331 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98977001600000003 2024-08-17T18:09:36.331 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.331 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.331 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.331 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029696+0000", 2024-08-17T18:09:36.331 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2041999999999998e-05 2024-08-17T18:09:36.331 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.331 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.331 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:09:36.331 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.029809+0000", 2024-08-17T18:09:36.331 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000113503 2024-08-17T18:09:36.332 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.332 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.332 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.332 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033156+0000", 2024-08-17T18:09:36.332 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0033461559999999999 2024-08-17T18:09:36.332 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.332 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.332 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.332 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036913+0000", 2024-08-17T18:09:36.332 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037575149999999999 2024-08-17T18:09:36.332 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.332 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.332 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.333 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036976+0000", 2024-08-17T18:09:36.333 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2580000000000001e-05 2024-08-17T18:09:36.333 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.333 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.333 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.333 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037020+0000", 2024-08-17T18:09:36.333 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4678000000000002e-05 2024-08-17T18:09:36.333 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.333 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.333 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.333 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037076+0000", 2024-08-17T18:09:36.333 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5449999999999999e-05 2024-08-17T18:09:36.333 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.334 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.334 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.334 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.334 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.334 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42566 46.e 46:733e1503:::benchmark_data_smithi130_267720_object42565:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:09:36.334 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.042517+0000", 2024-08-17T18:09:36.334 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 579.25905742999998, 2024-08-17T18:09:36.334 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99468539099999997, 2024-08-17T18:09:36.334 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.334 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.334 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.335 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:09:36.335 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:09:36.335 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42566 2024-08-17T18:09:36.335 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.335 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.335 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.335 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.335 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042517+0000", 2024-08-17T18:09:36.336 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.336 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.336 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.336 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.336 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042517+0000", 2024-08-17T18:09:36.336 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.336 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.336 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.336 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.336 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042516+0000", 2024-08-17T18:09:36.337 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:09:36.337 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.337 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.337 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.337 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042575+0000", 2024-08-17T18:09:36.337 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8802999999999999e-05 2024-08-17T18:09:36.337 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.337 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.337 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.337 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042588+0000", 2024-08-17T18:09:36.337 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2812e-05 2024-08-17T18:09:36.337 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.337 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.337 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.338 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042594+0000", 2024-08-17T18:09:36.338 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.55e-06 2024-08-17T18:09:36.338 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.338 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.338 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.338 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042634+0000", 2024-08-17T18:09:36.338 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9817999999999997e-05 2024-08-17T18:09:36.338 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.338 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.338 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.338 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.042640+0000", 2024-08-17T18:09:36.338 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7570000000000001e-06 2024-08-17T18:09:36.338 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.339 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.339 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.339 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.030766+0000", 2024-08-17T18:09:36.339 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98812632600000005 2024-08-17T18:09:36.339 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.339 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.339 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.339 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.030858+0000", 2024-08-17T18:09:36.339 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2035999999999998e-05 2024-08-17T18:09:36.339 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.339 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.339 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:09:36.339 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.030950+0000", 2024-08-17T18:09:36.339 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.1605000000000001e-05 2024-08-17T18:09:36.340 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.340 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.340 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.340 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033167+0000", 2024-08-17T18:09:36.340 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002216951 2024-08-17T18:09:36.340 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.340 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.340 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.340 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036943+0000", 2024-08-17T18:09:36.340 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037761840000000001 2024-08-17T18:09:36.340 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.340 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.340 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.340 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037108+0000", 2024-08-17T18:09:36.341 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016495400000000001 2024-08-17T18:09:36.341 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.341 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.341 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.341 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037137+0000", 2024-08-17T18:09:36.341 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8826000000000001e-05 2024-08-17T18:09:36.341 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.341 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.341 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.341 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037203+0000", 2024-08-17T18:09:36.341 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6042000000000002e-05 2024-08-17T18:09:36.341 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.341 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.342 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.342 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.342 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.342 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42587 46.e 46:755c4a73:::benchmark_data_smithi130_267720_object42586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:09:36.342 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.057082+0000", 2024-08-17T18:09:36.342 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 579.24449329499998, 2024-08-17T18:09:36.342 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98134466099999995, 2024-08-17T18:09:36.342 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.342 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.342 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.342 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:09:36.342 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:09:36.343 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42587 2024-08-17T18:09:36.343 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.343 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.343 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.343 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.343 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057082+0000", 2024-08-17T18:09:36.343 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.343 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.343 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.343 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.343 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057082+0000", 2024-08-17T18:09:36.343 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.343 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.343 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.344 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.344 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057080+0000", 2024-08-17T18:09:36.344 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:09:36.344 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.344 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.344 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.344 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057140+0000", 2024-08-17T18:09:36.344 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9277000000000002e-05 2024-08-17T18:09:36.344 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.344 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.344 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.344 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057152+0000", 2024-08-17T18:09:36.344 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2279000000000001e-05 2024-08-17T18:09:36.344 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057157+0000", 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9389999999999996e-06 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057223+0000", 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5721000000000003e-05 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.057232+0000", 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6209999999999999e-06 2024-08-17T18:09:36.345 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.346 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.346 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.346 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.031638+0000", 2024-08-17T18:09:36.346 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97440567300000003 2024-08-17T18:09:36.346 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.346 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.346 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.346 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.031716+0000", 2024-08-17T18:09:36.346 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7650000000000004e-05 2024-08-17T18:09:36.346 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.346 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.346 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:09:36.346 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.031819+0000", 2024-08-17T18:09:36.347 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010341899999999999 2024-08-17T18:09:36.347 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.347 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.347 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.347 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033981+0000", 2024-08-17T18:09:36.347 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0021622220000000001 2024-08-17T18:09:36.347 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.347 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.347 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.347 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.037227+0000", 2024-08-17T18:09:36.347 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0032460060000000001 2024-08-17T18:09:36.347 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.347 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.347 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.348 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.038264+0000", 2024-08-17T18:09:36.348 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0010368949999999999 2024-08-17T18:09:36.348 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.348 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.348 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.348 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.038321+0000", 2024-08-17T18:09:36.348 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7354e-05 2024-08-17T18:09:36.348 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.348 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.348 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.348 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.038426+0000", 2024-08-17T18:09:36.348 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000104776 2024-08-17T18:09:36.348 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.349 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.349 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.349 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.349 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.349 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:42593 46.e 46:77bdd57c:::benchmark_data_smithi130_267720_object42592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1263)", 2024-08-17T18:09:36.349 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T17:59:57.061397+0000", 2024-08-17T18:09:36.349 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 579.24017744399998, 2024-08-17T18:09:36.349 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98195155000000001, 2024-08-17T18:09:36.349 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.349 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.349 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.349 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:09:36.349 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:09:36.350 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 42593 2024-08-17T18:09:36.350 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.350 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.350 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.350 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.350 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061397+0000", 2024-08-17T18:09:36.350 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.350 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.350 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.350 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.350 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061397+0000", 2024-08-17T18:09:36.350 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.350 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.351 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.351 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.351 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061390+0000", 2024-08-17T18:09:36.351 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999924 2024-08-17T18:09:36.351 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.351 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.351 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.351 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061432+0000", 2024-08-17T18:09:36.351 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2111999999999999e-05 2024-08-17T18:09:36.351 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.351 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.351 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.351 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061445+0000", 2024-08-17T18:09:36.352 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3037000000000001e-05 2024-08-17T18:09:36.352 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.352 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.352 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.352 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061450+0000", 2024-08-17T18:09:36.352 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.4759999999999997e-06 2024-08-17T18:09:36.352 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.352 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.352 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.352 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061497+0000", 2024-08-17T18:09:36.352 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6696999999999997e-05 2024-08-17T18:09:36.352 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.352 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.353 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.353 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:57.061504+0000", 2024-08-17T18:09:36.353 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7100000000000001e-06 2024-08-17T18:09:36.353 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.353 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.353 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.353 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033248+0000", 2024-08-17T18:09:36.353 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97174373999999997 2024-08-17T18:09:36.353 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.353 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.353 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.353 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033325+0000", 2024-08-17T18:09:36.353 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7785000000000001e-05 2024-08-17T18:09:36.353 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.354 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.354 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:09:36.354 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.033416+0000", 2024-08-17T18:09:36.354 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0221000000000002e-05 2024-08-17T18:09:36.354 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.354 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.354 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.354 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.036950+0000", 2024-08-17T18:09:36.354 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0035344 2024-08-17T18:09:36.354 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.354 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.354 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.354 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.040390+0000", 2024-08-17T18:09:36.355 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0034401890000000002 2024-08-17T18:09:36.355 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.355 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.355 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.355 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.043240+0000", 2024-08-17T18:09:36.355 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028503019999999999 2024-08-17T18:09:36.355 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.355 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.355 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.355 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.043292+0000", 2024-08-17T18:09:36.355 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1598999999999998e-05 2024-08-17T18:09:36.355 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.355 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.356 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.356 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T17:59:58.043349+0000", 2024-08-17T18:09:36.356 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6907999999999998e-05 2024-08-17T18:09:36.356 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.356 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.356 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.356 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.356 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.356 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79514 46.a 46:572757ec:::benchmark_data_smithi130_267720_object79513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:09:36.356 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.619984+0000", 2024-08-17T18:09:36.356 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 545.68159114299999, 2024-08-17T18:09:36.356 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.39803696399999999, 2024-08-17T18:09:36.356 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.356 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.357 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.357 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:09:36.357 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:09:36.357 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79514 2024-08-17T18:09:36.357 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.357 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.357 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.357 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.357 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.619984+0000", 2024-08-17T18:09:36.357 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.357 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.357 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.357 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.357 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.619984+0000", 2024-08-17T18:09:36.358 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.358 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.358 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.358 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.358 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.619983+0000", 2024-08-17T18:09:36.358 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:09:36.358 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.358 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.358 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.358 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620039+0000", 2024-08-17T18:09:36.358 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6446999999999997e-05 2024-08-17T18:09:36.358 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.359 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.359 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.359 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620050+0000", 2024-08-17T18:09:36.359 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0359e-05 2024-08-17T18:09:36.359 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.359 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.359 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.359 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620055+0000", 2024-08-17T18:09:36.359 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0479999999999998e-06 2024-08-17T18:09:36.359 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.359 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.359 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.359 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620085+0000", 2024-08-17T18:09:36.360 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9850000000000001e-05 2024-08-17T18:09:36.360 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.360 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.360 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.360 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620160+0000", 2024-08-17T18:09:36.360 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.5499000000000003e-05 2024-08-17T18:09:36.360 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.360 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.360 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:09:36.360 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.620240+0000", 2024-08-17T18:09:36.360 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0418000000000002e-05 2024-08-17T18:09:36.360 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.622161+0000", 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001920342 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.627515+0000", 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0053542399999999997 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017941+0000", 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.390425929 2024-08-17T18:09:36.361 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.362 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.362 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.362 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017991+0000", 2024-08-17T18:09:36.362 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9988999999999999e-05 2024-08-17T18:09:36.362 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.362 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.363 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.363 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.018021+0000", 2024-08-17T18:09:36.363 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9762999999999999e-05 2024-08-17T18:09:36.363 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.363 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.363 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.363 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.363 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.363 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79523 46.a 46:590eeb35:::benchmark_data_smithi130_267720_object79522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:09:36.363 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.631753+0000", 2024-08-17T18:09:36.363 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 545.66982181699996, 2024-08-17T18:09:36.364 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38901796, 2024-08-17T18:09:36.364 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.364 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.364 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.364 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:09:36.364 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:09:36.364 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79523 2024-08-17T18:09:36.364 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.364 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.364 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.364 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.364 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631753+0000", 2024-08-17T18:09:36.365 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.365 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.365 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.365 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.365 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631753+0000", 2024-08-17T18:09:36.365 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.365 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.365 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.365 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.365 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631752+0000", 2024-08-17T18:09:36.365 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:09:36.365 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.365 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.366 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.366 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631818+0000", 2024-08-17T18:09:36.366 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5757000000000001e-05 2024-08-17T18:09:36.366 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.366 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.366 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.366 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631828+0000", 2024-08-17T18:09:36.366 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0200000000000001e-05 2024-08-17T18:09:36.366 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.366 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.366 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.366 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631832+0000", 2024-08-17T18:09:36.366 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0430000000000001e-06 2024-08-17T18:09:36.367 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.367 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.367 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.367 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631869+0000", 2024-08-17T18:09:36.367 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6730999999999998e-05 2024-08-17T18:09:36.367 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.367 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.367 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.367 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.631941+0000", 2024-08-17T18:09:36.367 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.1763999999999999e-05 2024-08-17T18:09:36.367 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.367 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.368 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:09:36.368 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632027+0000", 2024-08-17T18:09:36.368 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6599000000000002e-05 2024-08-17T18:09:36.368 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.368 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.368 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.368 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635101+0000", 2024-08-17T18:09:36.368 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0030730430000000001 2024-08-17T18:09:36.368 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.369 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.369 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.369 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.641996+0000", 2024-08-17T18:09:36.369 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0068956429999999999 2024-08-17T18:09:36.369 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.369 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.369 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.369 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020716+0000", 2024-08-17T18:09:36.369 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.378719474 2024-08-17T18:09:36.369 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.369 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.369 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.369 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020748+0000", 2024-08-17T18:09:36.370 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2026000000000001e-05 2024-08-17T18:09:36.370 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.370 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.370 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.370 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020771+0000", 2024-08-17T18:09:36.370 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3370000000000002e-05 2024-08-17T18:09:36.370 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.370 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.370 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.370 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.370 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.370 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79526 46.a 46:540ef56b:::benchmark_data_smithi130_267720_object79525:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:09:36.371 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:09:36.371 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 545.66929449899999, 2024-08-17T18:09:36.371 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38861050899999999, 2024-08-17T18:09:36.371 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.371 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.371 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.371 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:09:36.371 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:09:36.371 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79526 2024-08-17T18:09:36.371 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.371 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.371 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.373 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.373 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:09:36.373 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.373 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.373 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.373 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.373 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:09:36.373 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.373 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.373 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.374 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.374 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632280+0000", 2024-08-17T18:09:36.374 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:09:36.374 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.374 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.374 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.374 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632360+0000", 2024-08-17T18:09:36.374 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0008999999999999e-05 2024-08-17T18:09:36.374 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.374 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.374 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.374 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632369+0000", 2024-08-17T18:09:36.375 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6590000000000002e-06 2024-08-17T18:09:36.375 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.375 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.375 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.375 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.632373+0000", 2024-08-17T18:09:36.375 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8840000000000004e-06 2024-08-17T18:09:36.375 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.375 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.375 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.375 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.633861+0000", 2024-08-17T18:09:36.375 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001487326 2024-08-17T18:09:36.375 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.375 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.376 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.376 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.633938+0000", 2024-08-17T18:09:36.376 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7625000000000003e-05 2024-08-17T18:09:36.376 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.376 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.376 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:09:36.376 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.634026+0000", 2024-08-17T18:09:36.376 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7950999999999999e-05 2024-08-17T18:09:36.376 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.376 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.376 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.376 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635234+0000", 2024-08-17T18:09:36.377 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0012074340000000001 2024-08-17T18:09:36.377 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.377 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.377 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.377 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.642019+0000", 2024-08-17T18:09:36.377 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0067856890000000001 2024-08-17T18:09:36.377 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.377 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.377 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.377 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020838+0000", 2024-08-17T18:09:36.377 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37881851700000002 2024-08-17T18:09:36.377 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.377 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.378 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.378 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020870+0000", 2024-08-17T18:09:36.378 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2261999999999997e-05 2024-08-17T18:09:36.378 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.378 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.378 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.378 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.020891+0000", 2024-08-17T18:09:36.378 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0777000000000001e-05 2024-08-17T18:09:36.378 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.378 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.378 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.378 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.379 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.379 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79528 46.e 46:716b67ef:::benchmark_data_smithi130_267720_object79527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:09:36.379 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.635764+0000", 2024-08-17T18:09:36.379 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 545.66581068000005, 2024-08-17T18:09:36.379 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38226721899999999, 2024-08-17T18:09:36.379 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.379 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.379 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.379 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:09:36.379 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:09:36.379 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79528 2024-08-17T18:09:36.379 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.379 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635764+0000", 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635764+0000", 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635763+0000", 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:09:36.380 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.381 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.381 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.381 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635828+0000", 2024-08-17T18:09:36.381 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4821999999999997e-05 2024-08-17T18:09:36.381 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.381 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.381 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.381 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635838+0000", 2024-08-17T18:09:36.381 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9599999999999995e-06 2024-08-17T18:09:36.381 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.381 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.381 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.381 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635842+0000", 2024-08-17T18:09:36.381 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3280000000000002e-06 2024-08-17T18:09:36.382 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.382 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.382 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.382 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635865+0000", 2024-08-17T18:09:36.382 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3600000000000001e-05 2024-08-17T18:09:36.382 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.382 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.382 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.382 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635943+0000", 2024-08-17T18:09:36.382 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.8098999999999999e-05 2024-08-17T18:09:36.382 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.382 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.383 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:09:36.383 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.636030+0000", 2024-08-17T18:09:36.383 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7188000000000006e-05 2024-08-17T18:09:36.383 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.383 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.383 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.383 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.637969+0000", 2024-08-17T18:09:36.383 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001938148 2024-08-17T18:09:36.383 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.383 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.383 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.383 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.639658+0000", 2024-08-17T18:09:36.383 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0016891600000000001 2024-08-17T18:09:36.384 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.384 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.384 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.384 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017955+0000", 2024-08-17T18:09:36.384 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.378297194 2024-08-17T18:09:36.384 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.384 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.384 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.384 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.017992+0000", 2024-08-17T18:09:36.384 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6832999999999998e-05 2024-08-17T18:09:36.384 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.384 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.384 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.385 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.018031+0000", 2024-08-17T18:09:36.385 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9646e-05 2024-08-17T18:09:36.385 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.385 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.385 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.385 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.385 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.385 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79529 46.a 46:5d7561ca:::benchmark_data_smithi130_267720_object79528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:09:36.385 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.635869+0000", 2024-08-17T18:09:36.385 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 545.66570601900003, 2024-08-17T18:09:36.385 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38782661600000001, 2024-08-17T18:09:36.385 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.386 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.386 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.386 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:09:36.386 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:09:36.386 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79529 2024-08-17T18:09:36.386 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.386 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.386 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.386 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.386 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635869+0000", 2024-08-17T18:09:36.386 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.386 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.387 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.387 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.387 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635869+0000", 2024-08-17T18:09:36.387 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.387 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.387 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.387 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.387 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635868+0000", 2024-08-17T18:09:36.387 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:09:36.387 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.387 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.387 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.387 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635935+0000", 2024-08-17T18:09:36.388 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6509999999999998e-05 2024-08-17T18:09:36.388 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.388 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.388 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.388 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635942+0000", 2024-08-17T18:09:36.388 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3139999999999998e-06 2024-08-17T18:09:36.388 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.388 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.388 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.388 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635945+0000", 2024-08-17T18:09:36.388 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.7580000000000001e-06 2024-08-17T18:09:36.388 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.389 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.389 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.389 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.635983+0000", 2024-08-17T18:09:36.389 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8318000000000001e-05 2024-08-17T18:09:36.389 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.389 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.389 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.389 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.636088+0000", 2024-08-17T18:09:36.389 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000104718 2024-08-17T18:09:36.389 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.389 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.389 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:09:36.389 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.636213+0000", 2024-08-17T18:09:36.390 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000124852 2024-08-17T18:09:36.390 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.390 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.390 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.390 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.637790+0000", 2024-08-17T18:09:36.390 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0015775839999999999 2024-08-17T18:09:36.390 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.390 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.390 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.390 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.642450+0000", 2024-08-17T18:09:36.390 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0046600260000000003 2024-08-17T18:09:36.391 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.391 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.391 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.391 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.023638+0000", 2024-08-17T18:09:36.391 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38118750600000001 2024-08-17T18:09:36.391 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.391 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.391 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.392 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.023669+0000", 2024-08-17T18:09:36.392 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0998999999999999e-05 2024-08-17T18:09:36.392 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.392 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.392 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.392 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.023695+0000", 2024-08-17T18:09:36.392 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.6645e-05 2024-08-17T18:09:36.392 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.392 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.392 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.392 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.392 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.392 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79534 46.a 46:5daa6ba5:::benchmark_data_smithi130_267720_object79533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:09:36.392 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.643773+0000", 2024-08-17T18:09:36.393 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 545.65780191900001, 2024-08-17T18:09:36.393 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38289725099999999, 2024-08-17T18:09:36.393 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.393 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.393 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.393 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:09:36.393 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:09:36.393 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79534 2024-08-17T18:09:36.394 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.394 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.394 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.394 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.394 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643773+0000", 2024-08-17T18:09:36.394 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.394 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.394 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.394 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.394 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643773+0000", 2024-08-17T18:09:36.394 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.394 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.395 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.395 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.395 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643772+0000", 2024-08-17T18:09:36.395 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:09:36.395 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.395 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.395 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.395 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643824+0000", 2024-08-17T18:09:36.395 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2240000000000001e-05 2024-08-17T18:09:36.395 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.395 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.395 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.395 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643833+0000", 2024-08-17T18:09:36.396 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6770000000000001e-06 2024-08-17T18:09:36.396 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.396 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.396 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.396 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643837+0000", 2024-08-17T18:09:36.396 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8089999999999999e-06 2024-08-17T18:09:36.396 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.396 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.396 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.396 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643868+0000", 2024-08-17T18:09:36.396 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1560000000000003e-05 2024-08-17T18:09:36.396 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.397 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.397 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.397 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643961+0000", 2024-08-17T18:09:36.397 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2584999999999995e-05 2024-08-17T18:09:36.397 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.397 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.397 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:09:36.397 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.644059+0000", 2024-08-17T18:09:36.397 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7851999999999998e-05 2024-08-17T18:09:36.397 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.397 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.397 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.398 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.646074+0000", 2024-08-17T18:09:36.398 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020149550000000001 2024-08-17T18:09:36.398 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.398 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.398 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.398 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647702+0000", 2024-08-17T18:09:36.398 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001628811 2024-08-17T18:09:36.398 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.398 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.398 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.398 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.026581+0000", 2024-08-17T18:09:36.398 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.37887865300000001 2024-08-17T18:09:36.399 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.399 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.399 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.399 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.026633+0000", 2024-08-17T18:09:36.399 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2040000000000003e-05 2024-08-17T18:09:36.399 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.399 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.399 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.399 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.026670+0000", 2024-08-17T18:09:36.399 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7038e-05 2024-08-17T18:09:36.399 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.399 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.399 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.399 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.400 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.400 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36225.0:79535 46.4 46:2496b234:::benchmark_data_smithi130_267720_object79534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T18:09:36.400 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:09:36.400 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 545.65770940300001, 2024-08-17T18:09:36.400 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38560203399999998, 2024-08-17T18:09:36.400 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.400 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.400 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.400 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36225", 2024-08-17T18:09:36.400 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:09:36.400 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 79535 2024-08-17T18:09:36.400 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.400 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.400 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.401 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.401 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:09:36.401 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.401 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.401 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.401 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.401 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:09:36.401 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.401 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.401 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.401 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.401 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643865+0000", 2024-08-17T18:09:36.401 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:09:36.402 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.402 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.402 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.402 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643927+0000", 2024-08-17T18:09:36.402 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1629000000000002e-05 2024-08-17T18:09:36.402 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.402 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.402 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.402 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643933+0000", 2024-08-17T18:09:36.402 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5130000000000002e-06 2024-08-17T18:09:36.402 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.402 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.402 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.403 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643937+0000", 2024-08-17T18:09:36.403 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3479999999999997e-06 2024-08-17T18:09:36.403 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.403 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.403 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.403 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.643975+0000", 2024-08-17T18:09:36.403 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7771999999999997e-05 2024-08-17T18:09:36.403 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.403 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.403 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.403 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.644055+0000", 2024-08-17T18:09:36.403 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0144999999999999e-05 2024-08-17T18:09:36.403 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.404 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.404 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:09:36.404 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.644142+0000", 2024-08-17T18:09:36.404 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6539000000000006e-05 2024-08-17T18:09:36.404 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.404 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.404 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.404 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.646160+0000", 2024-08-17T18:09:36.404 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002018373 2024-08-17T18:09:36.404 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.404 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.404 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.404 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:30.647237+0000", 2024-08-17T18:09:36.404 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001076465 2024-08-17T18:09:36.405 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.405 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.405 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.405 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.029404+0000", 2024-08-17T18:09:36.405 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.38216758000000001 2024-08-17T18:09:36.405 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.405 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.405 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.405 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.029443+0000", 2024-08-17T18:09:36.405 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8846e-05 2024-08-17T18:09:36.405 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.405 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.405 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.405 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:00:31.029468+0000", 2024-08-17T18:09:36.406 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4355e-05 2024-08-17T18:09:36.406 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.406 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.406 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.406 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.406 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.406 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.46373.0:115177 47.e 47:7d674894:::benchmark_data_smithi130_302447_object19942:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:09:36.406 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:05:56.557182+0000", 2024-08-17T18:09:36.406 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 219.744392632, 2024-08-17T18:09:36.406 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0269707910000001, 2024-08-17T18:09:36.406 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.406 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:09:36.406 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.407 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.46373", 2024-08-17T18:09:36.407 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:09:36.407 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 115177 2024-08-17T18:09:36.407 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.407 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.407 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.407 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.407 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557182+0000", 2024-08-17T18:09:36.407 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.407 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.407 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.407 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.407 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557182+0000", 2024-08-17T18:09:36.408 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.408 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.408 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.408 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.408 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557182+0000", 2024-08-17T18:09:36.408 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:09:36.408 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.408 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.408 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.408 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557184+0000", 2024-08-17T18:09:36.408 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.623e-06 2024-08-17T18:09:36.408 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.408 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.409 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.409 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557188+0000", 2024-08-17T18:09:36.409 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3599999999999998e-06 2024-08-17T18:09:36.409 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.409 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.409 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.409 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557191+0000", 2024-08-17T18:09:36.409 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0070000000000002e-06 2024-08-17T18:09:36.409 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.409 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.409 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.409 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557236+0000", 2024-08-17T18:09:36.410 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4824999999999999e-05 2024-08-17T18:09:36.410 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.410 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.410 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.410 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557348+0000", 2024-08-17T18:09:36.410 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000112503 2024-08-17T18:09:36.410 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.410 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.410 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:09:36.410 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.557457+0000", 2024-08-17T18:09:36.410 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000108707 2024-08-17T18:09:36.410 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.410 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.559820+0000", 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0023626350000000001 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.568430+0000", 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0086101579999999997 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.077053+0000", 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.50862317199999996 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.411 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.412 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.412 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.077061+0000", 2024-08-17T18:09:36.412 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9440000000000005e-06 2024-08-17T18:09:36.412 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.412 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.412 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.412 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.584088+0000", 2024-08-17T18:09:36.412 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.507027487 2024-08-17T18:09:36.412 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.412 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.412 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.412 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.584153+0000", 2024-08-17T18:09:36.412 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4629999999999996e-05 2024-08-17T18:09:36.413 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.413 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.413 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.413 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.413 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.413 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.46373.0:115179 47.f 47:fa81ee3a:::benchmark_data_smithi130_302447_object19944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:09:36.413 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:05:56.564940+0000", 2024-08-17T18:09:36.413 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 219.73663535899999, 2024-08-17T18:09:36.413 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.024401452, 2024-08-17T18:09:36.413 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.413 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:09:36.413 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.413 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.46373", 2024-08-17T18:09:36.414 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:09:36.414 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 115179 2024-08-17T18:09:36.414 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.414 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.414 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.414 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.414 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.564940+0000", 2024-08-17T18:09:36.414 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.414 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.414 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.414 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.414 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.564940+0000", 2024-08-17T18:09:36.415 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.415 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.415 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.415 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.415 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.564939+0000", 2024-08-17T18:09:36.415 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:09:36.415 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.415 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.415 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.415 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.564943+0000", 2024-08-17T18:09:36.415 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7029999999999997e-06 2024-08-17T18:09:36.415 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.415 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.415 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.416 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.564954+0000", 2024-08-17T18:09:36.416 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1124999999999999e-05 2024-08-17T18:09:36.416 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.416 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.416 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.416 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.564959+0000", 2024-08-17T18:09:36.416 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2019999999999997e-06 2024-08-17T18:09:36.416 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.416 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.416 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.416 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.564985+0000", 2024-08-17T18:09:36.416 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.6480000000000001e-05 2024-08-17T18:09:36.417 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.417 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.417 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.417 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.565063+0000", 2024-08-17T18:09:36.417 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.8300999999999995e-05 2024-08-17T18:09:36.417 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.417 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.417 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T18:09:36.417 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.565134+0000", 2024-08-17T18:09:36.417 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0947999999999999e-05 2024-08-17T18:09:36.417 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.417 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.417 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.417 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.569055+0000", 2024-08-17T18:09:36.418 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003920649 2024-08-17T18:09:36.418 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.418 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.418 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.418 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.576387+0000", 2024-08-17T18:09:36.418 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.007332046 2024-08-17T18:09:36.418 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.418 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.418 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.418 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.077838+0000", 2024-08-17T18:09:36.418 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.50145124600000002 2024-08-17T18:09:36.418 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.419 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.419 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.419 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.077843+0000", 2024-08-17T18:09:36.419 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6630000000000001e-06 2024-08-17T18:09:36.419 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.419 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.419 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.419 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.589299+0000", 2024-08-17T18:09:36.419 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.51145641900000005 2024-08-17T18:09:36.419 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.419 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.419 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.419 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.589341+0000", 2024-08-17T18:09:36.420 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1548999999999998e-05 2024-08-17T18:09:36.420 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.420 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.420 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.420 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.420 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.420 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.46373.0:115181 47.f 47:ff784e5d:::benchmark_data_smithi130_302447_object19946:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:09:36.420 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:05:56.566481+0000", 2024-08-17T18:09:36.420 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 219.73509433000001, 2024-08-17T18:09:36.420 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.023022541, 2024-08-17T18:09:36.420 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.420 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:09:36.420 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.420 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.46373", 2024-08-17T18:09:36.421 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:09:36.421 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 115181 2024-08-17T18:09:36.421 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.421 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.421 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.421 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.421 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.566481+0000", 2024-08-17T18:09:36.421 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.421 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.421 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.421 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.421 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.566481+0000", 2024-08-17T18:09:36.421 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.422 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.422 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.422 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.422 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.566480+0000", 2024-08-17T18:09:36.422 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:09:36.422 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.422 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.422 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.422 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.566484+0000", 2024-08-17T18:09:36.422 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3810000000000001e-06 2024-08-17T18:09:36.422 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.422 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.422 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.422 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.566495+0000", 2024-08-17T18:09:36.423 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.079e-05 2024-08-17T18:09:36.423 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.423 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.423 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.423 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.566499+0000", 2024-08-17T18:09:36.423 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3470000000000001e-06 2024-08-17T18:09:36.423 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.423 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.423 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.423 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.567383+0000", 2024-08-17T18:09:36.423 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000884116 2024-08-17T18:09:36.423 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.423 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.423 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.424 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.567471+0000", 2024-08-17T18:09:36.424 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7812000000000006e-05 2024-08-17T18:09:36.424 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.424 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.424 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,3", 2024-08-17T18:09:36.424 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.567542+0000", 2024-08-17T18:09:36.424 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0338000000000003e-05 2024-08-17T18:09:36.424 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.424 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.424 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.424 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.569181+0000", 2024-08-17T18:09:36.424 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0016393460000000001 2024-08-17T18:09:36.424 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.424 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.425 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.425 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:56.584045+0000", 2024-08-17T18:09:36.425 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.014863955999999999 2024-08-17T18:09:36.425 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.425 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.425 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.425 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.077869+0000", 2024-08-17T18:09:36.425 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.493823868 2024-08-17T18:09:36.425 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.425 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.425 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.425 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.077875+0000", 2024-08-17T18:09:36.425 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4019999999999999e-06 2024-08-17T18:09:36.425 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.426 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.426 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.426 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.589472+0000", 2024-08-17T18:09:36.426 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.51159689799999997 2024-08-17T18:09:36.426 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.426 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.426 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.426 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:05:57.589503+0000", 2024-08-17T18:09:36.426 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1056000000000002e-05 2024-08-17T18:09:36.426 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.426 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.426 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.426 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.427 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.427 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:115453 48.f 48:f4eb53da:::benchmark_data_smithi130_327597_object115452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:09:36.427 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:08:57.106348+0000", 2024-08-17T18:09:36.427 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 39.195227136, 2024-08-17T18:09:36.427 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.643288114, 2024-08-17T18:09:36.427 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.427 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.427 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.427 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:09:36.427 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:09:36.427 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 115453 2024-08-17T18:09:36.427 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.427 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.428 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.428 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.428 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106348+0000", 2024-08-17T18:09:36.428 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.428 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.428 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.428 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.428 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106348+0000", 2024-08-17T18:09:36.428 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.428 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.428 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.428 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.428 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106347+0000", 2024-08-17T18:09:36.429 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:09:36.429 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.429 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.429 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.429 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106404+0000", 2024-08-17T18:09:36.429 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7111000000000003e-05 2024-08-17T18:09:36.429 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.429 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.429 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.429 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106414+0000", 2024-08-17T18:09:36.429 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0148e-05 2024-08-17T18:09:36.429 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.429 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.429 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.430 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106418+0000", 2024-08-17T18:09:36.430 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1910000000000002e-06 2024-08-17T18:09:36.430 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.430 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.430 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.430 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106459+0000", 2024-08-17T18:09:36.430 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1046e-05 2024-08-17T18:09:36.430 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.430 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.430 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.430 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106470+0000", 2024-08-17T18:09:36.430 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0722e-05 2024-08-17T18:09:36.430 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.430 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.431 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.431 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558373+0000", 2024-08-17T18:09:36.431 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.45190306899999999 2024-08-17T18:09:36.431 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.431 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.431 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.431 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558385+0000", 2024-08-17T18:09:36.431 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1593999999999999e-05 2024-08-17T18:09:36.431 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.431 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.431 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.431 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.727268+0000", 2024-08-17T18:09:36.431 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.16888346 2024-08-17T18:09:36.431 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.432 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.432 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.432 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.729887+0000", 2024-08-17T18:09:36.432 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002618521 2024-08-17T18:09:36.432 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.432 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.432 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1", 2024-08-17T18:09:36.432 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.730169+0000", 2024-08-17T18:09:36.432 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00028207800000000001 2024-08-17T18:09:36.432 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.432 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.432 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.432 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.744462+0000", 2024-08-17T18:09:36.432 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.01429271 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.745188+0000", 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00072621100000000004 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.749537+0000", 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0043489000000000002 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.749571+0000", 2024-08-17T18:09:36.433 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4696999999999997e-05 2024-08-17T18:09:36.434 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.434 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.434 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.434 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.749636+0000", 2024-08-17T18:09:36.434 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4455999999999994e-05 2024-08-17T18:09:36.434 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.434 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.434 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.434 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.434 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.434 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:115456 48.7 48:e9a5436a:::benchmark_data_smithi130_327597_object115455:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:09:36.434 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:08:57.109169+0000", 2024-08-17T18:09:36.435 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 39.192406011000003, 2024-08-17T18:09:36.435 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5023469999999999, 2024-08-17T18:09:36.435 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.435 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.435 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.435 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:09:36.435 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:09:36.435 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 115456 2024-08-17T18:09:36.435 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.435 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.435 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.435 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.435 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109169+0000", 2024-08-17T18:09:36.435 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.436 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.436 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.436 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.436 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109169+0000", 2024-08-17T18:09:36.436 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.436 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.436 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.436 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.436 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109168+0000", 2024-08-17T18:09:36.436 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:09:36.436 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.437 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.437 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.437 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109227+0000", 2024-08-17T18:09:36.437 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8847000000000001e-05 2024-08-17T18:09:36.437 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.437 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.437 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.437 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109237+0000", 2024-08-17T18:09:36.437 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6129999999999993e-06 2024-08-17T18:09:36.437 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.437 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.437 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.437 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109240+0000", 2024-08-17T18:09:36.438 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.501e-06 2024-08-17T18:09:36.438 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.438 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.438 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.438 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109271+0000", 2024-08-17T18:09:36.438 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0582e-05 2024-08-17T18:09:36.438 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.438 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.438 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.438 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109277+0000", 2024-08-17T18:09:36.438 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0100000000000001e-06 2024-08-17T18:09:36.438 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.438 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.439 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.439 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558408+0000", 2024-08-17T18:09:36.439 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.44913098299999998 2024-08-17T18:09:36.439 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.439 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.439 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.439 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558412+0000", 2024-08-17T18:09:36.439 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5669999999999996e-06 2024-08-17T18:09:36.439 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.439 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.439 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.439 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.587705+0000", 2024-08-17T18:09:36.440 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0292922280000001 2024-08-17T18:09:36.440 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.440 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.440 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.440 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.589250+0000", 2024-08-17T18:09:36.440 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0015458340000000001 2024-08-17T18:09:36.440 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.440 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.440 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:09:36.440 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.589440+0000", 2024-08-17T18:09:36.440 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000189276 2024-08-17T18:09:36.440 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.441 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.441 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.441 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.596555+0000", 2024-08-17T18:09:36.441 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0071154290000000004 2024-08-17T18:09:36.441 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.441 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.441 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.441 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.607974+0000", 2024-08-17T18:09:36.441 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.011418766 2024-08-17T18:09:36.441 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.441 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.441 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.441 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611424+0000", 2024-08-17T18:09:36.441 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0034497809999999999 2024-08-17T18:09:36.442 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.442 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.442 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.442 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611459+0000", 2024-08-17T18:09:36.442 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.5463000000000002e-05 2024-08-17T18:09:36.442 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.442 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.442 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.442 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611516+0000", 2024-08-17T18:09:36.442 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6674e-05 2024-08-17T18:09:36.442 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.442 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.442 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.443 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.443 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.443 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:115458 48.0 48:02c144b5:::benchmark_data_smithi130_327597_object115457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:09:36.443 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:08:57.111293+0000", 2024-08-17T18:09:36.443 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 39.190281997, 2024-08-17T18:09:36.443 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5002246669999999, 2024-08-17T18:09:36.443 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.443 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.443 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.443 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:09:36.443 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:09:36.443 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 115458 2024-08-17T18:09:36.443 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.444 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.444 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.444 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.444 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111293+0000", 2024-08-17T18:09:36.444 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.444 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.444 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.444 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.444 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111293+0000", 2024-08-17T18:09:36.444 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.444 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.444 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.444 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.444 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111292+0000", 2024-08-17T18:09:36.445 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:09:36.445 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.445 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.445 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.445 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111353+0000", 2024-08-17T18:09:36.445 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0925999999999999e-05 2024-08-17T18:09:36.445 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.445 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.445 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.445 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111361+0000", 2024-08-17T18:09:36.445 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7870000000000002e-06 2024-08-17T18:09:36.445 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.446 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.446 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.446 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111364+0000", 2024-08-17T18:09:36.446 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.5889999999999999e-06 2024-08-17T18:09:36.446 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.446 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.446 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.446 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111393+0000", 2024-08-17T18:09:36.446 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8082e-05 2024-08-17T18:09:36.446 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.446 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.446 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.447 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111398+0000", 2024-08-17T18:09:36.447 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5260000000000001e-06 2024-08-17T18:09:36.447 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.447 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.447 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.447 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558838+0000", 2024-08-17T18:09:36.447 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.44744001700000002 2024-08-17T18:09:36.447 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.447 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.447 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.447 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558846+0000", 2024-08-17T18:09:36.447 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2209999999999997e-06 2024-08-17T18:09:36.447 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.447 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.448 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.448 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.583382+0000", 2024-08-17T18:09:36.448 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0245358609999999 2024-08-17T18:09:36.448 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.448 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.448 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.448 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.585501+0000", 2024-08-17T18:09:36.448 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0021187269999999999 2024-08-17T18:09:36.448 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.448 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.448 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:09:36.448 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.585697+0000", 2024-08-17T18:09:36.448 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019644500000000001 2024-08-17T18:09:36.449 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.449 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.449 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.449 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.594573+0000", 2024-08-17T18:09:36.449 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0088757130000000004 2024-08-17T18:09:36.449 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.449 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.449 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.449 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611301+0000", 2024-08-17T18:09:36.449 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.016727606999999999 2024-08-17T18:09:36.449 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.449 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.449 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.449 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611426+0000", 2024-08-17T18:09:36.450 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012492300000000001 2024-08-17T18:09:36.450 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.450 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.450 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.450 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611462+0000", 2024-08-17T18:09:36.450 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6795000000000003e-05 2024-08-17T18:09:36.450 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.450 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.450 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.450 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611518+0000", 2024-08-17T18:09:36.450 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5151000000000001e-05 2024-08-17T18:09:36.450 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.450 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.450 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.451 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.451 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.451 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:115464 48.0 48:09f85e4e:::benchmark_data_smithi130_327597_object115463:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:09:36.451 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:08:57.117335+0000", 2024-08-17T18:09:36.451 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 39.184240349, 2024-08-17T18:09:36.451 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.499090252, 2024-08-17T18:09:36.451 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:09:36.451 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:09:36.451 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:09:36.451 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:09:36.451 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:09:36.451 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 115464 2024-08-17T18:09:36.451 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.452 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:09:36.452 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.452 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:09:36.452 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117335+0000", 2024-08-17T18:09:36.452 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.452 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.452 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.452 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:09:36.452 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117335+0000", 2024-08-17T18:09:36.452 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:09:36.452 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.453 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.453 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:09:36.453 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117334+0000", 2024-08-17T18:09:36.453 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:09:36.453 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.453 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.453 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:09:36.453 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117396+0000", 2024-08-17T18:09:36.453 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2404999999999996e-05 2024-08-17T18:09:36.453 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.453 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.453 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:09:36.454 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117406+0000", 2024-08-17T18:09:36.454 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6379999999999999e-06 2024-08-17T18:09:36.454 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.454 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.454 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:09:36.454 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117410+0000", 2024-08-17T18:09:36.454 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9700000000000001e-06 2024-08-17T18:09:36.454 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.454 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.454 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.454 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117445+0000", 2024-08-17T18:09:36.454 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4657999999999999e-05 2024-08-17T18:09:36.454 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.455 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.455 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.455 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117450+0000", 2024-08-17T18:09:36.455 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6030000000000004e-06 2024-08-17T18:09:36.455 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.455 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.455 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.455 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558868+0000", 2024-08-17T18:09:36.455 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.44141762299999998 2024-08-17T18:09:36.455 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.455 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.455 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:09:36.456 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558875+0000", 2024-08-17T18:09:36.456 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.4950000000000002e-06 2024-08-17T18:09:36.456 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.456 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.456 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:09:36.456 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.593273+0000", 2024-08-17T18:09:36.456 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0343973630000001 2024-08-17T18:09:36.456 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.456 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.456 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:09:36.456 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.593391+0000", 2024-08-17T18:09:36.456 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000118168 2024-08-17T18:09:36.456 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.456 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.457 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:09:36.457 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.593483+0000", 2024-08-17T18:09:36.457 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2351999999999999e-05 2024-08-17T18:09:36.457 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.457 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.457 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:09:36.457 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.594707+0000", 2024-08-17T18:09:36.457 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001224155 2024-08-17T18:09:36.457 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.457 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.457 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.457 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611339+0000", 2024-08-17T18:09:36.457 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.016631277 2024-08-17T18:09:36.458 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.458 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.458 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:09:36.458 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.616336+0000", 2024-08-17T18:09:36.458 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0049975769999999996 2024-08-17T18:09:36.458 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.458 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.458 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:09:36.458 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.616371+0000", 2024-08-17T18:09:36.458 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4779e-05 2024-08-17T18:09:36.458 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:09:36.458 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:09:36.459 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:09:36.459 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.616425+0000", 2024-08-17T18:09:36.459 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3881000000000001e-05 2024-08-17T18:09:36.459 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.459 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.459 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.459 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:09:36.459 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:09:36.459 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:09:36.459 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:36.482 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:36.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:36.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:36.584 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:36.606 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:36.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:36.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:36.708 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:36.732 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:36.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:36.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:36.835 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:36.857 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:36.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:36.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:36.959 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:36.995 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:36.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:36.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:37.097 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:37.119 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:37.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:37.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:37.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:36 smithi130 ceph-mon[110068]: pgmap v2395: 89 pgs: 1 active+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.9 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 948 op/s; 72864/341331 objects degraded (21.347%); 49258/341331 objects misplaced (14.431%); 11 MiB/s, 177 objects/s recovering 2024-08-17T18:09:37.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 72864/341331 objects degraded (21.347%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:37.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:36 smithi130 ceph-mon[105521]: pgmap v2395: 89 pgs: 1 active+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.9 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 948 op/s; 72864/341331 objects degraded (21.347%); 49258/341331 objects misplaced (14.431%); 11 MiB/s, 177 objects/s recovering 2024-08-17T18:09:37.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 72864/341331 objects degraded (21.347%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:37.221 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:37.253 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:36 smithi179 ceph-mon[93519]: pgmap v2395: 89 pgs: 1 active+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.9 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 948 op/s; 72864/341331 objects degraded (21.347%); 49258/341331 objects misplaced (14.431%); 11 MiB/s, 177 objects/s recovering 2024-08-17T18:09:37.253 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 72864/341331 objects degraded (21.347%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:37.266 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:37.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:37.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:37.278 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T18:09:37.368 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:37.399 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:37.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:37.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:37.501 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:37.523 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:37.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:37.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:37.625 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:37.677 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:37.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:37.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:37.779 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:37.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:37.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:37.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:37.922 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:37.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:37.946 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:37.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:38.047 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:38.085 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:38.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:38.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:38.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:38.211 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:38.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:38.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:38.313 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:38.356 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:38.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:38.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:38.458 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:38.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:38.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:38.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:38.582 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:38.605 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:38.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:38.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:38.706 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:38.730 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:38.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:38.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:38.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:38.869 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:38.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:38.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:38.971 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:39.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:39.017 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:39.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:39.118 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:39.141 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:39.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:39.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:39.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:38 smithi130 ceph-mon[105521]: pgmap v2396: 89 pgs: 1 active+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.9 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 529 op/s; 72864/341331 objects degraded (21.347%); 49258/341331 objects misplaced (14.431%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T18:09:39.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:38 smithi130 ceph-mon[110068]: pgmap v2396: 89 pgs: 1 active+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.9 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 529 op/s; 72864/341331 objects degraded (21.347%); 49258/341331 objects misplaced (14.431%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T18:09:39.243 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:39.282 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:39.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:39.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:39.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:38 smithi179 ceph-mon[93519]: pgmap v2396: 89 pgs: 1 active+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.9 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 529 op/s; 72864/341331 objects degraded (21.347%); 49258/341331 objects misplaced (14.431%); 4.3 MiB/s, 69 objects/s recovering 2024-08-17T18:09:39.383 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:39.415 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:39.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:39.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:39.517 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:39.540 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:39.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:39.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:39.641 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:39.664 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:39.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:39.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:39.766 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:39.787 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:39.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:39.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:39.888 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:39.910 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:39.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:39.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:40.012 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:40.053 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:40.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:40.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:40.083 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:09:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:09:39] "GET /metrics HTTP/1.1" 200 36673 "" "Prometheus/2.51.0" 2024-08-17T18:09:40.155 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:40.181 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:40.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:40.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:40.283 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:40.305 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:40.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:40.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:40.407 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:40.429 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:40.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:40.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:40.531 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:40.554 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:40.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:40.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:09:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:09:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:09:40.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:40.677 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:40.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:40.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:40.779 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:40.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:40.801 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:40.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:40.902 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:40.924 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:40.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:40.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:41.026 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:41.064 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:41.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:41.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:41.165 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:41.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:40 smithi130 ceph-mon[105521]: pgmap v2397: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 775 op/s; 70685/331689 objects degraded (21.311%); 47127/331689 objects misplaced (14.208%); 5.2 MiB/s, 83 objects/s recovering 2024-08-17T18:09:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:40 smithi130 ceph-mon[110068]: pgmap v2397: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 775 op/s; 70685/331689 objects degraded (21.311%); 47127/331689 objects misplaced (14.208%); 5.2 MiB/s, 83 objects/s recovering 2024-08-17T18:09:41.188 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:41.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:41.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:41.290 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:41.312 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:41.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:41.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:41.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:40 smithi179 ceph-mon[93519]: pgmap v2397: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 775 op/s; 70685/331689 objects degraded (21.311%); 47127/331689 objects misplaced (14.208%); 5.2 MiB/s, 83 objects/s recovering 2024-08-17T18:09:41.414 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:41.455 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:41.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:41.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:41.557 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:41.595 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:41.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:41.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:41.697 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:41.718 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:41.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:41.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:41.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:41.862 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:41.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:41.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:41.964 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:41.985 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:41.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:41.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:42.086 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:42.108 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:42.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:42.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:42.121 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:41 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 70685/331689 objects degraded (21.311%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:42.121 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:41 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 70685/331689 objects degraded (21.311%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:42.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T18:09:42.209 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:42.232 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:42.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:42.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:42.333 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:42.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:41 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 70685/331689 objects degraded (21.311%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:42.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:42.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:42.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:42.457 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:42.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:42.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:42.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:42.581 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:42.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:42.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:42.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:42.704 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:42.725 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:42.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:42.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:42.827 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:42.849 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:42.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:42.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:42.951 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:42.975 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:42.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:42.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:43.076 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:43.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:43.108 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:43.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:43.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:42 smithi130 ceph-mon[105521]: pgmap v2398: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 661 op/s; 69901/328416 objects degraded (21.284%); 47127/328416 objects misplaced (14.350%); 4.5 MiB/s, 71 objects/s recovering 2024-08-17T18:09:43.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:09:43.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:09:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:42 smithi130 ceph-mon[110068]: pgmap v2398: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 661 op/s; 69901/328416 objects degraded (21.284%); 47127/328416 objects misplaced (14.350%); 4.5 MiB/s, 71 objects/s recovering 2024-08-17T18:09:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:09:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:09:43.209 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:43.247 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:43.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:43.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:43.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:42 smithi179 ceph-mon[93519]: pgmap v2398: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.7 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 661 op/s; 69901/328416 objects degraded (21.284%); 47127/328416 objects misplaced (14.350%); 4.5 MiB/s, 71 objects/s recovering 2024-08-17T18:09:43.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:09:43.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:09:43.348 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:43.388 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:43.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:43.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:43.490 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:43.552 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:43.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:43.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:43.654 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:43.675 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:43.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:43.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:43.777 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:43.799 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:43.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:43.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:43.900 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:43.922 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:43.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:43.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:44.024 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:44.047 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:44.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:44.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:44.056 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:09:44.056 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:09:44.149 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:44.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:44.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:44.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:44.314 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:44.356 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:44.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:44.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:44.458 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:44.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:44.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:44.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:44.582 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:44.619 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:44.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:44.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:44.676 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:09:44.721 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:44.743 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:44.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:44.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:44.844 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:44.891 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:44.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:44.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:44.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:44 smithi130 ceph-mon[105521]: pgmap v2399: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.6 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 816 op/s; 68416/322005 objects degraded (21.247%); 45503/322005 objects misplaced (14.131%); 5.6 MiB/s, 90 objects/s recovering 2024-08-17T18:09:44.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:44 smithi130 ceph-mon[110068]: pgmap v2399: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.6 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 816 op/s; 68416/322005 objects degraded (21.247%); 45503/322005 objects misplaced (14.131%); 5.6 MiB/s, 90 objects/s recovering 2024-08-17T18:09:44.993 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:45.015 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:45.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:45.116 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:45.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:45.158 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:45.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:45.259 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:45.296 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:45.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:45.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:45.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:44 smithi179 ceph-mon[93519]: pgmap v2399: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.6 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 816 op/s; 68416/322005 objects degraded (21.247%); 45503/322005 objects misplaced (14.131%); 5.6 MiB/s, 90 objects/s recovering 2024-08-17T18:09:45.398 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:45.420 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:45.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:45.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:45.522 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:45.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:45.566 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:45.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:45.667 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:45.690 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:45.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:45.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:45.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:45.813 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:45.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:45.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:45.915 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:45.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:45.958 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:45.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:46.059 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:46.081 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:46.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:46.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:46.183 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:46.204 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:46.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:46.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:46.305 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:46.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:46.346 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:46.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:46.447 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:46.469 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:46.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:46.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:46.570 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:46.591 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:46.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:46.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:46.693 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:46.714 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:46.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:46.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:46.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:46.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:46.861 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:46.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:46.962 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:46.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:46.985 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:46.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:47.086 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:47.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:47.132 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:47.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:47.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:46 smithi130 ceph-mon[105521]: pgmap v2400: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.4 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 832 op/s; 67219/317043 objects degraded (21.202%); 45282/317043 objects misplaced (14.283%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T18:09:47.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 67219/317043 objects degraded (21.202%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:47.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:46 smithi130 ceph-mon[110068]: pgmap v2400: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.4 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 832 op/s; 67219/317043 objects degraded (21.202%); 45282/317043 objects misplaced (14.283%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T18:09:47.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 67219/317043 objects degraded (21.202%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:47.233 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:47.255 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:47.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:47.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:47.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:46 smithi179 ceph-mon[93519]: pgmap v2400: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.4 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 832 op/s; 67219/317043 objects degraded (21.202%); 45282/317043 objects misplaced (14.283%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T18:09:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 67219/317043 objects degraded (21.202%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:47.356 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:47.378 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:47.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:47.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:47.480 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:47.503 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:47.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:47.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:47.605 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:47.628 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:47.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:47.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:47.730 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:47.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:47.770 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:47.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:47.871 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:47.885 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:09:47.894 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:47.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:47.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:47.996 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:48.039 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:47 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/438518989' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:09:48.039 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:47 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/438518989' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:09:48.040 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:48.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:48.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:48.141 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:48.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:48.192 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:48.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:48.293 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:48.315 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:48.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:48.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:48.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:47 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/438518989' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:09:48.417 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:48.462 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:48.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:48.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:48.564 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:48.587 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:48.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:48.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:48.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:48.710 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:48.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:48.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:48.812 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:48.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:48.835 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:48.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:48.936 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:48.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:48.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:48.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:48.995 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:48 smithi130 ceph-mon[105521]: pgmap v2401: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.4 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 674 op/s; 67219/317043 objects degraded (21.202%); 45282/317043 objects misplaced (14.283%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T18:09:48.995 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:48 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:09:48.996 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:48 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/438518989' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:09:48.996 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:48 smithi130 ceph-mon[105521]: osdmap e1346: 8 total, 8 up, 4 in 2024-08-17T18:09:48.996 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:48 smithi130 ceph-mon[110068]: pgmap v2401: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.4 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 674 op/s; 67219/317043 objects degraded (21.202%); 45282/317043 objects misplaced (14.283%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T18:09:48.996 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:48 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:09:48.996 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:48 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/438518989' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:09:48.996 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:48 smithi130 ceph-mon[110068]: osdmap e1346: 8 total, 8 up, 4 in 2024-08-17T18:09:49.060 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:49.081 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:49.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:49.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:49.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:49.234 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:49.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:49.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:49.336 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:49.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:48 smithi179 ceph-mon[93519]: pgmap v2401: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.4 GiB data, 15 GiB used, 342 GiB / 358 GiB avail; 0 B/s wr, 674 op/s; 67219/317043 objects degraded (21.202%); 45282/317043 objects misplaced (14.283%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T18:09:49.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:48 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:09:49.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:48 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/438518989' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:09:49.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:48 smithi179 ceph-mon[93519]: osdmap e1346: 8 total, 8 up, 4 in 2024-08-17T18:09:49.358 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:49.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:49.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:49.460 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:49.501 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:49.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:49.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:49.603 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:49.625 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:49.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:49.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:49.727 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:49.770 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:49.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:49.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:49.859 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:09:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:09:49] "GET /metrics HTTP/1.1" 200 36812 "" "Prometheus/2.51.0" 2024-08-17T18:09:49.872 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:49.893 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:49.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:49.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:49.995 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:50.018 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:50.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:50.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:50.120 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:50.167 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:50.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:50.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:50.269 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:50.309 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:50.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:50.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:50.410 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:50.463 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:50.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:50.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:50.536 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:09:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:09:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:09:50.565 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:50.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:50.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:50.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:50.690 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:50.713 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:50.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:50.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:50.815 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:50.839 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:50.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:50.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:50.940 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:50.983 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:50.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:50.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:51.085 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:51.122 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:51.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:51.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:51.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:50 smithi130 ceph-mon[105521]: pgmap v2403: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.3 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 780 op/s; 65225/308250 objects degraded (21.160%); 43444/308250 objects misplaced (14.094%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T18:09:51.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:50 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/144575642' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:51.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:50 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1925694847' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:09:51.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:50 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:09:51.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:50 smithi130 ceph-mon[110068]: pgmap v2403: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.3 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 780 op/s; 65225/308250 objects degraded (21.160%); 43444/308250 objects misplaced (14.094%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T18:09:51.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:50 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/144575642' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:51.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:50 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1925694847' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:09:51.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:50 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:09:51.223 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:51.246 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:51.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:51.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:51.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:50 smithi179 ceph-mon[93519]: pgmap v2403: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.3 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 780 op/s; 65225/308250 objects degraded (21.160%); 43444/308250 objects misplaced (14.094%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T18:09:51.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:50 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/144575642' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:09:51.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:50 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1925694847' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:09:51.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:50 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:09:51.348 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:51.389 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:51.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:51.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:51.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:51.514 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:51.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:51.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:51.615 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:51.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:51.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:51.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:51.757 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:51.779 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:51.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:51.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:51.860 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:09:51.881 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:51.956 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:51.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:51.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:52.058 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:52.122 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:52.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:52.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:52.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:51 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:09:52.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:51 smithi130 ceph-mon[105521]: osdmap e1347: 8 total, 8 up, 4 in 2024-08-17T18:09:52.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 65225/308250 objects degraded (21.160%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:52.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:51 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:09:52.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:51 smithi130 ceph-mon[110068]: osdmap e1347: 8 total, 8 up, 4 in 2024-08-17T18:09:52.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 65225/308250 objects degraded (21.160%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:52.224 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:52.246 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:52.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:52.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:52.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:51 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:09:52.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:51 smithi179 ceph-mon[93519]: osdmap e1347: 8 total, 8 up, 4 in 2024-08-17T18:09:52.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 65225/308250 objects degraded (21.160%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:52.347 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:52.387 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:52.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:52.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:52.489 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:52.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:52.511 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:52.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:52.612 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:52.635 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:52.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:52.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:52.737 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:52.790 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:52.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:52.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:52.892 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:52.945 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:52.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:52.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:53.049 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:53.085 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:53.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:53.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:53.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:52 smithi130 ceph-mon[110068]: pgmap v2405: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.2 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 704 op/s; 64465/305094 objects degraded (21.130%); 43444/305094 objects misplaced (14.240%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T18:09:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:52 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:09:53.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:52 smithi130 ceph-mon[105521]: pgmap v2405: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.2 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 704 op/s; 64465/305094 objects degraded (21.130%); 43444/305094 objects misplaced (14.240%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T18:09:53.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:52 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:09:53.187 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:53.210 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:53.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:53.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:53.273 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:52 smithi179 ceph-mon[93519]: pgmap v2405: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.2 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 704 op/s; 64465/305094 objects degraded (21.130%); 43444/305094 objects misplaced (14.240%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T18:09:53.273 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:52 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:09:53.282 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T18:09:53.313 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:53.336 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:53.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:53.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:53.438 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:53.472 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:53.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:53.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:53.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:53.598 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:53.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:53.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:53.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:53.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:53.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:53.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:53.823 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:53.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:53.847 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:53.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:53.948 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:53.991 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:53.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:53.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:54.093 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:54.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:54.114 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:54.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:54.215 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:54.237 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:54.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:54.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:54.338 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:54.364 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:54.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:54.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:54.466 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:54.488 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:54.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:54.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:54.590 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:54.632 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:54.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:54.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:54.733 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:54.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:54.758 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:54.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:54.836 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:09:54.836 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:09:54.836 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:09:54.836 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:09:54.836 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:09:54.836 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:09:54.836 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:09:54.837 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:09:54.837 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:09:54.837 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:09:54.837 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:09:54.837 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:09:54.837 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:09:54.837 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:09:54.837 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:09:54.837 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:09:54.859 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:54.884 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:54.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:54.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:54.985 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:55.021 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:55.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:55.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:55.122 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:55.144 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:55.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:55.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:55.148 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:54 smithi130 ceph-mon[105521]: pgmap v2406: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.1 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 779 op/s; 62937/298323 objects degraded (21.097%); 41772/298323 objects misplaced (14.002%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T18:09:55.148 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:54 smithi130 ceph-mon[110068]: pgmap v2406: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.1 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 779 op/s; 62937/298323 objects degraded (21.097%); 41772/298323 objects misplaced (14.002%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T18:09:55.246 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:55.269 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:55.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:55.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:55.292 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:54 smithi179 ceph-mon[93519]: pgmap v2406: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.1 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 779 op/s; 62937/298323 objects degraded (21.097%); 41772/298323 objects misplaced (14.002%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T18:09:55.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:55.393 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:55.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:55.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:55.494 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:55.536 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:55.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:55.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:55.638 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:55.680 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:55.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:55.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:55.781 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:55.819 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:55.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:55.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:55.921 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:55.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:55.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:55.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:56.045 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:56.067 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:56.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:56.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:56.169 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:56.192 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:56.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:56.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:56.294 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:56.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:56.334 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:56.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:56.435 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:56.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:56.457 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:56.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:56.558 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:56.586 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:56.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:56.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:56.691 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:56.702 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T18:09:56.712 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:56.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:56.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:56.786 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:09:56.786 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:09:56.814 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:56.852 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:56.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:56.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:56.954 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:57.024 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:57.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:57.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:57.093 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:56 smithi130 ceph-mon[105521]: pgmap v2407: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.0 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 986 op/s; 61745/293349 objects degraded (21.048%); 41543/293349 objects misplaced (14.162%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T18:09:57.093 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 61745/293349 objects degraded (21.048%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:57.093 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:56 smithi130 ceph-mon[110068]: pgmap v2407: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.0 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 986 op/s; 61745/293349 objects degraded (21.048%); 41543/293349 objects misplaced (14.162%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T18:09:57.093 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 61745/293349 objects degraded (21.048%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:57.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:57.186 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:57.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:57.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:57.289 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:57.310 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:57.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:57.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:57.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:56 smithi179 ceph-mon[93519]: pgmap v2407: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.0 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 986 op/s; 61745/293349 objects degraded (21.048%); 41543/293349 objects misplaced (14.162%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T18:09:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 61745/293349 objects degraded (21.048%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:09:57.413 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:57.436 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:57.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:57.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:57.538 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:57.548 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T18:09:57.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:57.587 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:57.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:57.599 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:09:57.688 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:57.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:57.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:57.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:57.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:57.872 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:09:57.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:57.886 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:57.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:09:57.987 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:58.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:58.036 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:58.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:58.137 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:58.159 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:58.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:58.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:58.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:09:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:57 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:09:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:57 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:09:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:09:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:57 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:09:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:57 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:09:58.261 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:58.312 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:58.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:58.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:09:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:57 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:09:58.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:57 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:09:58.414 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:58.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:58.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:58.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:58.539 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:58.561 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:58.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:58.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:58.663 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:58.700 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:58.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:58.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:58.802 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:09:58.831 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:58.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:58.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:09:58.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:09:58.955 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:58.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:58.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:09:59.056 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:59.079 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:59.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:59.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:59.181 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:09:59.203 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:59.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:59.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:09:59.305 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:09:59.326 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:59.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:59.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:09:59.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:09:58 smithi179 ceph-mon[93519]: pgmap v2408: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.0 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 781 op/s; 61745/293349 objects degraded (21.048%); 41543/293349 objects misplaced (14.162%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T18:09:59.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:09:58 smithi130 ceph-mon[105521]: pgmap v2408: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.0 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 781 op/s; 61745/293349 objects degraded (21.048%); 41543/293349 objects misplaced (14.162%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T18:09:59.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:09:58 smithi130 ceph-mon[110068]: pgmap v2408: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 6.0 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 0 B/s wr, 781 op/s; 61745/293349 objects degraded (21.048%); 41543/293349 objects misplaced (14.162%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T18:09:59.428 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:59.451 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:59.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:59.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:59.553 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:09:59.575 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:59.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:59.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:09:59.677 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:09:59.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:59.716 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:59.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:09:59.817 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:09:59.838 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:09:59.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:59.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:09:59.940 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:09:59.983 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:09:59.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:09:59.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:00.071 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:09:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:09:59] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-08-17T18:10:00.085 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:00.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:00.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:00.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:00.208 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:00.231 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:00.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:00.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:00.332 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:00.354 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:00.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:00.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:00.456 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:00.491 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:00.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:00.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:10:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:10:00.593 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:00.615 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:00.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:00.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:00.717 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:00.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:00.757 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:00.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:00.858 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:00.902 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:00.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:00.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:01.004 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:01.061 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:01.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:01.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:01.163 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:01.188 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:01.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:01.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:01.290 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:01.313 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:01.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:01.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: pgmap v2409: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.8 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 778 op/s; 59819/284877 objects degraded (20.998%); 39719/284877 objects misplaced (13.943%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T18:10:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 59819/284877 objects degraded (20.998%), 11 pgs degraded 2024-08-17T18:10:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-08-17T18:10:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: [WRN] PG_DEGRADED: Degraded data redundancy: 59819/284877 objects degraded (20.998%), 11 pgs degraded 2024-08-17T18:10:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: pg 48.2 is active+undersized+degraded+remapped+backfilling, acting [1,2] 2024-08-17T18:10:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: pg 48.3 is active+undersized+degraded+remapped+backfill_wait, acting [0,7] 2024-08-17T18:10:01.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: pg 48.4 is active+undersized+degraded+remapped+backfill_wait, acting [0,2] 2024-08-17T18:10:01.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: pg 48.5 is active+undersized+degraded+remapped+backfill_wait, acting [2,7] 2024-08-17T18:10:01.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: pg 48.6 is active+undersized+degraded+remapped, acting [1,2] 2024-08-17T18:10:01.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: pg 48.9 is active+undersized+degraded+remapped+backfill_wait, acting [2,7] 2024-08-17T18:10:01.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: pg 48.b is active+undersized+degraded+remapped+backfill_wait, acting [7,0] 2024-08-17T18:10:01.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: pg 48.c is active+undersized+degraded+remapped+backfill_wait, acting [7,1] 2024-08-17T18:10:01.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: pg 48.d is active+undersized+degraded+remapped+backfill_wait, acting [7,2] 2024-08-17T18:10:01.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: pg 48.e is active+undersized+degraded+remapped+backfill_wait, acting [2,7] 2024-08-17T18:10:01.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: pg 48.f is active+undersized+degraded+remapped+backfill_wait, acting [7,1] 2024-08-17T18:10:01.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:00 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 59819/284877 objects degraded (20.998%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:01.415 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: pgmap v2409: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.8 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 778 op/s; 59819/284877 objects degraded (20.998%); 39719/284877 objects misplaced (13.943%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T18:10:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 59819/284877 objects degraded (20.998%), 11 pgs degraded 2024-08-17T18:10:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-08-17T18:10:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: [WRN] PG_DEGRADED: Degraded data redundancy: 59819/284877 objects degraded (20.998%), 11 pgs degraded 2024-08-17T18:10:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: pg 48.2 is active+undersized+degraded+remapped+backfilling, acting [1,2] 2024-08-17T18:10:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: pg 48.3 is active+undersized+degraded+remapped+backfill_wait, acting [0,7] 2024-08-17T18:10:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: pg 48.4 is active+undersized+degraded+remapped+backfill_wait, acting [0,2] 2024-08-17T18:10:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: pg 48.5 is active+undersized+degraded+remapped+backfill_wait, acting [2,7] 2024-08-17T18:10:01.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: pg 48.6 is active+undersized+degraded+remapped, acting [1,2] 2024-08-17T18:10:01.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: pg 48.9 is active+undersized+degraded+remapped+backfill_wait, acting [2,7] 2024-08-17T18:10:01.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: pg 48.b is active+undersized+degraded+remapped+backfill_wait, acting [7,0] 2024-08-17T18:10:01.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: pg 48.c is active+undersized+degraded+remapped+backfill_wait, acting [7,1] 2024-08-17T18:10:01.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: pg 48.d is active+undersized+degraded+remapped+backfill_wait, acting [7,2] 2024-08-17T18:10:01.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: pg 48.e is active+undersized+degraded+remapped+backfill_wait, acting [2,7] 2024-08-17T18:10:01.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: pg 48.f is active+undersized+degraded+remapped+backfill_wait, acting [7,1] 2024-08-17T18:10:01.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 59819/284877 objects degraded (20.998%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:01.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: pgmap v2409: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.8 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 778 op/s; 59819/284877 objects degraded (20.998%); 39719/284877 objects misplaced (13.943%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T18:10:01.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 59819/284877 objects degraded (20.998%), 11 pgs degraded 2024-08-17T18:10:01.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2024-08-17T18:10:01.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: [WRN] PG_DEGRADED: Degraded data redundancy: 59819/284877 objects degraded (20.998%), 11 pgs degraded 2024-08-17T18:10:01.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: pg 48.2 is active+undersized+degraded+remapped+backfilling, acting [1,2] 2024-08-17T18:10:01.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: pg 48.3 is active+undersized+degraded+remapped+backfill_wait, acting [0,7] 2024-08-17T18:10:01.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: pg 48.4 is active+undersized+degraded+remapped+backfill_wait, acting [0,2] 2024-08-17T18:10:01.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: pg 48.5 is active+undersized+degraded+remapped+backfill_wait, acting [2,7] 2024-08-17T18:10:01.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: pg 48.6 is active+undersized+degraded+remapped, acting [1,2] 2024-08-17T18:10:01.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: pg 48.9 is active+undersized+degraded+remapped+backfill_wait, acting [2,7] 2024-08-17T18:10:01.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: pg 48.b is active+undersized+degraded+remapped+backfill_wait, acting [7,0] 2024-08-17T18:10:01.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: pg 48.c is active+undersized+degraded+remapped+backfill_wait, acting [7,1] 2024-08-17T18:10:01.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: pg 48.d is active+undersized+degraded+remapped+backfill_wait, acting [7,2] 2024-08-17T18:10:01.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: pg 48.e is active+undersized+degraded+remapped+backfill_wait, acting [2,7] 2024-08-17T18:10:01.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: pg 48.f is active+undersized+degraded+remapped+backfill_wait, acting [7,1] 2024-08-17T18:10:01.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:00 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 59819/284877 objects degraded (20.998%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:01.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:01.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:01.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:01.540 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:01.562 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:01.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:01.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:01.618 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:01.665 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:01.688 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:01.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:01.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:01.790 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:01.811 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:01.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:01.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:01.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:01.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:01.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:01.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:02.058 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:02.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:02.082 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:02.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:02.184 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:02.206 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:02.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:02.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:02.308 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:02.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:02.345 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:02.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:02.446 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:02.483 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:02.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:02.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:02.585 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:02.643 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:02.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:02.744 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:02.766 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:02.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:02.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:02.817 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:02.869 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:02.891 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:02.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:02.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:02.993 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:03.015 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:03.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:03.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:03.094 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:02 smithi130 ceph-mon[110068]: pgmap v2410: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.7 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 727 op/s; 59129/282132 objects degraded (20.958%); 39719/282132 objects misplaced (14.078%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T18:10:03.094 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:02 smithi130 ceph-mon[105521]: pgmap v2410: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.7 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 727 op/s; 59129/282132 objects degraded (20.958%); 39719/282132 objects misplaced (14.078%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T18:10:03.117 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:03.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:03.173 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:03.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:03.274 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:03.295 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:03.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:03.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:03.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:02 smithi179 ceph-mon[93519]: pgmap v2410: 89 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.7 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 727 op/s; 59129/282132 objects degraded (20.958%); 39719/282132 objects misplaced (14.078%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T18:10:03.397 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:03.419 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:03.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:03.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:03.521 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:03.566 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:03.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:03.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:03.667 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:03.699 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:03.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:03.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:03.801 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:03.840 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:03.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:03.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:03.942 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:03.965 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:03.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:03.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:03.984 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:10:04.067 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:04.089 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:04.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:04.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:04.190 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:04.212 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:04.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:04.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:04.314 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:04.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:03 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2460333477' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:10:04.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:03 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:10:04.352 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:04.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:04.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:04.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:03 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2460333477' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:10:04.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:03 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:10:04.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:03 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2460333477' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:10:04.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:03 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:10:04.454 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:04.475 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:04.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:04.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:04.576 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:04.599 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:04.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:04.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:04.701 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:04.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:04.744 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:04.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:04.845 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:04.866 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:04.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:04.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:04.968 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:04.990 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:04.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:04.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:05.092 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:05.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:05.133 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:05.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:05.234 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:05.275 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:05.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:05.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:05.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:04 smithi179 ceph-mon[93519]: pgmap v2411: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 814 op/s; 57685/275745 objects degraded (20.920%); 38146/275745 objects misplaced (13.834%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T18:10:05.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:04 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:10:05.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:04 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:10:05.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:04 smithi179 ceph-mon[93519]: osdmap e1348: 8 total, 8 up, 4 in 2024-08-17T18:10:05.377 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:05.379 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:05.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:04 smithi130 ceph-mon[105521]: pgmap v2411: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 814 op/s; 57685/275745 objects degraded (20.920%); 38146/275745 objects misplaced (13.834%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T18:10:05.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:04 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:10:05.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:04 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:10:05.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:04 smithi130 ceph-mon[105521]: osdmap e1348: 8 total, 8 up, 4 in 2024-08-17T18:10:05.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:04 smithi130 ceph-mon[110068]: pgmap v2411: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 814 op/s; 57685/275745 objects degraded (20.920%); 38146/275745 objects misplaced (13.834%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T18:10:05.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:04 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:10:05.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:04 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:10:05.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:04 smithi130 ceph-mon[110068]: osdmap e1348: 8 total, 8 up, 4 in 2024-08-17T18:10:05.435 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:05.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:05.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:05.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:05.558 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:05.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:05.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:05.660 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:05.682 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:05.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:05.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:05.784 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:05.804 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:05.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:05.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:05.906 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:05.962 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:05.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:05.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:06.064 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:06.086 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:06.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:06.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:06.114 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:05 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 57685/275745 objects degraded (20.920%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:06.118 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:05 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 57685/275745 objects degraded (20.920%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:06.188 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:06.242 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:06.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:06.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:06.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:05 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 57685/275745 objects degraded (20.920%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:06.344 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:06.379 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:06.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:06.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:06.484 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:06.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:06.520 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:06.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:06.621 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:06.657 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:06.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:06.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:06.759 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:06.793 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:06.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:06.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:06.895 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:06.934 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:06.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:06.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:07.036 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:07.059 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:07.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:07.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:07.160 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:07.183 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:07.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:07.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:07.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:07.326 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:07.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:07.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:07.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:07 smithi179 ceph-mon[93519]: pgmap v2413: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 760 op/s; 56460/270528 objects degraded (20.870%); 37879/270528 objects misplaced (14.002%); 3.1 MiB/s, 48 objects/s recovering 2024-08-17T18:10:07.399 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:07 smithi130 ceph-mon[105521]: pgmap v2413: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 760 op/s; 56460/270528 objects degraded (20.870%); 37879/270528 objects misplaced (14.002%); 3.1 MiB/s, 48 objects/s recovering 2024-08-17T18:10:07.399 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:07 smithi130 ceph-mon[110068]: pgmap v2413: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 760 op/s; 56460/270528 objects degraded (20.870%); 37879/270528 objects misplaced (14.002%); 3.1 MiB/s, 48 objects/s recovering 2024-08-17T18:10:07.428 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:07.449 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:07.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:07.551 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:07.571 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:07.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:07.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:07.673 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:07.706 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:07.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:07.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:07.808 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:07.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:07.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:07.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:07.933 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:07.955 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:07.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:07.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:08.057 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:08.079 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:08.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:08.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:08.180 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:08.219 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:08.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:08.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:08.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:08.363 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:08.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:08.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:08.465 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:08.486 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:08.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:08.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:08.588 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:08.632 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:08.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:08.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:08.734 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:08.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:08.792 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:08.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:08.893 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:08.943 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:08.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:08.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:09.045 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:09.067 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:09.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:09.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:09.169 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:09.229 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:09.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:09.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:09.280 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:09 smithi130 ceph-mon[105521]: pgmap v2414: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 760 op/s; 56460/270528 objects degraded (20.870%); 37879/270528 objects misplaced (14.002%); 3.1 MiB/s, 48 objects/s recovering 2024-08-17T18:10:09.280 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:09 smithi130 ceph-mon[110068]: pgmap v2414: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 760 op/s; 56460/270528 objects degraded (20.870%); 37879/270528 objects misplaced (14.002%); 3.1 MiB/s, 48 objects/s recovering 2024-08-17T18:10:09.323 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:10:09.323 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:10:09.330 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:09.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:09 smithi179 ceph-mon[93519]: pgmap v2414: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.5 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 760 op/s; 56460/270528 objects degraded (20.870%); 37879/270528 objects misplaced (14.002%); 3.1 MiB/s, 48 objects/s recovering 2024-08-17T18:10:09.352 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:09.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:09.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:09.454 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:09.494 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:09.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:09.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:09.596 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:09.633 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:09.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:09.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:09.735 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:09.756 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:09.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:09.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:09.858 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:09.905 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:09.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:09.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:10.007 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:10.043 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:10.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:10.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:10.145 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:10.157 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:10:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:10:09] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-08-17T18:10:10.182 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:10.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:10.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:10.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:10.330 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:10.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:10.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:10.432 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:10.489 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:10.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:10.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:10.591 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:10.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:10:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:10:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:10:10.628 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:10.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:10.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:10.730 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:10.772 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:10.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:10.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:10.873 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:10.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:10.918 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:10.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:11.018 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T18:10:11.020 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:11.026 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T18:10:11.041 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:11.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:11.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:11.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:10:11.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:11.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:11.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:11.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:11.314 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:11 smithi179 ceph-mon[93519]: pgmap v2415: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.3 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 759 op/s; 54456/262068 objects degraded (20.779%); 35974/262068 objects misplaced (13.727%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T18:10:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:11 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3978054980' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:10:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:11 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:10:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 54456/262068 objects degraded (20.779%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:11.406 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:11.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:11.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:11.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:11 smithi130 ceph-mon[105521]: pgmap v2415: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.3 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 759 op/s; 54456/262068 objects degraded (20.779%); 35974/262068 objects misplaced (13.727%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T18:10:11.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:11 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3978054980' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:10:11.421 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:11 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:10:11.421 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 54456/262068 objects degraded (20.779%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:11.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:11 smithi130 ceph-mon[110068]: pgmap v2415: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.3 GiB data, 13 GiB used, 344 GiB / 358 GiB avail; 0 B/s wr, 759 op/s; 54456/262068 objects degraded (20.779%); 35974/262068 objects misplaced (13.727%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T18:10:11.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:11 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3978054980' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:10:11.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:11 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:10:11.421 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 54456/262068 objects degraded (20.779%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:11.508 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:11.531 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:11.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:11.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:11.633 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:11.656 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:11.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:11.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:11.757 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:11.802 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:11.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:11.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:11.858 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T18:10:11.904 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:11.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:11.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:11.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:12.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:12 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:10:12.059 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:12 smithi130 ceph-mon[105521]: osdmap e1349: 8 total, 8 up, 4 in 2024-08-17T18:10:12.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:12 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:10:12.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:12 smithi130 ceph-mon[110068]: osdmap e1349: 8 total, 8 up, 4 in 2024-08-17T18:10:12.079 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:12.100 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:12.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:12.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:12.202 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:12.223 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:12.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:12.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:12.325 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:12.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:12 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:10:12.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:12 smithi179 ceph-mon[93519]: osdmap e1349: 8 total, 8 up, 4 in 2024-08-17T18:10:12.373 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:12.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:12.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:12.474 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:12.495 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:12.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:12.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:12.597 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:12.620 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:12.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:12.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:12.722 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:12.769 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:12.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:12.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:12.870 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:12.909 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:12.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:12.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:13.011 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:13.033 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:13.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:13.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:13.135 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:13.156 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:13.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:13.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:13.258 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:13.281 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:13.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:13.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:13 smithi179 ceph-mon[93519]: pgmap v2417: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 689 op/s; 53724/259185 objects degraded (20.728%); 35974/259185 objects misplaced (13.880%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T18:10:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:10:13.383 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:13.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:13 smithi130 ceph-mon[105521]: pgmap v2417: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 689 op/s; 53724/259185 objects degraded (20.728%); 35974/259185 objects misplaced (13.880%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T18:10:13.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:10:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:13 smithi130 ceph-mon[110068]: pgmap v2417: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 689 op/s; 53724/259185 objects degraded (20.728%); 35974/259185 objects misplaced (13.880%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T18:10:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:10:13.420 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:13.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:13.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:13.522 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:13.544 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:13.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:13.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:13.646 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:13.668 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:13.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:13.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:13.769 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:13.791 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:13.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:13.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:13.893 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:13.926 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:13.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:13.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:14.028 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:14.079 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:14.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:14.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:14.180 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:14.201 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:14.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:14.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:14.303 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:14.342 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:14.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:14.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:14.444 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:14.465 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:14.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:14.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:14.567 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:14.588 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:14.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:14.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:14.689 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:14.734 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:14.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:14.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:14.836 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:14.857 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:14.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:14.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:14.959 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:14.980 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:14.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:14.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:15.082 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:15.124 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:15.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:15.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:15.226 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:15.268 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:15.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:15.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:15.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:15 smithi179 ceph-mon[93519]: pgmap v2418: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 725 op/s; 52372/252948 objects degraded (20.705%); 34457/252948 objects misplaced (13.622%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T18:10:15.370 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:15.391 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:15.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:15.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:15.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:15 smithi130 ceph-mon[105521]: pgmap v2418: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 725 op/s; 52372/252948 objects degraded (20.705%); 34457/252948 objects misplaced (13.622%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T18:10:15.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:15 smithi130 ceph-mon[110068]: pgmap v2418: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 725 op/s; 52372/252948 objects degraded (20.705%); 34457/252948 objects misplaced (13.622%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T18:10:15.493 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:15.514 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:15.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:15.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:15.615 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:15.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:15.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:15.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:15.757 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:15.778 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:15.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:15.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:15.880 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:15.903 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:15.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:15.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:16.005 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:16.065 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:16.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:16.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:16.156 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:10:16.156 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:10:16.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:16.208 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:16.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:16.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:16.210 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:10:16.210 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:10:16.210 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 52372/252948 objects degraded (20.705%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:16.210 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 52372/252948 objects degraded (20.705%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:16.249 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:10:16.249 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:10:16.291 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:10:16.292 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:10:16.310 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:16.320 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:10:16.321 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:10:16.332 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:16.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:16.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:16.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 52372/252948 objects degraded (20.705%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:16.350 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:10:16.350 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:10:16.381 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:10:16.381 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:10:16.415 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:10:16.416 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:10:16.434 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:16.474 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:16.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:16.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:16.576 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:16.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:16.632 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:16.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:16.733 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:16.754 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:16.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:16.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:16.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:16.891 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:16.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:16.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:16.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:17.040 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:17.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:17.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:17.141 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:17.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:17 smithi130 ceph-mon[105521]: pgmap v2419: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 737 op/s; 51261/248385 objects degraded (20.638%); 34245/248385 objects misplaced (13.787%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T18:10:17.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:17 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1546216977' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:10:17.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:17 smithi130 ceph-mon[110068]: pgmap v2419: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 737 op/s; 51261/248385 objects degraded (20.638%); 34245/248385 objects misplaced (13.787%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T18:10:17.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:17 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1546216977' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:10:17.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:17.175 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:17.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:17.276 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:17.317 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:17.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:17.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:17.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:17 smithi179 ceph-mon[93519]: pgmap v2419: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 737 op/s; 51261/248385 objects degraded (20.638%); 34245/248385 objects misplaced (13.787%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T18:10:17.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:17 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1546216977' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:10:17.419 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:17.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:17.441 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:17.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:17.542 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:17.565 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:17.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:17.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:17.667 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:17.690 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:17.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:17.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:17.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:17.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:17.814 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:17.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:17.888 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:17.888 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:17.915 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:17.957 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:17.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:17.984 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:17.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:18.085 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:18.128 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:18.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:18.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:18.230 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:18.278 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:18.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:18.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:18.380 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:18.402 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:18.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:18.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:18.503 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:18.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:18.548 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:18.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:18.649 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:18.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:18.705 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:18.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:18.806 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:18.828 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:18.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:18.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:18.930 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:18.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:18.968 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:18.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:19.069 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:19.091 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:19.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:19.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:19.193 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:19.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:19.234 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:19.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:19.335 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:19.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:19 smithi179 ceph-mon[93519]: pgmap v2420: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 737 op/s; 51261/248385 objects degraded (20.638%); 34245/248385 objects misplaced (13.787%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T18:10:19.353 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:19 smithi130 ceph-mon[110068]: pgmap v2420: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 737 op/s; 51261/248385 objects degraded (20.638%); 34245/248385 objects misplaced (13.787%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T18:10:19.356 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:19 smithi130 ceph-mon[105521]: pgmap v2420: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 5.1 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 737 op/s; 51261/248385 objects degraded (20.638%); 34245/248385 objects misplaced (13.787%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T18:10:19.378 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:19.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:19.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:19.480 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:19.502 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:19.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:19.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:19.604 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:19.626 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:19.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:19.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:19.727 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:19.767 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:19.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:19.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:19.869 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:19.892 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:19.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:19.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:19.924 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:10:19.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:20.040 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:20.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:20.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:20.142 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:10:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:10:19] "GET /metrics HTTP/1.1" 200 36815 "" "Prometheus/2.51.0" 2024-08-17T18:10:20.194 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:20.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:20.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:20.297 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:20.322 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:20.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:20.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:20.424 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:20.445 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:20.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:20.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:20.543 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:10:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:10:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:10:20.547 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:20.571 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:20.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:20.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:20.672 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:20.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:20.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:20.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:20.798 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:20.839 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:20.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:20.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:20.941 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:20.977 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:20.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:20.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:21.079 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:21.102 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:21.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:21.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:21.204 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:21.244 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:21.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:21.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:21.345 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:21.386 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:21.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:21.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:21.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:21 smithi130 ceph-mon[105521]: pgmap v2421: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 701 op/s; 49642/241014 objects degraded (20.597%); 32610/241014 objects misplaced (13.530%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T18:10:21.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 49642/241014 objects degraded (20.597%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:21 smithi130 ceph-mon[110068]: pgmap v2421: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 701 op/s; 49642/241014 objects degraded (20.597%); 32610/241014 objects misplaced (13.530%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T18:10:21.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 49642/241014 objects degraded (20.597%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:21.488 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:21.509 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:21.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:21.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:21 smithi179 ceph-mon[93519]: pgmap v2421: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.9 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 701 op/s; 49642/241014 objects degraded (20.597%); 32610/241014 objects misplaced (13.530%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T18:10:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 49642/241014 objects degraded (20.597%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:21.611 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:21.646 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:21.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:21.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:21.747 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:21.786 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:21.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:21.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:21.888 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:21.911 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:21.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:21.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:22.013 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:22.052 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:22.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:22.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:22.155 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:22.217 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:22.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:22.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:22.318 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:22.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:22.364 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:22.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:22.464 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:22.486 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:22.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:22.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:22.588 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:22.634 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:22.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:22.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:22.736 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:22.759 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:22.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:22.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:22.861 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:22.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:22.883 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:22.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:22.984 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:23.006 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:23.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:23.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:23.108 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:23.152 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:23.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:23.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:23.253 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:23.276 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:23.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:23.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:23.377 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:23.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:23 smithi130 ceph-mon[105521]: pgmap v2422: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 664 op/s; 48997/238419 objects degraded (20.551%); 32610/238419 objects misplaced (13.678%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T18:10:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:23 smithi130 ceph-mon[110068]: pgmap v2422: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 664 op/s; 48997/238419 objects degraded (20.551%); 32610/238419 objects misplaced (13.678%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T18:10:23.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:23.424 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:23.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:23.525 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:23.577 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:23.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:23.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:23 smithi179 ceph-mon[93519]: pgmap v2422: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.8 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 664 op/s; 48997/238419 objects degraded (20.551%); 32610/238419 objects misplaced (13.678%); 2.8 MiB/s, 44 objects/s recovering 2024-08-17T18:10:23.679 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:23.704 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:23.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:23.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:23.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:23.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:23.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:23.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:23.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:23.950 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:23.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:23.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:24.052 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:24.073 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:24.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:24.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:24.175 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:24.184 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:10:24.184 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:10:24.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:10:24.184 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:10:24.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.184 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195096 46.9 46:9e985471:::benchmark_data_smithi130_267720_object87113:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-08-17T18:10:24.184 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.852302+0000", 2024-08-17T18:10:24.184 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 451.32728320699999, 2024-08-17T18:10:24.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.309431236, 2024-08-17T18:10:24.184 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.185 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.185 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:10:24.185 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.185 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:10:24.185 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:10:24.185 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195096 2024-08-17T18:10:24.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.185 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852302+0000", 2024-08-17T18:10:24.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852302+0000", 2024-08-17T18:10:24.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852303+0000", 2024-08-17T18:10:24.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.33e-06 2024-08-17T18:10:24.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852308+0000", 2024-08-17T18:10:24.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4079999999999997e-06 2024-08-17T18:10:24.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852319+0000", 2024-08-17T18:10:24.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0382e-05 2024-08-17T18:10:24.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852325+0000", 2024-08-17T18:10:24.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4649999999999999e-06 2024-08-17T18:10:24.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852449+0000", 2024-08-17T18:10:24.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012352000000000001 2024-08-17T18:10:24.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:10:24.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852579+0000", 2024-08-17T18:10:24.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013025800000000001 2024-08-17T18:10:24.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,6", 2024-08-17T18:10:24.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.852699+0000", 2024-08-17T18:10:24.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011966799999999999 2024-08-17T18:10:24.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.856209+0000", 2024-08-17T18:10:24.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0035104649999999999 2024-08-17T18:10:24.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:10:24.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873340+0000", 2024-08-17T18:10:24.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017130537000000001 2024-08-17T18:10:24.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.190 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.190 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934040+0000", 2024-08-17T18:10:24.190 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.060700089999999998 2024-08-17T18:10:24.190 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.190 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.190 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.190 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934049+0000", 2024-08-17T18:10:24.190 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1239999999999994e-06 2024-08-17T18:10:24.190 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.190 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.190 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.190 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877962+0000", 2024-08-17T18:10:24.191 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94391303400000004 2024-08-17T18:10:24.191 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.191 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.191 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.191 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877971+0000", 2024-08-17T18:10:24.191 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0100000000000001e-06 2024-08-17T18:10:24.191 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.191 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.191 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.191 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.161671+0000", 2024-08-17T18:10:24.191 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.283700372 2024-08-17T18:10:24.191 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.191 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.192 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.161733+0000", 2024-08-17T18:10:24.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1573000000000002e-05 2024-08-17T18:10:24.192 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.192 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.192 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.192 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195105 46.9 46:94a33cb3:::benchmark_data_smithi130_267720_object87122:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-08-17T18:10:24.192 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.862150+0000", 2024-08-17T18:10:24.192 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 451.31743523699998, 2024-08-17T18:10:24.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3538093619999998, 2024-08-17T18:10:24.192 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.193 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.193 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:10:24.193 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.193 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:10:24.193 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:10:24.193 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195105 2024-08-17T18:10:24.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.193 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.193 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.193 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862150+0000", 2024-08-17T18:10:24.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862150+0000", 2024-08-17T18:10:24.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.194 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862151+0000", 2024-08-17T18:10:24.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3170000000000001e-06 2024-08-17T18:10:24.194 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862157+0000", 2024-08-17T18:10:24.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5040000000000002e-06 2024-08-17T18:10:24.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.195 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862169+0000", 2024-08-17T18:10:24.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2520999999999999e-05 2024-08-17T18:10:24.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.195 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862176+0000", 2024-08-17T18:10:24.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1269999999999997e-06 2024-08-17T18:10:24.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.868137+0000", 2024-08-17T18:10:24.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0059607250000000001 2024-08-17T18:10:24.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:10:24.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.868274+0000", 2024-08-17T18:10:24.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013746200000000001 2024-08-17T18:10:24.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,6", 2024-08-17T18:10:24.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.868404+0000", 2024-08-17T18:10:24.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012964699999999999 2024-08-17T18:10:24.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:10:24.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.879188+0000", 2024-08-17T18:10:24.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010783915 2024-08-17T18:10:24.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934076+0000", 2024-08-17T18:10:24.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.054887752999999997 2024-08-17T18:10:24.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934083+0000", 2024-08-17T18:10:24.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9859999999999997e-06 2024-08-17T18:10:24.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.198 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.198 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.198 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878000+0000", 2024-08-17T18:10:24.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94391712299999997 2024-08-17T18:10:24.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.198 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.198 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.198 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878009+0000", 2024-08-17T18:10:24.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.884e-06 2024-08-17T18:10:24.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.198 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.198 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.199 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.161985+0000", 2024-08-17T18:10:24.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2839767369999999 2024-08-17T18:10:24.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.199 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:10:24.199 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.162438+0000", 2024-08-17T18:10:24.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00045227700000000001 2024-08-17T18:10:24.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.199 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.199 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215917+0000", 2024-08-17T18:10:24.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.053479086000000002 2024-08-17T18:10:24.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.199 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215959+0000", 2024-08-17T18:10:24.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2298000000000001e-05 2024-08-17T18:10:24.200 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.200 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.200 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.200 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195107 46.9 46:9aacae76:::benchmark_data_smithi130_267720_object87124:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-08-17T18:10:24.200 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.862322+0000", 2024-08-17T18:10:24.200 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 451.31726294999999, 2024-08-17T18:10:24.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3730563999999998, 2024-08-17T18:10:24.200 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.200 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.201 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:10:24.201 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.201 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:10:24.201 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:10:24.201 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195107 2024-08-17T18:10:24.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.201 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.201 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.201 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862322+0000", 2024-08-17T18:10:24.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.202 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.202 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862322+0000", 2024-08-17T18:10:24.202 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.202 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.202 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.202 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.202 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862323+0000", 2024-08-17T18:10:24.202 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3200000000000005e-07 2024-08-17T18:10:24.202 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.202 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.202 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.202 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862326+0000", 2024-08-17T18:10:24.203 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3249999999999999e-06 2024-08-17T18:10:24.203 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.203 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.203 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.203 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862335+0000", 2024-08-17T18:10:24.203 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0149999999999992e-06 2024-08-17T18:10:24.203 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.203 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.203 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.203 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.862341+0000", 2024-08-17T18:10:24.203 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9909999999999999e-06 2024-08-17T18:10:24.203 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.203 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873392+0000", 2024-08-17T18:10:24.204 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011050658 2024-08-17T18:10:24.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:10:24.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873518+0000", 2024-08-17T18:10:24.204 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012659100000000001 2024-08-17T18:10:24.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,6", 2024-08-17T18:10:24.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873640+0000", 2024-08-17T18:10:24.204 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012183199999999999 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934107+0000", 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.060466796000000003 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934114+0000", 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0690000000000004e-06 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878038+0000", 2024-08-17T18:10:24.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94392365499999997 2024-08-17T18:10:24.206 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.206 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.206 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878047+0000", 2024-08-17T18:10:24.206 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9600000000000006e-06 2024-08-17T18:10:24.206 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.206 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.206 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215977+0000", 2024-08-17T18:10:24.206 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3379302630000001 2024-08-17T18:10:24.206 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.207 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:10:24.207 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216122+0000", 2024-08-17T18:10:24.207 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014515900000000001 2024-08-17T18:10:24.207 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.207 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.207 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.207 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.235324+0000", 2024-08-17T18:10:24.207 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019201732999999999 2024-08-17T18:10:24.207 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.207 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.207 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.207 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.235378+0000", 2024-08-17T18:10:24.207 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4721e-05 2024-08-17T18:10:24.208 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.208 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.208 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.208 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.208 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.208 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195109 46.9 46:941627c3:::benchmark_data_smithi130_267720_object87126:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-08-17T18:10:24.208 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.870700+0000", 2024-08-17T18:10:24.208 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 451.30888517599999, 2024-08-17T18:10:24.208 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3651783769999999, 2024-08-17T18:10:24.208 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.208 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.208 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:10:24.208 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.208 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:10:24.209 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:10:24.209 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195109 2024-08-17T18:10:24.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.209 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870700+0000", 2024-08-17T18:10:24.209 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870700+0000", 2024-08-17T18:10:24.209 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.210 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.210 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.210 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870701+0000", 2024-08-17T18:10:24.210 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.415e-06 2024-08-17T18:10:24.210 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.210 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.210 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870704+0000", 2024-08-17T18:10:24.210 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8899999999999999e-06 2024-08-17T18:10:24.210 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.210 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.211 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870712+0000", 2024-08-17T18:10:24.211 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2579999999999995e-06 2024-08-17T18:10:24.211 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.211 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.211 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.211 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.870718+0000", 2024-08-17T18:10:24.211 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.327e-06 2024-08-17T18:10:24.211 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.211 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.211 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.211 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.879223+0000", 2024-08-17T18:10:24.211 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0085048620000000002 2024-08-17T18:10:24.211 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.211 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.212 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:10:24.212 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.879355+0000", 2024-08-17T18:10:24.212 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013201599999999999 2024-08-17T18:10:24.212 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.212 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.212 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,6", 2024-08-17T18:10:24.212 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.879499+0000", 2024-08-17T18:10:24.212 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014413800000000001 2024-08-17T18:10:24.212 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.212 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.212 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.212 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934138+0000", 2024-08-17T18:10:24.212 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.05463925 2024-08-17T18:10:24.213 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.213 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.213 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.213 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934145+0000", 2024-08-17T18:10:24.214 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7399999999999998e-06 2024-08-17T18:10:24.214 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.214 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.214 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.214 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878105+0000", 2024-08-17T18:10:24.214 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94396076500000003 2024-08-17T18:10:24.214 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.214 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.214 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.214 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878115+0000", 2024-08-17T18:10:24.214 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3489999999999998e-06 2024-08-17T18:10:24.214 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.214 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.215 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.215 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216148+0000", 2024-08-17T18:10:24.215 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3380333520000001 2024-08-17T18:10:24.215 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.215 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.215 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:10:24.215 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216284+0000", 2024-08-17T18:10:24.215 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000135513 2024-08-17T18:10:24.215 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.215 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.215 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.215 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.235773+0000", 2024-08-17T18:10:24.215 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019489216 2024-08-17T18:10:24.215 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.216 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.216 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.216 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.235878+0000", 2024-08-17T18:10:24.216 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010528599999999999 2024-08-17T18:10:24.216 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.216 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.216 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.216 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.216 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.216 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195099 46.f 46:fad5c3c4:::benchmark_data_smithi130_267720_object87116:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:10:24.216 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.873814+0000", 2024-08-17T18:10:24.216 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 451.305771381, 2024-08-17T18:10:24.216 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.301145886, 2024-08-17T18:10:24.217 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.217 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.217 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:10:24.217 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.217 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:10:24.217 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:10:24.217 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195099 2024-08-17T18:10:24.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.217 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.217 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.217 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.217 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873814+0000", 2024-08-17T18:10:24.217 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.218 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.218 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.218 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873814+0000", 2024-08-17T18:10:24.218 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.218 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.218 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.218 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873816+0000", 2024-08-17T18:10:24.218 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9769999999999999e-06 2024-08-17T18:10:24.218 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.218 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.219 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873823+0000", 2024-08-17T18:10:24.221 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2590000000000003e-06 2024-08-17T18:10:24.221 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873834+0000", 2024-08-17T18:10:24.222 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1243000000000001e-05 2024-08-17T18:10:24.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873843+0000", 2024-08-17T18:10:24.222 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6079999999999996e-06 2024-08-17T18:10:24.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878460+0000", 2024-08-17T18:10:24.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046169899999999996 2024-08-17T18:10:24.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op must wait for map", 2024-08-17T18:10:24.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878467+0000", 2024-08-17T18:10:24.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9399999999999996e-06 2024-08-17T18:10:24.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.932773+0000", 2024-08-17T18:10:24.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.054306074000000003 2024-08-17T18:10:24.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.932783+0000", 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0214000000000001e-05 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877857+0000", 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94507391299999999 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877867+0000", 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0132000000000001e-05 2024-08-17T18:10:24.224 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.225 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.225 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.225 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.172642+0000", 2024-08-17T18:10:24.225 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2947751140000001 2024-08-17T18:10:24.225 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.225 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.225 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.225 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.174959+0000", 2024-08-17T18:10:24.225 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002317422 2024-08-17T18:10:24.225 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.225 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.225 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.225 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.226 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.226 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195108 46.f 46:f8e03b9e:::benchmark_data_smithi130_267720_object87125:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:10:24.226 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.873857+0000", 2024-08-17T18:10:24.226 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 451.30572836200002, 2024-08-17T18:10:24.226 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3014579089999998, 2024-08-17T18:10:24.226 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.226 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.226 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:10:24.226 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.226 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:10:24.226 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:10:24.226 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195108 2024-08-17T18:10:24.226 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.227 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.227 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.227 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873857+0000", 2024-08-17T18:10:24.227 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.227 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.227 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.227 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873857+0000", 2024-08-17T18:10:24.227 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.227 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.227 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.228 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873857+0000", 2024-08-17T18:10:24.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8599999999999998e-07 2024-08-17T18:10:24.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.228 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.228 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.228 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873859+0000", 2024-08-17T18:10:24.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2819999999999999e-06 2024-08-17T18:10:24.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.228 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.228 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.228 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873865+0000", 2024-08-17T18:10:24.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1310000000000003e-06 2024-08-17T18:10:24.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873870+0000", 2024-08-17T18:10:24.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5770000000000004e-06 2024-08-17T18:10:24.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878516+0000", 2024-08-17T18:10:24.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046464200000000001 2024-08-17T18:10:24.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting_for_map not empty", 2024-08-17T18:10:24.230 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878522+0000", 2024-08-17T18:10:24.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.259e-06 2024-08-17T18:10:24.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.230 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.230 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.932812+0000", 2024-08-17T18:10:24.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.054289865 2024-08-17T18:10:24.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.230 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.230 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.932818+0000", 2024-08-17T18:10:24.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7259999999999996e-06 2024-08-17T18:10:24.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877927+0000", 2024-08-17T18:10:24.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94510819099999999 2024-08-17T18:10:24.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877935+0000", 2024-08-17T18:10:24.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5280000000000003e-06 2024-08-17T18:10:24.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.232 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.175240+0000", 2024-08-17T18:10:24.232 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.297304512 2024-08-17T18:10:24.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.232 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.232 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.175314+0000", 2024-08-17T18:10:24.232 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4931999999999993e-05 2024-08-17T18:10:24.232 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.232 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.232 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.232 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195110 46.1 46:81ba4812:::benchmark_data_smithi130_267720_object87127:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:10:24.233 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.873876+0000", 2024-08-17T18:10:24.233 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 451.30570887900001, 2024-08-17T18:10:24.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.503486423, 2024-08-17T18:10:24.233 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.233 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.233 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:10:24.233 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.233 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:10:24.233 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:10:24.233 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195110 2024-08-17T18:10:24.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.233 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.233 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873876+0000", 2024-08-17T18:10:24.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873876+0000", 2024-08-17T18:10:24.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873876+0000", 2024-08-17T18:10:24.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.84e-07 2024-08-17T18:10:24.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.235 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.235 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.235 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.873878+0000", 2024-08-17T18:10:24.235 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9410000000000001e-06 2024-08-17T18:10:24.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.235 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.235 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.235 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.874329+0000", 2024-08-17T18:10:24.235 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00045083200000000002 2024-08-17T18:10:24.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.235 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.235 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.235 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.874337+0000", 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2009999999999999e-06 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934172+0000", 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.059834948999999998 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934179+0000", 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0949999999999998e-06 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877887+0000", 2024-08-17T18:10:24.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94370757699999996 2024-08-17T18:10:24.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877897+0000", 2024-08-17T18:10:24.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6600000000000007e-06 2024-08-17T18:10:24.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.204539+0000", 2024-08-17T18:10:24.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.32664223 2024-08-17T18:10:24.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:10:24.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.206374+0000", 2024-08-17T18:10:24.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001834638 2024-08-17T18:10:24.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,6,7", 2024-08-17T18:10:24.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.206574+0000", 2024-08-17T18:10:24.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020093300000000001 2024-08-17T18:10:24.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.218471+0000", 2024-08-17T18:10:24.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011896487000000001 2024-08-17T18:10:24.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:10:24.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.218487+0000", 2024-08-17T18:10:24.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6118999999999999e-05 2024-08-17T18:10:24.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.232569+0000", 2024-08-17T18:10:24.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014081425 2024-08-17T18:10:24.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.376529+0000", 2024-08-17T18:10:24.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14396075699999999 2024-08-17T18:10:24.240 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.240 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.240 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:10:24.240 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.376663+0000", 2024-08-17T18:10:24.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013359900000000001 2024-08-17T18:10:24.240 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.240 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.240 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.240 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377362+0000", 2024-08-17T18:10:24.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00069959600000000003 2024-08-17T18:10:24.240 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.240 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.241 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.241 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.241 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.241 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195111 46.9 46:9346673d:::benchmark_data_smithi130_267720_object87128:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:10:24.241 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.878090+0000", 2024-08-17T18:10:24.241 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 451.30149494599999, 2024-08-17T18:10:24.241 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4994862809999998, 2024-08-17T18:10:24.241 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.241 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.241 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:10:24.241 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.241 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:10:24.241 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:10:24.241 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195111 2024-08-17T18:10:24.242 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.242 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.242 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878090+0000", 2024-08-17T18:10:24.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.242 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.242 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878090+0000", 2024-08-17T18:10:24.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.242 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.242 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878092+0000", 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.548e-06 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878097+0000", 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.665e-06 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878115+0000", 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7829000000000002e-05 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.878123+0000", 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8029999999999997e-06 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934199+0000", 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.056076150999999998 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.934205+0000", 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3559999999999998e-06 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878140+0000", 2024-08-17T18:10:24.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.943934355 2024-08-17T18:10:24.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.878148+0000", 2024-08-17T18:10:24.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8399999999999995e-06 2024-08-17T18:10:24.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216331+0000", 2024-08-17T18:10:24.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3381836009999999 2024-08-17T18:10:24.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.246 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:10:24.246 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216449+0000", 2024-08-17T18:10:24.246 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000117873 2024-08-17T18:10:24.246 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.246 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,3,6", 2024-08-17T18:10:24.246 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216861+0000", 2024-08-17T18:10:24.246 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00041161799999999999 2024-08-17T18:10:24.246 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.246 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:10:24.246 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.229816+0000", 2024-08-17T18:10:24.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012955112 2024-08-17T18:10:24.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.232815+0000", 2024-08-17T18:10:24.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029990630000000002 2024-08-17T18:10:24.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.234938+0000", 2024-08-17T18:10:24.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002122779 2024-08-17T18:10:24.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.248 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377412+0000", 2024-08-17T18:10:24.248 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14247422800000001 2024-08-17T18:10:24.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.248 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:10:24.248 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377471+0000", 2024-08-17T18:10:24.248 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9032e-05 2024-08-17T18:10:24.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.248 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.248 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377576+0000", 2024-08-17T18:10:24.248 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010542799999999999 2024-08-17T18:10:24.249 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.249 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.249 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.249 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.249 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.249 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195121 46.1 46:8cbb5caf:::benchmark_data_smithi130_267720_object87138:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:10:24.249 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.946248+0000", 2024-08-17T18:10:24.249 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 451.23333673000002, 2024-08-17T18:10:24.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4315148899999999, 2024-08-17T18:10:24.249 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.249 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.249 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:10:24.249 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.249 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:10:24.250 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:10:24.250 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195121 2024-08-17T18:10:24.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.250 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.250 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.250 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.250 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946248+0000", 2024-08-17T18:10:24.250 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.250 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.250 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.250 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946248+0000", 2024-08-17T18:10:24.250 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946250+0000", 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4649999999999999e-06 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946256+0000", 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8839999999999999e-06 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946270+0000", 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3993e-05 2024-08-17T18:10:24.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946276+0000", 2024-08-17T18:10:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2400000000000004e-06 2024-08-17T18:10:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946334+0000", 2024-08-17T18:10:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8402000000000001e-05 2024-08-17T18:10:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.946345+0000", 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0645999999999999e-05 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877922+0000", 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.93157699800000004 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.877930+0000", 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7079999999999997e-06 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215774+0000", 2024-08-17T18:10:24.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.337844566 2024-08-17T18:10:24.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:10:24.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.215906+0000", 2024-08-17T18:10:24.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013154400000000001 2024-08-17T18:10:24.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,6,7", 2024-08-17T18:10:24.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.216043+0000", 2024-08-17T18:10:24.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000137288 2024-08-17T18:10:24.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.218520+0000", 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002477492 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.229810+0000", 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011289123999999999 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.234501+0000", 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046915749999999999 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377616+0000", 2024-08-17T18:10:24.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14311495099999999 2024-08-17T18:10:24.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:10:24.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377664+0000", 2024-08-17T18:10:24.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8047999999999998e-05 2024-08-17T18:10:24.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.377763+0000", 2024-08-17T18:10:24.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8966000000000001e-05 2024-08-17T18:10:24.256 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.256 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.257 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.257 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36225.0:195124 46.2 46:497fbb11:::benchmark_data_smithi130_267720_object87141:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1283)", 2024-08-17T18:10:24.257 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:02:52.956154+0000", 2024-08-17T18:10:24.257 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 451.22343088000002, 2024-08-17T18:10:24.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2343982279999999, 2024-08-17T18:10:24.257 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.257 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.257 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:10:24.257 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.257 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36225", 2024-08-17T18:10:24.257 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1891308598", 2024-08-17T18:10:24.257 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 195124 2024-08-17T18:10:24.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.258 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956154+0000", 2024-08-17T18:10:24.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956154+0000", 2024-08-17T18:10:24.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:24.260 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:24.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:24.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956155+0000", 2024-08-17T18:10:24.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1119999999999999e-06 2024-08-17T18:10:24.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956159+0000", 2024-08-17T18:10:24.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2679999999999996e-06 2024-08-17T18:10:24.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956170+0000", 2024-08-17T18:10:24.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0334e-05 2024-08-17T18:10:24.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956177+0000", 2024-08-17T18:10:24.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2849999999999997e-06 2024-08-17T18:10:24.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956244+0000", 2024-08-17T18:10:24.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6994000000000003e-05 2024-08-17T18:10:24.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:52.956257+0000", 2024-08-17T18:10:24.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3196e-05 2024-08-17T18:10:24.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.876745+0000", 2024-08-17T18:10:24.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.92048745399999998 2024-08-17T18:10:24.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.264 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:53.876755+0000", 2024-08-17T18:10:24.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0001e-05 2024-08-17T18:10:24.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.264 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.169422+0000", 2024-08-17T18:10:24.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.292667488 2024-08-17T18:10:24.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:10:24.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.169523+0000", 2024-08-17T18:10:24.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000101116 2024-08-17T18:10:24.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6,7", 2024-08-17T18:10:24.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.169908+0000", 2024-08-17T18:10:24.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000384936 2024-08-17T18:10:24.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.182867+0000", 2024-08-17T18:10:24.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012958374 2024-08-17T18:10:24.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:10:24.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.189357+0000", 2024-08-17T18:10:24.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0064902980000000002 2024-08-17T18:10:24.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.189592+0000", 2024-08-17T18:10:24.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00023549000000000001 2024-08-17T18:10:24.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.190415+0000", 2024-08-17T18:10:24.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00082231100000000005 2024-08-17T18:10:24.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:10:24.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.190452+0000", 2024-08-17T18:10:24.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7054999999999997e-05 2024-08-17T18:10:24.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:02:55.190552+0000", 2024-08-17T18:10:24.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000100516 2024-08-17T18:10:24.267 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.268 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.268 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.268 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:115193 47.1 47:8a1837dc:::benchmark_data_smithi130_302447_object19958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:10:24.268 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:56.586429+0000", 2024-08-17T18:10:24.268 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 267.59315608999998, 2024-08-17T18:10:24.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0156209679999999, 2024-08-17T18:10:24.268 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.268 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.268 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:10:24.268 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.268 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:10:24.269 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:10:24.269 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115193 2024-08-17T18:10:24.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.269 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586429+0000", 2024-08-17T18:10:24.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586429+0000", 2024-08-17T18:10:24.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586430+0000", 2024-08-17T18:10:24.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.364e-06 2024-08-17T18:10:24.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586435+0000", 2024-08-17T18:10:24.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3900000000000003e-06 2024-08-17T18:10:24.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586450+0000", 2024-08-17T18:10:24.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5209e-05 2024-08-17T18:10:24.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586457+0000", 2024-08-17T18:10:24.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0099999999999998e-06 2024-08-17T18:10:24.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586575+0000", 2024-08-17T18:10:24.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000118305 2024-08-17T18:10:24.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:10:24.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586714+0000", 2024-08-17T18:10:24.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013882799999999999 2024-08-17T18:10:24.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:10:24.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586850+0000", 2024-08-17T18:10:24.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000136198 2024-08-17T18:10:24.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.594476+0000", 2024-08-17T18:10:24.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0076255070000000001 2024-08-17T18:10:24.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:10:24.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.594483+0000", 2024-08-17T18:10:24.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5549999999999997e-06 2024-08-17T18:10:24.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.081391+0000", 2024-08-17T18:10:24.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.48690757499999998 2024-08-17T18:10:24.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.274 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.081400+0000", 2024-08-17T18:10:24.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4089999999999992e-06 2024-08-17T18:10:24.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.274 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.274 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.601973+0000", 2024-08-17T18:10:24.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.52057316499999995 2024-08-17T18:10:24.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.274 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.274 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.602050+0000", 2024-08-17T18:10:24.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6452999999999995e-05 2024-08-17T18:10:24.274 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.275 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.275 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.275 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:115174 47.4 47:2d651f61:::benchmark_data_smithi130_302447_object19939:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1317)", 2024-08-17T18:10:24.275 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:57.071033+0000", 2024-08-17T18:10:24.275 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 267.10855192999998, 2024-08-17T18:10:24.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.540335236, 2024-08-17T18:10:24.275 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.275 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.275 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:10:24.275 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.276 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:10:24.276 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:10:24.276 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115174 2024-08-17T18:10:24.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.276 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071033+0000", 2024-08-17T18:10:24.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071033+0000", 2024-08-17T18:10:24.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071035+0000", 2024-08-17T18:10:24.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.666e-06 2024-08-17T18:10:24.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071041+0000", 2024-08-17T18:10:24.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3199999999999996e-06 2024-08-17T18:10:24.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071076+0000", 2024-08-17T18:10:24.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5286000000000001e-05 2024-08-17T18:10:24.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071087+0000", 2024-08-17T18:10:24.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0327e-05 2024-08-17T18:10:24.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071159+0000", 2024-08-17T18:10:24.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2201999999999998e-05 2024-08-17T18:10:24.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op must wait for map", 2024-08-17T18:10:24.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071167+0000", 2024-08-17T18:10:24.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1329999999999999e-06 2024-08-17T18:10:24.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.075440+0000", 2024-08-17T18:10:24.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0042733340000000002 2024-08-17T18:10:24.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.075449+0000", 2024-08-17T18:10:24.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3699999999999995e-06 2024-08-17T18:10:24.280 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.280 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.280 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.280 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.605615+0000", 2024-08-17T18:10:24.280 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.53016654399999996 2024-08-17T18:10:24.280 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.280 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.280 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.280 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.611368+0000", 2024-08-17T18:10:24.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0057530539999999996 2024-08-17T18:10:24.281 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.281 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.281 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.281 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115437 48.8 48:1823c229:::benchmark_data_smithi130_327597_object115436:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:10:24.281 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.095352+0000", 2024-08-17T18:10:24.281 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.084233163999997, 2024-08-17T18:10:24.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5254109470000001, 2024-08-17T18:10:24.281 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.281 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.281 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:10:24.282 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.282 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:10:24.282 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:10:24.282 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115437 2024-08-17T18:10:24.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.282 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.282 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.282 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095352+0000", 2024-08-17T18:10:24.282 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095352+0000", 2024-08-17T18:10:24.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095352+0000", 2024-08-17T18:10:24.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2199999999999999e-07 2024-08-17T18:10:24.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095377+0000", 2024-08-17T18:10:24.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4745000000000001e-05 2024-08-17T18:10:24.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095385+0000", 2024-08-17T18:10:24.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6199999999999999e-06 2024-08-17T18:10:24.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095390+0000", 2024-08-17T18:10:24.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5860000000000004e-06 2024-08-17T18:10:24.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.100178+0000", 2024-08-17T18:10:24.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0047875629999999999 2024-08-17T18:10:24.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.100189+0000", 2024-08-17T18:10:24.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1124999999999999e-05 2024-08-17T18:10:24.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.564942+0000", 2024-08-17T18:10:24.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.46475288100000001 2024-08-17T18:10:24.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.564952+0000", 2024-08-17T18:10:24.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9809999999999998e-06 2024-08-17T18:10:24.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.620659+0000", 2024-08-17T18:10:24.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.05570713 2024-08-17T18:10:24.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.620763+0000", 2024-08-17T18:10:24.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010389400000000001 2024-08-17T18:10:24.287 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.287 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.287 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.287 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115441 48.2 48.c80b4ff2 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:10:24.287 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.095398+0000", 2024-08-17T18:10:24.287 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.084186778000003, 2024-08-17T18:10:24.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.46818560999999997, 2024-08-17T18:10:24.287 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.287 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.287 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:10:24.287 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.287 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:10:24.288 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:10:24.288 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115441 2024-08-17T18:10:24.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.288 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.288 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.288 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095398+0000", 2024-08-17T18:10:24.288 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.288 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.288 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095398+0000", 2024-08-17T18:10:24.288 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.289 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.289 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.289 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095399+0000", 2024-08-17T18:10:24.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7800000000000002e-07 2024-08-17T18:10:24.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.289 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.289 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.289 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095447+0000", 2024-08-17T18:10:24.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7991999999999997e-05 2024-08-17T18:10:24.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.289 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.289 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.289 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095454+0000", 2024-08-17T18:10:24.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3329999999999999e-06 2024-08-17T18:10:24.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095459+0000", 2024-08-17T18:10:24.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2000000000000002e-06 2024-08-17T18:10:24.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095530+0000", 2024-08-17T18:10:24.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0770000000000002e-05 2024-08-17T18:10:24.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095536+0000", 2024-08-17T18:10:24.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3890000000000002e-06 2024-08-17T18:10:24.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.563569+0000", 2024-08-17T18:10:24.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.46803287999999998 2024-08-17T18:10:24.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.563584+0000", 2024-08-17T18:10:24.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4568e-05 2024-08-17T18:10:24.291 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.292 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.292 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.292 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.292 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.292 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115442 48.8 48:189c3bef:::benchmark_data_smithi130_327597_object115441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:10:24.292 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.095476+0000", 2024-08-17T18:10:24.292 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.084109401999996, 2024-08-17T18:10:24.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5581986209999998, 2024-08-17T18:10:24.292 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.292 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.292 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:10:24.292 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.292 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115442 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095476+0000", 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095476+0000", 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.294 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.294 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095476+0000", 2024-08-17T18:10:24.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2300000000000002e-07 2024-08-17T18:10:24.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.294 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.294 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095499+0000", 2024-08-17T18:10:24.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2728e-05 2024-08-17T18:10:24.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.294 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.294 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095506+0000", 2024-08-17T18:10:24.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8770000000000004e-06 2024-08-17T18:10:24.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095511+0000", 2024-08-17T18:10:24.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0549999999999999e-06 2024-08-17T18:10:24.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.100585+0000", 2024-08-17T18:10:24.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0050741049999999998 2024-08-17T18:10:24.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.100593+0000", 2024-08-17T18:10:24.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5150000000000006e-06 2024-08-17T18:10:24.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.564972+0000", 2024-08-17T18:10:24.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.46437926800000001 2024-08-17T18:10:24.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.564979+0000", 2024-08-17T18:10:24.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1090000000000003e-06 2024-08-17T18:10:24.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.297 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.621508+0000", 2024-08-17T18:10:24.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0565289579999999 2024-08-17T18:10:24.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.297 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:10:24.297 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.621663+0000", 2024-08-17T18:10:24.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000155415 2024-08-17T18:10:24.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.297 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.297 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641744+0000", 2024-08-17T18:10:24.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020081096 2024-08-17T18:10:24.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T18:10:24.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641815+0000", 2024-08-17T18:10:24.298 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1219999999999999e-05 2024-08-17T18:10:24.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.653582+0000", 2024-08-17T18:10:24.298 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011766835 2024-08-17T18:10:24.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.299 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.653674+0000", 2024-08-17T18:10:24.299 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2016999999999997e-05 2024-08-17T18:10:24.299 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.299 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.299 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.299 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.299 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.299 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115444 48.2 48.295e5ec2 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:10:24.299 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.095517+0000", 2024-08-17T18:10:24.299 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.084067989999994, 2024-08-17T18:10:24.299 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.46849580200000002, 2024-08-17T18:10:24.299 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.299 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.299 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:10:24.300 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.300 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:10:24.300 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:10:24.300 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115444 2024-08-17T18:10:24.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.300 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.300 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.300 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.300 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095517+0000", 2024-08-17T18:10:24.300 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.300 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.300 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.301 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095517+0000", 2024-08-17T18:10:24.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.301 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.301 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095517+0000", 2024-08-17T18:10:24.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6499999999999999e-07 2024-08-17T18:10:24.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.301 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.301 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095541+0000", 2024-08-17T18:10:24.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3374999999999999e-05 2024-08-17T18:10:24.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095548+0000", 2024-08-17T18:10:24.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7569999999999998e-06 2024-08-17T18:10:24.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095553+0000", 2024-08-17T18:10:24.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0599999999999998e-06 2024-08-17T18:10:24.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095600+0000", 2024-08-17T18:10:24.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7701000000000003e-05 2024-08-17T18:10:24.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095606+0000", 2024-08-17T18:10:24.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8839999999999999e-06 2024-08-17T18:10:24.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.564005+0000", 2024-08-17T18:10:24.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.46839848499999998 2024-08-17T18:10:24.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.564013+0000", 2024-08-17T18:10:24.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0749999999999998e-06 2024-08-17T18:10:24.304 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.304 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.304 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.304 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115459 48.8 48:15f41a0f:::benchmark_data_smithi130_327597_object115458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:10:24.304 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.111325+0000", 2024-08-17T18:10:24.304 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.068260288000005, 2024-08-17T18:10:24.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5377689189999999, 2024-08-17T18:10:24.305 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.305 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.305 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:10:24.305 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.305 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:10:24.305 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:10:24.305 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115459 2024-08-17T18:10:24.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.305 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111325+0000", 2024-08-17T18:10:24.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111325+0000", 2024-08-17T18:10:24.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111326+0000", 2024-08-17T18:10:24.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.311e-06 2024-08-17T18:10:24.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.307 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.307 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111351+0000", 2024-08-17T18:10:24.307 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5377000000000001e-05 2024-08-17T18:10:24.307 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.307 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.307 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.307 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111364+0000", 2024-08-17T18:10:24.307 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2632e-05 2024-08-17T18:10:24.307 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.307 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.307 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.307 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111370+0000", 2024-08-17T18:10:24.307 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.75e-06 2024-08-17T18:10:24.308 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.308 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.308 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.308 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111428+0000", 2024-08-17T18:10:24.308 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8425999999999999e-05 2024-08-17T18:10:24.308 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.308 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.308 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.308 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111440+0000", 2024-08-17T18:10:24.308 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1911e-05 2024-08-17T18:10:24.308 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.308 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.308 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.308 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565000+0000", 2024-08-17T18:10:24.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.45356039399999998 2024-08-17T18:10:24.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565007+0000", 2024-08-17T18:10:24.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.917e-06 2024-08-17T18:10:24.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.621760+0000", 2024-08-17T18:10:24.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.056752672 2024-08-17T18:10:24.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.310 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:10:24.310 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.621924+0000", 2024-08-17T18:10:24.310 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016439199999999999 2024-08-17T18:10:24.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.310 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:10:24.310 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.622547+0000", 2024-08-17T18:10:24.310 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000622424 2024-08-17T18:10:24.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.310 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.310 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641874+0000", 2024-08-17T18:10:24.310 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019326829 2024-08-17T18:10:24.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.311 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:10:24.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641892+0000", 2024-08-17T18:10:24.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8221000000000001e-05 2024-08-17T18:10:24.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.311 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.648915+0000", 2024-08-17T18:10:24.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.007023339 2024-08-17T18:10:24.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.311 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:10:24.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.648981+0000", 2024-08-17T18:10:24.312 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5957000000000006e-05 2024-08-17T18:10:24.312 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.312 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649094+0000", 2024-08-17T18:10:24.312 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000112367 2024-08-17T18:10:24.312 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.312 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.312 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.312 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.312 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115460 48.8 48:17144d5f:::benchmark_data_smithi130_327597_object115459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:10:24.313 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.111385+0000", 2024-08-17T18:10:24.313 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.068199797999995, 2024-08-17T18:10:24.313 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.537943938, 2024-08-17T18:10:24.313 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.313 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.313 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:10:24.313 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.313 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:10:24.313 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:10:24.313 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115460 2024-08-17T18:10:24.313 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.313 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.313 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.313 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.314 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111385+0000", 2024-08-17T18:10:24.314 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.314 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.314 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.314 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.314 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111385+0000", 2024-08-17T18:10:24.314 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.314 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.314 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.314 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.314 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111385+0000", 2024-08-17T18:10:24.314 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4999999999999998e-07 2024-08-17T18:10:24.314 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.314 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111401+0000", 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5566e-05 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111406+0000", 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4610000000000001e-06 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111409+0000", 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.303e-06 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111469+0000", 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0007000000000003e-05 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111476+0000", 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3209999999999999e-06 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565029+0000", 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.45355303200000002 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565037+0000", 2024-08-17T18:10:24.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.32e-06 2024-08-17T18:10:24.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.635425+0000", 2024-08-17T18:10:24.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0703878899999999 2024-08-17T18:10:24.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:10:24.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.635606+0000", 2024-08-17T18:10:24.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018029 2024-08-17T18:10:24.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:10:24.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.635807+0000", 2024-08-17T18:10:24.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020181999999999999 2024-08-17T18:10:24.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:10:24.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641899+0000", 2024-08-17T18:10:24.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0060914810000000002 2024-08-17T18:10:24.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641956+0000", 2024-08-17T18:10:24.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7343999999999998e-05 2024-08-17T18:10:24.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649206+0000", 2024-08-17T18:10:24.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072501060000000001 2024-08-17T18:10:24.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.320 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:10:24.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649267+0000", 2024-08-17T18:10:24.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0939e-05 2024-08-17T18:10:24.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.320 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649329+0000", 2024-08-17T18:10:24.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1707999999999999e-05 2024-08-17T18:10:24.320 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.321 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.321 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.321 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.321 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.321 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115465 48.8 48:103437e2:::benchmark_data_smithi130_327597_object115464:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:10:24.321 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.117778+0000", 2024-08-17T18:10:24.321 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.061807032999994, 2024-08-17T18:10:24.321 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5317481179999999, 2024-08-17T18:10:24.321 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.321 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.321 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:10:24.321 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.321 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:10:24.321 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:10:24.322 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115465 2024-08-17T18:10:24.322 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.322 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.322 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.322 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.322 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117778+0000", 2024-08-17T18:10:24.322 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.322 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.322 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.322 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.322 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117778+0000", 2024-08-17T18:10:24.322 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.323 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117779+0000", 2024-08-17T18:10:24.323 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.311e-06 2024-08-17T18:10:24.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.323 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117811+0000", 2024-08-17T18:10:24.323 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1489999999999998e-05 2024-08-17T18:10:24.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117826+0000", 2024-08-17T18:10:24.324 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5783000000000001e-05 2024-08-17T18:10:24.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117835+0000", 2024-08-17T18:10:24.324 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0709999999999995e-06 2024-08-17T18:10:24.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117889+0000", 2024-08-17T18:10:24.324 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4355999999999999e-05 2024-08-17T18:10:24.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.325 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.325 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117897+0000", 2024-08-17T18:10:24.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.729e-06 2024-08-17T18:10:24.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.325 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.325 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.325 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565061+0000", 2024-08-17T18:10:24.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.44716407800000002 2024-08-17T18:10:24.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.325 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.325 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.325 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565068+0000", 2024-08-17T18:10:24.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3130000000000001e-06 2024-08-17T18:10:24.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.326 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.642010+0000", 2024-08-17T18:10:24.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0769422019999999 2024-08-17T18:10:24.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.326 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:10:24.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.642197+0000", 2024-08-17T18:10:24.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000186923 2024-08-17T18:10:24.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.326 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:10:24.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.642394+0000", 2024-08-17T18:10:24.327 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000196913 2024-08-17T18:10:24.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.327 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:10:24.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649113+0000", 2024-08-17T18:10:24.327 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0067189290000000002 2024-08-17T18:10:24.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.327 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649158+0000", 2024-08-17T18:10:24.327 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4758000000000001e-05 2024-08-17T18:10:24.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.328 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.328 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649420+0000", 2024-08-17T18:10:24.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00026217599999999998 2024-08-17T18:10:24.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.328 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:10:24.328 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649452+0000", 2024-08-17T18:10:24.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2178000000000003e-05 2024-08-17T18:10:24.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.328 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.328 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649526+0000", 2024-08-17T18:10:24.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3907999999999998e-05 2024-08-17T18:10:24.329 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.329 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.329 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.329 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.329 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.329 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115470 48.8 48:1640b7fc:::benchmark_data_smithi130_327597_object115469:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:10:24.329 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.130397+0000", 2024-08-17T18:10:24.329 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.049188086000001, 2024-08-17T18:10:24.329 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.519320972, 2024-08-17T18:10:24.329 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:10:24.329 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:10:24.329 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:10:24.329 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:10:24.330 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:10:24.330 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:10:24.330 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115470 2024-08-17T18:10:24.330 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.330 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:10:24.330 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.330 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:10:24.330 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130397+0000", 2024-08-17T18:10:24.330 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.330 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.330 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.330 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:10:24.330 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130397+0000", 2024-08-17T18:10:24.331 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:10:24.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.331 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.331 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:10:24.331 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130399+0000", 2024-08-17T18:10:24.331 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0169999999999999e-06 2024-08-17T18:10:24.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.331 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.331 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:10:24.331 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130441+0000", 2024-08-17T18:10:24.331 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2552000000000002e-05 2024-08-17T18:10:24.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.331 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.331 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:10:24.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130470+0000", 2024-08-17T18:10:24.332 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8713000000000001e-05 2024-08-17T18:10:24.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.332 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:10:24.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130484+0000", 2024-08-17T18:10:24.332 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3709999999999999e-05 2024-08-17T18:10:24.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.332 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130612+0000", 2024-08-17T18:10:24.332 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000128598 2024-08-17T18:10:24.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.333 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.333 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130624+0000", 2024-08-17T18:10:24.333 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1878999999999999e-05 2024-08-17T18:10:24.333 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.333 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.333 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565091+0000", 2024-08-17T18:10:24.333 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.43446660199999998 2024-08-17T18:10:24.333 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.333 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:10:24.333 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565098+0000", 2024-08-17T18:10:24.333 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2250000000000003e-06 2024-08-17T18:10:24.334 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.334 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.334 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:10:24.334 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.645503+0000", 2024-08-17T18:10:24.334 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0804047400000001 2024-08-17T18:10:24.334 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.334 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.334 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:10:24.334 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.645638+0000", 2024-08-17T18:10:24.334 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000135418 2024-08-17T18:10:24.334 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.334 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.335 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:10:24.335 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.645861+0000", 2024-08-17T18:10:24.335 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00022232299999999999 2024-08-17T18:10:24.335 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.335 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.335 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:10:24.335 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649120+0000", 2024-08-17T18:10:24.335 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032596449999999998 2024-08-17T18:10:24.335 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.335 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.335 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.335 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649377+0000", 2024-08-17T18:10:24.335 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00025634200000000001 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649566+0000", 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018959399999999999 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649604+0000", 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8015000000000001e-05 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649718+0000", 2024-08-17T18:10:24.336 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011359899999999999 2024-08-17T18:10:24.337 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.337 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.337 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.337 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:10:24.337 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:10:24.337 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:10:24.361 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:24.383 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:24.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:24.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:24.485 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:24.507 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:24.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:24.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:24.609 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:24.648 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:24.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:24.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:24.750 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:24.772 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:24.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:24.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:24.874 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:24.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:24.912 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:24.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:25.013 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:25.070 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:25.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:25.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:25.117 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:10:25.172 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:25.195 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:25.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:25.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:25.296 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:25.338 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:25.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:25.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:25 smithi130 ceph-mon[105521]: pgmap v2423: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 760 op/s; 47593/231804 objects degraded (20.532%); 31029/231804 objects misplaced (13.386%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T18:10:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:25 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3902950928' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:10:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:25 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:10:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:25 smithi130 ceph-mon[110068]: pgmap v2423: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 760 op/s; 47593/231804 objects degraded (20.532%); 31029/231804 objects misplaced (13.386%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T18:10:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:25 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3902950928' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:10:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:25 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:10:25.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:25.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:25.482 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:25.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:25.583 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:25.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:25 smithi179 ceph-mon[93519]: pgmap v2423: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.7 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 0 B/s wr, 760 op/s; 47593/231804 objects degraded (20.532%); 31029/231804 objects misplaced (13.386%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T18:10:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:25 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3902950928' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:10:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:25 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:10:25.627 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:25.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:25.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:25.729 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:25.768 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:25.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:25.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:25.870 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:25.874 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T18:10:25.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:25.912 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:25.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:26.014 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:26.036 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:26.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:26.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:26.138 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:26.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:26.181 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:26.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:26.282 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:26.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:26.341 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:26.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:26.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:26 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:10:26.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:26 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:10:26.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:26 smithi130 ceph-mon[105521]: osdmap e1350: 8 total, 8 up, 4 in 2024-08-17T18:10:26.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 47593/231804 objects degraded (20.532%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:26.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:26 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:10:26.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:26 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:10:26.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:26 smithi130 ceph-mon[110068]: osdmap e1350: 8 total, 8 up, 4 in 2024-08-17T18:10:26.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 47593/231804 objects degraded (20.532%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:26.442 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:26.486 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:26.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:26.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:26.588 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:26.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:26 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:10:26.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:26 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:10:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:26 smithi179 ceph-mon[93519]: osdmap e1350: 8 total, 8 up, 4 in 2024-08-17T18:10:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 47593/231804 objects degraded (20.532%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:10:26.610 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:26.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:26.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:26.712 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:26.735 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:26.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:26.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:26.837 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:26.858 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:26.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:26.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:26.960 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:26.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:26.982 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:26.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:27.083 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:27.129 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:27.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:27.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:27.232 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:27.254 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:27.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:27.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:27.356 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:27.377 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:27.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:27.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:27 smithi130 ceph-mon[105521]: pgmap v2425: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.6 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 727 op/s; 46326/226539 objects degraded (20.449%); 30786/226539 objects misplaced (13.590%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T18:10:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:27 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2256278810' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:10:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:27 smithi130 ceph-mon[110068]: pgmap v2425: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.6 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 727 op/s; 46326/226539 objects degraded (20.449%); 30786/226539 objects misplaced (13.590%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T18:10:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:27 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2256278810' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:10:27.479 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:27.528 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T18:10:27.533 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:27.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:27.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:27.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:27 smithi179 ceph-mon[93519]: pgmap v2425: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.6 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 727 op/s; 46326/226539 objects degraded (20.449%); 30786/226539 objects misplaced (13.590%); 3.0 MiB/s, 48 objects/s recovering 2024-08-17T18:10:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:27 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2256278810' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:10:27.635 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:27.673 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:27.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:27.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:27.775 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:27.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:27.798 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:27.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:27.899 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:27.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:27.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:27.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:28.051 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:28.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:28.106 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:28.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:28.207 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:28.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:28.228 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:28.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:28.330 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:28.367 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:28.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:28.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:28 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2256278810' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:10:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:28 smithi130 ceph-mon[105521]: osdmap e1351: 8 total, 8 up, 4 in 2024-08-17T18:10:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:10:28.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:28 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2256278810' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:10:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:28 smithi130 ceph-mon[110068]: osdmap e1351: 8 total, 8 up, 4 in 2024-08-17T18:10:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:28.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:10:28.469 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:28.491 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:28.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:28.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:28 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2256278810' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:10:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:28 smithi179 ceph-mon[93519]: osdmap e1351: 8 total, 8 up, 4 in 2024-08-17T18:10:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:28.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:10:28.593 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:28.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:28.635 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:28.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:28.736 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:28.758 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:28.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:28.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:28.860 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:28.882 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:28.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:28.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:28.983 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:29.021 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:29.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:29.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:29.099 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T18:10:29.123 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:29.212 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:29.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:29.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:29.314 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:29.337 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:29.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:29.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:29.438 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:29.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:29.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:29.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:29.563 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:29.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:29 smithi179 ceph-mon[93519]: pgmap v2427: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.6 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 602 op/s; 46326/226539 objects degraded (20.449%); 30786/226539 objects misplaced (13.590%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T18:10:29.595 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:29.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:29.657 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:29.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:29.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:29 smithi130 ceph-mon[105521]: pgmap v2427: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.6 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 602 op/s; 46326/226539 objects degraded (20.449%); 30786/226539 objects misplaced (13.590%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T18:10:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:29 smithi130 ceph-mon[110068]: pgmap v2427: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.6 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 602 op/s; 46326/226539 objects degraded (20.449%); 30786/226539 objects misplaced (13.590%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T18:10:29.693 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:29.758 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:29.780 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:29.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:29.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:29.882 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:29.904 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:29.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:29.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:30.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:30.027 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:30.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:30.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:30.047 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:10:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:10:29] "GET /metrics HTTP/1.1" 200 36812 "" "Prometheus/2.51.0" 2024-08-17T18:10:30.129 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:30.187 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:30.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:30.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:30.289 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:30.338 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:30.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:30.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:30.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:30.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:30.484 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:30.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:30.530 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:10:30.579 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:10:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:10:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:10:30.585 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:30.652 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:30.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:30.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:30.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:30 smithi130 ceph-mon[110068]: pgmap v2428: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 847 op/s; 44488/218061 objects degraded (20.402%); 29041/218061 objects misplaced (13.318%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T18:10:30.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:30 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:30 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:30 smithi130 ceph-mon[105521]: pgmap v2428: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 847 op/s; 44488/218061 objects degraded (20.402%); 29041/218061 objects misplaced (13.318%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T18:10:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:30 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:30 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:30.739 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:30.754 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:30.817 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:30.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:30.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:30.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:30 smithi179 ceph-mon[93519]: pgmap v2428: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.4 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 847 op/s; 44488/218061 objects degraded (20.402%); 29041/218061 objects misplaced (13.318%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T18:10:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:30 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:30 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:30.919 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:30.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:30.967 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:30.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:31.068 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:31.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:31.144 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:31.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:31.245 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:31.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:31.267 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:31.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:31.368 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:31.420 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:31.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:31.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:31.522 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:31.544 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:31.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:31.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:31.646 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:31.685 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:31.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:31.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:31.787 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:31.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:31.810 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:31.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:31.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 44488/218061 objects degraded (20.402%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:31 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:10:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:31 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:10:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:31 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:31.843 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 44488/218061 objects degraded (20.402%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:31.843 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:31 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:10:31.843 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:31 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:10:31.843 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:31 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:31.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 44488/218061 objects degraded (20.402%), 11 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:31.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:31 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:10:31.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:31 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:10:31.844 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:31 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:31.910 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:31.949 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:31.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:31.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:32.051 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:32.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:32.098 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:32.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:32.202 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:32.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:32.267 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:32.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:32.368 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:32.415 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:32.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:32.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:32.517 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:32.541 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:32.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:32.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:32.643 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:32.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:32.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:32.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:32.788 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:32.832 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:32.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:32.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:32.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:32.972 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:32.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:32.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:33.074 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:33.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:33.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:33.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:33.198 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:33.240 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:33.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:33.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:33.342 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:33.348 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:33 smithi130 ceph-mon[105521]: pgmap v2429: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.4 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 689 op/s; 43797/215247 objects degraded (20.347%); 29041/215247 objects misplaced (13.492%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T18:10:33.348 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:33 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/437225439' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:10:33.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:33 smithi130 ceph-mon[110068]: pgmap v2429: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.4 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 689 op/s; 43797/215247 objects degraded (20.347%); 29041/215247 objects misplaced (13.492%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T18:10:33.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:33 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/437225439' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:10:33.348 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:10:33.349 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:10:33.391 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:33.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:33.393 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:10:33.393 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:10:33.442 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:10:33.442 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:10:33.493 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:33.497 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:10:33.498 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:10:33.515 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:33.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:33.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:33.534 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:10:33.535 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:10:33.574 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:10:33.574 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:10:33.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:33 smithi179 ceph-mon[93519]: pgmap v2429: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.4 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 689 op/s; 43797/215247 objects degraded (20.347%); 29041/215247 objects misplaced (13.492%); 1.9 MiB/s, 29 objects/s recovering 2024-08-17T18:10:33.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:33 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/437225439' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:10:33.606 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:10:33.607 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:10:33.616 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:33.641 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:10:33.642 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:10:33.653 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:33.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:33.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:33.754 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:33.793 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:33.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:33.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:33.895 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:33.918 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:33.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:33.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:34.019 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:34.042 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:34.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:34.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:34.147 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:34.168 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:34.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:34.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:34.270 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:34.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:34.306 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:34.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:34.353 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T18:10:34.392 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:34.408 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:34.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:34.430 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:34.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:34.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:34.576 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:34.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:34.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:34.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:34 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4253926042' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:10:34.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:34 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:10:34.618 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:34 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4253926042' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:10:34.618 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:34 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:10:34.619 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:34 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4253926042' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:10:34.619 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:34 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:10:34.678 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:34.700 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:34.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:34.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:34.802 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:34.838 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:34.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:34.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:34.940 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:34.987 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:34.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:34.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:35.088 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:35.110 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:35.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:35.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:35.212 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:35.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:35.257 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:35.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:35.357 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T18:10:35.358 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:10:35.358 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:35.410 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:35.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:35.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:35.512 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:35.562 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:35.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:35.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:35.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:35.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:35 smithi130 ceph-mon[105521]: pgmap v2430: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.3 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 701 op/s; 42526/209046 objects degraded (20.343%); 27640/209046 objects misplaced (13.222%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T18:10:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:35 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:10:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:35 smithi130 ceph-mon[105521]: osdmap e1352: 8 total, 8 up, 4 in 2024-08-17T18:10:35.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:35 smithi130 ceph-mon[110068]: pgmap v2430: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.3 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 701 op/s; 42526/209046 objects degraded (20.343%); 27640/209046 objects misplaced (13.222%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T18:10:35.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:35 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:10:35.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:35 smithi130 ceph-mon[110068]: osdmap e1352: 8 total, 8 up, 4 in 2024-08-17T18:10:35.704 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:35.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:35.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:35.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:35.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:35.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:35.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:35 smithi179 ceph-mon[93519]: pgmap v2430: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.3 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 0 B/s wr, 701 op/s; 42526/209046 objects degraded (20.343%); 27640/209046 objects misplaced (13.222%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T18:10:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:35 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:10:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:35 smithi179 ceph-mon[93519]: osdmap e1352: 8 total, 8 up, 4 in 2024-08-17T18:10:35.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:35.950 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:35.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:35.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:36.052 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:36.093 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:36.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:36.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:36.195 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:36.232 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:36.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:36.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:36.333 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:36.378 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:36.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:36.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:36.479 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:36.502 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:36.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:36.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:36.566 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:36 smithi130 ceph-mon[105521]: pgmap v2432: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 894 op/s; 41390/204246 objects degraded (20.265%); 27400/204246 objects misplaced (13.415%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T18:10:36.566 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 42526/209046 objects degraded (20.343%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:36.566 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:36 smithi130 ceph-mon[110068]: pgmap v2432: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 894 op/s; 41390/204246 objects degraded (20.265%); 27400/204246 objects misplaced (13.415%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T18:10:36.566 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 42526/209046 objects degraded (20.343%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:36.604 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:36.625 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:36.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:36.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:36.727 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:36.748 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:36.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:36.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:36.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:36 smithi179 ceph-mon[93519]: pgmap v2432: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 894 op/s; 41390/204246 objects degraded (20.265%); 27400/204246 objects misplaced (13.415%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T18:10:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 42526/209046 objects degraded (20.343%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:36.850 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:36.888 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:36.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:36.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:36.989 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:37.010 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:10:37.010 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:10:37.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:37.039 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:37.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:37.140 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:37.161 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:37.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:37.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:37.263 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:37.309 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:37.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:37.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:37.411 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:37.446 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:37.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:37.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:37.548 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:37.569 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:37.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:37.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:37.671 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:37.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:37.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:37.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:37.793 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:37.815 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:37.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:37.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:37.917 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:37.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:37.939 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:37.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:38.002 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:10:38.040 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:38.093 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:38.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:38.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:38.196 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:38.247 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:38.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:38.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:38.349 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:38.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:38.392 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:38.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:38.493 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:38.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:38.545 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:38.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:38.646 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:38 smithi130 ceph-mon[105521]: pgmap v2433: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 742 op/s; 41390/204246 objects degraded (20.265%); 27400/204246 objects misplaced (13.415%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T18:10:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:38 smithi130 ceph-mon[110068]: pgmap v2433: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 742 op/s; 41390/204246 objects degraded (20.265%); 27400/204246 objects misplaced (13.415%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T18:10:38.668 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:38.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:38.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:38.770 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:38.792 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:38.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:38.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:38 smithi179 ceph-mon[93519]: pgmap v2433: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.2 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 742 op/s; 41390/204246 objects degraded (20.265%); 27400/204246 objects misplaced (13.415%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T18:10:38.893 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:38.916 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:38.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:38.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:39.018 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:39.040 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T18:10:39.040 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:39.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:39.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:39.142 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:39.164 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:39.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:39.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:39.259 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:39.266 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:39.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:39.381 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:39.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:39.482 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:39.535 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:39.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:39.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:39.638 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:39.661 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:39.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:39.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:39.763 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:39.784 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:39.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:39.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:39.886 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:39.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:10:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:10:39] "GET /metrics HTTP/1.1" 200 36812 "" "Prometheus/2.51.0" 2024-08-17T18:10:39.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:39.942 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:39.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:40.043 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:40.081 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:40.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:40.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:40.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:40.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:40.225 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:40.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:40.253 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:40.326 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:40.395 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:40.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:40.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:40.497 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:40.508 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:10:40.515 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:10:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:10:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:10:40.520 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:40.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:40.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:40.622 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:40.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:40.668 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:40.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:40.770 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:40.791 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:40.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:40.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:40.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:40 smithi179 ceph-mon[93519]: pgmap v2434: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 744 op/s; 39571/195702 objects degraded (20.220%); 25698/195702 objects misplaced (13.131%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T18:10:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:40 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2085188826' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:10:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:40 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:10:40.893 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:40.914 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:40.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:40.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:40.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:40 smithi130 ceph-mon[105521]: pgmap v2434: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 744 op/s; 39571/195702 objects degraded (20.220%); 25698/195702 objects misplaced (13.131%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T18:10:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:40 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2085188826' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:10:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:40 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:10:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:40 smithi130 ceph-mon[110068]: pgmap v2434: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 4.0 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 744 op/s; 39571/195702 objects degraded (20.220%); 25698/195702 objects misplaced (13.131%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T18:10:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:40 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2085188826' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:10:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:40 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:10:41.016 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:41.039 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:41.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:41.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:41.140 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:41.162 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:41.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:41.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:41.264 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:41.304 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:41.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:41.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:41.406 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:41.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:41.446 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:41.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:41.547 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:41.571 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:41.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:41.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:41.673 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:41.715 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:41.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:41.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:41.817 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:41 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:10:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:41 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:10:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:41 smithi179 ceph-mon[93519]: osdmap e1353: 8 total, 8 up, 4 in 2024-08-17T18:10:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:41 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 39571/195702 objects degraded (20.220%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:41.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:41.862 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:41.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:41 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:10:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:41 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:10:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:41 smithi130 ceph-mon[105521]: osdmap e1353: 8 total, 8 up, 4 in 2024-08-17T18:10:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:41 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 39571/195702 objects degraded (20.220%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:41 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:10:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:41 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:10:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:41 smithi130 ceph-mon[110068]: osdmap e1353: 8 total, 8 up, 4 in 2024-08-17T18:10:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:41 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 39571/195702 objects degraded (20.220%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:41.964 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:41.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:41.986 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:41.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:42.087 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:42.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:42.121 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:42.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:42.222 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:42.244 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:42.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:42.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:42.345 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:42.382 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:42.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:42.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:42.484 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:42.505 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:42.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:42.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:42.606 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:42.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:42.654 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:42.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:42.755 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:42.800 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:42.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:42.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:42.902 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:42.941 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:42.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:42.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:43.043 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:43.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:43.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:43.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:43.167 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:43.189 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:43.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:43.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:43.290 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:43.312 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:43.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:43.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:43.414 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:43.436 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:43.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:43.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:43.514 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:10:43.538 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:43.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:43 smithi130 ceph-mon[105521]: pgmap v2436: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.9 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 674 op/s; 38863/192852 objects degraded (20.152%); 25698/192852 objects misplaced (13.325%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T18:10:43.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:43.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:10:43.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:43 smithi130 ceph-mon[110068]: pgmap v2436: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.9 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 674 op/s; 38863/192852 objects degraded (20.152%); 25698/192852 objects misplaced (13.325%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T18:10:43.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:43.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:10:43.561 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:43.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:43.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:43.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:43 smithi179 ceph-mon[93519]: pgmap v2436: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.9 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 674 op/s; 38863/192852 objects degraded (20.152%); 25698/192852 objects misplaced (13.325%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T18:10:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:10:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:10:43.662 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:43.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:43.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:43.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:43.787 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:43.809 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:43.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:43.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:43.911 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:43.933 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:43.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:43.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:44.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:44.077 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:44.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:44.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:44.179 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:44.200 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:44.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:44.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:44.302 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:44.336 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:44.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:44.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:44.438 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:44.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:44.480 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:44.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:44.581 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:44.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:44.604 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:44.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:44.705 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:44.726 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:44.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:44.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:44.828 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:44.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:44.869 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:44.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:44.970 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:45.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:45.011 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:45.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:45.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:45.134 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:45.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:45.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:45.236 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:45.258 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:45.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:45.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:45.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:45.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:45.403 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:45.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:45.504 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:45.548 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:45.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:45.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:45.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:45 smithi179 ceph-mon[93519]: pgmap v2437: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.8 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 744 op/s; 37556/186645 objects degraded (20.122%); 24282/186645 objects misplaced (13.010%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T18:10:45.650 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:45.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:45 smithi130 ceph-mon[105521]: pgmap v2437: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.8 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 744 op/s; 37556/186645 objects degraded (20.122%); 24282/186645 objects misplaced (13.010%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T18:10:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:45 smithi130 ceph-mon[110068]: pgmap v2437: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.8 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 744 op/s; 37556/186645 objects degraded (20.122%); 24282/186645 objects misplaced (13.010%); 2.9 MiB/s, 46 objects/s recovering 2024-08-17T18:10:45.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:45.682 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:45.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:45.783 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:45.826 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:45.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:45.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:45.928 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:45.950 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:45.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:45.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:46.052 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:46.074 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:46.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:46.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:46.163 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:46.175 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:46.240 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:46.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:46.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:46.342 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:46.366 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:46.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:46.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:46.468 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:46.491 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:46.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:46.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:46.593 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:46.615 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:46.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:46.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:46.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 37556/186645 objects degraded (20.122%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 37556/186645 objects degraded (20.122%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:46.716 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:46.739 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:46.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:46.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:46.841 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 37556/186645 objects degraded (20.122%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:46.863 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:46.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:46.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:46.964 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:46.987 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:46.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:46.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:47.089 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:47.121 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:47.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:47.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:47.223 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:47.244 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:47.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:47.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:47.346 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:47.397 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:47.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:47.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:47.500 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:47.513 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:47 smithi130 ceph-mon[105521]: pgmap v2438: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.7 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 743 op/s; 36426/181938 objects degraded (20.021%); 24066/181938 objects misplaced (13.228%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T18:10:47.514 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:47 smithi130 ceph-mon[110068]: pgmap v2438: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.7 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 743 op/s; 36426/181938 objects degraded (20.021%); 24066/181938 objects misplaced (13.228%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T18:10:47.550 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:47.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:47.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:47.652 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:47.673 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:47.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:47.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:47.775 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:47.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:47.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:47.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:47 smithi179 ceph-mon[93519]: pgmap v2438: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.7 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 743 op/s; 36426/181938 objects degraded (20.021%); 24066/181938 objects misplaced (13.228%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T18:10:47.924 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:47.966 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:47.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:47.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:48.068 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:48.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:48.102 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:48.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:48.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:48.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:48.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:48.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:48.326 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:48.347 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:48.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:48.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:48.449 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:48.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:48.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:48.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:48.572 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:48.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:48.609 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:48.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:48.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:48 smithi130 ceph-mon[105521]: pgmap v2439: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.7 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 743 op/s; 36426/181938 objects degraded (20.021%); 24066/181938 objects misplaced (13.228%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T18:10:48.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:48 smithi130 ceph-mon[110068]: pgmap v2439: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.7 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 743 op/s; 36426/181938 objects degraded (20.021%); 24066/181938 objects misplaced (13.228%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T18:10:48.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:48.752 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:48.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:48.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:48.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:48 smithi179 ceph-mon[93519]: pgmap v2439: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.7 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 0 B/s wr, 743 op/s; 36426/181938 objects degraded (20.021%); 24066/181938 objects misplaced (13.228%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T18:10:48.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:48.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:48.877 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:48.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:48.979 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:49.010 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:49.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:49.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:49.097 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:10:49.098 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:10:49.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:49.133 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:49.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:49.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:49.235 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:49.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:49.259 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:49.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:49.361 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:49.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:49.401 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:49.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:49.501 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:49.523 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:49.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:49.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:49.624 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:49.659 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:49.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:49.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:49.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:49 smithi130 ceph-mon[105521]: from='client.36801 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:10:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:49 smithi130 ceph-mon[110068]: from='client.36801 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:10:49.760 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:49.807 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:49.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:49.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:49.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:49 smithi179 ceph-mon[93519]: from='client.36801 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:10:49.909 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:49.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:49.932 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:49.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:50.033 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:50.055 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:50.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:50.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:50.157 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:50.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:10:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:10:49] "GET /metrics HTTP/1.1" 200 36810 "" "Prometheus/2.51.0" 2024-08-17T18:10:50.185 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:50.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:50.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:50.286 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:50.311 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:50.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:50.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:50.413 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:50.440 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:50.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:50.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:50.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:50.579 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:50.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:50.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:50 smithi179 ceph-mon[93519]: pgmap v2440: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.5 GiB data, 10 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 726 op/s; 34795/173910 objects degraded (20.007%); 22426/173910 objects misplaced (12.895%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T18:10:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:10:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:10:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:10:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:50 smithi130 ceph-mon[105521]: pgmap v2440: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.5 GiB data, 10 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 726 op/s; 34795/173910 objects degraded (20.007%); 22426/173910 objects misplaced (12.895%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T18:10:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:50 smithi130 ceph-mon[110068]: pgmap v2440: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.5 GiB data, 10 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 726 op/s; 34795/173910 objects degraded (20.007%); 22426/173910 objects misplaced (12.895%); 2.6 MiB/s, 41 objects/s recovering 2024-08-17T18:10:50.682 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:50.704 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:50.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:50.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:50.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:50.828 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:50.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:50.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:50.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:50.967 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:50.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:50.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:51.068 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:51.090 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:51.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:51.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:51.192 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:51.236 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:51.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:51.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:51.337 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:51.360 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:51.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:51.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:51.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:51.502 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:51.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:51.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:51.604 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:51.643 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:51.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:51.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:51.745 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:51.768 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:51.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:51.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 34795/173910 objects degraded (20.007%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:51.869 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:51.892 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:51.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:51.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:51.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 34795/173910 objects degraded (20.007%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 34795/173910 objects degraded (20.007%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:51.994 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:52.016 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:52.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:52.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:52.117 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:52.139 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:52.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:52.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:52.240 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:52.262 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:52.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:52.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:52.364 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:52.404 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:52.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:52.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:52.422 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:52.507 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:52.529 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:52.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:52.631 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:52.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:52.654 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:52.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:52.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:52 smithi130 ceph-mon[105521]: pgmap v2441: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.5 GiB data, 9.8 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 665 op/s; 34045/171012 objects degraded (19.908%); 22426/171012 objects misplaced (13.114%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T18:10:52.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:52 smithi130 ceph-mon[110068]: pgmap v2441: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.5 GiB data, 9.8 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 665 op/s; 34045/171012 objects degraded (19.908%); 22426/171012 objects misplaced (13.114%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T18:10:52.755 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:52.777 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:52.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:52.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:52.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:52 smithi179 ceph-mon[93519]: pgmap v2441: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.5 GiB data, 9.8 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 665 op/s; 34045/171012 objects degraded (19.908%); 22426/171012 objects misplaced (13.114%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T18:10:52.879 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:52.902 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:52.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:52.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:53.003 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:53.025 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:53.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:53.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:53.127 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:53.148 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:53.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:53.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:53.250 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:53.272 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:53.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:53.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:53.374 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:53.428 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:53.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:53.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:53.532 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:53.585 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:53.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:53.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:53.687 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:53.733 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:53.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:53.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:53.835 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:53.858 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:53.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:53.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:53.959 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:53.980 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:53.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:54.082 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:54.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:54.140 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:54.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:54.241 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:54.242 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2439,"stamp":"2024-08-17T18:10:47.440634+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3973322725,"num_objects":60646,"num_object_clones":0,"num_object_copies":181938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":36426,"num_objects_misplaced":24066,"num_objects_unfound":0,"num_objects_dirty":60646,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":203584,"num_write_kb":8448016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9117,"num_bytes_recovered":597439885,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84003,"ondisk_log_size":84003,"up":267,"acting":256,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":10687744,"kb_used_data":7847456,"kb_used_omap":294,"kb_used_meta":2839897,"kb_avail":364292864,"statfs":{"total":383980142592,"available":373035892736,"internally_reserved":0,"allocated":8035794944,"data_stored":8018151499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":301759,"internal_metadata":2908054849},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-487325696,"num_objects":-7436,"num_object_clones":0,"num_object_copies":-22308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-4964,"num_objects_misplaced":-3334,"num_objects_unfound":0,"num_objects_dirty":-7436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7436,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":420,"num_bytes_recovered":27525120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003740"},"pg_stats":[{"pgid":"48.f","version":"1353'12645","reported_seq":22969,"reported_epoch":1353,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:43.076085+0000","last_change":"2024-08-17T18:09:29.893285+0000","last_active":"2024-08-17T18:10:43.076085+0000","last_peered":"2024-08-17T18:10:43.076085+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:28.981495+0000","last_became_peered":"2024-08-17T18:09:28.981495+0000","last_unstale":"2024-08-17T18:10:43.076085+0000","last_undegraded":"2024-08-17T18:09:28.850340+0000","last_fullsized":"2024-08-17T18:09:28.832563+0000","mapping_epoch":1343,"log_start":"1340'8457","ondisk_log_start":"1340'8457","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4188,"log_dups_size":0,"ondisk_log_size":4188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:54.654486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256180224,"num_objects":3909,"num_object_clones":0,"num_object_copies":11727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1327,"num_objects_misplaced":3909,"num_objects_unfound":0,"num_objects_dirty":3909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12645,"num_write_kb":529728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5130,"num_bytes_recovered":336199680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":3909}],"blocked_by":[],"up_primary":3,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.e","version":"1353'12815","reported_seq":12857,"reported_epoch":1353,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:44.815584+0000","last_change":"2024-08-17T18:09:29.965937+0000","last_active":"2024-08-17T18:10:44.815584+0000","last_peered":"2024-08-17T18:10:44.815584+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.950126+0000","last_became_peered":"2024-08-17T18:09:29.950126+0000","last_unstale":"2024-08-17T18:10:44.815584+0000","last_undegraded":"2024-08-17T18:09:29.863735+0000","last_fullsized":"2024-08-17T18:09:29.826958+0000","mapping_epoch":1344,"log_start":"1340'7848","ondisk_log_start":"1340'7848","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4967,"log_dups_size":0,"ondisk_log_size":4967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239665152,"num_objects":3657,"num_object_clones":0,"num_object_copies":10971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3657,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12815,"num_write_kb":527104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":3657}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.d","version":"1353'12639","reported_seq":12681,"reported_epoch":1353,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:43.068457+0000","last_change":"2024-08-17T18:09:29.930605+0000","last_active":"2024-08-17T18:10:43.068457+0000","last_peered":"2024-08-17T18:10:43.068457+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.918910+0000","last_became_peered":"2024-08-17T18:09:29.918910+0000","last_unstale":"2024-08-17T18:10:43.068457+0000","last_undegraded":"2024-08-17T18:09:29.832124+0000","last_fullsized":"2024-08-17T18:09:29.818954+0000","mapping_epoch":1344,"log_start":"1340'8524","ondisk_log_start":"1340'8524","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4115,"log_dups_size":0,"ondisk_log_size":4115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:00:55.077466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246677528,"num_objects":3765,"num_object_clones":0,"num_object_copies":11295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3765,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3765,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12639,"num_write_kb":524865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":3765}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.c","version":"1353'12507","reported_seq":12616,"reported_epoch":1353,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:43.095141+0000","last_change":"2024-08-17T18:09:28.996223+0000","last_active":"2024-08-17T18:10:43.095141+0000","last_peered":"2024-08-17T18:10:43.095141+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:28.914112+0000","last_became_peered":"2024-08-17T18:09:28.914112+0000","last_unstale":"2024-08-17T18:10:43.095141+0000","last_undegraded":"2024-08-17T18:09:28.840987+0000","last_fullsized":"2024-08-17T18:09:28.815757+0000","mapping_epoch":1343,"log_start":"1340'8346","ondisk_log_start":"1340'8346","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4161,"log_dups_size":0,"ondisk_log_size":4161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:55.890300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252510208,"num_objects":3853,"num_object_clones":0,"num_object_copies":11559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3845,"num_objects_misplaced":3845,"num_objects_unfound":0,"num_objects_dirty":3853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12507,"num_write_kb":523520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":3853}],"blocked_by":[],"up_primary":3,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.b","version":"1353'12776","reported_seq":12817,"reported_epoch":1353,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:43.132170+0000","last_change":"2024-08-17T18:09:29.921199+0000","last_active":"2024-08-17T18:10:43.132170+0000","last_peered":"2024-08-17T18:10:43.132170+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.912384+0000","last_became_peered":"2024-08-17T18:09:29.912384+0000","last_unstale":"2024-08-17T18:10:43.132170+0000","last_undegraded":"2024-08-17T18:09:29.829736+0000","last_fullsized":"2024-08-17T18:09:29.817434+0000","mapping_epoch":1344,"log_start":"1340'8664","ondisk_log_start":"1340'8664","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4112,"log_dups_size":0,"ondisk_log_size":4112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:22.733621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254148608,"num_objects":3878,"num_object_clones":0,"num_object_copies":11634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3878,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12776,"num_write_kb":532928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":3878}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.a","version":"1353'12728","reported_seq":12793,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:44.736933+0000","last_change":"2024-08-17T18:07:32.359061+0000","last_active":"2024-08-17T18:10:44.736933+0000","last_peered":"2024-08-17T18:10:44.736933+0000","last_clean":"2024-08-17T18:10:44.736933+0000","last_became_active":"2024-08-17T18:07:32.357107+0000","last_became_peered":"2024-08-17T18:07:32.357107+0000","last_unstale":"2024-08-17T18:10:44.736933+0000","last_undegraded":"2024-08-17T18:10:44.736933+0000","last_fullsized":"2024-08-17T18:10:44.736933+0000","mapping_epoch":1327,"log_start":"1340'7764","ondisk_log_start":"1340'7764","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4964,"log_dups_size":0,"ondisk_log_size":4964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:30:02.061787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239075328,"num_objects":3648,"num_object_clones":0,"num_object_copies":10944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12728,"num_write_kb":524032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.9","version":"1353'13027","reported_seq":13069,"reported_epoch":1353,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:44.698067+0000","last_change":"2024-08-17T18:09:29.968655+0000","last_active":"2024-08-17T18:10:44.698067+0000","last_peered":"2024-08-17T18:10:44.698067+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.948623+0000","last_became_peered":"2024-08-17T18:09:29.948623+0000","last_unstale":"2024-08-17T18:10:44.698067+0000","last_undegraded":"2024-08-17T18:09:29.855385+0000","last_fullsized":"2024-08-17T18:09:29.821477+0000","mapping_epoch":1344,"log_start":"1340'8090","ondisk_log_start":"1340'8090","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4937,"log_dups_size":0,"ondisk_log_size":4937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256180224,"num_objects":3909,"num_object_clones":0,"num_object_copies":11727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3909,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13027,"num_write_kb":541952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":3909}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.8","version":"1353'12795","reported_seq":12890,"reported_epoch":1353,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:43.981924+0000","last_change":"2024-08-17T18:09:29.904036+0000","last_active":"2024-08-17T18:10:43.981924+0000","last_peered":"2024-08-17T18:10:43.981924+0000","last_clean":"2024-08-17T18:09:27.840248+0000","last_became_active":"2024-08-17T18:09:28.942085+0000","last_became_peered":"2024-08-17T18:09:28.942085+0000","last_unstale":"2024-08-17T18:10:43.981924+0000","last_undegraded":"2024-08-17T18:10:43.981924+0000","last_fullsized":"2024-08-17T18:10:43.981924+0000","mapping_epoch":1343,"log_start":"1340'7861","ondisk_log_start":"1340'7861","created":1327,"last_epoch_clean":1337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4934,"log_dups_size":0,"ondisk_log_size":4934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243990528,"num_objects":3723,"num_object_clones":0,"num_object_copies":11169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3723,"num_objects_unfound":0,"num_objects_dirty":3723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12795,"num_write_kb":528576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,1],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":3723}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.7","version":"1353'12565","reported_seq":12629,"reported_epoch":1353,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:43.080201+0000","last_change":"2024-08-17T18:09:28.989307+0000","last_active":"2024-08-17T18:10:43.080201+0000","last_peered":"2024-08-17T18:10:43.080201+0000","last_clean":"2024-08-17T18:09:27.824322+0000","last_became_active":"2024-08-17T18:09:28.981075+0000","last_became_peered":"2024-08-17T18:09:28.981075+0000","last_unstale":"2024-08-17T18:10:43.080201+0000","last_undegraded":"2024-08-17T18:10:43.080201+0000","last_fullsized":"2024-08-17T18:10:43.080201+0000","mapping_epoch":1343,"log_start":"1340'8448","ondisk_log_start":"1340'8448","created":1327,"last_epoch_clean":1337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4117,"log_dups_size":0,"ondisk_log_size":4117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:12:08.456671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256835584,"num_objects":3919,"num_object_clones":0,"num_object_copies":11757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3919,"num_objects_unfound":0,"num_objects_dirty":3919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12565,"num_write_kb":527488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,1],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"1,2,7","objects":3919}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.6","version":"1353'12584","reported_seq":13875,"reported_epoch":1353,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:42.711493+0000","last_change":"2024-08-17T18:09:59.877439+0000","last_active":"2024-08-17T18:10:42.711493+0000","last_peered":"2024-08-17T18:10:42.711493+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:28.962573+0000","last_became_peered":"2024-08-17T18:09:28.962573+0000","last_unstale":"2024-08-17T18:10:42.711493+0000","last_undegraded":"2024-08-17T18:09:28.855040+0000","last_fullsized":"2024-08-17T18:09:28.816313+0000","mapping_epoch":1343,"log_start":"1330'2500","ondisk_log_start":"1330'2500","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":10084,"log_dups_size":0,"ondisk_log_size":10084,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251002880,"num_objects":3830,"num_object_clones":0,"num_object_copies":11490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3626,"num_objects_misplaced":3626,"num_objects_unfound":0,"num_objects_dirty":3830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12584,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":403,"num_bytes_recovered":26411008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":3830}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"48.5","version":"1353'12657","reported_seq":12700,"reported_epoch":1353,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:44.801968+0000","last_change":"2024-08-17T18:09:29.983372+0000","last_active":"2024-08-17T18:10:44.801968+0000","last_peered":"2024-08-17T18:10:44.801968+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.956933+0000","last_became_peered":"2024-08-17T18:09:29.956933+0000","last_unstale":"2024-08-17T18:10:44.801968+0000","last_undegraded":"2024-08-17T18:09:29.847436+0000","last_fullsized":"2024-08-17T18:09:29.819789+0000","mapping_epoch":1344,"log_start":"1340'7658","ondisk_log_start":"1340'7658","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4999,"log_dups_size":0,"ondisk_log_size":4999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244776960,"num_objects":3735,"num_object_clones":0,"num_object_copies":11205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3735,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12657,"num_write_kb":524544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":3735}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.4","version":"1353'12732","reported_seq":12797,"reported_epoch":1353,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:43.904016+0000","last_change":"2024-08-17T18:09:29.965770+0000","last_active":"2024-08-17T18:10:43.904016+0000","last_peered":"2024-08-17T18:10:43.904016+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.948027+0000","last_became_peered":"2024-08-17T18:09:29.948027+0000","last_unstale":"2024-08-17T18:10:43.904016+0000","last_undegraded":"2024-08-17T18:09:29.860142+0000","last_fullsized":"2024-08-17T18:09:29.828238+0000","mapping_epoch":1344,"log_start":"1340'7817","ondisk_log_start":"1340'7817","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4915,"log_dups_size":0,"ondisk_log_size":4915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241303552,"num_objects":3682,"num_object_clones":0,"num_object_copies":11046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3682,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12732,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":3682}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.3","version":"1353'12735","reported_seq":12799,"reported_epoch":1353,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:43.867743+0000","last_change":"2024-08-17T18:09:30.006916+0000","last_active":"2024-08-17T18:10:43.867743+0000","last_peered":"2024-08-17T18:10:43.867743+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.994996+0000","last_became_peered":"2024-08-17T18:09:29.994996+0000","last_unstale":"2024-08-17T18:10:43.867743+0000","last_undegraded":"2024-08-17T18:09:29.861017+0000","last_fullsized":"2024-08-17T18:09:29.830876+0000","mapping_epoch":1344,"log_start":"1340'7823","ondisk_log_start":"1340'7823","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4912,"log_dups_size":0,"ondisk_log_size":4912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242679808,"num_objects":3703,"num_object_clones":0,"num_object_copies":11109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3703,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12735,"num_write_kb":526016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":3703}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.2","version":"1353'12622","reported_seq":23258,"reported_epoch":1353,"state":"active+undersized+degraded+remapped+backfilling","last_fresh":"2024-08-17T18:10:42.728218+0000","last_change":"2024-08-17T18:09:29.894664+0000","last_active":"2024-08-17T18:10:42.728218+0000","last_peered":"2024-08-17T18:10:42.728218+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:28.981975+0000","last_became_peered":"2024-08-17T18:09:28.981975+0000","last_unstale":"2024-08-17T18:10:42.728218+0000","last_undegraded":"2024-08-17T18:09:29.001178+0000","last_fullsized":"2024-08-17T18:09:28.815541+0000","mapping_epoch":1343,"log_start":"1330'2600","ondisk_log_start":"1330'2600","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":10022,"log_dups_size":0,"ondisk_log_size":10022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249167872,"num_objects":3802,"num_object_clones":0,"num_object_copies":11406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1299,"num_objects_misplaced":1299,"num_objects_unfound":0,"num_objects_dirty":3802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12622,"num_write_kb":525568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3524,"num_bytes_recovered":230817792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":3802}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"48.1","version":"1353'12826","reported_seq":12860,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:43.052747+0000","last_change":"2024-08-17T18:07:32.360312+0000","last_active":"2024-08-17T18:10:43.052747+0000","last_peered":"2024-08-17T18:10:43.052747+0000","last_clean":"2024-08-17T18:10:43.052747+0000","last_became_active":"2024-08-17T18:07:32.360092+0000","last_became_peered":"2024-08-17T18:07:32.360092+0000","last_unstale":"2024-08-17T18:10:43.052747+0000","last_undegraded":"2024-08-17T18:10:43.052747+0000","last_fullsized":"2024-08-17T18:10:43.052747+0000","mapping_epoch":1327,"log_start":"1340'8622","ondisk_log_start":"1340'8622","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4204,"log_dups_size":0,"ondisk_log_size":4204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:41.580676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253100032,"num_objects":3862,"num_object_clones":0,"num_object_copies":11586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12826,"num_write_kb":534016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.0","version":"1353'12653","reported_seq":12725,"reported_epoch":1353,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:43.095390+0000","last_change":"2024-08-17T18:09:28.963421+0000","last_active":"2024-08-17T18:10:43.095390+0000","last_peered":"2024-08-17T18:10:43.095390+0000","last_clean":"2024-08-17T18:09:27.823675+0000","last_became_active":"2024-08-17T18:09:28.941589+0000","last_became_peered":"2024-08-17T18:09:28.941589+0000","last_unstale":"2024-08-17T18:10:43.095390+0000","last_undegraded":"2024-08-17T18:10:43.095390+0000","last_fullsized":"2024-08-17T18:10:43.095390+0000","mapping_epoch":1343,"log_start":"1340'8488","ondisk_log_start":"1340'8488","created":1327,"last_epoch_clean":1337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4165,"log_dups_size":0,"ondisk_log_size":4165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:36.800547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245432320,"num_objects":3745,"num_object_clones":0,"num_object_copies":11235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3745,"num_objects_unfound":0,"num_objects_dirty":3745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12653,"num_write_kb":524736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,1],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"1,2,7","objects":3745}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":2194,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516276+0000","last_change":"2024-08-17T18:09:28.870033+0000","last_active":"2024-08-17T18:10:40.516276+0000","last_peered":"2024-08-17T18:10:40.516276+0000","last_clean":"2024-08-17T18:10:40.516276+0000","last_became_active":"2024-08-17T18:09:28.869346+0000","last_became_peered":"2024-08-17T18:09:28.869346+0000","last_unstale":"2024-08-17T18:10:40.516276+0000","last_undegraded":"2024-08-17T18:10:40.516276+0000","last_fullsized":"2024-08-17T18:10:40.516276+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:23.168631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":489,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.512276+0000","last_change":"2024-08-17T18:09:29.841584+0000","last_active":"2024-08-17T18:10:40.512276+0000","last_peered":"2024-08-17T18:10:40.512276+0000","last_clean":"2024-08-17T18:10:40.512276+0000","last_became_active":"2024-08-17T18:09:29.840950+0000","last_became_peered":"2024-08-17T18:09:29.840950+0000","last_unstale":"2024-08-17T18:10:40.512276+0000","last_undegraded":"2024-08-17T18:10:40.512276+0000","last_fullsized":"2024-08-17T18:10:40.512276+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:43.035959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1984,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516949+0000","last_change":"2024-08-17T18:09:29.883587+0000","last_active":"2024-08-17T18:10:40.516949+0000","last_peered":"2024-08-17T18:10:40.516949+0000","last_clean":"2024-08-17T18:10:40.516949+0000","last_became_active":"2024-08-17T18:09:29.882222+0000","last_became_peered":"2024-08-17T18:09:29.882222+0000","last_unstale":"2024-08-17T18:10:40.516949+0000","last_undegraded":"2024-08-17T18:10:40.516949+0000","last_fullsized":"2024-08-17T18:10:40.516949+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:10.777993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2041,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.519685+0000","last_change":"2024-08-17T18:09:28.858824+0000","last_active":"2024-08-17T18:10:40.519685+0000","last_peered":"2024-08-17T18:10:40.519685+0000","last_clean":"2024-08-17T18:10:40.519685+0000","last_became_active":"2024-08-17T18:09:28.857856+0000","last_became_peered":"2024-08-17T18:09:28.857856+0000","last_unstale":"2024-08-17T18:10:40.519685+0000","last_undegraded":"2024-08-17T18:10:40.519685+0000","last_fullsized":"2024-08-17T18:10:40.519685+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:41:35.146166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":45,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518310+0000","last_change":"2024-08-17T18:09:29.851515+0000","last_active":"2024-08-17T18:10:40.518310+0000","last_peered":"2024-08-17T18:10:40.518310+0000","last_clean":"2024-08-17T18:10:40.518310+0000","last_became_active":"2024-08-17T18:09:29.851109+0000","last_became_peered":"2024-08-17T18:09:29.851109+0000","last_unstale":"2024-08-17T18:10:40.518310+0000","last_undegraded":"2024-08-17T18:10:40.518310+0000","last_fullsized":"2024-08-17T18:10:40.518310+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:42.054532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":65,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347112+0000","last_change":"2024-08-17T18:03:16.089811+0000","last_active":"2024-08-17T18:10:34.347112+0000","last_peered":"2024-08-17T18:10:34.347112+0000","last_clean":"2024-08-17T18:10:34.347112+0000","last_became_active":"2024-08-17T18:03:16.089571+0000","last_became_peered":"2024-08-17T18:03:16.089571+0000","last_unstale":"2024-08-17T18:10:34.347112+0000","last_undegraded":"2024-08-17T18:10:34.347112+0000","last_fullsized":"2024-08-17T18:10:34.347112+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:04.630361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":47,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347253+0000","last_change":"2024-08-17T18:08:57.577055+0000","last_active":"2024-08-17T18:10:34.347253+0000","last_peered":"2024-08-17T18:10:34.347253+0000","last_clean":"2024-08-17T18:10:34.347253+0000","last_became_active":"2024-08-17T18:08:57.573659+0000","last_became_peered":"2024-08-17T18:08:57.573659+0000","last_unstale":"2024-08-17T18:10:34.347253+0000","last_undegraded":"2024-08-17T18:10:34.347253+0000","last_fullsized":"2024-08-17T18:10:34.347253+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:01.686752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":829,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.521393+0000","last_change":"2024-08-17T18:09:29.852393+0000","last_active":"2024-08-17T18:10:40.521393+0000","last_peered":"2024-08-17T18:10:40.521393+0000","last_clean":"2024-08-17T18:10:40.521393+0000","last_became_active":"2024-08-17T18:09:29.851947+0000","last_became_peered":"2024-08-17T18:09:29.851947+0000","last_unstale":"2024-08-17T18:10:40.521393+0000","last_undegraded":"2024-08-17T18:10:40.521393+0000","last_fullsized":"2024-08-17T18:10:40.521393+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:49.145327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":2139,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518690+0000","last_change":"2024-08-17T18:09:29.863241+0000","last_active":"2024-08-17T18:10:40.518690+0000","last_peered":"2024-08-17T18:10:40.518690+0000","last_clean":"2024-08-17T18:10:40.518690+0000","last_became_active":"2024-08-17T18:09:29.862369+0000","last_became_peered":"2024-08-17T18:09:29.862369+0000","last_unstale":"2024-08-17T18:10:40.518690+0000","last_undegraded":"2024-08-17T18:10:40.518690+0000","last_fullsized":"2024-08-17T18:10:40.518690+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:30.518551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":576,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.525683+0000","last_change":"2024-08-17T18:09:29.896233+0000","last_active":"2024-08-17T18:10:40.525683+0000","last_peered":"2024-08-17T18:10:40.525683+0000","last_clean":"2024-08-17T18:10:40.525683+0000","last_became_active":"2024-08-17T18:09:29.895718+0000","last_became_peered":"2024-08-17T18:09:29.895718+0000","last_unstale":"2024-08-17T18:10:40.525683+0000","last_undegraded":"2024-08-17T18:10:40.525683+0000","last_fullsized":"2024-08-17T18:10:40.525683+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:03.278878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":49,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516722+0000","last_change":"2024-08-17T18:08:57.574015+0000","last_active":"2024-08-17T18:10:40.516722+0000","last_peered":"2024-08-17T18:10:40.516722+0000","last_clean":"2024-08-17T18:10:40.516722+0000","last_became_active":"2024-08-17T18:08:57.567973+0000","last_became_peered":"2024-08-17T18:08:57.567973+0000","last_unstale":"2024-08-17T18:10:40.516722+0000","last_undegraded":"2024-08-17T18:10:40.516722+0000","last_fullsized":"2024-08-17T18:10:40.516722+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:06.251764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1996,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518414+0000","last_change":"2024-08-17T18:09:29.917203+0000","last_active":"2024-08-17T18:10:40.518414+0000","last_peered":"2024-08-17T18:10:40.518414+0000","last_clean":"2024-08-17T18:10:40.518414+0000","last_became_active":"2024-08-17T18:09:29.916844+0000","last_became_peered":"2024-08-17T18:09:29.916844+0000","last_unstale":"2024-08-17T18:10:40.518414+0000","last_undegraded":"2024-08-17T18:10:40.518414+0000","last_fullsized":"2024-08-17T18:10:40.518414+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.453720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1902,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.519425+0000","last_change":"2024-08-17T18:09:28.859255+0000","last_active":"2024-08-17T18:10:40.519425+0000","last_peered":"2024-08-17T18:10:40.519425+0000","last_clean":"2024-08-17T18:10:40.519425+0000","last_became_active":"2024-08-17T18:09:28.858537+0000","last_became_peered":"2024-08-17T18:09:28.858537+0000","last_unstale":"2024-08-17T18:10:40.519425+0000","last_undegraded":"2024-08-17T18:10:40.519425+0000","last_fullsized":"2024-08-17T18:10:40.519425+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:15.364687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2162,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516920+0000","last_change":"2024-08-17T18:09:28.868303+0000","last_active":"2024-08-17T18:10:40.516920+0000","last_peered":"2024-08-17T18:10:40.516920+0000","last_clean":"2024-08-17T18:10:40.516920+0000","last_became_active":"2024-08-17T18:09:28.867989+0000","last_became_peered":"2024-08-17T18:09:28.867989+0000","last_unstale":"2024-08-17T18:10:40.516920+0000","last_undegraded":"2024-08-17T18:10:40.516920+0000","last_fullsized":"2024-08-17T18:10:40.516920+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:58:52.741685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":28,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.512061+0000","last_change":"2024-08-17T18:09:29.842129+0000","last_active":"2024-08-17T18:10:40.512061+0000","last_peered":"2024-08-17T18:10:40.512061+0000","last_clean":"2024-08-17T18:10:40.512061+0000","last_became_active":"2024-08-17T18:09:29.841848+0000","last_became_peered":"2024-08-17T18:09:29.841848+0000","last_unstale":"2024-08-17T18:10:40.512061+0000","last_undegraded":"2024-08-17T18:10:40.512061+0000","last_fullsized":"2024-08-17T18:10:40.512061+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:02.119134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1276,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516194+0000","last_change":"2024-08-17T18:09:29.846509+0000","last_active":"2024-08-17T18:10:40.516194+0000","last_peered":"2024-08-17T18:10:40.516194+0000","last_clean":"2024-08-17T18:10:40.516194+0000","last_became_active":"2024-08-17T18:09:29.844097+0000","last_became_peered":"2024-08-17T18:09:29.844097+0000","last_unstale":"2024-08-17T18:10:40.516194+0000","last_undegraded":"2024-08-17T18:10:40.516194+0000","last_fullsized":"2024-08-17T18:10:40.516194+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:32.340354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1990,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.512155+0000","last_change":"2024-08-17T18:09:29.841335+0000","last_active":"2024-08-17T18:10:40.512155+0000","last_peered":"2024-08-17T18:10:40.512155+0000","last_clean":"2024-08-17T18:10:40.512155+0000","last_became_active":"2024-08-17T18:09:29.840522+0000","last_became_peered":"2024-08-17T18:09:29.840522+0000","last_unstale":"2024-08-17T18:10:40.512155+0000","last_undegraded":"2024-08-17T18:10:40.512155+0000","last_fullsized":"2024-08-17T18:10:40.512155+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:08.015836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2168,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516364+0000","last_change":"2024-08-17T18:09:29.847494+0000","last_active":"2024-08-17T18:10:40.516364+0000","last_peered":"2024-08-17T18:10:40.516364+0000","last_clean":"2024-08-17T18:10:40.516364+0000","last_became_active":"2024-08-17T18:09:29.846973+0000","last_became_peered":"2024-08-17T18:09:29.846973+0000","last_unstale":"2024-08-17T18:10:40.516364+0000","last_undegraded":"2024-08-17T18:10:40.516364+0000","last_fullsized":"2024-08-17T18:10:40.516364+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:49.469921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1949,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.519141+0000","last_change":"2024-08-17T18:09:28.855045+0000","last_active":"2024-08-17T18:10:40.519141+0000","last_peered":"2024-08-17T18:10:40.519141+0000","last_clean":"2024-08-17T18:10:40.519141+0000","last_became_active":"2024-08-17T18:09:28.854607+0000","last_became_peered":"2024-08-17T18:09:28.854607+0000","last_unstale":"2024-08-17T18:10:40.519141+0000","last_undegraded":"2024-08-17T18:10:40.519141+0000","last_fullsized":"2024-08-17T18:10:40.519141+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:13.437895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":677,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347791+0000","last_change":"2024-08-17T18:09:29.866944+0000","last_active":"2024-08-17T18:10:34.347791+0000","last_peered":"2024-08-17T18:10:34.347791+0000","last_clean":"2024-08-17T18:10:34.347791+0000","last_became_active":"2024-08-17T18:09:29.866694+0000","last_became_peered":"2024-08-17T18:09:29.866694+0000","last_unstale":"2024-08-17T18:10:34.347791+0000","last_undegraded":"2024-08-17T18:10:34.347791+0000","last_fullsized":"2024-08-17T18:10:34.347791+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:00:41.342392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1340,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347716+0000","last_change":"2024-08-17T18:09:28.884262+0000","last_active":"2024-08-17T18:10:34.347716+0000","last_peered":"2024-08-17T18:10:34.347716+0000","last_clean":"2024-08-17T18:10:34.347716+0000","last_became_active":"2024-08-17T18:09:28.883734+0000","last_became_peered":"2024-08-17T18:09:28.883734+0000","last_unstale":"2024-08-17T18:10:34.347716+0000","last_undegraded":"2024-08-17T18:10:34.347716+0000","last_fullsized":"2024-08-17T18:10:34.347716+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:10.735504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2052,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516565+0000","last_change":"2024-08-17T18:08:57.571995+0000","last_active":"2024-08-17T18:10:40.516565+0000","last_peered":"2024-08-17T18:10:40.516565+0000","last_clean":"2024-08-17T18:10:40.516565+0000","last_became_active":"2024-08-17T18:08:57.567342+0000","last_became_peered":"2024-08-17T18:08:57.567342+0000","last_unstale":"2024-08-17T18:10:40.516565+0000","last_undegraded":"2024-08-17T18:10:40.516565+0000","last_fullsized":"2024-08-17T18:10:40.516565+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:23.769111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":504,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347342+0000","last_change":"2024-08-17T18:02:53.912837+0000","last_active":"2024-08-17T18:10:34.347342+0000","last_peered":"2024-08-17T18:10:34.347342+0000","last_clean":"2024-08-17T18:10:34.347342+0000","last_became_active":"2024-08-17T18:02:53.883152+0000","last_became_peered":"2024-08-17T18:02:53.883152+0000","last_unstale":"2024-08-17T18:10:34.347342+0000","last_undegraded":"2024-08-17T18:10:34.347342+0000","last_fullsized":"2024-08-17T18:10:34.347342+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:46.261612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":503,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347437+0000","last_change":"2024-08-17T18:09:29.845126+0000","last_active":"2024-08-17T18:10:34.347437+0000","last_peered":"2024-08-17T18:10:34.347437+0000","last_clean":"2024-08-17T18:10:34.347437+0000","last_became_active":"2024-08-17T18:09:29.844836+0000","last_became_peered":"2024-08-17T18:09:29.844836+0000","last_unstale":"2024-08-17T18:10:34.347437+0000","last_undegraded":"2024-08-17T18:10:34.347437+0000","last_fullsized":"2024-08-17T18:10:34.347437+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:44.821894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1289,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.515856+0000","last_change":"2024-08-17T18:02:53.890198+0000","last_active":"2024-08-17T18:10:40.515856+0000","last_peered":"2024-08-17T18:10:40.515856+0000","last_clean":"2024-08-17T18:10:40.515856+0000","last_became_active":"2024-08-17T18:02:53.887007+0000","last_became_peered":"2024-08-17T18:02:53.887007+0000","last_unstale":"2024-08-17T18:10:40.515856+0000","last_undegraded":"2024-08-17T18:10:40.515856+0000","last_fullsized":"2024-08-17T18:10:40.515856+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:25.555909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":594,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347841+0000","last_change":"2024-08-17T18:09:28.884187+0000","last_active":"2024-08-17T18:10:34.347841+0000","last_peered":"2024-08-17T18:10:34.347841+0000","last_clean":"2024-08-17T18:10:34.347841+0000","last_became_active":"2024-08-17T18:09:28.883939+0000","last_became_peered":"2024-08-17T18:09:28.883939+0000","last_unstale":"2024-08-17T18:10:34.347841+0000","last_undegraded":"2024-08-17T18:10:34.347841+0000","last_fullsized":"2024-08-17T18:10:34.347841+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:37:41.951099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1894,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347732+0000","last_change":"2024-08-17T18:09:29.893220+0000","last_active":"2024-08-17T18:10:34.347732+0000","last_peered":"2024-08-17T18:10:34.347732+0000","last_clean":"2024-08-17T18:10:34.347732+0000","last_became_active":"2024-08-17T18:09:29.865699+0000","last_became_peered":"2024-08-17T18:09:29.865699+0000","last_unstale":"2024-08-17T18:10:34.347732+0000","last_undegraded":"2024-08-17T18:10:34.347732+0000","last_fullsized":"2024-08-17T18:10:34.347732+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:40.205658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2009,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.513589+0000","last_change":"2024-08-17T18:09:28.934404+0000","last_active":"2024-08-17T18:10:40.513589+0000","last_peered":"2024-08-17T18:10:40.513589+0000","last_clean":"2024-08-17T18:10:40.513589+0000","last_became_active":"2024-08-17T18:09:28.934001+0000","last_became_peered":"2024-08-17T18:09:28.934001+0000","last_unstale":"2024-08-17T18:10:40.513589+0000","last_undegraded":"2024-08-17T18:10:40.513589+0000","last_fullsized":"2024-08-17T18:10:40.513589+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:04.676812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":2041,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517507+0000","last_change":"2024-08-17T18:09:29.884088+0000","last_active":"2024-08-17T18:10:40.517507+0000","last_peered":"2024-08-17T18:10:40.517507+0000","last_clean":"2024-08-17T18:10:40.517507+0000","last_became_active":"2024-08-17T18:09:29.883027+0000","last_became_peered":"2024-08-17T18:09:29.883027+0000","last_unstale":"2024-08-17T18:10:40.517507+0000","last_undegraded":"2024-08-17T18:10:40.517507+0000","last_fullsized":"2024-08-17T18:10:40.517507+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:22:07.777188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":2011,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517483+0000","last_change":"2024-08-17T18:09:28.935401+0000","last_active":"2024-08-17T18:10:40.517483+0000","last_peered":"2024-08-17T18:10:40.517483+0000","last_clean":"2024-08-17T18:10:40.517483+0000","last_became_active":"2024-08-17T18:09:28.933628+0000","last_became_peered":"2024-08-17T18:09:28.933628+0000","last_unstale":"2024-08-17T18:10:40.517483+0000","last_undegraded":"2024-08-17T18:10:40.517483+0000","last_fullsized":"2024-08-17T18:10:40.517483+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:21:12.714486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2123,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518848+0000","last_change":"2024-08-17T18:08:57.574718+0000","last_active":"2024-08-17T18:10:40.518848+0000","last_peered":"2024-08-17T18:10:40.518848+0000","last_clean":"2024-08-17T18:10:40.518848+0000","last_became_active":"2024-08-17T18:08:57.573515+0000","last_became_peered":"2024-08-17T18:08:57.573515+0000","last_unstale":"2024-08-17T18:10:40.518848+0000","last_undegraded":"2024-08-17T18:10:40.518848+0000","last_fullsized":"2024-08-17T18:10:40.518848+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:04.873717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1353,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347911+0000","last_change":"2024-08-17T18:08:57.568821+0000","last_active":"2024-08-17T18:10:34.347911+0000","last_peered":"2024-08-17T18:10:34.347911+0000","last_clean":"2024-08-17T18:10:34.347911+0000","last_became_active":"2024-08-17T18:08:57.568526+0000","last_became_peered":"2024-08-17T18:08:57.568526+0000","last_unstale":"2024-08-17T18:10:34.347911+0000","last_undegraded":"2024-08-17T18:10:34.347911+0000","last_fullsized":"2024-08-17T18:10:34.347911+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:26.449197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1843,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.515969+0000","last_change":"2024-08-17T18:08:57.571174+0000","last_active":"2024-08-17T18:10:40.515969+0000","last_peered":"2024-08-17T18:10:40.515969+0000","last_clean":"2024-08-17T18:10:40.515969+0000","last_became_active":"2024-08-17T18:08:57.567571+0000","last_became_peered":"2024-08-17T18:08:57.567571+0000","last_unstale":"2024-08-17T18:10:40.515969+0000","last_undegraded":"2024-08-17T18:10:40.515969+0000","last_fullsized":"2024-08-17T18:10:40.515969+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:29.976150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1812,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517147+0000","last_change":"2024-08-17T18:08:57.572368+0000","last_active":"2024-08-17T18:10:40.517147+0000","last_peered":"2024-08-17T18:10:40.517147+0000","last_clean":"2024-08-17T18:10:40.517147+0000","last_became_active":"2024-08-17T18:08:57.568382+0000","last_became_peered":"2024-08-17T18:08:57.568382+0000","last_unstale":"2024-08-17T18:10:40.517147+0000","last_undegraded":"2024-08-17T18:10:40.517147+0000","last_fullsized":"2024-08-17T18:10:40.517147+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:11:19.968079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1860,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516053+0000","last_change":"2024-08-17T18:08:57.572991+0000","last_active":"2024-08-17T18:10:40.516053+0000","last_peered":"2024-08-17T18:10:40.516053+0000","last_clean":"2024-08-17T18:10:40.516053+0000","last_became_active":"2024-08-17T18:08:57.571145+0000","last_became_peered":"2024-08-17T18:08:57.571145+0000","last_unstale":"2024-08-17T18:10:40.516053+0000","last_undegraded":"2024-08-17T18:10:40.516053+0000","last_fullsized":"2024-08-17T18:10:40.516053+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:39.963337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":816,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518213+0000","last_change":"2024-08-17T18:09:28.860288+0000","last_active":"2024-08-17T18:10:40.518213+0000","last_peered":"2024-08-17T18:10:40.518213+0000","last_clean":"2024-08-17T18:10:40.518213+0000","last_became_active":"2024-08-17T18:09:28.860004+0000","last_became_peered":"2024-08-17T18:09:28.860004+0000","last_unstale":"2024-08-17T18:10:40.518213+0000","last_undegraded":"2024-08-17T18:10:40.518213+0000","last_fullsized":"2024-08-17T18:10:40.518213+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:57:33.054055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":609,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.525896+0000","last_change":"2024-08-17T18:09:28.860277+0000","last_active":"2024-08-17T18:10:40.525896+0000","last_peered":"2024-08-17T18:10:40.525896+0000","last_clean":"2024-08-17T18:10:40.525896+0000","last_became_active":"2024-08-17T18:09:28.859999+0000","last_became_peered":"2024-08-17T18:09:28.859999+0000","last_unstale":"2024-08-17T18:10:40.525896+0000","last_undegraded":"2024-08-17T18:10:40.525896+0000","last_fullsized":"2024-08-17T18:10:40.525896+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:27.946526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1514,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518549+0000","last_change":"2024-08-17T18:09:29.863669+0000","last_active":"2024-08-17T18:10:40.518549+0000","last_peered":"2024-08-17T18:10:40.518549+0000","last_clean":"2024-08-17T18:10:40.518549+0000","last_became_active":"2024-08-17T18:09:29.862781+0000","last_became_peered":"2024-08-17T18:09:29.862781+0000","last_unstale":"2024-08-17T18:10:40.518549+0000","last_undegraded":"2024-08-17T18:10:40.518549+0000","last_fullsized":"2024-08-17T18:10:40.518549+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:14.073233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":95,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517305+0000","last_change":"2024-08-17T18:09:29.843574+0000","last_active":"2024-08-17T18:10:40.517305+0000","last_peered":"2024-08-17T18:10:40.517305+0000","last_clean":"2024-08-17T18:10:40.517305+0000","last_became_active":"2024-08-17T18:09:29.843097+0000","last_became_peered":"2024-08-17T18:09:29.843097+0000","last_unstale":"2024-08-17T18:10:40.517305+0000","last_undegraded":"2024-08-17T18:10:40.517305+0000","last_fullsized":"2024-08-17T18:10:40.517305+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:46.740888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1859,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516555+0000","last_change":"2024-08-17T18:09:29.847264+0000","last_active":"2024-08-17T18:10:40.516555+0000","last_peered":"2024-08-17T18:10:40.516555+0000","last_clean":"2024-08-17T18:10:40.516555+0000","last_became_active":"2024-08-17T18:09:29.846226+0000","last_became_peered":"2024-08-17T18:09:29.846226+0000","last_unstale":"2024-08-17T18:10:40.516555+0000","last_undegraded":"2024-08-17T18:10:40.516555+0000","last_fullsized":"2024-08-17T18:10:40.516555+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:44.135977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2059,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518426+0000","last_change":"2024-08-17T18:09:29.845209+0000","last_active":"2024-08-17T18:10:40.518426+0000","last_peered":"2024-08-17T18:10:40.518426+0000","last_clean":"2024-08-17T18:10:40.518426+0000","last_became_active":"2024-08-17T18:09:29.844910+0000","last_became_peered":"2024-08-17T18:09:29.844910+0000","last_unstale":"2024-08-17T18:10:40.518426+0000","last_undegraded":"2024-08-17T18:10:40.518426+0000","last_fullsized":"2024-08-17T18:10:40.518426+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:14.586354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1932,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.519600+0000","last_change":"2024-08-17T18:09:28.859050+0000","last_active":"2024-08-17T18:10:40.519600+0000","last_peered":"2024-08-17T18:10:40.519600+0000","last_clean":"2024-08-17T18:10:40.519600+0000","last_became_active":"2024-08-17T18:09:28.858214+0000","last_became_peered":"2024-08-17T18:09:28.858214+0000","last_unstale":"2024-08-17T18:10:40.519600+0000","last_undegraded":"2024-08-17T18:10:40.519600+0000","last_fullsized":"2024-08-17T18:10:40.519600+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:54.146592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2184,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517068+0000","last_change":"2024-08-17T18:08:57.572610+0000","last_active":"2024-08-17T18:10:40.517068+0000","last_peered":"2024-08-17T18:10:40.517068+0000","last_clean":"2024-08-17T18:10:40.517068+0000","last_became_active":"2024-08-17T18:08:57.572308+0000","last_became_peered":"2024-08-17T18:08:57.572308+0000","last_unstale":"2024-08-17T18:10:40.517068+0000","last_undegraded":"2024-08-17T18:10:40.517068+0000","last_fullsized":"2024-08-17T18:10:40.517068+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:11.642831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2168,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.515939+0000","last_change":"2024-08-17T18:09:28.842838+0000","last_active":"2024-08-17T18:10:40.515939+0000","last_peered":"2024-08-17T18:10:40.515939+0000","last_clean":"2024-08-17T18:10:40.515939+0000","last_became_active":"2024-08-17T18:09:28.842617+0000","last_became_peered":"2024-08-17T18:09:28.842617+0000","last_unstale":"2024-08-17T18:10:40.515939+0000","last_undegraded":"2024-08-17T18:10:40.515939+0000","last_fullsized":"2024-08-17T18:10:40.515939+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:00.016002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1374,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347197+0000","last_change":"2024-08-17T18:09:29.845011+0000","last_active":"2024-08-17T18:10:34.347197+0000","last_peered":"2024-08-17T18:10:34.347197+0000","last_clean":"2024-08-17T18:10:34.347197+0000","last_became_active":"2024-08-17T18:09:29.844358+0000","last_became_peered":"2024-08-17T18:09:29.844358+0000","last_unstale":"2024-08-17T18:10:34.347197+0000","last_undegraded":"2024-08-17T18:10:34.347197+0000","last_fullsized":"2024-08-17T18:10:34.347197+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:41:27.792901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1222,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347813+0000","last_change":"2024-08-17T18:09:28.878263+0000","last_active":"2024-08-17T18:10:34.347813+0000","last_peered":"2024-08-17T18:10:34.347813+0000","last_clean":"2024-08-17T18:10:34.347813+0000","last_became_active":"2024-08-17T18:09:28.878025+0000","last_became_peered":"2024-08-17T18:09:28.878025+0000","last_unstale":"2024-08-17T18:10:34.347813+0000","last_undegraded":"2024-08-17T18:10:34.347813+0000","last_fullsized":"2024-08-17T18:10:34.347813+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:37.211100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1960,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517271+0000","last_change":"2024-08-17T18:09:29.840134+0000","last_active":"2024-08-17T18:10:40.517271+0000","last_peered":"2024-08-17T18:10:40.517271+0000","last_clean":"2024-08-17T18:10:40.517271+0000","last_became_active":"2024-08-17T18:09:29.839848+0000","last_became_peered":"2024-08-17T18:09:29.839848+0000","last_unstale":"2024-08-17T18:10:40.517271+0000","last_undegraded":"2024-08-17T18:10:40.517271+0000","last_fullsized":"2024-08-17T18:10:40.517271+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:08:08.717851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":129,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347907+0000","last_change":"2024-08-17T18:09:28.847633+0000","last_active":"2024-08-17T18:10:34.347907+0000","last_peered":"2024-08-17T18:10:34.347907+0000","last_clean":"2024-08-17T18:10:34.347907+0000","last_became_active":"2024-08-17T18:09:28.847341+0000","last_became_peered":"2024-08-17T18:09:28.847341+0000","last_unstale":"2024-08-17T18:10:34.347907+0000","last_undegraded":"2024-08-17T18:10:34.347907+0000","last_fullsized":"2024-08-17T18:10:34.347907+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:51:59.625154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1714,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.346931+0000","last_change":"2024-08-17T18:09:00.609431+0000","last_active":"2024-08-17T18:10:34.346931+0000","last_peered":"2024-08-17T18:10:34.346931+0000","last_clean":"2024-08-17T18:10:34.346931+0000","last_became_active":"2024-08-17T18:09:00.609167+0000","last_became_peered":"2024-08-17T18:09:00.609167+0000","last_unstale":"2024-08-17T18:10:34.346931+0000","last_undegraded":"2024-08-17T18:10:34.346931+0000","last_fullsized":"2024-08-17T18:10:34.346931+0000","mapping_epoch":1338,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:39.579970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1840,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517857+0000","last_change":"2024-08-17T18:09:29.837502+0000","last_active":"2024-08-17T18:10:40.517857+0000","last_peered":"2024-08-17T18:10:40.517857+0000","last_clean":"2024-08-17T18:10:40.517857+0000","last_became_active":"2024-08-17T18:09:29.837243+0000","last_became_peered":"2024-08-17T18:09:29.837243+0000","last_unstale":"2024-08-17T18:10:40.517857+0000","last_undegraded":"2024-08-17T18:10:40.517857+0000","last_fullsized":"2024-08-17T18:10:40.517857+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:47.377505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2038,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.513723+0000","last_change":"2024-08-17T18:09:29.859415+0000","last_active":"2024-08-17T18:10:40.513723+0000","last_peered":"2024-08-17T18:10:40.513723+0000","last_clean":"2024-08-17T18:10:40.513723+0000","last_became_active":"2024-08-17T18:09:29.859138+0000","last_became_peered":"2024-08-17T18:09:29.859138+0000","last_unstale":"2024-08-17T18:10:40.513723+0000","last_undegraded":"2024-08-17T18:10:40.513723+0000","last_fullsized":"2024-08-17T18:10:40.513723+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:31.178915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":489,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517138+0000","last_change":"2024-08-17T18:09:29.883834+0000","last_active":"2024-08-17T18:10:40.517138+0000","last_peered":"2024-08-17T18:10:40.517138+0000","last_clean":"2024-08-17T18:10:40.517138+0000","last_became_active":"2024-08-17T18:09:29.882638+0000","last_became_peered":"2024-08-17T18:09:29.882638+0000","last_unstale":"2024-08-17T18:10:40.517138+0000","last_undegraded":"2024-08-17T18:10:40.517138+0000","last_fullsized":"2024-08-17T18:10:40.517138+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:07:10.188684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":614,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347205+0000","last_change":"2024-08-17T18:09:29.857250+0000","last_active":"2024-08-17T18:10:34.347205+0000","last_peered":"2024-08-17T18:10:34.347205+0000","last_clean":"2024-08-17T18:10:34.347205+0000","last_became_active":"2024-08-17T18:09:29.857038+0000","last_became_peered":"2024-08-17T18:09:29.857038+0000","last_unstale":"2024-08-17T18:10:34.347205+0000","last_undegraded":"2024-08-17T18:10:34.347205+0000","last_fullsized":"2024-08-17T18:10:34.347205+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:02:54.028575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":602,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.515309+0000","last_change":"2024-08-17T18:08:57.571270+0000","last_active":"2024-08-17T18:10:40.515309+0000","last_peered":"2024-08-17T18:10:40.515309+0000","last_clean":"2024-08-17T18:10:40.515309+0000","last_became_active":"2024-08-17T18:08:57.567670+0000","last_became_peered":"2024-08-17T18:08:57.567670+0000","last_unstale":"2024-08-17T18:10:40.515309+0000","last_undegraded":"2024-08-17T18:10:40.515309+0000","last_fullsized":"2024-08-17T18:10:40.515309+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:19:42.617321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1173,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517444+0000","last_change":"2024-08-17T18:09:29.839260+0000","last_active":"2024-08-17T18:10:40.517444+0000","last_peered":"2024-08-17T18:10:40.517444+0000","last_clean":"2024-08-17T18:10:40.517444+0000","last_became_active":"2024-08-17T18:09:29.838953+0000","last_became_peered":"2024-08-17T18:09:29.838953+0000","last_unstale":"2024-08-17T18:10:40.517444+0000","last_undegraded":"2024-08-17T18:10:40.517444+0000","last_fullsized":"2024-08-17T18:10:40.517444+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:44.014083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":762,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.532021+0000","last_change":"2024-08-17T18:09:29.851347+0000","last_active":"2024-08-17T18:10:40.532021+0000","last_peered":"2024-08-17T18:10:40.532021+0000","last_clean":"2024-08-17T18:10:40.532021+0000","last_became_active":"2024-08-17T18:09:29.851046+0000","last_became_peered":"2024-08-17T18:09:29.851046+0000","last_unstale":"2024-08-17T18:10:40.532021+0000","last_undegraded":"2024-08-17T18:10:40.532021+0000","last_fullsized":"2024-08-17T18:10:40.532021+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:21.350703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2039,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.521227+0000","last_change":"2024-08-17T18:09:29.912446+0000","last_active":"2024-08-17T18:10:40.521227+0000","last_peered":"2024-08-17T18:10:40.521227+0000","last_clean":"2024-08-17T18:10:40.521227+0000","last_became_active":"2024-08-17T18:09:29.910703+0000","last_became_peered":"2024-08-17T18:09:29.910703+0000","last_unstale":"2024-08-17T18:10:40.521227+0000","last_undegraded":"2024-08-17T18:10:40.521227+0000","last_fullsized":"2024-08-17T18:10:40.521227+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.250608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2205,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518937+0000","last_change":"2024-08-17T18:09:29.863361+0000","last_active":"2024-08-17T18:10:40.518937+0000","last_peered":"2024-08-17T18:10:40.518937+0000","last_clean":"2024-08-17T18:10:40.518937+0000","last_became_active":"2024-08-17T18:09:29.862503+0000","last_became_peered":"2024-08-17T18:09:29.862503+0000","last_unstale":"2024-08-17T18:10:40.518937+0000","last_undegraded":"2024-08-17T18:10:40.518937+0000","last_fullsized":"2024-08-17T18:10:40.518937+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:25.908232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":28,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.525585+0000","last_change":"2024-08-17T18:09:29.895991+0000","last_active":"2024-08-17T18:10:40.525585+0000","last_peered":"2024-08-17T18:10:40.525585+0000","last_clean":"2024-08-17T18:10:40.525585+0000","last_became_active":"2024-08-17T18:09:29.895274+0000","last_became_peered":"2024-08-17T18:09:29.895274+0000","last_unstale":"2024-08-17T18:10:40.525585+0000","last_undegraded":"2024-08-17T18:10:40.525585+0000","last_fullsized":"2024-08-17T18:10:40.525585+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:19:26.420016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":496,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347209+0000","last_change":"2024-08-17T18:09:29.844686+0000","last_active":"2024-08-17T18:10:34.347209+0000","last_peered":"2024-08-17T18:10:34.347209+0000","last_clean":"2024-08-17T18:10:34.347209+0000","last_became_active":"2024-08-17T18:09:29.844341+0000","last_became_peered":"2024-08-17T18:09:29.844341+0000","last_unstale":"2024-08-17T18:10:34.347209+0000","last_undegraded":"2024-08-17T18:10:34.347209+0000","last_fullsized":"2024-08-17T18:10:34.347209+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:38.872767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1333,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347819+0000","last_change":"2024-08-17T18:09:28.884310+0000","last_active":"2024-08-17T18:10:34.347819+0000","last_peered":"2024-08-17T18:10:34.347819+0000","last_clean":"2024-08-17T18:10:34.347819+0000","last_became_active":"2024-08-17T18:09:28.883921+0000","last_became_peered":"2024-08-17T18:09:28.883921+0000","last_unstale":"2024-08-17T18:10:34.347819+0000","last_undegraded":"2024-08-17T18:10:34.347819+0000","last_fullsized":"2024-08-17T18:10:34.347819+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:05.272563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":771,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.519515+0000","last_change":"2024-08-17T18:09:29.837238+0000","last_active":"2024-08-17T18:10:40.519515+0000","last_peered":"2024-08-17T18:10:40.519515+0000","last_clean":"2024-08-17T18:10:40.519515+0000","last_became_active":"2024-08-17T18:09:29.836760+0000","last_became_peered":"2024-08-17T18:09:29.836760+0000","last_unstale":"2024-08-17T18:10:40.519515+0000","last_undegraded":"2024-08-17T18:10:40.519515+0000","last_fullsized":"2024-08-17T18:10:40.519515+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:38:52.862967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1348,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347914+0000","last_change":"2024-08-17T18:09:29.871872+0000","last_active":"2024-08-17T18:10:34.347914+0000","last_peered":"2024-08-17T18:10:34.347914+0000","last_clean":"2024-08-17T18:10:34.347914+0000","last_became_active":"2024-08-17T18:09:29.845111+0000","last_became_peered":"2024-08-17T18:09:29.845111+0000","last_unstale":"2024-08-17T18:10:34.347914+0000","last_undegraded":"2024-08-17T18:10:34.347914+0000","last_fullsized":"2024-08-17T18:10:34.347914+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:48.183250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":245,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.346850+0000","last_change":"2024-08-17T18:09:29.844786+0000","last_active":"2024-08-17T18:10:34.346850+0000","last_peered":"2024-08-17T18:10:34.346850+0000","last_clean":"2024-08-17T18:10:34.346850+0000","last_became_active":"2024-08-17T18:09:29.844270+0000","last_became_peered":"2024-08-17T18:09:29.844270+0000","last_unstale":"2024-08-17T18:10:34.346850+0000","last_undegraded":"2024-08-17T18:10:34.346850+0000","last_fullsized":"2024-08-17T18:10:34.346850+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:51.866680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1364,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347825+0000","last_change":"2024-08-17T18:09:29.886391+0000","last_active":"2024-08-17T18:10:34.347825+0000","last_peered":"2024-08-17T18:10:34.347825+0000","last_clean":"2024-08-17T18:10:34.347825+0000","last_became_active":"2024-08-17T18:09:29.844642+0000","last_became_peered":"2024-08-17T18:09:29.844642+0000","last_unstale":"2024-08-17T18:10:34.347825+0000","last_undegraded":"2024-08-17T18:10:34.347825+0000","last_fullsized":"2024-08-17T18:10:34.347825+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:07.988100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":185,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518637+0000","last_change":"2024-08-17T18:09:28.941223+0000","last_active":"2024-08-17T18:10:40.518637+0000","last_peered":"2024-08-17T18:10:40.518637+0000","last_clean":"2024-08-17T18:10:40.518637+0000","last_became_active":"2024-08-17T18:09:28.940857+0000","last_became_peered":"2024-08-17T18:09:28.940857+0000","last_unstale":"2024-08-17T18:10:40.518637+0000","last_undegraded":"2024-08-17T18:10:40.518637+0000","last_fullsized":"2024-08-17T18:10:40.518637+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:48.809420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2036,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517945+0000","last_change":"2024-08-17T18:09:29.838873+0000","last_active":"2024-08-17T18:10:40.517945+0000","last_peered":"2024-08-17T18:10:40.517945+0000","last_clean":"2024-08-17T18:10:40.517945+0000","last_became_active":"2024-08-17T18:09:29.838560+0000","last_became_peered":"2024-08-17T18:09:29.838560+0000","last_unstale":"2024-08-17T18:10:40.517945+0000","last_undegraded":"2024-08-17T18:10:40.517945+0000","last_fullsized":"2024-08-17T18:10:40.517945+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:00.103741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1324,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347339+0000","last_change":"2024-08-17T18:09:28.844170+0000","last_active":"2024-08-17T18:10:34.347339+0000","last_peered":"2024-08-17T18:10:34.347339+0000","last_clean":"2024-08-17T18:10:34.347339+0000","last_became_active":"2024-08-17T18:09:28.843862+0000","last_became_peered":"2024-08-17T18:09:28.843862+0000","last_unstale":"2024-08-17T18:10:34.347339+0000","last_undegraded":"2024-08-17T18:10:34.347339+0000","last_fullsized":"2024-08-17T18:10:34.347339+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:43:19.483326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2159,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516694+0000","last_change":"2024-08-17T18:09:28.864923+0000","last_active":"2024-08-17T18:10:40.516694+0000","last_peered":"2024-08-17T18:10:40.516694+0000","last_clean":"2024-08-17T18:10:40.516694+0000","last_became_active":"2024-08-17T18:09:28.864707+0000","last_became_peered":"2024-08-17T18:09:28.864707+0000","last_unstale":"2024-08-17T18:10:40.516694+0000","last_undegraded":"2024-08-17T18:10:40.516694+0000","last_fullsized":"2024-08-17T18:10:40.516694+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:27.507329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2160,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517395+0000","last_change":"2024-08-17T18:09:28.846741+0000","last_active":"2024-08-17T18:10:40.517395+0000","last_peered":"2024-08-17T18:10:40.517395+0000","last_clean":"2024-08-17T18:10:40.517395+0000","last_became_active":"2024-08-17T18:09:28.846478+0000","last_became_peered":"2024-08-17T18:09:28.846478+0000","last_unstale":"2024-08-17T18:10:40.517395+0000","last_undegraded":"2024-08-17T18:10:40.517395+0000","last_fullsized":"2024-08-17T18:10:40.517395+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:34.247517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":730,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347696+0000","last_change":"2024-08-17T18:03:16.091420+0000","last_active":"2024-08-17T18:10:34.347696+0000","last_peered":"2024-08-17T18:10:34.347696+0000","last_clean":"2024-08-17T18:10:34.347696+0000","last_became_active":"2024-08-17T18:03:16.091113+0000","last_became_peered":"2024-08-17T18:03:16.091113+0000","last_unstale":"2024-08-17T18:10:34.347696+0000","last_undegraded":"2024-08-17T18:10:34.347696+0000","last_fullsized":"2024-08-17T18:10:34.347696+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:47.770911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1338,"reported_epoch":1352,"state":"active+clean","last_fresh":"2024-08-17T18:10:34.347808+0000","last_change":"2024-08-17T18:02:53.893927+0000","last_active":"2024-08-17T18:10:34.347808+0000","last_peered":"2024-08-17T18:10:34.347808+0000","last_clean":"2024-08-17T18:10:34.347808+0000","last_became_active":"2024-08-17T18:02:53.883188+0000","last_became_peered":"2024-08-17T18:02:53.883188+0000","last_unstale":"2024-08-17T18:10:34.347808+0000","last_undegraded":"2024-08-17T18:10:34.347808+0000","last_fullsized":"2024-08-17T18:10:34.347808+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:19.296965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2036,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.525757+0000","last_change":"2024-08-17T18:09:29.840086+0000","last_active":"2024-08-17T18:10:40.525757+0000","last_peered":"2024-08-17T18:10:40.525757+0000","last_clean":"2024-08-17T18:10:40.525757+0000","last_became_active":"2024-08-17T18:09:29.839525+0000","last_became_peered":"2024-08-17T18:09:29.839525+0000","last_unstale":"2024-08-17T18:10:40.525757+0000","last_undegraded":"2024-08-17T18:10:40.525757+0000","last_fullsized":"2024-08-17T18:10:40.525757+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:38.316235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":48,"num_pg":16,"stat_sum":{"num_bytes":3972726808,"num_objects":60620,"num_object_clones":0,"num_object_copies":181860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":36426,"num_objects_misplaced":24066,"num_objects_unfound":0,"num_objects_dirty":60620,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":203306,"num_write_kb":8445569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9075,"num_bytes_recovered":594477056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9700777984,"data_stored":9700769840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83796,"ondisk_log_size":83796,"up":48,"acting":37,"num_store_stats":5},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":10970,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":155648,"data_stored":27490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234598,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3870312,"kb_used_data":2976736,"kb_used_omap":1,"kb_used_meta":893566,"kb_avail":89874840,"statfs":{"total":95995035648,"available":92031836160,"internally_reserved":0,"allocated":3048177664,"data_stored":3043759569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915011928},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568496233,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757871,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1343,"seq":5768141078545,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328660,"kb_used_data":187084,"kb_used_omap":112,"kb_used_meta":141455,"kb_avail":93416492,"statfs":{"total":95995035648,"available":95658487808,"internally_reserved":0,"allocated":191574016,"data_stored":187171637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":115373,"internal_metadata":144850259},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405724,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3846940,"kb_used_data":2885268,"kb_used_omap":61,"kb_used_meta":961538,"kb_avail":89898212,"statfs":{"total":95995035648,"available":92055769088,"internally_reserved":0,"allocated":2954514432,"data_stored":2950108998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":984615864},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":0,"seq":2173253452258,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474949,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2641832,"kb_used_data":1798368,"kb_used_omap":119,"kb_used_meta":843336,"kb_avail":91103320,"statfs":{"total":95995035648,"available":93289799680,"internally_reserved":0,"allocated":1841528832,"data_stored":1837111295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":863576798},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019506,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1817509888,"data_stored":1817509888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1763115008,"data_stored":1763115008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2931822592,"data_stored":2931818520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":180748288,"data_stored":180748288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3007582208,"data_stored":3007578136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:10:54.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:10:54.268 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:54.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:54.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:54.370 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:54.393 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:54.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:54.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:54.494 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:54.519 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:54.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:54.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:54.620 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:54.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:54.642 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:54.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:54.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:54 smithi130 ceph-mon[105521]: pgmap v2442: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.4 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 772 op/s; 32692/165030 objects degraded (19.810%); 20926/165030 objects misplaced (12.680%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T18:10:54.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:54 smithi130 ceph-mon[110068]: pgmap v2442: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.4 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 772 op/s; 32692/165030 objects degraded (19.810%); 20926/165030 objects misplaced (12.680%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T18:10:54.743 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:54.764 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:54.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:54.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:54.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:54 smithi179 ceph-mon[93519]: pgmap v2442: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.4 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 772 op/s; 32692/165030 objects degraded (19.810%); 20926/165030 objects misplaced (12.680%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T18:10:54.866 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:54.908 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:54.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:54.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:55.010 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:55.033 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:55.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:55.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:55.134 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:55.157 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:55.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:55.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:55.258 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:55.314 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:55.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:55.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:55.416 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:55.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:55.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:55.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:55.540 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:55.581 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:55.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:55.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:55.599 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:10:55.600 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:10:55.649 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:10:55.649 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:10:55.683 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:55.695 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:10:55.695 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:10:55.720 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:55.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:55.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:55.736 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:10:55.736 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:10:55.774 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:10:55.774 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:10:55.817 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:10:55.817 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:10:55.822 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:55.853 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:10:55.853 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:10:55.872 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:10:55.872 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T18:10:55.872 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T18:10:55.872 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T18:10:55.872 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:10:55.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:55.874 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:55.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:55.886 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:10:55.886 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:10:55.975 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:55.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:55.999 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:55.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:56.100 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:56.138 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:56.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:56.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:56.240 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:56.277 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:56.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:56.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:56.379 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:56.400 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:56.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:56.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:56.501 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:56.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:56.524 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:56.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:56.625 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:56 smithi130 ceph-mon[105521]: pgmap v2443: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.3 GiB data, 9.4 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 731 op/s; 31568/160305 objects degraded (19.692%); 20699/160305 objects misplaced (12.912%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T18:10:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:56 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/776360264' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:10:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 31568/160305 objects degraded (19.692%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:56 smithi130 ceph-mon[110068]: pgmap v2443: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.3 GiB data, 9.4 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 731 op/s; 31568/160305 objects degraded (19.692%); 20699/160305 objects misplaced (12.912%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T18:10:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:56 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/776360264' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:10:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 31568/160305 objects degraded (19.692%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:56.671 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:56.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:56.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:56.772 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:56.814 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:56.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:56.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:56.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:56 smithi179 ceph-mon[93519]: pgmap v2443: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.3 GiB data, 9.4 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 731 op/s; 31568/160305 objects degraded (19.692%); 20699/160305 objects misplaced (12.912%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T18:10:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:56 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/776360264' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:10:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 31568/160305 objects degraded (19.692%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:10:56.915 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:10:56.937 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:56.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:56.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:10:57.038 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:57.059 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:57.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:57.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:57.161 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:57.183 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:57.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:57.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:57.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:57.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:57.320 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:57.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:57.421 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:57.464 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:57.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:57.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:57.566 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:57.587 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:57.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:57.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:57.689 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:57.735 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:57.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:57.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:57.836 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:57.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:10:57.858 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:57.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:57.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:57.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:10:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:10:57.960 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:57.997 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:57.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:57.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:58.099 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:58.145 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:58.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:58.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:58.247 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:58.269 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:58.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:58.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:58.371 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:58.412 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T18:10:58.420 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:58.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:58.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:58.522 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:10:58.544 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:58.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:58.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:10:58.599 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:58.646 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:58.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:58 smithi130 ceph-mon[105521]: pgmap v2444: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.3 GiB data, 9.4 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 600 op/s; 31568/160305 objects degraded (19.692%); 20699/160305 objects misplaced (12.912%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T18:10:58.671 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:58 smithi130 ceph-mon[110068]: pgmap v2444: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.3 GiB data, 9.4 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 600 op/s; 31568/160305 objects degraded (19.692%); 20699/160305 objects misplaced (12.912%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T18:10:58.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:58.696 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:58.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:58.797 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:10:58.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:58 smithi179 ceph-mon[93519]: pgmap v2444: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.3 GiB data, 9.4 GiB used, 348 GiB / 358 GiB avail; 0 B/s wr, 600 op/s; 31568/160305 objects degraded (19.692%); 20699/160305 objects misplaced (12.912%); 2.7 MiB/s, 43 objects/s recovering 2024-08-17T18:10:58.856 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:58.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:58.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:10:58.958 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:59.028 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:59.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:59.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:59.130 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:10:59.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:59.188 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:59.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:10:59.234 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:10:59.290 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:10:59.363 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:59.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:59.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:10:59.465 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:10:59.487 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:59.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:59.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:10:59.531 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:10:59.556 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:10:59.588 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:10:59.610 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:10:59.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:59.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:10:59.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:59.757 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:59.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:59.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:10:59.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:59 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/826379024' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:10:59.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:10:59 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:10:59.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:59 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/826379024' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:10:59.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:10:59 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:10:59.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:59 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/826379024' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:10:59.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:10:59 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:10:59.859 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:10:59.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:10:59.918 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:10:59.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:00.019 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:00.041 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:00.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:00.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:00.118 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:10:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:10:59] "GET /metrics HTTP/1.1" 200 36809 "" "Prometheus/2.51.0" 2024-08-17T18:11:00.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:00.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:00.182 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:00.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:00.283 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:00.304 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:00.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:00.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:00.406 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:00.428 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:00.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:00.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:00.529 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:00.549 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:11:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:11:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:11:00.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:00.572 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:00.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:00.672 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:00.722 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:00.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:00.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:00.824 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:00.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:00 smithi179 ceph-mon[93519]: pgmap v2445: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.1 GiB data, 9.1 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 827 op/s; 29955/152121 objects degraded (19.692%); 19152/152121 objects misplaced (12.590%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T18:11:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:00 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:11:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:00 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:11:00.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:00 smithi179 ceph-mon[93519]: osdmap e1354: 8 total, 8 up, 4 in 2024-08-17T18:11:00.876 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:00 smithi130 ceph-mon[105521]: pgmap v2445: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.1 GiB data, 9.1 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 827 op/s; 29955/152121 objects degraded (19.692%); 19152/152121 objects misplaced (12.590%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T18:11:00.876 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:00 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:11:00.876 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:00 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:11:00.876 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:00 smithi130 ceph-mon[105521]: osdmap e1354: 8 total, 8 up, 4 in 2024-08-17T18:11:00.876 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:00 smithi130 ceph-mon[110068]: pgmap v2445: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.1 GiB data, 9.1 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 827 op/s; 29955/152121 objects degraded (19.692%); 19152/152121 objects misplaced (12.590%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T18:11:00.876 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:00 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:11:00.877 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:00 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:11:00.877 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:00 smithi130 ceph-mon[110068]: osdmap e1354: 8 total, 8 up, 4 in 2024-08-17T18:11:00.878 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:00.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:00.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:00.980 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:01.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:01.002 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:01.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:01.103 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:01.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:01.143 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:01.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:01.244 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:01.294 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:01.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:01.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:01.395 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:01.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:01.436 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:01.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:01.537 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:01.577 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:01.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:01.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:01.679 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:01.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:01.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:01.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:01.823 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:01.835 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 29955/152121 objects degraded (19.692%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:01.835 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 29955/152121 objects degraded (19.692%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:01.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 29955/152121 objects degraded (19.692%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:01.844 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:01.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:01.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:01.946 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:01.968 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:01.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:01.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:02.070 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:02.092 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:02.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:02.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:02.194 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:02.215 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:02.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:02.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:02.318 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:02.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:02.362 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:02.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:02.463 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:02.497 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:02.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:02.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:02.599 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:02.621 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:02.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:02.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:02.722 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:02.764 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:02.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:02.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:02.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:02 smithi179 ceph-mon[93519]: pgmap v2447: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.0 GiB data, 8.9 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 715 op/s; 29330/149550 objects degraded (19.612%); 19152/149550 objects misplaced (12.806%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T18:11:02.865 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:02.905 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:02.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:02.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:02.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:02 smithi130 ceph-mon[105521]: pgmap v2447: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.0 GiB data, 8.9 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 715 op/s; 29330/149550 objects degraded (19.612%); 19152/149550 objects misplaced (12.806%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T18:11:02.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:02 smithi130 ceph-mon[110068]: pgmap v2447: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 3.0 GiB data, 8.9 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 715 op/s; 29330/149550 objects degraded (19.612%); 19152/149550 objects misplaced (12.806%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T18:11:03.007 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:03.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:03.037 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:03.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:03.138 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:03.141 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:11:03.160 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:03.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:03.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:03.262 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:03.283 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:03.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:03.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:03.385 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:03.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:03.407 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:03.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:03.507 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:03.532 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:03.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:03.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:03.633 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:03.655 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:03.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:03.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:03.757 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:03.798 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:03.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:03.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:03.899 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:03.923 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:03.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:03.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:04.025 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:04.046 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:04.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:04.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:04.148 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:04.171 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:04.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:04.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:04.273 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:04.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:04.295 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:04.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:04.396 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:04.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:04.418 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:04.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:04.519 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:04.557 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:04.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:04.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:04.659 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:04.679 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:04.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:04.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:04.781 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:04.829 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:04.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:04.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:04.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:04 smithi179 ceph-mon[93519]: pgmap v2448: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.9 GiB data, 8.8 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 718 op/s; 28211/143451 objects degraded (19.666%); 17817/143451 objects misplaced (12.420%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T18:11:04.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:04 smithi130 ceph-mon[105521]: pgmap v2448: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.9 GiB data, 8.8 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 718 op/s; 28211/143451 objects degraded (19.666%); 17817/143451 objects misplaced (12.420%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T18:11:04.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:04 smithi130 ceph-mon[110068]: pgmap v2448: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.9 GiB data, 8.8 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 718 op/s; 28211/143451 objects degraded (19.666%); 17817/143451 objects misplaced (12.420%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T18:11:04.931 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:04.973 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:04.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:04.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:05.074 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:05.095 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:05.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:05.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:05.197 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:05.232 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:05.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:05.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:05.334 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:05.356 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:05.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:05.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:05.458 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:05.478 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:05.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:05.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:05.580 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:05.605 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:11:05.606 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:11:05.629 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:05.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:05.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:05.731 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:05.772 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:05.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:05.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:05.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:05 smithi179 ceph-mon[93519]: from='client.36816 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:11:05.874 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:05.895 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:05.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:05.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:05.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:05 smithi130 ceph-mon[105521]: from='client.36816 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:11:05.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:05 smithi130 ceph-mon[110068]: from='client.36816 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:11:05.997 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:06.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:06.020 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:06.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:06.120 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:06.143 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:06.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:06.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:06.246 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:06.268 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:06.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:06.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:06.370 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:06.391 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:06.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:06.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:06.484 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2448,"stamp":"2024-08-17T18:11:03.449002+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3132561381,"num_objects":47817,"num_object_clones":0,"num_object_copies":143451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":28211,"num_objects_misplaced":17817,"num_objects_unfound":0,"num_objects_dirty":47817,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":216413,"num_write_kb":8448016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10018,"num_bytes_recovered":656487821,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83932,"ondisk_log_size":83932,"up":267,"acting":256,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":9221088,"kb_used_data":6335872,"kb_used_omap":294,"kb_used_meta":2884825,"kb_avail":365759520,"statfs":{"total":383980142592,"available":374537748480,"internally_reserved":0,"allocated":6487932928,"data_stored":6470275273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":301759,"internal_metadata":2954061121},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,0,3,7,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-471400448,"num_objects":-7193,"num_object_clones":0,"num_object_copies":-21579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-4481,"num_objects_misplaced":-3109,"num_objects_unfound":0,"num_objects_dirty":-7193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7193,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":377,"num_bytes_recovered":24707072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006371"},"pg_stats":[{"pgid":"48.f","version":"1354'13537","reported_seq":23862,"reported_epoch":1354,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:03.108934+0000","last_change":"2024-08-17T18:09:29.893285+0000","last_active":"2024-08-17T18:11:03.108934+0000","last_peered":"2024-08-17T18:11:03.108934+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:28.981495+0000","last_became_peered":"2024-08-17T18:09:28.981495+0000","last_unstale":"2024-08-17T18:11:03.108934+0000","last_undegraded":"2024-08-17T18:09:28.850340+0000","last_fullsized":"2024-08-17T18:09:28.832563+0000","mapping_epoch":1343,"log_start":"1345'9357","ondisk_log_start":"1345'9357","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4180,"log_dups_size":0,"ondisk_log_size":4180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:54.654486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197722112,"num_objects":3017,"num_object_clones":0,"num_object_copies":9051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1034,"num_objects_misplaced":3017,"num_objects_unfound":0,"num_objects_dirty":3017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13537,"num_write_kb":529728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5130,"num_bytes_recovered":336199680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":3017}],"blocked_by":[],"up_primary":3,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.e","version":"1354'13476","reported_seq":13521,"reported_epoch":1354,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:59.769575+0000","last_change":"2024-08-17T18:09:29.965937+0000","last_active":"2024-08-17T18:10:59.769575+0000","last_peered":"2024-08-17T18:10:59.769575+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.950126+0000","last_became_peered":"2024-08-17T18:09:29.950126+0000","last_unstale":"2024-08-17T18:10:59.769575+0000","last_undegraded":"2024-08-17T18:09:29.863735+0000","last_fullsized":"2024-08-17T18:09:29.826958+0000","mapping_epoch":1344,"log_start":"1340'8548","ondisk_log_start":"1340'8548","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4928,"log_dups_size":0,"ondisk_log_size":4928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196345856,"num_objects":2996,"num_object_clones":0,"num_object_copies":8988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2996,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13476,"num_write_kb":527104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2996}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.d","version":"1354'13504","reported_seq":13547,"reported_epoch":1354,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:03.120212+0000","last_change":"2024-08-17T18:09:29.930605+0000","last_active":"2024-08-17T18:11:03.120212+0000","last_peered":"2024-08-17T18:11:03.120212+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.918910+0000","last_became_peered":"2024-08-17T18:09:29.918910+0000","last_unstale":"2024-08-17T18:11:03.120212+0000","last_undegraded":"2024-08-17T18:09:29.832124+0000","last_fullsized":"2024-08-17T18:09:29.818954+0000","mapping_epoch":1344,"log_start":"1345'9324","ondisk_log_start":"1345'9324","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4180,"log_dups_size":0,"ondisk_log_size":4180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:00:55.077466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189988888,"num_objects":2900,"num_object_clones":0,"num_object_copies":8700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2900,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2900,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13504,"num_write_kb":524865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":2900}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.c","version":"1354'13417","reported_seq":13527,"reported_epoch":1354,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:02.973020+0000","last_change":"2024-08-17T18:09:28.996223+0000","last_active":"2024-08-17T18:11:02.973020+0000","last_peered":"2024-08-17T18:11:02.973020+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:28.914112+0000","last_became_peered":"2024-08-17T18:09:28.914112+0000","last_unstale":"2024-08-17T18:11:02.973020+0000","last_undegraded":"2024-08-17T18:09:28.840987+0000","last_fullsized":"2024-08-17T18:09:28.815757+0000","mapping_epoch":1343,"log_start":"1345'9246","ondisk_log_start":"1345'9246","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4171,"log_dups_size":0,"ondisk_log_size":4171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:55.890300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192872448,"num_objects":2943,"num_object_clones":0,"num_object_copies":8829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2935,"num_objects_misplaced":2935,"num_objects_unfound":0,"num_objects_dirty":2943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13417,"num_write_kb":523520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":2943}],"blocked_by":[],"up_primary":3,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.b","version":"1354'13654","reported_seq":13696,"reported_epoch":1354,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:03.115469+0000","last_change":"2024-08-17T18:09:29.921199+0000","last_active":"2024-08-17T18:11:03.115469+0000","last_peered":"2024-08-17T18:11:03.115469+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.912384+0000","last_became_peered":"2024-08-17T18:09:29.912384+0000","last_unstale":"2024-08-17T18:11:03.115469+0000","last_undegraded":"2024-08-17T18:09:29.829736+0000","last_fullsized":"2024-08-17T18:09:29.817434+0000","mapping_epoch":1344,"log_start":"1345'9464","ondisk_log_start":"1345'9464","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4190,"log_dups_size":0,"ondisk_log_size":4190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:22.733621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196608000,"num_objects":3000,"num_object_clones":0,"num_object_copies":9000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3000,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13654,"num_write_kb":532928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":3000}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.a","version":"1354'13400","reported_seq":13466,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.890523+0000","last_change":"2024-08-17T18:07:32.359061+0000","last_active":"2024-08-17T18:10:59.890523+0000","last_peered":"2024-08-17T18:10:59.890523+0000","last_clean":"2024-08-17T18:10:59.890523+0000","last_became_active":"2024-08-17T18:07:32.357107+0000","last_became_peered":"2024-08-17T18:07:32.357107+0000","last_unstale":"2024-08-17T18:10:59.890523+0000","last_undegraded":"2024-08-17T18:10:59.890523+0000","last_fullsized":"2024-08-17T18:10:59.890523+0000","mapping_epoch":1327,"log_start":"1340'8464","ondisk_log_start":"1340'8464","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4936,"log_dups_size":0,"ondisk_log_size":4936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:30:02.061787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195035136,"num_objects":2976,"num_object_clones":0,"num_object_copies":8928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13400,"num_write_kb":524032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.9","version":"1354'13713","reported_seq":13757,"reported_epoch":1354,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:59.897879+0000","last_change":"2024-08-17T18:09:29.968655+0000","last_active":"2024-08-17T18:10:59.897879+0000","last_peered":"2024-08-17T18:10:59.897879+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.948623+0000","last_became_peered":"2024-08-17T18:09:29.948623+0000","last_unstale":"2024-08-17T18:10:59.897879+0000","last_undegraded":"2024-08-17T18:09:29.855385+0000","last_fullsized":"2024-08-17T18:09:29.821477+0000","mapping_epoch":1344,"log_start":"1340'8790","ondisk_log_start":"1340'8790","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4923,"log_dups_size":0,"ondisk_log_size":4923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211222528,"num_objects":3223,"num_object_clones":0,"num_object_copies":9669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3223,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13713,"num_write_kb":541952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":3223}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.8","version":"1353'13453","reported_seq":13548,"reported_epoch":1353,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:58.914355+0000","last_change":"2024-08-17T18:09:29.904036+0000","last_active":"2024-08-17T18:10:58.914355+0000","last_peered":"2024-08-17T18:10:58.914355+0000","last_clean":"2024-08-17T18:09:27.840248+0000","last_became_active":"2024-08-17T18:09:28.942085+0000","last_became_peered":"2024-08-17T18:09:28.942085+0000","last_unstale":"2024-08-17T18:10:58.914355+0000","last_undegraded":"2024-08-17T18:10:58.914355+0000","last_fullsized":"2024-08-17T18:10:58.914355+0000","mapping_epoch":1343,"log_start":"1340'8561","ondisk_log_start":"1340'8561","created":1327,"last_epoch_clean":1337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4892,"log_dups_size":0,"ondisk_log_size":4892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200867840,"num_objects":3065,"num_object_clones":0,"num_object_copies":9195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3065,"num_objects_unfound":0,"num_objects_dirty":3065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13453,"num_write_kb":528576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,1],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":3065}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.7","version":"1354'13503","reported_seq":13568,"reported_epoch":1354,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:03.106277+0000","last_change":"2024-08-17T18:09:28.989307+0000","last_active":"2024-08-17T18:11:03.106277+0000","last_peered":"2024-08-17T18:11:03.106277+0000","last_clean":"2024-08-17T18:09:27.824322+0000","last_became_active":"2024-08-17T18:09:28.981075+0000","last_became_peered":"2024-08-17T18:09:28.981075+0000","last_unstale":"2024-08-17T18:11:03.106277+0000","last_undegraded":"2024-08-17T18:11:03.106277+0000","last_fullsized":"2024-08-17T18:11:03.106277+0000","mapping_epoch":1343,"log_start":"1345'9348","ondisk_log_start":"1345'9348","created":1327,"last_epoch_clean":1337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4155,"log_dups_size":0,"ondisk_log_size":4155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:12:08.456671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195362816,"num_objects":2981,"num_object_clones":0,"num_object_copies":8943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2981,"num_objects_unfound":0,"num_objects_dirty":2981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13503,"num_write_kb":527488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,1],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"1,2,7","objects":2981}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.6","version":"1354'13488","reported_seq":14781,"reported_epoch":1354,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:02.781538+0000","last_change":"2024-08-17T18:09:59.877439+0000","last_active":"2024-08-17T18:11:02.781538+0000","last_peered":"2024-08-17T18:11:02.781538+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:28.962573+0000","last_became_peered":"2024-08-17T18:09:28.962573+0000","last_unstale":"2024-08-17T18:11:02.781538+0000","last_undegraded":"2024-08-17T18:09:28.855040+0000","last_fullsized":"2024-08-17T18:09:28.816313+0000","mapping_epoch":1343,"log_start":"1331'3400","ondisk_log_start":"1331'3400","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":10088,"log_dups_size":0,"ondisk_log_size":10088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191758336,"num_objects":2926,"num_object_clones":0,"num_object_copies":8778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2767,"num_objects_misplaced":2767,"num_objects_unfound":0,"num_objects_dirty":2926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13488,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":403,"num_bytes_recovered":26411008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":2926}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"48.5","version":"1354'13343","reported_seq":13388,"reported_epoch":1354,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:59.882883+0000","last_change":"2024-08-17T18:09:29.983372+0000","last_active":"2024-08-17T18:10:59.882883+0000","last_peered":"2024-08-17T18:10:59.882883+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.956933+0000","last_became_peered":"2024-08-17T18:09:29.956933+0000","last_unstale":"2024-08-17T18:10:59.882883+0000","last_undegraded":"2024-08-17T18:09:29.847436+0000","last_fullsized":"2024-08-17T18:09:29.819789+0000","mapping_epoch":1344,"log_start":"1340'8358","ondisk_log_start":"1340'8358","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4985,"log_dups_size":0,"ondisk_log_size":4985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199819264,"num_objects":3049,"num_object_clones":0,"num_object_copies":9147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3049,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13343,"num_write_kb":524544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":3049}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.4","version":"1353'13386","reported_seq":13451,"reported_epoch":1353,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:58.950257+0000","last_change":"2024-08-17T18:09:29.965770+0000","last_active":"2024-08-17T18:10:58.950257+0000","last_peered":"2024-08-17T18:10:58.950257+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.948027+0000","last_became_peered":"2024-08-17T18:09:29.948027+0000","last_unstale":"2024-08-17T18:10:58.950257+0000","last_undegraded":"2024-08-17T18:09:29.860142+0000","last_fullsized":"2024-08-17T18:09:29.828238+0000","mapping_epoch":1344,"log_start":"1340'8517","ondisk_log_start":"1340'8517","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4869,"log_dups_size":0,"ondisk_log_size":4869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198443008,"num_objects":3028,"num_object_clones":0,"num_object_copies":9084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3028,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13386,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":3028}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.3","version":"1353'13381","reported_seq":13445,"reported_epoch":1353,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:10:58.896969+0000","last_change":"2024-08-17T18:09:30.006916+0000","last_active":"2024-08-17T18:10:58.896969+0000","last_peered":"2024-08-17T18:10:58.896969+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.994996+0000","last_became_peered":"2024-08-17T18:09:29.994996+0000","last_unstale":"2024-08-17T18:10:58.896969+0000","last_undegraded":"2024-08-17T18:09:29.861017+0000","last_fullsized":"2024-08-17T18:09:29.830876+0000","mapping_epoch":1344,"log_start":"1340'8523","ondisk_log_start":"1340'8523","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4858,"log_dups_size":0,"ondisk_log_size":4858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200343552,"num_objects":3057,"num_object_clones":0,"num_object_copies":9171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13381,"num_write_kb":526016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":3057}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.2","version":"1354'13546","reported_seq":26888,"reported_epoch":1354,"state":"active+undersized+degraded+remapped+backfilling","last_fresh":"2024-08-17T18:11:02.782173+0000","last_change":"2024-08-17T18:09:29.894664+0000","last_active":"2024-08-17T18:11:02.782173+0000","last_peered":"2024-08-17T18:11:02.782173+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:28.981975+0000","last_became_peered":"2024-08-17T18:09:28.981975+0000","last_unstale":"2024-08-17T18:11:02.782173+0000","last_undegraded":"2024-08-17T18:09:29.001178+0000","last_fullsized":"2024-08-17T18:09:28.815541+0000","mapping_epoch":1343,"log_start":"1331'3500","ondisk_log_start":"1331'3500","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":10046,"log_dups_size":0,"ondisk_log_size":10046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188612608,"num_objects":2878,"num_object_clones":0,"num_object_copies":8634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":222,"num_objects_misplaced":222,"num_objects_unfound":0,"num_objects_dirty":2878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13546,"num_write_kb":525568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4425,"num_bytes_recovered":289865728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":2878}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"48.1","version":"1354'13766","reported_seq":13801,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:11:03.145299+0000","last_change":"2024-08-17T18:07:32.360312+0000","last_active":"2024-08-17T18:11:03.145299+0000","last_peered":"2024-08-17T18:11:03.145299+0000","last_clean":"2024-08-17T18:11:03.145299+0000","last_became_active":"2024-08-17T18:07:32.360092+0000","last_became_peered":"2024-08-17T18:07:32.360092+0000","last_unstale":"2024-08-17T18:11:03.145299+0000","last_undegraded":"2024-08-17T18:11:03.145299+0000","last_fullsized":"2024-08-17T18:11:03.145299+0000","mapping_epoch":1327,"log_start":"1345'9622","ondisk_log_start":"1345'9622","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4144,"log_dups_size":0,"ondisk_log_size":4144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:41.580676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191496192,"num_objects":2922,"num_object_clones":0,"num_object_copies":8766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13766,"num_write_kb":534016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.0","version":"1354'13568","reported_seq":13641,"reported_epoch":1354,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:03.082533+0000","last_change":"2024-08-17T18:09:28.963421+0000","last_active":"2024-08-17T18:11:03.082533+0000","last_peered":"2024-08-17T18:11:03.082533+0000","last_clean":"2024-08-17T18:09:27.823675+0000","last_became_active":"2024-08-17T18:09:28.941589+0000","last_became_peered":"2024-08-17T18:09:28.941589+0000","last_unstale":"2024-08-17T18:11:03.082533+0000","last_undegraded":"2024-08-17T18:11:03.082533+0000","last_fullsized":"2024-08-17T18:11:03.082533+0000","mapping_epoch":1343,"log_start":"1345'9388","ondisk_log_start":"1345'9388","created":1327,"last_epoch_clean":1337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4180,"log_dups_size":0,"ondisk_log_size":4180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:36.800547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185466880,"num_objects":2830,"num_object_clones":0,"num_object_copies":8490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2830,"num_objects_unfound":0,"num_objects_dirty":2830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13568,"num_write_kb":524736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,1],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"1,2,7","objects":2830}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":2194,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516276+0000","last_change":"2024-08-17T18:09:28.870033+0000","last_active":"2024-08-17T18:10:40.516276+0000","last_peered":"2024-08-17T18:10:40.516276+0000","last_clean":"2024-08-17T18:10:40.516276+0000","last_became_active":"2024-08-17T18:09:28.869346+0000","last_became_peered":"2024-08-17T18:09:28.869346+0000","last_unstale":"2024-08-17T18:10:40.516276+0000","last_undegraded":"2024-08-17T18:10:40.516276+0000","last_fullsized":"2024-08-17T18:10:40.516276+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:23.168631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":491,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.554845+0000","last_change":"2024-08-17T18:09:29.841584+0000","last_active":"2024-08-17T18:10:59.554845+0000","last_peered":"2024-08-17T18:10:59.554845+0000","last_clean":"2024-08-17T18:10:59.554845+0000","last_became_active":"2024-08-17T18:09:29.840950+0000","last_became_peered":"2024-08-17T18:09:29.840950+0000","last_unstale":"2024-08-17T18:10:59.554845+0000","last_undegraded":"2024-08-17T18:10:59.554845+0000","last_fullsized":"2024-08-17T18:10:59.554845+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:43.035959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1986,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.554939+0000","last_change":"2024-08-17T18:09:29.883587+0000","last_active":"2024-08-17T18:10:59.554939+0000","last_peered":"2024-08-17T18:10:59.554939+0000","last_clean":"2024-08-17T18:10:59.554939+0000","last_became_active":"2024-08-17T18:09:29.882222+0000","last_became_peered":"2024-08-17T18:09:29.882222+0000","last_unstale":"2024-08-17T18:10:59.554939+0000","last_undegraded":"2024-08-17T18:10:59.554939+0000","last_fullsized":"2024-08-17T18:10:59.554939+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:10.777993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2043,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.559167+0000","last_change":"2024-08-17T18:09:28.858824+0000","last_active":"2024-08-17T18:10:59.559167+0000","last_peered":"2024-08-17T18:10:59.559167+0000","last_clean":"2024-08-17T18:10:59.559167+0000","last_became_active":"2024-08-17T18:09:28.857856+0000","last_became_peered":"2024-08-17T18:09:28.857856+0000","last_unstale":"2024-08-17T18:10:59.559167+0000","last_undegraded":"2024-08-17T18:10:59.559167+0000","last_fullsized":"2024-08-17T18:10:59.559167+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:41:35.146166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":47,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.555311+0000","last_change":"2024-08-17T18:09:29.851515+0000","last_active":"2024-08-17T18:10:59.555311+0000","last_peered":"2024-08-17T18:10:59.555311+0000","last_clean":"2024-08-17T18:10:59.555311+0000","last_became_active":"2024-08-17T18:09:29.851109+0000","last_became_peered":"2024-08-17T18:09:29.851109+0000","last_unstale":"2024-08-17T18:10:59.555311+0000","last_undegraded":"2024-08-17T18:10:59.555311+0000","last_fullsized":"2024-08-17T18:10:59.555311+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:42.054532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":66,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.504830+0000","last_change":"2024-08-17T18:03:16.089811+0000","last_active":"2024-08-17T18:10:40.504830+0000","last_peered":"2024-08-17T18:10:40.504830+0000","last_clean":"2024-08-17T18:10:40.504830+0000","last_became_active":"2024-08-17T18:03:16.089571+0000","last_became_peered":"2024-08-17T18:03:16.089571+0000","last_unstale":"2024-08-17T18:10:40.504830+0000","last_undegraded":"2024-08-17T18:10:40.504830+0000","last_fullsized":"2024-08-17T18:10:40.504830+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:04.630361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":48,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505548+0000","last_change":"2024-08-17T18:08:57.577055+0000","last_active":"2024-08-17T18:10:40.505548+0000","last_peered":"2024-08-17T18:10:40.505548+0000","last_clean":"2024-08-17T18:10:40.505548+0000","last_became_active":"2024-08-17T18:08:57.573659+0000","last_became_peered":"2024-08-17T18:08:57.573659+0000","last_unstale":"2024-08-17T18:10:40.505548+0000","last_undegraded":"2024-08-17T18:10:40.505548+0000","last_fullsized":"2024-08-17T18:10:40.505548+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:01.686752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":831,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.567679+0000","last_change":"2024-08-17T18:09:29.852393+0000","last_active":"2024-08-17T18:10:59.567679+0000","last_peered":"2024-08-17T18:10:59.567679+0000","last_clean":"2024-08-17T18:10:59.567679+0000","last_became_active":"2024-08-17T18:09:29.851947+0000","last_became_peered":"2024-08-17T18:09:29.851947+0000","last_unstale":"2024-08-17T18:10:59.567679+0000","last_undegraded":"2024-08-17T18:10:59.567679+0000","last_fullsized":"2024-08-17T18:10:59.567679+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:49.145327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":2139,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518690+0000","last_change":"2024-08-17T18:09:29.863241+0000","last_active":"2024-08-17T18:10:40.518690+0000","last_peered":"2024-08-17T18:10:40.518690+0000","last_clean":"2024-08-17T18:10:40.518690+0000","last_became_active":"2024-08-17T18:09:29.862369+0000","last_became_peered":"2024-08-17T18:09:29.862369+0000","last_unstale":"2024-08-17T18:10:40.518690+0000","last_undegraded":"2024-08-17T18:10:40.518690+0000","last_fullsized":"2024-08-17T18:10:40.518690+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:30.518551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":578,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.572907+0000","last_change":"2024-08-17T18:09:29.896233+0000","last_active":"2024-08-17T18:10:59.572907+0000","last_peered":"2024-08-17T18:10:59.572907+0000","last_clean":"2024-08-17T18:10:59.572907+0000","last_became_active":"2024-08-17T18:09:29.895718+0000","last_became_peered":"2024-08-17T18:09:29.895718+0000","last_unstale":"2024-08-17T18:10:59.572907+0000","last_undegraded":"2024-08-17T18:10:59.572907+0000","last_fullsized":"2024-08-17T18:10:59.572907+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:03.278878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":49,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516722+0000","last_change":"2024-08-17T18:08:57.574015+0000","last_active":"2024-08-17T18:10:40.516722+0000","last_peered":"2024-08-17T18:10:40.516722+0000","last_clean":"2024-08-17T18:10:40.516722+0000","last_became_active":"2024-08-17T18:08:57.567973+0000","last_became_peered":"2024-08-17T18:08:57.567973+0000","last_unstale":"2024-08-17T18:10:40.516722+0000","last_undegraded":"2024-08-17T18:10:40.516722+0000","last_fullsized":"2024-08-17T18:10:40.516722+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:06.251764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1998,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.560898+0000","last_change":"2024-08-17T18:09:29.917203+0000","last_active":"2024-08-17T18:10:59.560898+0000","last_peered":"2024-08-17T18:10:59.560898+0000","last_clean":"2024-08-17T18:10:59.560898+0000","last_became_active":"2024-08-17T18:09:29.916844+0000","last_became_peered":"2024-08-17T18:09:29.916844+0000","last_unstale":"2024-08-17T18:10:59.560898+0000","last_undegraded":"2024-08-17T18:10:59.560898+0000","last_fullsized":"2024-08-17T18:10:59.560898+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.453720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1904,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.555197+0000","last_change":"2024-08-17T18:09:28.859255+0000","last_active":"2024-08-17T18:10:59.555197+0000","last_peered":"2024-08-17T18:10:59.555197+0000","last_clean":"2024-08-17T18:10:59.555197+0000","last_became_active":"2024-08-17T18:09:28.858537+0000","last_became_peered":"2024-08-17T18:09:28.858537+0000","last_unstale":"2024-08-17T18:10:59.555197+0000","last_undegraded":"2024-08-17T18:10:59.555197+0000","last_fullsized":"2024-08-17T18:10:59.555197+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:15.364687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2163,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516955+0000","last_change":"2024-08-17T18:09:28.868303+0000","last_active":"2024-08-17T18:10:40.516955+0000","last_peered":"2024-08-17T18:10:40.516955+0000","last_clean":"2024-08-17T18:10:40.516955+0000","last_became_active":"2024-08-17T18:09:28.867989+0000","last_became_peered":"2024-08-17T18:09:28.867989+0000","last_unstale":"2024-08-17T18:10:40.516955+0000","last_undegraded":"2024-08-17T18:10:40.516955+0000","last_fullsized":"2024-08-17T18:10:40.516955+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:58:52.741685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":30,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.554584+0000","last_change":"2024-08-17T18:09:29.842129+0000","last_active":"2024-08-17T18:10:59.554584+0000","last_peered":"2024-08-17T18:10:59.554584+0000","last_clean":"2024-08-17T18:10:59.554584+0000","last_became_active":"2024-08-17T18:09:29.841848+0000","last_became_peered":"2024-08-17T18:09:29.841848+0000","last_unstale":"2024-08-17T18:10:59.554584+0000","last_undegraded":"2024-08-17T18:10:59.554584+0000","last_fullsized":"2024-08-17T18:10:59.554584+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:02.119134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1276,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516194+0000","last_change":"2024-08-17T18:09:29.846509+0000","last_active":"2024-08-17T18:10:40.516194+0000","last_peered":"2024-08-17T18:10:40.516194+0000","last_clean":"2024-08-17T18:10:40.516194+0000","last_became_active":"2024-08-17T18:09:29.844097+0000","last_became_peered":"2024-08-17T18:09:29.844097+0000","last_unstale":"2024-08-17T18:10:40.516194+0000","last_undegraded":"2024-08-17T18:10:40.516194+0000","last_fullsized":"2024-08-17T18:10:40.516194+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:32.340354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1992,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.565619+0000","last_change":"2024-08-17T18:09:29.841335+0000","last_active":"2024-08-17T18:10:59.565619+0000","last_peered":"2024-08-17T18:10:59.565619+0000","last_clean":"2024-08-17T18:10:59.565619+0000","last_became_active":"2024-08-17T18:09:29.840522+0000","last_became_peered":"2024-08-17T18:09:29.840522+0000","last_unstale":"2024-08-17T18:10:59.565619+0000","last_undegraded":"2024-08-17T18:10:59.565619+0000","last_fullsized":"2024-08-17T18:10:59.565619+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:08.015836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2168,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516364+0000","last_change":"2024-08-17T18:09:29.847494+0000","last_active":"2024-08-17T18:10:40.516364+0000","last_peered":"2024-08-17T18:10:40.516364+0000","last_clean":"2024-08-17T18:10:40.516364+0000","last_became_active":"2024-08-17T18:09:29.846973+0000","last_became_peered":"2024-08-17T18:09:29.846973+0000","last_unstale":"2024-08-17T18:10:40.516364+0000","last_undegraded":"2024-08-17T18:10:40.516364+0000","last_fullsized":"2024-08-17T18:10:40.516364+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:49.469921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1951,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.554989+0000","last_change":"2024-08-17T18:09:28.855045+0000","last_active":"2024-08-17T18:10:59.554989+0000","last_peered":"2024-08-17T18:10:59.554989+0000","last_clean":"2024-08-17T18:10:59.554989+0000","last_became_active":"2024-08-17T18:09:28.854607+0000","last_became_peered":"2024-08-17T18:09:28.854607+0000","last_unstale":"2024-08-17T18:10:59.554989+0000","last_undegraded":"2024-08-17T18:10:59.554989+0000","last_fullsized":"2024-08-17T18:10:59.554989+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:13.437895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":678,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505467+0000","last_change":"2024-08-17T18:09:29.866944+0000","last_active":"2024-08-17T18:10:40.505467+0000","last_peered":"2024-08-17T18:10:40.505467+0000","last_clean":"2024-08-17T18:10:40.505467+0000","last_became_active":"2024-08-17T18:09:29.866694+0000","last_became_peered":"2024-08-17T18:09:29.866694+0000","last_unstale":"2024-08-17T18:10:40.505467+0000","last_undegraded":"2024-08-17T18:10:40.505467+0000","last_fullsized":"2024-08-17T18:10:40.505467+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:00:41.342392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1341,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505509+0000","last_change":"2024-08-17T18:09:28.884262+0000","last_active":"2024-08-17T18:10:40.505509+0000","last_peered":"2024-08-17T18:10:40.505509+0000","last_clean":"2024-08-17T18:10:40.505509+0000","last_became_active":"2024-08-17T18:09:28.883734+0000","last_became_peered":"2024-08-17T18:09:28.883734+0000","last_unstale":"2024-08-17T18:10:40.505509+0000","last_undegraded":"2024-08-17T18:10:40.505509+0000","last_fullsized":"2024-08-17T18:10:40.505509+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:10.735504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2052,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516565+0000","last_change":"2024-08-17T18:08:57.571995+0000","last_active":"2024-08-17T18:10:40.516565+0000","last_peered":"2024-08-17T18:10:40.516565+0000","last_clean":"2024-08-17T18:10:40.516565+0000","last_became_active":"2024-08-17T18:08:57.567342+0000","last_became_peered":"2024-08-17T18:08:57.567342+0000","last_unstale":"2024-08-17T18:10:40.516565+0000","last_undegraded":"2024-08-17T18:10:40.516565+0000","last_fullsized":"2024-08-17T18:10:40.516565+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:23.769111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":505,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505656+0000","last_change":"2024-08-17T18:02:53.912837+0000","last_active":"2024-08-17T18:10:40.505656+0000","last_peered":"2024-08-17T18:10:40.505656+0000","last_clean":"2024-08-17T18:10:40.505656+0000","last_became_active":"2024-08-17T18:02:53.883152+0000","last_became_peered":"2024-08-17T18:02:53.883152+0000","last_unstale":"2024-08-17T18:10:40.505656+0000","last_undegraded":"2024-08-17T18:10:40.505656+0000","last_fullsized":"2024-08-17T18:10:40.505656+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:46.261612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":504,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505169+0000","last_change":"2024-08-17T18:09:29.845126+0000","last_active":"2024-08-17T18:10:40.505169+0000","last_peered":"2024-08-17T18:10:40.505169+0000","last_clean":"2024-08-17T18:10:40.505169+0000","last_became_active":"2024-08-17T18:09:29.844836+0000","last_became_peered":"2024-08-17T18:09:29.844836+0000","last_unstale":"2024-08-17T18:10:40.505169+0000","last_undegraded":"2024-08-17T18:10:40.505169+0000","last_fullsized":"2024-08-17T18:10:40.505169+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:44.821894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1289,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.515856+0000","last_change":"2024-08-17T18:02:53.890198+0000","last_active":"2024-08-17T18:10:40.515856+0000","last_peered":"2024-08-17T18:10:40.515856+0000","last_clean":"2024-08-17T18:10:40.515856+0000","last_became_active":"2024-08-17T18:02:53.887007+0000","last_became_peered":"2024-08-17T18:02:53.887007+0000","last_unstale":"2024-08-17T18:10:40.515856+0000","last_undegraded":"2024-08-17T18:10:40.515856+0000","last_fullsized":"2024-08-17T18:10:40.515856+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:25.555909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":595,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505814+0000","last_change":"2024-08-17T18:09:28.884187+0000","last_active":"2024-08-17T18:10:40.505814+0000","last_peered":"2024-08-17T18:10:40.505814+0000","last_clean":"2024-08-17T18:10:40.505814+0000","last_became_active":"2024-08-17T18:09:28.883939+0000","last_became_peered":"2024-08-17T18:09:28.883939+0000","last_unstale":"2024-08-17T18:10:40.505814+0000","last_undegraded":"2024-08-17T18:10:40.505814+0000","last_fullsized":"2024-08-17T18:10:40.505814+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:37:41.951099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1895,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505239+0000","last_change":"2024-08-17T18:09:29.893220+0000","last_active":"2024-08-17T18:10:40.505239+0000","last_peered":"2024-08-17T18:10:40.505239+0000","last_clean":"2024-08-17T18:10:40.505239+0000","last_became_active":"2024-08-17T18:09:29.865699+0000","last_became_peered":"2024-08-17T18:09:29.865699+0000","last_unstale":"2024-08-17T18:10:40.505239+0000","last_undegraded":"2024-08-17T18:10:40.505239+0000","last_fullsized":"2024-08-17T18:10:40.505239+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:40.205658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2011,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.555545+0000","last_change":"2024-08-17T18:09:28.934404+0000","last_active":"2024-08-17T18:10:59.555545+0000","last_peered":"2024-08-17T18:10:59.555545+0000","last_clean":"2024-08-17T18:10:59.555545+0000","last_became_active":"2024-08-17T18:09:28.934001+0000","last_became_peered":"2024-08-17T18:09:28.934001+0000","last_unstale":"2024-08-17T18:10:59.555545+0000","last_undegraded":"2024-08-17T18:10:59.555545+0000","last_fullsized":"2024-08-17T18:10:59.555545+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:04.676812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":2043,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.559653+0000","last_change":"2024-08-17T18:09:29.884088+0000","last_active":"2024-08-17T18:10:59.559653+0000","last_peered":"2024-08-17T18:10:59.559653+0000","last_clean":"2024-08-17T18:10:59.559653+0000","last_became_active":"2024-08-17T18:09:29.883027+0000","last_became_peered":"2024-08-17T18:09:29.883027+0000","last_unstale":"2024-08-17T18:10:59.559653+0000","last_undegraded":"2024-08-17T18:10:59.559653+0000","last_fullsized":"2024-08-17T18:10:59.559653+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:22:07.777188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":2013,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.564530+0000","last_change":"2024-08-17T18:09:28.935401+0000","last_active":"2024-08-17T18:10:59.564530+0000","last_peered":"2024-08-17T18:10:59.564530+0000","last_clean":"2024-08-17T18:10:59.564530+0000","last_became_active":"2024-08-17T18:09:28.933628+0000","last_became_peered":"2024-08-17T18:09:28.933628+0000","last_unstale":"2024-08-17T18:10:59.564530+0000","last_undegraded":"2024-08-17T18:10:59.564530+0000","last_fullsized":"2024-08-17T18:10:59.564530+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:21:12.714486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2123,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518848+0000","last_change":"2024-08-17T18:08:57.574718+0000","last_active":"2024-08-17T18:10:40.518848+0000","last_peered":"2024-08-17T18:10:40.518848+0000","last_clean":"2024-08-17T18:10:40.518848+0000","last_became_active":"2024-08-17T18:08:57.573515+0000","last_became_peered":"2024-08-17T18:08:57.573515+0000","last_unstale":"2024-08-17T18:10:40.518848+0000","last_undegraded":"2024-08-17T18:10:40.518848+0000","last_fullsized":"2024-08-17T18:10:40.518848+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:04.873717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1354,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505769+0000","last_change":"2024-08-17T18:08:57.568821+0000","last_active":"2024-08-17T18:10:40.505769+0000","last_peered":"2024-08-17T18:10:40.505769+0000","last_clean":"2024-08-17T18:10:40.505769+0000","last_became_active":"2024-08-17T18:08:57.568526+0000","last_became_peered":"2024-08-17T18:08:57.568526+0000","last_unstale":"2024-08-17T18:10:40.505769+0000","last_undegraded":"2024-08-17T18:10:40.505769+0000","last_fullsized":"2024-08-17T18:10:40.505769+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:26.449197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1843,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.515969+0000","last_change":"2024-08-17T18:08:57.571174+0000","last_active":"2024-08-17T18:10:40.515969+0000","last_peered":"2024-08-17T18:10:40.515969+0000","last_clean":"2024-08-17T18:10:40.515969+0000","last_became_active":"2024-08-17T18:08:57.567571+0000","last_became_peered":"2024-08-17T18:08:57.567571+0000","last_unstale":"2024-08-17T18:10:40.515969+0000","last_undegraded":"2024-08-17T18:10:40.515969+0000","last_fullsized":"2024-08-17T18:10:40.515969+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:29.976150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1812,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517147+0000","last_change":"2024-08-17T18:08:57.572368+0000","last_active":"2024-08-17T18:10:40.517147+0000","last_peered":"2024-08-17T18:10:40.517147+0000","last_clean":"2024-08-17T18:10:40.517147+0000","last_became_active":"2024-08-17T18:08:57.568382+0000","last_became_peered":"2024-08-17T18:08:57.568382+0000","last_unstale":"2024-08-17T18:10:40.517147+0000","last_undegraded":"2024-08-17T18:10:40.517147+0000","last_fullsized":"2024-08-17T18:10:40.517147+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:11:19.968079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1860,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516053+0000","last_change":"2024-08-17T18:08:57.572991+0000","last_active":"2024-08-17T18:10:40.516053+0000","last_peered":"2024-08-17T18:10:40.516053+0000","last_clean":"2024-08-17T18:10:40.516053+0000","last_became_active":"2024-08-17T18:08:57.571145+0000","last_became_peered":"2024-08-17T18:08:57.571145+0000","last_unstale":"2024-08-17T18:10:40.516053+0000","last_undegraded":"2024-08-17T18:10:40.516053+0000","last_fullsized":"2024-08-17T18:10:40.516053+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:39.963337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":818,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.555264+0000","last_change":"2024-08-17T18:09:28.860288+0000","last_active":"2024-08-17T18:10:59.555264+0000","last_peered":"2024-08-17T18:10:59.555264+0000","last_clean":"2024-08-17T18:10:59.555264+0000","last_became_active":"2024-08-17T18:09:28.860004+0000","last_became_peered":"2024-08-17T18:09:28.860004+0000","last_unstale":"2024-08-17T18:10:59.555264+0000","last_undegraded":"2024-08-17T18:10:59.555264+0000","last_fullsized":"2024-08-17T18:10:59.555264+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:57:33.054055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":611,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.572492+0000","last_change":"2024-08-17T18:09:28.860277+0000","last_active":"2024-08-17T18:10:59.572492+0000","last_peered":"2024-08-17T18:10:59.572492+0000","last_clean":"2024-08-17T18:10:59.572492+0000","last_became_active":"2024-08-17T18:09:28.859999+0000","last_became_peered":"2024-08-17T18:09:28.859999+0000","last_unstale":"2024-08-17T18:10:59.572492+0000","last_undegraded":"2024-08-17T18:10:59.572492+0000","last_fullsized":"2024-08-17T18:10:59.572492+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:27.946526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1515,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518583+0000","last_change":"2024-08-17T18:09:29.863669+0000","last_active":"2024-08-17T18:10:40.518583+0000","last_peered":"2024-08-17T18:10:40.518583+0000","last_clean":"2024-08-17T18:10:40.518583+0000","last_became_active":"2024-08-17T18:09:29.862781+0000","last_became_peered":"2024-08-17T18:09:29.862781+0000","last_unstale":"2024-08-17T18:10:40.518583+0000","last_undegraded":"2024-08-17T18:10:40.518583+0000","last_fullsized":"2024-08-17T18:10:40.518583+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:14.073233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":95,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517305+0000","last_change":"2024-08-17T18:09:29.843574+0000","last_active":"2024-08-17T18:10:40.517305+0000","last_peered":"2024-08-17T18:10:40.517305+0000","last_clean":"2024-08-17T18:10:40.517305+0000","last_became_active":"2024-08-17T18:09:29.843097+0000","last_became_peered":"2024-08-17T18:09:29.843097+0000","last_unstale":"2024-08-17T18:10:40.517305+0000","last_undegraded":"2024-08-17T18:10:40.517305+0000","last_fullsized":"2024-08-17T18:10:40.517305+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:46.740888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1859,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516555+0000","last_change":"2024-08-17T18:09:29.847264+0000","last_active":"2024-08-17T18:10:40.516555+0000","last_peered":"2024-08-17T18:10:40.516555+0000","last_clean":"2024-08-17T18:10:40.516555+0000","last_became_active":"2024-08-17T18:09:29.846226+0000","last_became_peered":"2024-08-17T18:09:29.846226+0000","last_unstale":"2024-08-17T18:10:40.516555+0000","last_undegraded":"2024-08-17T18:10:40.516555+0000","last_fullsized":"2024-08-17T18:10:40.516555+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:44.135977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2060,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518474+0000","last_change":"2024-08-17T18:09:29.845209+0000","last_active":"2024-08-17T18:10:40.518474+0000","last_peered":"2024-08-17T18:10:40.518474+0000","last_clean":"2024-08-17T18:10:40.518474+0000","last_became_active":"2024-08-17T18:09:29.844910+0000","last_became_peered":"2024-08-17T18:09:29.844910+0000","last_unstale":"2024-08-17T18:10:40.518474+0000","last_undegraded":"2024-08-17T18:10:40.518474+0000","last_fullsized":"2024-08-17T18:10:40.518474+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:14.586354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1934,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.559092+0000","last_change":"2024-08-17T18:09:28.859050+0000","last_active":"2024-08-17T18:10:59.559092+0000","last_peered":"2024-08-17T18:10:59.559092+0000","last_clean":"2024-08-17T18:10:59.559092+0000","last_became_active":"2024-08-17T18:09:28.858214+0000","last_became_peered":"2024-08-17T18:09:28.858214+0000","last_unstale":"2024-08-17T18:10:59.559092+0000","last_undegraded":"2024-08-17T18:10:59.559092+0000","last_fullsized":"2024-08-17T18:10:59.559092+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:54.146592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2184,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517068+0000","last_change":"2024-08-17T18:08:57.572610+0000","last_active":"2024-08-17T18:10:40.517068+0000","last_peered":"2024-08-17T18:10:40.517068+0000","last_clean":"2024-08-17T18:10:40.517068+0000","last_became_active":"2024-08-17T18:08:57.572308+0000","last_became_peered":"2024-08-17T18:08:57.572308+0000","last_unstale":"2024-08-17T18:10:40.517068+0000","last_undegraded":"2024-08-17T18:10:40.517068+0000","last_fullsized":"2024-08-17T18:10:40.517068+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:11.642831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2168,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.515939+0000","last_change":"2024-08-17T18:09:28.842838+0000","last_active":"2024-08-17T18:10:40.515939+0000","last_peered":"2024-08-17T18:10:40.515939+0000","last_clean":"2024-08-17T18:10:40.515939+0000","last_became_active":"2024-08-17T18:09:28.842617+0000","last_became_peered":"2024-08-17T18:09:28.842617+0000","last_unstale":"2024-08-17T18:10:40.515939+0000","last_undegraded":"2024-08-17T18:10:40.515939+0000","last_fullsized":"2024-08-17T18:10:40.515939+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:00.016002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1375,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505103+0000","last_change":"2024-08-17T18:09:29.845011+0000","last_active":"2024-08-17T18:10:40.505103+0000","last_peered":"2024-08-17T18:10:40.505103+0000","last_clean":"2024-08-17T18:10:40.505103+0000","last_became_active":"2024-08-17T18:09:29.844358+0000","last_became_peered":"2024-08-17T18:09:29.844358+0000","last_unstale":"2024-08-17T18:10:40.505103+0000","last_undegraded":"2024-08-17T18:10:40.505103+0000","last_fullsized":"2024-08-17T18:10:40.505103+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:41:27.792901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1223,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505585+0000","last_change":"2024-08-17T18:09:28.878263+0000","last_active":"2024-08-17T18:10:40.505585+0000","last_peered":"2024-08-17T18:10:40.505585+0000","last_clean":"2024-08-17T18:10:40.505585+0000","last_became_active":"2024-08-17T18:09:28.878025+0000","last_became_peered":"2024-08-17T18:09:28.878025+0000","last_unstale":"2024-08-17T18:10:40.505585+0000","last_undegraded":"2024-08-17T18:10:40.505585+0000","last_fullsized":"2024-08-17T18:10:40.505585+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:37.211100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1962,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.554892+0000","last_change":"2024-08-17T18:09:29.840134+0000","last_active":"2024-08-17T18:10:59.554892+0000","last_peered":"2024-08-17T18:10:59.554892+0000","last_clean":"2024-08-17T18:10:59.554892+0000","last_became_active":"2024-08-17T18:09:29.839848+0000","last_became_peered":"2024-08-17T18:09:29.839848+0000","last_unstale":"2024-08-17T18:10:59.554892+0000","last_undegraded":"2024-08-17T18:10:59.554892+0000","last_fullsized":"2024-08-17T18:10:59.554892+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:08:08.717851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":130,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505343+0000","last_change":"2024-08-17T18:09:28.847633+0000","last_active":"2024-08-17T18:10:40.505343+0000","last_peered":"2024-08-17T18:10:40.505343+0000","last_clean":"2024-08-17T18:10:40.505343+0000","last_became_active":"2024-08-17T18:09:28.847341+0000","last_became_peered":"2024-08-17T18:09:28.847341+0000","last_unstale":"2024-08-17T18:10:40.505343+0000","last_undegraded":"2024-08-17T18:10:40.505343+0000","last_fullsized":"2024-08-17T18:10:40.505343+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:51:59.625154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1715,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.504925+0000","last_change":"2024-08-17T18:09:00.609431+0000","last_active":"2024-08-17T18:10:40.504925+0000","last_peered":"2024-08-17T18:10:40.504925+0000","last_clean":"2024-08-17T18:10:40.504925+0000","last_became_active":"2024-08-17T18:09:00.609167+0000","last_became_peered":"2024-08-17T18:09:00.609167+0000","last_unstale":"2024-08-17T18:10:40.504925+0000","last_undegraded":"2024-08-17T18:10:40.504925+0000","last_fullsized":"2024-08-17T18:10:40.504925+0000","mapping_epoch":1338,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:39.579970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1842,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.565191+0000","last_change":"2024-08-17T18:09:29.837502+0000","last_active":"2024-08-17T18:10:59.565191+0000","last_peered":"2024-08-17T18:10:59.565191+0000","last_clean":"2024-08-17T18:10:59.565191+0000","last_became_active":"2024-08-17T18:09:29.837243+0000","last_became_peered":"2024-08-17T18:09:29.837243+0000","last_unstale":"2024-08-17T18:10:59.565191+0000","last_undegraded":"2024-08-17T18:10:59.565191+0000","last_fullsized":"2024-08-17T18:10:59.565191+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:47.377505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2040,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.555622+0000","last_change":"2024-08-17T18:09:29.859415+0000","last_active":"2024-08-17T18:10:59.555622+0000","last_peered":"2024-08-17T18:10:59.555622+0000","last_clean":"2024-08-17T18:10:59.555622+0000","last_became_active":"2024-08-17T18:09:29.859138+0000","last_became_peered":"2024-08-17T18:09:29.859138+0000","last_unstale":"2024-08-17T18:10:59.555622+0000","last_undegraded":"2024-08-17T18:10:59.555622+0000","last_fullsized":"2024-08-17T18:10:59.555622+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:31.178915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":491,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.555047+0000","last_change":"2024-08-17T18:09:29.883834+0000","last_active":"2024-08-17T18:10:59.555047+0000","last_peered":"2024-08-17T18:10:59.555047+0000","last_clean":"2024-08-17T18:10:59.555047+0000","last_became_active":"2024-08-17T18:09:29.882638+0000","last_became_peered":"2024-08-17T18:09:29.882638+0000","last_unstale":"2024-08-17T18:10:59.555047+0000","last_undegraded":"2024-08-17T18:10:59.555047+0000","last_fullsized":"2024-08-17T18:10:59.555047+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:07:10.188684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":615,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505225+0000","last_change":"2024-08-17T18:09:29.857250+0000","last_active":"2024-08-17T18:10:40.505225+0000","last_peered":"2024-08-17T18:10:40.505225+0000","last_clean":"2024-08-17T18:10:40.505225+0000","last_became_active":"2024-08-17T18:09:29.857038+0000","last_became_peered":"2024-08-17T18:09:29.857038+0000","last_unstale":"2024-08-17T18:10:40.505225+0000","last_undegraded":"2024-08-17T18:10:40.505225+0000","last_fullsized":"2024-08-17T18:10:40.505225+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:02:54.028575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":602,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.515309+0000","last_change":"2024-08-17T18:08:57.571270+0000","last_active":"2024-08-17T18:10:40.515309+0000","last_peered":"2024-08-17T18:10:40.515309+0000","last_clean":"2024-08-17T18:10:40.515309+0000","last_became_active":"2024-08-17T18:08:57.567670+0000","last_became_peered":"2024-08-17T18:08:57.567670+0000","last_unstale":"2024-08-17T18:10:40.515309+0000","last_undegraded":"2024-08-17T18:10:40.515309+0000","last_fullsized":"2024-08-17T18:10:40.515309+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:19:42.617321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1175,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.559443+0000","last_change":"2024-08-17T18:09:29.839260+0000","last_active":"2024-08-17T18:10:59.559443+0000","last_peered":"2024-08-17T18:10:59.559443+0000","last_clean":"2024-08-17T18:10:59.559443+0000","last_became_active":"2024-08-17T18:09:29.838953+0000","last_became_peered":"2024-08-17T18:09:29.838953+0000","last_unstale":"2024-08-17T18:10:59.559443+0000","last_undegraded":"2024-08-17T18:10:59.559443+0000","last_fullsized":"2024-08-17T18:10:59.559443+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:44.014083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":764,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.572641+0000","last_change":"2024-08-17T18:09:29.851347+0000","last_active":"2024-08-17T18:10:59.572641+0000","last_peered":"2024-08-17T18:10:59.572641+0000","last_clean":"2024-08-17T18:10:59.572641+0000","last_became_active":"2024-08-17T18:09:29.851046+0000","last_became_peered":"2024-08-17T18:09:29.851046+0000","last_unstale":"2024-08-17T18:10:59.572641+0000","last_undegraded":"2024-08-17T18:10:59.572641+0000","last_fullsized":"2024-08-17T18:10:59.572641+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:21.350703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2041,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.567514+0000","last_change":"2024-08-17T18:09:29.912446+0000","last_active":"2024-08-17T18:10:59.567514+0000","last_peered":"2024-08-17T18:10:59.567514+0000","last_clean":"2024-08-17T18:10:59.567514+0000","last_became_active":"2024-08-17T18:09:29.910703+0000","last_became_peered":"2024-08-17T18:09:29.910703+0000","last_unstale":"2024-08-17T18:10:59.567514+0000","last_undegraded":"2024-08-17T18:10:59.567514+0000","last_fullsized":"2024-08-17T18:10:59.567514+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.250608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2206,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518972+0000","last_change":"2024-08-17T18:09:29.863361+0000","last_active":"2024-08-17T18:10:40.518972+0000","last_peered":"2024-08-17T18:10:40.518972+0000","last_clean":"2024-08-17T18:10:40.518972+0000","last_became_active":"2024-08-17T18:09:29.862503+0000","last_became_peered":"2024-08-17T18:09:29.862503+0000","last_unstale":"2024-08-17T18:10:40.518972+0000","last_undegraded":"2024-08-17T18:10:40.518972+0000","last_fullsized":"2024-08-17T18:10:40.518972+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:25.908232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":30,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.565765+0000","last_change":"2024-08-17T18:09:29.895991+0000","last_active":"2024-08-17T18:10:59.565765+0000","last_peered":"2024-08-17T18:10:59.565765+0000","last_clean":"2024-08-17T18:10:59.565765+0000","last_became_active":"2024-08-17T18:09:29.895274+0000","last_became_peered":"2024-08-17T18:09:29.895274+0000","last_unstale":"2024-08-17T18:10:59.565765+0000","last_undegraded":"2024-08-17T18:10:59.565765+0000","last_fullsized":"2024-08-17T18:10:59.565765+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:19:26.420016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":497,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505615+0000","last_change":"2024-08-17T18:09:29.844686+0000","last_active":"2024-08-17T18:10:40.505615+0000","last_peered":"2024-08-17T18:10:40.505615+0000","last_clean":"2024-08-17T18:10:40.505615+0000","last_became_active":"2024-08-17T18:09:29.844341+0000","last_became_peered":"2024-08-17T18:09:29.844341+0000","last_unstale":"2024-08-17T18:10:40.505615+0000","last_undegraded":"2024-08-17T18:10:40.505615+0000","last_fullsized":"2024-08-17T18:10:40.505615+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:38.872767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1334,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505670+0000","last_change":"2024-08-17T18:09:28.884310+0000","last_active":"2024-08-17T18:10:40.505670+0000","last_peered":"2024-08-17T18:10:40.505670+0000","last_clean":"2024-08-17T18:10:40.505670+0000","last_became_active":"2024-08-17T18:09:28.883921+0000","last_became_peered":"2024-08-17T18:09:28.883921+0000","last_unstale":"2024-08-17T18:10:40.505670+0000","last_undegraded":"2024-08-17T18:10:40.505670+0000","last_fullsized":"2024-08-17T18:10:40.505670+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:05.272563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":773,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.559016+0000","last_change":"2024-08-17T18:09:29.837238+0000","last_active":"2024-08-17T18:10:59.559016+0000","last_peered":"2024-08-17T18:10:59.559016+0000","last_clean":"2024-08-17T18:10:59.559016+0000","last_became_active":"2024-08-17T18:09:29.836760+0000","last_became_peered":"2024-08-17T18:09:29.836760+0000","last_unstale":"2024-08-17T18:10:59.559016+0000","last_undegraded":"2024-08-17T18:10:59.559016+0000","last_fullsized":"2024-08-17T18:10:59.559016+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:38:52.862967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1349,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505423+0000","last_change":"2024-08-17T18:09:29.871872+0000","last_active":"2024-08-17T18:10:40.505423+0000","last_peered":"2024-08-17T18:10:40.505423+0000","last_clean":"2024-08-17T18:10:40.505423+0000","last_became_active":"2024-08-17T18:09:29.845111+0000","last_became_peered":"2024-08-17T18:09:29.845111+0000","last_unstale":"2024-08-17T18:10:40.505423+0000","last_undegraded":"2024-08-17T18:10:40.505423+0000","last_fullsized":"2024-08-17T18:10:40.505423+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:48.183250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":246,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.504826+0000","last_change":"2024-08-17T18:09:29.844786+0000","last_active":"2024-08-17T18:10:40.504826+0000","last_peered":"2024-08-17T18:10:40.504826+0000","last_clean":"2024-08-17T18:10:40.504826+0000","last_became_active":"2024-08-17T18:09:29.844270+0000","last_became_peered":"2024-08-17T18:09:29.844270+0000","last_unstale":"2024-08-17T18:10:40.504826+0000","last_undegraded":"2024-08-17T18:10:40.504826+0000","last_fullsized":"2024-08-17T18:10:40.504826+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:51.866680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1365,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505702+0000","last_change":"2024-08-17T18:09:29.886391+0000","last_active":"2024-08-17T18:10:40.505702+0000","last_peered":"2024-08-17T18:10:40.505702+0000","last_clean":"2024-08-17T18:10:40.505702+0000","last_became_active":"2024-08-17T18:09:29.844642+0000","last_became_peered":"2024-08-17T18:09:29.844642+0000","last_unstale":"2024-08-17T18:10:40.505702+0000","last_undegraded":"2024-08-17T18:10:40.505702+0000","last_fullsized":"2024-08-17T18:10:40.505702+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:07.988100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":185,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.518637+0000","last_change":"2024-08-17T18:09:28.941223+0000","last_active":"2024-08-17T18:10:40.518637+0000","last_peered":"2024-08-17T18:10:40.518637+0000","last_clean":"2024-08-17T18:10:40.518637+0000","last_became_active":"2024-08-17T18:09:28.940857+0000","last_became_peered":"2024-08-17T18:09:28.940857+0000","last_unstale":"2024-08-17T18:10:40.518637+0000","last_undegraded":"2024-08-17T18:10:40.518637+0000","last_fullsized":"2024-08-17T18:10:40.518637+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:48.809420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2038,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.565351+0000","last_change":"2024-08-17T18:09:29.838873+0000","last_active":"2024-08-17T18:10:59.565351+0000","last_peered":"2024-08-17T18:10:59.565351+0000","last_clean":"2024-08-17T18:10:59.565351+0000","last_became_active":"2024-08-17T18:09:29.838560+0000","last_became_peered":"2024-08-17T18:09:29.838560+0000","last_unstale":"2024-08-17T18:10:59.565351+0000","last_undegraded":"2024-08-17T18:10:59.565351+0000","last_fullsized":"2024-08-17T18:10:59.565351+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:00.103741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1325,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505356+0000","last_change":"2024-08-17T18:09:28.844170+0000","last_active":"2024-08-17T18:10:40.505356+0000","last_peered":"2024-08-17T18:10:40.505356+0000","last_clean":"2024-08-17T18:10:40.505356+0000","last_became_active":"2024-08-17T18:09:28.843862+0000","last_became_peered":"2024-08-17T18:09:28.843862+0000","last_unstale":"2024-08-17T18:10:40.505356+0000","last_undegraded":"2024-08-17T18:10:40.505356+0000","last_fullsized":"2024-08-17T18:10:40.505356+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:43:19.483326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2159,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.516694+0000","last_change":"2024-08-17T18:09:28.864923+0000","last_active":"2024-08-17T18:10:40.516694+0000","last_peered":"2024-08-17T18:10:40.516694+0000","last_clean":"2024-08-17T18:10:40.516694+0000","last_became_active":"2024-08-17T18:09:28.864707+0000","last_became_peered":"2024-08-17T18:09:28.864707+0000","last_unstale":"2024-08-17T18:10:40.516694+0000","last_undegraded":"2024-08-17T18:10:40.516694+0000","last_fullsized":"2024-08-17T18:10:40.516694+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:27.507329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2160,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.517395+0000","last_change":"2024-08-17T18:09:28.846741+0000","last_active":"2024-08-17T18:10:40.517395+0000","last_peered":"2024-08-17T18:10:40.517395+0000","last_clean":"2024-08-17T18:10:40.517395+0000","last_became_active":"2024-08-17T18:09:28.846478+0000","last_became_peered":"2024-08-17T18:09:28.846478+0000","last_unstale":"2024-08-17T18:10:40.517395+0000","last_undegraded":"2024-08-17T18:10:40.517395+0000","last_fullsized":"2024-08-17T18:10:40.517395+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:34.247517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":731,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505689+0000","last_change":"2024-08-17T18:03:16.091420+0000","last_active":"2024-08-17T18:10:40.505689+0000","last_peered":"2024-08-17T18:10:40.505689+0000","last_clean":"2024-08-17T18:10:40.505689+0000","last_became_active":"2024-08-17T18:03:16.091113+0000","last_became_peered":"2024-08-17T18:03:16.091113+0000","last_unstale":"2024-08-17T18:10:40.505689+0000","last_undegraded":"2024-08-17T18:10:40.505689+0000","last_fullsized":"2024-08-17T18:10:40.505689+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:47.770911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1339,"reported_epoch":1353,"state":"active+clean","last_fresh":"2024-08-17T18:10:40.505279+0000","last_change":"2024-08-17T18:02:53.893927+0000","last_active":"2024-08-17T18:10:40.505279+0000","last_peered":"2024-08-17T18:10:40.505279+0000","last_clean":"2024-08-17T18:10:40.505279+0000","last_became_active":"2024-08-17T18:02:53.883188+0000","last_became_peered":"2024-08-17T18:02:53.883188+0000","last_unstale":"2024-08-17T18:10:40.505279+0000","last_undegraded":"2024-08-17T18:10:40.505279+0000","last_fullsized":"2024-08-17T18:10:40.505279+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:19.296965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2038,"reported_epoch":1354,"state":"active+clean","last_fresh":"2024-08-17T18:10:59.572200+0000","last_change":"2024-08-17T18:09:29.840086+0000","last_active":"2024-08-17T18:10:59.572200+0000","last_peered":"2024-08-17T18:10:59.572200+0000","last_clean":"2024-08-17T18:10:59.572200+0000","last_became_active":"2024-08-17T18:09:29.839525+0000","last_became_peered":"2024-08-17T18:09:29.839525+0000","last_unstale":"2024-08-17T18:10:59.572200+0000","last_undegraded":"2024-08-17T18:10:59.572200+0000","last_fullsized":"2024-08-17T18:10:59.572200+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:38.316235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":48,"num_pg":16,"stat_sum":{"num_bytes":3131965464,"num_objects":47791,"num_object_clones":0,"num_object_copies":143373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":28211,"num_objects_misplaced":17817,"num_objects_unfound":0,"num_objects_dirty":47791,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":216135,"num_write_kb":8445569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9976,"num_bytes_recovered":653524992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7759142912,"data_stored":7759134768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83725,"ondisk_log_size":83725,"up":48,"acting":37,"num_store_stats":5},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":10970,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":155648,"data_stored":27490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234602,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3159480,"kb_used_data":2257584,"kb_used_omap":1,"kb_used_meta":901886,"kb_avail":90585672,"statfs":{"total":95995035648,"available":92759728128,"internally_reserved":0,"allocated":2311766016,"data_stored":2307340816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923531608},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568496237,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757875,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1343,"seq":5768141078548,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":350884,"kb_used_data":190748,"kb_used_omap":112,"kb_used_meta":160015,"kb_avail":93394268,"statfs":{"total":95995035648,"available":95635730432,"internally_reserved":0,"allocated":195325952,"data_stored":190916468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":115373,"internal_metadata":163855699},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405727,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3337436,"kb_used_data":2367444,"kb_used_omap":61,"kb_used_meta":969858,"kb_avail":90407716,"statfs":{"total":95995035648,"available":92577501184,"internally_reserved":0,"allocated":2424262656,"data_stored":2419857222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":993135544},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":0,"seq":2173253452262,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474952,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2373288,"kb_used_data":1520096,"kb_used_omap":119,"kb_used_meta":853064,"kb_avail":91371864,"statfs":{"total":95995035648,"available":93564788736,"internally_reserved":0,"allocated":1556578304,"data_stored":1552160767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":873538270},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019510,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1534328832,"data_stored":1534328832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1343881216,"data_stored":1343881216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2396786688,"data_stored":2396782616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":185204736,"data_stored":185204736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2298941440,"data_stored":2298937368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:11:06.487 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T18:11:06.492 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:06.607 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:06.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:06.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:06.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:06 smithi130 ceph-mon[105521]: pgmap v2449: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.8 GiB data, 8.6 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 718 op/s; 27103/138735 objects degraded (19.536%); 17600/138735 objects misplaced (12.686%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T18:11:06.702 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 27103/138735 objects degraded (19.536%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:06.702 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:06 smithi130 ceph-mon[110068]: pgmap v2449: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.8 GiB data, 8.6 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 718 op/s; 27103/138735 objects degraded (19.536%); 17600/138735 objects misplaced (12.686%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T18:11:06.702 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 27103/138735 objects degraded (19.536%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:06.710 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:06.763 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:06.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:06.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:06.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:06 smithi179 ceph-mon[93519]: pgmap v2449: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.8 GiB data, 8.6 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 718 op/s; 27103/138735 objects degraded (19.536%); 17600/138735 objects misplaced (12.686%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T18:11:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 27103/138735 objects degraded (19.536%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:06.865 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:06.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:06.933 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:06.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:07.029 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:11:07.029 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:11:07.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:07.059 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:11:07.060 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:11:07.060 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:11:07.060 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:11:07.060 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.060 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:147146 48.e 48.4bea78fe (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1343)", 2024-08-17T18:11:07.060 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:27.854502+0000", 2024-08-17T18:11:07.060 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 99.202764827999999, 2024-08-17T18:11:07.060 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.96373221899999995, 2024-08-17T18:11:07.060 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.060 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.060 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:11:07.060 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:11:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:11:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:11:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 147146 2024-08-17T18:11:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.854502+0000", 2024-08-17T18:11:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.854502+0000", 2024-08-17T18:11:07.061 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.854503+0000", 2024-08-17T18:11:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6139999999999999e-06 2024-08-17T18:11:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.854510+0000", 2024-08-17T18:11:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3879999999999997e-06 2024-08-17T18:11:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.062 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.854520+0000", 2024-08-17T18:11:07.063 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.995e-06 2024-08-17T18:11:07.063 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.063 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.063 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.063 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.854529+0000", 2024-08-17T18:11:07.063 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4439999999999997e-06 2024-08-17T18:11:07.063 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.063 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.063 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.063 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.855118+0000", 2024-08-17T18:11:07.063 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00058908799999999998 2024-08-17T18:11:07.063 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.063 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.855126+0000", 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8730000000000007e-06 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:28.818224+0000", 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.96309766799999996 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:28.818234+0000", 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0149000000000001e-05 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.064 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:147144 48.b e1345/1344 48:da1da724:::benchmark_data_smithi130_327597_object15179:head v 1345'9239, mlcod=0'0)", 2024-08-17T18:11:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:29.919655+0000", 2024-08-17T18:11:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 97.137611829999997, 2024-08-17T18:11:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.074477280000000007, 2024-08-17T18:11:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.065 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:29.919655+0000", 2024-08-17T18:11:07.066 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.066 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.066 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.066 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.066 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:29.919655+0000", 2024-08-17T18:11:07.066 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.066 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.066 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.066 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.066 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:29.919656+0000", 2024-08-17T18:11:07.066 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8599999999999997e-07 2024-08-17T18:11:07.066 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.066 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.067 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.067 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:29.919662+0000", 2024-08-17T18:11:07.067 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2090000000000002e-06 2024-08-17T18:11:07.067 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.067 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.067 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.067 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:29.919672+0000", 2024-08-17T18:11:07.067 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0396e-05 2024-08-17T18:11:07.067 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.067 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.067 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.067 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:29.919685+0000", 2024-08-17T18:11:07.067 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2918999999999999e-05 2024-08-17T18:11:07.067 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.068 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.068 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.068 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:29.919807+0000", 2024-08-17T18:11:07.068 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000122132 2024-08-17T18:11:07.068 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.068 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.068 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.068 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:29.919837+0000", 2024-08-17T18:11:07.068 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9862e-05 2024-08-17T18:11:07.068 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.068 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.068 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.068 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:29.994097+0000", 2024-08-17T18:11:07.068 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.074259864999999994 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:29.994132+0000", 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5011000000000001e-05 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:155511 48.b e1345/1344 48:db675d00:::benchmark_data_smithi130_327597_object23546:head v 1345'9759, mlcod=1345'9756)", 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:39.735760+0000", 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.321506256000006, 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.073475509999999994, 2024-08-17T18:11:07.069 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.070 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.070 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.070 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.070 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.070 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.070 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.735760+0000", 2024-08-17T18:11:07.070 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.070 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.070 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.070 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.070 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.735760+0000", 2024-08-17T18:11:07.070 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.070 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.735761+0000", 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8100000000000003e-07 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.735765+0000", 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.048e-06 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.735772+0000", 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8480000000000003e-06 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.071 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.735777+0000", 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6910000000000002e-06 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.740997+0000", 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005220021 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.741311+0000", 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00031386100000000002 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.072 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.073 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.073 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.748499+0000", 2024-08-17T18:11:07.073 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0071875869999999996 2024-08-17T18:11:07.073 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.073 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.073 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.073 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.809236+0000", 2024-08-17T18:11:07.073 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.060736973 2024-08-17T18:11:07.073 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.073 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.073 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.073 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.073 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.074 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:155505 48.2 e1345/1343 48:46e46ab8:::benchmark_data_smithi130_327597_object23540:head v 1345'9756, mlcod=1345'9753)", 2024-08-17T18:11:07.074 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:39.746404+0000", 2024-08-17T18:11:07.074 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.310862623000006, 2024-08-17T18:11:07.074 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.075260516, 2024-08-17T18:11:07.074 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.074 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.074 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.074 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.074 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.746404+0000", 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.746404+0000", 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.746404+0000", 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9599999999999999e-07 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.075 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.746408+0000", 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3639999999999999e-06 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.746415+0000", 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8970000000000002e-06 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.746420+0000", 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8339999999999998e-06 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.076 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.746727+0000", 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00030777199999999999 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.746745+0000", 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7972e-05 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.821039+0000", 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.074293398999999996 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.077 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.078 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.078 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.821664+0000", 2024-08-17T18:11:07.078 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00062578199999999997 2024-08-17T18:11:07.078 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.078 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.078 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.078 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.078 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.078 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:155510 48.2 e1345/1343 48:4f23e26e:::benchmark_data_smithi130_327597_object23545:head v 1345'9757, mlcod=1345'9753)", 2024-08-17T18:11:07.078 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:39.748583+0000", 2024-08-17T18:11:07.078 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.308683969000001, 2024-08-17T18:11:07.078 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.073165177999999997, 2024-08-17T18:11:07.078 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.748583+0000", 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.748583+0000", 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.079 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.080 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.748583+0000", 2024-08-17T18:11:07.080 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.44e-07 2024-08-17T18:11:07.080 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.080 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.080 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.080 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.748588+0000", 2024-08-17T18:11:07.080 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1039999999999998e-06 2024-08-17T18:11:07.080 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.080 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.080 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.080 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.808724+0000", 2024-08-17T18:11:07.080 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.060135322999999997 2024-08-17T18:11:07.080 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.081 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.081 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.081 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.808735+0000", 2024-08-17T18:11:07.081 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1562e-05 2024-08-17T18:11:07.081 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.081 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.081 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.081 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.808893+0000", 2024-08-17T18:11:07.081 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015769199999999999 2024-08-17T18:11:07.081 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.081 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.081 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.082 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.808914+0000", 2024-08-17T18:11:07.082 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1563e-05 2024-08-17T18:11:07.082 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.082 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.082 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.082 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.821669+0000", 2024-08-17T18:11:07.082 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012754468999999999 2024-08-17T18:11:07.082 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.082 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.082 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.082 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:39.821748+0000", 2024-08-17T18:11:07.082 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8920999999999999e-05 2024-08-17T18:11:07.082 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.082 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:155990 48.e e1345/1344 48:7255305a:::benchmark_data_smithi130_327597_object24025:head v 1345'9766, mlcod=1345'9764)", 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:40.338631+0000", 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 86.718635785999993, 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.093578369999999994, 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.338631+0000", 2024-08-17T18:11:07.083 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.084 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.084 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.084 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.084 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.338631+0000", 2024-08-17T18:11:07.084 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.084 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.084 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.084 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.084 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.338631+0000", 2024-08-17T18:11:07.084 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1699999999999999e-07 2024-08-17T18:11:07.084 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.084 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.085 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.085 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.338635+0000", 2024-08-17T18:11:07.085 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4460000000000001e-06 2024-08-17T18:11:07.085 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.085 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.085 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.085 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.338641+0000", 2024-08-17T18:11:07.085 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2879999999999998e-06 2024-08-17T18:11:07.085 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.085 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.085 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.085 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.338646+0000", 2024-08-17T18:11:07.085 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.904e-06 2024-08-17T18:11:07.085 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.086 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.086 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.086 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.338757+0000", 2024-08-17T18:11:07.086 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011094000000000001 2024-08-17T18:11:07.086 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.086 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.086 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.086 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.338775+0000", 2024-08-17T18:11:07.086 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8016000000000002e-05 2024-08-17T18:11:07.086 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.086 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.086 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.086 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.346458+0000", 2024-08-17T18:11:07.087 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0076828859999999999 2024-08-17T18:11:07.087 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.087 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.087 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.087 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432209+0000", 2024-08-17T18:11:07.087 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.085751472999999995 2024-08-17T18:11:07.087 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.087 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.087 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.087 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.087 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.087 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:155998 48.f e1345/1343 48:f3561aed:::benchmark_data_smithi130_327597_object24033:head v 1345'9761, mlcod=1345'9759)", 2024-08-17T18:11:07.087 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:40.342335+0000", 2024-08-17T18:11:07.087 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 86.714931210000003, 2024-08-17T18:11:07.088 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.089943804000000002, 2024-08-17T18:11:07.088 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.088 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.088 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.088 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.088 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.088 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.088 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.342335+0000", 2024-08-17T18:11:07.088 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.088 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.088 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.088 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.088 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.342335+0000", 2024-08-17T18:11:07.089 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.089 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.089 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.089 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.089 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.342336+0000", 2024-08-17T18:11:07.089 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2800000000000002e-07 2024-08-17T18:11:07.089 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.089 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.089 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.089 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.342342+0000", 2024-08-17T18:11:07.089 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3840000000000002e-06 2024-08-17T18:11:07.089 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.089 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.089 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.090 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.342350+0000", 2024-08-17T18:11:07.090 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7849999999999992e-06 2024-08-17T18:11:07.090 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.090 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.090 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.090 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.342355+0000", 2024-08-17T18:11:07.090 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0440000000000003e-06 2024-08-17T18:11:07.090 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.090 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.090 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.090 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.343014+0000", 2024-08-17T18:11:07.090 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00065939999999999998 2024-08-17T18:11:07.090 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.091 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.091 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.091 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.343032+0000", 2024-08-17T18:11:07.091 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.806e-05 2024-08-17T18:11:07.091 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.091 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.091 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.091 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.431923+0000", 2024-08-17T18:11:07.091 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.088890597000000002 2024-08-17T18:11:07.091 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.091 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.091 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.091 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432279+0000", 2024-08-17T18:11:07.092 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00035610600000000002 2024-08-17T18:11:07.092 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.092 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.092 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.092 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.092 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.092 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:155999 48.c e1345/1343 48:3104efe7:::benchmark_data_smithi130_327597_object24034:head v 1345'9692, mlcod=1345'9690)", 2024-08-17T18:11:07.092 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:40.344067+0000", 2024-08-17T18:11:07.092 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 86.713199306000007, 2024-08-17T18:11:07.092 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.087971454000000004, 2024-08-17T18:11:07.092 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.092 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.092 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.092 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.344067+0000", 2024-08-17T18:11:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.344067+0000", 2024-08-17T18:11:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.344068+0000", 2024-08-17T18:11:07.093 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3600000000000003e-07 2024-08-17T18:11:07.094 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.094 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.094 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.094 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.344074+0000", 2024-08-17T18:11:07.094 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5799999999999999e-06 2024-08-17T18:11:07.094 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.094 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.094 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.094 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.344082+0000", 2024-08-17T18:11:07.094 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5750000000000001e-06 2024-08-17T18:11:07.094 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.094 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.344088+0000", 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0050000000000001e-06 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.344281+0000", 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00019269099999999999 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.344299+0000", 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7986999999999999e-05 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.095 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.096 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.096 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.431901+0000", 2024-08-17T18:11:07.096 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.087602280000000005 2024-08-17T18:11:07.096 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.096 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.096 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.096 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432039+0000", 2024-08-17T18:11:07.096 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013760000000000001 2024-08-17T18:11:07.096 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.096 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.096 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.096 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.096 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.096 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:155995 48.6 e1345/1343 48:6cb9299c:::benchmark_data_smithi130_327597_object24030:head v 1345'9648, mlcod=1345'9646)", 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:40.344463+0000", 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 86.712803546999993, 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.087782676000000004, 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.344463+0000", 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.097 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.344463+0000", 2024-08-17T18:11:07.098 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.098 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.098 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.098 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.098 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.344464+0000", 2024-08-17T18:11:07.098 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3399999999999999e-07 2024-08-17T18:11:07.098 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.098 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.098 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.098 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.344469+0000", 2024-08-17T18:11:07.098 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.989e-06 2024-08-17T18:11:07.098 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.098 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.344479+0000", 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9429999999999995e-06 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.344484+0000", 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8540000000000002e-06 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.345564+0000", 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0010796619999999999 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.099 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.100 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.100 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.345582+0000", 2024-08-17T18:11:07.100 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8303000000000001e-05 2024-08-17T18:11:07.100 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.100 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.100 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.100 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432215+0000", 2024-08-17T18:11:07.100 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.086632106 2024-08-17T18:11:07.100 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.100 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.100 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.100 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432246+0000", 2024-08-17T18:11:07.100 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1185e-05 2024-08-17T18:11:07.100 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:155997 48.4 e1345/1344 48:2b43c011:::benchmark_data_smithi130_327597_object24032:head v 1345'9703, mlcod=1345'9701)", 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:40.345404+0000", 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 86.711862401999994, 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.086665291000000005, 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.101 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.345404+0000", 2024-08-17T18:11:07.102 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.102 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.102 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.102 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.102 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.345404+0000", 2024-08-17T18:11:07.103 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.103 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.103 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.103 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.103 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.345405+0000", 2024-08-17T18:11:07.103 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4499999999999995e-07 2024-08-17T18:11:07.103 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.103 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.103 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.103 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.345412+0000", 2024-08-17T18:11:07.103 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5270000000000004e-06 2024-08-17T18:11:07.103 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.103 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.103 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.104 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.345422+0000", 2024-08-17T18:11:07.104 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0608000000000001e-05 2024-08-17T18:11:07.104 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.104 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.104 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.104 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.345429+0000", 2024-08-17T18:11:07.104 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.314e-06 2024-08-17T18:11:07.104 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.104 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.104 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.104 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.345500+0000", 2024-08-17T18:11:07.104 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1730000000000006e-05 2024-08-17T18:11:07.104 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.345519+0000", 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8546000000000002e-05 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432042+0000", 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.086523500000000003 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432069+0000", 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.7121e-05 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.105 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.106 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.106 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.106 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.106 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:155996 48.8 e1345/1343 48:199b05d5:::benchmark_data_smithi130_327597_object24031:head v 1345'9796, mlcod=1345'9793)", 2024-08-17T18:11:07.106 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:40.347354+0000", 2024-08-17T18:11:07.106 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 86.709912967999998, 2024-08-17T18:11:07.106 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.089621398000000005, 2024-08-17T18:11:07.106 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.106 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.106 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.106 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.106 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.107 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.107 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.347354+0000", 2024-08-17T18:11:07.107 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.107 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.107 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.107 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.107 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.347354+0000", 2024-08-17T18:11:07.107 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.107 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.107 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.107 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.107 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.347355+0000", 2024-08-17T18:11:07.107 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0210000000000001e-06 2024-08-17T18:11:07.108 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.108 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.108 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.108 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.347362+0000", 2024-08-17T18:11:07.108 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.985e-06 2024-08-17T18:11:07.108 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.108 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.108 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.108 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432003+0000", 2024-08-17T18:11:07.108 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.084641257999999997 2024-08-17T18:11:07.108 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.108 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.108 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.108 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432037+0000", 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4418999999999997e-05 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432096+0000", 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8811999999999999e-05 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432121+0000", 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4459000000000001e-05 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.109 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.436863+0000", 2024-08-17T18:11:07.110 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0047422560000000002 2024-08-17T18:11:07.110 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.110 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.110 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.110 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.436975+0000", 2024-08-17T18:11:07.110 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000112188 2024-08-17T18:11:07.110 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.110 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.110 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.110 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.110 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.110 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:156003 48.7 e1345/1343 48:e72ffe6e:::benchmark_data_smithi130_327597_object24038:head v 1345'9653, mlcod=1345'9651)", 2024-08-17T18:11:07.110 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:40.349841+0000", 2024-08-17T18:11:07.110 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 86.707425185000005, 2024-08-17T18:11:07.111 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.084804152999999993, 2024-08-17T18:11:07.111 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.111 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.111 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.111 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.111 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.111 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.111 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.349841+0000", 2024-08-17T18:11:07.111 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.111 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.111 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.111 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.111 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.349841+0000", 2024-08-17T18:11:07.111 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.112 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.112 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.112 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.112 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.349842+0000", 2024-08-17T18:11:07.112 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.037e-06 2024-08-17T18:11:07.112 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.112 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.112 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.112 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.349850+0000", 2024-08-17T18:11:07.112 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.61e-06 2024-08-17T18:11:07.112 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.112 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.112 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.113 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.431917+0000", 2024-08-17T18:11:07.113 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.082066756000000005 2024-08-17T18:11:07.113 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.113 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.113 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.113 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.431947+0000", 2024-08-17T18:11:07.113 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9788999999999999e-05 2024-08-17T18:11:07.113 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.113 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.113 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.113 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432305+0000", 2024-08-17T18:11:07.113 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00035871600000000003 2024-08-17T18:11:07.113 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.114 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.114 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.114 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432322+0000", 2024-08-17T18:11:07.114 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6401000000000001e-05 2024-08-17T18:11:07.114 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.114 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.114 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.114 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.434613+0000", 2024-08-17T18:11:07.114 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0022916619999999999 2024-08-17T18:11:07.114 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.114 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.114 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.114 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.434646+0000", 2024-08-17T18:11:07.114 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2181999999999998e-05 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "pg_backfill(progress 48.2 e 1345/1345 lb 48:414bcffd:::benchmark_data_smithi130_327597_object76893:head)", 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:40.349939+0000", 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 86.707327859000003, 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.082371829999999993, 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.115 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.116 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.349939+0000", 2024-08-17T18:11:07.116 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.116 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.116 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.116 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.116 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.349939+0000", 2024-08-17T18:11:07.116 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.116 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.116 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.116 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.116 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.349939+0000", 2024-08-17T18:11:07.116 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4799999999999998e-07 2024-08-17T18:11:07.116 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.116 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.349948+0000", 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6309999999999992e-06 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432135+0000", 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.082187338999999998 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432144+0000", 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3049999999999993e-06 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.117 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.118 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.118 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432235+0000", 2024-08-17T18:11:07.118 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1682999999999996e-05 2024-08-17T18:11:07.118 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.118 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.118 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.118 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432250+0000", 2024-08-17T18:11:07.118 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4347e-05 2024-08-17T18:11:07.118 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.118 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.118 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.118 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:40.432311+0000", 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0977000000000002e-05 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:157238 48.d e1345/1344 48:b9c93be8:::benchmark_data_smithi130_327597_object25273:head v 1345'9792, mlcod=1345'9789)", 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:41.921681+0000", 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 85.135585672999994, 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.20086790900000001, 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.119 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.921681+0000", 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.921681+0000", 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.921681+0000", 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3799999999999998e-07 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.121 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.121 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.921685+0000", 2024-08-17T18:11:07.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5820000000000002e-06 2024-08-17T18:11:07.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.121 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.121 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.121 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.921691+0000", 2024-08-17T18:11:07.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5649999999999998e-06 2024-08-17T18:11:07.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.121 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.121 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.121 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.921696+0000", 2024-08-17T18:11:07.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6929999999999998e-06 2024-08-17T18:11:07.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.122 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.122 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.922104+0000", 2024-08-17T18:11:07.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00040751699999999998 2024-08-17T18:11:07.122 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.122 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.122 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.922122+0000", 2024-08-17T18:11:07.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7935e-05 2024-08-17T18:11:07.122 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.122 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.122 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.122447+0000", 2024-08-17T18:11:07.123 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.20032507799999999 2024-08-17T18:11:07.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.123 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.122549+0000", 2024-08-17T18:11:07.123 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010220100000000001 2024-08-17T18:11:07.123 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.123 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.123 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.123 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:157242 48.0 e1345/1343 48:024e1b0a:::benchmark_data_smithi130_327597_object25277:head v 1345'9779, mlcod=1345'9777)", 2024-08-17T18:11:07.123 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:41.923529+0000", 2024-08-17T18:11:07.123 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 85.133737742999998, 2024-08-17T18:11:07.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.200757777, 2024-08-17T18:11:07.124 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.124 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.124 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.124 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.124 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.124 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.124 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.923529+0000", 2024-08-17T18:11:07.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.124 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.124 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.124 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.125 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.923529+0000", 2024-08-17T18:11:07.125 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.125 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.125 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.125 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.125 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.923530+0000", 2024-08-17T18:11:07.125 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4300000000000003e-07 2024-08-17T18:11:07.125 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.125 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.125 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.125 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.923535+0000", 2024-08-17T18:11:07.125 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5679999999999999e-06 2024-08-17T18:11:07.125 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.125 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.923544+0000", 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1749999999999994e-06 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.923551+0000", 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5710000000000003e-06 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.923622+0000", 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1627999999999999e-05 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.126 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.127 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.127 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.923640+0000", 2024-08-17T18:11:07.127 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7444000000000001e-05 2024-08-17T18:11:07.127 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.127 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.127 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.127 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.124153+0000", 2024-08-17T18:11:07.127 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.200513567 2024-08-17T18:11:07.127 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.127 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.127 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.127 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.124287+0000", 2024-08-17T18:11:07.127 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013318100000000001 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:157240 48.4 e1345/1344 48:2729ea1c:::benchmark_data_smithi130_327597_object25275:head v 1345'9783, mlcod=1345'9781)", 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:41.924690+0000", 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 85.132576372000003, 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.20930695199999999, 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.128 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.129 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.924690+0000", 2024-08-17T18:11:07.129 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.129 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.129 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.129 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.129 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.924690+0000", 2024-08-17T18:11:07.129 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.129 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.129 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.129 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.129 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.924691+0000", 2024-08-17T18:11:07.129 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8000000000000005e-07 2024-08-17T18:11:07.129 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.129 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.130 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.130 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.924696+0000", 2024-08-17T18:11:07.130 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4240000000000001e-06 2024-08-17T18:11:07.130 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.130 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.130 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.130 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.924705+0000", 2024-08-17T18:11:07.130 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9230000000000008e-06 2024-08-17T18:11:07.130 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.130 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.130 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.130 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.924711+0000", 2024-08-17T18:11:07.130 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0240000000000003e-06 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.122758+0000", 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.198046995 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.122779+0000", 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0165000000000001e-05 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.133961+0000", 2024-08-17T18:11:07.131 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011182846 2024-08-17T18:11:07.132 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.132 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.133997+0000", 2024-08-17T18:11:07.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5794999999999999e-05 2024-08-17T18:11:07.132 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.132 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.132 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.132 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.132 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:157239 48.6 e1345/1343 48:6c115264:::benchmark_data_smithi130_327597_object25274:head v 1345'9727, mlcod=1345'9722)", 2024-08-17T18:11:07.132 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:41.925148+0000", 2024-08-17T18:11:07.132 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 85.132118930000004, 2024-08-17T18:11:07.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.199782449, 2024-08-17T18:11:07.133 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.133 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.133 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.133 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.133 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.133 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.133 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.925148+0000", 2024-08-17T18:11:07.133 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.133 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.133 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.133 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.133 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.925148+0000", 2024-08-17T18:11:07.133 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.134 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.134 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.134 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.134 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.925148+0000", 2024-08-17T18:11:07.134 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3e-07 2024-08-17T18:11:07.134 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.134 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.134 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.134 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.925154+0000", 2024-08-17T18:11:07.134 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0249999999999999e-06 2024-08-17T18:11:07.134 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.134 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.134 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.134 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.120671+0000", 2024-08-17T18:11:07.135 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.19551686099999999 2024-08-17T18:11:07.135 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.135 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.135 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.135 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.120689+0000", 2024-08-17T18:11:07.135 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8009999999999999e-05 2024-08-17T18:11:07.135 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.135 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.135 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.135 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.120896+0000", 2024-08-17T18:11:07.135 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020703099999999999 2024-08-17T18:11:07.135 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.135 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.136 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.136 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.120919+0000", 2024-08-17T18:11:07.136 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2816e-05 2024-08-17T18:11:07.136 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.136 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.136 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.136 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.124685+0000", 2024-08-17T18:11:07.136 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0037655000000000002 2024-08-17T18:11:07.136 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.136 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.136 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.136 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.124930+0000", 2024-08-17T18:11:07.136 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00024557600000000001 2024-08-17T18:11:07.136 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.137 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.137 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.137 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.137 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.137 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:157243 48.0 e1345/1343 48:0c9bd482:::benchmark_data_smithi130_327597_object25278:head v 1345'9780, mlcod=1345'9777)", 2024-08-17T18:11:07.137 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:41.925434+0000", 2024-08-17T18:11:07.137 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 85.131833071000003, 2024-08-17T18:11:07.137 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.198946449, 2024-08-17T18:11:07.137 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.137 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.137 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.137 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.137 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.137 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.925434+0000", 2024-08-17T18:11:07.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.138 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.138 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.138 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.925434+0000", 2024-08-17T18:11:07.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.138 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.138 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.138 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.925434+0000", 2024-08-17T18:11:07.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7100000000000002e-07 2024-08-17T18:11:07.138 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.925442+0000", 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6879999999999999e-06 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.120891+0000", 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.19544924799999999 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.120899+0000", 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3629999999999994e-06 2024-08-17T18:11:07.139 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.120944+0000", 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4744999999999999e-05 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.120965+0000", 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1087999999999999e-05 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.124292+0000", 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0033267489999999999 2024-08-17T18:11:07.140 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.141 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.141 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.141 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.124380+0000", 2024-08-17T18:11:07.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7997000000000005e-05 2024-08-17T18:11:07.141 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.141 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.141 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.141 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.141 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.141 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:157241 48.4 e1345/1344 48:2c99e2d1:::benchmark_data_smithi130_327597_object25276:head v 1345'9784, mlcod=1345'9781)", 2024-08-17T18:11:07.141 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:41.930389+0000", 2024-08-17T18:11:07.141 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 85.126877859000004, 2024-08-17T18:11:07.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.20922869299999999, 2024-08-17T18:11:07.141 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.142 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.142 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.142 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.142 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.142 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.142 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.930389+0000", 2024-08-17T18:11:07.142 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.142 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.142 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.142 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.930389+0000", 2024-08-17T18:11:07.142 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.142 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.930390+0000", 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3379999999999999e-06 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:41.930398+0000", 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5870000000000004e-06 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.120848+0000", 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.190450799 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.144 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.144 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.120860+0000", 2024-08-17T18:11:07.144 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1353e-05 2024-08-17T18:11:07.144 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.144 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.144 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.144 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.124626+0000", 2024-08-17T18:11:07.144 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0037664690000000002 2024-08-17T18:11:07.144 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.145 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.145 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.124645+0000", 2024-08-17T18:11:07.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8243000000000001e-05 2024-08-17T18:11:07.145 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.145 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.145 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.139531+0000", 2024-08-17T18:11:07.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014886623999999999 2024-08-17T18:11:07.145 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.145 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.145 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:42.139617+0000", 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6279999999999994e-05 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:217923 48.0 e1354/1343 48:0712102a:::benchmark_data_smithi130_327597_object85958:head v 1354'13605, mlcod=1354'13602)", 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:03.924633+0000", 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 3.1326334139999998, 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.075751517000000004, 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:11:07.146 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:11:07.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:03.924633+0000", 2024-08-17T18:11:07.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:11:07.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:03.924633+0000", 2024-08-17T18:11:07.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:11:07.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.147 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:11:07.147 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:03.924634+0000", 2024-08-17T18:11:07.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5899999999999996e-07 2024-08-17T18:11:07.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:11:07.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:03.924638+0000", 2024-08-17T18:11:07.148 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1350000000000002e-06 2024-08-17T18:11:07.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:11:07.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:03.924646+0000", 2024-08-17T18:11:07.148 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8790000000000003e-06 2024-08-17T18:11:07.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:11:07.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:03.924651+0000", 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6160000000000001e-06 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:03.932865+0000", 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0082136610000000006 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:03.932885+0000", 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9859000000000001e-05 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:11:07.149 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:04.000357+0000", 2024-08-17T18:11:07.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.067471699999999996 2024-08-17T18:11:07.150 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:11:07.150 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:11:07.150 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:11:07.150 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:04.000385+0000", 2024-08-17T18:11:07.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8107999999999999e-05 2024-08-17T18:11:07.150 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.150 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.150 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.150 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:11:07.150 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:11:07.150 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:11:07.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:07.151 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:07.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:07.252 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:07.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:07.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:07.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:07.375 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:07.427 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:07.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:07.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:07.529 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:07.550 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:07.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:07.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:07.651 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:07.672 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:07.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:07.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:07.774 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:07.796 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:07.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:07.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:07.897 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:07.919 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:07.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:07.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:08.021 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:08.042 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:08.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:08.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:08.144 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:08.165 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:08.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:08.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:08.267 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:08.289 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:08.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:08.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:08.390 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:08.411 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:08.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:08.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:08.513 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:08.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:08.551 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:08.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:08.652 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:08.674 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:08.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:08.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:08.775 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:08.798 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:08.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:08.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:08.899 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:08.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:08 smithi130 ceph-mon[105521]: pgmap v2450: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.8 GiB data, 8.6 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 718 op/s; 27103/138735 objects degraded (19.536%); 17600/138735 objects misplaced (12.686%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T18:11:08.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:08 smithi130 ceph-mon[110068]: pgmap v2450: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.8 GiB data, 8.6 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 718 op/s; 27103/138735 objects degraded (19.536%); 17600/138735 objects misplaced (12.686%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T18:11:09.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:08 smithi179 ceph-mon[93519]: pgmap v2450: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.8 GiB data, 8.6 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 718 op/s; 27103/138735 objects degraded (19.536%); 17600/138735 objects misplaced (12.686%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T18:11:09.147 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:09.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:09.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:09.249 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:09.271 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:09.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:09.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:09.373 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:09.394 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:09.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:09.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:09.496 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:09.518 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:09.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:09.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:09.621 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:09.647 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:09.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:09.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:09.749 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:09.803 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:09.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:09.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:09.905 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:09.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:09 smithi130 ceph-mon[110068]: osdmap e1355: 8 total, 8 up, 4 in 2024-08-17T18:11:09.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:09 smithi130 ceph-mon[105521]: osdmap e1355: 8 total, 8 up, 4 in 2024-08-17T18:11:09.922 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:11:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:11:09] "GET /metrics HTTP/1.1" 200 36809 "" "Prometheus/2.51.0" 2024-08-17T18:11:09.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:09.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:09.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:09.957 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:11:10.000 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T18:11:10.052 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:10.073 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:10.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:10.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:10.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:09 smithi179 ceph-mon[93519]: osdmap e1355: 8 total, 8 up, 4 in 2024-08-17T18:11:10.175 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:10.249 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:10.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:10.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:10.351 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:10.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:10.400 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:10.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:10.501 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:10.523 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:10.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:10.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:11:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:11:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:11:10.624 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:10.650 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:10.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:10.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:10.752 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:10.775 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:10.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:10.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:10.877 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:10.898 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:10.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:10.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:10.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:10 smithi130 ceph-mon[105521]: pgmap v2452: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.7 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 709 op/s; 25531/131004 objects degraded (19.489%); 16160/131004 objects misplaced (12.336%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T18:11:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:10 smithi130 ceph-mon[105521]: osdmap e1356: 8 total, 8 up, 4 in 2024-08-17T18:11:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:10 smithi130 ceph-mon[110068]: pgmap v2452: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.7 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 709 op/s; 25531/131004 objects degraded (19.489%); 16160/131004 objects misplaced (12.336%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T18:11:10.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:10 smithi130 ceph-mon[110068]: osdmap e1356: 8 total, 8 up, 4 in 2024-08-17T18:11:11.000 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:11.046 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:11.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:11.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:11.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:10 smithi179 ceph-mon[93519]: pgmap v2452: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 10 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.7 GiB data, 8.3 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 709 op/s; 25531/131004 objects degraded (19.489%); 16160/131004 objects misplaced (12.336%); 2.4 MiB/s, 37 objects/s recovering 2024-08-17T18:11:11.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:10 smithi179 ceph-mon[93519]: osdmap e1356: 8 total, 8 up, 4 in 2024-08-17T18:11:11.148 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:11.170 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:11.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:11.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:11.272 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:11.294 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:11.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:11.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:11.396 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:11.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:11.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:11.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:11.519 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:11.540 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:11.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:11.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:11.642 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:11.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:11.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:11.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:11.788 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:11.810 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:11.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:11.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:11.821 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 25531/131004 objects degraded (19.489%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:11.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 25531/131004 objects degraded (19.489%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:11.911 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:11.954 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:11.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:11.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:12.055 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:12.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:12.094 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:12.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:12.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 25531/131004 objects degraded (19.489%), 11 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:12.168 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:11:12.168 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:11:12.168 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:11:12.169 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:11:12.169 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:11:12.169 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:11:12.169 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:11:12.169 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:11:12.169 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:11:12.169 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:11:12.169 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:11:12.169 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:11:12.170 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:11:12.170 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:11:12.170 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:11:12.170 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:11:12.195 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:12.240 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:12.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:12.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:12.342 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:12.366 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:12.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:12.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:12.467 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:12.489 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:12.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:12.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:12.591 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:12.613 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:12.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:12.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:12.714 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:12.737 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:12.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:12.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:12.838 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:12.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:12.873 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:12.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:12.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:12 smithi130 ceph-mon[105521]: pgmap v2454: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.6 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 607 op/s; 24434/128871 objects degraded (18.960%); 16160/128871 objects misplaced (12.540%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T18:11:12.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:12 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3297480173' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:11:12.899 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:11:12.899 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:12 smithi130 ceph-mon[110068]: pgmap v2454: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.6 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 607 op/s; 24434/128871 objects degraded (18.960%); 16160/128871 objects misplaced (12.540%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T18:11:12.899 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:12 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3297480173' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:11:12.900 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:11:12.973 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:12.998 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:12.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:12.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:12 smithi179 ceph-mon[93519]: pgmap v2454: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 75 active+clean; 2.6 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 0 B/s wr, 607 op/s; 24434/128871 objects degraded (18.960%); 16160/128871 objects misplaced (12.540%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T18:11:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:12 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3297480173' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:11:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:11:13.099 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:13.141 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:13.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:13.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:13.243 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:13.276 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:13.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:13.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:13.378 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:13.401 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:13.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:13.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:13.503 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:13.526 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:13.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:13.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:13.628 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:13.651 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:13.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:13.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:13.753 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:13.778 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:13.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:13.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:13.880 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:13.902 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:13.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:13.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:14.003 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:14.025 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:14.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:14.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:14.127 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:14.149 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:14.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:14.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:14.251 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:14.272 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:14.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:14.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:14.374 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:14.396 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:14.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:14.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:14.499 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:14.522 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:14.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:14.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:14.624 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:14.645 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:14.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:14.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:14.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:14 smithi130 ceph-mon[105521]: pgmap v2455: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.5 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 612 op/s; 23605/124035 objects degraded (19.031%); 15227/124035 objects misplaced (12.276%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T18:11:14.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:14 smithi130 ceph-mon[110068]: pgmap v2455: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.5 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 612 op/s; 23605/124035 objects degraded (19.031%); 15227/124035 objects misplaced (12.276%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T18:11:14.747 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:14.769 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:14.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:14.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:14.871 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:14.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:14.909 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:14.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:15.009 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:15.053 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:15.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:15.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:15.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:14 smithi179 ceph-mon[93519]: pgmap v2455: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.5 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 612 op/s; 23605/124035 objects degraded (19.031%); 15227/124035 objects misplaced (12.276%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T18:11:15.155 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:15.177 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:15.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:15.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:15.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:15.301 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:15.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:15.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:15.403 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:15.425 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:15.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:15.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:15.526 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:15.549 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:15.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:15.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:15.650 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:15.692 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:15.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:15.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:15.793 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:15.837 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:15.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:15.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:15.939 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:15.984 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:15.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:15.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:16.086 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:16.126 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:16.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:16.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:16.228 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:16.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:16.268 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:16.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:16.369 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:16.418 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:16.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:16.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:16.520 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:16.565 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:16.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:16.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:16.666 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:16.688 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:16.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:16.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:16.790 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:16.813 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:16.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:16.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:16.915 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:16.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:16 smithi130 ceph-mon[105521]: pgmap v2456: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 799 op/s; 22226/119547 objects degraded (18.592%); 15034/119547 objects misplaced (12.576%); 9.0 MiB/s, 144 objects/s recovering 2024-08-17T18:11:16.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 22226/119547 objects degraded (18.592%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:16.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:16 smithi130 ceph-mon[110068]: pgmap v2456: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 799 op/s; 22226/119547 objects degraded (18.592%); 15034/119547 objects misplaced (12.576%); 9.0 MiB/s, 144 objects/s recovering 2024-08-17T18:11:16.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 22226/119547 objects degraded (18.592%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:16.960 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:16.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:16.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:17.062 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:17.084 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:17.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:17.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:17.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:16 smithi179 ceph-mon[93519]: pgmap v2456: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 799 op/s; 22226/119547 objects degraded (18.592%); 15034/119547 objects misplaced (12.576%); 9.0 MiB/s, 144 objects/s recovering 2024-08-17T18:11:17.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 22226/119547 objects degraded (18.592%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:17.157 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:11:17.186 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:17.207 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:17.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:17.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:17.309 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:17.378 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:17.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:17.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:17.480 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:17.545 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:17.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:17.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:17.625 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:11:17.648 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:17.669 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:17.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:17.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:17.771 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:17.828 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:17.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:17.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:17.930 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:17.952 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:17.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:17.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:18.054 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:18.109 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:18.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:18.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:18.164 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:11:18.164 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 22226/118923 objects degraded (18.689%), 10 pgs degraded, 10 pgs undersized","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2599,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1356,"num_osds":8,"num_up_osds":8,"osd_up_since":1723918167,"num_in_osds":4,"osd_in_since":1723918136,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":76},{"state_name":"active+undersized+degraded+remapped+backfill_wait","count":9},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+undersized+degraded+remapped+backfilling","count":1}],"num_pgs":89,"num_pools":5,"num_objects":39641,"data_bytes":2596739045,"bytes_used":8385003520,"bytes_avail":375595139072,"bytes_total":383980142592,"degraded_objects":22226,"degraded_total":118923,"degraded_ratio":0.18689404068178569,"misplaced_objects":15034,"misplaced_total":118923,"misplaced_ratio":0.12641793429361856,"recovering_objects_per_sec":108,"recovering_bytes_per_sec":7128393,"recovering_keys_per_sec":0,"num_objects_recovered":963,"num_bytes_recovered":63111168,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":526},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":32,"modified":"2024-08-17T18:10:27.432406+0000","services":{}},"progress_events":{"4861586d-5402-4f10-93bc-62f8c4bfdc13":{"message":"Global Recovery Event (5m)\n [=======================.....] (remaining: 57s)","progress":0.84269660711288452,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T18:11:18.211 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:18.234 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:18.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:18.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:18.338 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:18.385 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:18.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:18.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:18.487 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:18.509 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:18.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:18.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:18.611 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:18.635 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:18.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:18.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:18.654 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:11:18.737 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:18.776 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:18.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:18.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:18.879 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:18.902 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:18.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:18.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:18.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:18 smithi130 ceph-mon[105521]: pgmap v2457: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 526 op/s; 22226/118923 objects degraded (18.689%); 15034/118923 objects misplaced (12.642%); 6.8 MiB/s, 108 objects/s recovering 2024-08-17T18:11:18.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1694214127' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:11:18.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:18 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:11:18.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4018859118' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T18:11:18.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:18 smithi130 ceph-mon[110068]: pgmap v2457: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 526 op/s; 22226/118923 objects degraded (18.689%); 15034/118923 objects misplaced (12.642%); 6.8 MiB/s, 108 objects/s recovering 2024-08-17T18:11:18.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1694214127' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:11:18.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:18 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:11:18.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4018859118' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T18:11:19.004 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:19.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:19.027 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:19.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:19.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:18 smithi179 ceph-mon[93519]: pgmap v2457: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 526 op/s; 22226/118923 objects degraded (18.689%); 15034/118923 objects misplaced (12.642%); 6.8 MiB/s, 108 objects/s recovering 2024-08-17T18:11:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1694214127' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:11:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:18 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:11:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4018859118' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T18:11:19.128 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:19.151 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:19.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:19.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:19.252 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:19.293 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:19.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:19.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:19.395 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:19.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:19.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:19.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:19.518 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:19.525 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:11:19.592 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:19.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:19.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:19.694 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:19.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:19.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:19.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:19.817 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:19.840 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:19.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:19.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:19.942 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:19.964 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:19.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:19.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:20.066 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:20.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:19 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:11:20.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:19 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:11:20.067 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:19 smithi130 ceph-mon[105521]: osdmap e1357: 8 total, 8 up, 4 in 2024-08-17T18:11:20.067 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:19 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:11:20.067 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:19 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:11:20.067 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:19 smithi130 ceph-mon[110068]: osdmap e1357: 8 total, 8 up, 4 in 2024-08-17T18:11:20.068 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:11:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:11:19] "GET /metrics HTTP/1.1" 200 36806 "" "Prometheus/2.51.0" 2024-08-17T18:11:20.089 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:20.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:20.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:20.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:19 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:11:20.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:19 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:11:20.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:19 smithi179 ceph-mon[93519]: osdmap e1357: 8 total, 8 up, 4 in 2024-08-17T18:11:20.191 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:20.213 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:20.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:20.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:20.314 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:20.337 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:20.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:20.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:20.439 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:20.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:20.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:20.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:20.563 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:20.584 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:20.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:20.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:20.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:11:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:11:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:11:20.616 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T18:11:20.616 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:11:20.686 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:20.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:20.735 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:20.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:20.836 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:20.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:20.894 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:20.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:20 smithi130 ceph-mon[105521]: pgmap v2459: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.3 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 683 op/s; 20760/110814 objects degraded (18.734%); 13522/110814 objects misplaced (12.202%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T18:11:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:20 smithi130 ceph-mon[110068]: pgmap v2459: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.3 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 683 op/s; 20760/110814 objects degraded (18.734%); 13522/110814 objects misplaced (12.202%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T18:11:20.995 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:21.019 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:21.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:21.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:21.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:20 smithi179 ceph-mon[93519]: pgmap v2459: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.3 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 683 op/s; 20760/110814 objects degraded (18.734%); 13522/110814 objects misplaced (12.202%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T18:11:21.120 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:21.142 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:21.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:21.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:21.244 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:21.265 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:21.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:21.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:21.367 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:21.389 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:21.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:21.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:21.491 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:21.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:21.513 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:21.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:21.614 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:21.671 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:21.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:21.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:21.773 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:21.819 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:21.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:21.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:21.865 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 20760/110814 objects degraded (18.734%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:21.865 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 20760/110814 objects degraded (18.734%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:21.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:21.963 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:21.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:21.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:22.065 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:22.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 20760/110814 objects degraded (18.734%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:22.112 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:11:22.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:22.117 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:22.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:22.218 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:22.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:22.239 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:22.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:22.340 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:22.363 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:22.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:22.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:22.465 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:22.517 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:22.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:22.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:22.619 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:22.658 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:22.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:22.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:22.760 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:22.782 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:22.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:22.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:22.884 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:22.920 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:22.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:22.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:23.022 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:23.070 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:22 smithi130 ceph-mon[105521]: pgmap v2460: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.2 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 701 op/s; 19860/107799 objects degraded (18.423%); 13522/107799 objects misplaced (12.544%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T18:11:23.070 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:22 smithi130 ceph-mon[110068]: pgmap v2460: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.2 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 701 op/s; 19860/107799 objects degraded (18.423%); 13522/107799 objects misplaced (12.544%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T18:11:23.076 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:23.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:23.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:23.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:22 smithi179 ceph-mon[93519]: pgmap v2460: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.2 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 701 op/s; 19860/107799 objects degraded (18.423%); 13522/107799 objects misplaced (12.544%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T18:11:23.178 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:23.199 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:23.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:23.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:23.301 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:23.322 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:23.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:23.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:23.424 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:23.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:23.467 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:23.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:23.568 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:23.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:23.591 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:23.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:23.692 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:23.749 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:23.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:23.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:23.850 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:23.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:23.873 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:23.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:23.931 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:11:23.948 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:11:23.948 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:11:23.974 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:24.026 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:24.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:24.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:24.128 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:24.150 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:24.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:24.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:24.252 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:24.287 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:24.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:24.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:24.388 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:24.432 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:24.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:24.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:24.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:24.570 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:24.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:24.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:24.672 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:24.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:24.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:24.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:24.797 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:24.834 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:24.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:24.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:24.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:24 smithi130 ceph-mon[105521]: pgmap v2461: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.0 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 775 op/s; 18765/100755 objects degraded (18.624%); 12269/100755 objects misplaced (12.177%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T18:11:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:24 smithi130 ceph-mon[110068]: pgmap v2461: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.0 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 775 op/s; 18765/100755 objects degraded (18.624%); 12269/100755 objects misplaced (12.177%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T18:11:24.936 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:24.988 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:24.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:24.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:24.991 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T18:11:25.090 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:25.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:24 smithi179 ceph-mon[93519]: pgmap v2461: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 2.0 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 775 op/s; 18765/100755 objects degraded (18.624%); 12269/100755 objects misplaced (12.177%); 3.9 MiB/s, 62 objects/s recovering 2024-08-17T18:11:25.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:25.156 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:25.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:25.192 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:11:25.258 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:25.262 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:11:25.262 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:11:25.303 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:11:25.304 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:11:25.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:25.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:25.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:25.343 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:11:25.343 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:11:25.391 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:11:25.391 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:11:25.418 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:11:25.418 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:11:25.421 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:25.449 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:11:25.450 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:11:25.478 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:11:25.478 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:11:25.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:25.479 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:25.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:25.508 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:11:25.508 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:11:25.581 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:25.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:25.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:25.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:25.703 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:25.740 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:25.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:25.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:25.842 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:25.865 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:25.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:25.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:25.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:25 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4178957260' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:11:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:25 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4178957260' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:11:25.966 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:25.989 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:25.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:25.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:26.091 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:26.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:25 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4178957260' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:11:26.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:26.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:26.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:26.215 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:26.250 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:26.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:26.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:26.352 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:26.373 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:26.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:26.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:26.475 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:26.515 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:26.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:26.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:26.617 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:26.639 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:26.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:26.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:26.741 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:26.764 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:26.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:26.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:26.866 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:26.902 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:26.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:26.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:27.004 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:27.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:27.028 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:27.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:27.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:26 smithi179 ceph-mon[93519]: pgmap v2462: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.9 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 797 op/s; 17404/95598 objects degraded (18.205%); 12023/95598 objects misplaced (12.577%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T18:11:27.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17404/95598 objects degraded (18.205%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:27.129 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:27.151 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:27.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:27.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:27.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:26 smithi130 ceph-mon[105521]: pgmap v2462: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.9 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 797 op/s; 17404/95598 objects degraded (18.205%); 12023/95598 objects misplaced (12.577%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T18:11:27.153 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17404/95598 objects degraded (18.205%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:27.153 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:26 smithi130 ceph-mon[110068]: pgmap v2462: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.9 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 797 op/s; 17404/95598 objects degraded (18.205%); 12023/95598 objects misplaced (12.577%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T18:11:27.153 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17404/95598 objects degraded (18.205%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:27.253 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:27.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:27.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:27.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:27.376 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:27.397 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:27.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:27.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:27.499 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:27.521 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:27.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:27.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:27.623 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:27.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:27.660 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:27.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:27.761 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:27.800 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:27.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:27.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:27.902 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:27.923 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:27.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:27.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:28.025 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:28.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:28.064 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:28.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:28.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:11:28.165 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:28.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:11:28.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:11:28.189 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:28.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:28.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:28.290 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:28.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:28.314 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:28.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:28.416 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:28.439 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:28.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:28.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:28.541 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:28.580 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:28.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:28.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:28.682 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:28.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:28.708 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:28.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:28.809 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:28.850 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:28.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:28.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:28.951 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:28.995 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:28.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:28.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:29.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:28 smithi179 ceph-mon[93519]: pgmap v2463: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.9 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 17404/94815 objects degraded (18.356%); 12023/94815 objects misplaced (12.680%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T18:11:29.096 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:29.118 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:29.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:29.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:29.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:28 smithi130 ceph-mon[105521]: pgmap v2463: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.9 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 17404/94815 objects degraded (18.356%); 12023/94815 objects misplaced (12.680%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T18:11:29.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:28 smithi130 ceph-mon[110068]: pgmap v2463: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.9 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 17404/94815 objects degraded (18.356%); 12023/94815 objects misplaced (12.680%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T18:11:29.219 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:29.260 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:29.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:29.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:29.362 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:29.404 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:29.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:29.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:29.506 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:29.529 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:29.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:29.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:29.631 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:29.652 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:29.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:29.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:29.754 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:29.776 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:29.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:29.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:29.878 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:29.912 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:29.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:29.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:30.014 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:30.050 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:30.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:30.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:30.152 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:11:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:11:29] "GET /metrics HTTP/1.1" 200 36806 "" "Prometheus/2.51.0" 2024-08-17T18:11:30.175 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:30.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:30.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:30.277 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:30.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:30.319 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:30.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:30.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:30.460 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:30.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:30.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:30.562 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:30.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:30.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:30.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:30.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:11:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:11:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:11:30.687 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:30.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:30.723 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:30.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:30.824 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:30.861 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:30.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:30.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:30.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:30 smithi130 ceph-mon[105521]: pgmap v2464: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.8 GiB data, 6.6 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 15896/86901 objects degraded (18.292%); 10594/86901 objects misplaced (12.191%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T18:11:30.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:30 smithi130 ceph-mon[110068]: pgmap v2464: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.8 GiB data, 6.6 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 15896/86901 objects degraded (18.292%); 10594/86901 objects misplaced (12.191%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T18:11:30.963 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:30.985 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:30.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:30.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:31.087 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:31.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:30 smithi179 ceph-mon[93519]: pgmap v2464: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.8 GiB data, 6.6 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 803 op/s; 15896/86901 objects degraded (18.292%); 10594/86901 objects misplaced (12.191%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T18:11:31.140 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:31.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:31.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:31.242 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:31.264 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:31.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:31.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:31.366 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:31.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:31.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:31.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:31.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:31.549 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:31.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:31.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:31.653 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:31.675 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:31.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:31.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:31.777 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:31.800 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:31.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:31.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:31.902 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:31.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15896/86901 objects degraded (18.292%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:31.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:31 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:11:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15896/86901 objects degraded (18.292%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:31 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:11:31.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:31.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:31.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:32.075 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:32.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15896/86901 objects degraded (18.292%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:32.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:31 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:11:32.134 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:32.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:32.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:32.237 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:32.293 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:32.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:32.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:32.395 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:32.416 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:32.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:32.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:32.517 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:32.541 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:32.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:32.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:32.642 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:32.665 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:32.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:32.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:32.769 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:32.792 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:32.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:32.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:32 smithi179 ceph-mon[93519]: pgmap v2465: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.7 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 744 op/s; 15016/84009 objects degraded (17.874%); 10594/84009 objects misplaced (12.611%); 4.1 MiB/s, 66 objects/s recovering 2024-08-17T18:11:32.894 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:32.916 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:32.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:32.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:33.017 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:33.039 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:33.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:33.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:33.141 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:33.162 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:33.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:33.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:33.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:32 smithi130 ceph-mon[105521]: pgmap v2465: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.7 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 744 op/s; 15016/84009 objects degraded (17.874%); 10594/84009 objects misplaced (12.611%); 4.1 MiB/s, 66 objects/s recovering 2024-08-17T18:11:33.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:32 smithi130 ceph-mon[110068]: pgmap v2465: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.7 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 744 op/s; 15016/84009 objects degraded (17.874%); 10594/84009 objects misplaced (12.611%); 4.1 MiB/s, 66 objects/s recovering 2024-08-17T18:11:33.264 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:33.285 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:33.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:33.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:33.387 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:33.408 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:33.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:33.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:33.510 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:33.549 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:33.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:33.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:33.651 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:33.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:33.675 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:33.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:33.776 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:33.817 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:33.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:33.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:33.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:33.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:33.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:33.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:34.045 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:34.067 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:34.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:34.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:34.169 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:34.178 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:11:34.195 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:34.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:34.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:34.297 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:34.318 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:34.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:34.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:34.420 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:34.441 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:34.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:34.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:34.543 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:34.568 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:34.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:34.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:34.670 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:34.695 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:34.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:34.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:34.797 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:34.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:34.820 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:34.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:34.908 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:11:34.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:34.956 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:11:34.961 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:11:35.009 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:35.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:35.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:35.077 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:34 smithi130 ceph-mon[110068]: pgmap v2466: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 830 op/s; 14033/77904 objects degraded (18.013%); 9472/77904 objects misplaced (12.159%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T18:11:35.077 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:34 smithi130 ceph-mon[110068]: osdmap e1358: 8 total, 8 up, 4 in 2024-08-17T18:11:35.081 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:34 smithi130 ceph-mon[105521]: pgmap v2466: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 830 op/s; 14033/77904 objects degraded (18.013%); 9472/77904 objects misplaced (12.159%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T18:11:35.081 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:34 smithi130 ceph-mon[105521]: osdmap e1358: 8 total, 8 up, 4 in 2024-08-17T18:11:35.086 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:34 smithi179 ceph-mon[93519]: pgmap v2466: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 9 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.6 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 830 op/s; 14033/77904 objects degraded (18.013%); 9472/77904 objects misplaced (12.159%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T18:11:35.086 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:34 smithi179 ceph-mon[93519]: osdmap e1358: 8 total, 8 up, 4 in 2024-08-17T18:11:35.110 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:35.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:35.164 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:35.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:35.265 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:35.288 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:35.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:35.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:35.390 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:35.434 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:35.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:35.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:35.535 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:35.596 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:35.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:35.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:35.698 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:35.734 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:35.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:35.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:35.783 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:11:35.836 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:35.874 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:35.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:35.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:35.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:35 smithi130 ceph-mon[105521]: osdmap e1359: 8 total, 8 up, 4 in 2024-08-17T18:11:35.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:35 smithi130 ceph-mon[105521]: from='client.36837 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:11:35.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:35 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1896635694' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:11:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:35 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:11:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:35 smithi130 ceph-mon[110068]: osdmap e1359: 8 total, 8 up, 4 in 2024-08-17T18:11:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:35 smithi130 ceph-mon[110068]: from='client.36837 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:11:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:35 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1896635694' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:11:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:35 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:11:35.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:36.022 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:36.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:36.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:36.054 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:35 smithi179 ceph-mon[93519]: osdmap e1359: 8 total, 8 up, 4 in 2024-08-17T18:11:36.054 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:35 smithi179 ceph-mon[93519]: from='client.36837 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:11:36.054 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:35 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1896635694' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:11:36.054 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:35 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:11:36.124 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:36.148 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:36.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:36.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:36.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:36.286 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:36.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:36.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:36.388 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:36.428 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:36.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:36.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:36.530 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:36.545 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2468,"stamp":"2024-08-17T18:11:34.764909+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1661016037,"num_objects":25363,"num_object_clones":0,"num_object_copies":76089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13557,"num_objects_misplaced":9276,"num_objects_unfound":0,"num_objects_dirty":25363,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":238867,"num_write_kb":8448016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12036,"num_bytes_recovered":788739469,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78313,"ondisk_log_size":78313,"up":267,"acting":257,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":6444224,"kb_used_data":3496544,"kb_used_omap":294,"kb_used_meta":2947289,"kb_avail":368536384,"statfs":{"total":383980142592,"available":377381257216,"internally_reserved":0,"allocated":3580461056,"data_stored":3562703731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":301759,"internal_metadata":3018024257},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,5,1,5,6],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-426180608,"num_objects":-6503,"num_object_clones":0,"num_object_copies":-19509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3847,"num_objects_misplaced":-2747,"num_objects_unfound":0,"num_objects_dirty":-6503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6503,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":365,"num_bytes_recovered":23920640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.305160"},"pg_stats":[{"pgid":"48.f","version":"1357'14912","reported_seq":25240,"reported_epoch":1357,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:33.121076+0000","last_change":"2024-08-17T18:09:29.893285+0000","last_active":"2024-08-17T18:11:33.121076+0000","last_peered":"2024-08-17T18:11:33.121076+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:28.981495+0000","last_became_peered":"2024-08-17T18:09:28.981495+0000","last_unstale":"2024-08-17T18:11:33.121076+0000","last_undegraded":"2024-08-17T18:09:28.850340+0000","last_fullsized":"2024-08-17T18:09:28.832563+0000","mapping_epoch":1343,"log_start":"1347'10757","ondisk_log_start":"1347'10757","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4155,"log_dups_size":0,"ondisk_log_size":4155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:54.654486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107610112,"num_objects":1642,"num_object_clones":0,"num_object_copies":4926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":567,"num_objects_misplaced":1642,"num_objects_unfound":0,"num_objects_dirty":1642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14912,"num_write_kb":529728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5130,"num_bytes_recovered":336199680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":1642}],"blocked_by":[],"up_primary":3,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.e","version":"1357'14795","reported_seq":18317,"reported_epoch":1357,"state":"active+undersized+degraded+remapped+backfilling","last_fresh":"2024-08-17T18:11:30.065742+0000","last_change":"2024-08-17T18:11:08.169594+0000","last_active":"2024-08-17T18:11:30.065742+0000","last_peered":"2024-08-17T18:11:30.065742+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.950126+0000","last_became_peered":"2024-08-17T18:09:29.950126+0000","last_unstale":"2024-08-17T18:11:30.065742+0000","last_undegraded":"2024-08-17T18:09:29.863735+0000","last_fullsized":"2024-08-17T18:09:29.826958+0000","mapping_epoch":1344,"log_start":"1345'9848","ondisk_log_start":"1345'9848","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4947,"log_dups_size":0,"ondisk_log_size":4947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109903872,"num_objects":1677,"num_object_clones":0,"num_object_copies":5031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":411,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14795,"num_write_kb":527104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1735,"num_bytes_recovered":113704960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1677}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.d","version":"1357'14816","reported_seq":14862,"reported_epoch":1357,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:32.552271+0000","last_change":"2024-08-17T18:09:29.930605+0000","last_active":"2024-08-17T18:11:32.552271+0000","last_peered":"2024-08-17T18:11:32.552271+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.918910+0000","last_became_peered":"2024-08-17T18:09:29.918910+0000","last_unstale":"2024-08-17T18:11:32.552271+0000","last_undegraded":"2024-08-17T18:09:29.832124+0000","last_fullsized":"2024-08-17T18:09:29.818954+0000","mapping_epoch":1344,"log_start":"1347'10624","ondisk_log_start":"1347'10624","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4192,"log_dups_size":0,"ondisk_log_size":4192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:00:55.077466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104005656,"num_objects":1588,"num_object_clones":0,"num_object_copies":4764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1588,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1588,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14816,"num_write_kb":524865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":1588}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.c","version":"1357'14805","reported_seq":14918,"reported_epoch":1357,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:33.100656+0000","last_change":"2024-08-17T18:09:28.996223+0000","last_active":"2024-08-17T18:11:33.100656+0000","last_peered":"2024-08-17T18:11:33.100656+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:28.914112+0000","last_became_peered":"2024-08-17T18:09:28.914112+0000","last_unstale":"2024-08-17T18:11:33.100656+0000","last_undegraded":"2024-08-17T18:09:28.840987+0000","last_fullsized":"2024-08-17T18:09:28.815757+0000","mapping_epoch":1343,"log_start":"1347'10646","ondisk_log_start":"1347'10646","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4159,"log_dups_size":0,"ondisk_log_size":4159,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:55.890300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101908480,"num_objects":1555,"num_object_clones":0,"num_object_copies":4665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1551,"num_objects_misplaced":1551,"num_objects_unfound":0,"num_objects_dirty":1555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14805,"num_write_kb":523520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":917504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":1555}],"blocked_by":[],"up_primary":3,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.b","version":"1357'15051","reported_seq":15096,"reported_epoch":1357,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:32.580661+0000","last_change":"2024-08-17T18:09:29.921199+0000","last_active":"2024-08-17T18:11:32.580661+0000","last_peered":"2024-08-17T18:11:32.580661+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.912384+0000","last_became_peered":"2024-08-17T18:09:29.912384+0000","last_unstale":"2024-08-17T18:11:32.580661+0000","last_undegraded":"2024-08-17T18:09:29.829736+0000","last_fullsized":"2024-08-17T18:09:29.817434+0000","mapping_epoch":1344,"log_start":"1347'10864","ondisk_log_start":"1347'10864","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4187,"log_dups_size":0,"ondisk_log_size":4187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:22.733621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105054208,"num_objects":1603,"num_object_clones":0,"num_object_copies":4809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1603,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15051,"num_write_kb":532928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1603}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.a","version":"1357'14752","reported_seq":14825,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:30.058946+0000","last_change":"2024-08-17T18:07:32.359061+0000","last_active":"2024-08-17T18:11:30.058946+0000","last_peered":"2024-08-17T18:11:30.058946+0000","last_clean":"2024-08-17T18:11:30.058946+0000","last_became_active":"2024-08-17T18:07:32.357107+0000","last_became_peered":"2024-08-17T18:07:32.357107+0000","last_unstale":"2024-08-17T18:11:30.058946+0000","last_undegraded":"2024-08-17T18:11:30.058946+0000","last_fullsized":"2024-08-17T18:11:30.058946+0000","mapping_epoch":1327,"log_start":"1345'9764","ondisk_log_start":"1345'9764","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4988,"log_dups_size":0,"ondisk_log_size":4988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:30:02.061787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106430464,"num_objects":1624,"num_object_clones":0,"num_object_copies":4872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14752,"num_write_kb":524032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.9","version":"1357'15204","reported_seq":15254,"reported_epoch":1357,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:29.997760+0000","last_change":"2024-08-17T18:09:29.968655+0000","last_active":"2024-08-17T18:11:29.997760+0000","last_peered":"2024-08-17T18:11:29.997760+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.948623+0000","last_became_peered":"2024-08-17T18:09:29.948623+0000","last_unstale":"2024-08-17T18:11:29.997760+0000","last_undegraded":"2024-08-17T18:09:29.855385+0000","last_fullsized":"2024-08-17T18:09:29.821477+0000","mapping_epoch":1344,"log_start":"1345'10190","ondisk_log_start":"1345'10190","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":5014,"log_dups_size":0,"ondisk_log_size":5014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113508352,"num_objects":1732,"num_object_clones":0,"num_object_copies":5196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1732,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15204,"num_write_kb":541952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1732}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.8","version":"1358'14976","reported_seq":15082,"reported_epoch":1358,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:34.064038+0000","last_change":"2024-08-17T18:09:29.904036+0000","last_active":"2024-08-17T18:11:34.064038+0000","last_peered":"2024-08-17T18:11:34.064038+0000","last_clean":"2024-08-17T18:09:27.840248+0000","last_became_active":"2024-08-17T18:09:28.942085+0000","last_became_peered":"2024-08-17T18:09:28.942085+0000","last_unstale":"2024-08-17T18:11:34.064038+0000","last_undegraded":"2024-08-17T18:11:34.064038+0000","last_fullsized":"2024-08-17T18:11:34.064038+0000","mapping_epoch":1343,"log_start":"1345'10061","ondisk_log_start":"1345'10061","created":1327,"last_epoch_clean":1337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4915,"log_dups_size":0,"ondisk_log_size":4915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101056512,"num_objects":1542,"num_object_clones":0,"num_object_copies":4626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1542,"num_objects_unfound":0,"num_objects_dirty":1542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14976,"num_write_kb":528576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,1],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1542}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.7","version":"1357'14886","reported_seq":14954,"reported_epoch":1357,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:33.111927+0000","last_change":"2024-08-17T18:09:28.989307+0000","last_active":"2024-08-17T18:11:33.111927+0000","last_peered":"2024-08-17T18:11:33.111927+0000","last_clean":"2024-08-17T18:09:27.824322+0000","last_became_active":"2024-08-17T18:09:28.981075+0000","last_became_peered":"2024-08-17T18:09:28.981075+0000","last_unstale":"2024-08-17T18:11:33.111927+0000","last_undegraded":"2024-08-17T18:11:33.111927+0000","last_fullsized":"2024-08-17T18:11:33.111927+0000","mapping_epoch":1343,"log_start":"1347'10748","ondisk_log_start":"1347'10748","created":1327,"last_epoch_clean":1337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4138,"log_dups_size":0,"ondisk_log_size":4138,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:12:08.456671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104726528,"num_objects":1598,"num_object_clones":0,"num_object_copies":4794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1598,"num_objects_unfound":0,"num_objects_dirty":1598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14886,"num_write_kb":527488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,1],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"1,2,7","objects":1598}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.6","version":"1357'14874","reported_seq":16173,"reported_epoch":1357,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:32.570321+0000","last_change":"2024-08-17T18:09:59.877439+0000","last_active":"2024-08-17T18:11:32.570321+0000","last_peered":"2024-08-17T18:11:32.570321+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:28.962573+0000","last_became_peered":"2024-08-17T18:09:28.962573+0000","last_unstale":"2024-08-17T18:11:32.570321+0000","last_undegraded":"2024-08-17T18:09:28.855040+0000","last_fullsized":"2024-08-17T18:09:28.816313+0000","mapping_epoch":1343,"log_start":"1333'4800","ondisk_log_start":"1333'4800","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":10074,"log_dups_size":0,"ondisk_log_size":10074,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100925440,"num_objects":1540,"num_object_clones":0,"num_object_copies":4620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1458,"num_objects_misplaced":1458,"num_objects_unfound":0,"num_objects_dirty":1540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14874,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":403,"num_bytes_recovered":26411008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":1540}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"48.5","version":"1357'14704","reported_seq":14755,"reported_epoch":1357,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:30.012326+0000","last_change":"2024-08-17T18:09:29.983372+0000","last_active":"2024-08-17T18:11:30.012326+0000","last_peered":"2024-08-17T18:11:30.012326+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.956933+0000","last_became_peered":"2024-08-17T18:09:29.956933+0000","last_unstale":"2024-08-17T18:11:30.012326+0000","last_undegraded":"2024-08-17T18:09:29.847436+0000","last_fullsized":"2024-08-17T18:09:29.819789+0000","mapping_epoch":1344,"log_start":"1345'9758","ondisk_log_start":"1345'9758","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4946,"log_dups_size":0,"ondisk_log_size":4946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110624768,"num_objects":1688,"num_object_clones":0,"num_object_copies":5064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1688,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14704,"num_write_kb":524544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1688}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.4","version":"1358'14903","reported_seq":15115,"reported_epoch":1358,"state":"active+undersized+degraded+remapped+backfilling","last_fresh":"2024-08-17T18:11:34.090161+0000","last_change":"2024-08-17T18:11:33.539022+0000","last_active":"2024-08-17T18:11:34.090161+0000","last_peered":"2024-08-17T18:11:34.090161+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.948027+0000","last_became_peered":"2024-08-17T18:09:29.948027+0000","last_unstale":"2024-08-17T18:11:34.090161+0000","last_undegraded":"2024-08-17T18:09:29.860142+0000","last_fullsized":"2024-08-17T18:09:29.828238+0000","mapping_epoch":1344,"log_start":"1345'10017","ondisk_log_start":"1345'10017","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4886,"log_dups_size":0,"ondisk_log_size":4886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99024896,"num_objects":1511,"num_object_clones":0,"num_object_copies":4533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1444,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14903,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":68,"num_bytes_recovered":4456448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":1511}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.3","version":"1358'14923","reported_seq":14997,"reported_epoch":1358,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:34.029793+0000","last_change":"2024-08-17T18:09:30.006916+0000","last_active":"2024-08-17T18:11:34.029793+0000","last_peered":"2024-08-17T18:11:34.029793+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.994996+0000","last_became_peered":"2024-08-17T18:09:29.994996+0000","last_unstale":"2024-08-17T18:11:34.029793+0000","last_undegraded":"2024-08-17T18:09:29.861017+0000","last_fullsized":"2024-08-17T18:09:29.830876+0000","mapping_epoch":1344,"log_start":"1345'10023","ondisk_log_start":"1345'10023","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4900,"log_dups_size":0,"ondisk_log_size":4900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99287040,"num_objects":1515,"num_object_clones":0,"num_object_copies":4545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1515,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14923,"num_write_kb":526016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1515}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.2","version":"1357'14893","reported_seq":28896,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:32.052102+0000","last_change":"2024-08-17T18:11:09.658283+0000","last_active":"2024-08-17T18:11:32.052102+0000","last_peered":"2024-08-17T18:11:32.052102+0000","last_clean":"2024-08-17T18:11:32.052102+0000","last_became_active":"2024-08-17T18:11:09.615907+0000","last_became_peered":"2024-08-17T18:11:09.615907+0000","last_unstale":"2024-08-17T18:11:32.052102+0000","last_undegraded":"2024-08-17T18:11:32.052102+0000","last_fullsized":"2024-08-17T18:11:32.052102+0000","mapping_epoch":1355,"log_start":"1347'10573","ondisk_log_start":"1347'10573","created":1327,"last_epoch_clean":1356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4320,"log_dups_size":0,"ondisk_log_size":4320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:04.549996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100335616,"num_objects":1531,"num_object_clones":0,"num_object_copies":4593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14893,"num_write_kb":525568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4640,"num_bytes_recovered":303955968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"48.1","version":"1357'15182","reported_seq":15220,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:32.605777+0000","last_change":"2024-08-17T18:07:32.360312+0000","last_active":"2024-08-17T18:11:32.605777+0000","last_peered":"2024-08-17T18:11:32.605777+0000","last_clean":"2024-08-17T18:11:32.605777+0000","last_became_active":"2024-08-17T18:07:32.360092+0000","last_became_peered":"2024-08-17T18:07:32.360092+0000","last_unstale":"2024-08-17T18:11:32.605777+0000","last_undegraded":"2024-08-17T18:11:32.605777+0000","last_fullsized":"2024-08-17T18:11:32.605777+0000","mapping_epoch":1327,"log_start":"1348'11022","ondisk_log_start":"1348'11022","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4160,"log_dups_size":0,"ondisk_log_size":4160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:41.580676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98697216,"num_objects":1506,"num_object_clones":0,"num_object_copies":4518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15182,"num_write_kb":534016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.0","version":"1357'14913","reported_seq":14989,"reported_epoch":1357,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:32.580872+0000","last_change":"2024-08-17T18:09:28.963421+0000","last_active":"2024-08-17T18:11:32.580872+0000","last_peered":"2024-08-17T18:11:32.580872+0000","last_clean":"2024-08-17T18:09:27.823675+0000","last_became_active":"2024-08-17T18:09:28.941589+0000","last_became_peered":"2024-08-17T18:09:28.941589+0000","last_unstale":"2024-08-17T18:11:32.580872+0000","last_undegraded":"2024-08-17T18:11:32.580872+0000","last_fullsized":"2024-08-17T18:11:32.580872+0000","mapping_epoch":1343,"log_start":"1347'10788","ondisk_log_start":"1347'10788","created":1327,"last_epoch_clean":1337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4125,"log_dups_size":0,"ondisk_log_size":4125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:36.800547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97320960,"num_objects":1485,"num_object_clones":0,"num_object_copies":4455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1485,"num_objects_unfound":0,"num_objects_dirty":1485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14913,"num_write_kb":524736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,1],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"1,2,7","objects":1485}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":2205,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.774177+0000","last_change":"2024-08-17T18:09:28.870033+0000","last_active":"2024-08-17T18:11:33.774177+0000","last_peered":"2024-08-17T18:11:33.774177+0000","last_clean":"2024-08-17T18:11:33.774177+0000","last_became_active":"2024-08-17T18:09:28.869346+0000","last_became_peered":"2024-08-17T18:09:28.869346+0000","last_unstale":"2024-08-17T18:11:33.774177+0000","last_undegraded":"2024-08-17T18:11:33.774177+0000","last_fullsized":"2024-08-17T18:11:33.774177+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:23.168631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":497,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.659570+0000","last_change":"2024-08-17T18:09:29.841584+0000","last_active":"2024-08-17T18:11:18.659570+0000","last_peered":"2024-08-17T18:11:18.659570+0000","last_clean":"2024-08-17T18:11:18.659570+0000","last_became_active":"2024-08-17T18:09:29.840950+0000","last_became_peered":"2024-08-17T18:09:29.840950+0000","last_unstale":"2024-08-17T18:11:18.659570+0000","last_undegraded":"2024-08-17T18:11:18.659570+0000","last_fullsized":"2024-08-17T18:11:18.659570+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:43.035959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":1993,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.662600+0000","last_change":"2024-08-17T18:09:29.883587+0000","last_active":"2024-08-17T18:11:18.662600+0000","last_peered":"2024-08-17T18:11:18.662600+0000","last_clean":"2024-08-17T18:11:18.662600+0000","last_became_active":"2024-08-17T18:09:29.882222+0000","last_became_peered":"2024-08-17T18:09:29.882222+0000","last_unstale":"2024-08-17T18:11:18.662600+0000","last_undegraded":"2024-08-17T18:11:18.662600+0000","last_fullsized":"2024-08-17T18:11:18.662600+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:10.777993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2049,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.656455+0000","last_change":"2024-08-17T18:09:28.858824+0000","last_active":"2024-08-17T18:11:18.656455+0000","last_peered":"2024-08-17T18:11:18.656455+0000","last_clean":"2024-08-17T18:11:18.656455+0000","last_became_active":"2024-08-17T18:09:28.857856+0000","last_became_peered":"2024-08-17T18:09:28.857856+0000","last_unstale":"2024-08-17T18:11:18.656455+0000","last_undegraded":"2024-08-17T18:11:18.656455+0000","last_fullsized":"2024-08-17T18:11:18.656455+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:41:35.146166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":53,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.662121+0000","last_change":"2024-08-17T18:09:29.851515+0000","last_active":"2024-08-17T18:11:18.662121+0000","last_peered":"2024-08-17T18:11:18.662121+0000","last_clean":"2024-08-17T18:11:18.662121+0000","last_became_active":"2024-08-17T18:09:29.851109+0000","last_became_peered":"2024-08-17T18:09:29.851109+0000","last_unstale":"2024-08-17T18:11:18.662121+0000","last_undegraded":"2024-08-17T18:11:18.662121+0000","last_fullsized":"2024-08-17T18:11:18.662121+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:42.054532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":69,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665381+0000","last_change":"2024-08-17T18:03:16.089811+0000","last_active":"2024-08-17T18:11:09.665381+0000","last_peered":"2024-08-17T18:11:09.665381+0000","last_clean":"2024-08-17T18:11:09.665381+0000","last_became_active":"2024-08-17T18:03:16.089571+0000","last_became_peered":"2024-08-17T18:03:16.089571+0000","last_unstale":"2024-08-17T18:11:09.665381+0000","last_undegraded":"2024-08-17T18:11:09.665381+0000","last_fullsized":"2024-08-17T18:11:09.665381+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:04.630361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":51,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665590+0000","last_change":"2024-08-17T18:08:57.577055+0000","last_active":"2024-08-17T18:11:09.665590+0000","last_peered":"2024-08-17T18:11:09.665590+0000","last_clean":"2024-08-17T18:11:09.665590+0000","last_became_active":"2024-08-17T18:08:57.573659+0000","last_became_peered":"2024-08-17T18:08:57.573659+0000","last_unstale":"2024-08-17T18:11:09.665590+0000","last_undegraded":"2024-08-17T18:11:09.665590+0000","last_fullsized":"2024-08-17T18:11:09.665590+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:01.686752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":837,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.663137+0000","last_change":"2024-08-17T18:09:29.852393+0000","last_active":"2024-08-17T18:11:18.663137+0000","last_peered":"2024-08-17T18:11:18.663137+0000","last_clean":"2024-08-17T18:11:18.663137+0000","last_became_active":"2024-08-17T18:09:29.851947+0000","last_became_peered":"2024-08-17T18:09:29.851947+0000","last_unstale":"2024-08-17T18:11:18.663137+0000","last_undegraded":"2024-08-17T18:11:18.663137+0000","last_fullsized":"2024-08-17T18:11:18.663137+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:49.145327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":2150,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.775647+0000","last_change":"2024-08-17T18:09:29.863241+0000","last_active":"2024-08-17T18:11:33.775647+0000","last_peered":"2024-08-17T18:11:33.775647+0000","last_clean":"2024-08-17T18:11:33.775647+0000","last_became_active":"2024-08-17T18:09:29.862369+0000","last_became_peered":"2024-08-17T18:09:29.862369+0000","last_unstale":"2024-08-17T18:11:33.775647+0000","last_undegraded":"2024-08-17T18:11:33.775647+0000","last_fullsized":"2024-08-17T18:11:33.775647+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:30.518551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":584,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.661430+0000","last_change":"2024-08-17T18:09:29.896233+0000","last_active":"2024-08-17T18:11:18.661430+0000","last_peered":"2024-08-17T18:11:18.661430+0000","last_clean":"2024-08-17T18:11:18.661430+0000","last_became_active":"2024-08-17T18:09:29.895718+0000","last_became_peered":"2024-08-17T18:09:29.895718+0000","last_unstale":"2024-08-17T18:11:18.661430+0000","last_undegraded":"2024-08-17T18:11:18.661430+0000","last_fullsized":"2024-08-17T18:11:18.661430+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:03.278878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":60,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.784073+0000","last_change":"2024-08-17T18:08:57.574015+0000","last_active":"2024-08-17T18:11:33.784073+0000","last_peered":"2024-08-17T18:11:33.784073+0000","last_clean":"2024-08-17T18:11:33.784073+0000","last_became_active":"2024-08-17T18:08:57.567973+0000","last_became_peered":"2024-08-17T18:08:57.567973+0000","last_unstale":"2024-08-17T18:11:33.784073+0000","last_undegraded":"2024-08-17T18:11:33.784073+0000","last_fullsized":"2024-08-17T18:11:33.784073+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:06.251764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2004,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.662210+0000","last_change":"2024-08-17T18:09:29.917203+0000","last_active":"2024-08-17T18:11:18.662210+0000","last_peered":"2024-08-17T18:11:18.662210+0000","last_clean":"2024-08-17T18:11:18.662210+0000","last_became_active":"2024-08-17T18:09:29.916844+0000","last_became_peered":"2024-08-17T18:09:29.916844+0000","last_unstale":"2024-08-17T18:11:18.662210+0000","last_undegraded":"2024-08-17T18:11:18.662210+0000","last_fullsized":"2024-08-17T18:11:18.662210+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.453720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1910,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.656223+0000","last_change":"2024-08-17T18:09:28.859255+0000","last_active":"2024-08-17T18:11:18.656223+0000","last_peered":"2024-08-17T18:11:18.656223+0000","last_clean":"2024-08-17T18:11:18.656223+0000","last_became_active":"2024-08-17T18:09:28.858537+0000","last_became_peered":"2024-08-17T18:09:28.858537+0000","last_unstale":"2024-08-17T18:11:18.656223+0000","last_undegraded":"2024-08-17T18:11:18.656223+0000","last_fullsized":"2024-08-17T18:11:18.656223+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:15.364687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2173,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.774309+0000","last_change":"2024-08-17T18:09:28.868303+0000","last_active":"2024-08-17T18:11:33.774309+0000","last_peered":"2024-08-17T18:11:33.774309+0000","last_clean":"2024-08-17T18:11:33.774309+0000","last_became_active":"2024-08-17T18:09:28.867989+0000","last_became_peered":"2024-08-17T18:09:28.867989+0000","last_unstale":"2024-08-17T18:11:33.774309+0000","last_undegraded":"2024-08-17T18:11:33.774309+0000","last_fullsized":"2024-08-17T18:11:33.774309+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:58:52.741685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":36,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.659331+0000","last_change":"2024-08-17T18:09:29.842129+0000","last_active":"2024-08-17T18:11:18.659331+0000","last_peered":"2024-08-17T18:11:18.659331+0000","last_clean":"2024-08-17T18:11:18.659331+0000","last_became_active":"2024-08-17T18:09:29.841848+0000","last_became_peered":"2024-08-17T18:09:29.841848+0000","last_unstale":"2024-08-17T18:11:18.659331+0000","last_undegraded":"2024-08-17T18:11:18.659331+0000","last_fullsized":"2024-08-17T18:11:18.659331+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:02.119134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1287,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.774085+0000","last_change":"2024-08-17T18:09:29.846509+0000","last_active":"2024-08-17T18:11:33.774085+0000","last_peered":"2024-08-17T18:11:33.774085+0000","last_clean":"2024-08-17T18:11:33.774085+0000","last_became_active":"2024-08-17T18:09:29.844097+0000","last_became_peered":"2024-08-17T18:09:29.844097+0000","last_unstale":"2024-08-17T18:11:33.774085+0000","last_undegraded":"2024-08-17T18:11:33.774085+0000","last_fullsized":"2024-08-17T18:11:33.774085+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:32.340354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1998,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.659429+0000","last_change":"2024-08-17T18:09:29.841335+0000","last_active":"2024-08-17T18:11:18.659429+0000","last_peered":"2024-08-17T18:11:18.659429+0000","last_clean":"2024-08-17T18:11:18.659429+0000","last_became_active":"2024-08-17T18:09:29.840522+0000","last_became_peered":"2024-08-17T18:09:29.840522+0000","last_unstale":"2024-08-17T18:11:18.659429+0000","last_undegraded":"2024-08-17T18:11:18.659429+0000","last_fullsized":"2024-08-17T18:11:18.659429+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:08.015836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2178,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.774275+0000","last_change":"2024-08-17T18:09:29.847494+0000","last_active":"2024-08-17T18:11:33.774275+0000","last_peered":"2024-08-17T18:11:33.774275+0000","last_clean":"2024-08-17T18:11:33.774275+0000","last_became_active":"2024-08-17T18:09:29.846973+0000","last_became_peered":"2024-08-17T18:09:29.846973+0000","last_unstale":"2024-08-17T18:11:33.774275+0000","last_undegraded":"2024-08-17T18:11:33.774275+0000","last_fullsized":"2024-08-17T18:11:33.774275+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:49.469921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1957,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.655595+0000","last_change":"2024-08-17T18:09:28.855045+0000","last_active":"2024-08-17T18:11:18.655595+0000","last_peered":"2024-08-17T18:11:18.655595+0000","last_clean":"2024-08-17T18:11:18.655595+0000","last_became_active":"2024-08-17T18:09:28.854607+0000","last_became_peered":"2024-08-17T18:09:28.854607+0000","last_unstale":"2024-08-17T18:11:18.655595+0000","last_undegraded":"2024-08-17T18:11:18.655595+0000","last_fullsized":"2024-08-17T18:11:18.655595+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:13.437895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":681,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665972+0000","last_change":"2024-08-17T18:09:29.866944+0000","last_active":"2024-08-17T18:11:09.665972+0000","last_peered":"2024-08-17T18:11:09.665972+0000","last_clean":"2024-08-17T18:11:09.665972+0000","last_became_active":"2024-08-17T18:09:29.866694+0000","last_became_peered":"2024-08-17T18:09:29.866694+0000","last_unstale":"2024-08-17T18:11:09.665972+0000","last_undegraded":"2024-08-17T18:11:09.665972+0000","last_fullsized":"2024-08-17T18:11:09.665972+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:00:41.342392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1344,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665943+0000","last_change":"2024-08-17T18:09:28.884262+0000","last_active":"2024-08-17T18:11:09.665943+0000","last_peered":"2024-08-17T18:11:09.665943+0000","last_clean":"2024-08-17T18:11:09.665943+0000","last_became_active":"2024-08-17T18:09:28.883734+0000","last_became_peered":"2024-08-17T18:09:28.883734+0000","last_unstale":"2024-08-17T18:11:09.665943+0000","last_undegraded":"2024-08-17T18:11:09.665943+0000","last_fullsized":"2024-08-17T18:11:09.665943+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:10.735504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2063,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.783940+0000","last_change":"2024-08-17T18:08:57.571995+0000","last_active":"2024-08-17T18:11:33.783940+0000","last_peered":"2024-08-17T18:11:33.783940+0000","last_clean":"2024-08-17T18:11:33.783940+0000","last_became_active":"2024-08-17T18:08:57.567342+0000","last_became_peered":"2024-08-17T18:08:57.567342+0000","last_unstale":"2024-08-17T18:11:33.783940+0000","last_undegraded":"2024-08-17T18:11:33.783940+0000","last_fullsized":"2024-08-17T18:11:33.783940+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:23.769111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":508,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665671+0000","last_change":"2024-08-17T18:02:53.912837+0000","last_active":"2024-08-17T18:11:09.665671+0000","last_peered":"2024-08-17T18:11:09.665671+0000","last_clean":"2024-08-17T18:11:09.665671+0000","last_became_active":"2024-08-17T18:02:53.883152+0000","last_became_peered":"2024-08-17T18:02:53.883152+0000","last_unstale":"2024-08-17T18:11:09.665671+0000","last_undegraded":"2024-08-17T18:11:09.665671+0000","last_fullsized":"2024-08-17T18:11:09.665671+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:46.261612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":507,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665603+0000","last_change":"2024-08-17T18:09:29.845126+0000","last_active":"2024-08-17T18:11:09.665603+0000","last_peered":"2024-08-17T18:11:09.665603+0000","last_clean":"2024-08-17T18:11:09.665603+0000","last_became_active":"2024-08-17T18:09:29.844836+0000","last_became_peered":"2024-08-17T18:09:29.844836+0000","last_unstale":"2024-08-17T18:11:09.665603+0000","last_undegraded":"2024-08-17T18:11:09.665603+0000","last_fullsized":"2024-08-17T18:11:09.665603+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:44.821894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1300,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.774780+0000","last_change":"2024-08-17T18:02:53.890198+0000","last_active":"2024-08-17T18:11:33.774780+0000","last_peered":"2024-08-17T18:11:33.774780+0000","last_clean":"2024-08-17T18:11:33.774780+0000","last_became_active":"2024-08-17T18:02:53.887007+0000","last_became_peered":"2024-08-17T18:02:53.887007+0000","last_unstale":"2024-08-17T18:11:33.774780+0000","last_undegraded":"2024-08-17T18:11:33.774780+0000","last_fullsized":"2024-08-17T18:11:33.774780+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:25.555909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":598,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.666148+0000","last_change":"2024-08-17T18:09:28.884187+0000","last_active":"2024-08-17T18:11:09.666148+0000","last_peered":"2024-08-17T18:11:09.666148+0000","last_clean":"2024-08-17T18:11:09.666148+0000","last_became_active":"2024-08-17T18:09:28.883939+0000","last_became_peered":"2024-08-17T18:09:28.883939+0000","last_unstale":"2024-08-17T18:11:09.666148+0000","last_undegraded":"2024-08-17T18:11:09.666148+0000","last_fullsized":"2024-08-17T18:11:09.666148+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:37:41.951099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1898,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665964+0000","last_change":"2024-08-17T18:09:29.893220+0000","last_active":"2024-08-17T18:11:09.665964+0000","last_peered":"2024-08-17T18:11:09.665964+0000","last_clean":"2024-08-17T18:11:09.665964+0000","last_became_active":"2024-08-17T18:09:29.865699+0000","last_became_peered":"2024-08-17T18:09:29.865699+0000","last_unstale":"2024-08-17T18:11:09.665964+0000","last_undegraded":"2024-08-17T18:11:09.665964+0000","last_fullsized":"2024-08-17T18:11:09.665964+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:40.205658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2018,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.659814+0000","last_change":"2024-08-17T18:09:28.934404+0000","last_active":"2024-08-17T18:11:18.659814+0000","last_peered":"2024-08-17T18:11:18.659814+0000","last_clean":"2024-08-17T18:11:18.659814+0000","last_became_active":"2024-08-17T18:09:28.934001+0000","last_became_peered":"2024-08-17T18:09:28.934001+0000","last_unstale":"2024-08-17T18:11:18.659814+0000","last_undegraded":"2024-08-17T18:11:18.659814+0000","last_fullsized":"2024-08-17T18:11:18.659814+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:04.676812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":2049,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.663580+0000","last_change":"2024-08-17T18:09:29.884088+0000","last_active":"2024-08-17T18:11:18.663580+0000","last_peered":"2024-08-17T18:11:18.663580+0000","last_clean":"2024-08-17T18:11:18.663580+0000","last_became_active":"2024-08-17T18:09:29.883027+0000","last_became_peered":"2024-08-17T18:09:29.883027+0000","last_unstale":"2024-08-17T18:11:18.663580+0000","last_undegraded":"2024-08-17T18:11:18.663580+0000","last_fullsized":"2024-08-17T18:11:18.663580+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:22:07.777188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":2019,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.663419+0000","last_change":"2024-08-17T18:09:28.935401+0000","last_active":"2024-08-17T18:11:18.663419+0000","last_peered":"2024-08-17T18:11:18.663419+0000","last_clean":"2024-08-17T18:11:18.663419+0000","last_became_active":"2024-08-17T18:09:28.933628+0000","last_became_peered":"2024-08-17T18:09:28.933628+0000","last_unstale":"2024-08-17T18:11:18.663419+0000","last_undegraded":"2024-08-17T18:11:18.663419+0000","last_fullsized":"2024-08-17T18:11:18.663419+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:21:12.714486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2133,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.785438+0000","last_change":"2024-08-17T18:08:57.574718+0000","last_active":"2024-08-17T18:11:33.785438+0000","last_peered":"2024-08-17T18:11:33.785438+0000","last_clean":"2024-08-17T18:11:33.785438+0000","last_became_active":"2024-08-17T18:08:57.573515+0000","last_became_peered":"2024-08-17T18:08:57.573515+0000","last_unstale":"2024-08-17T18:11:33.785438+0000","last_undegraded":"2024-08-17T18:11:33.785438+0000","last_fullsized":"2024-08-17T18:11:33.785438+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:04.873717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1357,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.666148+0000","last_change":"2024-08-17T18:08:57.568821+0000","last_active":"2024-08-17T18:11:09.666148+0000","last_peered":"2024-08-17T18:11:09.666148+0000","last_clean":"2024-08-17T18:11:09.666148+0000","last_became_active":"2024-08-17T18:08:57.568526+0000","last_became_peered":"2024-08-17T18:08:57.568526+0000","last_unstale":"2024-08-17T18:11:09.666148+0000","last_undegraded":"2024-08-17T18:11:09.666148+0000","last_fullsized":"2024-08-17T18:11:09.666148+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:26.449197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1854,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.781170+0000","last_change":"2024-08-17T18:08:57.571174+0000","last_active":"2024-08-17T18:11:33.781170+0000","last_peered":"2024-08-17T18:11:33.781170+0000","last_clean":"2024-08-17T18:11:33.781170+0000","last_became_active":"2024-08-17T18:08:57.567571+0000","last_became_peered":"2024-08-17T18:08:57.567571+0000","last_unstale":"2024-08-17T18:11:33.781170+0000","last_undegraded":"2024-08-17T18:11:33.781170+0000","last_fullsized":"2024-08-17T18:11:33.781170+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:29.976150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1823,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.784216+0000","last_change":"2024-08-17T18:08:57.572368+0000","last_active":"2024-08-17T18:11:33.784216+0000","last_peered":"2024-08-17T18:11:33.784216+0000","last_clean":"2024-08-17T18:11:33.784216+0000","last_became_active":"2024-08-17T18:08:57.568382+0000","last_became_peered":"2024-08-17T18:08:57.568382+0000","last_unstale":"2024-08-17T18:11:33.784216+0000","last_undegraded":"2024-08-17T18:11:33.784216+0000","last_fullsized":"2024-08-17T18:11:33.784216+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:11:19.968079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1871,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.781253+0000","last_change":"2024-08-17T18:08:57.572991+0000","last_active":"2024-08-17T18:11:33.781253+0000","last_peered":"2024-08-17T18:11:33.781253+0000","last_clean":"2024-08-17T18:11:33.781253+0000","last_became_active":"2024-08-17T18:08:57.571145+0000","last_became_peered":"2024-08-17T18:08:57.571145+0000","last_unstale":"2024-08-17T18:11:33.781253+0000","last_undegraded":"2024-08-17T18:11:33.781253+0000","last_fullsized":"2024-08-17T18:11:33.781253+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:39.963337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":825,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.662066+0000","last_change":"2024-08-17T18:09:28.860288+0000","last_active":"2024-08-17T18:11:18.662066+0000","last_peered":"2024-08-17T18:11:18.662066+0000","last_clean":"2024-08-17T18:11:18.662066+0000","last_became_active":"2024-08-17T18:09:28.860004+0000","last_became_peered":"2024-08-17T18:09:28.860004+0000","last_unstale":"2024-08-17T18:11:18.662066+0000","last_undegraded":"2024-08-17T18:11:18.662066+0000","last_fullsized":"2024-08-17T18:11:18.662066+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:57:33.054055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":617,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.661667+0000","last_change":"2024-08-17T18:09:28.860277+0000","last_active":"2024-08-17T18:11:18.661667+0000","last_peered":"2024-08-17T18:11:18.661667+0000","last_clean":"2024-08-17T18:11:18.661667+0000","last_became_active":"2024-08-17T18:09:28.859999+0000","last_became_peered":"2024-08-17T18:09:28.859999+0000","last_unstale":"2024-08-17T18:11:18.661667+0000","last_undegraded":"2024-08-17T18:11:18.661667+0000","last_fullsized":"2024-08-17T18:11:18.661667+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:27.946526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1525,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.775514+0000","last_change":"2024-08-17T18:09:29.863669+0000","last_active":"2024-08-17T18:11:33.775514+0000","last_peered":"2024-08-17T18:11:33.775514+0000","last_clean":"2024-08-17T18:11:33.775514+0000","last_became_active":"2024-08-17T18:09:29.862781+0000","last_became_peered":"2024-08-17T18:09:29.862781+0000","last_unstale":"2024-08-17T18:11:33.775514+0000","last_undegraded":"2024-08-17T18:11:33.775514+0000","last_fullsized":"2024-08-17T18:11:33.775514+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:14.073233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":106,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.784298+0000","last_change":"2024-08-17T18:09:29.843574+0000","last_active":"2024-08-17T18:11:33.784298+0000","last_peered":"2024-08-17T18:11:33.784298+0000","last_clean":"2024-08-17T18:11:33.784298+0000","last_became_active":"2024-08-17T18:09:29.843097+0000","last_became_peered":"2024-08-17T18:09:29.843097+0000","last_unstale":"2024-08-17T18:11:33.784298+0000","last_undegraded":"2024-08-17T18:11:33.784298+0000","last_fullsized":"2024-08-17T18:11:33.784298+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:46.740888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1870,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.774475+0000","last_change":"2024-08-17T18:09:29.847264+0000","last_active":"2024-08-17T18:11:33.774475+0000","last_peered":"2024-08-17T18:11:33.774475+0000","last_clean":"2024-08-17T18:11:33.774475+0000","last_became_active":"2024-08-17T18:09:29.846226+0000","last_became_peered":"2024-08-17T18:09:29.846226+0000","last_unstale":"2024-08-17T18:11:33.774475+0000","last_undegraded":"2024-08-17T18:11:33.774475+0000","last_fullsized":"2024-08-17T18:11:33.774475+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:44.135977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2070,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.773704+0000","last_change":"2024-08-17T18:09:29.845209+0000","last_active":"2024-08-17T18:11:33.773704+0000","last_peered":"2024-08-17T18:11:33.773704+0000","last_clean":"2024-08-17T18:11:33.773704+0000","last_became_active":"2024-08-17T18:09:29.844910+0000","last_became_peered":"2024-08-17T18:09:29.844910+0000","last_unstale":"2024-08-17T18:11:33.773704+0000","last_undegraded":"2024-08-17T18:11:33.773704+0000","last_fullsized":"2024-08-17T18:11:33.773704+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:14.586354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1941,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.656415+0000","last_change":"2024-08-17T18:09:28.859050+0000","last_active":"2024-08-17T18:11:18.656415+0000","last_peered":"2024-08-17T18:11:18.656415+0000","last_clean":"2024-08-17T18:11:18.656415+0000","last_became_active":"2024-08-17T18:09:28.858214+0000","last_became_peered":"2024-08-17T18:09:28.858214+0000","last_unstale":"2024-08-17T18:11:18.656415+0000","last_undegraded":"2024-08-17T18:11:18.656415+0000","last_fullsized":"2024-08-17T18:11:18.656415+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:54.146592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2194,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.774459+0000","last_change":"2024-08-17T18:08:57.572610+0000","last_active":"2024-08-17T18:11:33.774459+0000","last_peered":"2024-08-17T18:11:33.774459+0000","last_clean":"2024-08-17T18:11:33.774459+0000","last_became_active":"2024-08-17T18:08:57.572308+0000","last_became_peered":"2024-08-17T18:08:57.572308+0000","last_unstale":"2024-08-17T18:11:33.774459+0000","last_undegraded":"2024-08-17T18:11:33.774459+0000","last_fullsized":"2024-08-17T18:11:33.774459+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:11.642831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2179,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.774863+0000","last_change":"2024-08-17T18:09:28.842838+0000","last_active":"2024-08-17T18:11:33.774863+0000","last_peered":"2024-08-17T18:11:33.774863+0000","last_clean":"2024-08-17T18:11:33.774863+0000","last_became_active":"2024-08-17T18:09:28.842617+0000","last_became_peered":"2024-08-17T18:09:28.842617+0000","last_unstale":"2024-08-17T18:11:33.774863+0000","last_undegraded":"2024-08-17T18:11:33.774863+0000","last_fullsized":"2024-08-17T18:11:33.774863+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:00.016002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1378,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665379+0000","last_change":"2024-08-17T18:09:29.845011+0000","last_active":"2024-08-17T18:11:09.665379+0000","last_peered":"2024-08-17T18:11:09.665379+0000","last_clean":"2024-08-17T18:11:09.665379+0000","last_became_active":"2024-08-17T18:09:29.844358+0000","last_became_peered":"2024-08-17T18:09:29.844358+0000","last_unstale":"2024-08-17T18:11:09.665379+0000","last_undegraded":"2024-08-17T18:11:09.665379+0000","last_fullsized":"2024-08-17T18:11:09.665379+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:41:27.792901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1226,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.666054+0000","last_change":"2024-08-17T18:09:28.878263+0000","last_active":"2024-08-17T18:11:09.666054+0000","last_peered":"2024-08-17T18:11:09.666054+0000","last_clean":"2024-08-17T18:11:09.666054+0000","last_became_active":"2024-08-17T18:09:28.878025+0000","last_became_peered":"2024-08-17T18:09:28.878025+0000","last_unstale":"2024-08-17T18:11:09.666054+0000","last_undegraded":"2024-08-17T18:11:09.666054+0000","last_fullsized":"2024-08-17T18:11:09.666054+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:37.211100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1969,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.659459+0000","last_change":"2024-08-17T18:09:29.840134+0000","last_active":"2024-08-17T18:11:18.659459+0000","last_peered":"2024-08-17T18:11:18.659459+0000","last_clean":"2024-08-17T18:11:18.659459+0000","last_became_active":"2024-08-17T18:09:29.839848+0000","last_became_peered":"2024-08-17T18:09:29.839848+0000","last_unstale":"2024-08-17T18:11:18.659459+0000","last_undegraded":"2024-08-17T18:11:18.659459+0000","last_fullsized":"2024-08-17T18:11:18.659459+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:08:08.717851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":133,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665754+0000","last_change":"2024-08-17T18:09:28.847633+0000","last_active":"2024-08-17T18:11:09.665754+0000","last_peered":"2024-08-17T18:11:09.665754+0000","last_clean":"2024-08-17T18:11:09.665754+0000","last_became_active":"2024-08-17T18:09:28.847341+0000","last_became_peered":"2024-08-17T18:09:28.847341+0000","last_unstale":"2024-08-17T18:11:09.665754+0000","last_undegraded":"2024-08-17T18:11:09.665754+0000","last_fullsized":"2024-08-17T18:11:09.665754+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:51:59.625154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1718,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665111+0000","last_change":"2024-08-17T18:09:00.609431+0000","last_active":"2024-08-17T18:11:09.665111+0000","last_peered":"2024-08-17T18:11:09.665111+0000","last_clean":"2024-08-17T18:11:09.665111+0000","last_became_active":"2024-08-17T18:09:00.609167+0000","last_became_peered":"2024-08-17T18:09:00.609167+0000","last_unstale":"2024-08-17T18:11:09.665111+0000","last_undegraded":"2024-08-17T18:11:09.665111+0000","last_fullsized":"2024-08-17T18:11:09.665111+0000","mapping_epoch":1338,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:39.579970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1848,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.663709+0000","last_change":"2024-08-17T18:09:29.837502+0000","last_active":"2024-08-17T18:11:18.663709+0000","last_peered":"2024-08-17T18:11:18.663709+0000","last_clean":"2024-08-17T18:11:18.663709+0000","last_became_active":"2024-08-17T18:09:29.837243+0000","last_became_peered":"2024-08-17T18:09:29.837243+0000","last_unstale":"2024-08-17T18:11:18.663709+0000","last_undegraded":"2024-08-17T18:11:18.663709+0000","last_fullsized":"2024-08-17T18:11:18.663709+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:47.377505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2046,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.659850+0000","last_change":"2024-08-17T18:09:29.859415+0000","last_active":"2024-08-17T18:11:18.659850+0000","last_peered":"2024-08-17T18:11:18.659850+0000","last_clean":"2024-08-17T18:11:18.659850+0000","last_became_active":"2024-08-17T18:09:29.859138+0000","last_became_peered":"2024-08-17T18:09:29.859138+0000","last_unstale":"2024-08-17T18:11:18.659850+0000","last_undegraded":"2024-08-17T18:11:18.659850+0000","last_fullsized":"2024-08-17T18:11:18.659850+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:31.178915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":497,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.663480+0000","last_change":"2024-08-17T18:09:29.883834+0000","last_active":"2024-08-17T18:11:18.663480+0000","last_peered":"2024-08-17T18:11:18.663480+0000","last_clean":"2024-08-17T18:11:18.663480+0000","last_became_active":"2024-08-17T18:09:29.882638+0000","last_became_peered":"2024-08-17T18:09:29.882638+0000","last_unstale":"2024-08-17T18:11:18.663480+0000","last_undegraded":"2024-08-17T18:11:18.663480+0000","last_fullsized":"2024-08-17T18:11:18.663480+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:07:10.188684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":618,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665320+0000","last_change":"2024-08-17T18:09:29.857250+0000","last_active":"2024-08-17T18:11:09.665320+0000","last_peered":"2024-08-17T18:11:09.665320+0000","last_clean":"2024-08-17T18:11:09.665320+0000","last_became_active":"2024-08-17T18:09:29.857038+0000","last_became_peered":"2024-08-17T18:09:29.857038+0000","last_unstale":"2024-08-17T18:11:09.665320+0000","last_undegraded":"2024-08-17T18:11:09.665320+0000","last_fullsized":"2024-08-17T18:11:09.665320+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:02:54.028575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":613,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.776082+0000","last_change":"2024-08-17T18:08:57.571270+0000","last_active":"2024-08-17T18:11:33.776082+0000","last_peered":"2024-08-17T18:11:33.776082+0000","last_clean":"2024-08-17T18:11:33.776082+0000","last_became_active":"2024-08-17T18:08:57.567670+0000","last_became_peered":"2024-08-17T18:08:57.567670+0000","last_unstale":"2024-08-17T18:11:33.776082+0000","last_undegraded":"2024-08-17T18:11:33.776082+0000","last_fullsized":"2024-08-17T18:11:33.776082+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:19:42.617321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1181,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.659586+0000","last_change":"2024-08-17T18:09:29.839260+0000","last_active":"2024-08-17T18:11:18.659586+0000","last_peered":"2024-08-17T18:11:18.659586+0000","last_clean":"2024-08-17T18:11:18.659586+0000","last_became_active":"2024-08-17T18:09:29.838953+0000","last_became_peered":"2024-08-17T18:09:29.838953+0000","last_unstale":"2024-08-17T18:11:18.659586+0000","last_undegraded":"2024-08-17T18:11:18.659586+0000","last_fullsized":"2024-08-17T18:11:18.659586+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:44.014083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":770,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.661815+0000","last_change":"2024-08-17T18:09:29.851347+0000","last_active":"2024-08-17T18:11:18.661815+0000","last_peered":"2024-08-17T18:11:18.661815+0000","last_clean":"2024-08-17T18:11:18.661815+0000","last_became_active":"2024-08-17T18:09:29.851046+0000","last_became_peered":"2024-08-17T18:09:29.851046+0000","last_unstale":"2024-08-17T18:11:18.661815+0000","last_undegraded":"2024-08-17T18:11:18.661815+0000","last_fullsized":"2024-08-17T18:11:18.661815+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:21.350703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2047,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.662956+0000","last_change":"2024-08-17T18:09:29.912446+0000","last_active":"2024-08-17T18:11:18.662956+0000","last_peered":"2024-08-17T18:11:18.662956+0000","last_clean":"2024-08-17T18:11:18.662956+0000","last_became_active":"2024-08-17T18:09:29.910703+0000","last_became_peered":"2024-08-17T18:09:29.910703+0000","last_unstale":"2024-08-17T18:11:18.662956+0000","last_undegraded":"2024-08-17T18:11:18.662956+0000","last_fullsized":"2024-08-17T18:11:18.662956+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.250608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2216,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.785330+0000","last_change":"2024-08-17T18:09:29.863361+0000","last_active":"2024-08-17T18:11:33.785330+0000","last_peered":"2024-08-17T18:11:33.785330+0000","last_clean":"2024-08-17T18:11:33.785330+0000","last_became_active":"2024-08-17T18:09:29.862503+0000","last_became_peered":"2024-08-17T18:09:29.862503+0000","last_unstale":"2024-08-17T18:11:33.785330+0000","last_undegraded":"2024-08-17T18:11:33.785330+0000","last_fullsized":"2024-08-17T18:11:33.785330+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:25.908232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":36,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.661417+0000","last_change":"2024-08-17T18:09:29.895991+0000","last_active":"2024-08-17T18:11:18.661417+0000","last_peered":"2024-08-17T18:11:18.661417+0000","last_clean":"2024-08-17T18:11:18.661417+0000","last_became_active":"2024-08-17T18:09:29.895274+0000","last_became_peered":"2024-08-17T18:09:29.895274+0000","last_unstale":"2024-08-17T18:11:18.661417+0000","last_undegraded":"2024-08-17T18:11:18.661417+0000","last_fullsized":"2024-08-17T18:11:18.661417+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:19:26.420016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":500,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665522+0000","last_change":"2024-08-17T18:09:29.844686+0000","last_active":"2024-08-17T18:11:09.665522+0000","last_peered":"2024-08-17T18:11:09.665522+0000","last_clean":"2024-08-17T18:11:09.665522+0000","last_became_active":"2024-08-17T18:09:29.844341+0000","last_became_peered":"2024-08-17T18:09:29.844341+0000","last_unstale":"2024-08-17T18:11:09.665522+0000","last_undegraded":"2024-08-17T18:11:09.665522+0000","last_fullsized":"2024-08-17T18:11:09.665522+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:38.872767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1337,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.666068+0000","last_change":"2024-08-17T18:09:28.884310+0000","last_active":"2024-08-17T18:11:09.666068+0000","last_peered":"2024-08-17T18:11:09.666068+0000","last_clean":"2024-08-17T18:11:09.666068+0000","last_became_active":"2024-08-17T18:09:28.883921+0000","last_became_peered":"2024-08-17T18:09:28.883921+0000","last_unstale":"2024-08-17T18:11:09.666068+0000","last_undegraded":"2024-08-17T18:11:09.666068+0000","last_fullsized":"2024-08-17T18:11:09.666068+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:05.272563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":779,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.656311+0000","last_change":"2024-08-17T18:09:29.837238+0000","last_active":"2024-08-17T18:11:18.656311+0000","last_peered":"2024-08-17T18:11:18.656311+0000","last_clean":"2024-08-17T18:11:18.656311+0000","last_became_active":"2024-08-17T18:09:29.836760+0000","last_became_peered":"2024-08-17T18:09:29.836760+0000","last_unstale":"2024-08-17T18:11:18.656311+0000","last_undegraded":"2024-08-17T18:11:18.656311+0000","last_fullsized":"2024-08-17T18:11:18.656311+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:38:52.862967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1352,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.666120+0000","last_change":"2024-08-17T18:09:29.871872+0000","last_active":"2024-08-17T18:11:09.666120+0000","last_peered":"2024-08-17T18:11:09.666120+0000","last_clean":"2024-08-17T18:11:09.666120+0000","last_became_active":"2024-08-17T18:09:29.845111+0000","last_became_peered":"2024-08-17T18:09:29.845111+0000","last_unstale":"2024-08-17T18:11:09.666120+0000","last_undegraded":"2024-08-17T18:11:09.666120+0000","last_fullsized":"2024-08-17T18:11:09.666120+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:48.183250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":249,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.664982+0000","last_change":"2024-08-17T18:09:29.844786+0000","last_active":"2024-08-17T18:11:09.664982+0000","last_peered":"2024-08-17T18:11:09.664982+0000","last_clean":"2024-08-17T18:11:09.664982+0000","last_became_active":"2024-08-17T18:09:29.844270+0000","last_became_peered":"2024-08-17T18:09:29.844270+0000","last_unstale":"2024-08-17T18:11:09.664982+0000","last_undegraded":"2024-08-17T18:11:09.664982+0000","last_fullsized":"2024-08-17T18:11:09.664982+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:51.866680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1368,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.666130+0000","last_change":"2024-08-17T18:09:29.886391+0000","last_active":"2024-08-17T18:11:09.666130+0000","last_peered":"2024-08-17T18:11:09.666130+0000","last_clean":"2024-08-17T18:11:09.666130+0000","last_became_active":"2024-08-17T18:09:29.844642+0000","last_became_peered":"2024-08-17T18:09:29.844642+0000","last_unstale":"2024-08-17T18:11:09.666130+0000","last_undegraded":"2024-08-17T18:11:09.666130+0000","last_fullsized":"2024-08-17T18:11:09.666130+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:07.988100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":195,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.780639+0000","last_change":"2024-08-17T18:09:28.941223+0000","last_active":"2024-08-17T18:11:33.780639+0000","last_peered":"2024-08-17T18:11:33.780639+0000","last_clean":"2024-08-17T18:11:33.780639+0000","last_became_active":"2024-08-17T18:09:28.940857+0000","last_became_peered":"2024-08-17T18:09:28.940857+0000","last_unstale":"2024-08-17T18:11:33.780639+0000","last_undegraded":"2024-08-17T18:11:33.780639+0000","last_fullsized":"2024-08-17T18:11:33.780639+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:48.809420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2044,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.663856+0000","last_change":"2024-08-17T18:09:29.838873+0000","last_active":"2024-08-17T18:11:18.663856+0000","last_peered":"2024-08-17T18:11:18.663856+0000","last_clean":"2024-08-17T18:11:18.663856+0000","last_became_active":"2024-08-17T18:09:29.838560+0000","last_became_peered":"2024-08-17T18:09:29.838560+0000","last_unstale":"2024-08-17T18:11:18.663856+0000","last_undegraded":"2024-08-17T18:11:18.663856+0000","last_fullsized":"2024-08-17T18:11:18.663856+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:00.103741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1328,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665379+0000","last_change":"2024-08-17T18:09:28.844170+0000","last_active":"2024-08-17T18:11:09.665379+0000","last_peered":"2024-08-17T18:11:09.665379+0000","last_clean":"2024-08-17T18:11:09.665379+0000","last_became_active":"2024-08-17T18:09:28.843862+0000","last_became_peered":"2024-08-17T18:09:28.843862+0000","last_unstale":"2024-08-17T18:11:09.665379+0000","last_undegraded":"2024-08-17T18:11:09.665379+0000","last_fullsized":"2024-08-17T18:11:09.665379+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:43:19.483326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2169,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.774166+0000","last_change":"2024-08-17T18:09:28.864923+0000","last_active":"2024-08-17T18:11:33.774166+0000","last_peered":"2024-08-17T18:11:33.774166+0000","last_clean":"2024-08-17T18:11:33.774166+0000","last_became_active":"2024-08-17T18:09:28.864707+0000","last_became_peered":"2024-08-17T18:09:28.864707+0000","last_unstale":"2024-08-17T18:11:33.774166+0000","last_undegraded":"2024-08-17T18:11:33.774166+0000","last_fullsized":"2024-08-17T18:11:33.774166+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:27.507329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2170,"reported_epoch":1358,"state":"active+clean","last_fresh":"2024-08-17T18:11:33.784388+0000","last_change":"2024-08-17T18:09:28.846741+0000","last_active":"2024-08-17T18:11:33.784388+0000","last_peered":"2024-08-17T18:11:33.784388+0000","last_clean":"2024-08-17T18:11:33.784388+0000","last_became_active":"2024-08-17T18:09:28.846478+0000","last_became_peered":"2024-08-17T18:09:28.846478+0000","last_unstale":"2024-08-17T18:11:33.784388+0000","last_undegraded":"2024-08-17T18:11:33.784388+0000","last_fullsized":"2024-08-17T18:11:33.784388+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:34.247517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":734,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665834+0000","last_change":"2024-08-17T18:03:16.091420+0000","last_active":"2024-08-17T18:11:09.665834+0000","last_peered":"2024-08-17T18:11:09.665834+0000","last_clean":"2024-08-17T18:11:09.665834+0000","last_became_active":"2024-08-17T18:03:16.091113+0000","last_became_peered":"2024-08-17T18:03:16.091113+0000","last_unstale":"2024-08-17T18:11:09.665834+0000","last_undegraded":"2024-08-17T18:11:09.665834+0000","last_fullsized":"2024-08-17T18:11:09.665834+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:47.770911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1342,"reported_epoch":1356,"state":"active+clean","last_fresh":"2024-08-17T18:11:09.665754+0000","last_change":"2024-08-17T18:02:53.893927+0000","last_active":"2024-08-17T18:11:09.665754+0000","last_peered":"2024-08-17T18:11:09.665754+0000","last_clean":"2024-08-17T18:11:09.665754+0000","last_became_active":"2024-08-17T18:02:53.883188+0000","last_became_peered":"2024-08-17T18:02:53.883188+0000","last_unstale":"2024-08-17T18:11:09.665754+0000","last_undegraded":"2024-08-17T18:11:09.665754+0000","last_fullsized":"2024-08-17T18:11:09.665754+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:19.296965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2044,"reported_epoch":1357,"state":"active+clean","last_fresh":"2024-08-17T18:11:18.661528+0000","last_change":"2024-08-17T18:09:29.840086+0000","last_active":"2024-08-17T18:11:18.661528+0000","last_peered":"2024-08-17T18:11:18.661528+0000","last_clean":"2024-08-17T18:11:18.661528+0000","last_became_active":"2024-08-17T18:09:29.839525+0000","last_became_peered":"2024-08-17T18:09:29.839525+0000","last_unstale":"2024-08-17T18:11:18.661528+0000","last_undegraded":"2024-08-17T18:11:18.661528+0000","last_fullsized":"2024-08-17T18:11:18.661528+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:38.316235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":48,"num_pg":16,"stat_sum":{"num_bytes":1660420120,"num_objects":25337,"num_object_clones":0,"num_object_copies":76011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13557,"num_objects_misplaced":9276,"num_objects_unfound":0,"num_objects_dirty":25337,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":238589,"num_write_kb":8445569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11994,"num_bytes_recovered":785776640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4238221312,"data_stored":4238213168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78106,"ondisk_log_size":78106,"up":48,"acting":38,"num_store_stats":5},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":10970,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":155648,"data_stored":27490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234608,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2113640,"kb_used_data":1211744,"kb_used_omap":1,"kb_used_meta":901886,"kb_avail":91631512,"statfs":{"total":95995035648,"available":93830668288,"internally_reserved":0,"allocated":1240825856,"data_stored":1236379291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":923531608},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568496243,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757881,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1343,"seq":5768141078554,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":378708,"kb_used_data":196364,"kb_used_omap":112,"kb_used_meta":182223,"kb_avail":93366444,"statfs":{"total":95995035648,"available":95607238656,"internally_reserved":0,"allocated":201076736,"data_stored":196645887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":115373,"internal_metadata":186596691},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405733,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2303964,"kb_used_data":1315924,"kb_used_omap":61,"kb_used_meta":987906,"kb_avail":91441188,"statfs":{"total":95995035648,"available":93635776512,"internally_reserved":0,"allocated":1347506176,"data_stored":1343072272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":1011616696},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":0,"seq":2173253452268,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474959,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1647912,"kb_used_data":772512,"kb_used_omap":119,"kb_used_meta":875272,"kb_avail":92097240,"statfs":{"total":95995035648,"available":94307573760,"internally_reserved":0,"allocated":791052288,"data_stored":786606281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":896279262},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,5,6],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019516,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":765591552,"data_stored":765591552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":741212160,"data_stored":741212160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1313345536,"data_stored":1313341464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":189202432,"data_stored":189202432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1228869632,"data_stored":1228865560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:11:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:36.584 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:36.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:36.685 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:36.708 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:36.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:36.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:36.810 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:36.858 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:36.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:36.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:36.960 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:37.005 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:37.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:37.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:37.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:36 smithi130 ceph-mon[110068]: pgmap v2469: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.5 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 843 op/s; 13211/74547 objects degraded (17.722%); 9276/74547 objects misplaced (12.443%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T18:11:37.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:36 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:11:37.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:36 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:11:37.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:36 smithi130 ceph-mon[110068]: osdmap e1360: 8 total, 8 up, 4 in 2024-08-17T18:11:37.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 13211/74547 objects degraded (17.722%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:37.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:36 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:11:37.059 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:36 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:11:37.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:36 smithi130 ceph-mon[105521]: pgmap v2469: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.5 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 843 op/s; 13211/74547 objects degraded (17.722%); 9276/74547 objects misplaced (12.443%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T18:11:37.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:36 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:11:37.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:36 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:11:37.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:36 smithi130 ceph-mon[105521]: osdmap e1360: 8 total, 8 up, 4 in 2024-08-17T18:11:37.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 13211/74547 objects degraded (17.722%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:37.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:36 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:11:37.060 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:36 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:11:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:36 smithi179 ceph-mon[93519]: pgmap v2469: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 76 active+clean; 1.5 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 843 op/s; 13211/74547 objects degraded (17.722%); 9276/74547 objects misplaced (12.443%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T18:11:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:36 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:11:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:36 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:11:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:36 smithi179 ceph-mon[93519]: osdmap e1360: 8 total, 8 up, 4 in 2024-08-17T18:11:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 13211/74547 objects degraded (17.722%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:36 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:11:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:36 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:11:37.110 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:37.132 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:37.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:37.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:37.195 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:11:37.233 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:37.255 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:37.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:37.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:37.356 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:37.400 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:37.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:37.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:37.502 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:37.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:37.551 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:37.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:37.652 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:37.675 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:37.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:37.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:37.776 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:37.798 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:37.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:37.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:37.900 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:37.937 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:37.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:37.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:38.038 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:38.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:38.103 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:38.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:38.204 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:38.249 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:38.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:38.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:38.351 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:38.373 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:38.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:38.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:38.474 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:38.514 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:38.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:38.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:38.616 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:38.658 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:38.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:38.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:38.760 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:38.800 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:38.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:38.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:38.902 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:38.941 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:38.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:38.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:39.042 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:39.065 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:39.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:39.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:39.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:38 smithi179 ceph-mon[93519]: pgmap v2471: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.5 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 598 op/s; 12800/73230 objects degraded (17.479%); 9276/73230 objects misplaced (12.667%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T18:11:39.166 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:39.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:38 smithi130 ceph-mon[105521]: pgmap v2471: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.5 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 598 op/s; 12800/73230 objects degraded (17.479%); 9276/73230 objects misplaced (12.667%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T18:11:39.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:38 smithi130 ceph-mon[110068]: pgmap v2471: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.5 GiB data, 6.0 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 598 op/s; 12800/73230 objects degraded (17.479%); 9276/73230 objects misplaced (12.667%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T18:11:39.207 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:39.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:39.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:39.309 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:39.332 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:39.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:39.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:39.433 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:39.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:39.455 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:39.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:39.550 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:11:39.556 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:39.578 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:39.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:39.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:39.680 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:39.702 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:39.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:39.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:39.804 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:39.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:39.825 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:39.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:39.926 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:39.975 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:39.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:39.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:40.077 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:40.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:40.101 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:40.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:40.116 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:11:40.163 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:11:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:11:39] "GET /metrics HTTP/1.1" 200 36806 "" "Prometheus/2.51.0" 2024-08-17T18:11:40.203 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:40.278 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:40.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:40.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:40.380 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:40.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:40.445 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:40.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:40.546 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:40.568 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:40.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:40.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:11:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:11:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:11:40.670 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:40.712 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:40.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:40.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:40.814 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:40.842 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:40.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:40.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:40.944 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:40.999 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:40.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:41.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:41.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:40 smithi179 ceph-mon[93519]: pgmap v2472: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.4 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 628 op/s; 10881/66576 objects degraded (16.344%); 8098/66576 objects misplaced (12.164%); 12 MiB/s, 187 objects/s recovering 2024-08-17T18:11:41.100 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:41.121 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:41.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:41.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:41.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:40 smithi130 ceph-mon[105521]: pgmap v2472: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.4 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 628 op/s; 10881/66576 objects degraded (16.344%); 8098/66576 objects misplaced (12.164%); 12 MiB/s, 187 objects/s recovering 2024-08-17T18:11:41.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:40 smithi130 ceph-mon[110068]: pgmap v2472: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.4 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 628 op/s; 10881/66576 objects degraded (16.344%); 8098/66576 objects misplaced (12.164%); 12 MiB/s, 187 objects/s recovering 2024-08-17T18:11:41.223 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:41.265 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:41.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:41.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:41.367 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:41.388 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:41.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:41.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:41.490 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:41.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:41.542 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:41.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:41.643 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:41.665 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:41.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:41.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:41.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:41.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:41.819 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:41.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:41.920 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:41.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:41.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:41.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:42.003 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:41 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 10881/66576 objects degraded (16.344%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:42.004 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:41 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 10881/66576 objects degraded (16.344%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:42.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:42.083 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:42.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:42.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:42.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:41 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 10881/66576 objects degraded (16.344%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:42.185 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:42.207 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:42.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:42.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:42.309 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:42.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:42.331 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:42.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:42.432 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:42.455 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:42.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:42.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:42.557 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:42.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:42.583 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:42.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:42.684 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:42.708 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:42.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:42.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:42.809 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:42.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:42.858 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:42.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:42.959 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:42.983 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:42.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:42.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:43.017 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:11:43.017 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:11:43.018 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:11:43.019 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T18:11:43.019 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T18:11:43.019 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T18:11:43.019 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:11:43.056 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:11:43.056 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:11:43.085 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:43.093 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:11:43.094 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:11:43.129 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:43.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:43.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:43.131 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:11:43.131 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:11:43.160 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:11:43.161 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:11:43.194 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:11:43.195 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:11:43.222 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:11:43.222 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:11:43.230 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:43.253 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:43.253 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:11:43.254 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:11:43.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:43.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:43.270 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:11:43.270 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:11:43.355 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:43.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:43.400 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:43.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:43.501 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:43.536 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:43.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:43.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:43 smithi179 ceph-mon[93519]: pgmap v2473: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.3 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 586 op/s; 10365/64341 objects degraded (16.109%); 8098/64341 objects misplaced (12.586%); 9.1 MiB/s, 145 objects/s recovering 2024-08-17T18:11:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:11:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:11:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:43 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2198229831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:11:43.638 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:43.661 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:43.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:43.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:43 smithi130 ceph-mon[105521]: pgmap v2473: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.3 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 586 op/s; 10365/64341 objects degraded (16.109%); 8098/64341 objects misplaced (12.586%); 9.1 MiB/s, 145 objects/s recovering 2024-08-17T18:11:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:11:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:11:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:43 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2198229831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:11:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:43 smithi130 ceph-mon[110068]: pgmap v2473: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.3 GiB data, 5.7 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 586 op/s; 10365/64341 objects degraded (16.109%); 8098/64341 objects misplaced (12.586%); 9.1 MiB/s, 145 objects/s recovering 2024-08-17T18:11:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:11:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:11:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:43 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2198229831' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:11:43.763 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:43.784 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:43.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:43.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:43.870 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T18:11:43.886 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:43.908 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:43.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:43.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:44.010 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:44.032 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:44.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:44.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:44.134 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:44.155 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:44.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:44.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:44.256 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:44.278 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:44.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:44.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:44.381 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:44.402 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:44.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:44.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:44.504 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:44.527 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:44.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:44.629 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:44.688 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:44.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:44.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:44.790 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:44.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:44.846 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:44.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:44.947 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:44.986 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:44.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:44.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:45.087 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:45.132 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:45.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:45.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:45.234 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:45.256 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:45.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:45.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:45.358 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:45.379 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:45.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:45.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:45.481 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:45.505 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:45.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:45.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:45.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:45 smithi179 ceph-mon[93519]: pgmap v2474: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.2 GiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 694 op/s; 9439/57957 objects degraded (16.286%); 7050/57957 objects misplaced (12.164%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T18:11:45.607 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:45.630 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:45.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:45.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:45.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:45 smithi130 ceph-mon[105521]: pgmap v2474: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.2 GiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 694 op/s; 9439/57957 objects degraded (16.286%); 7050/57957 objects misplaced (12.164%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T18:11:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:45 smithi130 ceph-mon[110068]: pgmap v2474: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.2 GiB data, 5.5 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 694 op/s; 9439/57957 objects degraded (16.286%); 7050/57957 objects misplaced (12.164%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T18:11:45.731 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:45.753 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:45.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:45.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:45.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:45.877 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:45.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:45.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:45.979 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:46.000 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:46.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:46.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:46.102 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:46.125 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:46.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:46.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:46.227 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:46.249 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:46.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:46.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:46.351 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:46.375 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:46.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:46.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:46.477 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:46.517 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:46.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:46.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:46.619 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:46.643 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:46.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:46.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:46.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:46 smithi130 ceph-mon[105521]: osdmap e1361: 8 total, 8 up, 4 in 2024-08-17T18:11:46.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 9439/57957 objects degraded (16.286%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:46 smithi130 ceph-mon[110068]: osdmap e1361: 8 total, 8 up, 4 in 2024-08-17T18:11:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 9439/57957 objects degraded (16.286%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:46.745 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:46.787 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:46.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:46.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:46 smithi179 ceph-mon[93519]: osdmap e1361: 8 total, 8 up, 4 in 2024-08-17T18:11:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 9439/57957 objects degraded (16.286%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:46.889 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:46.912 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:46.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:46.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:47.013 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:47.048 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:47.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:47.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:47.150 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:47.187 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:47.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:47.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:47.289 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:47.312 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:47.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:47.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:47.442 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:47.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:47.483 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:47.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:47.584 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:47.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:47.622 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:47.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:47.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:47 smithi130 ceph-mon[105521]: pgmap v2476: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.1 GiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 697 op/s; 8282/54273 objects degraded (15.260%); 6810/54273 objects misplaced (12.548%); 10 MiB/s, 160 objects/s recovering 2024-08-17T18:11:47.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:47 smithi130 ceph-mon[105521]: osdmap e1362: 8 total, 8 up, 4 in 2024-08-17T18:11:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:47 smithi130 ceph-mon[110068]: pgmap v2476: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.1 GiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 697 op/s; 8282/54273 objects degraded (15.260%); 6810/54273 objects misplaced (12.548%); 10 MiB/s, 160 objects/s recovering 2024-08-17T18:11:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:47 smithi130 ceph-mon[110068]: osdmap e1362: 8 total, 8 up, 4 in 2024-08-17T18:11:47.723 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:47.747 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:47.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:47.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:47 smithi179 ceph-mon[93519]: pgmap v2476: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 77 active+clean; 1.1 GiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 697 op/s; 8282/54273 objects degraded (15.260%); 6810/54273 objects misplaced (12.548%); 10 MiB/s, 160 objects/s recovering 2024-08-17T18:11:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:47 smithi179 ceph-mon[93519]: osdmap e1362: 8 total, 8 up, 4 in 2024-08-17T18:11:47.849 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:47.872 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:47.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:47.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:47.974 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:47.995 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:47.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:47.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:48.097 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:48.132 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:48.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:48.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:48.234 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:48.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:48.257 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:48.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:48.358 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:48.402 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:48.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:48.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:48.504 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:48.543 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:48.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:48.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:48.645 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:48.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:48.668 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:48.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:48.769 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:48.815 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:48.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:48.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:48.917 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:48.960 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:48.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:48.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:49.062 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:49.084 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:49.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:49.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:49.186 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:49.220 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:49.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:49.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:49.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:49.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:49.372 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:49.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:49.473 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:49.521 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:49.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:49.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:49.623 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:49.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:49 smithi130 ceph-mon[110068]: pgmap v2478: 89 pgs: 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 1.1 GiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 577 op/s; 8210/52707 objects degraded (15.577%); 6810/52707 objects misplaced (12.920%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T18:11:49.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:49 smithi130 ceph-mon[105521]: pgmap v2478: 89 pgs: 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 1.1 GiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 577 op/s; 8210/52707 objects degraded (15.577%); 6810/52707 objects misplaced (12.920%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T18:11:49.668 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:49.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:49.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:49.771 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:49.794 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:49.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:49.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:49.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:49 smithi179 ceph-mon[93519]: pgmap v2478: 89 pgs: 3 active+remapped+backfill_wait, 8 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 1.1 GiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 577 op/s; 8210/52707 objects degraded (15.577%); 6810/52707 objects misplaced (12.920%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T18:11:49.896 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:49.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:49.952 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:49.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:50.011 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:11:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:11:49] "GET /metrics HTTP/1.1" 200 36803 "" "Prometheus/2.51.0" 2024-08-17T18:11:50.053 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:50.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:50.102 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:50.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:50.203 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:50.237 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:50.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:50.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:50.339 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:50.373 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:50.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:50.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:50.475 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:50.516 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:50.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:50.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:11:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:11:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:11:50.617 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:50.663 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:50.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:50.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:50.765 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:50.775 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:11:50.777 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:11:50.790 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:50.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:50.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:50.891 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:50.914 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:50.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:50.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:51.016 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:51.055 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:51.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:51.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:51.157 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:51.196 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:51.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:51.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:51.298 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:51.345 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:51.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:51.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:51.385 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:11:51.447 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:51.482 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:51.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:51.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:51.583 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:51.624 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:51.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:51.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:51 smithi130 ceph-mon[110068]: pgmap v2479: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 7 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 975 MiB data, 5.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 728 op/s; 6601/46851 objects degraded (14.089%); 5081/46851 objects misplaced (10.845%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T18:11:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 6601/46851 objects degraded (14.089%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:51 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1446954835' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:11:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:51 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:11:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:51 smithi130 ceph-mon[105521]: pgmap v2479: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 7 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 975 MiB data, 5.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 728 op/s; 6601/46851 objects degraded (14.089%); 5081/46851 objects misplaced (10.845%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T18:11:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 6601/46851 objects degraded (14.089%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:51 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1446954835' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:11:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:51 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:11:51.726 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:51.752 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:51.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:51.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:51.761 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2479,"stamp":"2024-08-17T18:11:49.472287+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1022302181,"num_objects":15617,"num_object_clones":0,"num_object_copies":46851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6601,"num_objects_misplaced":5081,"num_objects_unfound":0,"num_objects_dirty":15617,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":248613,"num_write_kb":8448016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14280,"num_bytes_recovered":935802253,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76951,"ondisk_log_size":76951,"up":267,"acting":259,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":5305248,"kb_used_data":2337920,"kb_used_omap":294,"kb_used_meta":2966745,"kb_avail":369675360,"statfs":{"total":383980142592,"available":378547568640,"internally_reserved":0,"allocated":2394030080,"data_stored":2376143906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":301759,"internal_metadata":3037947201},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-382074880,"num_objects":-5830,"num_object_clones":0,"num_object_copies":-17490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3764,"num_objects_misplaced":-3017,"num_objects_unfound":0,"num_objects_dirty":-5830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5830,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1187,"num_bytes_recovered":77791232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.004502"},"pg_stats":[{"pgid":"48.f","version":"1362'15527","reported_seq":25860,"reported_epoch":1362,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:48.148931+0000","last_change":"2024-08-17T18:09:29.893285+0000","last_active":"2024-08-17T18:11:48.148931+0000","last_peered":"2024-08-17T18:11:48.148931+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:28.981495+0000","last_became_peered":"2024-08-17T18:09:28.981495+0000","last_unstale":"2024-08-17T18:11:48.148931+0000","last_undegraded":"2024-08-17T18:09:28.850340+0000","last_fullsized":"2024-08-17T18:09:28.832563+0000","mapping_epoch":1343,"log_start":"1349'11357","ondisk_log_start":"1349'11357","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4170,"log_dups_size":0,"ondisk_log_size":4170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:54.654486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67305472,"num_objects":1027,"num_object_clones":0,"num_object_copies":3081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":341,"num_objects_misplaced":1027,"num_objects_unfound":0,"num_objects_dirty":1027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15527,"num_write_kb":529728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5130,"num_bytes_recovered":336199680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":1027}],"blocked_by":[],"up_primary":3,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.e","version":"1362'15516","reported_seq":19806,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:47.058939+0000","last_change":"2024-08-17T18:11:34.827100+0000","last_active":"2024-08-17T18:11:47.058939+0000","last_peered":"2024-08-17T18:11:47.058939+0000","last_clean":"2024-08-17T18:11:47.058939+0000","last_became_active":"2024-08-17T18:11:34.776143+0000","last_became_peered":"2024-08-17T18:11:34.776143+0000","last_unstale":"2024-08-17T18:11:47.058939+0000","last_undegraded":"2024-08-17T18:11:47.058939+0000","last_fullsized":"2024-08-17T18:11:47.058939+0000","mapping_epoch":1358,"log_start":"1348'11229","ondisk_log_start":"1348'11229","created":1327,"last_epoch_clean":1359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4287,"log_dups_size":0,"ondisk_log_size":4287,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:56.638791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62652416,"num_objects":956,"num_object_clones":0,"num_object_copies":2868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15516,"num_write_kb":527104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2109,"num_bytes_recovered":138215424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"48.d","version":"1362'15415","reported_seq":15466,"reported_epoch":1362,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:48.144798+0000","last_change":"2024-08-17T18:09:29.930605+0000","last_active":"2024-08-17T18:11:48.144798+0000","last_peered":"2024-08-17T18:11:48.144798+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.918910+0000","last_became_peered":"2024-08-17T18:09:29.918910+0000","last_unstale":"2024-08-17T18:11:48.144798+0000","last_undegraded":"2024-08-17T18:09:29.832124+0000","last_fullsized":"2024-08-17T18:09:29.818954+0000","mapping_epoch":1344,"log_start":"1349'11224","ondisk_log_start":"1349'11224","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4191,"log_dups_size":0,"ondisk_log_size":4191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:00:55.077466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64749592,"num_objects":989,"num_object_clones":0,"num_object_copies":2967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":989,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":989,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":15415,"num_write_kb":524865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":989}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.c","version":"1362'15409","reported_seq":17394,"reported_epoch":1362,"state":"active+undersized+degraded+remapped+backfilling","last_fresh":"2024-08-17T18:11:48.150450+0000","last_change":"2024-08-17T18:11:45.184761+0000","last_active":"2024-08-17T18:11:48.150450+0000","last_peered":"2024-08-17T18:11:48.150450+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:28.914112+0000","last_became_peered":"2024-08-17T18:09:28.914112+0000","last_unstale":"2024-08-17T18:11:48.150450+0000","last_undegraded":"2024-08-17T18:09:28.840987+0000","last_fullsized":"2024-08-17T18:09:28.815757+0000","mapping_epoch":1343,"log_start":"1349'11246","ondisk_log_start":"1349'11246","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4163,"log_dups_size":0,"ondisk_log_size":4163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:55.890300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62324736,"num_objects":951,"num_object_clones":0,"num_object_copies":2853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":367,"num_objects_misplaced":367,"num_objects_unfound":0,"num_objects_dirty":951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15409,"num_write_kb":523520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":636,"num_bytes_recovered":41680896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"0","objects":1},{"shards":"1,7","objects":950}],"blocked_by":[],"up_primary":3,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.b","version":"1362'15682","reported_seq":15732,"reported_epoch":1362,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:48.130276+0000","last_change":"2024-08-17T18:09:29.921199+0000","last_active":"2024-08-17T18:11:48.130276+0000","last_peered":"2024-08-17T18:11:48.130276+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.912384+0000","last_became_peered":"2024-08-17T18:09:29.912384+0000","last_unstale":"2024-08-17T18:11:48.130276+0000","last_undegraded":"2024-08-17T18:09:29.829736+0000","last_fullsized":"2024-08-17T18:09:29.817434+0000","mapping_epoch":1344,"log_start":"1349'11564","ondisk_log_start":"1349'11564","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4118,"log_dups_size":0,"ondisk_log_size":4118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:22.733621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63700992,"num_objects":972,"num_object_clones":0,"num_object_copies":2916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":972,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15682,"num_write_kb":532928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":972}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.a","version":"1360'15340","reported_seq":15419,"reported_epoch":1360,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.137443+0000","last_change":"2024-08-17T18:07:32.359061+0000","last_active":"2024-08-17T18:11:45.137443+0000","last_peered":"2024-08-17T18:11:45.137443+0000","last_clean":"2024-08-17T18:11:45.137443+0000","last_became_active":"2024-08-17T18:07:32.357107+0000","last_became_peered":"2024-08-17T18:07:32.357107+0000","last_unstale":"2024-08-17T18:11:45.137443+0000","last_undegraded":"2024-08-17T18:11:45.137443+0000","last_fullsized":"2024-08-17T18:11:45.137443+0000","mapping_epoch":1327,"log_start":"1347'10364","ondisk_log_start":"1347'10364","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4976,"log_dups_size":0,"ondisk_log_size":4976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:30:02.061787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67895296,"num_objects":1036,"num_object_clones":0,"num_object_copies":3108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15340,"num_write_kb":524032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.9","version":"1360'15845","reported_seq":15903,"reported_epoch":1360,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:45.132158+0000","last_change":"2024-08-17T18:09:29.968655+0000","last_active":"2024-08-17T18:11:45.132158+0000","last_peered":"2024-08-17T18:11:45.132158+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.948623+0000","last_became_peered":"2024-08-17T18:09:29.948623+0000","last_unstale":"2024-08-17T18:11:45.132158+0000","last_undegraded":"2024-08-17T18:09:29.855385+0000","last_fullsized":"2024-08-17T18:09:29.821477+0000","mapping_epoch":1344,"log_start":"1347'10890","ondisk_log_start":"1347'10890","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4955,"log_dups_size":0,"ondisk_log_size":4955,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71499776,"num_objects":1091,"num_object_clones":0,"num_object_copies":3273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1091,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15845,"num_write_kb":541952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1091}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.8","version":"1362'15623","reported_seq":15737,"reported_epoch":1362,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:49.108343+0000","last_change":"2024-08-17T18:09:29.904036+0000","last_active":"2024-08-17T18:11:49.108343+0000","last_peered":"2024-08-17T18:11:49.108343+0000","last_clean":"2024-08-17T18:09:27.840248+0000","last_became_active":"2024-08-17T18:09:28.942085+0000","last_became_peered":"2024-08-17T18:09:28.942085+0000","last_unstale":"2024-08-17T18:11:49.108343+0000","last_undegraded":"2024-08-17T18:11:49.108343+0000","last_fullsized":"2024-08-17T18:11:49.108343+0000","mapping_epoch":1343,"log_start":"1347'10761","ondisk_log_start":"1347'10761","created":1327,"last_epoch_clean":1337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4862,"log_dups_size":0,"ondisk_log_size":4862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58654720,"num_objects":895,"num_object_clones":0,"num_object_copies":2685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":895,"num_objects_unfound":0,"num_objects_dirty":895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15623,"num_write_kb":528576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,1],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":895}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.7","version":"1362'15488","reported_seq":15561,"reported_epoch":1362,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:48.130820+0000","last_change":"2024-08-17T18:09:28.989307+0000","last_active":"2024-08-17T18:11:48.130820+0000","last_peered":"2024-08-17T18:11:48.130820+0000","last_clean":"2024-08-17T18:09:27.824322+0000","last_became_active":"2024-08-17T18:09:28.981075+0000","last_became_peered":"2024-08-17T18:09:28.981075+0000","last_unstale":"2024-08-17T18:11:48.130820+0000","last_undegraded":"2024-08-17T18:11:48.130820+0000","last_fullsized":"2024-08-17T18:11:48.130820+0000","mapping_epoch":1343,"log_start":"1349'11348","ondisk_log_start":"1349'11348","created":1327,"last_epoch_clean":1337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4140,"log_dups_size":0,"ondisk_log_size":4140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:12:08.456671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65273856,"num_objects":996,"num_object_clones":0,"num_object_copies":2988,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":996,"num_objects_unfound":0,"num_objects_dirty":996,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15488,"num_write_kb":527488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,1],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"1,2,7","objects":996}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.6","version":"1362'15492","reported_seq":16796,"reported_epoch":1362,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:48.119915+0000","last_change":"2024-08-17T18:09:59.877439+0000","last_active":"2024-08-17T18:11:48.119915+0000","last_peered":"2024-08-17T18:11:48.119915+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:28.962573+0000","last_became_peered":"2024-08-17T18:09:28.962573+0000","last_unstale":"2024-08-17T18:11:48.119915+0000","last_undegraded":"2024-08-17T18:09:28.855040+0000","last_fullsized":"2024-08-17T18:09:28.816313+0000","mapping_epoch":1343,"log_start":"1333'5400","ondisk_log_start":"1333'5400","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":10092,"log_dups_size":0,"ondisk_log_size":10092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60424192,"num_objects":922,"num_object_clones":0,"num_object_copies":2766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":878,"num_objects_misplaced":878,"num_objects_unfound":0,"num_objects_dirty":922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15492,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":403,"num_bytes_recovered":26411008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":922}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"48.5","version":"1360'15339","reported_seq":15397,"reported_epoch":1360,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:45.125688+0000","last_change":"2024-08-17T18:09:29.983372+0000","last_active":"2024-08-17T18:11:45.125688+0000","last_peered":"2024-08-17T18:11:45.125688+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.956933+0000","last_became_peered":"2024-08-17T18:09:29.956933+0000","last_unstale":"2024-08-17T18:11:45.125688+0000","last_undegraded":"2024-08-17T18:09:29.847436+0000","last_fullsized":"2024-08-17T18:09:29.819789+0000","mapping_epoch":1344,"log_start":"1347'10358","ondisk_log_start":"1347'10358","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4981,"log_dups_size":0,"ondisk_log_size":4981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69009408,"num_objects":1053,"num_object_clones":0,"num_object_copies":3159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1053,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15339,"num_write_kb":524544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1053}],"blocked_by":[],"up_primary":3,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.4","version":"1362'15411","reported_seq":18138,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:47.084640+0000","last_change":"2024-08-17T18:11:46.398030+0000","last_active":"2024-08-17T18:11:47.084640+0000","last_peered":"2024-08-17T18:11:47.084640+0000","last_clean":"2024-08-17T18:11:47.084640+0000","last_became_active":"2024-08-17T18:11:46.364183+0000","last_became_peered":"2024-08-17T18:11:46.364183+0000","last_unstale":"2024-08-17T18:11:47.084640+0000","last_undegraded":"2024-08-17T18:11:47.084640+0000","last_fullsized":"2024-08-17T18:11:47.084640+0000","mapping_epoch":1361,"log_start":"1348'11144","ondisk_log_start":"1348'11144","created":1327,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4267,"log_dups_size":0,"ondisk_log_size":4267,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:25:19.771405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65732608,"num_objects":1003,"num_object_clones":0,"num_object_copies":3009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15411,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1316,"num_bytes_recovered":86245376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"48.3","version":"1362'15528","reported_seq":15611,"reported_epoch":1362,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:49.112538+0000","last_change":"2024-08-17T18:09:30.006916+0000","last_active":"2024-08-17T18:11:49.112538+0000","last_peered":"2024-08-17T18:11:49.112538+0000","last_clean":"2024-08-17T18:07:31.337586+0000","last_became_active":"2024-08-17T18:09:29.994996+0000","last_became_peered":"2024-08-17T18:09:29.994996+0000","last_unstale":"2024-08-17T18:11:49.112538+0000","last_undegraded":"2024-08-17T18:09:29.861017+0000","last_fullsized":"2024-08-17T18:09:29.830876+0000","mapping_epoch":1344,"log_start":"1347'10623","ondisk_log_start":"1347'10623","created":1327,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4905,"log_dups_size":0,"ondisk_log_size":4905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59637760,"num_objects":910,"num_object_clones":0,"num_object_copies":2730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":910,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15528,"num_write_kb":526016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":910}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.2","version":"1362'15489","reported_seq":29502,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:47.072197+0000","last_change":"2024-08-17T18:11:09.658283+0000","last_active":"2024-08-17T18:11:47.072197+0000","last_peered":"2024-08-17T18:11:47.072197+0000","last_clean":"2024-08-17T18:11:47.072197+0000","last_became_active":"2024-08-17T18:11:09.615907+0000","last_became_peered":"2024-08-17T18:11:09.615907+0000","last_unstale":"2024-08-17T18:11:47.072197+0000","last_undegraded":"2024-08-17T18:11:47.072197+0000","last_fullsized":"2024-08-17T18:11:47.072197+0000","mapping_epoch":1355,"log_start":"1348'11173","ondisk_log_start":"1348'11173","created":1327,"last_epoch_clean":1356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4316,"log_dups_size":0,"ondisk_log_size":4316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:04.549996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61276160,"num_objects":935,"num_object_clones":0,"num_object_copies":2805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15489,"num_write_kb":525568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4640,"num_bytes_recovered":303955968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"48.1","version":"1362'15751","reported_seq":15794,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:48.142192+0000","last_change":"2024-08-17T18:07:32.360312+0000","last_active":"2024-08-17T18:11:48.142192+0000","last_peered":"2024-08-17T18:11:48.142192+0000","last_clean":"2024-08-17T18:11:48.142192+0000","last_became_active":"2024-08-17T18:07:32.360092+0000","last_became_peered":"2024-08-17T18:07:32.360092+0000","last_unstale":"2024-08-17T18:11:48.142192+0000","last_undegraded":"2024-08-17T18:11:48.142192+0000","last_fullsized":"2024-08-17T18:11:48.142192+0000","mapping_epoch":1327,"log_start":"1349'11622","ondisk_log_start":"1349'11622","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4129,"log_dups_size":0,"ondisk_log_size":4129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:41.580676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61407232,"num_objects":937,"num_object_clones":0,"num_object_copies":2811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15751,"num_write_kb":534016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.0","version":"1362'15480","reported_seq":15561,"reported_epoch":1362,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:11:48.144828+0000","last_change":"2024-08-17T18:09:28.963421+0000","last_active":"2024-08-17T18:11:48.144828+0000","last_peered":"2024-08-17T18:11:48.144828+0000","last_clean":"2024-08-17T18:09:27.823675+0000","last_became_active":"2024-08-17T18:09:28.941589+0000","last_became_peered":"2024-08-17T18:09:28.941589+0000","last_unstale":"2024-08-17T18:11:48.144828+0000","last_undegraded":"2024-08-17T18:11:48.144828+0000","last_fullsized":"2024-08-17T18:11:48.144828+0000","mapping_epoch":1343,"log_start":"1349'11288","ondisk_log_start":"1349'11288","created":1327,"last_epoch_clean":1337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4192,"log_dups_size":0,"ondisk_log_size":4192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:36.800547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60162048,"num_objects":918,"num_object_clones":0,"num_object_copies":2754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":918,"num_objects_unfound":0,"num_objects_dirty":918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15480,"num_write_kb":524736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,1],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"1,2,7","objects":918}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":2213,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.363847+0000","last_change":"2024-08-17T18:09:28.870033+0000","last_active":"2024-08-17T18:11:46.363847+0000","last_peered":"2024-08-17T18:11:46.363847+0000","last_clean":"2024-08-17T18:11:46.363847+0000","last_became_active":"2024-08-17T18:09:28.869346+0000","last_became_peered":"2024-08-17T18:09:28.869346+0000","last_unstale":"2024-08-17T18:11:46.363847+0000","last_undegraded":"2024-08-17T18:11:46.363847+0000","last_fullsized":"2024-08-17T18:11:46.363847+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:23.168631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":507,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.356570+0000","last_change":"2024-08-17T18:09:29.841584+0000","last_active":"2024-08-17T18:11:46.356570+0000","last_peered":"2024-08-17T18:11:46.356570+0000","last_clean":"2024-08-17T18:11:46.356570+0000","last_became_active":"2024-08-17T18:09:29.840950+0000","last_became_peered":"2024-08-17T18:09:29.840950+0000","last_unstale":"2024-08-17T18:11:46.356570+0000","last_undegraded":"2024-08-17T18:11:46.356570+0000","last_fullsized":"2024-08-17T18:11:46.356570+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:43.035959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":2003,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357178+0000","last_change":"2024-08-17T18:09:29.883587+0000","last_active":"2024-08-17T18:11:46.357178+0000","last_peered":"2024-08-17T18:11:46.357178+0000","last_clean":"2024-08-17T18:11:46.357178+0000","last_became_active":"2024-08-17T18:09:29.882222+0000","last_became_peered":"2024-08-17T18:09:29.882222+0000","last_unstale":"2024-08-17T18:11:46.357178+0000","last_undegraded":"2024-08-17T18:11:46.357178+0000","last_fullsized":"2024-08-17T18:11:46.357178+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:10.777993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2059,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.356830+0000","last_change":"2024-08-17T18:09:28.858824+0000","last_active":"2024-08-17T18:11:46.356830+0000","last_peered":"2024-08-17T18:11:46.356830+0000","last_clean":"2024-08-17T18:11:46.356830+0000","last_became_active":"2024-08-17T18:09:28.857856+0000","last_became_peered":"2024-08-17T18:09:28.857856+0000","last_unstale":"2024-08-17T18:11:46.356830+0000","last_undegraded":"2024-08-17T18:11:46.356830+0000","last_fullsized":"2024-08-17T18:11:46.356830+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:41:35.146166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":63,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357288+0000","last_change":"2024-08-17T18:09:29.851515+0000","last_active":"2024-08-17T18:11:46.357288+0000","last_peered":"2024-08-17T18:11:46.357288+0000","last_clean":"2024-08-17T18:11:46.357288+0000","last_became_active":"2024-08-17T18:09:29.851109+0000","last_became_peered":"2024-08-17T18:09:29.851109+0000","last_unstale":"2024-08-17T18:11:46.357288+0000","last_undegraded":"2024-08-17T18:11:46.357288+0000","last_fullsized":"2024-08-17T18:11:46.357288+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:42.054532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":74,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.352895+0000","last_change":"2024-08-17T18:03:16.089811+0000","last_active":"2024-08-17T18:11:45.352895+0000","last_peered":"2024-08-17T18:11:45.352895+0000","last_clean":"2024-08-17T18:11:45.352895+0000","last_became_active":"2024-08-17T18:03:16.089571+0000","last_became_peered":"2024-08-17T18:03:16.089571+0000","last_unstale":"2024-08-17T18:11:45.352895+0000","last_undegraded":"2024-08-17T18:11:45.352895+0000","last_fullsized":"2024-08-17T18:11:45.352895+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:04.630361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":56,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353586+0000","last_change":"2024-08-17T18:08:57.577055+0000","last_active":"2024-08-17T18:11:45.353586+0000","last_peered":"2024-08-17T18:11:45.353586+0000","last_clean":"2024-08-17T18:11:45.353586+0000","last_became_active":"2024-08-17T18:08:57.573659+0000","last_became_peered":"2024-08-17T18:08:57.573659+0000","last_unstale":"2024-08-17T18:11:45.353586+0000","last_undegraded":"2024-08-17T18:11:45.353586+0000","last_fullsized":"2024-08-17T18:11:45.353586+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:01.686752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":847,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357478+0000","last_change":"2024-08-17T18:09:29.852393+0000","last_active":"2024-08-17T18:11:46.357478+0000","last_peered":"2024-08-17T18:11:46.357478+0000","last_clean":"2024-08-17T18:11:46.357478+0000","last_became_active":"2024-08-17T18:09:29.851947+0000","last_became_peered":"2024-08-17T18:09:29.851947+0000","last_unstale":"2024-08-17T18:11:46.357478+0000","last_undegraded":"2024-08-17T18:11:46.357478+0000","last_fullsized":"2024-08-17T18:11:46.357478+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:49.145327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":2158,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.362858+0000","last_change":"2024-08-17T18:09:29.863241+0000","last_active":"2024-08-17T18:11:46.362858+0000","last_peered":"2024-08-17T18:11:46.362858+0000","last_clean":"2024-08-17T18:11:46.362858+0000","last_became_active":"2024-08-17T18:09:29.862369+0000","last_became_peered":"2024-08-17T18:09:29.862369+0000","last_unstale":"2024-08-17T18:11:46.362858+0000","last_undegraded":"2024-08-17T18:11:46.362858+0000","last_fullsized":"2024-08-17T18:11:46.362858+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:30.518551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":594,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357156+0000","last_change":"2024-08-17T18:09:29.896233+0000","last_active":"2024-08-17T18:11:46.357156+0000","last_peered":"2024-08-17T18:11:46.357156+0000","last_clean":"2024-08-17T18:11:46.357156+0000","last_became_active":"2024-08-17T18:09:29.895718+0000","last_became_peered":"2024-08-17T18:09:29.895718+0000","last_unstale":"2024-08-17T18:11:46.357156+0000","last_undegraded":"2024-08-17T18:11:46.357156+0000","last_fullsized":"2024-08-17T18:11:46.357156+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:03.278878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":68,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.360857+0000","last_change":"2024-08-17T18:08:57.574015+0000","last_active":"2024-08-17T18:11:46.360857+0000","last_peered":"2024-08-17T18:11:46.360857+0000","last_clean":"2024-08-17T18:11:46.360857+0000","last_became_active":"2024-08-17T18:08:57.567973+0000","last_became_peered":"2024-08-17T18:08:57.567973+0000","last_unstale":"2024-08-17T18:11:46.360857+0000","last_undegraded":"2024-08-17T18:11:46.360857+0000","last_fullsized":"2024-08-17T18:11:46.360857+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:06.251764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2014,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357321+0000","last_change":"2024-08-17T18:09:29.917203+0000","last_active":"2024-08-17T18:11:46.357321+0000","last_peered":"2024-08-17T18:11:46.357321+0000","last_clean":"2024-08-17T18:11:46.357321+0000","last_became_active":"2024-08-17T18:09:29.916844+0000","last_became_peered":"2024-08-17T18:09:29.916844+0000","last_unstale":"2024-08-17T18:11:46.357321+0000","last_undegraded":"2024-08-17T18:11:46.357321+0000","last_fullsized":"2024-08-17T18:11:46.357321+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.453720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1920,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.356734+0000","last_change":"2024-08-17T18:09:28.859255+0000","last_active":"2024-08-17T18:11:46.356734+0000","last_peered":"2024-08-17T18:11:46.356734+0000","last_clean":"2024-08-17T18:11:46.356734+0000","last_became_active":"2024-08-17T18:09:28.858537+0000","last_became_peered":"2024-08-17T18:09:28.858537+0000","last_unstale":"2024-08-17T18:11:46.356734+0000","last_undegraded":"2024-08-17T18:11:46.356734+0000","last_fullsized":"2024-08-17T18:11:46.356734+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:15.364687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2181,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.363604+0000","last_change":"2024-08-17T18:09:28.868303+0000","last_active":"2024-08-17T18:11:46.363604+0000","last_peered":"2024-08-17T18:11:46.363604+0000","last_clean":"2024-08-17T18:11:46.363604+0000","last_became_active":"2024-08-17T18:09:28.867989+0000","last_became_peered":"2024-08-17T18:09:28.867989+0000","last_unstale":"2024-08-17T18:11:46.363604+0000","last_undegraded":"2024-08-17T18:11:46.363604+0000","last_fullsized":"2024-08-17T18:11:46.363604+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:58:52.741685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":46,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.356347+0000","last_change":"2024-08-17T18:09:29.842129+0000","last_active":"2024-08-17T18:11:46.356347+0000","last_peered":"2024-08-17T18:11:46.356347+0000","last_clean":"2024-08-17T18:11:46.356347+0000","last_became_active":"2024-08-17T18:09:29.841848+0000","last_became_peered":"2024-08-17T18:09:29.841848+0000","last_unstale":"2024-08-17T18:11:46.356347+0000","last_undegraded":"2024-08-17T18:11:46.356347+0000","last_fullsized":"2024-08-17T18:11:46.356347+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:02.119134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1295,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.363745+0000","last_change":"2024-08-17T18:09:29.846509+0000","last_active":"2024-08-17T18:11:46.363745+0000","last_peered":"2024-08-17T18:11:46.363745+0000","last_clean":"2024-08-17T18:11:46.363745+0000","last_became_active":"2024-08-17T18:09:29.844097+0000","last_became_peered":"2024-08-17T18:09:29.844097+0000","last_unstale":"2024-08-17T18:11:46.363745+0000","last_undegraded":"2024-08-17T18:11:46.363745+0000","last_fullsized":"2024-08-17T18:11:46.363745+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:32.340354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2009,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.356557+0000","last_change":"2024-08-17T18:09:29.841335+0000","last_active":"2024-08-17T18:11:46.356557+0000","last_peered":"2024-08-17T18:11:46.356557+0000","last_clean":"2024-08-17T18:11:46.356557+0000","last_became_active":"2024-08-17T18:09:29.840522+0000","last_became_peered":"2024-08-17T18:09:29.840522+0000","last_unstale":"2024-08-17T18:11:46.356557+0000","last_undegraded":"2024-08-17T18:11:46.356557+0000","last_fullsized":"2024-08-17T18:11:46.356557+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:08.015836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2186,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.363894+0000","last_change":"2024-08-17T18:09:29.847494+0000","last_active":"2024-08-17T18:11:46.363894+0000","last_peered":"2024-08-17T18:11:46.363894+0000","last_clean":"2024-08-17T18:11:46.363894+0000","last_became_active":"2024-08-17T18:09:29.846973+0000","last_became_peered":"2024-08-17T18:09:29.846973+0000","last_unstale":"2024-08-17T18:11:46.363894+0000","last_undegraded":"2024-08-17T18:11:46.363894+0000","last_fullsized":"2024-08-17T18:11:46.363894+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:49.469921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1968,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.356617+0000","last_change":"2024-08-17T18:09:28.855045+0000","last_active":"2024-08-17T18:11:46.356617+0000","last_peered":"2024-08-17T18:11:46.356617+0000","last_clean":"2024-08-17T18:11:46.356617+0000","last_became_active":"2024-08-17T18:09:28.854607+0000","last_became_peered":"2024-08-17T18:09:28.854607+0000","last_unstale":"2024-08-17T18:11:46.356617+0000","last_undegraded":"2024-08-17T18:11:46.356617+0000","last_fullsized":"2024-08-17T18:11:46.356617+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:13.437895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":686,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353557+0000","last_change":"2024-08-17T18:09:29.866944+0000","last_active":"2024-08-17T18:11:45.353557+0000","last_peered":"2024-08-17T18:11:45.353557+0000","last_clean":"2024-08-17T18:11:45.353557+0000","last_became_active":"2024-08-17T18:09:29.866694+0000","last_became_peered":"2024-08-17T18:09:29.866694+0000","last_unstale":"2024-08-17T18:11:45.353557+0000","last_undegraded":"2024-08-17T18:11:45.353557+0000","last_fullsized":"2024-08-17T18:11:45.353557+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:00:41.342392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1349,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353398+0000","last_change":"2024-08-17T18:09:28.884262+0000","last_active":"2024-08-17T18:11:45.353398+0000","last_peered":"2024-08-17T18:11:45.353398+0000","last_clean":"2024-08-17T18:11:45.353398+0000","last_became_active":"2024-08-17T18:09:28.883734+0000","last_became_peered":"2024-08-17T18:09:28.883734+0000","last_unstale":"2024-08-17T18:11:45.353398+0000","last_undegraded":"2024-08-17T18:11:45.353398+0000","last_fullsized":"2024-08-17T18:11:45.353398+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:10.735504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2071,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.360705+0000","last_change":"2024-08-17T18:08:57.571995+0000","last_active":"2024-08-17T18:11:46.360705+0000","last_peered":"2024-08-17T18:11:46.360705+0000","last_clean":"2024-08-17T18:11:46.360705+0000","last_became_active":"2024-08-17T18:08:57.567342+0000","last_became_peered":"2024-08-17T18:08:57.567342+0000","last_unstale":"2024-08-17T18:11:46.360705+0000","last_undegraded":"2024-08-17T18:11:46.360705+0000","last_fullsized":"2024-08-17T18:11:46.360705+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:23.769111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":513,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353636+0000","last_change":"2024-08-17T18:02:53.912837+0000","last_active":"2024-08-17T18:11:45.353636+0000","last_peered":"2024-08-17T18:11:45.353636+0000","last_clean":"2024-08-17T18:11:45.353636+0000","last_became_active":"2024-08-17T18:02:53.883152+0000","last_became_peered":"2024-08-17T18:02:53.883152+0000","last_unstale":"2024-08-17T18:11:45.353636+0000","last_undegraded":"2024-08-17T18:11:45.353636+0000","last_fullsized":"2024-08-17T18:11:45.353636+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:46.261612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":512,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353061+0000","last_change":"2024-08-17T18:09:29.845126+0000","last_active":"2024-08-17T18:11:45.353061+0000","last_peered":"2024-08-17T18:11:45.353061+0000","last_clean":"2024-08-17T18:11:45.353061+0000","last_became_active":"2024-08-17T18:09:29.844836+0000","last_became_peered":"2024-08-17T18:09:29.844836+0000","last_unstale":"2024-08-17T18:11:45.353061+0000","last_undegraded":"2024-08-17T18:11:45.353061+0000","last_fullsized":"2024-08-17T18:11:45.353061+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:44.821894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1308,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.361863+0000","last_change":"2024-08-17T18:02:53.890198+0000","last_active":"2024-08-17T18:11:46.361863+0000","last_peered":"2024-08-17T18:11:46.361863+0000","last_clean":"2024-08-17T18:11:46.361863+0000","last_became_active":"2024-08-17T18:02:53.887007+0000","last_became_peered":"2024-08-17T18:02:53.887007+0000","last_unstale":"2024-08-17T18:11:46.361863+0000","last_undegraded":"2024-08-17T18:11:46.361863+0000","last_fullsized":"2024-08-17T18:11:46.361863+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:25.555909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":603,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353917+0000","last_change":"2024-08-17T18:09:28.884187+0000","last_active":"2024-08-17T18:11:45.353917+0000","last_peered":"2024-08-17T18:11:45.353917+0000","last_clean":"2024-08-17T18:11:45.353917+0000","last_became_active":"2024-08-17T18:09:28.883939+0000","last_became_peered":"2024-08-17T18:09:28.883939+0000","last_unstale":"2024-08-17T18:11:45.353917+0000","last_undegraded":"2024-08-17T18:11:45.353917+0000","last_fullsized":"2024-08-17T18:11:45.353917+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:37:41.951099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1903,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353479+0000","last_change":"2024-08-17T18:09:29.893220+0000","last_active":"2024-08-17T18:11:45.353479+0000","last_peered":"2024-08-17T18:11:45.353479+0000","last_clean":"2024-08-17T18:11:45.353479+0000","last_became_active":"2024-08-17T18:09:29.865699+0000","last_became_peered":"2024-08-17T18:09:29.865699+0000","last_unstale":"2024-08-17T18:11:45.353479+0000","last_undegraded":"2024-08-17T18:11:45.353479+0000","last_fullsized":"2024-08-17T18:11:45.353479+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:40.205658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2028,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.356721+0000","last_change":"2024-08-17T18:09:28.934404+0000","last_active":"2024-08-17T18:11:46.356721+0000","last_peered":"2024-08-17T18:11:46.356721+0000","last_clean":"2024-08-17T18:11:46.356721+0000","last_became_active":"2024-08-17T18:09:28.934001+0000","last_became_peered":"2024-08-17T18:09:28.934001+0000","last_unstale":"2024-08-17T18:11:46.356721+0000","last_undegraded":"2024-08-17T18:11:46.356721+0000","last_fullsized":"2024-08-17T18:11:46.356721+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:04.676812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":2060,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357488+0000","last_change":"2024-08-17T18:09:29.884088+0000","last_active":"2024-08-17T18:11:46.357488+0000","last_peered":"2024-08-17T18:11:46.357488+0000","last_clean":"2024-08-17T18:11:46.357488+0000","last_became_active":"2024-08-17T18:09:29.883027+0000","last_became_peered":"2024-08-17T18:09:29.883027+0000","last_unstale":"2024-08-17T18:11:46.357488+0000","last_undegraded":"2024-08-17T18:11:46.357488+0000","last_fullsized":"2024-08-17T18:11:46.357488+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:22:07.777188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":2030,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357450+0000","last_change":"2024-08-17T18:09:28.935401+0000","last_active":"2024-08-17T18:11:46.357450+0000","last_peered":"2024-08-17T18:11:46.357450+0000","last_clean":"2024-08-17T18:11:46.357450+0000","last_became_active":"2024-08-17T18:09:28.933628+0000","last_became_peered":"2024-08-17T18:09:28.933628+0000","last_unstale":"2024-08-17T18:11:46.357450+0000","last_undegraded":"2024-08-17T18:11:46.357450+0000","last_fullsized":"2024-08-17T18:11:46.357450+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:21:12.714486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2142,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.362954+0000","last_change":"2024-08-17T18:08:57.574718+0000","last_active":"2024-08-17T18:11:46.362954+0000","last_peered":"2024-08-17T18:11:46.362954+0000","last_clean":"2024-08-17T18:11:46.362954+0000","last_became_active":"2024-08-17T18:08:57.573515+0000","last_became_peered":"2024-08-17T18:08:57.573515+0000","last_unstale":"2024-08-17T18:11:46.362954+0000","last_undegraded":"2024-08-17T18:11:46.362954+0000","last_fullsized":"2024-08-17T18:11:46.362954+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:04.873717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1362,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353500+0000","last_change":"2024-08-17T18:08:57.568821+0000","last_active":"2024-08-17T18:11:45.353500+0000","last_peered":"2024-08-17T18:11:45.353500+0000","last_clean":"2024-08-17T18:11:45.353500+0000","last_became_active":"2024-08-17T18:08:57.568526+0000","last_became_peered":"2024-08-17T18:08:57.568526+0000","last_unstale":"2024-08-17T18:11:45.353500+0000","last_undegraded":"2024-08-17T18:11:45.353500+0000","last_fullsized":"2024-08-17T18:11:45.353500+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:26.449197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1862,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.363377+0000","last_change":"2024-08-17T18:08:57.571174+0000","last_active":"2024-08-17T18:11:46.363377+0000","last_peered":"2024-08-17T18:11:46.363377+0000","last_clean":"2024-08-17T18:11:46.363377+0000","last_became_active":"2024-08-17T18:08:57.567571+0000","last_became_peered":"2024-08-17T18:08:57.567571+0000","last_unstale":"2024-08-17T18:11:46.363377+0000","last_undegraded":"2024-08-17T18:11:46.363377+0000","last_fullsized":"2024-08-17T18:11:46.363377+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:29.976150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1831,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.361024+0000","last_change":"2024-08-17T18:08:57.572368+0000","last_active":"2024-08-17T18:11:46.361024+0000","last_peered":"2024-08-17T18:11:46.361024+0000","last_clean":"2024-08-17T18:11:46.361024+0000","last_became_active":"2024-08-17T18:08:57.568382+0000","last_became_peered":"2024-08-17T18:08:57.568382+0000","last_unstale":"2024-08-17T18:11:46.361024+0000","last_undegraded":"2024-08-17T18:11:46.361024+0000","last_fullsized":"2024-08-17T18:11:46.361024+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:11:19.968079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1879,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.363409+0000","last_change":"2024-08-17T18:08:57.572991+0000","last_active":"2024-08-17T18:11:46.363409+0000","last_peered":"2024-08-17T18:11:46.363409+0000","last_clean":"2024-08-17T18:11:46.363409+0000","last_became_active":"2024-08-17T18:08:57.571145+0000","last_became_peered":"2024-08-17T18:08:57.571145+0000","last_unstale":"2024-08-17T18:11:46.363409+0000","last_undegraded":"2024-08-17T18:11:46.363409+0000","last_fullsized":"2024-08-17T18:11:46.363409+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:39.963337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":835,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357213+0000","last_change":"2024-08-17T18:09:28.860288+0000","last_active":"2024-08-17T18:11:46.357213+0000","last_peered":"2024-08-17T18:11:46.357213+0000","last_clean":"2024-08-17T18:11:46.357213+0000","last_became_active":"2024-08-17T18:09:28.860004+0000","last_became_peered":"2024-08-17T18:09:28.860004+0000","last_unstale":"2024-08-17T18:11:46.357213+0000","last_undegraded":"2024-08-17T18:11:46.357213+0000","last_fullsized":"2024-08-17T18:11:46.357213+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:57:33.054055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":628,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357619+0000","last_change":"2024-08-17T18:09:28.860277+0000","last_active":"2024-08-17T18:11:46.357619+0000","last_peered":"2024-08-17T18:11:46.357619+0000","last_clean":"2024-08-17T18:11:46.357619+0000","last_became_active":"2024-08-17T18:09:28.859999+0000","last_became_peered":"2024-08-17T18:09:28.859999+0000","last_unstale":"2024-08-17T18:11:46.357619+0000","last_undegraded":"2024-08-17T18:11:46.357619+0000","last_fullsized":"2024-08-17T18:11:46.357619+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:27.946526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1533,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.362939+0000","last_change":"2024-08-17T18:09:29.863669+0000","last_active":"2024-08-17T18:11:46.362939+0000","last_peered":"2024-08-17T18:11:46.362939+0000","last_clean":"2024-08-17T18:11:46.362939+0000","last_became_active":"2024-08-17T18:09:29.862781+0000","last_became_peered":"2024-08-17T18:09:29.862781+0000","last_unstale":"2024-08-17T18:11:46.362939+0000","last_undegraded":"2024-08-17T18:11:46.362939+0000","last_fullsized":"2024-08-17T18:11:46.362939+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:14.073233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":114,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.362418+0000","last_change":"2024-08-17T18:09:29.843574+0000","last_active":"2024-08-17T18:11:46.362418+0000","last_peered":"2024-08-17T18:11:46.362418+0000","last_clean":"2024-08-17T18:11:46.362418+0000","last_became_active":"2024-08-17T18:09:29.843097+0000","last_became_peered":"2024-08-17T18:09:29.843097+0000","last_unstale":"2024-08-17T18:11:46.362418+0000","last_undegraded":"2024-08-17T18:11:46.362418+0000","last_fullsized":"2024-08-17T18:11:46.362418+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:46.740888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1878,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.364059+0000","last_change":"2024-08-17T18:09:29.847264+0000","last_active":"2024-08-17T18:11:46.364059+0000","last_peered":"2024-08-17T18:11:46.364059+0000","last_clean":"2024-08-17T18:11:46.364059+0000","last_became_active":"2024-08-17T18:09:29.846226+0000","last_became_peered":"2024-08-17T18:09:29.846226+0000","last_unstale":"2024-08-17T18:11:46.364059+0000","last_undegraded":"2024-08-17T18:11:46.364059+0000","last_fullsized":"2024-08-17T18:11:46.364059+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:44.135977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2078,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.359482+0000","last_change":"2024-08-17T18:09:29.845209+0000","last_active":"2024-08-17T18:11:46.359482+0000","last_peered":"2024-08-17T18:11:46.359482+0000","last_clean":"2024-08-17T18:11:46.359482+0000","last_became_active":"2024-08-17T18:09:29.844910+0000","last_became_peered":"2024-08-17T18:09:29.844910+0000","last_unstale":"2024-08-17T18:11:46.359482+0000","last_undegraded":"2024-08-17T18:11:46.359482+0000","last_fullsized":"2024-08-17T18:11:46.359482+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:14.586354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1951,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.356828+0000","last_change":"2024-08-17T18:09:28.859050+0000","last_active":"2024-08-17T18:11:46.356828+0000","last_peered":"2024-08-17T18:11:46.356828+0000","last_clean":"2024-08-17T18:11:46.356828+0000","last_became_active":"2024-08-17T18:09:28.858214+0000","last_became_peered":"2024-08-17T18:09:28.858214+0000","last_unstale":"2024-08-17T18:11:46.356828+0000","last_undegraded":"2024-08-17T18:11:46.356828+0000","last_fullsized":"2024-08-17T18:11:46.356828+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:54.146592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2202,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.363774+0000","last_change":"2024-08-17T18:08:57.572610+0000","last_active":"2024-08-17T18:11:46.363774+0000","last_peered":"2024-08-17T18:11:46.363774+0000","last_clean":"2024-08-17T18:11:46.363774+0000","last_became_active":"2024-08-17T18:08:57.572308+0000","last_became_peered":"2024-08-17T18:08:57.572308+0000","last_unstale":"2024-08-17T18:11:46.363774+0000","last_undegraded":"2024-08-17T18:11:46.363774+0000","last_fullsized":"2024-08-17T18:11:46.363774+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:11.642831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2187,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.361876+0000","last_change":"2024-08-17T18:09:28.842838+0000","last_active":"2024-08-17T18:11:46.361876+0000","last_peered":"2024-08-17T18:11:46.361876+0000","last_clean":"2024-08-17T18:11:46.361876+0000","last_became_active":"2024-08-17T18:09:28.842617+0000","last_became_peered":"2024-08-17T18:09:28.842617+0000","last_unstale":"2024-08-17T18:11:46.361876+0000","last_undegraded":"2024-08-17T18:11:46.361876+0000","last_fullsized":"2024-08-17T18:11:46.361876+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:00.016002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1383,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.352878+0000","last_change":"2024-08-17T18:09:29.845011+0000","last_active":"2024-08-17T18:11:45.352878+0000","last_peered":"2024-08-17T18:11:45.352878+0000","last_clean":"2024-08-17T18:11:45.352878+0000","last_became_active":"2024-08-17T18:09:29.844358+0000","last_became_peered":"2024-08-17T18:09:29.844358+0000","last_unstale":"2024-08-17T18:11:45.352878+0000","last_undegraded":"2024-08-17T18:11:45.352878+0000","last_fullsized":"2024-08-17T18:11:45.352878+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:41:27.792901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1231,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353424+0000","last_change":"2024-08-17T18:09:28.878263+0000","last_active":"2024-08-17T18:11:45.353424+0000","last_peered":"2024-08-17T18:11:45.353424+0000","last_clean":"2024-08-17T18:11:45.353424+0000","last_became_active":"2024-08-17T18:09:28.878025+0000","last_became_peered":"2024-08-17T18:09:28.878025+0000","last_unstale":"2024-08-17T18:11:45.353424+0000","last_undegraded":"2024-08-17T18:11:45.353424+0000","last_fullsized":"2024-08-17T18:11:45.353424+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:37.211100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1979,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.356942+0000","last_change":"2024-08-17T18:09:29.840134+0000","last_active":"2024-08-17T18:11:46.356942+0000","last_peered":"2024-08-17T18:11:46.356942+0000","last_clean":"2024-08-17T18:11:46.356942+0000","last_became_active":"2024-08-17T18:09:29.839848+0000","last_became_peered":"2024-08-17T18:09:29.839848+0000","last_unstale":"2024-08-17T18:11:46.356942+0000","last_undegraded":"2024-08-17T18:11:46.356942+0000","last_fullsized":"2024-08-17T18:11:46.356942+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:08:08.717851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":138,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353422+0000","last_change":"2024-08-17T18:09:28.847633+0000","last_active":"2024-08-17T18:11:45.353422+0000","last_peered":"2024-08-17T18:11:45.353422+0000","last_clean":"2024-08-17T18:11:45.353422+0000","last_became_active":"2024-08-17T18:09:28.847341+0000","last_became_peered":"2024-08-17T18:09:28.847341+0000","last_unstale":"2024-08-17T18:11:45.353422+0000","last_undegraded":"2024-08-17T18:11:45.353422+0000","last_fullsized":"2024-08-17T18:11:45.353422+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:51:59.625154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1723,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.352760+0000","last_change":"2024-08-17T18:09:00.609431+0000","last_active":"2024-08-17T18:11:45.352760+0000","last_peered":"2024-08-17T18:11:45.352760+0000","last_clean":"2024-08-17T18:11:45.352760+0000","last_became_active":"2024-08-17T18:09:00.609167+0000","last_became_peered":"2024-08-17T18:09:00.609167+0000","last_unstale":"2024-08-17T18:11:45.352760+0000","last_undegraded":"2024-08-17T18:11:45.352760+0000","last_fullsized":"2024-08-17T18:11:45.352760+0000","mapping_epoch":1338,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:39.579970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1859,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357961+0000","last_change":"2024-08-17T18:09:29.837502+0000","last_active":"2024-08-17T18:11:46.357961+0000","last_peered":"2024-08-17T18:11:46.357961+0000","last_clean":"2024-08-17T18:11:46.357961+0000","last_became_active":"2024-08-17T18:09:29.837243+0000","last_became_peered":"2024-08-17T18:09:29.837243+0000","last_unstale":"2024-08-17T18:11:46.357961+0000","last_undegraded":"2024-08-17T18:11:46.357961+0000","last_fullsized":"2024-08-17T18:11:46.357961+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:47.377505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2057,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.356764+0000","last_change":"2024-08-17T18:09:29.859415+0000","last_active":"2024-08-17T18:11:46.356764+0000","last_peered":"2024-08-17T18:11:46.356764+0000","last_clean":"2024-08-17T18:11:46.356764+0000","last_became_active":"2024-08-17T18:09:29.859138+0000","last_became_peered":"2024-08-17T18:09:29.859138+0000","last_unstale":"2024-08-17T18:11:46.356764+0000","last_undegraded":"2024-08-17T18:11:46.356764+0000","last_fullsized":"2024-08-17T18:11:46.356764+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:31.178915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":508,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357361+0000","last_change":"2024-08-17T18:09:29.883834+0000","last_active":"2024-08-17T18:11:46.357361+0000","last_peered":"2024-08-17T18:11:46.357361+0000","last_clean":"2024-08-17T18:11:46.357361+0000","last_became_active":"2024-08-17T18:09:29.882638+0000","last_became_peered":"2024-08-17T18:09:29.882638+0000","last_unstale":"2024-08-17T18:11:46.357361+0000","last_undegraded":"2024-08-17T18:11:46.357361+0000","last_fullsized":"2024-08-17T18:11:46.357361+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:07:10.188684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":623,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.354912+0000","last_change":"2024-08-17T18:09:29.857250+0000","last_active":"2024-08-17T18:11:45.354912+0000","last_peered":"2024-08-17T18:11:45.354912+0000","last_clean":"2024-08-17T18:11:45.354912+0000","last_became_active":"2024-08-17T18:09:29.857038+0000","last_became_peered":"2024-08-17T18:09:29.857038+0000","last_unstale":"2024-08-17T18:11:45.354912+0000","last_undegraded":"2024-08-17T18:11:45.354912+0000","last_fullsized":"2024-08-17T18:11:45.354912+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:02:54.028575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":621,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.363351+0000","last_change":"2024-08-17T18:08:57.571270+0000","last_active":"2024-08-17T18:11:46.363351+0000","last_peered":"2024-08-17T18:11:46.363351+0000","last_clean":"2024-08-17T18:11:46.363351+0000","last_became_active":"2024-08-17T18:08:57.567670+0000","last_became_peered":"2024-08-17T18:08:57.567670+0000","last_unstale":"2024-08-17T18:11:46.363351+0000","last_undegraded":"2024-08-17T18:11:46.363351+0000","last_fullsized":"2024-08-17T18:11:46.363351+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:19:42.617321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1192,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357044+0000","last_change":"2024-08-17T18:09:29.839260+0000","last_active":"2024-08-17T18:11:46.357044+0000","last_peered":"2024-08-17T18:11:46.357044+0000","last_clean":"2024-08-17T18:11:46.357044+0000","last_became_active":"2024-08-17T18:09:29.838953+0000","last_became_peered":"2024-08-17T18:09:29.838953+0000","last_unstale":"2024-08-17T18:11:46.357044+0000","last_undegraded":"2024-08-17T18:11:46.357044+0000","last_fullsized":"2024-08-17T18:11:46.357044+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:44.014083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":781,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357807+0000","last_change":"2024-08-17T18:09:29.851347+0000","last_active":"2024-08-17T18:11:46.357807+0000","last_peered":"2024-08-17T18:11:46.357807+0000","last_clean":"2024-08-17T18:11:46.357807+0000","last_became_active":"2024-08-17T18:09:29.851046+0000","last_became_peered":"2024-08-17T18:09:29.851046+0000","last_unstale":"2024-08-17T18:11:46.357807+0000","last_undegraded":"2024-08-17T18:11:46.357807+0000","last_fullsized":"2024-08-17T18:11:46.357807+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:21.350703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2058,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357198+0000","last_change":"2024-08-17T18:09:29.912446+0000","last_active":"2024-08-17T18:11:46.357198+0000","last_peered":"2024-08-17T18:11:46.357198+0000","last_clean":"2024-08-17T18:11:46.357198+0000","last_became_active":"2024-08-17T18:09:29.910703+0000","last_became_peered":"2024-08-17T18:09:29.910703+0000","last_unstale":"2024-08-17T18:11:46.357198+0000","last_undegraded":"2024-08-17T18:11:46.357198+0000","last_fullsized":"2024-08-17T18:11:46.357198+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.250608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2224,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.363060+0000","last_change":"2024-08-17T18:09:29.863361+0000","last_active":"2024-08-17T18:11:46.363060+0000","last_peered":"2024-08-17T18:11:46.363060+0000","last_clean":"2024-08-17T18:11:46.363060+0000","last_became_active":"2024-08-17T18:09:29.862503+0000","last_became_peered":"2024-08-17T18:09:29.862503+0000","last_unstale":"2024-08-17T18:11:46.363060+0000","last_undegraded":"2024-08-17T18:11:46.363060+0000","last_fullsized":"2024-08-17T18:11:46.363060+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:25.908232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":47,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357070+0000","last_change":"2024-08-17T18:09:29.895991+0000","last_active":"2024-08-17T18:11:46.357070+0000","last_peered":"2024-08-17T18:11:46.357070+0000","last_clean":"2024-08-17T18:11:46.357070+0000","last_became_active":"2024-08-17T18:09:29.895274+0000","last_became_peered":"2024-08-17T18:09:29.895274+0000","last_unstale":"2024-08-17T18:11:46.357070+0000","last_undegraded":"2024-08-17T18:11:46.357070+0000","last_fullsized":"2024-08-17T18:11:46.357070+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:19:26.420016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":505,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353049+0000","last_change":"2024-08-17T18:09:29.844686+0000","last_active":"2024-08-17T18:11:45.353049+0000","last_peered":"2024-08-17T18:11:45.353049+0000","last_clean":"2024-08-17T18:11:45.353049+0000","last_became_active":"2024-08-17T18:09:29.844341+0000","last_became_peered":"2024-08-17T18:09:29.844341+0000","last_unstale":"2024-08-17T18:11:45.353049+0000","last_undegraded":"2024-08-17T18:11:45.353049+0000","last_fullsized":"2024-08-17T18:11:45.353049+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:38.872767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1342,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353501+0000","last_change":"2024-08-17T18:09:28.884310+0000","last_active":"2024-08-17T18:11:45.353501+0000","last_peered":"2024-08-17T18:11:45.353501+0000","last_clean":"2024-08-17T18:11:45.353501+0000","last_became_active":"2024-08-17T18:09:28.883921+0000","last_became_peered":"2024-08-17T18:09:28.883921+0000","last_unstale":"2024-08-17T18:11:45.353501+0000","last_undegraded":"2024-08-17T18:11:45.353501+0000","last_fullsized":"2024-08-17T18:11:45.353501+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:05.272563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":790,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.356760+0000","last_change":"2024-08-17T18:09:29.837238+0000","last_active":"2024-08-17T18:11:46.356760+0000","last_peered":"2024-08-17T18:11:46.356760+0000","last_clean":"2024-08-17T18:11:46.356760+0000","last_became_active":"2024-08-17T18:09:29.836760+0000","last_became_peered":"2024-08-17T18:09:29.836760+0000","last_unstale":"2024-08-17T18:11:46.356760+0000","last_undegraded":"2024-08-17T18:11:46.356760+0000","last_fullsized":"2024-08-17T18:11:46.356760+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:38:52.862967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1357,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353740+0000","last_change":"2024-08-17T18:09:29.871872+0000","last_active":"2024-08-17T18:11:45.353740+0000","last_peered":"2024-08-17T18:11:45.353740+0000","last_clean":"2024-08-17T18:11:45.353740+0000","last_became_active":"2024-08-17T18:09:29.845111+0000","last_became_peered":"2024-08-17T18:09:29.845111+0000","last_unstale":"2024-08-17T18:11:45.353740+0000","last_undegraded":"2024-08-17T18:11:45.353740+0000","last_fullsized":"2024-08-17T18:11:45.353740+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:48.183250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":254,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.352675+0000","last_change":"2024-08-17T18:09:29.844786+0000","last_active":"2024-08-17T18:11:45.352675+0000","last_peered":"2024-08-17T18:11:45.352675+0000","last_clean":"2024-08-17T18:11:45.352675+0000","last_became_active":"2024-08-17T18:09:29.844270+0000","last_became_peered":"2024-08-17T18:09:29.844270+0000","last_unstale":"2024-08-17T18:11:45.352675+0000","last_undegraded":"2024-08-17T18:11:45.352675+0000","last_fullsized":"2024-08-17T18:11:45.352675+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:51.866680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1373,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353871+0000","last_change":"2024-08-17T18:09:29.886391+0000","last_active":"2024-08-17T18:11:45.353871+0000","last_peered":"2024-08-17T18:11:45.353871+0000","last_clean":"2024-08-17T18:11:45.353871+0000","last_became_active":"2024-08-17T18:09:29.844642+0000","last_became_peered":"2024-08-17T18:09:29.844642+0000","last_unstale":"2024-08-17T18:11:45.353871+0000","last_undegraded":"2024-08-17T18:11:45.353871+0000","last_fullsized":"2024-08-17T18:11:45.353871+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:07.988100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":203,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.359956+0000","last_change":"2024-08-17T18:09:28.941223+0000","last_active":"2024-08-17T18:11:46.359956+0000","last_peered":"2024-08-17T18:11:46.359956+0000","last_clean":"2024-08-17T18:11:46.359956+0000","last_became_active":"2024-08-17T18:09:28.940857+0000","last_became_peered":"2024-08-17T18:09:28.940857+0000","last_unstale":"2024-08-17T18:11:46.359956+0000","last_undegraded":"2024-08-17T18:11:46.359956+0000","last_fullsized":"2024-08-17T18:11:46.359956+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:48.809420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2054,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357977+0000","last_change":"2024-08-17T18:09:29.838873+0000","last_active":"2024-08-17T18:11:46.357977+0000","last_peered":"2024-08-17T18:11:46.357977+0000","last_clean":"2024-08-17T18:11:46.357977+0000","last_became_active":"2024-08-17T18:09:29.838560+0000","last_became_peered":"2024-08-17T18:09:29.838560+0000","last_unstale":"2024-08-17T18:11:46.357977+0000","last_undegraded":"2024-08-17T18:11:46.357977+0000","last_fullsized":"2024-08-17T18:11:46.357977+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:00.103741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1333,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.354958+0000","last_change":"2024-08-17T18:09:28.844170+0000","last_active":"2024-08-17T18:11:45.354958+0000","last_peered":"2024-08-17T18:11:45.354958+0000","last_clean":"2024-08-17T18:11:45.354958+0000","last_became_active":"2024-08-17T18:09:28.843862+0000","last_became_peered":"2024-08-17T18:09:28.843862+0000","last_unstale":"2024-08-17T18:11:45.354958+0000","last_undegraded":"2024-08-17T18:11:45.354958+0000","last_fullsized":"2024-08-17T18:11:45.354958+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:43:19.483326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2177,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.363341+0000","last_change":"2024-08-17T18:09:28.864923+0000","last_active":"2024-08-17T18:11:46.363341+0000","last_peered":"2024-08-17T18:11:46.363341+0000","last_clean":"2024-08-17T18:11:46.363341+0000","last_became_active":"2024-08-17T18:09:28.864707+0000","last_became_peered":"2024-08-17T18:09:28.864707+0000","last_unstale":"2024-08-17T18:11:46.363341+0000","last_undegraded":"2024-08-17T18:11:46.363341+0000","last_fullsized":"2024-08-17T18:11:46.363341+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:27.507329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2178,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.362522+0000","last_change":"2024-08-17T18:09:28.846741+0000","last_active":"2024-08-17T18:11:46.362522+0000","last_peered":"2024-08-17T18:11:46.362522+0000","last_clean":"2024-08-17T18:11:46.362522+0000","last_became_active":"2024-08-17T18:09:28.846478+0000","last_became_peered":"2024-08-17T18:09:28.846478+0000","last_unstale":"2024-08-17T18:11:46.362522+0000","last_undegraded":"2024-08-17T18:11:46.362522+0000","last_fullsized":"2024-08-17T18:11:46.362522+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:34.247517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":739,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353336+0000","last_change":"2024-08-17T18:03:16.091420+0000","last_active":"2024-08-17T18:11:45.353336+0000","last_peered":"2024-08-17T18:11:45.353336+0000","last_clean":"2024-08-17T18:11:45.353336+0000","last_became_active":"2024-08-17T18:03:16.091113+0000","last_became_peered":"2024-08-17T18:03:16.091113+0000","last_unstale":"2024-08-17T18:11:45.353336+0000","last_undegraded":"2024-08-17T18:11:45.353336+0000","last_fullsized":"2024-08-17T18:11:45.353336+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:47.770911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1347,"reported_epoch":1361,"state":"active+clean","last_fresh":"2024-08-17T18:11:45.353327+0000","last_change":"2024-08-17T18:02:53.893927+0000","last_active":"2024-08-17T18:11:45.353327+0000","last_peered":"2024-08-17T18:11:45.353327+0000","last_clean":"2024-08-17T18:11:45.353327+0000","last_became_active":"2024-08-17T18:02:53.883188+0000","last_became_peered":"2024-08-17T18:02:53.883188+0000","last_unstale":"2024-08-17T18:11:45.353327+0000","last_undegraded":"2024-08-17T18:11:45.353327+0000","last_fullsized":"2024-08-17T18:11:45.353327+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:19.296965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2055,"reported_epoch":1362,"state":"active+clean","last_fresh":"2024-08-17T18:11:46.357386+0000","last_change":"2024-08-17T18:09:29.840086+0000","last_active":"2024-08-17T18:11:46.357386+0000","last_peered":"2024-08-17T18:11:46.357386+0000","last_clean":"2024-08-17T18:11:46.357386+0000","last_became_active":"2024-08-17T18:09:29.839525+0000","last_became_peered":"2024-08-17T18:09:29.839525+0000","last_unstale":"2024-08-17T18:11:46.357386+0000","last_undegraded":"2024-08-17T18:11:46.357386+0000","last_fullsized":"2024-08-17T18:11:46.357386+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:38.316235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":48,"num_pg":16,"stat_sum":{"num_bytes":1021706264,"num_objects":15591,"num_object_clones":0,"num_object_copies":46773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6601,"num_objects_misplaced":5081,"num_objects_unfound":0,"num_objects_dirty":15591,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":248335,"num_write_kb":8445569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14238,"num_bytes_recovered":932839424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2771984384,"data_stored":2771976240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76744,"ondisk_log_size":76744,"up":48,"acting":40,"num_store_stats":5},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":10970,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":155648,"data_stored":27490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234611,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1661432,"kb_used_data":763696,"kb_used_omap":1,"kb_used_meta":897726,"kb_avail":92083720,"statfs":{"total":95995035648,"available":94293729280,"internally_reserved":0,"allocated":782024704,"data_stored":777542340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":919271768},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568496246,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757884,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1343,"seq":5768141078557,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":443236,"kb_used_data":234460,"kb_used_omap":112,"kb_used_meta":208591,"kb_avail":93301916,"statfs":{"total":95995035648,"available":95541161984,"internally_reserved":0,"allocated":240087040,"data_stored":235620392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":115373,"internal_metadata":213597523},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405736,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1829516,"kb_used_data":835780,"kb_used_omap":61,"kb_used_meta":993474,"kb_avail":91915636,"statfs":{"total":95995035648,"available":94121611264,"internally_reserved":0,"allocated":855838720,"data_stored":851383364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":1017318328},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":0,"seq":2173253452271,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474962,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1371064,"kb_used_data":503984,"kb_used_omap":119,"kb_used_meta":866952,"kb_avail":92374088,"statfs":{"total":95995035648,"available":94591066112,"internally_reserved":0,"allocated":516079616,"data_stored":511597810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":887759582},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019519,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":492306432,"data_stored":492306432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":473628672,"data_stored":473628672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":821694464,"data_stored":821690392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":228392960,"data_stored":228392960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":755961856,"data_stored":755957784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:11:51.765 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T18:11:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:51 smithi179 ceph-mon[93519]: pgmap v2479: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 7 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 975 MiB data, 5.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 728 op/s; 6601/46851 objects degraded (14.089%); 5081/46851 objects misplaced (10.845%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T18:11:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 6601/46851 objects degraded (14.089%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:51 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1446954835' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:11:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:51 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:11:51.853 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:51.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:51.901 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:51.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:51.997 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:11:52.002 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:52.065 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:52.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:52.166 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:52.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:52.192 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:52.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:52.293 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:52.338 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:52.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:52.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:52.440 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:52.463 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:52.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:52.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:52.565 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:52.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:52 smithi179 ceph-mon[93519]: from='client.46763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:11:52.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:52 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:11:52.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:52 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:11:52.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:52 smithi179 ceph-mon[93519]: osdmap e1363: 8 total, 8 up, 4 in 2024-08-17T18:11:52.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:52 smithi179 ceph-mon[93519]: pgmap v2481: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 7 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 936 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 504 op/s; 6193/45000 objects degraded (13.762%); 5081/45000 objects misplaced (11.291%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T18:11:52.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:52.627 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:52.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:52.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:52 smithi130 ceph-mon[105521]: from='client.46763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:11:52.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:52 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:11:52.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:52 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:11:52.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:52 smithi130 ceph-mon[105521]: osdmap e1363: 8 total, 8 up, 4 in 2024-08-17T18:11:52.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:52 smithi130 ceph-mon[105521]: pgmap v2481: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 7 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 936 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 504 op/s; 6193/45000 objects degraded (13.762%); 5081/45000 objects misplaced (11.291%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T18:11:52.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:52 smithi130 ceph-mon[110068]: from='client.46763 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:11:52.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:52 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:11:52.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:52 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:11:52.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:52 smithi130 ceph-mon[110068]: osdmap e1363: 8 total, 8 up, 4 in 2024-08-17T18:11:52.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:52 smithi130 ceph-mon[110068]: pgmap v2481: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 7 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 936 MiB data, 4.9 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 504 op/s; 6193/45000 objects degraded (13.762%); 5081/45000 objects misplaced (11.291%); 7.1 MiB/s, 113 objects/s recovering 2024-08-17T18:11:52.707 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:11:52.728 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:52.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:52.798 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:52.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:52.899 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:52.921 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:52.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:52.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:53.022 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:53.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:53.088 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:53.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:53.189 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:53.211 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:53.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:53.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:53.313 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:53.336 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:53.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:53.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:53.437 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:53.487 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:53.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:53.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:53.589 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:53.611 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:53.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:53.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:53.713 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:53.740 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:53.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:53.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:53.842 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:53.867 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:53.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:53.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:53.969 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:53.992 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:53.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:53.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:54.094 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:54.115 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:54.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:54.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:54.217 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:54.242 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:54.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:54.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:54.344 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:54.368 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:54.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:54.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:54.470 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:54.504 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:54.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:54.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:54.606 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:54.618 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:11:54.628 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:54.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:54.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:54.661 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:54 smithi130 ceph-mon[105521]: osdmap e1364: 8 total, 8 up, 4 in 2024-08-17T18:11:54.661 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:54 smithi130 ceph-mon[105521]: pgmap v2483: 89 pgs: 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 763 MiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 823 op/s; 4980/36696 objects degraded (13.571%); 3704/36696 objects misplaced (10.094%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T18:11:54.662 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:54 smithi130 ceph-mon[110068]: osdmap e1364: 8 total, 8 up, 4 in 2024-08-17T18:11:54.662 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:54 smithi130 ceph-mon[110068]: pgmap v2483: 89 pgs: 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 763 MiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 823 op/s; 4980/36696 objects degraded (13.571%); 3704/36696 objects misplaced (10.094%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T18:11:54.729 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:54.778 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:54.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:54.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:54.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:54 smithi179 ceph-mon[93519]: osdmap e1364: 8 total, 8 up, 4 in 2024-08-17T18:11:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:54 smithi179 ceph-mon[93519]: pgmap v2483: 89 pgs: 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 763 MiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 823 op/s; 4980/36696 objects degraded (13.571%); 3704/36696 objects misplaced (10.094%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T18:11:54.865 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:11:54.880 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:54.951 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:54.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:54.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:55.053 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:55.108 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:55.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:55.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:55.210 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:55.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:55.272 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:55.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:55.373 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:55.395 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:55.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:55.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:55.497 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:55.551 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:55.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:55.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:55.653 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:55 smithi130 ceph-mon[105521]: osdmap e1365: 8 total, 8 up, 4 in 2024-08-17T18:11:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:55 smithi130 ceph-mon[110068]: osdmap e1365: 8 total, 8 up, 4 in 2024-08-17T18:11:55.704 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:55.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:55.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:55.806 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:55.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:55.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:55.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:55.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:55 smithi179 ceph-mon[93519]: osdmap e1365: 8 total, 8 up, 4 in 2024-08-17T18:11:55.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:55.951 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:55.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:55.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:56.040 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:11:56.041 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:11:56.052 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:56.096 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:56.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:56.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:56.198 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:56.221 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:56.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:56.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:56.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:56.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:56.363 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:56.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:56.464 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:56.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:56.485 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:56.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:56.586 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:11:56.608 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:56.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:56.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:11:56.710 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:56.732 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:56.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:56.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:56.834 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:56 smithi179 ceph-mon[93519]: pgmap v2485: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 696 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 742 op/s; 3851/33483 objects degraded (11.501%); 3479/33483 objects misplaced (10.390%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T18:11:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 4980/36696 objects degraded (13.571%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:56.879 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:56.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:56.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:56.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:56 smithi130 ceph-mon[110068]: pgmap v2485: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 696 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 742 op/s; 3851/33483 objects degraded (11.501%); 3479/33483 objects misplaced (10.390%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T18:11:56.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 4980/36696 objects degraded (13.571%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:56.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:56 smithi130 ceph-mon[105521]: pgmap v2485: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+undersized+degraded+remapped+backfill_wait, 78 active+clean; 696 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 742 op/s; 3851/33483 objects degraded (11.501%); 3479/33483 objects misplaced (10.390%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T18:11:56.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 4980/36696 objects degraded (13.571%), 7 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T18:11:56.981 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:57.002 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:57.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:57.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:57.104 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:57.135 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:57.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:57.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:57.237 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:57.259 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:57.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:57.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:57.361 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:57.409 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:57.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:57.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:57.511 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:11:57.533 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:57.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:57.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:11:57.635 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:57.674 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:57.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:57.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:57.775 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:57.811 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:57.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:57.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:57.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:57.951 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:57.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:57.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:58.053 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:58.074 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:58.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:58.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:58.176 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:58.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:58.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:58.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:58.314 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:58.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:58.337 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:58.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:58.438 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:58.469 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:58.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:58.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:58.571 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:58.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:58.605 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:58.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:58.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:11:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:11:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:11:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:11:58.706 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:58.729 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:58.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:58.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:58.831 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:58.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:11:58.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:11:58.852 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:58.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:58.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:58.954 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:58.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:58.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:58.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:59.078 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:11:59.101 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:59.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:59.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:11:59.202 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:59.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:59.227 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:59.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:59.326 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T18:11:59.328 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:11:59.351 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:59.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:59.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:11:59.404 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:11:59.454 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:11:59.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:59.504 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:59.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:11:59.572 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:59 smithi130 ceph-mon[105521]: pgmap v2486: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 6 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 647 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 757 op/s; 3851/31131 objects degraded (12.370%); 3479/31131 objects misplaced (11.175%); 8.8 MiB/s, 140 objects/s recovering 2024-08-17T18:11:59.572 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:11:59 smithi130 ceph-mon[105521]: osdmap e1366: 8 total, 8 up, 4 in 2024-08-17T18:11:59.572 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:59 smithi130 ceph-mon[110068]: pgmap v2486: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 6 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 647 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 757 op/s; 3851/31131 objects degraded (12.370%); 3479/31131 objects misplaced (11.175%); 8.8 MiB/s, 140 objects/s recovering 2024-08-17T18:11:59.572 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:11:59 smithi130 ceph-mon[110068]: osdmap e1366: 8 total, 8 up, 4 in 2024-08-17T18:11:59.606 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:11:59.649 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:11:59.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:59.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:11:59.751 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:11:59.772 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:59.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:59.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:11:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:59 smithi179 ceph-mon[93519]: pgmap v2486: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 6 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 647 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 757 op/s; 3851/31131 objects degraded (12.370%); 3479/31131 objects misplaced (11.175%); 8.8 MiB/s, 140 objects/s recovering 2024-08-17T18:11:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:11:59 smithi179 ceph-mon[93519]: osdmap e1366: 8 total, 8 up, 4 in 2024-08-17T18:11:59.874 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:11:59.879 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:11:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:11:59] "GET /metrics HTTP/1.1" 200 36800 "" "Prometheus/2.51.0" 2024-08-17T18:11:59.895 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:11:59.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:11:59.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:11:59.997 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:00.018 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:00.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:00.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:00.120 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:00.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:00.142 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:00.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:00.243 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:00.268 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:00.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:00.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:00.370 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:00.397 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:00.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:00.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:00.498 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:00.530 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:00.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:00.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:00 smithi179 ceph-mon[93519]: osdmap e1367: 8 total, 8 up, 4 in 2024-08-17T18:12:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:12:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:12:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:12:00.631 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:00.655 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:00.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:00.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:00.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:00 smithi130 ceph-mon[105521]: osdmap e1367: 8 total, 8 up, 4 in 2024-08-17T18:12:00.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:00 smithi130 ceph-mon[110068]: osdmap e1367: 8 total, 8 up, 4 in 2024-08-17T18:12:00.757 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:00.798 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:00.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:00.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:00.902 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:00.923 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:00.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:00.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:01.025 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:01.046 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:01.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:01.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:01.148 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:01.170 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:01.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:01.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:01.271 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:01.293 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:01.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:01.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:01.395 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:01.426 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:01.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:01.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:01.528 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:01.561 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:01.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:01.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:01.663 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:01.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:01 smithi130 ceph-mon[105521]: pgmap v2489: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 5 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 525 MiB data, 4.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 635 op/s; 2516/25251 objects degraded (9.964%); 2427/25251 objects misplaced (9.612%); 9.6 MiB/s, 152 objects/s recovering 2024-08-17T18:12:01.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:01 smithi130 ceph-mon[105521]: osdmap e1368: 8 total, 8 up, 4 in 2024-08-17T18:12:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 2516/25251 objects degraded (9.964%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T18:12:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:01 smithi130 ceph-mon[110068]: pgmap v2489: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 5 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 525 MiB data, 4.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 635 op/s; 2516/25251 objects degraded (9.964%); 2427/25251 objects misplaced (9.612%); 9.6 MiB/s, 152 objects/s recovering 2024-08-17T18:12:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:01 smithi130 ceph-mon[110068]: osdmap e1368: 8 total, 8 up, 4 in 2024-08-17T18:12:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 2516/25251 objects degraded (9.964%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T18:12:01.684 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:01.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:01.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:01.786 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:01.809 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:01.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:01.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:01.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:01 smithi179 ceph-mon[93519]: pgmap v2489: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 5 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 525 MiB data, 4.1 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 635 op/s; 2516/25251 objects degraded (9.964%); 2427/25251 objects misplaced (9.612%); 9.6 MiB/s, 152 objects/s recovering 2024-08-17T18:12:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:01 smithi179 ceph-mon[93519]: osdmap e1368: 8 total, 8 up, 4 in 2024-08-17T18:12:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 2516/25251 objects degraded (9.964%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T18:12:01.911 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:01.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:01.933 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:01.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:02.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:02.056 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:02.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:02.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:02.158 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:02.179 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:02.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:02.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:02.281 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:02.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:02.304 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:02.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:02.405 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:02.427 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:02.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:02.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:02.529 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:02.554 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:02.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:02.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:02.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:02 smithi179 ceph-mon[93519]: osdmap e1369: 8 total, 8 up, 4 in 2024-08-17T18:12:02.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:02 smithi179 ceph-mon[93519]: pgmap v2492: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 5 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 502 MiB data, 4.0 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 583 op/s; 2324/24129 objects degraded (9.632%); 2427/24129 objects misplaced (10.058%); 6.0 MiB/s, 96 objects/s recovering 2024-08-17T18:12:02.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:02.852 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:02.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:02.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:02.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:02 smithi130 ceph-mon[110068]: osdmap e1369: 8 total, 8 up, 4 in 2024-08-17T18:12:02.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:02 smithi130 ceph-mon[110068]: pgmap v2492: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 5 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 502 MiB data, 4.0 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 583 op/s; 2324/24129 objects degraded (9.632%); 2427/24129 objects misplaced (10.058%); 6.0 MiB/s, 96 objects/s recovering 2024-08-17T18:12:02.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:02 smithi130 ceph-mon[105521]: osdmap e1369: 8 total, 8 up, 4 in 2024-08-17T18:12:02.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:02 smithi130 ceph-mon[105521]: pgmap v2492: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 5 active+undersized+degraded+remapped+backfill_wait, 79 active+clean; 502 MiB data, 4.0 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 583 op/s; 2324/24129 objects degraded (9.632%); 2427/24129 objects misplaced (10.058%); 6.0 MiB/s, 96 objects/s recovering 2024-08-17T18:12:02.953 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:02.975 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:02.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:02.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:03.077 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:03.100 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:03.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:03.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:03.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:03.225 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:03.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:03.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:03.327 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:03.348 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:03.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:03.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:03.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:03.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:03.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:03.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:03.574 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:03.607 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:03.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:03.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:03.709 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:03.733 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:03.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:03.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:03.835 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:03.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:03 smithi179 ceph-mon[93519]: osdmap e1370: 8 total, 8 up, 4 in 2024-08-17T18:12:03.856 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:03.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:03.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:03.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:03 smithi130 ceph-mon[105521]: osdmap e1370: 8 total, 8 up, 4 in 2024-08-17T18:12:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:03 smithi130 ceph-mon[110068]: osdmap e1370: 8 total, 8 up, 4 in 2024-08-17T18:12:03.958 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:03.981 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:03.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:03.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:03.992 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:04.083 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:04.106 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:04.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:04.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:04.207 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:04.230 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:04.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:04.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:04.331 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:04.353 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:04.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:04.455 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:04.476 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:04.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:04.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:04.577 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:04.612 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:04.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:04.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:04.697 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:04 smithi130 ceph-mon[105521]: osdmap e1371: 8 total, 8 up, 4 in 2024-08-17T18:12:04.697 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:04 smithi130 ceph-mon[105521]: pgmap v2495: 89 pgs: 1 peering, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 394 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 522 op/s; 1315/18975 objects degraded (6.930%); 1249/18975 objects misplaced (6.582%); 15 MiB/s, 235 objects/s recovering 2024-08-17T18:12:04.697 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:04 smithi130 ceph-mon[110068]: osdmap e1371: 8 total, 8 up, 4 in 2024-08-17T18:12:04.697 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:04 smithi130 ceph-mon[110068]: pgmap v2495: 89 pgs: 1 peering, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 394 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 522 op/s; 1315/18975 objects degraded (6.930%); 1249/18975 objects misplaced (6.582%); 15 MiB/s, 235 objects/s recovering 2024-08-17T18:12:04.714 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:04.738 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:04.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:04.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:04.840 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:04.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:04 smithi179 ceph-mon[93519]: osdmap e1371: 8 total, 8 up, 4 in 2024-08-17T18:12:04.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:04 smithi179 ceph-mon[93519]: pgmap v2495: 89 pgs: 1 peering, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 80 active+clean; 394 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 522 op/s; 1315/18975 objects degraded (6.930%); 1249/18975 objects misplaced (6.582%); 15 MiB/s, 235 objects/s recovering 2024-08-17T18:12:04.862 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:04.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:04.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:04.964 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:04.986 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:04.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:04.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:05.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:05.112 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:05.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:05.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:05.214 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:05.236 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:05.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:05.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:05.338 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:05.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:05.361 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:05.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:05.462 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:05.484 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:05.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:05.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:05.585 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:05.619 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:12:05.619 INFO:teuthology.orchestra.run.smithi130.stdout:{"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 1315/18975 objects degraded (6.930%), 5 pgs degraded, 5 pgs undersized","count":10},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2646,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1373,"num_osds":8,"num_up_osds":8,"osd_up_since":1723918167,"num_in_osds":4,"osd_in_since":1723918136,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":80},{"state_name":"active+undersized+degraded+remapped+backfill_wait","count":3},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+undersized+degraded+remapped+backfilling","count":2},{"state_name":"peering","count":1}],"num_pgs":89,"num_pools":5,"num_objects":6325,"data_bytes":413341669,"bytes_used":4222889984,"bytes_avail":379757252608,"bytes_total":383980142592,"inactive_pgs_ratio":0.01123595517128706,"degraded_objects":1315,"degraded_total":18975,"degraded_ratio":0.069301712779973643,"misplaced_objects":1249,"misplaced_total":18975,"misplaced_ratio":0.065823451910408434,"recovering_objects_per_sec":235,"recovering_bytes_per_sec":15425539,"recovering_keys_per_sec":0,"num_objects_recovered":943,"num_bytes_recovered":61734912,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":522},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi130.jmaidg","status":"up:active","gid":24548}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.130:8443/","prometheus":"http://172.21.15.130:9283/"}},"servicemap":{"epoch":32,"modified":"2024-08-17T18:10:27.432406+0000","services":{}},"progress_events":{"4861586d-5402-4f10-93bc-62f8c4bfdc13":{"message":"Global Recovery Event (6m)\n [=========================...] (remaining: 40s)","progress":0.89887642860412598,"add_to_ceph_s":true},"b7320a31-afc9-4864-88db-9352561792d2":{"message":"Upgrade to 19.0.0-5836-gcd8bdbeb (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2024-08-17T18:12:05.619 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:05 smithi130 ceph-mon[110068]: osdmap e1372: 8 total, 8 up, 4 in 2024-08-17T18:12:05.619 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:05 smithi130 ceph-mon[105521]: osdmap e1372: 8 total, 8 up, 4 in 2024-08-17T18:12:05.622 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:05.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:05.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:05.725 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:05.747 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:05.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:05.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:05.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:05 smithi179 ceph-mon[93519]: osdmap e1372: 8 total, 8 up, 4 in 2024-08-17T18:12:05.849 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:05.870 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:05.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:05.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:05.972 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:05.993 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:05.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:05.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:06.095 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:06.135 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:06.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:06.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:06.239 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:06.260 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:06.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:06.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:06.362 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:06.391 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:06.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:06.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:06.493 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:06.514 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:06.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:06.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:06.616 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:06.637 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:06.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:06.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:06.739 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:06.760 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:06.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:06.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:06 smithi179 ceph-mon[93519]: osdmap e1373: 8 total, 8 up, 4 in 2024-08-17T18:12:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:06 smithi179 ceph-mon[93519]: pgmap v2498: 89 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 81 active+clean; 373 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 514 op/s; 1219/17952 objects degraded (6.790%); 1146/17952 objects misplaced (6.384%); 16 MiB/s, 259 objects/s recovering 2024-08-17T18:12:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:06 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1420144037' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T18:12:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:06 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:06 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 1315/18975 objects degraded (6.930%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T18:12:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:06 smithi179 ceph-mon[93519]: osdmap e1374: 8 total, 8 up, 4 in 2024-08-17T18:12:06.862 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:06.883 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:06.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:06.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:06 smithi130 ceph-mon[105521]: osdmap e1373: 8 total, 8 up, 4 in 2024-08-17T18:12:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:06 smithi130 ceph-mon[105521]: pgmap v2498: 89 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 81 active+clean; 373 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 514 op/s; 1219/17952 objects degraded (6.790%); 1146/17952 objects misplaced (6.384%); 16 MiB/s, 259 objects/s recovering 2024-08-17T18:12:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:06 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1420144037' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T18:12:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:06 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:06 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 1315/18975 objects degraded (6.930%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T18:12:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:06 smithi130 ceph-mon[105521]: osdmap e1374: 8 total, 8 up, 4 in 2024-08-17T18:12:06.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:06 smithi130 ceph-mon[110068]: osdmap e1373: 8 total, 8 up, 4 in 2024-08-17T18:12:06.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:06 smithi130 ceph-mon[110068]: pgmap v2498: 89 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 81 active+clean; 373 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 514 op/s; 1219/17952 objects degraded (6.790%); 1146/17952 objects misplaced (6.384%); 16 MiB/s, 259 objects/s recovering 2024-08-17T18:12:06.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:06 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1420144037' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T18:12:06.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:06 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:06.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:06 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:06.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 1315/18975 objects degraded (6.930%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T18:12:06.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:06 smithi130 ceph-mon[110068]: osdmap e1374: 8 total, 8 up, 4 in 2024-08-17T18:12:06.984 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:07.007 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:07.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:07.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:07.109 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:07.130 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:07.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:07.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:07.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:07.262 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:07.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:07.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:07.363 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:07.366 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T18:12:07.366 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:12:07.384 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:07.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:07.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:07.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:07.509 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:07.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:07.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:07.611 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:07.632 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:07.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:07.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:07.733 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:07.755 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:07.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:07.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:07.857 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:07.879 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:07.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:07.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:07.981 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:08.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:08.004 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:08.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:08.105 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:08.127 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:08.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:08.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:08.229 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:08.251 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:08.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:08.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:08.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:08 smithi130 ceph-mon[105521]: osdmap e1375: 8 total, 8 up, 4 in 2024-08-17T18:12:08.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:12:08.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:12:08.311 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:08 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:08.311 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:08 smithi130 ceph-mon[110068]: osdmap e1375: 8 total, 8 up, 4 in 2024-08-17T18:12:08.311 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:12:08.311 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:12:08.311 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:08 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:08.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:08 smithi179 ceph-mon[93519]: osdmap e1375: 8 total, 8 up, 4 in 2024-08-17T18:12:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:12:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:12:08.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:08 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:08.353 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:08.376 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:08.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:08.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:08.478 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:08.502 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:08.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:08.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:08.604 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:08.642 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:08.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:08.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:08.744 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:08.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:08.780 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:08.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:08.881 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:08.929 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:08.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:08.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:09.026 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:09 smithi130 ceph-mon[105521]: pgmap v2501: 89 pgs: 3 active+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfill_wait, 84 active+clean; 335 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 236 op/s; 718/16137 objects degraded (4.449%); 1093/16137 objects misplaced (6.773%); 8.3 MiB/s, 132 objects/s recovering 2024-08-17T18:12:09.027 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:09 smithi130 ceph-mon[105521]: osdmap e1376: 8 total, 8 up, 4 in 2024-08-17T18:12:09.027 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:09 smithi130 ceph-mon[110068]: pgmap v2501: 89 pgs: 3 active+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfill_wait, 84 active+clean; 335 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 236 op/s; 718/16137 objects degraded (4.449%); 1093/16137 objects misplaced (6.773%); 8.3 MiB/s, 132 objects/s recovering 2024-08-17T18:12:09.027 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:09 smithi130 ceph-mon[110068]: osdmap e1376: 8 total, 8 up, 4 in 2024-08-17T18:12:09.031 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:09.054 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:09.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:09.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:09.156 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:09.178 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:09.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:09.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:09.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:09.301 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:09.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:09.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:09.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:09 smithi179 ceph-mon[93519]: pgmap v2501: 89 pgs: 3 active+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfill_wait, 84 active+clean; 335 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 236 op/s; 718/16137 objects degraded (4.449%); 1093/16137 objects misplaced (6.773%); 8.3 MiB/s, 132 objects/s recovering 2024-08-17T18:12:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:09 smithi179 ceph-mon[93519]: osdmap e1376: 8 total, 8 up, 4 in 2024-08-17T18:12:09.403 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:09.441 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:09.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:09.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:09.479 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:09.543 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:09.566 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:09.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:09.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:09.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:12:09.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:12:09.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:12:09.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:12:09.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:12:09.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:12:09.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:12:09.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:12:09.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:12:09.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:12:09.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:12:09.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:12:09.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:12:09.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:12:09.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:12:09.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:12:09.667 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:09.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:09.714 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:09.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:09.815 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:09.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:09.863 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:09.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:09.964 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:10.004 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:10.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:10.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:10.106 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:10.145 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:10.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:10.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:10.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:10 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3983473934' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:10.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:10 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3983473934' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:10.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:12:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:12:09] "GET /metrics HTTP/1.1" 200 36800 "" "Prometheus/2.51.0" 2024-08-17T18:12:10.248 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:10.271 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:10.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:10.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:10.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:10 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3983473934' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:10.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:12:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:12:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:12:10.373 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:10.396 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:10.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:10.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:10.409 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:10.497 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:10.520 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:10.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:10.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:10.622 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:10.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:10.647 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:10.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:10.748 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:10.769 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:10.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:10.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:10.871 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:10.910 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:10.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:10.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:11.011 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:11.034 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:11.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:11.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:11.136 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:11.176 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:11.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:11.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:11.278 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:11.279 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:11 smithi130 ceph-mon[105521]: pgmap v2503: 89 pgs: 89 active+clean; 295 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 309 op/s; 33 MiB/s, 526 objects/s recovering 2024-08-17T18:12:11.279 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:11 smithi130 ceph-mon[105521]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 718/16137 objects degraded (4.449%), 2 pgs degraded, 2 pgs undersized) 2024-08-17T18:12:11.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:11 smithi130 ceph-mon[110068]: pgmap v2503: 89 pgs: 89 active+clean; 295 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 309 op/s; 33 MiB/s, 526 objects/s recovering 2024-08-17T18:12:11.279 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:11 smithi130 ceph-mon[110068]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 718/16137 objects degraded (4.449%), 2 pgs degraded, 2 pgs undersized) 2024-08-17T18:12:11.330 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:11.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:11.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:11.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:11 smithi179 ceph-mon[93519]: pgmap v2503: 89 pgs: 89 active+clean; 295 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 309 op/s; 33 MiB/s, 526 objects/s recovering 2024-08-17T18:12:11.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:11 smithi179 ceph-mon[93519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 718/16137 objects degraded (4.449%), 2 pgs degraded, 2 pgs undersized) 2024-08-17T18:12:11.431 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:11.467 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:11.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:11.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:11.569 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:11.592 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:11.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:11.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:11.694 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:11.742 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:11.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:11.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:11.844 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:11.866 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:11.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:11.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:11.968 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:12.011 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:12.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:12.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:12.031 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:12:12.113 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:12.135 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:12.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:12.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:12.237 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:12.276 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:12.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:12.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:12.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:12 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4033086376' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:12.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:12 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:12.355 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:12 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4033086376' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:12.355 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:12 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:12.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:12 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4033086376' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:12.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:12 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:12.378 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:12.400 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:12.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:12.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:12.452 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:12:12.502 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:12.523 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:12.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:12.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:12.625 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:12.661 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Removed 131962 objects 2024-08-17T18:12:12.661 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Clean up completed and total clean up time :179.174 2024-08-17T18:12:12.663 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:12.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:12.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:12.766 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:12.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:12.788 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:12.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:12.889 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:12.910 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:12.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:12.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:13.012 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:13.033 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:13.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:13.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:13.135 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:13.157 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:13.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:13.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:13.259 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:13.281 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:13.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:13.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:13.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:13 smithi179 ceph-mon[93519]: pgmap v2504: 89 pgs: 89 active+clean; 284 MiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 237 op/s; 22 MiB/s, 353 objects/s recovering 2024-08-17T18:12:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:13 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:12:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:13 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:12:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:13 smithi179 ceph-mon[93519]: osdmap e1377: 8 total, 8 up, 4 in 2024-08-17T18:12:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:13.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:12:13.382 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:13.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:13.405 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:13.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:13.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:13 smithi130 ceph-mon[110068]: pgmap v2504: 89 pgs: 89 active+clean; 284 MiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 237 op/s; 22 MiB/s, 353 objects/s recovering 2024-08-17T18:12:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:13 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:12:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:13 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:12:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:13 smithi130 ceph-mon[110068]: osdmap e1377: 8 total, 8 up, 4 in 2024-08-17T18:12:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:12:13.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:13 smithi130 ceph-mon[105521]: pgmap v2504: 89 pgs: 89 active+clean; 284 MiB data, 3.7 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 237 op/s; 22 MiB/s, 353 objects/s recovering 2024-08-17T18:12:13.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:13 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:12:13.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:13 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:12:13.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:13 smithi130 ceph-mon[105521]: osdmap e1377: 8 total, 8 up, 4 in 2024-08-17T18:12:13.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:13.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:12:13.506 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:13.527 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:13.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:13.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:13.629 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:13.650 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:13.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:13.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:13.752 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:13.774 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:13.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:13.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:13.876 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:13.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:13.898 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:13.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:13.999 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:14.021 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:14.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:14.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:14.123 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:14.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:14.146 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:14.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:14.248 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:14.269 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:14.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:14.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:14.371 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:14.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:14.394 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:14.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:14.495 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:14.517 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:14.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:14.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:14.619 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:14.641 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:14.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:14.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:14.743 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:14.766 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:14.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:14.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:14.867 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:14.890 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:14.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:14.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:14.992 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:15.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:15.016 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:15.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:15.117 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:15.138 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:15.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:15.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:15.242 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:15.263 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:15.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:15.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:15.306 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:15 smithi130 ceph-mon[105521]: pgmap v2506: 89 pgs: 89 active+clean; 55 MiB data, 3.2 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 784 op/s; 20 MiB/s, 327 objects/s recovering 2024-08-17T18:12:15.306 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:15 smithi130 ceph-mon[110068]: pgmap v2506: 89 pgs: 89 active+clean; 55 MiB data, 3.2 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 784 op/s; 20 MiB/s, 327 objects/s recovering 2024-08-17T18:12:15.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:15 smithi179 ceph-mon[93519]: pgmap v2506: 89 pgs: 89 active+clean; 55 MiB data, 3.2 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 784 op/s; 20 MiB/s, 327 objects/s recovering 2024-08-17T18:12:15.365 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:15.380 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:12:15.381 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:12:15.391 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:15.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:15.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:15.493 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:15.518 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:15.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:15.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:15.530 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:15.620 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:15.641 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:15.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:15.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:15.743 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:15.768 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:15.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:15.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:15.870 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:15.892 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:15.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:15.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:15.948 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:12:15.994 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:16.016 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:16.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:16.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:16.118 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:16.140 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:16.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:16.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:16.242 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:16.263 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:16.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:16.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:16.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:16 smithi179 ceph-mon[93519]: from='client.46790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:12:16.348 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:16 smithi130 ceph-mon[105521]: from='client.46790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:12:16.348 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:16 smithi130 ceph-mon[110068]: from='client.46790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:12:16.366 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:16.392 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:16.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:16.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:16.494 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:16.516 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:16.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:16.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:16.617 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:16.643 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:16.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:16.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:16.706 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2506,"stamp":"2024-08-17T18:12:13.481421+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":57546701,"num_objects":895,"num_object_clones":0,"num_object_copies":2685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":895,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":263335,"num_write_kb":8448016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18685,"num_bytes_recovered":1224357285,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":70006,"ondisk_log_size":70006,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":3403060,"kb_used_data":395716,"kb_used_omap":294,"kb_used_meta":3007001,"kb_avail":371577548,"statfs":{"total":383980142592,"available":380495409152,"internally_reserved":0,"allocated":405213184,"data_stored":386883713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":301759,"internal_metadata":3079169345},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,7,4,12,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-333447192,"num_objects":-5089,"num_object_clones":0,"num_object_copies":-15267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1219,"num_objects_misplaced":-1146,"num_objects_unfound":0,"num_objects_dirty":-5089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5089,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2124,"num_bytes_recovered":139132952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.486782"},"pg_stats":[{"pgid":"48.f","version":"1377'16520","reported_seq":27827,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.123447+0000","last_change":"2024-08-17T18:12:04.482023+0000","last_active":"2024-08-17T18:12:12.123447+0000","last_peered":"2024-08-17T18:12:12.123447+0000","last_clean":"2024-08-17T18:12:12.123447+0000","last_became_active":"2024-08-17T18:12:04.446485+0000","last_became_peered":"2024-08-17T18:12:04.446485+0000","last_unstale":"2024-08-17T18:12:12.123447+0000","last_undegraded":"2024-08-17T18:12:12.123447+0000","last_fullsized":"2024-08-17T18:12:12.123447+0000","mapping_epoch":1371,"log_start":"1352'12257","ondisk_log_start":"1352'12257","created":1327,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4263,"log_dups_size":0,"ondisk_log_size":4263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:35.189754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2228224,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16520,"num_write_kb":529728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5534,"num_bytes_recovered":362676224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"48.e","version":"1377'16438","reported_seq":20758,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.110113+0000","last_change":"2024-08-17T18:11:34.827100+0000","last_active":"2024-08-17T18:12:12.110113+0000","last_peered":"2024-08-17T18:12:12.110113+0000","last_clean":"2024-08-17T18:12:12.110113+0000","last_became_active":"2024-08-17T18:11:34.776143+0000","last_became_peered":"2024-08-17T18:11:34.776143+0000","last_unstale":"2024-08-17T18:12:12.110113+0000","last_undegraded":"2024-08-17T18:12:12.110113+0000","last_fullsized":"2024-08-17T18:12:12.110113+0000","mapping_epoch":1358,"log_start":"1351'12129","ondisk_log_start":"1351'12129","created":1327,"last_epoch_clean":1359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4309,"log_dups_size":0,"ondisk_log_size":4309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:56.638791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2228224,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16438,"num_write_kb":527104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2109,"num_bytes_recovered":138215424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"48.d","version":"1377'16404","reported_seq":17136,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.580021+0000","last_change":"2024-08-17T18:12:06.029504+0000","last_active":"2024-08-17T18:12:12.580021+0000","last_peered":"2024-08-17T18:12:12.580021+0000","last_clean":"2024-08-17T18:12:12.580021+0000","last_became_active":"2024-08-17T18:12:06.001995+0000","last_became_peered":"2024-08-17T18:12:06.001995+0000","last_unstale":"2024-08-17T18:12:12.580021+0000","last_undegraded":"2024-08-17T18:12:12.580021+0000","last_fullsized":"2024-08-17T18:12:12.580021+0000","mapping_epoch":1373,"log_start":"1351'12224","ondisk_log_start":"1351'12224","created":1327,"last_epoch_clean":1374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4180,"log_dups_size":0,"ondisk_log_size":4180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:57:51.421974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":16404,"num_write_kb":524865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":326,"num_bytes_recovered":21299224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.c","version":"1376'16324","reported_seq":19314,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.084779+0000","last_change":"2024-08-17T18:11:54.451754+0000","last_active":"2024-08-17T18:12:12.084779+0000","last_peered":"2024-08-17T18:12:12.084779+0000","last_clean":"2024-08-17T18:12:12.084779+0000","last_became_active":"2024-08-17T18:11:54.419946+0000","last_became_peered":"2024-08-17T18:11:54.419946+0000","last_unstale":"2024-08-17T18:12:12.084779+0000","last_undegraded":"2024-08-17T18:12:12.084779+0000","last_fullsized":"2024-08-17T18:12:12.084779+0000","mapping_epoch":1364,"log_start":"1351'12046","ondisk_log_start":"1351'12046","created":1327,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4278,"log_dups_size":0,"ondisk_log_size":4278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:27.096147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2359296,"num_objects":36,"num_object_clones":0,"num_object_copies":108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16324,"num_write_kb":523520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":959,"num_bytes_recovered":62849024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"48.b","version":"1377'16654","reported_seq":17481,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.659118+0000","last_change":"2024-08-17T18:12:05.486409+0000","last_active":"2024-08-17T18:12:12.659118+0000","last_peered":"2024-08-17T18:12:12.659118+0000","last_clean":"2024-08-17T18:12:12.659118+0000","last_became_active":"2024-08-17T18:12:05.452250+0000","last_became_peered":"2024-08-17T18:12:05.452250+0000","last_unstale":"2024-08-17T18:12:12.659118+0000","last_undegraded":"2024-08-17T18:12:12.659118+0000","last_fullsized":"2024-08-17T18:12:12.659118+0000","mapping_epoch":1372,"log_start":"1352'12464","ondisk_log_start":"1352'12464","created":1327,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4190,"log_dups_size":0,"ondisk_log_size":4190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:33:26.993068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16654,"num_write_kb":532928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":374,"num_bytes_recovered":24510464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.a","version":"1376'16233","reported_seq":16345,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:10.129233+0000","last_change":"2024-08-17T18:07:32.359061+0000","last_active":"2024-08-17T18:12:10.129233+0000","last_peered":"2024-08-17T18:12:10.129233+0000","last_clean":"2024-08-17T18:12:10.129233+0000","last_became_active":"2024-08-17T18:07:32.357107+0000","last_became_peered":"2024-08-17T18:07:32.357107+0000","last_unstale":"2024-08-17T18:12:10.129233+0000","last_undegraded":"2024-08-17T18:12:10.129233+0000","last_fullsized":"2024-08-17T18:12:10.129233+0000","mapping_epoch":1327,"log_start":"1349'11264","ondisk_log_start":"1349'11264","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4969,"log_dups_size":0,"ondisk_log_size":4969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:30:02.061787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9371648,"num_objects":143,"num_object_clones":0,"num_object_copies":429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16233,"num_write_kb":524032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"48.9","version":"1377'16906","reported_seq":18392,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.129965+0000","last_change":"2024-08-17T18:11:59.488847+0000","last_active":"2024-08-17T18:12:12.129965+0000","last_peered":"2024-08-17T18:12:12.129965+0000","last_clean":"2024-08-17T18:12:12.129965+0000","last_became_active":"2024-08-17T18:11:59.376052+0000","last_became_peered":"2024-08-17T18:11:59.376052+0000","last_unstale":"2024-08-17T18:12:12.129965+0000","last_undegraded":"2024-08-17T18:12:12.129965+0000","last_fullsized":"2024-08-17T18:12:12.129965+0000","mapping_epoch":1366,"log_start":"1352'12649","ondisk_log_start":"1352'12649","created":1327,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4257,"log_dups_size":0,"ondisk_log_size":4257,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:04:43.444213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1966080,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16906,"num_write_kb":541952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":691,"num_bytes_recovered":45219840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"48.8","version":"1376'16293","reported_seq":17139,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:09.119066+0000","last_change":"2024-08-17T18:12:06.033568+0000","last_active":"2024-08-17T18:12:09.119066+0000","last_peered":"2024-08-17T18:12:09.119066+0000","last_clean":"2024-08-17T18:12:09.119066+0000","last_became_active":"2024-08-17T18:12:06.002547+0000","last_became_peered":"2024-08-17T18:12:06.002547+0000","last_unstale":"2024-08-17T18:12:09.119066+0000","last_undegraded":"2024-08-17T18:12:09.119066+0000","last_fullsized":"2024-08-17T18:12:09.119066+0000","mapping_epoch":1373,"log_start":"1349'11361","ondisk_log_start":"1349'11361","created":1327,"last_epoch_clean":1374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4932,"log_dups_size":0,"ondisk_log_size":4932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:02:38.489800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14745600,"num_objects":225,"num_object_clones":0,"num_object_copies":675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16293,"num_write_kb":528576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":346,"num_bytes_recovered":22609920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.7","version":"1377'16484","reported_seq":17269,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.652316+0000","last_change":"2024-08-17T18:12:08.031694+0000","last_active":"2024-08-17T18:12:12.652316+0000","last_peered":"2024-08-17T18:12:12.652316+0000","last_clean":"2024-08-17T18:12:12.652316+0000","last_became_active":"2024-08-17T18:12:08.010290+0000","last_became_peered":"2024-08-17T18:12:08.010290+0000","last_unstale":"2024-08-17T18:12:12.652316+0000","last_undegraded":"2024-08-17T18:12:12.652316+0000","last_fullsized":"2024-08-17T18:12:12.652316+0000","mapping_epoch":1375,"log_start":"1352'12348","ondisk_log_start":"1352'12348","created":1327,"last_epoch_clean":1376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4136,"log_dups_size":0,"ondisk_log_size":4136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:55:20.162698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16484,"num_write_kb":527488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":341,"num_bytes_recovered":22347776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.6","version":"1377'16380","reported_seq":19033,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.127440+0000","last_change":"2024-08-17T18:12:02.507076+0000","last_active":"2024-08-17T18:12:12.127440+0000","last_peered":"2024-08-17T18:12:12.127440+0000","last_clean":"2024-08-17T18:12:12.127440+0000","last_became_active":"2024-08-17T18:12:02.428216+0000","last_became_peered":"2024-08-17T18:12:02.428216+0000","last_unstale":"2024-08-17T18:12:12.127440+0000","last_undegraded":"2024-08-17T18:12:12.127440+0000","last_fullsized":"2024-08-17T18:12:12.127440+0000","mapping_epoch":1369,"log_start":"1351'12079","ondisk_log_start":"1351'12079","created":1327,"last_epoch_clean":1370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4301,"log_dups_size":0,"ondisk_log_size":4301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:12:45.279166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2228224,"num_objects":34,"num_object_clones":0,"num_object_copies":102,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":34,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16380,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":837,"num_bytes_recovered":54853632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"48.5","version":"1377'16354","reported_seq":17214,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.122682+0000","last_change":"2024-08-17T18:12:03.465642+0000","last_active":"2024-08-17T18:12:12.122682+0000","last_peered":"2024-08-17T18:12:12.122682+0000","last_clean":"2024-08-17T18:12:12.122682+0000","last_became_active":"2024-08-17T18:12:03.429464+0000","last_became_peered":"2024-08-17T18:12:03.429464+0000","last_unstale":"2024-08-17T18:12:12.122682+0000","last_undegraded":"2024-08-17T18:12:12.122682+0000","last_fullsized":"2024-08-17T18:12:12.122682+0000","mapping_epoch":1370,"log_start":"1351'12087","ondisk_log_start":"1351'12087","created":1327,"last_epoch_clean":1371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4267,"log_dups_size":0,"ondisk_log_size":4267,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:10.780073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2490368,"num_objects":38,"num_object_clones":0,"num_object_copies":114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16354,"num_write_kb":524544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":380,"num_bytes_recovered":24903680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"48.4","version":"1377'16378","reported_seq":19135,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.090782+0000","last_change":"2024-08-17T18:11:46.398030+0000","last_active":"2024-08-17T18:12:12.090782+0000","last_peered":"2024-08-17T18:12:12.090782+0000","last_clean":"2024-08-17T18:12:12.090782+0000","last_became_active":"2024-08-17T18:11:46.364183+0000","last_became_peered":"2024-08-17T18:11:46.364183+0000","last_unstale":"2024-08-17T18:12:12.090782+0000","last_undegraded":"2024-08-17T18:12:12.090782+0000","last_fullsized":"2024-08-17T18:12:12.090782+0000","mapping_epoch":1361,"log_start":"1351'12144","ondisk_log_start":"1351'12144","created":1327,"last_epoch_clean":1362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4234,"log_dups_size":0,"ondisk_log_size":4234,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:25:19.771405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2359296,"num_objects":36,"num_object_clones":0,"num_object_copies":108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16378,"num_write_kb":525248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1316,"num_bytes_recovered":86245376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"48.3","version":"1376'16217","reported_seq":17299,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:09.119565+0000","last_change":"2024-08-17T18:12:00.412843+0000","last_active":"2024-08-17T18:12:09.119565+0000","last_peered":"2024-08-17T18:12:09.119565+0000","last_clean":"2024-08-17T18:12:09.119565+0000","last_became_active":"2024-08-17T18:12:00.376839+0000","last_became_peered":"2024-08-17T18:12:00.376839+0000","last_unstale":"2024-08-17T18:12:09.119565+0000","last_undegraded":"2024-08-17T18:12:09.119565+0000","last_fullsized":"2024-08-17T18:12:09.119565+0000","mapping_epoch":1367,"log_start":"1349'11323","ondisk_log_start":"1349'11323","created":1327,"last_epoch_clean":1368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4894,"log_dups_size":0,"ondisk_log_size":4894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:34:08.799386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14483456,"num_objects":221,"num_object_clones":0,"num_object_copies":663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16217,"num_write_kb":526016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":480,"num_bytes_recovered":31457280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"48.2","version":"1377'16386","reported_seq":30427,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.106202+0000","last_change":"2024-08-17T18:11:09.658283+0000","last_active":"2024-08-17T18:12:12.106202+0000","last_peered":"2024-08-17T18:12:12.106202+0000","last_clean":"2024-08-17T18:12:12.106202+0000","last_became_active":"2024-08-17T18:11:09.615907+0000","last_became_peered":"2024-08-17T18:11:09.615907+0000","last_unstale":"2024-08-17T18:12:12.106202+0000","last_undegraded":"2024-08-17T18:12:12.106202+0000","last_fullsized":"2024-08-17T18:12:12.106202+0000","mapping_epoch":1355,"log_start":"1351'12073","ondisk_log_start":"1351'12073","created":1327,"last_epoch_clean":1356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4313,"log_dups_size":0,"ondisk_log_size":4313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:04.549996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2490368,"num_objects":38,"num_object_clones":0,"num_object_copies":114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16386,"num_write_kb":525568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4640,"num_bytes_recovered":303955968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"48.1","version":"1377'16688","reported_seq":16746,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.649722+0000","last_change":"2024-08-17T18:07:32.360312+0000","last_active":"2024-08-17T18:12:12.649722+0000","last_peered":"2024-08-17T18:12:12.649722+0000","last_clean":"2024-08-17T18:12:12.649722+0000","last_became_active":"2024-08-17T18:07:32.360092+0000","last_became_peered":"2024-08-17T18:07:32.360092+0000","last_unstale":"2024-08-17T18:12:12.649722+0000","last_undegraded":"2024-08-17T18:12:12.649722+0000","last_fullsized":"2024-08-17T18:12:12.649722+0000","mapping_epoch":1327,"log_start":"1352'12522","ondisk_log_start":"1352'12522","created":1327,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4166,"log_dups_size":0,"ondisk_log_size":4166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:41.580676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16688,"num_write_kb":534016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"48.0","version":"1377'16398","reported_seq":17127,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.622846+0000","last_change":"2024-08-17T18:12:07.020907+0000","last_active":"2024-08-17T18:12:12.622846+0000","last_peered":"2024-08-17T18:12:12.622846+0000","last_clean":"2024-08-17T18:12:12.622846+0000","last_became_active":"2024-08-17T18:12:06.999766+0000","last_became_peered":"2024-08-17T18:12:06.999766+0000","last_unstale":"2024-08-17T18:12:12.622846+0000","last_undegraded":"2024-08-17T18:12:12.622846+0000","last_fullsized":"2024-08-17T18:12:12.622846+0000","mapping_epoch":1374,"log_start":"1352'12288","ondisk_log_start":"1352'12288","created":1327,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:07:31.337586+0000","last_clean_scrub_stamp":"2024-08-17T18:07:31.337586+0000","objects_scrubbed":0,"log_size":4110,"log_dups_size":0,"ondisk_log_size":4110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:01:14.915650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16398,"num_write_kb":524736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":310,"num_bytes_recovered":20250624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":2241,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036629+0000","last_change":"2024-08-17T18:09:28.870033+0000","last_active":"2024-08-17T18:12:08.036629+0000","last_peered":"2024-08-17T18:12:08.036629+0000","last_clean":"2024-08-17T18:12:08.036629+0000","last_became_active":"2024-08-17T18:09:28.869346+0000","last_became_peered":"2024-08-17T18:09:28.869346+0000","last_unstale":"2024-08-17T18:12:08.036629+0000","last_undegraded":"2024-08-17T18:12:08.036629+0000","last_fullsized":"2024-08-17T18:12:08.036629+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:23.168631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":538,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.037230+0000","last_change":"2024-08-17T18:09:29.841584+0000","last_active":"2024-08-17T18:12:12.037230+0000","last_peered":"2024-08-17T18:12:12.037230+0000","last_clean":"2024-08-17T18:12:12.037230+0000","last_became_active":"2024-08-17T18:09:29.840950+0000","last_became_peered":"2024-08-17T18:09:29.840950+0000","last_unstale":"2024-08-17T18:12:12.037230+0000","last_undegraded":"2024-08-17T18:12:12.037230+0000","last_fullsized":"2024-08-17T18:12:12.037230+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:43.035959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":2033,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.084981+0000","last_change":"2024-08-17T18:09:29.883587+0000","last_active":"2024-08-17T18:12:12.084981+0000","last_peered":"2024-08-17T18:12:12.084981+0000","last_clean":"2024-08-17T18:12:12.084981+0000","last_became_active":"2024-08-17T18:09:29.882222+0000","last_became_peered":"2024-08-17T18:09:29.882222+0000","last_unstale":"2024-08-17T18:12:12.084981+0000","last_undegraded":"2024-08-17T18:12:12.084981+0000","last_fullsized":"2024-08-17T18:12:12.084981+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:10.777993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2090,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.039240+0000","last_change":"2024-08-17T18:09:28.858824+0000","last_active":"2024-08-17T18:12:12.039240+0000","last_peered":"2024-08-17T18:12:12.039240+0000","last_clean":"2024-08-17T18:12:12.039240+0000","last_became_active":"2024-08-17T18:09:28.857856+0000","last_became_peered":"2024-08-17T18:09:28.857856+0000","last_unstale":"2024-08-17T18:12:12.039240+0000","last_undegraded":"2024-08-17T18:12:12.039240+0000","last_fullsized":"2024-08-17T18:12:12.039240+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:41:35.146166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":94,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.038219+0000","last_change":"2024-08-17T18:09:29.851515+0000","last_active":"2024-08-17T18:12:12.038219+0000","last_peered":"2024-08-17T18:12:12.038219+0000","last_clean":"2024-08-17T18:12:12.038219+0000","last_became_active":"2024-08-17T18:09:29.851109+0000","last_became_peered":"2024-08-17T18:09:29.851109+0000","last_unstale":"2024-08-17T18:12:12.038219+0000","last_undegraded":"2024-08-17T18:12:12.038219+0000","last_fullsized":"2024-08-17T18:12:12.038219+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:42.054532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":89,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.008434+0000","last_change":"2024-08-17T18:03:16.089811+0000","last_active":"2024-08-17T18:12:08.008434+0000","last_peered":"2024-08-17T18:12:08.008434+0000","last_clean":"2024-08-17T18:12:08.008434+0000","last_became_active":"2024-08-17T18:03:16.089571+0000","last_became_peered":"2024-08-17T18:03:16.089571+0000","last_unstale":"2024-08-17T18:12:08.008434+0000","last_undegraded":"2024-08-17T18:12:08.008434+0000","last_fullsized":"2024-08-17T18:12:08.008434+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:04.630361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":71,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.008869+0000","last_change":"2024-08-17T18:08:57.577055+0000","last_active":"2024-08-17T18:12:08.008869+0000","last_peered":"2024-08-17T18:12:08.008869+0000","last_clean":"2024-08-17T18:12:08.008869+0000","last_became_active":"2024-08-17T18:08:57.573659+0000","last_became_peered":"2024-08-17T18:08:57.573659+0000","last_unstale":"2024-08-17T18:12:08.008869+0000","last_undegraded":"2024-08-17T18:12:08.008869+0000","last_fullsized":"2024-08-17T18:12:08.008869+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:01.686752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":878,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.040608+0000","last_change":"2024-08-17T18:09:29.852393+0000","last_active":"2024-08-17T18:12:12.040608+0000","last_peered":"2024-08-17T18:12:12.040608+0000","last_clean":"2024-08-17T18:12:12.040608+0000","last_became_active":"2024-08-17T18:09:29.851947+0000","last_became_peered":"2024-08-17T18:09:29.851947+0000","last_unstale":"2024-08-17T18:12:12.040608+0000","last_undegraded":"2024-08-17T18:12:12.040608+0000","last_fullsized":"2024-08-17T18:12:12.040608+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:49.145327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":2186,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.037094+0000","last_change":"2024-08-17T18:09:29.863241+0000","last_active":"2024-08-17T18:12:08.037094+0000","last_peered":"2024-08-17T18:12:08.037094+0000","last_clean":"2024-08-17T18:12:08.037094+0000","last_became_active":"2024-08-17T18:09:29.862369+0000","last_became_peered":"2024-08-17T18:09:29.862369+0000","last_unstale":"2024-08-17T18:12:08.037094+0000","last_undegraded":"2024-08-17T18:12:08.037094+0000","last_fullsized":"2024-08-17T18:12:08.037094+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:30.518551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":625,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.044190+0000","last_change":"2024-08-17T18:09:29.896233+0000","last_active":"2024-08-17T18:12:12.044190+0000","last_peered":"2024-08-17T18:12:12.044190+0000","last_clean":"2024-08-17T18:12:12.044190+0000","last_became_active":"2024-08-17T18:09:29.895718+0000","last_became_peered":"2024-08-17T18:09:29.895718+0000","last_unstale":"2024-08-17T18:12:12.044190+0000","last_undegraded":"2024-08-17T18:12:12.044190+0000","last_fullsized":"2024-08-17T18:12:12.044190+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:03.278878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":96,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036210+0000","last_change":"2024-08-17T18:08:57.574015+0000","last_active":"2024-08-17T18:12:08.036210+0000","last_peered":"2024-08-17T18:12:08.036210+0000","last_clean":"2024-08-17T18:12:08.036210+0000","last_became_active":"2024-08-17T18:08:57.567973+0000","last_became_peered":"2024-08-17T18:08:57.567973+0000","last_unstale":"2024-08-17T18:12:08.036210+0000","last_undegraded":"2024-08-17T18:12:08.036210+0000","last_fullsized":"2024-08-17T18:12:08.036210+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:06.251764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2045,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.038347+0000","last_change":"2024-08-17T18:09:29.917203+0000","last_active":"2024-08-17T18:12:12.038347+0000","last_peered":"2024-08-17T18:12:12.038347+0000","last_clean":"2024-08-17T18:12:12.038347+0000","last_became_active":"2024-08-17T18:09:29.916844+0000","last_became_peered":"2024-08-17T18:09:29.916844+0000","last_unstale":"2024-08-17T18:12:12.038347+0000","last_undegraded":"2024-08-17T18:12:12.038347+0000","last_fullsized":"2024-08-17T18:12:12.038347+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.453720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1951,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.039008+0000","last_change":"2024-08-17T18:09:28.859255+0000","last_active":"2024-08-17T18:12:12.039008+0000","last_peered":"2024-08-17T18:12:12.039008+0000","last_clean":"2024-08-17T18:12:12.039008+0000","last_became_active":"2024-08-17T18:09:28.858537+0000","last_became_peered":"2024-08-17T18:09:28.858537+0000","last_unstale":"2024-08-17T18:12:12.039008+0000","last_undegraded":"2024-08-17T18:12:12.039008+0000","last_fullsized":"2024-08-17T18:12:12.039008+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:15.364687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2209,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036632+0000","last_change":"2024-08-17T18:09:28.868303+0000","last_active":"2024-08-17T18:12:08.036632+0000","last_peered":"2024-08-17T18:12:08.036632+0000","last_clean":"2024-08-17T18:12:08.036632+0000","last_became_active":"2024-08-17T18:09:28.867989+0000","last_became_peered":"2024-08-17T18:09:28.867989+0000","last_unstale":"2024-08-17T18:12:08.036632+0000","last_undegraded":"2024-08-17T18:12:08.036632+0000","last_fullsized":"2024-08-17T18:12:08.036632+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:58:52.741685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":77,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.037013+0000","last_change":"2024-08-17T18:09:29.842129+0000","last_active":"2024-08-17T18:12:12.037013+0000","last_peered":"2024-08-17T18:12:12.037013+0000","last_clean":"2024-08-17T18:12:12.037013+0000","last_became_active":"2024-08-17T18:09:29.841848+0000","last_became_peered":"2024-08-17T18:09:29.841848+0000","last_unstale":"2024-08-17T18:12:12.037013+0000","last_undegraded":"2024-08-17T18:12:12.037013+0000","last_fullsized":"2024-08-17T18:12:12.037013+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:02.119134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1323,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036472+0000","last_change":"2024-08-17T18:09:29.846509+0000","last_active":"2024-08-17T18:12:08.036472+0000","last_peered":"2024-08-17T18:12:08.036472+0000","last_clean":"2024-08-17T18:12:08.036472+0000","last_became_active":"2024-08-17T18:09:29.844097+0000","last_became_peered":"2024-08-17T18:09:29.844097+0000","last_unstale":"2024-08-17T18:12:08.036472+0000","last_undegraded":"2024-08-17T18:12:08.036472+0000","last_fullsized":"2024-08-17T18:12:08.036472+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:32.340354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2039,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.037103+0000","last_change":"2024-08-17T18:09:29.841335+0000","last_active":"2024-08-17T18:12:12.037103+0000","last_peered":"2024-08-17T18:12:12.037103+0000","last_clean":"2024-08-17T18:12:12.037103+0000","last_became_active":"2024-08-17T18:09:29.840522+0000","last_became_peered":"2024-08-17T18:09:29.840522+0000","last_unstale":"2024-08-17T18:12:12.037103+0000","last_undegraded":"2024-08-17T18:12:12.037103+0000","last_fullsized":"2024-08-17T18:12:12.037103+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:08.015836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2214,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036696+0000","last_change":"2024-08-17T18:09:29.847494+0000","last_active":"2024-08-17T18:12:08.036696+0000","last_peered":"2024-08-17T18:12:08.036696+0000","last_clean":"2024-08-17T18:12:08.036696+0000","last_became_active":"2024-08-17T18:09:29.846973+0000","last_became_peered":"2024-08-17T18:09:29.846973+0000","last_unstale":"2024-08-17T18:12:08.036696+0000","last_undegraded":"2024-08-17T18:12:08.036696+0000","last_fullsized":"2024-08-17T18:12:08.036696+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:49.469921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1998,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.036747+0000","last_change":"2024-08-17T18:09:28.855045+0000","last_active":"2024-08-17T18:12:12.036747+0000","last_peered":"2024-08-17T18:12:12.036747+0000","last_clean":"2024-08-17T18:12:12.036747+0000","last_became_active":"2024-08-17T18:09:28.854607+0000","last_became_peered":"2024-08-17T18:09:28.854607+0000","last_unstale":"2024-08-17T18:12:12.036747+0000","last_undegraded":"2024-08-17T18:12:12.036747+0000","last_fullsized":"2024-08-17T18:12:12.036747+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:13.437895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":701,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.009037+0000","last_change":"2024-08-17T18:09:29.866944+0000","last_active":"2024-08-17T18:12:08.009037+0000","last_peered":"2024-08-17T18:12:08.009037+0000","last_clean":"2024-08-17T18:12:08.009037+0000","last_became_active":"2024-08-17T18:09:29.866694+0000","last_became_peered":"2024-08-17T18:09:29.866694+0000","last_unstale":"2024-08-17T18:12:08.009037+0000","last_undegraded":"2024-08-17T18:12:08.009037+0000","last_fullsized":"2024-08-17T18:12:08.009037+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:00:41.342392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1364,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.009115+0000","last_change":"2024-08-17T18:09:28.884262+0000","last_active":"2024-08-17T18:12:08.009115+0000","last_peered":"2024-08-17T18:12:08.009115+0000","last_clean":"2024-08-17T18:12:08.009115+0000","last_became_active":"2024-08-17T18:09:28.883734+0000","last_became_peered":"2024-08-17T18:09:28.883734+0000","last_unstale":"2024-08-17T18:12:08.009115+0000","last_undegraded":"2024-08-17T18:12:08.009115+0000","last_fullsized":"2024-08-17T18:12:08.009115+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:10.735504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2099,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036060+0000","last_change":"2024-08-17T18:08:57.571995+0000","last_active":"2024-08-17T18:12:08.036060+0000","last_peered":"2024-08-17T18:12:08.036060+0000","last_clean":"2024-08-17T18:12:08.036060+0000","last_became_active":"2024-08-17T18:08:57.567342+0000","last_became_peered":"2024-08-17T18:08:57.567342+0000","last_unstale":"2024-08-17T18:12:08.036060+0000","last_undegraded":"2024-08-17T18:12:08.036060+0000","last_fullsized":"2024-08-17T18:12:08.036060+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:23.769111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":528,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.008801+0000","last_change":"2024-08-17T18:02:53.912837+0000","last_active":"2024-08-17T18:12:08.008801+0000","last_peered":"2024-08-17T18:12:08.008801+0000","last_clean":"2024-08-17T18:12:08.008801+0000","last_became_active":"2024-08-17T18:02:53.883152+0000","last_became_peered":"2024-08-17T18:02:53.883152+0000","last_unstale":"2024-08-17T18:12:08.008801+0000","last_undegraded":"2024-08-17T18:12:08.008801+0000","last_fullsized":"2024-08-17T18:12:08.008801+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:46.261612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":527,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.008949+0000","last_change":"2024-08-17T18:09:29.845126+0000","last_active":"2024-08-17T18:12:08.008949+0000","last_peered":"2024-08-17T18:12:08.008949+0000","last_clean":"2024-08-17T18:12:08.008949+0000","last_became_active":"2024-08-17T18:09:29.844836+0000","last_became_peered":"2024-08-17T18:09:29.844836+0000","last_unstale":"2024-08-17T18:12:08.008949+0000","last_undegraded":"2024-08-17T18:12:08.008949+0000","last_fullsized":"2024-08-17T18:12:08.008949+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:44.821894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1336,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.037150+0000","last_change":"2024-08-17T18:02:53.890198+0000","last_active":"2024-08-17T18:12:08.037150+0000","last_peered":"2024-08-17T18:12:08.037150+0000","last_clean":"2024-08-17T18:12:08.037150+0000","last_became_active":"2024-08-17T18:02:53.887007+0000","last_became_peered":"2024-08-17T18:02:53.887007+0000","last_unstale":"2024-08-17T18:12:08.037150+0000","last_undegraded":"2024-08-17T18:12:08.037150+0000","last_fullsized":"2024-08-17T18:12:08.037150+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:25.555909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":618,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.009231+0000","last_change":"2024-08-17T18:09:28.884187+0000","last_active":"2024-08-17T18:12:08.009231+0000","last_peered":"2024-08-17T18:12:08.009231+0000","last_clean":"2024-08-17T18:12:08.009231+0000","last_became_active":"2024-08-17T18:09:28.883939+0000","last_became_peered":"2024-08-17T18:09:28.883939+0000","last_unstale":"2024-08-17T18:12:08.009231+0000","last_undegraded":"2024-08-17T18:12:08.009231+0000","last_fullsized":"2024-08-17T18:12:08.009231+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:37:41.951099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1918,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.009009+0000","last_change":"2024-08-17T18:09:29.893220+0000","last_active":"2024-08-17T18:12:08.009009+0000","last_peered":"2024-08-17T18:12:08.009009+0000","last_clean":"2024-08-17T18:12:08.009009+0000","last_became_active":"2024-08-17T18:09:29.865699+0000","last_became_peered":"2024-08-17T18:09:29.865699+0000","last_unstale":"2024-08-17T18:12:08.009009+0000","last_undegraded":"2024-08-17T18:12:08.009009+0000","last_fullsized":"2024-08-17T18:12:08.009009+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:40.205658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2058,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.037510+0000","last_change":"2024-08-17T18:09:28.934404+0000","last_active":"2024-08-17T18:12:12.037510+0000","last_peered":"2024-08-17T18:12:12.037510+0000","last_clean":"2024-08-17T18:12:12.037510+0000","last_became_active":"2024-08-17T18:09:28.934001+0000","last_became_peered":"2024-08-17T18:09:28.934001+0000","last_unstale":"2024-08-17T18:12:12.037510+0000","last_undegraded":"2024-08-17T18:12:12.037510+0000","last_fullsized":"2024-08-17T18:12:12.037510+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:04.676812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":2090,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.085201+0000","last_change":"2024-08-17T18:09:29.884088+0000","last_active":"2024-08-17T18:12:12.085201+0000","last_peered":"2024-08-17T18:12:12.085201+0000","last_clean":"2024-08-17T18:12:12.085201+0000","last_became_active":"2024-08-17T18:09:29.883027+0000","last_became_peered":"2024-08-17T18:09:29.883027+0000","last_unstale":"2024-08-17T18:12:12.085201+0000","last_undegraded":"2024-08-17T18:12:12.085201+0000","last_fullsized":"2024-08-17T18:12:12.085201+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:22:07.777188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":2060,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.038933+0000","last_change":"2024-08-17T18:09:28.935401+0000","last_active":"2024-08-17T18:12:12.038933+0000","last_peered":"2024-08-17T18:12:12.038933+0000","last_clean":"2024-08-17T18:12:12.038933+0000","last_became_active":"2024-08-17T18:09:28.933628+0000","last_became_peered":"2024-08-17T18:09:28.933628+0000","last_unstale":"2024-08-17T18:12:12.038933+0000","last_undegraded":"2024-08-17T18:12:12.038933+0000","last_fullsized":"2024-08-17T18:12:12.038933+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:21:12.714486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2170,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.037165+0000","last_change":"2024-08-17T18:08:57.574718+0000","last_active":"2024-08-17T18:12:08.037165+0000","last_peered":"2024-08-17T18:12:08.037165+0000","last_clean":"2024-08-17T18:12:08.037165+0000","last_became_active":"2024-08-17T18:08:57.573515+0000","last_became_peered":"2024-08-17T18:08:57.573515+0000","last_unstale":"2024-08-17T18:12:08.037165+0000","last_undegraded":"2024-08-17T18:12:08.037165+0000","last_fullsized":"2024-08-17T18:12:08.037165+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:04.873717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1377,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.009264+0000","last_change":"2024-08-17T18:08:57.568821+0000","last_active":"2024-08-17T18:12:08.009264+0000","last_peered":"2024-08-17T18:12:08.009264+0000","last_clean":"2024-08-17T18:12:08.009264+0000","last_became_active":"2024-08-17T18:08:57.568526+0000","last_became_peered":"2024-08-17T18:08:57.568526+0000","last_unstale":"2024-08-17T18:12:08.009264+0000","last_undegraded":"2024-08-17T18:12:08.009264+0000","last_fullsized":"2024-08-17T18:12:08.009264+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:26.449197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1890,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036234+0000","last_change":"2024-08-17T18:08:57.571174+0000","last_active":"2024-08-17T18:12:08.036234+0000","last_peered":"2024-08-17T18:12:08.036234+0000","last_clean":"2024-08-17T18:12:08.036234+0000","last_became_active":"2024-08-17T18:08:57.567571+0000","last_became_peered":"2024-08-17T18:08:57.567571+0000","last_unstale":"2024-08-17T18:12:08.036234+0000","last_undegraded":"2024-08-17T18:12:08.036234+0000","last_fullsized":"2024-08-17T18:12:08.036234+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:29.976150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1859,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036382+0000","last_change":"2024-08-17T18:08:57.572368+0000","last_active":"2024-08-17T18:12:08.036382+0000","last_peered":"2024-08-17T18:12:08.036382+0000","last_clean":"2024-08-17T18:12:08.036382+0000","last_became_active":"2024-08-17T18:08:57.568382+0000","last_became_peered":"2024-08-17T18:08:57.568382+0000","last_unstale":"2024-08-17T18:12:08.036382+0000","last_undegraded":"2024-08-17T18:12:08.036382+0000","last_fullsized":"2024-08-17T18:12:08.036382+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:11:19.968079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1907,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036337+0000","last_change":"2024-08-17T18:08:57.572991+0000","last_active":"2024-08-17T18:12:08.036337+0000","last_peered":"2024-08-17T18:12:08.036337+0000","last_clean":"2024-08-17T18:12:08.036337+0000","last_became_active":"2024-08-17T18:08:57.571145+0000","last_became_peered":"2024-08-17T18:08:57.571145+0000","last_unstale":"2024-08-17T18:12:08.036337+0000","last_undegraded":"2024-08-17T18:12:08.036337+0000","last_fullsized":"2024-08-17T18:12:08.036337+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:39.963337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":865,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.038129+0000","last_change":"2024-08-17T18:09:28.860288+0000","last_active":"2024-08-17T18:12:12.038129+0000","last_peered":"2024-08-17T18:12:12.038129+0000","last_clean":"2024-08-17T18:12:12.038129+0000","last_became_active":"2024-08-17T18:09:28.860004+0000","last_became_peered":"2024-08-17T18:09:28.860004+0000","last_unstale":"2024-08-17T18:12:12.038129+0000","last_undegraded":"2024-08-17T18:12:12.038129+0000","last_fullsized":"2024-08-17T18:12:12.038129+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:57:33.054055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":658,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.084558+0000","last_change":"2024-08-17T18:09:28.860277+0000","last_active":"2024-08-17T18:12:12.084558+0000","last_peered":"2024-08-17T18:12:12.084558+0000","last_clean":"2024-08-17T18:12:12.084558+0000","last_became_active":"2024-08-17T18:09:28.859999+0000","last_became_peered":"2024-08-17T18:09:28.859999+0000","last_unstale":"2024-08-17T18:12:12.084558+0000","last_undegraded":"2024-08-17T18:12:12.084558+0000","last_fullsized":"2024-08-17T18:12:12.084558+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:27.946526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1561,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036990+0000","last_change":"2024-08-17T18:09:29.863669+0000","last_active":"2024-08-17T18:12:08.036990+0000","last_peered":"2024-08-17T18:12:08.036990+0000","last_clean":"2024-08-17T18:12:08.036990+0000","last_became_active":"2024-08-17T18:09:29.862781+0000","last_became_peered":"2024-08-17T18:09:29.862781+0000","last_unstale":"2024-08-17T18:12:08.036990+0000","last_undegraded":"2024-08-17T18:12:08.036990+0000","last_fullsized":"2024-08-17T18:12:08.036990+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:14.073233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":142,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036470+0000","last_change":"2024-08-17T18:09:29.843574+0000","last_active":"2024-08-17T18:12:08.036470+0000","last_peered":"2024-08-17T18:12:08.036470+0000","last_clean":"2024-08-17T18:12:08.036470+0000","last_became_active":"2024-08-17T18:09:29.843097+0000","last_became_peered":"2024-08-17T18:09:29.843097+0000","last_unstale":"2024-08-17T18:12:08.036470+0000","last_undegraded":"2024-08-17T18:12:08.036470+0000","last_fullsized":"2024-08-17T18:12:08.036470+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:46.740888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1906,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036805+0000","last_change":"2024-08-17T18:09:29.847264+0000","last_active":"2024-08-17T18:12:08.036805+0000","last_peered":"2024-08-17T18:12:08.036805+0000","last_clean":"2024-08-17T18:12:08.036805+0000","last_became_active":"2024-08-17T18:09:29.846226+0000","last_became_peered":"2024-08-17T18:09:29.846226+0000","last_unstale":"2024-08-17T18:12:08.036805+0000","last_undegraded":"2024-08-17T18:12:08.036805+0000","last_fullsized":"2024-08-17T18:12:08.036805+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:44.135977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2106,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.035598+0000","last_change":"2024-08-17T18:09:29.845209+0000","last_active":"2024-08-17T18:12:08.035598+0000","last_peered":"2024-08-17T18:12:08.035598+0000","last_clean":"2024-08-17T18:12:08.035598+0000","last_became_active":"2024-08-17T18:09:29.844910+0000","last_became_peered":"2024-08-17T18:09:29.844910+0000","last_unstale":"2024-08-17T18:12:08.035598+0000","last_undegraded":"2024-08-17T18:12:08.035598+0000","last_fullsized":"2024-08-17T18:12:08.035598+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:14.586354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1981,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.039164+0000","last_change":"2024-08-17T18:09:28.859050+0000","last_active":"2024-08-17T18:12:12.039164+0000","last_peered":"2024-08-17T18:12:12.039164+0000","last_clean":"2024-08-17T18:12:12.039164+0000","last_became_active":"2024-08-17T18:09:28.858214+0000","last_became_peered":"2024-08-17T18:09:28.858214+0000","last_unstale":"2024-08-17T18:12:12.039164+0000","last_undegraded":"2024-08-17T18:12:12.039164+0000","last_fullsized":"2024-08-17T18:12:12.039164+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:54.146592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2230,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036786+0000","last_change":"2024-08-17T18:08:57.572610+0000","last_active":"2024-08-17T18:12:08.036786+0000","last_peered":"2024-08-17T18:12:08.036786+0000","last_clean":"2024-08-17T18:12:08.036786+0000","last_became_active":"2024-08-17T18:08:57.572308+0000","last_became_peered":"2024-08-17T18:08:57.572308+0000","last_unstale":"2024-08-17T18:12:08.036786+0000","last_undegraded":"2024-08-17T18:12:08.036786+0000","last_fullsized":"2024-08-17T18:12:08.036786+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:11.642831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2215,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.037154+0000","last_change":"2024-08-17T18:09:28.842838+0000","last_active":"2024-08-17T18:12:08.037154+0000","last_peered":"2024-08-17T18:12:08.037154+0000","last_clean":"2024-08-17T18:12:08.037154+0000","last_became_active":"2024-08-17T18:09:28.842617+0000","last_became_peered":"2024-08-17T18:09:28.842617+0000","last_unstale":"2024-08-17T18:12:08.037154+0000","last_undegraded":"2024-08-17T18:12:08.037154+0000","last_fullsized":"2024-08-17T18:12:08.037154+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:00.016002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1398,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.008234+0000","last_change":"2024-08-17T18:09:29.845011+0000","last_active":"2024-08-17T18:12:08.008234+0000","last_peered":"2024-08-17T18:12:08.008234+0000","last_clean":"2024-08-17T18:12:08.008234+0000","last_became_active":"2024-08-17T18:09:29.844358+0000","last_became_peered":"2024-08-17T18:09:29.844358+0000","last_unstale":"2024-08-17T18:12:08.008234+0000","last_undegraded":"2024-08-17T18:12:08.008234+0000","last_fullsized":"2024-08-17T18:12:08.008234+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:41:27.792901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1246,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.009176+0000","last_change":"2024-08-17T18:09:28.878263+0000","last_active":"2024-08-17T18:12:08.009176+0000","last_peered":"2024-08-17T18:12:08.009176+0000","last_clean":"2024-08-17T18:12:08.009176+0000","last_became_active":"2024-08-17T18:09:28.878025+0000","last_became_peered":"2024-08-17T18:09:28.878025+0000","last_unstale":"2024-08-17T18:12:08.009176+0000","last_undegraded":"2024-08-17T18:12:08.009176+0000","last_fullsized":"2024-08-17T18:12:08.009176+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:37.211100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2009,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.037662+0000","last_change":"2024-08-17T18:09:29.840134+0000","last_active":"2024-08-17T18:12:12.037662+0000","last_peered":"2024-08-17T18:12:12.037662+0000","last_clean":"2024-08-17T18:12:12.037662+0000","last_became_active":"2024-08-17T18:09:29.839848+0000","last_became_peered":"2024-08-17T18:09:29.839848+0000","last_unstale":"2024-08-17T18:12:12.037662+0000","last_undegraded":"2024-08-17T18:12:12.037662+0000","last_fullsized":"2024-08-17T18:12:12.037662+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:08:08.717851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":153,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.009167+0000","last_change":"2024-08-17T18:09:28.847633+0000","last_active":"2024-08-17T18:12:08.009167+0000","last_peered":"2024-08-17T18:12:08.009167+0000","last_clean":"2024-08-17T18:12:08.009167+0000","last_became_active":"2024-08-17T18:09:28.847341+0000","last_became_peered":"2024-08-17T18:09:28.847341+0000","last_unstale":"2024-08-17T18:12:08.009167+0000","last_undegraded":"2024-08-17T18:12:08.009167+0000","last_fullsized":"2024-08-17T18:12:08.009167+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:51:59.625154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1738,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.008129+0000","last_change":"2024-08-17T18:09:00.609431+0000","last_active":"2024-08-17T18:12:08.008129+0000","last_peered":"2024-08-17T18:12:08.008129+0000","last_clean":"2024-08-17T18:12:08.008129+0000","last_became_active":"2024-08-17T18:09:00.609167+0000","last_became_peered":"2024-08-17T18:09:00.609167+0000","last_unstale":"2024-08-17T18:12:08.008129+0000","last_undegraded":"2024-08-17T18:12:08.008129+0000","last_fullsized":"2024-08-17T18:12:08.008129+0000","mapping_epoch":1338,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:39.579970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1889,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.039317+0000","last_change":"2024-08-17T18:09:29.837502+0000","last_active":"2024-08-17T18:12:12.039317+0000","last_peered":"2024-08-17T18:12:12.039317+0000","last_clean":"2024-08-17T18:12:12.039317+0000","last_became_active":"2024-08-17T18:09:29.837243+0000","last_became_peered":"2024-08-17T18:09:29.837243+0000","last_unstale":"2024-08-17T18:12:12.039317+0000","last_undegraded":"2024-08-17T18:12:12.039317+0000","last_fullsized":"2024-08-17T18:12:12.039317+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:47.377505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2087,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.044286+0000","last_change":"2024-08-17T18:09:29.859415+0000","last_active":"2024-08-17T18:12:12.044286+0000","last_peered":"2024-08-17T18:12:12.044286+0000","last_clean":"2024-08-17T18:12:12.044286+0000","last_became_active":"2024-08-17T18:09:29.859138+0000","last_became_peered":"2024-08-17T18:09:29.859138+0000","last_unstale":"2024-08-17T18:12:12.044286+0000","last_undegraded":"2024-08-17T18:12:12.044286+0000","last_fullsized":"2024-08-17T18:12:12.044286+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:31.178915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":538,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.085123+0000","last_change":"2024-08-17T18:09:29.883834+0000","last_active":"2024-08-17T18:12:12.085123+0000","last_peered":"2024-08-17T18:12:12.085123+0000","last_clean":"2024-08-17T18:12:12.085123+0000","last_became_active":"2024-08-17T18:09:29.882638+0000","last_became_peered":"2024-08-17T18:09:29.882638+0000","last_unstale":"2024-08-17T18:12:12.085123+0000","last_undegraded":"2024-08-17T18:12:12.085123+0000","last_fullsized":"2024-08-17T18:12:12.085123+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:07:10.188684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":638,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.008321+0000","last_change":"2024-08-17T18:09:29.857250+0000","last_active":"2024-08-17T18:12:08.008321+0000","last_peered":"2024-08-17T18:12:08.008321+0000","last_clean":"2024-08-17T18:12:08.008321+0000","last_became_active":"2024-08-17T18:09:29.857038+0000","last_became_peered":"2024-08-17T18:09:29.857038+0000","last_unstale":"2024-08-17T18:12:08.008321+0000","last_undegraded":"2024-08-17T18:12:08.008321+0000","last_fullsized":"2024-08-17T18:12:08.008321+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:02:54.028575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":649,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036080+0000","last_change":"2024-08-17T18:08:57.571270+0000","last_active":"2024-08-17T18:12:08.036080+0000","last_peered":"2024-08-17T18:12:08.036080+0000","last_clean":"2024-08-17T18:12:08.036080+0000","last_became_active":"2024-08-17T18:08:57.567670+0000","last_became_peered":"2024-08-17T18:08:57.567670+0000","last_unstale":"2024-08-17T18:12:08.036080+0000","last_undegraded":"2024-08-17T18:12:08.036080+0000","last_fullsized":"2024-08-17T18:12:08.036080+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:19:42.617321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1222,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.037824+0000","last_change":"2024-08-17T18:09:29.839260+0000","last_active":"2024-08-17T18:12:12.037824+0000","last_peered":"2024-08-17T18:12:12.037824+0000","last_clean":"2024-08-17T18:12:12.037824+0000","last_became_active":"2024-08-17T18:09:29.838953+0000","last_became_peered":"2024-08-17T18:09:29.838953+0000","last_unstale":"2024-08-17T18:12:12.037824+0000","last_undegraded":"2024-08-17T18:12:12.037824+0000","last_fullsized":"2024-08-17T18:12:12.037824+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:44.014083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":811,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.084839+0000","last_change":"2024-08-17T18:09:29.851347+0000","last_active":"2024-08-17T18:12:12.084839+0000","last_peered":"2024-08-17T18:12:12.084839+0000","last_clean":"2024-08-17T18:12:12.084839+0000","last_became_active":"2024-08-17T18:09:29.851046+0000","last_became_peered":"2024-08-17T18:09:29.851046+0000","last_unstale":"2024-08-17T18:12:12.084839+0000","last_undegraded":"2024-08-17T18:12:12.084839+0000","last_fullsized":"2024-08-17T18:12:12.084839+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:21.350703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2088,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.037362+0000","last_change":"2024-08-17T18:09:29.912446+0000","last_active":"2024-08-17T18:12:12.037362+0000","last_peered":"2024-08-17T18:12:12.037362+0000","last_clean":"2024-08-17T18:12:12.037362+0000","last_became_active":"2024-08-17T18:09:29.910703+0000","last_became_peered":"2024-08-17T18:09:29.910703+0000","last_unstale":"2024-08-17T18:12:12.037362+0000","last_undegraded":"2024-08-17T18:12:12.037362+0000","last_fullsized":"2024-08-17T18:12:12.037362+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.250608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2252,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.037206+0000","last_change":"2024-08-17T18:09:29.863361+0000","last_active":"2024-08-17T18:12:08.037206+0000","last_peered":"2024-08-17T18:12:08.037206+0000","last_clean":"2024-08-17T18:12:08.037206+0000","last_became_active":"2024-08-17T18:09:29.862503+0000","last_became_peered":"2024-08-17T18:09:29.862503+0000","last_unstale":"2024-08-17T18:12:08.037206+0000","last_undegraded":"2024-08-17T18:12:08.037206+0000","last_fullsized":"2024-08-17T18:12:08.037206+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:25.908232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":77,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.044099+0000","last_change":"2024-08-17T18:09:29.895991+0000","last_active":"2024-08-17T18:12:12.044099+0000","last_peered":"2024-08-17T18:12:12.044099+0000","last_clean":"2024-08-17T18:12:12.044099+0000","last_became_active":"2024-08-17T18:09:29.895274+0000","last_became_peered":"2024-08-17T18:09:29.895274+0000","last_unstale":"2024-08-17T18:12:12.044099+0000","last_undegraded":"2024-08-17T18:12:12.044099+0000","last_fullsized":"2024-08-17T18:12:12.044099+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:19:26.420016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":520,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.008662+0000","last_change":"2024-08-17T18:09:29.844686+0000","last_active":"2024-08-17T18:12:08.008662+0000","last_peered":"2024-08-17T18:12:08.008662+0000","last_clean":"2024-08-17T18:12:08.008662+0000","last_became_active":"2024-08-17T18:09:29.844341+0000","last_became_peered":"2024-08-17T18:09:29.844341+0000","last_unstale":"2024-08-17T18:12:08.008662+0000","last_undegraded":"2024-08-17T18:12:08.008662+0000","last_fullsized":"2024-08-17T18:12:08.008662+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:38.872767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1357,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.009211+0000","last_change":"2024-08-17T18:09:28.884310+0000","last_active":"2024-08-17T18:12:08.009211+0000","last_peered":"2024-08-17T18:12:08.009211+0000","last_clean":"2024-08-17T18:12:08.009211+0000","last_became_active":"2024-08-17T18:09:28.883921+0000","last_became_peered":"2024-08-17T18:09:28.883921+0000","last_unstale":"2024-08-17T18:12:08.009211+0000","last_undegraded":"2024-08-17T18:12:08.009211+0000","last_fullsized":"2024-08-17T18:12:08.009211+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:05.272563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":820,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.039090+0000","last_change":"2024-08-17T18:09:29.837238+0000","last_active":"2024-08-17T18:12:12.039090+0000","last_peered":"2024-08-17T18:12:12.039090+0000","last_clean":"2024-08-17T18:12:12.039090+0000","last_became_active":"2024-08-17T18:09:29.836760+0000","last_became_peered":"2024-08-17T18:09:29.836760+0000","last_unstale":"2024-08-17T18:12:12.039090+0000","last_undegraded":"2024-08-17T18:12:12.039090+0000","last_fullsized":"2024-08-17T18:12:12.039090+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:38:52.862967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1372,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.009183+0000","last_change":"2024-08-17T18:09:29.871872+0000","last_active":"2024-08-17T18:12:08.009183+0000","last_peered":"2024-08-17T18:12:08.009183+0000","last_clean":"2024-08-17T18:12:08.009183+0000","last_became_active":"2024-08-17T18:09:29.845111+0000","last_became_peered":"2024-08-17T18:09:29.845111+0000","last_unstale":"2024-08-17T18:12:08.009183+0000","last_undegraded":"2024-08-17T18:12:08.009183+0000","last_fullsized":"2024-08-17T18:12:08.009183+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:48.183250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":269,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.008017+0000","last_change":"2024-08-17T18:09:29.844786+0000","last_active":"2024-08-17T18:12:08.008017+0000","last_peered":"2024-08-17T18:12:08.008017+0000","last_clean":"2024-08-17T18:12:08.008017+0000","last_became_active":"2024-08-17T18:09:29.844270+0000","last_became_peered":"2024-08-17T18:09:29.844270+0000","last_unstale":"2024-08-17T18:12:08.008017+0000","last_undegraded":"2024-08-17T18:12:08.008017+0000","last_fullsized":"2024-08-17T18:12:08.008017+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:51.866680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1388,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.009243+0000","last_change":"2024-08-17T18:09:29.886391+0000","last_active":"2024-08-17T18:12:08.009243+0000","last_peered":"2024-08-17T18:12:08.009243+0000","last_clean":"2024-08-17T18:12:08.009243+0000","last_became_active":"2024-08-17T18:09:29.844642+0000","last_became_peered":"2024-08-17T18:09:29.844642+0000","last_unstale":"2024-08-17T18:12:08.009243+0000","last_undegraded":"2024-08-17T18:12:08.009243+0000","last_fullsized":"2024-08-17T18:12:08.009243+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:07.988100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":231,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.035749+0000","last_change":"2024-08-17T18:09:28.941223+0000","last_active":"2024-08-17T18:12:08.035749+0000","last_peered":"2024-08-17T18:12:08.035749+0000","last_clean":"2024-08-17T18:12:08.035749+0000","last_became_active":"2024-08-17T18:09:28.940857+0000","last_became_peered":"2024-08-17T18:09:28.940857+0000","last_unstale":"2024-08-17T18:12:08.035749+0000","last_undegraded":"2024-08-17T18:12:08.035749+0000","last_fullsized":"2024-08-17T18:12:08.035749+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:48.809420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2085,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.039480+0000","last_change":"2024-08-17T18:09:29.838873+0000","last_active":"2024-08-17T18:12:12.039480+0000","last_peered":"2024-08-17T18:12:12.039480+0000","last_clean":"2024-08-17T18:12:12.039480+0000","last_became_active":"2024-08-17T18:09:29.838560+0000","last_became_peered":"2024-08-17T18:09:29.838560+0000","last_unstale":"2024-08-17T18:12:12.039480+0000","last_undegraded":"2024-08-17T18:12:12.039480+0000","last_fullsized":"2024-08-17T18:12:12.039480+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:00.103741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1348,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.008423+0000","last_change":"2024-08-17T18:09:28.844170+0000","last_active":"2024-08-17T18:12:08.008423+0000","last_peered":"2024-08-17T18:12:08.008423+0000","last_clean":"2024-08-17T18:12:08.008423+0000","last_became_active":"2024-08-17T18:09:28.843862+0000","last_became_peered":"2024-08-17T18:09:28.843862+0000","last_unstale":"2024-08-17T18:12:08.008423+0000","last_undegraded":"2024-08-17T18:12:08.008423+0000","last_fullsized":"2024-08-17T18:12:08.008423+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:43:19.483326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2205,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036466+0000","last_change":"2024-08-17T18:09:28.864923+0000","last_active":"2024-08-17T18:12:08.036466+0000","last_peered":"2024-08-17T18:12:08.036466+0000","last_clean":"2024-08-17T18:12:08.036466+0000","last_became_active":"2024-08-17T18:09:28.864707+0000","last_became_peered":"2024-08-17T18:09:28.864707+0000","last_unstale":"2024-08-17T18:12:08.036466+0000","last_undegraded":"2024-08-17T18:12:08.036466+0000","last_fullsized":"2024-08-17T18:12:08.036466+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:27.507329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2207,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.036539+0000","last_change":"2024-08-17T18:09:28.846741+0000","last_active":"2024-08-17T18:12:08.036539+0000","last_peered":"2024-08-17T18:12:08.036539+0000","last_clean":"2024-08-17T18:12:08.036539+0000","last_became_active":"2024-08-17T18:09:28.846478+0000","last_became_peered":"2024-08-17T18:09:28.846478+0000","last_unstale":"2024-08-17T18:12:08.036539+0000","last_undegraded":"2024-08-17T18:12:08.036539+0000","last_fullsized":"2024-08-17T18:12:08.036539+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:34.247517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":754,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.009010+0000","last_change":"2024-08-17T18:03:16.091420+0000","last_active":"2024-08-17T18:12:08.009010+0000","last_peered":"2024-08-17T18:12:08.009010+0000","last_clean":"2024-08-17T18:12:08.009010+0000","last_became_active":"2024-08-17T18:03:16.091113+0000","last_became_peered":"2024-08-17T18:03:16.091113+0000","last_unstale":"2024-08-17T18:12:08.009010+0000","last_undegraded":"2024-08-17T18:12:08.009010+0000","last_fullsized":"2024-08-17T18:12:08.009010+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:47.770911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1362,"reported_epoch":1376,"state":"active+clean","last_fresh":"2024-08-17T18:12:08.009056+0000","last_change":"2024-08-17T18:02:53.893927+0000","last_active":"2024-08-17T18:12:08.009056+0000","last_peered":"2024-08-17T18:12:08.009056+0000","last_clean":"2024-08-17T18:12:08.009056+0000","last_became_active":"2024-08-17T18:02:53.883188+0000","last_became_peered":"2024-08-17T18:02:53.883188+0000","last_unstale":"2024-08-17T18:12:08.009056+0000","last_undegraded":"2024-08-17T18:12:08.009056+0000","last_fullsized":"2024-08-17T18:12:08.009056+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:19.296965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2085,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.044265+0000","last_change":"2024-08-17T18:09:29.840086+0000","last_active":"2024-08-17T18:12:12.044265+0000","last_peered":"2024-08-17T18:12:12.044265+0000","last_clean":"2024-08-17T18:12:12.044265+0000","last_became_active":"2024-08-17T18:09:29.839525+0000","last_became_peered":"2024-08-17T18:09:29.839525+0000","last_unstale":"2024-08-17T18:12:12.044265+0000","last_undegraded":"2024-08-17T18:12:12.044265+0000","last_fullsized":"2024-08-17T18:12:12.044265+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:38.316235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":48,"num_pg":16,"stat_sum":{"num_bytes":56950784,"num_objects":869,"num_object_clones":0,"num_object_copies":2607,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":869,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":263057,"num_write_kb":8445569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18643,"num_bytes_recovered":1221394456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":387784704,"data_stored":387776560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":69799,"ondisk_log_size":69799,"up":48,"acting":48,"num_store_stats":5},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":10970,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":155648,"data_stored":27490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234616,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":906668,"kb_used_data":13088,"kb_used_omap":1,"kb_used_meta":893566,"kb_avail":92838484,"statfs":{"total":95995035648,"available":95066607616,"internally_reserved":0,"allocated":13402112,"data_stored":8809767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915011928},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568496251,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757889,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1343,"seq":5768141078562,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":262556,"kb_used_data":44112,"kb_used_omap":112,"kb_used_meta":218319,"kb_avail":93482596,"statfs":{"total":95995035648,"available":95726178304,"internally_reserved":0,"allocated":45170688,"data_stored":40589987,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":115373,"internal_metadata":223558995},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,3,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405741,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1160144,"kb_used_data":162436,"kb_used_omap":61,"kb_used_meta":997634,"kb_avail":92585008,"statfs":{"total":95995035648,"available":94807048192,"internally_reserved":0,"allocated":166334464,"data_stored":161762303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":1021578168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":0,"seq":2173253452276,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474966,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1073692,"kb_used_data":176080,"kb_used_omap":119,"kb_used_meta":897480,"kb_avail":92671460,"statfs":{"total":95995035648,"available":94895575040,"internally_reserved":0,"allocated":180305920,"data_stored":175721656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":919020254},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019524,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":144506880,"data_stored":144506880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":93716480,"data_stored":93716480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":118886400,"data_stored":118882328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":30674944,"data_stored":30670872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":48,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:12:16.744 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:16.766 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:16.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:16.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:16.868 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:16.889 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:16.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:16.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:16.991 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:17.013 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:17.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:17.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:17.115 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:17.141 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:17.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:17.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:17.243 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:17.267 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:17.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:17.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:17.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:17 smithi179 ceph-mon[93519]: pgmap v2507: 89 pgs: 89 active+clean; 18 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 633 op/s; 13 MiB/s, 211 objects/s recovering 2024-08-17T18:12:17.369 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:17.395 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:17.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:17.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:17.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:17 smithi130 ceph-mon[105521]: pgmap v2507: 89 pgs: 89 active+clean; 18 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 633 op/s; 13 MiB/s, 211 objects/s recovering 2024-08-17T18:12:17.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:17 smithi130 ceph-mon[110068]: pgmap v2507: 89 pgs: 89 active+clean; 18 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 633 op/s; 13 MiB/s, 211 objects/s recovering 2024-08-17T18:12:17.497 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:17.519 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:17.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:17.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:17.621 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:17.646 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:17.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:17.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:17.748 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:17.770 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:17.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:17.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:17.871 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:17.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:17.896 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:17.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:17.976 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-08-17T18:12:17.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-08-17T18:12:17.997 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:18.023 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:18.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:18.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:18.113 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:12:18.113 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T18:12:18.113 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 0 2024-08-17T18:12:18.113 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:12:18.124 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:18.149 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:18.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:18.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:18.251 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:18.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:18.273 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:18.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:18.374 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:18.400 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:18.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:18.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:18.502 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:18.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:18.524 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:18.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:18.625 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:18.651 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:18.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:18.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:18.753 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:18.775 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:18.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:18.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:18.877 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:18.903 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:18.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:18.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:19.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:19.030 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:19.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:19.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:19.104 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:19.132 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:19.156 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:19.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:19.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:19.258 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:19.279 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:19.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:19.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:19.360 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:19 smithi130 ceph-mon[105521]: pgmap v2508: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 564 op/s; 11 MiB/s, 178 objects/s recovering 2024-08-17T18:12:19.360 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:19 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:19.360 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:19 smithi130 ceph-mon[110068]: pgmap v2508: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 564 op/s; 11 MiB/s, 178 objects/s recovering 2024-08-17T18:12:19.360 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:19 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:19.382 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:19.408 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:19.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:19.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:19.510 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:19.535 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:19.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:19.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:19.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:12:19.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:12:19.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:12:19.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:12:19.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:12:19.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:12:19.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:12:19.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:12:19.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:12:19.579 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:12:19.579 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:12:19.580 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:12:19.580 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:12:19.580 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:12:19.580 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:12:19.580 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:12:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:19 smithi179 ceph-mon[93519]: pgmap v2508: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 564 op/s; 11 MiB/s, 178 objects/s recovering 2024-08-17T18:12:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:19 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:19.636 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:19.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:19.660 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:19.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:19.710 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:12:19.760 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:19.760 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:19.782 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:19.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:19.884 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:19.907 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:19.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:19.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:20.009 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:20.030 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:20.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:20.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:20.129 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:12:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:12:19] "GET /metrics HTTP/1.1" 200 36796 "" "Prometheus/2.51.0" 2024-08-17T18:12:20.132 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:20.157 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:20.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:20.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:20.259 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:20.280 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:20.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:20.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:20.333 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T18:12:20.382 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:20.407 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:20.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:20.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:20.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3037710690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:20.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3037710690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:20.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3037710690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:20.495 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:12:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:12:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:12:20.509 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:20.532 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:20.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:20.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:20.634 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:20.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:20.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:20.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:20.757 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:20.783 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:20.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:20.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:20.884 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:20.909 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:20.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:20.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:21.011 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:21.033 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:21.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:21.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:21.135 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:21.156 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:21.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:21.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:21.258 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:21.279 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:21.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:21.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:21.381 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:21.407 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:21.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:21.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:21.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:21 smithi130 ceph-mon[105521]: pgmap v2509: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 470 op/s 2024-08-17T18:12:21.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:21 smithi130 ceph-mon[110068]: pgmap v2509: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 470 op/s 2024-08-17T18:12:21.509 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:21.530 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:21.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:21.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:21 smithi179 ceph-mon[93519]: pgmap v2509: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 470 op/s 2024-08-17T18:12:21.632 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:21.658 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:21.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:21.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:21.760 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:21.781 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:21.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:21.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:21.883 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:21.904 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:21.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:21.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:22.006 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:22.032 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:22.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:22.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:22.134 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:22.161 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:22.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:22.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:22.263 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:22.287 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:22.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:22.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:22.388 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:22.412 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:22.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:22.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:22.514 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:22.536 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:22.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:22.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:22.638 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:22.662 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:22.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:22.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:22.764 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:22.786 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:22.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:22.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:22.836 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:22.838 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:22.889 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:22.910 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:22.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:22.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:23.012 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:23.037 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:23.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:23.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:23.139 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:23.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:23.163 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:23.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:23.264 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:23.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:23.291 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:23.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:23.327 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:23 smithi130 ceph-mon[105521]: pgmap v2510: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 452 op/s 2024-08-17T18:12:23.327 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:23 smithi130 ceph-mon[110068]: pgmap v2510: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 452 op/s 2024-08-17T18:12:23.391 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:23.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:23.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:23.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:23.498 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:12:23.520 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:23.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:23.564 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:23.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:23.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:23 smithi179 ceph-mon[93519]: pgmap v2510: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 452 op/s 2024-08-17T18:12:23.665 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:23.689 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:23.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:23.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:23.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:23.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:23.818 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:23.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:23.919 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:23.940 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:23.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:23.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:24.042 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:24.067 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:24.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:24.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:24.139 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:12:24.143 INFO:teuthology.orchestra.run.smithi130.stderr:pool 'unique_pool_3' removed 2024-08-17T18:12:24.169 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:24.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:24.209 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:24.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:24.310 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:24.331 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:24.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:24.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:24.361 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:24 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/866607513' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:12:24.361 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:24 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:12:24.361 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:24 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3395624466' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:12:24.361 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:24 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/866607513' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:12:24.361 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:24 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:12:24.361 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:24 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3395624466' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:12:24.433 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:24.454 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:24.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:24.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:24.556 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:24.577 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:24.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:24.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:24 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/866607513' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:12:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:24 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:12:24.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:24 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3395624466' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:12:24.678 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:24.700 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:24.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:24.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:24.801 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:24.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:24.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:24.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:24.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:24.953 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:24.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:24.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:25.055 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:25.078 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:25.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:25.179 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:25.204 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:25.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:25.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:25.306 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:25.329 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:25.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:25.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:25.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:25 smithi130 ceph-mon[105521]: pgmap v2511: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 395 op/s 2024-08-17T18:12:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:25 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:12:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:25 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:12:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:25 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3395624466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:12:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:25 smithi130 ceph-mon[105521]: osdmap e1378: 8 total, 8 up, 4 in 2024-08-17T18:12:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:25 smithi130 ceph-mon[110068]: pgmap v2511: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 395 op/s 2024-08-17T18:12:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:25 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:12:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:25 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:12:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:25 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3395624466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:12:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:25 smithi130 ceph-mon[110068]: osdmap e1378: 8 total, 8 up, 4 in 2024-08-17T18:12:25.431 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:25.452 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:25.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:25.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:25.554 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:25.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:25.581 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:25.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:25.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:25 smithi179 ceph-mon[93519]: pgmap v2511: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 395 op/s 2024-08-17T18:12:25.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:25 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:12:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:25 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:12:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:25 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3395624466' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:12:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:25 smithi179 ceph-mon[93519]: osdmap e1378: 8 total, 8 up, 4 in 2024-08-17T18:12:25.681 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:25.708 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:25.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:25.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:25.810 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:25.832 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:25.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:25.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:25.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:25.955 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:25.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:25.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:26.057 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:26.078 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:26.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:26.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:26.180 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:26.202 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:26.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:26.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:26.304 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:26.326 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:26.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:26.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:26.428 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:26.449 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:26.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:26.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:26.551 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:26.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:26.575 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:26.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:26.676 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:26.696 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T18:12:26.697 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T18:12:26.697 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-08-17T18:12:26.697 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_4 16 2024-08-17T18:12:26.698 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:26.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:26.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:26.801 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:26.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:26.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:26.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:26.924 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:26.950 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:26.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:26.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:27.052 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:27.076 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:27.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:27.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:27.135 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:12:27.178 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:27.202 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:27.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:27.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:27.304 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:27.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:27.328 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:27.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:27.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:27 smithi130 ceph-mon[110068]: pgmap v2513: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:27 smithi130 ceph-mon[105521]: pgmap v2513: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:27.429 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:27.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:27.458 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:27.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:27.559 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:27.581 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:27.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:27.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:27.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:27.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:27 smithi179 ceph-mon[93519]: pgmap v2513: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:27.686 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:27.708 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:27.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:27.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:27.810 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:27.836 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:27.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:27.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:27.937 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:27.959 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:27.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:27.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:28.061 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:28.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:28.087 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:28.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:28.188 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:28.210 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:28.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:28.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:28.312 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:28.335 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:28.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:28.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:28.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:12:28.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:12:28.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:28.463 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:28.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:28.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:28.565 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:28.589 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:28.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:28.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:28.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:12:28.691 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:28.713 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:28.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:28.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:28.815 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:28.837 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:28.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:28.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:28.901 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:12:28.901 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:12:28.939 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:28.965 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:28.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:28.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:29.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:29.088 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:29.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:29.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:29.190 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:29.215 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:29.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:29.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:29.317 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:29.342 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:29.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:29.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:29.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:29 smithi130 ceph-mon[110068]: pgmap v2514: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:29 smithi130 ceph-mon[105521]: pgmap v2514: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:29.444 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:29.468 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:29.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:29.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:29.569 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:29.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:29 smithi179 ceph-mon[93519]: pgmap v2514: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:29.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:29.593 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:29.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:29.596 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T18:12:29.695 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:29.718 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:29.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:29.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:29.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:29.841 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:29.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:29.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:29.884 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:12:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:12:29] "GET /metrics HTTP/1.1" 200 34365 "" "Prometheus/2.51.0" 2024-08-17T18:12:29.943 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:29.969 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:29.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:29.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:30.070 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:30.094 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:30.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:30.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:30.195 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:30.216 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:30.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:30.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:30.278 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:12:30.279 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:12:30.318 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:30.344 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:30.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:30.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:30.446 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:30.470 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:30.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:30.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:30.572 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:12:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:12:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:12:30.595 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:30.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:30.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:30.697 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:30.718 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:30.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:30.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:30.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:30.842 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:30.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:30.944 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:30.965 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:30.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:30.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:31.067 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:31.093 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:31.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:31.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:31.195 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:31.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:31.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:31.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:31.321 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:31.344 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:31.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:31.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:31 smithi130 ceph-mon[105521]: pgmap v2515: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:31.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:31 smithi130 ceph-mon[105521]: from='client.36897 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:12:31.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:31 smithi130 ceph-mon[110068]: pgmap v2515: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:31.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:31 smithi130 ceph-mon[110068]: from='client.36897 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:12:31.447 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:31.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:31.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:31.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:31.524 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2515,"stamp":"2024-08-17T18:12:29.485471+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2962829,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":241,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":3087936,"kb_used_data":45396,"kb_used_omap":294,"kb_used_meta":3042201,"kb_avail":371892672,"statfs":{"total":383980142592,"available":380818096128,"internally_reserved":0,"allocated":46485504,"data_stored":28118370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":301759,"internal_metadata":3115214145},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":29,"apply_latency_ms":29,"commit_latency_ns":29000000,"apply_latency_ns":29000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-263926,"num_write_kb":-8445569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-18643,"num_bytes_recovered":-1221394456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002557"},"pg_stats":[{"pgid":"3.1f","version":"71'4","reported_seq":2245,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.156184+0000","last_change":"2024-08-17T18:09:28.870033+0000","last_active":"2024-08-17T18:12:25.156184+0000","last_peered":"2024-08-17T18:12:25.156184+0000","last_clean":"2024-08-17T18:12:25.156184+0000","last_became_active":"2024-08-17T18:09:28.869346+0000","last_became_peered":"2024-08-17T18:09:28.869346+0000","last_unstale":"2024-08-17T18:12:25.156184+0000","last_undegraded":"2024-08-17T18:12:25.156184+0000","last_fullsized":"2024-08-17T18:12:25.156184+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:23.168631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":540,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.140722+0000","last_change":"2024-08-17T18:09:29.841584+0000","last_active":"2024-08-17T18:12:24.140722+0000","last_peered":"2024-08-17T18:12:24.140722+0000","last_clean":"2024-08-17T18:12:24.140722+0000","last_became_active":"2024-08-17T18:09:29.840950+0000","last_became_peered":"2024-08-17T18:09:29.840950+0000","last_unstale":"2024-08-17T18:12:24.140722+0000","last_undegraded":"2024-08-17T18:12:24.140722+0000","last_fullsized":"2024-08-17T18:12:24.140722+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:43.035959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":2035,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.144643+0000","last_change":"2024-08-17T18:09:29.883587+0000","last_active":"2024-08-17T18:12:24.144643+0000","last_peered":"2024-08-17T18:12:24.144643+0000","last_clean":"2024-08-17T18:12:24.144643+0000","last_became_active":"2024-08-17T18:09:29.882222+0000","last_became_peered":"2024-08-17T18:09:29.882222+0000","last_unstale":"2024-08-17T18:12:24.144643+0000","last_undegraded":"2024-08-17T18:12:24.144643+0000","last_fullsized":"2024-08-17T18:12:24.144643+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:10.777993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2092,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.141416+0000","last_change":"2024-08-17T18:09:28.858824+0000","last_active":"2024-08-17T18:12:24.141416+0000","last_peered":"2024-08-17T18:12:24.141416+0000","last_clean":"2024-08-17T18:12:24.141416+0000","last_became_active":"2024-08-17T18:09:28.857856+0000","last_became_peered":"2024-08-17T18:09:28.857856+0000","last_unstale":"2024-08-17T18:12:24.141416+0000","last_undegraded":"2024-08-17T18:12:24.141416+0000","last_fullsized":"2024-08-17T18:12:24.141416+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:41:35.146166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":96,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.143773+0000","last_change":"2024-08-17T18:09:29.851515+0000","last_active":"2024-08-17T18:12:24.143773+0000","last_peered":"2024-08-17T18:12:24.143773+0000","last_clean":"2024-08-17T18:12:24.143773+0000","last_became_active":"2024-08-17T18:09:29.851109+0000","last_became_peered":"2024-08-17T18:09:29.851109+0000","last_unstale":"2024-08-17T18:12:24.143773+0000","last_undegraded":"2024-08-17T18:12:24.143773+0000","last_fullsized":"2024-08-17T18:12:24.143773+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:42.054532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":90,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.037146+0000","last_change":"2024-08-17T18:03:16.089811+0000","last_active":"2024-08-17T18:12:12.037146+0000","last_peered":"2024-08-17T18:12:12.037146+0000","last_clean":"2024-08-17T18:12:12.037146+0000","last_became_active":"2024-08-17T18:03:16.089571+0000","last_became_peered":"2024-08-17T18:03:16.089571+0000","last_unstale":"2024-08-17T18:12:12.037146+0000","last_undegraded":"2024-08-17T18:12:12.037146+0000","last_fullsized":"2024-08-17T18:12:12.037146+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:04.630361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":72,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.038085+0000","last_change":"2024-08-17T18:08:57.577055+0000","last_active":"2024-08-17T18:12:12.038085+0000","last_peered":"2024-08-17T18:12:12.038085+0000","last_clean":"2024-08-17T18:12:12.038085+0000","last_became_active":"2024-08-17T18:08:57.573659+0000","last_became_peered":"2024-08-17T18:08:57.573659+0000","last_unstale":"2024-08-17T18:12:12.038085+0000","last_undegraded":"2024-08-17T18:12:12.038085+0000","last_fullsized":"2024-08-17T18:12:12.038085+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:01.686752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":880,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.143018+0000","last_change":"2024-08-17T18:09:29.852393+0000","last_active":"2024-08-17T18:12:24.143018+0000","last_peered":"2024-08-17T18:12:24.143018+0000","last_clean":"2024-08-17T18:12:24.143018+0000","last_became_active":"2024-08-17T18:09:29.851947+0000","last_became_peered":"2024-08-17T18:09:29.851947+0000","last_unstale":"2024-08-17T18:12:24.143018+0000","last_undegraded":"2024-08-17T18:12:24.143018+0000","last_fullsized":"2024-08-17T18:12:24.143018+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:49.145327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":2190,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.157092+0000","last_change":"2024-08-17T18:09:29.863241+0000","last_active":"2024-08-17T18:12:25.157092+0000","last_peered":"2024-08-17T18:12:25.157092+0000","last_clean":"2024-08-17T18:12:25.157092+0000","last_became_active":"2024-08-17T18:09:29.862369+0000","last_became_peered":"2024-08-17T18:09:29.862369+0000","last_unstale":"2024-08-17T18:12:25.157092+0000","last_undegraded":"2024-08-17T18:12:25.157092+0000","last_fullsized":"2024-08-17T18:12:25.157092+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:30.518551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":627,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.143295+0000","last_change":"2024-08-17T18:09:29.896233+0000","last_active":"2024-08-17T18:12:24.143295+0000","last_peered":"2024-08-17T18:12:24.143295+0000","last_clean":"2024-08-17T18:12:24.143295+0000","last_became_active":"2024-08-17T18:09:29.895718+0000","last_became_peered":"2024-08-17T18:09:29.895718+0000","last_unstale":"2024-08-17T18:12:24.143295+0000","last_undegraded":"2024-08-17T18:12:24.143295+0000","last_fullsized":"2024-08-17T18:12:24.143295+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:03.278878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":100,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.156775+0000","last_change":"2024-08-17T18:08:57.574015+0000","last_active":"2024-08-17T18:12:25.156775+0000","last_peered":"2024-08-17T18:12:25.156775+0000","last_clean":"2024-08-17T18:12:25.156775+0000","last_became_active":"2024-08-17T18:08:57.567973+0000","last_became_peered":"2024-08-17T18:08:57.567973+0000","last_unstale":"2024-08-17T18:12:25.156775+0000","last_undegraded":"2024-08-17T18:12:25.156775+0000","last_fullsized":"2024-08-17T18:12:25.156775+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:06.251764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2047,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.143885+0000","last_change":"2024-08-17T18:09:29.917203+0000","last_active":"2024-08-17T18:12:24.143885+0000","last_peered":"2024-08-17T18:12:24.143885+0000","last_clean":"2024-08-17T18:12:24.143885+0000","last_became_active":"2024-08-17T18:09:29.916844+0000","last_became_peered":"2024-08-17T18:09:29.916844+0000","last_unstale":"2024-08-17T18:12:24.143885+0000","last_undegraded":"2024-08-17T18:12:24.143885+0000","last_fullsized":"2024-08-17T18:12:24.143885+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.453720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1953,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.141158+0000","last_change":"2024-08-17T18:09:28.859255+0000","last_active":"2024-08-17T18:12:24.141158+0000","last_peered":"2024-08-17T18:12:24.141158+0000","last_clean":"2024-08-17T18:12:24.141158+0000","last_became_active":"2024-08-17T18:09:28.858537+0000","last_became_peered":"2024-08-17T18:09:28.858537+0000","last_unstale":"2024-08-17T18:12:24.141158+0000","last_undegraded":"2024-08-17T18:12:24.141158+0000","last_fullsized":"2024-08-17T18:12:24.141158+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:15.364687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2213,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.156079+0000","last_change":"2024-08-17T18:09:28.868303+0000","last_active":"2024-08-17T18:12:25.156079+0000","last_peered":"2024-08-17T18:12:25.156079+0000","last_clean":"2024-08-17T18:12:25.156079+0000","last_became_active":"2024-08-17T18:09:28.867989+0000","last_became_peered":"2024-08-17T18:09:28.867989+0000","last_unstale":"2024-08-17T18:12:25.156079+0000","last_undegraded":"2024-08-17T18:12:25.156079+0000","last_fullsized":"2024-08-17T18:12:25.156079+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:58:52.741685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":79,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.140467+0000","last_change":"2024-08-17T18:09:29.842129+0000","last_active":"2024-08-17T18:12:24.140467+0000","last_peered":"2024-08-17T18:12:24.140467+0000","last_clean":"2024-08-17T18:12:24.140467+0000","last_became_active":"2024-08-17T18:09:29.841848+0000","last_became_peered":"2024-08-17T18:09:29.841848+0000","last_unstale":"2024-08-17T18:12:24.140467+0000","last_undegraded":"2024-08-17T18:12:24.140467+0000","last_fullsized":"2024-08-17T18:12:24.140467+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:02.119134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1327,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.157164+0000","last_change":"2024-08-17T18:09:29.846509+0000","last_active":"2024-08-17T18:12:25.157164+0000","last_peered":"2024-08-17T18:12:25.157164+0000","last_clean":"2024-08-17T18:12:25.157164+0000","last_became_active":"2024-08-17T18:09:29.844097+0000","last_became_peered":"2024-08-17T18:09:29.844097+0000","last_unstale":"2024-08-17T18:12:25.157164+0000","last_undegraded":"2024-08-17T18:12:25.157164+0000","last_fullsized":"2024-08-17T18:12:25.157164+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:32.340354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2041,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.140577+0000","last_change":"2024-08-17T18:09:29.841335+0000","last_active":"2024-08-17T18:12:24.140577+0000","last_peered":"2024-08-17T18:12:24.140577+0000","last_clean":"2024-08-17T18:12:24.140577+0000","last_became_active":"2024-08-17T18:09:29.840522+0000","last_became_peered":"2024-08-17T18:09:29.840522+0000","last_unstale":"2024-08-17T18:12:24.140577+0000","last_undegraded":"2024-08-17T18:12:24.140577+0000","last_fullsized":"2024-08-17T18:12:24.140577+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:08.015836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2218,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.157265+0000","last_change":"2024-08-17T18:09:29.847494+0000","last_active":"2024-08-17T18:12:25.157265+0000","last_peered":"2024-08-17T18:12:25.157265+0000","last_clean":"2024-08-17T18:12:25.157265+0000","last_became_active":"2024-08-17T18:09:29.846973+0000","last_became_peered":"2024-08-17T18:09:29.846973+0000","last_unstale":"2024-08-17T18:12:25.157265+0000","last_undegraded":"2024-08-17T18:12:25.157265+0000","last_fullsized":"2024-08-17T18:12:25.157265+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:49.469921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2000,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.140479+0000","last_change":"2024-08-17T18:09:28.855045+0000","last_active":"2024-08-17T18:12:24.140479+0000","last_peered":"2024-08-17T18:12:24.140479+0000","last_clean":"2024-08-17T18:12:24.140479+0000","last_became_active":"2024-08-17T18:09:28.854607+0000","last_became_peered":"2024-08-17T18:09:28.854607+0000","last_unstale":"2024-08-17T18:12:24.140479+0000","last_undegraded":"2024-08-17T18:12:24.140479+0000","last_fullsized":"2024-08-17T18:12:24.140479+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:13.437895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":702,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.038545+0000","last_change":"2024-08-17T18:09:29.866944+0000","last_active":"2024-08-17T18:12:12.038545+0000","last_peered":"2024-08-17T18:12:12.038545+0000","last_clean":"2024-08-17T18:12:12.038545+0000","last_became_active":"2024-08-17T18:09:29.866694+0000","last_became_peered":"2024-08-17T18:09:29.866694+0000","last_unstale":"2024-08-17T18:12:12.038545+0000","last_undegraded":"2024-08-17T18:12:12.038545+0000","last_fullsized":"2024-08-17T18:12:12.038545+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:00:41.342392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1365,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.038228+0000","last_change":"2024-08-17T18:09:28.884262+0000","last_active":"2024-08-17T18:12:12.038228+0000","last_peered":"2024-08-17T18:12:12.038228+0000","last_clean":"2024-08-17T18:12:12.038228+0000","last_became_active":"2024-08-17T18:09:28.883734+0000","last_became_peered":"2024-08-17T18:09:28.883734+0000","last_unstale":"2024-08-17T18:12:12.038228+0000","last_undegraded":"2024-08-17T18:12:12.038228+0000","last_fullsized":"2024-08-17T18:12:12.038228+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:10.735504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2103,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.156477+0000","last_change":"2024-08-17T18:08:57.571995+0000","last_active":"2024-08-17T18:12:25.156477+0000","last_peered":"2024-08-17T18:12:25.156477+0000","last_clean":"2024-08-17T18:12:25.156477+0000","last_became_active":"2024-08-17T18:08:57.567342+0000","last_became_peered":"2024-08-17T18:08:57.567342+0000","last_unstale":"2024-08-17T18:12:25.156477+0000","last_undegraded":"2024-08-17T18:12:25.156477+0000","last_fullsized":"2024-08-17T18:12:25.156477+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:23.769111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":529,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.038102+0000","last_change":"2024-08-17T18:02:53.912837+0000","last_active":"2024-08-17T18:12:12.038102+0000","last_peered":"2024-08-17T18:12:12.038102+0000","last_clean":"2024-08-17T18:12:12.038102+0000","last_became_active":"2024-08-17T18:02:53.883152+0000","last_became_peered":"2024-08-17T18:02:53.883152+0000","last_unstale":"2024-08-17T18:12:12.038102+0000","last_undegraded":"2024-08-17T18:12:12.038102+0000","last_fullsized":"2024-08-17T18:12:12.038102+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:46.261612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":528,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.039242+0000","last_change":"2024-08-17T18:09:29.845126+0000","last_active":"2024-08-17T18:12:12.039242+0000","last_peered":"2024-08-17T18:12:12.039242+0000","last_clean":"2024-08-17T18:12:12.039242+0000","last_became_active":"2024-08-17T18:09:29.844836+0000","last_became_peered":"2024-08-17T18:09:29.844836+0000","last_unstale":"2024-08-17T18:12:12.039242+0000","last_undegraded":"2024-08-17T18:12:12.039242+0000","last_fullsized":"2024-08-17T18:12:12.039242+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:44.821894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1340,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.156848+0000","last_change":"2024-08-17T18:02:53.890198+0000","last_active":"2024-08-17T18:12:25.156848+0000","last_peered":"2024-08-17T18:12:25.156848+0000","last_clean":"2024-08-17T18:12:25.156848+0000","last_became_active":"2024-08-17T18:02:53.887007+0000","last_became_peered":"2024-08-17T18:02:53.887007+0000","last_unstale":"2024-08-17T18:12:25.156848+0000","last_undegraded":"2024-08-17T18:12:25.156848+0000","last_fullsized":"2024-08-17T18:12:25.156848+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:25.555909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":619,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.037528+0000","last_change":"2024-08-17T18:09:28.884187+0000","last_active":"2024-08-17T18:12:12.037528+0000","last_peered":"2024-08-17T18:12:12.037528+0000","last_clean":"2024-08-17T18:12:12.037528+0000","last_became_active":"2024-08-17T18:09:28.883939+0000","last_became_peered":"2024-08-17T18:09:28.883939+0000","last_unstale":"2024-08-17T18:12:12.037528+0000","last_undegraded":"2024-08-17T18:12:12.037528+0000","last_fullsized":"2024-08-17T18:12:12.037528+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:37:41.951099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1919,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.038521+0000","last_change":"2024-08-17T18:09:29.893220+0000","last_active":"2024-08-17T18:12:12.038521+0000","last_peered":"2024-08-17T18:12:12.038521+0000","last_clean":"2024-08-17T18:12:12.038521+0000","last_became_active":"2024-08-17T18:09:29.865699+0000","last_became_peered":"2024-08-17T18:09:29.865699+0000","last_unstale":"2024-08-17T18:12:12.038521+0000","last_undegraded":"2024-08-17T18:12:12.038521+0000","last_fullsized":"2024-08-17T18:12:12.038521+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:40.205658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2060,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.141526+0000","last_change":"2024-08-17T18:09:28.934404+0000","last_active":"2024-08-17T18:12:24.141526+0000","last_peered":"2024-08-17T18:12:24.141526+0000","last_clean":"2024-08-17T18:12:24.141526+0000","last_became_active":"2024-08-17T18:09:28.934001+0000","last_became_peered":"2024-08-17T18:09:28.934001+0000","last_unstale":"2024-08-17T18:12:24.141526+0000","last_undegraded":"2024-08-17T18:12:24.141526+0000","last_fullsized":"2024-08-17T18:12:24.141526+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:04.676812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":2092,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.145354+0000","last_change":"2024-08-17T18:09:29.884088+0000","last_active":"2024-08-17T18:12:24.145354+0000","last_peered":"2024-08-17T18:12:24.145354+0000","last_clean":"2024-08-17T18:12:24.145354+0000","last_became_active":"2024-08-17T18:09:29.883027+0000","last_became_peered":"2024-08-17T18:09:29.883027+0000","last_unstale":"2024-08-17T18:12:24.145354+0000","last_undegraded":"2024-08-17T18:12:24.145354+0000","last_fullsized":"2024-08-17T18:12:24.145354+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:22:07.777188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":2062,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.145828+0000","last_change":"2024-08-17T18:09:28.935401+0000","last_active":"2024-08-17T18:12:24.145828+0000","last_peered":"2024-08-17T18:12:24.145828+0000","last_clean":"2024-08-17T18:12:24.145828+0000","last_became_active":"2024-08-17T18:09:28.933628+0000","last_became_peered":"2024-08-17T18:09:28.933628+0000","last_unstale":"2024-08-17T18:12:24.145828+0000","last_undegraded":"2024-08-17T18:12:24.145828+0000","last_fullsized":"2024-08-17T18:12:24.145828+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:21:12.714486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2174,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.157678+0000","last_change":"2024-08-17T18:08:57.574718+0000","last_active":"2024-08-17T18:12:25.157678+0000","last_peered":"2024-08-17T18:12:25.157678+0000","last_clean":"2024-08-17T18:12:25.157678+0000","last_became_active":"2024-08-17T18:08:57.573515+0000","last_became_peered":"2024-08-17T18:08:57.573515+0000","last_unstale":"2024-08-17T18:12:25.157678+0000","last_undegraded":"2024-08-17T18:12:25.157678+0000","last_fullsized":"2024-08-17T18:12:25.157678+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:04.873717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1378,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.038436+0000","last_change":"2024-08-17T18:08:57.568821+0000","last_active":"2024-08-17T18:12:12.038436+0000","last_peered":"2024-08-17T18:12:12.038436+0000","last_clean":"2024-08-17T18:12:12.038436+0000","last_became_active":"2024-08-17T18:08:57.568526+0000","last_became_peered":"2024-08-17T18:08:57.568526+0000","last_unstale":"2024-08-17T18:12:12.038436+0000","last_undegraded":"2024-08-17T18:12:12.038436+0000","last_fullsized":"2024-08-17T18:12:12.038436+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:26.449197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1894,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.158091+0000","last_change":"2024-08-17T18:08:57.571174+0000","last_active":"2024-08-17T18:12:25.158091+0000","last_peered":"2024-08-17T18:12:25.158091+0000","last_clean":"2024-08-17T18:12:25.158091+0000","last_became_active":"2024-08-17T18:08:57.567571+0000","last_became_peered":"2024-08-17T18:08:57.567571+0000","last_unstale":"2024-08-17T18:12:25.158091+0000","last_undegraded":"2024-08-17T18:12:25.158091+0000","last_fullsized":"2024-08-17T18:12:25.158091+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:29.976150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1863,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.156903+0000","last_change":"2024-08-17T18:08:57.572368+0000","last_active":"2024-08-17T18:12:25.156903+0000","last_peered":"2024-08-17T18:12:25.156903+0000","last_clean":"2024-08-17T18:12:25.156903+0000","last_became_active":"2024-08-17T18:08:57.568382+0000","last_became_peered":"2024-08-17T18:08:57.568382+0000","last_unstale":"2024-08-17T18:12:25.156903+0000","last_undegraded":"2024-08-17T18:12:25.156903+0000","last_fullsized":"2024-08-17T18:12:25.156903+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:11:19.968079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1911,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.158182+0000","last_change":"2024-08-17T18:08:57.572991+0000","last_active":"2024-08-17T18:12:25.158182+0000","last_peered":"2024-08-17T18:12:25.158182+0000","last_clean":"2024-08-17T18:12:25.158182+0000","last_became_active":"2024-08-17T18:08:57.571145+0000","last_became_peered":"2024-08-17T18:08:57.571145+0000","last_unstale":"2024-08-17T18:12:25.158182+0000","last_undegraded":"2024-08-17T18:12:25.158182+0000","last_fullsized":"2024-08-17T18:12:25.158182+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:39.963337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":867,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.143668+0000","last_change":"2024-08-17T18:09:28.860288+0000","last_active":"2024-08-17T18:12:24.143668+0000","last_peered":"2024-08-17T18:12:24.143668+0000","last_clean":"2024-08-17T18:12:24.143668+0000","last_became_active":"2024-08-17T18:09:28.860004+0000","last_became_peered":"2024-08-17T18:09:28.860004+0000","last_unstale":"2024-08-17T18:12:24.143668+0000","last_undegraded":"2024-08-17T18:12:24.143668+0000","last_fullsized":"2024-08-17T18:12:24.143668+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:57:33.054055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":660,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.143493+0000","last_change":"2024-08-17T18:09:28.860277+0000","last_active":"2024-08-17T18:12:24.143493+0000","last_peered":"2024-08-17T18:12:24.143493+0000","last_clean":"2024-08-17T18:12:24.143493+0000","last_became_active":"2024-08-17T18:09:28.859999+0000","last_became_peered":"2024-08-17T18:09:28.859999+0000","last_unstale":"2024-08-17T18:12:24.143493+0000","last_undegraded":"2024-08-17T18:12:24.143493+0000","last_fullsized":"2024-08-17T18:12:24.143493+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:27.946526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1565,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.156931+0000","last_change":"2024-08-17T18:09:29.863669+0000","last_active":"2024-08-17T18:12:25.156931+0000","last_peered":"2024-08-17T18:12:25.156931+0000","last_clean":"2024-08-17T18:12:25.156931+0000","last_became_active":"2024-08-17T18:09:29.862781+0000","last_became_peered":"2024-08-17T18:09:29.862781+0000","last_unstale":"2024-08-17T18:12:25.156931+0000","last_undegraded":"2024-08-17T18:12:25.156931+0000","last_fullsized":"2024-08-17T18:12:25.156931+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:14.073233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":146,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.157004+0000","last_change":"2024-08-17T18:09:29.843574+0000","last_active":"2024-08-17T18:12:25.157004+0000","last_peered":"2024-08-17T18:12:25.157004+0000","last_clean":"2024-08-17T18:12:25.157004+0000","last_became_active":"2024-08-17T18:09:29.843097+0000","last_became_peered":"2024-08-17T18:09:29.843097+0000","last_unstale":"2024-08-17T18:12:25.157004+0000","last_undegraded":"2024-08-17T18:12:25.157004+0000","last_fullsized":"2024-08-17T18:12:25.157004+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:46.740888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1910,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.157434+0000","last_change":"2024-08-17T18:09:29.847264+0000","last_active":"2024-08-17T18:12:25.157434+0000","last_peered":"2024-08-17T18:12:25.157434+0000","last_clean":"2024-08-17T18:12:25.157434+0000","last_became_active":"2024-08-17T18:09:29.846226+0000","last_became_peered":"2024-08-17T18:09:29.846226+0000","last_unstale":"2024-08-17T18:12:25.157434+0000","last_undegraded":"2024-08-17T18:12:25.157434+0000","last_fullsized":"2024-08-17T18:12:25.157434+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:44.135977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2110,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.156168+0000","last_change":"2024-08-17T18:09:29.845209+0000","last_active":"2024-08-17T18:12:25.156168+0000","last_peered":"2024-08-17T18:12:25.156168+0000","last_clean":"2024-08-17T18:12:25.156168+0000","last_became_active":"2024-08-17T18:09:29.844910+0000","last_became_peered":"2024-08-17T18:09:29.844910+0000","last_unstale":"2024-08-17T18:12:25.156168+0000","last_undegraded":"2024-08-17T18:12:25.156168+0000","last_fullsized":"2024-08-17T18:12:25.156168+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:14.586354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1983,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.141335+0000","last_change":"2024-08-17T18:09:28.859050+0000","last_active":"2024-08-17T18:12:24.141335+0000","last_peered":"2024-08-17T18:12:24.141335+0000","last_clean":"2024-08-17T18:12:24.141335+0000","last_became_active":"2024-08-17T18:09:28.858214+0000","last_became_peered":"2024-08-17T18:09:28.858214+0000","last_unstale":"2024-08-17T18:12:24.141335+0000","last_undegraded":"2024-08-17T18:12:24.141335+0000","last_fullsized":"2024-08-17T18:12:24.141335+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:54.146592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2235,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.157787+0000","last_change":"2024-08-17T18:08:57.572610+0000","last_active":"2024-08-17T18:12:25.157787+0000","last_peered":"2024-08-17T18:12:25.157787+0000","last_clean":"2024-08-17T18:12:25.157787+0000","last_became_active":"2024-08-17T18:08:57.572308+0000","last_became_peered":"2024-08-17T18:08:57.572308+0000","last_unstale":"2024-08-17T18:12:25.157787+0000","last_undegraded":"2024-08-17T18:12:25.157787+0000","last_fullsized":"2024-08-17T18:12:25.157787+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:11.642831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2219,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.156188+0000","last_change":"2024-08-17T18:09:28.842838+0000","last_active":"2024-08-17T18:12:25.156188+0000","last_peered":"2024-08-17T18:12:25.156188+0000","last_clean":"2024-08-17T18:12:25.156188+0000","last_became_active":"2024-08-17T18:09:28.842617+0000","last_became_peered":"2024-08-17T18:09:28.842617+0000","last_unstale":"2024-08-17T18:12:25.156188+0000","last_undegraded":"2024-08-17T18:12:25.156188+0000","last_fullsized":"2024-08-17T18:12:25.156188+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:00.016002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1399,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.036861+0000","last_change":"2024-08-17T18:09:29.845011+0000","last_active":"2024-08-17T18:12:12.036861+0000","last_peered":"2024-08-17T18:12:12.036861+0000","last_clean":"2024-08-17T18:12:12.036861+0000","last_became_active":"2024-08-17T18:09:29.844358+0000","last_became_peered":"2024-08-17T18:09:29.844358+0000","last_unstale":"2024-08-17T18:12:12.036861+0000","last_undegraded":"2024-08-17T18:12:12.036861+0000","last_fullsized":"2024-08-17T18:12:12.036861+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:41:27.792901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1247,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.038295+0000","last_change":"2024-08-17T18:09:28.878263+0000","last_active":"2024-08-17T18:12:12.038295+0000","last_peered":"2024-08-17T18:12:12.038295+0000","last_clean":"2024-08-17T18:12:12.038295+0000","last_became_active":"2024-08-17T18:09:28.878025+0000","last_became_peered":"2024-08-17T18:09:28.878025+0000","last_unstale":"2024-08-17T18:12:12.038295+0000","last_undegraded":"2024-08-17T18:12:12.038295+0000","last_fullsized":"2024-08-17T18:12:12.038295+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:37.211100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2011,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.141545+0000","last_change":"2024-08-17T18:09:29.840134+0000","last_active":"2024-08-17T18:12:24.141545+0000","last_peered":"2024-08-17T18:12:24.141545+0000","last_clean":"2024-08-17T18:12:24.141545+0000","last_became_active":"2024-08-17T18:09:29.839848+0000","last_became_peered":"2024-08-17T18:09:29.839848+0000","last_unstale":"2024-08-17T18:12:24.141545+0000","last_undegraded":"2024-08-17T18:12:24.141545+0000","last_fullsized":"2024-08-17T18:12:24.141545+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:08:08.717851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":154,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.039352+0000","last_change":"2024-08-17T18:09:28.847633+0000","last_active":"2024-08-17T18:12:12.039352+0000","last_peered":"2024-08-17T18:12:12.039352+0000","last_clean":"2024-08-17T18:12:12.039352+0000","last_became_active":"2024-08-17T18:09:28.847341+0000","last_became_peered":"2024-08-17T18:09:28.847341+0000","last_unstale":"2024-08-17T18:12:12.039352+0000","last_undegraded":"2024-08-17T18:12:12.039352+0000","last_fullsized":"2024-08-17T18:12:12.039352+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:51:59.625154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1739,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.036713+0000","last_change":"2024-08-17T18:09:00.609431+0000","last_active":"2024-08-17T18:12:12.036713+0000","last_peered":"2024-08-17T18:12:12.036713+0000","last_clean":"2024-08-17T18:12:12.036713+0000","last_became_active":"2024-08-17T18:09:00.609167+0000","last_became_peered":"2024-08-17T18:09:00.609167+0000","last_unstale":"2024-08-17T18:12:12.036713+0000","last_undegraded":"2024-08-17T18:12:12.036713+0000","last_fullsized":"2024-08-17T18:12:12.036713+0000","mapping_epoch":1338,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:39.579970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1891,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.146555+0000","last_change":"2024-08-17T18:09:29.837502+0000","last_active":"2024-08-17T18:12:24.146555+0000","last_peered":"2024-08-17T18:12:24.146555+0000","last_clean":"2024-08-17T18:12:24.146555+0000","last_became_active":"2024-08-17T18:09:29.837243+0000","last_became_peered":"2024-08-17T18:09:29.837243+0000","last_unstale":"2024-08-17T18:12:24.146555+0000","last_undegraded":"2024-08-17T18:12:24.146555+0000","last_fullsized":"2024-08-17T18:12:24.146555+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:47.377505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2089,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.141536+0000","last_change":"2024-08-17T18:09:29.859415+0000","last_active":"2024-08-17T18:12:24.141536+0000","last_peered":"2024-08-17T18:12:24.141536+0000","last_clean":"2024-08-17T18:12:24.141536+0000","last_became_active":"2024-08-17T18:09:29.859138+0000","last_became_peered":"2024-08-17T18:09:29.859138+0000","last_unstale":"2024-08-17T18:12:24.141536+0000","last_undegraded":"2024-08-17T18:12:24.141536+0000","last_fullsized":"2024-08-17T18:12:24.141536+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:31.178915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":540,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.145091+0000","last_change":"2024-08-17T18:09:29.883834+0000","last_active":"2024-08-17T18:12:24.145091+0000","last_peered":"2024-08-17T18:12:24.145091+0000","last_clean":"2024-08-17T18:12:24.145091+0000","last_became_active":"2024-08-17T18:09:29.882638+0000","last_became_peered":"2024-08-17T18:09:29.882638+0000","last_unstale":"2024-08-17T18:12:24.145091+0000","last_undegraded":"2024-08-17T18:12:24.145091+0000","last_fullsized":"2024-08-17T18:12:24.145091+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:07:10.188684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":639,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.037569+0000","last_change":"2024-08-17T18:09:29.857250+0000","last_active":"2024-08-17T18:12:12.037569+0000","last_peered":"2024-08-17T18:12:12.037569+0000","last_clean":"2024-08-17T18:12:12.037569+0000","last_became_active":"2024-08-17T18:09:29.857038+0000","last_became_peered":"2024-08-17T18:09:29.857038+0000","last_unstale":"2024-08-17T18:12:12.037569+0000","last_undegraded":"2024-08-17T18:12:12.037569+0000","last_fullsized":"2024-08-17T18:12:12.037569+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:02:54.028575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":653,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.156704+0000","last_change":"2024-08-17T18:08:57.571270+0000","last_active":"2024-08-17T18:12:25.156704+0000","last_peered":"2024-08-17T18:12:25.156704+0000","last_clean":"2024-08-17T18:12:25.156704+0000","last_became_active":"2024-08-17T18:08:57.567670+0000","last_became_peered":"2024-08-17T18:08:57.567670+0000","last_unstale":"2024-08-17T18:12:25.156704+0000","last_undegraded":"2024-08-17T18:12:25.156704+0000","last_fullsized":"2024-08-17T18:12:25.156704+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:19:42.617321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1224,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.141694+0000","last_change":"2024-08-17T18:09:29.839260+0000","last_active":"2024-08-17T18:12:24.141694+0000","last_peered":"2024-08-17T18:12:24.141694+0000","last_clean":"2024-08-17T18:12:24.141694+0000","last_became_active":"2024-08-17T18:09:29.838953+0000","last_became_peered":"2024-08-17T18:09:29.838953+0000","last_unstale":"2024-08-17T18:12:24.141694+0000","last_undegraded":"2024-08-17T18:12:24.141694+0000","last_fullsized":"2024-08-17T18:12:24.141694+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:44.014083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":813,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.144645+0000","last_change":"2024-08-17T18:09:29.851347+0000","last_active":"2024-08-17T18:12:24.144645+0000","last_peered":"2024-08-17T18:12:24.144645+0000","last_clean":"2024-08-17T18:12:24.144645+0000","last_became_active":"2024-08-17T18:09:29.851046+0000","last_became_peered":"2024-08-17T18:09:29.851046+0000","last_unstale":"2024-08-17T18:12:24.144645+0000","last_undegraded":"2024-08-17T18:12:24.144645+0000","last_fullsized":"2024-08-17T18:12:24.144645+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:21.350703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2090,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.142859+0000","last_change":"2024-08-17T18:09:29.912446+0000","last_active":"2024-08-17T18:12:24.142859+0000","last_peered":"2024-08-17T18:12:24.142859+0000","last_clean":"2024-08-17T18:12:24.142859+0000","last_became_active":"2024-08-17T18:09:29.910703+0000","last_became_peered":"2024-08-17T18:09:29.910703+0000","last_unstale":"2024-08-17T18:12:24.142859+0000","last_undegraded":"2024-08-17T18:12:24.142859+0000","last_fullsized":"2024-08-17T18:12:24.142859+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.250608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2256,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.157772+0000","last_change":"2024-08-17T18:09:29.863361+0000","last_active":"2024-08-17T18:12:25.157772+0000","last_peered":"2024-08-17T18:12:25.157772+0000","last_clean":"2024-08-17T18:12:25.157772+0000","last_became_active":"2024-08-17T18:09:29.862503+0000","last_became_peered":"2024-08-17T18:09:29.862503+0000","last_unstale":"2024-08-17T18:12:25.157772+0000","last_undegraded":"2024-08-17T18:12:25.157772+0000","last_fullsized":"2024-08-17T18:12:25.157772+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:25.908232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":79,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.143203+0000","last_change":"2024-08-17T18:09:29.895991+0000","last_active":"2024-08-17T18:12:24.143203+0000","last_peered":"2024-08-17T18:12:24.143203+0000","last_clean":"2024-08-17T18:12:24.143203+0000","last_became_active":"2024-08-17T18:09:29.895274+0000","last_became_peered":"2024-08-17T18:09:29.895274+0000","last_unstale":"2024-08-17T18:12:24.143203+0000","last_undegraded":"2024-08-17T18:12:24.143203+0000","last_fullsized":"2024-08-17T18:12:24.143203+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:19:26.420016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":521,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.037263+0000","last_change":"2024-08-17T18:09:29.844686+0000","last_active":"2024-08-17T18:12:12.037263+0000","last_peered":"2024-08-17T18:12:12.037263+0000","last_clean":"2024-08-17T18:12:12.037263+0000","last_became_active":"2024-08-17T18:09:29.844341+0000","last_became_peered":"2024-08-17T18:09:29.844341+0000","last_unstale":"2024-08-17T18:12:12.037263+0000","last_undegraded":"2024-08-17T18:12:12.037263+0000","last_fullsized":"2024-08-17T18:12:12.037263+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:38.872767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1358,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.038368+0000","last_change":"2024-08-17T18:09:28.884310+0000","last_active":"2024-08-17T18:12:12.038368+0000","last_peered":"2024-08-17T18:12:12.038368+0000","last_clean":"2024-08-17T18:12:12.038368+0000","last_became_active":"2024-08-17T18:09:28.883921+0000","last_became_peered":"2024-08-17T18:09:28.883921+0000","last_unstale":"2024-08-17T18:12:12.038368+0000","last_undegraded":"2024-08-17T18:12:12.038368+0000","last_fullsized":"2024-08-17T18:12:12.038368+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:05.272563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":822,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.141256+0000","last_change":"2024-08-17T18:09:29.837238+0000","last_active":"2024-08-17T18:12:24.141256+0000","last_peered":"2024-08-17T18:12:24.141256+0000","last_clean":"2024-08-17T18:12:24.141256+0000","last_became_active":"2024-08-17T18:09:29.836760+0000","last_became_peered":"2024-08-17T18:09:29.836760+0000","last_unstale":"2024-08-17T18:12:24.141256+0000","last_undegraded":"2024-08-17T18:12:24.141256+0000","last_fullsized":"2024-08-17T18:12:24.141256+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:38:52.862967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1373,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.038631+0000","last_change":"2024-08-17T18:09:29.871872+0000","last_active":"2024-08-17T18:12:12.038631+0000","last_peered":"2024-08-17T18:12:12.038631+0000","last_clean":"2024-08-17T18:12:12.038631+0000","last_became_active":"2024-08-17T18:09:29.845111+0000","last_became_peered":"2024-08-17T18:09:29.845111+0000","last_unstale":"2024-08-17T18:12:12.038631+0000","last_undegraded":"2024-08-17T18:12:12.038631+0000","last_fullsized":"2024-08-17T18:12:12.038631+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:48.183250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":270,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.036665+0000","last_change":"2024-08-17T18:09:29.844786+0000","last_active":"2024-08-17T18:12:12.036665+0000","last_peered":"2024-08-17T18:12:12.036665+0000","last_clean":"2024-08-17T18:12:12.036665+0000","last_became_active":"2024-08-17T18:09:29.844270+0000","last_became_peered":"2024-08-17T18:09:29.844270+0000","last_unstale":"2024-08-17T18:12:12.036665+0000","last_undegraded":"2024-08-17T18:12:12.036665+0000","last_fullsized":"2024-08-17T18:12:12.036665+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:51.866680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1389,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.037541+0000","last_change":"2024-08-17T18:09:29.886391+0000","last_active":"2024-08-17T18:12:12.037541+0000","last_peered":"2024-08-17T18:12:12.037541+0000","last_clean":"2024-08-17T18:12:12.037541+0000","last_became_active":"2024-08-17T18:09:29.844642+0000","last_became_peered":"2024-08-17T18:09:29.844642+0000","last_unstale":"2024-08-17T18:12:12.037541+0000","last_undegraded":"2024-08-17T18:12:12.037541+0000","last_fullsized":"2024-08-17T18:12:12.037541+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:07.988100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":236,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.156014+0000","last_change":"2024-08-17T18:09:28.941223+0000","last_active":"2024-08-17T18:12:25.156014+0000","last_peered":"2024-08-17T18:12:25.156014+0000","last_clean":"2024-08-17T18:12:25.156014+0000","last_became_active":"2024-08-17T18:09:28.940857+0000","last_became_peered":"2024-08-17T18:09:28.940857+0000","last_unstale":"2024-08-17T18:12:25.156014+0000","last_undegraded":"2024-08-17T18:12:25.156014+0000","last_fullsized":"2024-08-17T18:12:25.156014+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:48.809420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2087,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.146918+0000","last_change":"2024-08-17T18:09:29.838873+0000","last_active":"2024-08-17T18:12:24.146918+0000","last_peered":"2024-08-17T18:12:24.146918+0000","last_clean":"2024-08-17T18:12:24.146918+0000","last_became_active":"2024-08-17T18:09:29.838560+0000","last_became_peered":"2024-08-17T18:09:29.838560+0000","last_unstale":"2024-08-17T18:12:24.146918+0000","last_undegraded":"2024-08-17T18:12:24.146918+0000","last_fullsized":"2024-08-17T18:12:24.146918+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:00.103741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1349,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.037695+0000","last_change":"2024-08-17T18:09:28.844170+0000","last_active":"2024-08-17T18:12:12.037695+0000","last_peered":"2024-08-17T18:12:12.037695+0000","last_clean":"2024-08-17T18:12:12.037695+0000","last_became_active":"2024-08-17T18:09:28.843862+0000","last_became_peered":"2024-08-17T18:09:28.843862+0000","last_unstale":"2024-08-17T18:12:12.037695+0000","last_undegraded":"2024-08-17T18:12:12.037695+0000","last_fullsized":"2024-08-17T18:12:12.037695+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:43:19.483326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2209,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.156093+0000","last_change":"2024-08-17T18:09:28.864923+0000","last_active":"2024-08-17T18:12:25.156093+0000","last_peered":"2024-08-17T18:12:25.156093+0000","last_clean":"2024-08-17T18:12:25.156093+0000","last_became_active":"2024-08-17T18:09:28.864707+0000","last_became_peered":"2024-08-17T18:09:28.864707+0000","last_unstale":"2024-08-17T18:12:25.156093+0000","last_undegraded":"2024-08-17T18:12:25.156093+0000","last_fullsized":"2024-08-17T18:12:25.156093+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:27.507329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2211,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:25.156024+0000","last_change":"2024-08-17T18:09:28.846741+0000","last_active":"2024-08-17T18:12:25.156024+0000","last_peered":"2024-08-17T18:12:25.156024+0000","last_clean":"2024-08-17T18:12:25.156024+0000","last_became_active":"2024-08-17T18:09:28.846478+0000","last_became_peered":"2024-08-17T18:09:28.846478+0000","last_unstale":"2024-08-17T18:12:25.156024+0000","last_undegraded":"2024-08-17T18:12:25.156024+0000","last_fullsized":"2024-08-17T18:12:25.156024+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:34.247517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":755,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.038152+0000","last_change":"2024-08-17T18:03:16.091420+0000","last_active":"2024-08-17T18:12:12.038152+0000","last_peered":"2024-08-17T18:12:12.038152+0000","last_clean":"2024-08-17T18:12:12.038152+0000","last_became_active":"2024-08-17T18:03:16.091113+0000","last_became_peered":"2024-08-17T18:03:16.091113+0000","last_unstale":"2024-08-17T18:12:12.038152+0000","last_undegraded":"2024-08-17T18:12:12.038152+0000","last_fullsized":"2024-08-17T18:12:12.038152+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:47.770911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1363,"reported_epoch":1377,"state":"active+clean","last_fresh":"2024-08-17T18:12:12.039349+0000","last_change":"2024-08-17T18:02:53.893927+0000","last_active":"2024-08-17T18:12:12.039349+0000","last_peered":"2024-08-17T18:12:12.039349+0000","last_clean":"2024-08-17T18:12:12.039349+0000","last_became_active":"2024-08-17T18:02:53.883188+0000","last_became_peered":"2024-08-17T18:02:53.883188+0000","last_unstale":"2024-08-17T18:12:12.039349+0000","last_undegraded":"2024-08-17T18:12:12.039349+0000","last_fullsized":"2024-08-17T18:12:12.039349+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:19.296965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2087,"reported_epoch":1378,"state":"active+clean","last_fresh":"2024-08-17T18:12:24.143368+0000","last_change":"2024-08-17T18:09:29.840086+0000","last_active":"2024-08-17T18:12:24.143368+0000","last_peered":"2024-08-17T18:12:24.143368+0000","last_clean":"2024-08-17T18:12:24.143368+0000","last_became_active":"2024-08-17T18:09:29.839525+0000","last_became_peered":"2024-08-17T18:09:29.839525+0000","last_unstale":"2024-08-17T18:12:24.143368+0000","last_undegraded":"2024-08-17T18:12:24.143368+0000","last_fullsized":"2024-08-17T18:12:24.143368+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:38.316235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":10970,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":155648,"data_stored":27490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234619,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":905080,"kb_used_data":11504,"kb_used_omap":1,"kb_used_meta":893566,"kb_avail":92840072,"statfs":{"total":95995035648,"available":95068233728,"internally_reserved":0,"allocated":11780096,"data_stored":7180212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915011928},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[]},{"osd":6,"up_from":0,"seq":197568496254,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757892,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1343,"seq":5768141078565,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":247404,"kb_used_data":10912,"kb_used_omap":112,"kb_used_meta":236367,"kb_avail":93497748,"statfs":{"total":95995035648,"available":95741693952,"internally_reserved":0,"allocated":11173888,"data_stored":6585648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":115373,"internal_metadata":242040147},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405744,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1034652,"kb_used_data":11472,"kb_used_omap":61,"kb_used_meta":1023106,"kb_avail":92710500,"statfs":{"total":95995035648,"available":94935552000,"internally_reserved":0,"allocated":11747328,"data_stored":7171716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":1047661496},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":0,"seq":2173253452279,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474970,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":900800,"kb_used_data":11508,"kb_used_omap":119,"kb_used_meta":889160,"kb_avail":92844352,"statfs":{"total":95995035648,"available":95072616448,"internally_reserved":0,"allocated":11784192,"data_stored":7180794,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":910500574},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019527,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:12:31.527 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T18:12:31.528 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:12:31.528 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-08-17T18:12:31.528 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.0 'trim stale osdmaps' 2024-08-17T18:12:31.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:31.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:31 smithi179 ceph-mon[93519]: pgmap v2515: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:31 smithi179 ceph-mon[93519]: from='client.36897 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:12:31.599 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:31.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:31.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:31.701 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:31.726 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:31.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:31.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:31.728 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:12:31.728 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:12:31.743 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:31.754 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:12:31.755 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:12:31.781 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:12:31.781 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:12:31.790 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:31.791 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:31.803 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:12:31.803 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:12:31.822 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:12:31.822 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:12:31.828 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:31.837 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:12:31.837 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:12:31.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:31.851 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:31.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:31.853 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:12:31.853 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:12:31.869 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:12:31.869 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:12:31.952 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:31.973 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:31.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:32.075 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:32.097 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:32.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:32.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:32.199 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:32.220 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:32.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:32.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:32.322 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:32.343 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:32.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:32.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:32.368 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2849982871' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:32.368 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2849982871' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:32.446 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:32.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:32.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:32.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:32.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2849982871' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:32.595 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:32.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:32.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:32.697 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:32.718 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:32.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:32.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:32.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:32.842 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:32.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:32.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:32.944 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:32.966 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:32.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:32.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:33.067 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:33.093 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:33.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:33.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:33.195 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:33.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:33.217 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:33.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:33.318 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:33.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:33.345 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:33.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:33.391 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:33 smithi130 ceph-mon[105521]: pgmap v2516: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:33.697 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:33 smithi130 ceph-mon[110068]: pgmap v2516: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:33.697 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:33.698 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:33 smithi179 ceph-mon[93519]: pgmap v2516: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:33.724 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:33.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:33.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:33.826 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:33.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:33.852 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:33.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:33.953 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:33.977 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:33.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:33.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:34.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:34.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:34.101 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:34.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:34.202 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:34.229 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:34.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:34.330 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:34.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:34.356 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:34.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:34.457 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:34.483 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:34.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:34.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:34.585 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:34.609 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:34.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:34.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:34.712 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:34.737 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:34.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:34.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:34.839 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:34.865 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:34.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:34.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:34.966 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:34.992 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:34.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:34.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:35.093 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:35.115 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:35.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:35.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:35.217 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:35.243 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:35.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:35.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:35.345 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:35.369 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:35.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:35.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:35.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:35 smithi130 ceph-mon[105521]: pgmap v2517: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:35 smithi130 ceph-mon[110068]: pgmap v2517: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:35.471 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:35.496 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:35.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:35.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:35 smithi179 ceph-mon[93519]: pgmap v2517: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:35.598 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:35.621 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:35.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:35.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:35.723 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:35.746 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:35.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:35.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:35.848 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:35.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:35.872 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:35.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:35.973 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:35.974 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:35.999 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:36.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:36.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:36.101 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:36.125 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:36.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:36.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:36.171 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:12:36.226 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:36.251 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:36.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:36.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:36.353 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:36.376 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:36.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:36.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:36.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:36 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1290484255' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:36.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:36 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1290484255' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:36.478 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:36.504 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:36.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:36.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:36 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1290484255' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:36.605 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:36.630 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:36.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:36.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:36.732 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:36.757 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:36.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:36.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:36.859 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:36.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:36.883 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:36.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:36.984 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:37.010 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:37.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:37.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:37.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:37.137 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:37.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:37.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:37.239 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:37.263 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:37.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:37.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:37.365 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:37.390 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:37.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:37.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:37.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:37 smithi130 ceph-mon[105521]: pgmap v2518: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:37.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:37 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:12:37.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:37 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T18:12:37.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:37 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1290484255' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:12:37.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:37 smithi130 ceph-mon[105521]: osdmap e1379: 8 total, 8 up, 4 in 2024-08-17T18:12:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:37 smithi130 ceph-mon[110068]: pgmap v2518: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:37 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:12:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:37 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T18:12:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:37 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1290484255' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:12:37.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:37 smithi130 ceph-mon[110068]: osdmap e1379: 8 total, 8 up, 4 in 2024-08-17T18:12:37.492 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:37.515 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:37.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:37.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:37 smithi179 ceph-mon[93519]: pgmap v2518: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:37 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:12:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:37 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T18:12:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:37 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1290484255' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:12:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:37 smithi179 ceph-mon[93519]: osdmap e1379: 8 total, 8 up, 4 in 2024-08-17T18:12:37.618 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:37.639 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:37.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:37.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:37.680 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:12:37.680 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:12:37.680 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:12:37.680 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:12:37.680 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.680 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:138777 48.a 48:580474fd:::benchmark_data_smithi130_327597_object6812:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1340)", 2024-08-17T18:12:37.680 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:19.250102+0000", 2024-08-17T18:12:37.680 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 198.39719219700001, 2024-08-17T18:12:37.680 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.20467937, 2024-08-17T18:12:37.680 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.681 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.681 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.681 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:12:37.681 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:12:37.681 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:12:37.681 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 138777 2024-08-17T18:12:37.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.681 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.681 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.681 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.681 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.250102+0000", 2024-08-17T18:12:37.681 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.681 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.250102+0000", 2024-08-17T18:12:37.682 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.682 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.250102+0000", 2024-08-17T18:12:37.682 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7599999999999997e-07 2024-08-17T18:12:37.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.682 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.250105+0000", 2024-08-17T18:12:37.682 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9390000000000002e-06 2024-08-17T18:12:37.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.683 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.683 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.683 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.250113+0000", 2024-08-17T18:12:37.683 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2169999999999994e-06 2024-08-17T18:12:37.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.683 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.683 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.683 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.250119+0000", 2024-08-17T18:12:37.683 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3650000000000003e-06 2024-08-17T18:12:37.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.683 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.683 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.683 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.439110+0000", 2024-08-17T18:12:37.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.18899086900000001 2024-08-17T18:12:37.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.684 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.684 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.439244+0000", 2024-08-17T18:12:37.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013408499999999999 2024-08-17T18:12:37.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.684 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T18:12:37.684 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.439379+0000", 2024-08-17T18:12:37.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000135511 2024-08-17T18:12:37.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.684 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.454646+0000", 2024-08-17T18:12:37.685 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015266309 2024-08-17T18:12:37.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:12:37.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.454658+0000", 2024-08-17T18:12:37.685 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.204e-05 2024-08-17T18:12:37.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.454691+0000", 2024-08-17T18:12:37.685 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3566999999999999e-05 2024-08-17T18:12:37.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.686 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.686 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.454741+0000", 2024-08-17T18:12:37.686 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9962e-05 2024-08-17T18:12:37.686 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.686 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.686 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.686 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.454781+0000", 2024-08-17T18:12:37.686 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9830000000000003e-05 2024-08-17T18:12:37.686 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.686 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.686 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.686 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.686 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.687 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:138772 48.2 e1340/1336 48:43bb1614:::benchmark_data_smithi130_327597_object6807:head v 1340'8698, mlcod=1340'8694)", 2024-08-17T18:12:37.687 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:19.250491+0000", 2024-08-17T18:12:37.687 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 198.396803143, 2024-08-17T18:12:37.687 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.202190909, 2024-08-17T18:12:37.687 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.687 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.687 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.687 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.687 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.687 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.687 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.250491+0000", 2024-08-17T18:12:37.688 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.688 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.688 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.688 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.688 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.250491+0000", 2024-08-17T18:12:37.688 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.688 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.688 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.688 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.688 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.250491+0000", 2024-08-17T18:12:37.688 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4799999999999998e-07 2024-08-17T18:12:37.688 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.688 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.688 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.250495+0000", 2024-08-17T18:12:37.689 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.228e-06 2024-08-17T18:12:37.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.250503+0000", 2024-08-17T18:12:37.689 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6830000000000008e-06 2024-08-17T18:12:37.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.250508+0000", 2024-08-17T18:12:37.689 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4190000000000001e-06 2024-08-17T18:12:37.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.690 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.690 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.690 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.259754+0000", 2024-08-17T18:12:37.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0092452960000000001 2024-08-17T18:12:37.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.690 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.690 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.690 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.259774+0000", 2024-08-17T18:12:37.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9851999999999999e-05 2024-08-17T18:12:37.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.690 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.694 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.694 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.450306+0000", 2024-08-17T18:12:37.694 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.19053282199999999 2024-08-17T18:12:37.694 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.694 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.694 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.694 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:19.452682+0000", 2024-08-17T18:12:37.694 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0023750609999999999 2024-08-17T18:12:37.694 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.694 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.694 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.694 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.695 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.695 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:220687 48.e 48:7b790c1c:::benchmark_data_smithi130_327597_object88722:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1354)", 2024-08-17T18:12:37.695 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:07.978857+0000", 2024-08-17T18:12:37.695 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 89.668436478999993, 2024-08-17T18:12:37.695 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.19067125500000001, 2024-08-17T18:12:37.695 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.695 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.695 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.695 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:12:37.695 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:12:37.695 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:12:37.695 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 220687 2024-08-17T18:12:37.695 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.696 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.696 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.696 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.978857+0000", 2024-08-17T18:12:37.696 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.696 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.696 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.696 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.978857+0000", 2024-08-17T18:12:37.696 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.696 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.696 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.697 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.978859+0000", 2024-08-17T18:12:37.697 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7239999999999999e-06 2024-08-17T18:12:37.697 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.697 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.697 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.697 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.978862+0000", 2024-08-17T18:12:37.697 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1659999999999998e-06 2024-08-17T18:12:37.697 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.697 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.697 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.697 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.978870+0000", 2024-08-17T18:12:37.697 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2060000000000007e-06 2024-08-17T18:12:37.697 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.698 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.698 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.698 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.978876+0000", 2024-08-17T18:12:37.698 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2839999999999997e-06 2024-08-17T18:12:37.698 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.698 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.698 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.698 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.998903+0000", 2024-08-17T18:12:37.698 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020027618000000001 2024-08-17T18:12:37.698 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.698 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.698 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.699 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.999028+0000", 2024-08-17T18:12:37.699 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000124435 2024-08-17T18:12:37.699 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.699 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.699 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:12:37.699 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.999152+0000", 2024-08-17T18:12:37.699 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012402 2024-08-17T18:12:37.699 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.699 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.699 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.699 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.004955+0000", 2024-08-17T18:12:37.699 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0058033360000000001 2024-08-17T18:12:37.699 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.699 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.700 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:12:37.700 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.018382+0000", 2024-08-17T18:12:37.700 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013427401 2024-08-17T18:12:37.700 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.700 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.700 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.700 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.020680+0000", 2024-08-17T18:12:37.700 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0022975869999999998 2024-08-17T18:12:37.700 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.700 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.700 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.700 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.169414+0000", 2024-08-17T18:12:37.701 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.14873363000000001 2024-08-17T18:12:37.701 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.701 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.701 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.701 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.169529+0000", 2024-08-17T18:12:37.701 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000114848 2024-08-17T18:12:37.701 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.701 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.701 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.701 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.701 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.701 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:220690 48.e 48:71b89b89:::benchmark_data_smithi130_327597_object88725:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1354)", 2024-08-17T18:12:37.701 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:07.986307+0000", 2024-08-17T18:12:37.702 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 89.660987250999995, 2024-08-17T18:12:37.702 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.188064279, 2024-08-17T18:12:37.702 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.702 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.702 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.702 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:12:37.702 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:12:37.702 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:12:37.702 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 220690 2024-08-17T18:12:37.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.702 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.702 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.702 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.702 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.986307+0000", 2024-08-17T18:12:37.703 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.703 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.986307+0000", 2024-08-17T18:12:37.703 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.703 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.986308+0000", 2024-08-17T18:12:37.703 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4780000000000001e-06 2024-08-17T18:12:37.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.986313+0000", 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4299999999999997e-06 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.986333+0000", 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9318e-05 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.986341+0000", 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7509999999999997e-06 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.705 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.705 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.001965+0000", 2024-08-17T18:12:37.705 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015623066 2024-08-17T18:12:37.705 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.705 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.705 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.705 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.002090+0000", 2024-08-17T18:12:37.705 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000125253 2024-08-17T18:12:37.705 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.705 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.705 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:12:37.705 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.002218+0000", 2024-08-17T18:12:37.705 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000128169 2024-08-17T18:12:37.705 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.706 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.706 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.706 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.004997+0000", 2024-08-17T18:12:37.706 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027788420000000001 2024-08-17T18:12:37.706 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.706 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.706 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.706 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.169691+0000", 2024-08-17T18:12:37.706 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.16469426000000001 2024-08-17T18:12:37.706 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.706 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.706 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:12:37.706 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.174207+0000", 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.004515504 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.174267+0000", 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0511999999999999e-05 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.174371+0000", 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000103696 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.707 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:220700 48.e 48:7400d571:::benchmark_data_smithi130_327597_object88735:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1354)", 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:07.996145+0000", 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 89.651148749000001, 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.18161851600000001, 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 220700 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.708 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.996145+0000", 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.996145+0000", 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.996145+0000", 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5499999999999999e-07 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.709 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.996147+0000", 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.885e-06 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.996153+0000", 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.665e-06 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:07.996157+0000", 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4039999999999997e-06 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.711 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.005025+0000", 2024-08-17T18:12:37.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0088681060000000006 2024-08-17T18:12:37.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.711 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.005146+0000", 2024-08-17T18:12:37.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012069399999999999 2024-08-17T18:12:37.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:12:37.711 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.005270+0000", 2024-08-17T18:12:37.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012409899999999999 2024-08-17T18:12:37.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.018449+0000", 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013179124 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.169736+0000", 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.15128617899999999 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.177599+0000", 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0078634510000000005 2024-08-17T18:12:37.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.713 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.713 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.177655+0000", 2024-08-17T18:12:37.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5763000000000001e-05 2024-08-17T18:12:37.713 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.713 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.713 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.177764+0000", 2024-08-17T18:12:37.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000108791 2024-08-17T18:12:37.713 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.713 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.713 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.713 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.714 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:220705 48.6 e1354/1343 48:6b6bc097:::benchmark_data_smithi130_327597_object88740:head v 1354'13721, mlcod=1354'13717)", 2024-08-17T18:12:37.714 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:08.015484+0000", 2024-08-17T18:12:37.714 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 89.631809313000005, 2024-08-17T18:12:37.714 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.191474809, 2024-08-17T18:12:37.714 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.714 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.714 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.714 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.714 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.714 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.714 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.015484+0000", 2024-08-17T18:12:37.714 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.015484+0000", 2024-08-17T18:12:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.015485+0000", 2024-08-17T18:12:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.82e-07 2024-08-17T18:12:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.715 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.015489+0000", 2024-08-17T18:12:37.716 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4170000000000002e-06 2024-08-17T18:12:37.716 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.716 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.716 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.716 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.015497+0000", 2024-08-17T18:12:37.716 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1550000000000001e-06 2024-08-17T18:12:37.716 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.716 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.716 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.716 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.015501+0000", 2024-08-17T18:12:37.716 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7979999999999996e-06 2024-08-17T18:12:37.716 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.716 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.716 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.169776+0000", 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.154274723 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.169806+0000", 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0000000000000001e-05 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.206917+0000", 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.037111140000000001 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.717 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.206959+0000", 2024-08-17T18:12:37.718 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2094000000000001e-05 2024-08-17T18:12:37.718 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.718 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.718 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.718 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.718 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.718 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221952 48.e 48:71c7ac05:::benchmark_data_smithi130_327597_object89987:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1356)", 2024-08-17T18:12:37.718 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:10.587168+0000", 2024-08-17T18:12:37.718 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.060125997, 2024-08-17T18:12:37.718 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.23552592899999999, 2024-08-17T18:12:37.718 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.719 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.719 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.719 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:12:37.719 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:12:37.719 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:12:37.719 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221952 2024-08-17T18:12:37.719 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.719 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.719 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.719 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.719 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587168+0000", 2024-08-17T18:12:37.719 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.719 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587168+0000", 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587169+0000", 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7499999999999998e-07 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587172+0000", 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9900000000000002e-06 2024-08-17T18:12:37.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587180+0000", 2024-08-17T18:12:37.721 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9959999999999993e-06 2024-08-17T18:12:37.721 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587185+0000", 2024-08-17T18:12:37.721 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.66e-06 2024-08-17T18:12:37.721 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587394+0000", 2024-08-17T18:12:37.722 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020831499999999999 2024-08-17T18:12:37.722 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.722 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.722 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.722 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587526+0000", 2024-08-17T18:12:37.722 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013261100000000001 2024-08-17T18:12:37.722 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.722 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.722 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:12:37.722 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587662+0000", 2024-08-17T18:12:37.722 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013568900000000001 2024-08-17T18:12:37.722 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.722 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.597178+0000", 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0095163620000000004 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.604898+0000", 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0077198370000000002 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.822513+0000", 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.21761467400000001 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.723 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.724 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.724 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.822572+0000", 2024-08-17T18:12:37.724 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8941999999999999e-05 2024-08-17T18:12:37.724 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.724 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.724 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.724 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.822694+0000", 2024-08-17T18:12:37.724 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000121878 2024-08-17T18:12:37.724 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.724 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.724 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.724 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.724 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.725 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221954 48.9 48:9d9e3aaa:::benchmark_data_smithi130_327597_object89989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1356)", 2024-08-17T18:12:37.725 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:10.587381+0000", 2024-08-17T18:12:37.725 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.059912636999996, 2024-08-17T18:12:37.725 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.241624437, 2024-08-17T18:12:37.725 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.725 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.725 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.725 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:12:37.725 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:12:37.725 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:12:37.725 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221954 2024-08-17T18:12:37.725 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.726 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.726 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.726 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587381+0000", 2024-08-17T18:12:37.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.726 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.726 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.726 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587381+0000", 2024-08-17T18:12:37.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.726 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.726 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587382+0000", 2024-08-17T18:12:37.727 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4900000000000005e-07 2024-08-17T18:12:37.727 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587386+0000", 2024-08-17T18:12:37.727 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0319999999999997e-06 2024-08-17T18:12:37.727 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587396+0000", 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6660000000000003e-06 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.587402+0000", 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.392e-06 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.590865+0000", 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0034627049999999999 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.590983+0000", 2024-08-17T18:12:37.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000118146 2024-08-17T18:12:37.729 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.729 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.729 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:12:37.729 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.591113+0000", 2024-08-17T18:12:37.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013029299999999999 2024-08-17T18:12:37.729 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.729 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.729 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.729 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.603968+0000", 2024-08-17T18:12:37.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012855337 2024-08-17T18:12:37.729 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.730 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.730 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:12:37.730 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.604146+0000", 2024-08-17T18:12:37.730 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017804799999999999 2024-08-17T18:12:37.730 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.730 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.730 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.730 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.828839+0000", 2024-08-17T18:12:37.730 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.224692262 2024-08-17T18:12:37.730 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.730 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.730 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.731 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.828894+0000", 2024-08-17T18:12:37.731 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5534000000000001e-05 2024-08-17T18:12:37.731 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.731 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.731 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.731 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.829006+0000", 2024-08-17T18:12:37.731 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000111273 2024-08-17T18:12:37.731 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.731 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.731 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.731 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.731 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.731 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221958 48.9 48:9ff55c01:::benchmark_data_smithi130_327597_object89993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1356)", 2024-08-17T18:12:37.732 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:10.592832+0000", 2024-08-17T18:12:37.732 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.054462104999999, 2024-08-17T18:12:37.732 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.23646325700000001, 2024-08-17T18:12:37.732 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.732 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.732 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.732 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:12:37.732 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:12:37.732 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:12:37.732 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221958 2024-08-17T18:12:37.732 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.732 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.732 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.732 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.592832+0000", 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.592832+0000", 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.592833+0000", 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2309999999999999e-06 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.592838+0000", 2024-08-17T18:12:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6609999999999999e-06 2024-08-17T18:12:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.592849+0000", 2024-08-17T18:12:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1601e-05 2024-08-17T18:12:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.592856+0000", 2024-08-17T18:12:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0659999999999998e-06 2024-08-17T18:12:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.734 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.735 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.735 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.598852+0000", 2024-08-17T18:12:37.735 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005995439 2024-08-17T18:12:37.735 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.735 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.735 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.735 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.598982+0000", 2024-08-17T18:12:37.735 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000130448 2024-08-17T18:12:37.735 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.735 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.735 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:12:37.735 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.599114+0000", 2024-08-17T18:12:37.735 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013195899999999999 2024-08-17T18:12:37.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.736 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.736 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:12:37.736 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.606845+0000", 2024-08-17T18:12:37.736 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0077307130000000002 2024-08-17T18:12:37.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.736 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.736 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.736 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.608536+0000", 2024-08-17T18:12:37.736 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0016909869999999999 2024-08-17T18:12:37.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.736 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.736 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.736 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.829044+0000", 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.220508333 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.829091+0000", 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7015999999999998e-05 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.829295+0000", 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020380299999999999 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.737 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.738 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.738 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221960 48.a 48:55d1dbd2:::benchmark_data_smithi130_327597_object89995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1356)", 2024-08-17T18:12:37.738 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:10.600368+0000", 2024-08-17T18:12:37.738 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 87.046925525999995, 2024-08-17T18:12:37.738 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22206870200000001, 2024-08-17T18:12:37.738 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.738 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.738 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.738 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:12:37.738 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:12:37.738 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:12:37.738 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221960 2024-08-17T18:12:37.738 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.738 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.739 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.739 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.739 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.600368+0000", 2024-08-17T18:12:37.739 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.739 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.739 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.739 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.739 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.600368+0000", 2024-08-17T18:12:37.739 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.739 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.739 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.739 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.739 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.600370+0000", 2024-08-17T18:12:37.739 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.79e-06 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.600376+0000", 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1530000000000002e-06 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.600389+0000", 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3029e-05 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.600398+0000", 2024-08-17T18:12:37.740 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0720000000000006e-06 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.600486+0000", 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7810000000000001e-05 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.600598+0000", 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000111853 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,7", 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.600707+0000", 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010943 2024-08-17T18:12:37.741 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.742 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.742 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:12:37.742 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.606826+0000", 2024-08-17T18:12:37.742 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0061188950000000001 2024-08-17T18:12:37.742 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.742 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.742 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.742 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.606887+0000", 2024-08-17T18:12:37.742 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1226e-05 2024-08-17T18:12:37.742 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.742 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.742 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.742 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.822338+0000", 2024-08-17T18:12:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.215450273 2024-08-17T18:12:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.822394+0000", 2024-08-17T18:12:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5884999999999998e-05 2024-08-17T18:12:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:10.822437+0000", 2024-08-17T18:12:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3285999999999999e-05 2024-08-17T18:12:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.743 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:238894 48.0 e1357/1343 48:01b524f1:::benchmark_data_smithi130_327597_object106929:head v 1357'14906, mlcod=1357'14904)", 2024-08-17T18:12:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:32.335906+0000", 2024-08-17T18:12:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 65.311387902000007, 2024-08-17T18:12:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.187274104, 2024-08-17T18:12:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.335906+0000", 2024-08-17T18:12:37.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.335906+0000", 2024-08-17T18:12:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.335907+0000", 2024-08-17T18:12:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4799999999999997e-07 2024-08-17T18:12:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.335912+0000", 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3199999999999999e-06 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.335922+0000", 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9529999999999994e-06 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.335928+0000", 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.981e-06 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.746 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.335976+0000", 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7757999999999999e-05 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.335998+0000", 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.287e-05 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.347792+0000", 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011793101 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.747 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.523180+0000", 2024-08-17T18:12:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.17538837299999999 2024-08-17T18:12:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:238893 48.8 e1357/1343 48:10be31f5:::benchmark_data_smithi130_327597_object106928:head v 1357'14931, mlcod=1357'14929)", 2024-08-17T18:12:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:32.339520+0000", 2024-08-17T18:12:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 65.307774226999996, 2024-08-17T18:12:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.18524884699999999, 2024-08-17T18:12:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.748 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.339520+0000", 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.339520+0000", 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.339520+0000", 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7400000000000002e-07 2024-08-17T18:12:37.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.339527+0000", 2024-08-17T18:12:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4760000000000003e-06 2024-08-17T18:12:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.339546+0000", 2024-08-17T18:12:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.925e-05 2024-08-17T18:12:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.339553+0000", 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3890000000000002e-06 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.339675+0000", 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012265799999999999 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.339698+0000", 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2447e-05 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.524716+0000", 2024-08-17T18:12:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.18501875700000001 2024-08-17T18:12:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.524768+0000", 2024-08-17T18:12:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1996000000000002e-05 2024-08-17T18:12:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.752 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:238897 48.e 48:7ab91d90:::benchmark_data_smithi130_327597_object106932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:12:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:32.340938+0000", 2024-08-17T18:12:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 65.306355346000004, 2024-08-17T18:12:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.18333833499999999, 2024-08-17T18:12:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:12:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:12:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:12:37.753 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 238897 2024-08-17T18:12:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.340938+0000", 2024-08-17T18:12:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.340938+0000", 2024-08-17T18:12:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.754 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.340940+0000", 2024-08-17T18:12:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8840000000000001e-06 2024-08-17T18:12:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.340946+0000", 2024-08-17T18:12:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3890000000000004e-06 2024-08-17T18:12:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.340958+0000", 2024-08-17T18:12:37.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2807e-05 2024-08-17T18:12:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.341016+0000", 2024-08-17T18:12:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7411999999999999e-05 2024-08-17T18:12:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.341077+0000", 2024-08-17T18:12:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1100999999999997e-05 2024-08-17T18:12:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.756 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.341193+0000", 2024-08-17T18:12:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011629999999999999 2024-08-17T18:12:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:12:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.341313+0000", 2024-08-17T18:12:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000119988 2024-08-17T18:12:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.343977+0000", 2024-08-17T18:12:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026635640000000002 2024-08-17T18:12:37.757 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:12:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.346552+0000", 2024-08-17T18:12:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025750899999999999 2024-08-17T18:12:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.524139+0000", 2024-08-17T18:12:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.17758724300000001 2024-08-17T18:12:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.758 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.524188+0000", 2024-08-17T18:12:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8758999999999999e-05 2024-08-17T18:12:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.524277+0000", 2024-08-17T18:12:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8797999999999999e-05 2024-08-17T18:12:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.759 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:238898 48.9 48:9c4a659d:::benchmark_data_smithi130_327597_object106933:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:12:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:32.341351+0000", 2024-08-17T18:12:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 65.305942591999994, 2024-08-17T18:12:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.182737013, 2024-08-17T18:12:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:12:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:12:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:12:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 238898 2024-08-17T18:12:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.760 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.341351+0000", 2024-08-17T18:12:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.341351+0000", 2024-08-17T18:12:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.761 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.341352+0000", 2024-08-17T18:12:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0049999999999999e-06 2024-08-17T18:12:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.341357+0000", 2024-08-17T18:12:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3810000000000001e-06 2024-08-17T18:12:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.762 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.341368+0000", 2024-08-17T18:12:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1528e-05 2024-08-17T18:12:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.341375+0000", 2024-08-17T18:12:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9059999999999996e-06 2024-08-17T18:12:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.763 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.341465+0000", 2024-08-17T18:12:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0458999999999996e-05 2024-08-17T18:12:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.341599+0000", 2024-08-17T18:12:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013327 2024-08-17T18:12:37.764 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:12:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.341713+0000", 2024-08-17T18:12:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011445100000000001 2024-08-17T18:12:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.344819+0000", 2024-08-17T18:12:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003105638 2024-08-17T18:12:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:12:37.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.346709+0000", 2024-08-17T18:12:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00189043 2024-08-17T18:12:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.523930+0000", 2024-08-17T18:12:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.17722045 2024-08-17T18:12:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.766 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.523984+0000", 2024-08-17T18:12:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4753000000000002e-05 2024-08-17T18:12:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.524088+0000", 2024-08-17T18:12:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000103742 2024-08-17T18:12:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:238994 48.9 48:998419c8:::benchmark_data_smithi130_327597_object107029:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:12:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:33.100882+0000", 2024-08-17T18:12:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 64.546411917, 2024-08-17T18:12:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.229325049, 2024-08-17T18:12:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:12:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:12:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:12:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 238994 2024-08-17T18:12:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.768 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.100882+0000", 2024-08-17T18:12:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.100882+0000", 2024-08-17T18:12:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.100884+0000", 2024-08-17T18:12:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9999999999999999e-06 2024-08-17T18:12:37.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.100889+0000", 2024-08-17T18:12:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3800000000000002e-06 2024-08-17T18:12:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.310166+0000", 2024-08-17T18:12:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.20927642299999999 2024-08-17T18:12:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.770 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.310180+0000", 2024-08-17T18:12:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4253000000000001e-05 2024-08-17T18:12:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.310250+0000", 2024-08-17T18:12:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9758000000000005e-05 2024-08-17T18:12:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.310388+0000", 2024-08-17T18:12:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013830299999999999 2024-08-17T18:12:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.310521+0000", 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013277300000000001 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.316320+0000", 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0057992490000000002 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.318131+0000", 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0018105510000000001 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.330041+0000", 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011910808 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.330100+0000", 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9021999999999998e-05 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.330207+0000", 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000106529 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.773 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:238997 48.2 e1357/1355 48:48c17ad2:::benchmark_data_smithi130_327597_object107032:head v 1357'14911, mlcod=1357'14909)", 2024-08-17T18:12:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:33.108479+0000", 2024-08-17T18:12:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 64.538814348000002, 2024-08-17T18:12:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.20896878599999999, 2024-08-17T18:12:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.774 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.108479+0000", 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.108479+0000", 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.108481+0000", 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.429e-06 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.108490+0000", 2024-08-17T18:12:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6439999999999995e-06 2024-08-17T18:12:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.310169+0000", 2024-08-17T18:12:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.20167821899999999 2024-08-17T18:12:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.310179+0000", 2024-08-17T18:12:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0708e-05 2024-08-17T18:12:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.776 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.310242+0000", 2024-08-17T18:12:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2553999999999998e-05 2024-08-17T18:12:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.310265+0000", 2024-08-17T18:12:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3149000000000001e-05 2024-08-17T18:12:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.317393+0000", 2024-08-17T18:12:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.007128255 2024-08-17T18:12:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.317448+0000", 2024-08-17T18:12:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4827999999999998e-05 2024-08-17T18:12:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:244994 48.5 48:afd58c00:::benchmark_data_smithi130_327597_object113029:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1360)", 2024-08-17T18:12:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:42.266322+0000", 2024-08-17T18:12:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 55.380971918, 2024-08-17T18:12:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.30333681200000001, 2024-08-17T18:12:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.778 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:12:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:12:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:12:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 244994 2024-08-17T18:12:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.266322+0000", 2024-08-17T18:12:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.779 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.266322+0000", 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.266323+0000", 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.452e-06 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.266329+0000", 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8019999999999997e-06 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.780 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.266340+0000", 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1119e-05 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.266348+0000", 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3220000000000004e-06 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.266414+0000", 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6416999999999998e-05 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.782 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.266535+0000", 2024-08-17T18:12:37.782 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000121161 2024-08-17T18:12:37.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.782 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.782 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:12:37.782 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.266656+0000", 2024-08-17T18:12:37.782 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000120532 2024-08-17T18:12:37.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.782 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.782 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:12:37.782 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.269543+0000", 2024-08-17T18:12:37.782 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002887471 2024-08-17T18:12:37.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.782 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.272456+0000", 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029132839999999999 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.569496+0000", 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.297039471 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.569549+0000", 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2911999999999998e-05 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.783 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.784 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.784 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.569659+0000", 2024-08-17T18:12:37.784 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000109869 2024-08-17T18:12:37.784 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.784 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.784 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.784 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.784 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop_reply(client.36675.0:244994 48.5 e1360/1344 ondisk, result = 0)", 2024-08-17T18:12:37.784 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:42.274433+0000", 2024-08-17T18:12:37.784 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 55.372860680999999, 2024-08-17T18:12:37.784 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.295230928, 2024-08-17T18:12:37.784 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.784 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.274433+0000", 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.274433+0000", 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.785 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.274434+0000", 2024-08-17T18:12:37.786 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9599999999999999e-07 2024-08-17T18:12:37.786 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.786 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.786 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.786 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.274438+0000", 2024-08-17T18:12:37.786 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9820000000000002e-06 2024-08-17T18:12:37.786 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.786 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.786 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.786 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.274449+0000", 2024-08-17T18:12:37.786 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1163e-05 2024-08-17T18:12:37.786 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.786 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.787 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.787 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.274456+0000", 2024-08-17T18:12:37.787 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.72e-06 2024-08-17T18:12:37.787 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.787 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.787 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.787 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.569485+0000", 2024-08-17T18:12:37.787 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.29502936299999999 2024-08-17T18:12:37.787 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.787 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.787 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.787 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.569491+0000", 2024-08-17T18:12:37.787 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4139999999999999e-06 2024-08-17T18:12:37.787 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.788 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.788 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.788 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.569664+0000", 2024-08-17T18:12:37.788 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017259 2024-08-17T18:12:37.788 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.788 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.788 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.788 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.788 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.788 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:244998 48.7 e1360/1343 48:e5add7ae:::benchmark_data_smithi130_327597_object113033:head v 1360'15241, mlcod=1360'15239)", 2024-08-17T18:12:37.788 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:42.274476+0000", 2024-08-17T18:12:37.788 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 55.372817818000001, 2024-08-17T18:12:37.788 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.30233234599999997, 2024-08-17T18:12:37.788 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.789 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.789 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.789 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.789 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.789 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.789 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.274476+0000", 2024-08-17T18:12:37.789 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.789 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.789 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.789 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.274476+0000", 2024-08-17T18:12:37.789 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.789 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.790 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.790 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.274476+0000", 2024-08-17T18:12:37.790 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4700000000000002e-07 2024-08-17T18:12:37.790 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.790 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.790 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.790 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.274480+0000", 2024-08-17T18:12:37.790 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9199999999999997e-06 2024-08-17T18:12:37.790 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.790 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.790 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.790 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.569368+0000", 2024-08-17T18:12:37.790 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.29488801199999998 2024-08-17T18:12:37.790 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.791 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.791 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.569384+0000", 2024-08-17T18:12:37.791 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5798e-05 2024-08-17T18:12:37.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.791 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.791 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.569460+0000", 2024-08-17T18:12:37.791 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5599000000000006e-05 2024-08-17T18:12:37.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.791 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.791 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.569487+0000", 2024-08-17T18:12:37.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.7583999999999999e-05 2024-08-17T18:12:37.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.792 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.792 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.792 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.572565+0000", 2024-08-17T18:12:37.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0030778730000000001 2024-08-17T18:12:37.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.792 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.792 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.792 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.576808+0000", 2024-08-17T18:12:37.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.004243213 2024-08-17T18:12:37.792 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.792 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.793 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.793 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.793 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.793 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:244986 48.e e1360/1358 48:767e6b03:::benchmark_data_smithi130_327597_object113021:head v 1360'15309, mlcod=1360'15306)", 2024-08-17T18:12:37.793 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:42.274531+0000", 2024-08-17T18:12:37.793 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 55.372762354000002, 2024-08-17T18:12:37.793 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.300193402, 2024-08-17T18:12:37.793 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:12:37.793 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:12:37.793 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:12:37.793 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:12:37.793 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.793 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:12:37.794 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.274531+0000", 2024-08-17T18:12:37.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.794 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.794 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:12:37.794 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.274531+0000", 2024-08-17T18:12:37.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:12:37.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.794 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.794 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:12:37.794 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.274532+0000", 2024-08-17T18:12:37.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9299999999999998e-07 2024-08-17T18:12:37.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.795 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.795 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:12:37.795 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.274538+0000", 2024-08-17T18:12:37.795 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0209999999999996e-06 2024-08-17T18:12:37.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.795 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.795 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:12:37.795 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.569385+0000", 2024-08-17T18:12:37.795 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.29484693200000001 2024-08-17T18:12:37.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.795 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.795 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:12:37.795 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.569398+0000", 2024-08-17T18:12:37.795 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.347e-05 2024-08-17T18:12:37.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.796 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:12:37.796 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.569630+0000", 2024-08-17T18:12:37.796 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000231828 2024-08-17T18:12:37.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.796 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:12:37.796 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.569651+0000", 2024-08-17T18:12:37.796 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0726000000000001e-05 2024-08-17T18:12:37.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.796 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.797 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:12:37.797 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.574656+0000", 2024-08-17T18:12:37.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0050049559999999996 2024-08-17T18:12:37.797 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:12:37.797 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:12:37.797 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:12:37.797 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.574725+0000", 2024-08-17T18:12:37.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8875999999999996e-05 2024-08-17T18:12:37.797 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.797 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.797 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.797 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:12:37.797 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:12:37.797 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:12:37.798 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:37.823 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:37.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:37.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:37.925 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:37.931 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:12:37.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:37.947 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:37.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:38.047 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:38.073 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:38.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:38.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:38.175 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:38.199 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:38.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:38.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:38.301 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:38.325 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:38.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:38.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:38.427 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:38.452 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:38.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:38.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:38.554 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:38.578 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:38.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:38.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:38.680 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:38.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:38.703 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:38.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:38.804 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:38.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:38.827 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:38.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:38.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:38.955 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:38.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:38.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:39.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:39.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:39.080 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:39.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:39.181 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:39.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:39.204 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:39.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:39.305 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:39.331 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:39.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:39.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:39.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:39 smithi130 ceph-mon[105521]: pgmap v2520: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:39 smithi130 ceph-mon[110068]: pgmap v2520: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:39.433 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:39.458 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:39.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:39.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:39.560 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:39.586 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:39.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:39.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:39.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:39 smithi179 ceph-mon[93519]: pgmap v2520: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:39.688 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:39.711 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:39.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:39.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:39.813 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:39.839 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:39.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:39.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:39.941 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:39.964 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:39.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:39.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:40.033 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:12:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:12:39] "GET /metrics HTTP/1.1" 200 34365 "" "Prometheus/2.51.0" 2024-08-17T18:12:40.066 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:40.091 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:40.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:40.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:40.193 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:40.218 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:40.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:40.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:40.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:40.342 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:40.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:40.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:40.444 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:40.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:40.470 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:40.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:40.541 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:12:40.542 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:12:40.542 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:12:40.542 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:12:40.542 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:12:40.542 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:12:40.542 INFO:teuthology.orchestra.run.smithi130.stderr:bench [] [] [] [] 2024-08-17T18:12:40.542 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:12:40.542 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:12:40.542 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:12:40.542 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator dump block 2024-08-17T18:12:40.542 INFO:teuthology.orchestra.run.smithi130.stderr:Error EINVAL: invalid command 2024-08-17T18:12:40.571 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:40.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:12:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:12:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:12:40.596 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:40.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:40.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:40.698 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:40.723 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:40.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:40.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:40.824 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:40.849 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:40.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:40.951 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:40.976 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:40.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:40.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:41.078 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:41.099 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:41.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:41.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:41.135 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:12:41.182 INFO:teuthology.orchestra.run.smithi130.stderr:pool 'unique_pool_4' created 2024-08-17T18:12:41.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:41.224 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:41.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:41.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:41.325 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:41.347 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:41.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:41.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:41 smithi130 ceph-mon[110068]: pgmap v2521: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:41 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1086353740' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T18:12:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:41 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T18:12:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:41 smithi130 ceph-mon[105521]: pgmap v2521: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:41 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1086353740' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T18:12:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:41 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T18:12:41.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:41.475 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:41.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:41.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:41.577 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:41.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:41 smithi179 ceph-mon[93519]: pgmap v2521: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:41.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:41 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1086353740' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T18:12:41.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:41 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T18:12:41.601 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:41.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:41.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:41.703 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:41.725 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:41.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:41.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:41.827 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:41.852 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:41.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:41.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:41.954 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:41.978 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:41.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:41.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:42.005 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-08-17T18:12:42.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:42.102 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:42.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:42.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:42.181 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T18:12:42.204 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:42.246 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:42.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:42.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:42.283 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:42 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-17T18:12:42.283 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:42 smithi130 ceph-mon[105521]: osdmap e1380: 8 total, 8 up, 4 in 2024-08-17T18:12:42.283 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:42 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-17T18:12:42.283 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:42 smithi130 ceph-mon[110068]: osdmap e1380: 8 total, 8 up, 4 in 2024-08-17T18:12:42.348 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:42.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:42.370 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:42.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:42.471 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:42.493 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:42.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:42.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:42.507 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:12:42.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:42 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-17T18:12:42.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:42 smithi179 ceph-mon[93519]: osdmap e1380: 8 total, 8 up, 4 in 2024-08-17T18:12:42.595 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:42.617 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:42.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:42.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:42.683 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:42.711 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:12:42.711 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T18:12:42.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-08-17T18:12:42.719 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:42.728 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:42.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:42.741 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:42.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:42.842 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:42.863 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:42.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:42.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:42.965 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:42.987 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:42.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:42.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:43.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:43.110 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:43.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:43.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:43.212 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:43.234 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:43.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:43.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:43.239 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:43 smithi130 ceph-mon[105521]: pgmap v2523: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:43.239 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:43 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:12:43.239 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:43 smithi130 ceph-mon[105521]: osdmap e1381: 8 total, 8 up, 4 in 2024-08-17T18:12:43.239 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:43.239 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:12:43.240 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:43 smithi130 ceph-mon[110068]: pgmap v2523: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:43.240 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:43 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:12:43.240 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:43 smithi130 ceph-mon[110068]: osdmap e1381: 8 total, 8 up, 4 in 2024-08-17T18:12:43.240 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:43.240 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:12:43.336 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:43.358 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:43.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:43.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:43.463 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:43.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:43.486 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:43.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:43.538 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:43 smithi179 ceph-mon[93519]: pgmap v2523: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:43.539 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:43 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:12:43.539 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:43 smithi179 ceph-mon[93519]: osdmap e1381: 8 total, 8 up, 4 in 2024-08-17T18:12:43.539 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:43.539 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:12:43.587 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:43.609 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:43.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:43.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:43.711 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:43.732 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:43.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:43.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:43.834 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:43.856 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:43.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:43.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:43.957 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:43.979 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:43.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:43.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:44.081 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:44.102 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:44.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:44.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:44.204 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:44.225 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:44.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:44.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:44.253 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:44 smithi179 ceph-mon[93519]: osdmap e1382: 8 total, 8 up, 4 in 2024-08-17T18:12:44.327 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:44.349 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:44.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:44.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:44.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:44 smithi130 ceph-mon[110068]: osdmap e1382: 8 total, 8 up, 4 in 2024-08-17T18:12:44.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:44 smithi130 ceph-mon[105521]: osdmap e1382: 8 total, 8 up, 4 in 2024-08-17T18:12:44.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:44.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:44.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:44.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:44.507 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:12:44.507 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:12:44.507 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T18:12:44.507 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:12:44.573 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:44.594 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:44.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:44.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:44.696 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:44.718 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:44.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:44.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:44.819 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:44.841 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:44.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:44.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:44.943 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:44.965 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:44.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:44.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:45.066 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:45.091 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:45.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:45.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:45.134 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:45.134 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:45.193 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:45.208 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T18:12:45.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:45.215 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:45.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:45.317 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:45.339 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:45.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:45.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:45.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:45 smithi130 ceph-mon[105521]: pgmap v2526: 89 pgs: 9 creating+peering, 3 unknown, 77 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:45.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:45 smithi130 ceph-mon[110068]: pgmap v2526: 89 pgs: 9 creating+peering, 3 unknown, 77 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:45.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:45.461 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:45.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:45.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:45.463 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:12:45.563 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:45.586 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:45 smithi179 ceph-mon[93519]: pgmap v2526: 89 pgs: 9 creating+peering, 3 unknown, 77 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:45.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:45.587 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:45.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:45.689 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:12:45.689 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:12:45.690 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:45.692 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:12:45.693 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:12:45.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:45.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:45.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:45.712 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:12:45.712 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:12:45.727 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:12:45.728 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:12:45.743 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:12:45.744 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:12:45.759 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:12:45.759 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:12:45.775 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:12:45.775 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:12:45.791 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:12:45.792 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:12:45.812 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:45.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:45.836 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:45.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:45.937 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:45.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:45.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:45.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:46.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:46.081 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:46.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:46.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:46.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:46.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:46.207 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:46.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:46.308 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:46.329 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:46.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:46.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:46.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:46 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2947734975' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:46.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:46 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2947734975' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:46.430 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:46.451 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:46.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:46.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:46.553 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:46.574 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:46.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:46.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:46.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:46 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2947734975' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:46.676 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:46.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:46.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:46.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:46.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:46.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:46.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:46.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:46.923 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:46.945 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:46.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:46.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:47.047 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:47.069 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:47.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:47.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:47.171 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:47.192 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:47.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:47.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:47.199 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:12:47.236 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:12:47.236 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:12:47.294 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:47.317 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:47.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:47.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:47.419 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:47.440 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:47.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:47.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:47.542 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:47.564 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:47.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:47.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:47.590 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:47 smithi179 ceph-mon[93519]: pgmap v2527: 89 pgs: 9 creating+peering, 80 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:47.590 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:47 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1332742007' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:12:47.666 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:47.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:47 smithi130 ceph-mon[105521]: pgmap v2527: 89 pgs: 9 creating+peering, 80 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:47.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:47 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1332742007' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:12:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:47 smithi130 ceph-mon[110068]: pgmap v2527: 89 pgs: 9 creating+peering, 80 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:47 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1332742007' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:12:47.688 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:47.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:47.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:47.790 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:47.811 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:47.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:47.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:47.934 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:47.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:47.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:48.036 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:48.059 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:48.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:48.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:48.109 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T18:12:48.161 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:48.182 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:48.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:48.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:48.283 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:48.306 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:48.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:48.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:48.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:48 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:12:48.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:48 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1332742007' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:12:48.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:48 smithi179 ceph-mon[93519]: osdmap e1383: 8 total, 8 up, 4 in 2024-08-17T18:12:48.368 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:12:48.408 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:48.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:48 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:12:48.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:48 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1332742007' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:12:48.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:48 smithi130 ceph-mon[105521]: osdmap e1383: 8 total, 8 up, 4 in 2024-08-17T18:12:48.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:48 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:12:48.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:48 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1332742007' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:12:48.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:48 smithi130 ceph-mon[110068]: osdmap e1383: 8 total, 8 up, 4 in 2024-08-17T18:12:48.429 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:48.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:48.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:48.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:48.554 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:48.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:48.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:48.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:48.677 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:48.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:48.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:48.779 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:48.801 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:48.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:48.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:48.880 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:12:48.903 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:48.924 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:48.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:48.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:49.026 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:49.047 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:49.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:49.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:49.149 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:49.170 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:49.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:49.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:49.272 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:49.293 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:49.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:49.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:49.395 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:49.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:49.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:49.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:49.519 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:49.541 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:49.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:49.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:49.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:49 smithi179 ceph-mon[93519]: pgmap v2529: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:49.642 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:49.663 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:49.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:49.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:49.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:49 smithi130 ceph-mon[105521]: pgmap v2529: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:49.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:49 smithi130 ceph-mon[110068]: pgmap v2529: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:49.759 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:49.765 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:49.788 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:49.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:49.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:49.890 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:49.912 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:49.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:49.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:50.000 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:12:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:12:49] "GET /metrics HTTP/1.1" 200 36715 "" "Prometheus/2.51.0" 2024-08-17T18:12:50.014 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:50.036 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:50.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:50.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:50.138 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:50.154 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:12:50.154 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:12:50.159 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:50.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:50.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:50.261 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:50.283 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:50.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:50.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:50.384 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:50.406 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:50.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:50.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:50.448 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:12:50.477 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:12:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:12:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:12:50.508 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:50.529 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:50.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:50.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:50.631 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:50.652 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:50.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:50.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:50.754 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:50.776 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:50.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:50.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:50.801 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T18:12:50.878 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:50.900 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:50.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:50.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:51.002 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:51.024 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:51.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:51.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:51.092 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:12:51.126 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:51.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:51.148 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:51.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:51.249 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:51.270 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:51.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:51.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:51.372 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:51.394 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:51.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:51.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:51.496 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:51.505 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:51 smithi179 ceph-mon[93519]: pgmap v2530: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:51.518 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:51.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:51.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:51.550 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:51 smithi130 ceph-mon[105521]: pgmap v2530: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:51.550 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:51 smithi130 ceph-mon[110068]: pgmap v2530: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:51.620 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:51.642 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:51.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:51.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:51.744 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:51.766 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:51.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:51.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:51.868 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:51.881 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:12:51.881 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:12:51.881 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:12:51.881 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:12:51.881 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:12:51.881 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:12:51.881 INFO:teuthology.orchestra.run.smithi130.stderr:bench [] [] [] [] 2024-08-17T18:12:51.882 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:12:51.882 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:12:51.882 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:12:51.882 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator dump block 2024-08-17T18:12:51.882 INFO:teuthology.orchestra.run.smithi130.stderr:Error EINVAL: invalid command 2024-08-17T18:12:51.889 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:51.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:51.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:51.991 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:52.012 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:52.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:52.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:52.114 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:52.137 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:52.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:52.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:52.223 INFO:teuthology.orchestra.run.smithi130.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-08-17T18:12:52.238 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:52.259 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:52.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:52.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:52 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/323523147' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:12:52.289 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:52 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:12:52.361 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:52.383 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:52.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:52.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:52.485 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:52.507 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:52.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:52.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:52.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:52 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/323523147' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:12:52.578 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:52 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:12:52.578 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:52 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/323523147' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:12:52.578 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:52 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:12:52.609 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:52.630 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:52.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:52.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:52.732 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:52.754 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:52.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:52.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:52.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:52.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:52.878 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:52.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:52.979 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:53.001 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:53.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:53.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:53.103 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:53.126 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:53.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:53.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:53.131 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:12:53.131 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:12:53.142 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:12:53.143 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T18:12:53.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-08-17T18:12:53.228 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:53.249 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:53.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:53.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:53 smithi179 ceph-mon[93519]: pgmap v2531: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:53 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T18:12:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:53 smithi179 ceph-mon[93519]: osdmap e1384: 8 total, 8 up, 4 in 2024-08-17T18:12:53.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:53 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:53.351 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:53.373 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:53.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:53.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:53.386 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:53 smithi130 ceph-mon[105521]: pgmap v2531: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:53.386 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:53 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T18:12:53.386 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:53 smithi130 ceph-mon[105521]: osdmap e1384: 8 total, 8 up, 4 in 2024-08-17T18:12:53.387 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:53 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:53.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:53 smithi130 ceph-mon[110068]: pgmap v2531: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:53.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:53 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T18:12:53.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:53 smithi130 ceph-mon[110068]: osdmap e1384: 8 total, 8 up, 4 in 2024-08-17T18:12:53.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:53 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:53.395 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:12:53.395 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:12:53.404 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:12:53.404 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:12:53.421 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:12:53.422 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:12:53.438 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:12:53.438 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:12:53.453 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:12:53.454 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:12:53.468 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:12:53.469 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:12:53.475 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:53.483 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:12:53.484 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:12:53.497 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:53.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:53.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:53.501 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:12:53.501 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:12:53.599 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:53.621 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:53.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:53.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:53.723 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:53.744 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:53.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:53.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:53.846 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T18:12:53.847 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:53.868 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:53.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:53.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:53.970 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:53.991 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:53.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:53.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:54.065 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:54.093 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:54.098 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:12:54.115 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:54.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:54.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:54.216 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:54.239 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:54.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:54.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:54.341 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:54.362 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:54.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:54.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:54.463 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:54.467 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/821257901' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:54.468 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/821257901' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:54.484 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:54.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:54.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:54.557 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/821257901' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:12:54.586 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:54.606 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:54.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:54.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:54.708 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:54.730 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:54.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:54.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:54.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:54.854 DEBUG:teuthology.orchestra.run.smithi130:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-08-17T18:12:54.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:54.855 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:54.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:54.856 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T18:12:54.856 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T18:12:54.901 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:hints = 1 2024-08-17T18:12:54.901 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T18:12:54.901 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Object prefix: benchmark_data_smithi130_364143 2024-08-17T18:12:54.901 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:12:54.901 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 0 0 0 0 0 0 - 0 2024-08-17T18:12:54.956 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:54.980 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:54.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:54.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:55.082 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:55.119 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:55.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:55.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:55.221 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:55.258 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:55.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:55.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:55 smithi179 ceph-mon[93519]: pgmap v2533: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:55 smithi179 ceph-mon[93519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:12:55.360 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:55.384 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:55.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:55.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:55.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:55.524 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:55.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:55.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:55.626 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:55.651 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:55.651 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:55 smithi130 ceph-mon[110068]: pgmap v2533: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:55.652 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:55 smithi130 ceph-mon[110068]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:12:55.652 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:55 smithi130 ceph-mon[105521]: pgmap v2533: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:55.652 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:55 smithi130 ceph-mon[105521]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:12:55.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:55.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:55.663 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:55.754 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:55.782 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:55.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:55.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:55.883 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:55.901 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 1 16 3259 3243 202.654 202.688 0.00274869 0.00492134 2024-08-17T18:12:55.922 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:55.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:55.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:56.016 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:12:56.017 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:12:56.017 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:12:56.017 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:12:56.017 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:12:56.024 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:56.050 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:56.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:56.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:56.152 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:56.178 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:56.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:56.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:56.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:56.305 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:56.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:56.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:56.407 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:56.432 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:56.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:56.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:56.492 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:12:56.533 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:56.580 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:56.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:56.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:56.682 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:56.724 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:56.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:56.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:56.770 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T18:12:56.827 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:56.851 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:56.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:56.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:56.902 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 2 16 6128 6112 190.973 179.312 0.00504545 0.0052239 2024-08-17T18:12:56.954 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:56.998 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:56.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:56.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:57.075 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:12:57.100 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:57.138 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:57.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:57.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:57.240 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:57.265 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:57.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:57.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:57.367 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:57.403 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:57.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:57.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:57.505 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:57.527 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:57.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:57.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:57 smithi179 ceph-mon[93519]: pgmap v2534: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:57.627 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:57 smithi130 ceph-mon[105521]: pgmap v2534: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:57.627 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:57 smithi130 ceph-mon[110068]: pgmap v2534: 89 pgs: 89 active+clean; 582 KiB data, 2.9 GiB used, 355 GiB / 358 GiB avail 2024-08-17T18:12:57.630 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:57.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:57.654 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:57.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:57.755 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:57.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:57.796 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:57.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:57.897 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:57.901 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 3 16 8694 8678 180.769 160.375 0.00675924 0.00552285 2024-08-17T18:12:57.923 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:57.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:57.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:58.026 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:58.066 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:58.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:58.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:58.083 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:12:58.160 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:12:58.160 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:12:58.160 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:12:58.160 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:12:58.160 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:12:58.160 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:12:58.160 INFO:teuthology.orchestra.run.smithi130.stderr:bench [] [] [] [] 2024-08-17T18:12:58.161 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:12:58.161 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:12:58.161 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:12:58.161 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator dump block 2024-08-17T18:12:58.161 INFO:teuthology.orchestra.run.smithi130.stderr:Error EINVAL: invalid command 2024-08-17T18:12:58.168 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:58.193 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:58.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:58.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:58.295 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:58.319 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:58.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:58.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:58.352 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:12:58.421 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:12:58.443 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:58.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:58.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:12:58.545 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:58.570 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:58.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:58.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:12:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:58 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2081062767' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:58 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:12:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:58 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2081062767' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:58 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:12:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:12:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:58 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2081062767' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:58.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:58 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:12:58.672 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:58.712 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:58.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:58.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:12:58.814 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:12:58.837 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:58.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:58.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:12:58.902 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 4 16 11167 11151 174.214 154.562 0.00528266 0.00573207 2024-08-17T18:12:58.939 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:58.977 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:58.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:58.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:59.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:59.115 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:59.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:59.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:59.217 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:12:59.243 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:59.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:59.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:12:59.344 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:12:59.381 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:59.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:59.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:12:59.484 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:12:59.508 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:12:59.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:59.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:12:59.610 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:12:59.613 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:59 smithi179 ceph-mon[93519]: pgmap v2535: 89 pgs: 89 active+clean; 233 MiB data, 3.2 GiB used, 354 GiB / 358 GiB avail; 23 MiB/s wr, 372 op/s 2024-08-17T18:12:59.613 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:59 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:12:59.613 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:59 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:12:59.613 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:12:59 smithi179 ceph-mon[93519]: osdmap e1385: 8 total, 8 up, 4 in 2024-08-17T18:12:59.622 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:12:59.622 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T18:12:59.622 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T18:12:59.622 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [] 2024-08-17T18:12:59.622 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:12:59.646 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:59.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:59.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:12:59.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:59 smithi130 ceph-mon[110068]: pgmap v2535: 89 pgs: 89 active+clean; 233 MiB data, 3.2 GiB used, 354 GiB / 358 GiB avail; 23 MiB/s wr, 372 op/s 2024-08-17T18:12:59.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:59 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:12:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:59 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:12:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:12:59 smithi130 ceph-mon[110068]: osdmap e1385: 8 total, 8 up, 4 in 2024-08-17T18:12:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:59 smithi130 ceph-mon[105521]: pgmap v2535: 89 pgs: 89 active+clean; 233 MiB data, 3.2 GiB used, 354 GiB / 358 GiB avail; 23 MiB/s wr, 372 op/s 2024-08-17T18:12:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:59 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:12:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:59 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:12:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:12:59 smithi130 ceph-mon[105521]: osdmap e1385: 8 total, 8 up, 4 in 2024-08-17T18:12:59.748 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:12:59.772 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:59.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:59.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:12:59.874 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:12:59.901 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 5 16 13651 13635 170.418 155.25 0.0044935 0.00586059 2024-08-17T18:12:59.910 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:12:59.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:12:59.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:00.011 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:00.035 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:00.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:00.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:00.137 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:00.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:12:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:12:59] "GET /metrics HTTP/1.1" 200 36755 "" "Prometheus/2.51.0" 2024-08-17T18:13:00.176 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:00.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:00.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:00.278 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:00.305 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:00.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:00.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:00.342 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:13:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:13:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:13:00.382 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T18:13:00.406 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:00.449 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:00.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:00.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:00.551 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:00.578 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:00.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:00.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:00.680 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:00.685 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:13:00.719 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:00.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:00.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:00.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:00.858 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:00.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:00.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:00.902 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 6 16 15880 15864 165.232 139.312 0.00683957 0.00604534 2024-08-17T18:13:00.959 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:00.984 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:00.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:00.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:01.086 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:01.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:01.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:01.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:01.208 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:01.229 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:01.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:01.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:01.331 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:01.353 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:01.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:01.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:01.455 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:01.479 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:01.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:01.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:01.581 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:01.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:01 smithi179 ceph-mon[93519]: pgmap v2537: 89 pgs: 89 active+clean; 521 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T18:13:01.605 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:01.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:01.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:01.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:01 smithi130 ceph-mon[110068]: pgmap v2537: 89 pgs: 89 active+clean; 521 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T18:13:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:01 smithi130 ceph-mon[105521]: pgmap v2537: 89 pgs: 89 active+clean; 521 MiB data, 3.9 GiB used, 354 GiB / 358 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T18:13:01.707 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:01.728 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:01.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:01.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:01.830 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:01.866 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:01.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:01.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:01.902 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 7 16 17097 17081 152.493 76.0625 0.00473094 0.00655019 2024-08-17T18:13:01.969 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:01.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:01.991 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:01.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:02.092 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:02.118 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:02.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:02.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:02.220 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:02.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:02.245 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:02.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:02.347 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:02.384 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:02.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:02.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:02.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:02.518 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:02.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:02.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:02.615 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:13:02.615 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:13:02.615 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T18:13:02.616 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:13:02.620 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:02.648 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:02.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:02.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:02.750 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:02.788 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:02.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:02.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:02.890 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:02.902 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 8 16 18642 18626 145.5 96.5625 0.00721222 0.00686552 2024-08-17T18:13:02.913 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:02.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:02.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:03.015 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:03.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:03.037 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:03.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:03.138 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:03.159 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:03.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:03.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:03.260 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:03.284 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:03.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:03.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:03.386 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:03.410 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:03.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:03.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:03.512 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:03.522 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T18:13:03.548 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:03.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:03.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:03.651 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:03.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:03 smithi130 ceph-mon[105521]: pgmap v2538: 89 pgs: 89 active+clean; 521 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 56 MiB/s wr, 897 op/s 2024-08-17T18:13:03.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:03 smithi130 ceph-mon[110068]: pgmap v2538: 89 pgs: 89 active+clean; 521 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 56 MiB/s wr, 897 op/s 2024-08-17T18:13:03.673 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:03.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:03.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:03.686 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:03 smithi179 ceph-mon[93519]: pgmap v2538: 89 pgs: 89 active+clean; 521 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 56 MiB/s wr, 897 op/s 2024-08-17T18:13:03.775 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:03.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:03.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:03.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:03.829 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:13:03.902 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 9 16 20097 20081 139.437 90.9375 0.00556149 0.00716362 2024-08-17T18:13:03.911 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:03.935 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:03.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:03.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:04.037 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:04.063 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:04.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:04.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:04.165 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:04.201 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:04.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:04.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:04.303 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:04.324 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:04.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:04.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:04.426 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:04.464 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:04.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:04.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:04.566 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:04.591 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:04.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:04.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:04.693 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:04.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:04.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:04.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:04.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:04.852 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:04.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:04.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:04.902 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 10 16 22007 21991 137.429 119.375 0.00502251 0.00726995 2024-08-17T18:13:04.954 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:04.976 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:04.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:04.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:05.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:05.101 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:05.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:05.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:05.203 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:05.240 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:05.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:05.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:05.341 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:05.375 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:05.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:05.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:05.477 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:05.500 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:05.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:05.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:05.602 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:05.630 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:05.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:05.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:05.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:05 smithi130 ceph-mon[105521]: pgmap v2539: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-08-17T18:13:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:05 smithi130 ceph-mon[110068]: pgmap v2539: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-08-17T18:13:05.697 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:05 smithi179 ceph-mon[93519]: pgmap v2539: 89 pgs: 89 active+clean; 1.0 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-08-17T18:13:05.732 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:05.755 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:05.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:05.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:05.857 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:05.893 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:05.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:05.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:05.902 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 11 16 23664 23648 134.35 103.562 0.0131714 0.00743746 2024-08-17T18:13:05.995 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:06.031 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:06.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:06.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:06.133 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:06.155 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:06.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:06.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:06.257 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:06.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:06.283 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:06.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:06.384 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:06.420 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:06.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:06.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:06.522 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:06.559 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:06.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:06.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:06.661 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:06.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:06.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:06.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:06.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:06.826 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:06.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:06.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:06.902 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 12 16 25461 25445 132.513 112.312 0.00225041 0.0075398 2024-08-17T18:13:06.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:06.966 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:06.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:06.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:07.040 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:13:07.040 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:13:07.040 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:13:07.040 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:13:07.040 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:13:07.068 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:07.109 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:07.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:07.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:07.211 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:07.237 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:07.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:07.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:07.339 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:07.387 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:07.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:07.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:07.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:07.512 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:07.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:07.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:07.614 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:07.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:07.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:07.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:07.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:07 smithi130 ceph-mon[105521]: pgmap v2540: 89 pgs: 89 active+clean; 1.1 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-17T18:13:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:07 smithi130 ceph-mon[110068]: pgmap v2540: 89 pgs: 89 active+clean; 1.1 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-17T18:13:07.757 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:07.787 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:07 smithi179 ceph-mon[93519]: pgmap v2540: 89 pgs: 89 active+clean; 1.1 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-08-17T18:13:07.798 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:07.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:07.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:07.822 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T18:13:07.899 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:07.902 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 13 16 27261 27245 130.973 112.5 0.0079334 0.00762831 2024-08-17T18:13:07.925 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:07.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:07.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:08.018 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:13:08.019 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T18:13:08.022 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-08-17T18:13:08.027 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:08.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:08.107 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:08.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:08.208 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:08.231 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:08.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:08.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:08.333 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:08.356 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:08.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:08.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:08.457 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:08.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:08.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:08.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:08.582 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:08.606 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:08.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:08.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:08.661 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:08 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:13:08.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:08 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:13:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:08 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:13:08.708 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:08.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:08.767 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:08.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:08.868 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:08.902 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 14 16 28935 28919 129.09 104.625 0.0102154 0.0077397 2024-08-17T18:13:08.917 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:08.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:08.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:09.019 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:09.029 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:13:09.043 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:09.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:09.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:09.099 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:13:09.100 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:13:09.123 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:13:09.135 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:13:09.136 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:13:09.144 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:09.172 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:13:09.172 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:13:09.193 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:09.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:09.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:09.212 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:13:09.212 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:13:09.240 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:13:09.241 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:13:09.270 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:13:09.270 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:13:09.296 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:09.299 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:13:09.300 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:13:09.318 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:09.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:09.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:09.331 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:13:09.332 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:13:09.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:09.467 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:09.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:09.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:09.505 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:09 smithi179 ceph-mon[93519]: pgmap v2541: 89 pgs: 89 active+clean; 1.4 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-08-17T18:13:09.505 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:09 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1691333255' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:13:09.569 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:09.598 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:09.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:09.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:09.605 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:09 smithi130 ceph-mon[105521]: pgmap v2541: 89 pgs: 89 active+clean; 1.4 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-08-17T18:13:09.605 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:09 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1691333255' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:13:09.605 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:09 smithi130 ceph-mon[110068]: pgmap v2541: 89 pgs: 89 active+clean; 1.4 GiB data, 6.9 GiB used, 351 GiB / 358 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-08-17T18:13:09.605 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:09 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1691333255' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:13:09.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:09.738 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:09.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:09.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:09.842 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:09.878 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:09.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:09.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:09.902 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 15 15 30573 30558 127.312 102.438 0.0106251 0.00784918 2024-08-17T18:13:09.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:13:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:13:09] "GET /metrics HTTP/1.1" 200 36755 "" "Prometheus/2.51.0" 2024-08-17T18:13:09.980 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:10.018 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:10.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:10.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:10.120 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:10.156 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:10.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:10.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:10.257 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:10.286 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:10.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:10.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:10.388 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:10.413 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:10.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:10.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:10.478 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:13:10.515 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:10.552 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:10.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:10.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:13:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:13:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:13:10.653 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:10.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:10.701 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:10.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:10.802 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:10.824 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:10.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:10.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:10.903 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 16 16 32107 32091 125.343 95.8125 0.00333121 0.00797228 2024-08-17T18:13:10.926 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:10.949 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:10.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:10.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:11.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:11.073 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:11.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:11.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:11.175 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:11.226 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:11.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:11.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:11.328 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:11.349 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:11.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:11.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:11.452 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:11.475 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:11.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:11.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:11.503 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:11 smithi130 ceph-mon[105521]: pgmap v2542: 89 pgs: 89 active+clean; 1.6 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-08-17T18:13:11.504 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:11 smithi130 ceph-mon[110068]: pgmap v2542: 89 pgs: 89 active+clean; 1.6 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-08-17T18:13:11.578 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:11.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:11 smithi179 ceph-mon[93519]: pgmap v2542: 89 pgs: 89 active+clean; 1.6 GiB data, 7.6 GiB used, 350 GiB / 358 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-08-17T18:13:11.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:11.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:11.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:11.704 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:11.727 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:11.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:11.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:11.829 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:11.889 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:11.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:11.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:11.903 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 17 16 33690 33674 123.788 98.9375 0.00802239 0.0080716 2024-08-17T18:13:11.992 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:12.016 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:12.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:12.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:12.118 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:12.142 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:12.142 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:13:12.142 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:13:12.143 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:13:12.143 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:13:12.143 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:13:12.143 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:13:12.143 INFO:teuthology.orchestra.run.smithi130.stderr:bench [] [] [] [] 2024-08-17T18:13:12.143 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:13:12.143 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:13:12.143 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:13:12.143 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore allocator dump block 2024-08-17T18:13:12.143 INFO:teuthology.orchestra.run.smithi130.stderr:Error EINVAL: invalid command 2024-08-17T18:13:12.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:12.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:12.245 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:12.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:12.284 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:12.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:12.385 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:12.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:12.432 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:12.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:12.533 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:12.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:12.578 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:12.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:12.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:13:12.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:13:12.679 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:12.705 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:12.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:12.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:12.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:13:12.807 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:12.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:12.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:12.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:12.847 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:13:12.847 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:13:12.903 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 18 16 35069 35053 121.698 86.1875 0.00635307 0.0082106 2024-08-17T18:13:12.933 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:12.956 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:12.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:12.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:13.058 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:13.059 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:13:13.060 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-08-17T18:13:13.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:13.096 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:13.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:13.197 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:13.233 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:13.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:13.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:13.335 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:13.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:13.359 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:13.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:13.460 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:13.484 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:13.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:13.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:13.518 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:13:13.550 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:13:13.569 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:13 smithi179 ceph-mon[93519]: pgmap v2543: 89 pgs: 89 active+clean; 1.6 GiB data, 8.1 GiB used, 349 GiB / 358 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-08-17T18:13:13.587 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:13.600 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T18:13:13.607 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:13 smithi130 ceph-mon[105521]: pgmap v2543: 89 pgs: 89 active+clean; 1.6 GiB data, 8.1 GiB used, 349 GiB / 358 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-08-17T18:13:13.607 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:13 smithi130 ceph-mon[110068]: pgmap v2543: 89 pgs: 89 active+clean; 1.6 GiB data, 8.1 GiB used, 349 GiB / 358 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-08-17T18:13:13.611 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:13.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:13.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:13.713 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:13.737 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:13.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:13.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:13.839 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:13.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:13.900 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:13.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:13.903 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 19 16 36349 36333 119.502 80 0.0242478 0.00836015 2024-08-17T18:13:14.001 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:14.046 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:14.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:14.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:14.148 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:14.191 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:14.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:14.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:14.293 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:14.330 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:14.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:14.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:14.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:14 smithi130 ceph-mon[105521]: pgmap v2544: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 349 GiB / 358 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-08-17T18:13:14.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:14 smithi130 ceph-mon[110068]: pgmap v2544: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 349 GiB / 358 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-08-17T18:13:14.431 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:13:14.433 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:14.460 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:14.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:14.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:14.559 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:14 smithi179 ceph-mon[93519]: pgmap v2544: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 349 GiB / 358 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-08-17T18:13:14.563 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:14.604 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:14.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:14.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:14.706 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:14.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:14.746 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:14.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:14.847 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:14.872 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:14.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:14.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:14.903 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:13:14.903885+0000 min lat: 0.00139804 max lat: 0.329999 avg lat: 0.00844821 2024-08-17T18:13:14.904 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:13:14.904 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 20 16 37873 37857 118.289 95.25 0.011754 0.00844821 2024-08-17T18:13:14.974 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:14.997 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:14.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:14.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:15.021 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:13:15.099 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:15.125 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:15.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:15.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:15.226 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:15.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:15.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:15.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:15.353 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:15.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:15.388 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:15.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:15.489 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:15.512 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:15.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:15.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:15.614 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:15.655 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:15.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:15.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:15.757 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:15.781 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:15.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:15.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:15.883 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:15.903 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 21 16 39170 39154 116.516 81.0625 0.0829921 0.008566 2024-08-17T18:13:15.904 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:15.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:15.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:16.006 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:16.042 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:16.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:16.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:16.144 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:16.170 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:16.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:16.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:16.272 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:16.315 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:16.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:16.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:16.417 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:16.444 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:16.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:16.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:16.546 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:16.570 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:16.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:16.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:16.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:16 smithi179 ceph-mon[93519]: pgmap v2545: 89 pgs: 89 active+clean; 2.1 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 95 MiB/s wr, 1.51k op/s 2024-08-17T18:13:16.672 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:16.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:16.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:16.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:16.795 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:16.834 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:16.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:16.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:16.904 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 22 16 40411 40395 114.745 77.5625 0.00639866 0.00870749 2024-08-17T18:13:16.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:16 smithi130 ceph-mon[105521]: pgmap v2545: 89 pgs: 89 active+clean; 2.1 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 95 MiB/s wr, 1.51k op/s 2024-08-17T18:13:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:16 smithi130 ceph-mon[110068]: pgmap v2545: 89 pgs: 89 active+clean; 2.1 GiB data, 9.7 GiB used, 348 GiB / 358 GiB avail; 95 MiB/s wr, 1.51k op/s 2024-08-17T18:13:16.936 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:16.959 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:16.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:17.061 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:17.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:17.101 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:17.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:17.202 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:17.223 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:17.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:17.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:17.324 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:17.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:17.361 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:17.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:17.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:17.499 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:17.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:17.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:17.601 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:17.611 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:13:17.611 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:13:17.624 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:17.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:17.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:17.726 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:17.748 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:17.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:17.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:17.849 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:17.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:17.871 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:17.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:17.904 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 23 16 41186 41170 111.862 48.4375 0.0142498 0.00883343 2024-08-17T18:13:17.972 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:18.014 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:18.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:18.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:18.061 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 7] out_osds: [5, 4, 6, 1] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2024-08-17T18:13:18.061 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T18:13:18.116 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:18.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:18.176 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:18.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:18.277 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:18.312 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T18:13:18.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:18.316 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:18.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:18.416 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:18.437 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:18.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:18.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:18.539 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:18.591 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:18.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:18.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:18.693 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:18.721 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:18.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:18.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:18.823 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:18.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:18 smithi179 ceph-mon[93519]: pgmap v2546: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 348 GiB / 358 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-08-17T18:13:18.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:18.860 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:18.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:18.904 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 24 16 42335 42319 110.193 71.8125 0.0141682 0.00906934 2024-08-17T18:13:18.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:18 smithi130 ceph-mon[105521]: pgmap v2546: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 348 GiB / 358 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-08-17T18:13:18.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:18 smithi130 ceph-mon[110068]: pgmap v2546: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 348 GiB / 358 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-08-17T18:13:18.961 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:18.997 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:18.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:18.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:19.099 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:19.122 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:19.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:19.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:19.152 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:13:19.225 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:19.264 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:19.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:19.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:19.367 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:19.406 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:19.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:19.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:19.508 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:19.533 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:19.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:19.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:19.635 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:19.658 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:19.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:19.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:19.759 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:19.800 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:19.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:19.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:19.902 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:19.904 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 25 16 43707 43691 109.215 85.75 0.00861357 0.00915029 2024-08-17T18:13:19.925 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:19.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:19.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:20.027 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:20.051 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:20.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:20.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:20.153 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:13:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:13:19] "GET /metrics HTTP/1.1" 200 36775 "" "Prometheus/2.51.0" 2024-08-17T18:13:20.175 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:20.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:20.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:20.277 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:20.299 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:20.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:20.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:20.402 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:20.425 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:20.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:20.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:20.451 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:13:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:13:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:13:20.451 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:20 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:20.451 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:20 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:20.527 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:20.551 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:20.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:20.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:20.653 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:20.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:20 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:20.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:20 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:20.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:20 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:20.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:20 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:20.694 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:20.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:20.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:20.796 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:20.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:20.837 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:20.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:20.904 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 26 16 45038 45022 108.214 83.1875 0.00422614 0.00920994 2024-08-17T18:13:20.942 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:20.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:20.966 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:20.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:21.067 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:21.091 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:21.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:21.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:21.139 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:13:21.139 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:13:21.193 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:21.231 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:21.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:21.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:21.332 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:21.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:21 smithi179 ceph-mon[93519]: pgmap v2547: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 95 MiB/s wr, 1.51k op/s 2024-08-17T18:13:21.354 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:21.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:21.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:21.456 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:21.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:21.484 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:21.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:21.585 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:21.624 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:21.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:21.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:21.659 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:21 smithi130 ceph-mon[105521]: pgmap v2547: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 95 MiB/s wr, 1.51k op/s 2024-08-17T18:13:21.660 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:21 smithi130 ceph-mon[110068]: pgmap v2547: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 347 GiB / 358 GiB avail; 95 MiB/s wr, 1.51k op/s 2024-08-17T18:13:21.727 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:21.767 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:21.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:21.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:21.869 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:21.875 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T18:13:21.906 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 27 16 45963 45947 106.342 57.8125 0.0198101 0.00939704 2024-08-17T18:13:21.908 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:21.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:21.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:22.010 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:22.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:22.036 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:22.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:22.137 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:22.179 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:22.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:22.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:22.195 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:13:22.280 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:22.304 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:22.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:22.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:22.406 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:22.430 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:22.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:22.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:22.532 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:22.565 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:22.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:22.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:22.667 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:22.711 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:22.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:22.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:22.812 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:22.854 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:22.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:22.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:22.905 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 28 16 47340 47324 105.617 86.0625 0.00964014 0.00946164 2024-08-17T18:13:22.956 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:22.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:22.997 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:22.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:23.098 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:23.124 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:23.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:23.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:23.226 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:23.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:23.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:23.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:23.351 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:23.391 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:23.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:23.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:23.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:23 smithi130 ceph-mon[105521]: pgmap v2548: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T18:13:23.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:23 smithi130 ceph-mon[110068]: pgmap v2548: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T18:13:23.478 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:23 smithi179 ceph-mon[93519]: pgmap v2548: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 347 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T18:13:23.493 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:23.521 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:23.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:23.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:23.623 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:23.646 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:23.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:23.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:23.748 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:23.771 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:23.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:23.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:23.872 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:23.899 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:13:23.908 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 29 16 48545 48529 104.564 75.3125 0.0043316 0.00955458 2024-08-17T18:13:23.936 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:23.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:23.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:23.965 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:13:23.966 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:13:24.001 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:13:24.002 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:13:24.037 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:13:24.037 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:13:24.037 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:24.074 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:13:24.075 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:13:24.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:24.099 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:24.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:24.104 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:13:24.104 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:13:24.134 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:13:24.134 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:13:24.145 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:13:24.146 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:13:24.165 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:13:24.165 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:13:24.193 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:13:24.193 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:13:24.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:24.237 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:13:24.257 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:24.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:24.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:24.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:24 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3824056499' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:13:24.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:24 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:13:24.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:24 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4232441155' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:13:24.359 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:24.382 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:24.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:24.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:24.477 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:24 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3824056499' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:13:24.477 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:24 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:13:24.477 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:24 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4232441155' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:13:24.478 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:24 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3824056499' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:13:24.478 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:24 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:13:24.478 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:24 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4232441155' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:13:24.484 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:24.507 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:24.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:24.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:24.609 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:24.633 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:24.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:24.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:24.735 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:24.779 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:24.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:24.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:24.881 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:24.905 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T18:13:24.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:24.907 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:24.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:24.908 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 30 16 49793 49777 103.679 78 0.00738369 0.0096401 2024-08-17T18:13:25.008 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:25.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:25.060 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:25.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:25.161 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:25.197 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:25.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:25.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:25.299 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:25.367 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:25.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:25.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:25 smithi130 ceph-mon[105521]: pgmap v2549: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T18:13:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:25 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:13:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:25 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:13:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:25 smithi130 ceph-mon[105521]: osdmap e1386: 8 total, 8 up, 4 in 2024-08-17T18:13:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:25 smithi130 ceph-mon[110068]: pgmap v2549: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T18:13:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:25 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:13:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:25 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:13:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:25 smithi130 ceph-mon[110068]: osdmap e1386: 8 total, 8 up, 4 in 2024-08-17T18:13:25.469 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:25.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:25.515 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:25.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:25 smithi179 ceph-mon[93519]: pgmap v2549: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 346 GiB / 358 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T18:13:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:25 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:13:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:25 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:13:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:25 smithi179 ceph-mon[93519]: osdmap e1386: 8 total, 8 up, 4 in 2024-08-17T18:13:25.616 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:25.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:25.643 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:25.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:25.744 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:25.785 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:25.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:25.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:25.887 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:25.908 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 31 16 51008 50992 102.782 75.9375 0.0226191 0.00972288 2024-08-17T18:13:25.931 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:25.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:25.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:26.033 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:26.074 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:26.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:26.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:26.176 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:26.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:26.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:26.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:26.315 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:26.338 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:26.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:26.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:26.439 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:26.463 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:26.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:26.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:26.565 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:26.605 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:26.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:26.707 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:26.730 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:26.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:26.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:26.832 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:26.855 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:26.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:26.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:26.913 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 32 16 52320 52304 102.132 82 0.0191739 0.0097857 2024-08-17T18:13:26.956 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:26.979 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:26.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:26.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:27.082 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:27.105 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:27.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:27.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:27.207 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:27.230 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:27.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:27.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:27.332 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:27.354 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:27.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:27.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:27.456 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:27.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:27.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:27.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:27.582 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:27.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:27 smithi179 ceph-mon[93519]: pgmap v2551: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-17T18:13:27.601 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:27 smithi130 ceph-mon[105521]: pgmap v2551: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-17T18:13:27.604 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:27 smithi130 ceph-mon[110068]: pgmap v2551: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 346 GiB / 358 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-08-17T18:13:27.605 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:27.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:27.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:27.707 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:27.729 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:27.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:27.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:27.738 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:13:27.831 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:27.854 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:27.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:27.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:27.905 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:13:27.910 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 33 16 53581 53565 101.425 78.8125 0.00559977 0.00985356 2024-08-17T18:13:27.956 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:28.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:28.012 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:28.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:28.113 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:28.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:28.164 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:28.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:28.265 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:28.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:28.312 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:28.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:28.385 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:13:28.413 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:28.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:28.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:28.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:28.535 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:28.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:28.582 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:28.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:28.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:13:28.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:13:28.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:13:28.683 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:28.707 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:28.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:28.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:28.809 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:28.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:28.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:28.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:28.910 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 34 16 54648 54632 100.399 66.6875 0.0123664 0.00995438 2024-08-17T18:13:28.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:28.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:28.993 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:28.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:29.094 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:29.144 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:29.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:29.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:29.247 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:29.270 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:29.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:29.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:29.372 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:29.415 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:29.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:29.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:29 smithi130 ceph-mon[105521]: pgmap v2552: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T18:13:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:29 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:29 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:29.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:29 smithi130 ceph-mon[110068]: pgmap v2552: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T18:13:29.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:29 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:29.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:29 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:29.518 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:29.562 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:29.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:29.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:29 smithi179 ceph-mon[93519]: pgmap v2552: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 345 GiB / 358 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-08-17T18:13:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:29 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:29.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:29 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:29.664 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:29.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:29.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:29.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:29.788 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:29.828 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:29.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:29.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:29.910 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 35 16 55789 55773 99.568 71.3125 0.019814 0.0100373 2024-08-17T18:13:29.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:13:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:13:29] "GET /metrics HTTP/1.1" 200 36778 "" "Prometheus/2.51.0" 2024-08-17T18:13:29.930 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:29.967 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:29.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:29.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:30.069 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:30.125 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:30.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:30.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:30.227 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:30.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:30.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:30.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:30.354 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:30.392 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:30.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:30.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:30.493 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:30.495 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:13:30.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:30.546 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:30.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:30 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:13:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:30 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:13:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:30 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:13:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:13:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:13:30.614 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:30 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:13:30.614 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:30 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:13:30.614 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:30 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:30 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:13:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:30 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:13:30.614 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:30 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:30.647 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:30.670 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:30.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:30.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:30.772 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:30.795 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:30.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:30.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:30.897 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:30.910 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 36 16 56856 56840 98.6546 66.6875 0.00964617 0.010131 2024-08-17T18:13:30.935 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:30.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:30.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:31.037 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:31.088 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:13:31.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:31.101 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:31.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:31.202 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:31.225 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:31.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:31.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:31.265 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T18:13:31.269 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:31 smithi130 ceph-mon[105521]: pgmap v2553: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-08-17T18:13:31.269 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3818475635' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:13:31.269 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:31 smithi130 ceph-mon[110068]: pgmap v2553: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-08-17T18:13:31.269 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3818475635' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:13:31.327 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:31.348 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:31.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:31.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:31.450 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:31.492 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:31.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:31.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:31 smithi179 ceph-mon[93519]: pgmap v2553: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-08-17T18:13:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3818475635' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:13:31.594 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:31.633 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:31.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:31.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:31.735 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:31.758 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:31.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:31.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:31.860 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:31.884 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:31.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:31.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:31.910 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 37 16 58020 58004 97.9545 72.75 0.0147129 0.0102027 2024-08-17T18:13:31.986 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:32.014 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:32.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:32.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:32.115 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:32.137 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:32.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:32.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:32.170 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T18:13:32.240 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:32.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:32.303 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:32.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:32.404 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:32.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3818475635' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:13:32.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:32 smithi130 ceph-mon[105521]: osdmap e1387: 8 total, 8 up, 4 in 2024-08-17T18:13:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3818475635' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:13:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:32 smithi130 ceph-mon[110068]: osdmap e1387: 8 total, 8 up, 4 in 2024-08-17T18:13:32.428 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:32.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:32.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:32.529 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:32.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:32.572 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:32.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:32.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3818475635' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:13:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:32 smithi179 ceph-mon[93519]: osdmap e1387: 8 total, 8 up, 4 in 2024-08-17T18:13:32.673 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:32.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:32.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:32.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:32.798 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:32.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:32.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:32.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:32.911 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 38 16 59127 59111 97.1974 69.1875 0.0123112 0.0102828 2024-08-17T18:13:32.923 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:32.960 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:32.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:32.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:33.062 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:33.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:33.109 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:33.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:33.210 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:33.232 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:33.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:33.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:33.334 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:33.357 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:33.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:33.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:33.448 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:33 smithi130 ceph-mon[105521]: pgmap v2555: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 58 MiB/s wr, 929 op/s 2024-08-17T18:13:33.448 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:33 smithi130 ceph-mon[110068]: pgmap v2555: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 58 MiB/s wr, 929 op/s 2024-08-17T18:13:33.458 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:33.481 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:33.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:33.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:33.583 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:33.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:33 smithi179 ceph-mon[93519]: pgmap v2555: 89 pgs: 89 active+clean; 3.3 GiB data, 13 GiB used, 345 GiB / 358 GiB avail; 58 MiB/s wr, 929 op/s 2024-08-17T18:13:33.605 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:33.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:33.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:33.670 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:13:33.671 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:13:33.671 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.671 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36960.0:59916 49.a e1387/1380 49:54032ff9:::benchmark_data_smithi130_364143_object59915:head v 1387'3888, mlcod=1387'3885)", 2024-08-17T18:13:33.671 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:33.595815+0000", 2024-08-17T18:13:33.671 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.015626745000000001, 2024-08-17T18:13:33.671 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015655478, 2024-08-17T18:13:33.671 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:13:33.671 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:13:33.671 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:13:33.672 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:13:33.672 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.672 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:13:33.672 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.595815+0000", 2024-08-17T18:13:33.672 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.672 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.672 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.672 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:13:33.672 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.595815+0000", 2024-08-17T18:13:33.672 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.672 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.673 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.673 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:13:33.673 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.595816+0000", 2024-08-17T18:13:33.673 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.155e-06 2024-08-17T18:13:33.673 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.673 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.673 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:13:33.673 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.595854+0000", 2024-08-17T18:13:33.673 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8352999999999997e-05 2024-08-17T18:13:33.673 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.673 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.674 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:13:33.674 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.595866+0000", 2024-08-17T18:13:33.674 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2139e-05 2024-08-17T18:13:33.674 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.674 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.674 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:13:33.674 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.595874+0000", 2024-08-17T18:13:33.674 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6109999999999996e-06 2024-08-17T18:13:33.675 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.675 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.675 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:13:33.675 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.602123+0000", 2024-08-17T18:13:33.675 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0062495479999999997 2024-08-17T18:13:33.675 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.675 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.675 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:13:33.676 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.602139+0000", 2024-08-17T18:13:33.676 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5352000000000001e-05 2024-08-17T18:13:33.676 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.676 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:13:33.676 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.676 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.676 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36960.0:59915 49.d e1387/1380 49:b835e300:::benchmark_data_smithi130_364143_object59914:head v 1387'3785, mlcod=1387'3781)", 2024-08-17T18:13:33.676 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:33.601932+0000", 2024-08-17T18:13:33.676 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0095096130000000001, 2024-08-17T18:13:33.676 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0095894970000000006, 2024-08-17T18:13:33.677 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:13:33.677 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:13:33.677 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:13:33.677 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:13:33.677 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.677 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:13:33.677 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.601932+0000", 2024-08-17T18:13:33.677 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.677 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.677 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.677 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:13:33.678 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.601932+0000", 2024-08-17T18:13:33.678 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.678 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.678 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.678 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:13:33.678 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.601932+0000", 2024-08-17T18:13:33.678 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.27e-07 2024-08-17T18:13:33.678 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.678 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.678 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:13:33.678 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.601957+0000", 2024-08-17T18:13:33.679 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4488999999999999e-05 2024-08-17T18:13:33.679 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.679 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.679 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:13:33.679 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.601963+0000", 2024-08-17T18:13:33.679 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6810000000000001e-06 2024-08-17T18:13:33.679 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.679 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.679 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:13:33.679 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.601968+0000", 2024-08-17T18:13:33.679 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.758e-06 2024-08-17T18:13:33.680 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.680 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.680 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:13:33.680 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.605758+0000", 2024-08-17T18:13:33.680 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003789883 2024-08-17T18:13:33.680 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.680 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.680 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:13:33.680 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.605778+0000", 2024-08-17T18:13:33.680 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0086999999999999e-05 2024-08-17T18:13:33.680 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.681 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:13:33.681 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.681 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.681 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36960.0:59918 49.a e1387/1380 49:52556340:::benchmark_data_smithi130_364143_object59917:head v 1387'3889, mlcod=1387'3885)", 2024-08-17T18:13:33.681 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:33.603045+0000", 2024-08-17T18:13:33.681 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0083963130000000007, 2024-08-17T18:13:33.681 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0085225779999999994, 2024-08-17T18:13:33.681 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:13:33.681 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:13:33.682 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:13:33.682 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:13:33.682 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:13:33.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.603045+0000", 2024-08-17T18:13:33.682 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.682 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:13:33.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.603045+0000", 2024-08-17T18:13:33.683 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.683 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.683 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:13:33.683 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.603046+0000", 2024-08-17T18:13:33.683 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0999999999999997e-07 2024-08-17T18:13:33.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.684 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:13:33.684 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.603228+0000", 2024-08-17T18:13:33.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000182172 2024-08-17T18:13:33.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.684 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:13:33.684 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.603243+0000", 2024-08-17T18:13:33.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4639e-05 2024-08-17T18:13:33.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:13:33.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.603250+0000", 2024-08-17T18:13:33.685 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9739999999999996e-06 2024-08-17T18:13:33.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:13:33.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.604448+0000", 2024-08-17T18:13:33.685 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001198057 2024-08-17T18:13:33.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.686 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:13:33.686 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.604467+0000", 2024-08-17T18:13:33.686 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9545e-05 2024-08-17T18:13:33.686 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.686 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:13:33.686 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.686 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.686 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.686 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:59924 49.7 49:ec96460e:::benchmark_data_smithi130_364143_object59923:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1387)", 2024-08-17T18:13:33.687 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:33.605964+0000", 2024-08-17T18:13:33.687 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0054777790000000003, 2024-08-17T18:13:33.687 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005656393, 2024-08-17T18:13:33.687 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:13:33.687 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:13:33.687 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:13:33.687 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:13:33.687 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:13:33.688 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:13:33.688 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 59924 2024-08-17T18:13:33.688 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.688 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:13:33.688 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.688 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:13:33.688 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.605964+0000", 2024-08-17T18:13:33.688 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.688 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.688 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:13:33.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.605964+0000", 2024-08-17T18:13:33.689 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:13:33.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.605965+0000", 2024-08-17T18:13:33.689 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2899999999999999e-06 2024-08-17T18:13:33.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:13:33.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606065+0000", 2024-08-17T18:13:33.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000100241 2024-08-17T18:13:33.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.690 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.690 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:13:33.690 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606075+0000", 2024-08-17T18:13:33.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7079999999999992e-06 2024-08-17T18:13:33.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.690 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.690 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:13:33.690 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606081+0000", 2024-08-17T18:13:33.691 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6069999999999996e-06 2024-08-17T18:13:33.691 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.691 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.691 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:13:33.691 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606271+0000", 2024-08-17T18:13:33.691 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018986799999999999 2024-08-17T18:13:33.691 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.691 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.691 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:13:33.691 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606417+0000", 2024-08-17T18:13:33.691 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014563400000000001 2024-08-17T18:13:33.692 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.692 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.692 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:13:33.692 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606548+0000", 2024-08-17T18:13:33.692 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000131407 2024-08-17T18:13:33.692 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.692 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:13:33.692 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.692 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.692 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.692 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:59925 49.1 49:8f5c3917:::benchmark_data_smithi130_364143_object59924:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1387)", 2024-08-17T18:13:33.693 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:33.606112+0000", 2024-08-17T18:13:33.693 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0053294379999999997, 2024-08-17T18:13:33.693 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0055684419999999998, 2024-08-17T18:13:33.693 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:13:33.693 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:13:33.693 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:13:33.694 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:13:33.694 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:13:33.694 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:13:33.694 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 59925 2024-08-17T18:13:33.694 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.694 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:13:33.694 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.694 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:13:33.695 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606112+0000", 2024-08-17T18:13:33.695 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.695 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.695 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.695 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:13:33.695 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606112+0000", 2024-08-17T18:13:33.695 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.695 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.695 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.695 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:13:33.695 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606113+0000", 2024-08-17T18:13:33.695 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3e-07 2024-08-17T18:13:33.695 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.696 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:13:33.696 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606131+0000", 2024-08-17T18:13:33.696 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8351000000000001e-05 2024-08-17T18:13:33.696 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.696 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:13:33.696 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606139+0000", 2024-08-17T18:13:33.696 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7849999999999992e-06 2024-08-17T18:13:33.696 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.697 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:13:33.697 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606144+0000", 2024-08-17T18:13:33.697 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4949999999999999e-06 2024-08-17T18:13:33.697 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.697 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.697 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:13:33.697 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.611057+0000", 2024-08-17T18:13:33.697 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.004912881 2024-08-17T18:13:33.697 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.697 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.697 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:13:33.698 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.611230+0000", 2024-08-17T18:13:33.698 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017273599999999999 2024-08-17T18:13:33.698 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.698 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.698 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:13:33.698 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.611381+0000", 2024-08-17T18:13:33.698 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015159099999999999 2024-08-17T18:13:33.698 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.698 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:13:33.699 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.699 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.699 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.699 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:59926 49.7 49:e69183f9:::benchmark_data_smithi130_364143_object59925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1387)", 2024-08-17T18:13:33.699 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:33.606158+0000", 2024-08-17T18:13:33.699 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0052829790000000001, 2024-08-17T18:13:33.699 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0055784479999999997, 2024-08-17T18:13:33.699 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:13:33.699 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:13:33.699 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:13:33.699 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:13:33.700 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:13:33.700 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:13:33.700 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 59926 2024-08-17T18:13:33.700 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.700 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:13:33.700 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.700 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:13:33.700 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606158+0000", 2024-08-17T18:13:33.700 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.700 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.700 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.701 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:13:33.701 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606158+0000", 2024-08-17T18:13:33.701 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.701 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.701 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.701 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:13:33.702 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606159+0000", 2024-08-17T18:13:33.702 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4799999999999999e-07 2024-08-17T18:13:33.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.702 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.702 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:13:33.702 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606177+0000", 2024-08-17T18:13:33.702 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7827999999999999e-05 2024-08-17T18:13:33.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.702 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:13:33.703 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606183+0000", 2024-08-17T18:13:33.703 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.494e-06 2024-08-17T18:13:33.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:13:33.703 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606188+0000", 2024-08-17T18:13:33.703 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.186e-06 2024-08-17T18:13:33.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.704 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:13:33.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.608593+0000", 2024-08-17T18:13:33.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0024043179999999999 2024-08-17T18:13:33.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.704 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:13:33.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.608739+0000", 2024-08-17T18:13:33.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014686500000000001 2024-08-17T18:13:33.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.705 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:13:33.705 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.608877+0000", 2024-08-17T18:13:33.705 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013774699999999999 2024-08-17T18:13:33.705 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.705 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:13:33.705 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.705 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.705 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.705 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36960.0:59923 49.9 e1387/1380 49:954d5059:::benchmark_data_smithi130_364143_object59922:head v 1387'3717, mlcod=1387'3715)", 2024-08-17T18:13:33.705 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:33.606500+0000", 2024-08-17T18:13:33.706 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0049409529999999997, 2024-08-17T18:13:33.706 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0052902440000000004, 2024-08-17T18:13:33.706 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:13:33.706 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:13:33.706 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:13:33.706 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:13:33.706 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.706 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:13:33.706 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606500+0000", 2024-08-17T18:13:33.706 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.706 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.707 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.707 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:13:33.707 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606500+0000", 2024-08-17T18:13:33.707 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.707 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.707 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.707 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:13:33.707 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606514+0000", 2024-08-17T18:13:33.707 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3444e-05 2024-08-17T18:13:33.707 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.707 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.707 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:13:33.708 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606558+0000", 2024-08-17T18:13:33.708 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4311999999999999e-05 2024-08-17T18:13:33.708 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.708 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.708 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:13:33.708 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606567+0000", 2024-08-17T18:13:33.708 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9819999999999997e-06 2024-08-17T18:13:33.708 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.708 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.708 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:13:33.708 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606573+0000", 2024-08-17T18:13:33.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3049999999999998e-06 2024-08-17T18:13:33.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.709 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.709 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:13:33.709 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609962+0000", 2024-08-17T18:13:33.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0033888109999999998 2024-08-17T18:13:33.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.709 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.709 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:13:33.709 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609985+0000", 2024-08-17T18:13:33.710 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2396999999999999e-05 2024-08-17T18:13:33.710 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.710 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:13:33.710 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.710 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36960.0:59928 49.a e1387/1380 49:518eb13b:::benchmark_data_smithi130_364143_object59927:head v 1387'3890, mlcod=1387'3886)", 2024-08-17T18:13:33.710 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:33.606684+0000", 2024-08-17T18:13:33.710 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0047573210000000001, 2024-08-17T18:13:33.710 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0051591529999999997, 2024-08-17T18:13:33.710 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:13:33.711 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:13:33.711 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:13:33.711 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:13:33.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:13:33.711 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606684+0000", 2024-08-17T18:13:33.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:13:33.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606684+0000", 2024-08-17T18:13:33.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.712 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:13:33.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606685+0000", 2024-08-17T18:13:33.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8999999999999996e-07 2024-08-17T18:13:33.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.712 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:13:33.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606774+0000", 2024-08-17T18:13:33.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9184000000000005e-05 2024-08-17T18:13:33.713 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.713 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:13:33.713 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606783+0000", 2024-08-17T18:13:33.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7360000000000007e-06 2024-08-17T18:13:33.713 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.713 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:13:33.713 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606788+0000", 2024-08-17T18:13:33.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6219999999999997e-06 2024-08-17T18:13:33.713 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.714 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.714 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:13:33.714 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.606999+0000", 2024-08-17T18:13:33.714 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00021113300000000001 2024-08-17T18:13:33.714 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.714 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.714 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:13:33.714 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.607017+0000", 2024-08-17T18:13:33.714 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7450999999999999e-05 2024-08-17T18:13:33.715 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.715 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:13:33.715 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.715 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.715 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.715 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36960.0:59920 49.d e1387/1380 49:b0a1e4a8:::benchmark_data_smithi130_364143_object59919:head v 1387'3786, mlcod=1387'3781)", 2024-08-17T18:13:33.715 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:33.609664+0000", 2024-08-17T18:13:33.715 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.001777629, 2024-08-17T18:13:33.715 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0022249069999999999, 2024-08-17T18:13:33.715 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:13:33.715 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:13:33.716 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:13:33.716 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:13:33.716 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.716 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:13:33.716 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609664+0000", 2024-08-17T18:13:33.716 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.716 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.716 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.716 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:13:33.716 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609664+0000", 2024-08-17T18:13:33.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.717 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:13:33.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609664+0000", 2024-08-17T18:13:33.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.16e-07 2024-08-17T18:13:33.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.717 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:13:33.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609694+0000", 2024-08-17T18:13:33.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9096000000000001e-05 2024-08-17T18:13:33.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.718 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.718 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:13:33.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609704+0000", 2024-08-17T18:13:33.718 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0597999999999999e-05 2024-08-17T18:13:33.718 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.718 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.718 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:13:33.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609711+0000", 2024-08-17T18:13:33.718 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4500000000000001e-06 2024-08-17T18:13:33.718 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.719 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.719 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:13:33.719 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609875+0000", 2024-08-17T18:13:33.719 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000164549 2024-08-17T18:13:33.719 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.719 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.719 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:13:33.719 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609894+0000", 2024-08-17T18:13:33.719 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9012e-05 2024-08-17T18:13:33.719 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.719 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:13:33.720 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.720 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.720 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36960.0:59922 49.4 e1387/1380 49:2d660476:::benchmark_data_smithi130_364143_object59921:head v 1387'3779, mlcod=1387'3777)", 2024-08-17T18:13:33.720 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:33.609737+0000", 2024-08-17T18:13:33.720 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0017043290000000001, 2024-08-17T18:13:33.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002195509, 2024-08-17T18:13:33.720 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:13:33.720 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:13:33.720 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:13:33.720 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:13:33.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:13:33.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609737+0000", 2024-08-17T18:13:33.721 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.721 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:13:33.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609737+0000", 2024-08-17T18:13:33.721 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.721 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:13:33.722 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609737+0000", 2024-08-17T18:13:33.722 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3299999999999998e-07 2024-08-17T18:13:33.722 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.722 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.722 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:13:33.722 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609757+0000", 2024-08-17T18:13:33.722 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9692000000000001e-05 2024-08-17T18:13:33.722 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.722 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.722 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:13:33.722 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609763+0000", 2024-08-17T18:13:33.723 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1240000000000002e-06 2024-08-17T18:13:33.723 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.723 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.723 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:13:33.723 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609767+0000", 2024-08-17T18:13:33.723 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3309999999999997e-06 2024-08-17T18:13:33.723 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.723 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.723 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:13:33.723 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.610065+0000", 2024-08-17T18:13:33.723 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00029765400000000001 2024-08-17T18:13:33.724 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.724 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.724 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:13:33.724 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.610081+0000", 2024-08-17T18:13:33.724 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6249999999999999e-05 2024-08-17T18:13:33.724 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.724 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:13:33.724 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.724 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.724 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.724 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:59929 49.1 49.3142b561 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1387)", 2024-08-17T18:13:33.724 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:33.609855+0000", 2024-08-17T18:13:33.725 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0015862770000000001, 2024-08-17T18:13:33.725 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021204280000000002, 2024-08-17T18:13:33.725 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:13:33.725 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:13:33.725 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "queued for pg", 2024-08-17T18:13:33.725 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:13:33.725 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:13:33.725 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:13:33.725 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 59929 2024-08-17T18:13:33.725 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.725 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:13:33.725 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.726 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:13:33.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609855+0000", 2024-08-17T18:13:33.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.726 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.726 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.726 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:13:33.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609855+0000", 2024-08-17T18:13:33.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.726 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.726 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.726 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:13:33.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609857+0000", 2024-08-17T18:13:33.727 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6160000000000001e-06 2024-08-17T18:13:33.727 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:13:33.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609877+0000", 2024-08-17T18:13:33.727 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0622999999999999e-05 2024-08-17T18:13:33.727 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:13:33.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609889+0000", 2024-08-17T18:13:33.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2075e-05 2024-08-17T18:13:33.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.728 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:13:33.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609897+0000", 2024-08-17T18:13:33.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.345e-06 2024-08-17T18:13:33.728 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.728 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:13:33.728 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.729 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:59931 49.7 49:e3644bee:::benchmark_data_smithi130_364143_object59930:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1387)", 2024-08-17T18:13:33.729 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:33.609990+0000", 2024-08-17T18:13:33.729 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.001451492, 2024-08-17T18:13:33.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0020364910000000001, 2024-08-17T18:13:33.729 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:13:33.729 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:13:33.729 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:13:33.729 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:13:33.729 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:13:33.729 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:13:33.729 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 59931 2024-08-17T18:13:33.729 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.730 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:13:33.730 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.730 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:13:33.730 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609990+0000", 2024-08-17T18:13:33.730 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.730 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.730 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.730 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:13:33.730 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609990+0000", 2024-08-17T18:13:33.730 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:13:33.730 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.731 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.731 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:13:33.731 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.609990+0000", 2024-08-17T18:13:33.731 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5000000000000002e-07 2024-08-17T18:13:33.731 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.731 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.731 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:13:33.731 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.610008+0000", 2024-08-17T18:13:33.731 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7421000000000001e-05 2024-08-17T18:13:33.731 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.731 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.732 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:13:33.732 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.610017+0000", 2024-08-17T18:13:33.732 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8410000000000006e-06 2024-08-17T18:13:33.732 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.732 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.732 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:13:33.732 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.610024+0000", 2024-08-17T18:13:33.732 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1350000000000003e-06 2024-08-17T18:13:33.732 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.732 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.732 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:13:33.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.611277+0000", 2024-08-17T18:13:33.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001252677 2024-08-17T18:13:33.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.733 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:13:33.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.611401+0000", 2024-08-17T18:13:33.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000124778 2024-08-17T18:13:33.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:13:33.733 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:13:33.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:13:33.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:33.611521+0000", 2024-08-17T18:13:33.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000120087 2024-08-17T18:13:33.734 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.734 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:13:33.734 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.734 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:13:33.734 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T18:13:33.734 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 12 2024-08-17T18:13:33.734 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:13:33.734 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:33.760 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:33.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:33.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:33.862 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:33.884 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:33.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:33.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:33.912 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 39 16 60273 60257 96.5397 71.625 0.0156863 0.0103536 2024-08-17T18:13:33.986 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:34.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:34.027 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:34.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:34.128 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:34.164 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:34.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:34.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:34.266 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:34.302 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:34.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:34.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:34.404 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:34.440 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:34.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:34.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:34.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:34.579 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:34.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:34.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:34.681 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:34.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:34.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:34.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:34.823 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:34.847 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:34.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:34.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:34.912 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:13:34.912115+0000 min lat: 0.00139804 max lat: 0.329999 avg lat: 0.0104172 2024-08-17T18:13:34.912 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:13:34.912 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 40 16 61441 61425 95.9512 73 0.00537259 0.0104172 2024-08-17T18:13:34.949 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:34.986 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:34.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:34.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:35.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:35.126 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:35.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:35.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:35.228 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:35.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:35.249 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:35.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:35.350 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:35.373 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:35.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:35.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:35.474 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:35.519 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:35.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:35.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:35.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:35 smithi179 ceph-mon[93519]: pgmap v2556: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T18:13:35.621 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:35.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:35.645 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:35.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:35.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:35 smithi130 ceph-mon[110068]: pgmap v2556: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T18:13:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:35 smithi130 ceph-mon[105521]: pgmap v2556: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T18:13:35.746 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:35.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:35.770 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:35.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:35.871 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:35.892 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:35.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:35.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:35.912 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 41 16 62461 62445 95.1657 63.75 0.0128064 0.0104899 2024-08-17T18:13:35.994 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:36.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:36.018 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:36.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:36.119 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:36.141 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:36.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:36.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:36.243 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:36.265 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:36.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:36.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:36.367 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:36.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:36.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:36.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:36.506 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:36.541 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:36.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:36.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:36.642 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:36.689 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:36.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:36.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:36.791 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:36.828 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:36.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:36.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:36.912 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 42 16 63512 63496 94.4638 65.6875 0.072952 0.0105806 2024-08-17T18:13:36.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:36.954 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:36.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:36.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:37.056 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:37.093 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:37.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:37.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:37.195 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:37.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:37.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:37.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:37.321 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:37.355 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:37.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:37.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:37.457 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:37.481 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:37.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:37.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:37.583 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:37.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:37 smithi179 ceph-mon[93519]: pgmap v2557: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T18:13:37.618 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:37.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:37.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:37.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:37 smithi130 ceph-mon[105521]: pgmap v2557: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T18:13:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:37 smithi130 ceph-mon[110068]: pgmap v2557: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 344 GiB / 358 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-08-17T18:13:37.720 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:37.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:37.743 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:37.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:37.844 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:37.867 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:37.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:37.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:37.912 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 43 16 64523 64507 93.7364 63.1875 0.00841233 0.0106627 2024-08-17T18:13:37.970 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:38.013 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:38.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:38.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:38.115 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:38.136 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:38.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:38.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:38.238 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:38.276 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:38.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:38.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:38.378 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:38.399 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:38.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:38.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:38.501 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:38.534 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:38.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:38.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:38.636 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:38.661 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:13:38.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:38.711 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:38.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:38.812 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:38.851 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:38.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:38.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:38.912 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 44 16 65502 65486 92.9966 61.1875 0.00858901 0.0107478 2024-08-17T18:13:38.953 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:38.974 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:38.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:38.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:39.076 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:39.113 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:39.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:39.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:39.214 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:39.237 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:39.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:39.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:39.339 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:39.362 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:39.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:39.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:39.464 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:39.501 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:39.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:39.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:39.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:39 smithi179 ceph-mon[93519]: pgmap v2558: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 72 MiB/s wr, 1.14k op/s 2024-08-17T18:13:39.603 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:39.625 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:39.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:39.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:39 smithi130 ceph-mon[105521]: pgmap v2558: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 72 MiB/s wr, 1.14k op/s 2024-08-17T18:13:39.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:39 smithi130 ceph-mon[110068]: pgmap v2558: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 343 GiB / 358 GiB avail; 72 MiB/s wr, 1.14k op/s 2024-08-17T18:13:39.727 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:39.769 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:39.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:39.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:39.871 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:39.898 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:39.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:39.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:39.912 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 45 16 66296 66280 92.0329 49.625 0.00918949 0.0108542 2024-08-17T18:13:40.001 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:40.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:40.025 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:40.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:40.126 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:40.148 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:40.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:40.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:13:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:13:39] "GET /metrics HTTP/1.1" 200 36778 "" "Prometheus/2.51.0" 2024-08-17T18:13:40.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:40.301 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:40.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:40.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:40.404 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:40.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:40.427 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:40.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:40.528 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:40.568 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:40.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:40.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:40.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:13:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:13:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:13:40.670 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:40.708 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:40.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:40.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:40.810 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:40.831 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:40.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:40.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:40.912 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 46 16 67081 67065 91.0988 49.0625 0.0131665 0.0109682 2024-08-17T18:13:40.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:40.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:40.955 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:40.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:41.056 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:41.098 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:41.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:41.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:41.200 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:41.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:41.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:41.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:41.325 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:41.348 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:41.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:41.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:41.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:41.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:41.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:41.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:41.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:41.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:41 smithi179 ceph-mon[93519]: pgmap v2559: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T18:13:41.596 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:41.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:41.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:41.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:41 smithi130 ceph-mon[105521]: pgmap v2559: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T18:13:41.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:41 smithi130 ceph-mon[110068]: pgmap v2559: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-08-17T18:13:41.697 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:41.720 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:41.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:41.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:41.822 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:41.846 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:41.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:41.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:41.912 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 47 16 67935 67919 90.2962 53.375 0.095157 0.011057 2024-08-17T18:13:41.948 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:41.984 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:41.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:41.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:42.086 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:42.109 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:42.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:42.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:42.146 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T18:13:42.211 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:42.233 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:42.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:42.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:42.335 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:42.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:42.373 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:42.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:42.474 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:42.531 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:42.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:42.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:42.633 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:42.685 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:42.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:42.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:42.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:42.844 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:42.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:42.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:42.912 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 48 16 68780 68764 89.5153 52.8125 0.0138981 0.0111668 2024-08-17T18:13:42.946 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:42.968 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:42.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:42.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:43.018 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:13:43.018 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:13:43.018 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:13:43.018 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:13:43.018 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:13:43.019 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:13:43.019 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:13:43.019 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:13:43.019 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:13:43.019 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:13:43.042 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:13:43.043 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:13:43.070 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:43.073 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:13:43.074 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:13:43.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:43.105 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:43.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:43.106 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:13:43.106 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:13:43.206 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:43.229 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:43.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:43.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:43.331 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:43.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:43.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:43.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:43.456 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:43.478 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:43.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:43.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:43.580 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:43 smithi130 ceph-mon[105521]: pgmap v2560: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T18:13:43.580 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:13:43.580 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:43 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/937896665' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:13:43.580 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:43 smithi130 ceph-mon[110068]: pgmap v2560: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T18:13:43.581 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:13:43.581 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:43 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/937896665' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:13:43.581 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:43.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:43 smithi179 ceph-mon[93519]: pgmap v2560: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 343 GiB / 358 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T18:13:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:13:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:43 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/937896665' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:13:43.622 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:43.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:43.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:43.724 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:43.746 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:43.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:43.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:43.848 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:43.872 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:43.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:43.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:43.913 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 49 16 69665 69649 88.8173 55.3125 0.0180339 0.0112526 2024-08-17T18:13:43.974 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:43.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:43.997 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:43.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:44.098 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:44.120 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:44.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:44.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:44.222 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:44.267 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:44.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:44.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:44.369 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:44.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:44.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:44.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:44.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:44.547 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:44.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:44.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:44.649 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:44.671 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:44.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:44.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:44.773 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:44.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:44.815 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:44.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:44.913 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 50 16 70702 70686 88.3371 64.8125 0.0100423 0.0113144 2024-08-17T18:13:44.915 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:44.952 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:44.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:44.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:45.054 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:45.099 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:13:45.127 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:45.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:45.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:45.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:45.284 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:45.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:45.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:45.325 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:13:45.386 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:45.429 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:45.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:45.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:45.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:45.568 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:45.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:45.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:45.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:45 smithi179 ceph-mon[93519]: pgmap v2561: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-08-17T18:13:45.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:45 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2431008904' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:13:45.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:45 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:13:45.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:45 smithi130 ceph-mon[105521]: pgmap v2561: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-08-17T18:13:45.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:45 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2431008904' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:13:45.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:45 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:13:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:45 smithi130 ceph-mon[110068]: pgmap v2561: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-08-17T18:13:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:45 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2431008904' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:13:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:45 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:13:45.671 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:45.694 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:45.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:45.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:45.796 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:45.836 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:45.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:45.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:45.913 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 51 16 71726 71710 87.8599 64 0.0112157 0.0113765 2024-08-17T18:13:45.939 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:45.960 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:45.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:45.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:46.062 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:46.105 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:46.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:46.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:46.207 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:46.248 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:46.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:46.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:46.350 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:46.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:46.389 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:46.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:46.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:46.513 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:46.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:46.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:46 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:13:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:46 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T18:13:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:46 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:13:46.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:46 smithi179 ceph-mon[93519]: osdmap e1388: 8 total, 8 up, 4 in 2024-08-17T18:13:46.615 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:46.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:46.657 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:46.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:46.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:46 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:13:46.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:46 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T18:13:46.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:46 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:13:46.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:46 smithi130 ceph-mon[105521]: osdmap e1388: 8 total, 8 up, 4 in 2024-08-17T18:13:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:46 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:13:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:46 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T18:13:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:46 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:13:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:46 smithi130 ceph-mon[110068]: osdmap e1388: 8 total, 8 up, 4 in 2024-08-17T18:13:46.670 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:13:46.758 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:46.802 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:46.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:46.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:46.904 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:46.913 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 52 16 72742 72726 87.3904 63.5 0.011947 0.0114356 2024-08-17T18:13:46.944 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:46.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:46.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:47.045 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:47.067 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:47.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:47.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:47.169 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:47.205 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:47.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:47.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:47.307 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:47.329 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:47.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:47.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:47.430 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:47.453 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:47.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:47.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:47.554 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:47.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:47.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:47.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:47.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:47 smithi130 ceph-mon[105521]: pgmap v2563: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 61 MiB/s wr, 970 op/s 2024-08-17T18:13:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:47 smithi130 ceph-mon[110068]: pgmap v2563: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 61 MiB/s wr, 970 op/s 2024-08-17T18:13:47.678 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:47.717 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:47.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:47.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:47.819 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:47.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:47.843 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:47.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:47 smithi179 ceph-mon[93519]: pgmap v2563: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 342 GiB / 358 GiB avail; 61 MiB/s wr, 970 op/s 2024-08-17T18:13:47.913 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 53 16 73677 73661 86.8441 58.4375 0.0134632 0.011509 2024-08-17T18:13:47.944 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:47.967 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:47.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:47.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:48.068 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:48.102 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:48.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:48.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:48.204 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:48.244 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:48.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:48.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:48.346 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:48.368 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:48.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:48.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:48.470 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:48.492 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:48.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:48.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:48.593 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:48.639 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:48.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:48.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:48.741 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:48.763 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:48.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:48.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:48.865 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:48.887 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:48.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:48.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:48.913 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 54 16 74658 74642 86.3713 61.3125 0.0180525 0.0115716 2024-08-17T18:13:48.989 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:49.042 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:49.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:49.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:49.144 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:49.182 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:49.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:49.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:49.283 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:49.305 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:49.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:49.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:49.309 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:13:49.344 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:13:49.344 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:13:49.360 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T18:13:49.407 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:49.430 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:49.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:49.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:49.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:49.567 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:49 smithi130 ceph-mon[105521]: pgmap v2564: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 58 MiB/s wr, 930 op/s 2024-08-17T18:13:49.568 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:49 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/595820806' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:13:49.568 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:49 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:13:49.568 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:49 smithi130 ceph-mon[110068]: pgmap v2564: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 58 MiB/s wr, 930 op/s 2024-08-17T18:13:49.568 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:49 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/595820806' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:13:49.568 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:49 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:13:49.570 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:49.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:49.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:49.672 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:49.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:49.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:49.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:49.797 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:49.834 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:49.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:49.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:49.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:49 smithi179 ceph-mon[93519]: pgmap v2564: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 341 GiB / 358 GiB avail; 58 MiB/s wr, 930 op/s 2024-08-17T18:13:49.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:49 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/595820806' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:13:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:49 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:13:49.914 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 55 16 75650 75634 85.9281 62 0.0107562 0.0116319 2024-08-17T18:13:49.936 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:49.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:49.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:49.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:50.060 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:50.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:50.083 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:50.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:13:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:13:49] "GET /metrics HTTP/1.1" 200 36780 "" "Prometheus/2.51.0" 2024-08-17T18:13:50.184 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:50.207 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:50.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:50.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:50.310 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:50.332 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:50.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:50.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:50.363 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T18:13:50.434 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:50.456 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:50.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:50.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:50.509 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:13:50.558 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:50.574 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:50 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:13:50.574 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:50 smithi130 ceph-mon[105521]: osdmap e1389: 8 total, 8 up, 4 in 2024-08-17T18:13:50.575 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:50 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:13:50.575 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:50 smithi130 ceph-mon[110068]: osdmap e1389: 8 total, 8 up, 4 in 2024-08-17T18:13:50.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:50 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:13:50.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:50 smithi179 ceph-mon[93519]: osdmap e1389: 8 total, 8 up, 4 in 2024-08-17T18:13:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:13:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:13:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:13:50.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:50.646 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:50.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:50.747 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:50.772 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:50.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:50.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:50.873 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:50.915 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 56 16 76604 76588 85.4575 59.625 0.0117611 0.0116956 2024-08-17T18:13:50.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:50.915 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:50.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:51.016 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:51.038 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:51.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:51.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:51.139 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:51.177 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:51.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:51.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:51.278 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:51.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:51.304 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:51.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:51.405 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:51.430 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:51.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:51.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:51.532 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:51.555 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:51.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:51.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:51.657 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:51 smithi130 ceph-mon[105521]: pgmap v2566: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T18:13:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:51 smithi130 ceph-mon[110068]: pgmap v2566: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T18:13:51.694 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:51.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:51.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:51.796 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:51.818 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:51.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:51.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:51 smithi179 ceph-mon[93519]: pgmap v2566: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T18:13:51.914 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 57 16 77522 77506 84.9648 57.375 0.0143322 0.0117637 2024-08-17T18:13:51.920 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:51.963 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:51.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:51.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:52.065 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:52.088 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:52.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:52.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:52.190 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:52.214 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:52.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:52.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:52.316 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:52.347 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:52.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:52.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:52.449 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:52.490 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:52.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:52.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:52.592 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:52.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:52.635 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:52.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:52.736 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:52.761 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:52.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:52.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:52.863 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:52.914 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 58 16 78526 78510 84.5818 62.75 0.0133604 0.0118171 2024-08-17T18:13:52.920 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:52.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:52.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:53.022 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:53.044 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:53.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:53.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:53.146 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:53.185 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:53.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:53.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:53.287 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:53.310 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:53.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:53.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:53.412 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:53.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:53.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:53.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:53.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:53.558 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:53.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:53.659 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:53.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:53 smithi130 ceph-mon[105521]: pgmap v2567: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 45 MiB/s wr, 712 op/s 2024-08-17T18:13:53.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:53 smithi130 ceph-mon[110068]: pgmap v2567: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 45 MiB/s wr, 712 op/s 2024-08-17T18:13:53.683 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:53.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:53.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:53.786 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:53.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T18:13:53.839 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:53.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:53.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:53.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:53 smithi179 ceph-mon[93519]: pgmap v2567: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 45 MiB/s wr, 712 op/s 2024-08-17T18:13:53.919 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 59 16 79558 79542 84.2382 64.5 0.0309344 0.0118653 2024-08-17T18:13:53.942 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:54.000 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:54.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:54.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:54.102 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:54.124 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:54.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:54.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:54.226 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:54.287 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:54.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:54.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:54.389 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:54.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:54.411 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:54.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:54.423 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:13:54.424 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:13:54.459 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:13:54.459 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:13:54.480 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:13:54.497 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:13:54.500 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:13:54.500 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:13:54.513 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:54.540 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:13:54.540 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:13:54.548 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:13:54.562 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:13:54.563 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:13:54.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:54.576 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:54.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:54.594 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:13:54.594 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:13:54.626 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:13:54.627 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:13:54.627 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1768444528' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:13:54.629 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1768444528' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:13:54.659 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:13:54.659 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:13:54.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:54.699 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:54.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:54.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:54.801 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:54.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1768444528' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:13:54.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:54.853 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:54.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:54.917 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:13:54.917309+0000 min lat: 0.00139804 max lat: 0.329999 avg lat: 0.0119418 2024-08-17T18:13:54.917 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:13:54.917 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 60 16 80393 80377 83.704 52.1875 0.00649977 0.0119418 2024-08-17T18:13:54.955 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:54.977 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:54.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:54.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:55.079 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:55.101 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:55.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:55.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:55.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:55.225 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:55.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:55.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:55.327 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:55.350 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:55.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:55.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:55.451 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:55.465 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:55 smithi130 ceph-mon[105521]: pgmap v2568: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T18:13:55.465 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:55 smithi130 ceph-mon[110068]: pgmap v2568: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T18:13:55.488 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:55.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:55.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:55.590 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:55.626 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:55.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:55.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:55.729 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:55.752 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:55.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:55.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:55.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:55 smithi179 ceph-mon[93519]: pgmap v2568: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T18:13:55.854 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:55.895 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:55.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:55.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:55.917 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 61 16 81440 81424 83.4045 65.4375 0.0199099 0.0119848 2024-08-17T18:13:55.997 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:56.037 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:56.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:56.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:56.141 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:56.173 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:56.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:56.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:56.275 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:56.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:56.313 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:56.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:56.414 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:56.465 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:56.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:56.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:56.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:56.590 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:56.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:56.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:56.619 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:56 smithi130 ceph-mon[105521]: pgmap v2569: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 61 MiB/s wr, 974 op/s 2024-08-17T18:13:56.619 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:56 smithi130 ceph-mon[110068]: pgmap v2569: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 61 MiB/s wr, 974 op/s 2024-08-17T18:13:56.692 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:56.716 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:56.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:56.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:56.818 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:56.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:56 smithi179 ceph-mon[93519]: pgmap v2569: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 61 MiB/s wr, 974 op/s 2024-08-17T18:13:56.856 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:56.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:56.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:56.917 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 62 16 82489 82473 83.1167 65.5625 0.0110188 0.0120255 2024-08-17T18:13:56.959 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:56.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:56.984 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:56.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:57.085 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:13:57.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:57.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:57.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:13:57.209 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:57.233 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:57.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:57.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:57.335 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:57.360 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:57.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:57.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:57.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:57.500 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:57.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:57.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:57.602 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:57.628 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:57.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:57.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:57.730 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:57.751 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:57.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:57.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:57.853 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:57.876 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:57.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:57.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:57.917 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 63 16 83495 83479 82.7953 62.875 0.011351 0.0120734 2024-08-17T18:13:57.978 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:58.002 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:58.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:58.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:58.103 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:58.126 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:58.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:58.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:58.228 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:58.250 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:58.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:58.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:58.352 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:58.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:58.395 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:58.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:58.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:13:58.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:58.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:13:58.496 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:58.537 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:58.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:58.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:58.639 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:58.672 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:58.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:58.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:58.774 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:13:58.814 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:58.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:58.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:13:58.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:13:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:13:58.916 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:13:58.917 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 64 16 84447 84431 82.4313 59.5 0.0152689 0.0121257 2024-08-17T18:13:58.954 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:58.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:58.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:13:59.056 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:59.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:59.098 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:59.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:59.199 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:59.220 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:59.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:59.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:59.322 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:13:59.345 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:59.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:59.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:13:59.447 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:13:59.505 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:59.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:59.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:13:59.607 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:13:59.624 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:13:59 smithi130 ceph-mon[105521]: pgmap v2570: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 61 MiB/s wr, 980 op/s 2024-08-17T18:13:59.624 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:13:59 smithi130 ceph-mon[110068]: pgmap v2570: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 61 MiB/s wr, 980 op/s 2024-08-17T18:13:59.654 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:13:59.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:59.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:13:59.756 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:13:59.780 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:59.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:59.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:13:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:13:59 smithi179 ceph-mon[93519]: pgmap v2570: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 61 MiB/s wr, 980 op/s 2024-08-17T18:13:59.882 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:13:59.904 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:13:59.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:13:59.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:13:59.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:13:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:13:59] "GET /metrics HTTP/1.1" 200 36781 "" "Prometheus/2.51.0" 2024-08-17T18:13:59.917 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 65 16 85502 85486 82.1775 65.9375 0.0221107 0.0121627 2024-08-17T18:14:00.006 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:00.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:00.048 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:00.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:00.149 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:00.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:14:00.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:00.191 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:00.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:00.293 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:00.316 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:00.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:00.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:00.417 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:00.439 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:00.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:00.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:00.543 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:00.591 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:00.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:00.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:00.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:14:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:14:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:14:00.693 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:00.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:00.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:00.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:00.817 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:00.839 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:00.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:00.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:00.917 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 66 16 86465 86449 81.8443 60.1875 0.0111096 0.0122131 2024-08-17T18:14:00.941 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:00.986 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:00.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:00.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:01.088 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:01.110 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:01.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:01.212 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:01.235 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:01.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:01.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:01.336 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:01.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:01.360 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:01.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:01.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:01.500 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:01.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:01.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:01.602 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:01.633 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:01 smithi130 ceph-mon[105521]: pgmap v2571: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 60 MiB/s wr, 967 op/s 2024-08-17T18:14:01.633 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:01 smithi130 ceph-mon[110068]: pgmap v2571: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 60 MiB/s wr, 967 op/s 2024-08-17T18:14:01.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:01.644 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:01.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:01.746 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:01.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:01.783 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:01.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:01 smithi179 ceph-mon[93519]: pgmap v2571: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 60 MiB/s wr, 967 op/s 2024-08-17T18:14:01.884 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:01.918 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 67 16 87397 87381 81.491 58.25 0.0135138 0.0122649 2024-08-17T18:14:01.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:01.928 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:01.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:02.030 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:02.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:02.069 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:02.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:02.171 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:02.194 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:02.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:02.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:02.296 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:02.340 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:02.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:02.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:02.381 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:14:02.382 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T18:14:02.442 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:02.465 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:02.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:02.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:02.507 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:02 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1188850463' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:14:02.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:02 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:14:02.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:02 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3686053814' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:14:02.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:02 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:14:02.508 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:02 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1188850463' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:14:02.508 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:02 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:14:02.509 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:02 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3686053814' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:14:02.509 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:02 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:14:02.566 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:02.605 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:02.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:02.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:02.707 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:02.743 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:02.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:02.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:02 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1188850463' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:14:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:02 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:14:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:02 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3686053814' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:14:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:02 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:14:02.845 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:02.883 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:02.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:02.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:02.919 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 68 16 88405 88389 81.2191 63 0.0102096 0.0123065 2024-08-17T18:14:02.986 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:03.011 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:03.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:03.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:03.113 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:03.135 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:03.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:03.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:03.237 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:03.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:03.274 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:03.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:03.283 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T18:14:03.375 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:03.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:03.398 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:03.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:03.499 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:03.540 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:03.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:03.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:03.642 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:03.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:03 smithi130 ceph-mon[105521]: pgmap v2572: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 51 MiB/s wr, 818 op/s 2024-08-17T18:14:03.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:03 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:14:03.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:03 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:14:03.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:03 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:14:03.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:03 smithi130 ceph-mon[105521]: osdmap e1390: 8 total, 8 up, 4 in 2024-08-17T18:14:03.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:03 smithi130 ceph-mon[110068]: pgmap v2572: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 51 MiB/s wr, 818 op/s 2024-08-17T18:14:03.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:03 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:14:03.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:03 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:14:03.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:03 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:14:03.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:03 smithi130 ceph-mon[110068]: osdmap e1390: 8 total, 8 up, 4 in 2024-08-17T18:14:03.706 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:03.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:03.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:03.808 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:03.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:03 smithi179 ceph-mon[93519]: pgmap v2572: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 51 MiB/s wr, 818 op/s 2024-08-17T18:14:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:03 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:14:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:03 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:14:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:03 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:14:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:03 smithi179 ceph-mon[93519]: osdmap e1390: 8 total, 8 up, 4 in 2024-08-17T18:14:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:03.850 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:03.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:03.919 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 69 16 89408 89392 80.9505 62.6875 0.0105003 0.0123482 2024-08-17T18:14:03.951 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:03.973 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:03.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:03.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:04.075 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:04.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:04.100 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:04.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:04.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:04.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:04.247 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:04.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:04.348 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:04.370 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:04.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:04.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:04.472 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:04.506 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:04.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:04.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:04.608 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:04.654 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:04.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:04.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:04.657 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:14:04.661 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:14:04.687 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:14:04.687 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:14:04.687 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:14:04.687 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:14:04.687 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.687 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:115193 47.1 47:8a1837dc:::benchmark_data_smithi130_302447_object19958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:14:04.688 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:56.586429+0000", 2024-08-17T18:14:04.688 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 488.09625893600003, 2024-08-17T18:14:04.688 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0156209679999999, 2024-08-17T18:14:04.688 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.688 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.688 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:14:04.688 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:04.688 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:14:04.688 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:14:04.688 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115193 2024-08-17T18:14:04.688 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.688 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586429+0000", 2024-08-17T18:14:04.689 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586429+0000", 2024-08-17T18:14:04.689 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586430+0000", 2024-08-17T18:14:04.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.364e-06 2024-08-17T18:14:04.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.690 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.690 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.690 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586435+0000", 2024-08-17T18:14:04.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3900000000000003e-06 2024-08-17T18:14:04.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.690 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.690 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.690 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586450+0000", 2024-08-17T18:14:04.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5209e-05 2024-08-17T18:14:04.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.691 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.691 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.691 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586457+0000", 2024-08-17T18:14:04.691 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0099999999999998e-06 2024-08-17T18:14:04.691 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.691 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.691 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.691 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586575+0000", 2024-08-17T18:14:04.691 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000118305 2024-08-17T18:14:04.691 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.691 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.691 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.691 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586714+0000", 2024-08-17T18:14:04.692 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013882799999999999 2024-08-17T18:14:04.692 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.692 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.692 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:14:04.692 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586850+0000", 2024-08-17T18:14:04.692 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000136198 2024-08-17T18:14:04.692 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.692 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.692 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.692 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.594476+0000", 2024-08-17T18:14:04.692 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0076255070000000001 2024-08-17T18:14:04.692 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.692 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.693 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:04.693 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.594483+0000", 2024-08-17T18:14:04.693 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5549999999999997e-06 2024-08-17T18:14:04.693 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.693 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.693 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.693 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.081391+0000", 2024-08-17T18:14:04.693 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.48690757499999998 2024-08-17T18:14:04.693 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.693 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.693 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:04.693 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.081400+0000", 2024-08-17T18:14:04.693 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4089999999999992e-06 2024-08-17T18:14:04.694 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.694 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.694 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.694 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.601973+0000", 2024-08-17T18:14:04.694 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.52057316499999995 2024-08-17T18:14:04.694 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.694 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.694 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.694 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.602050+0000", 2024-08-17T18:14:04.694 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6452999999999995e-05 2024-08-17T18:14:04.694 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.694 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.694 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.695 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.695 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.695 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:115174 47.4 47:2d651f61:::benchmark_data_smithi130_302447_object19939:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1317)", 2024-08-17T18:14:04.695 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:57.071033+0000", 2024-08-17T18:14:04.695 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 487.61165477600002, 2024-08-17T18:14:04.695 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.540335236, 2024-08-17T18:14:04.695 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.695 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.695 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:14:04.695 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:04.695 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:14:04.695 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:14:04.695 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115174 2024-08-17T18:14:04.695 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.696 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.696 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.696 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071033+0000", 2024-08-17T18:14:04.696 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.696 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.696 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.696 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071033+0000", 2024-08-17T18:14:04.696 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.696 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.696 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.696 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.697 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071035+0000", 2024-08-17T18:14:04.697 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.666e-06 2024-08-17T18:14:04.697 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.697 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.697 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.697 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071041+0000", 2024-08-17T18:14:04.697 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3199999999999996e-06 2024-08-17T18:14:04.697 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.697 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.697 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.697 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071076+0000", 2024-08-17T18:14:04.697 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5286000000000001e-05 2024-08-17T18:14:04.697 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.698 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.698 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.698 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071087+0000", 2024-08-17T18:14:04.698 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0327e-05 2024-08-17T18:14:04.698 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.698 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.698 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.698 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071159+0000", 2024-08-17T18:14:04.698 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2201999999999998e-05 2024-08-17T18:14:04.698 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.698 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.698 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op must wait for map", 2024-08-17T18:14:04.698 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.071167+0000", 2024-08-17T18:14:04.699 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1329999999999999e-06 2024-08-17T18:14:04.699 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.699 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.699 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.699 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.075440+0000", 2024-08-17T18:14:04.699 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0042733340000000002 2024-08-17T18:14:04.699 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.699 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.699 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:04.699 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.075449+0000", 2024-08-17T18:14:04.699 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3699999999999995e-06 2024-08-17T18:14:04.699 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.699 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.699 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.700 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.605615+0000", 2024-08-17T18:14:04.700 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.53016654399999996 2024-08-17T18:14:04.700 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.700 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.700 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.700 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.611368+0000", 2024-08-17T18:14:04.700 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0057530539999999996 2024-08-17T18:14:04.700 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.700 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.700 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.700 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.700 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.700 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115437 48.8 48:1823c229:::benchmark_data_smithi130_327597_object115436:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:14:04.701 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.095352+0000", 2024-08-17T18:14:04.701 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 307.58733601, 2024-08-17T18:14:04.701 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5254109470000001, 2024-08-17T18:14:04.701 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.701 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.701 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:14:04.701 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:04.701 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:04.701 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:04.701 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115437 2024-08-17T18:14:04.701 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.701 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.701 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.701 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.702 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095352+0000", 2024-08-17T18:14:04.702 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.702 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.702 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.702 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095352+0000", 2024-08-17T18:14:04.702 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.702 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.702 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.702 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095352+0000", 2024-08-17T18:14:04.702 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2199999999999999e-07 2024-08-17T18:14:04.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.703 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095377+0000", 2024-08-17T18:14:04.703 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4745000000000001e-05 2024-08-17T18:14:04.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.703 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095385+0000", 2024-08-17T18:14:04.703 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6199999999999999e-06 2024-08-17T18:14:04.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095390+0000", 2024-08-17T18:14:04.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5860000000000004e-06 2024-08-17T18:14:04.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.704 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.100178+0000", 2024-08-17T18:14:04.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0047875629999999999 2024-08-17T18:14:04.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.704 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:04.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.100189+0000", 2024-08-17T18:14:04.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1124999999999999e-05 2024-08-17T18:14:04.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.705 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.705 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.564942+0000", 2024-08-17T18:14:04.705 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.46475288100000001 2024-08-17T18:14:04.705 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.705 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.705 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:04.705 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.564952+0000", 2024-08-17T18:14:04.705 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9809999999999998e-06 2024-08-17T18:14:04.705 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.705 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.705 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.705 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.620659+0000", 2024-08-17T18:14:04.705 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.05570713 2024-08-17T18:14:04.706 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.706 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.706 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.706 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.620763+0000", 2024-08-17T18:14:04.706 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010389400000000001 2024-08-17T18:14:04.706 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.706 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.706 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.706 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.706 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.706 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115442 48.8 48:189c3bef:::benchmark_data_smithi130_327597_object115441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:14:04.706 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.095476+0000", 2024-08-17T18:14:04.706 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 307.58721224800001, 2024-08-17T18:14:04.706 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5581986209999998, 2024-08-17T18:14:04.707 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.707 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.707 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:14:04.707 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:04.707 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:04.707 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:04.707 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115442 2024-08-17T18:14:04.707 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.707 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.707 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.707 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.707 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095476+0000", 2024-08-17T18:14:04.707 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.708 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.708 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.708 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.708 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095476+0000", 2024-08-17T18:14:04.708 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.708 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.708 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.708 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.708 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095476+0000", 2024-08-17T18:14:04.708 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2300000000000002e-07 2024-08-17T18:14:04.708 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.708 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.708 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.708 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095499+0000", 2024-08-17T18:14:04.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2728e-05 2024-08-17T18:14:04.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.709 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.709 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.709 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095506+0000", 2024-08-17T18:14:04.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8770000000000004e-06 2024-08-17T18:14:04.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.709 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.709 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.709 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095511+0000", 2024-08-17T18:14:04.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0549999999999999e-06 2024-08-17T18:14:04.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.709 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.710 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.710 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.100585+0000", 2024-08-17T18:14:04.710 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0050741049999999998 2024-08-17T18:14:04.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.710 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:04.710 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.100593+0000", 2024-08-17T18:14:04.710 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5150000000000006e-06 2024-08-17T18:14:04.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.710 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.710 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.564972+0000", 2024-08-17T18:14:04.710 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.46437926800000001 2024-08-17T18:14:04.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:04.711 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.564979+0000", 2024-08-17T18:14:04.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1090000000000003e-06 2024-08-17T18:14:04.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.711 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.621508+0000", 2024-08-17T18:14:04.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0565289579999999 2024-08-17T18:14:04.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:14:04.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.621663+0000", 2024-08-17T18:14:04.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000155415 2024-08-17T18:14:04.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.712 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641744+0000", 2024-08-17T18:14:04.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020081096 2024-08-17T18:14:04.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.712 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T18:14:04.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641815+0000", 2024-08-17T18:14:04.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1219999999999999e-05 2024-08-17T18:14:04.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.713 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.713 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.653582+0000", 2024-08-17T18:14:04.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011766835 2024-08-17T18:14:04.713 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.713 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.713 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.653674+0000", 2024-08-17T18:14:04.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2016999999999997e-05 2024-08-17T18:14:04.713 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.713 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.713 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.713 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.714 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115459 48.8 48:15f41a0f:::benchmark_data_smithi130_327597_object115458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:14:04.714 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.111325+0000", 2024-08-17T18:14:04.714 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 307.57136313400002, 2024-08-17T18:14:04.714 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5377689189999999, 2024-08-17T18:14:04.714 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.714 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.714 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.714 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:04.714 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:04.714 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:04.714 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115459 2024-08-17T18:14:04.714 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.714 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.715 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.715 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.715 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111325+0000", 2024-08-17T18:14:04.715 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.715 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.715 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.715 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.715 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111325+0000", 2024-08-17T18:14:04.715 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.715 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.715 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.715 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.715 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111326+0000", 2024-08-17T18:14:04.716 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.311e-06 2024-08-17T18:14:04.716 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.716 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.716 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.716 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111351+0000", 2024-08-17T18:14:04.716 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5377000000000001e-05 2024-08-17T18:14:04.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.717 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111364+0000", 2024-08-17T18:14:04.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2632e-05 2024-08-17T18:14:04.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.717 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111370+0000", 2024-08-17T18:14:04.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.75e-06 2024-08-17T18:14:04.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.718 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.718 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111428+0000", 2024-08-17T18:14:04.718 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8425999999999999e-05 2024-08-17T18:14:04.718 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.718 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.718 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:04.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111440+0000", 2024-08-17T18:14:04.718 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1911e-05 2024-08-17T18:14:04.718 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.718 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.718 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565000+0000", 2024-08-17T18:14:04.719 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.45356039399999998 2024-08-17T18:14:04.719 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.719 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.719 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:04.719 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565007+0000", 2024-08-17T18:14:04.719 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.917e-06 2024-08-17T18:14:04.719 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.719 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.719 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.719 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.621760+0000", 2024-08-17T18:14:04.719 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.056752672 2024-08-17T18:14:04.719 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.719 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.720 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.720 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.621924+0000", 2024-08-17T18:14:04.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016439199999999999 2024-08-17T18:14:04.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.720 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.720 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:14:04.720 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.622547+0000", 2024-08-17T18:14:04.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000622424 2024-08-17T18:14:04.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.720 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.720 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.720 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641874+0000", 2024-08-17T18:14:04.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019326829 2024-08-17T18:14:04.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:04.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641892+0000", 2024-08-17T18:14:04.721 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8221000000000001e-05 2024-08-17T18:14:04.721 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.648915+0000", 2024-08-17T18:14:04.721 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.007023339 2024-08-17T18:14:04.721 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.648981+0000", 2024-08-17T18:14:04.722 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5957000000000006e-05 2024-08-17T18:14:04.722 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.722 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.722 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.722 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649094+0000", 2024-08-17T18:14:04.722 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000112367 2024-08-17T18:14:04.722 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.722 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.722 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.722 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.722 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.722 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115460 48.8 48:17144d5f:::benchmark_data_smithi130_327597_object115459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:14:04.722 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.111385+0000", 2024-08-17T18:14:04.723 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 307.57130264400001, 2024-08-17T18:14:04.723 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.537943938, 2024-08-17T18:14:04.723 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.723 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.723 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.723 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:04.723 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:04.723 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:04.723 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115460 2024-08-17T18:14:04.723 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.723 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.723 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.723 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.723 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111385+0000", 2024-08-17T18:14:04.724 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.724 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.724 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.724 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.724 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111385+0000", 2024-08-17T18:14:04.724 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.724 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.724 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.724 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.724 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111385+0000", 2024-08-17T18:14:04.724 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4999999999999998e-07 2024-08-17T18:14:04.724 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.724 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.724 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.725 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111401+0000", 2024-08-17T18:14:04.725 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5566e-05 2024-08-17T18:14:04.725 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.725 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.725 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.725 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111406+0000", 2024-08-17T18:14:04.725 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4610000000000001e-06 2024-08-17T18:14:04.725 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.725 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.725 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.725 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111409+0000", 2024-08-17T18:14:04.725 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.303e-06 2024-08-17T18:14:04.725 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.725 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.726 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111469+0000", 2024-08-17T18:14:04.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0007000000000003e-05 2024-08-17T18:14:04.726 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.726 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.726 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:04.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111476+0000", 2024-08-17T18:14:04.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3209999999999999e-06 2024-08-17T18:14:04.726 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.726 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.726 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565029+0000", 2024-08-17T18:14:04.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.45355303200000002 2024-08-17T18:14:04.727 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:04.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565037+0000", 2024-08-17T18:14:04.727 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.32e-06 2024-08-17T18:14:04.727 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.635425+0000", 2024-08-17T18:14:04.727 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0703878899999999 2024-08-17T18:14:04.727 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.635606+0000", 2024-08-17T18:14:04.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018029 2024-08-17T18:14:04.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.728 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:14:04.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.635807+0000", 2024-08-17T18:14:04.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020181999999999999 2024-08-17T18:14:04.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.728 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:04.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641899+0000", 2024-08-17T18:14:04.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0060914810000000002 2024-08-17T18:14:04.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.729 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.729 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.729 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641956+0000", 2024-08-17T18:14:04.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7343999999999998e-05 2024-08-17T18:14:04.729 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.729 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.729 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.729 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649206+0000", 2024-08-17T18:14:04.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072501060000000001 2024-08-17T18:14:04.729 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.729 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.729 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.729 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649267+0000", 2024-08-17T18:14:04.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0939e-05 2024-08-17T18:14:04.730 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.730 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.730 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.730 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649329+0000", 2024-08-17T18:14:04.730 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1707999999999999e-05 2024-08-17T18:14:04.730 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.730 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.730 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.730 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.730 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.730 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115465 48.8 48:103437e2:::benchmark_data_smithi130_327597_object115464:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:14:04.730 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.117778+0000", 2024-08-17T18:14:04.730 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 307.56490987900003, 2024-08-17T18:14:04.731 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5317481179999999, 2024-08-17T18:14:04.731 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.731 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.731 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.731 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:04.731 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:04.731 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:04.731 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115465 2024-08-17T18:14:04.731 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.731 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.731 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.731 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.731 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117778+0000", 2024-08-17T18:14:04.732 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.732 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.732 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.732 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.732 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117778+0000", 2024-08-17T18:14:04.732 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.732 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.732 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.732 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.732 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117779+0000", 2024-08-17T18:14:04.732 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.311e-06 2024-08-17T18:14:04.732 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.732 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117811+0000", 2024-08-17T18:14:04.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1489999999999998e-05 2024-08-17T18:14:04.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.733 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117826+0000", 2024-08-17T18:14:04.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5783000000000001e-05 2024-08-17T18:14:04.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.733 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117835+0000", 2024-08-17T18:14:04.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0709999999999995e-06 2024-08-17T18:14:04.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.734 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.734 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.734 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117889+0000", 2024-08-17T18:14:04.734 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4355999999999999e-05 2024-08-17T18:14:04.734 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.734 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.734 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:04.734 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117897+0000", 2024-08-17T18:14:04.734 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.729e-06 2024-08-17T18:14:04.734 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.734 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.734 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.734 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565061+0000", 2024-08-17T18:14:04.735 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.44716407800000002 2024-08-17T18:14:04.735 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.735 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.735 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:04.735 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565068+0000", 2024-08-17T18:14:04.735 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3130000000000001e-06 2024-08-17T18:14:04.735 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.735 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.735 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.736 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.642010+0000", 2024-08-17T18:14:04.736 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0769422019999999 2024-08-17T18:14:04.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.736 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.736 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.736 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.642197+0000", 2024-08-17T18:14:04.736 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000186923 2024-08-17T18:14:04.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.736 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.736 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:14:04.736 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.642394+0000", 2024-08-17T18:14:04.736 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000196913 2024-08-17T18:14:04.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.736 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.737 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:04.737 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649113+0000", 2024-08-17T18:14:04.737 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0067189290000000002 2024-08-17T18:14:04.737 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.737 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.737 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.737 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649158+0000", 2024-08-17T18:14:04.737 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4758000000000001e-05 2024-08-17T18:14:04.737 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.737 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.737 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.737 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649420+0000", 2024-08-17T18:14:04.737 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00026217599999999998 2024-08-17T18:14:04.737 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.738 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.738 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.738 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649452+0000", 2024-08-17T18:14:04.738 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2178000000000003e-05 2024-08-17T18:14:04.738 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.738 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.738 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.738 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649526+0000", 2024-08-17T18:14:04.738 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3907999999999998e-05 2024-08-17T18:14:04.738 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.738 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.739 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.739 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.739 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.739 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115470 48.8 48:1640b7fc:::benchmark_data_smithi130_327597_object115469:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:14:04.739 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.130397+0000", 2024-08-17T18:14:04.739 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 307.55229093200001, 2024-08-17T18:14:04.739 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.519320972, 2024-08-17T18:14:04.739 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.739 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.739 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.739 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:04.739 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:04.739 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:04.739 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115470 2024-08-17T18:14:04.740 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.740 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.740 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.740 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.740 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130397+0000", 2024-08-17T18:14:04.740 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.740 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.740 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.740 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.740 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130397+0000", 2024-08-17T18:14:04.740 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.740 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.740 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.740 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130399+0000", 2024-08-17T18:14:04.741 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0169999999999999e-06 2024-08-17T18:14:04.741 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.741 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.741 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130441+0000", 2024-08-17T18:14:04.741 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2552000000000002e-05 2024-08-17T18:14:04.741 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.741 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.741 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130470+0000", 2024-08-17T18:14:04.741 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8713000000000001e-05 2024-08-17T18:14:04.742 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.742 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.742 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.742 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130484+0000", 2024-08-17T18:14:04.742 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3709999999999999e-05 2024-08-17T18:14:04.742 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.742 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.742 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.742 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130612+0000", 2024-08-17T18:14:04.742 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000128598 2024-08-17T18:14:04.742 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.742 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.742 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:04.742 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130624+0000", 2024-08-17T18:14:04.743 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1878999999999999e-05 2024-08-17T18:14:04.743 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.743 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.743 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565091+0000", 2024-08-17T18:14:04.743 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.43446660199999998 2024-08-17T18:14:04.743 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.743 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:04.743 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565098+0000", 2024-08-17T18:14:04.743 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2250000000000003e-06 2024-08-17T18:14:04.743 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.744 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.645503+0000", 2024-08-17T18:14:04.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0804047400000001 2024-08-17T18:14:04.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.744 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.645638+0000", 2024-08-17T18:14:04.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000135418 2024-08-17T18:14:04.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.744 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:14:04.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.645861+0000", 2024-08-17T18:14:04.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00022232299999999999 2024-08-17T18:14:04.745 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.745 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:04.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649120+0000", 2024-08-17T18:14:04.745 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032596449999999998 2024-08-17T18:14:04.745 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.745 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649377+0000", 2024-08-17T18:14:04.746 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00025634200000000001 2024-08-17T18:14:04.746 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.746 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.746 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.746 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649566+0000", 2024-08-17T18:14:04.746 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018959399999999999 2024-08-17T18:14:04.747 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.747 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.747 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.747 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649604+0000", 2024-08-17T18:14:04.747 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8015000000000001e-05 2024-08-17T18:14:04.747 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.747 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.747 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.747 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649718+0000", 2024-08-17T18:14:04.748 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011359899999999999 2024-08-17T18:14:04.748 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.748 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.748 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.748 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.748 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.748 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:238973 48.8 48:1beb4fd9:::benchmark_data_smithi130_327597_object107008:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:14:04.748 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:32.588025+0000", 2024-08-17T18:14:04.748 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 152.09466318200001, 2024-08-17T18:14:04.748 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.51985719200000002, 2024-08-17T18:14:04.748 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.748 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.748 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.749 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:04.749 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:04.749 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:04.749 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 238973 2024-08-17T18:14:04.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.749 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.749 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.749 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588025+0000", 2024-08-17T18:14:04.749 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.749 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588025+0000", 2024-08-17T18:14:04.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.750 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.750 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588026+0000", 2024-08-17T18:14:04.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.457e-06 2024-08-17T18:14:04.750 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.750 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588032+0000", 2024-08-17T18:14:04.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5310000000000001e-06 2024-08-17T18:14:04.750 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588043+0000", 2024-08-17T18:14:04.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1503e-05 2024-08-17T18:14:04.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588050+0000", 2024-08-17T18:14:04.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0979999999999996e-06 2024-08-17T18:14:04.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588185+0000", 2024-08-17T18:14:04.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013512999999999999 2024-08-17T18:14:04.752 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.752 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.752 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.752 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588334+0000", 2024-08-17T18:14:04.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014820400000000001 2024-08-17T18:14:04.752 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.752 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.752 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2,3", 2024-08-17T18:14:04.752 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588476+0000", 2024-08-17T18:14:04.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014219800000000001 2024-08-17T18:14:04.752 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.752 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.752 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:04.752 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.593540+0000", 2024-08-17T18:14:04.753 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0050641920000000003 2024-08-17T18:14:04.753 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.753 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.753 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.753 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.601244+0000", 2024-08-17T18:14:04.753 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0077044410000000002 2024-08-17T18:14:04.753 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.753 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.753 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.753 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.601286+0000", 2024-08-17T18:14:04.753 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2089999999999999e-05 2024-08-17T18:14:04.753 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.753 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.753 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.754 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.107626+0000", 2024-08-17T18:14:04.754 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.50633972400000005 2024-08-17T18:14:04.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.754 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.754 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.107697+0000", 2024-08-17T18:14:04.754 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1056999999999994e-05 2024-08-17T18:14:04.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.754 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.754 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.107882+0000", 2024-08-17T18:14:04.754 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018456700000000001 2024-08-17T18:14:04.754 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.755 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.755 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.755 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.755 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.755 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:238978 48.8 48:18dd3d07:::benchmark_data_smithi130_327597_object107013:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:14:04.755 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:32.594272+0000", 2024-08-17T18:14:04.755 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 152.08841549799999, 2024-08-17T18:14:04.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.51382454700000002, 2024-08-17T18:14:04.755 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.755 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.755 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.755 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:04.755 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:04.755 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:04.756 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 238978 2024-08-17T18:14:04.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.756 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.756 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.756 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.594272+0000", 2024-08-17T18:14:04.756 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.756 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.756 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.594272+0000", 2024-08-17T18:14:04.756 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.594274+0000", 2024-08-17T18:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.705e-06 2024-08-17T18:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.594279+0000", 2024-08-17T18:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6750000000000001e-06 2024-08-17T18:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.594294+0000", 2024-08-17T18:14:04.757 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5429000000000001e-05 2024-08-17T18:14:04.758 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.758 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.758 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.758 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.594301+0000", 2024-08-17T18:14:04.758 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0890000000000002e-06 2024-08-17T18:14:04.758 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.758 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.758 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.758 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.594393+0000", 2024-08-17T18:14:04.758 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1648000000000001e-05 2024-08-17T18:14:04.758 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.758 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.758 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.594543+0000", 2024-08-17T18:14:04.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015047899999999999 2024-08-17T18:14:04.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.759 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2,3", 2024-08-17T18:14:04.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.594694+0000", 2024-08-17T18:14:04.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015072299999999999 2024-08-17T18:14:04.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.759 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:04.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.601301+0000", 2024-08-17T18:14:04.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0066069290000000001 2024-08-17T18:14:04.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.760 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.760 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.602644+0000", 2024-08-17T18:14:04.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0013434510000000001 2024-08-17T18:14:04.760 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.760 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.760 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.760 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.603244+0000", 2024-08-17T18:14:04.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00059939499999999996 2024-08-17T18:14:04.760 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.760 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.760 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.760 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.107929+0000", 2024-08-17T18:14:04.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.50468491299999996 2024-08-17T18:14:04.761 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.761 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.761 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.761 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.107980+0000", 2024-08-17T18:14:04.761 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1313000000000002e-05 2024-08-17T18:14:04.761 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.761 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.761 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.761 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.108097+0000", 2024-08-17T18:14:04.762 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000116798 2024-08-17T18:14:04.762 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.762 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.762 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.762 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.762 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.762 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:16100 49.3 49:ca7f6ebe:::benchmark_data_smithi130_364143_object16099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1385)", 2024-08-17T18:14:04.762 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:01.039335+0000", 2024-08-17T18:14:04.762 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 63.643352434000001, 2024-08-17T18:14:04.762 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.32985165799999999, 2024-08-17T18:14:04.762 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.762 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.763 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.763 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:04.763 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:14:04.763 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:14:04.763 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 16100 2024-08-17T18:14:04.763 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.763 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.763 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.763 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.763 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.039335+0000", 2024-08-17T18:14:04.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.763 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.763 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.763 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.764 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.039335+0000", 2024-08-17T18:14:04.764 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.764 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.764 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.764 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.764 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.039336+0000", 2024-08-17T18:14:04.764 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2399999999999998e-07 2024-08-17T18:14:04.764 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.764 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.764 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.764 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.039355+0000", 2024-08-17T18:14:04.764 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8879e-05 2024-08-17T18:14:04.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.039363+0000", 2024-08-17T18:14:04.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4619999999999996e-06 2024-08-17T18:14:04.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.039369+0000", 2024-08-17T18:14:04.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.66e-06 2024-08-17T18:14:04.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.766 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.040914+0000", 2024-08-17T18:14:04.766 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0015450640000000001 2024-08-17T18:14:04.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.766 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.766 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.766 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.041090+0000", 2024-08-17T18:14:04.766 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017615999999999999 2024-08-17T18:14:04.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.766 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.766 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T18:14:04.766 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.043161+0000", 2024-08-17T18:14:04.766 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002070821 2024-08-17T18:14:04.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.767 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:04.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.044208+0000", 2024-08-17T18:14:04.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0010466539999999999 2024-08-17T18:14:04.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.767 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.045234+0000", 2024-08-17T18:14:04.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001026587 2024-08-17T18:14:04.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.767 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.369021+0000", 2024-08-17T18:14:04.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.323787194 2024-08-17T18:14:04.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.768 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.768 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.768 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.369133+0000", 2024-08-17T18:14:04.768 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011192399999999999 2024-08-17T18:14:04.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.768 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.768 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.768 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.369187+0000", 2024-08-17T18:14:04.768 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3628999999999998e-05 2024-08-17T18:14:04.768 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.768 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.768 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.769 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.769 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:16109 49.3 49:c570ec3a:::benchmark_data_smithi130_364143_object16108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1385)", 2024-08-17T18:14:04.769 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:01.043098+0000", 2024-08-17T18:14:04.769 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 63.639589690999998, 2024-08-17T18:14:04.769 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.32624056400000001, 2024-08-17T18:14:04.769 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.769 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.769 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.769 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:04.769 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:14:04.769 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:14:04.769 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 16109 2024-08-17T18:14:04.770 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.770 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.770 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.770 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.770 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.043098+0000", 2024-08-17T18:14:04.770 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.770 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.770 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.770 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.770 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.043098+0000", 2024-08-17T18:14:04.770 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.770 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.770 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.043099+0000", 2024-08-17T18:14:04.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2049999999999999e-06 2024-08-17T18:14:04.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.043121+0000", 2024-08-17T18:14:04.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1617999999999999e-05 2024-08-17T18:14:04.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.043136+0000", 2024-08-17T18:14:04.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5155e-05 2024-08-17T18:14:04.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.772 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.772 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.043143+0000", 2024-08-17T18:14:04.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1760000000000004e-06 2024-08-17T18:14:04.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.772 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.772 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.044233+0000", 2024-08-17T18:14:04.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001090181 2024-08-17T18:14:04.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.772 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.044381+0000", 2024-08-17T18:14:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014733699999999999 2024-08-17T18:14:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T18:14:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.044513+0000", 2024-08-17T18:14:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013221700000000001 2024-08-17T18:14:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.046106+0000", 2024-08-17T18:14:04.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0015934479999999999 2024-08-17T18:14:04.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.774 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:04.774 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.046117+0000", 2024-08-17T18:14:04.774 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0959e-05 2024-08-17T18:14:04.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.774 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.774 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.369225+0000", 2024-08-17T18:14:04.774 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.323107592 2024-08-17T18:14:04.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.369304+0000", 2024-08-17T18:14:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8665999999999995e-05 2024-08-17T18:14:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:01.369339+0000", 2024-08-17T18:14:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5009999999999999e-05 2024-08-17T18:14:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:18627 49.7 49:e140e201:::benchmark_data_smithi130_364143_object18626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1385)", 2024-08-17T18:14:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:02.894733+0000", 2024-08-17T18:14:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 61.787954673000002, 2024-08-17T18:14:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.233556232, 2024-08-17T18:14:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:14:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:14:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 18627 2024-08-17T18:14:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.776 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.894733+0000", 2024-08-17T18:14:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.894733+0000", 2024-08-17T18:14:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.894735+0000", 2024-08-17T18:14:04.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.606e-06 2024-08-17T18:14:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.894755+0000", 2024-08-17T18:14:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0605999999999999e-05 2024-08-17T18:14:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.894766+0000", 2024-08-17T18:14:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1046999999999999e-05 2024-08-17T18:14:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.778 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.779 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.894773+0000", 2024-08-17T18:14:04.779 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4500000000000001e-06 2024-08-17T18:14:04.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.779 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.779 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.779 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.894879+0000", 2024-08-17T18:14:04.779 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000105899 2024-08-17T18:14:04.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.779 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.779 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.779 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.895013+0000", 2024-08-17T18:14:04.779 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000134688 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.895194+0000", 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018023799999999999 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.896272+0000", 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0010779520000000001 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.128118+0000", 2024-08-17T18:14:04.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.23184690499999999 2024-08-17T18:14:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.128187+0000", 2024-08-17T18:14:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8626000000000003e-05 2024-08-17T18:14:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.128245+0000", 2024-08-17T18:14:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7844999999999999e-05 2024-08-17T18:14:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.781 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.128289+0000", 2024-08-17T18:14:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4369999999999997e-05 2024-08-17T18:14:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36960.0:18619 49.c e1385/1380 49:3308e4a8:::benchmark_data_smithi130_364143_object18618:head v 1385'1202, mlcod=1385'1200)", 2024-08-17T18:14:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:02.895609+0000", 2024-08-17T18:14:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 61.787079112000001, 2024-08-17T18:14:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.23141420700000001, 2024-08-17T18:14:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.782 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.783 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.783 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.783 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.783 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.895609+0000", 2024-08-17T18:14:04.783 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.783 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.783 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.783 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.895609+0000", 2024-08-17T18:14:04.783 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.783 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.783 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.895609+0000", 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0900000000000001e-07 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.895629+0000", 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9941000000000001e-05 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.895634+0000", 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4240000000000001e-06 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.784 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.785 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.895638+0000", 2024-08-17T18:14:04.785 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7620000000000001e-06 2024-08-17T18:14:04.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.785 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.785 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.785 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.896947+0000", 2024-08-17T18:14:04.785 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0013094409999999999 2024-08-17T18:14:04.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.785 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.785 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.785 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.896968+0000", 2024-08-17T18:14:04.785 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0069e-05 2024-08-17T18:14:04.786 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.786 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.786 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.786 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.126984+0000", 2024-08-17T18:14:04.786 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.23001601799999999 2024-08-17T18:14:04.786 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.786 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.786 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.786 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.127023+0000", 2024-08-17T18:14:04.786 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9342999999999999e-05 2024-08-17T18:14:04.786 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.786 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.786 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.787 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.787 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.787 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36960.0:18621 49.c e1385/1380 49:3003e7a7:::benchmark_data_smithi130_364143_object18620:head v 1385'1203, mlcod=1385'1200)", 2024-08-17T18:14:04.787 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:02.896836+0000", 2024-08-17T18:14:04.787 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 61.785851841000003, 2024-08-17T18:14:04.787 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.23024609100000001, 2024-08-17T18:14:04.787 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.787 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.787 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.787 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.787 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.787 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.787 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.896836+0000", 2024-08-17T18:14:04.788 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.788 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.788 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.788 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.788 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.896836+0000", 2024-08-17T18:14:04.788 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.788 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.788 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.788 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.788 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.896837+0000", 2024-08-17T18:14:04.788 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6700000000000003e-07 2024-08-17T18:14:04.788 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.788 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.788 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.789 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.896857+0000", 2024-08-17T18:14:04.789 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0530000000000002e-05 2024-08-17T18:14:04.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.789 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.789 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.789 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.896866+0000", 2024-08-17T18:14:04.789 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3999999999999998e-06 2024-08-17T18:14:04.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.789 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.789 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.789 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.896873+0000", 2024-08-17T18:14:04.789 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0730000000000001e-06 2024-08-17T18:14:04.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.789 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.790 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.790 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898215+0000", 2024-08-17T18:14:04.790 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:04.791 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001342923 2024-08-17T18:14:04.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.791 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.791 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898233+0000", 2024-08-17T18:14:04.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7955999999999999e-05 2024-08-17T18:14:04.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.792 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.792 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.792 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.127060+0000", 2024-08-17T18:14:04.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22882659699999999 2024-08-17T18:14:04.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.792 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.792 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.792 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.127082+0000", 2024-08-17T18:14:04.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2045e-05 2024-08-17T18:14:04.792 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.792 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.792 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.793 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.793 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.793 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36960.0:18623 49.4 e1385/1380 49:2fb076ba:::benchmark_data_smithi130_364143_object18622:head v 1385'1127, mlcod=1385'1124)", 2024-08-17T18:14:04.793 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:02.897604+0000", 2024-08-17T18:14:04.793 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 61.785084001999998, 2024-08-17T18:14:04.793 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.229452563, 2024-08-17T18:14:04.793 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.793 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.793 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.793 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.793 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.793 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.793 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.897604+0000", 2024-08-17T18:14:04.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.794 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.794 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.794 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.897604+0000", 2024-08-17T18:14:04.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.794 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.794 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.794 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.897605+0000", 2024-08-17T18:14:04.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7000000000000004e-07 2024-08-17T18:14:04.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.794 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.795 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.795 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.897623+0000", 2024-08-17T18:14:04.795 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8278999999999999e-05 2024-08-17T18:14:04.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.795 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.795 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.795 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.897633+0000", 2024-08-17T18:14:04.795 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9059999999999997e-06 2024-08-17T18:14:04.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.795 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.795 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.795 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.897639+0000", 2024-08-17T18:14:04.795 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.054e-06 2024-08-17T18:14:04.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.796 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.796 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.897745+0000", 2024-08-17T18:14:04.796 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010587800000000001 2024-08-17T18:14:04.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.796 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.796 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.897762+0000", 2024-08-17T18:14:04.796 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7594000000000001e-05 2024-08-17T18:14:04.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.796 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.796 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.127029+0000", 2024-08-17T18:14:04.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.229266363 2024-08-17T18:14:04.797 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.797 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.797 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.797 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.127056+0000", 2024-08-17T18:14:04.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.7719e-05 2024-08-17T18:14:04.797 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.797 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.797 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.797 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.797 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.797 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36960.0:18632 49.a e1385/1380 49:5d333726:::benchmark_data_smithi130_364143_object18631:head v 1385'1176, mlcod=1385'1173)", 2024-08-17T18:14:04.797 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:02.898437+0000", 2024-08-17T18:14:04.798 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 61.784250878000002, 2024-08-17T18:14:04.798 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.228602414, 2024-08-17T18:14:04.798 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.798 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.798 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.798 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.798 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.798 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.798 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898437+0000", 2024-08-17T18:14:04.798 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.798 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.798 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.798 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898437+0000", 2024-08-17T18:14:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898438+0000", 2024-08-17T18:14:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5199999999999995e-07 2024-08-17T18:14:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898475+0000", 2024-08-17T18:14:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7144000000000002e-05 2024-08-17T18:14:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.799 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898486+0000", 2024-08-17T18:14:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1316e-05 2024-08-17T18:14:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898493+0000", 2024-08-17T18:14:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8140000000000003e-06 2024-08-17T18:14:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898546+0000", 2024-08-17T18:14:04.800 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2763000000000003e-05 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898565+0000", 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9284999999999999e-05 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.126999+0000", 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.228433682 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.127039+0000", 2024-08-17T18:14:04.801 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0558e-05 2024-08-17T18:14:04.802 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.802 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.802 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.802 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.802 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.802 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:18636 49.7 49:e0fdc833:::benchmark_data_smithi130_364143_object18635:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1385)", 2024-08-17T18:14:04.802 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:02.898510+0000", 2024-08-17T18:14:04.802 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 61.784177858, 2024-08-17T18:14:04.802 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.235438816, 2024-08-17T18:14:04.802 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.802 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.802 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.802 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:04.803 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:14:04.803 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:14:04.803 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 18636 2024-08-17T18:14:04.803 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.803 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.803 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.803 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.803 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898510+0000", 2024-08-17T18:14:04.803 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.803 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.803 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.803 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.803 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898510+0000", 2024-08-17T18:14:04.803 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.804 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.804 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.804 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.804 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898511+0000", 2024-08-17T18:14:04.804 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5680000000000001e-06 2024-08-17T18:14:04.804 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.804 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.804 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.804 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898531+0000", 2024-08-17T18:14:04.804 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9752e-05 2024-08-17T18:14:04.804 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.804 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.804 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.805 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898543+0000", 2024-08-17T18:14:04.805 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2156e-05 2024-08-17T18:14:04.805 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.805 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.805 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.805 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898551+0000", 2024-08-17T18:14:04.805 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3189999999999997e-06 2024-08-17T18:14:04.805 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.805 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.805 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.805 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898604+0000", 2024-08-17T18:14:04.805 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3631999999999997e-05 2024-08-17T18:14:04.805 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.805 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.806 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.806 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.127064+0000", 2024-08-17T18:14:04.806 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.228459418 2024-08-17T18:14:04.806 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.806 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.806 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:14:04.806 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.127174+0000", 2024-08-17T18:14:04.806 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0001105 2024-08-17T18:14:04.806 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.806 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.806 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:04.806 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.130674+0000", 2024-08-17T18:14:04.806 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0034996580000000001 2024-08-17T18:14:04.807 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.807 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.807 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.807 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.131083+0000", 2024-08-17T18:14:04.807 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00040871700000000001 2024-08-17T18:14:04.807 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.807 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.807 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:04.807 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.133793+0000", 2024-08-17T18:14:04.807 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027102839999999999 2024-08-17T18:14:04.807 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.807 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.807 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.808 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.133905+0000", 2024-08-17T18:14:04.808 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000112434 2024-08-17T18:14:04.808 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.808 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.808 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.808 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.133949+0000", 2024-08-17T18:14:04.808 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3377999999999997e-05 2024-08-17T18:14:04.808 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.808 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.808 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.808 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.808 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.808 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop_reply(client.36960.0:18627 49.7 e1385/1380 ondisk, result = 0)", 2024-08-17T18:14:04.808 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:02.898811+0000", 2024-08-17T18:14:04.809 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 61.783876868, 2024-08-17T18:14:04.809 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22931880099999999, 2024-08-17T18:14:04.809 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.809 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.809 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:14:04.809 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.809 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.809 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.809 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898811+0000", 2024-08-17T18:14:04.809 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.809 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.809 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.809 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.810 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898811+0000", 2024-08-17T18:14:04.810 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.810 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.810 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.810 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.810 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898812+0000", 2024-08-17T18:14:04.810 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7400000000000002e-07 2024-08-17T18:14:04.810 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.810 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.810 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.810 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.898816+0000", 2024-08-17T18:14:04.810 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7199999999999997e-06 2024-08-17T18:14:04.810 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.811 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.811 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.811 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.126965+0000", 2024-08-17T18:14:04.811 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22814884099999999 2024-08-17T18:14:04.811 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.811 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.811 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.811 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.126994+0000", 2024-08-17T18:14:04.811 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9224999999999999e-05 2024-08-17T18:14:04.811 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.811 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.811 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.811 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.128105+0000", 2024-08-17T18:14:04.811 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0011104089999999999 2024-08-17T18:14:04.812 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.812 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.812 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.812 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.128113+0000", 2024-08-17T18:14:04.812 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.678e-06 2024-08-17T18:14:04.812 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.812 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.812 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.812 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.128130+0000", 2024-08-17T18:14:04.812 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7153999999999999e-05 2024-08-17T18:14:04.812 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.812 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.812 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.813 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.813 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.813 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36960.0:18635 49.9 e1385/1380 49:9f1890fd:::benchmark_data_smithi130_364143_object18634:head v 1385'1150, mlcod=1385'1148)", 2024-08-17T18:14:04.813 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:13:02.899163+0000", 2024-08-17T18:14:04.813 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 61.783524427000003, 2024-08-17T18:14:04.813 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.22988726500000001, 2024-08-17T18:14:04.813 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:04.813 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:04.813 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:04.813 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:04.813 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.813 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:04.813 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.899163+0000", 2024-08-17T18:14:04.813 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.814 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.814 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.814 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:04.814 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.899163+0000", 2024-08-17T18:14:04.814 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:04.814 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.814 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.814 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:04.814 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.899164+0000", 2024-08-17T18:14:04.814 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.37e-07 2024-08-17T18:14:04.814 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.814 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.814 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:04.815 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:02.899195+0000", 2024-08-17T18:14:04.815 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1470000000000002e-05 2024-08-17T18:14:04.815 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.815 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.815 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:04.815 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.126965+0000", 2024-08-17T18:14:04.815 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.227769946 2024-08-17T18:14:04.815 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.815 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.815 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:04.815 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.126999+0000", 2024-08-17T18:14:04.815 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3441000000000001e-05 2024-08-17T18:14:04.815 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.816 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.816 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:04.816 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.127050+0000", 2024-08-17T18:14:04.816 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1591999999999997e-05 2024-08-17T18:14:04.816 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.816 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.816 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:04.816 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.127072+0000", 2024-08-17T18:14:04.816 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1746000000000002e-05 2024-08-17T18:14:04.816 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.816 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.816 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:04.816 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.129013+0000", 2024-08-17T18:14:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001940872 2024-08-17T18:14:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:13:03.129051+0000", 2024-08-17T18:14:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7660999999999997e-05 2024-08-17T18:14:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:04.817 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:04.817 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:14:04.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:04.819 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:04.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:04.919 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 70 16 90366 90350 80.6493 59.875 0.0167676 0.0123943 2024-08-17T18:14:04.920 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:04.958 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:04.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:04.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:05.060 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:05.081 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:05.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:05.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:05.182 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:05.205 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:05.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:05.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:05.307 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:05.330 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:05.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:05.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:05.432 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:05.464 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:05.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:05.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:05.566 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:05.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:05.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:05.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:05.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:05 smithi130 ceph-mon[105521]: pgmap v2574: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 62 MiB/s wr, 984 op/s 2024-08-17T18:14:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:05 smithi130 ceph-mon[110068]: pgmap v2574: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 62 MiB/s wr, 984 op/s 2024-08-17T18:14:05.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:05.712 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:05.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:05.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:05.814 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:05 smithi179 ceph-mon[93519]: pgmap v2574: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 62 MiB/s wr, 984 op/s 2024-08-17T18:14:05.859 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:05.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:05.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:05.919 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 71 16 91236 91220 80.2793 54.375 0.0154013 0.0124513 2024-08-17T18:14:05.962 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:05.998 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:05.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:05.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:06.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:14:06.100 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:06.125 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:06.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:06.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:06.227 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:06.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:06.249 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:06.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:06.350 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:06.373 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:06.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:06.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:06.475 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:06.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:06.499 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:06.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:06.600 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:06.629 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:06.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:06.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:06.731 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:06.753 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:06.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:06.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:06.855 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:06.877 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:06.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:06.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:06.919 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 72 16 92074 92058 79.8917 52.375 0.0182495 0.0125117 2024-08-17T18:14:06.978 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:07.010 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:07.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:07.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:07.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:07.135 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:07.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:07.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:07.236 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:07.258 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:07.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:07.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:07.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:07.398 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:07.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:07.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:07.500 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:07.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:07.525 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:07.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:07.626 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:07.647 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:07.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:07.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:07.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:07 smithi130 ceph-mon[105521]: pgmap v2575: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 988 op/s 2024-08-17T18:14:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:07 smithi130 ceph-mon[110068]: pgmap v2575: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 988 op/s 2024-08-17T18:14:07.749 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:07.772 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:07.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:07.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:07 smithi179 ceph-mon[93519]: pgmap v2575: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 62 MiB/s wr, 988 op/s 2024-08-17T18:14:07.873 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:07.895 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:07.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:07.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:07.919 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 73 16 92950 92934 79.5471 54.75 0.0373897 0.0125653 2024-08-17T18:14:07.997 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:08.018 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:08.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:08.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:08.121 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:08.157 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:08.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:08.257 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:08.278 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:08.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:08.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:08.380 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:08.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:08.419 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:08.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:08.520 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:08.541 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:08.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:08.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:08.642 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:08.665 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:08.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:08.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:08.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:08.808 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:08.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:08.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:08.910 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:08.919 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 74 16 93826 93810 79.2119 54.75 0.0255343 0.0126183 2024-08-17T18:14:08.931 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:08.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:08.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:09.033 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:09.055 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:09.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:09.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:09.157 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:09.181 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:09.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:09.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:09.283 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:09.305 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:09.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:09.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:09.406 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:09.428 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:09.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:09.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:09.530 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:09.571 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:09.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:09.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:09 smithi130 ceph-mon[105521]: pgmap v2576: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 61 MiB/s wr, 970 op/s 2024-08-17T18:14:09.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:09 smithi130 ceph-mon[110068]: pgmap v2576: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 61 MiB/s wr, 970 op/s 2024-08-17T18:14:09.673 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:09.712 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:09.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:09.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:09.755 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T18:14:09.814 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:09.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:09 smithi179 ceph-mon[93519]: pgmap v2576: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 61 MiB/s wr, 970 op/s 2024-08-17T18:14:09.877 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:09.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:09.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:09.919 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 75 16 94862 94846 79.0191 64.75 0.012236 0.01265 2024-08-17T18:14:09.979 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:10.002 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:10.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:10.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:10.104 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:10.150 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:10.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:10.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:10.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:14:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:14:09] "GET /metrics HTTP/1.1" 200 36781 "" "Prometheus/2.51.0" 2024-08-17T18:14:10.253 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:10.276 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:10.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:10.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:10.378 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:10.426 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:10.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:10.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:10.528 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:10.551 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:10.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:10.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:10.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:14:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:14:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:14:10.653 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:10.692 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:10.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:10.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:10.794 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:10.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:10.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:10.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:10.919 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 76 16 95885 95869 78.8206 63.9375 0.009601 0.0126815 2024-08-17T18:14:10.937 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:10.959 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:10.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:10.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:11.061 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:11.083 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:11.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:11.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:11.185 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:11.207 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:11.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:11.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:11.309 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:11.360 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:11.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:11.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:11.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:11.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:11.506 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:11.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:11.607 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:11.630 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:11.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:11.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:11.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:11 smithi130 ceph-mon[105521]: pgmap v2577: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 59 MiB/s wr, 951 op/s 2024-08-17T18:14:11.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:11 smithi130 ceph-mon[110068]: pgmap v2577: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 59 MiB/s wr, 951 op/s 2024-08-17T18:14:11.732 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:11.772 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:11.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:11.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:11.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:11 smithi179 ceph-mon[93519]: pgmap v2577: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 59 MiB/s wr, 951 op/s 2024-08-17T18:14:11.874 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:11.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:11.899 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:11.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:11.922 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 77 16 96917 96901 78.6346 64.5 0.0132376 0.0127123 2024-08-17T18:14:11.999 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:12.021 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:12.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:12.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:12.123 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:12.146 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:12.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:12.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:12.218 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:14:12.247 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:12.268 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:12.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:12.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:12.370 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:12.414 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:12.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:12.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:12.516 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:12.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:12.540 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:12.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:12.641 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:12.678 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:12.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:12.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:12.780 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:12.813 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:12.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:12.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:12.915 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:12.922 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 78 16 97758 97742 78.3003 52.5625 0.0225309 0.0127657 2024-08-17T18:14:12.937 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:12.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:12.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:13.038 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:13.060 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:13.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:13.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:13.162 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:13.206 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:13.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:13.308 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:13.329 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:13.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:13.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:13.431 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:13.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:13.453 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:13.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:13.554 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:13.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:13.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:13.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:13.579 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:13 smithi130 ceph-mon[105521]: pgmap v2578: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 59 MiB/s wr, 951 op/s 2024-08-17T18:14:13.579 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:13.579 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:14:13.580 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:13 smithi130 ceph-mon[110068]: pgmap v2578: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 59 MiB/s wr, 951 op/s 2024-08-17T18:14:13.580 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:13.580 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:14:13.677 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:13.716 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:13.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:13.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:13.818 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:13.840 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:13.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:13.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:13.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:13 smithi179 ceph-mon[93519]: pgmap v2578: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 59 MiB/s wr, 951 op/s 2024-08-17T18:14:13.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:13.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:14:13.920 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 79 16 98437 98421 77.8463 42.4375 0.0246168 0.0128399 2024-08-17T18:14:13.941 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:13.980 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:13.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:13.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:14.082 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:14.104 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:14.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:14.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:14.206 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:14.266 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:14.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:14.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:14.368 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:14.390 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:14.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:14.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:14.431 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:14:14.431 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:14:14.431 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:14:14.431 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:14:14.431 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:14:14.431 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:14:14.431 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:14:14.431 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:14:14.432 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:14:14.432 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:14:14.432 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:14:14.432 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:14:14.432 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:14:14.432 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:14:14.432 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:14:14.432 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:14:14.492 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:14.512 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:14.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:14.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:14.614 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:14.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:14.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:14.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:14.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:14 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2417092445' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:14:14.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:14 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2417092445' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:14:14.757 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:14.780 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:14.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:14.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:14.834 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:14:14.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:14 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2417092445' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:14:14.882 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:14.903 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:14.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:14.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:14.920 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:14:14.920493+0000 min lat: 0.00139804 max lat: 0.329999 avg lat: 0.012897 2024-08-17T18:14:14.920 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:14:14.920 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 80 16 99246 99230 77.5051 50.5625 0.0116827 0.012897 2024-08-17T18:14:15.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:15.026 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:15.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:15.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:15.128 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:15.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:15.179 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:15.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:15.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:15.302 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:15.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:15.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:15.404 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:15.447 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:15.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:15.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:15.482 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:15 smithi130 ceph-mon[105521]: pgmap v2579: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T18:14:15.482 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:15 smithi130 ceph-mon[110068]: pgmap v2579: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T18:14:15.550 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:15.572 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:15.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:15.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:15.673 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:15.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:15.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:15.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:15.823 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:15.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:15 smithi179 ceph-mon[93519]: pgmap v2579: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-08-17T18:14:15.859 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:15.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:15.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:15.920 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 81 16 100167 100151 77.2588 57.5625 0.016814 0.0129379 2024-08-17T18:14:15.961 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:15.982 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:15.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:15.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:16.084 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:16.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:16.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:16.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:16.209 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:16.232 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:16.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:16.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:16.333 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:16.356 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:16.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:16.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:16.376 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T18:14:16.458 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:16.479 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:16.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:16.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:16.581 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:16.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:16.606 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:16.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:16.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:16 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3950149960' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:14:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:16 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:14:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:16 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3950149960' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:14:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:16 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:14:16.707 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:16.749 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:16.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:16.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:16 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3950149960' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:14:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:16 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:14:16.851 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:16.895 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:16.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:16.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:16.920 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 82 16 101158 101142 77.0718 61.9375 0.0138384 0.0129695 2024-08-17T18:14:16.997 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:17.041 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:17.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:17.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:17.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:17.187 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:17.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:17.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:17.288 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:17.311 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:17.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:17.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:17.397 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:17 smithi130 ceph-mon[105521]: pgmap v2580: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 53 MiB/s wr, 855 op/s 2024-08-17T18:14:17.397 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:17 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:14:17.397 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:17 smithi130 ceph-mon[105521]: osdmap e1391: 8 total, 8 up, 4 in 2024-08-17T18:14:17.398 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:17 smithi130 ceph-mon[110068]: pgmap v2580: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 53 MiB/s wr, 855 op/s 2024-08-17T18:14:17.398 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:17 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:14:17.398 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:17 smithi130 ceph-mon[110068]: osdmap e1391: 8 total, 8 up, 4 in 2024-08-17T18:14:17.413 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:17.435 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:17.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:17.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:17.537 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:17.559 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:17.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:17.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:17.661 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:17.697 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:17.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:17.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:17.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:17.833 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:17.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:17.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:17.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:17 smithi179 ceph-mon[93519]: pgmap v2580: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 336 GiB / 358 GiB avail; 53 MiB/s wr, 855 op/s 2024-08-17T18:14:17.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:17 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:14:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:17 smithi179 ceph-mon[93519]: osdmap e1391: 8 total, 8 up, 4 in 2024-08-17T18:14:17.920 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 83 16 102219 102203 76.9421 66.3125 0.0139335 0.0129907 2024-08-17T18:14:17.935 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:17.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:17.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:17.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:18.059 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:18.098 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:18.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:18.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:18.200 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:18.236 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:18.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:18.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:18.338 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:18.376 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:18.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:18.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:18.478 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:18.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:18.524 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:18.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:18.624 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:18.662 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:18.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:18.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:18.764 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:18.787 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:18.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:18.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:18.888 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:18.910 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:18.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:18.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:18.920 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 84 16 103213 103197 76.7657 62.125 0.0141986 0.0130212 2024-08-17T18:14:19.013 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:19.066 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:19.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:19.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:19.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:19.204 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:19.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:19.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:19.307 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:19.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:19.355 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:19.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:19.380 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:14:19.456 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:19.479 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:19.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:19.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:19.534 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T18:14:19.535 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:14:19.580 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:19.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:19.659 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:19.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:19.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:19 smithi130 ceph-mon[105521]: pgmap v2582: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 57 MiB/s wr, 915 op/s 2024-08-17T18:14:19.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:19 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/914415009' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:14:19.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:19 smithi130 ceph-mon[110068]: pgmap v2582: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 57 MiB/s wr, 915 op/s 2024-08-17T18:14:19.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:19 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/914415009' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:14:19.760 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:19.782 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:19.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:19 smithi179 ceph-mon[93519]: pgmap v2582: 89 pgs: 89 active+clean; 6.1 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 57 MiB/s wr, 915 op/s 2024-08-17T18:14:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:19 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/914415009' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:14:19.884 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:19.922 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 85 16 104202 104186 76.5879 61.8125 0.00956447 0.0130518 2024-08-17T18:14:19.929 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:19.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:19.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:20.031 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:20.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:20.079 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:20.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:20.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:14:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:14:19] "GET /metrics HTTP/1.1" 200 36777 "" "Prometheus/2.51.0" 2024-08-17T18:14:20.180 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:20.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:20.219 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:20.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:20.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:20.375 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:20.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:20.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:20.477 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:20.499 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:20 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:14:20.499 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/914415009' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:14:20.499 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:20 smithi130 ceph-mon[105521]: osdmap e1392: 8 total, 8 up, 4 in 2024-08-17T18:14:20.499 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:20 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:14:20.499 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/914415009' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:14:20.500 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:20 smithi130 ceph-mon[110068]: osdmap e1392: 8 total, 8 up, 4 in 2024-08-17T18:14:20.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:20.530 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:20.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:20.576 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:14:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:20 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:14:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/914415009' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:14:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:20 smithi179 ceph-mon[93519]: osdmap e1392: 8 total, 8 up, 4 in 2024-08-17T18:14:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:14:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:14:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:14:20.631 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:20.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:20.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:20.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:20.822 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:20.836 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:14:20.836 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:14:20.844 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:20.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:20.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:20.925 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 86 16 105147 105131 76.383 59.0625 0.0162835 0.0130868 2024-08-17T18:14:20.946 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:20.969 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:20.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:20.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:21.071 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:21.093 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:21.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:21.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:21.194 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:21.251 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:21.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:21.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:21.353 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:21.376 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:21.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:21.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:21.478 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:21.516 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:21.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:21.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:21.617 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:21.636 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:21 smithi130 ceph-mon[105521]: pgmap v2584: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-17T18:14:21.638 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:21 smithi130 ceph-mon[110068]: pgmap v2584: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-17T18:14:21.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:21.663 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:21.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:21.680 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T18:14:21.735 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:14:21.765 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:21.787 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:21.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:21.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:21.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:21 smithi179 ceph-mon[93519]: pgmap v2584: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-08-17T18:14:21.889 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:21.926 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 87 16 106158 106142 76.2308 63.1875 0.0136873 0.0131131 2024-08-17T18:14:21.945 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:21.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:21.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:22.047 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:22.072 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:14:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:22.111 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:22.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:22.212 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:22.240 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:14:22.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:22.258 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:22.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:22.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:22.417 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:22.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:22.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:22.519 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:22.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:22.572 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:22.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:22.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:22 smithi130 ceph-mon[105521]: pgmap v2585: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 42 MiB/s wr, 677 op/s 2024-08-17T18:14:22.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:22 smithi130 ceph-mon[110068]: pgmap v2585: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 42 MiB/s wr, 677 op/s 2024-08-17T18:14:22.673 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:22.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:22.730 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:22.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:22.831 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:22.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:22 smithi179 ceph-mon[93519]: pgmap v2585: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 42 MiB/s wr, 677 op/s 2024-08-17T18:14:22.853 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:22.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:22.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:22.925 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 88 15 107089 107074 76.0265 58.25 0.0160782 0.0131487 2024-08-17T18:14:22.955 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:22.978 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:22.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:22.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:23.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:23.118 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:23.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:23.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:23.220 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:23.264 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:23.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:23.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:23.366 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:23.409 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:23.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:23.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:23.511 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:23.554 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:23.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:23.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:23.656 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:23.694 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:23.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:23.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:23.796 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:23.862 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:23.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:23.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:23.927 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 89 16 108131 108115 75.9013 65.0625 0.0172908 0.0131704 2024-08-17T18:14:23.964 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:23.987 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:23.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:23.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:24.088 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:24.151 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:24.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:24.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:24.253 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:24.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:24.295 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:24.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:24.396 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:24.418 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:24.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:24.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:24.520 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:24.541 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:24.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:24.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:24.643 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:24.666 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:24.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:24 smithi130 ceph-mon[110068]: pgmap v2586: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T18:14:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:24 smithi130 ceph-mon[105521]: pgmap v2586: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T18:14:24.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:24.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:24.768 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:24.815 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:24.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:24.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:24.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:24 smithi179 ceph-mon[93519]: pgmap v2586: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T18:14:24.917 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:24.927 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Total time run: 90.0136 2024-08-17T18:14:24.927 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Total writes made: 109131 2024-08-17T18:14:24.927 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Write size: 65536 2024-08-17T18:14:24.927 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Object size: 65536 2024-08-17T18:14:24.927 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Bandwidth (MB/sec): 75.774 2024-08-17T18:14:24.927 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev Bandwidth: 29.3025 2024-08-17T18:14:24.928 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max bandwidth (MB/sec): 202.688 2024-08-17T18:14:24.928 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min bandwidth (MB/sec): 42.4375 2024-08-17T18:14:24.928 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Average IOPS: 1212 2024-08-17T18:14:24.928 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev IOPS: 468.84 2024-08-17T18:14:24.928 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max IOPS: 3243 2024-08-17T18:14:24.928 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min IOPS: 679 2024-08-17T18:14:24.928 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Average Latency(s): 0.0131928 2024-08-17T18:14:24.928 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev Latency(s): 0.0120503 2024-08-17T18:14:24.928 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max latency(s): 0.329999 2024-08-17T18:14:24.928 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min latency(s): 0.00139804 2024-08-17T18:14:24.931 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T18:14:24.943 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:24.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:24.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:25.044 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:25.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:25.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:25.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:25.105 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:14:25.105 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:14:25.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:25.189 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:25.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:25.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:25.291 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:25.338 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:25.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:25.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:25.413 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:14:25.415 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:14:25.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:25.477 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:25.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:25.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:25.578 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:25.601 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:25.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:25.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:25.702 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:25.733 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:25.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:25.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:25.835 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:25.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:25 smithi179 ceph-mon[93519]: from='client.46946 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:14:25.872 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:25.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:25.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:25.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:25 smithi130 ceph-mon[105521]: from='client.46946 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:14:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:25 smithi130 ceph-mon[110068]: from='client.46946 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:14:25.974 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:26.008 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:26.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:26.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:26.110 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:26.144 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:26.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:26.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:26.245 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:26.268 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:26.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:26.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:26.370 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:26.393 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:26.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:26.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:26.495 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:26.517 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:26.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:26.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:26.619 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:26.655 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:26.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:26.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:26.757 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:26.779 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:26.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:26.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:26.881 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:26.903 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:26.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:26.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:27.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:27.028 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:27.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:27.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:27.130 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:27.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:27.167 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:27.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:27.268 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:27.290 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:27.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:27.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:27 smithi179 ceph-mon[93519]: pgmap v2587: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T18:14:27.392 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:27.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:27 smithi130 ceph-mon[105521]: pgmap v2587: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T18:14:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:27 smithi130 ceph-mon[110068]: pgmap v2587: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T18:14:27.434 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:27.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:27.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:27.536 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:27.557 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:27.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:27.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:27.660 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:27.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:27.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:27.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:27.794 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:27.815 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:27.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:27.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:27.917 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:27.939 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:27.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:27.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:28.041 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:28.082 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:28.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:28.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:28.184 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:28.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:28.207 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:28.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:28.308 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:28.329 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:28.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:28.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:28.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:14:28.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:14:28.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:14:28.431 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:28.466 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:28.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:28.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:28.567 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:28.605 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:28.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:28.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:28.707 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:28.730 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:28.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:28.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:28.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:28.867 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:28.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:28.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:28.968 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:28.991 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:28.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:28.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:29.093 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:29.114 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:29.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:29.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:29.216 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:29.237 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:29.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:29.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:29.339 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:29.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:29 smithi179 ceph-mon[93519]: pgmap v2588: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 102 B/s rd, 52 MiB/s wr, 976 op/s 2024-08-17T18:14:29.361 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:29.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:29.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:29.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:29 smithi130 ceph-mon[110068]: pgmap v2588: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 102 B/s rd, 52 MiB/s wr, 976 op/s 2024-08-17T18:14:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:29 smithi130 ceph-mon[105521]: pgmap v2588: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 102 B/s rd, 52 MiB/s wr, 976 op/s 2024-08-17T18:14:29.462 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:29.484 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:29.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:29.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:29.586 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:29.608 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:29.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:29.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:29.710 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:29.735 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:29.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:29.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:29.837 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:29.872 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:29.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:29.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:29.974 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:30.009 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:30.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:30.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:30.111 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:30.150 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:30.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:30.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:14:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:14:29] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-08-17T18:14:30.252 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:30.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:30.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:30.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:30.376 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:30.398 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:30.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:30.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:30.500 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:30.523 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:30.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:30.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:14:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:14:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:14:30.625 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:30.669 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:30.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:30.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:30.771 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:30.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:30.809 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:30.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:30.910 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:30.946 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:30.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:30.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:31.026 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2586,"stamp":"2024-08-17T18:14:23.518738+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6963009485,"num_objects":106264,"num_object_clones":0,"num_object_copies":318792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":106264,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":106516,"num_write_kb":6801679,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2962829,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":70645,"ondisk_log_size":70645,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":23721256,"kb_used_data":20247560,"kb_used_omap":294,"kb_used_meta":3473369,"kb_avail":351259352,"statfs":{"total":383980142592,"available":359689576448,"internally_reserved":0,"allocated":20733501440,"data_stored":20714719165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":301759,"internal_metadata":3556730177},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":557580288,"num_objects":8508,"num_object_clones":0,"num_object_copies":25524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8508,"num_write_kb":544512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001879"},"pg_stats":[{"pgid":"49.f","version":"1392'6679","reported_seq":6714,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:22.423814+0000","last_change":"2024-08-17T18:12:42.612957+0000","last_active":"2024-08-17T18:14:22.423814+0000","last_peered":"2024-08-17T18:14:22.423814+0000","last_clean":"2024-08-17T18:14:22.423814+0000","last_became_active":"2024-08-17T18:12:42.612660+0000","last_became_peered":"2024-08-17T18:12:42.612660+0000","last_unstale":"2024-08-17T18:14:22.423814+0000","last_undegraded":"2024-08-17T18:14:22.423814+0000","last_fullsized":"2024-08-17T18:14:22.423814+0000","mapping_epoch":1380,"log_start":"1385'2300","ondisk_log_start":"1385'2300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4379,"log_dups_size":0,"ondisk_log_size":4379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:26.876827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437714944,"num_objects":6679,"num_object_clones":0,"num_object_copies":20037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6679,"num_write_kb":427456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.e","version":"1392'6620","reported_seq":6655,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:22.435771+0000","last_change":"2024-08-17T18:12:42.612293+0000","last_active":"2024-08-17T18:14:22.435771+0000","last_peered":"2024-08-17T18:14:22.435771+0000","last_clean":"2024-08-17T18:14:22.435771+0000","last_became_active":"2024-08-17T18:12:42.611981+0000","last_became_peered":"2024-08-17T18:12:42.611981+0000","last_unstale":"2024-08-17T18:14:22.435771+0000","last_undegraded":"2024-08-17T18:14:22.435771+0000","last_fullsized":"2024-08-17T18:14:22.435771+0000","mapping_epoch":1380,"log_start":"1385'2300","ondisk_log_start":"1385'2300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4320,"log_dups_size":0,"ondisk_log_size":4320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:54:16.727653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":433848320,"num_objects":6620,"num_object_clones":0,"num_object_copies":19860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6620,"num_write_kb":423680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.d","version":"1392'6720","reported_seq":6755,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:22.359707+0000","last_change":"2024-08-17T18:12:42.192342+0000","last_active":"2024-08-17T18:14:22.359707+0000","last_peered":"2024-08-17T18:14:22.359707+0000","last_clean":"2024-08-17T18:14:22.359707+0000","last_became_active":"2024-08-17T18:12:42.191931+0000","last_became_peered":"2024-08-17T18:12:42.191931+0000","last_unstale":"2024-08-17T18:14:22.359707+0000","last_undegraded":"2024-08-17T18:14:22.359707+0000","last_fullsized":"2024-08-17T18:14:22.359707+0000","mapping_epoch":1380,"log_start":"1385'2400","ondisk_log_start":"1385'2400","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4320,"log_dups_size":0,"ondisk_log_size":4320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:03.046092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":440401920,"num_objects":6720,"num_object_clones":0,"num_object_copies":20160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6720,"num_write_kb":430080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.c","version":"1392'6628","reported_seq":6663,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:22.401178+0000","last_change":"2024-08-17T18:12:42.613558+0000","last_active":"2024-08-17T18:14:22.401178+0000","last_peered":"2024-08-17T18:14:22.401178+0000","last_clean":"2024-08-17T18:14:22.401178+0000","last_became_active":"2024-08-17T18:12:42.613083+0000","last_became_peered":"2024-08-17T18:12:42.613083+0000","last_unstale":"2024-08-17T18:14:22.401178+0000","last_undegraded":"2024-08-17T18:14:22.401178+0000","last_fullsized":"2024-08-17T18:14:22.401178+0000","mapping_epoch":1380,"log_start":"1385'2300","ondisk_log_start":"1385'2300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4328,"log_dups_size":0,"ondisk_log_size":4328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:54.163537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":434372608,"num_objects":6628,"num_object_clones":0,"num_object_copies":19884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6628,"num_write_kb":424192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.b","version":"1392'6561","reported_seq":6596,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:22.402781+0000","last_change":"2024-08-17T18:12:42.616433+0000","last_active":"2024-08-17T18:14:22.402781+0000","last_peered":"2024-08-17T18:14:22.402781+0000","last_clean":"2024-08-17T18:14:22.402781+0000","last_became_active":"2024-08-17T18:12:42.615691+0000","last_became_peered":"2024-08-17T18:12:42.615691+0000","last_unstale":"2024-08-17T18:14:22.402781+0000","last_undegraded":"2024-08-17T18:14:22.402781+0000","last_fullsized":"2024-08-17T18:14:22.402781+0000","mapping_epoch":1380,"log_start":"1385'2200","ondisk_log_start":"1385'2200","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4361,"log_dups_size":0,"ondisk_log_size":4361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:56.333635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429981696,"num_objects":6561,"num_object_clones":0,"num_object_copies":19683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6561,"num_write_kb":419904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.a","version":"1392'6902","reported_seq":6920,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:23.186709+0000","last_change":"2024-08-17T18:12:42.193306+0000","last_active":"2024-08-17T18:14:23.186709+0000","last_peered":"2024-08-17T18:14:23.186709+0000","last_clean":"2024-08-17T18:14:23.186709+0000","last_became_active":"2024-08-17T18:12:42.192965+0000","last_became_peered":"2024-08-17T18:12:42.192965+0000","last_unstale":"2024-08-17T18:14:23.186709+0000","last_undegraded":"2024-08-17T18:14:23.186709+0000","last_fullsized":"2024-08-17T18:14:23.186709+0000","mapping_epoch":1380,"log_start":"1385'2700","ondisk_log_start":"1385'2700","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4202,"log_dups_size":0,"ondisk_log_size":4202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:44:20.694736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":452329472,"num_objects":6902,"num_object_clones":0,"num_object_copies":20706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6902,"num_write_kb":441728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"49.9","version":"1392'6665","reported_seq":6685,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:23.152281+0000","last_change":"2024-08-17T18:12:42.612232+0000","last_active":"2024-08-17T18:14:23.152281+0000","last_peered":"2024-08-17T18:14:23.152281+0000","last_clean":"2024-08-17T18:14:23.152281+0000","last_became_active":"2024-08-17T18:12:42.611975+0000","last_became_peered":"2024-08-17T18:12:42.611975+0000","last_unstale":"2024-08-17T18:14:23.152281+0000","last_undegraded":"2024-08-17T18:14:23.152281+0000","last_fullsized":"2024-08-17T18:14:23.152281+0000","mapping_epoch":1380,"log_start":"1385'2500","ondisk_log_start":"1385'2500","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4165,"log_dups_size":0,"ondisk_log_size":4165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:26.661545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436797440,"num_objects":6665,"num_object_clones":0,"num_object_copies":19995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6665,"num_write_kb":426560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"49.8","version":"1392'6662","reported_seq":6682,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:23.168050+0000","last_change":"2024-08-17T18:12:42.613466+0000","last_active":"2024-08-17T18:14:23.168050+0000","last_peered":"2024-08-17T18:14:23.168050+0000","last_clean":"2024-08-17T18:14:23.168050+0000","last_became_active":"2024-08-17T18:12:42.613267+0000","last_became_peered":"2024-08-17T18:12:42.613267+0000","last_unstale":"2024-08-17T18:14:23.168050+0000","last_undegraded":"2024-08-17T18:14:23.168050+0000","last_fullsized":"2024-08-17T18:14:23.168050+0000","mapping_epoch":1380,"log_start":"1385'2500","ondisk_log_start":"1385'2500","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4162,"log_dups_size":0,"ondisk_log_size":4162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:55.951743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436600832,"num_objects":6662,"num_object_clones":0,"num_object_copies":19986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6662,"num_write_kb":426368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"49.7","version":"1391'6514","reported_seq":6547,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.270197+0000","last_change":"2024-08-17T18:12:42.616617+0000","last_active":"2024-08-17T18:14:19.270197+0000","last_peered":"2024-08-17T18:14:19.270197+0000","last_clean":"2024-08-17T18:14:19.270197+0000","last_became_active":"2024-08-17T18:12:42.615862+0000","last_became_peered":"2024-08-17T18:12:42.615862+0000","last_unstale":"2024-08-17T18:14:19.270197+0000","last_undegraded":"2024-08-17T18:14:19.270197+0000","last_fullsized":"2024-08-17T18:14:19.270197+0000","mapping_epoch":1380,"log_start":"1385'1600","ondisk_log_start":"1385'1600","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4914,"log_dups_size":0,"ondisk_log_size":4914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:13.095768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426901504,"num_objects":6514,"num_object_clones":0,"num_object_copies":19542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6514,"num_write_kb":416896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"49.6","version":"1392'6644","reported_seq":6679,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:22.443826+0000","last_change":"2024-08-17T18:12:42.616079+0000","last_active":"2024-08-17T18:14:22.443826+0000","last_peered":"2024-08-17T18:14:22.443826+0000","last_clean":"2024-08-17T18:14:22.443826+0000","last_became_active":"2024-08-17T18:12:42.615724+0000","last_became_peered":"2024-08-17T18:12:42.615724+0000","last_unstale":"2024-08-17T18:14:22.443826+0000","last_undegraded":"2024-08-17T18:14:22.443826+0000","last_fullsized":"2024-08-17T18:14:22.443826+0000","mapping_epoch":1380,"log_start":"1385'2300","ondisk_log_start":"1385'2300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4344,"log_dups_size":0,"ondisk_log_size":4344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:29.102069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435421184,"num_objects":6644,"num_object_clones":0,"num_object_copies":19932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6644,"num_write_kb":425216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.5","version":"1392'6673","reported_seq":6693,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:23.137037+0000","last_change":"2024-08-17T18:12:42.614361+0000","last_active":"2024-08-17T18:14:23.137037+0000","last_peered":"2024-08-17T18:14:23.137037+0000","last_clean":"2024-08-17T18:14:23.137037+0000","last_became_active":"2024-08-17T18:12:42.614132+0000","last_became_peered":"2024-08-17T18:12:42.614132+0000","last_unstale":"2024-08-17T18:14:23.137037+0000","last_undegraded":"2024-08-17T18:14:23.137037+0000","last_fullsized":"2024-08-17T18:14:23.137037+0000","mapping_epoch":1380,"log_start":"1385'2500","ondisk_log_start":"1385'2500","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4173,"log_dups_size":0,"ondisk_log_size":4173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:47:41.211349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437321728,"num_objects":6673,"num_object_clones":0,"num_object_copies":20019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6673,"num_write_kb":427072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"49.4","version":"1392'6677","reported_seq":6712,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:22.418902+0000","last_change":"2024-08-17T18:12:42.613238+0000","last_active":"2024-08-17T18:14:22.418902+0000","last_peered":"2024-08-17T18:14:22.418902+0000","last_clean":"2024-08-17T18:14:22.418902+0000","last_became_active":"2024-08-17T18:12:42.612693+0000","last_became_peered":"2024-08-17T18:12:42.612693+0000","last_unstale":"2024-08-17T18:14:22.418902+0000","last_undegraded":"2024-08-17T18:14:22.418902+0000","last_fullsized":"2024-08-17T18:14:22.418902+0000","mapping_epoch":1380,"log_start":"1385'2300","ondisk_log_start":"1385'2300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4377,"log_dups_size":0,"ondisk_log_size":4377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:03.449538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437583872,"num_objects":6677,"num_object_clones":0,"num_object_copies":20031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6677,"num_write_kb":427328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.3","version":"1391'6504","reported_seq":6536,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.266194+0000","last_change":"2024-08-17T18:12:42.617312+0000","last_active":"2024-08-17T18:14:19.266194+0000","last_peered":"2024-08-17T18:14:19.266194+0000","last_clean":"2024-08-17T18:14:19.266194+0000","last_became_active":"2024-08-17T18:12:42.616565+0000","last_became_peered":"2024-08-17T18:12:42.616565+0000","last_unstale":"2024-08-17T18:14:19.266194+0000","last_undegraded":"2024-08-17T18:14:19.266194+0000","last_fullsized":"2024-08-17T18:14:19.266194+0000","mapping_epoch":1380,"log_start":"1385'1600","ondisk_log_start":"1385'1600","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4904,"log_dups_size":0,"ondisk_log_size":4904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:24.331362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426246144,"num_objects":6504,"num_object_clones":0,"num_object_copies":19512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6504,"num_write_kb":416256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"49.2","version":"1392'6640","reported_seq":6675,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:22.411100+0000","last_change":"2024-08-17T18:12:42.613766+0000","last_active":"2024-08-17T18:14:22.411100+0000","last_peered":"2024-08-17T18:14:22.411100+0000","last_clean":"2024-08-17T18:14:22.411100+0000","last_became_active":"2024-08-17T18:12:42.613426+0000","last_became_peered":"2024-08-17T18:12:42.613426+0000","last_unstale":"2024-08-17T18:14:22.411100+0000","last_undegraded":"2024-08-17T18:14:22.411100+0000","last_fullsized":"2024-08-17T18:14:22.411100+0000","mapping_epoch":1380,"log_start":"1385'2300","ondisk_log_start":"1385'2300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4340,"log_dups_size":0,"ondisk_log_size":4340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:49.367275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435159040,"num_objects":6640,"num_object_clones":0,"num_object_copies":19920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6640,"num_write_kb":424960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.1","version":"1391'6551","reported_seq":6584,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.257728+0000","last_change":"2024-08-17T18:12:42.191991+0000","last_active":"2024-08-17T18:14:19.257728+0000","last_peered":"2024-08-17T18:14:19.257728+0000","last_clean":"2024-08-17T18:14:19.257728+0000","last_became_active":"2024-08-17T18:12:42.191700+0000","last_became_peered":"2024-08-17T18:12:42.191700+0000","last_unstale":"2024-08-17T18:14:19.257728+0000","last_undegraded":"2024-08-17T18:14:19.257728+0000","last_fullsized":"2024-08-17T18:14:19.257728+0000","mapping_epoch":1380,"log_start":"1385'1700","ondisk_log_start":"1385'1700","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4851,"log_dups_size":0,"ondisk_log_size":4851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:58:11.841245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429326336,"num_objects":6551,"num_object_clones":0,"num_object_copies":19653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6551,"num_write_kb":419264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"49.0","version":"1392'6598","reported_seq":6633,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:22.446556+0000","last_change":"2024-08-17T18:12:42.612356+0000","last_active":"2024-08-17T18:14:22.446556+0000","last_peered":"2024-08-17T18:14:22.446556+0000","last_clean":"2024-08-17T18:14:22.446556+0000","last_became_active":"2024-08-17T18:12:42.612054+0000","last_became_peered":"2024-08-17T18:12:42.612054+0000","last_unstale":"2024-08-17T18:14:22.446556+0000","last_undegraded":"2024-08-17T18:14:22.446556+0000","last_fullsized":"2024-08-17T18:14:22.446556+0000","mapping_epoch":1380,"log_start":"1385'2300","ondisk_log_start":"1385'2300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4298,"log_dups_size":0,"ondisk_log_size":4298,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:49.825180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432406528,"num_objects":6598,"num_object_clones":0,"num_object_copies":19794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6598,"num_write_kb":422272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":2272,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.379738+0000","last_change":"2024-08-17T18:09:28.870033+0000","last_active":"2024-08-17T18:14:16.379738+0000","last_peered":"2024-08-17T18:14:16.379738+0000","last_clean":"2024-08-17T18:14:16.379738+0000","last_became_active":"2024-08-17T18:09:28.869346+0000","last_became_peered":"2024-08-17T18:09:28.869346+0000","last_unstale":"2024-08-17T18:14:16.379738+0000","last_undegraded":"2024-08-17T18:14:16.379738+0000","last_fullsized":"2024-08-17T18:14:16.379738+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:23.168631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":569,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402522+0000","last_change":"2024-08-17T18:09:29.841584+0000","last_active":"2024-08-17T18:14:19.402522+0000","last_peered":"2024-08-17T18:14:19.402522+0000","last_clean":"2024-08-17T18:14:19.402522+0000","last_became_active":"2024-08-17T18:09:29.840950+0000","last_became_peered":"2024-08-17T18:09:29.840950+0000","last_unstale":"2024-08-17T18:14:19.402522+0000","last_undegraded":"2024-08-17T18:14:19.402522+0000","last_fullsized":"2024-08-17T18:14:19.402522+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:43.035959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":2064,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.403138+0000","last_change":"2024-08-17T18:09:29.883587+0000","last_active":"2024-08-17T18:14:19.403138+0000","last_peered":"2024-08-17T18:14:19.403138+0000","last_clean":"2024-08-17T18:14:19.403138+0000","last_became_active":"2024-08-17T18:09:29.882222+0000","last_became_peered":"2024-08-17T18:09:29.882222+0000","last_unstale":"2024-08-17T18:14:19.403138+0000","last_undegraded":"2024-08-17T18:14:19.403138+0000","last_fullsized":"2024-08-17T18:14:19.403138+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:10.777993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2121,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402976+0000","last_change":"2024-08-17T18:09:28.858824+0000","last_active":"2024-08-17T18:14:19.402976+0000","last_peered":"2024-08-17T18:14:19.402976+0000","last_clean":"2024-08-17T18:14:19.402976+0000","last_became_active":"2024-08-17T18:09:28.857856+0000","last_became_peered":"2024-08-17T18:09:28.857856+0000","last_unstale":"2024-08-17T18:14:19.402976+0000","last_undegraded":"2024-08-17T18:14:19.402976+0000","last_fullsized":"2024-08-17T18:14:19.402976+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:41:35.146166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":124,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402403+0000","last_change":"2024-08-17T18:09:29.851515+0000","last_active":"2024-08-17T18:14:19.402403+0000","last_peered":"2024-08-17T18:14:19.402403+0000","last_clean":"2024-08-17T18:14:19.402403+0000","last_became_active":"2024-08-17T18:09:29.851109+0000","last_became_peered":"2024-08-17T18:09:29.851109+0000","last_unstale":"2024-08-17T18:14:19.402403+0000","last_undegraded":"2024-08-17T18:14:19.402403+0000","last_fullsized":"2024-08-17T18:14:19.402403+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:42.054532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":104,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.376207+0000","last_change":"2024-08-17T18:03:16.089811+0000","last_active":"2024-08-17T18:14:16.376207+0000","last_peered":"2024-08-17T18:14:16.376207+0000","last_clean":"2024-08-17T18:14:16.376207+0000","last_became_active":"2024-08-17T18:03:16.089571+0000","last_became_peered":"2024-08-17T18:03:16.089571+0000","last_unstale":"2024-08-17T18:14:16.376207+0000","last_undegraded":"2024-08-17T18:14:16.376207+0000","last_fullsized":"2024-08-17T18:14:16.376207+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:04.630361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":86,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374162+0000","last_change":"2024-08-17T18:08:57.577055+0000","last_active":"2024-08-17T18:14:16.374162+0000","last_peered":"2024-08-17T18:14:16.374162+0000","last_clean":"2024-08-17T18:14:16.374162+0000","last_became_active":"2024-08-17T18:08:57.573659+0000","last_became_peered":"2024-08-17T18:08:57.573659+0000","last_unstale":"2024-08-17T18:14:16.374162+0000","last_undegraded":"2024-08-17T18:14:16.374162+0000","last_fullsized":"2024-08-17T18:14:16.374162+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:01.686752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":908,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402139+0000","last_change":"2024-08-17T18:09:29.852393+0000","last_active":"2024-08-17T18:14:19.402139+0000","last_peered":"2024-08-17T18:14:19.402139+0000","last_clean":"2024-08-17T18:14:19.402139+0000","last_became_active":"2024-08-17T18:09:29.851947+0000","last_became_peered":"2024-08-17T18:09:29.851947+0000","last_unstale":"2024-08-17T18:14:19.402139+0000","last_undegraded":"2024-08-17T18:14:19.402139+0000","last_fullsized":"2024-08-17T18:14:19.402139+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:49.145327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":2217,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.388000+0000","last_change":"2024-08-17T18:09:29.863241+0000","last_active":"2024-08-17T18:14:16.388000+0000","last_peered":"2024-08-17T18:14:16.388000+0000","last_clean":"2024-08-17T18:14:16.388000+0000","last_became_active":"2024-08-17T18:09:29.862369+0000","last_became_peered":"2024-08-17T18:09:29.862369+0000","last_unstale":"2024-08-17T18:14:16.388000+0000","last_undegraded":"2024-08-17T18:14:16.388000+0000","last_fullsized":"2024-08-17T18:14:16.388000+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:30.518551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":655,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.411166+0000","last_change":"2024-08-17T18:09:29.896233+0000","last_active":"2024-08-17T18:14:19.411166+0000","last_peered":"2024-08-17T18:14:19.411166+0000","last_clean":"2024-08-17T18:14:19.411166+0000","last_became_active":"2024-08-17T18:09:29.895718+0000","last_became_peered":"2024-08-17T18:09:29.895718+0000","last_unstale":"2024-08-17T18:14:19.411166+0000","last_undegraded":"2024-08-17T18:14:19.411166+0000","last_fullsized":"2024-08-17T18:14:19.411166+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:03.278878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":127,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.379018+0000","last_change":"2024-08-17T18:08:57.574015+0000","last_active":"2024-08-17T18:14:16.379018+0000","last_peered":"2024-08-17T18:14:16.379018+0000","last_clean":"2024-08-17T18:14:16.379018+0000","last_became_active":"2024-08-17T18:08:57.567973+0000","last_became_peered":"2024-08-17T18:08:57.567973+0000","last_unstale":"2024-08-17T18:14:16.379018+0000","last_undegraded":"2024-08-17T18:14:16.379018+0000","last_fullsized":"2024-08-17T18:14:16.379018+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:06.251764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2075,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402475+0000","last_change":"2024-08-17T18:09:29.917203+0000","last_active":"2024-08-17T18:14:19.402475+0000","last_peered":"2024-08-17T18:14:19.402475+0000","last_clean":"2024-08-17T18:14:19.402475+0000","last_became_active":"2024-08-17T18:09:29.916844+0000","last_became_peered":"2024-08-17T18:09:29.916844+0000","last_unstale":"2024-08-17T18:14:19.402475+0000","last_undegraded":"2024-08-17T18:14:19.402475+0000","last_fullsized":"2024-08-17T18:14:19.402475+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.453720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1982,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402810+0000","last_change":"2024-08-17T18:09:28.859255+0000","last_active":"2024-08-17T18:14:19.402810+0000","last_peered":"2024-08-17T18:14:19.402810+0000","last_clean":"2024-08-17T18:14:19.402810+0000","last_became_active":"2024-08-17T18:09:28.858537+0000","last_became_peered":"2024-08-17T18:09:28.858537+0000","last_unstale":"2024-08-17T18:14:19.402810+0000","last_undegraded":"2024-08-17T18:14:19.402810+0000","last_fullsized":"2024-08-17T18:14:19.402810+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:15.364687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2240,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.379018+0000","last_change":"2024-08-17T18:09:28.868303+0000","last_active":"2024-08-17T18:14:16.379018+0000","last_peered":"2024-08-17T18:14:16.379018+0000","last_clean":"2024-08-17T18:14:16.379018+0000","last_became_active":"2024-08-17T18:09:28.867989+0000","last_became_peered":"2024-08-17T18:09:28.867989+0000","last_unstale":"2024-08-17T18:14:16.379018+0000","last_undegraded":"2024-08-17T18:14:16.379018+0000","last_fullsized":"2024-08-17T18:14:16.379018+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:58:52.741685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":108,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402778+0000","last_change":"2024-08-17T18:09:29.842129+0000","last_active":"2024-08-17T18:14:19.402778+0000","last_peered":"2024-08-17T18:14:19.402778+0000","last_clean":"2024-08-17T18:14:19.402778+0000","last_became_active":"2024-08-17T18:09:29.841848+0000","last_became_peered":"2024-08-17T18:09:29.841848+0000","last_unstale":"2024-08-17T18:14:19.402778+0000","last_undegraded":"2024-08-17T18:14:19.402778+0000","last_fullsized":"2024-08-17T18:14:19.402778+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:02.119134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1354,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.379428+0000","last_change":"2024-08-17T18:09:29.846509+0000","last_active":"2024-08-17T18:14:16.379428+0000","last_peered":"2024-08-17T18:14:16.379428+0000","last_clean":"2024-08-17T18:14:16.379428+0000","last_became_active":"2024-08-17T18:09:29.844097+0000","last_became_peered":"2024-08-17T18:09:29.844097+0000","last_unstale":"2024-08-17T18:14:16.379428+0000","last_undegraded":"2024-08-17T18:14:16.379428+0000","last_fullsized":"2024-08-17T18:14:16.379428+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:32.340354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2070,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402334+0000","last_change":"2024-08-17T18:09:29.841335+0000","last_active":"2024-08-17T18:14:19.402334+0000","last_peered":"2024-08-17T18:14:19.402334+0000","last_clean":"2024-08-17T18:14:19.402334+0000","last_became_active":"2024-08-17T18:09:29.840522+0000","last_became_peered":"2024-08-17T18:09:29.840522+0000","last_unstale":"2024-08-17T18:14:19.402334+0000","last_undegraded":"2024-08-17T18:14:19.402334+0000","last_fullsized":"2024-08-17T18:14:19.402334+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:08.015836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2245,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.379845+0000","last_change":"2024-08-17T18:09:29.847494+0000","last_active":"2024-08-17T18:14:16.379845+0000","last_peered":"2024-08-17T18:14:16.379845+0000","last_clean":"2024-08-17T18:14:16.379845+0000","last_became_active":"2024-08-17T18:09:29.846973+0000","last_became_peered":"2024-08-17T18:09:29.846973+0000","last_unstale":"2024-08-17T18:14:16.379845+0000","last_undegraded":"2024-08-17T18:14:16.379845+0000","last_fullsized":"2024-08-17T18:14:16.379845+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:49.469921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2029,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402610+0000","last_change":"2024-08-17T18:09:28.855045+0000","last_active":"2024-08-17T18:14:19.402610+0000","last_peered":"2024-08-17T18:14:19.402610+0000","last_clean":"2024-08-17T18:14:19.402610+0000","last_became_active":"2024-08-17T18:09:28.854607+0000","last_became_peered":"2024-08-17T18:09:28.854607+0000","last_unstale":"2024-08-17T18:14:19.402610+0000","last_undegraded":"2024-08-17T18:14:19.402610+0000","last_fullsized":"2024-08-17T18:14:19.402610+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:13.437895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":716,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.376604+0000","last_change":"2024-08-17T18:09:29.866944+0000","last_active":"2024-08-17T18:14:16.376604+0000","last_peered":"2024-08-17T18:14:16.376604+0000","last_clean":"2024-08-17T18:14:16.376604+0000","last_became_active":"2024-08-17T18:09:29.866694+0000","last_became_peered":"2024-08-17T18:09:29.866694+0000","last_unstale":"2024-08-17T18:14:16.376604+0000","last_undegraded":"2024-08-17T18:14:16.376604+0000","last_fullsized":"2024-08-17T18:14:16.376604+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:00:41.342392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1379,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.375025+0000","last_change":"2024-08-17T18:09:28.884262+0000","last_active":"2024-08-17T18:14:16.375025+0000","last_peered":"2024-08-17T18:14:16.375025+0000","last_clean":"2024-08-17T18:14:16.375025+0000","last_became_active":"2024-08-17T18:09:28.883734+0000","last_became_peered":"2024-08-17T18:09:28.883734+0000","last_unstale":"2024-08-17T18:14:16.375025+0000","last_undegraded":"2024-08-17T18:14:16.375025+0000","last_fullsized":"2024-08-17T18:14:16.375025+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:10.735504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2130,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.378835+0000","last_change":"2024-08-17T18:08:57.571995+0000","last_active":"2024-08-17T18:14:16.378835+0000","last_peered":"2024-08-17T18:14:16.378835+0000","last_clean":"2024-08-17T18:14:16.378835+0000","last_became_active":"2024-08-17T18:08:57.567342+0000","last_became_peered":"2024-08-17T18:08:57.567342+0000","last_unstale":"2024-08-17T18:14:16.378835+0000","last_undegraded":"2024-08-17T18:14:16.378835+0000","last_fullsized":"2024-08-17T18:14:16.378835+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:23.769111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":543,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374251+0000","last_change":"2024-08-17T18:02:53.912837+0000","last_active":"2024-08-17T18:14:16.374251+0000","last_peered":"2024-08-17T18:14:16.374251+0000","last_clean":"2024-08-17T18:14:16.374251+0000","last_became_active":"2024-08-17T18:02:53.883152+0000","last_became_peered":"2024-08-17T18:02:53.883152+0000","last_unstale":"2024-08-17T18:14:16.374251+0000","last_undegraded":"2024-08-17T18:14:16.374251+0000","last_fullsized":"2024-08-17T18:14:16.374251+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:46.261612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":542,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374099+0000","last_change":"2024-08-17T18:09:29.845126+0000","last_active":"2024-08-17T18:14:16.374099+0000","last_peered":"2024-08-17T18:14:16.374099+0000","last_clean":"2024-08-17T18:14:16.374099+0000","last_became_active":"2024-08-17T18:09:29.844836+0000","last_became_peered":"2024-08-17T18:09:29.844836+0000","last_unstale":"2024-08-17T18:14:16.374099+0000","last_undegraded":"2024-08-17T18:14:16.374099+0000","last_fullsized":"2024-08-17T18:14:16.374099+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:44.821894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1367,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.379668+0000","last_change":"2024-08-17T18:02:53.890198+0000","last_active":"2024-08-17T18:14:16.379668+0000","last_peered":"2024-08-17T18:14:16.379668+0000","last_clean":"2024-08-17T18:14:16.379668+0000","last_became_active":"2024-08-17T18:02:53.887007+0000","last_became_peered":"2024-08-17T18:02:53.887007+0000","last_unstale":"2024-08-17T18:14:16.379668+0000","last_undegraded":"2024-08-17T18:14:16.379668+0000","last_fullsized":"2024-08-17T18:14:16.379668+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:25.555909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":633,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374720+0000","last_change":"2024-08-17T18:09:28.884187+0000","last_active":"2024-08-17T18:14:16.374720+0000","last_peered":"2024-08-17T18:14:16.374720+0000","last_clean":"2024-08-17T18:14:16.374720+0000","last_became_active":"2024-08-17T18:09:28.883939+0000","last_became_peered":"2024-08-17T18:09:28.883939+0000","last_unstale":"2024-08-17T18:14:16.374720+0000","last_undegraded":"2024-08-17T18:14:16.374720+0000","last_fullsized":"2024-08-17T18:14:16.374720+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:37:41.951099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1933,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.376561+0000","last_change":"2024-08-17T18:09:29.893220+0000","last_active":"2024-08-17T18:14:16.376561+0000","last_peered":"2024-08-17T18:14:16.376561+0000","last_clean":"2024-08-17T18:14:16.376561+0000","last_became_active":"2024-08-17T18:09:29.865699+0000","last_became_peered":"2024-08-17T18:09:29.865699+0000","last_unstale":"2024-08-17T18:14:16.376561+0000","last_undegraded":"2024-08-17T18:14:16.376561+0000","last_fullsized":"2024-08-17T18:14:16.376561+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:40.205658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2089,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402644+0000","last_change":"2024-08-17T18:09:28.934404+0000","last_active":"2024-08-17T18:14:19.402644+0000","last_peered":"2024-08-17T18:14:19.402644+0000","last_clean":"2024-08-17T18:14:19.402644+0000","last_became_active":"2024-08-17T18:09:28.934001+0000","last_became_peered":"2024-08-17T18:09:28.934001+0000","last_unstale":"2024-08-17T18:14:19.402644+0000","last_undegraded":"2024-08-17T18:14:19.402644+0000","last_fullsized":"2024-08-17T18:14:19.402644+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:04.676812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":2121,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.403322+0000","last_change":"2024-08-17T18:09:29.884088+0000","last_active":"2024-08-17T18:14:19.403322+0000","last_peered":"2024-08-17T18:14:19.403322+0000","last_clean":"2024-08-17T18:14:19.403322+0000","last_became_active":"2024-08-17T18:09:29.883027+0000","last_became_peered":"2024-08-17T18:09:29.883027+0000","last_unstale":"2024-08-17T18:14:19.403322+0000","last_undegraded":"2024-08-17T18:14:19.403322+0000","last_fullsized":"2024-08-17T18:14:19.403322+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:22:07.777188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":2091,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.405932+0000","last_change":"2024-08-17T18:09:28.935401+0000","last_active":"2024-08-17T18:14:19.405932+0000","last_peered":"2024-08-17T18:14:19.405932+0000","last_clean":"2024-08-17T18:14:19.405932+0000","last_became_active":"2024-08-17T18:09:28.933628+0000","last_became_peered":"2024-08-17T18:09:28.933628+0000","last_unstale":"2024-08-17T18:14:19.405932+0000","last_undegraded":"2024-08-17T18:14:19.405932+0000","last_fullsized":"2024-08-17T18:14:19.405932+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:21:12.714486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2201,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.388048+0000","last_change":"2024-08-17T18:08:57.574718+0000","last_active":"2024-08-17T18:14:16.388048+0000","last_peered":"2024-08-17T18:14:16.388048+0000","last_clean":"2024-08-17T18:14:16.388048+0000","last_became_active":"2024-08-17T18:08:57.573515+0000","last_became_peered":"2024-08-17T18:08:57.573515+0000","last_unstale":"2024-08-17T18:14:16.388048+0000","last_undegraded":"2024-08-17T18:14:16.388048+0000","last_fullsized":"2024-08-17T18:14:16.388048+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:04.873717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1392,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.375154+0000","last_change":"2024-08-17T18:08:57.568821+0000","last_active":"2024-08-17T18:14:16.375154+0000","last_peered":"2024-08-17T18:14:16.375154+0000","last_clean":"2024-08-17T18:14:16.375154+0000","last_became_active":"2024-08-17T18:08:57.568526+0000","last_became_peered":"2024-08-17T18:08:57.568526+0000","last_unstale":"2024-08-17T18:14:16.375154+0000","last_undegraded":"2024-08-17T18:14:16.375154+0000","last_fullsized":"2024-08-17T18:14:16.375154+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:26.449197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1921,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.380745+0000","last_change":"2024-08-17T18:08:57.571174+0000","last_active":"2024-08-17T18:14:16.380745+0000","last_peered":"2024-08-17T18:14:16.380745+0000","last_clean":"2024-08-17T18:14:16.380745+0000","last_became_active":"2024-08-17T18:08:57.567571+0000","last_became_peered":"2024-08-17T18:08:57.567571+0000","last_unstale":"2024-08-17T18:14:16.380745+0000","last_undegraded":"2024-08-17T18:14:16.380745+0000","last_fullsized":"2024-08-17T18:14:16.380745+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:29.976150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1890,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.379043+0000","last_change":"2024-08-17T18:08:57.572368+0000","last_active":"2024-08-17T18:14:16.379043+0000","last_peered":"2024-08-17T18:14:16.379043+0000","last_clean":"2024-08-17T18:14:16.379043+0000","last_became_active":"2024-08-17T18:08:57.568382+0000","last_became_peered":"2024-08-17T18:08:57.568382+0000","last_unstale":"2024-08-17T18:14:16.379043+0000","last_undegraded":"2024-08-17T18:14:16.379043+0000","last_fullsized":"2024-08-17T18:14:16.379043+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:11:19.968079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1938,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.380836+0000","last_change":"2024-08-17T18:08:57.572991+0000","last_active":"2024-08-17T18:14:16.380836+0000","last_peered":"2024-08-17T18:14:16.380836+0000","last_clean":"2024-08-17T18:14:16.380836+0000","last_became_active":"2024-08-17T18:08:57.571145+0000","last_became_peered":"2024-08-17T18:08:57.571145+0000","last_unstale":"2024-08-17T18:14:16.380836+0000","last_undegraded":"2024-08-17T18:14:16.380836+0000","last_fullsized":"2024-08-17T18:14:16.380836+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:39.963337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":896,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402384+0000","last_change":"2024-08-17T18:09:28.860288+0000","last_active":"2024-08-17T18:14:19.402384+0000","last_peered":"2024-08-17T18:14:19.402384+0000","last_clean":"2024-08-17T18:14:19.402384+0000","last_became_active":"2024-08-17T18:09:28.860004+0000","last_became_peered":"2024-08-17T18:09:28.860004+0000","last_unstale":"2024-08-17T18:14:19.402384+0000","last_undegraded":"2024-08-17T18:14:19.402384+0000","last_fullsized":"2024-08-17T18:14:19.402384+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:57:33.054055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":689,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.411374+0000","last_change":"2024-08-17T18:09:28.860277+0000","last_active":"2024-08-17T18:14:19.411374+0000","last_peered":"2024-08-17T18:14:19.411374+0000","last_clean":"2024-08-17T18:14:19.411374+0000","last_became_active":"2024-08-17T18:09:28.859999+0000","last_became_peered":"2024-08-17T18:09:28.859999+0000","last_unstale":"2024-08-17T18:14:19.411374+0000","last_undegraded":"2024-08-17T18:14:19.411374+0000","last_fullsized":"2024-08-17T18:14:19.411374+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:27.946526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1592,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.387735+0000","last_change":"2024-08-17T18:09:29.863669+0000","last_active":"2024-08-17T18:14:16.387735+0000","last_peered":"2024-08-17T18:14:16.387735+0000","last_clean":"2024-08-17T18:14:16.387735+0000","last_became_active":"2024-08-17T18:09:29.862781+0000","last_became_peered":"2024-08-17T18:09:29.862781+0000","last_unstale":"2024-08-17T18:14:16.387735+0000","last_undegraded":"2024-08-17T18:14:16.387735+0000","last_fullsized":"2024-08-17T18:14:16.387735+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:14.073233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":173,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.379149+0000","last_change":"2024-08-17T18:09:29.843574+0000","last_active":"2024-08-17T18:14:16.379149+0000","last_peered":"2024-08-17T18:14:16.379149+0000","last_clean":"2024-08-17T18:14:16.379149+0000","last_became_active":"2024-08-17T18:09:29.843097+0000","last_became_peered":"2024-08-17T18:09:29.843097+0000","last_unstale":"2024-08-17T18:14:16.379149+0000","last_undegraded":"2024-08-17T18:14:16.379149+0000","last_fullsized":"2024-08-17T18:14:16.379149+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:46.740888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1937,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.380115+0000","last_change":"2024-08-17T18:09:29.847264+0000","last_active":"2024-08-17T18:14:16.380115+0000","last_peered":"2024-08-17T18:14:16.380115+0000","last_clean":"2024-08-17T18:14:16.380115+0000","last_became_active":"2024-08-17T18:09:29.846226+0000","last_became_peered":"2024-08-17T18:09:29.846226+0000","last_unstale":"2024-08-17T18:14:16.380115+0000","last_undegraded":"2024-08-17T18:14:16.380115+0000","last_fullsized":"2024-08-17T18:14:16.380115+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:44.135977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2137,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.378503+0000","last_change":"2024-08-17T18:09:29.845209+0000","last_active":"2024-08-17T18:14:16.378503+0000","last_peered":"2024-08-17T18:14:16.378503+0000","last_clean":"2024-08-17T18:14:16.378503+0000","last_became_active":"2024-08-17T18:09:29.844910+0000","last_became_peered":"2024-08-17T18:09:29.844910+0000","last_unstale":"2024-08-17T18:14:16.378503+0000","last_undegraded":"2024-08-17T18:14:16.378503+0000","last_fullsized":"2024-08-17T18:14:16.378503+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:14.586354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2012,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402921+0000","last_change":"2024-08-17T18:09:28.859050+0000","last_active":"2024-08-17T18:14:19.402921+0000","last_peered":"2024-08-17T18:14:19.402921+0000","last_clean":"2024-08-17T18:14:19.402921+0000","last_became_active":"2024-08-17T18:09:28.858214+0000","last_became_peered":"2024-08-17T18:09:28.858214+0000","last_unstale":"2024-08-17T18:14:19.402921+0000","last_undegraded":"2024-08-17T18:14:19.402921+0000","last_fullsized":"2024-08-17T18:14:19.402921+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:54.146592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2261,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.379120+0000","last_change":"2024-08-17T18:08:57.572610+0000","last_active":"2024-08-17T18:14:16.379120+0000","last_peered":"2024-08-17T18:14:16.379120+0000","last_clean":"2024-08-17T18:14:16.379120+0000","last_became_active":"2024-08-17T18:08:57.572308+0000","last_became_peered":"2024-08-17T18:08:57.572308+0000","last_unstale":"2024-08-17T18:14:16.379120+0000","last_undegraded":"2024-08-17T18:14:16.379120+0000","last_fullsized":"2024-08-17T18:14:16.379120+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:11.642831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2246,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.379744+0000","last_change":"2024-08-17T18:09:28.842838+0000","last_active":"2024-08-17T18:14:16.379744+0000","last_peered":"2024-08-17T18:14:16.379744+0000","last_clean":"2024-08-17T18:14:16.379744+0000","last_became_active":"2024-08-17T18:09:28.842617+0000","last_became_peered":"2024-08-17T18:09:28.842617+0000","last_unstale":"2024-08-17T18:14:16.379744+0000","last_undegraded":"2024-08-17T18:14:16.379744+0000","last_fullsized":"2024-08-17T18:14:16.379744+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:00.016002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1413,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.377693+0000","last_change":"2024-08-17T18:09:29.845011+0000","last_active":"2024-08-17T18:14:16.377693+0000","last_peered":"2024-08-17T18:14:16.377693+0000","last_clean":"2024-08-17T18:14:16.377693+0000","last_became_active":"2024-08-17T18:09:29.844358+0000","last_became_peered":"2024-08-17T18:09:29.844358+0000","last_unstale":"2024-08-17T18:14:16.377693+0000","last_undegraded":"2024-08-17T18:14:16.377693+0000","last_fullsized":"2024-08-17T18:14:16.377693+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:41:27.792901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1261,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.375090+0000","last_change":"2024-08-17T18:09:28.878263+0000","last_active":"2024-08-17T18:14:16.375090+0000","last_peered":"2024-08-17T18:14:16.375090+0000","last_clean":"2024-08-17T18:14:16.375090+0000","last_became_active":"2024-08-17T18:09:28.878025+0000","last_became_peered":"2024-08-17T18:09:28.878025+0000","last_unstale":"2024-08-17T18:14:16.375090+0000","last_undegraded":"2024-08-17T18:14:16.375090+0000","last_fullsized":"2024-08-17T18:14:16.375090+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:37.211100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2040,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.410591+0000","last_change":"2024-08-17T18:09:29.840134+0000","last_active":"2024-08-17T18:14:19.410591+0000","last_peered":"2024-08-17T18:14:19.410591+0000","last_clean":"2024-08-17T18:14:19.410591+0000","last_became_active":"2024-08-17T18:09:29.839848+0000","last_became_peered":"2024-08-17T18:09:29.839848+0000","last_unstale":"2024-08-17T18:14:19.410591+0000","last_undegraded":"2024-08-17T18:14:19.410591+0000","last_fullsized":"2024-08-17T18:14:19.410591+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:08:08.717851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":168,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374367+0000","last_change":"2024-08-17T18:09:28.847633+0000","last_active":"2024-08-17T18:14:16.374367+0000","last_peered":"2024-08-17T18:14:16.374367+0000","last_clean":"2024-08-17T18:14:16.374367+0000","last_became_active":"2024-08-17T18:09:28.847341+0000","last_became_peered":"2024-08-17T18:09:28.847341+0000","last_unstale":"2024-08-17T18:14:16.374367+0000","last_undegraded":"2024-08-17T18:14:16.374367+0000","last_fullsized":"2024-08-17T18:14:16.374367+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:51:59.625154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1753,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.377648+0000","last_change":"2024-08-17T18:09:00.609431+0000","last_active":"2024-08-17T18:14:16.377648+0000","last_peered":"2024-08-17T18:14:16.377648+0000","last_clean":"2024-08-17T18:14:16.377648+0000","last_became_active":"2024-08-17T18:09:00.609167+0000","last_became_peered":"2024-08-17T18:09:00.609167+0000","last_unstale":"2024-08-17T18:14:16.377648+0000","last_undegraded":"2024-08-17T18:14:16.377648+0000","last_fullsized":"2024-08-17T18:14:16.377648+0000","mapping_epoch":1338,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:39.579970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1920,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.410582+0000","last_change":"2024-08-17T18:09:29.837502+0000","last_active":"2024-08-17T18:14:19.410582+0000","last_peered":"2024-08-17T18:14:19.410582+0000","last_clean":"2024-08-17T18:14:19.410582+0000","last_became_active":"2024-08-17T18:09:29.837243+0000","last_became_peered":"2024-08-17T18:09:29.837243+0000","last_unstale":"2024-08-17T18:14:19.410582+0000","last_undegraded":"2024-08-17T18:14:19.410582+0000","last_fullsized":"2024-08-17T18:14:19.410582+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:47.377505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2118,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402740+0000","last_change":"2024-08-17T18:09:29.859415+0000","last_active":"2024-08-17T18:14:19.402740+0000","last_peered":"2024-08-17T18:14:19.402740+0000","last_clean":"2024-08-17T18:14:19.402740+0000","last_became_active":"2024-08-17T18:09:29.859138+0000","last_became_peered":"2024-08-17T18:09:29.859138+0000","last_unstale":"2024-08-17T18:14:19.402740+0000","last_undegraded":"2024-08-17T18:14:19.402740+0000","last_fullsized":"2024-08-17T18:14:19.402740+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:31.178915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":569,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.403240+0000","last_change":"2024-08-17T18:09:29.883834+0000","last_active":"2024-08-17T18:14:19.403240+0000","last_peered":"2024-08-17T18:14:19.403240+0000","last_clean":"2024-08-17T18:14:19.403240+0000","last_became_active":"2024-08-17T18:09:29.882638+0000","last_became_peered":"2024-08-17T18:09:29.882638+0000","last_unstale":"2024-08-17T18:14:19.403240+0000","last_undegraded":"2024-08-17T18:14:19.403240+0000","last_fullsized":"2024-08-17T18:14:19.403240+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:07:10.188684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":653,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.373773+0000","last_change":"2024-08-17T18:09:29.857250+0000","last_active":"2024-08-17T18:14:16.373773+0000","last_peered":"2024-08-17T18:14:16.373773+0000","last_clean":"2024-08-17T18:14:16.373773+0000","last_became_active":"2024-08-17T18:09:29.857038+0000","last_became_peered":"2024-08-17T18:09:29.857038+0000","last_unstale":"2024-08-17T18:14:16.373773+0000","last_undegraded":"2024-08-17T18:14:16.373773+0000","last_fullsized":"2024-08-17T18:14:16.373773+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:02:54.028575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":680,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.380472+0000","last_change":"2024-08-17T18:08:57.571270+0000","last_active":"2024-08-17T18:14:16.380472+0000","last_peered":"2024-08-17T18:14:16.380472+0000","last_clean":"2024-08-17T18:14:16.380472+0000","last_became_active":"2024-08-17T18:08:57.567670+0000","last_became_peered":"2024-08-17T18:08:57.567670+0000","last_unstale":"2024-08-17T18:14:16.380472+0000","last_undegraded":"2024-08-17T18:14:16.380472+0000","last_fullsized":"2024-08-17T18:14:16.380472+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:19:42.617321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1253,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.410821+0000","last_change":"2024-08-17T18:09:29.839260+0000","last_active":"2024-08-17T18:14:19.410821+0000","last_peered":"2024-08-17T18:14:19.410821+0000","last_clean":"2024-08-17T18:14:19.410821+0000","last_became_active":"2024-08-17T18:09:29.838953+0000","last_became_peered":"2024-08-17T18:09:29.838953+0000","last_unstale":"2024-08-17T18:14:19.410821+0000","last_undegraded":"2024-08-17T18:14:19.410821+0000","last_fullsized":"2024-08-17T18:14:19.410821+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:44.014083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":842,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.411453+0000","last_change":"2024-08-17T18:09:29.851347+0000","last_active":"2024-08-17T18:14:19.411453+0000","last_peered":"2024-08-17T18:14:19.411453+0000","last_clean":"2024-08-17T18:14:19.411453+0000","last_became_active":"2024-08-17T18:09:29.851046+0000","last_became_peered":"2024-08-17T18:09:29.851046+0000","last_unstale":"2024-08-17T18:14:19.411453+0000","last_undegraded":"2024-08-17T18:14:19.411453+0000","last_fullsized":"2024-08-17T18:14:19.411453+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:21.350703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2119,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402038+0000","last_change":"2024-08-17T18:09:29.912446+0000","last_active":"2024-08-17T18:14:19.402038+0000","last_peered":"2024-08-17T18:14:19.402038+0000","last_clean":"2024-08-17T18:14:19.402038+0000","last_became_active":"2024-08-17T18:09:29.910703+0000","last_became_peered":"2024-08-17T18:09:29.910703+0000","last_unstale":"2024-08-17T18:14:19.402038+0000","last_undegraded":"2024-08-17T18:14:19.402038+0000","last_fullsized":"2024-08-17T18:14:19.402038+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.250608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2283,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.388126+0000","last_change":"2024-08-17T18:09:29.863361+0000","last_active":"2024-08-17T18:14:16.388126+0000","last_peered":"2024-08-17T18:14:16.388126+0000","last_clean":"2024-08-17T18:14:16.388126+0000","last_became_active":"2024-08-17T18:09:29.862503+0000","last_became_peered":"2024-08-17T18:09:29.862503+0000","last_unstale":"2024-08-17T18:14:16.388126+0000","last_undegraded":"2024-08-17T18:14:16.388126+0000","last_fullsized":"2024-08-17T18:14:16.388126+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:25.908232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":108,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.410751+0000","last_change":"2024-08-17T18:09:29.895991+0000","last_active":"2024-08-17T18:14:19.410751+0000","last_peered":"2024-08-17T18:14:19.410751+0000","last_clean":"2024-08-17T18:14:19.410751+0000","last_became_active":"2024-08-17T18:09:29.895274+0000","last_became_peered":"2024-08-17T18:09:29.895274+0000","last_unstale":"2024-08-17T18:14:19.410751+0000","last_undegraded":"2024-08-17T18:14:19.410751+0000","last_fullsized":"2024-08-17T18:14:19.410751+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:19:26.420016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":535,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.376281+0000","last_change":"2024-08-17T18:09:29.844686+0000","last_active":"2024-08-17T18:14:16.376281+0000","last_peered":"2024-08-17T18:14:16.376281+0000","last_clean":"2024-08-17T18:14:16.376281+0000","last_became_active":"2024-08-17T18:09:29.844341+0000","last_became_peered":"2024-08-17T18:09:29.844341+0000","last_unstale":"2024-08-17T18:14:16.376281+0000","last_undegraded":"2024-08-17T18:14:16.376281+0000","last_fullsized":"2024-08-17T18:14:16.376281+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:38.872767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1372,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.375135+0000","last_change":"2024-08-17T18:09:28.884310+0000","last_active":"2024-08-17T18:14:16.375135+0000","last_peered":"2024-08-17T18:14:16.375135+0000","last_clean":"2024-08-17T18:14:16.375135+0000","last_became_active":"2024-08-17T18:09:28.883921+0000","last_became_peered":"2024-08-17T18:09:28.883921+0000","last_unstale":"2024-08-17T18:14:16.375135+0000","last_undegraded":"2024-08-17T18:14:16.375135+0000","last_fullsized":"2024-08-17T18:14:16.375135+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:05.272563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":851,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402889+0000","last_change":"2024-08-17T18:09:29.837238+0000","last_active":"2024-08-17T18:14:19.402889+0000","last_peered":"2024-08-17T18:14:19.402889+0000","last_clean":"2024-08-17T18:14:19.402889+0000","last_became_active":"2024-08-17T18:09:29.836760+0000","last_became_peered":"2024-08-17T18:09:29.836760+0000","last_unstale":"2024-08-17T18:14:19.402889+0000","last_undegraded":"2024-08-17T18:14:19.402889+0000","last_fullsized":"2024-08-17T18:14:19.402889+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:38:52.862967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1387,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.376657+0000","last_change":"2024-08-17T18:09:29.871872+0000","last_active":"2024-08-17T18:14:16.376657+0000","last_peered":"2024-08-17T18:14:16.376657+0000","last_clean":"2024-08-17T18:14:16.376657+0000","last_became_active":"2024-08-17T18:09:29.845111+0000","last_became_peered":"2024-08-17T18:09:29.845111+0000","last_unstale":"2024-08-17T18:14:16.376657+0000","last_undegraded":"2024-08-17T18:14:16.376657+0000","last_fullsized":"2024-08-17T18:14:16.376657+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:48.183250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":284,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.377574+0000","last_change":"2024-08-17T18:09:29.844786+0000","last_active":"2024-08-17T18:14:16.377574+0000","last_peered":"2024-08-17T18:14:16.377574+0000","last_clean":"2024-08-17T18:14:16.377574+0000","last_became_active":"2024-08-17T18:09:29.844270+0000","last_became_peered":"2024-08-17T18:09:29.844270+0000","last_unstale":"2024-08-17T18:14:16.377574+0000","last_undegraded":"2024-08-17T18:14:16.377574+0000","last_fullsized":"2024-08-17T18:14:16.377574+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:51.866680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1403,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374723+0000","last_change":"2024-08-17T18:09:29.886391+0000","last_active":"2024-08-17T18:14:16.374723+0000","last_peered":"2024-08-17T18:14:16.374723+0000","last_clean":"2024-08-17T18:14:16.374723+0000","last_became_active":"2024-08-17T18:09:29.844642+0000","last_became_peered":"2024-08-17T18:09:29.844642+0000","last_unstale":"2024-08-17T18:14:16.374723+0000","last_undegraded":"2024-08-17T18:14:16.374723+0000","last_fullsized":"2024-08-17T18:14:16.374723+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:07.988100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":262,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.378671+0000","last_change":"2024-08-17T18:09:28.941223+0000","last_active":"2024-08-17T18:14:16.378671+0000","last_peered":"2024-08-17T18:14:16.378671+0000","last_clean":"2024-08-17T18:14:16.378671+0000","last_became_active":"2024-08-17T18:09:28.940857+0000","last_became_peered":"2024-08-17T18:09:28.940857+0000","last_unstale":"2024-08-17T18:14:16.378671+0000","last_undegraded":"2024-08-17T18:14:16.378671+0000","last_fullsized":"2024-08-17T18:14:16.378671+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:48.809420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2116,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.410090+0000","last_change":"2024-08-17T18:09:29.838873+0000","last_active":"2024-08-17T18:14:19.410090+0000","last_peered":"2024-08-17T18:14:19.410090+0000","last_clean":"2024-08-17T18:14:19.410090+0000","last_became_active":"2024-08-17T18:09:29.838560+0000","last_became_peered":"2024-08-17T18:09:29.838560+0000","last_unstale":"2024-08-17T18:14:19.410090+0000","last_undegraded":"2024-08-17T18:14:19.410090+0000","last_fullsized":"2024-08-17T18:14:19.410090+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:00.103741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1363,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.373906+0000","last_change":"2024-08-17T18:09:28.844170+0000","last_active":"2024-08-17T18:14:16.373906+0000","last_peered":"2024-08-17T18:14:16.373906+0000","last_clean":"2024-08-17T18:14:16.373906+0000","last_became_active":"2024-08-17T18:09:28.843862+0000","last_became_peered":"2024-08-17T18:09:28.843862+0000","last_unstale":"2024-08-17T18:14:16.373906+0000","last_undegraded":"2024-08-17T18:14:16.373906+0000","last_fullsized":"2024-08-17T18:14:16.373906+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:43:19.483326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2236,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.378870+0000","last_change":"2024-08-17T18:09:28.864923+0000","last_active":"2024-08-17T18:14:16.378870+0000","last_peered":"2024-08-17T18:14:16.378870+0000","last_clean":"2024-08-17T18:14:16.378870+0000","last_became_active":"2024-08-17T18:09:28.864707+0000","last_became_peered":"2024-08-17T18:09:28.864707+0000","last_unstale":"2024-08-17T18:14:16.378870+0000","last_undegraded":"2024-08-17T18:14:16.378870+0000","last_fullsized":"2024-08-17T18:14:16.378870+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:27.507329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2237,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.379240+0000","last_change":"2024-08-17T18:09:28.846741+0000","last_active":"2024-08-17T18:14:16.379240+0000","last_peered":"2024-08-17T18:14:16.379240+0000","last_clean":"2024-08-17T18:14:16.379240+0000","last_became_active":"2024-08-17T18:09:28.846478+0000","last_became_peered":"2024-08-17T18:09:28.846478+0000","last_unstale":"2024-08-17T18:14:16.379240+0000","last_undegraded":"2024-08-17T18:14:16.379240+0000","last_fullsized":"2024-08-17T18:14:16.379240+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:34.247517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":769,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374993+0000","last_change":"2024-08-17T18:03:16.091420+0000","last_active":"2024-08-17T18:14:16.374993+0000","last_peered":"2024-08-17T18:14:16.374993+0000","last_clean":"2024-08-17T18:14:16.374993+0000","last_became_active":"2024-08-17T18:03:16.091113+0000","last_became_peered":"2024-08-17T18:03:16.091113+0000","last_unstale":"2024-08-17T18:14:16.374993+0000","last_undegraded":"2024-08-17T18:14:16.374993+0000","last_fullsized":"2024-08-17T18:14:16.374993+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:47.770911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1377,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374289+0000","last_change":"2024-08-17T18:02:53.893927+0000","last_active":"2024-08-17T18:14:16.374289+0000","last_peered":"2024-08-17T18:14:16.374289+0000","last_clean":"2024-08-17T18:14:16.374289+0000","last_became_active":"2024-08-17T18:02:53.883188+0000","last_became_peered":"2024-08-17T18:02:53.883188+0000","last_unstale":"2024-08-17T18:14:16.374289+0000","last_undegraded":"2024-08-17T18:14:16.374289+0000","last_fullsized":"2024-08-17T18:14:16.374289+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:19.296965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2116,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.411240+0000","last_change":"2024-08-17T18:09:29.840086+0000","last_active":"2024-08-17T18:14:19.411240+0000","last_peered":"2024-08-17T18:14:19.411240+0000","last_clean":"2024-08-17T18:14:19.411240+0000","last_became_active":"2024-08-17T18:09:29.839525+0000","last_became_peered":"2024-08-17T18:09:29.839525+0000","last_unstale":"2024-08-17T18:14:19.411240+0000","last_undegraded":"2024-08-17T18:14:19.411240+0000","last_fullsized":"2024-08-17T18:14:19.411240+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:38.316235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":49,"num_pg":16,"stat_sum":{"num_bytes":6962413568,"num_objects":106238,"num_object_clones":0,"num_object_copies":318714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":106238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":106238,"num_write_kb":6799232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20770193408,"data_stored":20770193408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":70438,"ondisk_log_size":70438,"up":48,"acting":48,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":10970,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":155648,"data_stored":27490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234642,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6049688,"kb_used_data":5151952,"kb_used_omap":1,"kb_used_meta":897726,"kb_avail":87695464,"statfs":{"total":95995035648,"available":89800155136,"internally_reserved":0,"allocated":5275598848,"data_stored":5270894320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":919271768},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568496277,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757915,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1343,"seq":5768141078588,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5921928,"kb_used_data":5559808,"kb_used_omap":112,"kb_used_meta":361999,"kb_avail":87823224,"statfs":{"total":95995035648,"available":89930981376,"internally_reserved":0,"allocated":5693243392,"data_stored":5688550508,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":115373,"internal_metadata":370687315},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405767,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6557692,"kb_used_data":5380404,"kb_used_omap":61,"kb_used_meta":1177218,"kb_avail":87187460,"statfs":{"total":95995035648,"available":89279959040,"internally_reserved":0,"allocated":5509533696,"data_stored":5504849352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":1205472184},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":0,"seq":2173253452302,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474992,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5191948,"kb_used_data":4155396,"kb_used_omap":119,"kb_used_meta":1036424,"kb_avail":88553204,"statfs":{"total":95995035648,"available":90678480896,"internally_reserved":0,"allocated":4255125504,"data_stored":4250424985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":1061298910},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019550,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":49,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4257153024,"data_stored":4257153024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":49,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":5547294720,"data_stored":5547294720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":49,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":5682102272,"data_stored":5682102272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":49,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":5283643392,"data_stored":5283643392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:14:31.031 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:14:31.047 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:31.151 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:31.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:31.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:31.253 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:31.257 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:31 smithi130 ceph-mon[105521]: pgmap v2589: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 100 B/s rd, 42 MiB/s wr, 982 op/s 2024-08-17T18:14:31.257 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:31 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:14:31.258 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:31 smithi130 ceph-mon[110068]: pgmap v2589: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 100 B/s rd, 42 MiB/s wr, 982 op/s 2024-08-17T18:14:31.258 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:31 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:14:31.277 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:31.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:31.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:31.328 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:31 smithi179 ceph-mon[93519]: pgmap v2589: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 100 B/s rd, 42 MiB/s wr, 982 op/s 2024-08-17T18:14:31.328 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:31 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:14:31.378 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:31.435 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:31.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:31.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:31.537 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:31.559 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:31.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:31.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:31.661 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:31.684 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:31.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:31.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:31.786 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:31.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:31.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:31.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:31.924 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:31.946 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:31.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:32.048 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:32.080 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:32.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:32.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:32.182 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:32.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:32.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:32.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:32.315 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:32.339 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:32.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:32.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:32.441 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:32.464 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:32.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:32.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:32.566 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:32.590 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:32.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:32.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:32.691 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:32.718 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:32.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:32.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:32.819 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:32.841 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:32.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:32.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:32.943 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:32.966 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:32.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:32.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:33.068 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:33.089 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:33.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:33.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:33.191 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:33.225 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:33.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:33.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:33.327 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:33.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:33 smithi179 ceph-mon[93519]: pgmap v2590: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 85 B/s rd, 36 MiB/s wr, 830 op/s 2024-08-17T18:14:33.364 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:33.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:33.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:33.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:33 smithi130 ceph-mon[110068]: pgmap v2590: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 85 B/s rd, 36 MiB/s wr, 830 op/s 2024-08-17T18:14:33.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:33 smithi130 ceph-mon[105521]: pgmap v2590: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 335 GiB / 358 GiB avail; 85 B/s rd, 36 MiB/s wr, 830 op/s 2024-08-17T18:14:33.466 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:33.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:33.509 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:33.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:33.610 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:33.634 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:33.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:33.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:33.735 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:33.759 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:33.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:33.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:33.860 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:33.882 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:33.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:33.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:33.984 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:34.009 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:34.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:34.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:34.111 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:34.133 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:34.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:34.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:34.235 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:34.259 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:34.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:34.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:34.361 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:34.384 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:34.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:34.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:34.485 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:34.535 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:34.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:34.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:34.637 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:34.661 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:34.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:34.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:34.763 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:34.785 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:34.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:34.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:34.887 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:34.908 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:34.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:34.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:34.970 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:14:34.974 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:14:34.974 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:14:35.010 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:35.061 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:35.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:35.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:35.089 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:35 smithi130 ceph-mon[105521]: pgmap v2591: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 85 B/s rd, 36 MiB/s wr, 1.18k op/s 2024-08-17T18:14:35.090 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:35 smithi130 ceph-mon[110068]: pgmap v2591: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 85 B/s rd, 36 MiB/s wr, 1.18k op/s 2024-08-17T18:14:35.163 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:35.185 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:35.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:35.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:35.287 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:35.309 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:35 smithi179 ceph-mon[93519]: pgmap v2591: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 335 GiB / 358 GiB avail; 85 B/s rd, 36 MiB/s wr, 1.18k op/s 2024-08-17T18:14:35.310 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:35.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:35.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:35.412 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:35.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:35.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:35.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:35.538 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:35.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:35.584 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:35.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:35.685 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:35.739 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:35.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:35.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:35.841 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:35.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:35.864 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:35.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:35.965 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:36.021 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:36.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:36.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:36.123 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:36.146 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:36.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:36.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:36.248 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:36.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:36.271 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:36.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:36.306 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:14:36.306 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:14:36.338 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:14:36.339 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:14:36.372 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:14:36.372 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:14:36.372 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:36.393 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:36.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:36.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:36.409 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:14:36.409 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:14:36.441 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:14:36.441 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:14:36.480 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:14:36.480 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:14:36.495 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:36.510 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:14:36.510 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:14:36.517 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:36.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:36.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:36.544 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:14:36.544 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:14:36.618 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:36.662 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:36.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:36.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:36.763 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:36.801 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:36.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:36.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:36.903 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:36.925 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:36.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:36.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:37.026 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:37.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:36 smithi130 ceph-mon[105521]: pgmap v2592: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 85 B/s rd, 15 MiB/s wr, 930 op/s 2024-08-17T18:14:37.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:36 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:37.062 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:36 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:37.063 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:36 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2831685276' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:14:37.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:36 smithi130 ceph-mon[110068]: pgmap v2592: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 85 B/s rd, 15 MiB/s wr, 930 op/s 2024-08-17T18:14:37.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:36 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:37.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:36 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:37.063 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:36 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2831685276' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:14:37.065 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:37.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:37.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:36 smithi179 ceph-mon[93519]: pgmap v2592: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 85 B/s rd, 15 MiB/s wr, 930 op/s 2024-08-17T18:14:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:36 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:36 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:36 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2831685276' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:14:37.167 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:37.207 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:37.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:37.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:37.308 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:37.364 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:37.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:37.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:37.467 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:37.520 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:37.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:37.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:37.622 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:37.654 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:37.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:37.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:37.756 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:37.800 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:37.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:37.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:37.902 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:37.923 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:37.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:37.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:38.025 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:38.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T18:14:38.047 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:38.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:38.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:38.149 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:38.196 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:38.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:38.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:38.298 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:38.345 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:38.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:38.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:38.446 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:38.468 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:38.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:38.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:38.570 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:38.600 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:38.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:38.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:38.702 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:38.736 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:38.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:38.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:38.825 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:14:38.838 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:38.859 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:38.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:38.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:38.961 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:39.002 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:39.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:39.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:39.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:38 smithi179 ceph-mon[93519]: pgmap v2593: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 85 B/s rd, 10 MiB/s wr, 1.09k op/s 2024-08-17T18:14:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:38 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2174918117' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:14:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:38 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:14:39.104 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:39.126 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:39.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:39.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:39.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:38 smithi130 ceph-mon[110068]: pgmap v2593: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 85 B/s rd, 10 MiB/s wr, 1.09k op/s 2024-08-17T18:14:39.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:38 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2174918117' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:14:39.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:38 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:14:39.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:38 smithi130 ceph-mon[105521]: pgmap v2593: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 336 GiB / 358 GiB avail; 85 B/s rd, 10 MiB/s wr, 1.09k op/s 2024-08-17T18:14:39.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:38 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2174918117' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:14:39.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:38 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:14:39.228 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:39.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:39.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:39.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:39.352 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:39.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:39.389 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:39.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:39.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:39.513 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:39.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:39.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:39.614 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:39.660 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:39.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:39.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:39.762 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:39.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:39.803 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:39.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:39.904 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:39.927 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:39.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:39.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:40.004 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:39 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:14:40.004 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:39 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:14:40.005 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:39 smithi130 ceph-mon[110068]: osdmap e1393: 8 total, 8 up, 4 in 2024-08-17T18:14:40.005 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:39 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:14:40.005 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:39 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:14:40.005 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:39 smithi130 ceph-mon[105521]: osdmap e1393: 8 total, 8 up, 4 in 2024-08-17T18:14:40.005 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:14:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:14:39] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-08-17T18:14:40.029 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:40.050 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:40.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:40.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:40.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:39 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:14:40.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:39 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:14:40.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:39 smithi179 ceph-mon[93519]: osdmap e1393: 8 total, 8 up, 4 in 2024-08-17T18:14:40.152 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:40.189 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:40.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:40.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:40.293 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:40.329 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:40.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:40.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:40.431 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:40.454 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:40.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:40.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:40.555 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:14:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:14:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:14:40.617 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:40.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:40.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:40.718 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:40.752 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:40.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:40.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:40.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:40.876 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:40.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:40.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:40.978 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:41.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:41.000 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:41.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:41.089 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:40 smithi130 ceph-mon[110068]: pgmap v2595: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T18:14:41.089 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:40 smithi130 ceph-mon[105521]: pgmap v2595: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T18:14:41.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:40 smithi179 ceph-mon[93519]: pgmap v2595: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T18:14:41.101 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:41.122 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:41.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:41.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:41.224 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:41.246 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:41.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:41.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:41.348 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:41.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:41.396 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:41.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:41.497 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:41.519 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:41.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:41.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:41.621 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:41.673 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:41.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:41.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:41.736 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:14:41.777 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:41.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:41.843 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:41.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:41.944 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:41.966 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:41.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:41.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:41.991 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:14:41.993 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:14:42.048 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:14:42.068 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:42.089 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:42.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:42.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:42.191 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:42.235 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:42.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:42.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:42.336 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:42.358 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:42.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:42.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:42.460 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:42.505 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:42.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:42.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:42.606 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:42.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:42.646 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:42.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:42.747 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:42.787 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:42.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:42.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:42.889 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:42.914 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:42.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:42.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:43.016 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:43.017 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:14:43.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:43.058 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:43.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:42 smithi179 ceph-mon[93519]: pgmap v2596: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T18:14:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:42 smithi179 ceph-mon[93519]: from='client.37023 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:14:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:14:43.159 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:43.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:42 smithi130 ceph-mon[105521]: pgmap v2596: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T18:14:43.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:42 smithi130 ceph-mon[105521]: from='client.37023 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:14:43.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:14:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:42 smithi130 ceph-mon[110068]: pgmap v2596: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T18:14:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:42 smithi130 ceph-mon[110068]: from='client.37023 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:14:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:14:43.184 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:43.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:43.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:43.286 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:43.326 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:43.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:43.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:43.428 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:43.479 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:43.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:43.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:43.581 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:43.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:43.622 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:43.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:43.723 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:43.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:43.747 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:43.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:43.848 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:43.888 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:43.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:43.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:43.920 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:14:43.975 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2596,"stamp":"2024-08-17T18:14:41.527539+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6274947045,"num_objects":95766,"num_object_clones":0,"num_object_copies":287298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":95766,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":122802,"num_write_kb":6986832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":2962829,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":70131,"ondisk_log_size":70131,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":21816504,"kb_used_data":18310676,"kb_used_omap":294,"kb_used_meta":3505305,"kb_avail":353164104,"statfs":{"total":383980142592,"available":361640042496,"internally_reserved":0,"allocated":18750132224,"data_stored":18731307641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":301759,"internal_metadata":3589432641},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,4,4,3,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-673710080,"num_objects":-10280,"num_object_clones":0,"num_object_copies":-30840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-10280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10280,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006521"},"pg_stats":[{"pgid":"49.f","version":"1392'7615","reported_seq":7650,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:37.430412+0000","last_change":"2024-08-17T18:12:42.612957+0000","last_active":"2024-08-17T18:14:37.430412+0000","last_peered":"2024-08-17T18:14:37.430412+0000","last_clean":"2024-08-17T18:14:37.430412+0000","last_became_active":"2024-08-17T18:12:42.612660+0000","last_became_peered":"2024-08-17T18:12:42.612660+0000","last_unstale":"2024-08-17T18:14:37.430412+0000","last_undegraded":"2024-08-17T18:14:37.430412+0000","last_fullsized":"2024-08-17T18:14:37.430412+0000","mapping_epoch":1380,"log_start":"1386'3300","ondisk_log_start":"1386'3300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4315,"log_dups_size":0,"ondisk_log_size":4315,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:27:26.876827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394723328,"num_objects":6023,"num_object_clones":0,"num_object_copies":18069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7615,"num_write_kb":436416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.e","version":"1392'7586","reported_seq":7621,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:37.461295+0000","last_change":"2024-08-17T18:12:42.612293+0000","last_active":"2024-08-17T18:14:37.461295+0000","last_peered":"2024-08-17T18:14:37.461295+0000","last_clean":"2024-08-17T18:14:37.461295+0000","last_became_active":"2024-08-17T18:12:42.611981+0000","last_became_peered":"2024-08-17T18:12:42.611981+0000","last_unstale":"2024-08-17T18:14:37.461295+0000","last_undegraded":"2024-08-17T18:14:37.461295+0000","last_fullsized":"2024-08-17T18:14:37.461295+0000","mapping_epoch":1380,"log_start":"1386'3300","ondisk_log_start":"1386'3300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4286,"log_dups_size":0,"ondisk_log_size":4286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:54:16.727653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390463488,"num_objects":5958,"num_object_clones":0,"num_object_copies":17874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7586,"num_write_kb":433408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.d","version":"1392'7686","reported_seq":7722,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:37.459328+0000","last_change":"2024-08-17T18:12:42.192342+0000","last_active":"2024-08-17T18:14:37.459328+0000","last_peered":"2024-08-17T18:14:37.459328+0000","last_clean":"2024-08-17T18:14:37.459328+0000","last_became_active":"2024-08-17T18:12:42.191931+0000","last_became_peered":"2024-08-17T18:12:42.191931+0000","last_unstale":"2024-08-17T18:14:37.459328+0000","last_undegraded":"2024-08-17T18:14:37.459328+0000","last_fullsized":"2024-08-17T18:14:37.459328+0000","mapping_epoch":1380,"log_start":"1386'3400","ondisk_log_start":"1386'3400","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4286,"log_dups_size":0,"ondisk_log_size":4286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:03.046092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396296216,"num_objects":6048,"num_object_clones":0,"num_object_copies":18144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6048,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7686,"num_write_kb":439425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.c","version":"1392'7603","reported_seq":7638,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:37.465595+0000","last_change":"2024-08-17T18:12:42.613558+0000","last_active":"2024-08-17T18:14:37.465595+0000","last_peered":"2024-08-17T18:14:37.465595+0000","last_clean":"2024-08-17T18:14:37.465595+0000","last_became_active":"2024-08-17T18:12:42.613083+0000","last_became_peered":"2024-08-17T18:12:42.613083+0000","last_unstale":"2024-08-17T18:14:37.465595+0000","last_undegraded":"2024-08-17T18:14:37.465595+0000","last_fullsized":"2024-08-17T18:14:37.465595+0000","mapping_epoch":1380,"log_start":"1386'3300","ondisk_log_start":"1386'3300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4303,"log_dups_size":0,"ondisk_log_size":4303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:54.163537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390922240,"num_objects":5965,"num_object_clones":0,"num_object_copies":17895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7603,"num_write_kb":434176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.b","version":"1392'7510","reported_seq":7545,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:37.498417+0000","last_change":"2024-08-17T18:12:42.616433+0000","last_active":"2024-08-17T18:14:37.498417+0000","last_peered":"2024-08-17T18:14:37.498417+0000","last_clean":"2024-08-17T18:14:37.498417+0000","last_became_active":"2024-08-17T18:12:42.615691+0000","last_became_peered":"2024-08-17T18:12:42.615691+0000","last_unstale":"2024-08-17T18:14:37.498417+0000","last_undegraded":"2024-08-17T18:14:37.498417+0000","last_fullsized":"2024-08-17T18:14:37.498417+0000","mapping_epoch":1380,"log_start":"1386'3200","ondisk_log_start":"1386'3200","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4310,"log_dups_size":0,"ondisk_log_size":4310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:56.333635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386007040,"num_objects":5890,"num_object_clones":0,"num_object_copies":17670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7510,"num_write_kb":428800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.a","version":"1392'7843","reported_seq":7862,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.166711+0000","last_change":"2024-08-17T18:12:42.193306+0000","last_active":"2024-08-17T18:14:38.166711+0000","last_peered":"2024-08-17T18:14:38.166711+0000","last_clean":"2024-08-17T18:14:38.166711+0000","last_became_active":"2024-08-17T18:12:42.192965+0000","last_became_peered":"2024-08-17T18:12:42.192965+0000","last_unstale":"2024-08-17T18:14:38.166711+0000","last_undegraded":"2024-08-17T18:14:38.166711+0000","last_fullsized":"2024-08-17T18:14:38.166711+0000","mapping_epoch":1380,"log_start":"1386'3700","ondisk_log_start":"1386'3700","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4143,"log_dups_size":0,"ondisk_log_size":4143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:44:20.694736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404422656,"num_objects":6171,"num_object_clones":0,"num_object_copies":18513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7843,"num_write_kb":448448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"49.9","version":"1392'7658","reported_seq":7678,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.152913+0000","last_change":"2024-08-17T18:12:42.612232+0000","last_active":"2024-08-17T18:14:38.152913+0000","last_peered":"2024-08-17T18:14:38.152913+0000","last_clean":"2024-08-17T18:14:38.152913+0000","last_became_active":"2024-08-17T18:12:42.611975+0000","last_became_peered":"2024-08-17T18:12:42.611975+0000","last_unstale":"2024-08-17T18:14:38.152913+0000","last_undegraded":"2024-08-17T18:14:38.152913+0000","last_fullsized":"2024-08-17T18:14:38.152913+0000","mapping_epoch":1380,"log_start":"1386'3500","ondisk_log_start":"1386'3500","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4158,"log_dups_size":0,"ondisk_log_size":4158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:31:26.661545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388366336,"num_objects":5926,"num_object_clones":0,"num_object_copies":17778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7658,"num_write_kb":434688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"49.8","version":"1392'7626","reported_seq":7646,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.172856+0000","last_change":"2024-08-17T18:12:42.613466+0000","last_active":"2024-08-17T18:14:38.172856+0000","last_peered":"2024-08-17T18:14:38.172856+0000","last_clean":"2024-08-17T18:14:38.172856+0000","last_became_active":"2024-08-17T18:12:42.613267+0000","last_became_peered":"2024-08-17T18:12:42.613267+0000","last_unstale":"2024-08-17T18:14:38.172856+0000","last_undegraded":"2024-08-17T18:14:38.172856+0000","last_fullsized":"2024-08-17T18:14:38.172856+0000","mapping_epoch":1380,"log_start":"1386'3500","ondisk_log_start":"1386'3500","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4126,"log_dups_size":0,"ondisk_log_size":4126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:55.951743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386400256,"num_objects":5896,"num_object_clones":0,"num_object_copies":17688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5896,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7626,"num_write_kb":432704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"49.7","version":"1393'7796","reported_seq":7832,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:39.304105+0000","last_change":"2024-08-17T18:12:42.616617+0000","last_active":"2024-08-17T18:14:39.304105+0000","last_peered":"2024-08-17T18:14:39.304105+0000","last_clean":"2024-08-17T18:14:39.304105+0000","last_became_active":"2024-08-17T18:12:42.615862+0000","last_became_peered":"2024-08-17T18:12:42.615862+0000","last_unstale":"2024-08-17T18:14:39.304105+0000","last_undegraded":"2024-08-17T18:14:39.304105+0000","last_fullsized":"2024-08-17T18:14:39.304105+0000","mapping_epoch":1380,"log_start":"1385'2900","ondisk_log_start":"1385'2900","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4896,"log_dups_size":0,"ondisk_log_size":4896,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:13.095768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388759552,"num_objects":5932,"num_object_clones":0,"num_object_copies":17796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7796,"num_write_kb":439296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"49.6","version":"1392'7604","reported_seq":7639,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:37.466237+0000","last_change":"2024-08-17T18:12:42.616079+0000","last_active":"2024-08-17T18:14:37.466237+0000","last_peered":"2024-08-17T18:14:37.466237+0000","last_clean":"2024-08-17T18:14:37.466237+0000","last_became_active":"2024-08-17T18:12:42.615724+0000","last_became_peered":"2024-08-17T18:12:42.615724+0000","last_unstale":"2024-08-17T18:14:37.466237+0000","last_undegraded":"2024-08-17T18:14:37.466237+0000","last_fullsized":"2024-08-17T18:14:37.466237+0000","mapping_epoch":1380,"log_start":"1386'3300","ondisk_log_start":"1386'3300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4304,"log_dups_size":0,"ondisk_log_size":4304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:29.102069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394264576,"num_objects":6016,"num_object_clones":0,"num_object_copies":18048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7604,"num_write_kb":435840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.5","version":"1392'7680","reported_seq":7699,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.159581+0000","last_change":"2024-08-17T18:12:42.614361+0000","last_active":"2024-08-17T18:14:38.159581+0000","last_peered":"2024-08-17T18:14:38.159581+0000","last_clean":"2024-08-17T18:14:38.159581+0000","last_became_active":"2024-08-17T18:12:42.614132+0000","last_became_peered":"2024-08-17T18:12:42.614132+0000","last_unstale":"2024-08-17T18:14:38.159581+0000","last_undegraded":"2024-08-17T18:14:38.159581+0000","last_fullsized":"2024-08-17T18:14:38.159581+0000","mapping_epoch":1380,"log_start":"1386'3500","ondisk_log_start":"1386'3500","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4180,"log_dups_size":0,"ondisk_log_size":4180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:47:41.211349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387055616,"num_objects":5906,"num_object_clones":0,"num_object_copies":17718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7680,"num_write_kb":434752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"49.4","version":"1392'7604","reported_seq":7639,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:37.462106+0000","last_change":"2024-08-17T18:12:42.613238+0000","last_active":"2024-08-17T18:14:37.462106+0000","last_peered":"2024-08-17T18:14:37.462106+0000","last_clean":"2024-08-17T18:14:37.462106+0000","last_became_active":"2024-08-17T18:12:42.612693+0000","last_became_peered":"2024-08-17T18:12:42.612693+0000","last_unstale":"2024-08-17T18:14:37.462106+0000","last_undegraded":"2024-08-17T18:14:37.462106+0000","last_fullsized":"2024-08-17T18:14:37.462106+0000","mapping_epoch":1380,"log_start":"1386'3300","ondisk_log_start":"1386'3300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4304,"log_dups_size":0,"ondisk_log_size":4304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:03.449538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396623872,"num_objects":6052,"num_object_clones":0,"num_object_copies":18156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7604,"num_write_kb":436992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.3","version":"1393'7744","reported_seq":7781,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:39.285676+0000","last_change":"2024-08-17T18:12:42.617312+0000","last_active":"2024-08-17T18:14:39.285676+0000","last_peered":"2024-08-17T18:14:39.285676+0000","last_clean":"2024-08-17T18:14:39.285676+0000","last_became_active":"2024-08-17T18:12:42.616565+0000","last_became_peered":"2024-08-17T18:12:42.616565+0000","last_unstale":"2024-08-17T18:14:39.285676+0000","last_undegraded":"2024-08-17T18:14:39.285676+0000","last_fullsized":"2024-08-17T18:14:39.285676+0000","mapping_epoch":1380,"log_start":"1385'2900","ondisk_log_start":"1385'2900","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4844,"log_dups_size":0,"ondisk_log_size":4844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:00:24.331362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391380992,"num_objects":5972,"num_object_clones":0,"num_object_copies":17916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7744,"num_write_kb":438912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"49.2","version":"1392'7591","reported_seq":7626,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:37.474510+0000","last_change":"2024-08-17T18:12:42.613766+0000","last_active":"2024-08-17T18:14:37.474510+0000","last_peered":"2024-08-17T18:14:37.474510+0000","last_clean":"2024-08-17T18:14:37.474510+0000","last_became_active":"2024-08-17T18:12:42.613426+0000","last_became_peered":"2024-08-17T18:12:42.613426+0000","last_unstale":"2024-08-17T18:14:37.474510+0000","last_undegraded":"2024-08-17T18:14:37.474510+0000","last_fullsized":"2024-08-17T18:14:37.474510+0000","mapping_epoch":1380,"log_start":"1386'3300","ondisk_log_start":"1386'3300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4291,"log_dups_size":0,"ondisk_log_size":4291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:19:49.367275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395509760,"num_objects":6035,"num_object_clones":0,"num_object_copies":18105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7591,"num_write_kb":436032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"49.1","version":"1393'7856","reported_seq":7893,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:39.309515+0000","last_change":"2024-08-17T18:12:42.191991+0000","last_active":"2024-08-17T18:14:39.309515+0000","last_peered":"2024-08-17T18:14:39.309515+0000","last_clean":"2024-08-17T18:14:39.309515+0000","last_became_active":"2024-08-17T18:12:42.191700+0000","last_became_peered":"2024-08-17T18:12:42.191700+0000","last_unstale":"2024-08-17T18:14:39.309515+0000","last_undegraded":"2024-08-17T18:14:39.309515+0000","last_fullsized":"2024-08-17T18:14:39.309515+0000","mapping_epoch":1380,"log_start":"1385'3000","ondisk_log_start":"1385'3000","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4856,"log_dups_size":0,"ondisk_log_size":4856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:58:11.841245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388104192,"num_objects":5922,"num_object_clones":0,"num_object_copies":17766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7856,"num_write_kb":440896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"49.0","version":"1392'7522","reported_seq":7557,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:37.462294+0000","last_change":"2024-08-17T18:12:42.612356+0000","last_active":"2024-08-17T18:14:37.462294+0000","last_peered":"2024-08-17T18:14:37.462294+0000","last_clean":"2024-08-17T18:14:37.462294+0000","last_became_active":"2024-08-17T18:12:42.612054+0000","last_became_peered":"2024-08-17T18:12:42.612054+0000","last_unstale":"2024-08-17T18:14:37.462294+0000","last_undegraded":"2024-08-17T18:14:37.462294+0000","last_fullsized":"2024-08-17T18:14:37.462294+0000","mapping_epoch":1380,"log_start":"1386'3200","ondisk_log_start":"1386'3200","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:12:41.169621+0000","last_clean_scrub_stamp":"2024-08-17T18:12:41.169621+0000","objects_scrubbed":0,"log_size":4322,"log_dups_size":0,"ondisk_log_size":4322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:49.825180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395051008,"num_objects":6028,"num_object_clones":0,"num_object_copies":18084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7522,"num_write_kb":433600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":2276,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.841095+0000","last_change":"2024-08-17T18:09:28.870033+0000","last_active":"2024-08-17T18:14:38.841095+0000","last_peered":"2024-08-17T18:14:38.841095+0000","last_clean":"2024-08-17T18:14:38.841095+0000","last_became_active":"2024-08-17T18:09:28.869346+0000","last_became_peered":"2024-08-17T18:09:28.869346+0000","last_unstale":"2024-08-17T18:14:38.841095+0000","last_undegraded":"2024-08-17T18:14:38.841095+0000","last_fullsized":"2024-08-17T18:14:38.841095+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:23.168631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1140,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":569,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402522+0000","last_change":"2024-08-17T18:09:29.841584+0000","last_active":"2024-08-17T18:14:19.402522+0000","last_peered":"2024-08-17T18:14:19.402522+0000","last_clean":"2024-08-17T18:14:19.402522+0000","last_became_active":"2024-08-17T18:09:29.840950+0000","last_became_peered":"2024-08-17T18:09:29.840950+0000","last_unstale":"2024-08-17T18:14:19.402522+0000","last_undegraded":"2024-08-17T18:14:19.402522+0000","last_fullsized":"2024-08-17T18:14:19.402522+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:43.035959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":2064,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.403138+0000","last_change":"2024-08-17T18:09:29.883587+0000","last_active":"2024-08-17T18:14:19.403138+0000","last_peered":"2024-08-17T18:14:19.403138+0000","last_clean":"2024-08-17T18:14:19.403138+0000","last_became_active":"2024-08-17T18:09:29.882222+0000","last_became_peered":"2024-08-17T18:09:29.882222+0000","last_unstale":"2024-08-17T18:14:19.403138+0000","last_undegraded":"2024-08-17T18:14:19.403138+0000","last_fullsized":"2024-08-17T18:14:19.403138+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:10.777993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2121,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402976+0000","last_change":"2024-08-17T18:09:28.858824+0000","last_active":"2024-08-17T18:14:19.402976+0000","last_peered":"2024-08-17T18:14:19.402976+0000","last_clean":"2024-08-17T18:14:19.402976+0000","last_became_active":"2024-08-17T18:09:28.857856+0000","last_became_peered":"2024-08-17T18:09:28.857856+0000","last_unstale":"2024-08-17T18:14:19.402976+0000","last_undegraded":"2024-08-17T18:14:19.402976+0000","last_fullsized":"2024-08-17T18:14:19.402976+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:41:35.146166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":125,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402452+0000","last_change":"2024-08-17T18:09:29.851515+0000","last_active":"2024-08-17T18:14:19.402452+0000","last_peered":"2024-08-17T18:14:19.402452+0000","last_clean":"2024-08-17T18:14:19.402452+0000","last_became_active":"2024-08-17T18:09:29.851109+0000","last_became_peered":"2024-08-17T18:09:29.851109+0000","last_unstale":"2024-08-17T18:14:19.402452+0000","last_undegraded":"2024-08-17T18:14:19.402452+0000","last_fullsized":"2024-08-17T18:14:19.402452+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:42.054532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":104,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.376207+0000","last_change":"2024-08-17T18:03:16.089811+0000","last_active":"2024-08-17T18:14:16.376207+0000","last_peered":"2024-08-17T18:14:16.376207+0000","last_clean":"2024-08-17T18:14:16.376207+0000","last_became_active":"2024-08-17T18:03:16.089571+0000","last_became_peered":"2024-08-17T18:03:16.089571+0000","last_unstale":"2024-08-17T18:14:16.376207+0000","last_undegraded":"2024-08-17T18:14:16.376207+0000","last_fullsized":"2024-08-17T18:14:16.376207+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:04.630361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":86,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374162+0000","last_change":"2024-08-17T18:08:57.577055+0000","last_active":"2024-08-17T18:14:16.374162+0000","last_peered":"2024-08-17T18:14:16.374162+0000","last_clean":"2024-08-17T18:14:16.374162+0000","last_became_active":"2024-08-17T18:08:57.573659+0000","last_became_peered":"2024-08-17T18:08:57.573659+0000","last_unstale":"2024-08-17T18:14:16.374162+0000","last_undegraded":"2024-08-17T18:14:16.374162+0000","last_fullsized":"2024-08-17T18:14:16.374162+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:01.686752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":909,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402178+0000","last_change":"2024-08-17T18:09:29.852393+0000","last_active":"2024-08-17T18:14:19.402178+0000","last_peered":"2024-08-17T18:14:19.402178+0000","last_clean":"2024-08-17T18:14:19.402178+0000","last_became_active":"2024-08-17T18:09:29.851947+0000","last_became_peered":"2024-08-17T18:09:29.851947+0000","last_unstale":"2024-08-17T18:14:19.402178+0000","last_undegraded":"2024-08-17T18:14:19.402178+0000","last_fullsized":"2024-08-17T18:14:19.402178+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:49.145327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":2221,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.840605+0000","last_change":"2024-08-17T18:09:29.863241+0000","last_active":"2024-08-17T18:14:38.840605+0000","last_peered":"2024-08-17T18:14:38.840605+0000","last_clean":"2024-08-17T18:14:38.840605+0000","last_became_active":"2024-08-17T18:09:29.862369+0000","last_became_peered":"2024-08-17T18:09:29.862369+0000","last_unstale":"2024-08-17T18:14:38.840605+0000","last_undegraded":"2024-08-17T18:14:38.840605+0000","last_fullsized":"2024-08-17T18:14:38.840605+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:30.518551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":656,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.411198+0000","last_change":"2024-08-17T18:09:29.896233+0000","last_active":"2024-08-17T18:14:19.411198+0000","last_peered":"2024-08-17T18:14:19.411198+0000","last_clean":"2024-08-17T18:14:19.411198+0000","last_became_active":"2024-08-17T18:09:29.895718+0000","last_became_peered":"2024-08-17T18:09:29.895718+0000","last_unstale":"2024-08-17T18:14:19.411198+0000","last_undegraded":"2024-08-17T18:14:19.411198+0000","last_fullsized":"2024-08-17T18:14:19.411198+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:10:03.278878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":131,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.841412+0000","last_change":"2024-08-17T18:08:57.574015+0000","last_active":"2024-08-17T18:14:38.841412+0000","last_peered":"2024-08-17T18:14:38.841412+0000","last_clean":"2024-08-17T18:14:38.841412+0000","last_became_active":"2024-08-17T18:08:57.567973+0000","last_became_peered":"2024-08-17T18:08:57.567973+0000","last_unstale":"2024-08-17T18:14:38.841412+0000","last_undegraded":"2024-08-17T18:14:38.841412+0000","last_fullsized":"2024-08-17T18:14:38.841412+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:06.251764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2076,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402511+0000","last_change":"2024-08-17T18:09:29.917203+0000","last_active":"2024-08-17T18:14:19.402511+0000","last_peered":"2024-08-17T18:14:19.402511+0000","last_clean":"2024-08-17T18:14:19.402511+0000","last_became_active":"2024-08-17T18:09:29.916844+0000","last_became_peered":"2024-08-17T18:09:29.916844+0000","last_unstale":"2024-08-17T18:14:19.402511+0000","last_undegraded":"2024-08-17T18:14:19.402511+0000","last_fullsized":"2024-08-17T18:14:19.402511+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:18.453720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1982,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402810+0000","last_change":"2024-08-17T18:09:28.859255+0000","last_active":"2024-08-17T18:14:19.402810+0000","last_peered":"2024-08-17T18:14:19.402810+0000","last_clean":"2024-08-17T18:14:19.402810+0000","last_became_active":"2024-08-17T18:09:28.858537+0000","last_became_peered":"2024-08-17T18:09:28.858537+0000","last_unstale":"2024-08-17T18:14:19.402810+0000","last_undegraded":"2024-08-17T18:14:19.402810+0000","last_fullsized":"2024-08-17T18:14:19.402810+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:25:15.364687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2244,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.840634+0000","last_change":"2024-08-17T18:09:28.868303+0000","last_active":"2024-08-17T18:14:38.840634+0000","last_peered":"2024-08-17T18:14:38.840634+0000","last_clean":"2024-08-17T18:14:38.840634+0000","last_became_active":"2024-08-17T18:09:28.867989+0000","last_became_peered":"2024-08-17T18:09:28.867989+0000","last_unstale":"2024-08-17T18:14:38.840634+0000","last_undegraded":"2024-08-17T18:14:38.840634+0000","last_fullsized":"2024-08-17T18:14:38.840634+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:58:52.741685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":108,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402778+0000","last_change":"2024-08-17T18:09:29.842129+0000","last_active":"2024-08-17T18:14:19.402778+0000","last_peered":"2024-08-17T18:14:19.402778+0000","last_clean":"2024-08-17T18:14:19.402778+0000","last_became_active":"2024-08-17T18:09:29.841848+0000","last_became_peered":"2024-08-17T18:09:29.841848+0000","last_unstale":"2024-08-17T18:14:19.402778+0000","last_undegraded":"2024-08-17T18:14:19.402778+0000","last_fullsized":"2024-08-17T18:14:19.402778+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:02.119134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1358,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.840902+0000","last_change":"2024-08-17T18:09:29.846509+0000","last_active":"2024-08-17T18:14:38.840902+0000","last_peered":"2024-08-17T18:14:38.840902+0000","last_clean":"2024-08-17T18:14:38.840902+0000","last_became_active":"2024-08-17T18:09:29.844097+0000","last_became_peered":"2024-08-17T18:09:29.844097+0000","last_unstale":"2024-08-17T18:14:38.840902+0000","last_undegraded":"2024-08-17T18:14:38.840902+0000","last_fullsized":"2024-08-17T18:14:38.840902+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:32.340354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2070,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402334+0000","last_change":"2024-08-17T18:09:29.841335+0000","last_active":"2024-08-17T18:14:19.402334+0000","last_peered":"2024-08-17T18:14:19.402334+0000","last_clean":"2024-08-17T18:14:19.402334+0000","last_became_active":"2024-08-17T18:09:29.840522+0000","last_became_peered":"2024-08-17T18:09:29.840522+0000","last_unstale":"2024-08-17T18:14:19.402334+0000","last_undegraded":"2024-08-17T18:14:19.402334+0000","last_fullsized":"2024-08-17T18:14:19.402334+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:53:08.015836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2249,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.841384+0000","last_change":"2024-08-17T18:09:29.847494+0000","last_active":"2024-08-17T18:14:38.841384+0000","last_peered":"2024-08-17T18:14:38.841384+0000","last_clean":"2024-08-17T18:14:38.841384+0000","last_became_active":"2024-08-17T18:09:29.846973+0000","last_became_peered":"2024-08-17T18:09:29.846973+0000","last_unstale":"2024-08-17T18:14:38.841384+0000","last_undegraded":"2024-08-17T18:14:38.841384+0000","last_fullsized":"2024-08-17T18:14:38.841384+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:49.469921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2029,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402610+0000","last_change":"2024-08-17T18:09:28.855045+0000","last_active":"2024-08-17T18:14:19.402610+0000","last_peered":"2024-08-17T18:14:19.402610+0000","last_clean":"2024-08-17T18:14:19.402610+0000","last_became_active":"2024-08-17T18:09:28.854607+0000","last_became_peered":"2024-08-17T18:09:28.854607+0000","last_unstale":"2024-08-17T18:14:19.402610+0000","last_undegraded":"2024-08-17T18:14:19.402610+0000","last_fullsized":"2024-08-17T18:14:19.402610+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:13.437895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":716,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.376604+0000","last_change":"2024-08-17T18:09:29.866944+0000","last_active":"2024-08-17T18:14:16.376604+0000","last_peered":"2024-08-17T18:14:16.376604+0000","last_clean":"2024-08-17T18:14:16.376604+0000","last_became_active":"2024-08-17T18:09:29.866694+0000","last_became_peered":"2024-08-17T18:09:29.866694+0000","last_unstale":"2024-08-17T18:14:16.376604+0000","last_undegraded":"2024-08-17T18:14:16.376604+0000","last_fullsized":"2024-08-17T18:14:16.376604+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:00:41.342392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1379,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.375025+0000","last_change":"2024-08-17T18:09:28.884262+0000","last_active":"2024-08-17T18:14:16.375025+0000","last_peered":"2024-08-17T18:14:16.375025+0000","last_clean":"2024-08-17T18:14:16.375025+0000","last_became_active":"2024-08-17T18:09:28.883734+0000","last_became_peered":"2024-08-17T18:09:28.883734+0000","last_unstale":"2024-08-17T18:14:16.375025+0000","last_undegraded":"2024-08-17T18:14:16.375025+0000","last_fullsized":"2024-08-17T18:14:16.375025+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:10.735504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":2134,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.841271+0000","last_change":"2024-08-17T18:08:57.571995+0000","last_active":"2024-08-17T18:14:38.841271+0000","last_peered":"2024-08-17T18:14:38.841271+0000","last_clean":"2024-08-17T18:14:38.841271+0000","last_became_active":"2024-08-17T18:08:57.567342+0000","last_became_peered":"2024-08-17T18:08:57.567342+0000","last_unstale":"2024-08-17T18:14:38.841271+0000","last_undegraded":"2024-08-17T18:14:38.841271+0000","last_fullsized":"2024-08-17T18:14:38.841271+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:23.769111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":543,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374251+0000","last_change":"2024-08-17T18:02:53.912837+0000","last_active":"2024-08-17T18:14:16.374251+0000","last_peered":"2024-08-17T18:14:16.374251+0000","last_clean":"2024-08-17T18:14:16.374251+0000","last_became_active":"2024-08-17T18:02:53.883152+0000","last_became_peered":"2024-08-17T18:02:53.883152+0000","last_unstale":"2024-08-17T18:14:16.374251+0000","last_undegraded":"2024-08-17T18:14:16.374251+0000","last_fullsized":"2024-08-17T18:14:16.374251+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:46.261612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":542,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374099+0000","last_change":"2024-08-17T18:09:29.845126+0000","last_active":"2024-08-17T18:14:16.374099+0000","last_peered":"2024-08-17T18:14:16.374099+0000","last_clean":"2024-08-17T18:14:16.374099+0000","last_became_active":"2024-08-17T18:09:29.844836+0000","last_became_peered":"2024-08-17T18:09:29.844836+0000","last_unstale":"2024-08-17T18:14:16.374099+0000","last_undegraded":"2024-08-17T18:14:16.374099+0000","last_fullsized":"2024-08-17T18:14:16.374099+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:44.821894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1371,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.843531+0000","last_change":"2024-08-17T18:02:53.890198+0000","last_active":"2024-08-17T18:14:38.843531+0000","last_peered":"2024-08-17T18:14:38.843531+0000","last_clean":"2024-08-17T18:14:38.843531+0000","last_became_active":"2024-08-17T18:02:53.887007+0000","last_became_peered":"2024-08-17T18:02:53.887007+0000","last_unstale":"2024-08-17T18:14:38.843531+0000","last_undegraded":"2024-08-17T18:14:38.843531+0000","last_fullsized":"2024-08-17T18:14:38.843531+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:25.555909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":633,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374720+0000","last_change":"2024-08-17T18:09:28.884187+0000","last_active":"2024-08-17T18:14:16.374720+0000","last_peered":"2024-08-17T18:14:16.374720+0000","last_clean":"2024-08-17T18:14:16.374720+0000","last_became_active":"2024-08-17T18:09:28.883939+0000","last_became_peered":"2024-08-17T18:09:28.883939+0000","last_unstale":"2024-08-17T18:14:16.374720+0000","last_undegraded":"2024-08-17T18:14:16.374720+0000","last_fullsized":"2024-08-17T18:14:16.374720+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:37:41.951099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":1933,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.376561+0000","last_change":"2024-08-17T18:09:29.893220+0000","last_active":"2024-08-17T18:14:16.376561+0000","last_peered":"2024-08-17T18:14:16.376561+0000","last_clean":"2024-08-17T18:14:16.376561+0000","last_became_active":"2024-08-17T18:09:29.865699+0000","last_became_peered":"2024-08-17T18:09:29.865699+0000","last_unstale":"2024-08-17T18:14:16.376561+0000","last_undegraded":"2024-08-17T18:14:16.376561+0000","last_fullsized":"2024-08-17T18:14:16.376561+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:40.205658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2089,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402644+0000","last_change":"2024-08-17T18:09:28.934404+0000","last_active":"2024-08-17T18:14:19.402644+0000","last_peered":"2024-08-17T18:14:19.402644+0000","last_clean":"2024-08-17T18:14:19.402644+0000","last_became_active":"2024-08-17T18:09:28.934001+0000","last_became_peered":"2024-08-17T18:09:28.934001+0000","last_unstale":"2024-08-17T18:14:19.402644+0000","last_undegraded":"2024-08-17T18:14:19.402644+0000","last_fullsized":"2024-08-17T18:14:19.402644+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:04.676812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":2121,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.403322+0000","last_change":"2024-08-17T18:09:29.884088+0000","last_active":"2024-08-17T18:14:19.403322+0000","last_peered":"2024-08-17T18:14:19.403322+0000","last_clean":"2024-08-17T18:14:19.403322+0000","last_became_active":"2024-08-17T18:09:29.883027+0000","last_became_peered":"2024-08-17T18:09:29.883027+0000","last_unstale":"2024-08-17T18:14:19.403322+0000","last_undegraded":"2024-08-17T18:14:19.403322+0000","last_fullsized":"2024-08-17T18:14:19.403322+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:22:07.777188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":2091,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.405932+0000","last_change":"2024-08-17T18:09:28.935401+0000","last_active":"2024-08-17T18:14:19.405932+0000","last_peered":"2024-08-17T18:14:19.405932+0000","last_clean":"2024-08-17T18:14:19.405932+0000","last_became_active":"2024-08-17T18:09:28.933628+0000","last_became_peered":"2024-08-17T18:09:28.933628+0000","last_unstale":"2024-08-17T18:14:19.405932+0000","last_undegraded":"2024-08-17T18:14:19.405932+0000","last_fullsized":"2024-08-17T18:14:19.405932+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:21:12.714486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2205,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.840703+0000","last_change":"2024-08-17T18:08:57.574718+0000","last_active":"2024-08-17T18:14:38.840703+0000","last_peered":"2024-08-17T18:14:38.840703+0000","last_clean":"2024-08-17T18:14:38.840703+0000","last_became_active":"2024-08-17T18:08:57.573515+0000","last_became_peered":"2024-08-17T18:08:57.573515+0000","last_unstale":"2024-08-17T18:14:38.840703+0000","last_undegraded":"2024-08-17T18:14:38.840703+0000","last_fullsized":"2024-08-17T18:14:38.840703+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:04.873717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1392,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.375154+0000","last_change":"2024-08-17T18:08:57.568821+0000","last_active":"2024-08-17T18:14:16.375154+0000","last_peered":"2024-08-17T18:14:16.375154+0000","last_clean":"2024-08-17T18:14:16.375154+0000","last_became_active":"2024-08-17T18:08:57.568526+0000","last_became_peered":"2024-08-17T18:08:57.568526+0000","last_unstale":"2024-08-17T18:14:16.375154+0000","last_undegraded":"2024-08-17T18:14:16.375154+0000","last_fullsized":"2024-08-17T18:14:16.375154+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:53:26.449197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1925,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.843682+0000","last_change":"2024-08-17T18:08:57.571174+0000","last_active":"2024-08-17T18:14:38.843682+0000","last_peered":"2024-08-17T18:14:38.843682+0000","last_clean":"2024-08-17T18:14:38.843682+0000","last_became_active":"2024-08-17T18:08:57.567571+0000","last_became_peered":"2024-08-17T18:08:57.567571+0000","last_unstale":"2024-08-17T18:14:38.843682+0000","last_undegraded":"2024-08-17T18:14:38.843682+0000","last_fullsized":"2024-08-17T18:14:38.843682+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:29.976150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1894,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.841501+0000","last_change":"2024-08-17T18:08:57.572368+0000","last_active":"2024-08-17T18:14:38.841501+0000","last_peered":"2024-08-17T18:14:38.841501+0000","last_clean":"2024-08-17T18:14:38.841501+0000","last_became_active":"2024-08-17T18:08:57.568382+0000","last_became_peered":"2024-08-17T18:08:57.568382+0000","last_unstale":"2024-08-17T18:14:38.841501+0000","last_undegraded":"2024-08-17T18:14:38.841501+0000","last_fullsized":"2024-08-17T18:14:38.841501+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:11:19.968079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1942,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.843767+0000","last_change":"2024-08-17T18:08:57.572991+0000","last_active":"2024-08-17T18:14:38.843767+0000","last_peered":"2024-08-17T18:14:38.843767+0000","last_clean":"2024-08-17T18:14:38.843767+0000","last_became_active":"2024-08-17T18:08:57.571145+0000","last_became_peered":"2024-08-17T18:08:57.571145+0000","last_unstale":"2024-08-17T18:14:38.843767+0000","last_undegraded":"2024-08-17T18:14:38.843767+0000","last_fullsized":"2024-08-17T18:14:38.843767+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:39.963337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":896,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402384+0000","last_change":"2024-08-17T18:09:28.860288+0000","last_active":"2024-08-17T18:14:19.402384+0000","last_peered":"2024-08-17T18:14:19.402384+0000","last_clean":"2024-08-17T18:14:19.402384+0000","last_became_active":"2024-08-17T18:09:28.860004+0000","last_became_peered":"2024-08-17T18:09:28.860004+0000","last_unstale":"2024-08-17T18:14:19.402384+0000","last_undegraded":"2024-08-17T18:14:19.402384+0000","last_fullsized":"2024-08-17T18:14:19.402384+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:57:33.054055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":689,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.411374+0000","last_change":"2024-08-17T18:09:28.860277+0000","last_active":"2024-08-17T18:14:19.411374+0000","last_peered":"2024-08-17T18:14:19.411374+0000","last_clean":"2024-08-17T18:14:19.411374+0000","last_became_active":"2024-08-17T18:09:28.859999+0000","last_became_peered":"2024-08-17T18:09:28.859999+0000","last_unstale":"2024-08-17T18:14:19.411374+0000","last_undegraded":"2024-08-17T18:14:19.411374+0000","last_fullsized":"2024-08-17T18:14:19.411374+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:27.946526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1596,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.840302+0000","last_change":"2024-08-17T18:09:29.863669+0000","last_active":"2024-08-17T18:14:38.840302+0000","last_peered":"2024-08-17T18:14:38.840302+0000","last_clean":"2024-08-17T18:14:38.840302+0000","last_became_active":"2024-08-17T18:09:29.862781+0000","last_became_peered":"2024-08-17T18:09:29.862781+0000","last_unstale":"2024-08-17T18:14:38.840302+0000","last_undegraded":"2024-08-17T18:14:38.840302+0000","last_fullsized":"2024-08-17T18:14:38.840302+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:14.073233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":177,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.841586+0000","last_change":"2024-08-17T18:09:29.843574+0000","last_active":"2024-08-17T18:14:38.841586+0000","last_peered":"2024-08-17T18:14:38.841586+0000","last_clean":"2024-08-17T18:14:38.841586+0000","last_became_active":"2024-08-17T18:09:29.843097+0000","last_became_peered":"2024-08-17T18:09:29.843097+0000","last_unstale":"2024-08-17T18:14:38.841586+0000","last_undegraded":"2024-08-17T18:14:38.841586+0000","last_fullsized":"2024-08-17T18:14:38.841586+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:46.740888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1941,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.841598+0000","last_change":"2024-08-17T18:09:29.847264+0000","last_active":"2024-08-17T18:14:38.841598+0000","last_peered":"2024-08-17T18:14:38.841598+0000","last_clean":"2024-08-17T18:14:38.841598+0000","last_became_active":"2024-08-17T18:09:29.846226+0000","last_became_peered":"2024-08-17T18:09:29.846226+0000","last_unstale":"2024-08-17T18:14:38.841598+0000","last_undegraded":"2024-08-17T18:14:38.841598+0000","last_fullsized":"2024-08-17T18:14:38.841598+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:44.135977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2141,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.840192+0000","last_change":"2024-08-17T18:09:29.845209+0000","last_active":"2024-08-17T18:14:38.840192+0000","last_peered":"2024-08-17T18:14:38.840192+0000","last_clean":"2024-08-17T18:14:38.840192+0000","last_became_active":"2024-08-17T18:09:29.844910+0000","last_became_peered":"2024-08-17T18:09:29.844910+0000","last_unstale":"2024-08-17T18:14:38.840192+0000","last_undegraded":"2024-08-17T18:14:38.840192+0000","last_fullsized":"2024-08-17T18:14:38.840192+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:14.586354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2012,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402921+0000","last_change":"2024-08-17T18:09:28.859050+0000","last_active":"2024-08-17T18:14:19.402921+0000","last_peered":"2024-08-17T18:14:19.402921+0000","last_clean":"2024-08-17T18:14:19.402921+0000","last_became_active":"2024-08-17T18:09:28.858214+0000","last_became_peered":"2024-08-17T18:09:28.858214+0000","last_unstale":"2024-08-17T18:14:19.402921+0000","last_undegraded":"2024-08-17T18:14:19.402921+0000","last_fullsized":"2024-08-17T18:14:19.402921+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:54.146592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2266,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.840723+0000","last_change":"2024-08-17T18:08:57.572610+0000","last_active":"2024-08-17T18:14:38.840723+0000","last_peered":"2024-08-17T18:14:38.840723+0000","last_clean":"2024-08-17T18:14:38.840723+0000","last_became_active":"2024-08-17T18:08:57.572308+0000","last_became_peered":"2024-08-17T18:08:57.572308+0000","last_unstale":"2024-08-17T18:14:38.840723+0000","last_undegraded":"2024-08-17T18:14:38.840723+0000","last_fullsized":"2024-08-17T18:14:38.840723+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:11.642831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2250,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.843614+0000","last_change":"2024-08-17T18:09:28.842838+0000","last_active":"2024-08-17T18:14:38.843614+0000","last_peered":"2024-08-17T18:14:38.843614+0000","last_clean":"2024-08-17T18:14:38.843614+0000","last_became_active":"2024-08-17T18:09:28.842617+0000","last_became_peered":"2024-08-17T18:09:28.842617+0000","last_unstale":"2024-08-17T18:14:38.843614+0000","last_undegraded":"2024-08-17T18:14:38.843614+0000","last_fullsized":"2024-08-17T18:14:38.843614+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:00.016002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1413,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.377693+0000","last_change":"2024-08-17T18:09:29.845011+0000","last_active":"2024-08-17T18:14:16.377693+0000","last_peered":"2024-08-17T18:14:16.377693+0000","last_clean":"2024-08-17T18:14:16.377693+0000","last_became_active":"2024-08-17T18:09:29.844358+0000","last_became_peered":"2024-08-17T18:09:29.844358+0000","last_unstale":"2024-08-17T18:14:16.377693+0000","last_undegraded":"2024-08-17T18:14:16.377693+0000","last_fullsized":"2024-08-17T18:14:16.377693+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:41:27.792901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1261,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.375090+0000","last_change":"2024-08-17T18:09:28.878263+0000","last_active":"2024-08-17T18:14:16.375090+0000","last_peered":"2024-08-17T18:14:16.375090+0000","last_clean":"2024-08-17T18:14:16.375090+0000","last_became_active":"2024-08-17T18:09:28.878025+0000","last_became_peered":"2024-08-17T18:09:28.878025+0000","last_unstale":"2024-08-17T18:14:16.375090+0000","last_undegraded":"2024-08-17T18:14:16.375090+0000","last_fullsized":"2024-08-17T18:14:16.375090+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:56:37.211100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2040,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.410591+0000","last_change":"2024-08-17T18:09:29.840134+0000","last_active":"2024-08-17T18:14:19.410591+0000","last_peered":"2024-08-17T18:14:19.410591+0000","last_clean":"2024-08-17T18:14:19.410591+0000","last_became_active":"2024-08-17T18:09:29.839848+0000","last_became_peered":"2024-08-17T18:09:29.839848+0000","last_unstale":"2024-08-17T18:14:19.410591+0000","last_undegraded":"2024-08-17T18:14:19.410591+0000","last_fullsized":"2024-08-17T18:14:19.410591+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:08:08.717851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":168,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374367+0000","last_change":"2024-08-17T18:09:28.847633+0000","last_active":"2024-08-17T18:14:16.374367+0000","last_peered":"2024-08-17T18:14:16.374367+0000","last_clean":"2024-08-17T18:14:16.374367+0000","last_became_active":"2024-08-17T18:09:28.847341+0000","last_became_peered":"2024-08-17T18:09:28.847341+0000","last_unstale":"2024-08-17T18:14:16.374367+0000","last_undegraded":"2024-08-17T18:14:16.374367+0000","last_fullsized":"2024-08-17T18:14:16.374367+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:51:59.625154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1753,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.377648+0000","last_change":"2024-08-17T18:09:00.609431+0000","last_active":"2024-08-17T18:14:16.377648+0000","last_peered":"2024-08-17T18:14:16.377648+0000","last_clean":"2024-08-17T18:14:16.377648+0000","last_became_active":"2024-08-17T18:09:00.609167+0000","last_became_peered":"2024-08-17T18:09:00.609167+0000","last_unstale":"2024-08-17T18:14:16.377648+0000","last_undegraded":"2024-08-17T18:14:16.377648+0000","last_fullsized":"2024-08-17T18:14:16.377648+0000","mapping_epoch":1338,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:56:39.579970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1920,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.410582+0000","last_change":"2024-08-17T18:09:29.837502+0000","last_active":"2024-08-17T18:14:19.410582+0000","last_peered":"2024-08-17T18:14:19.410582+0000","last_clean":"2024-08-17T18:14:19.410582+0000","last_became_active":"2024-08-17T18:09:29.837243+0000","last_became_peered":"2024-08-17T18:09:29.837243+0000","last_unstale":"2024-08-17T18:14:19.410582+0000","last_undegraded":"2024-08-17T18:14:19.410582+0000","last_fullsized":"2024-08-17T18:14:19.410582+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:26:47.377505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2118,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402740+0000","last_change":"2024-08-17T18:09:29.859415+0000","last_active":"2024-08-17T18:14:19.402740+0000","last_peered":"2024-08-17T18:14:19.402740+0000","last_clean":"2024-08-17T18:14:19.402740+0000","last_became_active":"2024-08-17T18:09:29.859138+0000","last_became_peered":"2024-08-17T18:09:29.859138+0000","last_unstale":"2024-08-17T18:14:19.402740+0000","last_undegraded":"2024-08-17T18:14:19.402740+0000","last_fullsized":"2024-08-17T18:14:19.402740+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:31.178915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":569,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.403240+0000","last_change":"2024-08-17T18:09:29.883834+0000","last_active":"2024-08-17T18:14:19.403240+0000","last_peered":"2024-08-17T18:14:19.403240+0000","last_clean":"2024-08-17T18:14:19.403240+0000","last_became_active":"2024-08-17T18:09:29.882638+0000","last_became_peered":"2024-08-17T18:09:29.882638+0000","last_unstale":"2024-08-17T18:14:19.403240+0000","last_undegraded":"2024-08-17T18:14:19.403240+0000","last_fullsized":"2024-08-17T18:14:19.403240+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:07:10.188684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":653,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.373773+0000","last_change":"2024-08-17T18:09:29.857250+0000","last_active":"2024-08-17T18:14:16.373773+0000","last_peered":"2024-08-17T18:14:16.373773+0000","last_clean":"2024-08-17T18:14:16.373773+0000","last_became_active":"2024-08-17T18:09:29.857038+0000","last_became_peered":"2024-08-17T18:09:29.857038+0000","last_unstale":"2024-08-17T18:14:16.373773+0000","last_undegraded":"2024-08-17T18:14:16.373773+0000","last_fullsized":"2024-08-17T18:14:16.373773+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:02:54.028575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":684,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.842943+0000","last_change":"2024-08-17T18:08:57.571270+0000","last_active":"2024-08-17T18:14:38.842943+0000","last_peered":"2024-08-17T18:14:38.842943+0000","last_clean":"2024-08-17T18:14:38.842943+0000","last_became_active":"2024-08-17T18:08:57.567670+0000","last_became_peered":"2024-08-17T18:08:57.567670+0000","last_unstale":"2024-08-17T18:14:38.842943+0000","last_undegraded":"2024-08-17T18:14:38.842943+0000","last_fullsized":"2024-08-17T18:14:38.842943+0000","mapping_epoch":1335,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1336,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:19:42.617321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1253,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.410821+0000","last_change":"2024-08-17T18:09:29.839260+0000","last_active":"2024-08-17T18:14:19.410821+0000","last_peered":"2024-08-17T18:14:19.410821+0000","last_clean":"2024-08-17T18:14:19.410821+0000","last_became_active":"2024-08-17T18:09:29.838953+0000","last_became_peered":"2024-08-17T18:09:29.838953+0000","last_unstale":"2024-08-17T18:14:19.410821+0000","last_undegraded":"2024-08-17T18:14:19.410821+0000","last_fullsized":"2024-08-17T18:14:19.410821+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:44.014083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":842,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.411453+0000","last_change":"2024-08-17T18:09:29.851347+0000","last_active":"2024-08-17T18:14:19.411453+0000","last_peered":"2024-08-17T18:14:19.411453+0000","last_clean":"2024-08-17T18:14:19.411453+0000","last_became_active":"2024-08-17T18:09:29.851046+0000","last_became_peered":"2024-08-17T18:09:29.851046+0000","last_unstale":"2024-08-17T18:14:19.411453+0000","last_undegraded":"2024-08-17T18:14:19.411453+0000","last_fullsized":"2024-08-17T18:14:19.411453+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:21.350703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2119,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402038+0000","last_change":"2024-08-17T18:09:29.912446+0000","last_active":"2024-08-17T18:14:19.402038+0000","last_peered":"2024-08-17T18:14:19.402038+0000","last_clean":"2024-08-17T18:14:19.402038+0000","last_became_active":"2024-08-17T18:09:29.910703+0000","last_became_peered":"2024-08-17T18:09:29.910703+0000","last_unstale":"2024-08-17T18:14:19.402038+0000","last_undegraded":"2024-08-17T18:14:19.402038+0000","last_fullsized":"2024-08-17T18:14:19.402038+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:24.250608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2287,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.840799+0000","last_change":"2024-08-17T18:09:29.863361+0000","last_active":"2024-08-17T18:14:38.840799+0000","last_peered":"2024-08-17T18:14:38.840799+0000","last_clean":"2024-08-17T18:14:38.840799+0000","last_became_active":"2024-08-17T18:09:29.862503+0000","last_became_peered":"2024-08-17T18:09:29.862503+0000","last_unstale":"2024-08-17T18:14:38.840799+0000","last_undegraded":"2024-08-17T18:14:38.840799+0000","last_fullsized":"2024-08-17T18:14:38.840799+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:25.908232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":108,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.410751+0000","last_change":"2024-08-17T18:09:29.895991+0000","last_active":"2024-08-17T18:14:19.410751+0000","last_peered":"2024-08-17T18:14:19.410751+0000","last_clean":"2024-08-17T18:14:19.410751+0000","last_became_active":"2024-08-17T18:09:29.895274+0000","last_became_peered":"2024-08-17T18:09:29.895274+0000","last_unstale":"2024-08-17T18:14:19.410751+0000","last_undegraded":"2024-08-17T18:14:19.410751+0000","last_fullsized":"2024-08-17T18:14:19.410751+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:19:26.420016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":535,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.376281+0000","last_change":"2024-08-17T18:09:29.844686+0000","last_active":"2024-08-17T18:14:16.376281+0000","last_peered":"2024-08-17T18:14:16.376281+0000","last_clean":"2024-08-17T18:14:16.376281+0000","last_became_active":"2024-08-17T18:09:29.844341+0000","last_became_peered":"2024-08-17T18:09:29.844341+0000","last_unstale":"2024-08-17T18:14:16.376281+0000","last_undegraded":"2024-08-17T18:14:16.376281+0000","last_fullsized":"2024-08-17T18:14:16.376281+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:41:38.872767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1372,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.375135+0000","last_change":"2024-08-17T18:09:28.884310+0000","last_active":"2024-08-17T18:14:16.375135+0000","last_peered":"2024-08-17T18:14:16.375135+0000","last_clean":"2024-08-17T18:14:16.375135+0000","last_became_active":"2024-08-17T18:09:28.883921+0000","last_became_peered":"2024-08-17T18:09:28.883921+0000","last_unstale":"2024-08-17T18:14:16.375135+0000","last_undegraded":"2024-08-17T18:14:16.375135+0000","last_fullsized":"2024-08-17T18:14:16.375135+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:05.272563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":851,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.402889+0000","last_change":"2024-08-17T18:09:29.837238+0000","last_active":"2024-08-17T18:14:19.402889+0000","last_peered":"2024-08-17T18:14:19.402889+0000","last_clean":"2024-08-17T18:14:19.402889+0000","last_became_active":"2024-08-17T18:09:29.836760+0000","last_became_peered":"2024-08-17T18:09:29.836760+0000","last_unstale":"2024-08-17T18:14:19.402889+0000","last_undegraded":"2024-08-17T18:14:19.402889+0000","last_fullsized":"2024-08-17T18:14:19.402889+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:38:52.862967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1387,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.376657+0000","last_change":"2024-08-17T18:09:29.871872+0000","last_active":"2024-08-17T18:14:16.376657+0000","last_peered":"2024-08-17T18:14:16.376657+0000","last_clean":"2024-08-17T18:14:16.376657+0000","last_became_active":"2024-08-17T18:09:29.845111+0000","last_became_peered":"2024-08-17T18:09:29.845111+0000","last_unstale":"2024-08-17T18:14:16.376657+0000","last_undegraded":"2024-08-17T18:14:16.376657+0000","last_fullsized":"2024-08-17T18:14:16.376657+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:47:48.183250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":284,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.377574+0000","last_change":"2024-08-17T18:09:29.844786+0000","last_active":"2024-08-17T18:14:16.377574+0000","last_peered":"2024-08-17T18:14:16.377574+0000","last_clean":"2024-08-17T18:14:16.377574+0000","last_became_active":"2024-08-17T18:09:29.844270+0000","last_became_peered":"2024-08-17T18:09:29.844270+0000","last_unstale":"2024-08-17T18:14:16.377574+0000","last_undegraded":"2024-08-17T18:14:16.377574+0000","last_fullsized":"2024-08-17T18:14:16.377574+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:51.866680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1403,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374723+0000","last_change":"2024-08-17T18:09:29.886391+0000","last_active":"2024-08-17T18:14:16.374723+0000","last_peered":"2024-08-17T18:14:16.374723+0000","last_clean":"2024-08-17T18:14:16.374723+0000","last_became_active":"2024-08-17T18:09:29.844642+0000","last_became_peered":"2024-08-17T18:09:29.844642+0000","last_unstale":"2024-08-17T18:14:16.374723+0000","last_undegraded":"2024-08-17T18:14:16.374723+0000","last_fullsized":"2024-08-17T18:14:16.374723+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:04:07.988100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":267,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.840306+0000","last_change":"2024-08-17T18:09:28.941223+0000","last_active":"2024-08-17T18:14:38.840306+0000","last_peered":"2024-08-17T18:14:38.840306+0000","last_clean":"2024-08-17T18:14:38.840306+0000","last_became_active":"2024-08-17T18:09:28.940857+0000","last_became_peered":"2024-08-17T18:09:28.940857+0000","last_unstale":"2024-08-17T18:14:38.840306+0000","last_undegraded":"2024-08-17T18:14:38.840306+0000","last_fullsized":"2024-08-17T18:14:38.840306+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:22:48.809420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2116,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.410090+0000","last_change":"2024-08-17T18:09:29.838873+0000","last_active":"2024-08-17T18:14:19.410090+0000","last_peered":"2024-08-17T18:14:19.410090+0000","last_clean":"2024-08-17T18:14:19.410090+0000","last_became_active":"2024-08-17T18:09:29.838560+0000","last_became_peered":"2024-08-17T18:09:29.838560+0000","last_unstale":"2024-08-17T18:14:19.410090+0000","last_undegraded":"2024-08-17T18:14:19.410090+0000","last_fullsized":"2024-08-17T18:14:19.410090+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:00.103741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1363,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.373906+0000","last_change":"2024-08-17T18:09:28.844170+0000","last_active":"2024-08-17T18:14:16.373906+0000","last_peered":"2024-08-17T18:14:16.373906+0000","last_clean":"2024-08-17T18:14:16.373906+0000","last_became_active":"2024-08-17T18:09:28.843862+0000","last_became_peered":"2024-08-17T18:09:28.843862+0000","last_unstale":"2024-08-17T18:14:16.373906+0000","last_undegraded":"2024-08-17T18:14:16.373906+0000","last_fullsized":"2024-08-17T18:14:16.373906+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:43:19.483326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2240,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.840491+0000","last_change":"2024-08-17T18:09:28.864923+0000","last_active":"2024-08-17T18:14:38.840491+0000","last_peered":"2024-08-17T18:14:38.840491+0000","last_clean":"2024-08-17T18:14:38.840491+0000","last_became_active":"2024-08-17T18:09:28.864707+0000","last_became_peered":"2024-08-17T18:09:28.864707+0000","last_unstale":"2024-08-17T18:14:38.840491+0000","last_undegraded":"2024-08-17T18:14:38.840491+0000","last_fullsized":"2024-08-17T18:14:38.840491+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:27.507329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2242,"reported_epoch":1393,"state":"active+clean","last_fresh":"2024-08-17T18:14:38.841662+0000","last_change":"2024-08-17T18:09:28.846741+0000","last_active":"2024-08-17T18:14:38.841662+0000","last_peered":"2024-08-17T18:14:38.841662+0000","last_clean":"2024-08-17T18:14:38.841662+0000","last_became_active":"2024-08-17T18:09:28.846478+0000","last_became_peered":"2024-08-17T18:09:28.846478+0000","last_unstale":"2024-08-17T18:14:38.841662+0000","last_undegraded":"2024-08-17T18:14:38.841662+0000","last_fullsized":"2024-08-17T18:14:38.841662+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:34.247517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":769,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374993+0000","last_change":"2024-08-17T18:03:16.091420+0000","last_active":"2024-08-17T18:14:16.374993+0000","last_peered":"2024-08-17T18:14:16.374993+0000","last_clean":"2024-08-17T18:14:16.374993+0000","last_became_active":"2024-08-17T18:03:16.091113+0000","last_became_peered":"2024-08-17T18:03:16.091113+0000","last_unstale":"2024-08-17T18:14:16.374993+0000","last_undegraded":"2024-08-17T18:14:16.374993+0000","last_fullsized":"2024-08-17T18:14:16.374993+0000","mapping_epoch":1297,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:47.770911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1377,"reported_epoch":1391,"state":"active+clean","last_fresh":"2024-08-17T18:14:16.374289+0000","last_change":"2024-08-17T18:02:53.893927+0000","last_active":"2024-08-17T18:14:16.374289+0000","last_peered":"2024-08-17T18:14:16.374289+0000","last_clean":"2024-08-17T18:14:16.374289+0000","last_became_active":"2024-08-17T18:02:53.883188+0000","last_became_peered":"2024-08-17T18:02:53.883188+0000","last_unstale":"2024-08-17T18:14:16.374289+0000","last_undegraded":"2024-08-17T18:14:16.374289+0000","last_fullsized":"2024-08-17T18:14:16.374289+0000","mapping_epoch":1283,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1284,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:19.296965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2116,"reported_epoch":1392,"state":"active+clean","last_fresh":"2024-08-17T18:14:19.411240+0000","last_change":"2024-08-17T18:09:29.840086+0000","last_active":"2024-08-17T18:14:19.411240+0000","last_peered":"2024-08-17T18:14:19.411240+0000","last_clean":"2024-08-17T18:14:19.411240+0000","last_became_active":"2024-08-17T18:09:29.839525+0000","last_became_peered":"2024-08-17T18:09:29.839525+0000","last_unstale":"2024-08-17T18:14:19.411240+0000","last_undegraded":"2024-08-17T18:14:19.411240+0000","last_fullsized":"2024-08-17T18:14:19.411240+0000","mapping_epoch":1344,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:38.316235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":49,"num_pg":16,"stat_sum":{"num_bytes":6274351128,"num_objects":95740,"num_object_clones":0,"num_object_copies":287220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":95740,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":122524,"num_write_kb":6984385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18657325056,"data_stored":18657312840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":69924,"ondisk_log_size":69924,"up":48,"acting":48,"num_store_stats":4},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":10970,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":155648,"data_stored":27490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234645,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5522972,"kb_used_data":4629396,"kb_used_omap":1,"kb_used_meta":893566,"kb_avail":88222180,"statfs":{"total":95995035648,"available":90339512320,"internally_reserved":0,"allocated":4740501504,"data_stored":4735792904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":915011928},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":0,"seq":197568496280,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757918,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1343,"seq":5768141078591,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5413260,"kb_used_data":5028932,"kb_used_omap":112,"kb_used_meta":384207,"kb_avail":88331892,"statfs":{"total":95995035648,"available":90451857408,"internally_reserved":0,"allocated":5149626368,"data_stored":5144929412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":115373,"internal_metadata":393428307},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405771,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6032348,"kb_used_data":4850772,"kb_used_omap":61,"kb_used_meta":1181378,"kb_avail":87712804,"statfs":{"total":95995035648,"available":89817911296,"internally_reserved":0,"allocated":4967190528,"data_stored":4962491138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":1209732024},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":0,"seq":2173253452305,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471474996,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4847924,"kb_used_data":3801576,"kb_used_omap":119,"kb_used_meta":1046152,"kb_avail":88897228,"statfs":{"total":95995035648,"available":91030761472,"internally_reserved":0,"allocated":3892813824,"data_stored":3888094187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":1071260382},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,2,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019553,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":49,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3873705984,"data_stored":3873701912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":49,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4951834624,"data_stored":4951834624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":49,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":5127606272,"data_stored":5127602200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":49,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4704178176,"data_stored":4704174104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:14:43.979 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T18:14:43.979 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:14:43.979 INFO:tasks.thrashosds.thrasher:Adding osd 1 2024-08-17T18:14:43.980 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd in 1 2024-08-17T18:14:43.990 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:44.013 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:44.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:44.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:44.115 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:44.159 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:44.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:44.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:44.261 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:44.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:44.284 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:44.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:44.385 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:44.424 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:44.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:44.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:44.526 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:44.548 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:44.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:44.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:44.649 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:44.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:44.698 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:44.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:44.789 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:14:44.789 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:14:44.789 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:14:44.789 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:14:44.790 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.790 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:147146 48.e 48.4bea78fe (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1343)", 2024-08-17T18:14:44.790 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:09:27.854502+0000", 2024-08-17T18:14:44.790 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 316.88588412399997, 2024-08-17T18:14:44.790 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.96373221899999995, 2024-08-17T18:14:44.790 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.790 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.790 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:14:44.790 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.790 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.790 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.790 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 147146 2024-08-17T18:14:44.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.791 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.791 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.791 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.854502+0000", 2024-08-17T18:14:44.791 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.791 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.791 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.854502+0000", 2024-08-17T18:14:44.791 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.791 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.854503+0000", 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6139999999999999e-06 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.854510+0000", 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3879999999999997e-06 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.854520+0000", 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.995e-06 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.792 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.854529+0000", 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4439999999999997e-06 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.855118+0000", 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00058908799999999998 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:27.855126+0000", 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8730000000000007e-06 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.793 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.794 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:28.818224+0000", 2024-08-17T18:14:44.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.96309766799999996 2024-08-17T18:14:44.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.794 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.794 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.794 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:09:28.818234+0000", 2024-08-17T18:14:44.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0149000000000001e-05 2024-08-17T18:14:44.794 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.794 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.794 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.794 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.794 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221062 48.2 48:4d7247a0:::benchmark_data_smithi130_327597_object89097:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1355)", 2024-08-17T18:14:44.794 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:08.611643+0000", 2024-08-17T18:14:44.795 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 216.12874308900001, 2024-08-17T18:14:44.795 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0463803629999999, 2024-08-17T18:14:44.795 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.795 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.795 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:14:44.795 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.795 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.795 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.795 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221062 2024-08-17T18:14:44.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.795 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.795 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.796 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.611643+0000", 2024-08-17T18:14:44.796 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.796 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.796 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.611643+0000", 2024-08-17T18:14:44.796 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.796 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.796 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.611645+0000", 2024-08-17T18:14:44.796 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9690000000000001e-06 2024-08-17T18:14:44.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.611650+0000", 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.95e-06 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.611663+0000", 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.342e-05 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.611671+0000", 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5939999999999996e-06 2024-08-17T18:14:44.797 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.798 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.798 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.798 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.612950+0000", 2024-08-17T18:14:44.798 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00127928 2024-08-17T18:14:44.798 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.798 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.798 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.798 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.612967+0000", 2024-08-17T18:14:44.798 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7108999999999998e-05 2024-08-17T18:14:44.798 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.798 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.798 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.798 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.657969+0000", 2024-08-17T18:14:44.799 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.045002156 2024-08-17T18:14:44.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.799 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.799 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.799 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.658023+0000", 2024-08-17T18:14:44.799 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3885000000000003e-05 2024-08-17T18:14:44.799 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.799 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.799 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.799 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.799 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221097 48.2 48:48c7f2ea:::benchmark_data_smithi130_327597_object89132:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-08-17T18:14:44.799 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:08.645558+0000", 2024-08-17T18:14:44.799 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 216.09482741299999, 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.089144299, 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221097 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.645558+0000", 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.800 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.645558+0000", 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.645560+0000", 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5999999999999999e-06 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.645567+0000", 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9600000000000003e-06 2024-08-17T18:14:44.801 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.802 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.802 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.802 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.645581+0000", 2024-08-17T18:14:44.802 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4205e-05 2024-08-17T18:14:44.802 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.802 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.802 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.802 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.645589+0000", 2024-08-17T18:14:44.802 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3660000000000003e-06 2024-08-17T18:14:44.802 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.802 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.802 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.802 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.645651+0000", 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2531000000000002e-05 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.645660+0000", 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3760000000000008e-06 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.658512+0000", 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0128526179999999 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.803 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.658626+0000", 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011387099999999999 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.659319+0000", 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00069309999999999999 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.711869+0000", 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.052549891000000001 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.804 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.716934+0000", 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0050653130000000001 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.734553+0000", 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017619075000000001 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.734655+0000", 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000101467 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.805 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.734703+0000", 2024-08-17T18:14:44.806 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7926000000000002e-05 2024-08-17T18:14:44.806 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.806 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.806 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.806 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.806 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.806 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221118 48.2 48:4ea3e3a5:::benchmark_data_smithi130_327597_object89153:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-08-17T18:14:44.806 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:08.666708+0000", 2024-08-17T18:14:44.806 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 216.07367770299999, 2024-08-17T18:14:44.806 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0681728699999999, 2024-08-17T18:14:44.806 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.806 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.806 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.807 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.807 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.807 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.807 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221118 2024-08-17T18:14:44.807 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.807 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.807 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.807 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.807 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.666708+0000", 2024-08-17T18:14:44.807 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.807 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.807 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.807 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.807 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.666708+0000", 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.666710+0000", 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1009999999999999e-06 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.666716+0000", 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2399999999999998e-06 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.808 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.666728+0000", 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2928000000000001e-05 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.666736+0000", 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2699999999999999e-06 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.666853+0000", 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000117476 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.666863+0000", 2024-08-17T18:14:44.809 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3829999999999998e-06 2024-08-17T18:14:44.810 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.810 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.810 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.810 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.700484+0000", 2024-08-17T18:14:44.810 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0336215689999999 2024-08-17T18:14:44.810 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.810 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.810 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.810 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.700610+0000", 2024-08-17T18:14:44.810 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012550899999999999 2024-08-17T18:14:44.810 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.810 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.810 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:14:44.810 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.700740+0000", 2024-08-17T18:14:44.811 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013006199999999999 2024-08-17T18:14:44.811 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.811 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.811 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.811 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.711875+0000", 2024-08-17T18:14:44.811 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011134821 2024-08-17T18:14:44.811 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.811 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.811 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.811 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.716971+0000", 2024-08-17T18:14:44.811 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005096499 2024-08-17T18:14:44.811 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.811 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.811 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.734757+0000", 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017785823999999999 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.734843+0000", 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6386999999999998e-05 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.734881+0000", 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7800999999999999e-05 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.812 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221127 48.2 48:40caf3ea:::benchmark_data_smithi130_327597_object89162:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:08.671320+0000", 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 216.069066107, 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.063743409, 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221127 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.813 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.671320+0000", 2024-08-17T18:14:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.671320+0000", 2024-08-17T18:14:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.671321+0000", 2024-08-17T18:14:44.814 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5090000000000001e-06 2024-08-17T18:14:44.815 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.815 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.815 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.815 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.671326+0000", 2024-08-17T18:14:44.815 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7400000000000004e-06 2024-08-17T18:14:44.815 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.815 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.815 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.815 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.671340+0000", 2024-08-17T18:14:44.815 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3723999999999999e-05 2024-08-17T18:14:44.816 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.816 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.816 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.816 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.671347+0000", 2024-08-17T18:14:44.816 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8990000000000004e-06 2024-08-17T18:14:44.816 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.816 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.816 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.816 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.672575+0000", 2024-08-17T18:14:44.816 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0012283190000000001 2024-08-17T18:14:44.816 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.816 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.816 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.816 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.672583+0000", 2024-08-17T18:14:44.817 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0620000000000001e-06 2024-08-17T18:14:44.817 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.817 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.817 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.817 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.703166+0000", 2024-08-17T18:14:44.817 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.03058307 2024-08-17T18:14:44.817 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.817 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.817 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.817 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.703284+0000", 2024-08-17T18:14:44.817 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000118238 2024-08-17T18:14:44.817 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.817 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.817 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:14:44.818 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.703410+0000", 2024-08-17T18:14:44.818 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000125816 2024-08-17T18:14:44.818 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.818 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.818 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.818 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.711879+0000", 2024-08-17T18:14:44.818 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0084687510000000001 2024-08-17T18:14:44.818 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.818 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.818 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.818 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.717004+0000", 2024-08-17T18:14:44.818 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0051249340000000003 2024-08-17T18:14:44.818 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.819 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.819 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.819 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.734935+0000", 2024-08-17T18:14:44.819 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017931572999999999 2024-08-17T18:14:44.819 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.819 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.819 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.819 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735025+0000", 2024-08-17T18:14:44.819 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0044999999999996e-05 2024-08-17T18:14:44.819 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.819 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.819 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.819 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735063+0000", 2024-08-17T18:14:44.819 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7728999999999997e-05 2024-08-17T18:14:44.820 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.820 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.820 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.820 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.820 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.820 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221145 48.2 48:48681e13:::benchmark_data_smithi130_327597_object89180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-08-17T18:14:44.820 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:08.701105+0000", 2024-08-17T18:14:44.820 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 216.03928110699999, 2024-08-17T18:14:44.820 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0341293869999999, 2024-08-17T18:14:44.820 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.820 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.820 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.820 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.820 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221145 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.701105+0000", 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.701105+0000", 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.821 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.822 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.822 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.701107+0000", 2024-08-17T18:14:44.822 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.007e-06 2024-08-17T18:14:44.822 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.822 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.822 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.822 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.701114+0000", 2024-08-17T18:14:44.822 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.849e-06 2024-08-17T18:14:44.822 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.822 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.822 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.822 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.701128+0000", 2024-08-17T18:14:44.822 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4158000000000001e-05 2024-08-17T18:14:44.822 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.823 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.823 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.823 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.701136+0000", 2024-08-17T18:14:44.823 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1799999999999996e-06 2024-08-17T18:14:44.823 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.823 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.823 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.823 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.701217+0000", 2024-08-17T18:14:44.823 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1094999999999995e-05 2024-08-17T18:14:44.823 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.823 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.823 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.823 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.701226+0000", 2024-08-17T18:14:44.823 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6449999999999994e-06 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.705862+0000", 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0046361370000001 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.705990+0000", 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012773 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.706116+0000", 2024-08-17T18:14:44.824 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000126556 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.714427+0000", 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0083109780000000001 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.717038+0000", 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026108659999999999 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735114+0000", 2024-08-17T18:14:44.825 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018076045999999998 2024-08-17T18:14:44.826 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.826 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.826 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.826 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735195+0000", 2024-08-17T18:14:44.826 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1114999999999998e-05 2024-08-17T18:14:44.826 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.826 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.826 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.826 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735234+0000", 2024-08-17T18:14:44.826 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9025e-05 2024-08-17T18:14:44.826 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.826 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.826 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.826 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.827 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.827 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221151 48.2 48:48130c36:::benchmark_data_smithi130_327597_object89186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-08-17T18:14:44.827 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:08.703182+0000", 2024-08-17T18:14:44.827 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 216.037204124, 2024-08-17T18:14:44.827 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0322178390000001, 2024-08-17T18:14:44.827 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.827 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.827 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.827 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.827 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.827 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.827 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221151 2024-08-17T18:14:44.827 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.827 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.703182+0000", 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.703182+0000", 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.703183+0000", 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4789999999999999e-06 2024-08-17T18:14:44.828 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.829 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.829 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.829 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.703189+0000", 2024-08-17T18:14:44.829 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8119999999999996e-06 2024-08-17T18:14:44.829 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.829 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.829 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.829 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.703206+0000", 2024-08-17T18:14:44.829 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.666e-05 2024-08-17T18:14:44.829 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.829 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.829 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.829 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.703214+0000", 2024-08-17T18:14:44.829 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2360000000000004e-06 2024-08-17T18:14:44.830 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.830 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.830 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.830 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.703273+0000", 2024-08-17T18:14:44.830 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9358000000000003e-05 2024-08-17T18:14:44.830 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.830 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.830 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.830 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.703284+0000", 2024-08-17T18:14:44.830 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0523000000000001e-05 2024-08-17T18:14:44.830 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.830 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.830 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.709338+0000", 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0060540360000001 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.709438+0000", 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010011800000000001 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.709543+0000", 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000105259 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.831 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.832 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.721806+0000", 2024-08-17T18:14:44.832 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012263243 2024-08-17T18:14:44.832 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.832 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.832 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.832 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.729585+0000", 2024-08-17T18:14:44.832 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0077781780000000002 2024-08-17T18:14:44.832 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.832 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.832 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.832 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735274+0000", 2024-08-17T18:14:44.832 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005689614 2024-08-17T18:14:44.832 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.833 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.833 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.833 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735359+0000", 2024-08-17T18:14:44.833 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5197000000000004e-05 2024-08-17T18:14:44.833 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.833 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.833 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.833 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735400+0000", 2024-08-17T18:14:44.833 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0126000000000002e-05 2024-08-17T18:14:44.833 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.833 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.833 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.833 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.833 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.834 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221162 48.2 48:4006d510:::benchmark_data_smithi130_327597_object89197:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-08-17T18:14:44.834 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:08.716370+0000", 2024-08-17T18:14:44.834 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 216.02401544700001, 2024-08-17T18:14:44.834 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.019184439, 2024-08-17T18:14:44.834 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.834 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.834 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.834 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.834 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.834 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.834 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221162 2024-08-17T18:14:44.834 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.834 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.834 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.835 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.835 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.716370+0000", 2024-08-17T18:14:44.835 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.835 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.835 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.835 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.835 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.716370+0000", 2024-08-17T18:14:44.835 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.835 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.835 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.835 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.835 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.716372+0000", 2024-08-17T18:14:44.835 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8509999999999999e-06 2024-08-17T18:14:44.835 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.716375+0000", 2024-08-17T18:14:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8009999999999999e-06 2024-08-17T18:14:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.716383+0000", 2024-08-17T18:14:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.174e-06 2024-08-17T18:14:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.716388+0000", 2024-08-17T18:14:44.836 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.198e-06 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.718710+0000", 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0023218990000000001 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.718720+0000", 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.6229999999999992e-06 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.711907+0000", 2024-08-17T18:14:44.837 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.99318705799999996 2024-08-17T18:14:44.838 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.838 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.838 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.838 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.712018+0000", 2024-08-17T18:14:44.838 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011047100000000001 2024-08-17T18:14:44.838 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.838 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.838 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:14:44.838 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.712122+0000", 2024-08-17T18:14:44.838 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0001049 2024-08-17T18:14:44.838 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.838 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.838 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.838 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.724156+0000", 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012033828 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.729629+0000", 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0054726519999999997 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735443+0000", 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0058140989999999997 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.839 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735520+0000", 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.682e-05 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735555+0000", 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5064999999999997e-05 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221208 48.2 48:41562b52:::benchmark_data_smithi130_327597_object89243:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:08.772071+0000", 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 215.96831481000001, 2024-08-17T18:14:44.840 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.96364651300000004, 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221208 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.772071+0000", 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.841 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.842 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.772071+0000", 2024-08-17T18:14:44.842 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.842 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.842 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.842 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.842 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.772073+0000", 2024-08-17T18:14:44.842 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7969999999999999e-06 2024-08-17T18:14:44.842 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.842 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.842 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.842 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.772079+0000", 2024-08-17T18:14:44.842 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.874e-06 2024-08-17T18:14:44.842 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.842 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.772096+0000", 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7322000000000001e-05 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.772104+0000", 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5939999999999996e-06 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.772182+0000", 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8010999999999996e-05 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.843 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.844 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.844 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.772190+0000", 2024-08-17T18:14:44.844 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2700000000000004e-06 2024-08-17T18:14:44.844 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.844 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.844 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.844 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.714468+0000", 2024-08-17T18:14:44.844 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94227806800000002 2024-08-17T18:14:44.844 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.844 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.844 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.844 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.714571+0000", 2024-08-17T18:14:44.844 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000102631 2024-08-17T18:14:44.844 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.714683+0000", 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011274500000000001 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.724161+0000", 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0094779410000000001 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.729665+0000", 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0055038719999999999 2024-08-17T18:14:44.845 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.846 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.846 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.846 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735597+0000", 2024-08-17T18:14:44.846 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005931703 2024-08-17T18:14:44.846 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.846 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.846 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.846 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735679+0000", 2024-08-17T18:14:44.846 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2418999999999998e-05 2024-08-17T18:14:44.846 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.846 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.846 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.846 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735718+0000", 2024-08-17T18:14:44.846 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8266000000000003e-05 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221226 48.2 48:48a851ab:::benchmark_data_smithi130_327597_object89261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:08.793192+0000", 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 215.9471934, 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94269005900000002, 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.847 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221226 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.793192+0000", 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.793192+0000", 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.848 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.849 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.793194+0000", 2024-08-17T18:14:44.849 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.469e-06 2024-08-17T18:14:44.849 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.849 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.849 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.849 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.793198+0000", 2024-08-17T18:14:44.849 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9609999999999999e-06 2024-08-17T18:14:44.849 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.849 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.849 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.849 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.793212+0000", 2024-08-17T18:14:44.849 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4161e-05 2024-08-17T18:14:44.849 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.849 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.850 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.850 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.793220+0000", 2024-08-17T18:14:44.850 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7039999999999994e-06 2024-08-17T18:14:44.850 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.850 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.850 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.850 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.793297+0000", 2024-08-17T18:14:44.850 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7201999999999997e-05 2024-08-17T18:14:44.850 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.850 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.850 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.850 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.793307+0000", 2024-08-17T18:14:44.850 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5230000000000001e-06 2024-08-17T18:14:44.850 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.851 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.851 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.851 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.717067+0000", 2024-08-17T18:14:44.851 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.92376029999999998 2024-08-17T18:14:44.851 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.851 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.851 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.851 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.717168+0000", 2024-08-17T18:14:44.851 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000100987 2024-08-17T18:14:44.851 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.851 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.851 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:14:44.851 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.717277+0000", 2024-08-17T18:14:44.852 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000108817 2024-08-17T18:14:44.852 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.852 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.852 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.852 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.724166+0000", 2024-08-17T18:14:44.852 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.006889404 2024-08-17T18:14:44.852 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.852 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.852 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.852 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.729701+0000", 2024-08-17T18:14:44.852 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0055350959999999998 2024-08-17T18:14:44.852 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.852 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.852 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735757+0000", 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0060555000000000001 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735846+0000", 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9656999999999999e-05 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.735883+0000", 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6278000000000001e-05 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.853 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.854 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.854 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.854 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221237 48.2 48:4f1366e7:::benchmark_data_smithi130_327597_object89272:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-08-17T18:14:44.854 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:08.799663+0000", 2024-08-17T18:14:44.854 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 215.94072256600001, 2024-08-17T18:14:44.854 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94473499000000005, 2024-08-17T18:14:44.854 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.854 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.854 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.854 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.854 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.854 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.854 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221237 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.799663+0000", 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.799663+0000", 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.855 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.799664+0000", 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8999999999999997e-07 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.799669+0000", 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0649999999999998e-06 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.799675+0000", 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2249999999999997e-06 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.856 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.799679+0000", 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7189999999999999e-06 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.799710+0000", 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0726999999999999e-05 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.799713+0000", 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9160000000000003e-06 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.857 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.721907+0000", 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.92219345500000005 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.722063+0000", 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015651 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.722266+0000", 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020235500000000001 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.858 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.729544+0000", 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072783049999999997 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.736081+0000", 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0065372260000000001 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.744298+0000", 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.008217093 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.859 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.744351+0000", 2024-08-17T18:14:44.860 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2843999999999998e-05 2024-08-17T18:14:44.860 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.860 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.860 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.860 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.744398+0000", 2024-08-17T18:14:44.860 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.706e-05 2024-08-17T18:14:44.860 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.860 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.860 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.860 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.860 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.860 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:221243 48.2 48:48a223b9:::benchmark_data_smithi130_327597_object89278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:08.807552+0000", 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 215.93283345399999, 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.95222837100000002, 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 221243 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.861 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.807552+0000", 2024-08-17T18:14:44.862 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.862 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.862 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.862 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.862 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.807552+0000", 2024-08-17T18:14:44.862 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.862 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.862 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.862 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.862 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.807554+0000", 2024-08-17T18:14:44.862 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.429e-06 2024-08-17T18:14:44.862 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.862 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.807560+0000", 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7749999999999998e-06 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.807571+0000", 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1812e-05 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.807582+0000", 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0183000000000001e-05 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.863 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.807647+0000", 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5667000000000007e-05 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:08.807677+0000", 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9771e-05 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.724194+0000", 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.91651679900000005 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.864 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.724302+0000", 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010830199999999999 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.724413+0000", 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000110707 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.729596+0000", 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0051833319999999997 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.865 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.736118+0000", 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0065217260000000003 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.759375+0000", 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023257342 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.759738+0000", 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00036271399999999998 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.866 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:09.759781+0000", 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2812000000000003e-05 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:239295 48.e 48:74885da9:::benchmark_data_smithi130_327597_object107330:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1358)", 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:33.772868+0000", 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 190.967517961, 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.053549436, 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:14:44.867 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 239295 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.772868+0000", 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.772868+0000", 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.868 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.869 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.869 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.772869+0000", 2024-08-17T18:14:44.869 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5570000000000001e-06 2024-08-17T18:14:44.869 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.869 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.869 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.869 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.772879+0000", 2024-08-17T18:14:44.869 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3729999999999999e-06 2024-08-17T18:14:44.869 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.869 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.869 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.869 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.772892+0000", 2024-08-17T18:14:44.869 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2866e-05 2024-08-17T18:14:44.869 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.870 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:44.871 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.871 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.871 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.772899+0000", 2024-08-17T18:14:44.871 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1230000000000002e-06 2024-08-17T18:14:44.872 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.872 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.872 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.872 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.773567+0000", 2024-08-17T18:14:44.872 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00066773799999999997 2024-08-17T18:14:44.872 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.872 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.872 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.872 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.773574+0000", 2024-08-17T18:14:44.872 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.807e-06 2024-08-17T18:14:44.872 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.872 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.872 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.872 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.826330+0000", 2024-08-17T18:14:44.873 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.052755828 2024-08-17T18:14:44.873 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.873 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.873 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.873 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.826417+0000", 2024-08-17T18:14:44.873 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7144000000000005e-05 2024-08-17T18:14:44.873 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.873 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.873 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.873 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.873 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.873 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:239368 48.e 48:7845c0ff:::benchmark_data_smithi130_327597_object107403:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-08-17T18:14:44.873 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:33.823845+0000", 2024-08-17T18:14:44.874 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 190.91654118299999, 2024-08-17T18:14:44.874 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0268358689999999, 2024-08-17T18:14:44.874 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.874 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.874 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.874 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.874 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.874 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.874 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 239368 2024-08-17T18:14:44.874 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.874 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.874 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.874 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.823845+0000", 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.823845+0000", 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.823846+0000", 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5239999999999999e-06 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.875 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.823850+0000", 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.788e-06 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.823859+0000", 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4099999999999997e-06 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.823866+0000", 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5459999999999997e-06 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.876 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.823935+0000", 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8528999999999994e-05 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.823942+0000", 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7940000000000003e-06 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.827679+0000", 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.003736798 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.877 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.827786+0000", 2024-08-17T18:14:44.878 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010722600000000001 2024-08-17T18:14:44.878 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.878 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.878 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T18:14:44.878 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.828358+0000", 2024-08-17T18:14:44.878 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00057191900000000005 2024-08-17T18:14:44.878 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.878 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.878 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.878 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.840117+0000", 2024-08-17T18:14:44.878 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011758338 2024-08-17T18:14:44.878 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.878 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.878 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.840171+0000", 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4280000000000002e-05 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.850590+0000", 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010419159000000001 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.850640+0000", 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0232999999999998e-05 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.879 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.850681+0000", 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0324999999999998e-05 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:239369 48.e 48:7f73ca6a:::benchmark_data_smithi130_327597_object107404:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:33.824302+0000", 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 190.91608426900001, 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0357945740000001, 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.880 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 239369 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.824302+0000", 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.824302+0000", 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.881 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.824303+0000", 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9299999999999996e-07 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.824307+0000", 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3309999999999997e-06 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.824322+0000", 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4713e-05 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.882 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.883 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.883 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.824329+0000", 2024-08-17T18:14:44.883 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6739999999999997e-06 2024-08-17T18:14:44.883 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.883 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.883 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.883 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.824436+0000", 2024-08-17T18:14:44.883 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000106994 2024-08-17T18:14:44.883 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.883 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.883 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.883 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.824444+0000", 2024-08-17T18:14:44.883 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.272e-06 2024-08-17T18:14:44.883 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.837052+0000", 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0126083690000001 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.837186+0000", 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013439600000000001 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.837292+0000", 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010579 2024-08-17T18:14:44.884 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.885 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.885 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.885 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.840176+0000", 2024-08-17T18:14:44.885 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028837519999999998 2024-08-17T18:14:44.885 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.885 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.885 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.885 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.840243+0000", 2024-08-17T18:14:44.885 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7650000000000005e-05 2024-08-17T18:14:44.885 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.885 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.885 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.885 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.859992+0000", 2024-08-17T18:14:44.885 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019748279000000001 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.860048+0000", 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6585000000000001e-05 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.860096+0000", 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7876e-05 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.886 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:239380 48.e 48:70e81173:::benchmark_data_smithi130_327597_object107415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:33.832551+0000", 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 190.90783510899999, 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0276758029999999, 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 239380 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.887 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.832551+0000", 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.832551+0000", 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.832552+0000", 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1990000000000001e-06 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.888 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.832556+0000", 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5530000000000001e-06 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.832564+0000", 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6060000000000003e-06 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.832571+0000", 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9700000000000002e-06 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.889 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.835360+0000", 2024-08-17T18:14:44.890 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0027886170000000002 2024-08-17T18:14:44.890 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.890 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.890 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.890 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.835367+0000", 2024-08-17T18:14:44.890 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6190000000000002e-06 2024-08-17T18:14:44.890 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.890 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.890 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.890 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.840290+0000", 2024-08-17T18:14:44.890 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0049229420000001 2024-08-17T18:14:44.890 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.890 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.890 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.840421+0000", 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013053300000000001 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.840647+0000", 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00022585699999999999 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.845877+0000", 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0052300300000000001 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.891 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.892 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.845890+0000", 2024-08-17T18:14:44.892 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2850000000000001e-05 2024-08-17T18:14:44.892 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.892 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.892 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.892 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.860142+0000", 2024-08-17T18:14:44.892 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014252681 2024-08-17T18:14:44.892 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.892 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.892 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.892 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.860190+0000", 2024-08-17T18:14:44.892 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7645000000000002e-05 2024-08-17T18:14:44.892 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.892 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.860227+0000", 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6701e-05 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:239391 48.e 48:7ba2b194:::benchmark_data_smithi130_327597_object107426:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:33.846071+0000", 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 190.89431488, 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0142799220000001, 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.893 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.894 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.894 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.894 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.894 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 239391 2024-08-17T18:14:44.894 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.894 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.894 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.894 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.894 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.846071+0000", 2024-08-17T18:14:44.894 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.894 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.894 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.894 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.894 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.846071+0000", 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.846072+0000", 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4100000000000001e-06 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.846075+0000", 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.824e-06 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.895 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.846083+0000", 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9750000000000007e-06 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.846088+0000", 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8180000000000003e-06 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.847842+0000", 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0017543560000000001 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.896 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.847852+0000", 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3479999999999993e-06 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.843373+0000", 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.99552102399999998 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.843472+0000", 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9482999999999996e-05 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T18:14:44.897 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.843576+0000", 2024-08-17T18:14:44.898 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010385 2024-08-17T18:14:44.898 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.898 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.898 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.898 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.850483+0000", 2024-08-17T18:14:44.898 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0069070870000000001 2024-08-17T18:14:44.898 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.898 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.898 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.898 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.850519+0000", 2024-08-17T18:14:44.898 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6211999999999998e-05 2024-08-17T18:14:44.898 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.898 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.898 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.860268+0000", 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0097484570000000003 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.860315+0000", 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7015999999999998e-05 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.860351+0000", 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6062000000000001e-05 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.899 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:239399 48.e 48:75469e1f:::benchmark_data_smithi130_327597_object107434:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:33.852802+0000", 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 190.88758352299999, 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.008207066, 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 239399 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.900 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.852802+0000", 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.852802+0000", 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.852804+0000", 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3e-06 2024-08-17T18:14:44.901 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.852808+0000", 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0099999999999997e-06 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.852819+0000", 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1779e-05 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.852826+0000", 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7959999999999998e-06 2024-08-17T18:14:44.902 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.852882+0000", 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6203999999999999e-05 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.852890+0000", 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7670000000000003e-06 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.845922+0000", 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.99303188600000003 2024-08-17T18:14:44.903 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.846026+0000", 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000104267 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.846138+0000", 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000111945 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.850529+0000", 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0043909270000000002 2024-08-17T18:14:44.904 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.850558+0000", 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8365e-05 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.860909+0000", 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010351852 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.860968+0000", 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8820000000000003e-05 2024-08-17T18:14:44.905 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.906 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.906 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.906 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.861009+0000", 2024-08-17T18:14:44.906 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1148e-05 2024-08-17T18:14:44.906 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.906 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.906 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.906 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.906 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.906 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:239407 48.e 48:7aa6d702:::benchmark_data_smithi130_327597_object107442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-08-17T18:14:44.906 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:33.860927+0000", 2024-08-17T18:14:44.906 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 190.879458701, 2024-08-17T18:14:44.906 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0067034319999999, 2024-08-17T18:14:44.906 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.907 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.907 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.907 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.907 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.907 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.907 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 239407 2024-08-17T18:14:44.907 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.907 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.907 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.907 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.907 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.860927+0000", 2024-08-17T18:14:44.907 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.907 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.907 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.860927+0000", 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.860929+0000", 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4950000000000001e-06 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.860934+0000", 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0359999999999997e-06 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.908 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.860950+0000", 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5950000000000001e-05 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.860969+0000", 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9224000000000001e-05 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.862306+0000", 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001337363 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.909 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.862316+0000", 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9580000000000002e-06 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.848287+0000", 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98597034900000002 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.848384+0000", 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7617999999999999e-05 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.910 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.848486+0000", 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010173199999999999 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.850719+0000", 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0022329580000000002 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.855527+0000", 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0048085039999999999 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.911 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.912 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.867543+0000", 2024-08-17T18:14:44.912 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012015545000000001 2024-08-17T18:14:44.912 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.912 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.912 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.912 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.867587+0000", 2024-08-17T18:14:44.912 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4224000000000002e-05 2024-08-17T18:14:44.912 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.912 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.912 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.912 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.867631+0000", 2024-08-17T18:14:44.912 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3476000000000002e-05 2024-08-17T18:14:44.912 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.912 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:239460 48.e 48:79e4894c:::benchmark_data_smithi130_327597_object107495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:33.917894+0000", 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 190.82249184700001, 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94987701300000005, 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 239460 2024-08-17T18:14:44.913 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.917894+0000", 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.917894+0000", 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.917896+0000", 2024-08-17T18:14:44.914 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9250000000000002e-06 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.917902+0000", 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8499999999999999e-06 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.917915+0000", 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3518e-05 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.917923+0000", 2024-08-17T18:14:44.915 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4270000000000002e-06 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.917998+0000", 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4870999999999995e-05 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.918007+0000", 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7629999999999995e-06 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.850746+0000", 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.93273881800000003 2024-08-17T18:14:44.916 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.850857+0000", 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000110744 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,7", 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.850963+0000", 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000105666 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.859900+0000", 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0089374510000000008 2024-08-17T18:14:44.917 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.861020+0000", 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001120107 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.867701+0000", 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0066811099999999997 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.867740+0000", 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9004000000000001e-05 2024-08-17T18:14:44.918 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:14:44.919 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:14:44.919 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:14:44.919 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:34.867771+0000", 2024-08-17T18:14:44.919 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0759000000000001e-05 2024-08-17T18:14:44.919 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.919 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.919 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.919 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:14:44.919 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:14:44.919 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:14:44.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:44.920 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:44.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:45.021 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:45.057 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:45.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:45.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:44 smithi179 ceph-mon[93519]: pgmap v2597: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T18:14:45.159 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:45.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:44 smithi130 ceph-mon[105521]: pgmap v2597: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T18:14:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:44 smithi130 ceph-mon[110068]: pgmap v2597: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 337 GiB / 358 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T18:14:45.197 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:45.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:45.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:45.299 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:45.322 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:45.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:45.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:45.424 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:45.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:45.459 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:45.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:45.560 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:45.583 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:45.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:45.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:45.685 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:45.707 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:45.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:45.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:45.809 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:45.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:45.845 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:45.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:45.946 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:45.997 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:45.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:45.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:46.099 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:46.134 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:46.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:46.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:46.213 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:14:46.237 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:46.258 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:46.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:46.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:46.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:46.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:46.409 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:46.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:46.510 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:46.541 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:46.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:46.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:46.643 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:46.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:46.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:46.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:46.794 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:46.816 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:46.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:46.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:46.918 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:46.942 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:46.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:47.043 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:47.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:47.082 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:47.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:47.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:46 smithi130 ceph-mon[105521]: pgmap v2598: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T18:14:47.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:46 smithi130 ceph-mon[110068]: pgmap v2598: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T18:14:47.182 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:47.204 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:47.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:47.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:47.263 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T18:14:47.306 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:47.327 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:47.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:47.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:47.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:46 smithi179 ceph-mon[93519]: pgmap v2598: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.01k op/s 2024-08-17T18:14:47.429 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:47.451 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:47.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:47.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:47.553 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:47.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:47.575 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:47.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:47.676 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:47.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:47.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:47.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:47.798 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:47.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:47.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:47.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:47.923 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:47.961 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:47.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:47.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:48.063 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:48.096 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:48.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:48.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:48.198 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:48.221 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:48.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:48.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:48.323 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:48.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:48.361 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:48.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:48.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:48.484 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:48.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:48.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:48.586 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:48.608 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:48.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:48.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:48.710 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:48.744 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:48.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:48.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:48.846 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:48.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:48.869 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:48.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:48.970 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:48.993 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:48.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:48.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:49.094 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:49.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:49.133 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:49.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:49.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:48 smithi130 ceph-mon[110068]: pgmap v2599: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 730 op/s 2024-08-17T18:14:49.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:48 smithi130 ceph-mon[105521]: pgmap v2599: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 730 op/s 2024-08-17T18:14:49.233 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:49.265 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:49.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:49.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:49.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:48 smithi179 ceph-mon[93519]: pgmap v2599: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 730 op/s 2024-08-17T18:14:49.367 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:49.389 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:49.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:49.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:49.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:49.512 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:49.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:49.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:49.614 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:49.654 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:49.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:49.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:49.756 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:49.778 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:49.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:49.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:49.880 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:49.902 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:49.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:49.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:50.003 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:50.026 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:50.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:50.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:50.128 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:14:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:14:49] "GET /metrics HTTP/1.1" 200 36785 "" "Prometheus/2.51.0" 2024-08-17T18:14:50.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:50.168 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:50.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:50.269 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:50.292 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:50.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:50.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:50.394 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:50.438 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:50.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:50.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:50.540 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:50.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:50.568 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:50.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:50.576 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:14:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:14:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:14:50.670 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:50.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:50.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:50.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:50.794 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:50.844 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:50.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:50.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:50.946 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:51.018 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:51.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:51.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:51.120 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:51.126 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:50 smithi130 ceph-mon[105521]: pgmap v2600: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T18:14:51.127 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:50 smithi130 ceph-mon[110068]: pgmap v2600: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T18:14:51.142 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:51.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:51.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:51.243 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:51.284 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:51.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:51.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:50 smithi179 ceph-mon[93519]: pgmap v2600: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 338 GiB / 358 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T18:14:51.386 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:51.429 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:51.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:51.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:51.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:51.565 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:51.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:51.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:51.667 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:51.702 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:51.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:51.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:51.804 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:51.826 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:51.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:51.928 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:51.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:51.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:51.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:52.060 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:52.083 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:52.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:52.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:52.184 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:52.220 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:52.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:52.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:52.321 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:52.342 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:52.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:52.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:52.444 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:52.476 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:52.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:52.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:52.578 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:52.601 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:52.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:52.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:52.702 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:52.725 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:52.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:52.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:52.826 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:52.867 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:52.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:52.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:52.969 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:52.991 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:52.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:53.092 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:53.115 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:53.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:53.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:53.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:52 smithi130 ceph-mon[105521]: pgmap v2601: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 844 op/s 2024-08-17T18:14:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:52 smithi130 ceph-mon[110068]: pgmap v2601: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 844 op/s 2024-08-17T18:14:53.216 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:53.253 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:53.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:53.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:53.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:52 smithi179 ceph-mon[93519]: pgmap v2601: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 844 op/s 2024-08-17T18:14:53.355 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:53.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:53.395 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:53.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:53.496 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:53.527 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:53.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:53.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:53.629 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:53.651 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:53.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:53.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:53.753 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:53.774 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:53.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:53.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:53.876 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:53.899 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:53.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:53.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:54.000 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:54.031 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:54.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:54.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:54.132 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:54.156 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:54.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:54.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:54.258 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:54.282 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:54.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:54.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:54.384 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:54.420 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:54.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:54.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:54.522 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:54.559 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:54.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:54.662 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:54.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:54.697 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:54.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:54.798 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:54.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:54.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:54.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:54.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:54.956 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:54.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:54.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:55.058 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:55.097 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:55.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:55.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:55.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:54 smithi130 ceph-mon[105521]: pgmap v2602: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T18:14:55.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:54 smithi130 ceph-mon[110068]: pgmap v2602: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T18:14:55.200 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:55.222 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:55.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:55.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:55.324 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:55.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:54 smithi179 ceph-mon[93519]: pgmap v2602: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T18:14:55.360 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:55.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:55.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:55.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:55.499 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:55.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:55.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:55.601 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:55.638 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:55.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:55.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:55.739 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:55.762 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:55.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:55.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:55.864 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:55.870 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:14:55.886 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:55.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:55.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:55.988 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:56.028 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:56.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:56.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:56.130 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:56.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:55 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3516344463' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:14:56.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:55 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:14:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:55 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3516344463' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:14:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:55 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:14:56.168 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:56.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:56.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:56.270 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:56.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:56.312 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:56.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:56.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:55 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3516344463' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:14:56.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:55 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:14:56.413 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:56.453 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:56.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:56.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:56.555 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:56.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:56.608 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:56.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:56.709 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:56.732 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:56.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:56.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:56.834 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:56.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:56.869 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:56.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:56.970 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:57.005 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:57.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:57.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:57.107 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:57.141 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:57.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:57.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:57.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:56 smithi130 ceph-mon[105521]: pgmap v2603: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 955 op/s 2024-08-17T18:14:57.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:56 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:14:57.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:56 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T18:14:57.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:56 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:14:57.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:56 smithi130 ceph-mon[105521]: osdmap e1394: 8 total, 8 up, 4 in 2024-08-17T18:14:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:56 smithi130 ceph-mon[110068]: pgmap v2603: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 955 op/s 2024-08-17T18:14:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:56 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:14:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:56 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T18:14:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:56 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:14:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:56 smithi130 ceph-mon[110068]: osdmap e1394: 8 total, 8 up, 4 in 2024-08-17T18:14:57.243 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:57.265 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:57.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:57.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:56 smithi179 ceph-mon[93519]: pgmap v2603: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 955 op/s 2024-08-17T18:14:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:56 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:14:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:56 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T18:14:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:56 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:14:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:56 smithi179 ceph-mon[93519]: osdmap e1394: 8 total, 8 up, 4 in 2024-08-17T18:14:57.366 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:57.400 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:57.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:57.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:57.503 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:57.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:57.538 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:57.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:57.639 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:57.679 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:57.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:57.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:57.781 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:57.813 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:57.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:57.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:57.915 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:57.960 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:57.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:57.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:57.977 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:14:58.062 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:14:58.084 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:58.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:58.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:14:58.186 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:58.224 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:58.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:58.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:58.326 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:14:58.373 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:58.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:58.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:14:58.475 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:14:58.496 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:58.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:58.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:14:58.598 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:14:58.652 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:58.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:58.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:14:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:14:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:14:58.754 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:58.775 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:58.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:58.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:58.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:58.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:14:58.877 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:14:58.936 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:58.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:58.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:14:59.038 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:59.060 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:59.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:59.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:59.162 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:59.208 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:59.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:59.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:59.310 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:59.347 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:59.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:59.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:59.449 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:14:59.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:59.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:59.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:14:59.490 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:14:59.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:59.627 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:59.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:59.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:59.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:59 smithi130 ceph-mon[105521]: pgmap v2605: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T18:14:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:59 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:59 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:59 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:14:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:59 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:14:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:14:59 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:59 smithi130 ceph-mon[110068]: pgmap v2605: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T18:14:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:59 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:59 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:59 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:14:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:59 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:14:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:14:59 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:59.729 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:14:59.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:59.771 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:14:59.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:14:59.818 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:14:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:59 smithi179 ceph-mon[93519]: pgmap v2605: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 339 GiB / 358 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T18:14:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:59 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:59 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:59 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:14:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:59 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:14:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:14:59 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:14:59.872 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:14:59.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:14:59.894 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:14:59.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:14:59.995 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:00.018 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:00.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:00.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:00.120 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:00.129 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:15:00.129 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:15:00.129 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:14:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:14:59] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-08-17T18:15:00.142 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:00.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:00.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:00.163 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:15:00.163 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:15:00.205 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:15:00.205 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:15:00.239 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:15:00.240 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:15:00.243 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:00.266 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:00.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:00.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:00.267 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:15:00.268 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:15:00.295 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:15:00.296 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:15:00.331 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:15:00.331 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:15:00.363 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:15:00.364 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:15:00.367 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:00.385 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:00 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3126682902' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:15:00.385 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:00 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3126682902' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:15:00.389 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:00.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:00.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:00.491 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:00.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:00.534 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:00.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:00.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:00 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3126682902' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:15:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:15:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:15:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:15:00.635 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:00.671 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:00.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:00.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:00.773 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:00.814 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:00.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:00.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:00.915 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:00.953 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:00.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:00.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:01.055 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:01.089 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:01.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:01.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:01.191 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:01.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:01.234 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:01.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:01.335 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:01.372 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:01.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:01.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:01.474 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:01.496 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:01.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:01.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:01.598 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:01.620 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:01.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:01.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:01 smithi130 ceph-mon[105521]: pgmap v2606: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T18:15:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:01 smithi130 ceph-mon[110068]: pgmap v2606: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T18:15:01.722 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:01.761 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:01.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:01.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:01.804 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:01 smithi179 ceph-mon[93519]: pgmap v2606: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T18:15:01.863 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:01.899 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:01.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:01.999 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:02.022 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:02.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:02.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:02.124 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:02.146 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:02.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:02.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:02.248 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:02.287 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:02.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:02.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:02.388 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:02.410 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:02.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:02.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:02.512 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:02.534 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:02.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:02.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:02.636 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:02.674 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:02.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:02.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:02.776 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:02.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:02.812 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:02.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:02.913 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:02.935 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:02.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:02.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:03.037 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:03.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:03.079 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:03.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:03.180 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:03.202 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:03.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:03.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:03.281 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:03 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T18:15:03.202+0000 7fa856f19640 -1 mon.a@0(leader).osd e1394 definitely_dead 0 2024-08-17T18:15:03.305 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:03.327 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:03.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:03.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:03.391 INFO:teuthology.orchestra.run.smithi130.stderr:marked in osd.1. 2024-08-17T18:15:03.429 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:03.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:03.463 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:03.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:03.564 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:03.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:03.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:03.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:03.597 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:03 smithi130 ceph-mon[105521]: pgmap v2607: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T18:15:03.597 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:03 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1223626121' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T18:15:03.597 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:03 smithi130 ceph-mon[110068]: pgmap v2607: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T18:15:03.597 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:03 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1223626121' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T18:15:03.688 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:03.709 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:03.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:03.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:03.811 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:03.832 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:03.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:03.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:03.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:03 smithi179 ceph-mon[93519]: pgmap v2607: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 340 GiB / 358 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T18:15:03.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:03 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1223626121' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T18:15:03.934 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:03.956 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:03.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:03.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:04.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:04.078 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:04.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:04.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:04.180 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:04.202 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:04.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:04.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:04.223 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:15:04.304 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:04.326 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:04.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:04.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:04.427 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:04.450 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:04.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:04.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:04.551 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:04.574 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:04.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:04.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:04 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1223626121' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T18:15:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:04 smithi130 ceph-mon[105521]: osdmap e1395: 8 total, 8 up, 5 in 2024-08-17T18:15:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:04 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1223626121' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T18:15:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:04 smithi130 ceph-mon[110068]: osdmap e1395: 8 total, 8 up, 5 in 2024-08-17T18:15:04.675 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:04.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:04.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:04.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:04.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:04.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:04.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:04.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:04 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1223626121' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T18:15:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:04 smithi179 ceph-mon[93519]: osdmap e1395: 8 total, 8 up, 5 in 2024-08-17T18:15:04.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:04.944 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:04.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:04.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:05.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:05.067 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:05.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:05.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:05.169 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:05.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:05.192 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:05.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:05.293 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:05.314 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:05.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:05.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:05.416 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:05.453 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:05.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:05.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:05.555 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:05.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:05.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:05.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:05.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:05 smithi130 ceph-mon[105521]: pgmap v2609: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 1.17k op/s 2024-08-17T18:15:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:05 smithi130 ceph-mon[105521]: osdmap e1396: 8 total, 8 up, 5 in 2024-08-17T18:15:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:05 smithi130 ceph-mon[110068]: pgmap v2609: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 1.17k op/s 2024-08-17T18:15:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:05 smithi130 ceph-mon[110068]: osdmap e1396: 8 total, 8 up, 5 in 2024-08-17T18:15:05.678 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:05.700 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:05.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:05.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:05.802 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:05.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:05 smithi179 ceph-mon[93519]: pgmap v2609: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 1.17k op/s 2024-08-17T18:15:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:05 smithi179 ceph-mon[93519]: osdmap e1396: 8 total, 8 up, 5 in 2024-08-17T18:15:05.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:05.849 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:05.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:05.950 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:06.007 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:06.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:06.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:06.043 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:15:06.109 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:06.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:06.154 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:06.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:06.255 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:06.278 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:06.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:06.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:06.380 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:06.431 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:06.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:06.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:06.533 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:06.556 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:06.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:06.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:06.657 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:06.679 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:06.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:06.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:06.781 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:06.824 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:06.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:06.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:06.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:06 smithi179 ceph-mon[93519]: 49.b starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9163 2024-08-17T18:15:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:06 smithi179 ceph-mon[93519]: 49.d starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9345 2024-08-17T18:15:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:06 smithi179 ceph-mon[93519]: 49.6 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9288 2024-08-17T18:15:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:06 smithi179 ceph-mon[93519]: 49.4 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9325 2024-08-17T18:15:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:06 smithi179 ceph-mon[93519]: 49.1 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9443 2024-08-17T18:15:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:06 smithi179 ceph-mon[93519]: osdmap e1397: 8 total, 8 up, 5 in 2024-08-17T18:15:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:06 smithi179 ceph-mon[93519]: 49.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9262 2024-08-17T18:15:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:06 smithi179 ceph-mon[93519]: 49.2 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9281 2024-08-17T18:15:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:06 smithi179 ceph-mon[93519]: 49.7 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9338 2024-08-17T18:15:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:06 smithi179 ceph-mon[93519]: 49.e starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9266 2024-08-17T18:15:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:06 smithi179 ceph-mon[93519]: 49.f starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9243 2024-08-17T18:15:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:06 smithi179 ceph-mon[93519]: osdmap e1398: 8 total, 8 up, 5 in 2024-08-17T18:15:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[105521]: 49.b starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9163 2024-08-17T18:15:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[105521]: 49.d starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9345 2024-08-17T18:15:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[105521]: 49.6 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9288 2024-08-17T18:15:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[105521]: 49.4 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9325 2024-08-17T18:15:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[105521]: 49.1 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9443 2024-08-17T18:15:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[105521]: osdmap e1397: 8 total, 8 up, 5 in 2024-08-17T18:15:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[105521]: 49.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9262 2024-08-17T18:15:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[105521]: 49.2 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9281 2024-08-17T18:15:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[105521]: 49.7 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9338 2024-08-17T18:15:06.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[105521]: 49.e starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9266 2024-08-17T18:15:06.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[105521]: 49.f starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9243 2024-08-17T18:15:06.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[105521]: osdmap e1398: 8 total, 8 up, 5 in 2024-08-17T18:15:06.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[110068]: 49.b starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9163 2024-08-17T18:15:06.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[110068]: 49.d starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9345 2024-08-17T18:15:06.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[110068]: 49.6 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9288 2024-08-17T18:15:06.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[110068]: 49.4 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9325 2024-08-17T18:15:06.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[110068]: 49.1 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9443 2024-08-17T18:15:06.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[110068]: osdmap e1397: 8 total, 8 up, 5 in 2024-08-17T18:15:06.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[110068]: 49.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9262 2024-08-17T18:15:06.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[110068]: 49.2 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9281 2024-08-17T18:15:06.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[110068]: 49.7 starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9338 2024-08-17T18:15:06.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[110068]: 49.e starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9266 2024-08-17T18:15:06.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[110068]: 49.f starting backfill to osd.1 from (0'0,0'0] MAX to 1394'9243 2024-08-17T18:15:06.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:06 smithi130 ceph-mon[110068]: osdmap e1398: 8 total, 8 up, 5 in 2024-08-17T18:15:06.926 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:06.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:06.950 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:06.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:07.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:07.072 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:07.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:07.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:07.174 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:07.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:07.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:07.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:07.314 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:07.336 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:07.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:07.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:07.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:07.460 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:07.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:07.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:07.562 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:07.600 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:07.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:07.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:07.667 INFO:tasks.thrashosds.thrasher:Added osd 1 2024-08-17T18:15:07.703 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:07.726 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:07.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:07.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:07.828 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:07.831 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:07 smithi130 ceph-mon[105521]: pgmap v2612: 89 pgs: 3 remapped+peering, 8 peering, 78 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T18:15:07.831 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:07 smithi130 ceph-mon[105521]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T18:15:07.832 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:07 smithi130 ceph-mon[105521]: osdmap e1399: 8 total, 8 up, 5 in 2024-08-17T18:15:07.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:07 smithi130 ceph-mon[110068]: pgmap v2612: 89 pgs: 3 remapped+peering, 8 peering, 78 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T18:15:07.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:07 smithi130 ceph-mon[110068]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T18:15:07.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:07 smithi130 ceph-mon[110068]: osdmap e1399: 8 total, 8 up, 5 in 2024-08-17T18:15:07.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:07 smithi179 ceph-mon[93519]: pgmap v2612: 89 pgs: 3 remapped+peering, 8 peering, 78 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T18:15:07.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:07 smithi179 ceph-mon[93519]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T18:15:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:07 smithi179 ceph-mon[93519]: osdmap e1399: 8 total, 8 up, 5 in 2024-08-17T18:15:07.850 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:07.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:07.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:07.952 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:07.974 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:07.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:07.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:07.992 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:15:07.992 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:15:08.075 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:08.114 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:08.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:08.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:08.216 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:08.261 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:08.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:08.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:08.413 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:08.436 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:08.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:08.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:08.538 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:08.580 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:08.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:08.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:08.682 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:08.705 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:08.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:08.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:08.807 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:08.848 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:08.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:08.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:08.950 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:08.973 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:08.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:08.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:09.074 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:09.089 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T18:15:09.097 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:09.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:09.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:09.199 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:09.222 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:09.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:09.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:09.324 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:09.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:09.404 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:09.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:09.487 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:15:09.505 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:09.572 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:09.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:09.636 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:09 smithi130 ceph-mon[105521]: pgmap v2615: 89 pgs: 3 remapped+peering, 8 peering, 78 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 181 op/s 2024-08-17T18:15:09.638 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:09 smithi130 ceph-mon[110068]: pgmap v2615: 89 pgs: 3 remapped+peering, 8 peering, 78 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 181 op/s 2024-08-17T18:15:09.673 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:09.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:09.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:09.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:09.797 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:09.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:09.843 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:09.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:09.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:09 smithi179 ceph-mon[93519]: pgmap v2615: 89 pgs: 3 remapped+peering, 8 peering, 78 active+clean; 4.3 GiB data, 17 GiB used, 341 GiB / 358 GiB avail; 0 B/s wr, 181 op/s 2024-08-17T18:15:09.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:15:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:15:09] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-08-17T18:15:09.945 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:09.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:09.983 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:09.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:10.084 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:10.106 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:10.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:10.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:10.208 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:10.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:10.232 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:10.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:10.318 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:15:10.333 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:10.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:10.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:10.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:10.444 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:15:10.457 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:10.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:10.498 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:10.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:10 smithi179 ceph-mon[93519]: pgmap v2616: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 3 remapped+peering, 79 active+clean; 4.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 512 op/s; 28982/202911 objects misplaced (14.283%); 4.7 MiB/s, 75 objects/s recovering 2024-08-17T18:15:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:10 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2080304923' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:15:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:15:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:15:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:15:10.598 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:10.621 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:10.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:10.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:10.723 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:10.745 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:10.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:10.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:10.780 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:10 smithi130 ceph-mon[105521]: pgmap v2616: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 3 remapped+peering, 79 active+clean; 4.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 512 op/s; 28982/202911 objects misplaced (14.283%); 4.7 MiB/s, 75 objects/s recovering 2024-08-17T18:15:10.780 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:10 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2080304923' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:15:10.781 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:10 smithi130 ceph-mon[110068]: pgmap v2616: 89 pgs: 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 3 remapped+peering, 79 active+clean; 4.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 512 op/s; 28982/202911 objects misplaced (14.283%); 4.7 MiB/s, 75 objects/s recovering 2024-08-17T18:15:10.781 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:10 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2080304923' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:15:10.846 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:10.887 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:10.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:10.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:10.989 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:11.026 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:11.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:11.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:11.128 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:11.179 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:11.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:11.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:11.280 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:11.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:11.304 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:11.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:11.405 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:11.427 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:11.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:11.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:11.529 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:11.567 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:11.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:11.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:11.669 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:11.689 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:11 smithi130 ceph-mon[105521]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T18:15:11.689 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:11 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T18:15:11.689 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:11 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:15:11.689 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:11 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2080304923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:15:11.689 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:11 smithi130 ceph-mon[105521]: osdmap e1400: 8 total, 8 up, 5 in 2024-08-17T18:15:11.690 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:11 smithi130 ceph-mon[110068]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T18:15:11.690 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:11 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T18:15:11.690 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:11 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:15:11.690 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:11 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2080304923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:15:11.690 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:11 smithi130 ceph-mon[110068]: osdmap e1400: 8 total, 8 up, 5 in 2024-08-17T18:15:11.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:11.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:11.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:11.794 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:11.817 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:11.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:11.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:11 smithi179 ceph-mon[93519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T18:15:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:11 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T18:15:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:11 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:15:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:11 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2080304923' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:15:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:11 smithi179 ceph-mon[93519]: osdmap e1400: 8 total, 8 up, 5 in 2024-08-17T18:15:11.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:11.942 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:11.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:11.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:12.044 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:12.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:12.067 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:12.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:12.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:12.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:12.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:12.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:12.316 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:12.364 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:12.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:12.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:12.465 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:12.503 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:12.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:12.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:12.606 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:12.628 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:12.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:12.668 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 7, 1] out_osds: [5, 4, 6] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2024-08-17T18:15:12.668 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:15:12.669 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-08-17T18:15:12.669 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T18:15:12.669 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T18:15:12.669 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-08-17T18:15:12.730 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:12.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:12.779 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:12.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:12.880 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:12.901 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:12.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:12.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:13.003 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:13.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:13.044 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:13.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:13.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:13.168 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:13.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:13.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:13.270 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:13.292 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:13.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:13.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:13.394 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:13.433 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:13.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:13.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:13.535 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:13.572 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:13.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:13.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:13 smithi179 ceph-mon[93519]: pgmap v2618: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 4.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 41267/200712 objects misplaced (20.560%); 4.1 MiB/s, 64 objects/s recovering 2024-08-17T18:15:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:15:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:15:13.593 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 18:15:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=cleanup t=2024-08-17T18:15:13.13534773Z level=info msg="Completed cleanup jobs" duration=13.626103ms 2024-08-17T18:15:13.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:13 smithi130 ceph-mon[105521]: pgmap v2618: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 4.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 41267/200712 objects misplaced (20.560%); 4.1 MiB/s, 64 objects/s recovering 2024-08-17T18:15:13.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:15:13.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:15:13.662 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:13 smithi130 ceph-mon[110068]: pgmap v2618: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 4.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 533 op/s; 41267/200712 objects misplaced (20.560%); 4.1 MiB/s, 64 objects/s recovering 2024-08-17T18:15:13.662 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:15:13.662 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:15:13.674 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:13.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:13.696 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:13.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:13.732 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:15:13.797 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:13.818 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:13.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:13.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:13.920 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:13.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:13.979 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:13.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:14.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:14.149 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:14.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:14.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:14.251 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:14.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:14.293 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:14.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:14.394 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:14.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:14.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:14.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:14.518 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:14.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:14.542 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:14.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:14.643 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:14.670 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:14.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:14.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:14.772 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:14.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:14.795 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:14.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:14.897 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:14.919 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:14.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:14.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:15.021 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:15.062 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:15.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:15.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:15.154 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:15:15.154 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:15:15.165 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:15.223 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:15.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:15.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:15.325 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:15.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:15.371 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:15.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:15.428 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:15:15.473 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:15.495 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:15.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:15.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:15.596 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:15.646 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:15.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:15.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:15.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:15 smithi130 ceph-mon[105521]: pgmap v2619: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 722 op/s; 39663/194082 objects misplaced (20.436%); 8.8 MiB/s, 140 objects/s recovering 2024-08-17T18:15:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:15 smithi130 ceph-mon[110068]: pgmap v2619: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 722 op/s; 39663/194082 objects misplaced (20.436%); 8.8 MiB/s, 140 objects/s recovering 2024-08-17T18:15:15.749 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:15.790 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:15.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:15.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:15.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:15 smithi179 ceph-mon[93519]: pgmap v2619: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.9 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 722 op/s; 39663/194082 objects misplaced (20.436%); 8.8 MiB/s, 140 objects/s recovering 2024-08-17T18:15:15.878 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:15:15.888 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T18:15:15.892 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:15.982 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:15.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:15.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:16.084 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:16.149 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:16.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:16.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:16.251 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:16.273 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:16.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:16.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:16.375 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:16.439 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:16.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:16.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:16.540 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:16.562 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:16.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:16.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:16.664 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:16.687 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:16.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:16.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:16.789 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:16.832 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:16.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:16.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:16.934 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:16.956 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:16.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:16.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:17.058 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:17.103 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:17.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:17.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:17.205 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:17.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:17.227 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:17.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:17.328 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:17.366 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:17.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:17.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:17.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:17 smithi130 ceph-mon[105521]: pgmap v2620: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 837 op/s; 38237/188985 objects misplaced (20.233%); 7.7 MiB/s, 123 objects/s recovering 2024-08-17T18:15:17.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:17 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1624529770' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:15:17.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:17 smithi130 ceph-mon[110068]: pgmap v2620: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 837 op/s; 38237/188985 objects misplaced (20.233%); 7.7 MiB/s, 123 objects/s recovering 2024-08-17T18:15:17.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:17 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1624529770' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:15:17.468 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:17.490 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:17.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:17.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:17.592 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:17.632 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:17.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:17.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:17.734 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:17.756 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:17.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:17.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:17.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:17 smithi179 ceph-mon[93519]: pgmap v2620: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 837 op/s; 38237/188985 objects misplaced (20.233%); 7.7 MiB/s, 123 objects/s recovering 2024-08-17T18:15:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:17 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1624529770' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:15:17.858 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:17.880 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:17.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:17.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:17.982 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:18.005 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:18.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:18.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:18.098 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:15:18.098 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:15:18.098 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:15:18.098 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:15:18.098 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:15:18.098 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:15:18.098 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:15:18.098 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:15:18.099 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:15:18.099 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:15:18.099 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:15:18.099 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:15:18.099 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:15:18.099 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:15:18.099 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:15:18.099 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:15:18.107 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:18.146 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:18.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:18.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:18.247 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:18.270 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:18.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:18.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:18.372 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:18.394 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:18.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:18.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:18.496 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:18.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:18.534 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:18.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:18.635 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:18.674 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:18.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:18.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:18.776 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:18.797 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:18.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:18.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:18.899 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:18.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:18.943 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:18.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:19.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:19.082 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:19.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:19.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:19.184 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:19.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:19.206 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:19.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:19.307 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:19.330 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:19.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:19.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:19.432 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:19.475 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:19.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:19.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:19.577 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:19.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:19 smithi179 ceph-mon[93519]: pgmap v2621: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 710 op/s; 38237/188985 objects misplaced (20.233%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T18:15:19.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:19 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:15:19.598 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:19.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:19.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:19.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:19 smithi130 ceph-mon[105521]: pgmap v2621: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 710 op/s; 38237/188985 objects misplaced (20.233%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T18:15:19.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:19 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:15:19.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:19 smithi130 ceph-mon[110068]: pgmap v2621: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.8 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 710 op/s; 38237/188985 objects misplaced (20.233%); 6.6 MiB/s, 105 objects/s recovering 2024-08-17T18:15:19.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:19 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:15:19.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:19.742 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:19.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:19.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:19.844 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:19.874 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:19.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:19.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:19.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:20.014 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:20.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:20.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:20.116 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:20.139 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:20.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:20.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:15:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:15:19] "GET /metrics HTTP/1.1" 200 36813 "" "Prometheus/2.51.0" 2024-08-17T18:15:20.241 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:20.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:20.286 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:20.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:20.387 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:20.431 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:20.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:20.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:20.494 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:15:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:15:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:15:20.533 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:20.556 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:20.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:20.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:20.657 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:20.679 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:20.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:20.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:20.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:20.816 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:20.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:20.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:20.918 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:20.942 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:20.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:20.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:21.043 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:21.071 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:21.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:21.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:21.173 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:21.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:21.224 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:21.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:21.325 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:21.348 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:21.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:21.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:21.450 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:21.500 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:21.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:21.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:21.580 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:21 smithi130 ceph-mon[105521]: pgmap v2622: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 754 op/s; 36105/180273 objects misplaced (20.028%); 8.4 MiB/s, 135 objects/s recovering 2024-08-17T18:15:21.580 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:21 smithi130 ceph-mon[110068]: pgmap v2622: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 754 op/s; 36105/180273 objects misplaced (20.028%); 8.4 MiB/s, 135 objects/s recovering 2024-08-17T18:15:21.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:21 smithi179 ceph-mon[93519]: pgmap v2622: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 754 op/s; 36105/180273 objects misplaced (20.028%); 8.4 MiB/s, 135 objects/s recovering 2024-08-17T18:15:21.603 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:21.646 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:21.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:21.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:21.748 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:21.784 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:21.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:21.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:21.886 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:21.926 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:21.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:21.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:22.028 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:22.063 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:22.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:22.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:22.165 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:22.219 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:22.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:22.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:22.364 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:22.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:22.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:22.466 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:22.512 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:22.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:22.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:22.614 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:22.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:22.638 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:22.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:22.739 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:22.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:22.762 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:22.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:22.863 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:22.885 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:22.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:22.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:22.987 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:23.027 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:23.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:23.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:23.129 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:23.167 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:23.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:23.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:23.269 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:23.313 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:23.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:23.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:23.415 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:23.446 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:23 smithi130 ceph-mon[105521]: pgmap v2623: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 678 op/s; 35395/178143 objects misplaced (19.869%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T18:15:23.446 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:23 smithi130 ceph-mon[110068]: pgmap v2623: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 678 op/s; 35395/178143 objects misplaced (19.869%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T18:15:23.455 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:23.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:23.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:23.557 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:23.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:23 smithi179 ceph-mon[93519]: pgmap v2623: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.6 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 678 op/s; 35395/178143 objects misplaced (19.869%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T18:15:23.601 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:23.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:23.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:23.703 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:23.726 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:23.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:23.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:23.828 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:23.900 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:23.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:23.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:24.002 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:24.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:24.026 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:24.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:24.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:24.165 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:24.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:24.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:24.267 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:24.310 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:24.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:24.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:24.411 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:24.456 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:24.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:24.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:24.558 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:24.600 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:24.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:24.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:24.613 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:15:24.613 INFO:teuthology.orchestra.run.smithi130.stdout: "success": "" 2024-08-17T18:15:24.613 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:15:24.618 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:15:24.703 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:24.819 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:24.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:24.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:24.921 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:24.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:24.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:24.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:25.044 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:25.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:25.068 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:25.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:25.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:25.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:25.210 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:25.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:25.311 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:25.351 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:25.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:25.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:25.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:25 smithi130 ceph-mon[105521]: pgmap v2624: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 798 op/s; 33900/171969 objects misplaced (19.713%); 11 MiB/s, 169 objects/s recovering 2024-08-17T18:15:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:25 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/915256734' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:15:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:25 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:15:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:25 smithi130 ceph-mon[110068]: pgmap v2624: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 798 op/s; 33900/171969 objects misplaced (19.713%); 11 MiB/s, 169 objects/s recovering 2024-08-17T18:15:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:25 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/915256734' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:15:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:25 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:15:25.453 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:25.490 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:25.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:25.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:25.592 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:25 smithi179 ceph-mon[93519]: pgmap v2624: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 798 op/s; 33900/171969 objects misplaced (19.713%); 11 MiB/s, 169 objects/s recovering 2024-08-17T18:15:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:25 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/915256734' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:15:25.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:25 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:15:25.636 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:25.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:25.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:25.738 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:25.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:25.760 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:25.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:25.861 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:25.883 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:25.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:25.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:25.985 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:26.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:26.007 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:26.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:26.108 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:26.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:26.144 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:26.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:26.185 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:15:26.245 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:26.298 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:15:26.299 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:26 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:15:26.299 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:26 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:15:26.299 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:26 smithi130 ceph-mon[105521]: osdmap e1401: 8 total, 8 up, 5 in 2024-08-17T18:15:26.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:26 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:15:26.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:26 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:15:26.299 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:26 smithi130 ceph-mon[110068]: osdmap e1401: 8 total, 8 up, 5 in 2024-08-17T18:15:26.308 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:26.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:26.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:26.410 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:26.458 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:26.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:26.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:26.559 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:26 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:15:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:26 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:15:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:26 smithi179 ceph-mon[93519]: osdmap e1401: 8 total, 8 up, 5 in 2024-08-17T18:15:26.623 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:26.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:26.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:26.725 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:26.778 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:26.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:26.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:26.880 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:26.902 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:26.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:26.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:27.004 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:27.025 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:27.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:27.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:27.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:27.164 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:27.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:27.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:27.266 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:27.288 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:27.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:27.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:27.390 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:27.442 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:27.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:27.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:27.494 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:27 smithi130 ceph-mon[110068]: pgmap v2626: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 724 op/s; 32547/167238 objects misplaced (19.461%); 8.5 MiB/s, 136 objects/s recovering 2024-08-17T18:15:27.494 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:27 smithi130 ceph-mon[105521]: pgmap v2626: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 724 op/s; 32547/167238 objects misplaced (19.461%); 8.5 MiB/s, 136 objects/s recovering 2024-08-17T18:15:27.544 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:27.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:27.566 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:27.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:27.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:27 smithi179 ceph-mon[93519]: pgmap v2626: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 724 op/s; 32547/167238 objects misplaced (19.461%); 8.5 MiB/s, 136 objects/s recovering 2024-08-17T18:15:27.667 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:27.690 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:27.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:27.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:27.792 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:27.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:27.846 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:27.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:27.947 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:27.970 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:27.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:27.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:28.072 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:28.093 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:28.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:28.195 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:28.239 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:28.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:28.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:28.336 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:15:28.342 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:28.402 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:28.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:28.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:28.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:15:28.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:15:28.504 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:28.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:28.545 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:28.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:28.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:15:28.646 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:28.669 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:28.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:28.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:28.771 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:28.816 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:28.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:28.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:28.917 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:28.940 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:28.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:28.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:29.041 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:29.090 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:29.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:29.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:29.192 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:29.215 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:29.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:29.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:29.317 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:29.346 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:29 smithi130 ceph-mon[105521]: pgmap v2627: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 724 op/s; 32547/167238 objects misplaced (19.461%); 8.5 MiB/s, 136 objects/s recovering 2024-08-17T18:15:29.346 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:29 smithi130 ceph-mon[110068]: pgmap v2627: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 724 op/s; 32547/167238 objects misplaced (19.461%); 8.5 MiB/s, 136 objects/s recovering 2024-08-17T18:15:29.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:29.364 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:29.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:29.412 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:15:29.464 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:29.486 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:29.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:29.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:29.588 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:29.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:29 smithi179 ceph-mon[93519]: pgmap v2627: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.4 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 724 op/s; 32547/167238 objects misplaced (19.461%); 8.5 MiB/s, 136 objects/s recovering 2024-08-17T18:15:29.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:29.630 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:29.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:29.731 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:29.752 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:29.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:29.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:29.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:29.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:29.876 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:29.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:29.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:15:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:15:29] "GET /metrics HTTP/1.1" 200 36809 "" "Prometheus/2.51.0" 2024-08-17T18:15:29.977 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:30.024 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:30.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:30.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:30.056 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:15:30.056 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:15:30.056 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:15:30.056 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:15:30.056 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.056 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:115176 47.9 47:9407bc2a:::benchmark_data_smithi130_302447_object19941:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:15:30.056 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:56.561611+0000", 2024-08-17T18:15:30.056 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 573.12892584400004, 2024-08-17T18:15:30.057 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0437001399999999, 2024-08-17T18:15:30.057 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.057 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.057 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:15:30.057 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.057 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:15:30.057 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:15:30.057 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115176 2024-08-17T18:15:30.057 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.057 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.057 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.058 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.058 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.561611+0000", 2024-08-17T18:15:30.058 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.058 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.058 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.058 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.058 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.561611+0000", 2024-08-17T18:15:30.058 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.058 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.058 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.059 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.059 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.561612+0000", 2024-08-17T18:15:30.059 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.742e-06 2024-08-17T18:15:30.059 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.059 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.059 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.059 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.561619+0000", 2024-08-17T18:15:30.059 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.764e-06 2024-08-17T18:15:30.059 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.059 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.059 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.059 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.561633+0000", 2024-08-17T18:15:30.059 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4076999999999999e-05 2024-08-17T18:15:30.060 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.060 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.060 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.060 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.561640+0000", 2024-08-17T18:15:30.060 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9e-06 2024-08-17T18:15:30.060 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.060 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.060 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.060 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.561982+0000", 2024-08-17T18:15:30.060 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00034202400000000002 2024-08-17T18:15:30.060 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.060 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.060 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.061 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.562098+0000", 2024-08-17T18:15:30.061 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000115841 2024-08-17T18:15:30.061 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.061 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.061 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3", 2024-08-17T18:15:30.061 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.562216+0000", 2024-08-17T18:15:30.061 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011751700000000001 2024-08-17T18:15:30.061 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.061 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.061 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:15:30.061 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.568753+0000", 2024-08-17T18:15:30.061 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0065377830000000001 2024-08-17T18:15:30.061 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.061 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.062 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.062 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569428+0000", 2024-08-17T18:15:30.062 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00067508300000000004 2024-08-17T18:15:30.062 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.062 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.062 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.062 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.078952+0000", 2024-08-17T18:15:30.062 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.50952358900000005 2024-08-17T18:15:30.062 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.062 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.062 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:15:30.062 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.078963+0000", 2024-08-17T18:15:30.063 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1479e-05 2024-08-17T18:15:30.063 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.063 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.063 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.063 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.604892+0000", 2024-08-17T18:15:30.063 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.52592878799999998 2024-08-17T18:15:30.063 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.063 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.063 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.063 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.605311+0000", 2024-08-17T18:15:30.063 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000418553 2024-08-17T18:15:30.064 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.064 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.064 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.064 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.064 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.064 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:115183 47.8 47:10d7c7c7:::benchmark_data_smithi130_302447_object19948:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:15:30.064 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:56.569147+0000", 2024-08-17T18:15:30.064 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 573.12138985700005, 2024-08-17T18:15:30.064 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.033011946, 2024-08-17T18:15:30.064 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.064 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.064 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:15:30.064 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.065 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:15:30.065 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:15:30.065 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115183 2024-08-17T18:15:30.065 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.065 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.065 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.065 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.065 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569147+0000", 2024-08-17T18:15:30.065 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.065 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.065 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.065 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.065 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569147+0000", 2024-08-17T18:15:30.066 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.066 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.066 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.066 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.066 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569147+0000", 2024-08-17T18:15:30.066 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3600000000000002e-07 2024-08-17T18:15:30.066 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.067 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.067 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.067 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569151+0000", 2024-08-17T18:15:30.067 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9489999999999998e-06 2024-08-17T18:15:30.067 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.067 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.067 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.067 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569162+0000", 2024-08-17T18:15:30.067 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0145e-05 2024-08-17T18:15:30.068 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.068 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.068 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.068 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569168+0000", 2024-08-17T18:15:30.068 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4829999999999996e-06 2024-08-17T18:15:30.068 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.068 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.068 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.068 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569522+0000", 2024-08-17T18:15:30.068 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00035411600000000002 2024-08-17T18:15:30.068 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.068 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.068 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.068 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569634+0000", 2024-08-17T18:15:30.069 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000111377 2024-08-17T18:15:30.069 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.069 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.069 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:15:30.069 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569746+0000", 2024-08-17T18:15:30.069 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011209099999999999 2024-08-17T18:15:30.069 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.069 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.069 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:15:30.069 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.578576+0000", 2024-08-17T18:15:30.069 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0088299959999999997 2024-08-17T18:15:30.069 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.069 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.070 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.070 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.580322+0000", 2024-08-17T18:15:30.070 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001746526 2024-08-17T18:15:30.070 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.070 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.070 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.070 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.085525+0000", 2024-08-17T18:15:30.070 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.50520276099999994 2024-08-17T18:15:30.070 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.070 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.070 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:15:30.070 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.085533+0000", 2024-08-17T18:15:30.070 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2859999999999999e-06 2024-08-17T18:15:30.071 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.071 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.071 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.071 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.600605+0000", 2024-08-17T18:15:30.071 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.51507130999999995 2024-08-17T18:15:30.071 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.071 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.071 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.071 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.602159+0000", 2024-08-17T18:15:30.071 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0015540700000000001 2024-08-17T18:15:30.071 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.071 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.071 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.071 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.072 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.072 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:115188 47.a 47:5595e2c5:::benchmark_data_smithi130_302447_object19953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:15:30.072 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:56.569203+0000", 2024-08-17T18:15:30.072 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 573.12133327699996, 2024-08-17T18:15:30.072 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0298969609999999, 2024-08-17T18:15:30.072 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.072 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.072 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:15:30.072 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.072 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:15:30.072 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:15:30.072 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115188 2024-08-17T18:15:30.072 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.073 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.073 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.073 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.073 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569203+0000", 2024-08-17T18:15:30.073 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.073 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.073 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.073 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.073 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569203+0000", 2024-08-17T18:15:30.073 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.073 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.073 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.073 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.074 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569204+0000", 2024-08-17T18:15:30.074 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7e-07 2024-08-17T18:15:30.074 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.074 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.074 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.074 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569205+0000", 2024-08-17T18:15:30.074 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6649999999999999e-06 2024-08-17T18:15:30.074 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.074 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.074 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.074 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569214+0000", 2024-08-17T18:15:30.074 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.32e-06 2024-08-17T18:15:30.074 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.075 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.075 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.075 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569218+0000", 2024-08-17T18:15:30.075 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6020000000000002e-06 2024-08-17T18:15:30.075 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.075 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.075 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.075 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.570335+0000", 2024-08-17T18:15:30.075 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001116403 2024-08-17T18:15:30.075 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.075 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.076 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.076 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.570434+0000", 2024-08-17T18:15:30.076 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9745000000000001e-05 2024-08-17T18:15:30.076 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.076 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.076 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:15:30.076 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.570532+0000", 2024-08-17T18:15:30.076 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7367000000000005e-05 2024-08-17T18:15:30.076 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.076 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.077 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:15:30.077 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.578589+0000", 2024-08-17T18:15:30.077 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0080577349999999999 2024-08-17T18:15:30.077 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.077 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.077 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.077 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.580053+0000", 2024-08-17T18:15:30.077 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001463102 2024-08-17T18:15:30.077 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.077 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.077 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.077 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.088806+0000", 2024-08-17T18:15:30.077 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.50875339600000002 2024-08-17T18:15:30.078 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.078 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.078 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:15:30.078 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.088818+0000", 2024-08-17T18:15:30.078 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1970999999999999e-05 2024-08-17T18:15:30.078 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.078 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.078 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.078 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.598686+0000", 2024-08-17T18:15:30.078 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.50986771200000003 2024-08-17T18:15:30.078 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.078 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.078 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.079 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.599100+0000", 2024-08-17T18:15:30.079 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000414573 2024-08-17T18:15:30.079 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.079 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.079 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.079 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.079 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.079 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:115189 47.9 47:903d1e7c:::benchmark_data_smithi130_302447_object19954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:15:30.079 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:56.569357+0000", 2024-08-17T18:15:30.079 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 573.12117917299997, 2024-08-17T18:15:30.079 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.037520555, 2024-08-17T18:15:30.079 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.079 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.080 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:15:30.080 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.080 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:15:30.080 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:15:30.080 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115189 2024-08-17T18:15:30.080 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.080 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.080 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.080 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.080 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569357+0000", 2024-08-17T18:15:30.080 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.080 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.080 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.080 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.081 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569357+0000", 2024-08-17T18:15:30.081 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.081 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.081 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.081 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.081 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569358+0000", 2024-08-17T18:15:30.081 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9800000000000003e-07 2024-08-17T18:15:30.081 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.081 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.081 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.081 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569361+0000", 2024-08-17T18:15:30.081 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.3110000000000001e-06 2024-08-17T18:15:30.081 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.082 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.082 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.082 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569370+0000", 2024-08-17T18:15:30.082 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2320000000000007e-06 2024-08-17T18:15:30.082 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.082 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.082 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.082 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569376+0000", 2024-08-17T18:15:30.082 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9989999999999997e-06 2024-08-17T18:15:30.082 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.082 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.082 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.082 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569455+0000", 2024-08-17T18:15:30.083 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8801999999999995e-05 2024-08-17T18:15:30.083 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.083 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.083 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.083 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569553+0000", 2024-08-17T18:15:30.083 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7718000000000002e-05 2024-08-17T18:15:30.083 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.083 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.083 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3", 2024-08-17T18:15:30.083 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.569656+0000", 2024-08-17T18:15:30.083 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010297399999999999 2024-08-17T18:15:30.083 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.083 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.084 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:15:30.084 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.572594+0000", 2024-08-17T18:15:30.084 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029382890000000002 2024-08-17T18:15:30.084 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.084 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.084 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.084 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586487+0000", 2024-08-17T18:15:30.084 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013892432 2024-08-17T18:15:30.084 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.084 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.084 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.084 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.080683+0000", 2024-08-17T18:15:30.084 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.49419588399999997 2024-08-17T18:15:30.084 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.085 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.085 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:15:30.085 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.080694+0000", 2024-08-17T18:15:30.085 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1083e-05 2024-08-17T18:15:30.085 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.085 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.085 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.085 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.605603+0000", 2024-08-17T18:15:30.085 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.52490958399999998 2024-08-17T18:15:30.085 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.085 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.086 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.086 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.606878+0000", 2024-08-17T18:15:30.086 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0012746490000000001 2024-08-17T18:15:30.086 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.086 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.086 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.086 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.086 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.086 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:115191 47.9 47:9de83f78:::benchmark_data_smithi130_302447_object19956:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:15:30.086 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:56.580381+0000", 2024-08-17T18:15:30.086 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 573.11015560700002, 2024-08-17T18:15:30.086 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0266580750000001, 2024-08-17T18:15:30.086 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.087 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.087 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:15:30.087 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.087 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:15:30.087 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:15:30.087 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115191 2024-08-17T18:15:30.087 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.087 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.087 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.087 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.088 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.580381+0000", 2024-08-17T18:15:30.088 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.088 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.088 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.088 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.088 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.580381+0000", 2024-08-17T18:15:30.088 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.088 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.088 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.088 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.088 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.580383+0000", 2024-08-17T18:15:30.088 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0219999999999999e-06 2024-08-17T18:15:30.089 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.089 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.089 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.089 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.580390+0000", 2024-08-17T18:15:30.089 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8210000000000004e-06 2024-08-17T18:15:30.089 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.089 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.089 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.089 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.580407+0000", 2024-08-17T18:15:30.089 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7237000000000001e-05 2024-08-17T18:15:30.089 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.089 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.089 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.090 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.580415+0000", 2024-08-17T18:15:30.090 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7570000000000004e-06 2024-08-17T18:15:30.090 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.090 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.090 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.090 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.580472+0000", 2024-08-17T18:15:30.090 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7361000000000002e-05 2024-08-17T18:15:30.090 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.090 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.090 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.090 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.580606+0000", 2024-08-17T18:15:30.090 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013365999999999999 2024-08-17T18:15:30.090 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.091 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.091 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3", 2024-08-17T18:15:30.091 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.580739+0000", 2024-08-17T18:15:30.091 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013296599999999999 2024-08-17T18:15:30.091 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.091 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.091 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:15:30.091 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.585396+0000", 2024-08-17T18:15:30.091 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.004657085 2024-08-17T18:15:30.091 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.091 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.091 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.091 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.594455+0000", 2024-08-17T18:15:30.092 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0090589190000000003 2024-08-17T18:15:30.092 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.092 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.092 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.092 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.080728+0000", 2024-08-17T18:15:30.092 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.48627288499999999 2024-08-17T18:15:30.092 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.092 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.092 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:15:30.092 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.080735+0000", 2024-08-17T18:15:30.092 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3000000000000004e-06 2024-08-17T18:15:30.092 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.092 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.092 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.093 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.606980+0000", 2024-08-17T18:15:30.093 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.52624473299999996 2024-08-17T18:15:30.093 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.093 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.093 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.093 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.607039+0000", 2024-08-17T18:15:30.093 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9329000000000001e-05 2024-08-17T18:15:30.093 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.093 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.093 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.093 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.093 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.093 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:115192 47.9 47:90554a8f:::benchmark_data_smithi130_302447_object19957:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:15:30.094 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:56.586398+0000", 2024-08-17T18:15:30.094 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 573.10413868800003, 2024-08-17T18:15:30.094 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.020714213, 2024-08-17T18:15:30.094 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.094 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.094 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:15:30.094 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.094 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:15:30.094 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:15:30.094 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115192 2024-08-17T18:15:30.094 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.094 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.095 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.095 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.095 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586398+0000", 2024-08-17T18:15:30.095 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.095 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.095 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.095 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.095 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586398+0000", 2024-08-17T18:15:30.095 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.095 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.095 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.095 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.096 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586400+0000", 2024-08-17T18:15:30.096 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7689999999999999e-06 2024-08-17T18:15:30.096 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.096 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.096 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.096 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586407+0000", 2024-08-17T18:15:30.096 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2450000000000001e-06 2024-08-17T18:15:30.096 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.096 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.096 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.096 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586420+0000", 2024-08-17T18:15:30.096 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3216e-05 2024-08-17T18:15:30.096 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.097 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.097 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.097 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586428+0000", 2024-08-17T18:15:30.097 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8609999999999998e-06 2024-08-17T18:15:30.097 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.097 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.097 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.097 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586583+0000", 2024-08-17T18:15:30.097 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015472700000000001 2024-08-17T18:15:30.097 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.097 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.097 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.097 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586711+0000", 2024-08-17T18:15:30.098 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012831400000000001 2024-08-17T18:15:30.098 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.098 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.098 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3", 2024-08-17T18:15:30.098 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586857+0000", 2024-08-17T18:15:30.098 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014654499999999999 2024-08-17T18:15:30.098 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.098 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.098 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:15:30.098 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.594004+0000", 2024-08-17T18:15:30.098 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0071463220000000001 2024-08-17T18:15:30.098 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.098 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.099 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.099 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.597680+0000", 2024-08-17T18:15:30.099 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003675864 2024-08-17T18:15:30.099 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.099 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.099 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.099 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.080763+0000", 2024-08-17T18:15:30.099 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.48308351700000002 2024-08-17T18:15:30.099 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.099 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.099 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:15:30.099 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.080771+0000", 2024-08-17T18:15:30.100 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3300000000000001e-06 2024-08-17T18:15:30.100 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.100 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.100 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.100 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.607068+0000", 2024-08-17T18:15:30.100 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.52629757799999999 2024-08-17T18:15:30.100 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.100 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.100 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.100 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.607112+0000", 2024-08-17T18:15:30.100 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3924999999999997e-05 2024-08-17T18:15:30.101 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.101 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.101 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.101 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.101 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.101 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:115194 47.9 47:92fedd67:::benchmark_data_smithi130_302447_object19959:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:15:30.101 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:56.586481+0000", 2024-08-17T18:15:30.101 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 573.10405568800002, 2024-08-17T18:15:30.101 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.020697854, 2024-08-17T18:15:30.101 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.101 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.101 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:15:30.101 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.102 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:15:30.102 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:15:30.102 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115194 2024-08-17T18:15:30.102 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.102 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.102 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.102 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.102 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586481+0000", 2024-08-17T18:15:30.102 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.102 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.102 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.102 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.103 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586481+0000", 2024-08-17T18:15:30.103 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.103 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.103 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.103 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.103 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586481+0000", 2024-08-17T18:15:30.103 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0500000000000003e-07 2024-08-17T18:15:30.103 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.103 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.103 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.103 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586484+0000", 2024-08-17T18:15:30.103 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9919999999999999e-06 2024-08-17T18:15:30.103 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.104 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.104 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.104 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586494+0000", 2024-08-17T18:15:30.104 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2690000000000005e-06 2024-08-17T18:15:30.104 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.104 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.104 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.104 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586500+0000", 2024-08-17T18:15:30.104 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4080000000000003e-06 2024-08-17T18:15:30.104 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.104 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.104 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.105 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.590976+0000", 2024-08-17T18:15:30.105 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0044756049999999997 2024-08-17T18:15:30.105 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.105 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.105 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.105 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.591107+0000", 2024-08-17T18:15:30.105 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013118200000000001 2024-08-17T18:15:30.105 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.105 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.105 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3", 2024-08-17T18:15:30.105 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.591239+0000", 2024-08-17T18:15:30.105 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013204100000000001 2024-08-17T18:15:30.105 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.106 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.106 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:15:30.106 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.594080+0000", 2024-08-17T18:15:30.106 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028409239999999999 2024-08-17T18:15:30.106 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.106 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.106 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.106 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.597776+0000", 2024-08-17T18:15:30.106 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003696047 2024-08-17T18:15:30.106 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.106 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.106 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.107 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.080807+0000", 2024-08-17T18:15:30.107 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.48303063800000001 2024-08-17T18:15:30.107 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.107 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.107 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:15:30.107 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.080814+0000", 2024-08-17T18:15:30.107 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.515e-06 2024-08-17T18:15:30.107 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.107 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.107 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.107 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.607137+0000", 2024-08-17T18:15:30.107 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.52632332199999998 2024-08-17T18:15:30.108 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.108 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.108 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.108 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.607179+0000", 2024-08-17T18:15:30.108 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1306000000000001e-05 2024-08-17T18:15:30.108 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.108 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.108 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.108 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.108 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.108 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.46373.0:115195 47.a 47:5f2e532e:::benchmark_data_smithi130_302447_object19960:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1316)", 2024-08-17T18:15:30.108 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:05:56.586527+0000", 2024-08-17T18:15:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 573.10400986100001, 2024-08-17T18:15:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0233965119999999, 2024-08-17T18:15:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:15:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.46373", 2024-08-17T18:15:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/4189629603", 2024-08-17T18:15:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115195 2024-08-17T18:15:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.109 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586527+0000", 2024-08-17T18:15:30.110 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.110 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.110 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.110 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.110 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586527+0000", 2024-08-17T18:15:30.110 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.110 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.110 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.110 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.110 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586528+0000", 2024-08-17T18:15:30.110 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0800000000000003e-07 2024-08-17T18:15:30.110 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.110 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.111 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.111 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586532+0000", 2024-08-17T18:15:30.111 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3000000000000003e-06 2024-08-17T18:15:30.111 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.111 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.111 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.111 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586541+0000", 2024-08-17T18:15:30.111 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1519999999999998e-06 2024-08-17T18:15:30.111 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.111 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.111 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.111 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586548+0000", 2024-08-17T18:15:30.111 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7669999999999998e-06 2024-08-17T18:15:30.112 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.112 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.112 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.112 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586634+0000", 2024-08-17T18:15:30.112 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6081999999999993e-05 2024-08-17T18:15:30.112 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.112 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.112 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.112 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586732+0000", 2024-08-17T18:15:30.112 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8134999999999995e-05 2024-08-17T18:15:30.112 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.113 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.113 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:15:30.113 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.586900+0000", 2024-08-17T18:15:30.113 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000168281 2024-08-17T18:15:30.113 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.113 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.113 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.113 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.591968+0000", 2024-08-17T18:15:30.113 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0050673619999999997 2024-08-17T18:15:30.113 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.113 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.113 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:15:30.113 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:56.591978+0000", 2024-08-17T18:15:30.114 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0356999999999999e-05 2024-08-17T18:15:30.114 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.114 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.114 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.114 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.088859+0000", 2024-08-17T18:15:30.114 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.49688065100000001 2024-08-17T18:15:30.114 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.114 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.114 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:15:30.114 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.088869+0000", 2024-08-17T18:15:30.114 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0781000000000001e-05 2024-08-17T18:15:30.114 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.599869+0000", 2024-08-17T18:15:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.51100003100000002 2024-08-17T18:15:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:05:57.609923+0000", 2024-08-17T18:15:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010053705 2024-08-17T18:15:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.115 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.116 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115420 48.2 48:4523a0a7:::benchmark_data_smithi130_327597_object115419:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1336)", 2024-08-17T18:15:30.116 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.561920+0000", 2024-08-17T18:15:30.116 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 392.12861613699999, 2024-08-17T18:15:30.116 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.158898897, 2024-08-17T18:15:30.116 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.116 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.116 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:15:30.116 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.116 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:15:30.116 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:15:30.116 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115420 2024-08-17T18:15:30.116 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.116 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.116 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.117 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.117 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.561920+0000", 2024-08-17T18:15:30.117 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.117 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.117 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.117 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.117 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.561920+0000", 2024-08-17T18:15:30.117 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.117 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.117 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.117 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.117 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.561922+0000", 2024-08-17T18:15:30.117 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4529999999999999e-06 2024-08-17T18:15:30.118 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.118 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.118 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.118 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.561949+0000", 2024-08-17T18:15:30.118 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.7398000000000001e-05 2024-08-17T18:15:30.118 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.118 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.118 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.118 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.561959+0000", 2024-08-17T18:15:30.118 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.026e-05 2024-08-17T18:15:30.118 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.118 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.118 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.119 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.561969+0000", 2024-08-17T18:15:30.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9219999999999992e-06 2024-08-17T18:15:30.119 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.119 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.119 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.119 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.573044+0000", 2024-08-17T18:15:30.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011074785 2024-08-17T18:15:30.119 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.119 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.119 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:15:30.119 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.573051+0000", 2024-08-17T18:15:30.119 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6730000000000003e-06 2024-08-17T18:15:30.119 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.119 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.120 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.720724+0000", 2024-08-17T18:15:30.120 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.14767295 2024-08-17T18:15:30.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.120 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.120 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.720819+0000", 2024-08-17T18:15:30.120 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5456000000000002e-05 2024-08-17T18:15:30.120 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.120 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.120 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.120 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.120 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.121 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115423 48.2 48:43ffd269:::benchmark_data_smithi130_327597_object115422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1336)", 2024-08-17T18:15:30.121 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.561980+0000", 2024-08-17T18:15:30.121 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 392.128556726, 2024-08-17T18:15:30.121 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1848453830000001, 2024-08-17T18:15:30.121 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.121 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.121 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:15:30.121 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.121 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:15:30.121 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:15:30.121 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115423 2024-08-17T18:15:30.121 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.121 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.121 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.122 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.561980+0000", 2024-08-17T18:15:30.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.122 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.122 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.122 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.561980+0000", 2024-08-17T18:15:30.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.122 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.122 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.122 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.122 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.561980+0000", 2024-08-17T18:15:30.122 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0999999999999998e-07 2024-08-17T18:15:30.122 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.123 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562005+0000", 2024-08-17T18:15:30.123 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4368000000000001e-05 2024-08-17T18:15:30.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.123 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562013+0000", 2024-08-17T18:15:30.123 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9410000000000007e-06 2024-08-17T18:15:30.123 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.123 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.123 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.123 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562018+0000", 2024-08-17T18:15:30.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.728e-06 2024-08-17T18:15:30.124 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.124 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.124 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.124 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.573080+0000", 2024-08-17T18:15:30.124 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011061323 2024-08-17T18:15:30.124 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.124 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.124 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:15:30.124 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.573086+0000", 2024-08-17T18:15:30.125 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.387e-06 2024-08-17T18:15:30.125 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.125 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.125 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.125 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.721262+0000", 2024-08-17T18:15:30.125 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1481759519999999 2024-08-17T18:15:30.125 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.125 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.125 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T18:15:30.125 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.721857+0000", 2024-08-17T18:15:30.125 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00059529500000000003 2024-08-17T18:15:30.125 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.126 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.126 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.126 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.746730+0000", 2024-08-17T18:15:30.126 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.024872601000000001 2024-08-17T18:15:30.126 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.126 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.126 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.126 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.746825+0000", 2024-08-17T18:15:30.126 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5178000000000003e-05 2024-08-17T18:15:30.126 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.126 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.126 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.126 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.127 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.127 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115434 48.2 48:4a2a9bfb:::benchmark_data_smithi130_327597_object115433:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1336)", 2024-08-17T18:15:30.127 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.562027+0000", 2024-08-17T18:15:30.127 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 392.12850970599999, 2024-08-17T18:15:30.127 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.183716341, 2024-08-17T18:15:30.127 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.127 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.127 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:15:30.127 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.127 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:15:30.127 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:15:30.127 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115434 2024-08-17T18:15:30.127 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.128 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.128 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.128 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.128 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562027+0000", 2024-08-17T18:15:30.128 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.128 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.128 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.128 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.128 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562027+0000", 2024-08-17T18:15:30.128 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.128 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.128 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.128 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.129 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562027+0000", 2024-08-17T18:15:30.129 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9699999999999996e-07 2024-08-17T18:15:30.129 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.129 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.129 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.129 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562051+0000", 2024-08-17T18:15:30.129 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3289999999999999e-05 2024-08-17T18:15:30.129 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.129 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.130 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.130 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562058+0000", 2024-08-17T18:15:30.130 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2609999999999996e-06 2024-08-17T18:15:30.130 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.130 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.130 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.130 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562063+0000", 2024-08-17T18:15:30.130 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9089999999999999e-06 2024-08-17T18:15:30.130 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.130 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.130 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.130 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.573111+0000", 2024-08-17T18:15:30.130 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011048267000000001 2024-08-17T18:15:30.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.131 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.131 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:15:30.131 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.573117+0000", 2024-08-17T18:15:30.131 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1720000000000004e-06 2024-08-17T18:15:30.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.131 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.131 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.131 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.721892+0000", 2024-08-17T18:15:30.131 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1487752630000001 2024-08-17T18:15:30.131 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T18:15:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.723040+0000", 2024-08-17T18:15:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0011476030000000001 2024-08-17T18:15:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.745649+0000", 2024-08-17T18:15:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022608883 2024-08-17T18:15:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.132 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.133 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.745743+0000", 2024-08-17T18:15:30.133 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4196000000000004e-05 2024-08-17T18:15:30.133 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.133 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.133 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.133 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.133 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.133 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115441 48.2 48:4ff2d013:::benchmark_data_smithi130_327597_object115440:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1336)", 2024-08-17T18:15:30.133 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.562070+0000", 2024-08-17T18:15:30.133 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 392.12846657400002, 2024-08-17T18:15:30.133 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.184123099, 2024-08-17T18:15:30.133 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.134 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.134 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:15:30.134 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.134 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:15:30.134 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:15:30.134 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115441 2024-08-17T18:15:30.134 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.134 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.134 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.134 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.135 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562070+0000", 2024-08-17T18:15:30.135 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.135 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.135 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.135 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.135 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562070+0000", 2024-08-17T18:15:30.135 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.135 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.135 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.135 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.135 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562070+0000", 2024-08-17T18:15:30.135 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2500000000000001e-07 2024-08-17T18:15:30.135 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.136 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.136 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.136 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562094+0000", 2024-08-17T18:15:30.136 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3757999999999999e-05 2024-08-17T18:15:30.136 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.136 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.136 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.136 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562101+0000", 2024-08-17T18:15:30.136 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9020000000000002e-06 2024-08-17T18:15:30.136 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.136 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.136 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.136 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562106+0000", 2024-08-17T18:15:30.137 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7740000000000004e-06 2024-08-17T18:15:30.137 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.137 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.137 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.137 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.573142+0000", 2024-08-17T18:15:30.137 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011035796 2024-08-17T18:15:30.137 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.137 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.137 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:15:30.137 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.573148+0000", 2024-08-17T18:15:30.137 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3060000000000003e-06 2024-08-17T18:15:30.138 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.138 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.138 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.723080+0000", 2024-08-17T18:15:30.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1499324019999999 2024-08-17T18:15:30.138 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.138 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.138 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T18:15:30.138 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.723597+0000", 2024-08-17T18:15:30.138 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00051703899999999995 2024-08-17T18:15:30.138 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.138 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.139 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.139 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.746125+0000", 2024-08-17T18:15:30.139 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022527453999999999 2024-08-17T18:15:30.139 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.139 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.139 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.139 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.746193+0000", 2024-08-17T18:15:30.139 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8243000000000004e-05 2024-08-17T18:15:30.139 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.139 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.139 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.139 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.139 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.140 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115444 48.2 48:437a7a94:::benchmark_data_smithi130_327597_object115443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1336)", 2024-08-17T18:15:30.140 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.562112+0000", 2024-08-17T18:15:30.140 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 392.12842419100002, 2024-08-17T18:15:30.140 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1842510179999999, 2024-08-17T18:15:30.140 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.140 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.140 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:15:30.140 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.140 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:15:30.140 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:15:30.140 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115444 2024-08-17T18:15:30.140 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.140 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.140 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.141 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.141 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562112+0000", 2024-08-17T18:15:30.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.141 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.141 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.141 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.141 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562112+0000", 2024-08-17T18:15:30.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.141 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.141 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.141 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.141 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562113+0000", 2024-08-17T18:15:30.141 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4700000000000002e-07 2024-08-17T18:15:30.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.142 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.142 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.142 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562290+0000", 2024-08-17T18:15:30.142 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000176792 2024-08-17T18:15:30.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.142 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.142 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.142 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562297+0000", 2024-08-17T18:15:30.142 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8189999999999992e-06 2024-08-17T18:15:30.142 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.142 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.142 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.143 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562302+0000", 2024-08-17T18:15:30.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9119999999999997e-06 2024-08-17T18:15:30.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.143 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.573181+0000", 2024-08-17T18:15:30.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010878818 2024-08-17T18:15:30.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.143 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.143 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:15:30.143 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.573190+0000", 2024-08-17T18:15:30.143 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6500000000000002e-06 2024-08-17T18:15:30.143 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.144 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.144 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.144 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.723631+0000", 2024-08-17T18:15:30.144 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.150441544 2024-08-17T18:15:30.144 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.144 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.144 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.144 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.723760+0000", 2024-08-17T18:15:30.144 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000128279 2024-08-17T18:15:30.144 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.144 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.144 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:15:30.144 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.723930+0000", 2024-08-17T18:15:30.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017090700000000001 2024-08-17T18:15:30.145 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.145 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.145 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.734705+0000", 2024-08-17T18:15:30.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.01077491 2024-08-17T18:15:30.145 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.145 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:15:30.145 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.734722+0000", 2024-08-17T18:15:30.145 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6639000000000001e-05 2024-08-17T18:15:30.145 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.145 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.146 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.146 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.746229+0000", 2024-08-17T18:15:30.146 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011506781000000001 2024-08-17T18:15:30.146 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.146 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.146 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:15:30.146 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.746279+0000", 2024-08-17T18:15:30.146 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0040999999999997e-05 2024-08-17T18:15:30.146 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.146 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.146 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.146 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.746363+0000", 2024-08-17T18:15:30.146 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4479000000000002e-05 2024-08-17T18:15:30.146 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.147 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.147 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.147 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.147 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.147 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115445 48.6 48:60381a73:::benchmark_data_smithi130_327597_object115444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1336)", 2024-08-17T18:15:30.147 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.562310+0000", 2024-08-17T18:15:30.147 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 392.12822672700003, 2024-08-17T18:15:30.147 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1648159300000001, 2024-08-17T18:15:30.147 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.147 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.147 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:15:30.148 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.148 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:15:30.148 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:15:30.148 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115445 2024-08-17T18:15:30.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.148 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.148 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562310+0000", 2024-08-17T18:15:30.148 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.148 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.148 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.148 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.149 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562310+0000", 2024-08-17T18:15:30.149 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.149 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.149 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.149 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562310+0000", 2024-08-17T18:15:30.149 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7300000000000001e-07 2024-08-17T18:15:30.149 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.149 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.149 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.149 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562554+0000", 2024-08-17T18:15:30.149 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000243866 2024-08-17T18:15:30.150 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.150 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.150 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.150 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562563+0000", 2024-08-17T18:15:30.150 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6659999999999997e-06 2024-08-17T18:15:30.150 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.150 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.150 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.562569+0000", 2024-08-17T18:15:30.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9789999999999998e-06 2024-08-17T18:15:30.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.572947+0000", 2024-08-17T18:15:30.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010378289000000001 2024-08-17T18:15:30.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.151 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.151 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:15:30.151 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.572958+0000", 2024-08-17T18:15:30.151 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0621e-05 2024-08-17T18:15:30.151 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.152 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.712199+0000", 2024-08-17T18:15:30.152 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1392410420000001 2024-08-17T18:15:30.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.152 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.712381+0000", 2024-08-17T18:15:30.152 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018194899999999999 2024-08-17T18:15:30.152 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.152 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.152 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:15:30.152 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.712572+0000", 2024-08-17T18:15:30.152 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00019132400000000001 2024-08-17T18:15:30.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.153 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:15:30.153 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.719992+0000", 2024-08-17T18:15:30.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0074198729999999996 2024-08-17T18:15:30.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.153 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.153 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.720335+0000", 2024-08-17T18:15:30.153 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00034269500000000001 2024-08-17T18:15:30.153 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.153 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.153 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.154 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.726998+0000", 2024-08-17T18:15:30.154 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0066634770000000001 2024-08-17T18:15:30.154 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.154 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.154 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:15:30.154 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.727056+0000", 2024-08-17T18:15:30.154 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7606999999999999e-05 2024-08-17T18:15:30.154 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.154 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.154 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.154 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.727126+0000", 2024-08-17T18:15:30.154 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0068999999999996e-05 2024-08-17T18:15:30.155 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.155 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.155 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.155 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.155 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.155 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:238954 48.6 48:66c3c7b1:::benchmark_data_smithi130_327597_object106989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:15:30.155 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:32.569852+0000", 2024-08-17T18:15:30.155 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 237.12068481599999, 2024-08-17T18:15:30.155 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.53118338700000001, 2024-08-17T18:15:30.155 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.155 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.155 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:15:30.155 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.155 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:15:30.156 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:15:30.156 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 238954 2024-08-17T18:15:30.156 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.156 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.156 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.156 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.156 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.569852+0000", 2024-08-17T18:15:30.156 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.156 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.156 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.156 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.156 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.569852+0000", 2024-08-17T18:15:30.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.157 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.157 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.569853+0000", 2024-08-17T18:15:30.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.804e-06 2024-08-17T18:15:30.157 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.157 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.569857+0000", 2024-08-17T18:15:30.157 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4860000000000002e-06 2024-08-17T18:15:30.157 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.157 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.157 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.158 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.569865+0000", 2024-08-17T18:15:30.158 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4489999999999999e-06 2024-08-17T18:15:30.158 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.158 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.158 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.158 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.569870+0000", 2024-08-17T18:15:30.158 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.045e-06 2024-08-17T18:15:30.158 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.158 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.158 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.158 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.570436+0000", 2024-08-17T18:15:30.158 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00056585800000000003 2024-08-17T18:15:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.570552+0000", 2024-08-17T18:15:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000115841 2024-08-17T18:15:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T18:15:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.570697+0000", 2024-08-17T18:15:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014523499999999999 2024-08-17T18:15:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:15:30.159 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.576177+0000", 2024-08-17T18:15:30.160 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0054794209999999999 2024-08-17T18:15:30.160 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.160 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.160 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.160 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.579001+0000", 2024-08-17T18:15:30.160 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028241579999999998 2024-08-17T18:15:30.160 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.160 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.160 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.160 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.587406+0000", 2024-08-17T18:15:30.160 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0084045509999999997 2024-08-17T18:15:30.160 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.160 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.160 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.100825+0000", 2024-08-17T18:15:30.161 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.51341992299999994 2024-08-17T18:15:30.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:15:30.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.100911+0000", 2024-08-17T18:15:30.161 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5359999999999996e-05 2024-08-17T18:15:30.161 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.161 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.161 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.161 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.101035+0000", 2024-08-17T18:15:30.162 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000124256 2024-08-17T18:15:30.162 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.162 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.162 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.162 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.162 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.162 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:238971 48.c e1357/1343 48:31f99771:::benchmark_data_smithi130_327597_object107006:head v 1357'14804, mlcod=1357'14802)", 2024-08-17T18:15:30.162 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:32.588274+0000", 2024-08-17T18:15:30.162 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 237.10226249199999, 2024-08-17T18:15:30.162 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.51201612500000004, 2024-08-17T18:15:30.162 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.162 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.163 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:15:30.163 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.163 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.163 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.163 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588274+0000", 2024-08-17T18:15:30.163 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.163 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.163 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.163 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.163 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588274+0000", 2024-08-17T18:15:30.163 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.163 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.163 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.163 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.164 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588282+0000", 2024-08-17T18:15:30.164 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2050000000000002e-06 2024-08-17T18:15:30.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.164 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.164 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.164 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588291+0000", 2024-08-17T18:15:30.164 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1030000000000008e-06 2024-08-17T18:15:30.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.164 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.164 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.164 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588302+0000", 2024-08-17T18:15:30.164 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1062e-05 2024-08-17T18:15:30.164 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.165 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.165 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.165 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588309+0000", 2024-08-17T18:15:30.165 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8809999999999998e-06 2024-08-17T18:15:30.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.165 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.165 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.165 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588364+0000", 2024-08-17T18:15:30.165 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5237e-05 2024-08-17T18:15:30.165 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.165 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.165 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.165 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588387+0000", 2024-08-17T18:15:30.165 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2158999999999999e-05 2024-08-17T18:15:30.166 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:15:30.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.100247+0000", 2024-08-17T18:15:30.166 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.51185987799999999 2024-08-17T18:15:30.166 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.166 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.166 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.100290+0000", 2024-08-17T18:15:30.166 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3600000000000003e-05 2024-08-17T18:15:30.166 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.166 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.166 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.167 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.167 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:238970 48.7 e1357/1343 48:e5ad0cc4:::benchmark_data_smithi130_327597_object107005:head v 1357'14883, mlcod=1357'14880)", 2024-08-17T18:15:30.167 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:32.588332+0000", 2024-08-17T18:15:30.167 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 237.10220416199999, 2024-08-17T18:15:30.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.52261806399999999, 2024-08-17T18:15:30.167 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.167 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.167 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:15:30.167 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.167 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588332+0000", 2024-08-17T18:15:30.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588332+0000", 2024-08-17T18:15:30.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588333+0000", 2024-08-17T18:15:30.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1100000000000001e-07 2024-08-17T18:15:30.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588337+0000", 2024-08-17T18:15:30.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0890000000000002e-06 2024-08-17T18:15:30.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588344+0000", 2024-08-17T18:15:30.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.861e-06 2024-08-17T18:15:30.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588348+0000", 2024-08-17T18:15:30.170 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.583e-06 2024-08-17T18:15:30.170 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.170 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.170 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.170 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588414+0000", 2024-08-17T18:15:30.170 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5479000000000001e-05 2024-08-17T18:15:30.170 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.170 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.170 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.170 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588507+0000", 2024-08-17T18:15:30.170 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3225000000000003e-05 2024-08-17T18:15:30.170 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:15:30.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.110725+0000", 2024-08-17T18:15:30.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.52221840200000003 2024-08-17T18:15:30.171 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.110950+0000", 2024-08-17T18:15:30.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000225014 2024-08-17T18:15:30.171 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.171 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.171 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.171 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.172 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:238972 48.f e1357/1343 48:f748d370:::benchmark_data_smithi130_327597_object107007:head v 1357'14907, mlcod=1357'14905)", 2024-08-17T18:15:30.172 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:32.588449+0000", 2024-08-17T18:15:30.172 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 237.10208783300001, 2024-08-17T18:15:30.172 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.51838709000000005, 2024-08-17T18:15:30.172 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.172 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.172 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:15:30.172 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.172 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.172 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588449+0000", 2024-08-17T18:15:30.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588449+0000", 2024-08-17T18:15:30.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588449+0000", 2024-08-17T18:15:30.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.75e-07 2024-08-17T18:15:30.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588454+0000", 2024-08-17T18:15:30.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6190000000000002e-06 2024-08-17T18:15:30.174 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.174 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.174 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588461+0000", 2024-08-17T18:15:30.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3050000000000004e-06 2024-08-17T18:15:30.174 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.175 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.175 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.588466+0000", 2024-08-17T18:15:30.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8980000000000004e-06 2024-08-17T18:15:30.175 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.175 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.175 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.100245+0000", 2024-08-17T18:15:30.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.51177933200000003 2024-08-17T18:15:30.175 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.175 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.176 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.100271+0000", 2024-08-17T18:15:30.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5908999999999999e-05 2024-08-17T18:15:30.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.176 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:15:30.176 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.106786+0000", 2024-08-17T18:15:30.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0065148649999999999 2024-08-17T18:15:30.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.176 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.176 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.106836+0000", 2024-08-17T18:15:30.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9687e-05 2024-08-17T18:15:30.176 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.177 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.177 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.177 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.36675.0:238973 48.8 e1357/1343 48:1beb4fd9:::benchmark_data_smithi130_327597_object107008:head v 1357'14935, mlcod=1357'14933)", 2024-08-17T18:15:30.177 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:32.590810+0000", 2024-08-17T18:15:30.177 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 237.09972668399999, 2024-08-17T18:15:30.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.51666315100000004, 2024-08-17T18:15:30.177 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.177 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.177 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:15:30.178 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.178 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.590810+0000", 2024-08-17T18:15:30.178 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.178 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.590810+0000", 2024-08-17T18:15:30.178 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.590811+0000", 2024-08-17T18:15:30.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.032e-06 2024-08-17T18:15:30.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:32.590818+0000", 2024-08-17T18:15:30.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9419999999999998e-06 2024-08-17T18:15:30.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.100235+0000", 2024-08-17T18:15:30.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.509417393 2024-08-17T18:15:30.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.180 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.180 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.100248+0000", 2024-08-17T18:15:30.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2452e-05 2024-08-17T18:15:30.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.180 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.180 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.100304+0000", 2024-08-17T18:15:30.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6217000000000001e-05 2024-08-17T18:15:30.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.180 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.180 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.100329+0000", 2024-08-17T18:15:30.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5023000000000001e-05 2024-08-17T18:15:30.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.181 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:15:30.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.107437+0000", 2024-08-17T18:15:30.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0071080240000000001 2024-08-17T18:15:30.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.181 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:33.107473+0000", 2024-08-17T18:15:30.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6068000000000001e-05 2024-08-17T18:15:30.181 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.181 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.182 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.182 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:244987 48.6 48:6675ce07:::benchmark_data_smithi130_327597_object113022:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1360)", 2024-08-17T18:15:30.182 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:11:42.253056+0000", 2024-08-17T18:15:30.182 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 227.43748047299999, 2024-08-17T18:15:30.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.319074256, 2024-08-17T18:15:30.182 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:15:30.182 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:15:30.182 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:15:30.182 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:15:30.182 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:15:30.182 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:15:30.183 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 244987 2024-08-17T18:15:30.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.183 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:15:30.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:15:30.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.253056+0000", 2024-08-17T18:15:30.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:15:30.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.253056+0000", 2024-08-17T18:15:30.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:15:30.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:15:30.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.253057+0000", 2024-08-17T18:15:30.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1799999999999999e-06 2024-08-17T18:15:30.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:15:30.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.253060+0000", 2024-08-17T18:15:30.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0359999999999998e-06 2024-08-17T18:15:30.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:15:30.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.253069+0000", 2024-08-17T18:15:30.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3680000000000002e-06 2024-08-17T18:15:30.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:15:30.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.253074+0000", 2024-08-17T18:15:30.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0839999999999999e-06 2024-08-17T18:15:30.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:15:30.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.258518+0000", 2024-08-17T18:15:30.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0054439240000000002 2024-08-17T18:15:30.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:15:30.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.258697+0000", 2024-08-17T18:15:30.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017963699999999999 2024-08-17T18:15:30.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T18:15:30.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.258842+0000", 2024-08-17T18:15:30.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000144788 2024-08-17T18:15:30.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.277876+0000", 2024-08-17T18:15:30.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019033845000000001 2024-08-17T18:15:30.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:15:30.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.278102+0000", 2024-08-17T18:15:30.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000225691 2024-08-17T18:15:30.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.278521+0000", 2024-08-17T18:15:30.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00041956200000000001 2024-08-17T18:15:30.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:15:30.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.571949+0000", 2024-08-17T18:15:30.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.29342828700000001 2024-08-17T18:15:30.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:15:30.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.572017+0000", 2024-08-17T18:15:30.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7705000000000003e-05 2024-08-17T18:15:30.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:15:30.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:15:30.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:15:30.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:11:42.572130+0000", 2024-08-17T18:15:30.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000113149 2024-08-17T18:15:30.189 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.189 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.189 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.189 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:15:30.189 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:15:30.189 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:15:30.189 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:30.241 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:30.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:30.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:30.343 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:30.383 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:30.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:30.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:30.485 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:30.507 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:30.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:30.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:15:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:15:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:15:30.608 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:30.630 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:30.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:30.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:30.732 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:30.753 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:30.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:30.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:30.855 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:30.897 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:30.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:30.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:30.998 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:31.037 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:31.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:31.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:31.140 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:31.177 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:31.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:31.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:31.279 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:31.335 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:31.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:31.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:31.437 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:31.475 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:31.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:31.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:31.577 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:31.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:31 smithi179 ceph-mon[93519]: pgmap v2628: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 730 op/s; 30483/158331 objects misplaced (19.253%); 8.3 MiB/s, 132 objects/s recovering 2024-08-17T18:15:31.600 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:31.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:31.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:31.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:31 smithi130 ceph-mon[105521]: pgmap v2628: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 730 op/s; 30483/158331 objects misplaced (19.253%); 8.3 MiB/s, 132 objects/s recovering 2024-08-17T18:15:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:31 smithi130 ceph-mon[110068]: pgmap v2628: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 730 op/s; 30483/158331 objects misplaced (19.253%); 8.3 MiB/s, 132 objects/s recovering 2024-08-17T18:15:31.702 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:31.725 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:31.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:31.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:31.826 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:31.848 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:31.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:31.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:31.950 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:32.000 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:32.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:32.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:32.101 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:32.124 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:32.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:32.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:32.222 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T18:15:32.226 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:32.323 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:32.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:32.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:32.425 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:32.487 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:32.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:32.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:32.588 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:32.613 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:32.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:32.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:32.714 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:32.767 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:15:32.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:32.794 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:32.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:32.838 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 7, 1] out_osds: [5, 4, 6] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2024-08-17T18:15:32.838 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:15:32.839 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-08-17T18:15:32.839 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T18:15:32.839 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T18:15:32.839 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-08-17T18:15:32.840 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:15:32.895 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:32.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:32.918 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:32.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:33.019 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:33.059 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:33.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:33.160 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:33.218 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:33.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:33.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:33.320 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:33.344 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:33.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:33.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:33.445 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:33.467 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:33.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:33.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:33.526 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:33 smithi130 ceph-mon[105521]: pgmap v2629: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 729 op/s; 29786/156240 objects misplaced (19.064%); 8.3 MiB/s, 132 objects/s recovering 2024-08-17T18:15:33.527 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:33 smithi130 ceph-mon[110068]: pgmap v2629: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 729 op/s; 29786/156240 objects misplaced (19.064%); 8.3 MiB/s, 132 objects/s recovering 2024-08-17T18:15:33.568 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:33.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:33 smithi179 ceph-mon[93519]: pgmap v2629: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 729 op/s; 29786/156240 objects misplaced (19.064%); 8.3 MiB/s, 132 objects/s recovering 2024-08-17T18:15:33.608 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:33.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:33.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:33.709 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:33.730 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:33.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:33.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:33.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:33.856 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:33.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:33.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:33.958 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:33.979 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:33.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:33.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:34.063 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:15:34.063 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:15:34.064 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:15:34.064 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:15:34.064 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:15:34.064 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:15:34.064 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:15:34.064 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:15:34.064 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:15:34.064 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:15:34.064 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:15:34.064 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:15:34.064 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:15:34.065 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:15:34.065 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:15:34.065 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:15:34.081 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:34.104 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:34.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:34.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:34.206 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:34.228 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:34.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:34.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:34.329 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:34.351 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:34.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:34.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:34.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:34 smithi130 ceph-mon[105521]: osdmap e1402: 8 total, 8 up, 5 in 2024-08-17T18:15:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:34 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2517565207' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:15:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:34 smithi130 ceph-mon[105521]: osdmap e1403: 8 total, 8 up, 5 in 2024-08-17T18:15:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:34 smithi130 ceph-mon[110068]: osdmap e1402: 8 total, 8 up, 5 in 2024-08-17T18:15:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:34 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2517565207' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:15:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:34 smithi130 ceph-mon[110068]: osdmap e1403: 8 total, 8 up, 5 in 2024-08-17T18:15:34.453 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:34.474 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:34.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:34.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:34.576 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:34.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:34 smithi179 ceph-mon[93519]: osdmap e1402: 8 total, 8 up, 5 in 2024-08-17T18:15:34.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:34 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2517565207' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:15:34.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:34 smithi179 ceph-mon[93519]: osdmap e1403: 8 total, 8 up, 5 in 2024-08-17T18:15:34.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:34.619 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:34.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:34.720 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:34.759 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:34.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:34.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:34.860 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:34.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:34.901 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:34.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:35.002 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:35.038 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:35.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:35.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:35.140 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:35.190 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:35.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:35.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:35.292 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:35.328 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:35.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:35.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:35.398 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:15:35.431 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:35.485 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:35.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:35.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:35.522 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:35 smithi130 ceph-mon[105521]: pgmap v2631: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 763 op/s; 28480/150138 objects misplaced (18.969%); 9.3 MiB/s, 149 objects/s recovering 2024-08-17T18:15:35.523 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:35 smithi130 ceph-mon[110068]: pgmap v2631: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 763 op/s; 28480/150138 objects misplaced (18.969%); 9.3 MiB/s, 149 objects/s recovering 2024-08-17T18:15:35.586 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:35.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:35 smithi179 ceph-mon[93519]: pgmap v2631: 89 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 79 active+clean; 3.1 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 763 op/s; 28480/150138 objects misplaced (18.969%); 9.3 MiB/s, 149 objects/s recovering 2024-08-17T18:15:35.635 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:35.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:35.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:35.737 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:35.759 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:35.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:35.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:35.861 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:35.904 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:35.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:35.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:36.006 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:36.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:36.044 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:36.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:36.145 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:36.187 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:36.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:36.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:36.289 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:36.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:36.330 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:36.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:36.432 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:36.453 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:36.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:36.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:36.555 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:36.599 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:36.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:36.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:36.701 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:36.742 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:36.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:36.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:36.844 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:36.866 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:36.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:36.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:36.928 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:15:36.968 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:37.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:37.022 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:37.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:37.123 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:37.144 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:37.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:37.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:37.246 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:37.267 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:37.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:37.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:37.369 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:37.411 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:37.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:37.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:37.513 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:37.547 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:37.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:37.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:37.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:37 smithi179 ceph-mon[93519]: pgmap v2633: 89 pgs: 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 79 active+clean; 3.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 885 op/s; 26933/145971 objects misplaced (18.451%); 13 MiB/s, 201 objects/s recovering 2024-08-17T18:15:37.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:37 smithi130 ceph-mon[110068]: pgmap v2633: 89 pgs: 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 79 active+clean; 3.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 885 op/s; 26933/145971 objects misplaced (18.451%); 13 MiB/s, 201 objects/s recovering 2024-08-17T18:15:37.643 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:37 smithi130 ceph-mon[105521]: pgmap v2633: 89 pgs: 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 79 active+clean; 3.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 885 op/s; 26933/145971 objects misplaced (18.451%); 13 MiB/s, 201 objects/s recovering 2024-08-17T18:15:37.649 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:37.671 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:37.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:37.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:37.773 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:37.831 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:37.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:37.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:37.932 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:37.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:37.977 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:37.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:38.078 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:38.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:38.116 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:38.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:38.217 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:38.256 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:38.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:38.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:38.358 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:38.400 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:38.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:38.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:38.502 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:38.525 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:38.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:38.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:38.627 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:38.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:38.678 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:38.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:38.779 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:38.801 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:38.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:38.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:38.903 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:38.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:38.926 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:38.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:39.027 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:39.143 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:39.866 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:39 smithi179 ceph-mon[93519]: pgmap v2634: 89 pgs: 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 79 active+clean; 3.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 514 op/s; 26933/145971 objects misplaced (18.451%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T18:15:39.866 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:39 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/899597270' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:15:39.866 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:39 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:15:39.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:39.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:39.868 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:15:39.868 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:15:39.868 INFO:teuthology.orchestra.run.smithi130.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T18:15:39.868 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:15:39.869 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:15:39.869 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:39 smithi130 ceph-mon[105521]: pgmap v2634: 89 pgs: 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 79 active+clean; 3.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 514 op/s; 26933/145971 objects misplaced (18.451%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T18:15:39.869 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:39 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/899597270' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:15:39.869 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:39 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:15:39.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:39 smithi130 ceph-mon[110068]: pgmap v2634: 89 pgs: 2 active+remapped+backfilling, 8 active+remapped+backfill_wait, 79 active+clean; 3.0 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 514 op/s; 26933/145971 objects misplaced (18.451%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T18:15:39.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:39 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/899597270' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:15:39.869 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:39 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:15:39.968 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:40.008 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:40.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:40.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:40.078 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:15:40.078 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:15:40.079 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:15:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:15:39] "GET /metrics HTTP/1.1" 200 36809 "" "Prometheus/2.51.0" 2024-08-17T18:15:40.110 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:40.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:40.160 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:40.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:40.261 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:40.282 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:40.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:40.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:40.384 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:40.406 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:40.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:40.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:40.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:40 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:15:40.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:40 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:15:40.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:40 smithi130 ceph-mon[105521]: osdmap e1404: 8 total, 8 up, 5 in 2024-08-17T18:15:40.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:40 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:15:40.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:40 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:15:40.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:40 smithi130 ceph-mon[110068]: osdmap e1404: 8 total, 8 up, 5 in 2024-08-17T18:15:40.508 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:40.530 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:40.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:40.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:40 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:15:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:40 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:15:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:40 smithi179 ceph-mon[93519]: osdmap e1404: 8 total, 8 up, 5 in 2024-08-17T18:15:40.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:15:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:15:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:15:40.632 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:40.675 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:40.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:40.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:40.777 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:40.820 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:40.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:40.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:40.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:40.955 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:40.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:40.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:41.057 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:41.078 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:41.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:41.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:41.180 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:41.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:41.220 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:41.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:41.321 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:41.360 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:41.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:41.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:41.461 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:41.501 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:41.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:41.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:41.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:41 smithi179 ceph-mon[93519]: pgmap v2636: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 635 op/s; 25296/138147 objects misplaced (18.311%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T18:15:41.603 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:41.624 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:41.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:41.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:41.649 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:41 smithi130 ceph-mon[105521]: pgmap v2636: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 635 op/s; 25296/138147 objects misplaced (18.311%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T18:15:41.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:41 smithi130 ceph-mon[110068]: pgmap v2636: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 635 op/s; 25296/138147 objects misplaced (18.311%); 7.6 MiB/s, 121 objects/s recovering 2024-08-17T18:15:41.726 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:41.768 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:41.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:41.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:41.870 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:41.891 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:41.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:41.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:42.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:42.042 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:42.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:42.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:42.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:42.183 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:42.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:42.284 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:42.305 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:42.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:42.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:42.407 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:42.429 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:42.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:42.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:42.532 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:42.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:42.555 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:42.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:42.647 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T18:15:42.655 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:42.727 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:42.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:42.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:42.828 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:42.850 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:42.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:42.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:42.951 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:43.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:43.047 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:43.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:43.148 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:43.197 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:43.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:43.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:43.262 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:15:43.299 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:43.321 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:43.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:43.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:43.423 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:43.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:43.458 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:43.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:43.559 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:43.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:43 smithi179 ceph-mon[93519]: pgmap v2637: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 584 op/s; 24612/136095 objects misplaced (18.084%); 6.0 MiB/s, 95 objects/s recovering 2024-08-17T18:15:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:15:43.602 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:15:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:43.609 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:43.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:43 smithi130 ceph-mon[105521]: pgmap v2637: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 584 op/s; 24612/136095 objects misplaced (18.084%); 6.0 MiB/s, 95 objects/s recovering 2024-08-17T18:15:43.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:15:43.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:43 smithi130 ceph-mon[110068]: pgmap v2637: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.8 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 584 op/s; 24612/136095 objects misplaced (18.084%); 6.0 MiB/s, 95 objects/s recovering 2024-08-17T18:15:43.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:15:43.710 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:43.761 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:43.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:43.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:43.862 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:43.908 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:43.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:43.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:44.010 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:44.046 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:44.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:44.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:44.148 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:44.226 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:44.229 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:15:44.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:44.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:44.331 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:44.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:44.390 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:44.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:44.492 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:44.532 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:44.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:44.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:44.634 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:44.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:44.657 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:44.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:44.758 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:44.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:44.801 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:44.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:44.903 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:44.945 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:44.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:44.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:45.047 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:15:45.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:45.091 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:45.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:15:45.192 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:45.214 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:45.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:45.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:45.316 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:45.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:45.359 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:45.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:45.459 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:45.507 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:45.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:45.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:45.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:45 smithi179 ceph-mon[93519]: pgmap v2638: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 715 op/s; 23961/130194 objects misplaced (18.404%); 5.1 MiB/s, 82 objects/s recovering 2024-08-17T18:15:45.609 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:45.632 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:45.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:45.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:45.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:45 smithi130 ceph-mon[105521]: pgmap v2638: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 715 op/s; 23961/130194 objects misplaced (18.404%); 5.1 MiB/s, 82 objects/s recovering 2024-08-17T18:15:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:45 smithi130 ceph-mon[110068]: pgmap v2638: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 715 op/s; 23961/130194 objects misplaced (18.404%); 5.1 MiB/s, 82 objects/s recovering 2024-08-17T18:15:45.733 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:45.775 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:45.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:45.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:45.878 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:45.899 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:45.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:45.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:46.001 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:46.042 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:46.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:46.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:46.144 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:46.176 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:46.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:46.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:46.278 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:46.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:46.301 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:46.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:46.403 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:46.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:46.427 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:46.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:46.529 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:46.551 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:46.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:46.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:46.653 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:46.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:46.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:46.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:46.788 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:46.836 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:46.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:46.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:46.938 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:46.965 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:46.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:46.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:47.067 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:47.107 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:47.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:47.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:47.155 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 7, 1] out_osds: [5, 4, 6] dead_osds: [] live_osds: [1, 2, 4, 5, 6, 7, 0, 3] 2024-08-17T18:15:47.155 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:15:47.156 INFO:tasks.thrashosds.thrasher:Killing osd 2, live_osds are [1, 2, 4, 5, 6, 7, 0, 3] 2024-08-17T18:15:47.156 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-08-17T18:15:47.156 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.2 2024-08-17T18:15:47.209 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:47.231 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:47.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:47.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:47.332 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:47.375 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:47.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:47.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:47.477 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:47.518 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:47.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:47.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:47.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:47 smithi179 ceph-mon[93519]: pgmap v2639: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.6 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 678 op/s; 22325/125607 objects misplaced (17.774%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T18:15:47.620 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:47.643 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:47.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:47.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:47.666 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:15:47 smithi130 systemd[1]: Stopping Ceph osd.2 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:15:47.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:47 smithi130 ceph-mon[105521]: pgmap v2639: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.6 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 678 op/s; 22325/125607 objects misplaced (17.774%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T18:15:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:47 smithi130 ceph-mon[110068]: pgmap v2639: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.6 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 678 op/s; 22325/125607 objects misplaced (17.774%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T18:15:47.745 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:47.766 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:47.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:47.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:47.869 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:47.917 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:47.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:47.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:48.019 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:48.055 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:48.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:48.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:48.157 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:48.200 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:48.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:48.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:48.302 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:48.324 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:48.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:48.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:48.426 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:48.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:48.449 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:48.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:48.550 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:48.595 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:48.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:48.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:48.697 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:48.719 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:48.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:48.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:48.821 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:48.872 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:48.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:48.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:48.974 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:48.994 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:48.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:48.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:49.096 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:49.118 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:49.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:49.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:49.220 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:49.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:49.269 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:49.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:49.370 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:49.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:49.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:49.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:49.506 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:49.527 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:49.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:49.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:49 smithi179 ceph-mon[93519]: pgmap v2640: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.6 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 678 op/s; 22325/125607 objects misplaced (17.774%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T18:15:49.629 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:49.652 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:49.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:49.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:49.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:49 smithi130 ceph-mon[105521]: pgmap v2640: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.6 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 678 op/s; 22325/125607 objects misplaced (17.774%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T18:15:49.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:49 smithi130 ceph-mon[110068]: pgmap v2640: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.6 GiB data, 13 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 678 op/s; 22325/125607 objects misplaced (17.774%); 4.8 MiB/s, 76 objects/s recovering 2024-08-17T18:15:49.754 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:49.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:49.777 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:49.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:49.879 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:49.901 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:49.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:49.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:50.003 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:50.025 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:50.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:50.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:50.127 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:50.148 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:50.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:50.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:15:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:15:49] "GET /metrics HTTP/1.1" 200 36801 "" "Prometheus/2.51.0" 2024-08-17T18:15:50.251 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:50.287 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:50.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:50.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:50.389 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:50.413 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:50.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:50.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:50.494 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:15:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:15:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:15:50.514 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:50.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:50.564 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:50.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:50.665 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:50.700 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:50.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:50.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:50.801 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:50.823 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:50.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:50.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:50.925 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:50.953 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:50.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:50.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:51.055 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:51.076 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:51.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:51.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:51.178 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:51.212 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:51.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:51.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:51.313 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:51.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:51.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:51.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:51.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:51 smithi130 ceph-mon[105521]: pgmap v2641: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 760 op/s; 20799/117435 objects misplaced (17.711%); 6.9 MiB/s, 110 objects/s recovering 2024-08-17T18:15:51.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:51 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/796043374' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:15:51.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:51 smithi130 ceph-mon[110068]: pgmap v2641: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 760 op/s; 20799/117435 objects misplaced (17.711%); 6.9 MiB/s, 110 objects/s recovering 2024-08-17T18:15:51.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:51 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/796043374' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:15:51.436 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:51.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:51.476 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:51.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:51.578 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:51.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:51 smithi179 ceph-mon[93519]: pgmap v2641: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 760 op/s; 20799/117435 objects misplaced (17.711%); 6.9 MiB/s, 110 objects/s recovering 2024-08-17T18:15:51.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:51 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/796043374' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:15:51.615 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:51.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:51.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:51.717 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:51.764 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:51.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:51.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:51.866 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:51.900 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:51.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:51.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:52.002 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:52.044 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:52.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:52.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:52.123 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:15:52.124 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:15:52.124 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:15:52.124 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:15:52.124 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:15:52.124 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:15:52.124 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:15:52.124 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:15:52.125 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:15:52.125 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:15:52.125 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:15:52.125 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:15:52.125 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:15:52.125 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:15:52.125 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:15:52.125 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:15:52.147 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:52.168 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:52.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:52.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:52.270 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:52.292 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:52.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:52.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:52.394 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:52.433 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:52.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:52.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:52.535 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:52.556 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:52.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:52.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:52.658 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:52.681 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:52.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:52.783 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:52.824 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:52.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:52.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:52.926 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:52.970 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:52.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:52.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:53.072 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:53.094 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:53.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:53.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:53.195 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:53.241 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:53.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:53.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:53.343 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:53.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:53.385 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:53.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:53.486 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:53.507 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:53.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:53.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:53.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:53 smithi179 ceph-mon[93519]: pgmap v2642: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 629 op/s; 20145/115473 objects misplaced (17.446%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T18:15:53.609 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:53.630 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:53.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:53.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:53.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:53 smithi130 ceph-mon[105521]: pgmap v2642: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 629 op/s; 20145/115473 objects misplaced (17.446%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T18:15:53.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:53 smithi130 ceph-mon[110068]: pgmap v2642: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 629 op/s; 20145/115473 objects misplaced (17.446%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T18:15:53.731 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:53.773 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:53.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:53.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:53.875 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:53.897 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:53.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:53.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:53.999 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:54.043 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:54.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:54.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:54.145 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:54.183 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:54.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:54.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:54.285 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:54.306 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:54.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:54.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:54.408 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:54.430 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:54.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:54.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:54.532 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:54.553 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:54.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:54.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:54.655 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:54.701 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:54.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:54.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:54.803 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:54.838 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:54.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:54.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:54.940 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:54.963 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:54.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:54.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:55.065 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:55.112 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:55.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:55.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:55.213 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:55.257 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:55.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:55.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:55.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:55.407 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:55.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:55.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:55.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:55 smithi130 ceph-mon[105521]: pgmap v2643: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 734 op/s; 19490/109641 objects misplaced (17.776%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T18:15:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:55 smithi130 ceph-mon[110068]: pgmap v2643: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 734 op/s; 19490/109641 objects misplaced (17.776%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T18:15:55.509 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:55.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:55.553 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:55.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:55.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:55 smithi179 ceph-mon[93519]: pgmap v2643: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.2 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 734 op/s; 19490/109641 objects misplaced (17.776%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T18:15:55.654 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:55.703 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:55.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:55.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:55.792 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:15:55.793 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T18:15:55.793 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T18:15:55.793 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T18:15:55.793 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:15:55.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:55.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:55.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:55.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:55.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:55.968 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:55.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:55.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:56.070 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:56.109 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:56.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:56.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:56.211 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:56.233 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:56.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:56.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:56.335 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:56.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:56.371 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:56.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:56.472 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:56.495 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:56.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:56.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:56.596 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:56.637 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:56.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:56.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:56.739 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:56.781 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:56.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:56.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:56.829 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T18:15:56.884 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:56.905 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:56.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:56.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:57.006 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:57.027 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:57.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:57.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:57.129 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:57.151 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:57.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:57.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:57.253 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:57.275 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:57.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:57.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:57.376 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:57.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:57.398 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:57.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:57.500 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:57.521 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:57.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:57.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:57.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:57 smithi179 ceph-mon[93519]: pgmap v2644: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 693 op/s; 17935/105234 objects misplaced (17.043%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T18:15:57.623 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:57.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:57 smithi130 ceph-mon[105521]: pgmap v2644: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 693 op/s; 17935/105234 objects misplaced (17.043%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T18:15:57.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:57 smithi130 ceph-mon[110068]: pgmap v2644: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 693 op/s; 17935/105234 objects misplaced (17.043%); 6.2 MiB/s, 98 objects/s recovering 2024-08-17T18:15:57.667 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:57.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:57.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:57.769 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:57.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:57.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:57.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:57.912 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:57.954 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:57.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:57.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:58.055 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:58.077 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:15:58.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:58.078 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:58.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:58.178 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:58.221 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:58.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:58.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:58.323 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:58.357 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:58.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:58.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:58.459 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:58.483 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:58.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:58.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:58.585 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:15:58.607 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:58.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:58.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:15:58.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:15:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:58 smithi130 ceph-mon[105521]: osd.2 marked itself down and dead 2024-08-17T18:15:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:15:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:58 smithi130 ceph-mon[110068]: osd.2 marked itself down and dead 2024-08-17T18:15:58.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:15:58 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[130505]: 2024-08-17T18:15:58.305+0000 7f32594d4640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T18:15:58.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:15:58 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[130505]: 2024-08-17T18:15:58.305+0000 7f32594d4640 -1 osd.2 1404 *** Got signal Terminated *** 2024-08-17T18:15:58.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:15:58 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[130505]: 2024-08-17T18:15:58.305+0000 7f32594d4640 -1 osd.2 1404 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T18:15:58.709 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:58.730 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:58.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:58.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:58.832 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:58.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:15:58.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:58 smithi179 ceph-mon[93519]: osd.2 marked itself down and dead 2024-08-17T18:15:58.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:58.854 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:58.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:58.956 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:58.982 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:58.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:58.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:59.084 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:15:59.105 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:59.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:59.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:15:59.208 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:15:59.229 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:59.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:59.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:15:59.331 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:15:59.357 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:59.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:59.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:15:59.415 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:15:59.416 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:15:59 smithi130 podman[385476]: 2024-08-17 18:15:59.232607786 +0000 UTC m=+1.173965075 container died 05996ffbb485f9350c8059bfa9d5c6bea45433c3fdace90a921b5a04cd3c4614 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0) 2024-08-17T18:15:59.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:59 smithi130 ceph-mon[105521]: pgmap v2645: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 565 op/s; 17935/105234 objects misplaced (17.043%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T18:15:59.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:59 smithi130 ceph-mon[105521]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T18:15:59.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:59 smithi130 ceph-mon[105521]: osdmap e1405: 8 total, 7 up, 5 in 2024-08-17T18:15:59.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:15:59 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:15:59.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:59 smithi130 ceph-mon[110068]: pgmap v2645: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 565 op/s; 17935/105234 objects misplaced (17.043%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T18:15:59.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:59 smithi130 ceph-mon[110068]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T18:15:59.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:59 smithi130 ceph-mon[110068]: osdmap e1405: 8 total, 7 up, 5 in 2024-08-17T18:15:59.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:15:59 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:15:59.460 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:59.534 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:59.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:59.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:15:59.636 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:15:59.661 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:15:59.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:59.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:15:59.763 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:15:59.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:59.811 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:59.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:15:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:59 smithi179 ceph-mon[93519]: pgmap v2645: 89 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 80 active+clean; 2.1 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 565 op/s; 17935/105234 objects misplaced (17.043%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T18:15:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:59 smithi179 ceph-mon[93519]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T18:15:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:59 smithi179 ceph-mon[93519]: osdmap e1405: 8 total, 7 up, 5 in 2024-08-17T18:15:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:15:59 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:15:59.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:15:59.958 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:15:59.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:15:59.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:00.064 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:00.088 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:00.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:00.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:00.163 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:15:59 smithi130 podman[385476]: 2024-08-17 18:15:59.818466827 +0000 UTC m=+1.759824113 container cleanup 05996ffbb485f9350c8059bfa9d5c6bea45433c3fdace90a921b5a04cd3c4614 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-08-17T18:16:00.163 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:15:59 smithi130 bash[385476]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2 2024-08-17T18:16:00.163 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:15:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:15:59] "GET /metrics HTTP/1.1" 200 36799 "" "Prometheus/2.51.0" 2024-08-17T18:16:00.190 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:00.236 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:00.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:00.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:00.252 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:16:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:16:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:16:00.337 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:00.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:00.402 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:00.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:00.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[105521]: osdmap e1406: 8 total, 7 up, 5 in 2024-08-17T18:16:00.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[105521]: 49.7 continuing backfill to osd.1 from (1390'6300,1404'11652] MIN to 1404'11652 2024-08-17T18:16:00.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[105521]: 49.4 continuing backfill to osd.1 from (1390'6300,1404'11652] 49:2e646be4:::benchmark_data_smithi130_364143_object107586:head to 1404'11652 2024-08-17T18:16:00.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[105521]: 49.e continuing backfill to osd.1 from (1389'4900,1404'11607] MIN to 1404'11607 2024-08-17T18:16:00.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[105521]: 49.b continuing backfill to osd.1 from (1391'6200,1404'11423] MIN to 1404'11423 2024-08-17T18:16:00.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[105521]: 49.f continuing backfill to osd.1 from (1389'5000,1404'11674] MIN to 1404'11674 2024-08-17T18:16:00.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[105521]: 49.6 starting backfill to osd.1 from (1391'6400,1404'11623] MIN to 1404'11624 2024-08-17T18:16:00.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[105521]: 49.2 continuing backfill to osd.1 from (1389'4900,1404'11658] MIN to 1404'11658 2024-08-17T18:16:00.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[110068]: osdmap e1406: 8 total, 7 up, 5 in 2024-08-17T18:16:00.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[110068]: 49.7 continuing backfill to osd.1 from (1390'6300,1404'11652] MIN to 1404'11652 2024-08-17T18:16:00.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[110068]: 49.4 continuing backfill to osd.1 from (1390'6300,1404'11652] 49:2e646be4:::benchmark_data_smithi130_364143_object107586:head to 1404'11652 2024-08-17T18:16:00.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[110068]: 49.e continuing backfill to osd.1 from (1389'4900,1404'11607] MIN to 1404'11607 2024-08-17T18:16:00.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[110068]: 49.b continuing backfill to osd.1 from (1391'6200,1404'11423] MIN to 1404'11423 2024-08-17T18:16:00.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[110068]: 49.f continuing backfill to osd.1 from (1389'5000,1404'11674] MIN to 1404'11674 2024-08-17T18:16:00.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[110068]: 49.6 starting backfill to osd.1 from (1391'6400,1404'11623] MIN to 1404'11624 2024-08-17T18:16:00.420 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:00 smithi130 ceph-mon[110068]: 49.2 continuing backfill to osd.1 from (1389'4900,1404'11658] MIN to 1404'11658 2024-08-17T18:16:00.503 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:00.526 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:00.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:00.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:00 smithi179 ceph-mon[93519]: osdmap e1406: 8 total, 7 up, 5 in 2024-08-17T18:16:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:00 smithi179 ceph-mon[93519]: 49.7 continuing backfill to osd.1 from (1390'6300,1404'11652] MIN to 1404'11652 2024-08-17T18:16:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:00 smithi179 ceph-mon[93519]: 49.4 continuing backfill to osd.1 from (1390'6300,1404'11652] 49:2e646be4:::benchmark_data_smithi130_364143_object107586:head to 1404'11652 2024-08-17T18:16:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:00 smithi179 ceph-mon[93519]: 49.e continuing backfill to osd.1 from (1389'4900,1404'11607] MIN to 1404'11607 2024-08-17T18:16:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:00 smithi179 ceph-mon[93519]: 49.b continuing backfill to osd.1 from (1391'6200,1404'11423] MIN to 1404'11423 2024-08-17T18:16:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:00 smithi179 ceph-mon[93519]: 49.f continuing backfill to osd.1 from (1389'5000,1404'11674] MIN to 1404'11674 2024-08-17T18:16:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:00 smithi179 ceph-mon[93519]: 49.6 starting backfill to osd.1 from (1391'6400,1404'11623] MIN to 1404'11624 2024-08-17T18:16:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:00 smithi179 ceph-mon[93519]: 49.2 continuing backfill to osd.1 from (1389'4900,1404'11658] MIN to 1404'11658 2024-08-17T18:16:00.628 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:00.650 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:00.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:00.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:00.751 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:00.773 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:00.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:00.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:00.875 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:00.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:00.900 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:00.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:01.001 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:01.024 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:01.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:01.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:01.126 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:01.148 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:01.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:01.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:01.249 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:01.272 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:01.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:01.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:01.374 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:01.390 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:16:01.413 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:01.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:01.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:01.515 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:01.539 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:01.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:01.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:01.641 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:01.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:01 smithi130 ceph-mon[105521]: pgmap v2648: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 stale+active+remapped+backfill_wait, 3 active+undersized, 4 active+remapped+backfill_wait, 3 active+undersized+degraded, 74 active+clean; 2.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 763 op/s; 4309/97143 objects degraded (4.436%); 14325/97143 objects misplaced (14.746%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T18:16:01.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:01 smithi130 ceph-mon[105521]: Health check failed: Degraded data redundancy: 4309/97143 objects degraded (4.436%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:01.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:01 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4128901623' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:01.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:01 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:01.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:01 smithi130 ceph-mon[110068]: pgmap v2648: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 stale+active+remapped+backfill_wait, 3 active+undersized, 4 active+remapped+backfill_wait, 3 active+undersized+degraded, 74 active+clean; 2.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 763 op/s; 4309/97143 objects degraded (4.436%); 14325/97143 objects misplaced (14.746%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T18:16:01.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:01 smithi130 ceph-mon[110068]: Health check failed: Degraded data redundancy: 4309/97143 objects degraded (4.436%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:01.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:01 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4128901623' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:01.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:01 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:01.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:01.694 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:01.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:01.795 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:01.818 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:01.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:01.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:01.836 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:01 smithi179 ceph-mon[93519]: pgmap v2648: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 stale+active+remapped+backfill_wait, 3 active+undersized, 4 active+remapped+backfill_wait, 3 active+undersized+degraded, 74 active+clean; 2.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 763 op/s; 4309/97143 objects degraded (4.436%); 14325/97143 objects misplaced (14.746%); 5.9 MiB/s, 94 objects/s recovering 2024-08-17T18:16:01.836 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:01 smithi179 ceph-mon[93519]: Health check failed: Degraded data redundancy: 4309/97143 objects degraded (4.436%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:01.836 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:01 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4128901623' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:01.836 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:01 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:01.920 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:01.944 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:01.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:01.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:02.003 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:16:01 smithi130 podman[385575]: 2024-08-17 18:16:01.692478736 +0000 UTC m=+2.639412152 container remove 05996ffbb485f9350c8059bfa9d5c6bea45433c3fdace90a921b5a04cd3c4614 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS) 2024-08-17T18:16:02.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:02.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:02.090 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:02.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:02.191 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:02.214 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:02.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:02.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:02.283 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:02.316 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:02.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:02.342 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:02.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:02.442 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:02.465 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:02.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:02.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:02.567 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:02.597 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:02.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:02.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:02.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:02 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:16:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:02 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:16:02.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:02 smithi130 ceph-mon[105521]: osdmap e1407: 8 total, 7 up, 5 in 2024-08-17T18:16:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:02 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:16:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:02 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:16:02.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:02 smithi130 ceph-mon[110068]: osdmap e1407: 8 total, 7 up, 5 in 2024-08-17T18:16:02.699 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:02.745 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:02.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:02.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:02 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:16:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:02 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:16:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:02 smithi179 ceph-mon[93519]: osdmap e1407: 8 total, 7 up, 5 in 2024-08-17T18:16:02.847 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:02.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:02.891 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:02.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:02.992 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:03.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:03.028 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:03.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:03.129 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:03.168 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:03.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:03.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:03.250 INFO:teuthology.orchestra.run.smithi130.stderr:admin_socket: exception getting command descriptions: [Errno 111] Connection refused 2024-08-17T18:16:03.270 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:03.319 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:03.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:03.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:03.421 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:03.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:03.446 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:03.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:03.547 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:03.569 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:03.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:03.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:03.608 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:03 smithi130 ceph-mon[105521]: pgmap v2650: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 stale+active+remapped+backfill_wait, 14 active+undersized, 4 active+remapped+backfill_wait, 5 active+undersized+degraded, 61 active+clean; 2.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 449 op/s; 4311/97143 objects degraded (4.438%); 14325/97143 objects misplaced (14.746%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T18:16:03.608 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:03 smithi130 ceph-mon[110068]: pgmap v2650: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 stale+active+remapped+backfill_wait, 14 active+undersized, 4 active+remapped+backfill_wait, 5 active+undersized+degraded, 61 active+clean; 2.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 449 op/s; 4311/97143 objects degraded (4.438%); 14325/97143 objects misplaced (14.746%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T18:16:03.671 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:03.711 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:03.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:03.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:03.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:03 smithi179 ceph-mon[93519]: pgmap v2650: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 3 stale+active+remapped+backfill_wait, 14 active+undersized, 4 active+remapped+backfill_wait, 5 active+undersized+degraded, 61 active+clean; 2.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 449 op/s; 4311/97143 objects degraded (4.438%); 14325/97143 objects misplaced (14.746%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T18:16:03.856 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:03.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:03.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:03.958 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:03.981 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:03.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:03.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:04.083 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:04.105 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:04.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:04.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:04.207 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:04.229 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:04.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:04.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:04.331 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:04.354 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:04.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:04.453 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:16:04.453 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T18:16:04.455 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:04.478 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:04.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:04.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:04.580 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:04.605 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:04.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:04.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:04.707 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:04.735 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:04.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:04.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:04.751 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:16:04.772 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:16:04.837 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:04.858 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:04.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:04.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:04.960 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:05.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:05.037 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:05.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:05.138 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:05.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:05.184 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:05.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:05.285 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:05.312 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:05.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:05.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:05.405 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:16:05 smithi130 podman[386395]: 2024-08-17 18:16:04.912997179 +0000 UTC m=+0.103301050 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T18:16:05.413 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:05.455 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:05.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:05.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:05.557 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:05.579 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:05.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:05.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:05.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:05 smithi179 ceph-mon[93519]: pgmap v2651: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 922 op/s; 20057/88614 objects degraded (22.634%); 10406/88614 objects misplaced (11.743%); 12 MiB/s, 190 objects/s recovering 2024-08-17T18:16:05.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:05 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:05 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:05 smithi130 ceph-mon[105521]: pgmap v2651: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 922 op/s; 20057/88614 objects degraded (22.634%); 10406/88614 objects misplaced (11.743%); 12 MiB/s, 190 objects/s recovering 2024-08-17T18:16:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:05 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:05.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:05 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:05.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:05 smithi130 ceph-mon[110068]: pgmap v2651: 89 pgs: 2 active+undersized+degraded+remapped+backfilling, 5 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 922 op/s; 20057/88614 objects degraded (22.634%); 10406/88614 objects misplaced (11.743%); 12 MiB/s, 190 objects/s recovering 2024-08-17T18:16:05.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:05 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:05.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:05 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:05.669 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:16:05 smithi130 podman[386395]: 2024-08-17 18:16:05.500771819 +0000 UTC m=+0.691075691 container create 218409e091c77271ce3cc7a0ed28b54e74f0f4cbfc032c246b0050e221514370 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-17T18:16:05.681 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:05.727 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:05.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:05.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:05.829 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:05.867 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:05.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:05.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:05.968 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:16:05 smithi130 podman[386395]: 2024-08-17 18:16:05.859283365 +0000 UTC m=+1.049587251 container init 218409e091c77271ce3cc7a0ed28b54e74f0f4cbfc032c246b0050e221514370 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0) 2024-08-17T18:16:05.968 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:16:05 smithi130 podman[386395]: 2024-08-17 18:16:05.867167611 +0000 UTC m=+1.057471476 container start 218409e091c77271ce3cc7a0ed28b54e74f0f4cbfc032c246b0050e221514370 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-08-17T18:16:05.968 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:16:05 smithi130 podman[386395]: 2024-08-17 18:16:05.967536011 +0000 UTC m=+1.157839896 container attach 218409e091c77271ce3cc7a0ed28b54e74f0f4cbfc032c246b0050e221514370 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe) 2024-08-17T18:16:05.969 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:05.992 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:05.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:05.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:06.094 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:06.118 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:06.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:06.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:06.220 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:06.245 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:06.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:06.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:06.347 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:06.387 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:06.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:06.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:06.489 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:06.513 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:06.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:06.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:06.525 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 20057/88614 objects degraded (22.634%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:06.614 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:06.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:06.666 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:06.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:06.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:16:06 smithi130 conmon[386496]: conmon 218409e091c77271ce3c : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-218409e091c77271ce3cc7a0ed28b54e74f0f4cbfc032c246b0050e221514370.scope/memory.events 2024-08-17T18:16:06.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:16:06 smithi130 podman[386395]: 2024-08-17 18:16:06.421926103 +0000 UTC m=+1.612229973 container died 218409e091c77271ce3cc7a0ed28b54e74f0f4cbfc032c246b0050e221514370 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, RELEASE=main-cd8bdbe, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T18:16:06.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 20057/88614 objects degraded (22.634%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:06.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 20057/88614 objects degraded (22.634%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:06.728 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:06.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:06.790 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:16:06.791 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:16:06.791 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T18:16:06.791 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:16:06.830 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:06.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:06.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:06.932 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:06.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:06.978 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:06.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:07.079 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:07.102 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:07.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:07.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:07.204 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:07.226 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:07.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:07.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:07.328 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:07.351 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:07.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:07.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:07.454 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:07.478 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:07.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:07.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:07.492 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T18:16:07.537 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:16:07.538 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:16:07.545 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:16:07.546 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:16:07.549 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:16:07.580 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:07.583 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:16:07.583 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:16:07.584 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:07 smithi130 ceph-mon[105521]: pgmap v2652: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 656 op/s; 19422/84993 objects degraded (22.851%); 10093/84993 objects misplaced (11.875%); 10 MiB/s, 160 objects/s recovering 2024-08-17T18:16:07.584 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:07 smithi130 ceph-mon[110068]: pgmap v2652: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 656 op/s; 19422/84993 objects degraded (22.851%); 10093/84993 objects misplaced (11.875%); 10 MiB/s, 160 objects/s recovering 2024-08-17T18:16:07.603 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:07.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:07.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:07.613 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:16:07.614 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:16:07.648 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:16:07.648 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:16:07.679 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:16:07.680 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:16:07.706 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:07.714 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:16:07.714 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:16:07.748 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:07.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:07.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:07.827 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:16:07.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:07 smithi179 ceph-mon[93519]: pgmap v2652: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 656 op/s; 19422/84993 objects degraded (22.851%); 10093/84993 objects misplaced (11.875%); 10 MiB/s, 160 objects/s recovering 2024-08-17T18:16:07.850 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:07.890 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:07.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:07.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:07.992 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:08.030 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:08.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:08.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:08.132 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:08.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:08.171 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:08.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:08.272 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:08.312 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:08.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:08.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:08.413 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.2.service' 2024-08-17T18:16:08.414 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:08.461 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:08.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:08.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:08.496 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:16:08.496 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-08-17T18:16:08.562 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:08.587 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:08.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:08.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:08.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:08 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3671284524' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:16:08.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:08 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3671284524' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:16:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:08 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3671284524' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:16:08.689 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:08.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:08.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:08.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:08.812 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:08.836 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:08.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:08.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:08.937 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:08.972 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:08.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:08.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:09.074 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:09.112 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:09.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:09.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:09.214 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:09.250 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:09.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:09.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:09.351 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:09.377 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:09.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:09.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:09.479 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:09.512 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:09.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:09.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:09.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:09 smithi179 ceph-mon[93519]: pgmap v2653: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 577 op/s; 19422/84993 objects degraded (22.851%); 10093/84993 objects misplaced (11.875%); 8.8 MiB/s, 141 objects/s recovering 2024-08-17T18:16:09.613 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:09.636 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:09.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:09.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:09.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:09 smithi130 ceph-mon[105521]: pgmap v2653: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 577 op/s; 19422/84993 objects degraded (22.851%); 10093/84993 objects misplaced (11.875%); 8.8 MiB/s, 141 objects/s recovering 2024-08-17T18:16:09.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:09 smithi130 ceph-mon[110068]: pgmap v2653: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.7 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 577 op/s; 19422/84993 objects degraded (22.851%); 10093/84993 objects misplaced (11.875%); 8.8 MiB/s, 141 objects/s recovering 2024-08-17T18:16:09.738 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:09.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:09.762 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:09.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:09.863 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:09.870 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:16:09.871 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:16:09.910 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:16:09.910 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:16:09.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:09.913 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:09.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:10.014 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:10.024 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:16:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:16:09] "GET /metrics HTTP/1.1" 200 36799 "" "Prometheus/2.51.0" 2024-08-17T18:16:10.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:10.071 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:10.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:10.172 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:10.198 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:10.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:10.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:10.300 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:10.323 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:10.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:10.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:10.425 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:10.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:10.449 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:10.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:10.551 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:16:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:16:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:16:10.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:10.618 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:10.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:10.659 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T18:16:10.719 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:10.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:10.767 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:10.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:10.868 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:10.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:10.919 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:10.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:10.941 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:16:11.020 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:11.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:11.042 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:11.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:11.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:11.183 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:11.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:11.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:11.285 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:11.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:11.329 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:11.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:11.430 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:11.445 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:16:11.453 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:11.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:11.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:11.555 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:11.589 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:11.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:11.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:11.691 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:11.714 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:11.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:11.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:11.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:11.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:11 smithi130 ceph-mon[105521]: pgmap v2654: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 887 op/s; 15872/70518 objects degraded (22.508%); 8397/70518 objects misplaced (11.908%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T18:16:11.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:11 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4173761565' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:16:11.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:11 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:16:11.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15872/70518 objects degraded (22.508%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:11 smithi130 ceph-mon[110068]: pgmap v2654: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 887 op/s; 15872/70518 objects degraded (22.508%); 8397/70518 objects misplaced (11.908%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T18:16:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:11 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4173761565' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:16:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:11 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:16:11.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15872/70518 objects degraded (22.508%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:11 smithi179 ceph-mon[93519]: pgmap v2654: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 887 op/s; 15872/70518 objects degraded (22.508%); 8397/70518 objects misplaced (11.908%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T18:16:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:11 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4173761565' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:16:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:11 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:16:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15872/70518 objects degraded (22.508%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:11.854 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:11.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:11.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:11.872 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:16:11.956 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:12.015 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:12.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:12.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:12.118 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:12.141 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:12.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:12.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:12.243 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:12.287 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:12.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:12.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:12.390 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:12.427 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:12.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:12.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:12.529 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:12.563 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:12.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:12.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:12.588 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:12 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:16:12.588 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:12 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:16:12.588 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:12 smithi179 ceph-mon[93519]: osdmap e1408: 8 total, 7 up, 5 in 2024-08-17T18:16:12.589 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:12.589 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:16:12.665 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:12.688 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:12.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:12.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:12.721 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:12.790 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:12.815 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:12.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:12.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:12.847 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:16:12.847 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:16:12.864 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:12 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:16:12.864 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:12 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:16:12.864 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:12 smithi130 ceph-mon[105521]: osdmap e1408: 8 total, 7 up, 5 in 2024-08-17T18:16:12.864 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:12.864 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:16:12.864 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:12 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:16:12.864 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:12 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:16:12.865 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:12 smithi130 ceph-mon[110068]: osdmap e1408: 8 total, 7 up, 5 in 2024-08-17T18:16:12.865 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:12.865 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:16:12.916 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:12.938 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:12.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:12.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:13.040 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:13.091 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:13.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:13.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:13.193 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:13.235 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:13.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:13.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:13.338 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:13.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:13.362 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:13.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:13.463 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:13.486 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:13.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:13.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:13.496 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 7, 1] out_osds: [5, 4, 6] dead_osds: [2] live_osds: [1, 4, 5, 6, 7, 0, 3] 2024-08-17T18:16:13.497 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:16:13.497 INFO:tasks.thrashosds.thrasher:Adding osd 6 2024-08-17T18:16:13.497 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd in 6 2024-08-17T18:16:13.587 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:13.589 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T18:16:13.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:13.652 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:13.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:13.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:13 smithi130 ceph-mon[105521]: pgmap v2656: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 887 op/s; 15872/70518 objects degraded (22.508%); 8397/70518 objects misplaced (11.908%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T18:16:13.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:13 smithi130 ceph-mon[110068]: pgmap v2656: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 887 op/s; 15872/70518 objects degraded (22.508%); 8397/70518 objects misplaced (11.908%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T18:16:13.753 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:13.777 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:13.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:13.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:13 smithi179 ceph-mon[93519]: pgmap v2656: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.4 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 887 op/s; 15872/70518 objects degraded (22.508%); 8397/70518 objects misplaced (11.908%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T18:16:13.878 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:13.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:13.921 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:13.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:13.926 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:16:14.022 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:14.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:14.057 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:14.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:14.161 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:14.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:16:14.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:14.184 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:14.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:14.285 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:14.307 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:14.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:14.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:14.409 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:14.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:14.436 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:14.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:14.537 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:14.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:14.582 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:14.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:14.684 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:14.738 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:14.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:14.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:14.840 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:14.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:14.887 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:14.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:14.988 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:15.011 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:15.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:15.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:15.113 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:15.160 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:15.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:15.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:15.262 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:15.298 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:15.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:15.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:15.400 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:15.423 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:15.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:15.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:15.525 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:15.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:15.564 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:15.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:15.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:15 smithi179 ceph-mon[93519]: pgmap v2657: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.2 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 940 op/s; 13124/60390 objects degraded (21.732%); 7031/60390 objects misplaced (11.643%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T18:16:15.665 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:15.688 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:15.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:15.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:15.790 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:15.814 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:15.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:15.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:15.908 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:16:15.909 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:16:15.916 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:15.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:15 smithi130 ceph-mon[105521]: pgmap v2657: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.2 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 940 op/s; 13124/60390 objects degraded (21.732%); 7031/60390 objects misplaced (11.643%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T18:16:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:15 smithi130 ceph-mon[110068]: pgmap v2657: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.2 GiB data, 9.6 GiB used, 437 GiB / 447 GiB avail; 0 B/s wr, 940 op/s; 13124/60390 objects degraded (21.732%); 7031/60390 objects misplaced (11.643%); 4.5 MiB/s, 72 objects/s recovering 2024-08-17T18:16:15.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:15.956 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:15.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:16.057 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:16.081 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:16.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:16.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:16.182 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:16.205 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:16.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:16.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:16.306 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:16.346 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:16.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:16.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:16.448 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:16.471 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:16.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:16.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:16.573 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:16.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 13124/60390 objects degraded (21.732%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:16.596 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:16.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:16.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:16.661 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T18:16:16.698 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:16.738 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:16.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:16.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:16.840 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:16.872 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:16.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:16.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:16.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 13124/60390 objects degraded (21.732%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:16.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 13124/60390 objects degraded (21.732%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:16.960 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:16:16.974 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:17.010 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:17.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:17.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:17.113 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:17.137 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:17.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:17.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:17.238 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:17.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:17.277 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:17.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:17.378 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:17.414 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:17.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:17.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:17.515 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:17.547 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:17.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:17.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:17.650 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:17.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:17.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:17.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:17.788 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:17.809 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:17.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:17.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:17 smithi179 ceph-mon[93519]: pgmap v2658: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.2 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 934 op/s; 12532/56952 objects degraded (22.004%); 6734/56952 objects misplaced (11.824%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T18:16:17.910 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:17.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:17 smithi130 ceph-mon[105521]: pgmap v2658: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.2 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 934 op/s; 12532/56952 objects degraded (22.004%); 6734/56952 objects misplaced (11.824%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T18:16:17.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:17 smithi130 ceph-mon[110068]: pgmap v2658: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.2 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 934 op/s; 12532/56952 objects degraded (22.004%); 6734/56952 objects misplaced (11.824%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T18:16:17.934 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:17.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:17.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:18.036 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:18.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:18.074 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:18.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:18.175 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:18.222 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:18.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:18.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:18.324 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:18.357 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:18.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:18.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:18.459 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:18.483 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:18.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:18.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:18.585 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:18.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:18 smithi179 ceph-mon[93519]: pgmap v2659: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.2 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 934 op/s; 12532/56952 objects degraded (22.004%); 6734/56952 objects misplaced (11.824%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T18:16:18.606 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:18.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:18.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:18.708 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:18.729 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:18.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:18.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:18.831 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:18.852 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:18.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:18.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:18.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:18 smithi130 ceph-mon[105521]: pgmap v2659: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.2 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 934 op/s; 12532/56952 objects degraded (22.004%); 6734/56952 objects misplaced (11.824%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T18:16:18.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:18 smithi130 ceph-mon[110068]: pgmap v2659: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 1.2 GiB data, 9.4 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 934 op/s; 12532/56952 objects degraded (22.004%); 6734/56952 objects misplaced (11.824%); 4.4 MiB/s, 70 objects/s recovering 2024-08-17T18:16:18.954 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:18.961 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:16:18.961 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:16:18.977 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:18.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:18.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:19.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:19.101 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:19.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:19.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:19.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:19.227 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:19.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:19.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:19.329 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:19.352 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:19.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:19.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:19.454 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:19.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:19.477 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:19.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:19.578 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:19.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:19 smithi179 ceph-mon[93519]: osdmap e1409: 8 total, 7 up, 5 in 2024-08-17T18:16:19.601 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:19.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:19.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:19.658 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T18:16:19.703 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:19.726 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:19.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:19.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:19.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:19 smithi130 ceph-mon[105521]: osdmap e1409: 8 total, 7 up, 5 in 2024-08-17T18:16:19.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:19 smithi130 ceph-mon[110068]: osdmap e1409: 8 total, 7 up, 5 in 2024-08-17T18:16:19.827 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:19.848 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:19.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:19.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:19.932 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:16:19.950 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:19.971 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:19.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:19.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:20.073 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:20.098 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:20.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:20.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:20.140 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:16:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:16:19] "GET /metrics HTTP/1.1" 200 36808 "" "Prometheus/2.51.0" 2024-08-17T18:16:20.200 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:20.229 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:20.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:20.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:20.312 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:16:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:16:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:16:20.330 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:20.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:20.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:20.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:20.456 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:20.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:20.477 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:20.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:20.578 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:20.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:20 smithi179 ceph-mon[93519]: osdmap e1410: 8 total, 7 up, 5 in 2024-08-17T18:16:20.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:20 smithi179 ceph-mon[93519]: pgmap v2662: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 911 MiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.11k op/s; 9144/43803 objects degraded (20.875%); 5113/43803 objects misplaced (11.673%); 5.3 MiB/s, 85 objects/s recovering 2024-08-17T18:16:20.604 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:20.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:20.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:20.706 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:20.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:20.740 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:20.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:20.841 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:20.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:20.864 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:20.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:20.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:20 smithi130 ceph-mon[105521]: osdmap e1410: 8 total, 7 up, 5 in 2024-08-17T18:16:20.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:20 smithi130 ceph-mon[105521]: pgmap v2662: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 911 MiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.11k op/s; 9144/43803 objects degraded (20.875%); 5113/43803 objects misplaced (11.673%); 5.3 MiB/s, 85 objects/s recovering 2024-08-17T18:16:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:20 smithi130 ceph-mon[110068]: osdmap e1410: 8 total, 7 up, 5 in 2024-08-17T18:16:20.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:20 smithi130 ceph-mon[110068]: pgmap v2662: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 911 MiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.11k op/s; 9144/43803 objects degraded (20.875%); 5113/43803 objects misplaced (11.673%); 5.3 MiB/s, 85 objects/s recovering 2024-08-17T18:16:20.966 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:20.988 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:20.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:20.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:21.090 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:21.112 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:21.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:21.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:21.214 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:21.235 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:21.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:21.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:21.336 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:21.357 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:21.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:21.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:21.459 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:21.480 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:21.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:21.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:21.582 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:21.603 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:21.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:21.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:21.705 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:21.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:21.734 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:21.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:21.767 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:21 smithi179 ceph-mon[93519]: osdmap e1411: 8 total, 7 up, 5 in 2024-08-17T18:16:21.767 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 9144/43803 objects degraded (20.875%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:21.771 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:16:21.772 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:16:21.835 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:21.873 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:21.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:21.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:21.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:21 smithi130 ceph-mon[105521]: osdmap e1411: 8 total, 7 up, 5 in 2024-08-17T18:16:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 9144/43803 objects degraded (20.875%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:21 smithi130 ceph-mon[110068]: osdmap e1411: 8 total, 7 up, 5 in 2024-08-17T18:16:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 9144/43803 objects degraded (20.875%), 19 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:21.975 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:22.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:22.015 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:22.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:22.116 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:22.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:22.160 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:22.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:22.261 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:22.284 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:22.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:22.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:22.386 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:22.408 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:22.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:22.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:22.510 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:22.553 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:22.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:22.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:22.565 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T18:16:22.654 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:22.677 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:22.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:22.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:22.779 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:22.805 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:22.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:22.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:22 smithi179 ceph-mon[93519]: osdmap e1412: 8 total, 7 up, 5 in 2024-08-17T18:16:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:22 smithi179 ceph-mon[93519]: pgmap v2665: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 911 MiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.09k op/s; 9144/43803 objects degraded (20.875%); 5113/43803 objects misplaced (11.673%); 5.2 MiB/s, 83 objects/s recovering 2024-08-17T18:16:22.867 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:22 smithi130 ceph-mon[105521]: osdmap e1412: 8 total, 7 up, 5 in 2024-08-17T18:16:22.867 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:22 smithi130 ceph-mon[105521]: pgmap v2665: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 911 MiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.09k op/s; 9144/43803 objects degraded (20.875%); 5113/43803 objects misplaced (11.673%); 5.2 MiB/s, 83 objects/s recovering 2024-08-17T18:16:22.867 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:22 smithi130 ceph-mon[110068]: osdmap e1412: 8 total, 7 up, 5 in 2024-08-17T18:16:22.868 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:22 smithi130 ceph-mon[110068]: pgmap v2665: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 43 active+clean; 911 MiB data, 8.7 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1.09k op/s; 9144/43803 objects degraded (20.875%); 5113/43803 objects misplaced (11.673%); 5.2 MiB/s, 83 objects/s recovering 2024-08-17T18:16:22.872 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:16:22.907 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:22.929 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:22.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:22.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:23.031 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:23.054 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:23.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:23.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:23.156 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:23.201 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:23.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:23.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:23.304 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:23.330 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:23.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:23.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:23.432 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:23.475 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:23.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:23.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:23.577 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:23.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:23 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/747708654' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:16:23.617 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:23.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:23.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:23.718 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:23.761 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:23.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:23.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:23.787 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:16:23.788 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:16:23.788 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:16:23.788 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:16:23.788 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:16:23.788 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:16:23.788 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:16:23.788 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:16:23.788 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:16:23.788 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:16:23.789 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:16:23.789 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:16:23.789 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:16:23.789 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:16:23.789 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:16:23.789 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:23 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/747708654' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:16:23.790 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:23 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/747708654' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:16:23.863 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:23.885 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:23.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:23.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:23.987 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:24.009 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:24.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:24.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:24.111 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:24.133 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:24.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:24.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:24.235 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:24.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:24.274 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:24.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:24.375 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:24.410 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:24.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:24.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:24.512 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:24.536 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:24.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:24.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:24.638 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:24.667 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:24.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:24.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:24.769 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:24.775 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:24 smithi179 ceph-mon[93519]: pgmap v2666: 89 pgs: 1 active+clean+remapped, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 44 active+clean; 789 MiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 545 op/s; 6977/37911 objects degraded (18.404%); 3683/37911 objects misplaced (9.715%); 12 MiB/s, 198 objects/s recovering 2024-08-17T18:16:24.793 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:24.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:24.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:24.874 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:16:24.875 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:16:24.875 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 0 2024-08-17T18:16:24.875 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:16:24.894 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:24.915 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:24.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:24.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:24.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:24 smithi130 ceph-mon[105521]: pgmap v2666: 89 pgs: 1 active+clean+remapped, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 44 active+clean; 789 MiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 545 op/s; 6977/37911 objects degraded (18.404%); 3683/37911 objects misplaced (9.715%); 12 MiB/s, 198 objects/s recovering 2024-08-17T18:16:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:24 smithi130 ceph-mon[110068]: pgmap v2666: 89 pgs: 1 active+clean+remapped, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 44 active+clean; 789 MiB data, 8.6 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 545 op/s; 6977/37911 objects degraded (18.404%); 3683/37911 objects misplaced (9.715%); 12 MiB/s, 198 objects/s recovering 2024-08-17T18:16:25.017 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:25.038 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:25.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:25.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:25.140 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:25.163 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:25.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:25.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:25.181 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:25.265 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:25.289 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:25.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:25.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:25.391 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:25.412 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:25.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:25.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:25.514 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:25.538 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T18:16:25.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:25.540 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:25.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:25.640 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:25.666 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:25.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:25.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:25 smithi130 ceph-mon[105521]: osdmap e1413: 8 total, 7 up, 5 in 2024-08-17T18:16:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:25 smithi130 ceph-mon[110068]: osdmap e1413: 8 total, 7 up, 5 in 2024-08-17T18:16:25.768 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:25.794 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:25.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:25.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:25.831 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:16:25.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:25 smithi179 ceph-mon[93519]: osdmap e1413: 8 total, 7 up, 5 in 2024-08-17T18:16:25.896 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:25.928 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:25.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:25.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:26.030 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:26.067 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:26.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:26.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:26.169 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:26.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:26.211 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:26.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:26.312 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:26.343 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:26.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:26.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:26.445 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:26.468 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:26.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:26.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:26.570 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:26.606 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:26.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:26.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:26.708 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:26.730 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:26.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:26.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:26.832 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:26.838 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:26 smithi130 ceph-mon[105521]: osdmap e1414: 8 total, 7 up, 5 in 2024-08-17T18:16:26.838 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:26 smithi130 ceph-mon[105521]: pgmap v2669: 89 pgs: 1 active+clean+remapped, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 44 active+clean; 763 MiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 468 op/s; 6785/36687 objects degraded (18.494%); 3467/36687 objects misplaced (9.450%); 12 MiB/s, 198 objects/s recovering 2024-08-17T18:16:26.838 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 6977/37911 objects degraded (18.404%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:26.838 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:26 smithi130 ceph-mon[110068]: osdmap e1414: 8 total, 7 up, 5 in 2024-08-17T18:16:26.838 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:26 smithi130 ceph-mon[110068]: pgmap v2669: 89 pgs: 1 active+clean+remapped, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 44 active+clean; 763 MiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 468 op/s; 6785/36687 objects degraded (18.494%); 3467/36687 objects misplaced (9.450%); 12 MiB/s, 198 objects/s recovering 2024-08-17T18:16:26.838 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 6977/37911 objects degraded (18.404%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:26.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:26 smithi179 ceph-mon[93519]: osdmap e1414: 8 total, 7 up, 5 in 2024-08-17T18:16:26.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:26 smithi179 ceph-mon[93519]: pgmap v2669: 89 pgs: 1 active+clean+remapped, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 44 active+clean; 763 MiB data, 8.5 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 468 op/s; 6785/36687 objects degraded (18.494%); 3467/36687 objects misplaced (9.450%); 12 MiB/s, 198 objects/s recovering 2024-08-17T18:16:26.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 6977/37911 objects degraded (18.404%), 16 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:26.854 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:26.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:26.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:26.891 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:16:26.892 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:16:26.955 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:26.977 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:26.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:26.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:27.079 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:27.100 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:27.101 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:27.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:27.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:27.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:27.230 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:27.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:27.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:27.332 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:27.354 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:27.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:27.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:27.456 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:27.495 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:27.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:27.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:27.596 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:27.633 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:27.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:27.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:27.734 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:16:27.735 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:27.744 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:16:27.744 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:16:27.744 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:16:27.744 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:16:27.744 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:16:27.758 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:27.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:27.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:27.778 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:16:27.779 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:16:27.860 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:27.883 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:27.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:27.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:27.985 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:28.008 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:28.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:28.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:28.109 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:28.132 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:28.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:28.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:28.234 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:28.258 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:28.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:28.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:28.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:28.385 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:28.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:28.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:28.486 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:28.510 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:28.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:28.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:28.519 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T18:16:28.612 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:28.634 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:28.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:28.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:28.736 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:28.777 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:28.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:28.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:28.835 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:16:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:28 smithi179 ceph-mon[93519]: osdmap e1415: 8 total, 7 up, 5 in 2024-08-17T18:16:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:28 smithi179 ceph-mon[93519]: pgmap v2671: 89 pgs: 2 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 696 MiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 573 op/s; 5903/33480 objects degraded (17.631%); 3791/33480 objects misplaced (11.323%); 19 MiB/s, 306 objects/s recovering 2024-08-17T18:16:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:28.879 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:28.894 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:16:28.900 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:28.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:28.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:28 smithi130 ceph-mon[105521]: osdmap e1415: 8 total, 7 up, 5 in 2024-08-17T18:16:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:28 smithi130 ceph-mon[105521]: pgmap v2671: 89 pgs: 2 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 696 MiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 573 op/s; 5903/33480 objects degraded (17.631%); 3791/33480 objects misplaced (11.323%); 19 MiB/s, 306 objects/s recovering 2024-08-17T18:16:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:28.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:28 smithi130 ceph-mon[110068]: osdmap e1415: 8 total, 7 up, 5 in 2024-08-17T18:16:28.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:28 smithi130 ceph-mon[110068]: pgmap v2671: 89 pgs: 2 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 696 MiB data, 8.5 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 573 op/s; 5903/33480 objects degraded (17.631%); 3791/33480 objects misplaced (11.323%); 19 MiB/s, 306 objects/s recovering 2024-08-17T18:16:28.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:28.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:29.002 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:29.022 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:29.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:29.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:29.124 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:29.145 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:29.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:29.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:29.246 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:29.269 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:29.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:29.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:29.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:29.393 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:29.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:29.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:29.494 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:29.517 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:29.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:29.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:29.619 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:29.620 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:29 smithi179 ceph-mon[93519]: osdmap e1416: 8 total, 7 up, 5 in 2024-08-17T18:16:29.620 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:29 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:16:29.621 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:29 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:16:29.621 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:29 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:29.641 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:29.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:29.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:29.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:29 smithi130 ceph-mon[105521]: osdmap e1416: 8 total, 7 up, 5 in 2024-08-17T18:16:29.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:29 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:16:29.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:29 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:16:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:29 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:29 smithi130 ceph-mon[110068]: osdmap e1416: 8 total, 7 up, 5 in 2024-08-17T18:16:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:29 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:16:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:29 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:16:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:29 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:29.743 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:29.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:29.768 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:29.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:29.869 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:29.891 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:29.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:29.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:29.993 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:30.014 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:30.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:30.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:30.117 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:30.141 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:30.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:30.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:16:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:16:29] "GET /metrics HTTP/1.1" 200 36806 "" "Prometheus/2.51.0" 2024-08-17T18:16:30.242 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:30.263 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:30.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:30.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:30.365 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:30.388 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:30.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:30.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:30.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:30.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:30.513 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:30.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:30.550 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:16:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:16:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:16:30.614 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:30.635 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:30.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:30.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:30.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:30 smithi130 ceph-mon[105521]: pgmap v2673: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 538 MiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 797 op/s; 3488/25872 objects degraded (13.482%); 2522/25872 objects misplaced (9.748%); 21 MiB/s, 331 objects/s recovering 2024-08-17T18:16:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:30 smithi130 ceph-mon[105521]: osdmap e1417: 8 total, 7 up, 5 in 2024-08-17T18:16:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:30 smithi130 ceph-mon[110068]: pgmap v2673: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 538 MiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 797 op/s; 3488/25872 objects degraded (13.482%); 2522/25872 objects misplaced (9.748%); 21 MiB/s, 331 objects/s recovering 2024-08-17T18:16:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:30 smithi130 ceph-mon[110068]: osdmap e1417: 8 total, 7 up, 5 in 2024-08-17T18:16:30.737 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:30.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:30.778 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:30.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:30.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:30 smithi179 ceph-mon[93519]: pgmap v2673: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 538 MiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 797 op/s; 3488/25872 objects degraded (13.482%); 2522/25872 objects misplaced (9.748%); 21 MiB/s, 331 objects/s recovering 2024-08-17T18:16:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:30 smithi179 ceph-mon[93519]: osdmap e1417: 8 total, 7 up, 5 in 2024-08-17T18:16:30.879 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:30.880 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:30.919 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:30.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:30.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:31.021 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:31.044 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:31.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:31.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:31.118 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:16:31.118 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T18:16:31.118 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T18:16:31.118 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T18:16:31.118 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.118 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:115453 48.f 48:f4eb53da:::benchmark_data_smithi130_327597_object115452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:16:31.118 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:08:57.106348+0000", 2024-08-17T18:16:31.119 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 454.00976906800003, 2024-08-17T18:16:31.119 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.643288114, 2024-08-17T18:16:31.119 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.119 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.119 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.119 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:16:31.119 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:31.119 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 115453 2024-08-17T18:16:31.119 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.119 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.119 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.119 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.119 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106348+0000", 2024-08-17T18:16:31.119 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.120 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.120 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.120 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.120 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106348+0000", 2024-08-17T18:16:31.120 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.120 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.120 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.120 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.120 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106347+0000", 2024-08-17T18:16:31.120 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:16:31.120 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.120 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.121 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.121 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106404+0000", 2024-08-17T18:16:31.121 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7111000000000003e-05 2024-08-17T18:16:31.121 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.121 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.121 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.121 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106414+0000", 2024-08-17T18:16:31.121 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0148e-05 2024-08-17T18:16:31.121 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.121 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.121 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.121 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106418+0000", 2024-08-17T18:16:31.121 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1910000000000002e-06 2024-08-17T18:16:31.121 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.122 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.122 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.122 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106459+0000", 2024-08-17T18:16:31.122 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1046e-05 2024-08-17T18:16:31.122 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.122 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.122 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:16:31.122 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.106470+0000", 2024-08-17T18:16:31.122 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0722e-05 2024-08-17T18:16:31.122 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.122 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.122 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.123 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558373+0000", 2024-08-17T18:16:31.123 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.45190306899999999 2024-08-17T18:16:31.123 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.123 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.123 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:16:31.123 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558385+0000", 2024-08-17T18:16:31.123 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1593999999999999e-05 2024-08-17T18:16:31.123 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.123 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.123 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.123 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.727268+0000", 2024-08-17T18:16:31.123 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.16888346 2024-08-17T18:16:31.123 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.124 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.124 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.124 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.729887+0000", 2024-08-17T18:16:31.124 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002618521 2024-08-17T18:16:31.124 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.124 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.124 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1", 2024-08-17T18:16:31.124 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.730169+0000", 2024-08-17T18:16:31.124 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00028207800000000001 2024-08-17T18:16:31.124 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.124 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.125 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.125 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.744462+0000", 2024-08-17T18:16:31.125 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.01429271 2024-08-17T18:16:31.125 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.125 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.125 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.125 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.745188+0000", 2024-08-17T18:16:31.125 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00072621100000000004 2024-08-17T18:16:31.125 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.125 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.125 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.125 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.749537+0000", 2024-08-17T18:16:31.125 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0043489000000000002 2024-08-17T18:16:31.125 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.126 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.126 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.126 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.749571+0000", 2024-08-17T18:16:31.126 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4696999999999997e-05 2024-08-17T18:16:31.126 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.126 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.126 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.126 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.749636+0000", 2024-08-17T18:16:31.126 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4455999999999994e-05 2024-08-17T18:16:31.126 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.126 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.126 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.126 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.126 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.127 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:115456 48.7 48:e9a5436a:::benchmark_data_smithi130_327597_object115455:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:16:31.127 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:08:57.109169+0000", 2024-08-17T18:16:31.127 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 454.006947943, 2024-08-17T18:16:31.127 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5023469999999999, 2024-08-17T18:16:31.127 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.127 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.127 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.127 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:16:31.127 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:31.127 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 115456 2024-08-17T18:16:31.127 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.127 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.127 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.127 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.128 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109169+0000", 2024-08-17T18:16:31.128 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.128 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.128 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.128 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.128 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109169+0000", 2024-08-17T18:16:31.128 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.128 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.128 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.128 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.128 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109168+0000", 2024-08-17T18:16:31.128 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:16:31.128 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.129 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.129 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.129 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109227+0000", 2024-08-17T18:16:31.129 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8847000000000001e-05 2024-08-17T18:16:31.129 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.129 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.129 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.129 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109237+0000", 2024-08-17T18:16:31.129 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6129999999999993e-06 2024-08-17T18:16:31.129 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.130 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.130 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.130 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109240+0000", 2024-08-17T18:16:31.130 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.501e-06 2024-08-17T18:16:31.130 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.130 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.130 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.130 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109271+0000", 2024-08-17T18:16:31.130 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0582e-05 2024-08-17T18:16:31.130 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.130 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.130 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:16:31.130 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.109277+0000", 2024-08-17T18:16:31.131 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0100000000000001e-06 2024-08-17T18:16:31.131 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.131 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.131 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.131 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558408+0000", 2024-08-17T18:16:31.131 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.44913098299999998 2024-08-17T18:16:31.131 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.131 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.131 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:16:31.131 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558412+0000", 2024-08-17T18:16:31.131 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5669999999999996e-06 2024-08-17T18:16:31.131 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.131 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.132 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.132 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.587705+0000", 2024-08-17T18:16:31.132 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0292922280000001 2024-08-17T18:16:31.132 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.132 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.132 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.132 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.589250+0000", 2024-08-17T18:16:31.132 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0015458340000000001 2024-08-17T18:16:31.132 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.132 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.132 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:16:31.132 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.589440+0000", 2024-08-17T18:16:31.132 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000189276 2024-08-17T18:16:31.133 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.133 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.133 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.133 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.596555+0000", 2024-08-17T18:16:31.134 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0071154290000000004 2024-08-17T18:16:31.134 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.134 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.134 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.134 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.607974+0000", 2024-08-17T18:16:31.134 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.011418766 2024-08-17T18:16:31.134 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.134 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.134 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.134 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611424+0000", 2024-08-17T18:16:31.134 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0034497809999999999 2024-08-17T18:16:31.134 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.134 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.135 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.135 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611459+0000", 2024-08-17T18:16:31.135 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.5463000000000002e-05 2024-08-17T18:16:31.135 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.135 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.135 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.135 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611516+0000", 2024-08-17T18:16:31.135 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6674e-05 2024-08-17T18:16:31.135 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.135 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.135 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.135 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.135 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.136 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:115458 48.0 48:02c144b5:::benchmark_data_smithi130_327597_object115457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:16:31.136 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:08:57.111293+0000", 2024-08-17T18:16:31.136 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 454.004823929, 2024-08-17T18:16:31.136 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5002246669999999, 2024-08-17T18:16:31.136 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.136 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.136 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.136 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:16:31.136 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:31.136 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 115458 2024-08-17T18:16:31.136 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.136 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.136 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.136 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.137 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111293+0000", 2024-08-17T18:16:31.137 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.137 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.137 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.137 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.137 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111293+0000", 2024-08-17T18:16:31.137 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.137 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.137 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.137 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.137 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111292+0000", 2024-08-17T18:16:31.137 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:16:31.137 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.138 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.138 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.138 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111353+0000", 2024-08-17T18:16:31.138 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0925999999999999e-05 2024-08-17T18:16:31.138 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.138 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.138 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.138 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111361+0000", 2024-08-17T18:16:31.138 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7870000000000002e-06 2024-08-17T18:16:31.138 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.138 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.138 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.138 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111364+0000", 2024-08-17T18:16:31.138 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.5889999999999999e-06 2024-08-17T18:16:31.139 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.139 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.139 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.139 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111393+0000", 2024-08-17T18:16:31.139 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8082e-05 2024-08-17T18:16:31.139 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.139 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.142 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:16:31.142 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.111398+0000", 2024-08-17T18:16:31.142 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5260000000000001e-06 2024-08-17T18:16:31.142 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.142 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.143 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.143 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558838+0000", 2024-08-17T18:16:31.143 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.44744001700000002 2024-08-17T18:16:31.143 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.143 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.143 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:16:31.143 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558846+0000", 2024-08-17T18:16:31.143 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2209999999999997e-06 2024-08-17T18:16:31.143 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.143 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.143 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.143 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.583382+0000", 2024-08-17T18:16:31.143 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0245358609999999 2024-08-17T18:16:31.143 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.144 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.144 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.144 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.585501+0000", 2024-08-17T18:16:31.144 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0021187269999999999 2024-08-17T18:16:31.144 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.144 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.144 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:16:31.144 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.585697+0000", 2024-08-17T18:16:31.144 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019644500000000001 2024-08-17T18:16:31.144 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.144 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.144 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.144 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.594573+0000", 2024-08-17T18:16:31.144 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0088757130000000004 2024-08-17T18:16:31.145 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.145 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.145 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.145 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611301+0000", 2024-08-17T18:16:31.145 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.016727606999999999 2024-08-17T18:16:31.145 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.145 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.145 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.145 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611426+0000", 2024-08-17T18:16:31.145 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012492300000000001 2024-08-17T18:16:31.145 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.145 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.145 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.145 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611462+0000", 2024-08-17T18:16:31.146 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6795000000000003e-05 2024-08-17T18:16:31.146 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.146 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.146 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.146 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611518+0000", 2024-08-17T18:16:31.146 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5151000000000001e-05 2024-08-17T18:16:31.146 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.146 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.146 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.146 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.146 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.146 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:115464 48.0 48:09f85e4e:::benchmark_data_smithi130_327597_object115463:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:16:31.146 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:08:57.117335+0000", 2024-08-17T18:16:31.147 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 453.99878228099999, 2024-08-17T18:16:31.147 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.499090252, 2024-08-17T18:16:31.147 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.147 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.147 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.147 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:16:31.147 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:31.147 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 115464 2024-08-17T18:16:31.147 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.147 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.147 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.147 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.147 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117335+0000", 2024-08-17T18:16:31.148 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.148 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.148 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.148 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.148 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117335+0000", 2024-08-17T18:16:31.148 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.148 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.148 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.148 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.148 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117334+0000", 2024-08-17T18:16:31.148 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:16:31.148 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.148 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.148 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.149 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117396+0000", 2024-08-17T18:16:31.149 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2404999999999996e-05 2024-08-17T18:16:31.149 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.149 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.149 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.149 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117406+0000", 2024-08-17T18:16:31.149 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6379999999999999e-06 2024-08-17T18:16:31.149 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.149 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.149 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.149 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117410+0000", 2024-08-17T18:16:31.149 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9700000000000001e-06 2024-08-17T18:16:31.149 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.149 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.150 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.150 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117445+0000", 2024-08-17T18:16:31.150 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4657999999999999e-05 2024-08-17T18:16:31.150 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.150 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.150 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:16:31.150 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.117450+0000", 2024-08-17T18:16:31.150 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6030000000000004e-06 2024-08-17T18:16:31.150 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.150 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.150 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.150 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558868+0000", 2024-08-17T18:16:31.151 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.44141762299999998 2024-08-17T18:16:31.151 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.151 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.151 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:16:31.151 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:57.558875+0000", 2024-08-17T18:16:31.151 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.4950000000000002e-06 2024-08-17T18:16:31.151 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.151 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.151 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.151 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.593273+0000", 2024-08-17T18:16:31.151 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0343973630000001 2024-08-17T18:16:31.151 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.151 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.151 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.152 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.593391+0000", 2024-08-17T18:16:31.152 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000118168 2024-08-17T18:16:31.152 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.152 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.152 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:16:31.152 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.593483+0000", 2024-08-17T18:16:31.152 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2351999999999999e-05 2024-08-17T18:16:31.152 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.152 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.152 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.152 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.594707+0000", 2024-08-17T18:16:31.152 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001224155 2024-08-17T18:16:31.152 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.152 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.153 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.153 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.611339+0000", 2024-08-17T18:16:31.153 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.016631277 2024-08-17T18:16:31.153 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.153 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.153 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.153 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.616336+0000", 2024-08-17T18:16:31.153 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0049975769999999996 2024-08-17T18:16:31.153 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.153 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.153 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.153 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.616371+0000", 2024-08-17T18:16:31.153 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4779e-05 2024-08-17T18:16:31.153 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.154 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.154 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.154 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:08:58.616425+0000", 2024-08-17T18:16:31.154 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3881000000000001e-05 2024-08-17T18:16:31.154 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.154 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.154 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.154 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.154 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.154 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:238970 48.7 48:e5ad0cc4:::benchmark_data_smithi130_327597_object107005:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:16:31.154 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:11:32.587974+0000", 2024-08-17T18:16:31.154 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 298.52814236500001, 2024-08-17T18:16:31.154 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.52371970099999998, 2024-08-17T18:16:31.155 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.155 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.155 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.155 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:16:31.155 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:31.155 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 238970 2024-08-17T18:16:31.155 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.155 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.155 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.155 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.155 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.587974+0000", 2024-08-17T18:16:31.155 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.155 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.156 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.156 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.156 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.587974+0000", 2024-08-17T18:16:31.156 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.156 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.156 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.156 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.156 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.587974+0000", 2024-08-17T18:16:31.156 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:16:31.156 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.156 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.156 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.156 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.587977+0000", 2024-08-17T18:16:31.156 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.636e-06 2024-08-17T18:16:31.157 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.157 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.157 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.157 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.587983+0000", 2024-08-17T18:16:31.157 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7960000000000001e-06 2024-08-17T18:16:31.157 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.157 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.157 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.157 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.587986+0000", 2024-08-17T18:16:31.157 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1219999999999999e-06 2024-08-17T18:16:31.157 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.157 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.157 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.157 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588019+0000", 2024-08-17T18:16:31.158 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.26e-05 2024-08-17T18:16:31.158 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.158 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.158 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.158 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588138+0000", 2024-08-17T18:16:31.158 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00011885899999999999 2024-08-17T18:16:31.158 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.158 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.158 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,2,3", 2024-08-17T18:16:31.158 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588255+0000", 2024-08-17T18:16:31.158 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00011673200000000001 2024-08-17T18:16:31.158 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.158 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.159 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.159 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.590544+0000", 2024-08-17T18:16:31.159 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002289859 2024-08-17T18:16:31.159 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.159 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.159 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.159 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.594087+0000", 2024-08-17T18:16:31.159 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0035425790000000001 2024-08-17T18:16:31.159 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.159 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.159 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.159 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.599621+0000", 2024-08-17T18:16:31.159 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0055342430000000003 2024-08-17T18:16:31.159 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.160 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.160 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.160 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.111497+0000", 2024-08-17T18:16:31.160 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.51187599399999995 2024-08-17T18:16:31.160 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.160 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.160 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.160 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.111551+0000", 2024-08-17T18:16:31.160 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3338000000000003e-05 2024-08-17T18:16:31.160 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.160 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.160 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.160 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.111694+0000", 2024-08-17T18:16:31.160 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014359299999999999 2024-08-17T18:16:31.161 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.161 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.161 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.161 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.161 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.161 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:238971 48.c 48:31f99771:::benchmark_data_smithi130_327597_object107006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:16:31.161 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:11:32.587991+0000", 2024-08-17T18:16:31.161 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 298.52812555499997, 2024-08-17T18:16:31.161 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.51273529699999998, 2024-08-17T18:16:31.161 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.161 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.161 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.161 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:16:31.161 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:31.162 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 238971 2024-08-17T18:16:31.162 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.162 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.162 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.162 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.162 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.587991+0000", 2024-08-17T18:16:31.162 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.162 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.162 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.162 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.162 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.587991+0000", 2024-08-17T18:16:31.162 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.163 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.163 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.163 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.163 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.587991+0000", 2024-08-17T18:16:31.163 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:16:31.163 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.163 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.163 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.163 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.587992+0000", 2024-08-17T18:16:31.163 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4139999999999999e-06 2024-08-17T18:16:31.163 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.163 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.163 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.163 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.587995+0000", 2024-08-17T18:16:31.164 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0189999999999998e-06 2024-08-17T18:16:31.164 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.164 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.164 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.164 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.587997+0000", 2024-08-17T18:16:31.164 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0090000000000002e-06 2024-08-17T18:16:31.164 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.164 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.164 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.164 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588034+0000", 2024-08-17T18:16:31.164 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.625e-05 2024-08-17T18:16:31.164 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.164 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.165 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.165 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588124+0000", 2024-08-17T18:16:31.165 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0508999999999997e-05 2024-08-17T18:16:31.165 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.165 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.165 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1,3", 2024-08-17T18:16:31.165 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588201+0000", 2024-08-17T18:16:31.165 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7020000000000005e-05 2024-08-17T18:16:31.165 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.165 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.165 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.165 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.590150+0000", 2024-08-17T18:16:31.165 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019486589999999999 2024-08-17T18:16:31.166 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.166 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.166 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.166 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.594738+0000", 2024-08-17T18:16:31.166 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0045881300000000002 2024-08-17T18:16:31.166 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.166 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.166 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.166 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.599360+0000", 2024-08-17T18:16:31.166 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0046215680000000004 2024-08-17T18:16:31.166 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.166 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.166 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.100610+0000", 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.50125079400000006 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.100654+0000", 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3812e-05 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.100727+0000", 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.2354000000000006e-05 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.167 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.168 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.168 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.168 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:238972 48.f 48:f748d370:::benchmark_data_smithi130_327597_object107007:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:16:31.168 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:11:32.588052+0000", 2024-08-17T18:16:31.168 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 298.528064836, 2024-08-17T18:16:31.168 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.51957863400000004, 2024-08-17T18:16:31.168 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.168 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.168 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.168 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:16:31.168 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:31.168 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 238972 2024-08-17T18:16:31.168 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.169 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.169 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.169 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.169 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588052+0000", 2024-08-17T18:16:31.169 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.169 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.169 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.169 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.169 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588052+0000", 2024-08-17T18:16:31.169 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.169 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.169 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.169 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.169 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588051+0000", 2024-08-17T18:16:31.170 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:16:31.170 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.170 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.170 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.170 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588055+0000", 2024-08-17T18:16:31.170 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8120000000000001e-06 2024-08-17T18:16:31.170 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.170 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.170 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.170 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588062+0000", 2024-08-17T18:16:31.170 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0029999999999997e-06 2024-08-17T18:16:31.170 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.171 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.171 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.171 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588067+0000", 2024-08-17T18:16:31.171 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2639999999999999e-06 2024-08-17T18:16:31.171 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.171 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.171 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.171 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588108+0000", 2024-08-17T18:16:31.171 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0535999999999999e-05 2024-08-17T18:16:31.171 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.171 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.171 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.171 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588232+0000", 2024-08-17T18:16:31.172 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012370799999999999 2024-08-17T18:16:31.172 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.172 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.172 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1,3", 2024-08-17T18:16:31.172 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.588367+0000", 2024-08-17T18:16:31.172 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013500200000000001 2024-08-17T18:16:31.172 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.172 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.172 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.172 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.590860+0000", 2024-08-17T18:16:31.172 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024930989999999999 2024-08-17T18:16:31.172 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.172 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.173 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.173 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.594718+0000", 2024-08-17T18:16:31.173 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0038583160000000001 2024-08-17T18:16:31.173 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.173 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.173 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.173 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.599604+0000", 2024-08-17T18:16:31.173 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0048857620000000001 2024-08-17T18:16:31.173 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.173 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.173 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.173 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.107439+0000", 2024-08-17T18:16:31.174 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.50783477399999999 2024-08-17T18:16:31.174 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.174 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.174 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.174 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.107505+0000", 2024-08-17T18:16:31.174 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6266000000000006e-05 2024-08-17T18:16:31.174 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.174 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.174 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.174 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.107631+0000", 2024-08-17T18:16:31.174 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012569299999999999 2024-08-17T18:16:31.174 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.175 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.175 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.175 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.175 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.175 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:238976 48.7 48:e381fbea:::benchmark_data_smithi130_327597_object107011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:16:31.175 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:11:32.594090+0000", 2024-08-17T18:16:31.175 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 298.52202661299998, 2024-08-17T18:16:31.175 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.517743183, 2024-08-17T18:16:31.175 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.175 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.175 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.175 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:16:31.175 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:31.176 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 238976 2024-08-17T18:16:31.176 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.176 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.176 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.176 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.176 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.594090+0000", 2024-08-17T18:16:31.176 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.176 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.176 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.176 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.176 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.594090+0000", 2024-08-17T18:16:31.176 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.176 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.177 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.177 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.177 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.594090+0000", 2024-08-17T18:16:31.177 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:16:31.177 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.177 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.177 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.177 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.594093+0000", 2024-08-17T18:16:31.177 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8929999999999998e-06 2024-08-17T18:16:31.177 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.177 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.177 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.177 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.594099+0000", 2024-08-17T18:16:31.178 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9359999999999996e-06 2024-08-17T18:16:31.178 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.178 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.178 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.178 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.594102+0000", 2024-08-17T18:16:31.178 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9969999999999999e-06 2024-08-17T18:16:31.178 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.178 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.178 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.178 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.594146+0000", 2024-08-17T18:16:31.178 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3207999999999997e-05 2024-08-17T18:16:31.178 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.178 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.178 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.179 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.594272+0000", 2024-08-17T18:16:31.179 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012673900000000001 2024-08-17T18:16:31.179 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:31.181 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.181 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.181 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,2,3", 2024-08-17T18:16:31.181 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.594413+0000", 2024-08-17T18:16:31.181 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00014021899999999999 2024-08-17T18:16:31.181 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.181 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.181 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.181 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.596293+0000", 2024-08-17T18:16:31.181 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00188021 2024-08-17T18:16:31.181 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.182 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.182 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.182 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.601623+0000", 2024-08-17T18:16:31.182 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0053302549999999999 2024-08-17T18:16:31.182 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.182 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.182 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.182 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.603639+0000", 2024-08-17T18:16:31.182 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0020161559999999999 2024-08-17T18:16:31.182 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.182 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.182 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.183 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.111726+0000", 2024-08-17T18:16:31.183 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.50808725799999999 2024-08-17T18:16:31.183 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.183 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.183 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.183 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.111762+0000", 2024-08-17T18:16:31.183 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.5973000000000003e-05 2024-08-17T18:16:31.183 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.183 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.183 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.184 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.111833+0000", 2024-08-17T18:16:31.184 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0957999999999994e-05 2024-08-17T18:16:31.184 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.184 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.184 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.184 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.184 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.184 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:238984 48.7 48:e6e2b480:::benchmark_data_smithi130_327597_object107019:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:16:31.184 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:11:32.599638+0000", 2024-08-17T18:16:31.184 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 298.51647911999999, 2024-08-17T18:16:31.184 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.51241831699999996, 2024-08-17T18:16:31.184 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.184 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.185 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.185 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:16:31.185 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:31.185 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 238984 2024-08-17T18:16:31.185 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.185 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.185 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.185 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.185 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.599638+0000", 2024-08-17T18:16:31.185 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.185 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.185 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.185 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.185 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.599638+0000", 2024-08-17T18:16:31.186 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.186 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.186 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.186 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.186 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.599637+0000", 2024-08-17T18:16:31.186 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:16:31.186 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.186 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.186 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.186 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.599642+0000", 2024-08-17T18:16:31.186 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1730000000000003e-06 2024-08-17T18:16:31.186 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.186 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.187 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.187 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.599653+0000", 2024-08-17T18:16:31.187 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1117e-05 2024-08-17T18:16:31.187 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.187 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.187 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.187 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.599659+0000", 2024-08-17T18:16:31.187 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2110000000000003e-06 2024-08-17T18:16:31.187 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.187 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.187 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.187 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.599696+0000", 2024-08-17T18:16:31.187 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6600999999999997e-05 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.599826+0000", 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013040999999999999 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,2,3", 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.599967+0000", 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000140777 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.601719+0000", 2024-08-17T18:16:31.188 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0017525310000000001 2024-08-17T18:16:31.189 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.189 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.189 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.189 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.603956+0000", 2024-08-17T18:16:31.189 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0022365449999999999 2024-08-17T18:16:31.189 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.189 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.189 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.189 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.604167+0000", 2024-08-17T18:16:31.189 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000210972 2024-08-17T18:16:31.189 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.189 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.189 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.190 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.111874+0000", 2024-08-17T18:16:31.190 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.50770724499999997 2024-08-17T18:16:31.190 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.190 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.190 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.190 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.111926+0000", 2024-08-17T18:16:31.190 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2142000000000003e-05 2024-08-17T18:16:31.190 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.190 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.190 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.190 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.112056+0000", 2024-08-17T18:16:31.190 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012970200000000001 2024-08-17T18:16:31.190 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.191 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.191 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.191 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.191 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.191 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:238987 48.f 48:f0dddeb0:::benchmark_data_smithi130_327597_object107022:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:16:31.191 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:11:32.604361+0000", 2024-08-17T18:16:31.191 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 298.51175549099997, 2024-08-17T18:16:31.191 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.50908473200000004, 2024-08-17T18:16:31.191 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.191 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.192 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.192 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:16:31.192 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:31.192 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 238987 2024-08-17T18:16:31.192 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.192 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.192 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.192 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.192 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.604361+0000", 2024-08-17T18:16:31.192 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.192 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.192 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.192 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.193 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.604361+0000", 2024-08-17T18:16:31.193 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.193 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.193 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.193 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.193 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.604360+0000", 2024-08-17T18:16:31.193 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:16:31.193 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.193 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.193 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.193 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.604365+0000", 2024-08-17T18:16:31.193 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9239999999999998e-06 2024-08-17T18:16:31.193 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.193 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.194 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.194 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.604376+0000", 2024-08-17T18:16:31.194 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0366999999999999e-05 2024-08-17T18:16:31.194 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.194 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.194 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.194 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.604380+0000", 2024-08-17T18:16:31.194 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4830000000000001e-06 2024-08-17T18:16:31.194 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.194 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.194 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.194 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.604415+0000", 2024-08-17T18:16:31.194 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4600000000000001e-05 2024-08-17T18:16:31.195 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.195 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.195 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.195 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.604545+0000", 2024-08-17T18:16:31.195 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012995500000000001 2024-08-17T18:16:31.195 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.195 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.195 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1,3", 2024-08-17T18:16:31.195 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.604680+0000", 2024-08-17T18:16:31.195 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013503399999999999 2024-08-17T18:16:31.195 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.195 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.195 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.196 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.606802+0000", 2024-08-17T18:16:31.196 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002121932 2024-08-17T18:16:31.196 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.196 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.196 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.196 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.607308+0000", 2024-08-17T18:16:31.196 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00050666999999999997 2024-08-17T18:16:31.196 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.196 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.196 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.196 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.607480+0000", 2024-08-17T18:16:31.196 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000171411 2024-08-17T18:16:31.196 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.197 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.197 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.197 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.113269+0000", 2024-08-17T18:16:31.197 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.50578941700000002 2024-08-17T18:16:31.197 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.197 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.197 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.197 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.113319+0000", 2024-08-17T18:16:31.197 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9619e-05 2024-08-17T18:16:31.197 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.197 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.197 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.197 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.113446+0000", 2024-08-17T18:16:31.198 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012728300000000001 2024-08-17T18:16:31.198 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.198 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.198 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.198 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.198 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.198 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:238988 48.0 48:00eda41a:::benchmark_data_smithi130_327597_object107023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:16:31.198 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:11:32.605923+0000", 2024-08-17T18:16:31.198 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 298.51019366200001, 2024-08-17T18:16:31.198 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.50199315499999997, 2024-08-17T18:16:31.198 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.198 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.198 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.199 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:16:31.199 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:31.199 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 238988 2024-08-17T18:16:31.199 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.199 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.199 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.199 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.199 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.605923+0000", 2024-08-17T18:16:31.199 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.199 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.199 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.199 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.199 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.605923+0000", 2024-08-17T18:16:31.199 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.200 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.200 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.200 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.200 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.605922+0000", 2024-08-17T18:16:31.200 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:16:31.200 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.200 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.200 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.200 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.605926+0000", 2024-08-17T18:16:31.200 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.5090000000000002e-06 2024-08-17T18:16:31.200 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.200 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.200 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.200 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.605933+0000", 2024-08-17T18:16:31.201 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3640000000000001e-06 2024-08-17T18:16:31.201 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.201 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.201 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.201 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.605937+0000", 2024-08-17T18:16:31.201 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.45e-06 2024-08-17T18:16:31.201 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.201 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.201 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.201 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.605971+0000", 2024-08-17T18:16:31.201 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4544e-05 2024-08-17T18:16:31.201 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.201 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.202 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.202 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.606060+0000", 2024-08-17T18:16:31.202 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.8471e-05 2024-08-17T18:16:31.202 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.202 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.202 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,2,3", 2024-08-17T18:16:31.202 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.606137+0000", 2024-08-17T18:16:31.202 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7371000000000002e-05 2024-08-17T18:16:31.202 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.202 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.202 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.202 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.608060+0000", 2024-08-17T18:16:31.202 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00192318 2024-08-17T18:16:31.202 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.203 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.203 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.203 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.609176+0000", 2024-08-17T18:16:31.203 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0011154190000000001 2024-08-17T18:16:31.203 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.203 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.203 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.203 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.610017+0000", 2024-08-17T18:16:31.203 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00084169099999999999 2024-08-17T18:16:31.203 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.203 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.203 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.203 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.107814+0000", 2024-08-17T18:16:31.203 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.49779673899999999 2024-08-17T18:16:31.204 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.204 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.204 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.204 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.107850+0000", 2024-08-17T18:16:31.204 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.5816999999999999e-05 2024-08-17T18:16:31.204 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.204 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.204 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.204 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.107916+0000", 2024-08-17T18:16:31.204 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6552999999999998e-05 2024-08-17T18:16:31.204 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.204 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.204 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.205 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.205 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.205 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:238991 48.7 48:e161df05:::benchmark_data_smithi130_327597_object107026:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:16:31.205 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:11:32.608957+0000", 2024-08-17T18:16:31.205 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 298.50716000699998, 2024-08-17T18:16:31.205 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.50447804799999996, 2024-08-17T18:16:31.205 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.205 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.205 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.205 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:16:31.205 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:31.205 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 238991 2024-08-17T18:16:31.205 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.205 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.206 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.206 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.206 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.608957+0000", 2024-08-17T18:16:31.206 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.206 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.206 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.206 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.206 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.608957+0000", 2024-08-17T18:16:31.206 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.206 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.206 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.206 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.206 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.608956+0000", 2024-08-17T18:16:31.207 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:16:31.207 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.207 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.207 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.207 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.608960+0000", 2024-08-17T18:16:31.207 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2039999999999999e-06 2024-08-17T18:16:31.207 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.207 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.207 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.207 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.608970+0000", 2024-08-17T18:16:31.207 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.9040000000000004e-06 2024-08-17T18:16:31.207 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.207 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.207 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.208 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.608975+0000", 2024-08-17T18:16:31.208 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9080000000000003e-06 2024-08-17T18:16:31.208 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.208 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.208 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.208 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.609009+0000", 2024-08-17T18:16:31.208 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3491999999999997e-05 2024-08-17T18:16:31.208 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.208 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.208 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.208 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.609088+0000", 2024-08-17T18:16:31.208 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9493000000000006e-05 2024-08-17T18:16:31.208 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,2,3", 2024-08-17T18:16:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.609190+0000", 2024-08-17T18:16:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000101582 2024-08-17T18:16:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.611060+0000", 2024-08-17T18:16:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0018706079999999999 2024-08-17T18:16:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.611674+0000", 2024-08-17T18:16:31.209 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000613541 2024-08-17T18:16:31.210 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.210 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.210 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.210 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.611865+0000", 2024-08-17T18:16:31.210 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00019116900000000001 2024-08-17T18:16:31.210 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.210 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.210 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.210 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.113288+0000", 2024-08-17T18:16:31.210 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.50142292799999999 2024-08-17T18:16:31.210 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.210 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.211 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.211 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.113330+0000", 2024-08-17T18:16:31.211 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1789999999999998e-05 2024-08-17T18:16:31.211 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.211 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.211 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.211 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.113435+0000", 2024-08-17T18:16:31.211 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00010514300000000001 2024-08-17T18:16:31.211 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.211 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.211 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.211 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.211 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.211 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:238992 48.f 48:f24ecdd8:::benchmark_data_smithi130_327597_object107027:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:16:31.212 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:11:32.611977+0000", 2024-08-17T18:16:31.212 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 298.50414021799997, 2024-08-17T18:16:31.212 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.501582784, 2024-08-17T18:16:31.212 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.212 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.212 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.212 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:16:31.212 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:31.212 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 238992 2024-08-17T18:16:31.212 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.212 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.212 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.212 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.212 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.611977+0000", 2024-08-17T18:16:31.213 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.213 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.213 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.213 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.213 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.611977+0000", 2024-08-17T18:16:31.213 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.213 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.213 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.213 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.213 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.611976+0000", 2024-08-17T18:16:31.213 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:16:31.213 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.213 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.214 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.214 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.611980+0000", 2024-08-17T18:16:31.214 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0160000000000002e-06 2024-08-17T18:16:31.214 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.214 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.214 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.214 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.611988+0000", 2024-08-17T18:16:31.214 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.8390000000000007e-06 2024-08-17T18:16:31.214 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.214 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.214 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.214 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.611991+0000", 2024-08-17T18:16:31.214 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.252e-06 2024-08-17T18:16:31.215 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.215 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.215 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.215 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.612023+0000", 2024-08-17T18:16:31.215 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1739000000000002e-05 2024-08-17T18:16:31.215 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.215 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.215 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.215 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.612114+0000", 2024-08-17T18:16:31.215 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.1608999999999997e-05 2024-08-17T18:16:31.215 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.215 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.215 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1,3", 2024-08-17T18:16:31.215 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.612196+0000", 2024-08-17T18:16:31.216 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2144999999999994e-05 2024-08-17T18:16:31.216 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.216 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.216 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.216 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.615740+0000", 2024-08-17T18:16:31.216 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0035433819999999999 2024-08-17T18:16:31.216 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.216 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.216 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.216 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.615759+0000", 2024-08-17T18:16:31.216 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.9083e-05 2024-08-17T18:16:31.216 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.217 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.217 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.217 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.615948+0000", 2024-08-17T18:16:31.217 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00018937399999999999 2024-08-17T18:16:31.217 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.217 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.217 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.217 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.113477+0000", 2024-08-17T18:16:31.217 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.497528629 2024-08-17T18:16:31.217 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.217 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.217 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.217 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.113504+0000", 2024-08-17T18:16:31.218 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.7375000000000001e-05 2024-08-17T18:16:31.218 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.218 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.218 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.218 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.113559+0000", 2024-08-17T18:16:31.218 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5086000000000001e-05 2024-08-17T18:16:31.218 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.218 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.218 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.218 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.218 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.218 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36675.0:238993 48.f 48:fa497a2c:::benchmark_data_smithi130_327597_object107028:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-08-17T18:16:31.218 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:11:32.612099+0000", 2024-08-17T18:16:31.219 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 298.50401830800001, 2024-08-17T18:16:31.219 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.50157004800000005, 2024-08-17T18:16:31.219 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.219 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.219 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.219 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36675", 2024-08-17T18:16:31.219 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:31.219 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 238993 2024-08-17T18:16:31.219 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.219 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.219 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.219 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.219 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.612099+0000", 2024-08-17T18:16:31.219 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.220 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.220 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.220 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.220 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.612099+0000", 2024-08-17T18:16:31.220 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.220 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.220 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.220 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.220 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.612098+0000", 2024-08-17T18:16:31.220 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:16:31.220 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.220 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.220 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.221 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.612101+0000", 2024-08-17T18:16:31.221 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4580000000000002e-06 2024-08-17T18:16:31.221 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.221 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.221 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.221 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.612107+0000", 2024-08-17T18:16:31.221 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6930000000000004e-06 2024-08-17T18:16:31.221 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.221 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.221 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.221 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.612110+0000", 2024-08-17T18:16:31.221 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.371e-06 2024-08-17T18:16:31.221 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.221 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.222 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.222 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.614040+0000", 2024-08-17T18:16:31.222 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001929184 2024-08-17T18:16:31.222 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.222 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.222 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.222 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.614128+0000", 2024-08-17T18:16:31.222 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.8604999999999996e-05 2024-08-17T18:16:31.222 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.222 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.222 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1,3", 2024-08-17T18:16:31.222 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.614211+0000", 2024-08-17T18:16:31.222 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2663999999999993e-05 2024-08-17T18:16:31.223 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.223 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.223 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.223 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.615789+0000", 2024-08-17T18:16:31.223 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00157785 2024-08-17T18:16:31.223 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.223 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.223 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.223 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.615866+0000", 2024-08-17T18:16:31.223 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7395e-05 2024-08-17T18:16:31.223 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.223 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.223 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.223 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:32.616045+0000", 2024-08-17T18:16:31.224 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017870599999999999 2024-08-17T18:16:31.224 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.224 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.224 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.224 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.113582+0000", 2024-08-17T18:16:31.224 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.49753742299999998 2024-08-17T18:16:31.224 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.224 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.224 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.224 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.113615+0000", 2024-08-17T18:16:31.224 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2953000000000001e-05 2024-08-17T18:16:31.224 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.224 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.225 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.225 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:11:33.113669+0000", 2024-08-17T18:16:31.225 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3315e-05 2024-08-17T18:16:31.225 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.225 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.225 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.225 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.225 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.225 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36960.0:16116 49.5 49:afada5be:::benchmark_data_smithi130_364143_object16115:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1385)", 2024-08-17T18:16:31.225 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:13:01.046143+0000", 2024-08-17T18:16:31.225 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 210.06997386200001, 2024-08-17T18:16:31.225 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.32436821900000001, 2024-08-17T18:16:31.225 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.225 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.226 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.226 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36960", 2024-08-17T18:16:31.226 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:31.226 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 16116 2024-08-17T18:16:31.226 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.226 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.226 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.226 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.226 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046143+0000", 2024-08-17T18:16:31.226 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.226 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.226 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.226 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.227 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046143+0000", 2024-08-17T18:16:31.227 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.227 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.227 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.227 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.227 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046142+0000", 2024-08-17T18:16:31.227 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:16:31.227 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.227 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.227 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.227 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046205+0000", 2024-08-17T18:16:31.227 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2638999999999994e-05 2024-08-17T18:16:31.227 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.227 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.228 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.228 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046213+0000", 2024-08-17T18:16:31.228 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.4610000000000008e-06 2024-08-17T18:16:31.228 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.228 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.228 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.228 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046217+0000", 2024-08-17T18:16:31.228 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3419999999999999e-06 2024-08-17T18:16:31.228 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.228 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.228 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.228 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046250+0000", 2024-08-17T18:16:31.228 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3568000000000001e-05 2024-08-17T18:16:31.228 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.229 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.229 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.229 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046332+0000", 2024-08-17T18:16:31.229 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2097999999999999e-05 2024-08-17T18:16:31.229 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.229 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.229 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:16:31.229 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046417+0000", 2024-08-17T18:16:31.229 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.5106999999999997e-05 2024-08-17T18:16:31.229 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.229 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.229 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.229 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.047546+0000", 2024-08-17T18:16:31.230 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00112875 2024-08-17T18:16:31.230 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.230 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.230 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.230 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.049251+0000", 2024-08-17T18:16:31.230 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0017049680000000001 2024-08-17T18:16:31.230 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.230 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.230 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.230 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.370392+0000", 2024-08-17T18:16:31.230 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.321140598 2024-08-17T18:16:31.230 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.230 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.230 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.231 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.370469+0000", 2024-08-17T18:16:31.231 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7041999999999999e-05 2024-08-17T18:16:31.231 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.231 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.231 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.231 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.370511+0000", 2024-08-17T18:16:31.231 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2478000000000002e-05 2024-08-17T18:16:31.231 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.231 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.231 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.231 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.231 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.231 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36960.0:16120 49.5 49:ab6cbb4c:::benchmark_data_smithi130_364143_object16119:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1385)", 2024-08-17T18:16:31.231 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:13:01.046769+0000", 2024-08-17T18:16:31.232 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 210.069347784, 2024-08-17T18:16:31.232 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.32383754999999997, 2024-08-17T18:16:31.232 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.232 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.232 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.232 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36960", 2024-08-17T18:16:31.232 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:31.232 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 16120 2024-08-17T18:16:31.232 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.232 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.232 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.232 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.232 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046769+0000", 2024-08-17T18:16:31.232 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.233 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.233 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.233 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.233 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046769+0000", 2024-08-17T18:16:31.233 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.233 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.233 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.233 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.233 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046768+0000", 2024-08-17T18:16:31.233 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:16:31.233 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.233 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.233 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.233 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046838+0000", 2024-08-17T18:16:31.234 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0196000000000004e-05 2024-08-17T18:16:31.234 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.234 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.234 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.234 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046848+0000", 2024-08-17T18:16:31.234 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7329999999999998e-06 2024-08-17T18:16:31.234 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.234 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.234 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.234 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046853+0000", 2024-08-17T18:16:31.234 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0200000000000002e-06 2024-08-17T18:16:31.234 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.234 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.234 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.235 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.046956+0000", 2024-08-17T18:16:31.235 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000103244 2024-08-17T18:16:31.235 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.235 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.235 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.235 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.047051+0000", 2024-08-17T18:16:31.235 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5038999999999996e-05 2024-08-17T18:16:31.235 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.235 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.235 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:16:31.235 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.047178+0000", 2024-08-17T18:16:31.235 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012609200000000001 2024-08-17T18:16:31.235 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.236 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.236 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.236 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.047671+0000", 2024-08-17T18:16:31.236 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00049308999999999996 2024-08-17T18:16:31.236 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.236 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.236 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.236 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.049288+0000", 2024-08-17T18:16:31.236 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001617144 2024-08-17T18:16:31.236 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.236 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.236 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.236 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.370551+0000", 2024-08-17T18:16:31.236 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.32126297300000001 2024-08-17T18:16:31.237 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.237 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.237 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.237 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.370585+0000", 2024-08-17T18:16:31.237 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3943999999999999e-05 2024-08-17T18:16:31.237 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.237 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.237 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.237 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.370607+0000", 2024-08-17T18:16:31.237 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.1865000000000002e-05 2024-08-17T18:16:31.237 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.237 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.237 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.237 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.238 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.238 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36960.0:16126 49.8 49:1b4dbc68:::benchmark_data_smithi130_364143_object16125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1385)", 2024-08-17T18:16:31.238 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:13:01.050549+0000", 2024-08-17T18:16:31.238 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 210.065567773, 2024-08-17T18:16:31.238 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.32015333800000001, 2024-08-17T18:16:31.238 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.238 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.238 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.238 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36960", 2024-08-17T18:16:31.238 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:31.238 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 16126 2024-08-17T18:16:31.238 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.238 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.239 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.239 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.239 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.050549+0000", 2024-08-17T18:16:31.239 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.239 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.239 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.239 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.239 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.050549+0000", 2024-08-17T18:16:31.239 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.239 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.239 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.239 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.239 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.050548+0000", 2024-08-17T18:16:31.239 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:16:31.240 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.240 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.240 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.240 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.050615+0000", 2024-08-17T18:16:31.240 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6779999999999994e-05 2024-08-17T18:16:31.240 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.240 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.240 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.240 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.050624+0000", 2024-08-17T18:16:31.240 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3240000000000003e-06 2024-08-17T18:16:31.240 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.240 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.240 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.240 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.050627+0000", 2024-08-17T18:16:31.241 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2899999999999998e-06 2024-08-17T18:16:31.241 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.241 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.241 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.241 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.050664+0000", 2024-08-17T18:16:31.241 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7484000000000002e-05 2024-08-17T18:16:31.241 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.241 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.241 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.241 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.050786+0000", 2024-08-17T18:16:31.241 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000121365 2024-08-17T18:16:31.241 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.241 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.241 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 2,3", 2024-08-17T18:16:31.242 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.050925+0000", 2024-08-17T18:16:31.242 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013897099999999999 2024-08-17T18:16:31.242 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.242 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.242 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.242 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.051687+0000", 2024-08-17T18:16:31.242 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00076276200000000005 2024-08-17T18:16:31.242 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.242 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.242 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.242 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.052034+0000", 2024-08-17T18:16:31.242 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00034695 2024-08-17T18:16:31.242 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.243 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.243 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.243 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.370631+0000", 2024-08-17T18:16:31.243 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.31859703499999997 2024-08-17T18:16:31.243 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.243 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.243 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.243 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.370663+0000", 2024-08-17T18:16:31.243 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1905e-05 2024-08-17T18:16:31.243 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.243 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.243 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.243 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:13:01.370702+0000", 2024-08-17T18:16:31.244 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9128000000000002e-05 2024-08-17T18:16:31.244 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.244 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.244 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.244 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.244 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.244 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36960.0:186463 49.5 49:a8e4f8ef:::benchmark_data_smithi130_364143_object77329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1404)", 2024-08-17T18:16:31.244 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:15:58.351550+0000", 2024-08-17T18:16:31.244 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 32.764566412000001, 2024-08-17T18:16:31.244 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0452622110000001, 2024-08-17T18:16:31.244 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.244 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:16:31.244 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.244 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36960", 2024-08-17T18:16:31.245 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:31.245 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 186463 2024-08-17T18:16:31.245 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.245 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.245 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.245 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.245 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:15:58.351550+0000", 2024-08-17T18:16:31.245 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.245 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.245 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.245 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.245 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:15:58.351550+0000", 2024-08-17T18:16:31.245 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.245 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.246 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.246 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.246 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:15:58.351549+0000", 2024-08-17T18:16:31.246 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:16:31.246 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.246 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.246 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.246 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:15:58.351558+0000", 2024-08-17T18:16:31.246 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6940000000000001e-06 2024-08-17T18:16:31.246 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.246 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.246 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.246 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:15:58.351569+0000", 2024-08-17T18:16:31.247 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0973e-05 2024-08-17T18:16:31.247 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.247 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.247 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.247 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:15:58.351573+0000", 2024-08-17T18:16:31.247 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3619999999999999e-06 2024-08-17T18:16:31.247 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.247 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.247 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.247 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:15:58.351608+0000", 2024-08-17T18:16:31.247 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4502000000000003e-05 2024-08-17T18:16:31.247 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.247 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.248 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.248 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:15:58.351684+0000", 2024-08-17T18:16:31.248 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.6056999999999994e-05 2024-08-17T18:16:31.248 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.248 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.248 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:16:31.248 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:15:58.351750+0000", 2024-08-17T18:16:31.248 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6477999999999996e-05 2024-08-17T18:16:31.248 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.248 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.248 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.248 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:15:58.354326+0000", 2024-08-17T18:16:31.248 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025752119999999999 2024-08-17T18:16:31.248 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.249 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.249 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.249 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:15:58.365951+0000", 2024-08-17T18:16:31.249 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.011625699 2024-08-17T18:16:31.249 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.249 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.249 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:16:31.249 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:15:58.365957+0000", 2024-08-17T18:16:31.249 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6690000000000003e-06 2024-08-17T18:16:31.249 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.249 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.249 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.249 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:15:59.396728+0000", 2024-08-17T18:16:31.249 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0307709119999999 2024-08-17T18:16:31.250 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.250 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.250 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.250 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:15:59.396813+0000", 2024-08-17T18:16:31.250 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.4691000000000006e-05 2024-08-17T18:16:31.250 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.250 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.250 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.250 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.250 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.250 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36960.0:189025 49.9 49:9acbfbf2:::benchmark_data_smithi130_364143_object79891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1407)", 2024-08-17T18:16:31.250 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:16:02.248520+0000", 2024-08-17T18:16:31.250 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 28.867596524, 2024-08-17T18:16:31.250 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.089524655999999994, 2024-08-17T18:16:31.251 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.251 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.251 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.251 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36960", 2024-08-17T18:16:31.251 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:31.251 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 189025 2024-08-17T18:16:31.251 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.251 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.251 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.251 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.251 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.248520+0000", 2024-08-17T18:16:31.251 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.251 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.252 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.252 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.252 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.248520+0000", 2024-08-17T18:16:31.252 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.252 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.252 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.252 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.252 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.248519+0000", 2024-08-17T18:16:31.252 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:16:31.252 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.252 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.252 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.252 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.248524+0000", 2024-08-17T18:16:31.253 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3579999999999996e-06 2024-08-17T18:16:31.253 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.253 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.253 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.253 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.248530+0000", 2024-08-17T18:16:31.253 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7170000000000002e-06 2024-08-17T18:16:31.253 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.253 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.253 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.253 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.248534+0000", 2024-08-17T18:16:31.253 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.174e-06 2024-08-17T18:16:31.253 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.253 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.254 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.254 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.248558+0000", 2024-08-17T18:16:31.254 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4057e-05 2024-08-17T18:16:31.254 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.254 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.254 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.254 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.248628+0000", 2024-08-17T18:16:31.254 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0599999999999995e-05 2024-08-17T18:16:31.254 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.254 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.254 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1", 2024-08-17T18:16:31.254 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.248692+0000", 2024-08-17T18:16:31.254 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4014000000000001e-05 2024-08-17T18:16:31.255 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.255 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.255 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.255 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.252463+0000", 2024-08-17T18:16:31.255 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037710299999999999 2024-08-17T18:16:31.255 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.255 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.255 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.255 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.261077+0000", 2024-08-17T18:16:31.255 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0086135340000000008 2024-08-17T18:16:31.255 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.255 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.255 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.255 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.337970+0000", 2024-08-17T18:16:31.256 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.076893330999999995 2024-08-17T18:16:31.256 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.256 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.256 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.256 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.338008+0000", 2024-08-17T18:16:31.256 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7694000000000002e-05 2024-08-17T18:16:31.256 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.256 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.256 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.256 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.338045+0000", 2024-08-17T18:16:31.256 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7064000000000003e-05 2024-08-17T18:16:31.256 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.256 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.257 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.257 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.257 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.257 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36960.0:189027 49.9 49:9493f164:::benchmark_data_smithi130_364143_object79893:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1407)", 2024-08-17T18:16:31.257 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:16:02.250903+0000", 2024-08-17T18:16:31.257 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 28.865213409999999, 2024-08-17T18:16:31.257 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.087255555999999998, 2024-08-17T18:16:31.257 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:16:31.257 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:31.257 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:16:31.257 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36960", 2024-08-17T18:16:31.257 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:31.257 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 189027 2024-08-17T18:16:31.257 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.258 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:16:31.258 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.258 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:16:31.258 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.250903+0000", 2024-08-17T18:16:31.258 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.258 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.258 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.258 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:16:31.258 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.250903+0000", 2024-08-17T18:16:31.258 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:16:31.258 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.258 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.258 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:16:31.259 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.250903+0000", 2024-08-17T18:16:31.259 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:16:31.259 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.259 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.259 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:16:31.259 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.250907+0000", 2024-08-17T18:16:31.259 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.912e-06 2024-08-17T18:16:31.259 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.259 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.259 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:16:31.259 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.250914+0000", 2024-08-17T18:16:31.259 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.4510000000000003e-06 2024-08-17T18:16:31.259 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.259 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.260 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:16:31.260 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.250918+0000", 2024-08-17T18:16:31.260 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6260000000000001e-06 2024-08-17T18:16:31.260 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.260 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.260 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:16:31.260 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.250944+0000", 2024-08-17T18:16:31.260 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.6424e-05 2024-08-17T18:16:31.260 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.260 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.260 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:16:31.260 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.251012+0000", 2024-08-17T18:16:31.260 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7994e-05 2024-08-17T18:16:31.261 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.261 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.261 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1", 2024-08-17T18:16:31.261 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.251077+0000", 2024-08-17T18:16:31.261 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4350999999999994e-05 2024-08-17T18:16:31.261 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.261 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.261 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:16:31.261 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.252602+0000", 2024-08-17T18:16:31.261 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001525967 2024-08-17T18:16:31.261 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.261 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.261 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.261 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.267220+0000", 2024-08-17T18:16:31.262 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.014617163000000001 2024-08-17T18:16:31.262 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.262 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.262 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:31.262 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.338092+0000", 2024-08-17T18:16:31.262 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.070872076000000006 2024-08-17T18:16:31.262 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.262 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.262 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:16:31.262 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.338128+0000", 2024-08-17T18:16:31.262 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6208999999999998e-05 2024-08-17T18:16:31.262 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:16:31.262 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:16:31.262 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:16:31.263 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:02.338159+0000", 2024-08-17T18:16:31.263 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1056999999999998e-05 2024-08-17T18:16:31.263 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.263 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.263 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.263 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:16:31.263 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:16:31.263 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:16:31.286 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:31.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:31.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:31.388 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:31.410 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:31.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:31.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:31.511 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:31.534 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:31.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:31.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:31.636 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:31.659 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:31.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:31.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:31.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:31 smithi130 ceph-mon[105521]: osdmap e1418: 8 total, 7 up, 5 in 2024-08-17T18:16:31.747 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 3488/25872 objects degraded (13.482%), 14 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:31.747 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:31 smithi130 ceph-mon[110068]: osdmap e1418: 8 total, 7 up, 5 in 2024-08-17T18:16:31.747 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 3488/25872 objects degraded (13.482%), 14 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:31.761 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:31.793 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:31.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:31.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:31.895 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:31.917 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:31.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:31.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:32.019 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:32.029 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:31 smithi179 ceph-mon[93519]: osdmap e1418: 8 total, 7 up, 5 in 2024-08-17T18:16:32.030 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 3488/25872 objects degraded (13.482%), 14 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:32.041 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:32.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:32.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:32.054 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T18:16:32.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:32.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:32.165 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:32.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:32.266 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:32.289 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:32.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:32.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:32.391 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:32.413 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:32.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:32.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:32.515 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:32.537 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:32.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:32.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:32.617 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:16:32.639 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:32.661 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:32.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:32.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:32.763 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:32.785 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:32.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:32.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:32.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:32 smithi130 ceph-mon[105521]: pgmap v2676: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 538 MiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 632 op/s; 3488/25872 objects degraded (13.482%); 2522/25872 objects misplaced (9.748%); 13 MiB/s, 207 objects/s recovering 2024-08-17T18:16:32.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:32 smithi130 ceph-mon[105521]: osdmap e1419: 8 total, 7 up, 5 in 2024-08-17T18:16:32.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3364465408' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:32.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:32 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:32.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:32 smithi130 ceph-mon[110068]: pgmap v2676: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 538 MiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 632 op/s; 3488/25872 objects degraded (13.482%); 2522/25872 objects misplaced (9.748%); 13 MiB/s, 207 objects/s recovering 2024-08-17T18:16:32.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:32 smithi130 ceph-mon[110068]: osdmap e1419: 8 total, 7 up, 5 in 2024-08-17T18:16:32.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3364465408' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:32.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:32 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:32.887 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:32.908 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:32.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:32.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:33.010 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:33.052 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:33.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:33.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:33.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:32 smithi179 ceph-mon[93519]: pgmap v2676: 89 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 1 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 25 active+undersized, 2 active+remapped+backfill_wait, 12 active+undersized+degraded, 45 active+clean; 538 MiB data, 8.0 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 632 op/s; 3488/25872 objects degraded (13.482%); 2522/25872 objects misplaced (9.748%); 13 MiB/s, 207 objects/s recovering 2024-08-17T18:16:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:32 smithi179 ceph-mon[93519]: osdmap e1419: 8 total, 7 up, 5 in 2024-08-17T18:16:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3364465408' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:33.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:32 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:33.154 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:33.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T18:16:32.817+0000 7fa856f19640 -1 mon.a@0(leader).osd e1420 definitely_dead 0 2024-08-17T18:16:33.176 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:33.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:33.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:33.252 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:33.278 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:33.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:33.302 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:33.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:33.403 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:33.426 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:33.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:33.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:33.527 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:33.550 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:33.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:33.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:33.631 INFO:teuthology.orchestra.run.smithi130.stderr:marked in osd.6. 2024-08-17T18:16:33.652 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:33.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:33.675 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:33.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:33.776 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:33.797 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:33.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:33.900 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:33.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:33 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:16:33.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:33 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:16:33.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:33 smithi130 ceph-mon[105521]: osdmap e1420: 8 total, 7 up, 5 in 2024-08-17T18:16:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:33 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/451860798' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-08-17T18:16:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:33 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:16:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:33 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:16:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:33 smithi130 ceph-mon[110068]: osdmap e1420: 8 total, 7 up, 5 in 2024-08-17T18:16:33.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:33 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/451860798' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-08-17T18:16:33.922 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:33.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:33.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:34.023 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:34.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:34.046 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:34.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:34.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:33 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:16:34.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:33 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:16:34.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:33 smithi179 ceph-mon[93519]: osdmap e1420: 8 total, 7 up, 5 in 2024-08-17T18:16:34.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:33 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/451860798' entity='client.admin' cmd={"prefix": "osd in", "ids": ["6"]} : dispatch 2024-08-17T18:16:34.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:34.168 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:34.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:34.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:34.270 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:34.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:34.293 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:34.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:34.394 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:34.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:34.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:34.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:34.518 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:34.541 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:34.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:34.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:34.643 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:34.674 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:34.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:34.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:34.776 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:34.797 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:34.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:34.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:34.899 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:34.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:34 smithi130 ceph-mon[105521]: pgmap v2679: 89 pgs: 4 active+clean+remapped, 25 active+undersized, 1 active+remapped+backfill_wait, 12 active+undersized+degraded, 47 active+clean; 453 MiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 339 op/s; 2271/21792 objects degraded (10.421%); 2299/21792 objects misplaced (10.550%); 19 MiB/s, 297 objects/s recovering 2024-08-17T18:16:34.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:34 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/451860798' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-08-17T18:16:34.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:34 smithi130 ceph-mon[105521]: osdmap e1421: 8 total, 7 up, 6 in 2024-08-17T18:16:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:34 smithi130 ceph-mon[110068]: pgmap v2679: 89 pgs: 4 active+clean+remapped, 25 active+undersized, 1 active+remapped+backfill_wait, 12 active+undersized+degraded, 47 active+clean; 453 MiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 339 op/s; 2271/21792 objects degraded (10.421%); 2299/21792 objects misplaced (10.550%); 19 MiB/s, 297 objects/s recovering 2024-08-17T18:16:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:34 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/451860798' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-08-17T18:16:34.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:34 smithi130 ceph-mon[110068]: osdmap e1421: 8 total, 7 up, 6 in 2024-08-17T18:16:34.922 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:34.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:34.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:35.023 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:35.045 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:35.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:35.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:35.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:34 smithi179 ceph-mon[93519]: pgmap v2679: 89 pgs: 4 active+clean+remapped, 25 active+undersized, 1 active+remapped+backfill_wait, 12 active+undersized+degraded, 47 active+clean; 453 MiB data, 7.9 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 339 op/s; 2271/21792 objects degraded (10.421%); 2299/21792 objects misplaced (10.550%); 19 MiB/s, 297 objects/s recovering 2024-08-17T18:16:35.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:34 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/451860798' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2024-08-17T18:16:35.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:34 smithi179 ceph-mon[93519]: osdmap e1421: 8 total, 7 up, 6 in 2024-08-17T18:16:35.147 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:35.169 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:35.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:35.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:35.271 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:35.292 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:35.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:35.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:35.394 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:35.415 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:35.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:35.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:35.517 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:35.538 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:35.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:35.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:35.640 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:35.672 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:35.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:35.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:35.690 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:16:35.774 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:35.798 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:35.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:35.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:35.900 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:35.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:35 smithi130 ceph-mon[105521]: osdmap e1422: 8 total, 7 up, 6 in 2024-08-17T18:16:35.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:35 smithi130 ceph-mon[105521]: 49.2 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13236 2024-08-17T18:16:35.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:35 smithi130 ceph-mon[105521]: osdmap e1423: 8 total, 7 up, 6 in 2024-08-17T18:16:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:35 smithi130 ceph-mon[110068]: osdmap e1422: 8 total, 7 up, 6 in 2024-08-17T18:16:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:35 smithi130 ceph-mon[110068]: 49.2 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13236 2024-08-17T18:16:35.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:35 smithi130 ceph-mon[110068]: osdmap e1423: 8 total, 7 up, 6 in 2024-08-17T18:16:35.924 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:35.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:35.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:36.026 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:36.047 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:36.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:36.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:36.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:35 smithi179 ceph-mon[93519]: osdmap e1422: 8 total, 7 up, 6 in 2024-08-17T18:16:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:35 smithi179 ceph-mon[93519]: 49.2 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13236 2024-08-17T18:16:36.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:35 smithi179 ceph-mon[93519]: osdmap e1423: 8 total, 7 up, 6 in 2024-08-17T18:16:36.149 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:36.178 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:36.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:36.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:36.195 INFO:tasks.thrashosds.thrasher:Added osd 6 2024-08-17T18:16:36.280 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:36.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:36.325 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:36.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:36.426 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:36.460 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:36.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:36.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:36.562 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:36.584 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:36.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:36.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:36.685 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:36.733 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:36.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:36.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:36.835 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:36.859 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:36.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:36.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:36.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[105521]: pgmap v2682: 89 pgs: 4 peering, 4 active+clean+remapped, 25 active+undersized, 10 active+undersized+degraded, 46 active+clean; 438 MiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 398 op/s; 1747/21087 objects degraded (8.285%); 1797/21087 objects misplaced (8.522%); 26 MiB/s, 417 objects/s recovering 2024-08-17T18:16:36.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[105521]: 49.0 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13111 2024-08-17T18:16:36.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[105521]: 49.c starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13200 2024-08-17T18:16:36.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[105521]: 49.6 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13206 2024-08-17T18:16:36.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[105521]: 49.9 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13162 2024-08-17T18:16:36.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[105521]: 49.3 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13317 2024-08-17T18:16:36.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[105521]: 49.a starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13628 2024-08-17T18:16:36.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 1747/21087 objects degraded (8.285%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:36.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[105521]: osdmap e1424: 8 total, 7 up, 6 in 2024-08-17T18:16:36.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[110068]: pgmap v2682: 89 pgs: 4 peering, 4 active+clean+remapped, 25 active+undersized, 10 active+undersized+degraded, 46 active+clean; 438 MiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 398 op/s; 1747/21087 objects degraded (8.285%); 1797/21087 objects misplaced (8.522%); 26 MiB/s, 417 objects/s recovering 2024-08-17T18:16:36.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[110068]: 49.0 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13111 2024-08-17T18:16:36.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[110068]: 49.c starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13200 2024-08-17T18:16:36.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[110068]: 49.6 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13206 2024-08-17T18:16:36.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[110068]: 49.9 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13162 2024-08-17T18:16:36.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[110068]: 49.3 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13317 2024-08-17T18:16:36.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[110068]: 49.a starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13628 2024-08-17T18:16:36.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 1747/21087 objects degraded (8.285%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:36.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:36 smithi130 ceph-mon[110068]: osdmap e1424: 8 total, 7 up, 6 in 2024-08-17T18:16:36.961 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:36.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:36.999 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:37.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:36 smithi179 ceph-mon[93519]: pgmap v2682: 89 pgs: 4 peering, 4 active+clean+remapped, 25 active+undersized, 10 active+undersized+degraded, 46 active+clean; 438 MiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 398 op/s; 1747/21087 objects degraded (8.285%); 1797/21087 objects misplaced (8.522%); 26 MiB/s, 417 objects/s recovering 2024-08-17T18:16:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:36 smithi179 ceph-mon[93519]: 49.0 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13111 2024-08-17T18:16:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:36 smithi179 ceph-mon[93519]: 49.c starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13200 2024-08-17T18:16:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:36 smithi179 ceph-mon[93519]: 49.6 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13206 2024-08-17T18:16:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:36 smithi179 ceph-mon[93519]: 49.9 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13162 2024-08-17T18:16:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:36 smithi179 ceph-mon[93519]: 49.3 starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13317 2024-08-17T18:16:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:36 smithi179 ceph-mon[93519]: 49.a starting backfill to osd.6 from (0'0,0'0] MAX to 1420'13628 2024-08-17T18:16:37.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 1747/21087 objects degraded (8.285%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:37.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:36 smithi179 ceph-mon[93519]: osdmap e1424: 8 total, 7 up, 6 in 2024-08-17T18:16:37.101 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:37.125 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:37.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:37.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:37.227 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:37.252 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:37.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:37.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:37.354 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:37.377 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:37.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:37.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:37.479 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:37.519 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:37.520 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:16:37.520 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:16:37.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:37.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:37.555 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:16:37.556 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:16:37.558 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:16:37.604 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:16:37.604 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:16:37.621 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:37.638 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:16:37.638 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:16:37.655 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:37.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:37.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:37.673 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:16:37.674 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:16:37.706 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:16:37.706 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:16:37.738 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:16:37.738 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:16:37.757 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:37.780 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:37.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:37.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:37.882 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:37.906 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:37.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:37.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:38.008 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:38.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:38.067 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:38.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:38.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:38.211 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:38.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:38.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:38.313 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:38.336 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:38.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:38.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:38.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:38 smithi130 ceph-mon[110068]: osdmap e1425: 8 total, 7 up, 6 in 2024-08-17T18:16:38.391 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:38 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2585323684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:16:38.391 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:38 smithi130 ceph-mon[105521]: osdmap e1425: 8 total, 7 up, 6 in 2024-08-17T18:16:38.391 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:38 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2585323684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:16:38.438 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:38.463 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:38.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:38.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:38.565 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:38.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:38 smithi179 ceph-mon[93519]: osdmap e1425: 8 total, 7 up, 6 in 2024-08-17T18:16:38.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:38 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2585323684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:16:38.604 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:38.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:38.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:38.706 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:38.730 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:38.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:38.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:38.776 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:16:38.776 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:16:38.832 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:38.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:38.857 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:38.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:38.958 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:38.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:38.984 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:38.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:39.085 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:39.111 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:39.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:39.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:39.213 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:39.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:39.264 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:39.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:39.338 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:39 smithi130 ceph-mon[105521]: pgmap v2686: 89 pgs: 4 peering, 7 active+clean+remapped, 22 active+undersized, 9 active+undersized+degraded, 47 active+clean; 404 MiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 198 op/s; 1746/19443 objects degraded (8.980%); 1468/19443 objects misplaced (7.550%); 14 MiB/s, 221 objects/s recovering 2024-08-17T18:16:39.338 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:39 smithi130 ceph-mon[110068]: pgmap v2686: 89 pgs: 4 peering, 7 active+clean+remapped, 22 active+undersized, 9 active+undersized+degraded, 47 active+clean; 404 MiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 198 op/s; 1746/19443 objects degraded (8.980%); 1468/19443 objects misplaced (7.550%); 14 MiB/s, 221 objects/s recovering 2024-08-17T18:16:39.345 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:39.365 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:39.386 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:16:39.387 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:16:39.397 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:39.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:39.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:39.499 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:39.520 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:39.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:39.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:39.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:39 smithi179 ceph-mon[93519]: pgmap v2686: 89 pgs: 4 peering, 7 active+clean+remapped, 22 active+undersized, 9 active+undersized+degraded, 47 active+clean; 404 MiB data, 7.8 GiB used, 439 GiB / 447 GiB avail; 0 B/s wr, 198 op/s; 1746/19443 objects degraded (8.980%); 1468/19443 objects misplaced (7.550%); 14 MiB/s, 221 objects/s recovering 2024-08-17T18:16:39.622 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:39.644 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:39.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:39.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:39.745 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:39.766 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:39.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:39.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:39.868 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:39.890 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:39.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:39.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:16:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:16:39] "GET /metrics HTTP/1.1" 200 36806 "" "Prometheus/2.51.0" 2024-08-17T18:16:39.992 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:40.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:40.014 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:40.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:40.118 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:40.139 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:40.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:40.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:40.240 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:40.265 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:40.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:40.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:40.368 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:40.370 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:40 smithi130 ceph-mon[105521]: osdmap e1426: 8 total, 7 up, 6 in 2024-08-17T18:16:40.370 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:40 smithi130 ceph-mon[110068]: osdmap e1426: 8 total, 7 up, 6 in 2024-08-17T18:16:40.390 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:40.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:40.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:40.394 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T18:16:40.493 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:40.513 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:40.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:40.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:40 smithi179 ceph-mon[93519]: osdmap e1426: 8 total, 7 up, 6 in 2024-08-17T18:16:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:16:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:16:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:16:40.616 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:40.637 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:40.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:40.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:40.739 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:40.760 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:40.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:40.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:40.777 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:16:40.862 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:40.882 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:40.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:40.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:40.985 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:41.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:41.007 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:41.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:41.108 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:41.131 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:41.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:41.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:41.195 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 7, 1, 6] out_osds: [5, 4] dead_osds: [2] live_osds: [1, 4, 5, 6, 7, 0, 3] 2024-08-17T18:16:41.260 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:16:41.260 INFO:tasks.thrashosds.thrasher:reset_purged_snaps_last 2024-08-17T18:16:41.260 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.0 reset_purged_snaps_last 2024-08-17T18:16:41.261 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:41.284 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:41.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:41.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:41.386 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:41.407 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:41.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:41.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:41.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:41 smithi130 ceph-mon[110068]: pgmap v2688: 89 pgs: 1 active+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 3 active+remapped+backfill_wait, 12 active+undersized, 4 active+undersized+degraded, 51 active+clean; 280 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 798/13512 objects degraded (5.906%); 2595/13512 objects misplaced (19.205%); 15 MiB/s, 2 keys/s, 235 objects/s recovering 2024-08-17T18:16:41.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:41 smithi130 ceph-mon[110068]: osdmap e1427: 8 total, 7 up, 6 in 2024-08-17T18:16:41.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:41 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 798/13512 objects degraded (5.906%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:41.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:41 smithi130 ceph-mon[110068]: osdmap e1428: 8 total, 7 up, 6 in 2024-08-17T18:16:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:41 smithi130 ceph-mon[105521]: pgmap v2688: 89 pgs: 1 active+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 3 active+remapped+backfill_wait, 12 active+undersized, 4 active+undersized+degraded, 51 active+clean; 280 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 798/13512 objects degraded (5.906%); 2595/13512 objects misplaced (19.205%); 15 MiB/s, 2 keys/s, 235 objects/s recovering 2024-08-17T18:16:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:41 smithi130 ceph-mon[105521]: osdmap e1427: 8 total, 7 up, 6 in 2024-08-17T18:16:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:41 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 798/13512 objects degraded (5.906%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:41 smithi130 ceph-mon[105521]: osdmap e1428: 8 total, 7 up, 6 in 2024-08-17T18:16:41.509 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:41.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:41.531 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:41.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:41.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:41 smithi179 ceph-mon[93519]: pgmap v2688: 89 pgs: 1 active+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 3 active+remapped+backfill_wait, 12 active+undersized, 4 active+undersized+degraded, 51 active+clean; 280 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 798/13512 objects degraded (5.906%); 2595/13512 objects misplaced (19.205%); 15 MiB/s, 2 keys/s, 235 objects/s recovering 2024-08-17T18:16:41.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:41 smithi179 ceph-mon[93519]: osdmap e1427: 8 total, 7 up, 6 in 2024-08-17T18:16:41.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:41 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 798/13512 objects degraded (5.906%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:41.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:41 smithi179 ceph-mon[93519]: osdmap e1428: 8 total, 7 up, 6 in 2024-08-17T18:16:41.632 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:41.652 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:41.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:41.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:41.754 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:41.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:41.776 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:41.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:41.877 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:41.902 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:41.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:41.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:42.004 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:42.027 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:42.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:42.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:42.129 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:42.154 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:42.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:42.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:42.256 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:42.298 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:42.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:42.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:42.400 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:42.441 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:42.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:42.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:42.543 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:42.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:42.567 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:42.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:42.668 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:42.693 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:42.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:42.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:42.795 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:42.819 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:42.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:42.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:42.920 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:42.938 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:42.942 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:42.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:42.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:43.043 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:43.087 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:43.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:43.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:43.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:43 smithi130 ceph-mon[110068]: pgmap v2691: 89 pgs: 1 active+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 3 active+remapped+backfill_wait, 12 active+undersized, 4 active+undersized+degraded, 51 active+clean; 280 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 439 op/s; 798/13512 objects degraded (5.906%); 2595/13512 objects misplaced (19.205%); 7.5 MiB/s, 2 keys/s, 122 objects/s recovering 2024-08-17T18:16:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:43 smithi130 ceph-mon[110068]: osdmap e1429: 8 total, 7 up, 6 in 2024-08-17T18:16:43.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:16:43.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:43 smithi130 ceph-mon[105521]: pgmap v2691: 89 pgs: 1 active+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 3 active+remapped+backfill_wait, 12 active+undersized, 4 active+undersized+degraded, 51 active+clean; 280 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 439 op/s; 798/13512 objects degraded (5.906%); 2595/13512 objects misplaced (19.205%); 7.5 MiB/s, 2 keys/s, 122 objects/s recovering 2024-08-17T18:16:43.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:43 smithi130 ceph-mon[105521]: osdmap e1429: 8 total, 7 up, 6 in 2024-08-17T18:16:43.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:16:43.189 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:43.213 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:43.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:43.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:43.315 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:43.338 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:43.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:43.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:43.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:43 smithi179 ceph-mon[93519]: pgmap v2691: 89 pgs: 1 active+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 16 active+clean+remapped, 3 active+remapped+backfill_wait, 12 active+undersized, 4 active+undersized+degraded, 51 active+clean; 280 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 439 op/s; 798/13512 objects degraded (5.906%); 2595/13512 objects misplaced (19.205%); 7.5 MiB/s, 2 keys/s, 122 objects/s recovering 2024-08-17T18:16:43.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:43 smithi179 ceph-mon[93519]: osdmap e1429: 8 total, 7 up, 6 in 2024-08-17T18:16:43.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:16:43.439 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:43.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:43.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:43.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:43.564 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:43.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:43.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:43.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:43.687 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:43.735 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:43.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:43.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:43.837 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:43.878 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:43.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:43.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:43.980 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:44.002 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:44.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:44.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:44.104 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:44.152 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:44.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:44.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:44.254 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:44.277 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:44.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:44.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:44.296 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:44.378 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:44.400 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:44.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:44.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:44.502 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:44.524 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:44.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:44.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:44.626 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:44.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:44.670 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:44.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:44.771 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:44.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:44.816 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:44.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:44.917 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:44.919 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:44.940 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:44.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:44.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:44.951 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Removed 109131 objects 2024-08-17T18:16:44.951 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Clean up completed and total clean up time :140.021 2024-08-17T18:16:45.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:45.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:45.065 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:45.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:45.080 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:16:45.166 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:45 smithi130 ceph-mon[110068]: pgmap v2693: 89 pgs: 1 active+undersized+remapped, 17 active+clean+remapped, 12 active+undersized, 4 active+undersized+degraded, 55 active+clean; 137 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 607 op/s; 201/6609 objects degraded (3.041%); 938/6609 objects misplaced (14.193%); 14 MiB/s, 223 objects/s recovering 2024-08-17T18:16:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:45 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2399705778' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:16:45.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:45 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:16:45.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:45 smithi130 ceph-mon[105521]: pgmap v2693: 89 pgs: 1 active+undersized+remapped, 17 active+clean+remapped, 12 active+undersized, 4 active+undersized+degraded, 55 active+clean; 137 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 607 op/s; 201/6609 objects degraded (3.041%); 938/6609 objects misplaced (14.193%); 14 MiB/s, 223 objects/s recovering 2024-08-17T18:16:45.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:45 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2399705778' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:16:45.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:45 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:16:45.189 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:45.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:45.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:45.291 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:45.313 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:45.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:45.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:45.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:45 smithi179 ceph-mon[93519]: pgmap v2693: 89 pgs: 1 active+undersized+remapped, 17 active+clean+remapped, 12 active+undersized, 4 active+undersized+degraded, 55 active+clean; 137 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 607 op/s; 201/6609 objects degraded (3.041%); 938/6609 objects misplaced (14.193%); 14 MiB/s, 223 objects/s recovering 2024-08-17T18:16:45.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:45 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2399705778' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:16:45.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:45 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:16:45.415 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:45.437 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:45.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:45.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:45.539 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:45.560 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:45.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:45.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:45.661 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:45.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:45.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:45.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:45.787 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:45.812 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:45.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:45.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:45.914 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:45.939 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:45.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:45.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:46.041 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:46.064 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:46.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:46.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:46.166 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:46.190 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:46.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:46.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:46.292 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:46.304 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:16:46.305 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:16:46.314 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:46.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:46.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:46.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:46 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:16:46.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:46 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:16:46.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:46 smithi179 ceph-mon[93519]: osdmap e1430: 8 total, 7 up, 6 in 2024-08-17T18:16:46.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 201/6609 objects degraded (3.041%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:46.416 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:46.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:46 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:16:46.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:46 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:16:46.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:46 smithi130 ceph-mon[105521]: osdmap e1430: 8 total, 7 up, 6 in 2024-08-17T18:16:46.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 201/6609 objects degraded (3.041%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:46.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:46 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:46 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:46 smithi130 ceph-mon[110068]: osdmap e1430: 8 total, 7 up, 6 in 2024-08-17T18:16:46.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 201/6609 objects degraded (3.041%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:46.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:46.443 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:46.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:46.543 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:46.565 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:46.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:46.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:46.667 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:46.689 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:46.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:46.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:46.790 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:46.812 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:46.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:46.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:46.914 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:46.939 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:46.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:46.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:46.981 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-08-17T18:16:46.981 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-08-17T18:16:47.041 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:47.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:47.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:47.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:47.168 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:47.193 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:47.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:47.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:47.295 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:47.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:47.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:47.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:47.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:47 smithi179 ceph-mon[93519]: pgmap v2695: 89 pgs: 18 active+clean+remapped, 12 active+undersized, 4 active+undersized+degraded, 55 active+clean; 116 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 480 op/s; 201/5637 objects degraded (3.566%); 772/5637 objects misplaced (13.695%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T18:16:47.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:47 smithi130 ceph-mon[110068]: pgmap v2695: 89 pgs: 18 active+clean+remapped, 12 active+undersized, 4 active+undersized+degraded, 55 active+clean; 116 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 480 op/s; 201/5637 objects degraded (3.566%); 772/5637 objects misplaced (13.695%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T18:16:47.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:47 smithi130 ceph-mon[105521]: pgmap v2695: 89 pgs: 18 active+clean+remapped, 12 active+undersized, 4 active+undersized+degraded, 55 active+clean; 116 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 480 op/s; 201/5637 objects degraded (3.566%); 772/5637 objects misplaced (13.695%); 8.9 MiB/s, 142 objects/s recovering 2024-08-17T18:16:47.419 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:47.440 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:47.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:47.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:47.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:47.564 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:47.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:47.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:47.666 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:47.687 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:47.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:47.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:47.789 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:47.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:47.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:47.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:47.912 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:47.938 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:47.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:47.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:48.040 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:48.063 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:48.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:48.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:48.165 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:48.188 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:48.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:48.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:48.216 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:16:48.290 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:48.315 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:48.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:48.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:48.417 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:48.438 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:48.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:48.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:48.540 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:48.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:48.563 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:48.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:48.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:48.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:48.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:48.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:48.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:48.809 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:48.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:48.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:48.911 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:48.937 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:48.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:48.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:49.039 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:49.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:49.061 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:49.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:49.162 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:49.186 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:49.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:49.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:49.288 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:49.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:49.316 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:49.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:49.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:49 smithi179 ceph-mon[93519]: pgmap v2696: 89 pgs: 18 active+clean+remapped, 12 active+undersized, 4 active+undersized+degraded, 55 active+clean; 64 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 530 op/s; 201/3147 objects degraded (6.387%); 278/3147 objects misplaced (8.834%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T18:16:49.409 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:49 smithi130 ceph-mon[105521]: pgmap v2696: 89 pgs: 18 active+clean+remapped, 12 active+undersized, 4 active+undersized+degraded, 55 active+clean; 64 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 530 op/s; 201/3147 objects degraded (6.387%); 278/3147 objects misplaced (8.834%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T18:16:49.409 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:49 smithi130 ceph-mon[110068]: pgmap v2696: 89 pgs: 18 active+clean+remapped, 12 active+undersized, 4 active+undersized+degraded, 55 active+clean; 64 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 530 op/s; 201/3147 objects degraded (6.387%); 278/3147 objects misplaced (8.834%); 7.5 MiB/s, 120 objects/s recovering 2024-08-17T18:16:49.417 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:49.438 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:49.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:49.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:49.540 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:49.562 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:49.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:49.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:49.664 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:49.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:49.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:49.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:49.793 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:49.815 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:49.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:49.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:49.916 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:49.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:16:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:16:49] "GET /metrics HTTP/1.1" 200 36815 "" "Prometheus/2.51.0" 2024-08-17T18:16:49.938 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:49.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:49.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:50.040 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:50.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:50.062 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:50.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:50.163 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:50.184 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:50.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:50.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:50.286 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:50.311 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:50.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:50.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:50.413 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:50.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:50.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:50.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:50.520 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:16:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:16:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:16:50.536 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:50.558 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:50.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:50.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:50.660 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:50.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:50.683 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:50.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:50.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:50.805 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:50.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:50.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:50.842 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T18:16:50.908 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:50.930 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:50.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:50.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:51.032 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:51.054 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:51.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:51.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:51.156 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:51.177 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:51.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:51.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:51.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:51.299 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:51.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:51.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:51.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:51 smithi179 ceph-mon[93519]: pgmap v2697: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T18:16:51.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:51.402 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:51.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:51 smithi130 ceph-mon[105521]: pgmap v2697: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T18:16:51.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:51.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:51 smithi130 ceph-mon[110068]: pgmap v2697: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T18:16:51.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T18:16:51.424 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:51.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:51.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:51.526 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:51.548 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:51.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:51.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:51.617 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.2 reset_purged_snaps_last 2024-08-17T18:16:51.650 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:51.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:51.672 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:51.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:51.774 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:51.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:51.795 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:51.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:51.896 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:51.918 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:51.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:51.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:52.020 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:52.041 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:52.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:52.074 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:52.144 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:52.166 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:52.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:52.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:52.272 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:52.294 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:52.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:52.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:52.395 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:52.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:52.419 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:52.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:52.520 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:52.542 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:52.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:52.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:52.644 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:52.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:52.667 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:52.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:52.768 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:52.789 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:52.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:52.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:52.891 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:52.913 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:52.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:52.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:53.015 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:53.037 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:53.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:53.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:53.139 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:53.162 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:53.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:53.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:53.231 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:16:53.231 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:16:53.252 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:16:53.252 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:16:53.252 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:16:53.257 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:53.263 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:53.284 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:16:53.284 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:16:53.285 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:53.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:53.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:53.302 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:16:53.302 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:16:53.317 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:16:53.318 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:16:53.333 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:16:53.333 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:16:53.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:53 smithi179 ceph-mon[93519]: pgmap v2698: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 375 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%); 5.1 MiB/s, 82 objects/s recovering 2024-08-17T18:16:53.348 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:16:53.349 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:16:53.349 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:53 smithi130 ceph-mon[105521]: pgmap v2698: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 375 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%); 5.1 MiB/s, 82 objects/s recovering 2024-08-17T18:16:53.349 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:53 smithi130 ceph-mon[110068]: pgmap v2698: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 375 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%); 5.1 MiB/s, 82 objects/s recovering 2024-08-17T18:16:53.387 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:53.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:53.409 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:53.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:53.510 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:53.531 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:53.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:53.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:53.633 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:53.657 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:53.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:53.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:53.758 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:53.782 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:53.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:53.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:53.884 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:53.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:53.906 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:53.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:54.007 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:54.030 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:54.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:54.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:54.132 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:54.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:54.154 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:54.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:54.255 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:54.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:54.282 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:54.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:54.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1924896237' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:16:54.383 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:54.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:54.410 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:54.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:54.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1924896237' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:16:54.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1924896237' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:16:54.511 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:54.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:54.534 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:54.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:54.635 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:54.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:54.657 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:54.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:54.758 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:54.784 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:54.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:54.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:54.854 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:16:54.855 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:16:54.885 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:54.907 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:54.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:54.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:55.009 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:55.034 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:55.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:55.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:55.111 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:55 smithi130 ceph-mon[105521]: pgmap v2699: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 217 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:16:55.112 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:55 smithi130 ceph-mon[110068]: pgmap v2699: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 217 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:16:55.136 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:55.158 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:55.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:55.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:55.260 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:55.286 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:55.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:55.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:55 smithi179 ceph-mon[93519]: pgmap v2699: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 217 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:16:55.371 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:55.388 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:55.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:55.414 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:55.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:55.515 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:55.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:55.538 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:55.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:55.640 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:55.665 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:55.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:55.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:55.767 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:55.788 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:55.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:55.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:55.890 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:55.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:55.913 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:55.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:56.014 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:56.036 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:56.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:56.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:56.042 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:16:56.138 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:56.160 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:56.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:56.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:56.262 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:56.288 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:56.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:56.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:56.390 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:56.411 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:56.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:56.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:56.513 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:56.539 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:56.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:56.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:56.641 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:56.666 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:56.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:56.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:56.768 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:56.789 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:56.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:56.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:56.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:16:56.891 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:56.917 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:56.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:56.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:57.019 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:57.044 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:57.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:57.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:57.099 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:16:57.145 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:57.171 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:57.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:57.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:57.273 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:57.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:57.296 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:57.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:57.397 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:16:57.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:57 smithi130 ceph-mon[105521]: pgmap v2700: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 176 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:16:57.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:57 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2310237820' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:57.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:57 smithi130 ceph-mon[110068]: pgmap v2700: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 176 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:16:57.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:57 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2310237820' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:57.423 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:57.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:57.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:16:57.525 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:57.547 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:57.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:57.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:57 smithi179 ceph-mon[93519]: pgmap v2700: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 176 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:16:57.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:57 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2310237820' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:16:57.640 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:16:57.640 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:16:57.640 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:16:57.640 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:16:57.641 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.641 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115437 48.8 48:1823c229:::benchmark_data_smithi130_327597_object115436:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:16:57.641 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.095352+0000", 2024-08-17T18:16:57.641 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 480.52298662499999, 2024-08-17T18:16:57.641 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5254109470000001, 2024-08-17T18:16:57.641 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.641 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.641 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:16:57.641 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.641 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:16:57.641 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:57.641 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115437 2024-08-17T18:16:57.641 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.642 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.642 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.642 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.642 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095352+0000", 2024-08-17T18:16:57.642 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.642 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.642 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.642 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.642 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095352+0000", 2024-08-17T18:16:57.642 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.642 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.642 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.642 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.643 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095352+0000", 2024-08-17T18:16:57.643 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2199999999999999e-07 2024-08-17T18:16:57.643 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.643 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.643 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.643 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095377+0000", 2024-08-17T18:16:57.643 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4745000000000001e-05 2024-08-17T18:16:57.643 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.643 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.643 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.643 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095385+0000", 2024-08-17T18:16:57.643 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6199999999999999e-06 2024-08-17T18:16:57.643 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.644 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.644 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.644 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095390+0000", 2024-08-17T18:16:57.644 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5860000000000004e-06 2024-08-17T18:16:57.644 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.644 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.644 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.644 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.100178+0000", 2024-08-17T18:16:57.644 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0047875629999999999 2024-08-17T18:16:57.644 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.644 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.644 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.644 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.100189+0000", 2024-08-17T18:16:57.644 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1124999999999999e-05 2024-08-17T18:16:57.645 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.645 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.645 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.645 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.564942+0000", 2024-08-17T18:16:57.645 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.46475288100000001 2024-08-17T18:16:57.645 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.645 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.645 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.645 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.564952+0000", 2024-08-17T18:16:57.645 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9809999999999998e-06 2024-08-17T18:16:57.645 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.645 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.645 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.646 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.620659+0000", 2024-08-17T18:16:57.646 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.05570713 2024-08-17T18:16:57.646 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.646 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.646 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.646 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.620763+0000", 2024-08-17T18:16:57.646 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010389400000000001 2024-08-17T18:16:57.646 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.646 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.646 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.646 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.646 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.646 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115442 48.8 48:189c3bef:::benchmark_data_smithi130_327597_object115441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:16:57.647 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.095476+0000", 2024-08-17T18:16:57.647 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 480.522862863, 2024-08-17T18:16:57.647 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5581986209999998, 2024-08-17T18:16:57.647 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.647 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.647 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:16:57.647 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.647 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:16:57.647 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:57.647 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115442 2024-08-17T18:16:57.647 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.647 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.647 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.648 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.648 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095476+0000", 2024-08-17T18:16:57.648 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.648 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.648 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.648 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.648 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095476+0000", 2024-08-17T18:16:57.648 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.648 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.648 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.648 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.648 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095476+0000", 2024-08-17T18:16:57.648 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2300000000000002e-07 2024-08-17T18:16:57.648 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.649 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.649 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.649 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095499+0000", 2024-08-17T18:16:57.649 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2728e-05 2024-08-17T18:16:57.649 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.649 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.649 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.649 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095506+0000", 2024-08-17T18:16:57.649 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8770000000000004e-06 2024-08-17T18:16:57.649 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.649 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.649 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.649 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.095511+0000", 2024-08-17T18:16:57.649 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0549999999999999e-06 2024-08-17T18:16:57.650 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.650 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.650 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.650 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.100585+0000", 2024-08-17T18:16:57.650 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0050741049999999998 2024-08-17T18:16:57.650 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.650 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.650 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.650 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.100593+0000", 2024-08-17T18:16:57.650 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5150000000000006e-06 2024-08-17T18:16:57.650 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.650 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.650 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.651 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.564972+0000", 2024-08-17T18:16:57.651 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.46437926800000001 2024-08-17T18:16:57.651 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.651 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.651 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.651 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.564979+0000", 2024-08-17T18:16:57.651 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1090000000000003e-06 2024-08-17T18:16:57.651 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.651 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.651 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.651 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.621508+0000", 2024-08-17T18:16:57.651 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0565289579999999 2024-08-17T18:16:57.651 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.651 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.652 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:16:57.652 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.621663+0000", 2024-08-17T18:16:57.652 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000155415 2024-08-17T18:16:57.652 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.652 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.652 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.652 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641744+0000", 2024-08-17T18:16:57.652 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020081096 2024-08-17T18:16:57.652 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.652 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.652 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for degraded object", 2024-08-17T18:16:57.652 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641815+0000", 2024-08-17T18:16:57.652 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1219999999999999e-05 2024-08-17T18:16:57.652 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.653 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.653 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.653 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.653582+0000", 2024-08-17T18:16:57.653 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011766835 2024-08-17T18:16:57.653 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.653 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.653 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.653 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.653674+0000", 2024-08-17T18:16:57.653 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2016999999999997e-05 2024-08-17T18:16:57.653 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.653 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.653 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.653 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.654 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.654 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115459 48.8 48:15f41a0f:::benchmark_data_smithi130_327597_object115458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:16:57.654 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.111325+0000", 2024-08-17T18:16:57.654 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 480.50701374900001, 2024-08-17T18:16:57.654 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5377689189999999, 2024-08-17T18:16:57.654 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.654 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.654 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:57.654 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.654 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:16:57.654 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:57.654 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115459 2024-08-17T18:16:57.655 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.655 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.655 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.655 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.655 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111325+0000", 2024-08-17T18:16:57.655 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.655 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.655 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.655 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.655 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111325+0000", 2024-08-17T18:16:57.655 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.655 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.655 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.655 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.656 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111326+0000", 2024-08-17T18:16:57.656 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.311e-06 2024-08-17T18:16:57.656 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.656 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.656 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.656 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111351+0000", 2024-08-17T18:16:57.656 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5377000000000001e-05 2024-08-17T18:16:57.656 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.656 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.656 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.656 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111364+0000", 2024-08-17T18:16:57.656 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2632e-05 2024-08-17T18:16:57.656 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.657 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.657 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.657 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111370+0000", 2024-08-17T18:16:57.657 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.75e-06 2024-08-17T18:16:57.657 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.657 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.657 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.657 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111428+0000", 2024-08-17T18:16:57.657 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8425999999999999e-05 2024-08-17T18:16:57.658 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.658 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.658 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.658 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111440+0000", 2024-08-17T18:16:57.658 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1911e-05 2024-08-17T18:16:57.658 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.658 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.658 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.658 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565000+0000", 2024-08-17T18:16:57.658 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.45356039399999998 2024-08-17T18:16:57.658 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.658 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.658 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.659 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565007+0000", 2024-08-17T18:16:57.659 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.917e-06 2024-08-17T18:16:57.659 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.659 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.659 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.659 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.621760+0000", 2024-08-17T18:16:57.659 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.056752672 2024-08-17T18:16:57.659 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.659 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.659 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.659 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.621924+0000", 2024-08-17T18:16:57.659 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016439199999999999 2024-08-17T18:16:57.659 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.660 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.660 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:16:57.660 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.622547+0000", 2024-08-17T18:16:57.660 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000622424 2024-08-17T18:16:57.660 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.660 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.660 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.660 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641874+0000", 2024-08-17T18:16:57.660 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019326829 2024-08-17T18:16:57.660 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.660 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.660 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.660 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641892+0000", 2024-08-17T18:16:57.660 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8221000000000001e-05 2024-08-17T18:16:57.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.661 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.661 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.661 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.648915+0000", 2024-08-17T18:16:57.661 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.007023339 2024-08-17T18:16:57.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.661 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.661 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:16:57.661 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.648981+0000", 2024-08-17T18:16:57.661 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5957000000000006e-05 2024-08-17T18:16:57.661 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.661 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.661 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.661 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649094+0000", 2024-08-17T18:16:57.662 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000112367 2024-08-17T18:16:57.662 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.662 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.662 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.662 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.662 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.662 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115460 48.8 48:17144d5f:::benchmark_data_smithi130_327597_object115459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:16:57.663 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.111385+0000", 2024-08-17T18:16:57.663 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 480.506953259, 2024-08-17T18:16:57.663 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.537943938, 2024-08-17T18:16:57.663 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.663 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.663 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:57.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.663 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:16:57.664 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:57.664 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115460 2024-08-17T18:16:57.664 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.665 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.665 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.665 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.665 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111385+0000", 2024-08-17T18:16:57.665 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.665 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.665 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.665 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.665 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111385+0000", 2024-08-17T18:16:57.665 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.665 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.665 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.665 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.665 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111385+0000", 2024-08-17T18:16:57.666 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4999999999999998e-07 2024-08-17T18:16:57.666 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.666 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.666 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.666 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111401+0000", 2024-08-17T18:16:57.666 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5566e-05 2024-08-17T18:16:57.666 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.666 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.666 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.666 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111406+0000", 2024-08-17T18:16:57.666 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4610000000000001e-06 2024-08-17T18:16:57.666 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.666 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.667 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.667 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111409+0000", 2024-08-17T18:16:57.667 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.303e-06 2024-08-17T18:16:57.667 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.667 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.667 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.667 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111469+0000", 2024-08-17T18:16:57.667 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0007000000000003e-05 2024-08-17T18:16:57.667 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.667 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.667 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.667 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.111476+0000", 2024-08-17T18:16:57.667 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3209999999999999e-06 2024-08-17T18:16:57.668 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.668 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.668 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.668 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565029+0000", 2024-08-17T18:16:57.668 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.45355303200000002 2024-08-17T18:16:57.668 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.668 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.668 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.668 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565037+0000", 2024-08-17T18:16:57.668 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.32e-06 2024-08-17T18:16:57.668 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.668 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.668 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.668 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.635425+0000", 2024-08-17T18:16:57.669 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0703878899999999 2024-08-17T18:16:57.669 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.669 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.669 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.669 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.635606+0000", 2024-08-17T18:16:57.669 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018029 2024-08-17T18:16:57.669 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.669 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.669 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:16:57.669 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.635807+0000", 2024-08-17T18:16:57.669 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00020181999999999999 2024-08-17T18:16:57.669 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.669 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.670 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.670 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641899+0000", 2024-08-17T18:16:57.670 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0060914810000000002 2024-08-17T18:16:57.670 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.670 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.670 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.670 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.641956+0000", 2024-08-17T18:16:57.670 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7343999999999998e-05 2024-08-17T18:16:57.670 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.670 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.670 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.670 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649206+0000", 2024-08-17T18:16:57.670 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072501060000000001 2024-08-17T18:16:57.671 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.671 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.671 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:16:57.671 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649267+0000", 2024-08-17T18:16:57.671 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0939e-05 2024-08-17T18:16:57.671 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.671 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.671 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.671 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649329+0000", 2024-08-17T18:16:57.671 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1707999999999999e-05 2024-08-17T18:16:57.671 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.671 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.671 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.672 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.672 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.672 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115465 48.8 48:103437e2:::benchmark_data_smithi130_327597_object115464:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:16:57.672 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.117778+0000", 2024-08-17T18:16:57.672 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 480.50056049400001, 2024-08-17T18:16:57.672 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5317481179999999, 2024-08-17T18:16:57.672 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.672 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.672 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:57.673 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.673 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:16:57.673 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:57.673 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115465 2024-08-17T18:16:57.673 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.673 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.673 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.673 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.673 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117778+0000", 2024-08-17T18:16:57.673 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.673 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.673 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.673 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.673 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117778+0000", 2024-08-17T18:16:57.674 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.674 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.674 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.674 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.674 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117779+0000", 2024-08-17T18:16:57.674 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.311e-06 2024-08-17T18:16:57.674 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.674 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.674 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.674 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117811+0000", 2024-08-17T18:16:57.674 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1489999999999998e-05 2024-08-17T18:16:57.674 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.674 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.675 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.675 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117826+0000", 2024-08-17T18:16:57.675 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5783000000000001e-05 2024-08-17T18:16:57.675 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.675 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.675 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.675 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117835+0000", 2024-08-17T18:16:57.675 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0709999999999995e-06 2024-08-17T18:16:57.675 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.675 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.675 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.675 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117889+0000", 2024-08-17T18:16:57.675 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4355999999999999e-05 2024-08-17T18:16:57.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.676 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.676 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.676 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.117897+0000", 2024-08-17T18:16:57.676 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.729e-06 2024-08-17T18:16:57.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.676 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.676 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.676 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565061+0000", 2024-08-17T18:16:57.676 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.44716407800000002 2024-08-17T18:16:57.676 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.676 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.676 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.676 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565068+0000", 2024-08-17T18:16:57.677 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3130000000000001e-06 2024-08-17T18:16:57.677 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.677 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.677 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.677 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.642010+0000", 2024-08-17T18:16:57.677 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0769422019999999 2024-08-17T18:16:57.677 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.677 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.677 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.677 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.642197+0000", 2024-08-17T18:16:57.677 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000186923 2024-08-17T18:16:57.677 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.677 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.678 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:16:57.678 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.642394+0000", 2024-08-17T18:16:57.678 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000196913 2024-08-17T18:16:57.678 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.678 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.678 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.678 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649113+0000", 2024-08-17T18:16:57.678 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0067189290000000002 2024-08-17T18:16:57.678 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.678 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.678 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.678 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649158+0000", 2024-08-17T18:16:57.678 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4758000000000001e-05 2024-08-17T18:16:57.679 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.679 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.679 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.679 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649420+0000", 2024-08-17T18:16:57.679 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00026217599999999998 2024-08-17T18:16:57.679 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.679 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.679 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:16:57.679 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649452+0000", 2024-08-17T18:16:57.679 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2178000000000003e-05 2024-08-17T18:16:57.679 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.679 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.679 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.679 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649526+0000", 2024-08-17T18:16:57.680 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3907999999999998e-05 2024-08-17T18:16:57.680 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.680 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.680 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.680 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.680 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.680 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36675.0:115470 48.8 48:1640b7fc:::benchmark_data_smithi130_327597_object115469:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T18:16:57.680 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:08:57.130397+0000", 2024-08-17T18:16:57.680 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 480.48794154699999, 2024-08-17T18:16:57.680 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.519320972, 2024-08-17T18:16:57.680 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.680 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.680 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:57.680 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.681 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36675", 2024-08-17T18:16:57.681 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2548689656", 2024-08-17T18:16:57.681 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 115470 2024-08-17T18:16:57.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.681 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.681 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.681 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.681 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130397+0000", 2024-08-17T18:16:57.681 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.681 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.681 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.681 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.681 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130397+0000", 2024-08-17T18:16:57.682 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.682 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130399+0000", 2024-08-17T18:16:57.682 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0169999999999999e-06 2024-08-17T18:16:57.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.682 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.682 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130441+0000", 2024-08-17T18:16:57.682 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2552000000000002e-05 2024-08-17T18:16:57.682 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.682 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.682 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.683 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130470+0000", 2024-08-17T18:16:57.683 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.8713000000000001e-05 2024-08-17T18:16:57.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.683 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.683 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.683 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130484+0000", 2024-08-17T18:16:57.683 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3709999999999999e-05 2024-08-17T18:16:57.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.683 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.683 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.683 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130612+0000", 2024-08-17T18:16:57.683 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000128598 2024-08-17T18:16:57.683 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.684 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.684 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.130624+0000", 2024-08-17T18:16:57.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1878999999999999e-05 2024-08-17T18:16:57.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.684 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.684 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565091+0000", 2024-08-17T18:16:57.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.43446660199999998 2024-08-17T18:16:57.684 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.684 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.684 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.684 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:57.565098+0000", 2024-08-17T18:16:57.684 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2250000000000003e-06 2024-08-17T18:16:57.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.645503+0000", 2024-08-17T18:16:57.685 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0804047400000001 2024-08-17T18:16:57.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.685 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.645638+0000", 2024-08-17T18:16:57.685 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000135418 2024-08-17T18:16:57.685 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.685 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.685 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2", 2024-08-17T18:16:57.686 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.645861+0000", 2024-08-17T18:16:57.686 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00022232299999999999 2024-08-17T18:16:57.686 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.686 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.686 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.686 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649120+0000", 2024-08-17T18:16:57.686 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032596449999999998 2024-08-17T18:16:57.686 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.686 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.686 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.686 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649377+0000", 2024-08-17T18:16:57.686 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00025634200000000001 2024-08-17T18:16:57.686 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.687 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.687 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.687 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649566+0000", 2024-08-17T18:16:57.687 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00018959399999999999 2024-08-17T18:16:57.687 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.687 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.687 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:16:57.687 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649604+0000", 2024-08-17T18:16:57.687 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8015000000000001e-05 2024-08-17T18:16:57.687 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.687 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.687 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.687 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:08:58.649718+0000", 2024-08-17T18:16:57.688 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011359899999999999 2024-08-17T18:16:57.688 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.688 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.688 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.688 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.688 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.688 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186448 49.7 49:edcceba0:::benchmark_data_smithi130_364143_object77314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1404)", 2024-08-17T18:16:57.688 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.324714+0000", 2024-08-17T18:16:57.688 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 59.293624319000003, 2024-08-17T18:16:57.688 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0974611990000001, 2024-08-17T18:16:57.688 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.688 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.688 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:16:57.689 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.689 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:16:57.689 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:57.689 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186448 2024-08-17T18:16:57.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.689 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.324714+0000", 2024-08-17T18:16:57.689 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.689 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.689 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.689 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.689 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.324714+0000", 2024-08-17T18:16:57.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.690 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.690 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.690 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.324716+0000", 2024-08-17T18:16:57.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7090000000000001e-06 2024-08-17T18:16:57.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.690 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.690 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.690 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.324722+0000", 2024-08-17T18:16:57.690 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8150000000000002e-06 2024-08-17T18:16:57.690 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.690 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.691 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.691 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.324738+0000", 2024-08-17T18:16:57.691 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6381000000000001e-05 2024-08-17T18:16:57.691 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.691 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.691 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.691 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.324746+0000", 2024-08-17T18:16:57.691 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8839999999999994e-06 2024-08-17T18:16:57.691 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.691 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.691 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.691 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.324807+0000", 2024-08-17T18:16:57.691 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1282e-05 2024-08-17T18:16:57.691 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.692 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.692 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.692 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.324937+0000", 2024-08-17T18:16:57.692 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000129598 2024-08-17T18:16:57.692 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.692 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.692 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2,3", 2024-08-17T18:16:57.692 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.325112+0000", 2024-08-17T18:16:57.692 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017490499999999999 2024-08-17T18:16:57.692 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.692 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.692 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.692 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.342426+0000", 2024-08-17T18:16:57.692 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017314385000000002 2024-08-17T18:16:57.693 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.693 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.693 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.693 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.342465+0000", 2024-08-17T18:16:57.693 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8911000000000001e-05 2024-08-17T18:16:57.693 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.693 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.693 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.693 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.357445+0000", 2024-08-17T18:16:57.693 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014979983000000001 2024-08-17T18:16:57.693 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.693 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.693 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.693 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384068+0000", 2024-08-17T18:16:57.694 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.026623529999999999 2024-08-17T18:16:57.694 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.694 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.694 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.694 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384078+0000", 2024-08-17T18:16:57.694 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3559999999999999e-06 2024-08-17T18:16:57.694 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.694 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.694 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.694 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.422102+0000", 2024-08-17T18:16:57.700 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.038024335 2024-08-17T18:16:57.700 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.700 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.700 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.700 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.422175+0000", 2024-08-17T18:16:57.700 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3125000000000002e-05 2024-08-17T18:16:57.700 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.701 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.701 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.701 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.701 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.701 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186454 49.7 49:ebbf7ff5:::benchmark_data_smithi130_364143_object77320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1404)", 2024-08-17T18:16:57.701 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.326257+0000", 2024-08-17T18:16:57.701 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 59.292081011000001, 2024-08-17T18:16:57.701 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.096470598, 2024-08-17T18:16:57.701 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.701 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.701 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:16:57.701 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.701 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:16:57.701 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:57.702 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186454 2024-08-17T18:16:57.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.702 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.702 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.702 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.702 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.326257+0000", 2024-08-17T18:16:57.702 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.702 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.702 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.702 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.326257+0000", 2024-08-17T18:16:57.702 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.702 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.703 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.326258+0000", 2024-08-17T18:16:57.703 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9000000000000005e-07 2024-08-17T18:16:57.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.703 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.326263+0000", 2024-08-17T18:16:57.703 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3880000000000002e-06 2024-08-17T18:16:57.703 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.703 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.703 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.703 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.326273+0000", 2024-08-17T18:16:57.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0273e-05 2024-08-17T18:16:57.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.704 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.326280+0000", 2024-08-17T18:16:57.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5370000000000003e-06 2024-08-17T18:16:57.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.704 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.704 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.338892+0000", 2024-08-17T18:16:57.704 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012612013 2024-08-17T18:16:57.704 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.704 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.705 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.705 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.339039+0000", 2024-08-17T18:16:57.705 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014789100000000001 2024-08-17T18:16:57.705 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.705 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.705 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2,3", 2024-08-17T18:16:57.705 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.339188+0000", 2024-08-17T18:16:57.705 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000148066 2024-08-17T18:16:57.705 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.705 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.705 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.705 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.357382+0000", 2024-08-17T18:16:57.705 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018194155 2024-08-17T18:16:57.706 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.706 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.706 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.706 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.357452+0000", 2024-08-17T18:16:57.706 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0562000000000006e-05 2024-08-17T18:16:57.706 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.706 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.706 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.706 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.362860+0000", 2024-08-17T18:16:57.706 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0054077889999999997 2024-08-17T18:16:57.706 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.706 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.706 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.707 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384102+0000", 2024-08-17T18:16:57.707 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021241640999999999 2024-08-17T18:16:57.707 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.707 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.707 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.707 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384108+0000", 2024-08-17T18:16:57.707 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2899999999999999e-06 2024-08-17T18:16:57.707 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.707 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.707 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.707 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.422668+0000", 2024-08-17T18:16:57.707 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.038559987 2024-08-17T18:16:57.707 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.708 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.708 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.708 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.422728+0000", 2024-08-17T18:16:57.708 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0016000000000003e-05 2024-08-17T18:16:57.708 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.708 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.708 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.708 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.708 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.708 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186459 49.7 49:e729d3d2:::benchmark_data_smithi130_364143_object77325:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1404)", 2024-08-17T18:16:57.708 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.343801+0000", 2024-08-17T18:16:57.708 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 59.274537660999997, 2024-08-17T18:16:57.708 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.079049277, 2024-08-17T18:16:57.709 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.709 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.709 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:16:57.709 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.709 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:16:57.709 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:57.709 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186459 2024-08-17T18:16:57.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.709 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.709 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.709 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.709 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.343801+0000", 2024-08-17T18:16:57.709 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.709 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.710 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.710 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.343801+0000", 2024-08-17T18:16:57.710 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.710 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.710 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.343802+0000", 2024-08-17T18:16:57.710 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.491e-06 2024-08-17T18:16:57.710 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.710 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.710 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.710 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.343809+0000", 2024-08-17T18:16:57.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7410000000000003e-06 2024-08-17T18:16:57.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.711 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.343826+0000", 2024-08-17T18:16:57.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6773999999999999e-05 2024-08-17T18:16:57.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.711 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.711 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.343833+0000", 2024-08-17T18:16:57.711 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3159999999999999e-06 2024-08-17T18:16:57.711 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.711 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.343928+0000", 2024-08-17T18:16:57.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5310999999999996e-05 2024-08-17T18:16:57.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.712 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.344083+0000", 2024-08-17T18:16:57.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015473900000000001 2024-08-17T18:16:57.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.712 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.712 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2,3", 2024-08-17T18:16:57.712 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.344216+0000", 2024-08-17T18:16:57.712 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013315500000000001 2024-08-17T18:16:57.712 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.713 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.713 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.357424+0000", 2024-08-17T18:16:57.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013208184 2024-08-17T18:16:57.713 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.713 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.713 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.357457+0000", 2024-08-17T18:16:57.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.2975999999999997e-05 2024-08-17T18:16:57.713 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.713 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.713 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.713 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384134+0000", 2024-08-17T18:16:57.713 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.026676208 2024-08-17T18:16:57.714 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.714 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.714 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.714 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384139+0000", 2024-08-17T18:16:57.714 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6740000000000002e-06 2024-08-17T18:16:57.714 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.714 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.714 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.714 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.422757+0000", 2024-08-17T18:16:57.714 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.038617967 2024-08-17T18:16:57.714 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.714 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.714 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.715 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.422850+0000", 2024-08-17T18:16:57.715 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2740999999999999e-05 2024-08-17T18:16:57.715 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.715 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.715 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.715 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.715 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.715 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186468 49.7 49:e5240c4a:::benchmark_data_smithi130_364143_object77334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1405)", 2024-08-17T18:16:57.715 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.384573+0000", 2024-08-17T18:16:57.715 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 59.233765042999998, 2024-08-17T18:16:57.715 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.116594978, 2024-08-17T18:16:57.715 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.715 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.716 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:57.716 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.716 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:16:57.716 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:57.716 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186468 2024-08-17T18:16:57.716 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.716 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.716 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.716 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.716 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384573+0000", 2024-08-17T18:16:57.716 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.716 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.716 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.716 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384573+0000", 2024-08-17T18:16:57.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.717 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384575+0000", 2024-08-17T18:16:57.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6220000000000001e-06 2024-08-17T18:16:57.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.717 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.717 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.717 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384581+0000", 2024-08-17T18:16:57.717 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.614e-06 2024-08-17T18:16:57.717 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.718 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.718 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384592+0000", 2024-08-17T18:16:57.718 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1756000000000001e-05 2024-08-17T18:16:57.718 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.718 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.718 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384601+0000", 2024-08-17T18:16:57.718 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.081e-06 2024-08-17T18:16:57.718 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.718 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.718 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.718 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384848+0000", 2024-08-17T18:16:57.719 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00024697199999999998 2024-08-17T18:16:57.719 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.719 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.719 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.719 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384857+0000", 2024-08-17T18:16:57.719 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6440000000000006e-06 2024-08-17T18:16:57.719 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.719 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.719 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.719 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.422875+0000", 2024-08-17T18:16:57.719 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.038018052 2024-08-17T18:16:57.719 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.719 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.719 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.720 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.422981+0000", 2024-08-17T18:16:57.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000106283 2024-08-17T18:16:57.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.720 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.720 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,3", 2024-08-17T18:16:57.720 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.423093+0000", 2024-08-17T18:16:57.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000112092 2024-08-17T18:16:57.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.720 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.720 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.720 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.430383+0000", 2024-08-17T18:16:57.720 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072895210000000002 2024-08-17T18:16:57.720 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.434283+0000", 2024-08-17T18:16:57.721 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039003509999999998 2024-08-17T18:16:57.721 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.500942+0000", 2024-08-17T18:16:57.721 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.066658929000000006 2024-08-17T18:16:57.721 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.721 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.721 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:16:57.721 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.501031+0000", 2024-08-17T18:16:57.721 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8568999999999999e-05 2024-08-17T18:16:57.722 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.722 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.722 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.722 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.501168+0000", 2024-08-17T18:16:57.722 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013749199999999999 2024-08-17T18:16:57.722 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.722 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.722 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.722 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.722 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.722 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186469 49.7 49:ed0a96db:::benchmark_data_smithi130_364143_object77335:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1405)", 2024-08-17T18:16:57.722 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.391950+0000", 2024-08-17T18:16:57.722 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 59.226388569000001, 2024-08-17T18:16:57.723 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1122878460000001, 2024-08-17T18:16:57.723 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.723 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.723 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:57.723 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.723 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:16:57.723 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:57.723 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186469 2024-08-17T18:16:57.723 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.723 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.723 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.723 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.723 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391950+0000", 2024-08-17T18:16:57.723 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.724 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.724 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.724 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.724 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391950+0000", 2024-08-17T18:16:57.724 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.724 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.724 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.724 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.724 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391952+0000", 2024-08-17T18:16:57.724 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7910000000000001e-06 2024-08-17T18:16:57.724 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.724 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.724 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.725 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391959+0000", 2024-08-17T18:16:57.725 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1339999999999998e-06 2024-08-17T18:16:57.725 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.725 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.725 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.725 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391976+0000", 2024-08-17T18:16:57.725 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6892999999999999e-05 2024-08-17T18:16:57.725 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.725 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.725 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.725 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391987+0000", 2024-08-17T18:16:57.725 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0910000000000001e-05 2024-08-17T18:16:57.725 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.725 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.726 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392049+0000", 2024-08-17T18:16:57.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2512000000000001e-05 2024-08-17T18:16:57.726 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.726 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.726 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392059+0000", 2024-08-17T18:16:57.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7799999999999995e-06 2024-08-17T18:16:57.726 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.726 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.726 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.726 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.427062+0000", 2024-08-17T18:16:57.726 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0350030320000001 2024-08-17T18:16:57.727 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.427190+0000", 2024-08-17T18:16:57.727 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012799799999999999 2024-08-17T18:16:57.727 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,3", 2024-08-17T18:16:57.727 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.427329+0000", 2024-08-17T18:16:57.727 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013924299999999999 2024-08-17T18:16:57.727 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.727 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.727 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.434307+0000", 2024-08-17T18:16:57.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0069778030000000003 2024-08-17T18:16:57.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.728 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.448038+0000", 2024-08-17T18:16:57.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013730671 2024-08-17T18:16:57.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.728 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.728 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.504018+0000", 2024-08-17T18:16:57.728 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.055980038000000003 2024-08-17T18:16:57.728 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.728 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.729 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:16:57.729 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.504124+0000", 2024-08-17T18:16:57.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010654699999999999 2024-08-17T18:16:57.729 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.729 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.729 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.729 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.504238+0000", 2024-08-17T18:16:57.729 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011349399999999999 2024-08-17T18:16:57.729 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.729 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.729 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.729 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.729 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.730 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186472 49.4 49:2362d057:::benchmark_data_smithi130_364143_object77338:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1405)", 2024-08-17T18:16:57.730 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.398143+0000", 2024-08-17T18:16:57.730 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 59.220195298, 2024-08-17T18:16:57.730 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.085828469, 2024-08-17T18:16:57.730 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.730 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.730 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:57.730 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.730 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:16:57.730 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:57.730 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186472 2024-08-17T18:16:57.730 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.730 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.731 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.731 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.731 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.398143+0000", 2024-08-17T18:16:57.731 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.731 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.731 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.731 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.731 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.398143+0000", 2024-08-17T18:16:57.731 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.731 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.731 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.731 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.732 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.398146+0000", 2024-08-17T18:16:57.732 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4219999999999999e-06 2024-08-17T18:16:57.732 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.732 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.732 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.732 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.398153+0000", 2024-08-17T18:16:57.732 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6750000000000002e-06 2024-08-17T18:16:57.732 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.732 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.732 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.732 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.398176+0000", 2024-08-17T18:16:57.732 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2869000000000001e-05 2024-08-17T18:16:57.732 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.732 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.398186+0000", 2024-08-17T18:16:57.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5440000000000004e-06 2024-08-17T18:16:57.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.733 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.733 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.398262+0000", 2024-08-17T18:16:57.733 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6522999999999999e-05 2024-08-17T18:16:57.733 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.733 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.733 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.734 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:57.735 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.398273+0000", 2024-08-17T18:16:57.735 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.128e-05 2024-08-17T18:16:57.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.736 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.736 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.736 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.429385+0000", 2024-08-17T18:16:57.736 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0311118889999999 2024-08-17T18:16:57.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.736 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.736 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.736 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.429521+0000", 2024-08-17T18:16:57.736 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013570699999999999 2024-08-17T18:16:57.736 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.736 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.736 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,3", 2024-08-17T18:16:57.737 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.429657+0000", 2024-08-17T18:16:57.737 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013647 2024-08-17T18:16:57.737 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.737 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.737 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.737 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.434851+0000", 2024-08-17T18:16:57.737 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0051938870000000003 2024-08-17T18:16:57.737 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.737 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.737 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.737 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.444330+0000", 2024-08-17T18:16:57.737 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0094781810000000005 2024-08-17T18:16:57.737 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.738 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.738 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.738 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.483760+0000", 2024-08-17T18:16:57.738 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.039430448999999999 2024-08-17T18:16:57.738 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.738 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.738 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:16:57.738 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.483865+0000", 2024-08-17T18:16:57.738 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010471000000000001 2024-08-17T18:16:57.738 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.738 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.738 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.738 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.483972+0000", 2024-08-17T18:16:57.739 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000106863 2024-08-17T18:16:57.739 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.739 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.739 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.739 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.739 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.739 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:210491 49.1 49:874895de:::benchmark_data_smithi130_364143_object101357:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1418)", 2024-08-17T18:16:57.739 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:30.693264+0000", 2024-08-17T18:16:57.739 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 26.925074457000001, 2024-08-17T18:16:57.739 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.95089874100000005, 2024-08-17T18:16:57.739 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.739 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.740 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:57.740 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.740 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:16:57.740 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:57.740 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 210491 2024-08-17T18:16:57.740 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.740 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.740 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.740 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.740 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.693264+0000", 2024-08-17T18:16:57.740 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.740 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.740 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.740 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.693264+0000", 2024-08-17T18:16:57.741 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.741 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.741 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.741 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.693266+0000", 2024-08-17T18:16:57.741 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8300000000000001e-06 2024-08-17T18:16:57.741 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.741 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.741 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.741 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.693271+0000", 2024-08-17T18:16:57.741 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6110000000000001e-06 2024-08-17T18:16:57.741 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.742 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.742 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.742 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.693286+0000", 2024-08-17T18:16:57.742 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4722000000000001e-05 2024-08-17T18:16:57.742 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.742 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.742 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.742 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.693294+0000", 2024-08-17T18:16:57.742 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4660000000000002e-06 2024-08-17T18:16:57.742 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.742 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.742 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.742 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.693350+0000", 2024-08-17T18:16:57.743 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6413999999999999e-05 2024-08-17T18:16:57.743 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.743 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.743 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.693359+0000", 2024-08-17T18:16:57.743 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7320000000000004e-06 2024-08-17T18:16:57.743 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.743 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.743 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.639979+0000", 2024-08-17T18:16:57.743 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94662018800000003 2024-08-17T18:16:57.743 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.743 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.640147+0000", 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000167725 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,3", 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.640314+0000", 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000167307 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.643977+0000", 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003662668 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.744 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.644056+0000", 2024-08-17T18:16:57.745 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9498000000000004e-05 2024-08-17T18:16:57.745 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.745 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.644070+0000", 2024-08-17T18:16:57.745 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3474e-05 2024-08-17T18:16:57.745 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.745 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.745 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:16:57.745 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.644120+0000", 2024-08-17T18:16:57.745 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0201999999999998e-05 2024-08-17T18:16:57.746 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.746 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.746 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.746 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.644163+0000", 2024-08-17T18:16:57.746 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2904000000000001e-05 2024-08-17T18:16:57.746 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.746 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.746 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.746 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.746 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.746 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:210500 49.1 49:86a4c68c:::benchmark_data_smithi130_364143_object101366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1418)", 2024-08-17T18:16:57.746 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:30.699860+0000", 2024-08-17T18:16:57.746 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 26.918478299, 2024-08-17T18:16:57.746 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.95458005400000001, 2024-08-17T18:16:57.747 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.747 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.747 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:57.747 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.747 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:16:57.747 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:57.747 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 210500 2024-08-17T18:16:57.747 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.747 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.747 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.747 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.747 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.699860+0000", 2024-08-17T18:16:57.747 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.748 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.748 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.748 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.748 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.699860+0000", 2024-08-17T18:16:57.748 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.748 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.748 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.748 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.748 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.699862+0000", 2024-08-17T18:16:57.748 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.855e-06 2024-08-17T18:16:57.748 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.748 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.748 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.749 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.699868+0000", 2024-08-17T18:16:57.749 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7799999999999997e-06 2024-08-17T18:16:57.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.749 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.749 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.699880+0000", 2024-08-17T18:16:57.749 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2703e-05 2024-08-17T18:16:57.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.749 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.749 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.699887+0000", 2024-08-17T18:16:57.749 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9999999999999999e-06 2024-08-17T18:16:57.749 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.749 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.699954+0000", 2024-08-17T18:16:57.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6919000000000001e-05 2024-08-17T18:16:57.750 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.750 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.699964+0000", 2024-08-17T18:16:57.750 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.2939999999999994e-06 2024-08-17T18:16:57.750 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.750 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.750 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.750 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.644215+0000", 2024-08-17T18:16:57.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94425180399999997 2024-08-17T18:16:57.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.644355+0000", 2024-08-17T18:16:57.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013960099999999999 2024-08-17T18:16:57.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.751 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,3", 2024-08-17T18:16:57.751 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.644504+0000", 2024-08-17T18:16:57.751 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014872900000000001 2024-08-17T18:16:57.751 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.751 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.752 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.752 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.651102+0000", 2024-08-17T18:16:57.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.006597945 2024-08-17T18:16:57.752 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.752 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.752 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.752 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.651230+0000", 2024-08-17T18:16:57.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012783 2024-08-17T18:16:57.752 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.752 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.752 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.752 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.654322+0000", 2024-08-17T18:16:57.752 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0030927870000000001 2024-08-17T18:16:57.753 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.753 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.753 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:16:57.753 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.654393+0000", 2024-08-17T18:16:57.753 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0648999999999994e-05 2024-08-17T18:16:57.753 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.753 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.753 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.753 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.654440+0000", 2024-08-17T18:16:57.753 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7157999999999998e-05 2024-08-17T18:16:57.753 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.753 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.753 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.753 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.754 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.754 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:210509 49.1 49:8c960b75:::benchmark_data_smithi130_364143_object101375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1418)", 2024-08-17T18:16:57.754 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:30.708510+0000", 2024-08-17T18:16:57.754 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 26.909828117, 2024-08-17T18:16:57.754 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94617751299999997, 2024-08-17T18:16:57.754 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.754 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.754 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:57.754 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.754 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:16:57.754 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:57.754 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 210509 2024-08-17T18:16:57.754 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.755 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.755 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.755 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.755 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.708510+0000", 2024-08-17T18:16:57.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.755 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.755 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.755 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.755 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.708510+0000", 2024-08-17T18:16:57.755 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.755 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.755 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.755 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.756 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.708512+0000", 2024-08-17T18:16:57.756 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9750000000000001e-06 2024-08-17T18:16:57.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.756 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.756 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.708517+0000", 2024-08-17T18:16:57.756 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2480000000000004e-06 2024-08-17T18:16:57.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.756 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.756 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.708529+0000", 2024-08-17T18:16:57.756 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1601e-05 2024-08-17T18:16:57.756 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.756 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.757 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.757 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.708536+0000", 2024-08-17T18:16:57.757 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8809999999999998e-06 2024-08-17T18:16:57.757 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.757 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.757 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.757 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.708583+0000", 2024-08-17T18:16:57.757 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6659000000000002e-05 2024-08-17T18:16:57.757 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.757 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.758 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.758 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.708591+0000", 2024-08-17T18:16:57.758 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.909e-06 2024-08-17T18:16:57.758 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.758 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.758 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.758 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.647753+0000", 2024-08-17T18:16:57.758 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.93916201700000002 2024-08-17T18:16:57.758 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.758 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.758 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.647875+0000", 2024-08-17T18:16:57.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012228900000000001 2024-08-17T18:16:57.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.759 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,3", 2024-08-17T18:16:57.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.647992+0000", 2024-08-17T18:16:57.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011730099999999999 2024-08-17T18:16:57.759 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.759 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.759 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.759 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.654518+0000", 2024-08-17T18:16:57.759 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.006525543 2024-08-17T18:16:57.760 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.760 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.760 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.760 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.654584+0000", 2024-08-17T18:16:57.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5809000000000006e-05 2024-08-17T18:16:57.760 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.760 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.760 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.760 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.654598+0000", 2024-08-17T18:16:57.760 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4953999999999999e-05 2024-08-17T18:16:57.760 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.760 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.760 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:16:57.760 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.654643+0000", 2024-08-17T18:16:57.761 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4326000000000002e-05 2024-08-17T18:16:57.761 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.761 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.761 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.761 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.654688+0000", 2024-08-17T18:16:57.761 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5000999999999998e-05 2024-08-17T18:16:57.761 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.761 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.761 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.761 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.761 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.761 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:210543 49.1 49:8b6f2de5:::benchmark_data_smithi130_364143_object101409:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1418)", 2024-08-17T18:16:57.761 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:30.735723+0000", 2024-08-17T18:16:57.762 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 26.882615639000001, 2024-08-17T18:16:57.762 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.92764929299999999, 2024-08-17T18:16:57.762 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.762 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.762 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:57.762 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.762 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:16:57.762 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:57.762 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 210543 2024-08-17T18:16:57.762 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.762 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.762 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.762 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.763 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.735723+0000", 2024-08-17T18:16:57.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.763 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.763 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.763 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.763 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.735723+0000", 2024-08-17T18:16:57.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.763 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.763 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.763 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.763 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.735724+0000", 2024-08-17T18:16:57.763 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6440000000000001e-06 2024-08-17T18:16:57.763 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.764 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.764 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.764 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.735731+0000", 2024-08-17T18:16:57.764 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1120000000000001e-06 2024-08-17T18:16:57.764 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.764 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.764 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.764 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.735744+0000", 2024-08-17T18:16:57.764 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.349e-05 2024-08-17T18:16:57.764 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.764 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.764 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.764 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.735751+0000", 2024-08-17T18:16:57.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2130000000000002e-06 2024-08-17T18:16:57.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.736159+0000", 2024-08-17T18:16:57.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00040776099999999997 2024-08-17T18:16:57.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.765 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.765 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.765 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.736167+0000", 2024-08-17T18:16:57.765 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5120000000000008e-06 2024-08-17T18:16:57.765 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.766 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.766 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.766 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.651278+0000", 2024-08-17T18:16:57.766 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.91511017400000005 2024-08-17T18:16:57.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.766 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.766 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.766 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.651407+0000", 2024-08-17T18:16:57.766 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000129221 2024-08-17T18:16:57.766 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.766 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.766 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,3", 2024-08-17T18:16:57.766 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.651484+0000", 2024-08-17T18:16:57.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7558999999999994e-05 2024-08-17T18:16:57.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.767 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.657354+0000", 2024-08-17T18:16:57.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0058690920000000002 2024-08-17T18:16:57.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.767 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.767 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.657399+0000", 2024-08-17T18:16:57.767 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5813999999999999e-05 2024-08-17T18:16:57.767 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.767 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.768 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.768 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.663286+0000", 2024-08-17T18:16:57.768 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0058865159999999996 2024-08-17T18:16:57.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.768 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.768 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:16:57.768 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.663334+0000", 2024-08-17T18:16:57.768 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7899000000000003e-05 2024-08-17T18:16:57.768 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.768 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.768 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.768 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.663372+0000", 2024-08-17T18:16:57.768 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8285999999999999e-05 2024-08-17T18:16:57.768 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.769 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.769 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.769 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.769 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.769 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:210552 49.1 49:80c80336:::benchmark_data_smithi130_364143_object101418:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1418)", 2024-08-17T18:16:57.769 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:30.747612+0000", 2024-08-17T18:16:57.769 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 26.870726839, 2024-08-17T18:16:57.769 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.91591789099999998, 2024-08-17T18:16:57.769 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.769 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.769 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:57.769 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.769 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:16:57.769 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:57.770 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 210552 2024-08-17T18:16:57.770 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.770 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.770 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.770 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.770 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.747612+0000", 2024-08-17T18:16:57.770 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.770 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.770 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.770 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.770 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.747612+0000", 2024-08-17T18:16:57.770 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.770 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.747613+0000", 2024-08-17T18:16:57.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7990000000000001e-06 2024-08-17T18:16:57.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.747618+0000", 2024-08-17T18:16:57.771 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1270000000000002e-06 2024-08-17T18:16:57.771 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.771 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.771 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.771 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.747630+0000", 2024-08-17T18:16:57.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1533e-05 2024-08-17T18:16:57.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.772 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.772 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.747637+0000", 2024-08-17T18:16:57.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0380000000000002e-06 2024-08-17T18:16:57.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.772 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.772 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.747753+0000", 2024-08-17T18:16:57.772 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000115876 2024-08-17T18:16:57.772 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.772 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:30.747761+0000", 2024-08-17T18:16:57.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.926e-06 2024-08-17T18:16:57.773 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.654735+0000", 2024-08-17T18:16:57.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.90697426400000003 2024-08-17T18:16:57.773 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.773 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.773 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.773 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.654822+0000", 2024-08-17T18:16:57.773 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7115999999999997e-05 2024-08-17T18:16:57.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.774 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,3", 2024-08-17T18:16:57.774 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.654902+0000", 2024-08-17T18:16:57.774 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9460000000000002e-05 2024-08-17T18:16:57.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.774 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.774 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.663379+0000", 2024-08-17T18:16:57.774 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0084769600000000004 2024-08-17T18:16:57.774 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.774 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.774 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.663415+0000", 2024-08-17T18:16:57.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6165999999999999e-05 2024-08-17T18:16:57.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.775 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.663454+0000", 2024-08-17T18:16:57.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9648999999999999e-05 2024-08-17T18:16:57.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.775 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:16:57.775 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.663498+0000", 2024-08-17T18:16:57.775 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3686999999999997e-05 2024-08-17T18:16:57.775 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.775 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.776 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.776 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:31.663529+0000", 2024-08-17T18:16:57.776 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.129e-05 2024-08-17T18:16:57.776 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.776 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.776 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.776 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.776 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.776 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:211982 49.3 49:c754feed:::benchmark_data_smithi130_364143_object102848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1420)", 2024-08-17T18:16:57.776 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:33.639055+0000", 2024-08-17T18:16:57.776 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 23.979283708000001, 2024-08-17T18:16:57.776 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1370433850000001, 2024-08-17T18:16:57.776 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.777 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.777 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:16:57.777 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.777 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:16:57.777 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:57.777 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 211982 2024-08-17T18:16:57.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.777 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.777 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.777 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639055+0000", 2024-08-17T18:16:57.777 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.777 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.777 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.778 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.778 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639055+0000", 2024-08-17T18:16:57.778 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.778 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.778 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.778 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.778 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639056+0000", 2024-08-17T18:16:57.778 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7600000000000001e-06 2024-08-17T18:16:57.778 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.778 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.778 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.778 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639063+0000", 2024-08-17T18:16:57.778 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.212e-06 2024-08-17T18:16:57.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.779 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.779 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.779 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639079+0000", 2024-08-17T18:16:57.779 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6379999999999999e-05 2024-08-17T18:16:57.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.779 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.779 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.779 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639087+0000", 2024-08-17T18:16:57.779 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9270000000000005e-06 2024-08-17T18:16:57.779 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.779 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.779 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.780 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639143+0000", 2024-08-17T18:16:57.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6100999999999997e-05 2024-08-17T18:16:57.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.780 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.780 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.780 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639271+0000", 2024-08-17T18:16:57.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012814599999999999 2024-08-17T18:16:57.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.780 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.780 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 7", 2024-08-17T18:16:57.780 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639398+0000", 2024-08-17T18:16:57.780 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012684799999999999 2024-08-17T18:16:57.780 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.780 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.781 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.661999+0000", 2024-08-17T18:16:57.781 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022601033 2024-08-17T18:16:57.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.781 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.781 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.662009+0000", 2024-08-17T18:16:57.781 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0073e-05 2024-08-17T18:16:57.781 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.781 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.781 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.781 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.638599+0000", 2024-08-17T18:16:57.781 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.97658970499999997 2024-08-17T18:16:57.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.782 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.782 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.782 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.638616+0000", 2024-08-17T18:16:57.782 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6643e-05 2024-08-17T18:16:57.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.782 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.782 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.782 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.765739+0000", 2024-08-17T18:16:57.782 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1271230050000001 2024-08-17T18:16:57.782 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.782 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.782 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:16:57.783 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.765865+0000", 2024-08-17T18:16:57.783 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012649699999999999 2024-08-17T18:16:57.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.783 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.783 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.783 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.774953+0000", 2024-08-17T18:16:57.783 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.009087655 2024-08-17T18:16:57.783 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.783 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.783 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.783 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.776098+0000", 2024-08-17T18:16:57.783 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0011454 2024-08-17T18:16:57.783 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.783 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.784 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.784 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.784 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.784 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:211990 49.3 49:c9339f7c:::benchmark_data_smithi130_364143_object102856:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1421)", 2024-08-17T18:16:57.784 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:33.670314+0000", 2024-08-17T18:16:57.784 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 23.948024765, 2024-08-17T18:16:57.784 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.105048381, 2024-08-17T18:16:57.784 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.784 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.784 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:57.784 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.784 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:16:57.785 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:57.785 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 211990 2024-08-17T18:16:57.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.785 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.785 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.785 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.785 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.670314+0000", 2024-08-17T18:16:57.785 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.785 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.785 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.785 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.670314+0000", 2024-08-17T18:16:57.785 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.785 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.786 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.786 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.786 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.670316+0000", 2024-08-17T18:16:57.786 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2120000000000002e-06 2024-08-17T18:16:57.786 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.786 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.786 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.786 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.670323+0000", 2024-08-17T18:16:57.786 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4019999999999996e-06 2024-08-17T18:16:57.786 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.786 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.786 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.786 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.670342+0000", 2024-08-17T18:16:57.787 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8309e-05 2024-08-17T18:16:57.787 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.787 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.787 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.787 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.670352+0000", 2024-08-17T18:16:57.787 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0461e-05 2024-08-17T18:16:57.787 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.787 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.787 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.787 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.671105+0000", 2024-08-17T18:16:57.787 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00075322500000000005 2024-08-17T18:16:57.787 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.788 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.788 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.788 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.671113+0000", 2024-08-17T18:16:57.788 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4240000000000004e-06 2024-08-17T18:16:57.788 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.788 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.788 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.788 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.638766+0000", 2024-08-17T18:16:57.788 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.96765296999999995 2024-08-17T18:16:57.788 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.788 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.788 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.788 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.638775+0000", 2024-08-17T18:16:57.788 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0839999999999998e-06 2024-08-17T18:16:57.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.789 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.789 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.789 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.766145+0000", 2024-08-17T18:16:57.789 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.127370099 2024-08-17T18:16:57.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.789 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.789 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.789 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.766281+0000", 2024-08-17T18:16:57.789 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013627600000000001 2024-08-17T18:16:57.789 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.789 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.789 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 6,7", 2024-08-17T18:16:57.789 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.766416+0000", 2024-08-17T18:16:57.790 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013520700000000001 2024-08-17T18:16:57.790 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.790 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.790 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.790 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.771055+0000", 2024-08-17T18:16:57.790 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046387140000000004 2024-08-17T18:16:57.790 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.790 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.790 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.790 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.771113+0000", 2024-08-17T18:16:57.790 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7899000000000002e-05 2024-08-17T18:16:57.790 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.790 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.791 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.791 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.775200+0000", 2024-08-17T18:16:57.791 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0040870150000000003 2024-08-17T18:16:57.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.791 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:16:57.791 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.775261+0000", 2024-08-17T18:16:57.791 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1561000000000002e-05 2024-08-17T18:16:57.791 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.791 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.791 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.791 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.775362+0000", 2024-08-17T18:16:57.791 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000100523 2024-08-17T18:16:57.792 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.792 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.792 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.792 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.792 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.792 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:211997 49.3 49:c8cc8bff:::benchmark_data_smithi130_364143_object102863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1421)", 2024-08-17T18:16:57.792 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:33.674887+0000", 2024-08-17T18:16:57.792 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 23.94345182, 2024-08-17T18:16:57.792 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1006669499999999, 2024-08-17T18:16:57.792 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:16:57.792 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:16:57.792 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:16:57.792 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:16:57.793 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:16:57.793 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:16:57.793 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 211997 2024-08-17T18:16:57.793 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.793 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:16:57.793 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.793 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:16:57.793 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674887+0000", 2024-08-17T18:16:57.793 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.793 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.793 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.793 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:16:57.793 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674887+0000", 2024-08-17T18:16:57.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:16:57.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.794 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.794 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:16:57.794 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674888+0000", 2024-08-17T18:16:57.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.497e-06 2024-08-17T18:16:57.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.794 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.794 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:16:57.794 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674894+0000", 2024-08-17T18:16:57.794 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8660000000000002e-06 2024-08-17T18:16:57.794 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.794 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.795 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:16:57.795 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674910+0000", 2024-08-17T18:16:57.795 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5832e-05 2024-08-17T18:16:57.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.795 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.795 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:16:57.795 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674918+0000", 2024-08-17T18:16:57.795 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8990000000000001e-06 2024-08-17T18:16:57.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.795 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.795 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.795 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674971+0000", 2024-08-17T18:16:57.795 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3819000000000001e-05 2024-08-17T18:16:57.795 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.796 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.796 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674980+0000", 2024-08-17T18:16:57.796 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5439999999999999e-06 2024-08-17T18:16:57.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.796 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.796 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.638825+0000", 2024-08-17T18:16:57.796 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.96384535000000005 2024-08-17T18:16:57.796 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.796 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.796 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:16:57.796 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.638833+0000", 2024-08-17T18:16:57.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1679999999999998e-06 2024-08-17T18:16:57.797 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.797 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.797 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:16:57.797 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.768782+0000", 2024-08-17T18:16:57.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1299494729999999 2024-08-17T18:16:57.797 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.797 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.797 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:16:57.797 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.768892+0000", 2024-08-17T18:16:57.797 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000109677 2024-08-17T18:16:57.797 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.797 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.798 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 6,7", 2024-08-17T18:16:57.798 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.768965+0000", 2024-08-17T18:16:57.798 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3170999999999995e-05 2024-08-17T18:16:57.798 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.798 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.798 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.798 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.771156+0000", 2024-08-17T18:16:57.798 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021906619999999999 2024-08-17T18:16:57.798 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.798 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.798 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:16:57.798 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.771230+0000", 2024-08-17T18:16:57.798 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4429000000000002e-05 2024-08-17T18:16:57.798 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.799 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.799 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:16:57.799 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.775410+0000", 2024-08-17T18:16:57.799 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0041797450000000003 2024-08-17T18:16:57.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.799 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.799 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:16:57.799 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.775463+0000", 2024-08-17T18:16:57.799 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3559e-05 2024-08-17T18:16:57.799 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:16:57.799 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:16:57.799 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:16:57.799 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.775554+0000", 2024-08-17T18:16:57.800 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0259000000000004e-05 2024-08-17T18:16:57.800 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.800 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.800 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.800 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:16:57.800 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:16:57.800 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:16:57.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:57.802 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:57.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:57.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:57.932 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:57.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:57.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:58.034 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:58.059 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:58.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:58.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:58.161 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:58.182 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:58.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:58.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:58.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:58.305 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:58.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:58.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:58.386 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:58 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:16:58.386 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:58 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2310237820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:16:58.386 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:58 smithi130 ceph-mon[105521]: osdmap e1431: 8 total, 7 up, 6 in 2024-08-17T18:16:58.386 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:58.386 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:16:58.386 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:58 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:16:58.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:58 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2310237820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:16:58.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:58 smithi130 ceph-mon[110068]: osdmap e1431: 8 total, 7 up, 6 in 2024-08-17T18:16:58.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:58.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:16:58.406 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:16:58.431 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:58.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:58.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:16:58.533 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:58.555 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:58.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:58.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:58 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:16:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:58 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2310237820' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:16:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:58 smithi179 ceph-mon[93519]: osdmap e1431: 8 total, 7 up, 6 in 2024-08-17T18:16:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:16:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:16:58.656 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:58.678 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:58.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:58.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:16:58.780 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:16:58.806 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:58.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:58.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:16:58.908 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:58.929 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:58.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:58.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:59.031 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:59.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:59.057 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:59.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:59.158 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:59.180 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:59.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:59.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:59.282 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:59.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:59.305 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:59.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:59.406 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:59.432 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:59.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:59.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:59.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:16:59.555 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:59.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:59.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:16:59.620 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:16:59 smithi130 ceph-mon[105521]: pgmap v2702: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 102 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:16:59.620 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:16:59 smithi130 ceph-mon[110068]: pgmap v2702: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 102 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:16:59.657 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:16:59.682 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:16:59.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:59.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:16:59.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:16:59.805 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:59.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:59.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:16:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:16:59 smithi179 ceph-mon[93519]: pgmap v2702: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 102 op/s; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:16:59.908 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:16:59.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:16:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:16:59] "GET /metrics HTTP/1.1" 200 36776 "" "Prometheus/2.51.0" 2024-08-17T18:16:59.930 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:16:59.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:16:59.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:00.031 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:00.058 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:00.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:00.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:00.160 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:00.181 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:00.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:00.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:00.283 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:00.309 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:00.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:00.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:00.392 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T18:17:00.410 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:00.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:00.435 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:00.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:00.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:00.562 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:00.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:00.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:00 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:17:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:17:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:17:00.636 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:00 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:00.637 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:00 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:00.664 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:00.690 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:00.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:00.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:00.744 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:17:00.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:00.817 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:00.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:00.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:00.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:00.944 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:00.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:00.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:01.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:01.069 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:01.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:01.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:01.171 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:01.193 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:01.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:01.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:01.295 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:01.320 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:01.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:01.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:01.397 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:17:01.398 INFO:tasks.thrashosds.thrasher:Failed to reset_purged_snaps_last, ignoring 2024-08-17T18:17:01.398 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.3 reset_purged_snaps_last 2024-08-17T18:17:01.423 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:01.431 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:01.443 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:01.454 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:01.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:01.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:01.556 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:01.579 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:01.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:01.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:01 smithi130 ceph-mon[105521]: pgmap v2703: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:01 smithi130 ceph-mon[110068]: pgmap v2703: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:01.681 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:01.703 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:01.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:01.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:01.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:01.828 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:01.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:01.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:01.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:01 smithi179 ceph-mon[93519]: pgmap v2703: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:01.932 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:01.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:01.956 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:01.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:02.057 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:02.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:17:02.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:02.082 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:02.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:02.183 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:02.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:02.208 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:02.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:02.309 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:02.333 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:02.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:02.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:02.435 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:02.459 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:02.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:02.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:02.561 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:02.584 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:02.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:02.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:02.686 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:02.708 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:02.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:02.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:02.810 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:02.831 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:02.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:02.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:02.933 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:02.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:02.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:02.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:03.060 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:03.085 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:03.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:03.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:03.187 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:03.209 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:03.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:03.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:03.253 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:03.253 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:03.311 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:03.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:03.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:03.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:03.429 INFO:teuthology.orchestra.run.smithi130.stderr:pool 'unique_pool_4' removed 2024-08-17T18:17:03.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:03.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:03.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:03.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:03.471 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:03 smithi130 ceph-mon[105521]: pgmap v2704: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:03.472 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:03 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/229615602' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:17:03.472 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:03 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:17:03.472 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:03 smithi130 ceph-mon[110068]: pgmap v2704: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:03.472 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:03 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/229615602' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:17:03.472 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:03 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:17:03.562 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:03.584 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:03.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:03.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:03.685 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:03.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:03.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:03.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:03.813 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:03.838 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:03.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:03.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:03.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:03 smithi179 ceph-mon[93519]: pgmap v2704: 89 pgs: 18 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 55 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:03.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:03 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/229615602' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:17:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:03 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:17:03.939 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:03.961 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:03.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:03.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:04.063 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:04.085 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:04.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:04.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:04.186 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:04.209 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:04.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:04.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:04.311 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:04.334 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:04.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:04.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:04.435 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:04.454 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:17:04.455 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:17:04.455 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:17:04.455 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:17:04.455 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:17:04.455 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:17:04.457 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:17:04.457 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:17:04.457 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:17:04.458 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:17:04.458 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:17:04.458 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:17:04.458 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:17:04.458 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:17:04.458 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:17:04.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:04.459 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:04.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:04.560 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:04.582 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:04.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:04.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:04.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:04 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:17:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:04 smithi130 ceph-mon[105521]: osdmap e1432: 8 total, 7 up, 6 in 2024-08-17T18:17:04.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:04 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/295171929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:17:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:04 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:17:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:04 smithi130 ceph-mon[110068]: osdmap e1432: 8 total, 7 up, 6 in 2024-08-17T18:17:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:04 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/295171929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:17:04.683 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:04.705 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:04.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:04.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:04.807 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:04.828 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:04.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:04.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:04 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:17:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:04 smithi179 ceph-mon[93519]: osdmap e1432: 8 total, 7 up, 6 in 2024-08-17T18:17:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:04 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/295171929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:17:04.930 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:04.952 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:04.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:04.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:05.054 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:05.075 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:05.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:05.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:05.177 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:05.198 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:05.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:05.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:05.300 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:05.325 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:05.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:05.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:05.427 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:05.448 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:05.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:05.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:05.550 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:05.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:05.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:05.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:05.677 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:05.702 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:05.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:05.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:05.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:05.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:05.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:05.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:05.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:05 smithi179 ceph-mon[93519]: pgmap v2706: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:05.863 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:05 smithi130 ceph-mon[105521]: pgmap v2706: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:05.864 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:05 smithi130 ceph-mon[110068]: pgmap v2706: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:05.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:05.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:05.956 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:05.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:06.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:06.078 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:06.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:06.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:06.180 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:06.202 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:06.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:06.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:06.304 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:06.327 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:06.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:06.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:06.428 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:06.450 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:06.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:06.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:06.552 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:06.578 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:06.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:06.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:06.680 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:06.701 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:06.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:06.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:06.785 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:06.785 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:06.804 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:06.828 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:06.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:06.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:06.930 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:06.956 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:06.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:06.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:07.058 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:07.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:07.082 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:07.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:07.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:07.206 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:07.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:07.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:07.214 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T18:17:07.214 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T18:17:07.215 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-08-17T18:17:07.215 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_5 16 2024-08-17T18:17:07.309 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:07.335 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:07.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:07.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:07.437 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:07.458 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:07.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:07.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:07.560 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:07.586 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:07.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:07.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:07.660 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:07 smithi130 ceph-mon[105521]: pgmap v2707: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:07.660 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:07 smithi130 ceph-mon[110068]: pgmap v2707: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:07.685 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:17:07.685 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:17:07.688 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:07.710 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:07.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:07.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:07.812 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:07.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:07.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:07.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:07.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:07 smithi179 ceph-mon[93519]: pgmap v2707: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:07.937 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:07.959 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:07.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:07.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:08.061 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:08.087 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:08.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:08.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:08.189 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:08.211 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:08.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:08.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:08.245 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:17:08.245 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:17:08.313 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:08.335 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:08.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:08.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:08.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:08.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:08.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:08.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:08.564 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:08.585 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:08.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:08.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:08.687 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:08.713 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:08.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:08.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:08.814 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:08.836 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:08.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:08.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:08.938 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:08.963 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:08.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:08.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:09.067 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:09.088 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:09.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:09.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:09.191 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:09.216 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:09.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:09.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:09.319 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:09.341 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:09.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:09.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:09.443 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:09.469 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:09.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:09.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:09.533 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:09 smithi130 ceph-mon[105521]: pgmap v2708: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:09.533 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:09 smithi130 ceph-mon[110068]: pgmap v2708: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:09.571 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:09.597 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:09.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:09.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:09.686 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:17:09.698 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:09.752 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:09.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:09.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:09 smithi179 ceph-mon[93519]: pgmap v2708: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:09.854 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:09.880 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:09.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:09.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:09.983 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:10.008 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:10.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:10.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:10.054 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:17:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:17:09] "GET /metrics HTTP/1.1" 200 36776 "" "Prometheus/2.51.0" 2024-08-17T18:17:10.111 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:10.133 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:10.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:10.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:10.234 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:10.258 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:10.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:10.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:10.360 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:10.384 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:10.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:10.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:10.442 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:17:10.475 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T18:17:10.485 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:10.508 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:10.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:10.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:10 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1798663929' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:17:10.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:10 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:17:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:17:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:17:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:17:10.611 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:10.633 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:10.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:10.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:10.707 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:10 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1798663929' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:17:10.707 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:10 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:17:10.707 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:10 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1798663929' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:17:10.707 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:10 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:17:10.735 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:10.758 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:10.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:10.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:10.860 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:10.881 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:10.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:10.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:10.983 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:11.004 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:11.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:11.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:11.106 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:11.132 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:11.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:11.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:11.234 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:11.259 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:11.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:11.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:11.361 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:11.377 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.7 reset_purged_snaps_last 2024-08-17T18:17:11.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:11.384 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:11.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:11.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:11.507 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:11.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:11.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:11.609 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:11.635 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:11.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:11.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:11.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:11 smithi130 ceph-mon[105521]: pgmap v2709: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:11 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:17:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:11 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:17:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:11 smithi130 ceph-mon[105521]: osdmap e1433: 8 total, 7 up, 6 in 2024-08-17T18:17:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:11 smithi130 ceph-mon[110068]: pgmap v2709: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:11 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:17:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:11 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:17:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:11 smithi130 ceph-mon[110068]: osdmap e1433: 8 total, 7 up, 6 in 2024-08-17T18:17:11.737 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:11.761 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:11.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:11.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:11 smithi179 ceph-mon[93519]: pgmap v2709: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:11 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:17:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:11 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:17:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:11 smithi179 ceph-mon[93519]: osdmap e1433: 8 total, 7 up, 6 in 2024-08-17T18:17:11.864 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:11.885 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:11.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:11.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:11.943 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:11.986 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:11.986 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:11.987 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:12.010 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:12.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:12.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:12.112 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:12.133 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:12.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:12.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:12.235 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:12.738 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:12.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:12.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:12.841 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:12.867 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:12.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:12.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:12.969 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:12.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:12.993 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:12.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:13.020 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:13.094 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:13.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:13.122 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:13.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:13.223 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:13.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:13.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:13.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:13.352 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:13.377 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:13.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:13.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:13.480 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:13.505 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:13.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:13.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:13.607 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:13.631 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:13.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:13.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:13.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:13 smithi130 ceph-mon[110068]: pgmap v2711: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:13.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:13.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:17:13.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:13 smithi130 ceph-mon[105521]: pgmap v2711: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:13.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:13.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:17:13.733 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:13.758 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:13.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:13.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:13 smithi179 ceph-mon[93519]: pgmap v2711: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:17:13.860 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:13.872 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:17:13.885 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:13.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:13.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:13.987 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:14.012 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:14.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:14.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:14.114 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:14.139 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:14.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:14.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:14.241 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:14.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:14.268 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:14.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:14.369 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:14.393 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:14.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:14.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:14.495 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:14.516 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:14.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:14.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:14.618 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:14.644 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:14.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:14.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:14.746 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:14.771 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:14.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:14.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:14.873 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:14.894 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:14.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:14.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:14.998 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:15.020 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:15.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:15.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:15.122 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:15.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:15.145 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:15.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:15.246 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:15.271 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:15.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:15.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:15.373 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:15.395 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:15.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:15.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:15.409 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:15 smithi130 ceph-mon[105521]: pgmap v2712: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:15.410 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:15 smithi130 ceph-mon[110068]: pgmap v2712: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:15.497 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:15.511 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:17:15.511 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:17:15.519 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:15.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:15.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:15.526 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:17:15.526 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:17:15.527 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:17:15.540 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:17:15.540 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:17:15.556 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:17:15.556 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:17:15.572 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:17:15.572 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:17:15.588 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:17:15.589 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:17:15.604 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:17:15.604 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:17:15.622 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:15.645 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:15.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:15.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:15.748 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:15.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:15.776 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:15.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:15 smithi179 ceph-mon[93519]: pgmap v2712: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:15.877 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:15.902 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:15.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:15.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:16.004 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:16.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:16.027 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:16.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:16.128 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:16.154 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:16.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:16.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:16.256 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:16.278 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:16.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:16.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:16.380 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:16.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:16.406 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:16.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:16.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:16 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/789521672' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:17:16.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:16 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/789521672' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:17:16.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:16.531 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:16.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:16.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:16.551 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:17:16.552 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:17:16.633 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:16.658 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:16.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:16.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:16.760 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:16.786 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:16.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:16.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:16 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/789521672' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:17:16.887 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:16.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:16.913 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:16.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:17.014 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:17.039 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:17.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:17.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:17.141 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:17.166 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:17.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:17.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:17.268 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:17.291 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:17.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:17.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:17.393 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:17.414 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:17.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:17.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:17.475 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:17 smithi130 ceph-mon[105521]: pgmap v2713: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:17.475 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:17 smithi130 ceph-mon[110068]: pgmap v2713: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:17.498 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:17.516 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:17.542 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:17.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:17.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:17.644 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:17.665 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:17.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:17.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:17.767 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:17.794 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:17.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:17.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:17 smithi179 ceph-mon[93519]: pgmap v2713: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:17.896 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:17.922 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:17.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:17.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:18.023 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:18.049 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:18.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:18.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:18.151 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:18.173 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:18.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:18.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:18.275 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:18.301 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:18.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:18.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:18.367 INFO:teuthology.orchestra.run.smithi130.stderr:pool 'unique_pool_5' created 2024-08-17T18:17:18.403 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:18.427 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:18.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:18.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:18.529 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:18.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:18.552 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:18.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:18.553 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:17:18.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2218944689' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T18:17:18.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:18 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T18:17:18.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2218944689' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T18:17:18.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:18 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T18:17:18.654 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:18.680 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:18.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:18.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:18.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:18.803 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:18.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:18.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2218944689' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T18:17:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:18 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T18:17:18.904 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:18.929 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:18.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:18.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:19.031 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:19.056 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:19.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:19.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:19.157 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:19.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:19.180 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:19.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:19.281 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:19.307 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:19.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:19.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:19.409 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:19.431 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:19.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:19.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:19.533 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:19.559 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:19.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:19.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:19.615 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:19 smithi130 ceph-mon[105521]: pgmap v2714: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:19.615 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:19 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-17T18:17:19.615 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:19 smithi130 ceph-mon[105521]: osdmap e1434: 8 total, 7 up, 6 in 2024-08-17T18:17:19.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:19 smithi130 ceph-mon[110068]: pgmap v2714: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:19.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:19 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-17T18:17:19.615 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:19 smithi130 ceph-mon[110068]: osdmap e1434: 8 total, 7 up, 6 in 2024-08-17T18:17:19.661 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:19.684 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:19.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:19.786 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:19.810 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:19.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:19.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:19.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:19 smithi179 ceph-mon[93519]: pgmap v2714: 73 pgs: 12 active+clean+remapped, 12 active+undersized, 2 active+undersized+degraded, 47 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:19 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-17T18:17:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:19 smithi179 ceph-mon[93519]: osdmap e1434: 8 total, 7 up, 6 in 2024-08-17T18:17:19.912 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:19.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:17:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:17:19] "GET /metrics HTTP/1.1" 200 34373 "" "Prometheus/2.51.0" 2024-08-17T18:17:19.935 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:19.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:19.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:20.037 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:20.061 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:20.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:20.163 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:20.185 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:20.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:20.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:20.287 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:20.310 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:20.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:20.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:20.412 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:20.437 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:20.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:20.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:20.540 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:20.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:20 smithi130 ceph-mon[105521]: osdmap e1435: 8 total, 7 up, 6 in 2024-08-17T18:17:20.557 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:20 smithi130 ceph-mon[110068]: osdmap e1435: 8 total, 7 up, 6 in 2024-08-17T18:17:20.562 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:20.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:20.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:20.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:20 smithi179 ceph-mon[93519]: osdmap e1435: 8 total, 7 up, 6 in 2024-08-17T18:17:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:17:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:17:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:17:20.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:20.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:20.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:20.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:20.788 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:20.809 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:20.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:20.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:20.867 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:17:20.867 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T18:17:20.867 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T18:17:20.867 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T18:17:20.867 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:17:20.911 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:20.932 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:20.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:20.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:21.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:21.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:21.056 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:21.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:21.158 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:21.179 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:21.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:21.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:21.280 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:21.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:21.303 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:21.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:21.388 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:17:21.405 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:21.427 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:21.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:21.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:21.529 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:21.550 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:21.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:21.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:21.583 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:21 smithi130 ceph-mon[105521]: pgmap v2717: 89 pgs: 12 unknown, 12 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 49 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:21.583 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:21 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:17:21.583 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:21 smithi130 ceph-mon[105521]: osdmap e1436: 8 total, 7 up, 6 in 2024-08-17T18:17:21.584 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:21 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2290166457' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:17:21.584 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:21 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:17:21.584 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:21 smithi130 ceph-mon[110068]: pgmap v2717: 89 pgs: 12 unknown, 12 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 49 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:21.584 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:21 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:17:21.584 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:21 smithi130 ceph-mon[110068]: osdmap e1436: 8 total, 7 up, 6 in 2024-08-17T18:17:21.584 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:21 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2290166457' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:17:21.584 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:21 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:17:21.608 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T18:17:21.652 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:21.673 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:21.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:21.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:21.775 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:21.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:21 smithi179 ceph-mon[93519]: pgmap v2717: 89 pgs: 12 unknown, 12 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 49 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:21.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:21 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:17:21.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:21 smithi179 ceph-mon[93519]: osdmap e1436: 8 total, 7 up, 6 in 2024-08-17T18:17:21.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:21 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2290166457' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:17:21.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:21 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:17:21.796 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:21.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:21.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:21.897 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:21.918 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:21.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:21.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:22.020 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:22.042 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:22.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:22.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:22.143 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:22.165 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:22.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:22.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:22.267 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:22.290 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:22.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:22.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:22.391 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:22.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:22.414 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:22.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:22.481 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:22 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:17:22.482 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:22 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:17:22.482 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:22 smithi130 ceph-mon[105521]: osdmap e1437: 8 total, 7 up, 6 in 2024-08-17T18:17:22.482 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:22 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:17:22.482 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:22 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:17:22.482 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:22 smithi130 ceph-mon[110068]: osdmap e1437: 8 total, 7 up, 6 in 2024-08-17T18:17:22.514 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:22.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:22.536 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:22.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:22.637 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:22.660 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:22.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:22.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:22.762 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:22.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:22.798 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:22.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:22 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:17:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:22 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:17:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:22 smithi179 ceph-mon[93519]: osdmap e1437: 8 total, 7 up, 6 in 2024-08-17T18:17:22.848 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-08-17T18:17:22.899 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:22.927 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:22.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:22.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:23.029 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:23.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:23.059 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:23.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:23.160 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:23.182 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:23.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:23.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:23.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:23.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:23.306 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:23.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:23.408 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:23.429 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:23.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:23.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:23.531 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:23.553 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:23.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:23.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:23.655 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:23.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:23 smithi130 ceph-mon[105521]: pgmap v2720: 89 pgs: 12 unknown, 12 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 49 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:23.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:23 smithi130 ceph-mon[110068]: pgmap v2720: 89 pgs: 12 unknown, 12 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 49 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:23.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:23.678 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:23.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:23.770 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.1 reset_purged_snaps_last 2024-08-17T18:17:23.779 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:23.804 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:23.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:23.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:23 smithi179 ceph-mon[93519]: pgmap v2720: 89 pgs: 12 unknown, 12 active+clean+remapped, 14 active+undersized, 2 active+undersized+degraded, 49 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:23.891 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:23.892 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:23.894 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:23.906 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:23.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:23.936 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:23.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:24.037 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:24.058 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:24.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:24.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:24.160 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:24.182 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:24.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:24.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:24.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:17:24.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:24.312 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:24.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:24.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:24.413 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:24.440 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:24.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:24.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:24.542 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:24.567 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:24.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:24.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:24.669 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:24.694 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:24.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:24.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:24.796 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:24.818 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:24.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:24.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:24.920 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:24.941 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:24.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:24.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:25.043 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:25.069 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:25.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:25.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:25.170 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:25.195 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:25.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:25.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:25.297 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:25.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:25.319 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:25.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:25.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:25.442 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:25.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:25.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:25.544 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:25.565 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:25.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:25.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:25 smithi130 ceph-mon[105521]: pgmap v2721: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:25 smithi130 ceph-mon[110068]: pgmap v2721: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:25.668 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:25.690 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:25.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:25.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:25.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:25.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:25.819 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:25.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:25 smithi179 ceph-mon[93519]: pgmap v2721: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:25.920 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:25.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:25.943 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:25.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:26.044 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:26.070 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:26.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:26.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:26.172 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:26.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:26.198 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:26.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:26.299 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:26.325 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:26.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:26.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:26.427 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:26.449 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:26.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:26.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:26.551 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:26.574 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:26.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:26.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:26.675 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:26.701 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:26.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:26.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:26.803 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:26.825 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:26.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:26.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:26.927 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:26.948 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:26.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:26.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:27.050 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:27.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:27.071 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:27.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:27.172 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:27.198 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:27.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:27.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:27.300 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:27.322 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:27.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:27.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:27.424 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:27.445 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:27.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:27.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:27.547 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:27.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:27.574 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:27.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:27.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:27 smithi130 ceph-mon[105521]: pgmap v2722: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:17:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:27 smithi130 ceph-mon[110068]: pgmap v2722: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:17:27.674 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:27.699 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:27.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:27.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:27.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:27.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:27.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:27.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:27.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:27 smithi179 ceph-mon[93519]: pgmap v2722: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:27.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:17:27.897 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:17:27.897 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:17:27.897 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:17:27.897 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:17:27.897 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:17:27.912 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:17:27.912 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:17:27.924 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:27.928 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:17:27.928 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:17:27.944 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:17:27.944 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:17:27.949 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:27.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:27.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:27.959 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:17:27.960 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:17:27.975 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:17:27.975 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:17:28.051 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:28.072 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:28.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:28.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:28.174 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:28.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:28.203 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:28.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:28.303 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:28.325 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:28.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:28.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:28.401 INFO:teuthology.orchestra.run.smithi130.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-08-17T18:17:28.427 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:28.453 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:28.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:28.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:28.555 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:28.581 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:28.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:28.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:28.636 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:28 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3704532315' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:17:28.636 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:28 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4031026711' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:17:28.636 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:28 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3704532315' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:17:28.636 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:28 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4031026711' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:17:28.682 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:28.704 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:28.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:28.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:28.806 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:28.827 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:28.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:28.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:28.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:28 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3704532315' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:17:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:28 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4031026711' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T18:17:28.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:28.955 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:28.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:28.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:29.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:29.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:29.079 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:29.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:29.180 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:29.202 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:29.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:29.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:29.289 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:29.289 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:29.304 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:29.327 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:29.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:29.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:29.430 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:29.451 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:29.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:29.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:29.553 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:29.575 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:29.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:29.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:29 smithi130 ceph-mon[105521]: pgmap v2723: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4031026711' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T18:17:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:29 smithi130 ceph-mon[105521]: osdmap e1438: 8 total, 7 up, 6 in 2024-08-17T18:17:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:29 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:17:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:29 smithi130 ceph-mon[110068]: pgmap v2723: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4031026711' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T18:17:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:29 smithi130 ceph-mon[110068]: osdmap e1438: 8 total, 7 up, 6 in 2024-08-17T18:17:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:29 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:17:29.677 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:29.699 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:29.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:29.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:29.727 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:17:29.727 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:17:29.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:29 smithi179 ceph-mon[93519]: pgmap v2723: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:29.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4031026711' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T18:17:29.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:29 smithi179 ceph-mon[93519]: osdmap e1438: 8 total, 7 up, 6 in 2024-08-17T18:17:29.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:29 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:17:29.801 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:29.823 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:29.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:29.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:29.925 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:29.947 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:29.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:29.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:30.047 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:17:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:17:29] "GET /metrics HTTP/1.1" 200 36711 "" "Prometheus/2.51.0" 2024-08-17T18:17:30.049 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:30.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:30.071 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:30.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:30.172 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:30.193 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:30.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:30.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:30.295 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:30.317 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:30.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:30.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:30.355 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:17:30.355 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:17:30.419 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:30.440 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:30.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:30.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:30.542 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:30.563 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:30.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:30.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:30.577 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:30 smithi179 ceph-mon[93519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:17:30.578 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:17:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:17:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:17:30.665 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:30.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:30 smithi130 ceph-mon[105521]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:17:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:30 smithi130 ceph-mon[110068]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:17:30.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:30.688 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:30.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:30.789 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:30.811 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:30.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:30.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:30.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:30.935 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:30.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:30.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:31.037 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:31.058 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:31.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:31.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:31.160 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:31.181 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:31.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:31.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:31.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:31.305 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:31.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:31.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:31.407 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:31.428 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:31.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:31.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:31.530 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:31.550 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:31 smithi179 ceph-mon[93519]: pgmap v2725: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:31.552 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:31.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:31.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:31.654 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:31.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:31.676 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:31.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:31.728 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:17:31.749 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:31 smithi130 ceph-mon[105521]: pgmap v2725: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:31.750 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:31 smithi130 ceph-mon[110068]: pgmap v2725: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:31.778 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:31.799 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:31.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:31.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:31.901 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:31.922 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:31.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:31.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:32.024 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:32.045 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:32.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:32.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:32.071 DEBUG:teuthology.orchestra.run.smithi130:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-08-17T18:17:32.089 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T18:17:32.089 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T18:17:32.133 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:hints = 1 2024-08-17T18:17:32.134 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T18:17:32.134 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Object prefix: benchmark_data_smithi130_398002 2024-08-17T18:17:32.134 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:17:32.134 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 0 0 0 0 0 0 - 0 2024-08-17T18:17:32.148 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:32.190 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:32.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:32.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:32.292 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:32.323 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:32.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:32.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:32.425 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:32.467 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:32.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:32.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:32.569 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:32.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:32.597 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:32.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:32.698 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:32.734 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:32.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:32.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:32.836 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:32.870 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:32.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:32.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:32.972 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:33.010 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:33.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:33.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:33.112 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:33.134 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 1 16 4338 4322 270.087 270.125 0.00573051 0.00369475 2024-08-17T18:17:33.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:33.151 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:33.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:33.252 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:33.284 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:33.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:33.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:33.386 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:33.414 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:33.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:33.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:33.516 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:33.560 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:33.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:33.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:33.666 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:33.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:33.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:33.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:33.742 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:33 smithi130 ceph-mon[110068]: pgmap v2726: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:33.743 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:33 smithi130 ceph-mon[105521]: pgmap v2726: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:33.797 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:33.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:33.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:33.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:33.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:33 smithi179 ceph-mon[93519]: pgmap v2726: 89 pgs: 12 active+clean+remapped, 21 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 2/78 objects degraded (2.564%); 8/78 objects misplaced (10.256%) 2024-08-17T18:17:33.936 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:33.970 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:33.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:33.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:34.072 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:34.108 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T18:17:34.122 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:34.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:34.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:34.134 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 2 16 8228 8212 256.594 243.125 0.00452713 0.00388998 2024-08-17T18:17:34.224 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:34.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:34.270 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:34.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:34.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:34.398 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:34.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:34.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:34.500 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:34.534 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:34.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:34.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:34.636 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:34.666 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:34.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:34.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:34.768 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:34.804 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:34.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:34.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:34.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:34.936 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:34.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:34.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:35.039 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:35.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:35.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:35.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:35.134 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 3 16 11651 11635 242.369 213.938 0.0201371 0.00411398 2024-08-17T18:17:35.168 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:35.200 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:35.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:35.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:35.298 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:35 smithi130 ceph-mon[105521]: pgmap v2727: 89 pgs: 12 active+clean+remapped, 15 active+undersized, 8 active+undersized+degraded, 54 active+clean; 157 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 16 MiB/s wr, 250 op/s; 1476/7599 objects degraded (19.424%); 8/7599 objects misplaced (0.105%) 2024-08-17T18:17:35.328 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:35 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:35.328 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:35 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:35.328 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:35 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 1476/7599 objects degraded (19.424%), 8 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:35.329 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:35 smithi130 ceph-mon[110068]: pgmap v2727: 89 pgs: 12 active+clean+remapped, 15 active+undersized, 8 active+undersized+degraded, 54 active+clean; 157 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 16 MiB/s wr, 250 op/s; 1476/7599 objects degraded (19.424%); 8/7599 objects misplaced (0.105%) 2024-08-17T18:17:35.329 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:35 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:35.329 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:35 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:35.329 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:35 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 1476/7599 objects degraded (19.424%), 8 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:35.329 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:35.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:35 smithi179 ceph-mon[93519]: pgmap v2727: 89 pgs: 12 active+clean+remapped, 15 active+undersized, 8 active+undersized+degraded, 54 active+clean; 157 MiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 16 MiB/s wr, 250 op/s; 1476/7599 objects degraded (19.424%); 8/7599 objects misplaced (0.105%) 2024-08-17T18:17:35.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:35 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:35.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:35 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:35.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:35 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 1476/7599 objects degraded (19.424%), 8 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:35.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:35.360 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:35.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:35.416 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:35.462 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:35.492 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:35.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:35.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:35.594 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:35.622 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:35.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:35.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:35.724 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:35.784 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:35.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:35.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:35.887 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:35.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:35.925 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:35.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:36.026 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:36.027 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:17:36.056 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:36.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:36.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:36.134 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 4 16 14880 14864 232.225 201.812 0.00923892 0.0042987 2024-08-17T18:17:36.158 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:36.188 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:36.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:36.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:36.290 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:36.327 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:36.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:36.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:36.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:36 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2733484328' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:17:36.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:36 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:17:36.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:36 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2733484328' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:17:36.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:36 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:17:36.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:36 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2733484328' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:17:36.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:36 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:17:36.429 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:36.458 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:36.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:36.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:36.560 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:36.596 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:36.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:36.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:36.698 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:36.737 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:36.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:36.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:36.839 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:36.869 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:36.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:36.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:36.971 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:37.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:37.004 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:37.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:37.105 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:37.134 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 5 16 18100 18084 226.025 201.25 0.00968049 0.00441856 2024-08-17T18:17:37.139 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:37.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:37.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:37.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:37 smithi130 ceph-mon[105521]: pgmap v2728: 89 pgs: 12 active+clean+remapped, 13 active+undersized, 10 active+undersized+degraded, 54 active+clean; 309 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 31 MiB/s wr, 493 op/s; 2674/14898 objects degraded (17.949%); 8/14898 objects misplaced (0.054%) 2024-08-17T18:17:37.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:37 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:17:37.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:37 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:17:37.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:37 smithi130 ceph-mon[105521]: osdmap e1439: 8 total, 7 up, 6 in 2024-08-17T18:17:37.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:37 smithi130 ceph-mon[110068]: pgmap v2728: 89 pgs: 12 active+clean+remapped, 13 active+undersized, 10 active+undersized+degraded, 54 active+clean; 309 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 31 MiB/s wr, 493 op/s; 2674/14898 objects degraded (17.949%); 8/14898 objects misplaced (0.054%) 2024-08-17T18:17:37.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:37 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:17:37.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:37 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:17:37.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:37 smithi130 ceph-mon[110068]: osdmap e1439: 8 total, 7 up, 6 in 2024-08-17T18:17:37.242 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:37.273 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:37.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:37.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:37.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:37 smithi179 ceph-mon[93519]: pgmap v2728: 89 pgs: 12 active+clean+remapped, 13 active+undersized, 10 active+undersized+degraded, 54 active+clean; 309 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 31 MiB/s wr, 493 op/s; 2674/14898 objects degraded (17.949%); 8/14898 objects misplaced (0.054%) 2024-08-17T18:17:37.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:37 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:17:37.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:37 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:17:37.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:37 smithi179 ceph-mon[93519]: osdmap e1439: 8 total, 7 up, 6 in 2024-08-17T18:17:37.376 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:37.398 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:37.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:37.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:37.500 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:37.534 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:37.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:37.667 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:37.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:37.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:37.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:37.799 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:37.829 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:37.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:37.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:37.931 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:37.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:37.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:37.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:38.077 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:38.106 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:38.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:38.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:38.134 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 6 16 21361 21345 222.32 203.812 0.00227278 0.00449301 2024-08-17T18:17:38.207 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:38.239 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:38.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:38.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:38.341 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:38.376 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:38.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:38.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:38.478 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:38.507 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:38.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:38.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:38.609 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:38.652 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:38.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:38.754 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:38.790 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:38.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:38.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:38.892 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:38.929 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:38.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:38.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:39.031 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:39.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:39.064 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:39.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:39.128 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:39 smithi130 ceph-mon[105521]: pgmap v2730: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 438 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 48 MiB/s wr, 762 op/s; 3675/21093 objects degraded (17.423%); 8/21093 objects misplaced (0.038%) 2024-08-17T18:17:39.128 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:39 smithi130 ceph-mon[110068]: pgmap v2730: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 438 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 48 MiB/s wr, 762 op/s; 3675/21093 objects degraded (17.423%); 8/21093 objects misplaced (0.038%) 2024-08-17T18:17:39.134 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 7 16 24395 24379 217.647 189.625 0.00902606 0.00458927 2024-08-17T18:17:39.165 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:39.198 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:39.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:39.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:39.300 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:39.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:39.341 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:39.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:39.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:39 smithi179 ceph-mon[93519]: pgmap v2730: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 438 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 48 MiB/s wr, 762 op/s; 3675/21093 objects degraded (17.423%); 8/21093 objects misplaced (0.038%) 2024-08-17T18:17:39.442 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:39.478 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:39.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:39.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:39.580 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:39.584 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:39.608 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:39.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:39.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:39.620 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.6 reset_purged_snaps_last 2024-08-17T18:17:39.710 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:39.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:39.741 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:39.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:39.842 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:39.873 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:39.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:39.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:39.888 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:17:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:17:39] "GET /metrics HTTP/1.1" 200 36711 "" "Prometheus/2.51.0" 2024-08-17T18:17:39.975 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:40.006 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:40.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:40.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:40.108 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:40.135 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 8 16 27123 27107 211.752 170.5 0.00861697 0.00471737 2024-08-17T18:17:40.137 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:40.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:40.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:40.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:40 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 11959/65034 objects degraded (18.389%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:40.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:40 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 11959/65034 objects degraded (18.389%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:40.239 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:40.270 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:40.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:40.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:40.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:40 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 11959/65034 objects degraded (18.389%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:40.343 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:17:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:17:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:17:40.372 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:40.404 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:40.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:40.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:40.506 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:40.543 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:40.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:40.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:40.550 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:17:40.645 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:40.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:40.688 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:40.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:40.789 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:40.817 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:40.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:40.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:40.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:40.945 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:40.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:40.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:41.047 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:41.078 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:41.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:41.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:41.135 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 9 16 29532 29516 204.951 150.562 0.0046903 0.00487426 2024-08-17T18:17:41.181 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:41.209 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:41.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:41.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:41.311 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:41.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:41 smithi179 ceph-mon[93519]: pgmap v2731: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 1.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 135 MiB/s wr, 2.16k op/s; 11959/65034 objects degraded (18.389%); 8/65034 objects misplaced (0.012%) 2024-08-17T18:17:41.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:41.353 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:41.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:41.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:41 smithi130 ceph-mon[105521]: pgmap v2731: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 1.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 135 MiB/s wr, 2.16k op/s; 11959/65034 objects degraded (18.389%); 8/65034 objects misplaced (0.012%) 2024-08-17T18:17:41.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:41 smithi130 ceph-mon[110068]: pgmap v2731: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 1.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 135 MiB/s wr, 2.16k op/s; 11959/65034 objects degraded (18.389%); 8/65034 objects misplaced (0.012%) 2024-08-17T18:17:41.455 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:41.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:41.492 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:41.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:41.593 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:41.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:41.615 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:41.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:41.716 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:41.745 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:41.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:41.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:41.847 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:41.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:41.889 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:41.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:41.990 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:42.024 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:42.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:42.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:42.126 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:42.135 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 10 16 31834 31818 198.843 143.875 0.0084553 0.00502411 2024-08-17T18:17:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:42.163 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:42.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:42.264 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:42.301 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:42.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:42.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:42.403 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:42.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:42.428 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:42.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:42.530 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:42.557 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:42.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:42.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:42.659 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:42.687 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:42.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:42.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:42.789 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:42.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:42.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:42.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:42.922 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:42.952 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:42.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:42.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:43.054 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:43.085 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:43.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:43.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:43.135 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 11 16 33855 33839 192.248 126.312 0.0569235 0.00517632 2024-08-17T18:17:43.186 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:43.217 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:43.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:43.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:43.319 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:43.340 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:43.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:43.442 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:43.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:43.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:43.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:43.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:43.603 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:43.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:43.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:43.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:43 smithi130 ceph-mon[105521]: pgmap v2732: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 1.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 135 MiB/s wr, 2.16k op/s; 11959/65034 objects degraded (18.389%); 8/65034 objects misplaced (0.012%) 2024-08-17T18:17:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:17:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:43 smithi130 ceph-mon[110068]: pgmap v2732: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 1.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 135 MiB/s wr, 2.16k op/s; 11959/65034 objects degraded (18.389%); 8/65034 objects misplaced (0.012%) 2024-08-17T18:17:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:17:43.705 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:43.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:43.727 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:43.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:43.828 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:43 smithi179 ceph-mon[93519]: pgmap v2732: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 1.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 135 MiB/s wr, 2.16k op/s; 11959/65034 objects degraded (18.389%); 8/65034 objects misplaced (0.012%) 2024-08-17T18:17:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:17:43.865 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:43.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:43.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:43.967 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:44.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:44.004 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:44.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:44.105 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:44.135 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 12 16 36164 36148 188.252 144.312 0.00955793 0.00530725 2024-08-17T18:17:44.139 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:44.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:44.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:44.241 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:44.272 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:44.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:44.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:44.374 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:44.397 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:44.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:44.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:44.499 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:44.533 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:44.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:44.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:44.635 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:44.673 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:44.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:44.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:44.775 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:44.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:44.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:44.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:44.911 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:44.945 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:44.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:44.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:45.047 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:45.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:45.077 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:45.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:45.135 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 13 16 38375 38359 184.4 138.188 0.0095243 0.00541812 2024-08-17T18:17:45.178 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:45.208 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:45.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:45.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:45.310 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:45.341 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:45.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:45.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:45.443 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:45.478 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:45.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:45.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:45.580 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:45.618 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:45.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:45.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:45.623 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:45 smithi130 ceph-mon[105521]: pgmap v2733: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 1.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 180 MiB/s wr, 2.89k op/s; 17658/94260 objects degraded (18.733%); 8/94260 objects misplaced (0.008%) 2024-08-17T18:17:45.624 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:45 smithi130 ceph-mon[110068]: pgmap v2733: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 1.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 180 MiB/s wr, 2.89k op/s; 17658/94260 objects degraded (18.733%); 8/94260 objects misplaced (0.008%) 2024-08-17T18:17:45.721 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:45.759 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:45.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:45.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:45 smithi179 ceph-mon[93519]: pgmap v2733: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 1.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 180 MiB/s wr, 2.89k op/s; 17658/94260 objects degraded (18.733%); 8/94260 objects misplaced (0.008%) 2024-08-17T18:17:45.860 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:45.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:45.897 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:45.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:45.998 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:46.020 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:46.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:46.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:46.121 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:46.135 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 14 16 40538 40522 180.884 135.188 0.00528581 0.00552347 2024-08-17T18:17:46.152 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:46.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:46.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:46.254 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:46.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:46.276 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:46.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:46.377 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:46.400 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:46.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:46.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:46.502 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:46.531 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:46.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:46.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:46.633 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:46.656 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:46.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:46.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:46.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17658/94260 objects degraded (18.733%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:46.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17658/94260 objects degraded (18.733%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:46.758 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:46.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:46.781 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:46.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17658/94260 objects degraded (18.733%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:46.882 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:46.911 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:46.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:46.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:47.013 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:47.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:47.042 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:47.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:47.135 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 15 16 41965 41949 174.771 89.1875 0.00417218 0.0057168 2024-08-17T18:17:47.143 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:47.173 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:47.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:47.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:47.275 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:47.308 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:47.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:47.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:47.410 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:47.440 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:47.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:47.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:47.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:47.578 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:47.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:47.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:47.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:47 smithi130 ceph-mon[105521]: pgmap v2734: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 2.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 183 MiB/s wr, 2.93k op/s; 19077/102924 objects degraded (18.535%); 8/102924 objects misplaced (0.008%) 2024-08-17T18:17:47.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:47 smithi130 ceph-mon[110068]: pgmap v2734: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 2.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 183 MiB/s wr, 2.93k op/s; 19077/102924 objects degraded (18.535%); 8/102924 objects misplaced (0.008%) 2024-08-17T18:17:47.680 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:47.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:47.711 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:47.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:47.812 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:47.837 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:47.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:47.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:47.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:47 smithi179 ceph-mon[93519]: pgmap v2734: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 2.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 183 MiB/s wr, 2.93k op/s; 19077/102924 objects degraded (18.535%); 8/102924 objects misplaced (0.008%) 2024-08-17T18:17:47.939 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:47.978 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:47.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:47.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:48.080 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:48.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:48.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:48.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:48.135 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 16 16 44115 44099 172.245 134.375 0.00739606 0.00580023 2024-08-17T18:17:48.208 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:48.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:48.237 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:48.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:48.338 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:48.366 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:48.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:48.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:48.468 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:48.501 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:48.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:48.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:48.603 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:48.628 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:48.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:48.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:48.730 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:48.759 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:48.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:48.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:48.861 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:48.893 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:48.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:48.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:48.995 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:49.032 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:49.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:49.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:49.135 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:49.137 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 17 16 46211 46195 169.818 131 0.00781517 0.0058838 2024-08-17T18:17:49.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:49.175 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:49.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:49.278 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:49.347 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:49.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:49.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:49.450 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:49.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:49.496 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:49.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:49.540 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:49 smithi130 ceph-mon[105521]: pgmap v2735: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 2.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 166 MiB/s wr, 2.65k op/s; 19720/106779 objects degraded (18.468%); 8/106779 objects misplaced (0.007%) 2024-08-17T18:17:49.540 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:49 smithi130 ceph-mon[110068]: pgmap v2735: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 2.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 166 MiB/s wr, 2.65k op/s; 19720/106779 objects degraded (18.468%); 8/106779 objects misplaced (0.007%) 2024-08-17T18:17:49.597 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:49.641 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:49.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:49.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:49.743 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:49.771 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:49.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:49.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:49 smithi179 ceph-mon[93519]: pgmap v2735: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 2.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 166 MiB/s wr, 2.65k op/s; 19720/106779 objects degraded (18.468%); 8/106779 objects misplaced (0.007%) 2024-08-17T18:17:49.873 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:49.899 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:49.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:49.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:50.001 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:50.007 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:17:50.007 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:17:50.045 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:50.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:50.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:50.047 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:17:50.047 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:17:50.048 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:17:50.064 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:50.073 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:17:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:17:49] "GET /metrics HTTP/1.1" 200 36779 "" "Prometheus/2.51.0" 2024-08-17T18:17:50.084 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:17:50.084 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:17:50.121 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:17:50.121 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:17:50.137 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 18 16 48052 48036 166.763 115.062 0.00873146 0.00599103 2024-08-17T18:17:50.147 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:50.153 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:17:50.153 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:17:50.174 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:50.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:50.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:50.186 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:17:50.186 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:17:50.214 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:17:50.214 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:17:50.276 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:50.317 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:50.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:50.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:50.419 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:50.447 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:50.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:50.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:50.549 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:50.579 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:50.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:50.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:50.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:50 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2924020920' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:17:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:17:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:17:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:17:50.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:50 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2924020920' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:17:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:50 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2924020920' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:17:50.681 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:50.702 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:50.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:50.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:50.807 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:50.845 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:50.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:50.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:50.946 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:50.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:50.982 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:50.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:51.083 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:51.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:51.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:51.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:51.137 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 19 16 49938 49922 164.189 117.875 0.00309566 0.00608574 2024-08-17T18:17:51.215 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:51.243 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:51.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:51.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:51.345 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:51.380 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:51.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:51.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:51.482 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:51.519 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:51.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:51.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:51.622 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:51.666 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:51.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:51 smithi130 ceph-mon[105521]: pgmap v2736: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 2.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 182 MiB/s wr, 2.91k op/s; 23497/125766 objects degraded (18.683%); 8/125766 objects misplaced (0.006%) 2024-08-17T18:17:51.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 23497/125766 objects degraded (18.683%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:51 smithi130 ceph-mon[110068]: pgmap v2736: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 2.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 182 MiB/s wr, 2.91k op/s; 23497/125766 objects degraded (18.683%); 8/125766 objects misplaced (0.006%) 2024-08-17T18:17:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 23497/125766 objects degraded (18.683%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:51.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:51.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:51.769 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:51.801 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:51.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:51.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:51.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:51 smithi179 ceph-mon[93519]: pgmap v2736: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 2.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 182 MiB/s wr, 2.91k op/s; 23497/125766 objects degraded (18.683%); 8/125766 objects misplaced (0.006%) 2024-08-17T18:17:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 23497/125766 objects degraded (18.683%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:51.903 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:51.937 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:51.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:51.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:52.039 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:52.074 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:52.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:52.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:52.139 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:17:52.137746+0000 min lat: 0.00107633 max lat: 0.143133 avg lat: 0.00615176 2024-08-17T18:17:52.139 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:17:52.140 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 20 16 51998 51982 162.417 128.75 0.00302997 0.00615176 2024-08-17T18:17:52.176 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:52.219 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:17:52.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:52.221 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:52.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:52.256 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:17:52.256 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:17:52.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.256 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:52142 50.2 50:49f0a9ae:::benchmark_data_smithi130_398002_object52141:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1439)", 2024-08-17T18:17:52.256 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:17:52.243412+0000", 2024-08-17T18:17:52.256 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.010801597, 2024-08-17T18:17:52.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010836161, 2024-08-17T18:17:52.256 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:17:52.256 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:17:52.256 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:17:52.256 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:17:52.257 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:17:52.257 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:17:52.257 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 52142 2024-08-17T18:17:52.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.257 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:17:52.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.257 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:17:52.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.243412+0000", 2024-08-17T18:17:52.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:17:52.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.257 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:17:52.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.243412+0000", 2024-08-17T18:17:52.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:17:52.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:17:52.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.243414+0000", 2024-08-17T18:17:52.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4990000000000001e-06 2024-08-17T18:17:52.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:17:52.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.243440+0000", 2024-08-17T18:17:52.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5327e-05 2024-08-17T18:17:52.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.259 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:17:52.259 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.243458+0000", 2024-08-17T18:17:52.259 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7893e-05 2024-08-17T18:17:52.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.259 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:17:52.259 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.243467+0000", 2024-08-17T18:17:52.259 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3209999999999994e-06 2024-08-17T18:17:52.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:17:52.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.246037+0000", 2024-08-17T18:17:52.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025701769999999999 2024-08-17T18:17:52.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:17:52.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.246196+0000", 2024-08-17T18:17:52.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000158796 2024-08-17T18:17:52.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:17:52.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.246373+0000", 2024-08-17T18:17:52.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000177646 2024-08-17T18:17:52.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:17:52.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.249724+0000", 2024-08-17T18:17:52.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0033507110000000001 2024-08-17T18:17:52.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:17:52.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.249764+0000", 2024-08-17T18:17:52.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9907999999999998e-05 2024-08-17T18:17:52.261 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:17:52.261 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:17:52.261 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:17:52.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.262 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:52150 50.c e1439/1434 50:3723ab59:::benchmark_data_smithi130_398002_object52149:head v 1439'3192, mlcod=1439'3190)", 2024-08-17T18:17:52.262 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:17:52.253386+0000", 2024-08-17T18:17:52.262 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.00082786199999999996, 2024-08-17T18:17:52.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00094483399999999997, 2024-08-17T18:17:52.262 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:17:52.262 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:17:52.262 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:17:52.262 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:17:52.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.253386+0000", 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.253386+0000", 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.253387+0000", 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5e-06 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:17:52.264 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.253434+0000", 2024-08-17T18:17:52.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6678000000000003e-05 2024-08-17T18:17:52.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:17:52.264 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.253457+0000", 2024-08-17T18:17:52.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3430000000000001e-05 2024-08-17T18:17:52.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:17:52.264 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.253467+0000", 2024-08-17T18:17:52.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7200000000000001e-06 2024-08-17T18:17:52.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:17:52.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.253571+0000", 2024-08-17T18:17:52.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000103947 2024-08-17T18:17:52.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:17:52.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:17:52.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:17:52.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:17:52.253592+0000", 2024-08-17T18:17:52.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1016e-05 2024-08-17T18:17:52.266 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:17:52.266 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:17:52.266 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:17:52.266 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:17:52.266 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T18:17:52.266 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 2 2024-08-17T18:17:52.266 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:17:52.321 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:52.349 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:52.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:52.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:52.451 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:52.485 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:52.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:52.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:52.588 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:52.618 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:52.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:52.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:52.721 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:52.752 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:52.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:52.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:52.854 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:52.892 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:52.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:52.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:52.994 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:53.032 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:53.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:53.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:53.134 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:53.138 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 21 16 53919 53903 160.393 120.062 0.00621658 0.00622875 2024-08-17T18:17:53.162 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:53.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:53.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:53.268 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:53.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:53.315 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:53.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:53.416 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:53.456 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:53.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:53.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:53.558 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:53.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:53.591 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:53.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:53.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:53 smithi130 ceph-mon[105521]: pgmap v2737: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 2.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 119 MiB/s wr, 1.90k op/s; 24729/133338 objects degraded (18.546%); 8/133338 objects misplaced (0.006%) 2024-08-17T18:17:53.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:53 smithi130 ceph-mon[110068]: pgmap v2737: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 2.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 119 MiB/s wr, 1.90k op/s; 24729/133338 objects degraded (18.546%); 8/133338 objects misplaced (0.006%) 2024-08-17T18:17:53.692 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:53.727 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:53.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:53.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:53.828 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:53.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:53 smithi179 ceph-mon[93519]: pgmap v2737: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 2.7 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 119 MiB/s wr, 1.90k op/s; 24729/133338 objects degraded (18.546%); 8/133338 objects misplaced (0.006%) 2024-08-17T18:17:53.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:53.869 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:53.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:53.937 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:17:53.937 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:17:53.970 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:54.000 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:54.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:54.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:54.101 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:54.129 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:54.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:54.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:54.138 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 22 16 55762 55746 158.338 115.188 0.00881236 0.0063108 2024-08-17T18:17:54.231 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:54.261 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:54.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:54.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:54.363 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:54.406 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:54.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:54.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:54.508 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:54.536 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:54.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:54.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:54.638 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:54.665 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:54.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:54.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:54.767 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:54.794 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:54.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:54.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:54.896 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:54.922 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:54.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:54.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:55.024 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:55.051 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:55.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:55.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:55.138 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 23 16 57591 57575 156.423 114.312 0.0175902 0.00638749 2024-08-17T18:17:55.153 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:55.191 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:55.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:55.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:55.269 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T18:17:55.293 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:55.322 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:55.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:55.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:55.408 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:17:55.424 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:55.451 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:55.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:55.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:55.553 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:55.578 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:55.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:55.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:55 smithi130 ceph-mon[105521]: pgmap v2738: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 157 MiB/s wr, 2.50k op/s; 29007/155217 objects degraded (18.688%); 8/155217 objects misplaced (0.005%) 2024-08-17T18:17:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:55 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/112537813' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:17:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:55 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:17:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:55 smithi130 ceph-mon[110068]: pgmap v2738: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 157 MiB/s wr, 2.50k op/s; 29007/155217 objects degraded (18.688%); 8/155217 objects misplaced (0.005%) 2024-08-17T18:17:55.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:55 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/112537813' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:17:55.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:55 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:17:55.680 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:55.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:55.712 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:55.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:55.813 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:55 smithi179 ceph-mon[93519]: pgmap v2738: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 157 MiB/s wr, 2.50k op/s; 29007/155217 objects degraded (18.688%); 8/155217 objects misplaced (0.005%) 2024-08-17T18:17:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:55 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/112537813' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:17:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:55 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:17:55.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:55.844 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:55.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:55.939 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:17:55.946 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:55.974 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:55.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:55.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:56.076 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:56.100 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:56.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:56.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:56.140 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 24 16 59334 59318 154.438 108.938 0.0137346 0.00646855 2024-08-17T18:17:56.202 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:56.258 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:56.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:56.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:56.359 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:56.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:56.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:56.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:56.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:56.543 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:56.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:56.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:56.645 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:56.683 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:56.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:56.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:56.785 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:56.807 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:56.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:56.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:56.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:56 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:17:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:56 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:17:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:56 smithi179 ceph-mon[93519]: osdmap e1440: 8 total, 7 up, 6 in 2024-08-17T18:17:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 29007/155217 objects degraded (18.688%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:56.909 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:56.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:56 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:17:56.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:56 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:17:56.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:56 smithi130 ceph-mon[110068]: osdmap e1440: 8 total, 7 up, 6 in 2024-08-17T18:17:56.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 29007/155217 objects degraded (18.688%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:56.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:56 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:17:56.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:56 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:17:56.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:56 smithi130 ceph-mon[105521]: osdmap e1440: 8 total, 7 up, 6 in 2024-08-17T18:17:56.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 29007/155217 objects degraded (18.688%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:17:56.946 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:56.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:56.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:57.048 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:17:57.075 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:57.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:57.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:17:57.140 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 25 16 60975 60959 152.363 102.562 0.00908987 0.00655842 2024-08-17T18:17:57.176 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:57.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:57.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:57.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:57.315 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:57.340 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:57.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:57.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:57.442 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:57.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:57.478 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:57.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:57.579 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:57.607 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:57.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:57.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:57.708 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:57.737 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:57.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:57.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:57.838 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:17:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:57 smithi179 ceph-mon[93519]: pgmap v2740: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 124 MiB/s wr, 1.99k op/s; 30186/162543 objects degraded (18.571%); 8/162543 objects misplaced (0.005%) 2024-08-17T18:17:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:57 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:17:57.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:57.869 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:57.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:17:57.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:57 smithi130 ceph-mon[105521]: pgmap v2740: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 124 MiB/s wr, 1.99k op/s; 30186/162543 objects degraded (18.571%); 8/162543 objects misplaced (0.005%) 2024-08-17T18:17:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:57 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:57.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:17:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:57 smithi130 ceph-mon[110068]: pgmap v2740: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 124 MiB/s wr, 1.99k op/s; 30186/162543 objects degraded (18.571%); 8/162543 objects misplaced (0.005%) 2024-08-17T18:17:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:57 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:17:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:17:57.970 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:58.009 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:58.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:58.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:58.111 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:58.140 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 26 16 62649 62633 150.526 104.625 0.00857172 0.00663875 2024-08-17T18:17:58.147 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:58.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:58.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:58.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:58.285 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:58.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:58.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:58.387 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:58.422 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:58.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:58.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:58.524 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:58.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:58.565 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:58.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:58.666 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:58.698 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:58.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:58.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:58.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:58.833 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:58.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:58.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:17:58.935 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:58.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:58.966 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:58.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:59.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:17:59.101 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:59.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:59.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:17:59.140 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 27 16 64320 64304 148.819 104.438 0.0101519 0.00671505 2024-08-17T18:17:59.203 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:59.239 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:59.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:59.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:59.341 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:17:59.368 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:59.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:59.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:17:59.469 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:59.497 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:59.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:59.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:59.600 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:17:59.643 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:59.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:17:59.669 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:17:59 smithi130 ceph-mon[105521]: pgmap v2741: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 123 MiB/s wr, 1.97k op/s; 30793/165972 objects degraded (18.553%); 8/165972 objects misplaced (0.005%) 2024-08-17T18:17:59.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:17:59 smithi130 ceph-mon[110068]: pgmap v2741: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 123 MiB/s wr, 1.97k op/s; 30793/165972 objects degraded (18.553%); 8/165972 objects misplaced (0.005%) 2024-08-17T18:17:59.745 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:17:59.773 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:17:59.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:59.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:17:59.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:17:59 smithi179 ceph-mon[93519]: pgmap v2741: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.4 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 123 MiB/s wr, 1.97k op/s; 30793/165972 objects degraded (18.553%); 8/165972 objects misplaced (0.005%) 2024-08-17T18:17:59.874 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:17:59.919 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:17:59.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:17:59.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:00.003 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:17:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:17:59] "GET /metrics HTTP/1.1" 200 36783 "" "Prometheus/2.51.0" 2024-08-17T18:18:00.019 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:18:00.022 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:00.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:00.077 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:00.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:00.142 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 28 16 65887 65871 146.991 97.9375 0.00824475 0.00679797 2024-08-17T18:18:00.179 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:00.210 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:00.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:00.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:00.312 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:00.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:00.352 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:00.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:00.453 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:00.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:00.495 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:00.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:00.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:18:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:18:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:18:00.596 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:00.645 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:00.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:00.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:00.748 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:00.778 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:00.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:00.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:00.880 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:00.921 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:00.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:00.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:01.023 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:01.057 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:01.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:01.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:01.142 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 29 15 67540 67525 145.487 103.375 0.00661048 0.00686936 2024-08-17T18:18:01.160 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:01.197 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:01.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:01.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:01.299 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:01.321 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:01.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:01.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:01.377 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:18:01.423 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:01.449 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:01.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:01.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:01.551 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:01.584 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:01.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:01.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:01.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:01 smithi130 ceph-mon[105521]: pgmap v2742: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 118 MiB/s wr, 1.89k op/s; 34057/182526 objects degraded (18.659%); 8/182526 objects misplaced (0.004%) 2024-08-17T18:18:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 34057/182526 objects degraded (18.659%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:01 smithi130 ceph-mon[110068]: pgmap v2742: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 118 MiB/s wr, 1.89k op/s; 34057/182526 objects degraded (18.659%); 8/182526 objects misplaced (0.004%) 2024-08-17T18:18:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 34057/182526 objects degraded (18.659%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:01.685 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:01.714 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:01.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:01.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:01.816 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:01.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:01 smithi179 ceph-mon[93519]: pgmap v2742: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 118 MiB/s wr, 1.89k op/s; 34057/182526 objects degraded (18.659%); 8/182526 objects misplaced (0.004%) 2024-08-17T18:18:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 34057/182526 objects degraded (18.659%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:01.852 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:01.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:01.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:01.954 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:01.992 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:01.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:01.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:02.094 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:02.129 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:02.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:02.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:02.142 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 30 16 69104 69088 143.894 97.6875 0.00605815 0.00694504 2024-08-17T18:18:02.232 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:02.262 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:02.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:02.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:02.364 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:02.392 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:02.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:02.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:02.494 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:02.525 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:02.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:02.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:02.627 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:02.655 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:02.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:02.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:02.758 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:02.782 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:02.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:02.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:02.867 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:18:02.884 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:02.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:02.912 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:02.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:03.013 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:03.041 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:03.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:03.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:03.142 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 31 16 70758 70742 142.587 103.375 0.00644615 0.00700844 2024-08-17T18:18:03.143 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:03.173 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:03.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:03.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:03.274 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:03.318 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:03.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:03.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:03.366 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:18:03.420 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:03.465 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:03.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:03.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:03.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:03.598 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:03.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:03.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:03.629 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:03 smithi130 ceph-mon[105521]: pgmap v2743: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 115 MiB/s wr, 1.85k op/s; 35103/188757 objects degraded (18.597%); 8/188757 objects misplaced (0.004%) 2024-08-17T18:18:03.629 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:03 smithi130 ceph-mon[110068]: pgmap v2743: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 115 MiB/s wr, 1.85k op/s; 35103/188757 objects degraded (18.597%); 8/188757 objects misplaced (0.004%) 2024-08-17T18:18:03.700 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:03.731 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:03.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:03.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:03.833 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:03.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:03 smithi179 ceph-mon[93519]: pgmap v2743: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 3.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 115 MiB/s wr, 1.85k op/s; 35103/188757 objects degraded (18.597%); 8/188757 objects misplaced (0.004%) 2024-08-17T18:18:03.861 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:03.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:03.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:03.963 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:03.987 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:03.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:03.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:04.089 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:04.121 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:04.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:04.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:04.142 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 32 16 72331 72315 141.203 98.3125 0.00912128 0.00707766 2024-08-17T18:18:04.223 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:04.261 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:04.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:04.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:04.363 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:04.394 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:04.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:04.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:04.496 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:04.526 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:04.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:04.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:04.628 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:04.667 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:04.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:04.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:04.769 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:04.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:04.800 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:04.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:04.901 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:04.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:04.950 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:04.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:05.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:05.081 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:05.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:05.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:05.142 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 33 16 73888 73872 139.873 97.3125 0.00335239 0.00713836 2024-08-17T18:18:05.182 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:05.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:05.204 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:05.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:05.306 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:05.333 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:05.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:05.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:05.435 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:05.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:05.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:05.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:05.539 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:18:05.539 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T18:18:05.539 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T18:18:05.539 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T18:18:05.539 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:18:05.564 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:05.605 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:05.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:05.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:05.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:05 smithi130 ceph-mon[105521]: pgmap v2744: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s; 38676/207174 objects degraded (18.668%); 8/207174 objects misplaced (0.004%) 2024-08-17T18:18:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:05 smithi130 ceph-mon[110068]: pgmap v2744: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s; 38676/207174 objects degraded (18.668%); 8/207174 objects misplaced (0.004%) 2024-08-17T18:18:05.707 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:05.747 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:05.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:05.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:05.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:05 smithi179 ceph-mon[93519]: pgmap v2744: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.2 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s; 38676/207174 objects degraded (18.668%); 8/207174 objects misplaced (0.004%) 2024-08-17T18:18:05.849 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:05.881 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:05.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:05.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:05.983 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:06.009 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:06.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:06.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:06.111 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:06.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:06.141 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:06.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:06.143 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 34 16 75253 75237 138.267 85.3125 0.00977028 0.00722819 2024-08-17T18:18:06.242 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:06.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:06.278 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:06.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:06.379 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:06.407 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:06.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:06.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:06.509 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:06.546 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:06.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:06.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:06.648 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:06.678 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:06.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:06.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:06.780 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:06.808 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:06.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:06.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:06.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 38676/207174 objects degraded (18.668%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:06.910 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:06.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 38676/207174 objects degraded (18.668%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:06.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 38676/207174 objects degraded (18.668%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:06.938 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:06.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:06.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:07.040 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:07.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:07.078 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:07.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:07.143 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 35 16 76868 76852 137.2 100.938 0.0227199 0.0072841 2024-08-17T18:18:07.179 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:07.209 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:07.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:07.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:07.310 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:07.350 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:07.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:07.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:07.452 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:07.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:07.532 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:07.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:07.633 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:07.662 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:07.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:07.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:07.764 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:07.795 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 7, 1, 6] out_osds: [5, 4] dead_osds: [2] live_osds: [1, 4, 5, 6, 7, 0, 3] 2024-08-17T18:18:07.796 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:18:07.796 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2024-08-17T18:18:07.796 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:18:07.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:07.799 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:07.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:07.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:07 smithi179 ceph-mon[93519]: pgmap v2745: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 103 MiB/s wr, 1.65k op/s; 39670/213072 objects degraded (18.618%); 8/213072 objects misplaced (0.004%) 2024-08-17T18:18:07.881 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:07 smithi130 ceph-mon[105521]: pgmap v2745: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 103 MiB/s wr, 1.65k op/s; 39670/213072 objects degraded (18.618%); 8/213072 objects misplaced (0.004%) 2024-08-17T18:18:07.882 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:07 smithi130 ceph-mon[110068]: pgmap v2745: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.3 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 103 MiB/s wr, 1.65k op/s; 39670/213072 objects degraded (18.618%); 8/213072 objects misplaced (0.004%) 2024-08-17T18:18:07.900 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:07.927 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:07.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:07.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:08.029 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:08.069 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:08.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:08.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:08.143 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 36 16 78190 78174 135.684 82.625 0.0186953 0.0073651 2024-08-17T18:18:08.171 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:08.200 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:08.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:08.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:08.301 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:08.322 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:08.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:08.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:08.424 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:08.454 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:08.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:08.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:08.556 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:08.579 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:08.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:08.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:08.681 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:08.703 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:08.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:08.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:08.805 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:08.832 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:08.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:08.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:08.934 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:08.965 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:08.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:08.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:09.067 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:09.125 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:09.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:09.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:09.143 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 37 16 79481 79465 134.198 80.6875 0.00476352 0.00744698 2024-08-17T18:18:09.228 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:09.255 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:09.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:09.357 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:09.392 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:09.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:09.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:09.494 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:09.525 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:09.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:09.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:09.627 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:09.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:09.666 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:09.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:09.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:09 smithi130 ceph-mon[105521]: pgmap v2746: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 93 MiB/s wr, 1.48k op/s; 40160/215955 objects degraded (18.596%); 8/215955 objects misplaced (0.004%) 2024-08-17T18:18:09.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:09 smithi130 ceph-mon[110068]: pgmap v2746: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 93 MiB/s wr, 1.48k op/s; 40160/215955 objects degraded (18.596%); 8/215955 objects misplaced (0.004%) 2024-08-17T18:18:09.767 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:09.794 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:09.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:09.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:09 smithi179 ceph-mon[93519]: pgmap v2746: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.4 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 93 MiB/s wr, 1.48k op/s; 40160/215955 objects degraded (18.596%); 8/215955 objects misplaced (0.004%) 2024-08-17T18:18:09.897 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:09.931 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:09.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:09.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:10.034 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:10.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:10.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:10.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:10.143 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 38 16 80928 80912 133.046 90.4375 0.0107657 0.00751167 2024-08-17T18:18:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:18:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:18:09] "GET /metrics HTTP/1.1" 200 36783 "" "Prometheus/2.51.0" 2024-08-17T18:18:10.168 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:10.205 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:10.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:10.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:10.307 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:10.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:10.345 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:10.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:10.446 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:10.476 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:10.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:10.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:10.578 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:10.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:18:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:18:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:18:10.613 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:10.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:10.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:10.714 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:10.750 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:10.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:10.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:10.852 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:10.882 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:10.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:10.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:10.984 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:11.014 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:11.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:11.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:11.116 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:11.144 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 39 16 82353 82337 131.918 89.0625 0.00871933 0.00757561 2024-08-17T18:18:11.145 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:11.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:11.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:11.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:11.298 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:11.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:11.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:11.400 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:11.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:11.444 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:11.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:11.497 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:11 smithi130 ceph-mon[110068]: pgmap v2747: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.7 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 111 MiB/s wr, 1.77k op/s; 42886/229812 objects degraded (18.661%); 8/229812 objects misplaced (0.003%) 2024-08-17T18:18:11.497 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 42886/229812 objects degraded (18.661%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:11.498 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:11 smithi130 ceph-mon[105521]: pgmap v2747: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.7 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 111 MiB/s wr, 1.77k op/s; 42886/229812 objects degraded (18.661%); 8/229812 objects misplaced (0.003%) 2024-08-17T18:18:11.498 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 42886/229812 objects degraded (18.661%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:11.545 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:11.573 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:11.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:11.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:11.675 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:11.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:11.703 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:11.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:11.805 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:11 smithi179 ceph-mon[93519]: pgmap v2747: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.7 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 111 MiB/s wr, 1.77k op/s; 42886/229812 objects degraded (18.661%); 8/229812 objects misplaced (0.003%) 2024-08-17T18:18:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 42886/229812 objects degraded (18.661%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:11.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:11.848 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:11.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:11.949 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:11.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:11.978 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:11.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:12.079 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:12.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:12.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:12.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:12.144 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:18:12.143966+0000 min lat: 0.00107633 max lat: 0.145068 avg lat: 0.00765672 2024-08-17T18:18:12.144 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:18:12.144 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 40 16 83572 83556 130.525 76.1875 0.00692571 0.00765672 2024-08-17T18:18:12.209 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:12.249 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:12.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:12.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:12.351 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:12.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:12.390 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:12.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:12.491 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:12.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:12.520 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:12.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:12.589 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:18:12.589 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:18:12.589 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:18:12.590 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:18:12.621 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:12.627 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:18:12.627 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:18:12.627 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:18:12.650 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:12.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:12.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:12.658 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:18:12.659 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:18:12.688 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:18:12.688 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:18:12.717 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:18:12.717 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:18:12.746 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:18:12.747 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:18:12.751 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:12.774 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:18:12.774 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:18:12.777 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:12.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:12.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:18:12.878 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:12.907 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:12.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:12.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:13.009 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:13.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:13.039 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:13.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:13.140 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:13.143 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 41 16 84774 84758 129.174 75.125 0.0264438 0.00773603 2024-08-17T18:18:13.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:13.169 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:13.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:13.270 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:13.297 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:13.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:13.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:13.398 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:13.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:13.428 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:13.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:13.529 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:13.573 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:13.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:13.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:13.675 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:13.700 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:13.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:13.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:13.716 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T18:18:13.751 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:13 smithi130 ceph-mon[110068]: pgmap v2748: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 91 MiB/s wr, 1.46k op/s; 43770/235179 objects degraded (18.611%); 8/235179 objects misplaced (0.003%) 2024-08-17T18:18:13.751 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:13 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1757929814' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:18:13.752 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:13 smithi130 ceph-mon[105521]: pgmap v2748: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 91 MiB/s wr, 1.46k op/s; 43770/235179 objects degraded (18.611%); 8/235179 objects misplaced (0.003%) 2024-08-17T18:18:13.752 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:13 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1757929814' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:18:13.801 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:13 smithi179 ceph-mon[93519]: pgmap v2748: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 4.8 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 91 MiB/s wr, 1.46k op/s; 43770/235179 objects degraded (18.611%); 8/235179 objects misplaced (0.003%) 2024-08-17T18:18:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:13 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1757929814' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:18:13.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:13.853 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:13.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:13.954 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:13.979 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:13.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:13.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:14.080 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:14.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:14.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:14.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:14.145 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 42 16 85978 85962 127.885 75.25 0.0079813 0.0078149 2024-08-17T18:18:14.209 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:14.236 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:14.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:14.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:14.338 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:14.376 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:14.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:14.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:14.478 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:14.506 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:14.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:14.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:14.607 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:14.637 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:14.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:14.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:14.739 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:14.768 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:14.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:14.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:14.805 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:18:14.806 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:18:14.869 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:14.906 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:14.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:14.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:15.008 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:15.034 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:15.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:15.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:15.136 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:15.145 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 43 16 87405 87389 126.985 89.1875 0.0113612 0.0078707 2024-08-17T18:18:15.164 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:15.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:15.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:15.266 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:15.311 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:15.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:15.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:15.413 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:15.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:15.460 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:15.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:15.561 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:15.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:15.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:15.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:15.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:15 smithi130 ceph-mon[105521]: pgmap v2749: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 107 MiB/s wr, 1.71k op/s; 46732/250173 objects degraded (18.680%); 8/250173 objects misplaced (0.003%) 2024-08-17T18:18:15.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:15 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:15.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:15 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:15.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:15 smithi130 ceph-mon[110068]: pgmap v2749: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 107 MiB/s wr, 1.71k op/s; 46732/250173 objects degraded (18.680%); 8/250173 objects misplaced (0.003%) 2024-08-17T18:18:15.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:15 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:15.670 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:15 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:15.687 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:15.710 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:15.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:15.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:15.756 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:18:15.767 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:18:15.812 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:15.836 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:15.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:15.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:15.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:15 smithi179 ceph-mon[93519]: pgmap v2749: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 107 MiB/s wr, 1.71k op/s; 46732/250173 objects degraded (18.680%); 8/250173 objects misplaced (0.003%) 2024-08-17T18:18:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:15 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:15 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:15.937 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:15.964 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:15.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:15.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:16.065 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:16.104 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:16.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:16.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:16.146 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 44 16 88484 88468 125.632 67.4375 0.0137341 0.00795415 2024-08-17T18:18:16.205 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:16.233 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:16.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:16.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:16.334 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:16.375 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:16.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:16.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:16.477 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:16.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:16.507 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:16.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:16.608 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:16.643 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:16.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:16.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:18:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:18:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 46732/250173 objects degraded (18.680%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:16 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1631359013' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:18:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:16 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:18:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:18:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:18:16.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:16.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 46732/250173 objects degraded (18.680%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:16.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:16 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1631359013' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:18:16.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:16 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:18:16.744 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:16.791 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:16.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:16.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:16.807 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:18:16.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:18:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:18:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 46732/250173 objects degraded (18.680%), 11 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:16 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1631359013' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:18:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:16 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:18:16.893 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:16.919 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:16.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:16.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:16.988 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:18:17.021 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:17.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:17.074 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:17.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:17.146 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 45 16 89799 89783 124.666 82.1875 0.0181776 0.00801664 2024-08-17T18:18:17.175 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:17.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:17.213 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:17.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:17.314 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:17.345 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:17.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:17.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:17.447 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:17.485 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:17.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:17.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:17.586 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:17.616 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:17.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:17.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:17.717 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:17.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:17.746 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:17.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:17 smithi179 ceph-mon[93519]: pgmap v2750: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s; 47603/255261 objects degraded (18.649%); 8/255261 objects misplaced (0.003%) 2024-08-17T18:18:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:17 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:18:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:17 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:18:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:17 smithi179 ceph-mon[93519]: osdmap e1441: 8 total, 7 up, 6 in 2024-08-17T18:18:17.847 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:17.875 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:17.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:17.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:17 smithi130 ceph-mon[105521]: pgmap v2750: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s; 47603/255261 objects degraded (18.649%); 8/255261 objects misplaced (0.003%) 2024-08-17T18:18:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:17 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:18:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:17 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:18:17.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:17 smithi130 ceph-mon[105521]: osdmap e1441: 8 total, 7 up, 6 in 2024-08-17T18:18:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:17 smithi130 ceph-mon[110068]: pgmap v2750: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 83 MiB/s wr, 1.33k op/s; 47603/255261 objects degraded (18.649%); 8/255261 objects misplaced (0.003%) 2024-08-17T18:18:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:17 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:18:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:17 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:18:17.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:17 smithi130 ceph-mon[110068]: osdmap e1441: 8 total, 7 up, 6 in 2024-08-17T18:18:17.977 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:18.010 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:18.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:18.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:18.112 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:18.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:18.146 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:18.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:18.146 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 46 16 91152 91136 123.794 84.5625 0.00462536 0.00807332 2024-08-17T18:18:18.247 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:18.279 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:18.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:18.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:18.381 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:18.408 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:18.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:18.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:18.510 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:18.542 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:18.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:18.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:18.644 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:18.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:18 smithi130 ceph-mon[105521]: pgmap v2752: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 47998/257652 objects degraded (18.629%); 8/257652 objects misplaced (0.003%) 2024-08-17T18:18:18.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:18 smithi130 ceph-mon[110068]: pgmap v2752: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 47998/257652 objects degraded (18.629%); 8/257652 objects misplaced (0.003%) 2024-08-17T18:18:18.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:18.687 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:18.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:18.788 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:18.823 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:18.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:18.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:18.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:18 smithi179 ceph-mon[93519]: pgmap v2752: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.2 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s; 47998/257652 objects degraded (18.629%); 8/257652 objects misplaced (0.003%) 2024-08-17T18:18:18.924 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:18.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:18.955 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:18.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:19.056 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:19.086 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:19.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:19.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:19.146 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 47 16 92520 92504 122.979 85.5 0.008042 0.00812683 2024-08-17T18:18:19.188 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:19.233 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:19.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:19.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:19.335 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:19.363 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:19.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:19.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:19.465 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:19.492 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:19.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:19.502 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:18:19.502 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:18:19.594 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:19.641 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:19.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:19.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:19.743 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:19.771 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:18:19.783 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:19.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:19.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:19.886 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:19.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:19.927 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:19.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:20.028 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:20.056 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:20.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:20.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:20.147 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 48 16 93839 93823 122.135 82.4375 0.00970676 0.00818315 2024-08-17T18:18:20.158 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:18:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:18:19] "GET /metrics HTTP/1.1" 200 36785 "" "Prometheus/2.51.0" 2024-08-17T18:18:20.186 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:20.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:20.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:20.288 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:20.318 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:20.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:20.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:20.420 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:20.422 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T18:18:20.454 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:20.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:20.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:20.468 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:18:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:18:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:18:20.556 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:20.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:20.590 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:20.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:20.691 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:20.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:20.728 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:20.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:20.829 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:20 smithi179 ceph-mon[93519]: pgmap v2753: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 82 MiB/s wr, 1.32k op/s; 50324/269415 objects degraded (18.679%); 8/269415 objects misplaced (0.003%) 2024-08-17T18:18:20.869 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:20.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:20.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:20.906 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:18:20.913 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:20 smithi130 ceph-mon[105521]: pgmap v2753: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 82 MiB/s wr, 1.32k op/s; 50324/269415 objects degraded (18.679%); 8/269415 objects misplaced (0.003%) 2024-08-17T18:18:20.913 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:20 smithi130 ceph-mon[110068]: pgmap v2753: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.5 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 82 MiB/s wr, 1.32k op/s; 50324/269415 objects degraded (18.679%); 8/269415 objects misplaced (0.003%) 2024-08-17T18:18:20.971 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:21.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:21.016 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:21.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:21.052 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:18:21.117 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:21.148 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 49 16 95107 95091 121.254 79.25 0.0177201 0.00824199 2024-08-17T18:18:21.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:21.161 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:21.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:21.262 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:21.292 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:21.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:21.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:21.394 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:21.423 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:21.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:21.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:21.525 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:21.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:21.559 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:21.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:21.583 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:18:21.587 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:18:21.660 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:21.698 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:21.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:21.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:21.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:21.832 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:21.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:21.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:21.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 50324/269415 objects degraded (18.679%), 11 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:21.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 50324/269415 objects degraded (18.679%), 11 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 50324/269415 objects degraded (18.679%), 11 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:21.934 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:21.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:21.963 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:21.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:22.064 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:22.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:22.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:22.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:22.148 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 50 16 96356 96340 120.39 78.0625 0.00795194 0.00830173 2024-08-17T18:18:22.198 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:22.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:22.231 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:22.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:22.332 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:22.361 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:22.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:22.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:22.463 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:22.495 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:22.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:22.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:22.597 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:22.641 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:22.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:22.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:22.742 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:22.773 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:22.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:22.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:22.813 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:22 smithi179 ceph-mon[93519]: from='client.37221 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:18:22.813 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:22 smithi179 ceph-mon[93519]: pgmap v2754: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s; 51131/274272 objects degraded (18.642%); 8/274272 objects misplaced (0.003%) 2024-08-17T18:18:22.875 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:22.878 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2753,"stamp":"2024-08-17T18:18:19.611243+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5884352461,"num_objects":89805,"num_object_clones":0,"num_object_copies":269415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":50324,"num_objects_misplaced":8,"num_objects_unfound":0,"num_objects_dirty":89805,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":90057,"num_write_kb":5748303,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":4149043,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89986,"ondisk_log_size":89986,"up":232,"acting":244,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":289,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":21792400,"kb_used_data":15711832,"kb_used_omap":350,"kb_used_meta":6080097,"kb_avail":540678512,"statfs":{"total":575970213888,"available":553654796288,"internally_reserved":0,"allocated":16088915968,"data_stored":16058939967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":359385,"internal_metadata":6226019367},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":865140736,"num_objects":13201,"num_object_clones":0,"num_object_copies":39603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7438,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13201,"num_write_kb":844864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006291"},"pg_stats":[{"pgid":"50.f","version":"1441'5638","reported_seq":5653,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:18.238561+0000","last_change":"2024-08-17T18:17:19.384830+0000","last_active":"2024-08-17T18:18:18.238561+0000","last_peered":"2024-08-17T18:18:18.238561+0000","last_clean":"2024-08-17T18:18:18.238561+0000","last_became_active":"2024-08-17T18:17:19.384515+0000","last_became_peered":"2024-08-17T18:17:19.384515+0000","last_unstale":"2024-08-17T18:18:18.238561+0000","last_undegraded":"2024-08-17T18:18:18.238561+0000","last_fullsized":"2024-08-17T18:18:18.238561+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":1435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5638,"log_dups_size":0,"ondisk_log_size":5638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:15:39.468947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369491968,"num_objects":5638,"num_object_clones":0,"num_object_copies":16914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5638,"num_write_kb":360832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"50.e","version":"1441'5611","reported_seq":5623,"reported_epoch":1441,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:18:17.882963+0000","last_change":"2024-08-17T18:17:32.145360+0000","last_active":"2024-08-17T18:18:17.882963+0000","last_peered":"2024-08-17T18:18:17.882963+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:17:19.382860+0000","last_became_peered":"2024-08-17T18:17:19.382860+0000","last_unstale":"2024-08-17T18:18:17.882963+0000","last_undegraded":"2024-08-17T18:17:32.142638+0000","last_fullsized":"2024-08-17T18:17:19.380093+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5611,"log_dups_size":0,"ondisk_log_size":5611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:44:09.028710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367722496,"num_objects":5611,"num_object_clones":0,"num_object_copies":16833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5611,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5611,"num_write_kb":359104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":5611}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"50.d","version":"1440'5374","reported_seq":5394,"reported_epoch":1440,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:18:14.701446+0000","last_change":"2024-08-17T18:17:32.162669+0000","last_active":"2024-08-17T18:18:14.701446+0000","last_peered":"2024-08-17T18:18:14.701446+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:17:19.384283+0000","last_became_peered":"2024-08-17T18:17:19.384283+0000","last_unstale":"2024-08-17T18:18:14.701446+0000","last_undegraded":"2024-08-17T18:17:32.149264+0000","last_fullsized":"2024-08-17T18:17:19.380544+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5374,"log_dups_size":0,"ondisk_log_size":5374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:54:41.887432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352190464,"num_objects":5374,"num_object_clones":0,"num_object_copies":16122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5374,"num_write_kb":343936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[{"shards":"0,3","objects":5374}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"50.c","version":"1441'5714","reported_seq":5729,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:17.978566+0000","last_change":"2024-08-17T18:17:19.384247+0000","last_active":"2024-08-17T18:18:17.978566+0000","last_peered":"2024-08-17T18:18:17.978566+0000","last_clean":"2024-08-17T18:18:17.978566+0000","last_became_active":"2024-08-17T18:17:19.383749+0000","last_became_peered":"2024-08-17T18:17:19.383749+0000","last_unstale":"2024-08-17T18:18:17.978566+0000","last_undegraded":"2024-08-17T18:18:17.978566+0000","last_fullsized":"2024-08-17T18:18:17.978566+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":1435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5714,"log_dups_size":0,"ondisk_log_size":5714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:28.983167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374472704,"num_objects":5714,"num_object_clones":0,"num_object_copies":17142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5714,"num_write_kb":365696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"50.b","version":"1441'5827","reported_seq":5842,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:18.238322+0000","last_change":"2024-08-17T18:17:19.385042+0000","last_active":"2024-08-17T18:18:18.238322+0000","last_peered":"2024-08-17T18:18:18.238322+0000","last_clean":"2024-08-17T18:18:18.238322+0000","last_became_active":"2024-08-17T18:17:19.384703+0000","last_became_peered":"2024-08-17T18:17:19.384703+0000","last_unstale":"2024-08-17T18:18:18.238322+0000","last_undegraded":"2024-08-17T18:18:18.238322+0000","last_fullsized":"2024-08-17T18:18:18.238322+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":1435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5827,"log_dups_size":0,"ondisk_log_size":5827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:22:33.611833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381878272,"num_objects":5827,"num_object_clones":0,"num_object_copies":17481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5827,"num_write_kb":372928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"50.a","version":"1440'5476","reported_seq":5499,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:18:14.677646+0000","last_change":"2024-08-17T18:17:19.386502+0000","last_active":"2024-08-17T18:18:14.677646+0000","last_peered":"2024-08-17T18:18:14.677646+0000","last_clean":"2024-08-17T18:18:14.677646+0000","last_became_active":"2024-08-17T18:17:19.386207+0000","last_became_peered":"2024-08-17T18:17:19.386207+0000","last_unstale":"2024-08-17T18:18:14.677646+0000","last_undegraded":"2024-08-17T18:18:14.677646+0000","last_fullsized":"2024-08-17T18:18:14.677646+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":1435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5476,"log_dups_size":0,"ondisk_log_size":5476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:34.892939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358875136,"num_objects":5476,"num_object_clones":0,"num_object_copies":16428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5476,"num_write_kb":350464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"50.9","version":"1441'5674","reported_seq":5689,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:18.010037+0000","last_change":"2024-08-17T18:17:19.384518+0000","last_active":"2024-08-17T18:18:18.010037+0000","last_peered":"2024-08-17T18:18:18.010037+0000","last_clean":"2024-08-17T18:18:18.010037+0000","last_became_active":"2024-08-17T18:17:19.383673+0000","last_became_peered":"2024-08-17T18:17:19.383673+0000","last_unstale":"2024-08-17T18:18:18.010037+0000","last_undegraded":"2024-08-17T18:18:18.010037+0000","last_fullsized":"2024-08-17T18:18:18.010037+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":1435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5674,"log_dups_size":0,"ondisk_log_size":5674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:22:15.140596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371851264,"num_objects":5674,"num_object_clones":0,"num_object_copies":17022,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5674,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5674,"num_write_kb":363136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"50.8","version":"1441'5625","reported_seq":5637,"reported_epoch":1441,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:18:17.983235+0000","last_change":"2024-08-17T18:17:32.142070+0000","last_active":"2024-08-17T18:18:17.983235+0000","last_peered":"2024-08-17T18:18:17.983235+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:17:19.382385+0000","last_became_peered":"2024-08-17T18:17:19.382385+0000","last_unstale":"2024-08-17T18:18:17.983235+0000","last_undegraded":"2024-08-17T18:17:32.138639+0000","last_fullsized":"2024-08-17T18:17:19.379952+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5625,"log_dups_size":0,"ondisk_log_size":5625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:33:36.309533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368640000,"num_objects":5625,"num_object_clones":0,"num_object_copies":16875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5625,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5625,"num_write_kb":360000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":5625}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"50.7","version":"1441'5628","reported_seq":5640,"reported_epoch":1441,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:18:18.251535+0000","last_change":"2024-08-17T18:17:32.140070+0000","last_active":"2024-08-17T18:18:18.251535+0000","last_peered":"2024-08-17T18:18:18.251535+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:17:19.385158+0000","last_became_peered":"2024-08-17T18:17:19.385158+0000","last_unstale":"2024-08-17T18:18:18.251535+0000","last_undegraded":"2024-08-17T18:17:32.139911+0000","last_fullsized":"2024-08-17T18:17:19.381948+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5628,"log_dups_size":0,"ondisk_log_size":5628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:35.269970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368836608,"num_objects":5628,"num_object_clones":0,"num_object_copies":16884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5628,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5628,"num_write_kb":360192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":5628}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"50.6","version":"1441'5656","reported_seq":5668,"reported_epoch":1441,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:18:18.250550+0000","last_change":"2024-08-17T18:17:32.139599+0000","last_active":"2024-08-17T18:18:18.250550+0000","last_peered":"2024-08-17T18:18:18.250550+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:17:19.385385+0000","last_became_peered":"2024-08-17T18:17:19.385385+0000","last_unstale":"2024-08-17T18:18:18.250550+0000","last_undegraded":"2024-08-17T18:17:32.138301+0000","last_fullsized":"2024-08-17T18:17:19.381291+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5656,"log_dups_size":0,"ondisk_log_size":5656,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:44.037472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370671616,"num_objects":5656,"num_object_clones":0,"num_object_copies":16968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5656,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5656,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":5656}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"50.5","version":"1440'5596","reported_seq":5616,"reported_epoch":1440,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:18:16.691197+0000","last_change":"2024-08-17T18:17:32.145421+0000","last_active":"2024-08-17T18:18:16.691197+0000","last_peered":"2024-08-17T18:18:16.691197+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:17:19.381775+0000","last_became_peered":"2024-08-17T18:17:19.381775+0000","last_unstale":"2024-08-17T18:18:16.691197+0000","last_undegraded":"2024-08-17T18:17:32.144903+0000","last_fullsized":"2024-08-17T18:17:19.377753+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5596,"log_dups_size":0,"ondisk_log_size":5596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:01:26.647888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366739456,"num_objects":5596,"num_object_clones":0,"num_object_copies":16788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5596,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5596,"num_write_kb":358144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":5596}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"50.4","version":"1441'5743","reported_seq":5764,"reported_epoch":1441,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:18:17.961142+0000","last_change":"2024-08-17T18:17:32.142755+0000","last_active":"2024-08-17T18:18:17.961142+0000","last_peered":"2024-08-17T18:18:17.961142+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:17:19.383734+0000","last_became_peered":"2024-08-17T18:17:19.383734+0000","last_unstale":"2024-08-17T18:18:17.961142+0000","last_undegraded":"2024-08-17T18:17:32.137407+0000","last_fullsized":"2024-08-17T18:17:19.373508+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5743,"log_dups_size":0,"ondisk_log_size":5743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:31:40.452433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376373248,"num_objects":5743,"num_object_clones":0,"num_object_copies":17229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5743,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5743,"num_write_kb":367552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":5743}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"50.3","version":"1441'5630","reported_seq":5652,"reported_epoch":1441,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:18:17.933458+0000","last_change":"2024-08-17T18:17:32.143238+0000","last_active":"2024-08-17T18:18:17.933458+0000","last_peered":"2024-08-17T18:18:17.933458+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:17:19.383205+0000","last_became_peered":"2024-08-17T18:17:19.383205+0000","last_unstale":"2024-08-17T18:18:17.933458+0000","last_undegraded":"2024-08-17T18:17:32.142664+0000","last_fullsized":"2024-08-17T18:17:19.373508+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5630,"log_dups_size":0,"ondisk_log_size":5630,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:39.481448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368967680,"num_objects":5630,"num_object_clones":0,"num_object_copies":16890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5630,"num_write_kb":360320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":5630}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"50.2","version":"1440'5622","reported_seq":5645,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:18:16.686870+0000","last_change":"2024-08-17T18:17:19.384865+0000","last_active":"2024-08-17T18:18:16.686870+0000","last_peered":"2024-08-17T18:18:16.686870+0000","last_clean":"2024-08-17T18:18:16.686870+0000","last_became_active":"2024-08-17T18:17:19.384532+0000","last_became_peered":"2024-08-17T18:17:19.384532+0000","last_unstale":"2024-08-17T18:18:16.686870+0000","last_undegraded":"2024-08-17T18:18:16.686870+0000","last_fullsized":"2024-08-17T18:18:16.686870+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":1435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5622,"log_dups_size":0,"ondisk_log_size":5622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:42.808268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368443392,"num_objects":5622,"num_object_clones":0,"num_object_copies":16866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5622,"num_write_kb":359808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"50.1","version":"1440'5506","reported_seq":5529,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:18:14.668737+0000","last_change":"2024-08-17T18:17:19.384889+0000","last_active":"2024-08-17T18:18:14.668737+0000","last_peered":"2024-08-17T18:18:14.668737+0000","last_clean":"2024-08-17T18:18:14.668737+0000","last_became_active":"2024-08-17T18:17:19.384602+0000","last_became_peered":"2024-08-17T18:17:19.384602+0000","last_unstale":"2024-08-17T18:18:14.668737+0000","last_undegraded":"2024-08-17T18:18:14.668737+0000","last_fullsized":"2024-08-17T18:18:14.668737+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":1435,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5506,"log_dups_size":0,"ondisk_log_size":5506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:54:25.310389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360841216,"num_objects":5506,"num_object_clones":0,"num_object_copies":16518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5506,"num_write_kb":352384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"50.0","version":"1440'5459","reported_seq":5479,"reported_epoch":1440,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:18:14.686308+0000","last_change":"2024-08-17T18:17:32.148016+0000","last_active":"2024-08-17T18:18:14.686308+0000","last_peered":"2024-08-17T18:18:14.686308+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:17:19.382402+0000","last_became_peered":"2024-08-17T18:17:19.382402+0000","last_unstale":"2024-08-17T18:18:14.686308+0000","last_undegraded":"2024-08-17T18:17:32.147838+0000","last_fullsized":"2024-08-17T18:17:19.379827+0000","mapping_epoch":1434,"log_start":"0'0","ondisk_log_start":"0'0","created":1434,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5459,"log_dups_size":0,"ondisk_log_size":5459,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:19:29.309617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357761024,"num_objects":5459,"num_object_clones":0,"num_object_copies":16377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5459,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5459,"num_write_kb":349376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":5459}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":2388,"reported_epoch":1440,"state":"active+clean+remapped","last_fresh":"2024-08-17T18:17:55.417015+0000","last_change":"2024-08-17T18:16:35.665645+0000","last_active":"2024-08-17T18:17:55.417015+0000","last_peered":"2024-08-17T18:17:55.417015+0000","last_clean":"2024-08-17T18:17:55.417015+0000","last_became_active":"2024-08-17T18:16:35.644929+0000","last_became_peered":"2024-08-17T18:16:35.644929+0000","last_unstale":"2024-08-17T18:17:55.417015+0000","last_undegraded":"2024-08-17T18:17:55.417015+0000","last_fullsized":"2024-08-17T18:17:55.417015+0000","mapping_epoch":1422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:15:21.011118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,3],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":29,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.404423+0000","last_change":"2024-08-17T18:16:34.646543+0000","last_active":"2024-08-17T18:17:55.404423+0000","last_peered":"2024-08-17T18:17:55.404423+0000","last_clean":"2024-08-17T18:17:55.404423+0000","last_became_active":"2024-08-17T18:16:34.646169+0000","last_became_peered":"2024-08-17T18:16:34.646169+0000","last_unstale":"2024-08-17T18:17:55.404423+0000","last_undegraded":"2024-08-17T18:17:55.404423+0000","last_fullsized":"2024-08-17T18:17:55.404423+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:17:40.371130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":2181,"reported_epoch":1441,"state":"active+clean+remapped","last_fresh":"2024-08-17T18:18:17.006287+0000","last_change":"2024-08-17T18:16:35.661471+0000","last_active":"2024-08-17T18:18:17.006287+0000","last_peered":"2024-08-17T18:18:17.006287+0000","last_clean":"2024-08-17T18:18:17.006287+0000","last_became_active":"2024-08-17T18:16:35.661003+0000","last_became_peered":"2024-08-17T18:16:35.661003+0000","last_unstale":"2024-08-17T18:18:17.006287+0000","last_undegraded":"2024-08-17T18:18:17.006287+0000","last_fullsized":"2024-08-17T18:18:17.006287+0000","mapping_epoch":1422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:41.250294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,1],"avail_no_missing":["3","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2224,"reported_epoch":1441,"state":"active+undersized","last_fresh":"2024-08-17T18:18:17.005748+0000","last_change":"2024-08-17T18:15:59.380510+0000","last_active":"2024-08-17T18:18:17.005748+0000","last_peered":"2024-08-17T18:18:17.005748+0000","last_clean":"2024-08-17T18:15:39.316148+0000","last_became_active":"2024-08-17T18:15:59.380510+0000","last_became_peered":"2024-08-17T18:15:59.380510+0000","last_unstale":"2024-08-17T18:18:17.005748+0000","last_undegraded":"2024-08-17T18:18:17.005748+0000","last_fullsized":"2024-08-17T18:15:59.370492+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:47:13.832651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":33,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.402560+0000","last_change":"2024-08-17T18:16:34.663280+0000","last_active":"2024-08-17T18:17:55.402560+0000","last_peered":"2024-08-17T18:17:55.402560+0000","last_clean":"2024-08-17T18:17:55.402560+0000","last_became_active":"2024-08-17T18:16:34.659095+0000","last_became_peered":"2024-08-17T18:16:34.659095+0000","last_unstale":"2024-08-17T18:17:55.402560+0000","last_undegraded":"2024-08-17T18:17:55.402560+0000","last_fullsized":"2024-08-17T18:17:55.402560+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:24:30.740039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":160,"reported_epoch":1440,"state":"active+undersized","last_fresh":"2024-08-17T18:17:55.405452+0000","last_change":"2024-08-17T18:15:59.369392+0000","last_active":"2024-08-17T18:17:55.405452+0000","last_peered":"2024-08-17T18:17:55.405452+0000","last_clean":"2024-08-17T18:15:39.323293+0000","last_became_active":"2024-08-17T18:15:59.369392+0000","last_became_peered":"2024-08-17T18:15:59.369392+0000","last_unstale":"2024-08-17T18:17:55.405452+0000","last_undegraded":"2024-08-17T18:17:55.405452+0000","last_fullsized":"2024-08-17T18:15:59.363378+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:22.398498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":120,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.409080+0000","last_change":"2024-08-17T18:16:34.660835+0000","last_active":"2024-08-17T18:17:55.409080+0000","last_peered":"2024-08-17T18:17:55.409080+0000","last_clean":"2024-08-17T18:17:55.409080+0000","last_became_active":"2024-08-17T18:16:34.651622+0000","last_became_peered":"2024-08-17T18:16:34.651622+0000","last_unstale":"2024-08-17T18:17:55.409080+0000","last_undegraded":"2024-08-17T18:17:55.409080+0000","last_fullsized":"2024-08-17T18:17:55.409080+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:09.762908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1019,"reported_epoch":1441,"state":"active+undersized","last_fresh":"2024-08-17T18:18:17.007451+0000","last_change":"2024-08-17T18:15:59.390736+0000","last_active":"2024-08-17T18:18:17.007451+0000","last_peered":"2024-08-17T18:18:17.007451+0000","last_clean":"2024-08-17T18:15:39.316051+0000","last_became_active":"2024-08-17T18:15:59.390736+0000","last_became_peered":"2024-08-17T18:15:59.390736+0000","last_unstale":"2024-08-17T18:18:17.007451+0000","last_undegraded":"2024-08-17T18:18:17.007451+0000","last_fullsized":"2024-08-17T18:15:59.373180+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:57:03.772238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":2323,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.417126+0000","last_change":"2024-08-17T18:15:04.437252+0000","last_active":"2024-08-17T18:17:55.417126+0000","last_peered":"2024-08-17T18:17:55.417126+0000","last_clean":"2024-08-17T18:17:55.417126+0000","last_became_active":"2024-08-17T18:15:04.435092+0000","last_became_peered":"2024-08-17T18:15:04.435092+0000","last_unstale":"2024-08-17T18:17:55.417126+0000","last_undegraded":"2024-08-17T18:17:55.417126+0000","last_fullsized":"2024-08-17T18:17:55.417126+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:57.983669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":105,"reported_epoch":1440,"state":"active+undersized","last_fresh":"2024-08-17T18:17:55.408898+0000","last_change":"2024-08-17T18:15:59.486274+0000","last_active":"2024-08-17T18:17:55.408898+0000","last_peered":"2024-08-17T18:17:55.408898+0000","last_clean":"2024-08-17T18:15:39.327548+0000","last_became_active":"2024-08-17T18:15:59.486274+0000","last_became_peered":"2024-08-17T18:15:59.486274+0000","last_unstale":"2024-08-17T18:17:55.408898+0000","last_undegraded":"2024-08-17T18:17:55.408898+0000","last_fullsized":"2024-08-17T18:15:59.366115+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:02.372303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":105,"reported_epoch":1440,"state":"active+undersized","last_fresh":"2024-08-17T18:17:55.407712+0000","last_change":"2024-08-17T18:15:59.375880+0000","last_active":"2024-08-17T18:17:55.407712+0000","last_peered":"2024-08-17T18:17:55.407712+0000","last_clean":"2024-08-17T18:15:39.344912+0000","last_became_active":"2024-08-17T18:15:59.375880+0000","last_became_peered":"2024-08-17T18:15:59.375880+0000","last_unstale":"2024-08-17T18:17:55.407712+0000","last_undegraded":"2024-08-17T18:17:55.407712+0000","last_fullsized":"2024-08-17T18:15:59.365584+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:48:09.760258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2193,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:17.004133+0000","last_change":"2024-08-17T18:16:34.659923+0000","last_active":"2024-08-17T18:18:17.004133+0000","last_peered":"2024-08-17T18:18:17.004133+0000","last_clean":"2024-08-17T18:18:17.004133+0000","last_became_active":"2024-08-17T18:16:34.659285+0000","last_became_peered":"2024-08-17T18:16:34.659285+0000","last_unstale":"2024-08-17T18:18:17.004133+0000","last_undegraded":"2024-08-17T18:18:17.004133+0000","last_fullsized":"2024-08-17T18:18:17.004133+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:38.573213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":108,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.407359+0000","last_change":"2024-08-17T18:16:34.655726+0000","last_active":"2024-08-17T18:17:55.407359+0000","last_peered":"2024-08-17T18:17:55.407359+0000","last_clean":"2024-08-17T18:17:55.407359+0000","last_became_active":"2024-08-17T18:16:34.654917+0000","last_became_peered":"2024-08-17T18:16:34.654917+0000","last_unstale":"2024-08-17T18:17:55.407359+0000","last_undegraded":"2024-08-17T18:17:55.407359+0000","last_fullsized":"2024-08-17T18:17:55.407359+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:55:57.599896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2340,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.417576+0000","last_change":"2024-08-17T18:09:28.868303+0000","last_active":"2024-08-17T18:17:55.417576+0000","last_peered":"2024-08-17T18:17:55.417576+0000","last_clean":"2024-08-17T18:17:55.417576+0000","last_became_active":"2024-08-17T18:09:28.867989+0000","last_became_peered":"2024-08-17T18:09:28.867989+0000","last_unstale":"2024-08-17T18:17:55.417576+0000","last_undegraded":"2024-08-17T18:17:55.417576+0000","last_fullsized":"2024-08-17T18:17:55.417576+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:58:52.741685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2040,"reported_epoch":1440,"state":"active+undersized","last_fresh":"2024-08-17T18:17:55.418804+0000","last_change":"2024-08-17T18:15:59.376723+0000","last_active":"2024-08-17T18:17:55.418804+0000","last_peered":"2024-08-17T18:17:55.418804+0000","last_clean":"2024-08-17T18:15:39.321395+0000","last_became_active":"2024-08-17T18:15:59.376723+0000","last_became_peered":"2024-08-17T18:15:59.376723+0000","last_unstale":"2024-08-17T18:17:55.418804+0000","last_undegraded":"2024-08-17T18:17:55.418804+0000","last_fullsized":"2024-08-17T18:15:59.367823+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:27.886834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":29,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.402845+0000","last_change":"2024-08-17T18:16:34.655323+0000","last_active":"2024-08-17T18:17:55.402845+0000","last_peered":"2024-08-17T18:17:55.402845+0000","last_clean":"2024-08-17T18:17:55.402845+0000","last_became_active":"2024-08-17T18:16:34.654936+0000","last_became_peered":"2024-08-17T18:16:34.654936+0000","last_unstale":"2024-08-17T18:17:55.402845+0000","last_undegraded":"2024-08-17T18:17:55.402845+0000","last_fullsized":"2024-08-17T18:17:55.402845+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:09:15.976414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2187,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:17.003254+0000","last_change":"2024-08-17T18:16:34.645544+0000","last_active":"2024-08-17T18:18:17.003254+0000","last_peered":"2024-08-17T18:18:17.003254+0000","last_clean":"2024-08-17T18:18:17.003254+0000","last_became_active":"2024-08-17T18:16:34.645125+0000","last_became_peered":"2024-08-17T18:16:34.645125+0000","last_unstale":"2024-08-17T18:18:17.003254+0000","last_undegraded":"2024-08-17T18:18:17.003254+0000","last_fullsized":"2024-08-17T18:18:17.003254+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:53.156455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2359,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.417210+0000","last_change":"2024-08-17T18:16:34.647907+0000","last_active":"2024-08-17T18:17:55.417210+0000","last_peered":"2024-08-17T18:17:55.417210+0000","last_clean":"2024-08-17T18:17:55.417210+0000","last_became_active":"2024-08-17T18:16:34.647464+0000","last_became_peered":"2024-08-17T18:16:34.647464+0000","last_unstale":"2024-08-17T18:17:55.417210+0000","last_undegraded":"2024-08-17T18:17:55.417210+0000","last_fullsized":"2024-08-17T18:17:55.417210+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:49:20.345586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2135,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:17.005309+0000","last_change":"2024-08-17T18:16:34.644366+0000","last_active":"2024-08-17T18:18:17.005309+0000","last_peered":"2024-08-17T18:18:17.005309+0000","last_clean":"2024-08-17T18:18:17.005309+0000","last_became_active":"2024-08-17T18:16:34.643994+0000","last_became_peered":"2024-08-17T18:16:34.643994+0000","last_unstale":"2024-08-17T18:18:17.005309+0000","last_undegraded":"2024-08-17T18:18:17.005309+0000","last_fullsized":"2024-08-17T18:18:17.005309+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:51.953343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":775,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.405660+0000","last_change":"2024-08-17T18:15:04.433108+0000","last_active":"2024-08-17T18:17:55.405660+0000","last_peered":"2024-08-17T18:17:55.405660+0000","last_clean":"2024-08-17T18:17:55.405660+0000","last_became_active":"2024-08-17T18:15:04.432766+0000","last_became_peered":"2024-08-17T18:15:04.432766+0000","last_unstale":"2024-08-17T18:17:55.405660+0000","last_undegraded":"2024-08-17T18:17:55.405660+0000","last_fullsized":"2024-08-17T18:17:55.405660+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:38.299190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1428,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.405690+0000","last_change":"2024-08-17T18:09:28.884262+0000","last_active":"2024-08-17T18:17:55.405690+0000","last_peered":"2024-08-17T18:17:55.405690+0000","last_clean":"2024-08-17T18:17:55.405690+0000","last_became_active":"2024-08-17T18:09:28.883734+0000","last_became_peered":"2024-08-17T18:09:28.883734+0000","last_unstale":"2024-08-17T18:17:55.405690+0000","last_undegraded":"2024-08-17T18:17:55.405690+0000","last_fullsized":"2024-08-17T18:17:55.405690+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:10.735504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":111,"reported_epoch":1440,"state":"active+clean+remapped","last_fresh":"2024-08-17T18:17:55.407545+0000","last_change":"2024-08-17T18:16:35.644804+0000","last_active":"2024-08-17T18:17:55.407545+0000","last_peered":"2024-08-17T18:17:55.407545+0000","last_clean":"2024-08-17T18:17:55.407545+0000","last_became_active":"2024-08-17T18:16:35.644110+0000","last_became_peered":"2024-08-17T18:16:35.644110+0000","last_unstale":"2024-08-17T18:17:55.407545+0000","last_undegraded":"2024-08-17T18:17:55.407545+0000","last_fullsized":"2024-08-17T18:17:55.407545+0000","mapping_epoch":1422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:07:42.832047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,0],"avail_no_missing":["1","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":621,"reported_epoch":1440,"state":"active+clean+remapped","last_fresh":"2024-08-17T18:17:55.405369+0000","last_change":"2024-08-17T18:16:35.650645+0000","last_active":"2024-08-17T18:17:55.405369+0000","last_peered":"2024-08-17T18:17:55.405369+0000","last_clean":"2024-08-17T18:17:55.405369+0000","last_became_active":"2024-08-17T18:16:35.641796+0000","last_became_peered":"2024-08-17T18:16:35.641796+0000","last_unstale":"2024-08-17T18:17:55.405369+0000","last_undegraded":"2024-08-17T18:17:55.405369+0000","last_fullsized":"2024-08-17T18:17:55.405369+0000","mapping_epoch":1422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:00:45.652684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14232,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6,0],"avail_no_missing":["7","0","6"],"object_location_counts":[{"shards":"0,6,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":31,"reported_epoch":1440,"state":"active+clean+remapped","last_fresh":"2024-08-17T18:17:55.402317+0000","last_change":"2024-08-17T18:16:35.643801+0000","last_active":"2024-08-17T18:17:55.402317+0000","last_peered":"2024-08-17T18:17:55.402317+0000","last_clean":"2024-08-17T18:17:55.402317+0000","last_became_active":"2024-08-17T18:16:35.643397+0000","last_became_peered":"2024-08-17T18:16:35.643397+0000","last_unstale":"2024-08-17T18:17:55.402317+0000","last_undegraded":"2024-08-17T18:17:55.402317+0000","last_fullsized":"2024-08-17T18:17:55.402317+0000","mapping_epoch":1422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:12.411295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,7],"avail_no_missing":["6","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":29,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.403708+0000","last_change":"2024-08-17T18:16:34.642605+0000","last_active":"2024-08-17T18:17:55.403708+0000","last_peered":"2024-08-17T18:17:55.403708+0000","last_clean":"2024-08-17T18:17:55.403708+0000","last_became_active":"2024-08-17T18:16:34.642086+0000","last_became_peered":"2024-08-17T18:16:34.642086+0000","last_unstale":"2024-08-17T18:17:55.403708+0000","last_undegraded":"2024-08-17T18:17:55.403708+0000","last_fullsized":"2024-08-17T18:17:55.403708+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:39:01.107809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":31,"reported_epoch":1440,"state":"active+clean+remapped","last_fresh":"2024-08-17T18:17:55.402911+0000","last_change":"2024-08-17T18:16:35.638951+0000","last_active":"2024-08-17T18:17:55.402911+0000","last_peered":"2024-08-17T18:17:55.402911+0000","last_clean":"2024-08-17T18:17:55.402911+0000","last_became_active":"2024-08-17T18:16:35.638504+0000","last_became_peered":"2024-08-17T18:16:35.638504+0000","last_unstale":"2024-08-17T18:17:55.402911+0000","last_undegraded":"2024-08-17T18:17:55.402911+0000","last_fullsized":"2024-08-17T18:17:55.402911+0000","mapping_epoch":1422,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:15:14.122188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7,3],"avail_no_missing":["6","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":109,"reported_epoch":1440,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:17:55.409386+0000","last_change":"2024-08-17T18:15:59.367304+0000","last_active":"2024-08-17T18:17:55.409386+0000","last_peered":"2024-08-17T18:17:55.409386+0000","last_clean":"2024-08-17T18:15:39.345292+0000","last_became_active":"2024-08-17T18:15:59.367304+0000","last_became_peered":"2024-08-17T18:15:59.367304+0000","last_unstale":"2024-08-17T18:17:55.409386+0000","last_undegraded":"2024-08-17T18:15:59.365830+0000","last_fullsized":"2024-08-17T18:15:59.365714+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:33.229238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2189,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:17.003484+0000","last_change":"2024-08-17T18:09:28.934404+0000","last_active":"2024-08-17T18:18:17.003484+0000","last_peered":"2024-08-17T18:18:17.003484+0000","last_clean":"2024-08-17T18:18:17.003484+0000","last_became_active":"2024-08-17T18:09:28.934001+0000","last_became_peered":"2024-08-17T18:09:28.934001+0000","last_unstale":"2024-08-17T18:18:17.003484+0000","last_undegraded":"2024-08-17T18:18:17.003484+0000","last_fullsized":"2024-08-17T18:18:17.003484+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:04.676812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":108,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.410947+0000","last_change":"2024-08-17T18:16:34.652748+0000","last_active":"2024-08-17T18:17:55.410947+0000","last_peered":"2024-08-17T18:17:55.410947+0000","last_clean":"2024-08-17T18:17:55.410947+0000","last_became_active":"2024-08-17T18:16:34.652439+0000","last_became_peered":"2024-08-17T18:16:34.652439+0000","last_unstale":"2024-08-17T18:17:55.410947+0000","last_undegraded":"2024-08-17T18:17:55.410947+0000","last_fullsized":"2024-08-17T18:17:55.410947+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:11:37.922525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":2191,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:17.005250+0000","last_change":"2024-08-17T18:09:28.935401+0000","last_active":"2024-08-17T18:18:17.005250+0000","last_peered":"2024-08-17T18:18:17.005250+0000","last_clean":"2024-08-17T18:18:17.005250+0000","last_became_active":"2024-08-17T18:09:28.933628+0000","last_became_peered":"2024-08-17T18:09:28.933628+0000","last_unstale":"2024-08-17T18:18:17.005250+0000","last_undegraded":"2024-08-17T18:18:17.005250+0000","last_fullsized":"2024-08-17T18:18:17.005250+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:21:12.714486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2308,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.417228+0000","last_change":"2024-08-17T18:15:04.435383+0000","last_active":"2024-08-17T18:17:55.417228+0000","last_peered":"2024-08-17T18:17:55.417228+0000","last_clean":"2024-08-17T18:17:55.417228+0000","last_became_active":"2024-08-17T18:15:04.434100+0000","last_became_peered":"2024-08-17T18:15:04.434100+0000","last_unstale":"2024-08-17T18:17:55.417228+0000","last_undegraded":"2024-08-17T18:17:55.417228+0000","last_fullsized":"2024-08-17T18:17:55.417228+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:24.856737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1451,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.405850+0000","last_change":"2024-08-17T18:15:04.433053+0000","last_active":"2024-08-17T18:17:55.405850+0000","last_peered":"2024-08-17T18:17:55.405850+0000","last_clean":"2024-08-17T18:17:55.405850+0000","last_became_active":"2024-08-17T18:15:04.432721+0000","last_became_peered":"2024-08-17T18:15:04.432721+0000","last_unstale":"2024-08-17T18:17:55.405850+0000","last_undegraded":"2024-08-17T18:17:55.405850+0000","last_fullsized":"2024-08-17T18:17:55.405850+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:52:17.355991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":105,"reported_epoch":1440,"state":"active+undersized","last_fresh":"2024-08-17T18:17:55.409359+0000","last_change":"2024-08-17T18:15:59.375690+0000","last_active":"2024-08-17T18:17:55.409359+0000","last_peered":"2024-08-17T18:17:55.409359+0000","last_clean":"2024-08-17T18:15:39.344817+0000","last_became_active":"2024-08-17T18:15:59.375690+0000","last_became_peered":"2024-08-17T18:15:59.375690+0000","last_unstale":"2024-08-17T18:17:55.409359+0000","last_undegraded":"2024-08-17T18:17:55.409359+0000","last_fullsized":"2024-08-17T18:15:59.366709+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:15:01.033842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":105,"reported_epoch":1440,"state":"active+undersized","last_fresh":"2024-08-17T18:17:55.407588+0000","last_change":"2024-08-17T18:15:59.375264+0000","last_active":"2024-08-17T18:17:55.407588+0000","last_peered":"2024-08-17T18:17:55.407588+0000","last_clean":"2024-08-17T18:15:39.327529+0000","last_became_active":"2024-08-17T18:15:59.375264+0000","last_became_peered":"2024-08-17T18:15:59.375264+0000","last_unstale":"2024-08-17T18:17:55.407588+0000","last_undegraded":"2024-08-17T18:17:55.407588+0000","last_fullsized":"2024-08-17T18:15:59.365404+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:02:00.253365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":101,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.408776+0000","last_change":"2024-08-17T18:15:04.434047+0000","last_active":"2024-08-17T18:17:55.408776+0000","last_peered":"2024-08-17T18:17:55.408776+0000","last_clean":"2024-08-17T18:17:55.408776+0000","last_became_active":"2024-08-17T18:15:04.433748+0000","last_became_peered":"2024-08-17T18:15:04.433748+0000","last_unstale":"2024-08-17T18:17:55.408776+0000","last_undegraded":"2024-08-17T18:17:55.408776+0000","last_fullsized":"2024-08-17T18:17:55.408776+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:05.446870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1006,"reported_epoch":1441,"state":"active+clean+remapped","last_fresh":"2024-08-17T18:18:17.003882+0000","last_change":"2024-08-17T18:16:35.644066+0000","last_active":"2024-08-17T18:18:17.003882+0000","last_peered":"2024-08-17T18:18:17.003882+0000","last_clean":"2024-08-17T18:18:17.003882+0000","last_became_active":"2024-08-17T18:16:35.640910+0000","last_became_peered":"2024-08-17T18:16:35.640910+0000","last_unstale":"2024-08-17T18:18:17.003882+0000","last_undegraded":"2024-08-17T18:18:17.003882+0000","last_fullsized":"2024-08-17T18:18:17.003882+0000","mapping_epoch":1422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:59:49.207593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,7],"avail_no_missing":["3","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":105,"reported_epoch":1440,"state":"active+undersized","last_fresh":"2024-08-17T18:17:55.409186+0000","last_change":"2024-08-17T18:15:59.486591+0000","last_active":"2024-08-17T18:17:55.409186+0000","last_peered":"2024-08-17T18:17:55.409186+0000","last_clean":"2024-08-17T18:15:39.344715+0000","last_became_active":"2024-08-17T18:15:59.486591+0000","last_became_peered":"2024-08-17T18:15:59.486591+0000","last_unstale":"2024-08-17T18:17:55.409186+0000","last_undegraded":"2024-08-17T18:17:55.409186+0000","last_fullsized":"2024-08-17T18:15:59.366425+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:55.021730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1706,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.416959+0000","last_change":"2024-08-17T18:16:34.650941+0000","last_active":"2024-08-17T18:17:55.416959+0000","last_peered":"2024-08-17T18:17:55.416959+0000","last_clean":"2024-08-17T18:17:55.416959+0000","last_became_active":"2024-08-17T18:16:34.650594+0000","last_became_peered":"2024-08-17T18:16:34.650594+0000","last_unstale":"2024-08-17T18:17:55.416959+0000","last_undegraded":"2024-08-17T18:17:55.416959+0000","last_fullsized":"2024-08-17T18:17:55.416959+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:21:27.095827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":119,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.407500+0000","last_change":"2024-08-17T18:16:34.869990+0000","last_active":"2024-08-17T18:17:55.407500+0000","last_peered":"2024-08-17T18:17:55.407500+0000","last_clean":"2024-08-17T18:17:55.407500+0000","last_became_active":"2024-08-17T18:16:34.667609+0000","last_became_peered":"2024-08-17T18:16:34.667609+0000","last_unstale":"2024-08-17T18:17:55.407500+0000","last_undegraded":"2024-08-17T18:17:55.407500+0000","last_fullsized":"2024-08-17T18:17:55.407500+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:04:08.790527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":101,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.411570+0000","last_change":"2024-08-17T18:15:04.435826+0000","last_active":"2024-08-17T18:17:55.411570+0000","last_peered":"2024-08-17T18:17:55.411570+0000","last_clean":"2024-08-17T18:17:55.411570+0000","last_became_active":"2024-08-17T18:15:04.435301+0000","last_became_peered":"2024-08-17T18:15:04.435301+0000","last_unstale":"2024-08-17T18:17:55.411570+0000","last_undegraded":"2024-08-17T18:17:55.411570+0000","last_fullsized":"2024-08-17T18:17:55.411570+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:14:03.286433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":111,"reported_epoch":1440,"state":"active+clean+remapped","last_fresh":"2024-08-17T18:17:55.407243+0000","last_change":"2024-08-17T18:16:35.646462+0000","last_active":"2024-08-17T18:17:55.407243+0000","last_peered":"2024-08-17T18:17:55.407243+0000","last_clean":"2024-08-17T18:17:55.407243+0000","last_became_active":"2024-08-17T18:16:35.646089+0000","last_became_peered":"2024-08-17T18:16:35.646089+0000","last_unstale":"2024-08-17T18:17:55.407243+0000","last_undegraded":"2024-08-17T18:17:55.407243+0000","last_fullsized":"2024-08-17T18:17:55.407243+0000","mapping_epoch":1422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:48.445288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,0],"avail_no_missing":["1","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2116,"reported_epoch":1441,"state":"active+undersized","last_fresh":"2024-08-17T18:18:17.005572+0000","last_change":"2024-08-17T18:15:59.380078+0000","last_active":"2024-08-17T18:18:17.005572+0000","last_peered":"2024-08-17T18:18:17.005572+0000","last_clean":"2024-08-17T18:15:42.676229+0000","last_became_active":"2024-08-17T18:15:59.380078+0000","last_became_peered":"2024-08-17T18:15:59.380078+0000","last_unstale":"2024-08-17T18:18:17.005572+0000","last_undegraded":"2024-08-17T18:18:17.005572+0000","last_fullsized":"2024-08-17T18:15:59.370183+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:24.262096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2372,"reported_epoch":1440,"state":"active+undersized","last_fresh":"2024-08-17T18:17:55.417660+0000","last_change":"2024-08-17T18:15:59.376946+0000","last_active":"2024-08-17T18:17:55.417660+0000","last_peered":"2024-08-17T18:17:55.417660+0000","last_clean":"2024-08-17T18:15:39.321813+0000","last_became_active":"2024-08-17T18:15:59.376946+0000","last_became_peered":"2024-08-17T18:15:59.376946+0000","last_unstale":"2024-08-17T18:17:55.417660+0000","last_undegraded":"2024-08-17T18:17:55.417660+0000","last_fullsized":"2024-08-17T18:15:59.368107+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:18.303802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2346,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.415157+0000","last_change":"2024-08-17T18:09:28.842838+0000","last_active":"2024-08-17T18:17:55.415157+0000","last_peered":"2024-08-17T18:17:55.415157+0000","last_clean":"2024-08-17T18:17:55.415157+0000","last_became_active":"2024-08-17T18:09:28.842617+0000","last_became_peered":"2024-08-17T18:09:28.842617+0000","last_unstale":"2024-08-17T18:17:55.415157+0000","last_undegraded":"2024-08-17T18:17:55.415157+0000","last_fullsized":"2024-08-17T18:17:55.415157+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:00.016002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1472,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.405151+0000","last_change":"2024-08-17T18:15:04.422793+0000","last_active":"2024-08-17T18:17:55.405151+0000","last_peered":"2024-08-17T18:17:55.405151+0000","last_clean":"2024-08-17T18:17:55.405151+0000","last_became_active":"2024-08-17T18:15:04.422359+0000","last_became_peered":"2024-08-17T18:15:04.422359+0000","last_unstale":"2024-08-17T18:17:55.405151+0000","last_undegraded":"2024-08-17T18:17:55.405151+0000","last_fullsized":"2024-08-17T18:17:55.405151+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:09:43.126229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":29,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.402599+0000","last_change":"2024-08-17T18:16:34.643455+0000","last_active":"2024-08-17T18:17:55.402599+0000","last_peered":"2024-08-17T18:17:55.402599+0000","last_clean":"2024-08-17T18:17:55.402599+0000","last_became_active":"2024-08-17T18:16:34.642572+0000","last_became_peered":"2024-08-17T18:16:34.642572+0000","last_unstale":"2024-08-17T18:17:55.402599+0000","last_undegraded":"2024-08-17T18:17:55.402599+0000","last_fullsized":"2024-08-17T18:17:55.402599+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:36:26.372505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2154,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:17.004022+0000","last_change":"2024-08-17T18:16:34.668505+0000","last_active":"2024-08-17T18:18:17.004022+0000","last_peered":"2024-08-17T18:18:17.004022+0000","last_clean":"2024-08-17T18:18:17.004022+0000","last_became_active":"2024-08-17T18:16:34.668076+0000","last_became_peered":"2024-08-17T18:16:34.668076+0000","last_unstale":"2024-08-17T18:18:17.004022+0000","last_undegraded":"2024-08-17T18:18:17.004022+0000","last_fullsized":"2024-08-17T18:18:17.004022+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:15:13.851229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":224,"reported_epoch":1440,"state":"active+undersized+degraded","last_fresh":"2024-08-17T18:17:55.405484+0000","last_change":"2024-08-17T18:15:59.376458+0000","last_active":"2024-08-17T18:17:55.405484+0000","last_peered":"2024-08-17T18:17:55.405484+0000","last_clean":"2024-08-17T18:15:39.323317+0000","last_became_active":"2024-08-17T18:15:59.376458+0000","last_became_peered":"2024-08-17T18:15:59.376458+0000","last_unstale":"2024-08-17T18:17:55.405484+0000","last_undegraded":"2024-08-17T18:15:59.363659+0000","last_fullsized":"2024-08-17T18:15:59.363523+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:53.800723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1857,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.405030+0000","last_change":"2024-08-17T18:16:37.078361+0000","last_active":"2024-08-17T18:17:55.405030+0000","last_peered":"2024-08-17T18:17:55.405030+0000","last_clean":"2024-08-17T18:17:55.405030+0000","last_became_active":"2024-08-17T18:16:37.077943+0000","last_became_peered":"2024-08-17T18:16:37.077943+0000","last_unstale":"2024-08-17T18:17:55.405030+0000","last_undegraded":"2024-08-17T18:17:55.405030+0000","last_fullsized":"2024-08-17T18:17:55.405030+0000","mapping_epoch":1424,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:08:30.810350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":101,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.411647+0000","last_change":"2024-08-17T18:15:04.438927+0000","last_active":"2024-08-17T18:17:55.411647+0000","last_peered":"2024-08-17T18:17:55.411647+0000","last_clean":"2024-08-17T18:17:55.411647+0000","last_became_active":"2024-08-17T18:15:04.438356+0000","last_became_peered":"2024-08-17T18:15:04.438356+0000","last_unstale":"2024-08-17T18:17:55.411647+0000","last_undegraded":"2024-08-17T18:17:55.411647+0000","last_fullsized":"2024-08-17T18:17:55.411647+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:43:46.372749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2225,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:17.003570+0000","last_change":"2024-08-17T18:15:04.418596+0000","last_active":"2024-08-17T18:18:17.003570+0000","last_peered":"2024-08-17T18:18:17.003570+0000","last_clean":"2024-08-17T18:18:17.003570+0000","last_became_active":"2024-08-17T18:15:04.418011+0000","last_became_peered":"2024-08-17T18:15:04.418011+0000","last_unstale":"2024-08-17T18:18:17.003570+0000","last_undegraded":"2024-08-17T18:18:17.003570+0000","last_fullsized":"2024-08-17T18:18:17.003570+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:32.118859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":29,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.402552+0000","last_change":"2024-08-17T18:16:34.650527+0000","last_active":"2024-08-17T18:17:55.402552+0000","last_peered":"2024-08-17T18:17:55.402552+0000","last_clean":"2024-08-17T18:17:55.402552+0000","last_became_active":"2024-08-17T18:16:34.650228+0000","last_became_peered":"2024-08-17T18:16:34.650228+0000","last_unstale":"2024-08-17T18:17:55.402552+0000","last_undegraded":"2024-08-17T18:17:55.402552+0000","last_fullsized":"2024-08-17T18:17:55.402552+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:30:07.593560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":719,"reported_epoch":1440,"state":"active+undersized","last_fresh":"2024-08-17T18:17:55.405223+0000","last_change":"2024-08-17T18:15:59.365847+0000","last_active":"2024-08-17T18:17:55.405223+0000","last_peered":"2024-08-17T18:17:55.405223+0000","last_clean":"2024-08-17T18:15:39.323257+0000","last_became_active":"2024-08-17T18:15:59.365847+0000","last_became_peered":"2024-08-17T18:15:59.365847+0000","last_unstale":"2024-08-17T18:17:55.405223+0000","last_undegraded":"2024-08-17T18:17:55.405223+0000","last_fullsized":"2024-08-17T18:15:59.363316+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:14:07.754350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":111,"reported_epoch":1440,"state":"active+clean+remapped","last_fresh":"2024-08-17T18:17:55.426009+0000","last_change":"2024-08-17T18:16:35.642427+0000","last_active":"2024-08-17T18:17:55.426009+0000","last_peered":"2024-08-17T18:17:55.426009+0000","last_clean":"2024-08-17T18:17:55.426009+0000","last_became_active":"2024-08-17T18:16:35.642042+0000","last_became_peered":"2024-08-17T18:16:35.642042+0000","last_unstale":"2024-08-17T18:17:55.426009+0000","last_undegraded":"2024-08-17T18:17:55.426009+0000","last_fullsized":"2024-08-17T18:17:55.426009+0000","mapping_epoch":1422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:12.832266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,0],"avail_no_missing":["1","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":29,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.402447+0000","last_change":"2024-08-17T18:16:34.727445+0000","last_active":"2024-08-17T18:17:55.402447+0000","last_peered":"2024-08-17T18:17:55.402447+0000","last_clean":"2024-08-17T18:17:55.402447+0000","last_became_active":"2024-08-17T18:16:34.674278+0000","last_became_peered":"2024-08-17T18:16:34.674278+0000","last_unstale":"2024-08-17T18:17:55.402447+0000","last_undegraded":"2024-08-17T18:17:55.402447+0000","last_fullsized":"2024-08-17T18:17:55.402447+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:57.760933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":955,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:17.009954+0000","last_change":"2024-08-17T18:16:34.651207+0000","last_active":"2024-08-17T18:18:17.009954+0000","last_peered":"2024-08-17T18:18:17.009954+0000","last_clean":"2024-08-17T18:18:17.009954+0000","last_became_active":"2024-08-17T18:16:34.650895+0000","last_became_peered":"2024-08-17T18:16:34.650895+0000","last_unstale":"2024-08-17T18:18:17.009954+0000","last_undegraded":"2024-08-17T18:18:17.009954+0000","last_fullsized":"2024-08-17T18:18:17.009954+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:08:10.447087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2229,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:17.007280+0000","last_change":"2024-08-17T18:15:04.441353+0000","last_active":"2024-08-17T18:18:17.007280+0000","last_peered":"2024-08-17T18:18:17.007280+0000","last_clean":"2024-08-17T18:18:17.007280+0000","last_became_active":"2024-08-17T18:15:04.423663+0000","last_became_peered":"2024-08-17T18:15:04.423663+0000","last_unstale":"2024-08-17T18:18:17.007280+0000","last_undegraded":"2024-08-17T18:18:17.007280+0000","last_fullsized":"2024-08-17T18:18:17.007280+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:14:33.820349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2390,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.417313+0000","last_change":"2024-08-17T18:15:04.436670+0000","last_active":"2024-08-17T18:17:55.417313+0000","last_peered":"2024-08-17T18:17:55.417313+0000","last_clean":"2024-08-17T18:17:55.417313+0000","last_became_active":"2024-08-17T18:15:04.435497+0000","last_became_peered":"2024-08-17T18:15:04.435497+0000","last_unstale":"2024-08-17T18:17:55.417313+0000","last_undegraded":"2024-08-17T18:17:55.417313+0000","last_fullsized":"2024-08-17T18:17:55.417313+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:43:53.427302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":31,"reported_epoch":1440,"state":"active+clean+remapped","last_fresh":"2024-08-17T18:17:55.403617+0000","last_change":"2024-08-17T18:16:35.643836+0000","last_active":"2024-08-17T18:17:55.403617+0000","last_peered":"2024-08-17T18:17:55.403617+0000","last_clean":"2024-08-17T18:17:55.403617+0000","last_became_active":"2024-08-17T18:16:35.643464+0000","last_became_peered":"2024-08-17T18:16:35.643464+0000","last_unstale":"2024-08-17T18:17:55.403617+0000","last_undegraded":"2024-08-17T18:17:55.403617+0000","last_fullsized":"2024-08-17T18:17:55.403617+0000","mapping_epoch":1422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:34:43.495624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,0],"avail_no_missing":["6","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":29,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.402365+0000","last_change":"2024-08-17T18:16:34.727319+0000","last_active":"2024-08-17T18:17:55.402365+0000","last_peered":"2024-08-17T18:17:55.402365+0000","last_clean":"2024-08-17T18:17:55.402365+0000","last_became_active":"2024-08-17T18:16:34.673905+0000","last_became_peered":"2024-08-17T18:16:34.673905+0000","last_unstale":"2024-08-17T18:17:55.402365+0000","last_undegraded":"2024-08-17T18:17:55.402365+0000","last_fullsized":"2024-08-17T18:17:55.402365+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:05.107752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1435,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.405857+0000","last_change":"2024-08-17T18:16:34.651447+0000","last_active":"2024-08-17T18:17:55.405857+0000","last_peered":"2024-08-17T18:17:55.405857+0000","last_clean":"2024-08-17T18:17:55.405857+0000","last_became_active":"2024-08-17T18:16:34.647419+0000","last_became_peered":"2024-08-17T18:16:34.647419+0000","last_unstale":"2024-08-17T18:17:55.405857+0000","last_undegraded":"2024-08-17T18:17:55.405857+0000","last_fullsized":"2024-08-17T18:17:55.405857+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:44.760211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":108,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.409670+0000","last_change":"2024-08-17T18:16:34.655452+0000","last_active":"2024-08-17T18:17:55.409670+0000","last_peered":"2024-08-17T18:17:55.409670+0000","last_clean":"2024-08-17T18:17:55.409670+0000","last_became_active":"2024-08-17T18:16:34.654518+0000","last_became_peered":"2024-08-17T18:16:34.654518+0000","last_unstale":"2024-08-17T18:17:55.409670+0000","last_undegraded":"2024-08-17T18:17:55.409670+0000","last_fullsized":"2024-08-17T18:17:55.409670+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:10:26.866276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1450,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.405975+0000","last_change":"2024-08-17T18:15:04.447609+0000","last_active":"2024-08-17T18:17:55.405975+0000","last_peered":"2024-08-17T18:17:55.405975+0000","last_clean":"2024-08-17T18:17:55.405975+0000","last_became_active":"2024-08-17T18:15:04.432772+0000","last_became_peered":"2024-08-17T18:15:04.432772+0000","last_unstale":"2024-08-17T18:17:55.405975+0000","last_undegraded":"2024-08-17T18:17:55.405975+0000","last_fullsized":"2024-08-17T18:17:55.405975+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:59:46.128541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":353,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.405009+0000","last_change":"2024-08-17T18:16:34.648983+0000","last_active":"2024-08-17T18:17:55.405009+0000","last_peered":"2024-08-17T18:17:55.405009+0000","last_clean":"2024-08-17T18:17:55.405009+0000","last_became_active":"2024-08-17T18:16:34.648264+0000","last_became_peered":"2024-08-17T18:16:34.648264+0000","last_unstale":"2024-08-17T18:17:55.405009+0000","last_undegraded":"2024-08-17T18:17:55.405009+0000","last_fullsized":"2024-08-17T18:17:55.405009+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:43.112614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1466,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.407209+0000","last_change":"2024-08-17T18:15:04.445387+0000","last_active":"2024-08-17T18:17:55.407209+0000","last_peered":"2024-08-17T18:17:55.407209+0000","last_clean":"2024-08-17T18:17:55.407209+0000","last_became_active":"2024-08-17T18:15:04.432775+0000","last_became_peered":"2024-08-17T18:15:04.432775+0000","last_unstale":"2024-08-17T18:17:55.407209+0000","last_undegraded":"2024-08-17T18:17:55.407209+0000","last_fullsized":"2024-08-17T18:17:55.407209+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:32:50.936356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":39,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.404462+0000","last_change":"2024-08-17T18:16:34.660944+0000","last_active":"2024-08-17T18:17:55.404462+0000","last_peered":"2024-08-17T18:17:55.404462+0000","last_clean":"2024-08-17T18:17:55.404462+0000","last_became_active":"2024-08-17T18:16:34.646790+0000","last_became_peered":"2024-08-17T18:16:34.646790+0000","last_unstale":"2024-08-17T18:17:55.404462+0000","last_undegraded":"2024-08-17T18:17:55.404462+0000","last_fullsized":"2024-08-17T18:17:55.404462+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:49:25.544254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2222,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:17.005413+0000","last_change":"2024-08-17T18:15:04.426282+0000","last_active":"2024-08-17T18:18:17.005413+0000","last_peered":"2024-08-17T18:18:17.005413+0000","last_clean":"2024-08-17T18:18:17.005413+0000","last_became_active":"2024-08-17T18:15:04.425713+0000","last_became_peered":"2024-08-17T18:15:04.425713+0000","last_unstale":"2024-08-17T18:18:17.005413+0000","last_undegraded":"2024-08-17T18:18:17.005413+0000","last_fullsized":"2024-08-17T18:18:17.005413+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:54:54.594901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1422,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.405363+0000","last_change":"2024-08-17T18:16:34.641621+0000","last_active":"2024-08-17T18:17:55.405363+0000","last_peered":"2024-08-17T18:17:55.405363+0000","last_clean":"2024-08-17T18:17:55.405363+0000","last_became_active":"2024-08-17T18:16:34.641239+0000","last_became_peered":"2024-08-17T18:16:34.641239+0000","last_unstale":"2024-08-17T18:17:55.405363+0000","last_undegraded":"2024-08-17T18:17:55.405363+0000","last_fullsized":"2024-08-17T18:17:55.405363+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:19.066766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2336,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.416912+0000","last_change":"2024-08-17T18:09:28.864923+0000","last_active":"2024-08-17T18:17:55.416912+0000","last_peered":"2024-08-17T18:17:55.416912+0000","last_clean":"2024-08-17T18:17:55.416912+0000","last_became_active":"2024-08-17T18:09:28.864707+0000","last_became_peered":"2024-08-17T18:09:28.864707+0000","last_unstale":"2024-08-17T18:17:55.416912+0000","last_undegraded":"2024-08-17T18:17:55.416912+0000","last_fullsized":"2024-08-17T18:17:55.416912+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:20:27.507329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2337,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T18:17:55.421322+0000","last_change":"2024-08-17T18:09:28.846741+0000","last_active":"2024-08-17T18:17:55.421322+0000","last_peered":"2024-08-17T18:17:55.421322+0000","last_clean":"2024-08-17T18:17:55.421322+0000","last_became_active":"2024-08-17T18:09:28.846478+0000","last_became_peered":"2024-08-17T18:09:28.846478+0000","last_unstale":"2024-08-17T18:17:55.421322+0000","last_undegraded":"2024-08-17T18:17:55.421322+0000","last_fullsized":"2024-08-17T18:17:55.421322+0000","mapping_epoch":1343,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:34.247517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":837,"reported_epoch":1440,"state":"active+clean+remapped","last_fresh":"2024-08-17T18:17:55.405486+0000","last_change":"2024-08-17T18:16:35.640621+0000","last_active":"2024-08-17T18:17:55.405486+0000","last_peered":"2024-08-17T18:17:55.405486+0000","last_clean":"2024-08-17T18:17:55.405486+0000","last_became_active":"2024-08-17T18:16:35.640155+0000","last_became_peered":"2024-08-17T18:16:35.640155+0000","last_unstale":"2024-08-17T18:17:55.405486+0000","last_undegraded":"2024-08-17T18:17:55.405486+0000","last_fullsized":"2024-08-17T18:17:55.405486+0000","mapping_epoch":1422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:57:17.060031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6,0],"avail_no_missing":["7","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1451,"reported_epoch":1440,"state":"active+clean+remapped","last_fresh":"2024-08-17T18:17:55.405467+0000","last_change":"2024-08-17T18:16:35.742578+0000","last_active":"2024-08-17T18:17:55.405467+0000","last_peered":"2024-08-17T18:17:55.405467+0000","last_clean":"2024-08-17T18:17:55.405467+0000","last_became_active":"2024-08-17T18:16:35.655950+0000","last_became_peered":"2024-08-17T18:16:35.655950+0000","last_unstale":"2024-08-17T18:17:55.405467+0000","last_undegraded":"2024-08-17T18:17:55.405467+0000","last_fullsized":"2024-08-17T18:17:55.405467+0000","mapping_epoch":1422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:26.839281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6,0],"avail_no_missing":["7","0","6"],"object_location_counts":[{"shards":"0,6,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2222,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T18:18:17.009810+0000","last_change":"2024-08-17T18:15:04.420087+0000","last_active":"2024-08-17T18:18:17.009810+0000","last_peered":"2024-08-17T18:18:17.009810+0000","last_clean":"2024-08-17T18:18:17.009810+0000","last_became_active":"2024-08-17T18:15:04.419599+0000","last_became_peered":"2024-08-17T18:15:04.419599+0000","last_unstale":"2024-08-17T18:18:17.009810+0000","last_undegraded":"2024-08-17T18:18:17.009810+0000","last_fullsized":"2024-08-17T18:18:17.009810+0000","mapping_epoch":1395,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:42.792932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":50,"num_pg":16,"stat_sum":{"num_bytes":5883756544,"num_objects":89779,"num_object_clones":0,"num_object_copies":269337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":50322,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":89779,"num_write_kb":5745856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14350614528,"data_stored":14350614528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89779,"ondisk_log_size":89779,"up":39,"acting":39,"num_store_stats":5},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":4132576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":21,"acting":22,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":8,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":16410,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":27398,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":84,"acting":91,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":85,"acting":89,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234689,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4404332,"kb_used_data":2902048,"kb_used_omap":1,"kb_used_meta":1502206,"kb_avail":89340820,"statfs":{"total":95995035648,"available":91484999680,"internally_reserved":0,"allocated":2971697152,"data_stored":2966647521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1538259288},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496324,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3841128,"kb_used_data":3255392,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":89904024,"statfs":{"total":95995035648,"available":92061720576,"internally_reserved":0,"allocated":3333521408,"data_stored":3328472246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208757962,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1343,"seq":5768141078635,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3070104,"kb_used_data":2550220,"kb_used_omap":112,"kb_used_meta":519759,"kb_avail":90675048,"statfs":{"total":95995035648,"available":92851249152,"internally_reserved":0,"allocated":2611425280,"data_stored":2606391367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":115373,"internal_metadata":532233555},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":538,"seq":2310692405786,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3021704,"kb_used_data":1786112,"kb_used_omap":61,"kb_used_meta":1235522,"kb_avail":90723448,"statfs":{"total":95995035648,"available":92900810752,"internally_reserved":0,"allocated":1828978688,"data_stored":1824204841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62536,"internal_metadata":1265175480},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452349,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2749120,"kb_used_data":1785912,"kb_used_omap":54,"kb_used_meta":963145,"kb_avail":90996032,"statfs":{"total":95995035648,"available":93179936768,"internally_reserved":0,"allocated":1828773888,"data_stored":1823750509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":986260878},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471475039,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4706012,"kb_used_data":3432148,"kb_used_omap":119,"kb_used_meta":1273736,"kb_avail":89039140,"statfs":{"total":95995035648,"available":91176079360,"internally_reserved":0,"allocated":3514519552,"data_stored":3509473483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":1304306398},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019597,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":50,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3549757440,"data_stored":3549757440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":50,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1837432832,"data_stored":1837432832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":50,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2615541760,"data_stored":2615541760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":50,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3349807104,"data_stored":3349807104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":50,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2998075392,"data_stored":2998075392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:18:22.883 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T18:18:22.883 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_5 pg_num 26 2024-08-17T18:18:22.899 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:22.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:22.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:22.935 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:22 smithi130 ceph-mon[105521]: from='client.37221 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:18:22.935 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:22 smithi130 ceph-mon[105521]: pgmap v2754: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s; 51131/274272 objects degraded (18.642%); 8/274272 objects misplaced (0.003%) 2024-08-17T18:18:22.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:22 smithi130 ceph-mon[110068]: from='client.37221 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:18:22.935 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:22 smithi130 ceph-mon[110068]: pgmap v2754: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 81 MiB/s wr, 1.30k op/s; 51131/274272 objects degraded (18.642%); 8/274272 objects misplaced (0.003%) 2024-08-17T18:18:23.001 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:23.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:23.032 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:23.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:23.109 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:18:23.109 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:18:23.132 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:23.148 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 51 16 97618 97602 119.576 78.875 0.0165938 0.00835855 2024-08-17T18:18:23.164 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:23.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:23.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:23.265 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:23.289 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:23.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:23.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:23.391 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:23.420 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:23.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:23.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:23.522 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:23.556 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:18:23.560 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:23.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:23.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:23.619 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:18:23.620 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:18:23.652 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:18:23.653 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:18:23.655 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:18:23.662 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:23.678 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:18:23.688 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:18:23.688 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:18:23.689 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:23.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:23.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:23.724 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:18:23.725 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:18:23.753 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:18:23.753 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:18:23.776 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:18:23.776 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:18:23.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:23.808 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:18:23.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:23 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4039035304' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:18:23.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:23 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4039035304' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:18:23.809 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:18:23.820 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:23.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:23.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:23.824 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:23 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4039035304' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:18:23.860 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T18:18:23.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:23.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:23.958 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:23.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:24.059 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:24.087 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:24.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:24.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:24.149 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 52 16 98834 98818 118.738 76 0.00421073 0.00841705 2024-08-17T18:18:24.189 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:24.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:24.232 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:24.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:24.302 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:18:24.333 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:24.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:24.378 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:24.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:24.479 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:24.510 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:24.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:24.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:24.612 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:24.649 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:24.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:24.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:24.752 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:24.791 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:24.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:24.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:24.893 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:24.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:24 smithi130 ceph-mon[105521]: pgmap v2755: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s; 54018/289011 objects degraded (18.691%); 8/289011 objects misplaced (0.003%) 2024-08-17T18:18:24.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:24 smithi130 ceph-mon[110068]: pgmap v2755: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s; 54018/289011 objects degraded (18.691%); 8/289011 objects misplaced (0.003%) 2024-08-17T18:18:24.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:24.926 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:24.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:25.027 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:25.057 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:25.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:25.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:25.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:24 smithi179 ceph-mon[93519]: pgmap v2755: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 5.9 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s; 54018/289011 objects degraded (18.691%); 8/289011 objects misplaced (0.003%) 2024-08-17T18:18:25.149 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 53 16 100135 100119 118.032 81.3125 0.0142317 0.00846742 2024-08-17T18:18:25.159 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:25.192 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:25.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:25.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:25.294 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:25.331 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:25.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:25.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:25.433 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:25.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:25.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:25.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:25.564 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:25.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:25.599 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:25.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:25.700 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:25.716 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:18:25.716 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:18:25.745 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:25.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:25.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:25.847 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:25.898 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:25.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:26.000 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:26.031 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:26.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:26.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:26.133 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:26.150 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 54 16 101442 101426 117.356 81.6875 0.0160302 0.00851633 2024-08-17T18:18:26.173 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:26.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:26.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:26.275 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:26.306 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:26.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:26.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:26.408 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:26.454 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:26.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:26.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:26.555 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:26.585 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:26.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:26.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:26.687 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:26.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:26.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:26.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:26.817 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:26.852 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:26.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:26.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:26.954 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:26.991 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:26.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:26.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:27.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:26 smithi179 ceph-mon[93519]: pgmap v2756: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s; 54782/293562 objects degraded (18.661%); 8/293562 objects misplaced (0.003%) 2024-08-17T18:18:27.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 54782/293562 objects degraded (18.661%), 11 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:27.093 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:27.124 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:27.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:27.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:27.150 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 55 16 102705 102689 116.657 78.9375 0.00883763 0.00856779 2024-08-17T18:18:27.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:26 smithi130 ceph-mon[105521]: pgmap v2756: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s; 54782/293562 objects degraded (18.661%); 8/293562 objects misplaced (0.003%) 2024-08-17T18:18:27.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 54782/293562 objects degraded (18.661%), 11 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:27.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:26 smithi130 ceph-mon[110068]: pgmap v2756: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 80 MiB/s wr, 1.28k op/s; 54782/293562 objects degraded (18.661%); 8/293562 objects misplaced (0.003%) 2024-08-17T18:18:27.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 54782/293562 objects degraded (18.661%), 11 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:27.226 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:27.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:27.260 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:27.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:27.361 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:27.388 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:27.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:27.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:27.489 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:27.525 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:27.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:27.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:27.627 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:27.660 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:27.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:27.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:27.717 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:18:27.762 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:27.793 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:27.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:27.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:27.895 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:27.918 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:27.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:27.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:27.922 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:18:27.922 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:18:27.922 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:18:27.922 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:18:27.922 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:18:28.020 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:28.069 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:28.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:28.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:28.150 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 56 16 103786 103770 115.781 67.5625 0.0144481 0.00863201 2024-08-17T18:18:28.171 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:28.205 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:28.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:28.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:28.307 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:28.345 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:28.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:28.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:28.396 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:28.396 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:18:28.396 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:28.396 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:18:28.447 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:28.487 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:28.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:28.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:28.589 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:28.616 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:28.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:28.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:28.718 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:28.727 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:28.728 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:18:28.761 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T18:18:28.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:28.763 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:28.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:28.863 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:28.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:28.892 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:28.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:28.993 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:29.026 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:29.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:29.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:29.123 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:18:29.129 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:29.150 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 57 16 104890 104874 114.96 69 0.00744364 0.008694 2024-08-17T18:18:29.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:29.167 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:29.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:29.253 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:18:29.268 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:29.296 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:29.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:29.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:29.362 INFO:teuthology.orchestra.run.smithi130.stderr:set pool 50 pg_num to 26 2024-08-17T18:18:29.365 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:18:29.398 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:29.428 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:29.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:29.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:29.530 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:29.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:29 smithi130 ceph-mon[105521]: pgmap v2757: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 75 MiB/s wr, 1.20k op/s; 55197/295944 objects degraded (18.651%); 8/295944 objects misplaced (0.003%) 2024-08-17T18:18:29.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1464755288' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T18:18:29.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/239391362' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:18:29.556 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:29 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:18:29.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:29 smithi130 ceph-mon[110068]: pgmap v2757: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 75 MiB/s wr, 1.20k op/s; 55197/295944 objects degraded (18.651%); 8/295944 objects misplaced (0.003%) 2024-08-17T18:18:29.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1464755288' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T18:18:29.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/239391362' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:18:29.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:29 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:18:29.558 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:29.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:29.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:29.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:29 smithi179 ceph-mon[93519]: pgmap v2757: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 6.0 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 75 MiB/s wr, 1.20k op/s; 55197/295944 objects degraded (18.651%); 8/295944 objects misplaced (0.003%) 2024-08-17T18:18:29.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1464755288' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T18:18:29.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/239391362' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:18:29.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:29 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:18:29.660 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:29.697 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:29.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:29.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:29.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:29.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:29.844 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:29.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:29.945 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:29.976 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:29.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:29.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:30.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:30.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:30.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:30.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:30.151 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 58 16 106153 106137 114.339 78.9375 0.00749338 0.00874134 2024-08-17T18:18:30.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:18:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:18:29] "GET /metrics HTTP/1.1" 200 36781 "" "Prometheus/2.51.0" 2024-08-17T18:18:30.209 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:30.243 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:30.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:30.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:30.345 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:30.376 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:30.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:30.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:30.391 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:18:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:18:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:18:30.478 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:30.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:30.545 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:30.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:30.646 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:30.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:30 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:18:30.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:30 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1464755288' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-08-17T18:18:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:30 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:18:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:30 smithi130 ceph-mon[110068]: osdmap e1442: 8 total, 7 up, 6 in 2024-08-17T18:18:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:30 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T18:18:30.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:30 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T18:18:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:30 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:18:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:30 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1464755288' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-08-17T18:18:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:30 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:18:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:30 smithi130 ceph-mon[105521]: osdmap e1442: 8 total, 7 up, 6 in 2024-08-17T18:18:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:30 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T18:18:30.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:30 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T18:18:30.668 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:30.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:30.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:30.701 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:30 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:18:30.702 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:30 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1464755288' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-08-17T18:18:30.702 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:30 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:18:30.702 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:30 smithi179 ceph-mon[93519]: osdmap e1442: 8 total, 7 up, 6 in 2024-08-17T18:18:30.702 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:30 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T18:18:30.702 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:30 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T18:18:30.770 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:30.790 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:30.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:30.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:30.892 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:30.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:30.914 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:30.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:31.015 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:31.037 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:31.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:31.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:31.139 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:31.151 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 59 16 106494 106478 112.762 21.3125 0.0076545 0.00876478 2024-08-17T18:18:31.160 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:31.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:31.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:31.262 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:31.285 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:31.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:31.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:31.387 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:31.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:31.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:31.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:31.441 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:18:31.441 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T18:18:31.441 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T18:18:31.441 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [] 2024-08-17T18:18:31.441 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:18:31.519 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:31.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:31.556 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:31.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:31.594 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:31 smithi130 ceph-mon[110068]: pgmap v2759: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 6.3 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s; 57507/307494 objects degraded (18.702%); 8/307494 objects misplaced (0.003%) 2024-08-17T18:18:31.595 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:31 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T18:18:31.595 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:31 smithi130 ceph-mon[110068]: osdmap e1443: 8 total, 7 up, 6 in 2024-08-17T18:18:31.595 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 57507/307494 objects degraded (18.702%), 11 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:31.595 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:31 smithi130 ceph-mon[105521]: pgmap v2759: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 6.3 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s; 57507/307494 objects degraded (18.702%); 8/307494 objects misplaced (0.003%) 2024-08-17T18:18:31.595 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:31 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T18:18:31.595 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:31 smithi130 ceph-mon[105521]: osdmap e1443: 8 total, 7 up, 6 in 2024-08-17T18:18:31.596 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 57507/307494 objects degraded (18.702%), 11 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:31.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:31.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:31.687 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:31.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:31.788 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:31.813 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:31.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:31.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:31.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:31 smithi179 ceph-mon[93519]: pgmap v2759: 89 pgs: 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 6.3 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 79 MiB/s wr, 1.27k op/s; 57507/307494 objects degraded (18.702%); 8/307494 objects misplaced (0.003%) 2024-08-17T18:18:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:31 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T18:18:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:31 smithi179 ceph-mon[93519]: osdmap e1443: 8 total, 7 up, 6 in 2024-08-17T18:18:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 57507/307494 objects degraded (18.702%), 11 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:31.915 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:31.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:31.952 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:31.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:32.053 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:32.083 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:32.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:32.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:32.151 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:18:32.151351+0000 min lat: 0.00107633 max lat: 1.08107 avg lat: 0.00891375 2024-08-17T18:18:32.151 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:18:32.151 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 60 16 107698 107682 112.137 75.25 0.00512701 0.00891375 2024-08-17T18:18:32.185 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:32.223 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:32.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:32.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:32.277 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T18:18:32.324 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:32.357 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:18:32.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:32.366 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:32.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:32.467 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:32.509 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:32.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:32.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:32.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:32 smithi179 ceph-mon[93519]: osdmap e1444: 8 total, 7 up, 6 in 2024-08-17T18:18:32.611 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:32.643 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:32.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:32.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:32.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:32 smithi130 ceph-mon[105521]: osdmap e1444: 8 total, 7 up, 6 in 2024-08-17T18:18:32.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:32 smithi130 ceph-mon[110068]: osdmap e1444: 8 total, 7 up, 6 in 2024-08-17T18:18:32.681 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:18:32.745 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:32.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:32.791 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:32.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:32.892 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:32.922 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:32.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:32.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:33.024 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:33.064 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:33.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:33.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:33.151 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 61 16 109288 109272 111.928 99.375 0.00737633 0.00893003 2024-08-17T18:18:33.165 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:33.198 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:33.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:33.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:33.300 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:33.348 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:33.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:33.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:33.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:33.479 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:33.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:33.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:33.486 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:18:33.486 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:18:33.509 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:18:33.519 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:18:33.519 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:18:33.523 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:18:33.576 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:33 smithi130 ceph-mon[105521]: pgmap v2762: 99 pgs: 10 unknown, 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 58263/311985 objects degraded (18.675%); 8/311985 objects misplaced (0.003%) 2024-08-17T18:18:33.576 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:33 smithi130 ceph-mon[105521]: 50.1 scrub starts 2024-08-17T18:18:33.576 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:33 smithi130 ceph-mon[105521]: 50.2 scrub starts 2024-08-17T18:18:33.576 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:33 smithi130 ceph-mon[110068]: pgmap v2762: 99 pgs: 10 unknown, 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 58263/311985 objects degraded (18.675%); 8/311985 objects misplaced (0.003%) 2024-08-17T18:18:33.576 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:33 smithi130 ceph-mon[110068]: 50.1 scrub starts 2024-08-17T18:18:33.576 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:33 smithi130 ceph-mon[110068]: 50.2 scrub starts 2024-08-17T18:18:33.577 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:18:33.577 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:18:33.581 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:33.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:33 smithi179 ceph-mon[93519]: pgmap v2762: 99 pgs: 10 unknown, 12 active+clean+remapped, 12 active+undersized, 11 active+undersized+degraded, 54 active+clean; 6.3 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s; 58263/311985 objects degraded (18.675%); 8/311985 objects misplaced (0.003%) 2024-08-17T18:18:33.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:33 smithi179 ceph-mon[93519]: 50.1 scrub starts 2024-08-17T18:18:33.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:33 smithi179 ceph-mon[93519]: 50.2 scrub starts 2024-08-17T18:18:33.605 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:18:33.605 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:18:33.609 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:33.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:33.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:33.633 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:18:33.633 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:18:33.667 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:18:33.667 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:18:33.697 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:18:33.698 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:18:33.711 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:33.738 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:33.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:33.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:33.840 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:33.878 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:33.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:33.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:33.979 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:34.011 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:34.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:34.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:34.114 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:34.151 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 62 16 110901 110885 111.748 100.812 0.0291637 0.00894375 2024-08-17T18:18:34.155 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:34.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:34.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:34.256 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:34.286 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:34.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:34.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:34.388 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:34.421 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:34.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:34.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:34.523 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:34.552 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:34.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:34.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:34.654 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:34.659 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:34 smithi179 ceph-mon[93519]: 50.11 scrub starts 2024-08-17T18:18:34.659 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:34 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3084625090' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:18:34.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:34 smithi130 ceph-mon[105521]: 50.11 scrub starts 2024-08-17T18:18:34.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:34 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3084625090' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:18:34.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:34 smithi130 ceph-mon[110068]: 50.11 scrub starts 2024-08-17T18:18:34.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:34 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3084625090' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:18:34.681 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:18:34.682 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:18:34.696 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:34.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:34.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:34.798 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:34.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:34.829 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:34.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:34.930 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:34.963 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:34.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:34.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:35.065 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:35.095 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:35.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:35.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:35.151 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 63 16 112655 112639 111.714 109.625 0.0186932 0.00894698 2024-08-17T18:18:35.197 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:35.240 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:35.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:35.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:35.342 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:35.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:35.376 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:35.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:35.468 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T18:18:35.478 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:35.499 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:18:35.500 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:18:35.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:35.528 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:35.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:35.629 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:35.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:35 smithi130 ceph-mon[105521]: pgmap v2763: 99 pgs: 2 unknown, 12 active+clean+remapped, 12 active+undersized, 17 active+undersized+degraded, 56 active+clean; 6.6 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 98 MiB/s wr, 1.57k op/s; 60621/324177 objects degraded (18.700%); 8/324177 objects misplaced (0.002%); 21 KiB/s, 0 objects/s recovering 2024-08-17T18:18:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:35 smithi130 ceph-mon[105521]: 50.12 scrub starts 2024-08-17T18:18:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:35 smithi130 ceph-mon[110068]: pgmap v2763: 99 pgs: 2 unknown, 12 active+clean+remapped, 12 active+undersized, 17 active+undersized+degraded, 56 active+clean; 6.6 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 98 MiB/s wr, 1.57k op/s; 60621/324177 objects degraded (18.700%); 8/324177 objects misplaced (0.002%); 21 KiB/s, 0 objects/s recovering 2024-08-17T18:18:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:35 smithi130 ceph-mon[110068]: 50.12 scrub starts 2024-08-17T18:18:35.667 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:35.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:35.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:35.769 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:35.809 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:35.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:35.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:35.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:35 smithi179 ceph-mon[93519]: pgmap v2763: 99 pgs: 2 unknown, 12 active+clean+remapped, 12 active+undersized, 17 active+undersized+degraded, 56 active+clean; 6.6 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 98 MiB/s wr, 1.57k op/s; 60621/324177 objects degraded (18.700%); 8/324177 objects misplaced (0.002%); 21 KiB/s, 0 objects/s recovering 2024-08-17T18:18:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:35 smithi179 ceph-mon[93519]: 50.12 scrub starts 2024-08-17T18:18:35.844 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:18:35.911 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:35.944 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:35.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:35.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:36.046 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:36.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:36.072 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:36.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:36.154 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 64 16 114285 114269 111.56 101.875 0.0136011 0.00895918 2024-08-17T18:18:36.173 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:36.198 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:36.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:36.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:36.299 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:36.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:36.336 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:36.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:36.437 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:36.480 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:36.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:36.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:36.581 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:36.616 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:36.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:36.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:36.643 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 7, 1, 6] out_osds: [5, 4] dead_osds: [2] live_osds: [1, 4, 5, 6, 7, 0, 3] 2024-08-17T18:18:36.643 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:18:36.643 INFO:tasks.thrashosds.thrasher:Adding osd 4 2024-08-17T18:18:36.643 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd in 4 2024-08-17T18:18:36.679 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 60621/324177 objects degraded (18.700%), 17 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:36.679 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 60621/324177 objects degraded (18.700%), 17 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:36.718 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:36.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:36.754 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:36.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:36.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 60621/324177 objects degraded (18.700%), 17 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:36.855 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:36.884 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:36.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:36.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:36.985 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:37.011 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:37.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:37.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:37.074 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:18:37.114 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:37.152 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 65 16 115647 115631 111.152 85.125 0.0186825 0.00899206 2024-08-17T18:18:37.156 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:37.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:37.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:37.258 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:37.295 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:37.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:37.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:37.397 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:37.428 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:37.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:37.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:37.441 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:18:37.462 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:37 smithi130 ceph-mon[105521]: pgmap v2764: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 6.7 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 71 MiB/s wr, 1.14k op/s; 61412/328845 objects degraded (18.675%); 8/328845 objects misplaced (0.002%); 41 KiB/s, 0 objects/s recovering 2024-08-17T18:18:37.462 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:37 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3484517515' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:18:37.462 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:37 smithi130 ceph-mon[110068]: pgmap v2764: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 6.7 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 71 MiB/s wr, 1.14k op/s; 61412/328845 objects degraded (18.675%); 8/328845 objects misplaced (0.002%); 41 KiB/s, 0 objects/s recovering 2024-08-17T18:18:37.462 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:37 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3484517515' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:18:37.501 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:18:37.529 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:37.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:37.590 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:37.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:37.691 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:37.718 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:37.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:37.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:37.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:37.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:37 smithi179 ceph-mon[93519]: pgmap v2764: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 6.7 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 71 MiB/s wr, 1.14k op/s; 61412/328845 objects degraded (18.675%); 8/328845 objects misplaced (0.002%); 41 KiB/s, 0 objects/s recovering 2024-08-17T18:18:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:37 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3484517515' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:18:37.860 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:37.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:37.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:37.962 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:37.990 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:37.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:37.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:38.092 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:38.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:38.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:38.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:38.152 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 66 16 117026 117010 110.774 86.1875 0.00580435 0.00901505 2024-08-17T18:18:38.215 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:38.236 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:38.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:38.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:38.337 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:38.359 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:38.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:38.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:38.461 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:38.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:38.499 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:38.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:38.602 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:38.633 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:38.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:38.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:38 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:18:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:38 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3484517515' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:18:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:38 smithi130 ceph-mon[105521]: osdmap e1445: 8 total, 7 up, 6 in 2024-08-17T18:18:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:38 smithi130 ceph-mon[105521]: 50.1 scrub starts 2024-08-17T18:18:38.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:38 smithi130 ceph-mon[105521]: 50.9 scrub starts 2024-08-17T18:18:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:38 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:18:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:38 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3484517515' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:18:38.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:38 smithi130 ceph-mon[110068]: osdmap e1445: 8 total, 7 up, 6 in 2024-08-17T18:18:38.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:38 smithi130 ceph-mon[110068]: 50.1 scrub starts 2024-08-17T18:18:38.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:38 smithi130 ceph-mon[110068]: 50.9 scrub starts 2024-08-17T18:18:38.735 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:38.776 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:38.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:38.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:38.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:38 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:18:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:38 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3484517515' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:18:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:38 smithi179 ceph-mon[93519]: osdmap e1445: 8 total, 7 up, 6 in 2024-08-17T18:18:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:38 smithi179 ceph-mon[93519]: 50.1 scrub starts 2024-08-17T18:18:38.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:38 smithi179 ceph-mon[93519]: 50.9 scrub starts 2024-08-17T18:18:38.877 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:38.916 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:38.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:38.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:39.017 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:39.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:39.046 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:39.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:39.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:39.152 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 67 16 118152 118136 110.171 70.375 0.00530602 0.00907229 2024-08-17T18:18:39.176 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:39.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:39.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:39.278 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:39.323 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:39.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:39.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:39.424 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:39.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:39.515 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:39.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:39.616 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:39.640 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:39.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:39.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:39.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:39 smithi130 ceph-mon[105521]: pgmap v2766: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 6.8 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 58 MiB/s wr, 921 op/s; 61910/331965 objects degraded (18.650%); 8/331965 objects misplaced (0.002%); 35 KiB/s, 0 objects/s recovering 2024-08-17T18:18:39.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:39 smithi130 ceph-mon[105521]: 50.11 scrub starts 2024-08-17T18:18:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:39 smithi130 ceph-mon[105521]: 50.1 scrub ok 2024-08-17T18:18:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:39 smithi130 ceph-mon[105521]: 50.2 scrub starts 2024-08-17T18:18:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:39 smithi130 ceph-mon[110068]: pgmap v2766: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 6.8 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 58 MiB/s wr, 921 op/s; 61910/331965 objects degraded (18.650%); 8/331965 objects misplaced (0.002%); 35 KiB/s, 0 objects/s recovering 2024-08-17T18:18:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:39 smithi130 ceph-mon[110068]: 50.11 scrub starts 2024-08-17T18:18:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:39 smithi130 ceph-mon[110068]: 50.1 scrub ok 2024-08-17T18:18:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:39 smithi130 ceph-mon[110068]: 50.2 scrub starts 2024-08-17T18:18:39.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:39 smithi179 ceph-mon[93519]: pgmap v2766: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 6.8 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 58 MiB/s wr, 921 op/s; 61910/331965 objects degraded (18.650%); 8/331965 objects misplaced (0.002%); 35 KiB/s, 0 objects/s recovering 2024-08-17T18:18:39.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:39 smithi179 ceph-mon[93519]: 50.11 scrub starts 2024-08-17T18:18:39.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:39 smithi179 ceph-mon[93519]: 50.1 scrub ok 2024-08-17T18:18:39.722 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:39 smithi179 ceph-mon[93519]: 50.2 scrub starts 2024-08-17T18:18:39.742 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:39.777 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:39.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:39.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:39.878 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:39.918 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:39.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:39.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:40.011 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:18:40.011 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:18:40.020 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:40.060 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:40.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:40.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:40.153 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 68 16 119252 119236 109.562 68.75 0.0235858 0.00912275 2024-08-17T18:18:40.162 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:18:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:18:39] "GET /metrics HTTP/1.1" 200 36781 "" "Prometheus/2.51.0" 2024-08-17T18:18:40.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:40.199 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:40.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:40.300 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:40.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:40.333 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:40.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:40.342 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:18:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:18:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:18:40.434 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:40.465 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:40.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:40.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:40.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:40.595 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:40.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:40.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:40.697 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:40.731 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:40.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:40.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:40.750 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:40 smithi179 ceph-mon[93519]: 50.9 scrub ok 2024-08-17T18:18:40.751 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:40 smithi179 ceph-mon[93519]: 50.11 scrub ok 2024-08-17T18:18:40.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T18:18:40.833 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:40.865 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:40.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:40.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:40.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:40 smithi130 ceph-mon[105521]: 50.9 scrub ok 2024-08-17T18:18:40.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:40 smithi130 ceph-mon[105521]: 50.11 scrub ok 2024-08-17T18:18:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:40 smithi130 ceph-mon[110068]: 50.9 scrub ok 2024-08-17T18:18:40.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:40 smithi130 ceph-mon[110068]: 50.11 scrub ok 2024-08-17T18:18:40.966 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:40.999 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:41.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:41.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:41.102 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:41.139 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:41.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:41.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:41.153 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 69 16 120713 120697 109.297 91.3125 0.00959557 0.00914531 2024-08-17T18:18:41.181 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:18:41.241 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:41.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:41.309 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:41.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:41.410 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:41.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:41.456 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:41.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:41.557 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:41.612 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:41 smithi179 ceph-mon[93519]: pgmap v2767: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.1 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 96 MiB/s wr, 1.53k op/s; 65420/349845 objects degraded (18.700%); 8/349845 objects misplaced (0.002%); 31 KiB/s, 0 objects/s recovering 2024-08-17T18:18:41.612 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:41 smithi179 ceph-mon[93519]: 50.2 scrub ok 2024-08-17T18:18:41.612 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:41 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 65420/349845 objects degraded (18.700%), 18 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:41.612 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:41 smithi179 ceph-mon[93519]: 50.12 scrub starts 2024-08-17T18:18:41.612 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:41 smithi179 ceph-mon[93519]: 50.19 scrub starts 2024-08-17T18:18:41.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:41.614 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:41.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:41.715 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:41.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:41.749 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:41.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:41.850 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:41.871 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:41.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:41.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:41.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:41 smithi130 ceph-mon[105521]: pgmap v2767: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.1 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 96 MiB/s wr, 1.53k op/s; 65420/349845 objects degraded (18.700%); 8/349845 objects misplaced (0.002%); 31 KiB/s, 0 objects/s recovering 2024-08-17T18:18:41.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:41 smithi130 ceph-mon[105521]: 50.2 scrub ok 2024-08-17T18:18:41.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:41 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 65420/349845 objects degraded (18.700%), 18 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:41.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:41 smithi130 ceph-mon[105521]: 50.12 scrub starts 2024-08-17T18:18:41.886 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:41 smithi130 ceph-mon[105521]: 50.19 scrub starts 2024-08-17T18:18:41.886 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:41 smithi130 ceph-mon[110068]: pgmap v2767: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.1 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 96 MiB/s wr, 1.53k op/s; 65420/349845 objects degraded (18.700%); 8/349845 objects misplaced (0.002%); 31 KiB/s, 0 objects/s recovering 2024-08-17T18:18:41.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:41 smithi130 ceph-mon[110068]: 50.2 scrub ok 2024-08-17T18:18:41.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:41 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 65420/349845 objects degraded (18.700%), 18 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:41.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:41 smithi130 ceph-mon[110068]: 50.12 scrub starts 2024-08-17T18:18:41.887 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:41 smithi130 ceph-mon[110068]: 50.19 scrub starts 2024-08-17T18:18:41.974 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:42.014 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:42.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:42.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:42.115 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:42.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:42.147 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:42.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:42.152 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 70 16 122070 122054 108.947 84.8125 0.0066979 0.0091741 2024-08-17T18:18:42.248 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:42.283 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:42.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:42.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:42.385 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:42.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:42.419 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:42.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:42.520 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:42.549 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:42.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:42.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:42.651 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:42.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:42.684 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:42.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:42.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:42 smithi179 ceph-mon[93519]: 50.12 scrub ok 2024-08-17T18:18:42.719 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:18:42.785 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:42.823 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:42.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:42.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:42 smithi130 ceph-mon[105521]: 50.12 scrub ok 2024-08-17T18:18:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:18:42.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:42 smithi130 ceph-mon[110068]: 50.12 scrub ok 2024-08-17T18:18:42.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:18:42.924 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:42.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:42.956 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:42.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:43.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:43.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:43.095 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:43.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:43.153 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 71 16 123156 123140 108.369 67.875 0.00438819 0.00922338 2024-08-17T18:18:43.197 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:43.210 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:18:43.210 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:18:43.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:43.220 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:43.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:43.321 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:43.352 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:43.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:43.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:43.454 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:43.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:43.479 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:43.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:43.580 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:43.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:43.603 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:43.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:43.704 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:43.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:43.729 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:43.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:43.830 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:43 smithi179 ceph-mon[93519]: pgmap v2768: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.1 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 65420/349845 objects degraded (18.700%); 8/349845 objects misplaced (0.002%); 26 KiB/s, 0 objects/s recovering 2024-08-17T18:18:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:43 smithi179 ceph-mon[93519]: 50.19 scrub ok 2024-08-17T18:18:43.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:43.865 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:43.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:43.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:43 smithi130 ceph-mon[105521]: pgmap v2768: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.1 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 65420/349845 objects degraded (18.700%); 8/349845 objects misplaced (0.002%); 26 KiB/s, 0 objects/s recovering 2024-08-17T18:18:43.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:43 smithi130 ceph-mon[105521]: 50.19 scrub ok 2024-08-17T18:18:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:43 smithi130 ceph-mon[110068]: pgmap v2768: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.1 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 79 MiB/s wr, 1.26k op/s; 65420/349845 objects degraded (18.700%); 8/349845 objects misplaced (0.002%); 26 KiB/s, 0 objects/s recovering 2024-08-17T18:18:43.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:43 smithi130 ceph-mon[110068]: 50.19 scrub ok 2024-08-17T18:18:43.946 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T18:18:43.967 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:44.002 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:44.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:44.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:44.104 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:44.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:44.136 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:44.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:44.153 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 72 16 123980 123964 107.579 51.5 0.0080838 0.00929061 2024-08-17T18:18:44.237 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:44.270 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:18:44.277 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:44.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:44.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:44.379 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:44.415 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:44.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:44.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:44.517 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:44.545 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:44.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:44.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:44.647 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:44.682 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:44.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:44.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:44.784 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:44.808 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:44.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:44.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:44.910 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:44.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:44.940 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:44.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:45.041 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:45.076 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:45.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:45.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:45.153 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 73 16 125488 125472 107.396 94.25 0.00734337 0.00930596 2024-08-17T18:18:45.179 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:45.213 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:45.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:45.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:45.315 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:45.340 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:45.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:45.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:45.442 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:45.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:45.464 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:45.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:45.565 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:45.578 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:45 smithi179 ceph-mon[93519]: pgmap v2769: 99 pgs: 1 active+clean+scrubbing, 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 56 active+clean; 7.5 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 68864/368508 objects degraded (18.687%); 8/368508 objects misplaced (0.002%); 13 KiB/s, 0 objects/s recovering 2024-08-17T18:18:45.592 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:45.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:45.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:45.694 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:45.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:45.729 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:45.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:45.830 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:45.860 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:45.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:45.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:45.904 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:45 smithi130 ceph-mon[105521]: pgmap v2769: 99 pgs: 1 active+clean+scrubbing, 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 56 active+clean; 7.5 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 68864/368508 objects degraded (18.687%); 8/368508 objects misplaced (0.002%); 13 KiB/s, 0 objects/s recovering 2024-08-17T18:18:45.904 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:45 smithi130 ceph-mon[110068]: pgmap v2769: 99 pgs: 1 active+clean+scrubbing, 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 56 active+clean; 7.5 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 68864/368508 objects degraded (18.687%); 8/368508 objects misplaced (0.002%); 13 KiB/s, 0 objects/s recovering 2024-08-17T18:18:45.962 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:46.002 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:46.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:46.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:46.104 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:46.131 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:46.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:46.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:46.153 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 74 16 126725 126709 106.99 77.3125 0.0051977 0.00934242 2024-08-17T18:18:46.233 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:46.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:46.272 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:46.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:46.274 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:18:46.275 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:18:46.373 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:46.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:46.402 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:46.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:46.418 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:18:46.503 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:46.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:46.532 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:46.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:46.633 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:46.672 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:46.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:46.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:46.774 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:46.803 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:46.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:46.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 68864/368508 objects degraded (18.687%), 18 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:46.905 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:46.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 68864/368508 objects degraded (18.687%), 18 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:46.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 68864/368508 objects degraded (18.687%), 18 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:46.942 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:46.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:47.043 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:47.059 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T18:18:47.077 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:47.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:47.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:47.153 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 75 16 128240 128224 106.826 94.6875 0.00874873 0.00935686 2024-08-17T18:18:47.180 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:47.213 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:47.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:47.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:47.315 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:47.348 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:47.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:47.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:47.449 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:47.460 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:18:47.497 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:47.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:47.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:47.575 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:47 smithi130 ceph-mon[105521]: pgmap v2770: 99 pgs: 1 active+clean+scrubbing, 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 56 active+clean; 7.5 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 68864/368508 objects degraded (18.687%); 8/368508 objects misplaced (0.002%) 2024-08-17T18:18:47.576 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:47 smithi130 ceph-mon[110068]: pgmap v2770: 99 pgs: 1 active+clean+scrubbing, 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 56 active+clean; 7.5 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 68864/368508 objects degraded (18.687%); 8/368508 objects misplaced (0.002%) 2024-08-17T18:18:47.599 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:47.626 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:47.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:47.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:47.728 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:47.766 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:47.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:47.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:47.791 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:47 smithi179 ceph-mon[93519]: pgmap v2770: 99 pgs: 1 active+clean+scrubbing, 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 56 active+clean; 7.5 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 68864/368508 objects degraded (18.687%); 8/368508 objects misplaced (0.002%) 2024-08-17T18:18:47.868 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:47.904 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:47.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:47.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:47.945 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:18:48.006 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:48.038 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:48.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:48.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:48.140 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:48.153 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 76 16 129823 129807 106.722 98.9375 0.00786763 0.00936582 2024-08-17T18:18:48.167 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:48.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:48.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:48.269 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:48.297 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:48.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:48.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:48.398 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:48.426 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:48.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:48.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:48.528 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:48.557 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:48.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:48.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:48.659 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:48.688 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:48.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:48.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:48.790 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:48.817 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:48.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:48.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:48.919 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:48.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:48.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:48.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:49.059 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:49.083 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:49.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:49.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:49.154 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 77 16 131243 131227 106.488 88.75 0.00874881 0.00938654 2024-08-17T18:18:49.185 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:49.218 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:49.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:49.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:49.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:49.359 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:49.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:49.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:49.415 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:18:49.416 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:18:49.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T18:18:49.138+0000 7fa856f19640 -1 mon.a@0(leader).osd e1445 definitely_dead 0 2024-08-17T18:18:49.451 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:18:49.451 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:18:49.453 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:18:49.461 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:49.483 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:49.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:49.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:49.484 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:18:49.485 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:18:49.496 INFO:teuthology.orchestra.run.smithi130.stderr:marked in osd.4. 2024-08-17T18:18:49.520 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:18:49.522 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:18:49.522 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:18:49.547 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:18:49.547 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:18:49.565 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:18:49.565 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:18:49.581 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:18:49.581 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:18:49.584 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:49.605 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:49.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:49.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:49.706 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:49.727 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:49.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:49.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:49.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:49 smithi130 ceph-mon[105521]: pgmap v2771: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.5 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 69240/370611 objects degraded (18.683%); 8/370611 objects misplaced (0.002%) 2024-08-17T18:18:49.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:49 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/184984610' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-08-17T18:18:49.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:49 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-08-17T18:18:49.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:49 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/323681856' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:18:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:49 smithi130 ceph-mon[110068]: pgmap v2771: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.5 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 69240/370611 objects degraded (18.683%); 8/370611 objects misplaced (0.002%) 2024-08-17T18:18:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:49 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/184984610' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-08-17T18:18:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:49 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-08-17T18:18:49.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:49 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/323681856' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:18:49.829 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:49 smithi179 ceph-mon[93519]: pgmap v2771: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.5 GiB data, 26 GiB used, 510 GiB / 536 GiB avail; 69240/370611 objects degraded (18.683%); 8/370611 objects misplaced (0.002%) 2024-08-17T18:18:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:49 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/184984610' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-08-17T18:18:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:49 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-08-17T18:18:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:49 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/323681856' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:18:49.851 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:49.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:49.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:49.953 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:49.977 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:49.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:49.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:50.079 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:50.104 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:50.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:50.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:50.154 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 78 16 131733 131717 105.516 30.625 0.00364591 0.00939581 2024-08-17T18:18:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:18:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:18:49] "GET /metrics HTTP/1.1" 200 36796 "" "Prometheus/2.51.0" 2024-08-17T18:18:50.206 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:50.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:50.232 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:50.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:50.243 INFO:tasks.thrashosds.thrasher:Added osd 4 2024-08-17T18:18:50.334 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:50.357 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:50.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:50.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:50.459 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:50.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:50.483 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:50.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:50.520 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:18:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:18:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:18:50.584 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:50.605 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:50.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:50.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:50.707 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:50.730 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:50.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:50.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:50.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:50 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-08-17T18:18:50.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:50 smithi179 ceph-mon[93519]: osdmap e1446: 8 total, 7 up, 7 in 2024-08-17T18:18:50.789 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:50 smithi179 ceph-mon[93519]: osdmap e1447: 8 total, 7 up, 7 in 2024-08-17T18:18:50.813 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:18:50.813 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:18:50.813 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:18:50.814 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:18:50.814 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:18:50.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:50.853 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:50.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:50.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:50.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:50 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-08-17T18:18:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:50 smithi130 ceph-mon[105521]: osdmap e1446: 8 total, 7 up, 7 in 2024-08-17T18:18:50.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:50 smithi130 ceph-mon[105521]: osdmap e1447: 8 total, 7 up, 7 in 2024-08-17T18:18:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:50 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-08-17T18:18:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:50 smithi130 ceph-mon[110068]: osdmap e1446: 8 total, 7 up, 7 in 2024-08-17T18:18:50.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:50 smithi130 ceph-mon[110068]: osdmap e1447: 8 total, 7 up, 7 in 2024-08-17T18:18:50.955 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:50.977 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:50.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:50.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:51.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:51.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:51.102 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:51.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:51.154 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 79 16 131733 131717 104.18 0 - 0.00939581 2024-08-17T18:18:51.204 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:51.226 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:51.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:51.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:51.328 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:51.351 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:51.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:51.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:51.453 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:51.470 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T18:18:51.479 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:51.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:51.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:51.581 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:51.604 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:51.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:51.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:51.706 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:51.741 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:51.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:51.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:51 smithi179 ceph-mon[93519]: pgmap v2773: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.9 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 72558/388152 objects degraded (18.693%); 8/388152 objects misplaced (0.002%) 2024-08-17T18:18:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 72558/388152 objects degraded (18.693%), 18 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:51 smithi179 ceph-mon[93519]: osdmap e1448: 8 total, 7 up, 7 in 2024-08-17T18:18:51.843 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:51.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:51.878 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:51.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:51.906 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:18:51.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:51 smithi130 ceph-mon[105521]: pgmap v2773: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.9 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 72558/388152 objects degraded (18.693%); 8/388152 objects misplaced (0.002%) 2024-08-17T18:18:51.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 72558/388152 objects degraded (18.693%), 18 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:51 smithi130 ceph-mon[105521]: osdmap e1448: 8 total, 7 up, 7 in 2024-08-17T18:18:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:51 smithi130 ceph-mon[110068]: pgmap v2773: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.9 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 72558/388152 objects degraded (18.693%); 8/388152 objects misplaced (0.002%) 2024-08-17T18:18:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 72558/388152 objects degraded (18.693%), 18 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:51 smithi130 ceph-mon[110068]: osdmap e1448: 8 total, 7 up, 7 in 2024-08-17T18:18:51.979 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:52.022 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:52.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:52.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:52.124 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:52.154 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:2024-08-17T18:18:52.154307+0000 min lat: 0.00107633 max lat: 2.21256 avg lat: 0.00966544 2024-08-17T18:18:52.154 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T18:18:52.154 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 80 16 132421 132405 103.416 21.5 0.00347522 0.00966544 2024-08-17T18:18:52.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:52.159 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:52.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:52.260 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:52.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:52.297 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:52.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:52.398 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:52.427 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:18:52.428 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:18:52.431 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:52.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:52.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:52.532 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:52.565 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:52.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:52.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:52.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[105521]: 50.0 starting backfill to osd.4 from (0'0,0'0] MAX to 1445'7470 2024-08-17T18:18:52.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[105521]: 50.d starting backfill to osd.4 from (0'0,0'0] MAX to 1445'8140 2024-08-17T18:18:52.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[105521]: 50.a starting backfill to osd.4 from (0'0,0'0] MAX to 1445'8283 2024-08-17T18:18:52.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[105521]: 50.10 starting backfill to osd.4 from (0'0,0'0] MAX to 1445'7473 2024-08-17T18:18:52.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[105521]: 50.f starting backfill to osd.4 from (0'0,0'0] MAX to 1445'8202 2024-08-17T18:18:52.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[105521]: 50.b starting backfill to osd.4 from (0'0,0'0] MAX to 1445'8298 2024-08-17T18:18:52.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[105521]: 50.2 starting backfill to osd.4 from (0'0,0'0] MAX to 1445'7491 2024-08-17T18:18:52.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[105521]: 50.12 starting backfill to osd.4 from (0'0,0'0] MAX to 1445'7510 2024-08-17T18:18:52.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[105521]: pgmap v2776: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.9 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 29 MiB/s wr, 461 op/s; 72558/388152 objects degraded (18.693%); 8/388152 objects misplaced (0.002%) 2024-08-17T18:18:52.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[110068]: 50.0 starting backfill to osd.4 from (0'0,0'0] MAX to 1445'7470 2024-08-17T18:18:52.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[110068]: 50.d starting backfill to osd.4 from (0'0,0'0] MAX to 1445'8140 2024-08-17T18:18:52.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[110068]: 50.a starting backfill to osd.4 from (0'0,0'0] MAX to 1445'8283 2024-08-17T18:18:52.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[110068]: 50.10 starting backfill to osd.4 from (0'0,0'0] MAX to 1445'7473 2024-08-17T18:18:52.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[110068]: 50.f starting backfill to osd.4 from (0'0,0'0] MAX to 1445'8202 2024-08-17T18:18:52.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[110068]: 50.b starting backfill to osd.4 from (0'0,0'0] MAX to 1445'8298 2024-08-17T18:18:52.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[110068]: 50.2 starting backfill to osd.4 from (0'0,0'0] MAX to 1445'7491 2024-08-17T18:18:52.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[110068]: 50.12 starting backfill to osd.4 from (0'0,0'0] MAX to 1445'7510 2024-08-17T18:18:52.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:52 smithi130 ceph-mon[110068]: pgmap v2776: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.9 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 29 MiB/s wr, 461 op/s; 72558/388152 objects degraded (18.693%); 8/388152 objects misplaced (0.002%) 2024-08-17T18:18:52.668 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:52.704 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:52.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:52.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:52.806 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:52.828 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:52.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:52.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:52.829 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:52 smithi179 ceph-mon[93519]: 50.0 starting backfill to osd.4 from (0'0,0'0] MAX to 1445'7470 2024-08-17T18:18:52.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:52 smithi179 ceph-mon[93519]: 50.d starting backfill to osd.4 from (0'0,0'0] MAX to 1445'8140 2024-08-17T18:18:52.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:52 smithi179 ceph-mon[93519]: 50.a starting backfill to osd.4 from (0'0,0'0] MAX to 1445'8283 2024-08-17T18:18:52.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:52 smithi179 ceph-mon[93519]: 50.10 starting backfill to osd.4 from (0'0,0'0] MAX to 1445'7473 2024-08-17T18:18:52.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:52 smithi179 ceph-mon[93519]: 50.f starting backfill to osd.4 from (0'0,0'0] MAX to 1445'8202 2024-08-17T18:18:52.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:52 smithi179 ceph-mon[93519]: 50.b starting backfill to osd.4 from (0'0,0'0] MAX to 1445'8298 2024-08-17T18:18:52.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:52 smithi179 ceph-mon[93519]: 50.2 starting backfill to osd.4 from (0'0,0'0] MAX to 1445'7491 2024-08-17T18:18:52.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:52 smithi179 ceph-mon[93519]: 50.12 starting backfill to osd.4 from (0'0,0'0] MAX to 1445'7510 2024-08-17T18:18:52.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:52 smithi179 ceph-mon[93519]: pgmap v2776: 99 pgs: 12 active+clean+remapped, 12 active+undersized, 18 active+undersized+degraded, 57 active+clean; 7.9 GiB data, 27 GiB used, 509 GiB / 536 GiB avail; 29 MiB/s wr, 461 op/s; 72558/388152 objects degraded (18.693%); 8/388152 objects misplaced (0.002%) 2024-08-17T18:18:52.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:52.967 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:52.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:52.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:53.069 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:53.110 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:53.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:53.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:53.154 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 81 16 133676 133660 103.107 78.4375 0.0106186 0.0096946 2024-08-17T18:18:53.211 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:53.244 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:53.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:53.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:53.346 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:53.377 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:53.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:53.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:53.479 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:53.510 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:53.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:53.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:53.612 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:53.649 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:53.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:53.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:53.751 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:53.791 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:53.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:53.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:53.893 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:53.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:53.934 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:53.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:54.035 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:54.078 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:54.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:54.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:54.154 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 82 16 135051 135035 102.898 85.9375 0.0175502 0.00971383 2024-08-17T18:18:54.180 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:54.219 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:54.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:54.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:54.321 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:54.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:54.360 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:54.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:54.429 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:18:54.461 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:54.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:54.499 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:54.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:54.599 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:18:54.599 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T18:18:54.599 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T18:18:54.600 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T18:18:54.600 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.600 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36960.0:211976 49.0 49.eb210650 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1421)", 2024-08-17T18:18:54.600 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:16:33.650996+0000", 2024-08-17T18:18:54.600 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 140.94548765900001, 2024-08-17T18:18:54.600 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98308165800000002, 2024-08-17T18:18:54.600 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.600 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:18:54.600 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:18:54.600 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36960", 2024-08-17T18:18:54.600 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:18:54.600 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 211976 2024-08-17T18:18:54.600 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.601 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.601 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.601 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.601 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.650996+0000", 2024-08-17T18:18:54.601 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.601 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.601 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.601 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.601 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.650996+0000", 2024-08-17T18:18:54.601 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.601 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.601 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.602 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.602 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.650995+0000", 2024-08-17T18:18:54.602 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:18:54.602 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.602 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.602 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.602 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.650999+0000", 2024-08-17T18:18:54.602 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5879999999999999e-06 2024-08-17T18:18:54.602 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.602 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.602 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.603 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.651006+0000", 2024-08-17T18:18:54.603 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7700000000000004e-06 2024-08-17T18:18:54.603 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.603 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.603 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.603 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.651010+0000", 2024-08-17T18:18:54.603 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5539999999999999e-06 2024-08-17T18:18:54.603 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.603 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.603 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.603 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.651040+0000", 2024-08-17T18:18:54.603 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9504000000000001e-05 2024-08-17T18:18:54.603 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.604 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.604 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:18:54.604 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.651046+0000", 2024-08-17T18:18:54.604 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5890000000000002e-06 2024-08-17T18:18:54.604 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.604 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.604 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.604 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:34.634065+0000", 2024-08-17T18:18:54.604 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98301972900000001 2024-08-17T18:18:54.604 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.604 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.604 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.604 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:34.634077+0000", 2024-08-17T18:18:54.605 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.22e-05 2024-08-17T18:18:54.605 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.605 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.605 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.605 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.605 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.605 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36960.0:211995 49.0 49.89f3ff50 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1421)", 2024-08-17T18:18:54.605 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:16:33.674686+0000", 2024-08-17T18:18:54.605 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 140.92179784800001, 2024-08-17T18:18:54.605 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.95975966000000001, 2024-08-17T18:18:54.605 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.605 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:18:54.605 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:18:54.606 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36960", 2024-08-17T18:18:54.606 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:18:54.606 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 211995 2024-08-17T18:18:54.606 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.606 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.606 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.606 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.606 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674686+0000", 2024-08-17T18:18:54.606 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.606 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.606 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.606 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.607 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674686+0000", 2024-08-17T18:18:54.607 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.607 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.607 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.607 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.607 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674684+0000", 2024-08-17T18:18:54.607 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:18:54.607 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.607 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.607 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.607 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674690+0000", 2024-08-17T18:18:54.607 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3000000000000001e-06 2024-08-17T18:18:54.607 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.608 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.608 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.608 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674700+0000", 2024-08-17T18:18:54.608 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0628999999999999e-05 2024-08-17T18:18:54.608 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.608 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.608 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.608 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674706+0000", 2024-08-17T18:18:54.608 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2730000000000002e-06 2024-08-17T18:18:54.608 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.608 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.608 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.608 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674743+0000", 2024-08-17T18:18:54.609 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7249000000000002e-05 2024-08-17T18:18:54.609 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.609 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.609 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:18:54.609 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674750+0000", 2024-08-17T18:18:54.609 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.8909999999999998e-06 2024-08-17T18:18:54.609 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.609 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.609 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.609 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:34.634437+0000", 2024-08-17T18:18:54.609 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.95968738099999995 2024-08-17T18:18:54.609 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.609 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.610 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.610 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:34.634445+0000", 2024-08-17T18:18:54.610 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2169999999999994e-06 2024-08-17T18:18:54.610 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.610 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.610 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.610 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.610 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.610 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:73865 50.9 50:9e9dc258:::benchmark_data_smithi130_398002_object73864:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-08-17T18:18:54.610 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:05.099627+0000", 2024-08-17T18:18:54.610 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 49.496856102000002, 2024-08-17T18:18:54.610 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.14423280799999999, 2024-08-17T18:18:54.610 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.611 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:18:54.611 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:18:54.611 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:18:54.611 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:18:54.611 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 73865 2024-08-17T18:18:54.611 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.611 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.611 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.611 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.611 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.099627+0000", 2024-08-17T18:18:54.611 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.611 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.612 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.612 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.612 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.099627+0000", 2024-08-17T18:18:54.612 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.612 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.612 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.612 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.612 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.099626+0000", 2024-08-17T18:18:54.612 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:18:54.612 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.612 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.612 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.612 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.099703+0000", 2024-08-17T18:18:54.613 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.6724000000000006e-05 2024-08-17T18:18:54.613 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.613 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.613 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.613 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.099716+0000", 2024-08-17T18:18:54.613 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2601e-05 2024-08-17T18:18:54.613 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.613 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.613 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.613 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.099721+0000", 2024-08-17T18:18:54.613 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7629999999999997e-06 2024-08-17T18:18:54.613 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.613 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.614 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.614 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.099793+0000", 2024-08-17T18:18:54.614 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.1972999999999997e-05 2024-08-17T18:18:54.614 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.614 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.614 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:54.614 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.099867+0000", 2024-08-17T18:18:54.614 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3478000000000004e-05 2024-08-17T18:18:54.614 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.614 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.614 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,7", 2024-08-17T18:18:54.615 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.099960+0000", 2024-08-17T18:18:54.615 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:54.616 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.3259999999999998e-05 2024-08-17T18:18:54.616 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.616 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.616 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:18:54.616 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.102408+0000", 2024-08-17T18:18:54.616 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002447702 2024-08-17T18:18:54.617 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.617 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.617 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.617 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.102441+0000", 2024-08-17T18:18:54.617 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3609999999999998e-05 2024-08-17T18:18:54.617 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.617 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.617 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.617 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.243772+0000", 2024-08-17T18:18:54.617 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.141330437 2024-08-17T18:18:54.617 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.617 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.618 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:18:54.618 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.243824+0000", 2024-08-17T18:18:54.618 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2596999999999998e-05 2024-08-17T18:18:54.618 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.618 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.618 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.618 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.243860+0000", 2024-08-17T18:18:54.618 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.5760000000000003e-05 2024-08-17T18:18:54.618 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.618 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.618 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.618 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.619 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.619 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.47132.0:73866 50.4 e1440/1434 50:288c03de:::benchmark_data_smithi130_398002_object73865:head v 1440'4667, mlcod=1440'4665)", 2024-08-17T18:18:54.619 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:05.104166+0000", 2024-08-17T18:18:54.619 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 49.492317346, 2024-08-17T18:18:54.619 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.140285039, 2024-08-17T18:18:54.619 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.619 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:18:54.619 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.619 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.619 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.619 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104166+0000", 2024-08-17T18:18:54.619 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.619 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.619 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.620 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.620 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104166+0000", 2024-08-17T18:18:54.620 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.620 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.620 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.620 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.620 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104165+0000", 2024-08-17T18:18:54.620 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:18:54.620 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.620 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.620 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.620 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104224+0000", 2024-08-17T18:18:54.620 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.834e-05 2024-08-17T18:18:54.621 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.621 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.621 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.621 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104232+0000", 2024-08-17T18:18:54.621 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.8170000000000005e-06 2024-08-17T18:18:54.621 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.621 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.621 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.621 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104237+0000", 2024-08-17T18:18:54.621 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1819999999999999e-06 2024-08-17T18:18:54.621 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.621 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.621 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.621 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104263+0000", 2024-08-17T18:18:54.622 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.6721e-05 2024-08-17T18:18:54.622 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.622 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.622 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:54.622 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104277+0000", 2024-08-17T18:18:54.622 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3623e-05 2024-08-17T18:18:54.622 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.622 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.622 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:18:54.622 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.244408+0000", 2024-08-17T18:18:54.622 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.14013062600000001 2024-08-17T18:18:54.622 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.623 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.623 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.623 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.244451+0000", 2024-08-17T18:18:54.623 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3497e-05 2024-08-17T18:18:54.623 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.623 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.623 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.623 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.623 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.623 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:73871 50.8 50:1505f02e:::benchmark_data_smithi130_398002_object73870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-08-17T18:18:54.623 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:05.104735+0000", 2024-08-17T18:18:54.624 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 49.49174876, 2024-08-17T18:18:54.624 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.13697667499999999, 2024-08-17T18:18:54.624 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.624 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:18:54.624 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:18:54.624 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:18:54.624 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:18:54.624 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 73871 2024-08-17T18:18:54.624 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.624 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.624 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.624 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.625 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104735+0000", 2024-08-17T18:18:54.625 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.625 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.625 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.625 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.625 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104735+0000", 2024-08-17T18:18:54.625 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.625 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.625 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.625 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.625 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104734+0000", 2024-08-17T18:18:54.625 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:18:54.625 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.626 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.626 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.626 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104778+0000", 2024-08-17T18:18:54.626 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4100000000000001e-05 2024-08-17T18:18:54.626 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.626 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.626 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.626 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104794+0000", 2024-08-17T18:18:54.626 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.6308e-05 2024-08-17T18:18:54.626 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.626 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.626 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.627 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104799+0000", 2024-08-17T18:18:54.627 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2920000000000003e-06 2024-08-17T18:18:54.627 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.627 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.627 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.627 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104834+0000", 2024-08-17T18:18:54.627 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4379999999999999e-05 2024-08-17T18:18:54.627 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.627 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.627 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:54.627 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104911+0000", 2024-08-17T18:18:54.627 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.6966999999999997e-05 2024-08-17T18:18:54.627 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.628 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.628 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0", 2024-08-17T18:18:54.628 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104991+0000", 2024-08-17T18:18:54.628 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9856000000000003e-05 2024-08-17T18:18:54.628 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.628 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.628 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:18:54.628 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.240874+0000", 2024-08-17T18:18:54.628 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.135883226 2024-08-17T18:18:54.628 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.628 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.628 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.628 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.241579+0000", 2024-08-17T18:18:54.628 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00070528500000000005 2024-08-17T18:18:54.629 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.629 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.629 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:18:54.629 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.241627+0000", 2024-08-17T18:18:54.629 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.833e-05 2024-08-17T18:18:54.629 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.629 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.629 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.629 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.241711+0000", 2024-08-17T18:18:54.629 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3959e-05 2024-08-17T18:18:54.629 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.629 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.629 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.630 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.630 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.630 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:73873 50.9 50:9a2ad551:::benchmark_data_smithi130_398002_object73872:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-08-17T18:18:54.630 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:05.104828+0000", 2024-08-17T18:18:54.630 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 49.491655840999996, 2024-08-17T18:18:54.630 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.13915735600000001, 2024-08-17T18:18:54.630 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.630 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:18:54.630 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:18:54.630 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:18:54.630 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:18:54.630 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 73873 2024-08-17T18:18:54.630 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.631 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.631 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.631 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.631 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104828+0000", 2024-08-17T18:18:54.631 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.631 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.631 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.631 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.631 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104828+0000", 2024-08-17T18:18:54.631 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.631 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.631 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.631 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.632 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104827+0000", 2024-08-17T18:18:54.632 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:18:54.632 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.632 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.632 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.632 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104886+0000", 2024-08-17T18:18:54.632 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9351000000000001e-05 2024-08-17T18:18:54.632 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.632 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.632 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.632 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104896+0000", 2024-08-17T18:18:54.632 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.8689999999999999e-06 2024-08-17T18:18:54.633 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.633 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.633 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.633 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104900+0000", 2024-08-17T18:18:54.633 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2409999999999997e-06 2024-08-17T18:18:54.633 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.633 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.633 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.633 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.104934+0000", 2024-08-17T18:18:54.633 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3569000000000003e-05 2024-08-17T18:18:54.633 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.633 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.633 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:54.634 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.105014+0000", 2024-08-17T18:18:54.634 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0216999999999994e-05 2024-08-17T18:18:54.634 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.634 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.634 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,7", 2024-08-17T18:18:54.634 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.105091+0000", 2024-08-17T18:18:54.634 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.6625999999999994e-05 2024-08-17T18:18:54.634 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.634 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.634 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:18:54.634 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.107257+0000", 2024-08-17T18:18:54.634 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0021665090000000001 2024-08-17T18:18:54.635 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.635 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.635 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.635 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.107926+0000", 2024-08-17T18:18:54.635 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00066894800000000005 2024-08-17T18:18:54.635 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.635 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.635 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.635 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.243911+0000", 2024-08-17T18:18:54.635 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.13598437899999999 2024-08-17T18:18:54.635 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.635 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.635 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:18:54.635 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.243953+0000", 2024-08-17T18:18:54.636 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2716999999999998e-05 2024-08-17T18:18:54.636 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.636 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.636 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.636 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.243985+0000", 2024-08-17T18:18:54.636 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1508999999999999e-05 2024-08-17T18:18:54.636 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.636 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.636 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.636 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.636 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.636 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:73874 50.c 50:385f2f21:::benchmark_data_smithi130_398002_object73873:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-08-17T18:18:54.637 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:05.105653+0000", 2024-08-17T18:18:54.637 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 49.490830191999997, 2024-08-17T18:18:54.637 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.138896834, 2024-08-17T18:18:54.637 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.637 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:18:54.637 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:18:54.637 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:18:54.637 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:18:54.637 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 73874 2024-08-17T18:18:54.638 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.638 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.638 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.638 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.638 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.105653+0000", 2024-08-17T18:18:54.638 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.638 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.638 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.638 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.638 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.105653+0000", 2024-08-17T18:18:54.638 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.638 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.638 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.638 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.639 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.105653+0000", 2024-08-17T18:18:54.639 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:18:54.639 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.639 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.639 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.639 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.105733+0000", 2024-08-17T18:18:54.639 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0389999999999994e-05 2024-08-17T18:18:54.639 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.639 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.639 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.639 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.105742+0000", 2024-08-17T18:18:54.639 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0880000000000001e-06 2024-08-17T18:18:54.640 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.640 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.640 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.640 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.105747+0000", 2024-08-17T18:18:54.640 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9599999999999999e-06 2024-08-17T18:18:54.640 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.640 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.640 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.640 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.105786+0000", 2024-08-17T18:18:54.640 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8847000000000003e-05 2024-08-17T18:18:54.640 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.640 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.640 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:54.640 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.105882+0000", 2024-08-17T18:18:54.641 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6644000000000005e-05 2024-08-17T18:18:54.641 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.641 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.641 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1", 2024-08-17T18:18:54.641 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.105996+0000", 2024-08-17T18:18:54.641 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000113313 2024-08-17T18:18:54.641 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.641 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.641 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.641 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.241574+0000", 2024-08-17T18:18:54.641 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.13557794100000001 2024-08-17T18:18:54.641 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.641 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.642 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.642 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.244448+0000", 2024-08-17T18:18:54.642 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0028746510000000002 2024-08-17T18:18:54.642 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.642 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.642 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:18:54.642 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.244459+0000", 2024-08-17T18:18:54.642 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0200000000000001e-05 2024-08-17T18:18:54.642 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.642 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.642 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:18:54.642 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.244501+0000", 2024-08-17T18:18:54.642 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2864000000000002e-05 2024-08-17T18:18:54.643 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.643 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.643 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.643 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.244550+0000", 2024-08-17T18:18:54.643 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.863e-05 2024-08-17T18:18:54.643 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.643 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.643 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.643 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.643 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.644 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:73879 50.9 50:94195b0e:::benchmark_data_smithi130_398002_object73878:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-08-17T18:18:54.644 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:05.108041+0000", 2024-08-17T18:18:54.644 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 49.488442722000002, 2024-08-17T18:18:54.644 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.13819862399999999, 2024-08-17T18:18:54.644 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.644 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:18:54.644 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:18:54.644 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:18:54.644 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:18:54.644 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 73879 2024-08-17T18:18:54.644 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.644 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.644 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.645 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.645 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.108041+0000", 2024-08-17T18:18:54.645 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.645 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.645 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.645 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.645 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.108041+0000", 2024-08-17T18:18:54.645 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.645 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.645 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.645 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.645 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.108040+0000", 2024-08-17T18:18:54.645 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:18:54.646 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.646 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.646 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.646 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.108110+0000", 2024-08-17T18:18:54.646 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0050999999999997e-05 2024-08-17T18:18:54.646 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.646 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.646 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.646 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.108120+0000", 2024-08-17T18:18:54.646 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0278000000000001e-05 2024-08-17T18:18:54.646 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.646 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.647 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.647 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.108125+0000", 2024-08-17T18:18:54.647 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8840000000000002e-06 2024-08-17T18:18:54.647 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.647 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.647 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.647 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.108159+0000", 2024-08-17T18:18:54.647 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3896999999999997e-05 2024-08-17T18:18:54.647 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.647 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.647 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:54.647 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.240852+0000", 2024-08-17T18:18:54.647 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.13269314199999999 2024-08-17T18:18:54.647 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.648 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.648 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,7", 2024-08-17T18:18:54.648 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.241010+0000", 2024-08-17T18:18:54.648 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00015836100000000001 2024-08-17T18:18:54.648 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.648 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.648 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:18:54.648 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.243997+0000", 2024-08-17T18:18:54.648 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0029863189999999999 2024-08-17T18:18:54.648 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.648 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.648 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.648 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.244437+0000", 2024-08-17T18:18:54.649 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000440891 2024-08-17T18:18:54.649 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.649 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.649 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.649 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.246155+0000", 2024-08-17T18:18:54.649 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0017179910000000001 2024-08-17T18:18:54.649 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.649 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.649 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:18:54.649 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.246203+0000", 2024-08-17T18:18:54.649 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7725000000000001e-05 2024-08-17T18:18:54.649 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.649 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.649 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.650 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.246239+0000", 2024-08-17T18:18:54.650 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6149000000000002e-05 2024-08-17T18:18:54.650 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.650 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.650 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.650 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.650 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.650 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:73880 50.c 50:3f4dfdd7:::benchmark_data_smithi130_398002_object73879:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-08-17T18:18:54.650 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:05.109888+0000", 2024-08-17T18:18:54.650 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 49.486595842, 2024-08-17T18:18:54.650 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.14104760799999999, 2024-08-17T18:18:54.650 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.650 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:18:54.651 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:18:54.651 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:18:54.651 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:18:54.651 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 73880 2024-08-17T18:18:54.651 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.651 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.651 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.651 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.651 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.109888+0000", 2024-08-17T18:18:54.651 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.651 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.651 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.651 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.652 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.109888+0000", 2024-08-17T18:18:54.652 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.652 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.652 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.652 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.652 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.109886+0000", 2024-08-17T18:18:54.652 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:18:54.652 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.652 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.652 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.652 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.109942+0000", 2024-08-17T18:18:54.652 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5234e-05 2024-08-17T18:18:54.652 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.652 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.653 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.653 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.240756+0000", 2024-08-17T18:18:54.653 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.130814706 2024-08-17T18:18:54.653 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.653 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.653 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.653 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.240769+0000", 2024-08-17T18:18:54.653 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3132999999999999e-05 2024-08-17T18:18:54.653 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.653 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.653 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.653 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.241608+0000", 2024-08-17T18:18:54.653 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000838421 2024-08-17T18:18:54.654 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.654 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.654 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:54.654 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.241734+0000", 2024-08-17T18:18:54.654 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000126088 2024-08-17T18:18:54.654 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.654 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.654 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1", 2024-08-17T18:18:54.654 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.241866+0000", 2024-08-17T18:18:54.654 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000131996 2024-08-17T18:18:54.654 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.654 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.654 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.655 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.246930+0000", 2024-08-17T18:18:54.655 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0050640370000000004 2024-08-17T18:18:54.655 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.655 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.655 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:18:54.655 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.249499+0000", 2024-08-17T18:18:54.655 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0025692369999999998 2024-08-17T18:18:54.655 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.655 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.655 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.655 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.250855+0000", 2024-08-17T18:18:54.655 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001355247 2024-08-17T18:18:54.655 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.655 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.656 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:18:54.656 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.250885+0000", 2024-08-17T18:18:54.656 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.0037000000000001e-05 2024-08-17T18:18:54.656 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.656 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.656 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.656 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:05.250935+0000", 2024-08-17T18:18:54.656 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0668999999999999e-05 2024-08-17T18:18:54.656 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.656 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.656 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.656 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.656 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.657 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:106455 50.9 50:9b4a0a99:::benchmark_data_smithi130_398002_object106454:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1442)", 2024-08-17T18:18:54.657 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:30.377679+0000", 2024-08-17T18:18:54.657 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 24.218804677000001, 2024-08-17T18:18:54.657 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.24222401800000001, 2024-08-17T18:18:54.657 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.657 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:18:54.657 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:18:54.657 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:18:54.657 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:18:54.657 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 106455 2024-08-17T18:18:54.657 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.657 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.657 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.657 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.658 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.377679+0000", 2024-08-17T18:18:54.658 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.658 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.658 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.658 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.658 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.377679+0000", 2024-08-17T18:18:54.658 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.658 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.658 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.658 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.658 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.377677+0000", 2024-08-17T18:18:54.658 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999976 2024-08-17T18:18:54.658 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.659 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.659 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.659 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.377740+0000", 2024-08-17T18:18:54.659 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3807999999999996e-05 2024-08-17T18:18:54.659 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.659 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.659 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.659 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.377754+0000", 2024-08-17T18:18:54.659 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3390999999999999e-05 2024-08-17T18:18:54.659 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.659 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.659 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.659 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.377760+0000", 2024-08-17T18:18:54.659 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4640000000000003e-06 2024-08-17T18:18:54.660 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.660 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.660 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.660 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.377809+0000", 2024-08-17T18:18:54.660 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9286000000000002e-05 2024-08-17T18:18:54.660 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.660 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.660 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:54.662 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.377932+0000", 2024-08-17T18:18:54.662 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012256600000000001 2024-08-17T18:18:54.662 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.662 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.662 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,7", 2024-08-17T18:18:54.662 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.378065+0000", 2024-08-17T18:18:54.662 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000132776 2024-08-17T18:18:54.663 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.663 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.663 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:18:54.663 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.381914+0000", 2024-08-17T18:18:54.663 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0038489219999999998 2024-08-17T18:18:54.663 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.663 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.663 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.663 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.382754+0000", 2024-08-17T18:18:54.663 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00084013499999999999 2024-08-17T18:18:54.663 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.663 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.663 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.664 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.619820+0000", 2024-08-17T18:18:54.664 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.23706576200000001 2024-08-17T18:18:54.664 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.664 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.664 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.664 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.619903+0000", 2024-08-17T18:18:54.664 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3175000000000002e-05 2024-08-17T18:18:54.664 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.664 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.664 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.664 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.664 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.664 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:106471 50.c 50:31ee0cbd:::benchmark_data_smithi130_398002_object106470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1442)", 2024-08-17T18:18:54.664 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:30.391908+0000", 2024-08-17T18:18:54.665 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 24.204575195, 2024-08-17T18:18:54.665 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.227818879, 2024-08-17T18:18:54.665 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.665 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:18:54.665 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:18:54.665 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:18:54.665 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:18:54.665 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 106471 2024-08-17T18:18:54.665 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.665 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.665 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.665 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.665 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.391908+0000", 2024-08-17T18:18:54.666 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.666 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.666 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.666 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.666 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.391908+0000", 2024-08-17T18:18:54.666 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.666 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.666 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.667 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.667 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.391907+0000", 2024-08-17T18:18:54.667 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:18:54.667 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.667 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.667 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.667 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.391990+0000", 2024-08-17T18:18:54.667 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3735999999999999e-05 2024-08-17T18:18:54.667 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.667 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.668 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.668 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.392003+0000", 2024-08-17T18:18:54.668 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2160999999999999e-05 2024-08-17T18:18:54.668 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.668 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.668 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.668 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.392019+0000", 2024-08-17T18:18:54.668 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.666e-05 2024-08-17T18:18:54.668 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.668 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.668 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.669 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.392058+0000", 2024-08-17T18:18:54.669 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8405999999999998e-05 2024-08-17T18:18:54.669 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.669 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.669 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:54.669 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.392182+0000", 2024-08-17T18:18:54.669 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012404899999999999 2024-08-17T18:18:54.669 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.669 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.669 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1", 2024-08-17T18:18:54.669 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.392327+0000", 2024-08-17T18:18:54.669 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000145563 2024-08-17T18:18:54.669 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.670 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.670 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.670 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.611433+0000", 2024-08-17T18:18:54.670 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.219105511 2024-08-17T18:18:54.670 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.670 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.670 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.670 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.611490+0000", 2024-08-17T18:18:54.670 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6996000000000001e-05 2024-08-17T18:18:54.670 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.670 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.670 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:18:54.671 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.619270+0000", 2024-08-17T18:18:54.671 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.007779874 2024-08-17T18:18:54.671 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.671 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.671 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:18:54.671 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.619686+0000", 2024-08-17T18:18:54.671 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00041607999999999999 2024-08-17T18:18:54.671 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.671 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.671 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.672 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.619727+0000", 2024-08-17T18:18:54.672 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1399000000000001e-05 2024-08-17T18:18:54.672 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.672 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.672 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.672 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.672 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.673 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:106472 50.e 50:74fd320d:::benchmark_data_smithi130_398002_object106471:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1442)", 2024-08-17T18:18:54.673 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:30.392032+0000", 2024-08-17T18:18:54.673 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 24.204451764000002, 2024-08-17T18:18:54.673 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.21222959499999999, 2024-08-17T18:18:54.673 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.673 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:18:54.673 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:18:54.673 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:18:54.673 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:18:54.673 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 106472 2024-08-17T18:18:54.673 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.673 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.674 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.674 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.674 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.392032+0000", 2024-08-17T18:18:54.674 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.674 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.674 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.674 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.674 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.392032+0000", 2024-08-17T18:18:54.674 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.674 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.674 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.674 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.675 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.392031+0000", 2024-08-17T18:18:54.675 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:18:54.675 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.675 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.675 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.675 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.392065+0000", 2024-08-17T18:18:54.675 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4001000000000002e-05 2024-08-17T18:18:54.675 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.675 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.675 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.675 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.392072+0000", 2024-08-17T18:18:54.675 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6059999999999999e-06 2024-08-17T18:18:54.676 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.676 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.676 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.676 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.392076+0000", 2024-08-17T18:18:54.676 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0520000000000003e-06 2024-08-17T18:18:54.676 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.676 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.676 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.676 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.392119+0000", 2024-08-17T18:18:54.676 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.278e-05 2024-08-17T18:18:54.676 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.676 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.676 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:54.676 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.392274+0000", 2024-08-17T18:18:54.677 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00015584400000000001 2024-08-17T18:18:54.677 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.677 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.677 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0", 2024-08-17T18:18:54.677 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.392406+0000", 2024-08-17T18:18:54.677 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013147199999999999 2024-08-17T18:18:54.677 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.677 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.677 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.677 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.404867+0000", 2024-08-17T18:18:54.677 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.012461566 2024-08-17T18:18:54.677 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.678 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.678 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:18:54.678 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.604126+0000", 2024-08-17T18:18:54.678 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.19925884499999999 2024-08-17T18:18:54.678 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.678 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.678 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:18:54.678 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.604185+0000", 2024-08-17T18:18:54.678 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8689e-05 2024-08-17T18:18:54.678 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.678 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.678 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.678 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.604261+0000", 2024-08-17T18:18:54.679 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.6342999999999997e-05 2024-08-17T18:18:54.679 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.679 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.679 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.679 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.679 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.679 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.47132.0:106470 50.4 e1442/1434)", 2024-08-17T18:18:54.679 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:30.396030+0000", 2024-08-17T18:18:54.679 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 24.200453079999999, 2024-08-17T18:18:54.679 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.215247312, 2024-08-17T18:18:54.679 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.679 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:18:54.679 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.680 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.680 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.680 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.396030+0000", 2024-08-17T18:18:54.680 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.680 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.680 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.680 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.680 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.396030+0000", 2024-08-17T18:18:54.680 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.680 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.680 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.680 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.680 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.396029+0000", 2024-08-17T18:18:54.681 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:18:54.681 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.681 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.681 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.681 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.396148+0000", 2024-08-17T18:18:54.681 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000119469 2024-08-17T18:18:54.681 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.681 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.681 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.681 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.396165+0000", 2024-08-17T18:18:54.681 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.6702e-05 2024-08-17T18:18:54.681 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.681 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.682 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.682 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.396174+0000", 2024-08-17T18:18:54.682 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3270000000000001e-06 2024-08-17T18:18:54.682 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.682 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.682 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.682 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.611256+0000", 2024-08-17T18:18:54.682 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.21508217700000001 2024-08-17T18:18:54.682 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.682 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.682 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.682 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.611278+0000", 2024-08-17T18:18:54.683 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.1985e-05 2024-08-17T18:18:54.683 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.683 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.683 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.683 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.683 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.683 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:106474 50.c 50:3e7d699e:::benchmark_data_smithi130_398002_object106473:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1442)", 2024-08-17T18:18:54.683 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:30.396129+0000", 2024-08-17T18:18:54.683 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 24.200354712999999, 2024-08-17T18:18:54.683 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.25288739199999999, 2024-08-17T18:18:54.683 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.683 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:18:54.683 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:18:54.684 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:18:54.684 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:18:54.684 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 106474 2024-08-17T18:18:54.684 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.684 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.684 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.684 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.684 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.396129+0000", 2024-08-17T18:18:54.684 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.684 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.684 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.684 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.684 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.396129+0000", 2024-08-17T18:18:54.684 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.685 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.685 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.685 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.685 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.396106+0000", 2024-08-17T18:18:54.685 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999776 2024-08-17T18:18:54.685 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.685 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.685 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.685 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.396167+0000", 2024-08-17T18:18:54.685 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0770999999999998e-05 2024-08-17T18:18:54.685 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.685 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.686 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.686 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.396180+0000", 2024-08-17T18:18:54.686 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3094e-05 2024-08-17T18:18:54.686 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.686 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.686 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.686 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.396186+0000", 2024-08-17T18:18:54.686 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6049999999999997e-06 2024-08-17T18:18:54.686 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.686 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.686 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.686 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.611527+0000", 2024-08-17T18:18:54.686 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.21534076599999999 2024-08-17T18:18:54.686 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.687 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.687 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:54.687 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.611679+0000", 2024-08-17T18:18:54.687 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000152643 2024-08-17T18:18:54.687 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.687 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.687 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1", 2024-08-17T18:18:54.687 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.612368+0000", 2024-08-17T18:18:54.687 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00068829500000000001 2024-08-17T18:18:54.687 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.687 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.687 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.687 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.639061+0000", 2024-08-17T18:18:54.688 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.026692983999999999 2024-08-17T18:18:54.688 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.688 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.688 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:18:54.688 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.646130+0000", 2024-08-17T18:18:54.688 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0070691770000000003 2024-08-17T18:18:54.688 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.688 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.688 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:18:54.688 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.648927+0000", 2024-08-17T18:18:54.688 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0027973970000000001 2024-08-17T18:18:54.688 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.688 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.689 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:18:54.689 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.648973+0000", 2024-08-17T18:18:54.689 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6218999999999999e-05 2024-08-17T18:18:54.689 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.689 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.689 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.689 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.649016+0000", 2024-08-17T18:18:54.689 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2740000000000001e-05 2024-08-17T18:18:54.689 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.689 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.689 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.689 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.689 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.689 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop_reply(client.47132.0:106455 50.9 e1442/1434)", 2024-08-17T18:18:54.690 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:30.397391+0000", 2024-08-17T18:18:54.690 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 24.199092556, 2024-08-17T18:18:54.690 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.22299272000000001, 2024-08-17T18:18:54.690 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.690 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:18:54.690 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.690 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.690 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.690 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.397391+0000", 2024-08-17T18:18:54.691 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.691 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.691 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.691 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.691 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.397391+0000", 2024-08-17T18:18:54.691 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.691 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.691 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.691 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.691 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.397390+0000", 2024-08-17T18:18:54.691 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:18:54.691 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.692 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.692 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.692 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.397396+0000", 2024-08-17T18:18:54.692 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1970000000000001e-06 2024-08-17T18:18:54.692 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.692 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.692 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.692 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.397410+0000", 2024-08-17T18:18:54.692 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3399e-05 2024-08-17T18:18:54.692 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.692 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.692 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.692 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.397415+0000", 2024-08-17T18:18:54.693 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0919999999999997e-06 2024-08-17T18:18:54.693 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.693 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.693 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.693 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.620368+0000", 2024-08-17T18:18:54.693 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.222953243 2024-08-17T18:18:54.693 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.693 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.693 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.693 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.620384+0000", 2024-08-17T18:18:54.693 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5735000000000001e-05 2024-08-17T18:18:54.693 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.693 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.694 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.694 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.694 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.694 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.47132.0:106477 50.0 e1442/1434)", 2024-08-17T18:18:54.694 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:30.403106+0000", 2024-08-17T18:18:54.694 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 24.193377344000002, 2024-08-17T18:18:54.694 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.165401519, 2024-08-17T18:18:54.694 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.694 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:18:54.694 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.694 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.694 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.695 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403106+0000", 2024-08-17T18:18:54.695 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.695 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.695 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.695 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.695 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403106+0000", 2024-08-17T18:18:54.695 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.695 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.695 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.695 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.695 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403106+0000", 2024-08-17T18:18:54.695 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:18:54.695 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.695 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.696 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.696 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403139+0000", 2024-08-17T18:18:54.696 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3572999999999999e-05 2024-08-17T18:18:54.696 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.696 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.696 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.696 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403146+0000", 2024-08-17T18:18:54.696 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6510000000000004e-06 2024-08-17T18:18:54.696 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.696 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.696 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.696 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403151+0000", 2024-08-17T18:18:54.696 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2379999999999997e-06 2024-08-17T18:18:54.697 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.697 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.697 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.697 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.568451+0000", 2024-08-17T18:18:54.697 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.165299363 2024-08-17T18:18:54.697 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.697 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.697 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.697 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.568508+0000", 2024-08-17T18:18:54.697 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7071999999999998e-05 2024-08-17T18:18:54.697 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.697 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.697 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.698 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.698 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.698 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop_reply(client.47132.0:106471 50.c e1442/1434 ondisk, result = 0)", 2024-08-17T18:18:54.698 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:30.403291+0000", 2024-08-17T18:18:54.698 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 24.193192613000001, 2024-08-17T18:18:54.698 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.20814380800000001, 2024-08-17T18:18:54.698 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.698 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "started", 2024-08-17T18:18:54.698 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.698 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.698 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.698 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403291+0000", 2024-08-17T18:18:54.699 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.699 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.699 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.699 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.699 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403291+0000", 2024-08-17T18:18:54.699 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.699 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.699 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.699 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.699 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403290+0000", 2024-08-17T18:18:54.699 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:18:54.699 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.699 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.700 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.700 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403294+0000", 2024-08-17T18:18:54.700 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1300000000000001e-06 2024-08-17T18:18:54.700 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.700 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.700 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.700 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403302+0000", 2024-08-17T18:18:54.700 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7630000000000006e-06 2024-08-17T18:18:54.700 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.700 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.700 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.700 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403306+0000", 2024-08-17T18:18:54.701 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.489e-06 2024-08-17T18:18:54.701 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.701 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.701 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.701 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.611420+0000", 2024-08-17T18:18:54.701 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.20811463499999999 2024-08-17T18:18:54.701 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.701 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.701 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:54.701 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.611427+0000", 2024-08-17T18:18:54.701 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.002e-06 2024-08-17T18:18:54.701 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.701 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.702 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.702 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.611435+0000", 2024-08-17T18:18:54.702 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.1600000000000001e-06 2024-08-17T18:18:54.702 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.702 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.702 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.702 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.702 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.702 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop_reply(client.47132.0:106471 50.c e1442/1434 ondisk, result = 0)", 2024-08-17T18:18:54.702 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:30.403385+0000", 2024-08-17T18:18:54.702 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 24.193098444, 2024-08-17T18:18:54.702 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.20810637300000001, 2024-08-17T18:18:54.703 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.703 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "started", 2024-08-17T18:18:54.703 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.703 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.703 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.703 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403385+0000", 2024-08-17T18:18:54.703 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.703 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.703 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.703 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.703 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403385+0000", 2024-08-17T18:18:54.703 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.703 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.704 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.704 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.704 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403384+0000", 2024-08-17T18:18:54.704 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:18:54.704 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.704 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.704 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.704 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403391+0000", 2024-08-17T18:18:54.704 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7510000000000002e-06 2024-08-17T18:18:54.704 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.704 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.704 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.704 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403400+0000", 2024-08-17T18:18:54.705 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2220000000000008e-06 2024-08-17T18:18:54.705 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.705 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.705 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.705 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.403405+0000", 2024-08-17T18:18:54.705 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5770000000000004e-06 2024-08-17T18:18:54.705 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.705 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.705 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.705 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.611480+0000", 2024-08-17T18:18:54.705 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.20807541800000001 2024-08-17T18:18:54.705 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.705 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.706 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:54.706 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.611485+0000", 2024-08-17T18:18:54.706 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.113e-06 2024-08-17T18:18:54.706 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.706 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.706 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.706 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.611491+0000", 2024-08-17T18:18:54.706 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9340000000000003e-06 2024-08-17T18:18:54.706 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.706 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.706 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.706 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.706 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.706 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.47132.0:106478 50.0 e1442/1434)", 2024-08-17T18:18:54.707 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:30.411856+0000", 2024-08-17T18:18:54.707 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 24.184627548000002, 2024-08-17T18:18:54.707 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.15679435899999999, 2024-08-17T18:18:54.707 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.707 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:18:54.707 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.707 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.707 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.707 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.411856+0000", 2024-08-17T18:18:54.707 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.707 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.707 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.707 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.707 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.411856+0000", 2024-08-17T18:18:54.708 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.708 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.708 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.708 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.708 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.411856+0000", 2024-08-17T18:18:54.708 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:18:54.708 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.708 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.708 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.708 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.411913+0000", 2024-08-17T18:18:54.708 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7531000000000002e-05 2024-08-17T18:18:54.708 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.708 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.709 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.709 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.411934+0000", 2024-08-17T18:18:54.709 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0713e-05 2024-08-17T18:18:54.709 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.709 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.709 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.709 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.411943+0000", 2024-08-17T18:18:54.709 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4839999999999993e-06 2024-08-17T18:18:54.709 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.709 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.709 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.709 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.568596+0000", 2024-08-17T18:18:54.709 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.156652976 2024-08-17T18:18:54.710 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.710 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.710 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.710 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.568650+0000", 2024-08-17T18:18:54.710 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3897999999999998e-05 2024-08-17T18:18:54.710 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.710 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.710 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.710 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.710 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.710 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:106455 50.19 50:9b4a0a99:::benchmark_data_smithi130_398002_object106454:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1443)", 2024-08-17T18:18:54.710 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:30.541757+0000", 2024-08-17T18:18:54.710 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 24.054726218999999, 2024-08-17T18:18:54.711 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.91456280199999995, 2024-08-17T18:18:54.711 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:54.711 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:18:54.711 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:18:54.711 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:18:54.711 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:18:54.711 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 106455 2024-08-17T18:18:54.711 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.711 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:54.711 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.711 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:54.711 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.541757+0000", 2024-08-17T18:18:54.711 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.712 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.712 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.712 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:54.712 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.541757+0000", 2024-08-17T18:18:54.712 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:54.712 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.712 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.712 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:54.712 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.541755+0000", 2024-08-17T18:18:54.712 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999976 2024-08-17T18:18:54.712 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.712 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.712 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:54.712 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.541812+0000", 2024-08-17T18:18:54.713 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7299000000000001e-05 2024-08-17T18:18:54.713 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.713 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.713 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:54.713 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.541830+0000", 2024-08-17T18:18:54.713 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.8125999999999999e-05 2024-08-17T18:18:54.713 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.713 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.713 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:54.713 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.541839+0000", 2024-08-17T18:18:54.713 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.4509999999999992e-06 2024-08-17T18:18:54.713 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.713 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.713 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.714 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.620408+0000", 2024-08-17T18:18:54.714 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.078569359000000005 2024-08-17T18:18:54.714 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.714 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.714 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:18:54.714 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.620415+0000", 2024-08-17T18:18:54.714 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6440000000000003e-06 2024-08-17T18:18:54.714 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.714 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.714 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:54.714 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:31.456262+0000", 2024-08-17T18:18:54.714 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.83584671099999996 2024-08-17T18:18:54.715 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:54.715 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:54.715 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:18:54.715 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:31.456320+0000", 2024-08-17T18:18:54.715 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8400000000000003e-05 2024-08-17T18:18:54.715 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.715 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.715 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.715 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:54.716 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:54.716 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:18:54.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:54.717 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:54.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:54.718 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:18:54.732 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:54 smithi130 ceph-mon[105521]: pgmap v2777: 99 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.1 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 100 MiB/s wr, 1.61k op/s; 74585/399561 objects degraded (18.667%); 41676/399561 objects misplaced (10.430%); 1.7 MiB/s, 1 keys/s, 28 objects/s recovering 2024-08-17T18:18:54.732 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1807613448' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:18:54.732 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:54 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:18:54.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:54 smithi130 ceph-mon[110068]: pgmap v2777: 99 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.1 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 100 MiB/s wr, 1.61k op/s; 74585/399561 objects degraded (18.667%); 41676/399561 objects misplaced (10.430%); 1.7 MiB/s, 1 keys/s, 28 objects/s recovering 2024-08-17T18:18:54.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1807613448' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:18:54.732 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:54 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:18:54.819 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:54.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:54 smithi179 ceph-mon[93519]: pgmap v2777: 99 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.1 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 100 MiB/s wr, 1.61k op/s; 74585/399561 objects degraded (18.667%); 41676/399561 objects misplaced (10.430%); 1.7 MiB/s, 1 keys/s, 28 objects/s recovering 2024-08-17T18:18:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1807613448' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:18:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:54 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:18:54.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:54.861 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:54.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:54.962 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:55.000 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:55.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:55.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:55.102 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:55.149 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:55.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:55.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:55.154 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 83 16 136352 136336 102.637 81.3125 0.0116766 0.00973865 2024-08-17T18:18:55.244 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 7, 1, 6, 4] out_osds: [5] dead_osds: [2] live_osds: [1, 4, 5, 6, 7, 0, 3] 2024-08-17T18:18:55.244 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:18:55.244 INFO:tasks.thrashosds.thrasher:Reviving osd 2 2024-08-17T18:18:55.244 INFO:tasks.cephadm.osd.2:Restarting osd.2 (starting--it wasn't running)... 2024-08-17T18:18:55.244 DEBUG:teuthology.orchestra.run.smithi130:osd.2> sudo journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.2.service 2024-08-17T18:18:55.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl start ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.2 2024-08-17T18:18:55.251 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:55.281 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:55.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:55.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:55.382 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:55.411 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:55.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:55.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:55.417 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:55 smithi130 systemd[1]: Starting Ceph osd.2 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:18:55.513 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:55.543 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:55.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:55.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:55.610 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T18:18:55.645 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:55.682 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:55.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:55.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:55.749 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:18:55.784 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:55.823 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:55.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:55.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:55.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:55 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:18:55.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:55 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:18:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:55 smithi179 ceph-mon[93519]: osdmap e1449: 8 total, 7 up, 7 in 2024-08-17T18:18:55.878 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:55 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:18:55.878 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:55 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:18:55.878 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:55 smithi130 ceph-mon[105521]: osdmap e1449: 8 total, 7 up, 7 in 2024-08-17T18:18:55.879 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:55 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:18:55.879 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:55 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:18:55.879 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:55 smithi130 ceph-mon[110068]: osdmap e1449: 8 total, 7 up, 7 in 2024-08-17T18:18:55.925 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:55.963 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:55.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:55.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:56.008 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:18:56.065 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:56.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:56.103 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:56.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:56.145 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:55 smithi130 podman[406722]: 2024-08-17 18:18:55.781234008 +0000 UTC m=+0.042039702 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T18:18:56.145 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:55 smithi130 podman[406722]: 2024-08-17 18:18:55.970887743 +0000 UTC m=+0.231693416 container create fc6c472ee0005d3f7b631ca51d6372b006a97c418f8e65ff3b9d2179b729ccd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-08-17T18:18:56.155 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 84 16 137650 137634 102.381 81.125 0.0109544 0.0097629 2024-08-17T18:18:56.203 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:56.245 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:56.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:56.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:56.347 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:56.383 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:56.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:56.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:56.485 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:56.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:56.526 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:56.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:56.627 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:56.658 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:56.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:56.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:56.760 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:56.803 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:56.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:56.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:56.836 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:56 smithi130 ceph-mon[105521]: pgmap v2779: 99 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.1 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 40 MiB/s wr, 633 op/s; 74585/399561 objects degraded (18.667%); 41676/399561 objects misplaced (10.430%); 1.7 MiB/s, 1 keys/s, 28 objects/s recovering 2024-08-17T18:18:56.837 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 74585/399561 objects degraded (18.667%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:56.837 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:56 smithi130 ceph-mon[110068]: pgmap v2779: 99 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.1 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 40 MiB/s wr, 633 op/s; 74585/399561 objects degraded (18.667%); 41676/399561 objects misplaced (10.430%); 1.7 MiB/s, 1 keys/s, 28 objects/s recovering 2024-08-17T18:18:56.837 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 74585/399561 objects degraded (18.667%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:56.837 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:56 smithi130 podman[406722]: 2024-08-17 18:18:56.552028269 +0000 UTC m=+0.812833953 container init fc6c472ee0005d3f7b631ca51d6372b006a97c418f8e65ff3b9d2179b729ccd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux ) 2024-08-17T18:18:56.837 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:56 smithi130 podman[406722]: 2024-08-17 18:18:56.566131825 +0000 UTC m=+0.826937500 container start fc6c472ee0005d3f7b631ca51d6372b006a97c418f8e65ff3b9d2179b729ccd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.36.0) 2024-08-17T18:18:56.838 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:56 smithi130 podman[406722]: 2024-08-17 18:18:56.660256118 +0000 UTC m=+0.921061793 container attach fc6c472ee0005d3f7b631ca51d6372b006a97c418f8e65ff3b9d2179b729ccd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2) 2024-08-17T18:18:56.838 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[406781]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:18:56.838 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:56 smithi130 bash[406722]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:18:56.838 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[406781]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:18:56.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:56 smithi179 ceph-mon[93519]: pgmap v2779: 99 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.1 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 40 MiB/s wr, 633 op/s; 74585/399561 objects degraded (18.667%); 41676/399561 objects misplaced (10.430%); 1.7 MiB/s, 1 keys/s, 28 objects/s recovering 2024-08-17T18:18:56.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 74585/399561 objects degraded (18.667%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:18:56.906 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:56.944 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:56.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:57.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:57.085 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:57.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:57.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:57.155 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 85 16 138969 138953 102.146 82.4375 0.0119084 0.00978573 2024-08-17T18:18:57.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:56 smithi130 bash[406722]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:18:57.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[406781]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T18:18:57.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[406781]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:18:57.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:56 smithi130 bash[406722]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T18:18:57.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:56 smithi130 bash[406722]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:18:57.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[406781]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:18:57.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:56 smithi130 bash[406722]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T18:18:57.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:57.209 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:57.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:57.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:57.311 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:57.352 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:57.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:57.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:57.454 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:57.487 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:57.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:57.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:57.589 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:57.628 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:57.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:57.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:57.634 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:18:57.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:57 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[406781]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T18:18:57.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:57 smithi130 bash[406722]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T18:18:57.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:57 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[406781]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-08-17T18:18:57.667 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:57 smithi130 bash[406722]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-08-17T18:18:57.729 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:57.765 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:57.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:57.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:57.867 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:18:57.909 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:57.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:18:58.011 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:58.040 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:58.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:58.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:58.141 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:58.155 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 86 16 140263 140247 101.898 80.875 0.0122583 0.00980954 2024-08-17T18:18:58.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:58.173 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:58.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:58.274 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:58.297 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:58.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:58.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:58.357 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:58 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[406781]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-08-17T18:18:58.358 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:58 smithi130 bash[406722]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-08-17T18:18:58.358 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:58 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[406781]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-08-17T18:18:58.358 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:58 smithi130 bash[406722]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-08-17T18:18:58.358 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:58 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[406781]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T18:18:58.358 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:58 smithi130 bash[406722]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T18:18:58.358 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:58 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[406781]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T18:18:58.358 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:58 smithi130 bash[406722]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T18:18:58.358 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:58 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate[406781]: --> ceph-volume lvm activate successful for osd ID: 2 2024-08-17T18:18:58.358 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:58 smithi130 bash[406722]: --> ceph-volume lvm activate successful for osd ID: 2 2024-08-17T18:18:58.399 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:58.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:58.433 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:58.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:58.534 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:18:58.568 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:58.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:58.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:18:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:58 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:18:58.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:58 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:58.672 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:18:58.673 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:58 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:18:58.673 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:18:58.673 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:58 smithi130 podman[406722]: 2024-08-17 18:18:58.402770929 +0000 UTC m=+2.663576600 container died fc6c472ee0005d3f7b631ca51d6372b006a97c418f8e65ff3b9d2179b729ccd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-08-17T18:18:58.673 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:58.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:58.699 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:58.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:58.800 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:18:58.835 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:58.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:58.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:18:58.937 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:58.967 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:58.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:58.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:59.068 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:59.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:59.092 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:59.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:59.155 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 87 16 141347 141331 101.506 67.75 0.0103954 0.00984733 2024-08-17T18:18:59.193 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:18:59.223 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:59.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:59.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:18:59.325 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:59.348 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:59.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:59.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:59.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:18:59 smithi130 ceph-mon[105521]: pgmap v2780: 99 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.2 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 74985/403311 objects degraded (18.592%); 42526/403311 objects misplaced (10.544%); 1.5 MiB/s, 1 keys/s, 24 objects/s recovering 2024-08-17T18:18:59.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:18:59 smithi130 ceph-mon[110068]: pgmap v2780: 99 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.2 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 74985/403311 objects degraded (18.592%); 42526/403311 objects misplaced (10.544%); 1.5 MiB/s, 1 keys/s, 24 objects/s recovering 2024-08-17T18:18:59.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:59.481 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:59.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:59.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:59.500 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:18:59.500 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T18:18:59.500 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.500 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.47132.0:140534 50.a e1449/1447 50:5c341473:::benchmark_data_smithi130_398002_object140533:head v 1449'8855, mlcod=1449'8852)", 2024-08-17T18:18:59.500 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:58.417879+0000", 2024-08-17T18:18:59.501 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.012090304, 2024-08-17T18:18:59.501 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.012117139000000001, 2024-08-17T18:18:59.501 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:59.501 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "started", 2024-08-17T18:18:59.501 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:59.501 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.501 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:59.501 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.417879+0000", 2024-08-17T18:18:59.501 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:59.501 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.501 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.501 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:59.501 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.417879+0000", 2024-08-17T18:18:59.502 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:59.502 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.502 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.502 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:59.502 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.417879+0000", 2024-08-17T18:18:59.502 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:18:59.502 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.502 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.502 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:59.502 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.417912+0000", 2024-08-17T18:18:59.502 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3572999999999999e-05 2024-08-17T18:18:59.502 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.502 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.503 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:59.503 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.417919+0000", 2024-08-17T18:18:59.503 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.0110000000000003e-06 2024-08-17T18:18:59.503 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.503 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.503 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:59.503 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.417923+0000", 2024-08-17T18:18:59.503 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4249999999999998e-06 2024-08-17T18:18:59.503 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.503 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.503 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:59.503 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.419752+0000", 2024-08-17T18:18:59.504 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001829362 2024-08-17T18:18:59.504 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.504 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.504 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:59.504 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.419772+0000", 2024-08-17T18:18:59.504 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0040000000000001e-05 2024-08-17T18:18:59.504 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:59.504 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:59.504 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:59.504 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.504 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.504 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.47132.0:140540 50.12 e1449/1447 50:4f8357ea:::benchmark_data_smithi130_398002_object140539:head v 1449'7794, mlcod=1449'7792)", 2024-08-17T18:18:59.504 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:58.419636+0000", 2024-08-17T18:18:59.505 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.010333406, 2024-08-17T18:18:59.505 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.010418805999999999, 2024-08-17T18:18:59.505 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:59.505 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "started", 2024-08-17T18:18:59.505 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:59.505 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.505 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:59.505 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.419636+0000", 2024-08-17T18:18:59.505 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:59.505 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.505 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.505 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:59.505 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.419636+0000", 2024-08-17T18:18:59.506 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:59.506 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.506 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.506 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:59.506 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.419635+0000", 2024-08-17T18:18:59.506 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:18:59.506 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.506 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.506 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:59.506 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.419688+0000", 2024-08-17T18:18:59.506 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2296000000000002e-05 2024-08-17T18:18:59.506 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.506 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.507 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:59.507 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.419699+0000", 2024-08-17T18:18:59.507 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1375e-05 2024-08-17T18:18:59.507 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.507 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.507 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:59.507 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.419704+0000", 2024-08-17T18:18:59.507 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7990000000000001e-06 2024-08-17T18:18:59.507 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.507 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.507 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:59.507 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.421593+0000", 2024-08-17T18:18:59.507 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0018888380000000001 2024-08-17T18:18:59.508 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.508 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.508 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:59.508 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.421613+0000", 2024-08-17T18:18:59.508 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0564000000000002e-05 2024-08-17T18:18:59.508 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:59.508 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:59.508 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:59.508 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.508 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.508 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.47132.0:140536 50.a e1449/1447 50:5dc2a75d:::benchmark_data_smithi130_398002_object140535:head v 1449'8856, mlcod=1449'8852)", 2024-08-17T18:18:59.508 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:58.420152+0000", 2024-08-17T18:18:59.508 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.0098176949999999995, 2024-08-17T18:18:59.509 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0099599960000000005, 2024-08-17T18:18:59.509 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:59.509 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "started", 2024-08-17T18:18:59.509 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:59.509 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.509 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:59.509 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.420152+0000", 2024-08-17T18:18:59.509 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:59.509 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.509 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.509 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:59.509 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.420152+0000", 2024-08-17T18:18:59.509 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:59.510 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.510 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.510 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:59.510 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.420151+0000", 2024-08-17T18:18:59.510 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:18:59.510 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.510 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.510 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:59.510 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.420187+0000", 2024-08-17T18:18:59.510 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6165999999999999e-05 2024-08-17T18:18:59.510 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.510 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.510 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:59.511 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.420205+0000", 2024-08-17T18:18:59.511 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.7739000000000001e-05 2024-08-17T18:18:59.511 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.511 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.511 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:59.511 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.420210+0000", 2024-08-17T18:18:59.511 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6619999999999996e-06 2024-08-17T18:18:59.511 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.511 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.511 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:59.511 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.423678+0000", 2024-08-17T18:18:59.511 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0034685129999999999 2024-08-17T18:18:59.511 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.511 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.512 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:59.512 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.423702+0000", 2024-08-17T18:18:59.512 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3249000000000001e-05 2024-08-17T18:18:59.512 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:59.512 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:59.512 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:59.512 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.512 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.512 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.47132.0:140546 50.a e1449/1447 50:544d9bc1:::benchmark_data_smithi130_398002_object140545:head v 1449'8857, mlcod=1449'8852)", 2024-08-17T18:18:59.512 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:58.426592+0000", 2024-08-17T18:18:59.512 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.0033779090000000001, 2024-08-17T18:18:59.512 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0035767170000000001, 2024-08-17T18:18:59.512 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:59.513 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "started", 2024-08-17T18:18:59.513 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:59.513 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.513 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:59.513 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.426592+0000", 2024-08-17T18:18:59.513 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:59.513 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.513 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.513 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:59.513 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.426592+0000", 2024-08-17T18:18:59.513 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:59.513 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.513 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.513 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:59.514 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.426591+0000", 2024-08-17T18:18:59.514 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:18:59.514 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.514 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.514 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:59.514 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.426636+0000", 2024-08-17T18:18:59.514 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5398999999999997e-05 2024-08-17T18:18:59.514 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.514 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.514 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:59.514 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.426656+0000", 2024-08-17T18:18:59.514 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.9522e-05 2024-08-17T18:18:59.514 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.515 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.515 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:59.515 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.426663+0000", 2024-08-17T18:18:59.515 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.6229999999999997e-06 2024-08-17T18:18:59.515 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.515 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.515 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:59.515 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.426907+0000", 2024-08-17T18:18:59.515 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00024374800000000001 2024-08-17T18:18:59.515 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.515 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.515 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:59.516 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.426928+0000", 2024-08-17T18:18:59.516 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0780000000000001e-05 2024-08-17T18:18:59.516 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:59.516 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:59.516 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:59.516 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.516 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.516 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.47132.0:140553 50.a e1449/1447 50:5c063fa5:::benchmark_data_smithi130_398002_object140552:head v 1449'8858, mlcod=1449'8852)", 2024-08-17T18:18:59.516 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:58.428433+0000", 2024-08-17T18:18:59.516 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.0015366449999999999, 2024-08-17T18:18:59.516 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001794962, 2024-08-17T18:18:59.516 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:59.516 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "started", 2024-08-17T18:18:59.517 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:59.517 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.517 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:59.517 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.428433+0000", 2024-08-17T18:18:59.517 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:59.517 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.517 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.517 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:59.517 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.428433+0000", 2024-08-17T18:18:59.517 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:59.517 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.517 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.517 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:59.517 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.428432+0000", 2024-08-17T18:18:59.518 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:18:59.518 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.518 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.518 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:59.518 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.428477+0000", 2024-08-17T18:18:59.518 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5272000000000003e-05 2024-08-17T18:18:59.518 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.518 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.518 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:59.518 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.428491+0000", 2024-08-17T18:18:59.518 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3499000000000001e-05 2024-08-17T18:18:59.518 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.518 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.519 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:59.519 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.428497+0000", 2024-08-17T18:18:59.519 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3829999999999997e-06 2024-08-17T18:18:59.519 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.519 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.519 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:59.519 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.430039+0000", 2024-08-17T18:18:59.519 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0015413740000000001 2024-08-17T18:18:59.519 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.519 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.519 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:59.519 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.430074+0000", 2024-08-17T18:18:59.519 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.5644e-05 2024-08-17T18:18:59.520 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:59.520 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:59.520 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:59.520 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.520 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.520 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:140555 50.17 50:ed879109:::benchmark_data_smithi130_398002_object140554:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1449)", 2024-08-17T18:18:59.520 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:58.429368+0000", 2024-08-17T18:18:59.520 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.00060103700000000001, 2024-08-17T18:18:59.520 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00091956099999999995, 2024-08-17T18:18:59.520 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:18:59.520 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:18:59.520 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:18:59.520 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:18:59.521 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:18:59.521 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 140555 2024-08-17T18:18:59.521 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.521 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:18:59.521 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.521 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:18:59.521 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.429368+0000", 2024-08-17T18:18:59.521 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:59.521 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.521 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.521 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:18:59.521 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.429368+0000", 2024-08-17T18:18:59.521 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:18:59.522 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.522 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.522 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:18:59.522 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.429367+0000", 2024-08-17T18:18:59.522 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:18:59.522 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.522 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.522 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:18:59.522 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.429417+0000", 2024-08-17T18:18:59.522 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0212e-05 2024-08-17T18:18:59.522 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.522 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.522 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:18:59.523 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.429434+0000", 2024-08-17T18:18:59.523 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.7759000000000001e-05 2024-08-17T18:18:59.523 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.523 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.523 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:18:59.523 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.429441+0000", 2024-08-17T18:18:59.523 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.917e-06 2024-08-17T18:18:59.523 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.523 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.523 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:18:59.523 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.429489+0000", 2024-08-17T18:18:59.523 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7225000000000003e-05 2024-08-17T18:18:59.523 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.523 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.524 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:18:59.524 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.429641+0000", 2024-08-17T18:18:59.524 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00015226000000000001 2024-08-17T18:18:59.524 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:18:59.524 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:18:59.524 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0", 2024-08-17T18:18:59.524 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:58.430109+0000", 2024-08-17T18:18:59.524 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00046837400000000003 2024-08-17T18:18:59.524 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:59.524 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:18:59.524 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:59.524 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:18:59.524 INFO:teuthology.orchestra.run.smithi179.stdout: ], 2024-08-17T18:18:59.525 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 6 2024-08-17T18:18:59.525 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:18:59.583 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:18:59.617 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:18:59.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:59.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:18:59.719 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:18:59.763 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:59.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:59.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:18:59.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:18:59 smithi179 ceph-mon[93519]: pgmap v2780: 99 pgs: 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.2 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 74985/403311 objects degraded (18.592%); 42526/403311 objects misplaced (10.544%); 1.5 MiB/s, 1 keys/s, 24 objects/s recovering 2024-08-17T18:18:59.865 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:18:59.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:18:59.906 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:18:59.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:18:59.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:18:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:18:59] "GET /metrics HTTP/1.1" 200 36808 "" "Prometheus/2.51.0" 2024-08-17T18:18:59.917 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:18:59 smithi130 podman[407111]: 2024-08-17 18:18:59.595825862 +0000 UTC m=+1.510801994 container remove fc6c472ee0005d3f7b631ca51d6372b006a97c418f8e65ff3b9d2179b729ccd6 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-activate, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T18:19:00.007 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:00.044 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:00.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:00.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:00.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:00.156 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 88 16 142612 142596 101.251 79.0625 0.0126294 0.00987134 2024-08-17T18:19:00.172 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:00.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:00.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:00.179 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:19:00.274 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:00.307 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:00.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:00.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:00.409 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:00.442 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:00.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:00.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:00.544 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:00.573 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:00.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:00.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:19:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:19:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:19:00.675 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:00.708 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:00.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:00.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:00.810 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:00.851 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:00.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:00.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:00.931 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:19:00.931 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:19:00.942 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:19:00.942 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:19:00.943 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:19:00.953 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:00.978 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:19:00.979 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:19:00.983 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:00.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:00.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:01.015 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:19:01.015 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:19:01.045 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:19:01.046 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:19:01.085 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:01.086 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:19:01.087 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:19:01.115 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:01.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:01.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:01.121 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:19:01.122 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:19:01.156 INFO:tasks.radosbench.radosbench.0.smithi130.stdout: 89 16 143858 143842 100.987 77.875 0.0257092 0.00989757 2024-08-17T18:19:01.217 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:01.253 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:01.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:01.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:01.354 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:01.384 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:01.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:01.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:01.485 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:01.525 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:01.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:01.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:01 smithi179 ceph-mon[93519]: pgmap v2781: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.5 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 77434/419301 objects degraded (18.467%); 43744/419301 objects misplaced (10.433%); 7.5 MiB/s, 1 keys/s, 120 objects/s recovering 2024-08-17T18:19:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:01 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/807719285' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:19:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 77434/419301 objects degraded (18.467%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:01.627 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:01.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:01.662 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:01.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:01 smithi130 ceph-mon[105521]: pgmap v2781: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.5 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 77434/419301 objects degraded (18.467%); 43744/419301 objects misplaced (10.433%); 7.5 MiB/s, 1 keys/s, 120 objects/s recovering 2024-08-17T18:19:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:01 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/807719285' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:19:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 77434/419301 objects degraded (18.467%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:01 smithi130 ceph-mon[110068]: pgmap v2781: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.5 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s; 77434/419301 objects degraded (18.467%); 43744/419301 objects misplaced (10.433%); 7.5 MiB/s, 1 keys/s, 120 objects/s recovering 2024-08-17T18:19:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:01 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/807719285' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:19:01.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 77434/419301 objects degraded (18.467%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:01.763 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:01.798 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:01.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:01.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:01.900 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:01.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:01.938 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:01.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:02.039 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:02.075 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:02.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:02.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:02.113 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T18:19:02.156 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Total time run: 90.0086 2024-08-17T18:19:02.156 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Total writes made: 145063 2024-08-17T18:19:02.156 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Write size: 65536 2024-08-17T18:19:02.156 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Object size: 65536 2024-08-17T18:19:02.157 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Bandwidth (MB/sec): 100.729 2024-08-17T18:19:02.157 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev Bandwidth: 44.0102 2024-08-17T18:19:02.157 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max bandwidth (MB/sec): 270.125 2024-08-17T18:19:02.157 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min bandwidth (MB/sec): 0 2024-08-17T18:19:02.157 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Average IOPS: 1611 2024-08-17T18:19:02.157 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev IOPS: 704.163 2024-08-17T18:19:02.157 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max IOPS: 4322 2024-08-17T18:19:02.157 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min IOPS: 0 2024-08-17T18:19:02.157 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Average Latency(s): 0.00992367 2024-08-17T18:19:02.157 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Stddev Latency(s): 0.0260131 2024-08-17T18:19:02.157 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Max latency(s): 2.21256 2024-08-17T18:19:02.157 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Min latency(s): 0.00107633 2024-08-17T18:19:02.161 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T18:19:02.177 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:02.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:02.265 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:02.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:02.367 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:02.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:02.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:02.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:02.507 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:02.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:02.541 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:02.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:02.581 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:19:02.642 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:02.677 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:02.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:02.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:02.779 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:02.814 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:02.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:02.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:02.916 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:02.949 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:02.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:02.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:02.999 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:19:02.999 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:19:03.051 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:03.087 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:03.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:03.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:03.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:03.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:03.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:03.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:03.320 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:03.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:03.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:03.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:03.457 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:03.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:03.493 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:03.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:03.529 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:03 smithi130 ceph-mon[105521]: pgmap v2782: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.5 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 77434/419301 objects degraded (18.467%); 43744/419301 objects misplaced (10.433%); 6.1 MiB/s, 0 keys/s, 98 objects/s recovering 2024-08-17T18:19:03.530 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:03 smithi130 ceph-mon[110068]: pgmap v2782: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.5 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 77434/419301 objects degraded (18.467%); 43744/419301 objects misplaced (10.433%); 6.1 MiB/s, 0 keys/s, 98 objects/s recovering 2024-08-17T18:19:03.530 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:19:03 smithi130 podman[407508]: 2024-08-17 18:19:03.20387334 +0000 UTC m=+0.220829769 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T18:19:03.530 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:19:03 smithi130 podman[407508]: 2024-08-17 18:19:03.374613516 +0000 UTC m=+0.391569937 container create 294e7a7bd278c7b3a13f0e1b4b9c0a793fc64bff3f7cecc3644635b586ac1f8e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-08-17T18:19:03.594 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:03.625 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:03.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:03.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:03.727 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:03.773 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:03.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:03.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:03.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:03 smithi179 ceph-mon[93519]: pgmap v2782: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.5 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 77434/419301 objects degraded (18.467%); 43744/419301 objects misplaced (10.433%); 6.1 MiB/s, 0 keys/s, 98 objects/s recovering 2024-08-17T18:19:03.874 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:03.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:03.902 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:03.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:04.003 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:04.022 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T18:19:04.040 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:04.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:04.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:04.142 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:04.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:19:03 smithi130 podman[407508]: 2024-08-17 18:19:03.871832924 +0000 UTC m=+0.888789349 container init 294e7a7bd278c7b3a13f0e1b4b9c0a793fc64bff3f7cecc3644635b586ac1f8e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T18:19:04.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:19:03 smithi130 podman[407508]: 2024-08-17 18:19:03.88442149 +0000 UTC m=+0.901377910 container start 294e7a7bd278c7b3a13f0e1b4b9c0a793fc64bff3f7cecc3644635b586ac1f8e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0) 2024-08-17T18:19:04.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:19:04 smithi130 bash[407508]: 294e7a7bd278c7b3a13f0e1b4b9c0a793fc64bff3f7cecc3644635b586ac1f8e 2024-08-17T18:19:04.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:19:04 smithi130 systemd[1]: Started Ceph osd.2 for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T18:19:04.168 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:04.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:04.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:04.270 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:04.307 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:04.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:04.401 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:19:04.408 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:04.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:04.437 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:04.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:04.538 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:04.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:04.573 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:04.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:04.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:04 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3368282955' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:19:04.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:04 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:19:04.653 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:19:04.653 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:19:04.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:04 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3368282955' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:19:04.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:04 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:19:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:04 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3368282955' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:19:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:04 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:19:04.674 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:04.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:04.710 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:04.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:04.811 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:04.845 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:04.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:04.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:04.948 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:04.982 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:04.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:04.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:05.000 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:19:05.083 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:05.119 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:05.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:05.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:05.221 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:05.259 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:05.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:05.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:05.361 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:05.386 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:05.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:05.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:05.390 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T18:19:05.488 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:05.519 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:19:05.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:05.531 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:05.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:05.632 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:05.666 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:19:05 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[407557]: 2024-08-17T18:19:05.540+0000 7f7256c97640 -1 Falling back to public interface 2024-08-17T18:19:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:05 smithi130 ceph-mon[105521]: pgmap v2783: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 102 B/s rd, 74 MiB/s wr, 1.27k op/s; 78741/432063 objects degraded (18.224%); 45282/432063 objects misplaced (10.480%); 11 MiB/s, 174 objects/s recovering 2024-08-17T18:19:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:05 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:19:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:05 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:19:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:05 smithi130 ceph-mon[105521]: osdmap e1450: 8 total, 7 up, 7 in 2024-08-17T18:19:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:05 smithi130 ceph-mon[110068]: pgmap v2783: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 102 B/s rd, 74 MiB/s wr, 1.27k op/s; 78741/432063 objects degraded (18.224%); 45282/432063 objects misplaced (10.480%); 11 MiB/s, 174 objects/s recovering 2024-08-17T18:19:05.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:05 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:19:05.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:05 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:19:05.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:05 smithi130 ceph-mon[110068]: osdmap e1450: 8 total, 7 up, 7 in 2024-08-17T18:19:05.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:05.679 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:05.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:05.781 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:05.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:05.825 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:05.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:05.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:05 smithi179 ceph-mon[93519]: pgmap v2783: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 102 B/s rd, 74 MiB/s wr, 1.27k op/s; 78741/432063 objects degraded (18.224%); 45282/432063 objects misplaced (10.480%); 11 MiB/s, 174 objects/s recovering 2024-08-17T18:19:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:05 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:19:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:05 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:19:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:05 smithi179 ceph-mon[93519]: osdmap e1450: 8 total, 7 up, 7 in 2024-08-17T18:19:05.869 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:19:05.926 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:05.959 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:05.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:05.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:06.061 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:06.096 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:06.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:06.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:06.198 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:06.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:06.242 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:06.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:06.343 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:06.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:06.398 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:06.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:06.499 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:06.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:06.536 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:06.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:06.637 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:06.655 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 78741/432063 objects degraded (18.224%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:06.655 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 78741/432063 objects degraded (18.224%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:06.664 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:06.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:06.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:06.766 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:06.805 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:06.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:06.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:06.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 78741/432063 objects degraded (18.224%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:06.907 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:06.927 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:19:06.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:06.954 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:06.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:07.055 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:07.093 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:07.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:07.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:07.195 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:07.229 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:07.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:07.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:07.331 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:07.364 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:07.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:07.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:07.466 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:07.487 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:07.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:07.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:07.589 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:07.624 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:07.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:07.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:07.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:07 smithi130 ceph-mon[105521]: pgmap v2785: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 102 B/s rd, 74 MiB/s wr, 1.27k op/s; 78741/432063 objects degraded (18.224%); 45282/432063 objects misplaced (10.480%); 11 MiB/s, 173 objects/s recovering 2024-08-17T18:19:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:07 smithi130 ceph-mon[110068]: pgmap v2785: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 102 B/s rd, 74 MiB/s wr, 1.27k op/s; 78741/432063 objects degraded (18.224%); 45282/432063 objects misplaced (10.480%); 11 MiB/s, 173 objects/s recovering 2024-08-17T18:19:07.726 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:07.755 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:07.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:07.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:07 smithi179 ceph-mon[93519]: pgmap v2785: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.8 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 102 B/s rd, 74 MiB/s wr, 1.27k op/s; 78741/432063 objects degraded (18.224%); 45282/432063 objects misplaced (10.480%); 11 MiB/s, 173 objects/s recovering 2024-08-17T18:19:07.857 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:07.887 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:07.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:07.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:07.989 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:08.022 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:08.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:08.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:08.124 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:08.125 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:19:08.157 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:08.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:08.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:08.258 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:08.308 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:08.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:08.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:08.409 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:08.440 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:08.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:08.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:08.541 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:08.588 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:08.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:08.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:08.689 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:08.729 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:08.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:08.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:08.831 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:08.866 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:08.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:08.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:08.969 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:09.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:09.010 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:09.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:09.111 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:09.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:09.152 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:09.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:09.253 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:09.291 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:09.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:09.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:09.393 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:09.427 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:09.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:09.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:09.529 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:09.550 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:09.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:09.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:09.577 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:19:09.577 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:19:09.652 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:09.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:09 smithi130 ceph-mon[105521]: pgmap v2786: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.7 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 102 B/s rd, 67 MiB/s wr, 1.28k op/s; 78357/428721 objects degraded (18.277%); 44552/428721 objects misplaced (10.392%); 11 MiB/s, 173 objects/s recovering 2024-08-17T18:19:09.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:09 smithi130 ceph-mon[110068]: pgmap v2786: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.7 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 102 B/s rd, 67 MiB/s wr, 1.28k op/s; 78357/428721 objects degraded (18.277%); 44552/428721 objects misplaced (10.392%); 11 MiB/s, 173 objects/s recovering 2024-08-17T18:19:09.688 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:09.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:09.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:09.790 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:09.820 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:09.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:09.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:09.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:09 smithi179 ceph-mon[93519]: pgmap v2786: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.7 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 102 B/s rd, 67 MiB/s wr, 1.28k op/s; 78357/428721 objects degraded (18.277%); 44552/428721 objects misplaced (10.392%); 11 MiB/s, 173 objects/s recovering 2024-08-17T18:19:09.922 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:09.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:09.960 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:09.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:10.061 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:10.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:10.090 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:10.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:10.170 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:19:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:19:09] "GET /metrics HTTP/1.1" 200 36808 "" "Prometheus/2.51.0" 2024-08-17T18:19:10.191 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:10.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:10.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:10.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:10.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T18:19:10.325 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:10.353 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:10.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:10.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:10.455 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:10.491 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:10.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:10.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:19:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:19:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:19:10.593 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:10.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:10.647 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:10.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:10.748 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:10.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:10.786 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:10.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:10.887 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:10.926 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:10.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:10.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:11.028 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:11.061 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:11.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:11.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:11.163 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:11.205 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:11.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:11.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:11.308 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:11.345 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:11.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:11.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:11.447 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:11.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:11.473 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:11.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:11.574 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:11.608 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:11.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:11.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:11 smithi130 ceph-mon[105521]: pgmap v2787: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.4 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 102 B/s rd, 33 MiB/s wr, 1.29k op/s; 74435/412578 objects degraded (18.041%); 43264/412578 objects misplaced (10.486%); 11 MiB/s, 168 objects/s recovering 2024-08-17T18:19:11.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 74435/412578 objects degraded (18.041%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:11 smithi130 ceph-mon[110068]: pgmap v2787: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.4 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 102 B/s rd, 33 MiB/s wr, 1.29k op/s; 74435/412578 objects degraded (18.041%); 43264/412578 objects misplaced (10.486%); 11 MiB/s, 168 objects/s recovering 2024-08-17T18:19:11.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 74435/412578 objects degraded (18.041%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:11.710 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:11.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:11.744 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:11.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:11 smithi179 ceph-mon[93519]: pgmap v2787: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.4 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 102 B/s rd, 33 MiB/s wr, 1.29k op/s; 74435/412578 objects degraded (18.041%); 43264/412578 objects misplaced (10.486%); 11 MiB/s, 168 objects/s recovering 2024-08-17T18:19:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 74435/412578 objects degraded (18.041%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:11.845 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:11.886 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:11.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:11.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:11.988 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:12.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:12.381 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:12.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:12.483 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:12.529 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:12.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:12.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:12.615 INFO:teuthology.orchestra.run.smithi130.stderr:no valid command found; 10 closest matches: 2024-08-17T18:19:12.615 INFO:teuthology.orchestra.run.smithi130.stderr:0 2024-08-17T18:19:12.615 INFO:teuthology.orchestra.run.smithi130.stderr:1 2024-08-17T18:19:12.615 INFO:teuthology.orchestra.run.smithi130.stderr:2 2024-08-17T18:19:12.616 INFO:teuthology.orchestra.run.smithi130.stderr:abort 2024-08-17T18:19:12.616 INFO:teuthology.orchestra.run.smithi130.stderr:assert 2024-08-17T18:19:12.616 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs debug_inject_read_zeros 2024-08-17T18:19:12.616 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs files list 2024-08-17T18:19:12.616 INFO:teuthology.orchestra.run.smithi130.stderr:bluefs stats 2024-08-17T18:19:12.616 INFO:teuthology.orchestra.run.smithi130.stderr:bluestore bluefs device info [] 2024-08-17T18:19:12.616 INFO:teuthology.orchestra.run.smithi130.stderr:config diff 2024-08-17T18:19:12.616 INFO:teuthology.orchestra.run.smithi130.stderr:admin_socket: invalid command 2024-08-17T18:19:12.631 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:12.664 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:12.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:12.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:12.766 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:12.805 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:12.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:12.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:12.908 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:12.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:12.942 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:12.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:13.043 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:13.080 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:13.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:13.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:13.182 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:13.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:13.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:13.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:13.320 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:13.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:13.356 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:13.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:13.458 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:13.487 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:13.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:13.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:13.589 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:13.591 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T18:19:13.592 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-2, ['dump_ops_in_flight'] 2024-08-17T18:19:13.618 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:13.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:13.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:13 smithi130 ceph-mon[105521]: pgmap v2788: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.4 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 102 B/s rd, 33 MiB/s wr, 1.29k op/s; 74435/412578 objects degraded (18.041%); 43264/412578 objects misplaced (10.486%); 11 MiB/s, 168 objects/s recovering 2024-08-17T18:19:13.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:13.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:19:13.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:13 smithi130 ceph-mon[110068]: pgmap v2788: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.4 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 102 B/s rd, 33 MiB/s wr, 1.29k op/s; 74435/412578 objects degraded (18.041%); 43264/412578 objects misplaced (10.486%); 11 MiB/s, 168 objects/s recovering 2024-08-17T18:19:13.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:13.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:19:13.720 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:13.755 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:13.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:13.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:13 smithi179 ceph-mon[93519]: pgmap v2788: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.4 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 102 B/s rd, 33 MiB/s wr, 1.29k op/s; 74435/412578 objects degraded (18.041%); 43264/412578 objects misplaced (10.486%); 11 MiB/s, 168 objects/s recovering 2024-08-17T18:19:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:19:13.857 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:13.900 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:13.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:13.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:13.965 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:19:14.002 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:14.028 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:14.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:14.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:14.044 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:19:14.129 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:14.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:14.166 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:14.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:14.267 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:14.295 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:14.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:14.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:14.396 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:14.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:14.425 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:14.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:14.526 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:14.564 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:14.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:14.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:14.667 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:14.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:14.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:14.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:14.800 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:14.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:14.853 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:14.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:14.955 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:15.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:15.005 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:15.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:15.106 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:15.133 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:15.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:15.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:15.235 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:15.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:15.269 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:15.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:15.370 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:15.396 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:15.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:15.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:15.498 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:15.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:15.529 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:15.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:15.631 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:15.657 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:15.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:15.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:15 smithi130 ceph-mon[110068]: pgmap v2789: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.0 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 678 KiB/s wr, 1.28k op/s; 70484/394206 objects degraded (17.880%); 41294/394206 objects misplaced (10.475%); 9.3 MiB/s, 149 objects/s recovering 2024-08-17T18:19:15.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:15 smithi130 ceph-mon[105521]: pgmap v2789: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.0 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 678 KiB/s wr, 1.28k op/s; 70484/394206 objects degraded (17.880%); 41294/394206 objects misplaced (10.475%); 9.3 MiB/s, 149 objects/s recovering 2024-08-17T18:19:15.759 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:15.796 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:15.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:15.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:15 smithi179 ceph-mon[93519]: pgmap v2789: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.0 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 678 KiB/s wr, 1.28k op/s; 70484/394206 objects degraded (17.880%); 41294/394206 objects misplaced (10.475%); 9.3 MiB/s, 149 objects/s recovering 2024-08-17T18:19:15.898 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:15.928 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:15.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:15.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:16.030 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:16.058 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:16.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:16.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:16.160 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:16.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:16.193 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:16.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:16.294 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:16.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:16.351 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:16.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:16.452 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:16.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:16.487 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:16.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:16.588 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:16.636 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:16.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:16.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:16.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 70484/394206 objects degraded (17.880%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:19:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 70484/394206 objects degraded (17.880%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:16.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:19:16.738 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:16.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:16.775 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:16.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:16.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 70484/394206 objects degraded (17.880%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:16.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:19:16.876 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:16.907 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:16.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:16.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:17.009 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:17.042 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:17.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:17.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:17.144 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:17.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:17.178 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:17.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:17.279 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:17.315 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:17.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:17.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:17.417 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:17.453 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:17.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:17.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:17.556 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:17.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:17.579 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:17.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:17.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:17 smithi130 ceph-mon[110068]: pgmap v2790: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.0 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 603 KiB/s wr, 1.14k op/s; 70484/394206 objects degraded (17.880%); 41294/394206 objects misplaced (10.475%); 8.3 MiB/s, 132 objects/s recovering 2024-08-17T18:19:17.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:17 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:17.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:17 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:17 smithi130 ceph-mon[105521]: pgmap v2790: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.0 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 603 KiB/s wr, 1.14k op/s; 70484/394206 objects degraded (17.880%); 41294/394206 objects misplaced (10.475%); 8.3 MiB/s, 132 objects/s recovering 2024-08-17T18:19:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:17 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:17 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:17.680 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:17.710 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:17.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:17.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:17.812 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:17.839 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:17 smithi179 ceph-mon[93519]: pgmap v2790: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.0 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 603 KiB/s wr, 1.14k op/s; 70484/394206 objects degraded (17.880%); 41294/394206 objects misplaced (10.475%); 8.3 MiB/s, 132 objects/s recovering 2024-08-17T18:19:17.839 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:17 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:17.840 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:17 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:17.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:17.846 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:17.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:17.947 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:17.988 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:17.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:17.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:18.089 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:18.123 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:18.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:18.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:18.225 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:18.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:18.271 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:18.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:18.372 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:18.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:18.416 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:18.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:18.517 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:18.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:18.545 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:18.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:18.593 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T18:19:18.646 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:18.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:18.703 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:18.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:18.804 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:18.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:18.840 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:18.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:18.941 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:18.970 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:18.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:18.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:19.072 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:19.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:19.112 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:19.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:19.214 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:19.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:19.244 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:19.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:19.345 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:19.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:19.382 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:19.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:19.483 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:19.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:19.506 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:19.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:19.607 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:19.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:19.648 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:19.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:19.659 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:19 smithi130 ceph-mon[105521]: pgmap v2791: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.0 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 565 KiB/s wr, 1.16k op/s; 70142/390945 objects degraded (17.942%); 40549/390945 objects misplaced (10.372%); 7.8 MiB/s, 124 objects/s recovering 2024-08-17T18:19:19.659 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:19 smithi130 ceph-mon[110068]: pgmap v2791: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.0 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 565 KiB/s wr, 1.16k op/s; 70142/390945 objects degraded (17.942%); 40549/390945 objects misplaced (10.372%); 7.8 MiB/s, 124 objects/s recovering 2024-08-17T18:19:19.749 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:19.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:19.787 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:19.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:19 smithi179 ceph-mon[93519]: pgmap v2791: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 8.0 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 565 KiB/s wr, 1.16k op/s; 70142/390945 objects degraded (17.942%); 40549/390945 objects misplaced (10.372%); 7.8 MiB/s, 124 objects/s recovering 2024-08-17T18:19:19.888 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:19.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:19:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:19:19] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-08-17T18:19:19.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:19.921 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:19.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:20.022 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:20.035 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:19:20.035 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:19:20.035 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:19:20.035 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:19:20.035 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:19:20.035 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:19:20.035 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:19:20.035 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:19:20.036 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:19:20.036 INFO:teuthology.orchestra.run.smithi130.stderr:Error ENXIO: problem getting command descriptions from osd.2 2024-08-17T18:19:20.036 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:19:20.036 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:19:20.036 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:19:20.036 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:19:20.036 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:19:20.047 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:20.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:20.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:20.148 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:20.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:20.170 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:20.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:20.275 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:20.307 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:20.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:20.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:20.342 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:19:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:19:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:19:20.409 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:20.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:20.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:20.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:20.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:20.567 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:20.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:20.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:20.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:20 smithi130 ceph-mon[105521]: osdmap e1451: 8 total, 7 up, 7 in 2024-08-17T18:19:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3074413544' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:19:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:20 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:20.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:20 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:20.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:20 smithi130 ceph-mon[110068]: osdmap e1451: 8 total, 7 up, 7 in 2024-08-17T18:19:20.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3074413544' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:19:20.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:20 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:20.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:20 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:20.670 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:20.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:20.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:20.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:20.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:20 smithi179 ceph-mon[93519]: osdmap e1451: 8 total, 7 up, 7 in 2024-08-17T18:19:20.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3074413544' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:19:20.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:20 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:20.730 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:20 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:20.812 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:20.858 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:20.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:20.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:20.960 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:21.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:21.006 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:21.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:21.107 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:21.137 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:21.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:21.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:21.239 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:21.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:21.286 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:21.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:21.387 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:21.441 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T18:19:21.442 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T18:19:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:21.459 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:21.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:21.560 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:21.589 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:21.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:21.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:21 smithi179 ceph-mon[93519]: pgmap v2793: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 7.7 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 0 B/s wr, 1.17k op/s; 66862/377508 objects degraded (17.711%); 39527/377508 objects misplaced (10.471%); 8.8 MiB/s, 140 objects/s recovering 2024-08-17T18:19:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:21 smithi179 ceph-mon[93519]: osdmap e1452: 8 total, 7 up, 7 in 2024-08-17T18:19:21.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 66862/377508 objects degraded (17.711%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:21 smithi130 ceph-mon[105521]: pgmap v2793: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 7.7 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 0 B/s wr, 1.17k op/s; 66862/377508 objects degraded (17.711%); 39527/377508 objects misplaced (10.471%); 8.8 MiB/s, 140 objects/s recovering 2024-08-17T18:19:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:21 smithi130 ceph-mon[105521]: osdmap e1452: 8 total, 7 up, 7 in 2024-08-17T18:19:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 66862/377508 objects degraded (17.711%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:21.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:21 smithi130 ceph-mon[110068]: pgmap v2793: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 7.7 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 0 B/s wr, 1.17k op/s; 66862/377508 objects degraded (17.711%); 39527/377508 objects misplaced (10.471%); 8.8 MiB/s, 140 objects/s recovering 2024-08-17T18:19:21.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:21 smithi130 ceph-mon[110068]: osdmap e1452: 8 total, 7 up, 7 in 2024-08-17T18:19:21.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 66862/377508 objects degraded (17.711%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:21.691 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:21.733 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:21.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:21.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:21.835 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:21.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:21.882 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:21.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:21.983 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:22.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:22.023 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:22.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:22.123 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:22.151 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:22.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:22.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:22.253 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:22.287 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:22.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:22.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:22.389 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:22.421 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:22.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:22.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:22.523 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:22.551 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:22.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:22.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:22.653 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:22.700 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:22.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:22.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:22.744 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:19:22.802 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:22.854 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:22.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:22.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:22.956 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:22.985 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:22.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:22.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:23.087 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:23.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:23.127 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:23.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:23.228 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:23.280 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:23.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:23.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:23.382 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:23.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:23.415 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:23.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:23.416 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:19:23 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[407557]: 2024-08-17T18:19:23.123+0000 7f7256c97640 -1 osd.2 1404 log_to_monitors true 2024-08-17T18:19:23.438 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:19:23.442 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:19:23.485 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:19:23.485 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:19:23.485 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.485 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:169807 50.17 e1453/1443 50:ebad6ee4:::benchmark_data_smithi130_398002_object24741:head v 1453'8583, mlcod=1452'8580)", 2024-08-17T18:19:23.485 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:23.461180+0000", 2024-08-17T18:19:23.485 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.021915746, 2024-08-17T18:19:23.485 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021943435000000001, 2024-08-17T18:19:23.485 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:23.485 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:23.485 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:19:23.485 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:23.486 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.486 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:23.486 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.461180+0000", 2024-08-17T18:19:23.486 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:23.486 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.486 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.486 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:23.486 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.461180+0000", 2024-08-17T18:19:23.486 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:23.486 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.486 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.486 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:23.486 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.461181+0000", 2024-08-17T18:19:23.487 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0199999999999998e-07 2024-08-17T18:19:23.487 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.487 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.487 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:23.487 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.461185+0000", 2024-08-17T18:19:23.487 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.843e-06 2024-08-17T18:19:23.487 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.487 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.487 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:23.487 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.461190+0000", 2024-08-17T18:19:23.487 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0699999999999997e-06 2024-08-17T18:19:23.487 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.487 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.487 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:23.488 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.461194+0000", 2024-08-17T18:19:23.488 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7000000000000002e-06 2024-08-17T18:19:23.488 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.488 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.488 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:23.488 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.472625+0000", 2024-08-17T18:19:23.488 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011430923000000001 2024-08-17T18:19:23.488 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.488 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.488 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:23.488 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.472869+0000", 2024-08-17T18:19:23.488 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000244319 2024-08-17T18:19:23.488 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.489 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:23.489 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.489 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.489 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.489 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:169812 50.e e1453/1434 50:75f0a662:::benchmark_data_smithi130_398002_object24746:head v 1453'10396, mlcod=1452'10392)", 2024-08-17T18:19:23.489 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:23.464925+0000", 2024-08-17T18:19:23.489 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.018171244, 2024-08-17T18:19:23.489 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018254056000000001, 2024-08-17T18:19:23.489 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:23.489 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:23.489 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:19:23.489 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:23.490 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.490 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:23.490 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.464925+0000", 2024-08-17T18:19:23.490 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:23.490 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.490 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.490 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:23.490 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.464925+0000", 2024-08-17T18:19:23.490 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:23.490 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.490 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.490 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:23.491 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.464925+0000", 2024-08-17T18:19:23.491 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1999999999999999e-07 2024-08-17T18:19:23.491 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.491 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.491 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:23.491 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.464930+0000", 2024-08-17T18:19:23.491 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1510000000000003e-06 2024-08-17T18:19:23.491 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.491 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.491 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:23.491 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.464938+0000", 2024-08-17T18:19:23.491 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.729e-06 2024-08-17T18:19:23.491 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.491 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.464944+0000", 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4620000000000004e-06 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.476022+0000", 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011077946 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.476066+0000", 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3945e-05 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.492 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:23.493 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.493 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.493 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.493 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:169813 50.5 e1453/1443 50:a76199e9:::benchmark_data_smithi130_398002_object24747:head v 1453'8654, mlcod=1452'8652)", 2024-08-17T18:19:23.493 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:23.475594+0000", 2024-08-17T18:19:23.493 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0075020720000000003, 2024-08-17T18:19:23.493 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0076332930000000002, 2024-08-17T18:19:23.493 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:23.493 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:23.493 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:19:23.494 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:23.494 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.494 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:23.494 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.475594+0000", 2024-08-17T18:19:23.494 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:23.494 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.494 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.494 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:23.494 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.475594+0000", 2024-08-17T18:19:23.494 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:23.494 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.494 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.494 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.475595+0000", 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4e-07 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.475602+0000", 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6469999999999998e-06 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.475617+0000", 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4807000000000001e-05 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.495 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:23.496 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.475624+0000", 2024-08-17T18:19:23.496 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0639999999999996e-06 2024-08-17T18:19:23.496 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.496 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.496 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:23.496 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.475869+0000", 2024-08-17T18:19:23.496 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00024455000000000001 2024-08-17T18:19:23.496 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.496 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.496 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:23.496 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.475889+0000", 2024-08-17T18:19:23.496 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0701000000000001e-05 2024-08-17T18:19:23.496 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.496 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:23.497 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.497 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.497 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.497 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:169818 50.19 e1453/1443 50:9e271a7c:::benchmark_data_smithi130_398002_object24752:head v 1453'8669, mlcod=1452'8667)", 2024-08-17T18:19:23.497 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:23.477589+0000", 2024-08-17T18:19:23.497 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0055072849999999998, 2024-08-17T18:19:23.497 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005686623, 2024-08-17T18:19:23.497 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:23.497 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:23.497 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:19:23.497 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:23.497 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.497 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:23.498 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.477589+0000", 2024-08-17T18:19:23.498 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:23.498 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.498 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.498 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:23.498 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.477589+0000", 2024-08-17T18:19:23.498 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:23.498 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.498 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.498 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:23.498 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.477590+0000", 2024-08-17T18:19:23.498 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0950000000000001e-06 2024-08-17T18:19:23.498 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.499 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.499 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:23.499 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.477597+0000", 2024-08-17T18:19:23.499 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3089999999999998e-06 2024-08-17T18:19:23.499 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.499 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.499 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:23.499 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.477609+0000", 2024-08-17T18:19:23.500 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2135e-05 2024-08-17T18:19:23.500 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.500 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.500 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:23.500 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.477617+0000", 2024-08-17T18:19:23.500 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.763e-06 2024-08-17T18:19:23.500 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.501 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.501 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:23.501 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.478993+0000", 2024-08-17T18:19:23.501 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001375579 2024-08-17T18:19:23.501 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.501 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.501 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:23.501 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.479211+0000", 2024-08-17T18:19:23.501 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000218684 2024-08-17T18:19:23.501 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.501 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:23.501 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.501 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.501 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.502 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:169819 50.7 e1453/1443 50:e173e085:::benchmark_data_smithi130_398002_object24753:head v 1453'8583, mlcod=1452'8581)", 2024-08-17T18:19:23.502 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:23.477683+0000", 2024-08-17T18:19:23.502 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0054128930000000002, 2024-08-17T18:19:23.502 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0056401380000000003, 2024-08-17T18:19:23.502 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:23.502 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:23.502 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:19:23.502 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:23.502 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.502 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:23.502 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.477683+0000", 2024-08-17T18:19:23.502 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:23.502 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.503 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.503 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:23.503 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.477683+0000", 2024-08-17T18:19:23.503 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:23.503 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.503 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.503 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:23.503 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.477684+0000", 2024-08-17T18:19:23.503 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.68e-07 2024-08-17T18:19:23.503 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.503 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.503 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:23.503 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.477688+0000", 2024-08-17T18:19:23.504 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3839999999999999e-06 2024-08-17T18:19:23.504 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.504 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.504 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:23.504 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.477697+0000", 2024-08-17T18:19:23.504 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7469999999999994e-06 2024-08-17T18:19:23.504 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.504 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.504 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:23.504 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.477702+0000", 2024-08-17T18:19:23.504 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7300000000000002e-06 2024-08-17T18:19:23.504 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.504 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.504 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:23.505 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.478958+0000", 2024-08-17T18:19:23.505 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0012550619999999999 2024-08-17T18:19:23.505 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.505 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.505 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:23.505 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.478978+0000", 2024-08-17T18:19:23.505 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0815999999999999e-05 2024-08-17T18:19:23.505 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.505 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:23.505 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.505 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.505 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.505 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:169804 50.11 50:8ad0f1c5:::benchmark_data_smithi130_398002_object24738:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1453)", 2024-08-17T18:19:23.505 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:23.442297+0000", 2024-08-17T18:19:23.506 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.040798848999999998, 2024-08-17T18:19:23.506 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.041077005, 2024-08-17T18:19:23.506 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:23.506 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:23.506 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:19:23.506 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:23.506 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:23.506 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:23.506 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 169804 2024-08-17T18:19:23.507 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.507 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:23.507 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.507 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:23.507 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.442297+0000", 2024-08-17T18:19:23.507 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:23.507 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.507 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.507 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:23.507 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.442297+0000", 2024-08-17T18:19:23.507 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.442298+0000", 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.035e-06 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.442302+0000", 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2549999999999999e-06 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.442312+0000", 2024-08-17T18:19:23.508 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9189999999999994e-06 2024-08-17T18:19:23.509 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.509 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.509 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:23.509 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.442320+0000", 2024-08-17T18:19:23.509 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3329999999999999e-06 2024-08-17T18:19:23.509 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.509 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.509 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:23.509 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.442367+0000", 2024-08-17T18:19:23.509 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7451999999999999e-05 2024-08-17T18:19:23.510 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.510 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.510 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op must wait for map", 2024-08-17T18:19:23.510 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.442373+0000", 2024-08-17T18:19:23.510 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0090000000000004e-06 2024-08-17T18:19:23.510 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.510 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.510 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:23.510 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.452377+0000", 2024-08-17T18:19:23.510 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010003557999999999 2024-08-17T18:19:23.510 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.510 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.511 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:23.511 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.452520+0000", 2024-08-17T18:19:23.511 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000142903 2024-08-17T18:19:23.511 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.511 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.511 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:19:23.511 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.453009+0000", 2024-08-17T18:19:23.511 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00048963900000000005 2024-08-17T18:19:23.511 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.511 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.511 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:23.511 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.463163+0000", 2024-08-17T18:19:23.511 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010153627 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.465040+0000", 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0018768770000000001 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:169805 50.e e1453/1434 50:7dc4f505:::benchmark_data_smithi130_398002_object24739:head v 1453'10395, mlcod=1452'10392)", 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:23.447716+0000", 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.035380116000000003, 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.03573602, 2024-08-17T18:19:23.512 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:23.513 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:23.513 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:19:23.513 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:23.513 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.513 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:23.513 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.447716+0000", 2024-08-17T18:19:23.513 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:23.513 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.513 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.513 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:23.513 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.447716+0000", 2024-08-17T18:19:23.513 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:23.513 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.513 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.514 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:23.514 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.447716+0000", 2024-08-17T18:19:23.514 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2099999999999996e-07 2024-08-17T18:19:23.514 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.514 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.514 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:23.514 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.447724+0000", 2024-08-17T18:19:23.514 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6529999999999994e-06 2024-08-17T18:19:23.514 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.514 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.514 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:23.514 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.447733+0000", 2024-08-17T18:19:23.514 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4360000000000002e-06 2024-08-17T18:19:23.515 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.515 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.515 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:23.515 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.447739+0000", 2024-08-17T18:19:23.515 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1009999999999997e-06 2024-08-17T18:19:23.515 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.515 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.515 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:23.515 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.470337+0000", 2024-08-17T18:19:23.515 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022598819999999999 2024-08-17T18:19:23.515 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:23.515 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:23.515 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:23.515 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.470358+0000", 2024-08-17T18:19:23.516 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0316999999999999e-05 2024-08-17T18:19:23.516 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.516 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:23.516 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.516 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:23.516 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T18:19:23.516 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 7 2024-08-17T18:19:23.516 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:19:23.517 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:23.547 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:23.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:23.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:23.649 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:23.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:23 smithi130 ceph-mon[105521]: pgmap v2795: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 7.7 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 66862/377508 objects degraded (17.711%); 39527/377508 objects misplaced (10.471%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T18:19:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:23 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/281025190' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:19:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:23 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:19:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:23 smithi130 ceph-mon[105521]: from='osd.2 [v2:172.21.15.130:6818/950156771,v1:172.21.15.130:6819/950156771]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T18:19:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:23 smithi130 ceph-mon[105521]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T18:19:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:23 smithi130 ceph-mon[110068]: pgmap v2795: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 7.7 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 66862/377508 objects degraded (17.711%); 39527/377508 objects misplaced (10.471%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T18:19:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:23 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/281025190' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:19:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:23 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:19:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:23 smithi130 ceph-mon[110068]: from='osd.2 [v2:172.21.15.130:6818/950156771,v1:172.21.15.130:6819/950156771]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T18:19:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:23 smithi130 ceph-mon[110068]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T18:19:23.676 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:23.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:23.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:23.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:23 smithi179 ceph-mon[93519]: pgmap v2795: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 12 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 7.7 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 66862/377508 objects degraded (17.711%); 39527/377508 objects misplaced (10.471%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T18:19:23.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:23 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/281025190' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:19:23.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:23 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:19:23.716 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:23 smithi179 ceph-mon[93519]: from='osd.2 [v2:172.21.15.130:6818/950156771,v1:172.21.15.130:6819/950156771]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T18:19:23.717 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:23 smithi179 ceph-mon[93519]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T18:19:23.778 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:23.805 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:23.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:23.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:23.907 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:23.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:23.947 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:23.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:24.048 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:24.081 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:24.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:24.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:24.183 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:24.211 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:24.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:24.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:24.291 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[407557]: 2024-08-17T18:19:24.076+0000 7f724aa39640 -1 osd.2 1453 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T18:19:24.313 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:24.344 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:24.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:24.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:24.446 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:24.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:24.480 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:24.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:24.581 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:24.630 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:24.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:24.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:24.639 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:19:24.640 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:19:24.640 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[105521]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T18:19:24.640 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[105521]: osdmap e1453: 8 total, 7 up, 7 in 2024-08-17T18:19:24.640 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[105521]: from='osd.2 [v2:172.21.15.130:6818/950156771,v1:172.21.15.130:6819/950156771]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T18:19:24.640 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[105521]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T18:19:24.640 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[105521]: from='osd.2 ' entity='osd.2' 2024-08-17T18:19:24.640 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:24.640 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:24.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:19:24.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:19:24.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[110068]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T18:19:24.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[110068]: osdmap e1453: 8 total, 7 up, 7 in 2024-08-17T18:19:24.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[110068]: from='osd.2 [v2:172.21.15.130:6818/950156771,v1:172.21.15.130:6819/950156771]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T18:19:24.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[110068]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T18:19:24.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[110068]: from='osd.2 ' entity='osd.2' 2024-08-17T18:19:24.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:24.642 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:24 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:24.734 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:24.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:24.774 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:24.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:24 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:19:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:24 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:19:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:24 smithi179 ceph-mon[93519]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T18:19:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:24 smithi179 ceph-mon[93519]: osdmap e1453: 8 total, 7 up, 7 in 2024-08-17T18:19:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:24 smithi179 ceph-mon[93519]: from='osd.2 [v2:172.21.15.130:6818/950156771,v1:172.21.15.130:6819/950156771]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T18:19:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:24 smithi179 ceph-mon[93519]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi130", "root=default"]} : dispatch 2024-08-17T18:19:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:24 smithi179 ceph-mon[93519]: from='osd.2 ' entity='osd.2' 2024-08-17T18:19:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:24 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:24 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:24.876 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:24.914 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:24.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:24.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:25.016 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:25.058 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:25.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:25.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:25.161 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:25.182 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:25.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:25.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:25.285 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:25.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:25.308 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:25.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:25.409 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:25.432 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:25.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:25.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:25.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:25.547 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T18:19:25.556 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:25.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:25.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:25.658 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:25.682 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:25.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:25.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:25.783 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:25.805 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:25.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:25.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:25.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:25 smithi179 ceph-mon[93519]: pgmap v2797: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 13 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 7.4 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.54k op/s; 63556/363168 objects degraded (17.500%); 37995/363168 objects misplaced (10.462%); 15 MiB/s, 238 objects/s recovering 2024-08-17T18:19:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:25 smithi179 ceph-mon[93519]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T18:19:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:25 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T18:19:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:25 smithi179 ceph-mon[93519]: osd.2 [v2:172.21.15.130:6818/950156771,v1:172.21.15.130:6819/950156771] boot 2024-08-17T18:19:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:25 smithi179 ceph-mon[93519]: osdmap e1454: 8 total, 8 up, 7 in 2024-08-17T18:19:25.906 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:25.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:25 smithi130 ceph-mon[105521]: pgmap v2797: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 13 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 7.4 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.54k op/s; 63556/363168 objects degraded (17.500%); 37995/363168 objects misplaced (10.462%); 15 MiB/s, 238 objects/s recovering 2024-08-17T18:19:25.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:25 smithi130 ceph-mon[105521]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T18:19:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:25 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T18:19:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:25 smithi130 ceph-mon[105521]: osd.2 [v2:172.21.15.130:6818/950156771,v1:172.21.15.130:6819/950156771] boot 2024-08-17T18:19:25.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:25 smithi130 ceph-mon[105521]: osdmap e1454: 8 total, 8 up, 7 in 2024-08-17T18:19:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:25 smithi130 ceph-mon[110068]: pgmap v2797: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 13 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 7.4 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.54k op/s; 63556/363168 objects degraded (17.500%); 37995/363168 objects misplaced (10.462%); 15 MiB/s, 238 objects/s recovering 2024-08-17T18:19:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:25 smithi130 ceph-mon[110068]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T18:19:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:25 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T18:19:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:25 smithi130 ceph-mon[110068]: osd.2 [v2:172.21.15.130:6818/950156771,v1:172.21.15.130:6819/950156771] boot 2024-08-17T18:19:25.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:25 smithi130 ceph-mon[110068]: osdmap e1454: 8 total, 8 up, 7 in 2024-08-17T18:19:25.928 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:25.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:25.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:26.030 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:26.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:26.052 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:26.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:26.153 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:26.176 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:26.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:26.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:26.278 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:26.300 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:26.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:26.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:26.358 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:19:26.402 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:26.424 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:26.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:26.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:26.525 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:26.548 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:26.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:26.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:26.650 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:26.674 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:26.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:26.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:26.776 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:26.798 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:26.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:26.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:26.900 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:26.924 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:26.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:26.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:27.025 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:27.066 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:27.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:27.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:27.129 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:19:27.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:27.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:27.233 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:27.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:27.334 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:27.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:27 smithi179 ceph-mon[93519]: pgmap v2799: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 13 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 7.4 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 63556/363168 objects degraded (17.500%); 37995/363168 objects misplaced (10.462%); 8.0 MiB/s, 127 objects/s recovering 2024-08-17T18:19:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:27 smithi179 ceph-mon[93519]: osdmap e1455: 8 total, 8 up, 7 in 2024-08-17T18:19:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:27 smithi179 ceph-mon[93519]: 50.10 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8717 2024-08-17T18:19:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:27 smithi179 ceph-mon[93519]: 50.0 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8725 2024-08-17T18:19:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:27 smithi179 ceph-mon[93519]: 50.10 continuing backfill to osd.4 from (1438'900,1453'8717] 50:09efc84d:::benchmark_data_smithi130_398002_object91543:head to 1453'8717 2024-08-17T18:19:27.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:27 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 63556/363168 objects degraded (17.500%), 16 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:27.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:27.372 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:27.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:27.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:27 smithi130 ceph-mon[105521]: pgmap v2799: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 13 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 7.4 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 63556/363168 objects degraded (17.500%); 37995/363168 objects misplaced (10.462%); 8.0 MiB/s, 127 objects/s recovering 2024-08-17T18:19:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:27 smithi130 ceph-mon[105521]: osdmap e1455: 8 total, 8 up, 7 in 2024-08-17T18:19:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:27 smithi130 ceph-mon[105521]: 50.10 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8717 2024-08-17T18:19:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:27 smithi130 ceph-mon[105521]: 50.0 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8725 2024-08-17T18:19:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:27 smithi130 ceph-mon[105521]: 50.10 continuing backfill to osd.4 from (1438'900,1453'8717] 50:09efc84d:::benchmark_data_smithi130_398002_object91543:head to 1453'8717 2024-08-17T18:19:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:27 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 63556/363168 objects degraded (17.500%), 16 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:27 smithi130 ceph-mon[110068]: pgmap v2799: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 13 active+clean+remapped, 5 active+remapped+backfill_wait, 7 active+undersized, 14 active+undersized+degraded, 58 active+clean; 7.4 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 63556/363168 objects degraded (17.500%); 37995/363168 objects misplaced (10.462%); 8.0 MiB/s, 127 objects/s recovering 2024-08-17T18:19:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:27 smithi130 ceph-mon[110068]: osdmap e1455: 8 total, 8 up, 7 in 2024-08-17T18:19:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:27 smithi130 ceph-mon[110068]: 50.10 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8717 2024-08-17T18:19:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:27 smithi130 ceph-mon[110068]: 50.0 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8725 2024-08-17T18:19:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:27 smithi130 ceph-mon[110068]: 50.10 continuing backfill to osd.4 from (1438'900,1453'8717] 50:09efc84d:::benchmark_data_smithi130_398002_object91543:head to 1453'8717 2024-08-17T18:19:27.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:27 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 63556/363168 objects degraded (17.500%), 16 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T18:19:27.473 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:27.506 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:27.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:27.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:27.608 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:27.639 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:27.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:27.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:27.742 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:27.779 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:27.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:27.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:27.880 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:27.921 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:27.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:27.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:28.023 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:28.055 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:28.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:28.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:28.157 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:28.187 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:28.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:28.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:28.289 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:28.292 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[105521]: 50.13 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8675 2024-08-17T18:19:28.292 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[105521]: 50.3 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8617 2024-08-17T18:19:28.292 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[105521]: 50.14 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8784 2024-08-17T18:19:28.292 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[105521]: 50.4 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8730 2024-08-17T18:19:28.292 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[105521]: 50.e starting backfill to osd.2 from (0'0,0'0] MAX to 1453'10492 2024-08-17T18:19:28.292 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[105521]: 50.6 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8695 2024-08-17T18:19:28.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[105521]: osdmap e1456: 8 total, 8 up, 7 in 2024-08-17T18:19:28.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[105521]: 50.15 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8685 2024-08-17T18:19:28.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[105521]: 50.16 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8550 2024-08-17T18:19:28.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[105521]: 50.17 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8633 2024-08-17T18:19:28.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[105521]: 50.7 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8623 2024-08-17T18:19:28.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[105521]: 50.5 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8704 2024-08-17T18:19:28.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:28.293 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:19:28.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[110068]: 50.13 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8675 2024-08-17T18:19:28.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[110068]: 50.3 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8617 2024-08-17T18:19:28.293 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[110068]: 50.14 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8784 2024-08-17T18:19:28.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[110068]: 50.4 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8730 2024-08-17T18:19:28.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[110068]: 50.e starting backfill to osd.2 from (0'0,0'0] MAX to 1453'10492 2024-08-17T18:19:28.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[110068]: 50.6 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8695 2024-08-17T18:19:28.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[110068]: osdmap e1456: 8 total, 8 up, 7 in 2024-08-17T18:19:28.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[110068]: 50.15 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8685 2024-08-17T18:19:28.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[110068]: 50.16 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8550 2024-08-17T18:19:28.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[110068]: 50.17 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8633 2024-08-17T18:19:28.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[110068]: 50.7 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8623 2024-08-17T18:19:28.294 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[110068]: 50.5 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8704 2024-08-17T18:19:28.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:28.295 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:19:28.316 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:28.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:28.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:28 smithi179 ceph-mon[93519]: 50.13 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8675 2024-08-17T18:19:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:28 smithi179 ceph-mon[93519]: 50.3 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8617 2024-08-17T18:19:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:28 smithi179 ceph-mon[93519]: 50.14 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8784 2024-08-17T18:19:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:28 smithi179 ceph-mon[93519]: 50.4 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8730 2024-08-17T18:19:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:28 smithi179 ceph-mon[93519]: 50.e starting backfill to osd.2 from (0'0,0'0] MAX to 1453'10492 2024-08-17T18:19:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:28 smithi179 ceph-mon[93519]: 50.6 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8695 2024-08-17T18:19:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:28 smithi179 ceph-mon[93519]: osdmap e1456: 8 total, 8 up, 7 in 2024-08-17T18:19:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:28 smithi179 ceph-mon[93519]: 50.15 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8685 2024-08-17T18:19:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:28 smithi179 ceph-mon[93519]: 50.16 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8550 2024-08-17T18:19:28.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:28 smithi179 ceph-mon[93519]: 50.17 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8633 2024-08-17T18:19:28.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:28 smithi179 ceph-mon[93519]: 50.7 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8623 2024-08-17T18:19:28.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:28 smithi179 ceph-mon[93519]: 50.5 starting backfill to osd.2 from (0'0,0'0] MAX to 1453'8704 2024-08-17T18:19:28.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:28.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:19:28.418 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:28.443 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:28.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:28.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:28.545 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:28.586 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:28.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:28.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:28.688 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:28.728 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:28.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:28.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:28.832 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:28.875 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:28.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:28.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:28.976 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:29.006 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:29.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:29.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:29.108 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:29.140 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:29.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:29.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:29.166 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:19:29.166 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:19:29.166 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.166 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:173115 50.4 e1456/1455 50:26a6cde8:::benchmark_data_smithi130_398002_object28049:head v 1456'8803, mlcod=1456'8801)", 2024-08-17T18:19:29.166 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:29.161838+0000", 2024-08-17T18:19:29.166 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0023562269999999998, 2024-08-17T18:19:29.166 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0023847579999999998, 2024-08-17T18:19:29.166 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161838+0000", 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161838+0000", 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.167 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:29.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161839+0000", 2024-08-17T18:19:29.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8000000000000004e-07 2024-08-17T18:19:29.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:29.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161846+0000", 2024-08-17T18:19:29.168 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5610000000000001e-06 2024-08-17T18:19:29.168 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.168 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.168 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:29.168 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161863+0000", 2024-08-17T18:19:29.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6898999999999999e-05 2024-08-17T18:19:29.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:29.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161870+0000", 2024-08-17T18:19:29.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7880000000000001e-06 2024-08-17T18:19:29.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.169 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.169 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:29.169 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.162015+0000", 2024-08-17T18:19:29.169 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014482399999999999 2024-08-17T18:19:29.169 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.170 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.170 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:29.170 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.162034+0000", 2024-08-17T18:19:29.170 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9134999999999999e-05 2024-08-17T18:19:29.170 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:29.170 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:29.170 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:29.170 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.170 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.170 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:173116 50.3 e1456/1455 50:c10cb6ca:::benchmark_data_smithi130_398002_object28050:head v 1456'8678, mlcod=1456'8676)", 2024-08-17T18:19:29.170 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:29.161893+0000", 2024-08-17T18:19:29.170 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0023004969999999999, 2024-08-17T18:19:29.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0023891490000000001, 2024-08-17T18:19:29.171 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:29.171 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:29.171 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:19:29.171 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:29.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:29.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161893+0000", 2024-08-17T18:19:29.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:29.171 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.171 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.171 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:29.171 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161893+0000", 2024-08-17T18:19:29.171 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161894+0000", 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.0899999999999997e-07 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161897+0000", 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7309999999999999e-06 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161906+0000", 2024-08-17T18:19:29.172 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7839999999999992e-06 2024-08-17T18:19:29.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:29.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161916+0000", 2024-08-17T18:19:29.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5810000000000003e-06 2024-08-17T18:19:29.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:29.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161970+0000", 2024-08-17T18:19:29.173 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4512000000000003e-05 2024-08-17T18:19:29.173 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.173 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.173 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:29.173 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161989+0000", 2024-08-17T18:19:29.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8615999999999999e-05 2024-08-17T18:19:29.174 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:29.174 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:29.174 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:29.174 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.174 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.174 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:173124 50.15 e1456/1455 50:abef0377:::benchmark_data_smithi130_398002_object28058:head v 1456'8738, mlcod=1456'8736)", 2024-08-17T18:19:29.174 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:29.161909+0000", 2024-08-17T18:19:29.174 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.002285189, 2024-08-17T18:19:29.174 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0024181279999999999, 2024-08-17T18:19:29.174 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:29.175 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:29.175 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:19:29.175 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:29.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.175 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:29.175 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161909+0000", 2024-08-17T18:19:29.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:29.175 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.176 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:29.176 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161909+0000", 2024-08-17T18:19:29.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:29.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.176 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:29.176 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161909+0000", 2024-08-17T18:19:29.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.68e-07 2024-08-17T18:19:29.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:29.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161916+0000", 2024-08-17T18:19:29.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3239999999999999e-06 2024-08-17T18:19:29.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.180 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:29.180 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161925+0000", 2024-08-17T18:19:29.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7270000000000002e-06 2024-08-17T18:19:29.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.180 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:29.180 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161935+0000", 2024-08-17T18:19:29.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4790000000000002e-06 2024-08-17T18:19:29.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.180 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:29.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.162505+0000", 2024-08-17T18:19:29.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00057018300000000004 2024-08-17T18:19:29.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.181 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:29.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.162527+0000", 2024-08-17T18:19:29.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1605999999999999e-05 2024-08-17T18:19:29.181 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:29.181 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:29.181 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:29.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.181 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:173118 50.13 e1456/1455 50:ca9f7ca8:::benchmark_data_smithi130_398002_object28052:head v 1456'8728, mlcod=1456'8726)", 2024-08-17T18:19:29.181 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:29.161932+0000", 2024-08-17T18:19:29.182 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0022623080000000002, 2024-08-17T18:19:29.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002420256, 2024-08-17T18:19:29.182 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:29.182 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:29.182 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:19:29.182 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:29.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:29.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161932+0000", 2024-08-17T18:19:29.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:29.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:29.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161932+0000", 2024-08-17T18:19:29.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:29.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:29.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161932+0000", 2024-08-17T18:19:29.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9999999999999999e-07 2024-08-17T18:19:29.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.183 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:29.183 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161935+0000", 2024-08-17T18:19:29.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.1049999999999999e-06 2024-08-17T18:19:29.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:29.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161940+0000", 2024-08-17T18:19:29.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2639999999999999e-06 2024-08-17T18:19:29.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:29.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.161944+0000", 2024-08-17T18:19:29.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4620000000000001e-06 2024-08-17T18:19:29.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:29.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.163974+0000", 2024-08-17T18:19:29.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002029973 2024-08-17T18:19:29.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:29.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.163992+0000", 2024-08-17T18:19:29.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8051e-05 2024-08-17T18:19:29.185 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:29.185 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:29.185 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:29.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.185 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:173122 50.10 e1456/1454)", 2024-08-17T18:19:29.185 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:29.163847+0000", 2024-08-17T18:19:29.186 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.00034666800000000001, 2024-08-17T18:19:29.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00053528600000000005, 2024-08-17T18:19:29.186 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:29.186 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:29.186 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "queued for pg", 2024-08-17T18:19:29.186 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:29.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:29.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.163847+0000", 2024-08-17T18:19:29.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:29.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:29.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.163847+0000", 2024-08-17T18:19:29.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:29.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:29.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.163848+0000", 2024-08-17T18:19:29.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5899999999999995e-07 2024-08-17T18:19:29.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:29.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.163854+0000", 2024-08-17T18:19:29.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5620000000000003e-06 2024-08-17T18:19:29.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:29.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.163863+0000", 2024-08-17T18:19:29.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3570000000000004e-06 2024-08-17T18:19:29.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:29.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:29.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:29.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:29.163869+0000", 2024-08-17T18:19:29.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4960000000000001e-06 2024-08-17T18:19:29.188 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:29.188 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:29.188 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:29.188 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:29.188 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T18:19:29.189 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 5 2024-08-17T18:19:29.189 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:19:29.241 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:29.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:29.292 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:29.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:29.393 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:29.422 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:29.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:29.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:29.524 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:29.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:29.555 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:29.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:29.656 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:29 smithi130 ceph-mon[105521]: pgmap v2802: 99 pgs: 3 peering, 1 active+undersized+degraded+remapped+backfilling, 2 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized, 12 active+undersized+degraded, 61 active+clean; 7.3 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 55917/361245 objects degraded (15.479%); 37582/361245 objects misplaced (10.403%) 2024-08-17T18:19:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:29 smithi130 ceph-mon[110068]: pgmap v2802: 99 pgs: 3 peering, 1 active+undersized+degraded+remapped+backfilling, 2 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized, 12 active+undersized+degraded, 61 active+clean; 7.3 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 55917/361245 objects degraded (15.479%); 37582/361245 objects misplaced (10.403%) 2024-08-17T18:19:29.692 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:29.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:29.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:29.794 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:29.828 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:29.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:29.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:29.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:29 smithi179 ceph-mon[93519]: pgmap v2802: 99 pgs: 3 peering, 1 active+undersized+degraded+remapped+backfilling, 2 remapped+peering, 1 active+undersized+degraded+remapped+backfill_wait, 10 active+clean+remapped, 5 active+remapped+backfill_wait, 4 active+undersized, 12 active+undersized+degraded, 61 active+clean; 7.3 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 55917/361245 objects degraded (15.479%); 37582/361245 objects misplaced (10.403%) 2024-08-17T18:19:29.887 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:19:29.930 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:29.956 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:29.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:29.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:30.035 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:19:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:19:29] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-08-17T18:19:30.057 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:30.106 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:30.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:30.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:30.208 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:30.249 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:30.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:30.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:30.351 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:30.378 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:30.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:30.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:30.479 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:30.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:30.509 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:30.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:30.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:19:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:19:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:19:30.610 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:30.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:30.643 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:30.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:30.744 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:30.786 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:30.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:30.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:30.888 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:30.932 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:30.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:30.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:31.034 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:31.060 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:31.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:31.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:31.162 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:31.199 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:31.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:31.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:31.303 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:31.331 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:31.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:31.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:31.433 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:31.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:31.475 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:31.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:31.486 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:19:31.576 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:31.622 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:31.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:31.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:31 smithi130 ceph-mon[105521]: pgmap v2803: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+degraded+remapped+backfilling, 2 remapped+peering, 10 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 75 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 53947/354672 objects degraded (15.210%); 36487/354672 objects misplaced (10.288%); 23 MiB/s, 361 objects/s recovering 2024-08-17T18:19:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 53947/354672 objects degraded (15.210%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:31 smithi130 ceph-mon[110068]: pgmap v2803: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+degraded+remapped+backfilling, 2 remapped+peering, 10 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 75 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 53947/354672 objects degraded (15.210%); 36487/354672 objects misplaced (10.288%); 23 MiB/s, 361 objects/s recovering 2024-08-17T18:19:31.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 53947/354672 objects degraded (15.210%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:31.724 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:31.762 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:31.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:31.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:31 smithi179 ceph-mon[93519]: pgmap v2803: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+degraded+remapped+backfilling, 2 remapped+peering, 10 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 75 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 53947/354672 objects degraded (15.210%); 36487/354672 objects misplaced (10.288%); 23 MiB/s, 361 objects/s recovering 2024-08-17T18:19:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 53947/354672 objects degraded (15.210%), 13 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:31.865 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:31.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:31.898 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:31.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:31.999 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:32.025 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:32.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:32.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:32.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:32.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:32.175 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:32.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:32.276 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:32.329 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:32.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:32.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:32.431 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:32.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:32.459 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:32.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:32.560 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:32.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:32.620 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:32.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:32.705 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:19:32.706 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:19:32.706 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:19:32.706 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:19:32.721 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:32.740 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:19:32.740 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:19:32.749 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:32.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:32.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:32.774 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:19:32.774 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:19:32.805 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:19:32.806 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:19:32.833 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:19:32.833 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:19:32.851 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:32.866 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:19:32.866 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:19:32.878 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:32.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:32.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:32.897 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:19:32.897 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:19:32.980 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:33.013 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:33.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:33.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:33.116 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:33.147 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:33.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:33.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:33.249 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:33.284 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:33.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:33.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:33.386 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:33.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:33.426 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:33.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:33.527 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:33.563 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:33.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:33.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:33.665 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:33 smithi130 ceph-mon[105521]: pgmap v2804: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+degraded+remapped+backfilling, 2 remapped+peering, 10 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 75 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 427 op/s; 53947/354672 objects degraded (15.210%); 36487/354672 objects misplaced (10.288%); 20 MiB/s, 327 objects/s recovering 2024-08-17T18:19:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:33 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/163094737' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:19:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:33 smithi130 ceph-mon[110068]: pgmap v2804: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+degraded+remapped+backfilling, 2 remapped+peering, 10 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 75 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 427 op/s; 53947/354672 objects degraded (15.210%); 36487/354672 objects misplaced (10.288%); 20 MiB/s, 327 objects/s recovering 2024-08-17T18:19:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:33 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/163094737' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:19:33.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:33.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:33.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:33.796 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:33.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:33.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:33.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:33.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:33 smithi179 ceph-mon[93519]: pgmap v2804: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+degraded+remapped+backfilling, 2 remapped+peering, 10 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 75 active+clean; 7.2 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 427 op/s; 53947/354672 objects degraded (15.210%); 36487/354672 objects misplaced (10.288%); 20 MiB/s, 327 objects/s recovering 2024-08-17T18:19:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:33 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/163094737' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:19:33.933 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:33.969 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:33.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:33.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:34.070 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:34.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:34.107 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:34.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:34.209 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:34.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:34.247 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:34.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:34.348 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:34.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:34.383 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:34.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:34.484 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:34.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:34.506 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:34.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:34.607 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:34.632 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:34.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:34.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:34.734 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:34.774 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:34.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:34.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:34.876 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:34.907 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:34.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:34.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:35.009 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:35.053 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:35.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:35.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:35.155 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:35.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:35.186 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:35.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:35.287 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:35.319 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:35.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:35.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:35.421 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:35.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:35.459 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:35.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:35.548 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 7, 1, 6, 4] out_osds: [5] dead_osds: [] live_osds: [1, 4, 5, 6, 7, 0, 3, 2] 2024-08-17T18:19:35.548 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:19:35.549 INFO:tasks.thrashosds.thrasher:Adding osd 5 2024-08-17T18:19:35.549 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd in 5 2024-08-17T18:19:35.560 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:35.650 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:35.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:35.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:35 smithi130 ceph-mon[105521]: pgmap v2805: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 959 op/s; 58384/340143 objects degraded (17.165%); 35039/340143 objects misplaced (10.301%); 23 MiB/s, 370 objects/s recovering 2024-08-17T18:19:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:35 smithi130 ceph-mon[110068]: pgmap v2805: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 959 op/s; 58384/340143 objects degraded (17.165%); 35039/340143 objects misplaced (10.301%); 23 MiB/s, 370 objects/s recovering 2024-08-17T18:19:35.753 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:35.776 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:35.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:35.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:35.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:35 smithi179 ceph-mon[93519]: pgmap v2805: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 959 op/s; 58384/340143 objects degraded (17.165%); 35039/340143 objects misplaced (10.301%); 23 MiB/s, 370 objects/s recovering 2024-08-17T18:19:35.878 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:35.908 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:35.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:35.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:36.010 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:36.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:36.041 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:36.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:36.141 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:36.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:36.180 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:36.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:36.281 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:36.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:36.318 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:36.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:36.419 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:36.463 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:36.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:36.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:36.565 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:36.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:36.595 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:36.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:36.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 58384/340143 objects degraded (17.165%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:36.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 58384/340143 objects degraded (17.165%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:36.696 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:36.736 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:36.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:36.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:36.838 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:36.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 58384/340143 objects degraded (17.165%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:36.874 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:36.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:36.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:36.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:37.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:37.016 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:37.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:37.117 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:37.142 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:37.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:37.244 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:37.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:37.273 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:37.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:37.374 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:37.401 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:37.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:37.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:37.504 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:37.528 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:37.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:37.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:37.630 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:37.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:37 smithi130 ceph-mon[105521]: pgmap v2806: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 58384/340143 objects degraded (17.165%); 35039/340143 objects misplaced (10.301%); 19 MiB/s, 307 objects/s recovering 2024-08-17T18:19:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:37 smithi130 ceph-mon[110068]: pgmap v2806: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 58384/340143 objects degraded (17.165%); 35039/340143 objects misplaced (10.301%); 19 MiB/s, 307 objects/s recovering 2024-08-17T18:19:37.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:37.676 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:37.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:37.778 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:37.817 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:37.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:37.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:37 smithi179 ceph-mon[93519]: pgmap v2806: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 797 op/s; 58384/340143 objects degraded (17.165%); 35039/340143 objects misplaced (10.301%); 19 MiB/s, 307 objects/s recovering 2024-08-17T18:19:37.919 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:37.954 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:37.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:37.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:38.055 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:38.096 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:38.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:38.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:38.198 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:38.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:38.229 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:38.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:38.330 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:38.373 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:38.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:38.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:38.475 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:38.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:38.505 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:38.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:38.606 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:38.631 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:38.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:38.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:38.733 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:38.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:38.762 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:38.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:38.863 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:38.896 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:38.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:38.997 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:39.041 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:39.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:39.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:39.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:39.181 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:39.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:39.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:39.282 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:39.311 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:39.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:39.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:39.413 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:39.457 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:39.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:39.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:39.559 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:39.592 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:39.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:39.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:39.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:39 smithi130 ceph-mon[105521]: pgmap v2807: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 58078/337350 objects degraded (17.216%); 34414/337350 objects misplaced (10.201%); 17 MiB/s, 278 objects/s recovering 2024-08-17T18:19:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:39 smithi130 ceph-mon[110068]: pgmap v2807: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 58078/337350 objects degraded (17.216%); 34414/337350 objects misplaced (10.201%); 17 MiB/s, 278 objects/s recovering 2024-08-17T18:19:39.693 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:39.739 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:39.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:39.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:39.841 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:39.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:39 smithi179 ceph-mon[93519]: pgmap v2807: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 58078/337350 objects degraded (17.216%); 34414/337350 objects misplaced (10.201%); 17 MiB/s, 278 objects/s recovering 2024-08-17T18:19:39.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:39.871 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:39.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:39.972 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:39.999 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:40.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:40.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:40.036 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:19:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:19:39] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.51.0" 2024-08-17T18:19:40.101 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:40.130 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:40.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:40.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:40.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:40.285 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:40.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:40.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:40.319 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:19:40.319 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:19:40.387 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:40.417 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:40.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:40.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:40.518 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:40.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:40.554 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:40.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:19:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:19:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:19:40.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:40.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:40.686 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:40.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:40.787 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:40.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:40.828 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:40.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:40.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:40.960 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:40.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:40.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:40.998 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T18:19:41.012 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:19:41.062 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:41.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:41.107 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:41.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:41.208 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:41.234 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:41.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:41.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:41.336 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:41.365 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:41.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:41.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:41.467 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:41.530 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:41.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:41.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:41.592 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:41 smithi130 ceph-mon[105521]: pgmap v2808: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.6 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s; 55452/325179 objects degraded (17.053%); 33526/325179 objects misplaced (10.310%); 19 MiB/s, 309 objects/s recovering 2024-08-17T18:19:41.593 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:41 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 55452/325179 objects degraded (17.053%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:41.593 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:41 smithi130 ceph-mon[110068]: pgmap v2808: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.6 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s; 55452/325179 objects degraded (17.053%); 33526/325179 objects misplaced (10.310%); 19 MiB/s, 309 objects/s recovering 2024-08-17T18:19:41.593 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:41 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 55452/325179 objects degraded (17.053%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:41.632 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:41.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:41.691 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:41.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:41.745 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:19:41.792 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:41.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:41.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:41.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:41.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:41 smithi179 ceph-mon[93519]: pgmap v2808: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.6 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.00k op/s; 55452/325179 objects degraded (17.053%); 33526/325179 objects misplaced (10.310%); 19 MiB/s, 309 objects/s recovering 2024-08-17T18:19:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:41 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 55452/325179 objects degraded (17.053%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:41.937 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:41.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:41.992 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:41.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:42.093 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:42.138 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:42.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:42.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:42.240 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:42.271 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:42.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:42.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:42.373 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:42.403 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:42.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:42.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:42.443 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:19:42.505 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:42.535 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:42.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:42.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:42.637 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:42.683 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:42.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:42.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:42.785 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:42.814 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:42.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:42.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:42 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3221038147' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:19:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:42 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:19:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:19:42.916 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:42 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3221038147' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:19:42.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:42 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:19:42.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:19:42.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:42 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3221038147' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:19:42.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:42 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:19:42.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:19:42.948 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:42.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:42.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:43.080 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:43.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:43.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:43.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:43.214 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:43.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:43.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:43.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:43.352 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:43.379 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:43.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:43.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:43.480 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:43.512 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:43.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:43.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:43.614 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:43.646 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:43.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:43.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:43.720 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:43 smithi130 ceph-mon[105521]: pgmap v2809: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.6 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 55452/325179 objects degraded (17.053%); 33526/325179 objects misplaced (10.310%); 8.0 MiB/s, 128 objects/s recovering 2024-08-17T18:19:43.720 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:43 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:19:43.720 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:43 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:19:43.721 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:43 smithi130 ceph-mon[105521]: osdmap e1457: 8 total, 8 up, 7 in 2024-08-17T18:19:43.721 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:43 smithi130 ceph-mon[110068]: pgmap v2809: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.6 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 55452/325179 objects degraded (17.053%); 33526/325179 objects misplaced (10.310%); 8.0 MiB/s, 128 objects/s recovering 2024-08-17T18:19:43.721 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:43 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:19:43.721 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:43 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:19:43.721 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:43 smithi130 ceph-mon[110068]: osdmap e1457: 8 total, 8 up, 7 in 2024-08-17T18:19:43.748 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:43.777 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:43.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:43.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:43.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:43 smithi179 ceph-mon[93519]: pgmap v2809: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.6 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 818 op/s; 55452/325179 objects degraded (17.053%); 33526/325179 objects misplaced (10.310%); 8.0 MiB/s, 128 objects/s recovering 2024-08-17T18:19:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:43 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:19:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:43 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:19:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:43 smithi179 ceph-mon[93519]: osdmap e1457: 8 total, 8 up, 7 in 2024-08-17T18:19:43.879 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:43.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:43.911 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:43.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:44.012 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:44.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:44.038 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:44.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:44.139 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:44.162 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:44.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:44.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:44.263 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:44.286 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:44.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:44.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:44.387 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:44.409 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:44.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:44.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:44.511 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:44.552 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:44.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:44.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:44.654 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:44.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:44.683 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:44.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:44.784 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:44.818 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:44.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:44.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:44.921 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:44.948 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:44.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:44.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:45.050 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:45.082 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:45.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:45.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:45.184 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:45.209 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:45.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:45.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:45.311 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:45.334 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:45.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:45.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:45.436 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:45.475 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:45.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:45.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:45.577 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:45.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:45.611 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:45.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:45.712 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:45.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:45.740 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:45.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:45.841 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:45.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:45 smithi179 ceph-mon[93519]: pgmap v2811: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 962 op/s; 52712/311235 objects degraded (16.936%); 32054/311235 objects misplaced (10.299%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T18:19:45.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:45.883 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:45.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:45.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:45 smithi130 ceph-mon[105521]: pgmap v2811: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 962 op/s; 52712/311235 objects degraded (16.936%); 32054/311235 objects misplaced (10.299%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T18:19:45.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:45 smithi130 ceph-mon[110068]: pgmap v2811: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 962 op/s; 52712/311235 objects degraded (16.936%); 32054/311235 objects misplaced (10.299%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T18:19:45.984 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:46.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:46.032 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:46.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:46.133 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:46.165 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:46.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:46.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:46.267 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:46.296 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:46.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:46.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:46.398 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:46.430 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:46.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:46.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:46.532 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:46.573 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:46.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:46.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:46.675 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:46.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:46.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:46.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:46.798 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:46.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:46.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:46.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:46.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 52712/311235 objects degraded (16.936%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:46.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 52712/311235 objects degraded (16.936%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:46.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 52712/311235 objects degraded (16.936%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:46.936 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:46.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:46.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:46.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:47.059 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:47.086 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:47.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:47.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:47.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:47.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:47.225 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:47.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:47.326 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:47.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:47.359 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:47.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:47.460 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:47.489 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:47.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:47.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:47.590 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:47.612 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:47.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:47.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:47.713 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:47.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:47.760 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:47.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:47.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:47 smithi179 ceph-mon[93519]: pgmap v2812: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 962 op/s; 52712/311235 objects degraded (16.936%); 32054/311235 objects misplaced (10.299%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T18:19:47.860 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:47.899 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:47.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:47.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:47.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:47 smithi130 ceph-mon[105521]: pgmap v2812: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 962 op/s; 52712/311235 objects degraded (16.936%); 32054/311235 objects misplaced (10.299%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T18:19:47.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:47 smithi130 ceph-mon[110068]: pgmap v2812: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 962 op/s; 52712/311235 objects degraded (16.936%); 32054/311235 objects misplaced (10.299%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T18:19:48.000 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:48.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:48.040 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:48.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:48.141 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:48.180 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:48.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:48.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:48.282 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:48.310 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:48.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:48.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:48.412 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:48.452 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:48.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:48.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:48.554 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:48.592 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:48.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:48.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:48.694 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:48.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:48.731 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:48.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:48.832 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:48.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:48.861 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:48.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:48.940 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:19:48.947 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:19:48.962 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:48.989 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:48.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:48.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:49.090 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:49.123 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:49.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:49.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:49.225 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:49.257 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:49.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:49.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:49.359 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:49.388 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:49.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:49.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:49.490 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:49.521 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:49.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:49.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:49.579 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:19:49.623 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:49.683 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:49.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:49.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:49.725 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:49 smithi130 ceph-mon[105521]: pgmap v2813: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 947 op/s; 52471/308919 objects degraded (16.985%); 31523/308919 objects misplaced (10.204%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T18:19:49.725 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:49 smithi130 ceph-mon[110068]: pgmap v2813: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 947 op/s; 52471/308919 objects degraded (16.985%); 31523/308919 objects misplaced (10.204%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T18:19:49.785 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:49.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:49 smithi179 ceph-mon[93519]: pgmap v2813: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 947 op/s; 52471/308919 objects degraded (16.985%); 31523/308919 objects misplaced (10.204%); 9.3 MiB/s, 148 objects/s recovering 2024-08-17T18:19:49.886 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:49.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:49.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:49.990 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:50.037 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:50.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:50.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:50.139 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:50.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:19:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:19:49] "GET /metrics HTTP/1.1" 200 36814 "" "Prometheus/2.51.0" 2024-08-17T18:19:50.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:50.178 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:50.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:50.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:50.303 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:50.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:50.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:50.404 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:50.437 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:50.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:50.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:50.471 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:19:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:19:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:19:50.472 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:50 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T18:19:50.400+0000 7fa856f19640 -1 mon.a@0(leader).osd e1457 definitely_dead 0 2024-08-17T18:19:50.540 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:50.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:50.563 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:50.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:50.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:50.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:50.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:50.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:50.788 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:50.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:50 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/181943611' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T18:19:50.805 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:50 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T18:19:50.813 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:50 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/181943611' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T18:19:50.813 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:50 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T18:19:50.813 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:50 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/181943611' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T18:19:50.814 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:50 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T18:19:50.814 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:50.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:50.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:50.896 INFO:teuthology.orchestra.run.smithi130.stderr:marked in osd.5. 2024-08-17T18:19:50.916 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:50.937 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:50.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:50.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:51.039 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:51.061 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:51.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:51.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:51.163 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:51.184 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:51.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:51.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:51.286 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:51.308 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:51.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:51.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:51.410 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:51.436 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:51.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:51.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:51.538 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:51.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:51.561 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:51.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:51.662 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:51.694 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:51.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:51.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:51.797 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:51.826 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:51.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:51.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:51.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:51 smithi179 ceph-mon[93519]: pgmap v2814: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 873 op/s; 50065/298971 objects degraded (16.746%); 30795/298971 objects misplaced (10.300%); 9.9 MiB/s, 159 objects/s recovering 2024-08-17T18:19:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:51 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-17T18:19:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:51 smithi179 ceph-mon[93519]: osdmap e1458: 8 total, 8 up, 8 in 2024-08-17T18:19:51.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 50065/298971 objects degraded (16.746%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:51.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:51 smithi130 ceph-mon[105521]: pgmap v2814: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 873 op/s; 50065/298971 objects degraded (16.746%); 30795/298971 objects misplaced (10.300%); 9.9 MiB/s, 159 objects/s recovering 2024-08-17T18:19:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:51 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-17T18:19:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:51 smithi130 ceph-mon[105521]: osdmap e1458: 8 total, 8 up, 8 in 2024-08-17T18:19:51.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 50065/298971 objects degraded (16.746%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:51 smithi130 ceph-mon[110068]: pgmap v2814: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 873 op/s; 50065/298971 objects degraded (16.746%); 30795/298971 objects misplaced (10.300%); 9.9 MiB/s, 159 objects/s recovering 2024-08-17T18:19:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:51 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-17T18:19:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:51 smithi130 ceph-mon[110068]: osdmap e1458: 8 total, 8 up, 8 in 2024-08-17T18:19:51.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 50065/298971 objects degraded (16.746%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:51.927 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:51.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:51.962 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:51.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:52.023 INFO:tasks.thrashosds.thrasher:Added osd 5 2024-08-17T18:19:52.063 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:52.094 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:52.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:52.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:52.196 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:52.225 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:52.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:52.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:52.327 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:52.361 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:52.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:52.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:52.463 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:52.529 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:52.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:52.630 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:52.672 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:52.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:52.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:52.774 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:52.816 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:52.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:52.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.14 continuing backfill to osd.2 from (1438'300,1457'9435] MIN to 1457'9435 2024-08-17T18:19:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.14 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9435 2024-08-17T18:19:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.4 continuing backfill to osd.2 from (1438'300,1457'9435] MIN to 1457'9435 2024-08-17T18:19:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.4 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9435 2024-08-17T18:19:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.15 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9320 2024-08-17T18:19:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.2 continuing backfill to osd.4 from (1438'900,1457'9501] MIN to 1457'9501 2024-08-17T18:19:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.2 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9501 2024-08-17T18:19:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.0 continuing backfill to osd.2 from (1438'900,1457'9373] 50:013c81a5:::benchmark_data_smithi130_398002_object38821:head to 1457'9373 2024-08-17T18:19:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.5 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9349 2024-08-17T18:19:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.0 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9373 2024-08-17T18:19:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: osdmap e1459: 8 total, 8 up, 8 in 2024-08-17T18:19:52.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.12 starting backfill to osd.4 from (1438'900,1457'9439] MIN to 1457'9440 2024-08-17T18:19:52.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.b starting backfill to osd.4 from (1440'5476,1457'12160] MIN to 1457'12161 2024-08-17T18:19:52.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.10 continuing backfill to osd.2 from (1438'900,1457'9415] 50:080b503b:::benchmark_data_smithi130_398002_object116412:head to 1457'9415 2024-08-17T18:19:52.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.10 continuing backfill to osd.4 from (1438'900,1457'9415] 50:09efc84d:::benchmark_data_smithi130_398002_object91543:head to 1457'9415 2024-08-17T18:19:52.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.10 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9415 2024-08-17T18:19:52.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.17 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9329 2024-08-17T18:19:52.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.7 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9317 2024-08-17T18:19:52.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.d starting backfill to osd.5 from (0'0,0'0] MAX to 1457'11927 2024-08-17T18:19:52.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.12 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9440 2024-08-17T18:19:52.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:52 smithi179 ceph-mon[93519]: 50.b starting backfill to osd.5 from (0'0,0'0] MAX to 1457'12161 2024-08-17T18:19:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.14 continuing backfill to osd.2 from (1438'300,1457'9435] MIN to 1457'9435 2024-08-17T18:19:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.14 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9435 2024-08-17T18:19:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.4 continuing backfill to osd.2 from (1438'300,1457'9435] MIN to 1457'9435 2024-08-17T18:19:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.4 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9435 2024-08-17T18:19:52.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.15 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9320 2024-08-17T18:19:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.2 continuing backfill to osd.4 from (1438'900,1457'9501] MIN to 1457'9501 2024-08-17T18:19:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.2 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9501 2024-08-17T18:19:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.0 continuing backfill to osd.2 from (1438'900,1457'9373] 50:013c81a5:::benchmark_data_smithi130_398002_object38821:head to 1457'9373 2024-08-17T18:19:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.5 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9349 2024-08-17T18:19:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.0 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9373 2024-08-17T18:19:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: osdmap e1459: 8 total, 8 up, 8 in 2024-08-17T18:19:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.12 starting backfill to osd.4 from (1438'900,1457'9439] MIN to 1457'9440 2024-08-17T18:19:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.b starting backfill to osd.4 from (1440'5476,1457'12160] MIN to 1457'12161 2024-08-17T18:19:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.10 continuing backfill to osd.2 from (1438'900,1457'9415] 50:080b503b:::benchmark_data_smithi130_398002_object116412:head to 1457'9415 2024-08-17T18:19:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.10 continuing backfill to osd.4 from (1438'900,1457'9415] 50:09efc84d:::benchmark_data_smithi130_398002_object91543:head to 1457'9415 2024-08-17T18:19:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.10 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9415 2024-08-17T18:19:52.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.17 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9329 2024-08-17T18:19:52.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.7 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9317 2024-08-17T18:19:52.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.d starting backfill to osd.5 from (0'0,0'0] MAX to 1457'11927 2024-08-17T18:19:52.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.12 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9440 2024-08-17T18:19:52.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[110068]: 50.b starting backfill to osd.5 from (0'0,0'0] MAX to 1457'12161 2024-08-17T18:19:52.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.14 continuing backfill to osd.2 from (1438'300,1457'9435] MIN to 1457'9435 2024-08-17T18:19:52.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.14 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9435 2024-08-17T18:19:52.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.4 continuing backfill to osd.2 from (1438'300,1457'9435] MIN to 1457'9435 2024-08-17T18:19:52.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.4 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9435 2024-08-17T18:19:52.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.15 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9320 2024-08-17T18:19:52.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.2 continuing backfill to osd.4 from (1438'900,1457'9501] MIN to 1457'9501 2024-08-17T18:19:52.919 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.2 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9501 2024-08-17T18:19:52.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.0 continuing backfill to osd.2 from (1438'900,1457'9373] 50:013c81a5:::benchmark_data_smithi130_398002_object38821:head to 1457'9373 2024-08-17T18:19:52.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.5 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9349 2024-08-17T18:19:52.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.0 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9373 2024-08-17T18:19:52.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: osdmap e1459: 8 total, 8 up, 8 in 2024-08-17T18:19:52.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.12 starting backfill to osd.4 from (1438'900,1457'9439] MIN to 1457'9440 2024-08-17T18:19:52.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.b starting backfill to osd.4 from (1440'5476,1457'12160] MIN to 1457'12161 2024-08-17T18:19:52.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.10 continuing backfill to osd.2 from (1438'900,1457'9415] 50:080b503b:::benchmark_data_smithi130_398002_object116412:head to 1457'9415 2024-08-17T18:19:52.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.10 continuing backfill to osd.4 from (1438'900,1457'9415] 50:09efc84d:::benchmark_data_smithi130_398002_object91543:head to 1457'9415 2024-08-17T18:19:52.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.10 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9415 2024-08-17T18:19:52.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.17 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9329 2024-08-17T18:19:52.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.7 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9317 2024-08-17T18:19:52.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.d starting backfill to osd.5 from (0'0,0'0] MAX to 1457'11927 2024-08-17T18:19:52.920 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.12 starting backfill to osd.5 from (0'0,0'0] MAX to 1457'9440 2024-08-17T18:19:52.921 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:52 smithi130 ceph-mon[105521]: 50.b starting backfill to osd.5 from (0'0,0'0] MAX to 1457'12161 2024-08-17T18:19:52.921 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:52.964 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:52.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:52.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:53.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:53.111 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:53.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:53.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:53.213 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:53.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:53.253 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:53.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:53.354 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:53.397 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:53.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:53.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:53.499 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:53.537 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:53.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:53.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:53.639 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:53.675 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:53.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:53.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:53.777 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:53.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:53.825 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:53.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:53.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:53 smithi179 ceph-mon[93519]: pgmap v2817: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 50065/298971 objects degraded (16.746%); 30795/298971 objects misplaced (10.300%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T18:19:53.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:53 smithi130 ceph-mon[105521]: pgmap v2817: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 50065/298971 objects degraded (16.746%); 30795/298971 objects misplaced (10.300%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T18:19:53.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:53 smithi130 ceph-mon[110068]: pgmap v2817: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 78 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 50065/298971 objects degraded (16.746%); 30795/298971 objects misplaced (10.300%); 6.7 MiB/s, 107 objects/s recovering 2024-08-17T18:19:53.926 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:53.948 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:53.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:53.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:54.050 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:54.069 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:19:54.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:54.107 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:54.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:54.208 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:54.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:54.251 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:54.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:54.352 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:54.385 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:54.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:54.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:54.487 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:54.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:54.524 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:54.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:54.625 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:54.655 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:54.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:54.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:54.757 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:54.787 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:54.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:54.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:54.869 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:19:54.869 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:19:54.890 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:54.908 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:19:54.909 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:19:54.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:54.935 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:54.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:54.946 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:19:54.946 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:19:54.986 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:19:54.987 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:19:55.017 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:19:55.017 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:19:55.036 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:55.048 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:19:55.049 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:19:55.086 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:19:55.086 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:19:55.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:55.087 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:55.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:55.122 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:19:55.122 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:19:55.188 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:55.224 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:55.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:55.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:55.326 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:55.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:55.362 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:55.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:55.463 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:55.499 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:55.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:55.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:55.604 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:55.638 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:55.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:55.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:55.739 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:55.793 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:55.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:55.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:55 smithi179 ceph-mon[93519]: pgmap v2818: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 activating, 1 activating+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 77 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 51380/289257 objects degraded (17.763%); 53329/289257 objects misplaced (18.437%); 18 MiB/s, 288 objects/s recovering 2024-08-17T18:19:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:55 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/257428003' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:19:55.859 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:55 smithi130 ceph-mon[105521]: pgmap v2818: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 activating, 1 activating+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 77 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 51380/289257 objects degraded (17.763%); 53329/289257 objects misplaced (18.437%); 18 MiB/s, 288 objects/s recovering 2024-08-17T18:19:55.859 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:55 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/257428003' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:19:55.859 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:55 smithi130 ceph-mon[110068]: pgmap v2818: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 activating, 1 activating+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 77 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 51380/289257 objects degraded (17.763%); 53329/289257 objects misplaced (18.437%); 18 MiB/s, 288 objects/s recovering 2024-08-17T18:19:55.859 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:55 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/257428003' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:19:55.896 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:55.945 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:55.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:55.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:56.047 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:56.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:56.084 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:56.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:56.175 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:19:56.175 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:19:56.175 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:19:56.175 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:19:56.175 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.175 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186468 49.7 49:e5240c4a:::benchmark_data_smithi130_364143_object77334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1405)", 2024-08-17T18:19:56.175 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.384573+0000", 2024-08-17T18:19:56.176 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 237.78734407600001, 2024-08-17T18:19:56.176 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.116594978, 2024-08-17T18:19:56.176 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.176 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.176 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.176 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.176 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:19:56.176 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:19:56.176 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186468 2024-08-17T18:19:56.176 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.176 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.176 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384573+0000", 2024-08-17T18:19:56.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384573+0000", 2024-08-17T18:19:56.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.177 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.177 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.177 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.177 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384575+0000", 2024-08-17T18:19:56.177 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6220000000000001e-06 2024-08-17T18:19:56.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.178 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384581+0000", 2024-08-17T18:19:56.178 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.614e-06 2024-08-17T18:19:56.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.178 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.178 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.178 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384592+0000", 2024-08-17T18:19:56.178 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1756000000000001e-05 2024-08-17T18:19:56.178 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384601+0000", 2024-08-17T18:19:56.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.081e-06 2024-08-17T18:19:56.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384848+0000", 2024-08-17T18:19:56.179 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00024697199999999998 2024-08-17T18:19:56.179 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.179 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.179 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.179 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.384857+0000", 2024-08-17T18:19:56.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6440000000000006e-06 2024-08-17T18:19:56.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.180 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.180 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.422875+0000", 2024-08-17T18:19:56.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.038018052 2024-08-17T18:19:56.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.180 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.180 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.422981+0000", 2024-08-17T18:19:56.180 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000106283 2024-08-17T18:19:56.180 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.180 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.180 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,3", 2024-08-17T18:19:56.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.423093+0000", 2024-08-17T18:19:56.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000112092 2024-08-17T18:19:56.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.181 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.430383+0000", 2024-08-17T18:19:56.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072895210000000002 2024-08-17T18:19:56.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.181 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.181 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.181 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.434283+0000", 2024-08-17T18:19:56.181 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039003509999999998 2024-08-17T18:19:56.181 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.500942+0000", 2024-08-17T18:19:56.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.066658929000000006 2024-08-17T18:19:56.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.501031+0000", 2024-08-17T18:19:56.182 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8568999999999999e-05 2024-08-17T18:19:56.182 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.182 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.182 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.182 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.501168+0000", 2024-08-17T18:19:56.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013749199999999999 2024-08-17T18:19:56.183 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.183 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.183 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.183 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.183 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.183 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186469 49.7 49:ed0a96db:::benchmark_data_smithi130_364143_object77335:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1405)", 2024-08-17T18:19:56.183 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.391950+0000", 2024-08-17T18:19:56.183 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 237.779967602, 2024-08-17T18:19:56.183 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1122878460000001, 2024-08-17T18:19:56.183 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.183 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.183 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.183 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.184 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:19:56.184 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:19:56.184 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186469 2024-08-17T18:19:56.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.184 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391950+0000", 2024-08-17T18:19:56.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.184 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.184 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.184 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.184 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391950+0000", 2024-08-17T18:19:56.184 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391952+0000", 2024-08-17T18:19:56.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7910000000000001e-06 2024-08-17T18:19:56.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.185 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391959+0000", 2024-08-17T18:19:56.185 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1339999999999998e-06 2024-08-17T18:19:56.185 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.185 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.185 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391976+0000", 2024-08-17T18:19:56.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6892999999999999e-05 2024-08-17T18:19:56.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391987+0000", 2024-08-17T18:19:56.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0910000000000001e-05 2024-08-17T18:19:56.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.186 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.186 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392049+0000", 2024-08-17T18:19:56.186 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.2512000000000001e-05 2024-08-17T18:19:56.186 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.186 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392059+0000", 2024-08-17T18:19:56.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7799999999999995e-06 2024-08-17T18:19:56.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.427062+0000", 2024-08-17T18:19:56.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0350030320000001 2024-08-17T18:19:56.187 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.187 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.187 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.187 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.427190+0000", 2024-08-17T18:19:56.187 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012799799999999999 2024-08-17T18:19:56.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,3", 2024-08-17T18:19:56.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.427329+0000", 2024-08-17T18:19:56.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013924299999999999 2024-08-17T18:19:56.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.188 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.434307+0000", 2024-08-17T18:19:56.188 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0069778030000000003 2024-08-17T18:19:56.188 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.188 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.188 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.448038+0000", 2024-08-17T18:19:56.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013730671 2024-08-17T18:19:56.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.504018+0000", 2024-08-17T18:19:56.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.055980038000000003 2024-08-17T18:19:56.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.189 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.189 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.504124+0000", 2024-08-17T18:19:56.189 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010654699999999999 2024-08-17T18:19:56.189 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.189 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.190 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:56.191 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.191 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.504238+0000", 2024-08-17T18:19:56.191 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011349399999999999 2024-08-17T18:19:56.191 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.191 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.191 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.191 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.191 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.191 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:211982 49.3 49:c754feed:::benchmark_data_smithi130_364143_object102848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1420)", 2024-08-17T18:19:56.191 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:33.639055+0000", 2024-08-17T18:19:56.191 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 202.532862741, 2024-08-17T18:19:56.191 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1370433850000001, 2024-08-17T18:19:56.192 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.192 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.192 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:19:56.192 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.192 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:19:56.192 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:19:56.192 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 211982 2024-08-17T18:19:56.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.192 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.192 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.192 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.192 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639055+0000", 2024-08-17T18:19:56.192 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.192 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639055+0000", 2024-08-17T18:19:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639056+0000", 2024-08-17T18:19:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7600000000000001e-06 2024-08-17T18:19:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639063+0000", 2024-08-17T18:19:56.193 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.212e-06 2024-08-17T18:19:56.194 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639079+0000", 2024-08-17T18:19:56.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6379999999999999e-05 2024-08-17T18:19:56.194 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.194 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639087+0000", 2024-08-17T18:19:56.194 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9270000000000005e-06 2024-08-17T18:19:56.194 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.194 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.194 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639143+0000", 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6100999999999997e-05 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639271+0000", 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012814599999999999 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 7", 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.639398+0000", 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012684799999999999 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.195 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.661999+0000", 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022601033 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.662009+0000", 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0073e-05 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.638599+0000", 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.97658970499999997 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.196 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.638616+0000", 2024-08-17T18:19:56.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6643e-05 2024-08-17T18:19:56.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.765739+0000", 2024-08-17T18:19:56.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1271230050000001 2024-08-17T18:19:56.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.197 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:19:56.197 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.765865+0000", 2024-08-17T18:19:56.197 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012649699999999999 2024-08-17T18:19:56.197 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.197 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.198 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.198 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.774953+0000", 2024-08-17T18:19:56.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.009087655 2024-08-17T18:19:56.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.198 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.198 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.198 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.776098+0000", 2024-08-17T18:19:56.198 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0011454 2024-08-17T18:19:56.198 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.198 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.198 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.198 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.198 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.199 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:211990 49.3 49:c9339f7c:::benchmark_data_smithi130_364143_object102856:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1421)", 2024-08-17T18:19:56.199 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:33.670314+0000", 2024-08-17T18:19:56.199 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 202.50160379799999, 2024-08-17T18:19:56.199 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.105048381, 2024-08-17T18:19:56.199 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.199 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.199 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.199 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.199 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:19:56.199 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:19:56.199 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 211990 2024-08-17T18:19:56.199 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.199 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.670314+0000", 2024-08-17T18:19:56.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.670314+0000", 2024-08-17T18:19:56.200 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.200 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.200 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.200 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.200 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.670316+0000", 2024-08-17T18:19:56.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2120000000000002e-06 2024-08-17T18:19:56.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.201 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.201 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.670323+0000", 2024-08-17T18:19:56.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4019999999999996e-06 2024-08-17T18:19:56.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.201 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.201 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.670342+0000", 2024-08-17T18:19:56.201 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8309e-05 2024-08-17T18:19:56.201 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.201 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.202 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.202 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.670352+0000", 2024-08-17T18:19:56.202 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0461e-05 2024-08-17T18:19:56.202 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.202 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.202 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.202 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.671105+0000", 2024-08-17T18:19:56.202 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00075322500000000005 2024-08-17T18:19:56.202 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.202 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.202 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.202 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.671113+0000", 2024-08-17T18:19:56.203 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4240000000000004e-06 2024-08-17T18:19:56.203 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.203 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.203 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.203 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.638766+0000", 2024-08-17T18:19:56.203 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.96765296999999995 2024-08-17T18:19:56.203 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.203 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.203 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.203 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.638775+0000", 2024-08-17T18:19:56.203 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0839999999999998e-06 2024-08-17T18:19:56.203 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.203 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.766145+0000", 2024-08-17T18:19:56.204 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.127370099 2024-08-17T18:19:56.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.766281+0000", 2024-08-17T18:19:56.204 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013627600000000001 2024-08-17T18:19:56.204 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.204 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.204 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 6,7", 2024-08-17T18:19:56.204 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.766416+0000", 2024-08-17T18:19:56.204 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013520700000000001 2024-08-17T18:19:56.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.771055+0000", 2024-08-17T18:19:56.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046387140000000004 2024-08-17T18:19:56.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.205 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.771113+0000", 2024-08-17T18:19:56.205 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.7899000000000002e-05 2024-08-17T18:19:56.205 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.205 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.205 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.206 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.775200+0000", 2024-08-17T18:19:56.206 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0040870150000000003 2024-08-17T18:19:56.206 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.206 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.206 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.775261+0000", 2024-08-17T18:19:56.206 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1561000000000002e-05 2024-08-17T18:19:56.206 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.206 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.206 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.206 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.775362+0000", 2024-08-17T18:19:56.206 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000100523 2024-08-17T18:19:56.207 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.207 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.207 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.207 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.207 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.207 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:211997 49.3 49:c8cc8bff:::benchmark_data_smithi130_364143_object102863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1421)", 2024-08-17T18:19:56.207 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:33.674887+0000", 2024-08-17T18:19:56.207 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 202.49703085300001, 2024-08-17T18:19:56.207 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1006669499999999, 2024-08-17T18:19:56.207 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.207 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.207 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.207 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.207 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:19:56.208 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:19:56.208 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 211997 2024-08-17T18:19:56.208 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.208 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.208 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.208 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.208 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674887+0000", 2024-08-17T18:19:56.208 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.208 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.208 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.208 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.208 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674887+0000", 2024-08-17T18:19:56.208 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674888+0000", 2024-08-17T18:19:56.209 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.497e-06 2024-08-17T18:19:56.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.209 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.209 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674894+0000", 2024-08-17T18:19:56.209 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8660000000000002e-06 2024-08-17T18:19:56.209 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.209 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.210 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674910+0000", 2024-08-17T18:19:56.210 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5832e-05 2024-08-17T18:19:56.210 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.210 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.210 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674918+0000", 2024-08-17T18:19:56.210 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8990000000000001e-06 2024-08-17T18:19:56.210 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.210 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.210 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.210 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674971+0000", 2024-08-17T18:19:56.211 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3819000000000001e-05 2024-08-17T18:19:56.211 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.211 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.211 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.211 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.674980+0000", 2024-08-17T18:19:56.211 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5439999999999999e-06 2024-08-17T18:19:56.211 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.211 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.211 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.211 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.638825+0000", 2024-08-17T18:19:56.211 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.96384535000000005 2024-08-17T18:19:56.211 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.211 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.211 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.212 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.638833+0000", 2024-08-17T18:19:56.212 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1679999999999998e-06 2024-08-17T18:19:56.212 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.212 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.212 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.212 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.768782+0000", 2024-08-17T18:19:56.212 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1299494729999999 2024-08-17T18:19:56.212 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.212 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.212 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.212 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.768892+0000", 2024-08-17T18:19:56.212 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000109677 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 6,7", 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.768965+0000", 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3170999999999995e-05 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.771156+0000", 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021906619999999999 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.771230+0000", 2024-08-17T18:19:56.213 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4429000000000002e-05 2024-08-17T18:19:56.214 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.214 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.214 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.214 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.775410+0000", 2024-08-17T18:19:56.214 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0041797450000000003 2024-08-17T18:19:56.214 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.214 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.214 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.214 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.775463+0000", 2024-08-17T18:19:56.214 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3559e-05 2024-08-17T18:19:56.214 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.214 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.214 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.215 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.775554+0000", 2024-08-17T18:19:56.215 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0259000000000004e-05 2024-08-17T18:19:56.215 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.215 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.215 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.215 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.215 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.215 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:131715 50.d 50:b2005028:::benchmark_data_smithi130_398002_object131714:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1445)", 2024-08-17T18:19:56.215 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:18:49.522448+0000", 2024-08-17T18:19:56.215 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 66.649469398999997, 2024-08-17T18:19:56.215 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1376242639999998, 2024-08-17T18:19:56.215 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.215 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.216 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.216 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.216 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.216 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.216 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 131715 2024-08-17T18:19:56.216 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.216 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.216 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.216 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.216 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.522448+0000", 2024-08-17T18:19:56.216 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.216 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.216 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.217 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.217 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.522448+0000", 2024-08-17T18:19:56.217 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.217 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.217 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.217 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.522450+0000", 2024-08-17T18:19:56.217 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1390000000000002e-06 2024-08-17T18:19:56.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.217 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.217 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.217 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.522477+0000", 2024-08-17T18:19:56.217 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.6767999999999999e-05 2024-08-17T18:19:56.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.218 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.218 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.522501+0000", 2024-08-17T18:19:56.218 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.4348000000000001e-05 2024-08-17T18:19:56.218 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.218 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.218 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.522513+0000", 2024-08-17T18:19:56.218 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1504e-05 2024-08-17T18:19:56.218 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.218 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.219 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.522661+0000", 2024-08-17T18:19:56.219 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014841499999999999 2024-08-17T18:19:56.219 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.219 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.219 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.219 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.522673+0000", 2024-08-17T18:19:56.219 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.218e-05 2024-08-17T18:19:56.219 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.219 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.219 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.220 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.509241+0000", 2024-08-17T18:19:56.220 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98656729399999998 2024-08-17T18:19:56.220 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.220 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.221 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.221 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.509250+0000", 2024-08-17T18:19:56.221 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3989999999999993e-06 2024-08-17T18:19:56.221 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.221 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.221 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.221 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.650253+0000", 2024-08-17T18:19:56.221 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.141003239 2024-08-17T18:19:56.221 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.221 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.221 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.221 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.650423+0000", 2024-08-17T18:19:56.222 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016955599999999999 2024-08-17T18:19:56.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,4", 2024-08-17T18:19:56.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.650591+0000", 2024-08-17T18:19:56.222 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000167682 2024-08-17T18:19:56.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.659746+0000", 2024-08-17T18:19:56.222 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0091559359999999999 2024-08-17T18:19:56.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.659800+0000", 2024-08-17T18:19:56.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3628999999999998e-05 2024-08-17T18:19:56.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.659887+0000", 2024-08-17T18:19:56.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7072999999999998e-05 2024-08-17T18:19:56.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.659947+0000", 2024-08-17T18:19:56.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9939999999999999e-05 2024-08-17T18:19:56.224 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.224 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.224 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.224 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.660072+0000", 2024-08-17T18:19:56.224 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012516200000000001 2024-08-17T18:19:56.224 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.224 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.224 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.224 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.224 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.224 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:131724 50.d 50:babd9d5c:::benchmark_data_smithi130_398002_object131723:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1446)", 2024-08-17T18:19:56.224 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:18:49.541030+0000", 2024-08-17T18:19:56.224 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 66.630887802000004, 2024-08-17T18:19:56.224 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1260811369999999, 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 131724 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.541030+0000", 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.225 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.226 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.226 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.541030+0000", 2024-08-17T18:19:56.226 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.226 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.226 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.226 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.226 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.541031+0000", 2024-08-17T18:19:56.226 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.782e-06 2024-08-17T18:19:56.226 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.226 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.226 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.226 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.541055+0000", 2024-08-17T18:19:56.226 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.3798000000000002e-05 2024-08-17T18:19:56.227 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.227 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.227 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.541068+0000", 2024-08-17T18:19:56.227 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3066e-05 2024-08-17T18:19:56.227 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.227 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.227 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.541080+0000", 2024-08-17T18:19:56.227 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1401e-05 2024-08-17T18:19:56.227 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.227 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.227 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.541143+0000", 2024-08-17T18:19:56.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3477000000000002e-05 2024-08-17T18:19:56.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.228 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.228 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.228 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.541154+0000", 2024-08-17T18:19:56.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0861e-05 2024-08-17T18:19:56.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.228 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.228 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.228 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.509502+0000", 2024-08-17T18:19:56.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.968347756 2024-08-17T18:19:56.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.228 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.510039+0000", 2024-08-17T18:19:56.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00053716300000000001 2024-08-17T18:19:56.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.653185+0000", 2024-08-17T18:19:56.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.143145761 2024-08-17T18:19:56.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.653315+0000", 2024-08-17T18:19:56.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013045799999999999 2024-08-17T18:19:56.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.230 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,4", 2024-08-17T18:19:56.230 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.653421+0000", 2024-08-17T18:19:56.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000106289 2024-08-17T18:19:56.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.230 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.230 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.659872+0000", 2024-08-17T18:19:56.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0064508150000000004 2024-08-17T18:19:56.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.230 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.666653+0000", 2024-08-17T18:19:56.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0067805160000000003 2024-08-17T18:19:56.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.666761+0000", 2024-08-17T18:19:56.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010863599999999999 2024-08-17T18:19:56.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.666954+0000", 2024-08-17T18:19:56.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00019265800000000001 2024-08-17T18:19:56.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.232 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.232 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.667111+0000", 2024-08-17T18:19:56.232 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015669999999999999 2024-08-17T18:19:56.232 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.232 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.232 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.232 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:131729 50.d 50:b061c124:::benchmark_data_smithi130_398002_object131728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1446)", 2024-08-17T18:19:56.232 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:18:49.546159+0000", 2024-08-17T18:19:56.232 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 66.625758480000002, 2024-08-17T18:19:56.232 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1212690379999999, 2024-08-17T18:19:56.232 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.232 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.233 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.233 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.233 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.233 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.233 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 131729 2024-08-17T18:19:56.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.233 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.233 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.233 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.546159+0000", 2024-08-17T18:19:56.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.546159+0000", 2024-08-17T18:19:56.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.546160+0000", 2024-08-17T18:19:56.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1489999999999999e-06 2024-08-17T18:19:56.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.235 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.546181+0000", 2024-08-17T18:19:56.235 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0978000000000001e-05 2024-08-17T18:19:56.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.235 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.235 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.235 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.546192+0000", 2024-08-17T18:19:56.235 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1095e-05 2024-08-17T18:19:56.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.235 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.235 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.235 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.546197+0000", 2024-08-17T18:19:56.235 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3260000000000003e-06 2024-08-17T18:19:56.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.546256+0000", 2024-08-17T18:19:56.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8754e-05 2024-08-17T18:19:56.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:49.546269+0000", 2024-08-17T18:19:56.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2312e-05 2024-08-17T18:19:56.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.510222+0000", 2024-08-17T18:19:56.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.96395316200000003 2024-08-17T18:19:56.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.510232+0000", 2024-08-17T18:19:56.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0064000000000001e-05 2024-08-17T18:19:56.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.657148+0000", 2024-08-17T18:19:56.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1469161370000001 2024-08-17T18:19:56.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.657314+0000", 2024-08-17T18:19:56.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016560899999999999 2024-08-17T18:19:56.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,4", 2024-08-17T18:19:56.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.657472+0000", 2024-08-17T18:19:56.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015861200000000001 2024-08-17T18:19:56.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.666660+0000", 2024-08-17T18:19:56.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0091879509999999998 2024-08-17T18:19:56.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.666706+0000", 2024-08-17T18:19:56.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5497000000000002e-05 2024-08-17T18:19:56.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.667201+0000", 2024-08-17T18:19:56.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00049492700000000004 2024-08-17T18:19:56.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.240 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.667339+0000", 2024-08-17T18:19:56.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013830400000000001 2024-08-17T18:19:56.240 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.240 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.240 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.240 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.667428+0000", 2024-08-17T18:19:56.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9160999999999996e-05 2024-08-17T18:19:56.240 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.240 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.240 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.240 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.240 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.240 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:131689 50.a 50:5ce7e74f:::benchmark_data_smithi130_398002_object131688:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1447)", 2024-08-17T18:19:56.241 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:18:50.505569+0000", 2024-08-17T18:19:56.241 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 65.666348386999999, 2024-08-17T18:19:56.241 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.166286573, 2024-08-17T18:19:56.241 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.241 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.241 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:19:56.241 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.241 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.241 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.241 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 131689 2024-08-17T18:19:56.241 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.241 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.241 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505569+0000", 2024-08-17T18:19:56.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.242 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.242 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505569+0000", 2024-08-17T18:19:56.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.242 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.242 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505571+0000", 2024-08-17T18:19:56.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9649999999999998e-06 2024-08-17T18:19:56.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505596+0000", 2024-08-17T18:19:56.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5238000000000001e-05 2024-08-17T18:19:56.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505609+0000", 2024-08-17T18:19:56.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.234e-05 2024-08-17T18:19:56.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505617+0000", 2024-08-17T18:19:56.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8249999999999994e-06 2024-08-17T18:19:56.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.508579+0000", 2024-08-17T18:19:56.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029614200000000002 2024-08-17T18:19:56.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.508589+0000", 2024-08-17T18:19:56.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7219999999999994e-06 2024-08-17T18:19:56.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.671762+0000", 2024-08-17T18:19:56.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1631733829999999 2024-08-17T18:19:56.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.671856+0000", 2024-08-17T18:19:56.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3679999999999998e-05 2024-08-17T18:19:56.245 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.245 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.245 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.246 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:131700 50.a 50:56bb5287:::benchmark_data_smithi130_398002_object131699:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1447)", 2024-08-17T18:19:56.246 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:18:50.505690+0000", 2024-08-17T18:19:56.246 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 65.666227114999998, 2024-08-17T18:19:56.246 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1665457189999999, 2024-08-17T18:19:56.246 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.246 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.246 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:19:56.246 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.246 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.246 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.246 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 131700 2024-08-17T18:19:56.246 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.246 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505690+0000", 2024-08-17T18:19:56.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505690+0000", 2024-08-17T18:19:56.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505691+0000", 2024-08-17T18:19:56.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6400000000000003e-07 2024-08-17T18:19:56.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.248 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.248 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505710+0000", 2024-08-17T18:19:56.248 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9578000000000001e-05 2024-08-17T18:19:56.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.248 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.248 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505717+0000", 2024-08-17T18:19:56.248 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8940000000000004e-06 2024-08-17T18:19:56.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.248 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.249 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505723+0000", 2024-08-17T18:19:56.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2959999999999998e-06 2024-08-17T18:19:56.249 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.249 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.249 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.249 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.508610+0000", 2024-08-17T18:19:56.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002887906 2024-08-17T18:19:56.249 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.249 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.249 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.249 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.508617+0000", 2024-08-17T18:19:56.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4389999999999997e-06 2024-08-17T18:19:56.249 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.250 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.250 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.250 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.672150+0000", 2024-08-17T18:19:56.250 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.163532835 2024-08-17T18:19:56.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.250 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.250 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.250 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.672236+0000", 2024-08-17T18:19:56.250 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.6306999999999999e-05 2024-08-17T18:19:56.250 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.250 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.250 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.250 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.251 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:131709 50.a 50:5d0d8860:::benchmark_data_smithi130_398002_object131708:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1447)", 2024-08-17T18:19:56.251 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:18:50.505730+0000", 2024-08-17T18:19:56.251 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 65.666187793000006, 2024-08-17T18:19:56.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1666546, 2024-08-17T18:19:56.251 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.251 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.251 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:19:56.251 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.251 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.251 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.251 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 131709 2024-08-17T18:19:56.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.251 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505730+0000", 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505730+0000", 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505730+0000", 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4400000000000001e-07 2024-08-17T18:19:56.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505749+0000", 2024-08-17T18:19:56.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9188999999999999e-05 2024-08-17T18:19:56.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505756+0000", 2024-08-17T18:19:56.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.939e-06 2024-08-17T18:19:56.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505762+0000", 2024-08-17T18:19:56.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4990000000000002e-06 2024-08-17T18:19:56.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.508638+0000", 2024-08-17T18:19:56.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002876132 2024-08-17T18:19:56.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.508644+0000", 2024-08-17T18:19:56.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3729999999999998e-06 2024-08-17T18:19:56.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.672265+0000", 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1636209280000001 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.672384+0000", 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000119096 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:131716 50.0 50:070f9984:::benchmark_data_smithi130_398002_object131715:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1447)", 2024-08-17T18:19:56.255 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:18:50.505769+0000", 2024-08-17T18:19:56.256 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 65.666148519999993, 2024-08-17T18:19:56.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1102353709999999, 2024-08-17T18:19:56.256 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.256 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.256 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.256 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.256 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.256 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.256 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 131716 2024-08-17T18:19:56.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.256 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505769+0000", 2024-08-17T18:19:56.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.257 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505769+0000", 2024-08-17T18:19:56.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.257 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505769+0000", 2024-08-17T18:19:56.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2399999999999999e-07 2024-08-17T18:19:56.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505945+0000", 2024-08-17T18:19:56.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000175203 2024-08-17T18:19:56.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505954+0000", 2024-08-17T18:19:56.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3500000000000003e-06 2024-08-17T18:19:56.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.259 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.259 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505961+0000", 2024-08-17T18:19:56.259 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8839999999999997e-06 2024-08-17T18:19:56.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.259 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.259 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.511366+0000", 2024-08-17T18:19:56.259 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0054052249999999996 2024-08-17T18:19:56.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.259 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.259 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.511373+0000", 2024-08-17T18:19:56.259 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1659999999999997e-06 2024-08-17T18:19:56.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.610885+0000", 2024-08-17T18:19:56.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.099511627 2024-08-17T18:19:56.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.611009+0000", 2024-08-17T18:19:56.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012428499999999999 2024-08-17T18:19:56.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 4,6", 2024-08-17T18:19:56.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.611344+0000", 2024-08-17T18:19:56.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000334586 2024-08-17T18:19:56.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.614118+0000", 2024-08-17T18:19:56.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002774786 2024-08-17T18:19:56.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.615729+0000", 2024-08-17T18:19:56.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0016105270000000001 2024-08-17T18:19:56.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.615908+0000", 2024-08-17T18:19:56.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00017939400000000001 2024-08-17T18:19:56.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.615944+0000", 2024-08-17T18:19:56.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.5710000000000002e-05 2024-08-17T18:19:56.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.616004+0000", 2024-08-17T18:19:56.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0204000000000002e-05 2024-08-17T18:19:56.262 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.263 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.263 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.265 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:131719 50.a 50:534758a9:::benchmark_data_smithi130_398002_object131718:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1447)", 2024-08-17T18:19:56.265 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:18:50.505970+0000", 2024-08-17T18:19:56.265 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 65.665947614000004, 2024-08-17T18:19:56.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.173217226, 2024-08-17T18:19:56.265 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.265 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.265 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.265 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.265 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.266 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.266 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 131719 2024-08-17T18:19:56.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.266 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505970+0000", 2024-08-17T18:19:56.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505970+0000", 2024-08-17T18:19:56.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505970+0000", 2024-08-17T18:19:56.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9100000000000004e-07 2024-08-17T18:19:56.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505989+0000", 2024-08-17T18:19:56.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8715e-05 2024-08-17T18:19:56.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.505996+0000", 2024-08-17T18:19:56.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3889999999999999e-06 2024-08-17T18:19:56.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.268 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.506001+0000", 2024-08-17T18:19:56.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9189999999999998e-06 2024-08-17T18:19:56.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.268 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.508664+0000", 2024-08-17T18:19:56.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026628419999999999 2024-08-17T18:19:56.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.508671+0000", 2024-08-17T18:19:56.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1010000000000003e-06 2024-08-17T18:19:56.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.672420+0000", 2024-08-17T18:19:56.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.163748349 2024-08-17T18:19:56.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.672557+0000", 2024-08-17T18:19:56.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000137157 2024-08-17T18:19:56.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 4,6,7", 2024-08-17T18:19:56.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.672685+0000", 2024-08-17T18:19:56.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012778299999999999 2024-08-17T18:19:56.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.675907+0000", 2024-08-17T18:19:56.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032222539999999999 2024-08-17T18:19:56.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.678112+0000", 2024-08-17T18:19:56.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0022052830000000002 2024-08-17T18:19:56.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.678603+0000", 2024-08-17T18:19:56.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00049051300000000004 2024-08-17T18:19:56.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.678811+0000", 2024-08-17T18:19:56.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000208323 2024-08-17T18:19:56.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.678883+0000", 2024-08-17T18:19:56.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2498999999999998e-05 2024-08-17T18:19:56.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.272 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.272 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.679187+0000", 2024-08-17T18:19:56.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00030360800000000002 2024-08-17T18:19:56.272 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.272 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.272 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.273 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:131733 50.0 50:0692d62b:::benchmark_data_smithi130_398002_object131732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1447)", 2024-08-17T18:19:56.273 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:18:50.506008+0000", 2024-08-17T18:19:56.273 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 65.665909685000003, 2024-08-17T18:19:56.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.110173882, 2024-08-17T18:19:56.273 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.273 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.273 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.273 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.273 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.273 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.273 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 131733 2024-08-17T18:19:56.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.273 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.506008+0000", 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.506008+0000", 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.506008+0000", 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6600000000000002e-07 2024-08-17T18:19:56.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.275 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.275 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.506027+0000", 2024-08-17T18:19:56.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8708000000000001e-05 2024-08-17T18:19:56.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.275 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.275 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.506033+0000", 2024-08-17T18:19:56.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1800000000000001e-06 2024-08-17T18:19:56.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.275 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.275 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.506038+0000", 2024-08-17T18:19:56.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1449999999999999e-06 2024-08-17T18:19:56.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.511398+0000", 2024-08-17T18:19:56.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0053599049999999999 2024-08-17T18:19:56.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:50.511440+0000", 2024-08-17T18:19:56.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1644999999999999e-05 2024-08-17T18:19:56.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.612762+0000", 2024-08-17T18:19:56.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.101322578 2024-08-17T18:19:56.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.612906+0000", 2024-08-17T18:19:56.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014332900000000001 2024-08-17T18:19:56.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 4,6", 2024-08-17T18:19:56.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.613134+0000", 2024-08-17T18:19:56.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00022877300000000001 2024-08-17T18:19:56.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.614237+0000", 2024-08-17T18:19:56.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00110205 2024-08-17T18:19:56.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.616046+0000", 2024-08-17T18:19:56.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0018094109999999999 2024-08-17T18:19:56.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.616085+0000", 2024-08-17T18:19:56.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8634999999999999e-05 2024-08-17T18:19:56.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.616122+0000", 2024-08-17T18:19:56.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7756000000000002e-05 2024-08-17T18:19:56.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:18:51.616182+0000", 2024-08-17T18:19:56.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9301e-05 2024-08-17T18:19:56.279 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.280 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.280 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.280 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.280 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.280 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:166571 50.d 50:bb365157:::benchmark_data_smithi130_398002_object21505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1451)", 2024-08-17T18:19:56.280 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:20.103116+0000", 2024-08-17T18:19:56.280 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 36.068801409000002, 2024-08-17T18:19:56.280 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.117960624, 2024-08-17T18:19:56.280 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.280 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.280 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.280 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.280 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.281 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.281 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 166571 2024-08-17T18:19:56.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.281 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:20.103116+0000", 2024-08-17T18:19:56.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:20.103116+0000", 2024-08-17T18:19:56.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.282 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.282 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:20.103117+0000", 2024-08-17T18:19:56.282 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1680000000000001e-06 2024-08-17T18:19:56.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.282 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.282 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:20.103122+0000", 2024-08-17T18:19:56.282 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3270000000000002e-06 2024-08-17T18:19:56.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.282 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.282 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:20.103132+0000", 2024-08-17T18:19:56.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0541e-05 2024-08-17T18:19:56.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:20.103139+0000", 2024-08-17T18:19:56.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6730000000000003e-06 2024-08-17T18:19:56.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:20.103187+0000", 2024-08-17T18:19:56.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8019999999999998e-05 2024-08-17T18:19:56.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:20.103278+0000", 2024-08-17T18:19:56.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.1465999999999995e-05 2024-08-17T18:19:56.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,4", 2024-08-17T18:19:56.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:20.103359+0000", 2024-08-17T18:19:56.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0630999999999994e-05 2024-08-17T18:19:56.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:20.105898+0000", 2024-08-17T18:19:56.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025386969999999999 2024-08-17T18:19:56.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:20.106786+0000", 2024-08-17T18:19:56.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00088799599999999995 2024-08-17T18:19:56.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:20.220977+0000", 2024-08-17T18:19:56.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.114191932 2024-08-17T18:19:56.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:20.221017+0000", 2024-08-17T18:19:56.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9728000000000003e-05 2024-08-17T18:19:56.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:20.221077+0000", 2024-08-17T18:19:56.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9444999999999998e-05 2024-08-17T18:19:56.286 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.286 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.286 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.287 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:169863 50.a 50:5ab688eb:::benchmark_data_smithi130_398002_object24797:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1453)", 2024-08-17T18:19:56.287 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:23.540111+0000", 2024-08-17T18:19:56.287 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 32.631806679, 2024-08-17T18:19:56.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.110230966, 2024-08-17T18:19:56.287 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.287 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.287 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.287 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.287 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.287 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.287 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 169863 2024-08-17T18:19:56.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.288 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.288 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.288 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.540111+0000", 2024-08-17T18:19:56.288 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.288 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.288 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.540111+0000", 2024-08-17T18:19:56.288 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.288 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.289 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.540112+0000", 2024-08-17T18:19:56.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.344e-06 2024-08-17T18:19:56.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.289 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.289 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.289 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.540117+0000", 2024-08-17T18:19:56.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6369999999999998e-06 2024-08-17T18:19:56.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.289 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.289 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.289 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.540134+0000", 2024-08-17T18:19:56.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7254000000000001e-05 2024-08-17T18:19:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.540142+0000", 2024-08-17T18:19:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8509999999999999e-06 2024-08-17T18:19:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.541431+0000", 2024-08-17T18:19:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001289342 2024-08-17T18:19:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.541590+0000", 2024-08-17T18:19:56.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015838000000000001 2024-08-17T18:19:56.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 4,6,7", 2024-08-17T18:19:56.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.541736+0000", 2024-08-17T18:19:56.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000146234 2024-08-17T18:19:56.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.548686+0000", 2024-08-17T18:19:56.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0069502219999999998 2024-08-17T18:19:56.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.292 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.292 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.292 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.548728+0000", 2024-08-17T18:19:56.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2098000000000002e-05 2024-08-17T18:19:56.292 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.292 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.292 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.292 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.548766+0000", 2024-08-17T18:19:56.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7951999999999998e-05 2024-08-17T18:19:56.292 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.292 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.292 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.292 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.649909+0000", 2024-08-17T18:19:56.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.10114324600000001 2024-08-17T18:19:56.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.293 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.293 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.650034+0000", 2024-08-17T18:19:56.293 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012479599999999999 2024-08-17T18:19:56.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.293 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.293 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.650342+0000", 2024-08-17T18:19:56.293 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00030760999999999999 2024-08-17T18:19:56.293 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.293 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.293 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.294 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:169864 50.a 50:55c44339:::benchmark_data_smithi130_398002_object24798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1453)", 2024-08-17T18:19:56.294 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:23.540259+0000", 2024-08-17T18:19:56.294 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 32.631658889999997, 2024-08-17T18:19:56.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.11031039300000001, 2024-08-17T18:19:56.294 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.294 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.294 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.294 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.294 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.294 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.294 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 169864 2024-08-17T18:19:56.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.295 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.540259+0000", 2024-08-17T18:19:56.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.540259+0000", 2024-08-17T18:19:56.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.540260+0000", 2024-08-17T18:19:56.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0270000000000001e-06 2024-08-17T18:19:56.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.540264+0000", 2024-08-17T18:19:56.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6310000000000002e-06 2024-08-17T18:19:56.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.540328+0000", 2024-08-17T18:19:56.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3715999999999998e-05 2024-08-17T18:19:56.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.297 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.540337+0000", 2024-08-17T18:19:56.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8980000000000002e-06 2024-08-17T18:19:56.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.297 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.297 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.548818+0000", 2024-08-17T18:19:56.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0084807179999999999 2024-08-17T18:19:56.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.297 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.297 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.297 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.548928+0000", 2024-08-17T18:19:56.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000110728 2024-08-17T18:19:56.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 4,6,7", 2024-08-17T18:19:56.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.549053+0000", 2024-08-17T18:19:56.298 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000124714 2024-08-17T18:19:56.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.552426+0000", 2024-08-17T18:19:56.298 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0033730409999999998 2024-08-17T18:19:56.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.552758+0000", 2024-08-17T18:19:56.299 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00033221900000000003 2024-08-17T18:19:56.299 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.299 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.299 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.299 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.553506+0000", 2024-08-17T18:19:56.299 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00074789400000000003 2024-08-17T18:19:56.299 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.299 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.299 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.299 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.650376+0000", 2024-08-17T18:19:56.299 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.096869944999999999 2024-08-17T18:19:56.299 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.300 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.300 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.300 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.650433+0000", 2024-08-17T18:19:56.300 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6852999999999999e-05 2024-08-17T18:19:56.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.300 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.300 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.300 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.650569+0000", 2024-08-17T18:19:56.300 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013600900000000001 2024-08-17T18:19:56.300 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.300 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.300 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.300 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.301 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:169877 50.d 50:b13192a7:::benchmark_data_smithi130_398002_object24811:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1453)", 2024-08-17T18:19:56.301 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:23.572791+0000", 2024-08-17T18:19:56.301 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 32.599125978000004, 2024-08-17T18:19:56.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.111565186, 2024-08-17T18:19:56.301 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.301 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.301 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.301 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.301 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.301 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.301 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 169877 2024-08-17T18:19:56.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.301 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.572791+0000", 2024-08-17T18:19:56.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.572791+0000", 2024-08-17T18:19:56.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.572792+0000", 2024-08-17T18:19:56.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3900000000000003e-07 2024-08-17T18:19:56.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.572802+0000", 2024-08-17T18:19:56.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8749999999999995e-06 2024-08-17T18:19:56.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.572811+0000", 2024-08-17T18:19:56.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0080000000000008e-06 2024-08-17T18:19:56.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.572818+0000", 2024-08-17T18:19:56.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6189999999999996e-06 2024-08-17T18:19:56.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.572874+0000", 2024-08-17T18:19:56.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6607999999999997e-05 2024-08-17T18:19:56.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.573024+0000", 2024-08-17T18:19:56.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014900800000000001 2024-08-17T18:19:56.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 3,4", 2024-08-17T18:19:56.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.573182+0000", 2024-08-17T18:19:56.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000158727 2024-08-17T18:19:56.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.576705+0000", 2024-08-17T18:19:56.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0035225479999999999 2024-08-17T18:19:56.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.305 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.305 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.305 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.603553+0000", 2024-08-17T18:19:56.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.026847869999999999 2024-08-17T18:19:56.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.684165+0000", 2024-08-17T18:19:56.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.080612775999999997 2024-08-17T18:19:56.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.684239+0000", 2024-08-17T18:19:56.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3798000000000001e-05 2024-08-17T18:19:56.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.307 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:23.684357+0000", 2024-08-17T18:19:56.307 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011741 2024-08-17T18:19:56.307 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.307 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.307 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.307 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.307 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.307 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:171244 50.10 50:0fb5351d:::benchmark_data_smithi130_398002_object26178:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1453)", 2024-08-17T18:19:56.307 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:25.018786+0000", 2024-08-17T18:19:56.307 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 31.153131276, 2024-08-17T18:19:56.307 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1174052809999999, 2024-08-17T18:19:56.307 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.307 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.307 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.308 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.308 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.308 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.308 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 171244 2024-08-17T18:19:56.308 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.308 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.308 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.308 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.308 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.018786+0000", 2024-08-17T18:19:56.308 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.308 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.308 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.018786+0000", 2024-08-17T18:19:56.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.018787+0000", 2024-08-17T18:19:56.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.879999999999999e-07 2024-08-17T18:19:56.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.018792+0000", 2024-08-17T18:19:56.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.42e-06 2024-08-17T18:19:56.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.310 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.310 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.018952+0000", 2024-08-17T18:19:56.310 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00016043399999999999 2024-08-17T18:19:56.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.310 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.310 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.018959+0000", 2024-08-17T18:19:56.310 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9550000000000003e-06 2024-08-17T18:19:56.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.310 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.042338+0000", 2024-08-17T18:19:56.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023378567999999999 2024-08-17T18:19:56.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.311 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.042345+0000", 2024-08-17T18:19:56.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.131e-06 2024-08-17T18:19:56.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.311 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.131332+0000", 2024-08-17T18:19:56.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.08898775 2024-08-17T18:19:56.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.312 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.131473+0000", 2024-08-17T18:19:56.312 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014050299999999999 2024-08-17T18:19:56.312 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.312 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,4,6", 2024-08-17T18:19:56.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.131949+0000", 2024-08-17T18:19:56.312 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00047580700000000002 2024-08-17T18:19:56.312 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.312 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.133987+0000", 2024-08-17T18:19:56.313 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002037929 2024-08-17T18:19:56.313 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.313 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.313 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.313 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.134028+0000", 2024-08-17T18:19:56.313 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1641000000000003e-05 2024-08-17T18:19:56.313 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.313 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.313 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.313 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.134039+0000", 2024-08-17T18:19:56.313 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0638999999999999e-05 2024-08-17T18:19:56.313 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.313 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.314 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.314 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.136058+0000", 2024-08-17T18:19:56.314 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0020195019999999998 2024-08-17T18:19:56.314 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.314 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.314 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.314 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.136096+0000", 2024-08-17T18:19:56.314 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.7904000000000002e-05 2024-08-17T18:19:56.314 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.314 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.314 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.314 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.136191+0000", 2024-08-17T18:19:56.314 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5110000000000002e-05 2024-08-17T18:19:56.315 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.315 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.315 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.315 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:171260 50.10 50:0cc4001f:::benchmark_data_smithi130_398002_object26194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1454)", 2024-08-17T18:19:56.315 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:25.058072+0000", 2024-08-17T18:19:56.315 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 31.113845108, 2024-08-17T18:19:56.315 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0785421500000001, 2024-08-17T18:19:56.315 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:19:56.315 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:19:56.315 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:19:56.315 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:19:56.315 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:19:56.316 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:19:56.316 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 171260 2024-08-17T18:19:56.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.316 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:19:56.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:19:56.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.058072+0000", 2024-08-17T18:19:56.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:19:56.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.058072+0000", 2024-08-17T18:19:56.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:19:56.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:19:56.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.058074+0000", 2024-08-17T18:19:56.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.708e-06 2024-08-17T18:19:56.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:19:56.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.058079+0000", 2024-08-17T18:19:56.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7149999999999997e-06 2024-08-17T18:19:56.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:19:56.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.058094+0000", 2024-08-17T18:19:56.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4851000000000001e-05 2024-08-17T18:19:56.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:19:56.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.058101+0000", 2024-08-17T18:19:56.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3660000000000003e-06 2024-08-17T18:19:56.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.058155+0000", 2024-08-17T18:19:56.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4030999999999998e-05 2024-08-17T18:19:56.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:19:56.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.058163+0000", 2024-08-17T18:19:56.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4600000000000003e-06 2024-08-17T18:19:56.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:19:56.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.134058+0000", 2024-08-17T18:19:56.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0758949 2024-08-17T18:19:56.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:19:56.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.134146+0000", 2024-08-17T18:19:56.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.8061999999999998e-05 2024-08-17T18:19:56.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.320 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,4,6", 2024-08-17T18:19:56.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.134471+0000", 2024-08-17T18:19:56.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00032436599999999998 2024-08-17T18:19:56.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.320 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:19:56.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.136208+0000", 2024-08-17T18:19:56.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0017375139999999999 2024-08-17T18:19:56.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.320 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.321 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.136256+0000", 2024-08-17T18:19:56.321 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.7911999999999998e-05 2024-08-17T18:19:56.321 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.321 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.321 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.321 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.136352+0000", 2024-08-17T18:19:56.321 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5678000000000001e-05 2024-08-17T18:19:56.321 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.321 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.321 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:19:56.321 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.136426+0000", 2024-08-17T18:19:56.321 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4307000000000005e-05 2024-08-17T18:19:56.321 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.321 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.322 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:19:56.322 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.136490+0000", 2024-08-17T18:19:56.322 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3612e-05 2024-08-17T18:19:56.322 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:19:56.322 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:19:56.322 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:19:56.322 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.136614+0000", 2024-08-17T18:19:56.322 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012466800000000001 2024-08-17T18:19:56.322 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.322 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.322 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.322 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:19:56.322 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:19:56.323 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:19:56.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:56.323 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:56.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:56.424 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:56.459 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:56.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:56.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:56.561 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:56.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:56.598 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:56.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:56.699 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:56.728 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:56.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:56.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:56.830 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:56.865 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:56.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:56.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:56.967 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:57.003 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:57.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:57.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:57.024 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 7, 1, 6, 4, 5] out_osds: [] dead_osds: [] live_osds: [1, 4, 5, 6, 7, 0, 3, 2] 2024-08-17T18:19:57.024 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:19:57.025 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2024-08-17T18:19:57.025 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T18:19:57.025 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T18:19:57.025 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 config set filestore_inject_stall 3 2024-08-17T18:19:57.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:56 smithi130 ceph-mon[105521]: pgmap v2819: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 activating, 1 activating+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 77 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 51380/289257 objects degraded (17.763%); 53329/289257 objects misplaced (18.437%); 18 MiB/s, 288 objects/s recovering 2024-08-17T18:19:57.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:56 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:57.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:56 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:57.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 51380/289257 objects degraded (17.763%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:57.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:56 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:19:57.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:56 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:19:57.051 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:56 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:57.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:56 smithi130 ceph-mon[110068]: pgmap v2819: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 activating, 1 activating+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 77 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 51380/289257 objects degraded (17.763%); 53329/289257 objects misplaced (18.437%); 18 MiB/s, 288 objects/s recovering 2024-08-17T18:19:57.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:56 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:57.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:56 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:57.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 51380/289257 objects degraded (17.763%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:57.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:56 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:19:57.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:56 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:19:57.052 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:56 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:56 smithi179 ceph-mon[93519]: pgmap v2819: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 activating+degraded+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 activating, 1 activating+remapped, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 77 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 51380/289257 objects degraded (17.763%); 53329/289257 objects misplaced (18.437%); 18 MiB/s, 288 objects/s recovering 2024-08-17T18:19:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:56 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:56 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 51380/289257 objects degraded (17.763%), 17 pgs degraded (PG_DEGRADED) 2024-08-17T18:19:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:56 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:19:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:56 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:19:57.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:56 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:19:57.104 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:57.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:57.171 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:57.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:57.272 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:19:57.329 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:57.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:57.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:19:57.431 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:57.469 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:57.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:57.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:57.571 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:57.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:57.611 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:57.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:57.712 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:57.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:57.756 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:57.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:57.857 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:57.895 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:57.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:57.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:57.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:19:57.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:19:57.996 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:58.030 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:58.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:58.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:58.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:19:58.132 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:58.169 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:58.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:58.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:58.270 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:58.306 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:58.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:58.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:58.408 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:58.440 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:58.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:58.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:58.474 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T18:19:58.542 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:58.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:58.582 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:58.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:58.683 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:19:58.721 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:58.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:58.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:19:58.811 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:19:58 smithi130 ceph-mon[105521]: pgmap v2820: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 51113/286791 objects degraded (17.822%); 52216/286791 objects misplaced (18.207%); 11 MiB/s, 181 objects/s recovering 2024-08-17T18:19:58.823 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:58.863 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:58.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:58.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:58.965 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:59.039 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:59.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:59.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:59.057 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:19:59.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:19:58 smithi179 ceph-mon[93519]: pgmap v2820: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 51113/286791 objects degraded (17.822%); 52216/286791 objects misplaced (18.207%); 11 MiB/s, 181 objects/s recovering 2024-08-17T18:19:59.097 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:19:59.141 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:19:59.161 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:19:58 smithi130 ceph-mon[110068]: pgmap v2820: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 507 op/s; 51113/286791 objects degraded (17.822%); 52216/286791 objects misplaced (18.207%); 11 MiB/s, 181 objects/s recovering 2024-08-17T18:19:59.178 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:59.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:59.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:19:59.279 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:19:59.334 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:59.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:59.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:19:59.436 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:19:59.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:59.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:59.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:19:59.564 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:19:59.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:19:59.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:59.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:19:59.704 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:19:59.743 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:59.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:59.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:19:59.845 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:19:59.890 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:19:59.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:19:59.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:19:59.984 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:19:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:19:59] "GET /metrics HTTP/1.1" 200 36816 "" "Prometheus/2.51.0" 2024-08-17T18:19:59.991 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:00.020 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:00.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:00.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:00.122 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:00.161 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:00.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:00.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:00.262 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:00.308 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:00.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:00.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:00.361 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:20:00.410 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:00.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:00.444 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:00.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:00.545 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:00.585 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:00.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:00.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:00.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:20:00.215Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:20:00.687 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:00.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:00.725 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:00.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:00.826 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:00.837 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:20:00.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:00.871 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:00.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:00.972 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:01.005 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:01.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:01.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pgmap v2821: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 48596/276087 objects degraded (17.602%); 50244/276087 objects misplaced (18.199%); 19 MiB/s, 308 objects/s recovering 2024-08-17T18:20:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 48596/276087 objects degraded (17.602%), 15 pgs degraded 2024-08-17T18:20:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-08-17T18:20:01.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: [WRN] PG_DEGRADED: Degraded data redundancy: 48596/276087 objects degraded (17.602%), 15 pgs degraded 2024-08-17T18:20:01.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.3 is active+undersized+degraded+remapped+backfill_wait, acting [3,6] 2024-08-17T18:20:01.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.4 is active+undersized+degraded+remapped+backfill_wait, acting [3,6] 2024-08-17T18:20:01.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.5 is active+undersized+degraded+remapped+backfill_wait, acting [1,0] 2024-08-17T18:20:01.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.6 is active+undersized+degraded+remapped+backfill_wait, acting [7,6] 2024-08-17T18:20:01.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.7 is active+undersized+degraded+remapped+backfill_wait, acting [7,0] 2024-08-17T18:20:01.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.8 is active+recovering+undersized+degraded+remapped, acting [6,0] 2024-08-17T18:20:01.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.d is active+undersized+degraded+remapped+backfilling, acting [0,3] 2024-08-17T18:20:01.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.e is active+undersized+degraded+remapped+backfill_wait, acting [6,0] 2024-08-17T18:20:01.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.10 is active+undersized+degraded+remapped+backfill_wait, acting [0,6] 2024-08-17T18:20:01.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.13 is active+undersized+degraded+remapped+backfill_wait, acting [3,6] 2024-08-17T18:20:01.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.14 is active+undersized+degraded+remapped+backfill_wait, acting [3,6] 2024-08-17T18:20:01.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.15 is active+undersized+degraded+remapped+backfill_wait, acting [1,0] 2024-08-17T18:20:01.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.16 is active+undersized+degraded+remapped+backfill_wait, acting [7,6] 2024-08-17T18:20:01.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.17 is active+undersized+degraded+remapped+backfill_wait, acting [7,0] 2024-08-17T18:20:01.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: pg 50.18 is active+recovery_wait+undersized+degraded+remapped, acting [6,0] 2024-08-17T18:20:01.095 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:00 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3908691303' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:20:01.107 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:01.154 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:01.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:01.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pgmap v2821: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 48596/276087 objects degraded (17.602%); 50244/276087 objects misplaced (18.199%); 19 MiB/s, 308 objects/s recovering 2024-08-17T18:20:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 48596/276087 objects degraded (17.602%), 15 pgs degraded 2024-08-17T18:20:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-08-17T18:20:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: [WRN] PG_DEGRADED: Degraded data redundancy: 48596/276087 objects degraded (17.602%), 15 pgs degraded 2024-08-17T18:20:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.3 is active+undersized+degraded+remapped+backfill_wait, acting [3,6] 2024-08-17T18:20:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.4 is active+undersized+degraded+remapped+backfill_wait, acting [3,6] 2024-08-17T18:20:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.5 is active+undersized+degraded+remapped+backfill_wait, acting [1,0] 2024-08-17T18:20:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.6 is active+undersized+degraded+remapped+backfill_wait, acting [7,6] 2024-08-17T18:20:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.7 is active+undersized+degraded+remapped+backfill_wait, acting [7,0] 2024-08-17T18:20:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.8 is active+recovering+undersized+degraded+remapped, acting [6,0] 2024-08-17T18:20:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.d is active+undersized+degraded+remapped+backfilling, acting [0,3] 2024-08-17T18:20:01.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.e is active+undersized+degraded+remapped+backfill_wait, acting [6,0] 2024-08-17T18:20:01.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.10 is active+undersized+degraded+remapped+backfill_wait, acting [0,6] 2024-08-17T18:20:01.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.13 is active+undersized+degraded+remapped+backfill_wait, acting [3,6] 2024-08-17T18:20:01.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.14 is active+undersized+degraded+remapped+backfill_wait, acting [3,6] 2024-08-17T18:20:01.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.15 is active+undersized+degraded+remapped+backfill_wait, acting [1,0] 2024-08-17T18:20:01.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.16 is active+undersized+degraded+remapped+backfill_wait, acting [7,6] 2024-08-17T18:20:01.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.17 is active+undersized+degraded+remapped+backfill_wait, acting [7,0] 2024-08-17T18:20:01.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: pg 50.18 is active+recovery_wait+undersized+degraded+remapped, acting [6,0] 2024-08-17T18:20:01.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3908691303' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:20:01.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pgmap v2821: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 48596/276087 objects degraded (17.602%); 50244/276087 objects misplaced (18.199%); 19 MiB/s, 308 objects/s recovering 2024-08-17T18:20:01.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set; Degraded data redundancy: 48596/276087 objects degraded (17.602%), 15 pgs degraded 2024-08-17T18:20:01.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2024-08-17T18:20:01.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: [WRN] PG_DEGRADED: Degraded data redundancy: 48596/276087 objects degraded (17.602%), 15 pgs degraded 2024-08-17T18:20:01.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.3 is active+undersized+degraded+remapped+backfill_wait, acting [3,6] 2024-08-17T18:20:01.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.4 is active+undersized+degraded+remapped+backfill_wait, acting [3,6] 2024-08-17T18:20:01.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.5 is active+undersized+degraded+remapped+backfill_wait, acting [1,0] 2024-08-17T18:20:01.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.6 is active+undersized+degraded+remapped+backfill_wait, acting [7,6] 2024-08-17T18:20:01.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.7 is active+undersized+degraded+remapped+backfill_wait, acting [7,0] 2024-08-17T18:20:01.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.8 is active+recovering+undersized+degraded+remapped, acting [6,0] 2024-08-17T18:20:01.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.d is active+undersized+degraded+remapped+backfilling, acting [0,3] 2024-08-17T18:20:01.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.e is active+undersized+degraded+remapped+backfill_wait, acting [6,0] 2024-08-17T18:20:01.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.10 is active+undersized+degraded+remapped+backfill_wait, acting [0,6] 2024-08-17T18:20:01.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.13 is active+undersized+degraded+remapped+backfill_wait, acting [3,6] 2024-08-17T18:20:01.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.14 is active+undersized+degraded+remapped+backfill_wait, acting [3,6] 2024-08-17T18:20:01.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.15 is active+undersized+degraded+remapped+backfill_wait, acting [1,0] 2024-08-17T18:20:01.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.16 is active+undersized+degraded+remapped+backfill_wait, acting [7,6] 2024-08-17T18:20:01.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.17 is active+undersized+degraded+remapped+backfill_wait, acting [7,0] 2024-08-17T18:20:01.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: pg 50.18 is active+recovery_wait+undersized+degraded+remapped, acting [6,0] 2024-08-17T18:20:01.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:00 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3908691303' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:20:01.256 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:01.294 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:01.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:01.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:01.395 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:01.428 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:01.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:01.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:01.530 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:01.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:01.568 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:01.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:01.670 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:01.700 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:20:01.707 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:01.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:01.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:01.809 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:01.843 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:01.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:01.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:01.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:01 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:20:01.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:01 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3908691303' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:20:01.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:01 smithi130 ceph-mon[105521]: osdmap e1460: 8 total, 8 up, 8 in 2024-08-17T18:20:01.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 48596/276087 objects degraded (17.602%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:01 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:20:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:01 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3908691303' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:20:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:01 smithi130 ceph-mon[110068]: osdmap e1460: 8 total, 8 up, 8 in 2024-08-17T18:20:01.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 48596/276087 objects degraded (17.602%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:01.946 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:01.979 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:01.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:01.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:02.081 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:01 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:20:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:01 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3908691303' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:20:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:01 smithi179 ceph-mon[93519]: osdmap e1460: 8 total, 8 up, 8 in 2024-08-17T18:20:02.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 48596/276087 objects degraded (17.602%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:02.112 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:02.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:02.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:02.214 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:02.254 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:02.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:02.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:02.358 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:02.390 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:02.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:02.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:02.492 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:02.537 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:02.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:02.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:02.639 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:02.654 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:20:02.654 INFO:teuthology.orchestra.run.smithi130.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T18:20:02.654 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:20:02.676 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:02.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:02.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:02.778 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:02.817 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:02.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:02.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:02.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:02.954 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:02.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:02.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:03.055 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:03.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:02 smithi179 ceph-mon[93519]: pgmap v2823: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 48596/276087 objects degraded (17.602%); 50244/276087 objects misplaced (18.199%); 18 MiB/s, 282 objects/s recovering 2024-08-17T18:20:03.097 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:03.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:03.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:03.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:02 smithi130 ceph-mon[105521]: pgmap v2823: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 48596/276087 objects degraded (17.602%); 50244/276087 objects misplaced (18.199%); 18 MiB/s, 282 objects/s recovering 2024-08-17T18:20:03.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:02 smithi130 ceph-mon[110068]: pgmap v2823: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 48596/276087 objects degraded (17.602%); 50244/276087 objects misplaced (18.199%); 18 MiB/s, 282 objects/s recovering 2024-08-17T18:20:03.198 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:03.241 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:03.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:03.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:03.343 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:03.383 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:03.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:03.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:03.485 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:03.534 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:03.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:03.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:03.637 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:03.649 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:20:03.680 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:03.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:03.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:03.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:03.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:03.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:03.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:03.924 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:03.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:03.970 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:03.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:04.071 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:04.114 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:04.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:04.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:04.216 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:04.253 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:04.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:04.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:04.355 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:04.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:04.399 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:04.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:04.500 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:04.540 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:04.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:04.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:04.642 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:04.678 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:04.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:04.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:04.780 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:04.820 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:04.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:04.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:04.922 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:04.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:04.961 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:04.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:05.062 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:05.098 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:05.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:05.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:05.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:04 smithi130 ceph-mon[105521]: pgmap v2824: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 45889/262872 objects degraded (17.457%); 47273/262872 objects misplaced (17.983%); 17 MiB/s, 269 objects/s recovering 2024-08-17T18:20:05.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:04 smithi130 ceph-mon[110068]: pgmap v2824: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 45889/262872 objects degraded (17.457%); 47273/262872 objects misplaced (17.983%); 17 MiB/s, 269 objects/s recovering 2024-08-17T18:20:05.199 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:05.234 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:05.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:05.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:05.335 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:05.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:04 smithi179 ceph-mon[93519]: pgmap v2824: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 45889/262872 objects degraded (17.457%); 47273/262872 objects misplaced (17.983%); 17 MiB/s, 269 objects/s recovering 2024-08-17T18:20:05.370 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:05.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:05.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:05.472 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:05.509 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:05.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:05.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:05.611 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:05.651 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:05.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:05.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:05.753 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:05.793 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:05.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:05.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:05.895 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:05.937 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:05.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:05.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:06.039 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:06.080 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:06.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:06.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:06.182 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:06.224 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:06.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:06.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:06.325 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:06.368 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:06.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:06.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:06.470 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:06.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:06.519 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:06.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:06.620 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:06.670 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:06.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:06.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:06.772 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:06.812 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:06.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:06.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:06.914 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:06.951 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:06.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:06.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:06.986 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:06 smithi130 ceph-mon[105521]: pgmap v2825: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 45889/262872 objects degraded (17.457%); 47273/262872 objects misplaced (17.983%); 17 MiB/s, 269 objects/s recovering 2024-08-17T18:20:06.986 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 45889/262872 objects degraded (17.457%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:06.986 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:06 smithi130 ceph-mon[110068]: pgmap v2825: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 45889/262872 objects degraded (17.457%); 47273/262872 objects misplaced (17.983%); 17 MiB/s, 269 objects/s recovering 2024-08-17T18:20:06.986 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 45889/262872 objects degraded (17.457%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:07.054 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:07.094 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:07.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:07.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:07.195 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:07.222 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:07.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:07.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:07.299 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:20:07.300 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:20:07.311 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:20:07.311 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:20:07.323 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:07.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:06 smithi179 ceph-mon[93519]: pgmap v2825: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 45889/262872 objects degraded (17.457%); 47273/262872 objects misplaced (17.983%); 17 MiB/s, 269 objects/s recovering 2024-08-17T18:20:07.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 45889/262872 objects degraded (17.457%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:07.351 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:20:07.351 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:20:07.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:07.377 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:07.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:07.408 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:20:07.409 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:20:07.417 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:20:07.417 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:20:07.418 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.418 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:206567 50.14 50:2d6a1e3c:::benchmark_data_smithi130_398002_object61501:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1460)", 2024-08-17T18:20:07.418 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:20:07.346431+0000", 2024-08-17T18:20:07.418 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.006592746, 2024-08-17T18:20:07.418 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0066219870000000002, 2024-08-17T18:20:07.418 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:07.418 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:07.418 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "waiting for sub ops", 2024-08-17T18:20:07.418 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:07.418 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:20:07.418 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:20:07.418 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 206567 2024-08-17T18:20:07.418 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.419 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:07.419 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.419 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:07.419 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.346431+0000", 2024-08-17T18:20:07.419 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:07.419 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.419 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.419 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:07.419 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.346431+0000", 2024-08-17T18:20:07.419 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:07.419 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.419 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.419 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:07.419 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.346433+0000", 2024-08-17T18:20:07.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0150000000000002e-06 2024-08-17T18:20:07.420 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.420 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.420 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:07.420 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.346440+0000", 2024-08-17T18:20:07.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5479999999999998e-06 2024-08-17T18:20:07.420 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.420 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.420 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:07.420 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.346458+0000", 2024-08-17T18:20:07.420 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8722000000000001e-05 2024-08-17T18:20:07.420 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.420 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.420 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:07.421 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.346468+0000", 2024-08-17T18:20:07.421 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3500000000000003e-06 2024-08-17T18:20:07.421 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.421 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.421 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:07.421 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.346563+0000", 2024-08-17T18:20:07.421 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5402999999999994e-05 2024-08-17T18:20:07.421 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.421 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.421 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:07.421 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.346683+0000", 2024-08-17T18:20:07.421 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012007599999999999 2024-08-17T18:20:07.421 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.421 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.422 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,5,6", 2024-08-17T18:20:07.422 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.347258+0000", 2024-08-17T18:20:07.422 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00057424500000000003 2024-08-17T18:20:07.422 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.422 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.422 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:07.422 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.352368+0000", 2024-08-17T18:20:07.422 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0051107790000000002 2024-08-17T18:20:07.422 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.422 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.422 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:07.422 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.352408+0000", 2024-08-17T18:20:07.422 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9242999999999997e-05 2024-08-17T18:20:07.423 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:07.423 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:07.423 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:07.423 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.423 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.423 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:206566 50.1 e1460/1443 50:80115a38:::benchmark_data_smithi130_398002_object61500:head v 1460'9913, mlcod=1460'9911)", 2024-08-17T18:20:07.423 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:20:07.348098+0000", 2024-08-17T18:20:07.423 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.0049261009999999996, 2024-08-17T18:20:07.423 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0050333519999999996, 2024-08-17T18:20:07.423 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:07.423 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:07.423 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:20:07.423 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:07.424 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.424 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:07.424 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.348098+0000", 2024-08-17T18:20:07.424 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:07.424 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.424 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.424 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:07.424 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.348098+0000", 2024-08-17T18:20:07.424 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:07.424 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.424 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.424 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:07.424 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.348098+0000", 2024-08-17T18:20:07.424 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6899999999999998e-07 2024-08-17T18:20:07.425 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.425 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.425 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:07.425 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.348104+0000", 2024-08-17T18:20:07.425 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3430000000000003e-06 2024-08-17T18:20:07.425 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.425 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.425 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:07.425 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.348111+0000", 2024-08-17T18:20:07.425 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.041e-06 2024-08-17T18:20:07.425 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.425 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.425 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:07.426 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.348116+0000", 2024-08-17T18:20:07.426 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0810000000000001e-06 2024-08-17T18:20:07.426 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.426 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.426 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:07.426 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.348212+0000", 2024-08-17T18:20:07.426 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5942999999999999e-05 2024-08-17T18:20:07.426 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.426 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.426 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:07.426 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.348232+0000", 2024-08-17T18:20:07.426 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0309000000000002e-05 2024-08-17T18:20:07.426 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:07.427 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:07.427 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:07.427 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.427 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.427 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:206568 50.d e1460/1458 50:b8c501ba:::benchmark_data_smithi130_398002_object61502:head v 1460'12791, mlcod=1460'12789)", 2024-08-17T18:20:07.427 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:20:07.352091+0000", 2024-08-17T18:20:07.427 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 0.00093306699999999999, 2024-08-17T18:20:07.427 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0010892790000000001, 2024-08-17T18:20:07.427 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:07.427 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:07.427 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "started", 2024-08-17T18:20:07.427 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:07.427 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.428 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:07.428 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.352091+0000", 2024-08-17T18:20:07.428 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:07.428 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.428 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.428 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:07.428 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.352091+0000", 2024-08-17T18:20:07.428 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:07.428 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.428 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.428 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:07.428 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.352092+0000", 2024-08-17T18:20:07.428 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.8100000000000001e-07 2024-08-17T18:20:07.428 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.429 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.429 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:07.429 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.352099+0000", 2024-08-17T18:20:07.429 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2819999999999999e-06 2024-08-17T18:20:07.429 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.429 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.429 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:07.429 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.352113+0000", 2024-08-17T18:20:07.429 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3878e-05 2024-08-17T18:20:07.429 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.429 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.429 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:07.429 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.352120+0000", 2024-08-17T18:20:07.429 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4959999999999999e-06 2024-08-17T18:20:07.430 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.430 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.430 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:07.430 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.352605+0000", 2024-08-17T18:20:07.430 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00048415600000000002 2024-08-17T18:20:07.430 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.430 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.430 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:07.430 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.352658+0000", 2024-08-17T18:20:07.430 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3542000000000003e-05 2024-08-17T18:20:07.430 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:07.430 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:07.431 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:07.431 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.431 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.431 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop_reply(client.47132.0:206567 50.14 e1460/1458)", 2024-08-17T18:20:07.431 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:20:07.353166+0000", 2024-08-17T18:20:07.431 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 4294967295.9998574, 2024-08-17T18:20:07.431 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9570000000000001e-05, 2024-08-17T18:20:07.431 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:07.431 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:07.431 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "queued for pg", 2024-08-17T18:20:07.431 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:07.431 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.431 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:07.431 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.353166+0000", 2024-08-17T18:20:07.432 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:07.432 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.432 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.432 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:07.432 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.353166+0000", 2024-08-17T18:20:07.432 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:07.432 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.432 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.432 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:07.432 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.353167+0000", 2024-08-17T18:20:07.432 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5099999999999998e-07 2024-08-17T18:20:07.432 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.432 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.432 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:07.433 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.353171+0000", 2024-08-17T18:20:07.433 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.472e-06 2024-08-17T18:20:07.433 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.433 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.433 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:07.433 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.353181+0000", 2024-08-17T18:20:07.433 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0416e-05 2024-08-17T18:20:07.433 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.433 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.433 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:07.433 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.353188+0000", 2024-08-17T18:20:07.433 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6499999999999999e-06 2024-08-17T18:20:07.433 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:07.433 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:07.434 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:07.434 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.434 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.434 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_repop(client.47132.0:206572 50.f e1460/1447)", 2024-08-17T18:20:07.434 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:20:07.353219+0000", 2024-08-17T18:20:07.434 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 4294967295.9998045, 2024-08-17T18:20:07.434 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8786999999999999e-05, 2024-08-17T18:20:07.434 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:07.434 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:07.434 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "queued for pg", 2024-08-17T18:20:07.434 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:07.434 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.434 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:07.435 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.353219+0000", 2024-08-17T18:20:07.435 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:07.435 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.435 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.435 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:07.435 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.353219+0000", 2024-08-17T18:20:07.435 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:07.435 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.435 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.435 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:07.435 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.353220+0000", 2024-08-17T18:20:07.435 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.2199999999999999e-07 2024-08-17T18:20:07.435 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.436 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.436 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:07.436 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.353224+0000", 2024-08-17T18:20:07.436 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1280000000000001e-06 2024-08-17T18:20:07.436 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.436 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.436 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:07.436 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.353231+0000", 2024-08-17T18:20:07.436 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2969999999999998e-06 2024-08-17T18:20:07.436 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:07.436 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:07.436 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:07.436 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:20:07.353236+0000", 2024-08-17T18:20:07.436 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2310000000000004e-06 2024-08-17T18:20:07.437 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:07.437 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:07.437 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:07.437 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:07.437 INFO:teuthology.orchestra.run.smithi130.stdout: ], 2024-08-17T18:20:07.437 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 5 2024-08-17T18:20:07.437 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:20:07.451 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:20:07.451 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:20:07.478 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:07.488 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:20:07.488 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:20:07.509 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:07.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:07.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:07.526 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:20:07.526 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:20:07.554 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:20:07.554 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:20:07.611 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:07.648 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:07.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:07.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:07.750 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:07.792 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:07.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:07.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:07.894 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:07.935 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:07.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:07.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:08.036 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:08.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:08.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:08.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:08.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:07 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2685865758' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:20:08.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:07 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2685865758' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:20:08.168 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:08.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:08.204 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:08.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:08.305 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:08.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:08.339 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:08.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:08.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:07 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2685865758' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:20:08.440 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:08.475 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:08.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:08.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:08.577 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:08.710 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:08.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:08.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:08.812 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:08.853 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:08.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:08.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:08.856 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 7, 1, 6, 4, 5] out_osds: [] dead_osds: [] live_osds: [1, 4, 5, 6, 7, 0, 3, 2] 2024-08-17T18:20:08.856 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:20:08.856 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 2, 3, 7, 1, 6, 4, 5] 2024-08-17T18:20:08.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd out 0 2024-08-17T18:20:08.957 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:08.978 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:08.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:08.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:09.080 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:09.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:09.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:09.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:09.152 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:08 smithi130 ceph-mon[110068]: pgmap v2826: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 45590/260307 objects degraded (17.514%); 46161/260307 objects misplaced (17.733%); 17 MiB/s, 269 objects/s recovering 2024-08-17T18:20:09.152 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:08 smithi130 ceph-mon[105521]: pgmap v2826: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 45590/260307 objects degraded (17.514%); 46161/260307 objects misplaced (17.733%); 17 MiB/s, 269 objects/s recovering 2024-08-17T18:20:09.216 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:09.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:09.264 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:09.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:09.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:08 smithi179 ceph-mon[93519]: pgmap v2826: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 45590/260307 objects degraded (17.514%); 46161/260307 objects misplaced (17.733%); 17 MiB/s, 269 objects/s recovering 2024-08-17T18:20:09.364 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:09.404 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:09.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:09.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:09.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:09.544 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:09.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:09.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:09.646 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:09.683 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:09.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:09.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:09.785 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:09.818 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:09.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:09.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:09.920 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:09.952 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:09.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:09.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:10.053 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:10.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:10.086 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:10.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:20:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:20:09] "GET /metrics HTTP/1.1" 200 36816 "" "Prometheus/2.51.0" 2024-08-17T18:20:10.187 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:10.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:10.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:10.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:10.321 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:10.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:10.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:10.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:10.455 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:10.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:10.493 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:10.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:20:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:20:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:20:10.594 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:10.639 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:10.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:10.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:10.741 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:10.777 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:10.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:10.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:10.879 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:10.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:10.914 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:10.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:11.015 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:11.058 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:11.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:11.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:11.135 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:10 smithi130 ceph-mon[105521]: pgmap v2827: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 43183/249294 objects degraded (17.322%); 44258/249294 objects misplaced (17.753%); 16 MiB/s, 255 objects/s recovering 2024-08-17T18:20:11.135 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:10 smithi130 ceph-mon[110068]: pgmap v2827: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 43183/249294 objects degraded (17.322%); 44258/249294 objects misplaced (17.753%); 16 MiB/s, 255 objects/s recovering 2024-08-17T18:20:11.160 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:11.191 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:11.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:11.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:11.293 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:11.335 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:11.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:11.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:11.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:10 smithi179 ceph-mon[93519]: pgmap v2827: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 892 op/s; 43183/249294 objects degraded (17.322%); 44258/249294 objects misplaced (17.753%); 16 MiB/s, 255 objects/s recovering 2024-08-17T18:20:11.437 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:11.478 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:11.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:11.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:11.580 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:11.612 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:11.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:11.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:11.714 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:11.748 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:11.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:11.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:11.850 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:11.888 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:11.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:11.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:11.989 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:12.022 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:12.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:12.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:12.125 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:12.168 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 43183/249294 objects degraded (17.322%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:12.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 43183/249294 objects degraded (17.322%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:12.169 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:12.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:12.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:12.271 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:12.294 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:12.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:12.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:12.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 43183/249294 objects degraded (17.322%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:12.396 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:12.425 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:12.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:12.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:12.526 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:12.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:12.572 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:12.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:12.673 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:12.709 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:12.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:12.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:12.811 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:12.851 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:12.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:12.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:12.953 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:12.994 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:12.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:12.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:13.096 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:13.134 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:13.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:13.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:13.230 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:20:13.236 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:13.263 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:13.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:13.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:13.365 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:13.393 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:13.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:13.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:13.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:13 smithi130 ceph-mon[105521]: pgmap v2828: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 43183/249294 objects degraded (17.322%); 44258/249294 objects misplaced (17.753%); 15 MiB/s, 236 objects/s recovering 2024-08-17T18:20:13.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:13 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:20:13.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:20:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:13 smithi130 ceph-mon[110068]: pgmap v2828: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 43183/249294 objects degraded (17.322%); 44258/249294 objects misplaced (17.753%); 15 MiB/s, 236 objects/s recovering 2024-08-17T18:20:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:13 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:20:13.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:20:13.495 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:13.533 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:13.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:13.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:13.635 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:13.679 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:13.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:13.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:13.781 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:13.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:13.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:13.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:13 smithi179 ceph-mon[93519]: pgmap v2828: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 824 op/s; 43183/249294 objects degraded (17.322%); 44258/249294 objects misplaced (17.753%); 15 MiB/s, 236 objects/s recovering 2024-08-17T18:20:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:13 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:20:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:20:13.923 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:13.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:13.964 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:13.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:14.065 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:14.088 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:14.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:14.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:14.189 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:14.221 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:14.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:14.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:14.322 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:14.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:14.355 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:14.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:14.456 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:14.495 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:14.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:14.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:14.597 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:14.635 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:14.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:14.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:14.738 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:14.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:14.787 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:14.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:14.888 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:14.917 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:14.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:14.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:15.019 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:15.049 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:15.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:15.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:15.151 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:15.186 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:15.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:15.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:15.200 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:20:15.288 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:15.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:15.342 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:15.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:15.443 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:15.470 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:15.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:15.572 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:15.603 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:15.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:15.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:15.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:15 smithi130 ceph-mon[105521]: pgmap v2829: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 40596/236382 objects degraded (17.174%); 41384/236382 objects misplaced (17.507%); 20 MiB/s, 320 objects/s recovering 2024-08-17T18:20:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:15 smithi130 ceph-mon[110068]: pgmap v2829: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 40596/236382 objects degraded (17.174%); 41384/236382 objects misplaced (17.507%); 20 MiB/s, 320 objects/s recovering 2024-08-17T18:20:15.705 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:15.737 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:15.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:15.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:15.839 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:15.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:15 smithi179 ceph-mon[93519]: pgmap v2829: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 40596/236382 objects degraded (17.174%); 41384/236382 objects misplaced (17.507%); 20 MiB/s, 320 objects/s recovering 2024-08-17T18:20:15.879 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:15.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:15.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:15.980 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:16.023 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:16.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:16.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:16.125 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:16.155 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:16.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:16.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:16.257 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:16.281 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:16.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:16.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:16.384 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:16.432 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:16.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:16.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:16.534 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:16.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:16.574 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:16.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:16.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 40596/236382 objects degraded (17.174%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:16.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 40596/236382 objects degraded (17.174%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:16.675 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:16.705 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:16.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:16.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:16.808 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:16.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 40596/236382 objects degraded (17.174%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:16.844 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:16.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:16.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:16.946 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:16.979 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:16.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:16.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:17.081 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:17.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:17.122 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:17.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:17.223 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:17.263 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:17.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:17.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:17.365 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:17.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:17.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:17.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:17.506 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:17.544 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:17.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:17.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:17.645 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:17.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:17 smithi130 ceph-mon[105521]: pgmap v2830: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 40596/236382 objects degraded (17.174%); 41384/236382 objects misplaced (17.507%); 13 MiB/s, 211 objects/s recovering 2024-08-17T18:20:17.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:17 smithi130 ceph-mon[110068]: pgmap v2830: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 40596/236382 objects degraded (17.174%); 41384/236382 objects misplaced (17.507%); 13 MiB/s, 211 objects/s recovering 2024-08-17T18:20:17.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:17.684 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:17.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:17.748 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T18:20:17.785 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:17.811 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:17.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:17.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:17.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:17 smithi179 ceph-mon[93519]: pgmap v2830: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 40596/236382 objects degraded (17.174%); 41384/236382 objects misplaced (17.507%); 13 MiB/s, 211 objects/s recovering 2024-08-17T18:20:17.913 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:17.946 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:17.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:17.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:18.032 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:20:18.048 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:18.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:18.078 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:18.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:18.179 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:18.209 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:18.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:18.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:18.311 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:18.348 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:18.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:18.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:18.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:18.487 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:18.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:18.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:18.589 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:18.621 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:18.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:18.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:18.723 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:18.751 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:18.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:18.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:18.853 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:18.885 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:18.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:18.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:18.987 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:19.035 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:19.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:19.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:19.137 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:19.173 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:19.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:19.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:19.275 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:19.308 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:19.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:19.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:19.410 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:19.449 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:19.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:19.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:19.551 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:19.576 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:19.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:19.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:19.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:19 smithi130 ceph-mon[105521]: pgmap v2831: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 40337/234036 objects degraded (17.235%); 40338/234036 objects misplaced (17.236%); 13 MiB/s, 211 objects/s recovering 2024-08-17T18:20:19.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:19 smithi130 ceph-mon[110068]: pgmap v2831: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 40337/234036 objects degraded (17.235%); 40338/234036 objects misplaced (17.236%); 13 MiB/s, 211 objects/s recovering 2024-08-17T18:20:19.678 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:19.704 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:19.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:19.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:19.806 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:19.827 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:19.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:19.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:19.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:19 smithi179 ceph-mon[93519]: pgmap v2831: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 40337/234036 objects degraded (17.235%); 40338/234036 objects misplaced (17.236%); 13 MiB/s, 211 objects/s recovering 2024-08-17T18:20:19.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:19.979 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:19.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:19.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:20.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:20.124 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:20.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:20.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:20:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:20:19] "GET /metrics HTTP/1.1" 200 36817 "" "Prometheus/2.51.0" 2024-08-17T18:20:20.226 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:20.255 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:20.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:20.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:20.356 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:20.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:20.394 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:20.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:20.495 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:20.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:20.539 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:20.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:20:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:20:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:20:20.640 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:20.685 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:20.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:20.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:20.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:20.828 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:20.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:20.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:20.930 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:20.973 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:20.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:20.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:21.075 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:21.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:21.109 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:21.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:21.210 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:21.252 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:21.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:21.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:21.354 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:21.395 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:21.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:21.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:21.414 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:20:21.496 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:21.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:21.527 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:21.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:21.628 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:21.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:21 smithi130 ceph-mon[105521]: pgmap v2832: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 38095/224205 objects degraded (16.991%); 38696/224205 objects misplaced (17.259%); 20 MiB/s, 323 objects/s recovering 2024-08-17T18:20:21.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 38095/224205 objects degraded (16.991%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:21 smithi130 ceph-mon[110068]: pgmap v2832: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 38095/224205 objects degraded (16.991%); 38696/224205 objects misplaced (17.259%); 20 MiB/s, 323 objects/s recovering 2024-08-17T18:20:21.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 38095/224205 objects degraded (16.991%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:21.674 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:21.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:21.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:21.776 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:21.818 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:21.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:21.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:21 smithi179 ceph-mon[93519]: pgmap v2832: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 38095/224205 objects degraded (16.991%); 38696/224205 objects misplaced (17.259%); 20 MiB/s, 323 objects/s recovering 2024-08-17T18:20:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 38095/224205 objects degraded (16.991%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:21.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:21.959 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:21.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:21.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:22.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:22.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:22.112 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:22.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:22.213 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:22.258 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:22.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:22.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:22.360 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:22.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:22.399 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:22.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:22.501 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:22.544 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:22.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:22.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:22.646 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:22.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:22.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:22.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:22.746 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:20:22.789 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:22.822 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:22.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:22.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:22.924 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:22.966 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:22.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:22.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:23.068 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:23.115 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:23.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:23.217 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:23.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:23.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:23.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:23.351 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:23.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:23.386 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:23.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:23.488 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:23.528 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:23.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:23.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:23.629 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:23.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:23.660 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:23.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:23.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:23 smithi130 ceph-mon[105521]: pgmap v2833: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 38095/224205 objects degraded (16.991%); 38696/224205 objects misplaced (17.259%); 14 MiB/s, 220 objects/s recovering 2024-08-17T18:20:23.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:23 smithi130 ceph-mon[110068]: pgmap v2833: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 38095/224205 objects degraded (16.991%); 38696/224205 objects misplaced (17.259%); 14 MiB/s, 220 objects/s recovering 2024-08-17T18:20:23.761 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:23.799 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:23.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:23.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:23.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:23 smithi179 ceph-mon[93519]: pgmap v2833: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 696 op/s; 38095/224205 objects degraded (16.991%); 38696/224205 objects misplaced (17.259%); 14 MiB/s, 220 objects/s recovering 2024-08-17T18:20:23.901 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:23.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:23.934 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:23.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:24.035 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:24.083 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:24.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:24.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:24.185 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:24.225 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:24.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:24.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:24.327 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:24.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:24.377 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:24.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:24.428 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:20:24.479 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:24.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:24.528 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:24.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:24.629 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:24.657 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:24.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:24.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:24.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:24 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/610126406' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:20:24.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:24 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:20:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:24 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/610126406' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:20:24.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:24 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:20:24.759 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:24.808 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:24.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:24.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:24.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:24 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/610126406' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:20:24.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:24 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:20:24.910 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:24.945 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:24.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:24.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:25.047 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:25.086 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:25.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:25.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:25.188 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:25.229 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:25.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:25.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:25.331 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:25.375 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:25.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:25.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:25.477 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:25.511 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:25.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:25.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:25.613 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:25.654 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:25.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:25.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:25.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:25 smithi130 ceph-mon[105521]: pgmap v2834: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 35552/211227 objects degraded (16.831%); 35970/211227 objects misplaced (17.029%); 20 MiB/s, 324 objects/s recovering 2024-08-17T18:20:25.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:25 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:20:25.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:25 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:20:25.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:25 smithi130 ceph-mon[105521]: osdmap e1461: 8 total, 8 up, 8 in 2024-08-17T18:20:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:25 smithi130 ceph-mon[110068]: pgmap v2834: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 35552/211227 objects degraded (16.831%); 35970/211227 objects misplaced (17.029%); 20 MiB/s, 324 objects/s recovering 2024-08-17T18:20:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:25 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:20:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:25 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:20:25.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:25 smithi130 ceph-mon[110068]: osdmap e1461: 8 total, 8 up, 8 in 2024-08-17T18:20:25.756 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:25.804 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:25.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:25.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:25.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:25 smithi179 ceph-mon[93519]: pgmap v2834: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 35552/211227 objects degraded (16.831%); 35970/211227 objects misplaced (17.029%); 20 MiB/s, 324 objects/s recovering 2024-08-17T18:20:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:25 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:20:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:25 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:20:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:25 smithi179 ceph-mon[93519]: osdmap e1461: 8 total, 8 up, 8 in 2024-08-17T18:20:25.906 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:25.942 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:25.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:25.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:26.043 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:26.071 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:26.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:26.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:26.173 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:26.206 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:26.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:26.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:26.308 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:26.339 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:26.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:26.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:26.441 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:26.488 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:26.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:26.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:26.590 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:26.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:26.640 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:26.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:26.695 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 35552/211227 objects degraded (16.831%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:26.696 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 35552/211227 objects degraded (16.831%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:26.741 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:26.771 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:26.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:26.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:26.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 35552/211227 objects degraded (16.831%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T18:20:26.873 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:26.908 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:26.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:26.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:27.010 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:27.053 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:27.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:27.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:27.155 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:27.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:27.182 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:27.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:27.283 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:27.324 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:27.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:27.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:27.426 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:27.463 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:27.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:27.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:27.471 INFO:teuthology.orchestra.run.smithi130.stderr:marked out osd.0. 2024-08-17T18:20:27.565 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:27.582 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:27 smithi130 ceph-mon[110068]: pgmap v2836: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 35552/211227 objects degraded (16.831%); 35970/211227 objects misplaced (17.029%); 16 MiB/s, 259 objects/s recovering 2024-08-17T18:20:27.582 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:27 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2815857429' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T18:20:27.582 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:27 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T18:20:27.582 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:27 smithi130 ceph-mon[110068]: Client client.admin marked osd.0 out, while it was still marked up 2024-08-17T18:20:27.582 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:20:27.582 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:20:27.582 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:27 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T18:20:27.277+0000 7fa856f19640 -1 mon.a@0(leader).osd e1461 definitely_dead 0 2024-08-17T18:20:27.582 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:27 smithi130 ceph-mon[105521]: pgmap v2836: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 35552/211227 objects degraded (16.831%); 35970/211227 objects misplaced (17.029%); 16 MiB/s, 259 objects/s recovering 2024-08-17T18:20:27.582 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:27 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2815857429' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T18:20:27.583 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:27 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T18:20:27.583 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:27 smithi130 ceph-mon[105521]: Client client.admin marked osd.0 out, while it was still marked up 2024-08-17T18:20:27.583 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:20:27.583 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:20:27.587 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:27.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:27.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:27.689 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:27.700 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:20:27.700 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:20:27.713 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:27.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:27.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:27.722 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:20:27.722 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:20:27.744 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:20:27.744 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:20:27.765 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:20:27.766 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:20:27.783 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:20:27.783 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:20:27.802 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:20:27.802 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:20:27.815 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:27.816 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:20:27.816 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:20:27.821 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:20:27.821 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:20:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:27 smithi179 ceph-mon[93519]: pgmap v2836: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 837 op/s; 35552/211227 objects degraded (16.831%); 35970/211227 objects misplaced (17.029%); 16 MiB/s, 259 objects/s recovering 2024-08-17T18:20:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:27 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2815857429' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T18:20:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:27 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T18:20:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:27 smithi179 ceph-mon[93519]: Client client.admin marked osd.0 out, while it was still marked up 2024-08-17T18:20:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:20:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:20:27.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:27.844 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:27.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:27.845 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:20:27.845 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:20:27.945 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:27.969 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:27.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:27.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:28.052 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:20:28.071 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:28.094 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:28.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:28.195 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:28.221 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:28.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:28.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:28.323 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:28.346 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:28.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:28.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:28.448 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:28.473 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:28.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:28.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:28.575 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:28.596 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T18:20:28.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:28.625 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:28.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:28.727 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:28.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:28.757 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:28.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:28 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-08-17T18:20:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:28 smithi179 ceph-mon[93519]: osdmap e1462: 8 total, 8 up, 7 in 2024-08-17T18:20:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:28 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2507630358' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:20:28.858 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:28.881 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:28.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:28.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:28.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:28 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-08-17T18:20:28.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:28 smithi130 ceph-mon[110068]: osdmap e1462: 8 total, 8 up, 7 in 2024-08-17T18:20:28.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:28 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2507630358' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:20:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:28 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-08-17T18:20:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:28 smithi130 ceph-mon[105521]: osdmap e1462: 8 total, 8 up, 7 in 2024-08-17T18:20:28.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:28 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2507630358' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:20:28.983 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:29.005 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:29.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:29.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:29.107 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:29.130 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:29.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:29.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:29.232 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:29.255 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:29.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:29.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:29.357 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:29.380 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:29.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:29.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:29.482 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:29.512 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:29.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:29.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:29.613 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:29.638 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:29.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:29.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:29.740 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:29.768 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:29.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:29.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:29.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[105521]: pgmap v2838: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 35251/208683 objects degraded (16.892%); 34876/208683 objects misplaced (16.712%); 9.7 MiB/s, 156 objects/s recovering 2024-08-17T18:20:29.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[105521]: 50.a continuing backfill to osd.4 from (1444'6738,1461'14086] 50:5d2ff1c7:::benchmark_data_smithi130_398002_object100703:head to 1461'14086 2024-08-17T18:20:29.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[105521]: 50.7 starting backfill to osd.2 from (1438'200,1457'9317] MIN to 1461'10226 2024-08-17T18:20:29.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[105521]: osdmap e1463: 8 total, 8 up, 7 in 2024-08-17T18:20:29.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[105521]: 50.e starting backfill to osd.1 from (0'0,0'0] MAX to 1461'13751 2024-08-17T18:20:29.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[105521]: 50.17 starting backfill to osd.2 from (1438'200,1457'9329] MIN to 1461'10269 2024-08-17T18:20:29.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[105521]: 50.15 starting backfill to osd.2 from (1438'900,1457'9320] MIN to 1461'10304 2024-08-17T18:20:29.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[105521]: 50.5 starting backfill to osd.2 from (1438'900,1457'9349] MIN to 1461'10344 2024-08-17T18:20:29.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[105521]: 50.d continuing backfill to osd.5 from (1441'6510,1461'13834] 50:bb94d5c0:::benchmark_data_smithi130_398002_object86715:head to 1461'13834 2024-08-17T18:20:29.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[105521]: 50.e starting backfill to osd.2 from (1445'7288,1461'13750] MIN to 1461'13751 2024-08-17T18:20:29.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[105521]: 50.7 continuing backfill to osd.5 from (1438'200,1461'10226] MIN to 1461'10226 2024-08-17T18:20:29.821 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[105521]: 50.17 starting backfill to osd.5 from (1438'394,1461'10268] MIN to 1461'10269 2024-08-17T18:20:29.821 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[105521]: 50.5 continuing backfill to osd.5 from (1438'900,1461'10344] MIN to 1461'10344 2024-08-17T18:20:29.821 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[105521]: 50.15 continuing backfill to osd.5 from (1438'900,1461'10304] MIN to 1461'10304 2024-08-17T18:20:29.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[110068]: pgmap v2838: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 35251/208683 objects degraded (16.892%); 34876/208683 objects misplaced (16.712%); 9.7 MiB/s, 156 objects/s recovering 2024-08-17T18:20:29.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[110068]: 50.a continuing backfill to osd.4 from (1444'6738,1461'14086] 50:5d2ff1c7:::benchmark_data_smithi130_398002_object100703:head to 1461'14086 2024-08-17T18:20:29.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[110068]: 50.7 starting backfill to osd.2 from (1438'200,1457'9317] MIN to 1461'10226 2024-08-17T18:20:29.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[110068]: osdmap e1463: 8 total, 8 up, 7 in 2024-08-17T18:20:29.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[110068]: 50.e starting backfill to osd.1 from (0'0,0'0] MAX to 1461'13751 2024-08-17T18:20:29.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[110068]: 50.17 starting backfill to osd.2 from (1438'200,1457'9329] MIN to 1461'10269 2024-08-17T18:20:29.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[110068]: 50.15 starting backfill to osd.2 from (1438'900,1457'9320] MIN to 1461'10304 2024-08-17T18:20:29.821 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[110068]: 50.5 starting backfill to osd.2 from (1438'900,1457'9349] MIN to 1461'10344 2024-08-17T18:20:29.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[110068]: 50.d continuing backfill to osd.5 from (1441'6510,1461'13834] 50:bb94d5c0:::benchmark_data_smithi130_398002_object86715:head to 1461'13834 2024-08-17T18:20:29.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[110068]: 50.e starting backfill to osd.2 from (1445'7288,1461'13750] MIN to 1461'13751 2024-08-17T18:20:29.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[110068]: 50.7 continuing backfill to osd.5 from (1438'200,1461'10226] MIN to 1461'10226 2024-08-17T18:20:29.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[110068]: 50.17 starting backfill to osd.5 from (1438'394,1461'10268] MIN to 1461'10269 2024-08-17T18:20:29.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[110068]: 50.5 continuing backfill to osd.5 from (1438'900,1461'10344] MIN to 1461'10344 2024-08-17T18:20:29.822 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-mon[110068]: 50.15 continuing backfill to osd.5 from (1438'900,1461'10304] MIN to 1461'10304 2024-08-17T18:20:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:29 smithi179 ceph-mon[93519]: pgmap v2838: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 12 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 78 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 35251/208683 objects degraded (16.892%); 34876/208683 objects misplaced (16.712%); 9.7 MiB/s, 156 objects/s recovering 2024-08-17T18:20:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:29 smithi179 ceph-mon[93519]: 50.a continuing backfill to osd.4 from (1444'6738,1461'14086] 50:5d2ff1c7:::benchmark_data_smithi130_398002_object100703:head to 1461'14086 2024-08-17T18:20:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:29 smithi179 ceph-mon[93519]: 50.7 starting backfill to osd.2 from (1438'200,1457'9317] MIN to 1461'10226 2024-08-17T18:20:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:29 smithi179 ceph-mon[93519]: osdmap e1463: 8 total, 8 up, 7 in 2024-08-17T18:20:29.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:29 smithi179 ceph-mon[93519]: 50.e starting backfill to osd.1 from (0'0,0'0] MAX to 1461'13751 2024-08-17T18:20:29.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:29 smithi179 ceph-mon[93519]: 50.17 starting backfill to osd.2 from (1438'200,1457'9329] MIN to 1461'10269 2024-08-17T18:20:29.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:29 smithi179 ceph-mon[93519]: 50.15 starting backfill to osd.2 from (1438'900,1457'9320] MIN to 1461'10304 2024-08-17T18:20:29.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:29 smithi179 ceph-mon[93519]: 50.5 starting backfill to osd.2 from (1438'900,1457'9349] MIN to 1461'10344 2024-08-17T18:20:29.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:29 smithi179 ceph-mon[93519]: 50.d continuing backfill to osd.5 from (1441'6510,1461'13834] 50:bb94d5c0:::benchmark_data_smithi130_398002_object86715:head to 1461'13834 2024-08-17T18:20:29.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:29 smithi179 ceph-mon[93519]: 50.e starting backfill to osd.2 from (1445'7288,1461'13750] MIN to 1461'13751 2024-08-17T18:20:29.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:29 smithi179 ceph-mon[93519]: 50.7 continuing backfill to osd.5 from (1438'200,1461'10226] MIN to 1461'10226 2024-08-17T18:20:29.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:29 smithi179 ceph-mon[93519]: 50.17 starting backfill to osd.5 from (1438'394,1461'10268] MIN to 1461'10269 2024-08-17T18:20:29.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:29 smithi179 ceph-mon[93519]: 50.5 continuing backfill to osd.5 from (1438'900,1461'10344] MIN to 1461'10344 2024-08-17T18:20:29.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:29 smithi179 ceph-mon[93519]: 50.15 continuing backfill to osd.5 from (1438'900,1461'10304] MIN to 1461'10304 2024-08-17T18:20:29.871 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:29.901 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:29.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:29.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:30.003 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:30.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:30.039 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:30.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:30.140 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:30.168 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:20:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:20:29] "GET /metrics HTTP/1.1" 200 36812 "" "Prometheus/2.51.0" 2024-08-17T18:20:30.178 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:30.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:30.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:30.280 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:30.312 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:30.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:30.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:30.414 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:30.458 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:30.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:30.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:30.487 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:20:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:20:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:20:30.560 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:30.582 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:30.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:30.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:30.684 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:30.719 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:30.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:30.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:30.821 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:30 smithi179 ceph-mon[93519]: 50.11 starting backfill to osd.1 from (0'0,0'0] MAX to 1461'10334 2024-08-17T18:20:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:30 smithi179 ceph-mon[93519]: osdmap e1464: 8 total, 8 up, 7 in 2024-08-17T18:20:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:30 smithi179 ceph-mon[93519]: 50.1 starting backfill to osd.1 from (0'0,0'0] MAX to 1461'10424 2024-08-17T18:20:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:30 smithi179 ceph-mon[93519]: 50.8 starting backfill to osd.3 from (0'0,0'0] MAX to 1461'10346 2024-08-17T18:20:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:30 smithi179 ceph-mon[93519]: 50.19 starting backfill to osd.1 from (0'0,0'0] MAX to 1461'10331 2024-08-17T18:20:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:30 smithi179 ceph-mon[93519]: 50.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1461'10309 2024-08-17T18:20:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:30 smithi179 ceph-mon[93519]: 50.c starting backfill to osd.2 from (0'0,0'0] MAX to 1461'13998 2024-08-17T18:20:30.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:30 smithi179 ceph-mon[93519]: 50.18 starting backfill to osd.3 from (0'0,0'0] MAX to 1461'10148 2024-08-17T18:20:30.862 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:30.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:30.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:30.878 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[105521]: 50.11 starting backfill to osd.1 from (0'0,0'0] MAX to 1461'10334 2024-08-17T18:20:30.878 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[105521]: osdmap e1464: 8 total, 8 up, 7 in 2024-08-17T18:20:30.878 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[105521]: 50.1 starting backfill to osd.1 from (0'0,0'0] MAX to 1461'10424 2024-08-17T18:20:30.879 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[105521]: 50.8 starting backfill to osd.3 from (0'0,0'0] MAX to 1461'10346 2024-08-17T18:20:30.879 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[105521]: 50.19 starting backfill to osd.1 from (0'0,0'0] MAX to 1461'10331 2024-08-17T18:20:30.879 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[105521]: 50.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1461'10309 2024-08-17T18:20:30.879 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[105521]: 50.c starting backfill to osd.2 from (0'0,0'0] MAX to 1461'13998 2024-08-17T18:20:30.879 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[105521]: 50.18 starting backfill to osd.3 from (0'0,0'0] MAX to 1461'10148 2024-08-17T18:20:30.879 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[110068]: 50.11 starting backfill to osd.1 from (0'0,0'0] MAX to 1461'10334 2024-08-17T18:20:30.879 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[110068]: osdmap e1464: 8 total, 8 up, 7 in 2024-08-17T18:20:30.879 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[110068]: 50.1 starting backfill to osd.1 from (0'0,0'0] MAX to 1461'10424 2024-08-17T18:20:30.879 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[110068]: 50.8 starting backfill to osd.3 from (0'0,0'0] MAX to 1461'10346 2024-08-17T18:20:30.879 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[110068]: 50.19 starting backfill to osd.1 from (0'0,0'0] MAX to 1461'10331 2024-08-17T18:20:30.880 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[110068]: 50.9 starting backfill to osd.1 from (0'0,0'0] MAX to 1461'10309 2024-08-17T18:20:30.880 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[110068]: 50.c starting backfill to osd.2 from (0'0,0'0] MAX to 1461'13998 2024-08-17T18:20:30.880 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:30 smithi130 ceph-mon[110068]: 50.18 starting backfill to osd.3 from (0'0,0'0] MAX to 1461'10148 2024-08-17T18:20:30.965 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:31.000 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:31.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:31.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:31.101 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:31.127 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:31.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:31.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:31.229 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:31.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:31.261 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:31.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:31.363 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:31.398 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:31.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:31.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:31.499 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:31.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:31.539 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:31.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:31.640 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:31.683 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:31.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:31.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:31.785 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:31.824 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:31.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:31.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:31.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:31 smithi179 ceph-mon[93519]: pgmap v2841: 99 pgs: 2 activating+undersized+remapped, 1 active+undersized+degraded+remapped+backfilling, 2 activating, 18 peering, 8 remapped+peering, 8 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 55 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 21810/199848 objects degraded (10.913%); 36924/199848 objects misplaced (18.476%); 19 MiB/s, 307 objects/s recovering 2024-08-17T18:20:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:31 smithi179 ceph-mon[93519]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-17T18:20:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:31 smithi179 ceph-mon[93519]: osdmap e1465: 8 total, 8 up, 7 in 2024-08-17T18:20:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 21810/199848 objects degraded (10.913%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:31.908 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:31 smithi130 ceph-mon[105521]: pgmap v2841: 99 pgs: 2 activating+undersized+remapped, 1 active+undersized+degraded+remapped+backfilling, 2 activating, 18 peering, 8 remapped+peering, 8 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 55 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 21810/199848 objects degraded (10.913%); 36924/199848 objects misplaced (18.476%); 19 MiB/s, 307 objects/s recovering 2024-08-17T18:20:31.908 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:31 smithi130 ceph-mon[105521]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-17T18:20:31.908 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:31 smithi130 ceph-mon[105521]: osdmap e1465: 8 total, 8 up, 7 in 2024-08-17T18:20:31.909 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 21810/199848 objects degraded (10.913%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:31.909 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:31 smithi130 ceph-mon[110068]: pgmap v2841: 99 pgs: 2 activating+undersized+remapped, 1 active+undersized+degraded+remapped+backfilling, 2 activating, 18 peering, 8 remapped+peering, 8 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 55 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 722 op/s; 21810/199848 objects degraded (10.913%); 36924/199848 objects misplaced (18.476%); 19 MiB/s, 307 objects/s recovering 2024-08-17T18:20:31.909 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:31 smithi130 ceph-mon[110068]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-17T18:20:31.909 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:31 smithi130 ceph-mon[110068]: osdmap e1465: 8 total, 8 up, 7 in 2024-08-17T18:20:31.909 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 21810/199848 objects degraded (10.913%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:31.926 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:31.959 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:31.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:31.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:32.061 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:32.115 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:32.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:32.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:32.216 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:32.257 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:32.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:32.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:32.359 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:32.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:32.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:32.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:32.491 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:20:32.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:32.558 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:20:32.576 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:32.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:32.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:32.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:32 smithi130 ceph-mon[110068]: osdmap e1466: 8 total, 8 up, 7 in 2024-08-17T18:20:32.670 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:32 smithi130 ceph-mon[105521]: osdmap e1466: 8 total, 8 up, 7 in 2024-08-17T18:20:32.678 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:32.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:32.730 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:32.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:32.831 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:32.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:32 smithi179 ceph-mon[93519]: osdmap e1466: 8 total, 8 up, 7 in 2024-08-17T18:20:32.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:32.887 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:32.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:32.988 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:33.020 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:33.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:33.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:33.122 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:33.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:33.158 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:33.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:33.258 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:33.309 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:33.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:33.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:33.411 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:33.445 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:33.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:33.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:33.547 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:33.584 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:33.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:33.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:33.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:33 smithi130 ceph-mon[105521]: pgmap v2844: 99 pgs: 2 activating+undersized+remapped, 1 active+undersized+degraded+remapped+backfilling, 2 activating, 18 peering, 8 remapped+peering, 8 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 55 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 21810/199848 objects degraded (10.913%); 36924/199848 objects misplaced (18.476%); 25 MiB/s, 404 objects/s recovering 2024-08-17T18:20:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:33 smithi130 ceph-mon[110068]: pgmap v2844: 99 pgs: 2 activating+undersized+remapped, 1 active+undersized+degraded+remapped+backfilling, 2 activating, 18 peering, 8 remapped+peering, 8 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 55 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 21810/199848 objects degraded (10.913%); 36924/199848 objects misplaced (18.476%); 25 MiB/s, 404 objects/s recovering 2024-08-17T18:20:33.686 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:33.719 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:33.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:33.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:33.821 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:33.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:33 smithi179 ceph-mon[93519]: pgmap v2844: 99 pgs: 2 activating+undersized+remapped, 1 active+undersized+degraded+remapped+backfilling, 2 activating, 18 peering, 8 remapped+peering, 8 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 55 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 21810/199848 objects degraded (10.913%); 36924/199848 objects misplaced (18.476%); 25 MiB/s, 404 objects/s recovering 2024-08-17T18:20:33.863 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:33.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:33.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:33.964 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:33.998 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:33.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:33.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:34.100 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:34.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:34.129 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:34.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:34.230 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:34.266 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:34.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:34.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:34.368 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:34.403 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:34.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:34.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:34.449 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:20:34.505 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:34.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:34.566 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:34.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:34.667 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:34.753 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:34.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:34.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:34.776 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:34 smithi130 ceph-mon[105521]: pgmap v2845: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 28410/192165 objects degraded (14.784%); 61679/192165 objects misplaced (32.097%); 21 MiB/s, 457 objects/s recovering 2024-08-17T18:20:34.776 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:34 smithi130 ceph-mon[105521]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-08-17T18:20:34.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:34 smithi130 ceph-mon[110068]: pgmap v2845: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 28410/192165 objects degraded (14.784%); 61679/192165 objects misplaced (32.097%); 21 MiB/s, 457 objects/s recovering 2024-08-17T18:20:34.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:34 smithi130 ceph-mon[110068]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-08-17T18:20:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:34 smithi179 ceph-mon[93519]: pgmap v2845: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 28410/192165 objects degraded (14.784%); 61679/192165 objects misplaced (32.097%); 21 MiB/s, 457 objects/s recovering 2024-08-17T18:20:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:34 smithi179 ceph-mon[93519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-08-17T18:20:34.855 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:34.887 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:34.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:34.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:34.989 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:35.027 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:35.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:35.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:35.129 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:35.208 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:35.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:35.310 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:35.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:35.355 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:35.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:35.456 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:35.519 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:35.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:35.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:35.621 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:35.668 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:35.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:35.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:35.770 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:35.810 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:35.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:35.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:35.912 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:35.949 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:35.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:35.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:36.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:36.081 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:36.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:36.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:36.183 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:36.225 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:36.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:36.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:36.329 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:36.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:36.379 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:36.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:36.480 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:36.519 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:36.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:36.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:36.621 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:36.660 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:36.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:36.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:36.762 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:36.801 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:36.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:36.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:36.903 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:36.936 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:36.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:36.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:36.953 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 7, 1, 6, 4, 5] out_osds: [0] dead_osds: [] live_osds: [1, 4, 5, 6, 7, 0, 3, 2] 2024-08-17T18:20:36.953 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:20:36.954 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T18:20:36.954 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.805094 2024-08-17T18:20:36.954 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 2 0.8050944066973458 2024-08-17T18:20:36.989 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:36 smithi130 ceph-mon[105521]: pgmap v2846: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 28410/192165 objects degraded (14.784%); 61679/192165 objects misplaced (32.097%); 12 MiB/s, 292 objects/s recovering 2024-08-17T18:20:36.989 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 28410/192165 objects degraded (14.784%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:36.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:36 smithi130 ceph-mon[110068]: pgmap v2846: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 28410/192165 objects degraded (14.784%); 61679/192165 objects misplaced (32.097%); 12 MiB/s, 292 objects/s recovering 2024-08-17T18:20:36.990 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 28410/192165 objects degraded (14.784%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:37.037 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:37.079 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:37.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:37.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:37.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:36 smithi179 ceph-mon[93519]: pgmap v2846: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 28410/192165 objects degraded (14.784%); 61679/192165 objects misplaced (32.097%); 12 MiB/s, 292 objects/s recovering 2024-08-17T18:20:37.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 28410/192165 objects degraded (14.784%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:37.181 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:37.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:37.271 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:37.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:37.372 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:37.420 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:37.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:37.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:37.521 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:37.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:37.570 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:37.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:37.671 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:37.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:37.697 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:37.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:37.798 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:37.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:37.842 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:37.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:37.943 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:37.978 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:37.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:37.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:38.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:38.120 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:38.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:38.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:38.221 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:38.263 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:38.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:38.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:38.365 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:38.407 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:38.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:38.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:38.509 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:38.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:38.531 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:38.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:38.632 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:38.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:38.667 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:38.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:38.710 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:20:38.768 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:38.774 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:20:38.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:38.831 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:38.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:38.932 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:38.973 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:38.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:38.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:39.075 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:38 smithi179 ceph-mon[93519]: pgmap v2847: 99 pgs: 1 active+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 28119/189726 objects degraded (14.821%); 60400/189726 objects misplaced (31.835%); 9.1 MiB/s, 227 objects/s recovering 2024-08-17T18:20:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:38 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/857479672' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:20:39.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:38 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:20:39.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:39.107 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:39.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:39.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:38 smithi130 ceph-mon[105521]: pgmap v2847: 99 pgs: 1 active+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 28119/189726 objects degraded (14.821%); 60400/189726 objects misplaced (31.835%); 9.1 MiB/s, 227 objects/s recovering 2024-08-17T18:20:39.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:38 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/857479672' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:20:39.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:38 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:20:39.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:38 smithi130 ceph-mon[110068]: pgmap v2847: 99 pgs: 1 active+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 421 op/s; 28119/189726 objects degraded (14.821%); 60400/189726 objects misplaced (31.835%); 9.1 MiB/s, 227 objects/s recovering 2024-08-17T18:20:39.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:38 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/857479672' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:20:39.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:38 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:20:39.209 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:39.242 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:39.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:39.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:39.344 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:39.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:39.378 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:39.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:39.479 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:39.517 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:39.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:39.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:39.619 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:39.659 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:39.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:39.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:39.761 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:39.801 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:39.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:39.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:39.902 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:39.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:39.951 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:39.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:39.956 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:20:39.956 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:20:39.957 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:20:39.957 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:20:39.957 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.957 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186456 49.6 49:68186510:::benchmark_data_smithi130_364143_object77322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1404)", 2024-08-17T18:20:39.957 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.331750+0000", 2024-08-17T18:20:39.957 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 281.61412093400003, 2024-08-17T18:20:39.957 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.259376829, 2024-08-17T18:20:39.957 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:39.957 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:39.957 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:20:39.957 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:39.957 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:39.957 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:39.957 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186456 2024-08-17T18:20:39.958 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.958 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:39.958 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.958 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:39.958 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.331750+0000", 2024-08-17T18:20:39.958 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:39.958 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.958 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.958 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:39.958 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.331750+0000", 2024-08-17T18:20:39.958 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:39.958 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.958 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.959 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:39.959 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.331751+0000", 2024-08-17T18:20:39.959 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0599999999999999e-07 2024-08-17T18:20:39.959 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.959 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.959 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:39.959 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.331755+0000", 2024-08-17T18:20:39.959 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.765e-06 2024-08-17T18:20:39.959 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.959 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.959 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:39.959 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.331763+0000", 2024-08-17T18:20:39.959 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1869999999999997e-06 2024-08-17T18:20:39.960 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.960 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.960 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:39.960 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.331769+0000", 2024-08-17T18:20:39.960 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1990000000000003e-06 2024-08-17T18:20:39.960 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.960 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.960 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.960 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.350180+0000", 2024-08-17T18:20:39.960 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.018410268 2024-08-17T18:20:39.960 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.960 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.960 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:39.961 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.350739+0000", 2024-08-17T18:20:39.961 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00055904100000000003 2024-08-17T18:20:39.961 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.961 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.961 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2,7", 2024-08-17T18:20:39.961 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.350880+0000", 2024-08-17T18:20:39.961 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014124 2024-08-17T18:20:39.961 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.961 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.961 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:20:39.961 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.363243+0000", 2024-08-17T18:20:39.961 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012363254000000001 2024-08-17T18:20:39.961 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.962 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.962 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:39.962 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.363850+0000", 2024-08-17T18:20:39.962 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000607034 2024-08-17T18:20:39.962 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.962 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.962 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:39.962 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.364152+0000", 2024-08-17T18:20:39.962 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00030168099999999999 2024-08-17T18:20:39.962 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.962 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.962 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.963 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.379833+0000", 2024-08-17T18:20:39.963 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015680565 2024-08-17T18:20:39.963 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.963 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.963 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:39.963 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.379844+0000", 2024-08-17T18:20:39.963 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1233000000000001e-05 2024-08-17T18:20:39.963 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.963 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.963 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.963 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.591062+0000", 2024-08-17T18:20:39.963 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2112186899999999 2024-08-17T18:20:39.964 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.964 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.964 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:39.964 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.591127+0000", 2024-08-17T18:20:39.964 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4865999999999999e-05 2024-08-17T18:20:39.964 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:39.964 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:39.964 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:39.964 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.964 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.964 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186464 49.6 49:6c9357ba:::benchmark_data_smithi130_364143_object77330:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1404)", 2024-08-17T18:20:39.964 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.352267+0000", 2024-08-17T18:20:39.964 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 281.593604438, 2024-08-17T18:20:39.964 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.253199599, 2024-08-17T18:20:39.965 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:39.965 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:39.965 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:20:39.965 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:39.965 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:39.965 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:39.965 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186464 2024-08-17T18:20:39.965 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.965 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:39.965 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.965 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:39.965 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.352267+0000", 2024-08-17T18:20:39.965 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:39.966 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.966 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.966 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:39.966 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.352267+0000", 2024-08-17T18:20:39.966 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:39.966 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.966 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.966 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:39.966 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.352268+0000", 2024-08-17T18:20:39.966 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4649999999999999e-06 2024-08-17T18:20:39.966 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.966 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.966 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:39.967 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.352274+0000", 2024-08-17T18:20:39.967 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5269999999999998e-06 2024-08-17T18:20:39.967 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.967 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.967 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:39.967 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.352292+0000", 2024-08-17T18:20:39.967 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7855000000000001e-05 2024-08-17T18:20:39.967 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.967 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.967 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:39.967 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.352301+0000", 2024-08-17T18:20:39.967 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7110000000000001e-06 2024-08-17T18:20:39.967 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.968 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.968 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.968 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.367745+0000", 2024-08-17T18:20:39.968 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015443989 2024-08-17T18:20:39.968 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.968 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.968 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:39.968 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.368010+0000", 2024-08-17T18:20:39.968 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00026594699999999999 2024-08-17T18:20:39.968 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.968 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.968 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,2,7", 2024-08-17T18:20:39.968 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.368287+0000", 2024-08-17T18:20:39.969 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000276111 2024-08-17T18:20:39.969 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.969 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.969 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.969 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391624+0000", 2024-08-17T18:20:39.969 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023337051000000001 2024-08-17T18:20:39.969 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.969 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.969 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:39.969 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391636+0000", 2024-08-17T18:20:39.969 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.276e-05 2024-08-17T18:20:39.969 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.969 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.969 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.970 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.591790+0000", 2024-08-17T18:20:39.970 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.20015323 2024-08-17T18:20:39.970 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.970 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.970 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:20:39.970 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.591931+0000", 2024-08-17T18:20:39.970 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014178699999999999 2024-08-17T18:20:39.970 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.970 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.970 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.970 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.605401+0000", 2024-08-17T18:20:39.970 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013469778999999999 2024-08-17T18:20:39.970 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.971 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.971 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:39.971 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.605467+0000", 2024-08-17T18:20:39.971 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5387000000000003e-05 2024-08-17T18:20:39.971 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:39.971 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:39.971 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:39.971 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.971 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.971 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186447 49.f 49:f5e043ca:::benchmark_data_smithi130_364143_object77313:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1405)", 2024-08-17T18:20:39.971 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.369979+0000", 2024-08-17T18:20:39.971 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 281.575892458, 2024-08-17T18:20:39.972 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1152148909999999, 2024-08-17T18:20:39.972 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:39.972 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:39.972 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:20:39.972 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:39.972 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:39.972 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:39.972 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186447 2024-08-17T18:20:39.972 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.972 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:39.972 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.972 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:39.972 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.369979+0000", 2024-08-17T18:20:39.973 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:39.973 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.973 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.973 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:39.973 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.369979+0000", 2024-08-17T18:20:39.973 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:39.973 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.973 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.973 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:39.973 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.369979+0000", 2024-08-17T18:20:39.973 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9200000000000001e-07 2024-08-17T18:20:39.974 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.974 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.974 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:39.974 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.369982+0000", 2024-08-17T18:20:39.974 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.5469999999999999e-06 2024-08-17T18:20:39.974 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.974 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.974 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:39.974 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.369989+0000", 2024-08-17T18:20:39.974 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9840000000000004e-06 2024-08-17T18:20:39.974 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.974 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.974 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:39.975 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.369994+0000", 2024-08-17T18:20:39.975 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.383e-06 2024-08-17T18:20:39.975 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.975 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.975 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.975 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370071+0000", 2024-08-17T18:20:39.975 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6489999999999994e-05 2024-08-17T18:20:39.975 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.975 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.975 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op must wait for map", 2024-08-17T18:20:39.975 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370078+0000", 2024-08-17T18:20:39.975 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9349999999999997e-06 2024-08-17T18:20:39.975 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.976 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.976 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.976 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392807+0000", 2024-08-17T18:20:39.976 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022728809999999999 2024-08-17T18:20:39.976 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.976 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.976 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:39.976 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392815+0000", 2024-08-17T18:20:39.976 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1610000000000003e-06 2024-08-17T18:20:39.976 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.976 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.976 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.977 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.485110+0000", 2024-08-17T18:20:39.977 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.092295236 2024-08-17T18:20:39.977 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.977 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.977 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:39.977 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.485194+0000", 2024-08-17T18:20:39.977 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3852999999999998e-05 2024-08-17T18:20:39.977 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:39.977 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:39.977 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:39.977 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.977 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.977 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186457 49.e 49:7158c5aa:::benchmark_data_smithi130_364143_object77323:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1405)", 2024-08-17T18:20:39.978 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.370020+0000", 2024-08-17T18:20:39.978 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 281.575851769, 2024-08-17T18:20:39.978 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1465918470000001, 2024-08-17T18:20:39.978 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:39.978 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:39.978 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:20:39.978 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:39.978 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:39.978 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:39.978 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186457 2024-08-17T18:20:39.978 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.978 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:39.978 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.979 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:39.979 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370020+0000", 2024-08-17T18:20:39.979 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:39.979 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.979 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.979 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:39.979 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370020+0000", 2024-08-17T18:20:39.979 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:39.979 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.979 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.979 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:39.979 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370020+0000", 2024-08-17T18:20:39.979 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3700000000000001e-07 2024-08-17T18:20:39.979 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.980 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.980 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:39.980 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370022+0000", 2024-08-17T18:20:39.980 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0600000000000002e-06 2024-08-17T18:20:39.980 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.980 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.980 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:39.980 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370028+0000", 2024-08-17T18:20:39.980 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1340000000000001e-06 2024-08-17T18:20:39.980 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.980 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.980 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:39.981 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370033+0000", 2024-08-17T18:20:39.981 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.887e-06 2024-08-17T18:20:39.981 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.981 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.981 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.981 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391670+0000", 2024-08-17T18:20:39.981 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021637111000000001 2024-08-17T18:20:39.981 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.981 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.981 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:39.981 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391678+0000", 2024-08-17T18:20:39.981 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9729999999999997e-06 2024-08-17T18:20:39.981 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.981 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.982 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.982 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.516532+0000", 2024-08-17T18:20:39.982 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.124853801 2024-08-17T18:20:39.982 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.982 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.982 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:39.982 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.516612+0000", 2024-08-17T18:20:39.982 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.9443999999999994e-05 2024-08-17T18:20:39.982 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:39.982 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:39.982 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:39.982 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.982 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.982 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186462 49.f 49:f5861bec:::benchmark_data_smithi130_364143_object77328:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1405)", 2024-08-17T18:20:39.983 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.370041+0000", 2024-08-17T18:20:39.983 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 281.575830516, 2024-08-17T18:20:39.983 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1238796959999999, 2024-08-17T18:20:39.983 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:39.983 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:39.983 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:20:39.983 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:39.983 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:39.983 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:39.983 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186462 2024-08-17T18:20:39.983 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.983 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:39.983 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.984 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:39.984 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370041+0000", 2024-08-17T18:20:39.984 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:39.984 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.984 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.984 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:39.984 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370041+0000", 2024-08-17T18:20:39.984 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:39.984 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.984 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.984 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:39.984 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370041+0000", 2024-08-17T18:20:39.985 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.0699999999999998e-07 2024-08-17T18:20:39.985 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.985 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.985 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:39.985 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370043+0000", 2024-08-17T18:20:39.985 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6780000000000001e-06 2024-08-17T18:20:39.985 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.985 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.985 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:39.985 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370049+0000", 2024-08-17T18:20:39.985 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.1720000000000004e-06 2024-08-17T18:20:39.985 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.985 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.986 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:39.986 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370054+0000", 2024-08-17T18:20:39.986 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5709999999999999e-06 2024-08-17T18:20:39.986 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.986 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.986 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.986 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370126+0000", 2024-08-17T18:20:39.987 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1877000000000003e-05 2024-08-17T18:20:39.988 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.988 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.988 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting_for_map not empty", 2024-08-17T18:20:39.988 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370131+0000", 2024-08-17T18:20:39.988 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0019999999999997e-06 2024-08-17T18:20:39.988 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.988 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.988 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.988 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392838+0000", 2024-08-17T18:20:39.988 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022707267 2024-08-17T18:20:39.988 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.988 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.989 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:39.989 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392844+0000", 2024-08-17T18:20:39.989 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.161e-06 2024-08-17T18:20:39.989 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.989 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.989 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.989 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.493831+0000", 2024-08-17T18:20:39.989 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.100987347 2024-08-17T18:20:39.989 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.989 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.989 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:39.989 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.493921+0000", 2024-08-17T18:20:39.989 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9214000000000003e-05 2024-08-17T18:20:39.990 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:39.990 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:39.990 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:39.990 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.990 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.990 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186465 49.e 49:78c6a962:::benchmark_data_smithi130_364143_object77331:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1405)", 2024-08-17T18:20:39.990 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.370060+0000", 2024-08-17T18:20:39.990 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 281.57581153299998, 2024-08-17T18:20:39.990 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2212996860000001, 2024-08-17T18:20:39.990 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:39.990 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:39.990 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:20:39.990 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:39.991 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:39.991 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:39.991 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186465 2024-08-17T18:20:39.991 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.991 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:39.991 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.991 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:39.991 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370060+0000", 2024-08-17T18:20:39.991 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:39.991 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.991 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.991 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:39.991 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370060+0000", 2024-08-17T18:20:39.991 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:39.992 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.992 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.992 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:39.992 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370060+0000", 2024-08-17T18:20:39.992 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.84e-07 2024-08-17T18:20:39.992 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.992 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.992 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:39.992 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370062+0000", 2024-08-17T18:20:39.992 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6029999999999999e-06 2024-08-17T18:20:39.992 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.992 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.992 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:39.992 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370067+0000", 2024-08-17T18:20:39.993 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5400000000000003e-06 2024-08-17T18:20:39.993 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.993 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.993 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:39.993 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.370072+0000", 2024-08-17T18:20:39.993 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3769999999999998e-06 2024-08-17T18:20:39.993 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.993 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.993 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.993 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391708+0000", 2024-08-17T18:20:39.993 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021635867999999999 2024-08-17T18:20:39.993 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.993 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.994 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:39.994 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391715+0000", 2024-08-17T18:20:39.994 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8959999999999997e-06 2024-08-17T18:20:39.994 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.994 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.994 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:39.994 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.516998+0000", 2024-08-17T18:20:39.994 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1252833820000001 2024-08-17T18:20:39.994 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.994 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.994 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:39.994 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.517105+0000", 2024-08-17T18:20:39.994 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000106808 2024-08-17T18:20:39.995 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.995 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.995 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:20:39.995 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.517617+0000", 2024-08-17T18:20:39.995 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00051267800000000003 2024-08-17T18:20:39.995 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.995 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.995 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:39.995 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.533197+0000", 2024-08-17T18:20:39.995 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015579268 2024-08-17T18:20:39.995 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.995 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.995 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:20:39.995 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.539293+0000", 2024-08-17T18:20:39.996 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0060962229999999996 2024-08-17T18:20:39.996 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.996 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.996 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:39.996 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.591207+0000", 2024-08-17T18:20:39.996 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.051914271999999997 2024-08-17T18:20:39.996 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.996 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.996 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:20:39.996 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.591258+0000", 2024-08-17T18:20:39.996 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.0519000000000002e-05 2024-08-17T18:20:39.996 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.996 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.997 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:39.997 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.591360+0000", 2024-08-17T18:20:39.997 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010186799999999999 2024-08-17T18:20:39.997 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:39.997 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:39.997 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:39.997 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.997 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.997 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186467 49.6 49:68b596c6:::benchmark_data_smithi130_364143_object77333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1405)", 2024-08-17T18:20:39.997 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.374055+0000", 2024-08-17T18:20:39.997 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 281.57181603999999, 2024-08-17T18:20:39.997 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.242002885, 2024-08-17T18:20:39.997 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:39.997 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:39.998 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:20:39.998 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:39.998 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:39.998 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:39.998 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186467 2024-08-17T18:20:39.998 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.998 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:39.998 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.998 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:39.998 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.374055+0000", 2024-08-17T18:20:39.998 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:39.998 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.998 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.999 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:39.999 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.374055+0000", 2024-08-17T18:20:39.999 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:39.999 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.999 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.999 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:39.999 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.374057+0000", 2024-08-17T18:20:39.999 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.8279999999999999e-06 2024-08-17T18:20:39.999 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:39.999 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:39.999 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:39.999 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.374063+0000", 2024-08-17T18:20:39.999 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.772e-06 2024-08-17T18:20:40.000 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.000 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.000 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:40.000 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.374074+0000", 2024-08-17T18:20:40.000 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1276e-05 2024-08-17T18:20:40.000 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.000 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.000 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:40.000 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.374081+0000", 2024-08-17T18:20:40.000 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8580000000000002e-06 2024-08-17T18:20:40.000 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.000 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.000 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.000 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391948+0000", 2024-08-17T18:20:40.001 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017866770000000001 2024-08-17T18:20:40.001 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.001 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.001 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.001 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391956+0000", 2024-08-17T18:20:40.001 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1389999999999995e-06 2024-08-17T18:20:40.001 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.001 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.001 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.001 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.591961+0000", 2024-08-17T18:20:40.001 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2000054520000001 2024-08-17T18:20:40.001 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.001 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.001 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:40.002 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.592080+0000", 2024-08-17T18:20:40.002 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000118069 2024-08-17T18:20:40.002 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.003 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.003 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:20:40.003 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.592195+0000", 2024-08-17T18:20:40.003 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000115257 2024-08-17T18:20:40.003 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.003 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.003 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:20:40.003 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.605372+0000", 2024-08-17T18:20:40.003 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013177082999999999 2024-08-17T18:20:40.003 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.003 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.004 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.004 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.605498+0000", 2024-08-17T18:20:40.004 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012611499999999999 2024-08-17T18:20:40.004 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.004 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.004 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.004 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.615840+0000", 2024-08-17T18:20:40.004 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010341724 2024-08-17T18:20:40.004 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.004 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.004 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:20:40.004 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.615903+0000", 2024-08-17T18:20:40.004 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.3042999999999999e-05 2024-08-17T18:20:40.005 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.005 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.005 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:40.005 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.616058+0000", 2024-08-17T18:20:40.005 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000155499 2024-08-17T18:20:40.005 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.005 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.005 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.005 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.005 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.005 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186470 49.6 49:6754e8f4:::benchmark_data_smithi130_364143_object77336:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1405)", 2024-08-17T18:20:40.005 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.391966+0000", 2024-08-17T18:20:40.005 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 281.55390525400003, 2024-08-17T18:20:40.006 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2243784360000001, 2024-08-17T18:20:40.006 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:40.006 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:40.006 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:20:40.006 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:40.006 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:40.006 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:40.006 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186470 2024-08-17T18:20:40.006 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.006 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:40.006 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.006 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:40.006 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391966+0000", 2024-08-17T18:20:40.007 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.007 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.007 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.007 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:40.007 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391966+0000", 2024-08-17T18:20:40.007 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.007 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.007 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.007 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:40.007 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391968+0000", 2024-08-17T18:20:40.007 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6309999999999999e-06 2024-08-17T18:20:40.007 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.007 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.008 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:40.008 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391972+0000", 2024-08-17T18:20:40.008 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5800000000000002e-06 2024-08-17T18:20:40.008 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.008 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.008 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:40.008 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391984+0000", 2024-08-17T18:20:40.008 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1858000000000001e-05 2024-08-17T18:20:40.008 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.008 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.008 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:40.008 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.391992+0000", 2024-08-17T18:20:40.008 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7880000000000007e-06 2024-08-17T18:20:40.008 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.009 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.009 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.009 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392067+0000", 2024-08-17T18:20:40.009 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4548000000000005e-05 2024-08-17T18:20:40.009 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.009 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.009 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.009 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392075+0000", 2024-08-17T18:20:40.009 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5809999999999997e-06 2024-08-17T18:20:40.009 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.009 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.009 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.009 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.601690+0000", 2024-08-17T18:20:40.009 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2096144600000001 2024-08-17T18:20:40.010 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.010 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.010 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:40.010 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.601820+0000", 2024-08-17T18:20:40.010 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013061400000000001 2024-08-17T18:20:40.010 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.010 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.010 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:20:40.010 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.601969+0000", 2024-08-17T18:20:40.010 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000148877 2024-08-17T18:20:40.010 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.010 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.011 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.011 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.605591+0000", 2024-08-17T18:20:40.011 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0036217329999999998 2024-08-17T18:20:40.011 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.011 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.011 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:20:40.011 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.616070+0000", 2024-08-17T18:20:40.011 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010479265 2024-08-17T18:20:40.011 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.011 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.011 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.011 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.616201+0000", 2024-08-17T18:20:40.011 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000131309 2024-08-17T18:20:40.012 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.012 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.012 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:20:40.012 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.616255+0000", 2024-08-17T18:20:40.012 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3142e-05 2024-08-17T18:20:40.012 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.012 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.012 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:40.012 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.616345+0000", 2024-08-17T18:20:40.012 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.0049999999999993e-05 2024-08-17T18:20:40.012 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.012 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.012 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.012 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.013 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.013 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:186471 49.0 49:00a0737a:::benchmark_data_smithi130_364143_object77337:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1405)", 2024-08-17T18:20:40.013 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:15:58.392014+0000", 2024-08-17T18:20:40.013 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 281.55385739299999, 2024-08-17T18:20:40.013 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0970431060000001, 2024-08-17T18:20:40.013 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:40.013 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:40.013 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:20:40.013 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:40.013 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:40.013 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:40.013 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 186471 2024-08-17T18:20:40.013 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.013 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:40.014 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.014 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:40.014 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392014+0000", 2024-08-17T18:20:40.014 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.014 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.014 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.014 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:40.014 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392014+0000", 2024-08-17T18:20:40.014 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.014 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.014 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.014 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:40.014 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392014+0000", 2024-08-17T18:20:40.014 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4499999999999997e-07 2024-08-17T18:20:40.015 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.015 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.015 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:40.015 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392017+0000", 2024-08-17T18:20:40.015 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0619999999999999e-06 2024-08-17T18:20:40.015 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.015 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.015 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:40.015 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392024+0000", 2024-08-17T18:20:40.015 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.097e-06 2024-08-17T18:20:40.015 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.015 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.015 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:40.015 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392029+0000", 2024-08-17T18:20:40.016 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.2059999999999998e-06 2024-08-17T18:20:40.016 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.016 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.016 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.016 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392323+0000", 2024-08-17T18:20:40.016 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000293904 2024-08-17T18:20:40.016 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.016 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.016 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.016 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:58.392751+0000", 2024-08-17T18:20:40.016 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00042786899999999999 2024-08-17T18:20:40.016 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.016 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.017 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.017 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.482539+0000", 2024-08-17T18:20:40.017 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.089788236 2024-08-17T18:20:40.017 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.017 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.017 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:40.017 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.482662+0000", 2024-08-17T18:20:40.017 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012271100000000001 2024-08-17T18:20:40.017 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.017 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.017 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 7", 2024-08-17T18:20:40.017 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.482794+0000", 2024-08-17T18:20:40.017 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000132566 2024-08-17T18:20:40.018 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.018 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.018 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.018 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.488818+0000", 2024-08-17T18:20:40.018 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0060241189999999997 2024-08-17T18:20:40.018 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.018 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.018 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:20:40.018 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.488832+0000", 2024-08-17T18:20:40.018 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3434999999999999e-05 2024-08-17T18:20:40.018 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.018 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.018 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:20:40.018 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.488970+0000", 2024-08-17T18:20:40.019 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000137978 2024-08-17T18:20:40.019 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.019 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.019 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:40.019 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:15:59.489057+0000", 2024-08-17T18:20:40.019 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.7478000000000005e-05 2024-08-17T18:20:40.019 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.019 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.019 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.019 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.019 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.019 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:209783 49.b 49:d7c6a4cc:::benchmark_data_smithi130_364143_object100649:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1414)", 2024-08-17T18:20:40.019 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:27.561259+0000", 2024-08-17T18:20:40.020 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 252.38461266900001, 2024-08-17T18:20:40.020 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1021654350000001, 2024-08-17T18:20:40.020 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:40.020 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:40.020 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:20:40.020 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:40.020 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:40.020 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:40.020 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 209783 2024-08-17T18:20:40.020 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.020 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:40.020 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.020 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:40.020 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.561259+0000", 2024-08-17T18:20:40.021 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.021 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.021 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.021 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:40.021 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.561259+0000", 2024-08-17T18:20:40.021 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.021 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.021 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.021 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:40.021 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.561260+0000", 2024-08-17T18:20:40.021 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2810000000000001e-06 2024-08-17T18:20:40.021 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.021 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.021 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:40.022 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.561265+0000", 2024-08-17T18:20:40.022 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9470000000000002e-06 2024-08-17T18:20:40.022 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.022 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.022 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:40.022 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.561277+0000", 2024-08-17T18:20:40.022 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2225999999999999e-05 2024-08-17T18:20:40.022 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.022 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.022 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:40.022 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.561284+0000", 2024-08-17T18:20:40.023 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.1609999999999997e-06 2024-08-17T18:20:40.023 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.023 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.023 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.023 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.561368+0000", 2024-08-17T18:20:40.023 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3677000000000005e-05 2024-08-17T18:20:40.023 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.023 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.023 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:40.023 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.561498+0000", 2024-08-17T18:20:40.023 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000129658 2024-08-17T18:20:40.023 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.024 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.024 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,1", 2024-08-17T18:20:40.024 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.561623+0000", 2024-08-17T18:20:40.024 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000125241 2024-08-17T18:20:40.024 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.024 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.024 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:20:40.024 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.571531+0000", 2024-08-17T18:20:40.024 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0099085460000000007 2024-08-17T18:20:40.024 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.024 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.024 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.024 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.571985+0000", 2024-08-17T18:20:40.024 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00045376299999999999 2024-08-17T18:20:40.025 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.025 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.025 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.025 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.579768+0000", 2024-08-17T18:20:40.025 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0077827139999999996 2024-08-17T18:20:40.025 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.025 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.025 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.025 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.579779+0000", 2024-08-17T18:20:40.025 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0699999999999999e-05 2024-08-17T18:20:40.025 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.025 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.025 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.025 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:28.663348+0000", 2024-08-17T18:20:40.026 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0835692020000001 2024-08-17T18:20:40.026 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.026 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.026 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:40.026 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:28.663424+0000", 2024-08-17T18:20:40.026 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.6318999999999999e-05 2024-08-17T18:20:40.026 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.026 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.026 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.026 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.026 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.026 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:209825 49.b 49:d711087f:::benchmark_data_smithi130_364143_object100691:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1415)", 2024-08-17T18:20:40.026 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:27.608015+0000", 2024-08-17T18:20:40.026 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 252.33785605200001, 2024-08-17T18:20:40.027 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.095000526, 2024-08-17T18:20:40.027 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:40.027 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:40.027 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:20:40.027 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:40.027 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:40.027 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:40.027 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 209825 2024-08-17T18:20:40.027 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.027 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:40.027 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.027 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:40.027 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.608015+0000", 2024-08-17T18:20:40.027 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.028 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.028 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.028 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:40.028 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.608015+0000", 2024-08-17T18:20:40.028 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.028 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.028 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.028 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:40.028 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.608016+0000", 2024-08-17T18:20:40.028 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.71e-07 2024-08-17T18:20:40.028 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.028 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.028 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:40.028 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.608020+0000", 2024-08-17T18:20:40.029 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.172e-06 2024-08-17T18:20:40.029 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.029 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.029 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:40.029 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.608034+0000", 2024-08-17T18:20:40.029 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3562e-05 2024-08-17T18:20:40.029 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.029 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.029 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:40.029 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.608040+0000", 2024-08-17T18:20:40.029 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4160000000000001e-06 2024-08-17T18:20:40.029 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.029 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.029 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.030 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.608085+0000", 2024-08-17T18:20:40.030 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.4388000000000003e-05 2024-08-17T18:20:40.030 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.030 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.030 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.030 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:27.608092+0000", 2024-08-17T18:20:40.030 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.9820000000000002e-06 2024-08-17T18:20:40.030 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.030 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.030 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.030 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:28.680747+0000", 2024-08-17T18:20:40.030 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0726550880000001 2024-08-17T18:20:40.030 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.030 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.031 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:40.031 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:28.680868+0000", 2024-08-17T18:20:40.031 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012140400000000001 2024-08-17T18:20:40.031 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.031 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.031 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,1", 2024-08-17T18:20:40.031 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:28.681006+0000", 2024-08-17T18:20:40.031 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013726100000000001 2024-08-17T18:20:40.031 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.031 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.031 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:20:40.031 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:28.692086+0000", 2024-08-17T18:20:40.031 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011080237 2024-08-17T18:20:40.032 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.032 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.032 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.032 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:28.692175+0000", 2024-08-17T18:20:40.032 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9013999999999999e-05 2024-08-17T18:20:40.032 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.032 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.032 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.032 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:28.702861+0000", 2024-08-17T18:20:40.032 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010686553 2024-08-17T18:20:40.032 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.032 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.032 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:20:40.032 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:28.702913+0000", 2024-08-17T18:20:40.033 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1198e-05 2024-08-17T18:20:40.033 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.033 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.033 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:40.033 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:28.703016+0000", 2024-08-17T18:20:40.033 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00010338 2024-08-17T18:20:40.033 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.033 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.033 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.033 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.033 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.033 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:211984 49.c 49:385517eb:::benchmark_data_smithi130_364143_object102850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1420)", 2024-08-17T18:20:40.033 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:33.662067+0000", 2024-08-17T18:20:40.033 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 246.283804726, 2024-08-17T18:20:40.034 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1599780079999999, 2024-08-17T18:20:40.034 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:40.034 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:40.034 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:20:40.034 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:40.034 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:40.034 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:40.034 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 211984 2024-08-17T18:20:40.034 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.034 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:40.035 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.035 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:40.035 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.662067+0000", 2024-08-17T18:20:40.035 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.035 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.035 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.035 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:40.035 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.662067+0000", 2024-08-17T18:20:40.035 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.035 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.035 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.035 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:40.035 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.662068+0000", 2024-08-17T18:20:40.036 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.525e-06 2024-08-17T18:20:40.036 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.036 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.036 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:40.036 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.662072+0000", 2024-08-17T18:20:40.036 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8619999999999996e-06 2024-08-17T18:20:40.036 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.036 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.036 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:40.036 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.662080+0000", 2024-08-17T18:20:40.036 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.0970000000000006e-06 2024-08-17T18:20:40.037 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.037 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.037 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:40.037 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.662091+0000", 2024-08-17T18:20:40.037 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1002000000000001e-05 2024-08-17T18:20:40.037 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.037 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.037 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.037 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.662145+0000", 2024-08-17T18:20:40.037 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.4311999999999998e-05 2024-08-17T18:20:40.037 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.037 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.037 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.038 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.662153+0000", 2024-08-17T18:20:40.038 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7239999999999992e-06 2024-08-17T18:20:40.038 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.038 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.038 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.038 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.634385+0000", 2024-08-17T18:20:40.038 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.97223178200000004 2024-08-17T18:20:40.038 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.038 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.038 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.038 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.634400+0000", 2024-08-17T18:20:40.038 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4586000000000001e-05 2024-08-17T18:20:40.039 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.039 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.039 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.039 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.810606+0000", 2024-08-17T18:20:40.039 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.176205935 2024-08-17T18:20:40.039 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.039 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.039 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:40.039 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.810708+0000", 2024-08-17T18:20:40.039 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000102917 2024-08-17T18:20:40.039 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.040 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.040 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T18:20:40.040 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.810821+0000", 2024-08-17T18:20:40.040 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011222600000000001 2024-08-17T18:20:40.040 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.040 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.040 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.040 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.814112+0000", 2024-08-17T18:20:40.040 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032909039999999999 2024-08-17T18:20:40.040 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.040 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.040 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.040 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.821731+0000", 2024-08-17T18:20:40.041 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0076196279999999998 2024-08-17T18:20:40.041 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.041 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.041 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:20:40.041 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.821879+0000", 2024-08-17T18:20:40.041 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00014813399999999999 2024-08-17T18:20:40.041 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.041 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.041 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:20:40.041 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.821936+0000", 2024-08-17T18:20:40.041 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.6450000000000003e-05 2024-08-17T18:20:40.041 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.041 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.041 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:40.042 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.822045+0000", 2024-08-17T18:20:40.042 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000108924 2024-08-17T18:20:40.042 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.042 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.042 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.042 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.042 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.042 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:211988 49.c 49:3cb744cb:::benchmark_data_smithi130_364143_object102854:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1421)", 2024-08-17T18:20:40.042 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:33.665208+0000", 2024-08-17T18:20:40.042 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 246.28066391600001, 2024-08-17T18:20:40.042 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1577635640000001, 2024-08-17T18:20:40.042 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:40.042 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:40.043 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:20:40.043 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:40.043 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:40.043 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:40.043 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 211988 2024-08-17T18:20:40.043 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.043 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:40.043 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.043 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:40.043 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.665208+0000", 2024-08-17T18:20:40.043 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.043 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.043 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.044 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:40.044 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.665208+0000", 2024-08-17T18:20:40.044 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.044 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.044 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.044 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:40.044 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.665209+0000", 2024-08-17T18:20:40.044 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.048e-06 2024-08-17T18:20:40.044 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.044 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.044 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:40.044 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.665212+0000", 2024-08-17T18:20:40.044 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.9620000000000002e-06 2024-08-17T18:20:40.044 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.045 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.045 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:40.045 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.665219+0000", 2024-08-17T18:20:40.045 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5959999999999998e-06 2024-08-17T18:20:40.045 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.045 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.045 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:40.045 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.665226+0000", 2024-08-17T18:20:40.045 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3209999999999999e-06 2024-08-17T18:20:40.045 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.045 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.045 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.045 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.665510+0000", 2024-08-17T18:20:40.045 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00028342800000000001 2024-08-17T18:20:40.046 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.046 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.046 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.046 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.665517+0000", 2024-08-17T18:20:40.046 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2830000000000004e-06 2024-08-17T18:20:40.046 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:39 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:20:40.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:39 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:20:40.046 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:39 smithi130 ceph-mon[105521]: osdmap e1467: 8 total, 8 up, 7 in 2024-08-17T18:20:40.047 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:20:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:20:39] "GET /metrics HTTP/1.1" 200 36812 "" "Prometheus/2.51.0" 2024-08-17T18:20:40.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:39 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:20:40.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:39 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:20:40.047 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:39 smithi130 ceph-mon[110068]: osdmap e1467: 8 total, 8 up, 7 in 2024-08-17T18:20:40.048 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.048 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.048 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.634433+0000", 2024-08-17T18:20:40.048 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.96891625100000001 2024-08-17T18:20:40.048 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.048 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.048 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.049 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.634441+0000", 2024-08-17T18:20:40.049 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7579999999999992e-06 2024-08-17T18:20:40.049 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.049 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.049 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.049 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.814146+0000", 2024-08-17T18:20:40.049 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1797045239999999 2024-08-17T18:20:40.049 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.049 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.049 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:40.049 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.814262+0000", 2024-08-17T18:20:40.049 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011624 2024-08-17T18:20:40.049 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.049 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.050 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,6", 2024-08-17T18:20:40.050 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.814397+0000", 2024-08-17T18:20:40.050 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000134652 2024-08-17T18:20:40.050 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.050 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.050 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.050 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.821803+0000", 2024-08-17T18:20:40.050 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0074067050000000004 2024-08-17T18:20:40.050 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.050 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.050 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:20:40.050 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.822054+0000", 2024-08-17T18:20:40.050 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00025028700000000002 2024-08-17T18:20:40.051 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.051 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.051 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.051 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.822092+0000", 2024-08-17T18:20:40.051 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8549e-05 2024-08-17T18:20:40.051 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.051 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.051 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:20:40.051 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.822857+0000", 2024-08-17T18:20:40.051 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00076526600000000001 2024-08-17T18:20:40.051 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.051 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.051 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:40.051 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.822971+0000", 2024-08-17T18:20:40.052 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000113694 2024-08-17T18:20:40.052 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.052 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.052 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.052 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.052 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.052 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:211989 49.6 49:699887fc:::benchmark_data_smithi130_364143_object102855:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1421)", 2024-08-17T18:20:40.052 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:33.669668+0000", 2024-08-17T18:20:40.052 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 246.276203178, 2024-08-17T18:20:40.052 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1215277750000001, 2024-08-17T18:20:40.052 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:40.052 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:40.052 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:20:40.053 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:40.053 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:40.053 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:40.053 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 211989 2024-08-17T18:20:40.053 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.053 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:40.053 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.053 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:40.053 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.669668+0000", 2024-08-17T18:20:40.053 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.053 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.053 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.053 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.669668+0000", 2024-08-17T18:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.669670+0000", 2024-08-17T18:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5209999999999999e-06 2024-08-17T18:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.669675+0000", 2024-08-17T18:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.181e-06 2024-08-17T18:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.054 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.055 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:40.055 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.669689+0000", 2024-08-17T18:20:40.055 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3664e-05 2024-08-17T18:20:40.055 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.055 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.055 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:40.055 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.669697+0000", 2024-08-17T18:20:40.055 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1049999999999995e-06 2024-08-17T18:20:40.055 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.055 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.055 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.055 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.669756+0000", 2024-08-17T18:20:40.055 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9704000000000003e-05 2024-08-17T18:20:40.056 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.056 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.056 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.056 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:33.669768+0000", 2024-08-17T18:20:40.056 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.128e-05 2024-08-17T18:20:40.058 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.058 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.058 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.058 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.636418+0000", 2024-08-17T18:20:40.058 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.966649808 2024-08-17T18:20:40.058 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.058 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.058 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.058 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.636430+0000", 2024-08-17T18:20:40.058 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2088e-05 2024-08-17T18:20:40.058 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.058 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.058 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.059 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.786588+0000", 2024-08-17T18:20:40.059 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1501584659999999 2024-08-17T18:20:40.059 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.059 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.059 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:40.059 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.786716+0000", 2024-08-17T18:20:40.059 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012789600000000001 2024-08-17T18:20:40.059 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.059 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.059 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 1,6,7", 2024-08-17T18:20:40.059 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.786933+0000", 2024-08-17T18:20:40.059 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00021723900000000001 2024-08-17T18:20:40.059 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.060 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.060 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.060 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.790696+0000", 2024-08-17T18:20:40.060 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0037626389999999999 2024-08-17T18:20:40.060 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.060 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.060 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.060 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.790813+0000", 2024-08-17T18:20:40.060 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00011726 2024-08-17T18:20:40.060 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.060 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.060 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.060 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.791003+0000", 2024-08-17T18:20:40.061 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000189791 2024-08-17T18:20:40.061 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.061 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.061 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:20:40.061 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.791049+0000", 2024-08-17T18:20:40.061 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6131999999999998e-05 2024-08-17T18:20:40.061 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.061 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.061 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:20:40.061 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.791095+0000", 2024-08-17T18:20:40.061 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.6e-05 2024-08-17T18:20:40.061 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.061 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.061 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:40.062 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.791196+0000", 2024-08-17T18:20:40.062 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000101001 2024-08-17T18:20:40.062 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.062 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.062 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.062 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.062 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.062 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:211976 49.0 49:0a6084d7:::benchmark_data_smithi130_364143_object102842:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1422)", 2024-08-17T18:20:40.062 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:34.639979+0000", 2024-08-17T18:20:40.062 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 245.305892774, 2024-08-17T18:20:40.062 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1641510340000001, 2024-08-17T18:20:40.062 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:40.062 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:20:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 211976 2024-08-17T18:20:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.639979+0000", 2024-08-17T18:20:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.063 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:40.064 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.639979+0000", 2024-08-17T18:20:40.064 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.064 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.064 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.064 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:40.064 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.639980+0000", 2024-08-17T18:20:40.064 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.593e-06 2024-08-17T18:20:40.064 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.064 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.064 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:40.064 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.639986+0000", 2024-08-17T18:20:40.064 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0480000000000004e-06 2024-08-17T18:20:40.064 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.065 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.065 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:40.065 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.639997+0000", 2024-08-17T18:20:40.065 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0322e-05 2024-08-17T18:20:40.065 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.065 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.065 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:40.065 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.640008+0000", 2024-08-17T18:20:40.065 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1739000000000001e-05 2024-08-17T18:20:40.065 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.065 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.065 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.065 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.640081+0000", 2024-08-17T18:20:40.066 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2158999999999998e-05 2024-08-17T18:20:40.066 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.066 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.066 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.066 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.640092+0000", 2024-08-17T18:20:40.066 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1481000000000001e-05 2024-08-17T18:20:40.066 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.066 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.066 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.066 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.804040+0000", 2024-08-17T18:20:40.066 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1639479429999999 2024-08-17T18:20:40.066 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.066 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.067 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:40.067 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.804130+0000", 2024-08-17T18:20:40.067 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9748999999999997e-05 2024-08-17T18:20:40.067 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.067 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.067 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.067 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.067 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.067 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.36960.0:211995 49.0 49:0affcf91:::benchmark_data_smithi130_364143_object102861:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1422)", 2024-08-17T18:20:40.067 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:16:34.640038+0000", 2024-08-17T18:20:40.067 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 245.305833573, 2024-08-17T18:20:40.067 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1799087130000001, 2024-08-17T18:20:40.067 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:40.068 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:40.068 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:20:40.068 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:40.068 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.36960", 2024-08-17T18:20:40.068 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:20:40.068 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 211995 2024-08-17T18:20:40.068 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.068 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:40.068 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.068 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:40.068 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.640038+0000", 2024-08-17T18:20:40.068 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.068 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.069 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.069 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:40.069 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.640038+0000", 2024-08-17T18:20:40.069 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.069 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.069 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.069 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:40.069 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.640039+0000", 2024-08-17T18:20:40.069 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5000000000000002e-07 2024-08-17T18:20:40.069 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.069 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.069 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:40.069 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.640042+0000", 2024-08-17T18:20:40.070 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.6899999999999998e-06 2024-08-17T18:20:40.070 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.070 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.070 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:40.070 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.640051+0000", 2024-08-17T18:20:40.070 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.2209999999999997e-06 2024-08-17T18:20:40.070 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.070 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.070 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:40.070 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.640058+0000", 2024-08-17T18:20:40.070 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3050000000000004e-06 2024-08-17T18:20:40.070 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.070 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.071 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.071 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.640131+0000", 2024-08-17T18:20:40.071 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.3305000000000004e-05 2024-08-17T18:20:40.071 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.071 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.071 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.071 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:34.640139+0000", 2024-08-17T18:20:40.071 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.8180000000000004e-06 2024-08-17T18:20:40.071 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.071 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.071 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.071 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.804326+0000", 2024-08-17T18:20:40.071 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1641875349999999 2024-08-17T18:20:40.071 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.804429+0000", 2024-08-17T18:20:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000102885 2024-08-17T18:20:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 6,7", 2024-08-17T18:20:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.804547+0000", 2024-08-17T18:20:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000117237 2024-08-17T18:20:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.809924+0000", 2024-08-17T18:20:40.072 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0053773149999999997 2024-08-17T18:20:40.073 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.073 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.073 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.073 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.809963+0000", 2024-08-17T18:20:40.073 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8680000000000002e-05 2024-08-17T18:20:40.073 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.073 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.073 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:20:40.073 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.819718+0000", 2024-08-17T18:20:40.073 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0097557939999999999 2024-08-17T18:20:40.073 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.073 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.073 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:20:40.073 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.819818+0000", 2024-08-17T18:20:40.074 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.9924000000000001e-05 2024-08-17T18:20:40.074 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.074 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.074 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:40.074 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:16:35.819947+0000", 2024-08-17T18:20:40.074 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012825400000000001 2024-08-17T18:20:40.074 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.074 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.074 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.074 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.074 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.074 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:171243 50.13 50:ca2d4bf7:::benchmark_data_smithi130_398002_object26177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1453)", 2024-08-17T18:20:40.074 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:25.018994+0000", 2024-08-17T18:20:40.075 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 74.926877684000004, 2024-08-17T18:20:40.075 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.2328543710000002, 2024-08-17T18:20:40.075 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:40.075 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:40.075 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "reached pg", 2024-08-17T18:20:40.075 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:40.075 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:20:40.075 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:20:40.075 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 171243 2024-08-17T18:20:40.075 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.075 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:40.075 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.075 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:40.076 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.018994+0000", 2024-08-17T18:20:40.076 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.076 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.076 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.076 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:40.076 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.018994+0000", 2024-08-17T18:20:40.076 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.076 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.076 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.076 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:40.076 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.018995+0000", 2024-08-17T18:20:40.076 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.147e-06 2024-08-17T18:20:40.076 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.076 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.077 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:40.077 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.018999+0000", 2024-08-17T18:20:40.077 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.8739999999999997e-06 2024-08-17T18:20:40.077 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.077 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.077 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:40.077 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.019012+0000", 2024-08-17T18:20:40.077 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3733999999999999e-05 2024-08-17T18:20:40.077 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.077 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.077 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:40.077 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.019021+0000", 2024-08-17T18:20:40.077 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.3820000000000004e-06 2024-08-17T18:20:40.077 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.078 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.078 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.078 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.019905+0000", 2024-08-17T18:20:40.078 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00088383299999999999 2024-08-17T18:20:40.078 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.078 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.078 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:40.078 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.020034+0000", 2024-08-17T18:20:40.078 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00012950799999999999 2024-08-17T18:20:40.078 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.078 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.078 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 6", 2024-08-17T18:20:40.078 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.020499+0000", 2024-08-17T18:20:40.079 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00046465500000000002 2024-08-17T18:20:40.079 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.079 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.079 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.079 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.039720+0000", 2024-08-17T18:20:40.079 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019220911 2024-08-17T18:20:40.079 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.079 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.079 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.079 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.039732+0000", 2024-08-17T18:20:40.079 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2303999999999999e-05 2024-08-17T18:20:40.079 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.079 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.080 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.080 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.028360+0000", 2024-08-17T18:20:40.080 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.98862822299999997 2024-08-17T18:20:40.080 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.080 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.080 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.080 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.028368+0000", 2024-08-17T18:20:40.080 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1380000000000007e-06 2024-08-17T18:20:40.080 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.080 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.080 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.080 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.251734+0000", 2024-08-17T18:20:40.080 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2233653979999999 2024-08-17T18:20:40.080 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.081 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.081 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:40.081 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.251848+0000", 2024-08-17T18:20:40.081 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000114264 2024-08-17T18:20:40.081 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.081 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.081 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.081 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.081 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.081 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:171264 50.3 50:c1ed1595:::benchmark_data_smithi130_398002_object26198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1454)", 2024-08-17T18:20:40.081 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:25.064232+0000", 2024-08-17T18:20:40.081 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 74.881639332000006, 2024-08-17T18:20:40.081 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.203966581, 2024-08-17T18:20:40.082 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:40.082 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:40.082 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:20:40.082 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:40.082 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:20:40.082 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:20:40.082 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 171264 2024-08-17T18:20:40.082 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.082 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:40.082 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.082 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:40.082 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.064232+0000", 2024-08-17T18:20:40.082 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.083 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.083 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.083 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:40.083 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.064232+0000", 2024-08-17T18:20:40.083 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.083 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.083 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.083 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:40.083 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.064233+0000", 2024-08-17T18:20:40.083 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.235e-06 2024-08-17T18:20:40.083 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.083 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.083 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:40.083 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.064238+0000", 2024-08-17T18:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.3819999999999997e-06 2024-08-17T18:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.064248+0000", 2024-08-17T18:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0494e-05 2024-08-17T18:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.064255+0000", 2024-08-17T18:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5030000000000002e-06 2024-08-17T18:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.084 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.064315+0000", 2024-08-17T18:20:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0105000000000001e-05 2024-08-17T18:20:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.064322+0000", 2024-08-17T18:20:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2119999999999997e-06 2024-08-17T18:20:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.028320+0000", 2024-08-17T18:20:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.96399826600000005 2024-08-17T18:20:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.085 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.086 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.086 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.028331+0000", 2024-08-17T18:20:40.086 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0292000000000001e-05 2024-08-17T18:20:40.086 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.086 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.086 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.086 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.262647+0000", 2024-08-17T18:20:40.086 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2343165700000001 2024-08-17T18:20:40.086 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.086 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.086 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:40.086 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.262805+0000", 2024-08-17T18:20:40.086 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00015826699999999999 2024-08-17T18:20:40.087 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.087 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.087 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:20:40.087 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.263351+0000", 2024-08-17T18:20:40.087 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00054586700000000001 2024-08-17T18:20:40.087 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.087 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.087 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:20:40.087 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.266945+0000", 2024-08-17T18:20:40.087 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0035932849999999999 2024-08-17T18:20:40.087 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.087 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.087 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.087 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.266979+0000", 2024-08-17T18:20:40.088 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.4876999999999998e-05 2024-08-17T18:20:40.088 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.088 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.088 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.088 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.268007+0000", 2024-08-17T18:20:40.088 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0010276230000000001 2024-08-17T18:20:40.088 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.088 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.088 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:20:40.088 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.268109+0000", 2024-08-17T18:20:40.088 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000102015 2024-08-17T18:20:40.088 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.088 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.088 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:40.089 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.268199+0000", 2024-08-17T18:20:40.089 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.9587999999999997e-05 2024-08-17T18:20:40.089 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.089 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.089 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.089 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.089 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.089 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:171273 50.14 50:2f449355:::benchmark_data_smithi130_398002_object26207:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1454)", 2024-08-17T18:20:40.089 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:25.079617+0000", 2024-08-17T18:20:40.089 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 74.866254670000004, 2024-08-17T18:20:40.089 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.1879477340000002, 2024-08-17T18:20:40.089 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:40.089 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:40.089 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:20:40.090 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:40.090 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:20:40.090 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:20:40.090 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 171273 2024-08-17T18:20:40.090 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.090 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:40.090 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.090 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:40.090 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.079617+0000", 2024-08-17T18:20:40.090 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.090 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.090 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.090 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:40.090 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.079617+0000", 2024-08-17T18:20:40.091 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.091 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.091 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.091 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:40.091 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.079618+0000", 2024-08-17T18:20:40.091 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.141e-06 2024-08-17T18:20:40.091 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.091 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.091 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:40.091 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.079622+0000", 2024-08-17T18:20:40.091 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1359999999999999e-06 2024-08-17T18:20:40.091 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.091 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.091 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:40.092 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.079632+0000", 2024-08-17T18:20:40.092 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0373e-05 2024-08-17T18:20:40.092 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.092 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.092 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:40.092 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.079638+0000", 2024-08-17T18:20:40.092 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.3279999999999996e-06 2024-08-17T18:20:40.092 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.092 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.092 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.092 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.079696+0000", 2024-08-17T18:20:40.092 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8331000000000001e-05 2024-08-17T18:20:40.093 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.093 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.093 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.093 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:25.079705+0000", 2024-08-17T18:20:40.093 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.3880000000000006e-06 2024-08-17T18:20:40.093 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.093 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.093 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.093 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.024735+0000", 2024-08-17T18:20:40.093 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.94503001799999997 2024-08-17T18:20:40.093 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.093 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.093 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.093 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.024749+0000", 2024-08-17T18:20:40.094 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3657e-05 2024-08-17T18:20:40.094 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.094 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.094 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.094 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.263567+0000", 2024-08-17T18:20:40.094 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2388181410000001 2024-08-17T18:20:40.094 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.094 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.094 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:40.094 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.263706+0000", 2024-08-17T18:20:40.094 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00013868800000000001 2024-08-17T18:20:40.094 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.094 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:20:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.264186+0000", 2024-08-17T18:20:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00048022000000000001 2024-08-17T18:20:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:20:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.266756+0000", 2024-08-17T18:20:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025697210000000001 2024-08-17T18:20:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.266854+0000", 2024-08-17T18:20:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7714999999999995e-05 2024-08-17T18:20:40.095 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.096 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.096 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.096 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.267408+0000", 2024-08-17T18:20:40.096 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00055436899999999998 2024-08-17T18:20:40.096 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.096 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.096 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:20:40.096 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.267467+0000", 2024-08-17T18:20:40.096 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9443e-05 2024-08-17T18:20:40.096 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.096 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.096 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:40.096 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.267564+0000", 2024-08-17T18:20:40.097 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.7065000000000007e-05 2024-08-17T18:20:40.097 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.097 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.097 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.097 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.097 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.097 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:171277 50.13 50:cc6141c1:::benchmark_data_smithi130_398002_object26211:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1455)", 2024-08-17T18:20:40.097 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:19:26.140572+0000", 2024-08-17T18:20:40.097 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 73.805299868999995, 2024-08-17T18:20:40.097 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.127395479, 2024-08-17T18:20:40.097 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:20:40.097 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:20:40.097 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:20:40.098 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:20:40.098 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:20:40.098 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:20:40.098 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 171277 2024-08-17T18:20:40.098 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.098 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:20:40.098 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.098 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:20:40.098 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.140572+0000", 2024-08-17T18:20:40.098 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.098 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.098 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.098 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:20:40.098 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.140572+0000", 2024-08-17T18:20:40.099 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:20:40.099 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.099 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.099 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:20:40.099 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.140573+0000", 2024-08-17T18:20:40.099 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.294e-06 2024-08-17T18:20:40.099 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.099 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.099 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:20:40.099 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.140578+0000", 2024-08-17T18:20:40.099 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9200000000000003e-06 2024-08-17T18:20:40.099 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.099 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.099 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:20:40.100 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.140589+0000", 2024-08-17T18:20:40.100 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1688999999999999e-05 2024-08-17T18:20:40.100 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.100 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.100 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:20:40.100 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.140596+0000", 2024-08-17T18:20:40.100 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0610000000000001e-06 2024-08-17T18:20:40.100 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.100 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.100 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.100 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.140668+0000", 2024-08-17T18:20:40.100 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2387999999999999e-05 2024-08-17T18:20:40.100 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.100 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for peered", 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:26.140681+0000", 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.2850999999999999e-05 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.252456+0000", 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.111775733 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.252590+0000", 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000133795 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.101 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.102 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 2,6", 2024-08-17T18:20:40.102 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.253882+0000", 2024-08-17T18:20:40.102 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001291691 2024-08-17T18:20:40.102 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.102 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.102 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:20:40.102 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.266694+0000", 2024-08-17T18:20:40.102 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.012812338 2024-08-17T18:20:40.102 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.102 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.102 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.102 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.266734+0000", 2024-08-17T18:20:40.102 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 3.9409000000000002e-05 2024-08-17T18:20:40.102 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.267733+0000", 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0009993370000000001 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.267867+0000", 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000133609 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:19:27.267967+0000", 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000100364 2024-08-17T18:20:40.103 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.104 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.104 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.104 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:20:40.104 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:20:40.104 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:20:40.104 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:40.105 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:39 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:20:40.105 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:39 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:20:40.105 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:39 smithi179 ceph-mon[93519]: osdmap e1467: 8 total, 8 up, 7 in 2024-08-17T18:20:40.143 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:40.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:40.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:40.245 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:40.277 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:40.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:40.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:40.379 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:40.417 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:40.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:40.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:40.518 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:40.554 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:40.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:40.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:20:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:20:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:20:40.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:40.688 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:40.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:40.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:40.790 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:40.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:40.836 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:40.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:40.938 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:40.981 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:40.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:40.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:41.083 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:41.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:40 smithi179 ceph-mon[93519]: pgmap v2849: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 812 op/s; 26573/179892 objects degraded (14.772%); 58044/179892 objects misplaced (32.266%); 9.9 MiB/s, 281 objects/s recovering 2024-08-17T18:20:41.102 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:40 smithi130 ceph-mon[105521]: pgmap v2849: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 812 op/s; 26573/179892 objects degraded (14.772%); 58044/179892 objects misplaced (32.266%); 9.9 MiB/s, 281 objects/s recovering 2024-08-17T18:20:41.104 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:40 smithi130 ceph-mon[110068]: pgmap v2849: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 812 op/s; 26573/179892 objects degraded (14.772%); 58044/179892 objects misplaced (32.266%); 9.9 MiB/s, 281 objects/s recovering 2024-08-17T18:20:41.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:41.126 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:41.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:41.227 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:41.266 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:41.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:41.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:41.368 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:41.405 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:41.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:41.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:41.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:41.546 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:41.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:41.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:41.648 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:41.676 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:41.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:41.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:41.691 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T18:20:41.778 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:41.828 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:41.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:41.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:41.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:41 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 26573/179892 objects degraded (14.772%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:41.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:41 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 26573/179892 objects degraded (14.772%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:41.929 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:41.985 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:41.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:41.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:42.087 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:42.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:41 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 26573/179892 objects degraded (14.772%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:42.116 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:42.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:42.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:42.218 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:42.271 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:42.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:42.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:42.376 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:42.421 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:42.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:42.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:42.527 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:42.558 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:42.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:42.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:42.659 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:42.691 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:42.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:42.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:42.793 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:42.817 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:20:42.823 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:42.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:42.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:42.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:42 smithi130 ceph-mon[105521]: pgmap v2850: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 26573/179892 objects degraded (14.772%); 58044/179892 objects misplaced (32.266%); 8.1 MiB/s, 230 objects/s recovering 2024-08-17T18:20:42.918 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:20:42.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:42 smithi130 ceph-mon[110068]: pgmap v2850: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 26573/179892 objects degraded (14.772%); 58044/179892 objects misplaced (32.266%); 8.1 MiB/s, 230 objects/s recovering 2024-08-17T18:20:42.919 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:20:42.924 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:42.967 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:42.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:42.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:43.069 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:43.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:42 smithi179 ceph-mon[93519]: pgmap v2850: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 2 active+recovery_wait+degraded+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 26573/179892 objects degraded (14.772%); 58044/179892 objects misplaced (32.266%); 8.1 MiB/s, 230 objects/s recovering 2024-08-17T18:20:43.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:20:43.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:43.120 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:43.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:43.221 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:43.255 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:43.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:43.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:43.356 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:43.385 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:43.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:43.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:43.488 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:43.515 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:43.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:43.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:43.618 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:43.673 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:43.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:43.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:43.775 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:43.820 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:43.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:43.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:43.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:43.961 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:43.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:43.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:44.064 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:44.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:44.095 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:44.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:44.196 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:44.232 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:20:44.240 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:44.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:44.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:44.342 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:44.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:44.369 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:44.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:44.381 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:20:44.470 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:44.684 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:20:44.684 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:20:44.685 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:20:44.685 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:20:44.685 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:20:44.686 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:20:44.686 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:20:44.686 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:20:44.698 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:20:44.698 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:20:44.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:44.739 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:44.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:44.739 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:20:44.740 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:20:44.770 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:20:44.771 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:20:44.802 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:20:44.803 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:20:44.840 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:44.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:44.871 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:44.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:44 smithi130 ceph-mon[105521]: pgmap v2851: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 24980/168807 objects degraded (14.798%); 54061/168807 objects misplaced (32.025%); 4.1 MiB/s, 112 objects/s recovering 2024-08-17T18:20:44.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:44 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2521382771' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:20:44.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:44 smithi130 ceph-mon[110068]: pgmap v2851: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 24980/168807 objects degraded (14.798%); 54061/168807 objects misplaced (32.025%); 4.1 MiB/s, 112 objects/s recovering 2024-08-17T18:20:44.918 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:44 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2521382771' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:20:44.972 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:45.013 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:45.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:45.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:45.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:44 smithi179 ceph-mon[93519]: pgmap v2851: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 24980/168807 objects degraded (14.798%); 54061/168807 objects misplaced (32.025%); 4.1 MiB/s, 112 objects/s recovering 2024-08-17T18:20:45.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:44 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2521382771' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:20:45.115 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:45.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:45.152 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:45.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:45.253 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:45.297 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:45.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:45.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:45.399 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:45.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:45.429 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:45.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:45.530 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:45.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:45.569 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:45.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:45.670 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:45.695 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:45.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:45.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:45.797 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:45.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:45.838 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:45.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:45.940 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:45.976 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:45.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:45.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:46.078 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:46.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:46.116 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:46.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:46.217 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:46.261 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:46.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:46.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:46.363 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:46.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:46.400 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:46.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:46.501 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:46.553 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:46.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:46.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:46.655 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:46.690 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:46.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:46.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:46.792 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:46.835 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:46.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:46.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:46.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:46 smithi130 ceph-mon[105521]: pgmap v2852: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 24980/168807 objects degraded (14.798%); 54061/168807 objects misplaced (32.025%); 4.1 MiB/s, 112 objects/s recovering 2024-08-17T18:20:46.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 24980/168807 objects degraded (14.798%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:46 smithi130 ceph-mon[110068]: pgmap v2852: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 24980/168807 objects degraded (14.798%); 54061/168807 objects misplaced (32.025%); 4.1 MiB/s, 112 objects/s recovering 2024-08-17T18:20:46.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 24980/168807 objects degraded (14.798%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:46.937 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:46.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:46.977 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:46.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:47.078 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:47.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:46 smithi179 ceph-mon[93519]: pgmap v2852: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 778 op/s; 24980/168807 objects degraded (14.798%); 54061/168807 objects misplaced (32.025%); 4.1 MiB/s, 112 objects/s recovering 2024-08-17T18:20:47.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 24980/168807 objects degraded (14.798%), 15 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:47.120 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:47.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:47.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:47.222 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:47.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:47.260 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:47.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:47.361 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:47.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:47.398 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:47.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:47.500 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:47.539 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:47.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:47.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:47.641 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:47.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:47.698 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:47.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:47.799 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:47.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:47.844 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:47.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:47.945 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:47.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:47.982 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:47.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:48.083 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:48.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:48.119 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:48.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:48.220 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:48.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:48.266 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:48.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:48.368 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:48.399 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:48.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:48.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:48.501 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:48.560 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:48.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:48.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:48.662 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:48.692 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:48.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:48.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:48.793 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:48.831 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:48.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:48.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:48.933 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:48.960 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:48.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:48.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:48.969 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:48 smithi130 ceph-mon[105521]: pgmap v2853: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 24732/166644 objects degraded (14.841%); 52945/166644 objects misplaced (31.771%); 4.1 MiB/s, 112 objects/s recovering 2024-08-17T18:20:48.969 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:48 smithi130 ceph-mon[110068]: pgmap v2853: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 24732/166644 objects degraded (14.841%); 52945/166644 objects misplaced (31.771%); 4.1 MiB/s, 112 objects/s recovering 2024-08-17T18:20:49.063 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:49.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:48 smithi179 ceph-mon[93519]: pgmap v2853: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 769 op/s; 24732/166644 objects degraded (14.841%); 52945/166644 objects misplaced (31.771%); 4.1 MiB/s, 112 objects/s recovering 2024-08-17T18:20:49.099 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:49.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:49.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:49.201 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:49.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:49.252 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:49.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:49.353 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:49.367 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:20:49.367 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:20:49.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:49.384 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:49.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:49.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:49.522 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:49.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:49.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:49.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:20:49.606 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:20:49.624 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:49.704 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:49.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:49.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:49.807 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:49.868 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:49.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:49.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:49.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:20:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:20:49] "GET /metrics HTTP/1.1" 200 36821 "" "Prometheus/2.51.0" 2024-08-17T18:20:49.971 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:50.011 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:50.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:50.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:50.113 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:50.154 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:50.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:50.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:50.256 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:50.284 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:50.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:50.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:50.309 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T18:20:50.385 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:50.430 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:50.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:50.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:50.532 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:50.589 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:50.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:50.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:50.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:20:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:20:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:20:50.691 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:50.728 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:50.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:50.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:50.830 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:50.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:50.878 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:50.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:50.979 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:51.018 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:51.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:51.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:51.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:50 smithi179 ceph-mon[93519]: pgmap v2854: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 23011/157014 objects degraded (14.655%); 50252/157014 objects misplaced (32.005%); 7.1 MiB/s, 122 objects/s recovering 2024-08-17T18:20:51.121 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:51.155 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:51.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:51.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:51.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:50 smithi130 ceph-mon[105521]: pgmap v2854: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 23011/157014 objects degraded (14.655%); 50252/157014 objects misplaced (32.005%); 7.1 MiB/s, 122 objects/s recovering 2024-08-17T18:20:51.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:50 smithi130 ceph-mon[110068]: pgmap v2854: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 23011/157014 objects degraded (14.655%); 50252/157014 objects misplaced (32.005%); 7.1 MiB/s, 122 objects/s recovering 2024-08-17T18:20:51.257 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:51.297 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:51.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:51.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:51.399 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:51.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:51.425 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:51.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:51.526 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:51.572 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:51.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:51.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:51.674 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:51.705 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:51.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:51.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:51.807 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:51.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:51.852 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:51.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:51.953 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:51.994 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:51.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:51.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:52.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 23011/157014 objects degraded (14.655%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:52.096 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:52.132 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:52.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:52.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:52.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 23011/157014 objects degraded (14.655%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:52.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 23011/157014 objects degraded (14.655%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:52.234 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:52.269 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:52.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:52.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:52.371 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:52.408 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:52.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:52.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:52.509 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:52.541 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:52.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:52.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:52.643 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:52.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:52.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:52.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:52.796 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:52.819 INFO:teuthology.orchestra.run.smithi130.stderr:set osd.2 primary-affinity to 0.805094 (8527622) 2024-08-17T18:20:52.829 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:52.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:52.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:52.931 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:52.953 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:52.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:52.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:53.055 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:53.079 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:53.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:53.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:53.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:52 smithi179 ceph-mon[93519]: pgmap v2855: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 23011/157014 objects degraded (14.655%); 50252/157014 objects misplaced (32.005%); 6.4 MiB/s, 111 objects/s recovering 2024-08-17T18:20:53.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:52 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3385450089' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.8050944066973458} : dispatch 2024-08-17T18:20:53.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:52 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.8050944066973458} : dispatch 2024-08-17T18:20:53.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:52 smithi130 ceph-mon[105521]: pgmap v2855: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 23011/157014 objects degraded (14.655%); 50252/157014 objects misplaced (32.005%); 6.4 MiB/s, 111 objects/s recovering 2024-08-17T18:20:53.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:52 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3385450089' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.8050944066973458} : dispatch 2024-08-17T18:20:53.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:52 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.8050944066973458} : dispatch 2024-08-17T18:20:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:52 smithi130 ceph-mon[110068]: pgmap v2855: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+remapped, 11 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.2 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 23011/157014 objects degraded (14.655%); 50252/157014 objects misplaced (32.005%); 6.4 MiB/s, 111 objects/s recovering 2024-08-17T18:20:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:52 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3385450089' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.8050944066973458} : dispatch 2024-08-17T18:20:53.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:52 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 0.8050944066973458} : dispatch 2024-08-17T18:20:53.182 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:53.205 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:53.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:53.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:53.306 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:53.330 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:53.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:53.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:53.431 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:53.455 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:53.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:53.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:53.557 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:53.581 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:53.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:53.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:53.683 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:53.705 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:53.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:53.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:53.807 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:53.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:53.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:53.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:53.932 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:53.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:53.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:53.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:54.060 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:54.086 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:54.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:54.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:54.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:53 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.8050944066973458}]': finished 2024-08-17T18:20:54.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:53 smithi179 ceph-mon[93519]: osdmap e1468: 8 total, 8 up, 7 in 2024-08-17T18:20:54.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:53 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.8050944066973458}]': finished 2024-08-17T18:20:54.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:53 smithi130 ceph-mon[105521]: osdmap e1468: 8 total, 8 up, 7 in 2024-08-17T18:20:54.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:53 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.8050944066973458}]': finished 2024-08-17T18:20:54.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:53 smithi130 ceph-mon[110068]: osdmap e1468: 8 total, 8 up, 7 in 2024-08-17T18:20:54.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:54.210 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:54.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:54.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:54.312 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:54.335 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:54.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:54.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:54.437 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:54.463 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:54.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:54.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:54.564 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:54.586 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:54.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:54.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:54.655 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:20:54.655 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:20:54.688 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:54.710 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:54.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:54.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:54.813 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:54.839 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:54.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:54.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:54.941 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:54.964 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:54.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:54.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:55.066 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:55.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:55.094 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:55.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:55.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:54 smithi130 ceph-mon[105521]: pgmap v2857: 99 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 1 active+remapped, 4 remapped+peering, 9 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 15409/146328 objects degraded (10.530%); 42087/146328 objects misplaced (28.762%); 10 MiB/s, 160 objects/s recovering 2024-08-17T18:20:55.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:54 smithi130 ceph-mon[105521]: osdmap e1469: 8 total, 8 up, 7 in 2024-08-17T18:20:55.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:54 smithi130 ceph-mon[105521]: 50.3 continuing backfill to osd.2 from (1439'1324,1467'10782] MIN to 1467'10782 2024-08-17T18:20:55.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:54 smithi130 ceph-mon[105521]: 50.13 continuing backfill to osd.2 from (1439'1512,1467'10859] MIN to 1467'10859 2024-08-17T18:20:55.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:54 smithi130 ceph-mon[110068]: pgmap v2857: 99 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 1 active+remapped, 4 remapped+peering, 9 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 15409/146328 objects degraded (10.530%); 42087/146328 objects misplaced (28.762%); 10 MiB/s, 160 objects/s recovering 2024-08-17T18:20:55.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:54 smithi130 ceph-mon[110068]: osdmap e1469: 8 total, 8 up, 7 in 2024-08-17T18:20:55.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:54 smithi130 ceph-mon[110068]: 50.3 continuing backfill to osd.2 from (1439'1324,1467'10782] MIN to 1467'10782 2024-08-17T18:20:55.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:54 smithi130 ceph-mon[110068]: 50.13 continuing backfill to osd.2 from (1439'1512,1467'10859] MIN to 1467'10859 2024-08-17T18:20:55.195 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:55.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:55.218 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:55.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:55.319 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:55.341 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:55.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:55.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:54 smithi179 ceph-mon[93519]: pgmap v2857: 99 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 1 active+remapped, 4 remapped+peering, 9 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 73 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 749 op/s; 15409/146328 objects degraded (10.530%); 42087/146328 objects misplaced (28.762%); 10 MiB/s, 160 objects/s recovering 2024-08-17T18:20:55.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:54 smithi179 ceph-mon[93519]: osdmap e1469: 8 total, 8 up, 7 in 2024-08-17T18:20:55.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:54 smithi179 ceph-mon[93519]: 50.3 continuing backfill to osd.2 from (1439'1324,1467'10782] MIN to 1467'10782 2024-08-17T18:20:55.344 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:54 smithi179 ceph-mon[93519]: 50.13 continuing backfill to osd.2 from (1439'1512,1467'10859] MIN to 1467'10859 2024-08-17T18:20:55.444 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:55.473 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:55.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:55.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:55.575 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:55.598 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:55.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:55.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:55.700 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:55.722 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:55.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:55.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:55.824 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:55.853 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:55.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:55.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:55.956 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:55.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:55.979 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:55.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:56.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:56.108 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:56.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:56.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:56.166 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:55 smithi130 ceph-mon[110068]: 50.18 continuing backfill to osd.3 from (1439'1063,1467'10712] MIN to 1467'10712 2024-08-17T18:20:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:55 smithi130 ceph-mon[110068]: 50.8 starting backfill to osd.3 from (1439'1699,1467'10929] MIN to 1467'10931 2024-08-17T18:20:56.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:55 smithi130 ceph-mon[110068]: osdmap e1470: 8 total, 8 up, 7 in 2024-08-17T18:20:56.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:55 smithi130 ceph-mon[105521]: 50.18 continuing backfill to osd.3 from (1439'1063,1467'10712] MIN to 1467'10712 2024-08-17T18:20:56.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:55 smithi130 ceph-mon[105521]: 50.8 starting backfill to osd.3 from (1439'1699,1467'10929] MIN to 1467'10931 2024-08-17T18:20:56.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:55 smithi130 ceph-mon[105521]: osdmap e1470: 8 total, 8 up, 7 in 2024-08-17T18:20:56.211 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:56.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:56.238 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:56.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:56.339 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:56.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:55 smithi179 ceph-mon[93519]: 50.18 continuing backfill to osd.3 from (1439'1063,1467'10712] MIN to 1467'10712 2024-08-17T18:20:56.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:55 smithi179 ceph-mon[93519]: 50.8 starting backfill to osd.3 from (1439'1699,1467'10929] MIN to 1467'10931 2024-08-17T18:20:56.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:55 smithi179 ceph-mon[93519]: osdmap e1470: 8 total, 8 up, 7 in 2024-08-17T18:20:56.362 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:56.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:56.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:56.464 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:56.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:56.495 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:56.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:56.596 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:56.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:56.627 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:56.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:56.728 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:56.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:56.751 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:56.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:56.853 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:20:56.883 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:56.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:56.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:20:56.985 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:57.013 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:57.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:57.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:57.115 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:57.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:57.141 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:57.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:57.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:56 smithi130 ceph-mon[105521]: pgmap v2860: 99 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 2 peering, 1 active+remapped, 4 remapped+peering, 9 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 71 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 15409/146328 objects degraded (10.530%); 42087/146328 objects misplaced (28.762%); 9.4 MiB/s, 149 objects/s recovering 2024-08-17T18:20:57.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:56 smithi130 ceph-mon[105521]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T18:20:57.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:56 smithi130 ceph-mon[105521]: osdmap e1471: 8 total, 8 up, 7 in 2024-08-17T18:20:57.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 15409/146328 objects degraded (10.530%), 10 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:57.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:56 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:20:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:56 smithi130 ceph-mon[110068]: pgmap v2860: 99 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 2 peering, 1 active+remapped, 4 remapped+peering, 9 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 71 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 15409/146328 objects degraded (10.530%); 42087/146328 objects misplaced (28.762%); 9.4 MiB/s, 149 objects/s recovering 2024-08-17T18:20:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:56 smithi130 ceph-mon[110068]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T18:20:57.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:56 smithi130 ceph-mon[110068]: osdmap e1471: 8 total, 8 up, 7 in 2024-08-17T18:20:57.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 15409/146328 objects degraded (10.530%), 10 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:57.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:56 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:20:57.242 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:57.267 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:57.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:57.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:56 smithi179 ceph-mon[93519]: pgmap v2860: 99 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 2 peering, 1 active+remapped, 4 remapped+peering, 9 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 71 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 593 op/s; 15409/146328 objects degraded (10.530%); 42087/146328 objects misplaced (28.762%); 9.4 MiB/s, 149 objects/s recovering 2024-08-17T18:20:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:56 smithi179 ceph-mon[93519]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T18:20:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:56 smithi179 ceph-mon[93519]: osdmap e1471: 8 total, 8 up, 7 in 2024-08-17T18:20:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 15409/146328 objects degraded (10.530%), 10 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T18:20:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:56 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:20:57.370 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:57.391 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:57.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:57.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:57.493 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:57.516 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:57.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:57.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:57.618 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:57.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:57.642 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:57.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:57.743 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:57.767 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:57.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:57.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:57.869 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:57.907 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:57.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:57.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:58.010 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:58.033 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:58.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:58.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:58.135 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:20:58.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:57 smithi130 ceph-mon[105521]: osdmap e1472: 8 total, 8 up, 7 in 2024-08-17T18:20:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:57 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:20:58.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:20:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:57 smithi130 ceph-mon[110068]: osdmap e1472: 8 total, 8 up, 7 in 2024-08-17T18:20:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:57 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:20:58.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:20:58.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:58.168 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:58.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:20:58.199 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:57 smithi179 ceph-mon[93519]: osdmap e1472: 8 total, 8 up, 7 in 2024-08-17T18:20:58.199 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:57 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:20:58.199 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:20:58.270 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:58.296 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:58.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:58.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:58.398 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:58.425 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:58.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:58.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:58.527 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:58.554 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:58.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:58.656 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:58.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:58.684 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:58.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:58.785 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:20:58.813 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:58.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:58.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:20:58.915 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:58.944 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:58.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:58.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:20:59.022 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:58 smithi130 ceph-mon[105521]: pgmap v2863: 99 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 1 active+remapped, 2 peering, 4 remapped+peering, 9 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 71 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 42 op/s; 15355/145812 objects degraded (10.531%); 41811/145812 objects misplaced (28.675%) 2024-08-17T18:20:59.022 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:58 smithi130 ceph-mon[105521]: osdmap e1473: 8 total, 8 up, 7 in 2024-08-17T18:20:59.022 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:58 smithi130 ceph-mon[110068]: pgmap v2863: 99 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 1 active+remapped, 2 peering, 4 remapped+peering, 9 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 71 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 42 op/s; 15355/145812 objects degraded (10.531%); 41811/145812 objects misplaced (28.675%) 2024-08-17T18:20:59.023 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:58 smithi130 ceph-mon[110068]: osdmap e1473: 8 total, 8 up, 7 in 2024-08-17T18:20:59.046 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:59.079 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:59.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:59.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:59.181 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:59.201 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 7, 1, 6, 4, 5] out_osds: [0] dead_osds: [] live_osds: [1, 4, 5, 6, 7, 0, 3, 2] 2024-08-17T18:20:59.201 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:20:59.202 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [2, 3, 7, 1, 6, 4, 5] 2024-08-17T18:20:59.202 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd out 5 2024-08-17T18:20:59.208 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:59.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:59.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:59.252 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:58 smithi179 ceph-mon[93519]: pgmap v2863: 99 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped+backfilling, 1 active+remapped, 2 peering, 4 remapped+peering, 9 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 71 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 42 op/s; 15355/145812 objects degraded (10.531%); 41811/145812 objects misplaced (28.675%) 2024-08-17T18:20:59.252 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:58 smithi179 ceph-mon[93519]: osdmap e1473: 8 total, 8 up, 7 in 2024-08-17T18:20:59.310 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:20:59.332 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:59.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:59.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:20:59.434 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:20:59.470 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:59.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:59.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:20:59.524 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:20:59.524 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:20:59.524 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:20:59.524 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:20:59.527 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:20:59.527 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:20:59.551 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:20:59.552 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:20:59.572 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:20:59.578 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:20:59.578 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:20:59.597 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:59.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:59.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:20:59.601 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:20:59.601 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:20:59.628 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:20:59.628 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:20:59.647 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:20:59.648 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:20:59.698 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:59.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:59.723 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:59.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:59.825 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:20:59.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:59.847 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:20:59.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:20:59.886 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:20:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:20:59] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-08-17T18:20:59.948 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:20:59.986 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:20:59.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:20:59.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:00.088 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:00.125 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:00.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:00.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:59 smithi130 ceph-mon[105521]: osdmap e1474: 8 total, 8 up, 7 in 2024-08-17T18:21:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:59 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1327955740' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:21:00.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:20:59 smithi130 ceph-mon[105521]: osdmap e1475: 8 total, 8 up, 7 in 2024-08-17T18:21:00.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:59 smithi130 ceph-mon[110068]: osdmap e1474: 8 total, 8 up, 7 in 2024-08-17T18:21:00.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:59 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1327955740' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:21:00.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:20:59 smithi130 ceph-mon[110068]: osdmap e1475: 8 total, 8 up, 7 in 2024-08-17T18:21:00.214 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:59 smithi179 ceph-mon[93519]: osdmap e1474: 8 total, 8 up, 7 in 2024-08-17T18:21:00.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:59 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1327955740' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:21:00.215 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:20:59 smithi179 ceph-mon[93519]: osdmap e1475: 8 total, 8 up, 7 in 2024-08-17T18:21:00.226 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:00.252 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:00.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:00.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:00.354 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:00.385 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:00.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:00.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:00.487 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:00.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:00.517 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:00.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:21:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:21:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:21:00.617 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:00.650 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:00.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:00.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:00.685 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:21:00.686 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T18:21:00.686 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T18:21:00.686 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T18:21:00.686 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:21:00.752 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:00.777 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:00.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:00.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:00.878 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:00.902 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:00.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:00.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:01.004 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:01.033 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:01.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:01.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:01.135 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:01.165 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:01.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:01.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:00 smithi130 ceph-mon[105521]: pgmap v2866: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped, 9 active+undersized+degraded+remapped+backfill_wait, 8 active+remapped+backfill_wait, 75 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 115 op/s; 19610/144936 objects degraded (13.530%); 41140/144936 objects misplaced (28.385%); 71 MiB/s, 1.14k objects/s recovering 2024-08-17T18:21:01.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:00 smithi130 ceph-mon[105521]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T18:21:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:00 smithi130 ceph-mon[110068]: pgmap v2866: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped, 9 active+undersized+degraded+remapped+backfill_wait, 8 active+remapped+backfill_wait, 75 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 115 op/s; 19610/144936 objects degraded (13.530%); 41140/144936 objects misplaced (28.385%); 71 MiB/s, 1.14k objects/s recovering 2024-08-17T18:21:01.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:00 smithi130 ceph-mon[110068]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T18:21:01.246 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:00 smithi179 ceph-mon[93519]: pgmap v2866: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped, 9 active+undersized+degraded+remapped+backfill_wait, 8 active+remapped+backfill_wait, 75 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 115 op/s; 19610/144936 objects degraded (13.530%); 41140/144936 objects misplaced (28.385%); 71 MiB/s, 1.14k objects/s recovering 2024-08-17T18:21:01.246 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:00 smithi179 ceph-mon[93519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T18:21:01.267 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:01.299 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:01.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:01.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:01.401 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:01.426 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:01.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:01.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:01.482 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T18:21:01.528 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:01.552 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:01.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:01.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:01.654 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:01.685 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:01.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:01.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:01.787 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:01.821 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:01.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:01.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:01.924 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:01.956 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:01.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:01.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:02.058 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:02.082 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:02.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:02.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:02.118 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 19610/144936 objects degraded (13.530%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:21:02.119 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 19610/144936 objects degraded (13.530%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:21:02.183 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:02.205 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:02.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:02.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:02.232 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 19610/144936 objects degraded (13.530%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:21:02.307 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:02.333 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:02.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:02.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:02.434 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:02.446 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:21:02.456 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:02.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:02.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:02.501 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:21:02.558 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:02.581 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:02.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:02.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:02.683 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:02.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:02.713 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:02.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:02.815 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:02.852 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:02.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:02.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:02.954 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:02.981 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:02.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:02.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:03.083 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:03.115 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:03.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:03.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:03.218 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:03.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:03.247 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:03.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:03.348 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:03.374 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:03.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:03.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:03.476 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:03.501 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:03.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:03.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:03.566 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:21:03.603 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:03.626 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:03.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:03.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:03.728 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:03.752 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:03.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:03.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:03 smithi179 ceph-mon[93519]: pgmap v2868: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped, 9 active+undersized+degraded+remapped+backfill_wait, 8 active+remapped+backfill_wait, 75 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 19610/144936 objects degraded (13.530%); 41140/144936 objects misplaced (28.385%); 59 MiB/s, 940 objects/s recovering 2024-08-17T18:21:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:03 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:03 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:03 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2321163991' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:21:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:03 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:21:03.853 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:03.883 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:03.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:03.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:03.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:03 smithi130 ceph-mon[105521]: pgmap v2868: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped, 9 active+undersized+degraded+remapped+backfill_wait, 8 active+remapped+backfill_wait, 75 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 19610/144936 objects degraded (13.530%); 41140/144936 objects misplaced (28.385%); 59 MiB/s, 940 objects/s recovering 2024-08-17T18:21:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:03 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:03 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:03 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2321163991' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:21:03.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:03 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:21:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:03 smithi130 ceph-mon[110068]: pgmap v2868: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped, 9 active+undersized+degraded+remapped+backfill_wait, 8 active+remapped+backfill_wait, 75 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 96 op/s; 19610/144936 objects degraded (13.530%); 41140/144936 objects misplaced (28.385%); 59 MiB/s, 940 objects/s recovering 2024-08-17T18:21:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:03 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:03 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:03 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2321163991' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:21:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:03 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:21:03.924 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:21:03.985 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:04.016 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:04.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:04.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:04.118 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:04.148 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:04.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:04.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:04.250 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:04.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:04.284 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:04.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:04.385 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:04.409 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:04.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:04.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:04.511 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:04.533 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:04.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:04.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:04.638 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:04.665 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:04.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:04.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:04.766 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:04.791 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:04.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:04.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:04.833 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:04 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:21:04.833 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:04 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:21:04.833 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:04 smithi130 ceph-mon[105521]: osdmap e1476: 8 total, 8 up, 7 in 2024-08-17T18:21:04.833 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:04 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:21:04.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:04 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:21:04.834 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:04 smithi130 ceph-mon[110068]: osdmap e1476: 8 total, 8 up, 7 in 2024-08-17T18:21:04.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:04 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:21:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:04 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:21:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:04 smithi179 ceph-mon[93519]: osdmap e1476: 8 total, 8 up, 7 in 2024-08-17T18:21:04.893 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:04.916 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:04.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:04.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:05.018 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:05.041 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:05.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:05.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:05.143 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:05.170 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:05.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:05.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:05.272 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:05.301 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:05.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:05.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:05.402 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:05.424 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:05.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:05.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:05.526 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:05.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:05.551 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:05.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:05.652 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:05.677 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:05.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:05.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:05.780 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:05.804 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:05.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:05.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:05.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:05 smithi179 ceph-mon[93519]: pgmap v2870: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 18789/138435 objects degraded (13.572%); 38897/138435 objects misplaced (28.098%); 53 MiB/s, 845 objects/s recovering 2024-08-17T18:21:05.906 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:05.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:05 smithi130 ceph-mon[105521]: pgmap v2870: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 18789/138435 objects degraded (13.572%); 38897/138435 objects misplaced (28.098%); 53 MiB/s, 845 objects/s recovering 2024-08-17T18:21:05.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:05 smithi130 ceph-mon[110068]: pgmap v2870: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 18789/138435 objects degraded (13.572%); 38897/138435 objects misplaced (28.098%); 53 MiB/s, 845 objects/s recovering 2024-08-17T18:21:05.936 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:05.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:05.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:06.037 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:06.059 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:06.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:06.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:06.162 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:06.187 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:06.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:06.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:06.289 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:06.321 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:06.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:06.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:06.423 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:06.455 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:06.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:06.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:06.556 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:06.581 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:06.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:06.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:06.683 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:06.710 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:06.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:06.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:06.811 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:06.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 18789/138435 objects degraded (13.572%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:21:06.845 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:06.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:06.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:06.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 18789/138435 objects degraded (13.572%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:21:06.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 18789/138435 objects degraded (13.572%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:21:06.947 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:06.979 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:06.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:06.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:07.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:07.117 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:07.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:07.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:07.219 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:07.249 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:07.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:07.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:07.351 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:07.374 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:07.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:07.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:07.476 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:07.506 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:07.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:07.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:07.608 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:07.634 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:07.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:07.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:07.736 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:07.757 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:07.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:07.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:07.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:07 smithi179 ceph-mon[93519]: pgmap v2871: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 338 op/s; 18689/138006 objects degraded (13.542%); 38787/138006 objects misplaced (28.105%); 4.4 MiB/s, 69 objects/s recovering 2024-08-17T18:21:07.859 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:07.885 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:07.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:07.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:07.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:07 smithi130 ceph-mon[105521]: pgmap v2871: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 338 op/s; 18689/138006 objects degraded (13.542%); 38787/138006 objects misplaced (28.105%); 4.4 MiB/s, 69 objects/s recovering 2024-08-17T18:21:07.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:07 smithi130 ceph-mon[110068]: pgmap v2871: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 338 op/s; 18689/138006 objects degraded (13.542%); 38787/138006 objects misplaced (28.105%); 4.4 MiB/s, 69 objects/s recovering 2024-08-17T18:21:07.986 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:08.015 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:08.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:08.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:08.117 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:08.142 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:08.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:08.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:08.244 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:08.271 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:08.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:08.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:08.373 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:08.403 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:08.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:08.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:08.505 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:08.528 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:08.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:08.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:08.630 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:08.651 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:08.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:08.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:08.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:08 smithi130 ceph-mon[105521]: pgmap v2872: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 18519/136728 objects degraded (13.544%); 38137/136728 objects misplaced (27.893%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T18:21:08.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:08 smithi130 ceph-mon[110068]: pgmap v2872: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 18519/136728 objects degraded (13.544%); 38137/136728 objects misplaced (27.893%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T18:21:08.753 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:08.794 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:08.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:08.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:08.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:08 smithi179 ceph-mon[93519]: pgmap v2872: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 341 op/s; 18519/136728 objects degraded (13.544%); 38137/136728 objects misplaced (27.893%); 3.7 MiB/s, 59 objects/s recovering 2024-08-17T18:21:08.896 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:08.918 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:08.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:08.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:09.020 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:09.044 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:09.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:09.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:09.146 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:09.168 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:09.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:09.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:09.269 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:09.305 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:09.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:09.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:09.407 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:09.437 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:09.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:09.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:09.539 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:09.571 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:09.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:09.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:09.674 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:09.701 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:09.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:09.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:09.803 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:09.827 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:09.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:09.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:09.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:09.950 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:09.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:09.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:09.969 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:21:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:21:09] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-08-17T18:21:10.052 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:10.083 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:10.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:10.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:10.185 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:10.216 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:10.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:10.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:10.317 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:10.341 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:10.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:10.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:10.443 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:10.465 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:10.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:10.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:10.567 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:10.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:21:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:21:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:21:10.596 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:10.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:10.699 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:10.723 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:10.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:10.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:10.824 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:10.848 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:10.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:10.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:10.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:10 smithi130 ceph-mon[110068]: pgmap v2873: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 17708/130644 objects degraded (13.554%); 36905/130644 objects misplaced (28.249%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T18:21:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:10 smithi130 ceph-mon[105521]: pgmap v2873: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 17708/130644 objects degraded (13.554%); 36905/130644 objects misplaced (28.249%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T18:21:10.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:10 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T18:21:10.766+0000 7fa856f19640 -1 mon.a@0(leader).osd e1476 definitely_dead 0 2024-08-17T18:21:10.949 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:10.973 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:10.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:10.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:11.075 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:11.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:10 smithi179 ceph-mon[93519]: pgmap v2873: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 17708/130644 objects degraded (13.554%); 36905/130644 objects misplaced (28.249%); 3.7 MiB/s, 58 objects/s recovering 2024-08-17T18:21:11.101 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:11.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:11.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:11.202 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:11.216 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:21:11.216 INFO:teuthology.orchestra.run.smithi130.stdout: "size": 20, 2024-08-17T18:21:11.216 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 600, 2024-08-17T18:21:11.216 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [ 2024-08-17T18:21:11.216 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.216 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:242809 50.18 50:1af01796:::benchmark_data_smithi130_398002_object97743:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1475)", 2024-08-17T18:21:11.216 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:00.462479+0000", 2024-08-17T18:21:11.216 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 9.8676659049999991, 2024-08-17T18:21:11.217 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.95795769500000005, 2024-08-17T18:21:11.217 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.217 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.217 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.217 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.217 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.217 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.217 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 242809 2024-08-17T18:21:11.217 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.217 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.217 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.217 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.217 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.462479+0000", 2024-08-17T18:21:11.218 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.218 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.218 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.218 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.462479+0000", 2024-08-17T18:21:11.218 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.218 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.218 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.218 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.462481+0000", 2024-08-17T18:21:11.218 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.212e-06 2024-08-17T18:21:11.218 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.218 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.219 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.219 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.462485+0000", 2024-08-17T18:21:11.219 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.5220000000000001e-06 2024-08-17T18:21:11.219 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.219 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.219 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.219 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.462499+0000", 2024-08-17T18:21:11.219 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3475000000000001e-05 2024-08-17T18:21:11.219 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.219 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.219 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.219 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.462504+0000", 2024-08-17T18:21:11.219 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5929999999999996e-06 2024-08-17T18:21:11.220 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.220 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.220 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.220 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.514296+0000", 2024-08-17T18:21:11.220 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.051791667 2024-08-17T18:21:11.220 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.220 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.220 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.220 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.517328+0000", 2024-08-17T18:21:11.220 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0030321250000000001 2024-08-17T18:21:11.220 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.220 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.220 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.221 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.542022+0000", 2024-08-17T18:21:11.221 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.024693487 2024-08-17T18:21:11.221 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.221 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.221 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.221 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.544581+0000", 2024-08-17T18:21:11.221 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025593399999999998 2024-08-17T18:21:11.221 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.221 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.221 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.221 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.381911+0000", 2024-08-17T18:21:11.221 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.83733024099999998 2024-08-17T18:21:11.221 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.384498+0000", 2024-08-17T18:21:11.222 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025871689999999998 2024-08-17T18:21:11.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.389835+0000", 2024-08-17T18:21:11.222 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0053371679999999998 2024-08-17T18:21:11.222 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.222 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.222 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.222 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.410212+0000", 2024-08-17T18:21:11.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020376378000000001 2024-08-17T18:21:11.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.410732+0000", 2024-08-17T18:21:11.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00051968299999999995 2024-08-17T18:21:11.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.223 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.223 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.412102+0000", 2024-08-17T18:21:11.223 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0013703980000000001 2024-08-17T18:21:11.223 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.223 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.224 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.224 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.413276+0000", 2024-08-17T18:21:11.224 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00117438 2024-08-17T18:21:11.224 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.224 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.224 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.224 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.415159+0000", 2024-08-17T18:21:11.224 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001882636 2024-08-17T18:21:11.224 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.224 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.225 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.225 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.420437+0000", 2024-08-17T18:21:11.225 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0052782209999999996 2024-08-17T18:21:11.225 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.225 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.225 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.225 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.225 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.225 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:242885 50.18 50:19ec1fb8:::benchmark_data_smithi130_398002_object97819:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1475)", 2024-08-17T18:21:11.225 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:00.651646+0000", 2024-08-17T18:21:11.225 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 9.6784998560000002, 2024-08-17T18:21:11.225 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.80336147300000005, 2024-08-17T18:21:11.225 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.226 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.226 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.226 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.226 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.226 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.226 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 242885 2024-08-17T18:21:11.226 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.226 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.226 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.226 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.226 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.651646+0000", 2024-08-17T18:21:11.226 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.226 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.227 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.227 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.227 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.651646+0000", 2024-08-17T18:21:11.227 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.228 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.228 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.228 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.228 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.651647+0000", 2024-08-17T18:21:11.228 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7129999999999999e-06 2024-08-17T18:21:11.229 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.229 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.229 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.229 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.651663+0000", 2024-08-17T18:21:11.229 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6032000000000002e-05 2024-08-17T18:21:11.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.230 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.230 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.651677+0000", 2024-08-17T18:21:11.230 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3494e-05 2024-08-17T18:21:11.230 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.230 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.230 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.651683+0000", 2024-08-17T18:21:11.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.6540000000000002e-06 2024-08-17T18:21:11.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.654570+0000", 2024-08-17T18:21:11.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028866400000000002 2024-08-17T18:21:11.231 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.231 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.231 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.231 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.658477+0000", 2024-08-17T18:21:11.231 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039074210000000003 2024-08-17T18:21:11.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.232 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.232 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.681416+0000", 2024-08-17T18:21:11.232 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022938351999999999 2024-08-17T18:21:11.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.232 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.232 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.685081+0000", 2024-08-17T18:21:11.232 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0036647369999999999 2024-08-17T18:21:11.232 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.232 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.232 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.233 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.429517+0000", 2024-08-17T18:21:11.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.74443685699999995 2024-08-17T18:21:11.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.233 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.233 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.431879+0000", 2024-08-17T18:21:11.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0023620680000000002 2024-08-17T18:21:11.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.233 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.233 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.233 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.436587+0000", 2024-08-17T18:21:11.233 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0047079299999999999 2024-08-17T18:21:11.233 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.445013+0000", 2024-08-17T18:21:11.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0084257710000000003 2024-08-17T18:21:11.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.445579+0000", 2024-08-17T18:21:11.234 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00056618800000000002 2024-08-17T18:21:11.234 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.234 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.234 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.234 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.446457+0000", 2024-08-17T18:21:11.235 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00087750100000000004 2024-08-17T18:21:11.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.235 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.235 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.235 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.447596+0000", 2024-08-17T18:21:11.235 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001139323 2024-08-17T18:21:11.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.235 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.235 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.235 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.449364+0000", 2024-08-17T18:21:11.235 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001767369 2024-08-17T18:21:11.235 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.235 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.236 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.236 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.455007+0000", 2024-08-17T18:21:11.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0056434229999999998 2024-08-17T18:21:11.236 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.236 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.236 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.236 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.236 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.236 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:242913 50.18 50:1892e5cb:::benchmark_data_smithi130_398002_object97847:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1475)", 2024-08-17T18:21:11.236 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:00.710830+0000", 2024-08-17T18:21:11.236 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 9.6193151629999996, 2024-08-17T18:21:11.236 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.86958853999999997, 2024-08-17T18:21:11.237 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.237 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.237 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.237 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.237 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.237 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.237 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 242913 2024-08-17T18:21:11.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.237 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.237 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.237 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.237 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.710830+0000", 2024-08-17T18:21:11.237 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.237 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.710830+0000", 2024-08-17T18:21:11.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.710832+0000", 2024-08-17T18:21:11.238 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.748e-06 2024-08-17T18:21:11.238 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.238 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.238 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.238 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.710837+0000", 2024-08-17T18:21:11.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1399999999999999e-06 2024-08-17T18:21:11.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.710851+0000", 2024-08-17T18:21:11.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3478e-05 2024-08-17T18:21:11.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.239 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.239 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.710859+0000", 2024-08-17T18:21:11.239 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5429999999999994e-06 2024-08-17T18:21:11.239 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.239 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.240 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.240 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.716829+0000", 2024-08-17T18:21:11.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0059701429999999998 2024-08-17T18:21:11.240 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.240 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.240 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.240 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.724044+0000", 2024-08-17T18:21:11.240 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0072152259999999999 2024-08-17T18:21:11.241 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.241 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.241 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.241 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.740383+0000", 2024-08-17T18:21:11.241 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016338338000000001 2024-08-17T18:21:11.241 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.241 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.241 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.743543+0000", 2024-08-17T18:21:11.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003160047 2024-08-17T18:21:11.242 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.242 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.242 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.242 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.523819+0000", 2024-08-17T18:21:11.242 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.78027565399999999 2024-08-17T18:21:11.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.526340+0000", 2024-08-17T18:21:11.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025213050000000002 2024-08-17T18:21:11.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.243 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.532935+0000", 2024-08-17T18:21:11.243 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0065954330000000004 2024-08-17T18:21:11.243 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.243 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.243 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.566789+0000", 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.033854215 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.568042+0000", 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001252328 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.570262+0000", 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0022205910000000001 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.244 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.572261+0000", 2024-08-17T18:21:11.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0019986980000000001 2024-08-17T18:21:11.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.573919+0000", 2024-08-17T18:21:11.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0016580779999999999 2024-08-17T18:21:11.245 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.245 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.245 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.245 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.580419+0000", 2024-08-17T18:21:11.245 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0064995749999999996 2024-08-17T18:21:11.245 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.246 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.246 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.246 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.246 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.246 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:242961 50.18 50:18ceb6d9:::benchmark_data_smithi130_398002_object97895:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1475)", 2024-08-17T18:21:11.246 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:00.796272+0000", 2024-08-17T18:21:11.246 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 9.5338737029999994, 2024-08-17T18:21:11.246 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.79543741300000004, 2024-08-17T18:21:11.246 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.246 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.246 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.246 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.246 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.247 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.247 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 242961 2024-08-17T18:21:11.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.247 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.796272+0000", 2024-08-17T18:21:11.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.247 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.247 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.247 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.247 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.796272+0000", 2024-08-17T18:21:11.247 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.248 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.248 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.796273+0000", 2024-08-17T18:21:11.248 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.0449999999999999e-06 2024-08-17T18:21:11.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.248 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.248 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.796278+0000", 2024-08-17T18:21:11.248 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9849999999999997e-06 2024-08-17T18:21:11.248 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.248 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.248 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.249 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.796289+0000", 2024-08-17T18:21:11.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1433999999999999e-05 2024-08-17T18:21:11.249 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.249 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.249 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.249 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.796296+0000", 2024-08-17T18:21:11.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5660000000000003e-06 2024-08-17T18:21:11.249 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.249 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.249 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.249 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.806313+0000", 2024-08-17T18:21:11.249 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.010017702999999999 2024-08-17T18:21:11.249 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.250 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.250 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.250 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.809208+0000", 2024-08-17T18:21:11.250 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028942690000000001 2024-08-17T18:21:11.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.250 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.250 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.250 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.845295+0000", 2024-08-17T18:21:11.250 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.036087374999999998 2024-08-17T18:21:11.250 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.250 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.250 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.250 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.848285+0000", 2024-08-17T18:21:11.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029901530000000002 2024-08-17T18:21:11.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.251 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.547633+0000", 2024-08-17T18:21:11.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.69934753100000002 2024-08-17T18:21:11.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.251 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.550440+0000", 2024-08-17T18:21:11.251 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028075610000000001 2024-08-17T18:21:11.251 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.251 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.251 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.558095+0000", 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0076546399999999999 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.581305+0000", 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023210002 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.581905+0000", 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00060005099999999999 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.252 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.582772+0000", 2024-08-17T18:21:11.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00086750800000000002 2024-08-17T18:21:11.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.583984+0000", 2024-08-17T18:21:11.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0012112620000000001 2024-08-17T18:21:11.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.253 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.253 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.253 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.585785+0000", 2024-08-17T18:21:11.253 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001801722 2024-08-17T18:21:11.253 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.254 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.254 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.591709+0000", 2024-08-17T18:21:11.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0059236059999999997 2024-08-17T18:21:11.254 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.254 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.254 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.254 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.254 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.254 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:242965 50.18 50:1b0c89f5:::benchmark_data_smithi130_398002_object97899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1475)", 2024-08-17T18:21:11.254 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:00.800272+0000", 2024-08-17T18:21:11.254 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 9.5298735249999993, 2024-08-17T18:21:11.254 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.87740884100000005, 2024-08-17T18:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 242965 2024-08-17T18:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.800272+0000", 2024-08-17T18:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.255 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.800272+0000", 2024-08-17T18:21:11.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.256 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.800274+0000", 2024-08-17T18:21:11.256 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6729999999999999e-06 2024-08-17T18:21:11.256 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.256 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.256 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.800279+0000", 2024-08-17T18:21:11.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.1649999999999997e-06 2024-08-17T18:21:11.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.257 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.800294+0000", 2024-08-17T18:21:11.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5296999999999999e-05 2024-08-17T18:21:11.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.257 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.257 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.800303+0000", 2024-08-17T18:21:11.257 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5169999999999999e-06 2024-08-17T18:21:11.257 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.257 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.809504+0000", 2024-08-17T18:21:11.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0092017379999999992 2024-08-17T18:21:11.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.812743+0000", 2024-08-17T18:21:11.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003238824 2024-08-17T18:21:11.258 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.258 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.258 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.258 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.860938+0000", 2024-08-17T18:21:11.258 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.048194538000000002 2024-08-17T18:21:11.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.259 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.259 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:00.863471+0000", 2024-08-17T18:21:11.259 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025331300000000002 2024-08-17T18:21:11.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.259 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.259 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.599024+0000", 2024-08-17T18:21:11.259 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.73555292100000003 2024-08-17T18:21:11.259 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.259 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.259 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.601691+0000", 2024-08-17T18:21:11.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002667586 2024-08-17T18:21:11.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.607304+0000", 2024-08-17T18:21:11.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0056127709999999999 2024-08-17T18:21:11.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.260 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.260 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.260 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.616562+0000", 2024-08-17T18:21:11.260 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0092576099999999995 2024-08-17T18:21:11.260 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.617745+0000", 2024-08-17T18:21:11.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0011831540000000001 2024-08-17T18:21:11.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.261 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.618915+0000", 2024-08-17T18:21:11.261 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0011698850000000001 2024-08-17T18:21:11.261 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.261 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.261 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.670142+0000", 2024-08-17T18:21:11.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.05122757 2024-08-17T18:21:11.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.671591+0000", 2024-08-17T18:21:11.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0014488319999999999 2024-08-17T18:21:11.262 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.262 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.262 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.262 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.677681+0000", 2024-08-17T18:21:11.262 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0060896300000000004 2024-08-17T18:21:11.262 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.263 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.263 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.263 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.263 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.263 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:243250 50.18 50:1d4f40eb:::benchmark_data_smithi130_398002_object98184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1475)", 2024-08-17T18:21:11.263 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:01.298839+0000", 2024-08-17T18:21:11.263 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 9.0313061339999994, 2024-08-17T18:21:11.263 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.78612321799999996, 2024-08-17T18:21:11.263 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.263 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.263 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.264 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.264 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.264 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.264 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 243250 2024-08-17T18:21:11.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.264 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.264 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.298839+0000", 2024-08-17T18:21:11.264 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.264 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.264 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.264 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.298839+0000", 2024-08-17T18:21:11.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.298841+0000", 2024-08-17T18:21:11.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7439999999999999e-06 2024-08-17T18:21:11.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.265 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.265 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.265 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.298846+0000", 2024-08-17T18:21:11.265 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.8899999999999998e-06 2024-08-17T18:21:11.265 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.298857+0000", 2024-08-17T18:21:11.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1408e-05 2024-08-17T18:21:11.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.266 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.298863+0000", 2024-08-17T18:21:11.266 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.0410000000000003e-06 2024-08-17T18:21:11.266 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.266 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.266 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.371292+0000", 2024-08-17T18:21:11.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.072428724999999999 2024-08-17T18:21:11.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.374507+0000", 2024-08-17T18:21:11.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032147260000000002 2024-08-17T18:21:11.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.267 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.267 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.406389+0000", 2024-08-17T18:21:11.267 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.031882299000000003 2024-08-17T18:21:11.267 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.267 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.268 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.409402+0000", 2024-08-17T18:21:11.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0030127219999999998 2024-08-17T18:21:11.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.268 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.052627+0000", 2024-08-17T18:21:11.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.64322531100000002 2024-08-17T18:21:11.268 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.268 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.268 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.268 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.055240+0000", 2024-08-17T18:21:11.268 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026124439999999998 2024-08-17T18:21:11.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.060755+0000", 2024-08-17T18:21:11.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0055158919999999997 2024-08-17T18:21:11.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.068077+0000", 2024-08-17T18:21:11.269 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0073219000000000001 2024-08-17T18:21:11.269 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.269 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.269 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.269 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.072503+0000", 2024-08-17T18:21:11.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0044252500000000004 2024-08-17T18:21:11.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.073611+0000", 2024-08-17T18:21:11.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001108721 2024-08-17T18:21:11.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.270 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.270 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.077573+0000", 2024-08-17T18:21:11.270 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0039612570000000001 2024-08-17T18:21:11.270 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.270 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.079410+0000", 2024-08-17T18:21:11.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001837041 2024-08-17T18:21:11.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.271 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.271 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.084962+0000", 2024-08-17T18:21:11.271 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0055528469999999996 2024-08-17T18:21:11.271 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.271 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.271 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.271 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.271 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.272 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:243276 50.18 50:1d067428:::benchmark_data_smithi130_398002_object98210:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1475)", 2024-08-17T18:21:11.272 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:01.375122+0000", 2024-08-17T18:21:11.272 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 8.9550230810000002, 2024-08-17T18:21:11.272 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.82812326599999997, 2024-08-17T18:21:11.272 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.272 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.272 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.272 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.272 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.272 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.272 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 243276 2024-08-17T18:21:11.272 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.272 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.272 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.375122+0000", 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.375122+0000", 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.375124+0000", 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.996e-06 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.273 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.274 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.274 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.375129+0000", 2024-08-17T18:21:11.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.527e-06 2024-08-17T18:21:11.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.274 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.274 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.375145+0000", 2024-08-17T18:21:11.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6387000000000001e-05 2024-08-17T18:21:11.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.274 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.274 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.274 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.375154+0000", 2024-08-17T18:21:11.274 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4249999999999998e-06 2024-08-17T18:21:11.274 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.441163+0000", 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.066009709 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.444117+0000", 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029534940000000001 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.467553+0000", 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.02343572 2024-08-17T18:21:11.275 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.470713+0000", 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0031604910000000001 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.145489+0000", 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.67477546200000005 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.150923+0000", 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0054348010000000004 2024-08-17T18:21:11.276 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.157937+0000", 2024-08-17T18:21:11.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0070141650000000002 2024-08-17T18:21:11.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.178885+0000", 2024-08-17T18:21:11.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020947757000000001 2024-08-17T18:21:11.277 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.277 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.277 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.277 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.192581+0000", 2024-08-17T18:21:11.277 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.013695541 2024-08-17T18:21:11.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.193513+0000", 2024-08-17T18:21:11.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0009324 2024-08-17T18:21:11.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.278 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.195294+0000", 2024-08-17T18:21:11.278 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0017804310000000001 2024-08-17T18:21:11.278 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.278 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.278 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.197268+0000", 2024-08-17T18:21:11.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0019748190000000001 2024-08-17T18:21:11.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.279 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.279 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.203246+0000", 2024-08-17T18:21:11.279 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0059771410000000001 2024-08-17T18:21:11.279 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.279 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.279 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.279 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.279 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.279 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:243288 50.18 50:19f6a648:::benchmark_data_smithi130_398002_object98222:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1475)", 2024-08-17T18:21:11.280 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:01.393258+0000", 2024-08-17T18:21:11.280 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 8.9368877649999998, 2024-08-17T18:21:11.280 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.86785709700000002, 2024-08-17T18:21:11.280 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.280 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.280 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.280 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.280 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.280 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.280 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 243288 2024-08-17T18:21:11.280 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.280 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.393258+0000", 2024-08-17T18:21:11.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.393258+0000", 2024-08-17T18:21:11.281 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.281 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.281 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.281 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.281 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.393260+0000", 2024-08-17T18:21:11.282 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.131e-06 2024-08-17T18:21:11.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.282 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.282 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.393266+0000", 2024-08-17T18:21:11.282 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.823e-06 2024-08-17T18:21:11.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.282 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.282 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.393284+0000", 2024-08-17T18:21:11.282 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7997999999999999e-05 2024-08-17T18:21:11.282 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.282 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.393293+0000", 2024-08-17T18:21:11.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.5380000000000008e-06 2024-08-17T18:21:11.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.471065+0000", 2024-08-17T18:21:11.283 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.077771953000000005 2024-08-17T18:21:11.283 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.283 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.283 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.283 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.474216+0000", 2024-08-17T18:21:11.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0031505080000000002 2024-08-17T18:21:11.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.511376+0000", 2024-08-17T18:21:11.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.037160805999999998 2024-08-17T18:21:11.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.284 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:01.514883+0000", 2024-08-17T18:21:11.284 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0035069039999999999 2024-08-17T18:21:11.284 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.284 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.284 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.235777+0000", 2024-08-17T18:21:11.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.72089351400000001 2024-08-17T18:21:11.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.238325+0000", 2024-08-17T18:21:11.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002548481 2024-08-17T18:21:11.285 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.285 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.285 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.285 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.244040+0000", 2024-08-17T18:21:11.285 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0057144040000000002 2024-08-17T18:21:11.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.251042+0000", 2024-08-17T18:21:11.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0070019879999999998 2024-08-17T18:21:11.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.286 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.252128+0000", 2024-08-17T18:21:11.286 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0010868150000000001 2024-08-17T18:21:11.286 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.286 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.286 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.252679+0000", 2024-08-17T18:21:11.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00055088700000000001 2024-08-17T18:21:11.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.253553+0000", 2024-08-17T18:21:11.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00087336599999999999 2024-08-17T18:21:11.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.287 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.287 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.287 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.255502+0000", 2024-08-17T18:21:11.287 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0019495249999999999 2024-08-17T18:21:11.287 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.288 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.288 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.261115+0000", 2024-08-17T18:21:11.288 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005612456 2024-08-17T18:21:11.288 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.288 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.288 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.288 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.288 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.288 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:243854 50.18 50:1b9ec846:::benchmark_data_smithi130_398002_object98788:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1475)", 2024-08-17T18:21:11.288 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:02.419081+0000", 2024-08-17T18:21:11.288 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 7.9110640559999998, 2024-08-17T18:21:11.288 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.82266397400000002, 2024-08-17T18:21:11.288 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.289 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.289 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.289 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.289 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.289 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.289 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 243854 2024-08-17T18:21:11.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.289 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.289 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.289 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.289 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.419081+0000", 2024-08-17T18:21:11.289 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.289 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.419081+0000", 2024-08-17T18:21:11.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.419083+0000", 2024-08-17T18:21:11.290 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 2.0279999999999999e-06 2024-08-17T18:21:11.290 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.290 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.290 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.290 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.419090+0000", 2024-08-17T18:21:11.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.4219999999999997e-06 2024-08-17T18:21:11.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.419105+0000", 2024-08-17T18:21:11.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5716e-05 2024-08-17T18:21:11.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.291 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.419115+0000", 2024-08-17T18:21:11.291 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 9.4630000000000007e-06 2024-08-17T18:21:11.291 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.291 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.291 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.292 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.495400+0000", 2024-08-17T18:21:11.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.076284609000000003 2024-08-17T18:21:11.292 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.292 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.292 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.292 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.498605+0000", 2024-08-17T18:21:11.292 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032059089999999998 2024-08-17T18:21:11.292 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.292 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.292 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.293 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.552710+0000", 2024-08-17T18:21:11.293 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.054104575000000002 2024-08-17T18:21:11.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.293 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.293 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:02.555808+0000", 2024-08-17T18:21:11.293 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0030976290000000002 2024-08-17T18:21:11.293 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.293 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.293 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.293 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.193153+0000", 2024-08-17T18:21:11.293 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.63734556099999995 2024-08-17T18:21:11.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.294 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.294 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.195770+0000", 2024-08-17T18:21:11.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026166739999999998 2024-08-17T18:21:11.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.294 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.294 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.201100+0000", 2024-08-17T18:21:11.294 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0053297400000000003 2024-08-17T18:21:11.294 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.294 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.294 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.216801+0000", 2024-08-17T18:21:11.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.015701804 2024-08-17T18:21:11.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.217745+0000", 2024-08-17T18:21:11.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00094391199999999998 2024-08-17T18:21:11.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.295 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.295 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.295 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.219355+0000", 2024-08-17T18:21:11.295 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001609786 2024-08-17T18:21:11.295 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.235454+0000", 2024-08-17T18:21:11.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016098835999999998 2024-08-17T18:21:11.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.236941+0000", 2024-08-17T18:21:11.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001486795 2024-08-17T18:21:11.296 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.296 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.296 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.296 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.241745+0000", 2024-08-17T18:21:11.296 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0048045149999999997 2024-08-17T18:21:11.297 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.297 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.297 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.297 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.297 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.297 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:244286 50.18 50:1c8f0e8e:::benchmark_data_smithi130_398002_object99220:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1475)", 2024-08-17T18:21:11.297 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:03.297305+0000", 2024-08-17T18:21:11.297 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 7.0328404950000003, 2024-08-17T18:21:11.297 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.78274325600000005, 2024-08-17T18:21:11.297 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.297 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.297 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.297 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.297 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.298 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.298 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 244286 2024-08-17T18:21:11.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.298 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.297305+0000", 2024-08-17T18:21:11.298 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.298 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.298 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.298 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.297305+0000", 2024-08-17T18:21:11.298 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.298 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.299 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.299 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.299 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.297306+0000", 2024-08-17T18:21:11.299 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1030000000000001e-06 2024-08-17T18:21:11.299 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.299 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.299 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.299 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.297311+0000", 2024-08-17T18:21:11.299 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.079e-06 2024-08-17T18:21:11.299 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.299 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.299 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.299 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.297326+0000", 2024-08-17T18:21:11.299 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4501999999999999e-05 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.297333+0000", 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.2110000000000001e-06 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.318608+0000", 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021275171999999998 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.321146+0000", 2024-08-17T18:21:11.300 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025377820000000001 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.343445+0000", 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.022299007999999999 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.345877+0000", 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0024321149999999999 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.047983+0000", 2024-08-17T18:21:11.301 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.70210629800000002 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.051141+0000", 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003158269 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.056978+0000", 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0058361089999999999 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.067981+0000", 2024-08-17T18:21:11.302 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011003199999999999 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.068457+0000", 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00047639900000000002 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.069344+0000", 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00088652000000000002 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.070611+0000", 2024-08-17T18:21:11.303 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0012669980000000001 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.072688+0000", 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0020773559999999998 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.080048+0000", 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0073601350000000003 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.304 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.305 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:244322 50.18 50:1a2abc04:::benchmark_data_smithi130_398002_object99256:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1475)", 2024-08-17T18:21:11.305 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:03.400599+0000", 2024-08-17T18:21:11.305 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 6.9295460479999997, 2024-08-17T18:21:11.305 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.76204544100000005, 2024-08-17T18:21:11.305 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.305 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.305 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.305 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.305 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.305 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.305 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 244322 2024-08-17T18:21:11.305 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.306 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.400599+0000", 2024-08-17T18:21:11.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.306 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.400599+0000", 2024-08-17T18:21:11.306 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.306 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.306 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.306 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.307 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.400601+0000", 2024-08-17T18:21:11.307 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.759e-06 2024-08-17T18:21:11.307 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.307 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.307 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.307 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.400610+0000", 2024-08-17T18:21:11.307 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.4929999999999998e-06 2024-08-17T18:21:11.307 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.307 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.307 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.307 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.400623+0000", 2024-08-17T18:21:11.307 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3502e-05 2024-08-17T18:21:11.307 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.308 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.308 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.308 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.400630+0000", 2024-08-17T18:21:11.308 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.08e-06 2024-08-17T18:21:11.308 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.308 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.308 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.308 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.441386+0000", 2024-08-17T18:21:11.308 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.040756144000000001 2024-08-17T18:21:11.308 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.308 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.308 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.308 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.444684+0000", 2024-08-17T18:21:11.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032980710000000001 2024-08-17T18:21:11.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.465497+0000", 2024-08-17T18:21:11.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020812192 2024-08-17T18:21:11.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.309 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.468317+0000", 2024-08-17T18:21:11.309 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028208489999999998 2024-08-17T18:21:11.309 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.309 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.309 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.310 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.113356+0000", 2024-08-17T18:21:11.310 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.64503869800000002 2024-08-17T18:21:11.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.310 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.310 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.115554+0000", 2024-08-17T18:21:11.310 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0021976159999999999 2024-08-17T18:21:11.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.310 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.310 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.310 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.124771+0000", 2024-08-17T18:21:11.310 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0092170329999999995 2024-08-17T18:21:11.310 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.153288+0000", 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.028517442 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.154291+0000", 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0010025380000000001 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.155312+0000", 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0010215179999999999 2024-08-17T18:21:11.311 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.155755+0000", 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00044263499999999998 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.157188+0000", 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001433361 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.162645+0000", 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0054565100000000004 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.312 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.313 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.313 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.313 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.313 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:244467 50.18 50:1fa6d9a5:::benchmark_data_smithi130_398002_object99401:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1476)", 2024-08-17T18:21:11.313 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:03.689810+0000", 2024-08-17T18:21:11.313 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 6.6403350919999999, 2024-08-17T18:21:11.313 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.76470730399999998, 2024-08-17T18:21:11.313 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.313 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.313 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.313 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.313 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.313 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.314 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 244467 2024-08-17T18:21:11.314 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.314 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.314 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.314 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.314 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.689810+0000", 2024-08-17T18:21:11.314 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.314 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.314 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.314 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.314 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.689810+0000", 2024-08-17T18:21:11.314 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.314 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.689812+0000", 2024-08-17T18:21:11.315 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.6899999999999999e-06 2024-08-17T18:21:11.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.689828+0000", 2024-08-17T18:21:11.315 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5722999999999998e-05 2024-08-17T18:21:11.315 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.315 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.315 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.315 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.689841+0000", 2024-08-17T18:21:11.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3477e-05 2024-08-17T18:21:11.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.689849+0000", 2024-08-17T18:21:11.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.7840000000000004e-06 2024-08-17T18:21:11.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.316 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.316 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.316 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.721598+0000", 2024-08-17T18:21:11.316 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.031749057999999997 2024-08-17T18:21:11.316 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.724527+0000", 2024-08-17T18:21:11.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029288999999999999 2024-08-17T18:21:11.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.772009+0000", 2024-08-17T18:21:11.317 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.047482569000000002 2024-08-17T18:21:11.317 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.317 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.317 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.317 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.775097+0000", 2024-08-17T18:21:11.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003087682 2024-08-17T18:21:11.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.428414+0000", 2024-08-17T18:21:11.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.65331723799999997 2024-08-17T18:21:11.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.318 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.318 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.318 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.430510+0000", 2024-08-17T18:21:11.318 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002095679 2024-08-17T18:21:11.318 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.435664+0000", 2024-08-17T18:21:11.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0051539760000000002 2024-08-17T18:21:11.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.441681+0000", 2024-08-17T18:21:11.319 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0060169179999999996 2024-08-17T18:21:11.319 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.319 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.319 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.319 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.447328+0000", 2024-08-17T18:21:11.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0056467490000000004 2024-08-17T18:21:11.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.320 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.448460+0000", 2024-08-17T18:21:11.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001132266 2024-08-17T18:21:11.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.320 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.320 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.448902+0000", 2024-08-17T18:21:11.320 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00044207900000000001 2024-08-17T18:21:11.320 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.320 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.321 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.321 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.450183+0000", 2024-08-17T18:21:11.321 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001280612 2024-08-17T18:21:11.321 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.321 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.321 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.321 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.454518+0000", 2024-08-17T18:21:11.321 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0043349039999999997 2024-08-17T18:21:11.321 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.321 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.321 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.321 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.321 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.321 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:244601 50.18 50:1f4c1669:::benchmark_data_smithi130_398002_object99535:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1476)", 2024-08-17T18:21:11.322 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:03.933847+0000", 2024-08-17T18:21:11.322 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 6.3962979129999997, 2024-08-17T18:21:11.322 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.80952116500000004, 2024-08-17T18:21:11.322 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.322 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.322 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.322 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.322 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.322 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.322 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 244601 2024-08-17T18:21:11.322 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.322 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.322 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.323 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.933847+0000", 2024-08-17T18:21:11.323 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.323 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.933847+0000", 2024-08-17T18:21:11.323 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.323 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.323 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.323 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.323 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.933849+0000", 2024-08-17T18:21:11.323 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.9479999999999998e-06 2024-08-17T18:21:11.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.933856+0000", 2024-08-17T18:21:11.324 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.8240000000000002e-06 2024-08-17T18:21:11.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.933869+0000", 2024-08-17T18:21:11.324 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.294e-05 2024-08-17T18:21:11.324 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.324 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.324 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.324 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.933877+0000", 2024-08-17T18:21:11.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.605e-06 2024-08-17T18:21:11.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.325 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.325 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.325 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.982847+0000", 2024-08-17T18:21:11.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.048970008000000002 2024-08-17T18:21:11.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.325 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.325 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.325 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:03.985845+0000", 2024-08-17T18:21:11.325 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0029978629999999999 2024-08-17T18:21:11.325 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.325 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.037750+0000", 2024-08-17T18:21:11.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.051905057999999997 2024-08-17T18:21:11.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.326 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.040863+0000", 2024-08-17T18:21:11.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.003113368 2024-08-17T18:21:11.326 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.326 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.326 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.326 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.692547+0000", 2024-08-17T18:21:11.326 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.65168388200000005 2024-08-17T18:21:11.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.327 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.694854+0000", 2024-08-17T18:21:11.327 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0023074609999999998 2024-08-17T18:21:11.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.327 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.699873+0000", 2024-08-17T18:21:11.327 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0050188050000000003 2024-08-17T18:21:11.327 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.327 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.327 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.327 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.720398+0000", 2024-08-17T18:21:11.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.020524634 2024-08-17T18:21:11.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.328 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.328 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.734705+0000", 2024-08-17T18:21:11.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.014307624999999999 2024-08-17T18:21:11.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.328 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.328 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.735685+0000", 2024-08-17T18:21:11.328 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0009791820000000001 2024-08-17T18:21:11.328 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.328 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.328 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.329 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.737239+0000", 2024-08-17T18:21:11.329 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0015540700000000001 2024-08-17T18:21:11.329 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.329 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.329 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.329 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.738772+0000", 2024-08-17T18:21:11.329 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001532842 2024-08-17T18:21:11.329 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.329 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.329 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.329 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.743369+0000", 2024-08-17T18:21:11.329 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0045970500000000001 2024-08-17T18:21:11.329 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.330 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.330 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.330 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.330 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.330 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:244942 50.18 50:1f917443:::benchmark_data_smithi130_398002_object99876:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1476)", 2024-08-17T18:21:11.330 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:04.661013+0000", 2024-08-17T18:21:11.330 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 5.6691322809999996, 2024-08-17T18:21:11.330 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.85257542100000006, 2024-08-17T18:21:11.330 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.330 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.330 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.330 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.330 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.331 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.331 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 244942 2024-08-17T18:21:11.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.331 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.331 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.331 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.331 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.661013+0000", 2024-08-17T18:21:11.331 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.331 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.331 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.331 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.661013+0000", 2024-08-17T18:21:11.331 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.331 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.332 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.661014+0000", 2024-08-17T18:21:11.332 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3379999999999999e-06 2024-08-17T18:21:11.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.332 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.661019+0000", 2024-08-17T18:21:11.332 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.9649999999999999e-06 2024-08-17T18:21:11.332 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.332 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.332 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.332 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.661031+0000", 2024-08-17T18:21:11.333 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1517e-05 2024-08-17T18:21:11.333 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.333 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.333 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.661038+0000", 2024-08-17T18:21:11.333 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.5039999999999996e-06 2024-08-17T18:21:11.333 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.333 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.333 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.680252+0000", 2024-08-17T18:21:11.333 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019213411 2024-08-17T18:21:11.333 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.333 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.333 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.334 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.682723+0000", 2024-08-17T18:21:11.334 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0024712240000000002 2024-08-17T18:21:11.334 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.334 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.335 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.335 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.717163+0000", 2024-08-17T18:21:11.335 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.034440176000000003 2024-08-17T18:21:11.335 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.335 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.335 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.335 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.720115+0000", 2024-08-17T18:21:11.335 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002952227 2024-08-17T18:21:11.335 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.335 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.468092+0000", 2024-08-17T18:21:11.336 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.74797700899999997 2024-08-17T18:21:11.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.470107+0000", 2024-08-17T18:21:11.336 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0020145240000000002 2024-08-17T18:21:11.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.336 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.336 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.336 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.474515+0000", 2024-08-17T18:21:11.336 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.004408464 2024-08-17T18:21:11.336 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.337 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.337 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.337 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.501648+0000", 2024-08-17T18:21:11.337 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.027132823 2024-08-17T18:21:11.337 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.337 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.337 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.337 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.505068+0000", 2024-08-17T18:21:11.337 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0034201750000000001 2024-08-17T18:21:11.337 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.337 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.337 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.337 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.506625+0000", 2024-08-17T18:21:11.338 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0015563 2024-08-17T18:21:11.338 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.338 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.338 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.338 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.507501+0000", 2024-08-17T18:21:11.338 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00087661399999999995 2024-08-17T18:21:11.338 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.338 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.338 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.338 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.509042+0000", 2024-08-17T18:21:11.338 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0015410070000000001 2024-08-17T18:21:11.338 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.338 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.339 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.339 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.513589+0000", 2024-08-17T18:21:11.339 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0045461429999999999 2024-08-17T18:21:11.339 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.339 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.339 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.339 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.339 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.339 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:244943 50.18 50:186237b9:::benchmark_data_smithi130_398002_object99877:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1476)", 2024-08-17T18:21:11.339 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:04.661063+0000", 2024-08-17T18:21:11.339 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 5.6690823039999998, 2024-08-17T18:21:11.339 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.89832396000000003, 2024-08-17T18:21:11.339 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.340 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.340 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.340 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.340 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.340 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.340 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 244943 2024-08-17T18:21:11.340 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.340 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.340 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.340 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.340 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.661063+0000", 2024-08-17T18:21:11.340 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.340 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.341 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.341 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.341 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.661063+0000", 2024-08-17T18:21:11.341 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.341 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.341 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.341 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.341 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.661064+0000", 2024-08-17T18:21:11.341 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.8999999999999996e-07 2024-08-17T18:21:11.341 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.341 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.341 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.341 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.661065+0000", 2024-08-17T18:21:11.341 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.579e-06 2024-08-17T18:21:11.342 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.342 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.342 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.342 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.661071+0000", 2024-08-17T18:21:11.342 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.507e-06 2024-08-17T18:21:11.342 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.342 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.342 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.342 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.661075+0000", 2024-08-17T18:21:11.342 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 4.1409999999999998e-06 2024-08-17T18:21:11.342 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.342 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.343 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.343 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.683019+0000", 2024-08-17T18:21:11.343 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.021944139000000001 2024-08-17T18:21:11.343 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.343 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.343 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.343 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.685526+0000", 2024-08-17T18:21:11.343 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025073180000000001 2024-08-17T18:21:11.343 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.343 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.343 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.343 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.730684+0000", 2024-08-17T18:21:11.343 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.045157572 2024-08-17T18:21:11.344 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.344 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.344 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.344 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.733209+0000", 2024-08-17T18:21:11.344 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025252159999999998 2024-08-17T18:21:11.344 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.344 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.344 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.344 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.520213+0000", 2024-08-17T18:21:11.344 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.78700387999999999 2024-08-17T18:21:11.344 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.344 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.344 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.344 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.522496+0000", 2024-08-17T18:21:11.345 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0022829920000000002 2024-08-17T18:21:11.345 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.345 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.345 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.345 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.526897+0000", 2024-08-17T18:21:11.345 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0044013719999999997 2024-08-17T18:21:11.345 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.345 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.345 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.345 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.550523+0000", 2024-08-17T18:21:11.345 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023625983 2024-08-17T18:21:11.345 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.345 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.345 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.346 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.551656+0000", 2024-08-17T18:21:11.346 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0011327819999999999 2024-08-17T18:21:11.346 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.346 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.346 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.346 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.552167+0000", 2024-08-17T18:21:11.346 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00051057100000000003 2024-08-17T18:21:11.346 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.346 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.346 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.346 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.553014+0000", 2024-08-17T18:21:11.346 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00084681899999999996 2024-08-17T18:21:11.346 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.347 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.347 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.347 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.554568+0000", 2024-08-17T18:21:11.347 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001554885 2024-08-17T18:21:11.347 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.347 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.347 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.347 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.559387+0000", 2024-08-17T18:21:11.347 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0048186139999999997 2024-08-17T18:21:11.347 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.347 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.347 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.347 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.348 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.348 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:244978 50.18 50:1b1bb1cf:::benchmark_data_smithi130_398002_object99912:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1476)", 2024-08-17T18:21:11.348 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:04.773012+0000", 2024-08-17T18:21:11.348 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 5.5571333049999998, 2024-08-17T18:21:11.348 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.79623477399999998, 2024-08-17T18:21:11.348 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.348 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.348 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.348 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.348 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.348 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.348 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 244978 2024-08-17T18:21:11.348 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.348 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.349 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.349 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.349 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.773012+0000", 2024-08-17T18:21:11.349 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.349 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.349 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.349 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.349 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.773012+0000", 2024-08-17T18:21:11.349 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.349 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.349 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.349 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.349 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.773014+0000", 2024-08-17T18:21:11.350 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5099999999999999e-06 2024-08-17T18:21:11.350 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.350 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.350 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.350 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.773020+0000", 2024-08-17T18:21:11.350 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.9880000000000001e-06 2024-08-17T18:21:11.350 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.350 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.350 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.350 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.773031+0000", 2024-08-17T18:21:11.350 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.1576000000000001e-05 2024-08-17T18:21:11.350 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.351 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.351 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.351 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.773038+0000", 2024-08-17T18:21:11.351 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.0389999999999999e-06 2024-08-17T18:21:11.351 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.351 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.351 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.351 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.802152+0000", 2024-08-17T18:21:11.351 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.029113727999999998 2024-08-17T18:21:11.351 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.351 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.351 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.351 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.804677+0000", 2024-08-17T18:21:11.351 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0025247400000000001 2024-08-17T18:21:11.352 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.352 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.352 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.352 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.834212+0000", 2024-08-17T18:21:11.352 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.029535266000000001 2024-08-17T18:21:11.352 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.352 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.352 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.352 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:04.837489+0000", 2024-08-17T18:21:11.352 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032767550000000001 2024-08-17T18:21:11.352 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.352 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.352 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.353 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.536860+0000", 2024-08-17T18:21:11.353 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.69937119700000006 2024-08-17T18:21:11.353 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.353 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.353 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.353 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.539291+0000", 2024-08-17T18:21:11.353 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0024314459999999999 2024-08-17T18:21:11.353 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.353 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.353 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.353 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.543984+0000", 2024-08-17T18:21:11.353 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.004692519 2024-08-17T18:21:11.353 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.353 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.354 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.354 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.560129+0000", 2024-08-17T18:21:11.354 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.016145535999999999 2024-08-17T18:21:11.354 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.354 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.354 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.354 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.560579+0000", 2024-08-17T18:21:11.354 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00044957100000000001 2024-08-17T18:21:11.354 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.354 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.354 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.354 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.561375+0000", 2024-08-17T18:21:11.354 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000795694 2024-08-17T18:21:11.355 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.355 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.562459+0000", 2024-08-17T18:21:11.355 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0010843560000000001 2024-08-17T18:21:11.355 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.355 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.564084+0000", 2024-08-17T18:21:11.355 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001625266 2024-08-17T18:21:11.355 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.355 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.355 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.355 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:05.569247+0000", 2024-08-17T18:21:11.356 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0051625869999999997 2024-08-17T18:21:11.356 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.356 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.356 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.356 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.356 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.356 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:245827 50.18 50:1be96cec:::benchmark_data_smithi130_398002_object100761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1476)", 2024-08-17T18:21:11.356 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:06.565558+0000", 2024-08-17T18:21:11.356 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 3.764587589, 2024-08-17T18:21:11.356 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.80462174099999995, 2024-08-17T18:21:11.356 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.356 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.356 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.357 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.357 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.357 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.357 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 245827 2024-08-17T18:21:11.357 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.357 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.357 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.357 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.357 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.565558+0000", 2024-08-17T18:21:11.357 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.357 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.357 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.357 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.357 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.565558+0000", 2024-08-17T18:21:11.358 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.358 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.358 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.565560+0000", 2024-08-17T18:21:11.358 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.745e-06 2024-08-17T18:21:11.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.358 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.358 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.358 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.565566+0000", 2024-08-17T18:21:11.358 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.7660000000000001e-06 2024-08-17T18:21:11.358 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.359 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.359 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.359 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.565582+0000", 2024-08-17T18:21:11.359 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5628e-05 2024-08-17T18:21:11.359 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.359 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.359 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.359 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.565590+0000", 2024-08-17T18:21:11.359 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5699999999999993e-06 2024-08-17T18:21:11.359 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.359 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.359 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.359 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.626739+0000", 2024-08-17T18:21:11.359 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.061148464999999999 2024-08-17T18:21:11.360 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.360 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.360 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.360 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.631405+0000", 2024-08-17T18:21:11.360 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0046662270000000002 2024-08-17T18:21:11.360 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.360 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.360 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.360 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.673912+0000", 2024-08-17T18:21:11.360 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.042506616999999997 2024-08-17T18:21:11.360 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.360 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.360 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.360 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.676315+0000", 2024-08-17T18:21:11.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0024033380000000001 2024-08-17T18:21:11.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.361 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.328166+0000", 2024-08-17T18:21:11.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.65185062900000001 2024-08-17T18:21:11.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.361 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.361 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.330626+0000", 2024-08-17T18:21:11.361 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002460678 2024-08-17T18:21:11.361 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.361 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.362 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.335754+0000", 2024-08-17T18:21:11.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0051272940000000001 2024-08-17T18:21:11.362 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.362 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.360247+0000", 2024-08-17T18:21:11.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.024492835000000001 2024-08-17T18:21:11.362 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.362 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.362 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.362 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.360713+0000", 2024-08-17T18:21:11.362 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00046642299999999999 2024-08-17T18:21:11.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.361493+0000", 2024-08-17T18:21:11.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00077950700000000003 2024-08-17T18:21:11.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.363 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.363077+0000", 2024-08-17T18:21:11.363 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001584645 2024-08-17T18:21:11.363 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.363 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.363 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.364936+0000", 2024-08-17T18:21:11.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0018592140000000001 2024-08-17T18:21:11.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.364 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.364 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.370180+0000", 2024-08-17T18:21:11.364 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0052431600000000002 2024-08-17T18:21:11.364 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.364 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.364 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.364 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.364 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.364 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:245865 50.18 50:18b6c2e4:::benchmark_data_smithi130_398002_object100799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1476)", 2024-08-17T18:21:11.364 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:06.634127+0000", 2024-08-17T18:21:11.365 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 3.6960186889999997, 2024-08-17T18:21:11.365 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.76541941899999999, 2024-08-17T18:21:11.365 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.365 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.365 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.365 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.365 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.365 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.365 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 245865 2024-08-17T18:21:11.365 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.365 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.365 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.365 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.366 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.634127+0000", 2024-08-17T18:21:11.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.366 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.366 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.366 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.634127+0000", 2024-08-17T18:21:11.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.366 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.366 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.366 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.366 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.634128+0000", 2024-08-17T18:21:11.366 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.584e-06 2024-08-17T18:21:11.366 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.367 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.367 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.367 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.634134+0000", 2024-08-17T18:21:11.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.677e-06 2024-08-17T18:21:11.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.367 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.367 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.367 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.634151+0000", 2024-08-17T18:21:11.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.7487e-05 2024-08-17T18:21:11.367 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.367 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.367 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.367 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.634159+0000", 2024-08-17T18:21:11.367 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 7.4619999999999999e-06 2024-08-17T18:21:11.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.677534+0000", 2024-08-17T18:21:11.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.043375190000000001 2024-08-17T18:21:11.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.368 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.682410+0000", 2024-08-17T18:21:11.368 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0048756049999999999 2024-08-17T18:21:11.368 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.368 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.368 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.715122+0000", 2024-08-17T18:21:11.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.032712499999999999 2024-08-17T18:21:11.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.369 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.717729+0000", 2024-08-17T18:21:11.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0026065709999999998 2024-08-17T18:21:11.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.369 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.369 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.376514+0000", 2024-08-17T18:21:11.369 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.65878523899999997 2024-08-17T18:21:11.369 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.369 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.378594+0000", 2024-08-17T18:21:11.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002079961 2024-08-17T18:21:11.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.382949+0000", 2024-08-17T18:21:11.370 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0043549210000000003 2024-08-17T18:21:11.370 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.370 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.370 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.370 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.391017+0000", 2024-08-17T18:21:11.371 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0080684950000000002 2024-08-17T18:21:11.371 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.371 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.371 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.373 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.391929+0000", 2024-08-17T18:21:11.373 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00091117100000000001 2024-08-17T18:21:11.373 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.373 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.373 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.373 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.392358+0000", 2024-08-17T18:21:11.373 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00042928900000000002 2024-08-17T18:21:11.373 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.373 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.373 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.374 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.393129+0000", 2024-08-17T18:21:11.374 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00077078800000000001 2024-08-17T18:21:11.374 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.374 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.374 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.374 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.394648+0000", 2024-08-17T18:21:11.374 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00151951 2024-08-17T18:21:11.374 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.374 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.374 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.374 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.399546+0000", 2024-08-17T18:21:11.374 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0048979690000000003 2024-08-17T18:21:11.374 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:245899 50.18 50:19d8341e:::benchmark_data_smithi130_398002_object100833:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1476)", 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:06.706002+0000", 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 3.6241430340000003, 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.76771683000000002, 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.375 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 245899 2024-08-17T18:21:11.376 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.376 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.376 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.376 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.376 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.706002+0000", 2024-08-17T18:21:11.376 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.376 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.376 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.376 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.376 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.706002+0000", 2024-08-17T18:21:11.376 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.376 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.376 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.376 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.377 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.706004+0000", 2024-08-17T18:21:11.377 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.553e-06 2024-08-17T18:21:11.377 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.377 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.377 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.377 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.706009+0000", 2024-08-17T18:21:11.377 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 5.5620000000000003e-06 2024-08-17T18:21:11.377 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.377 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.377 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.377 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.706023+0000", 2024-08-17T18:21:11.377 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.3088000000000001e-05 2024-08-17T18:21:11.377 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.377 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.378 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.378 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.706031+0000", 2024-08-17T18:21:11.378 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.1599999999999998e-06 2024-08-17T18:21:11.378 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.378 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.378 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.378 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.729080+0000", 2024-08-17T18:21:11.378 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023049371999999999 2024-08-17T18:21:11.378 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.378 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.378 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.378 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.732898+0000", 2024-08-17T18:21:11.379 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0038177279999999998 2024-08-17T18:21:11.379 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.379 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.379 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.379 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.750313+0000", 2024-08-17T18:21:11.379 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.017415039 2024-08-17T18:21:11.379 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.379 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.379 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.379 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:06.753554+0000", 2024-08-17T18:21:11.379 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0032410640000000001 2024-08-17T18:21:11.379 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.380 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.380 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.380 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.427526+0000", 2024-08-17T18:21:11.380 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.67397218599999997 2024-08-17T18:21:11.380 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.380 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.380 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.380 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.429921+0000", 2024-08-17T18:21:11.380 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0023945250000000002 2024-08-17T18:21:11.380 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.380 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.380 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.380 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.434694+0000", 2024-08-17T18:21:11.381 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0047732290000000004 2024-08-17T18:21:11.381 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.381 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.381 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.381 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.454375+0000", 2024-08-17T18:21:11.381 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.019681315000000001 2024-08-17T18:21:11.381 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.381 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.381 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.381 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.465395+0000", 2024-08-17T18:21:11.381 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.011019394 2024-08-17T18:21:11.381 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.381 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.382 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.382 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.466313+0000", 2024-08-17T18:21:11.382 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00091827900000000004 2024-08-17T18:21:11.382 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.382 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.382 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.382 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.467358+0000", 2024-08-17T18:21:11.382 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.001044894 2024-08-17T18:21:11.382 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.382 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.382 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.382 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.469003+0000", 2024-08-17T18:21:11.382 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0016449259999999999 2024-08-17T18:21:11.383 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.383 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.383 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.383 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:07.473719+0000", 2024-08-17T18:21:11.383 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0047165159999999996 2024-08-17T18:21:11.383 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.383 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.383 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.383 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.383 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.383 INFO:teuthology.orchestra.run.smithi130.stdout: "description": "osd_op(client.47132.0:246827 50.18 50:188a90da:::benchmark_data_smithi130_398002_object101761:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1476)", 2024-08-17T18:21:11.383 INFO:teuthology.orchestra.run.smithi130.stdout: "initiated_at": "2024-08-17T18:21:08.469835+0000", 2024-08-17T18:21:11.383 INFO:teuthology.orchestra.run.smithi130.stdout: "age": 1.8603100079999999, 2024-08-17T18:21:11.383 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.76223359700000004, 2024-08-17T18:21:11.384 INFO:teuthology.orchestra.run.smithi130.stdout: "continuous": false, 2024-08-17T18:21:11.384 INFO:teuthology.orchestra.run.smithi130.stdout: "type_data": { 2024-08-17T18:21:11.384 INFO:teuthology.orchestra.run.smithi130.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:21:11.384 INFO:teuthology.orchestra.run.smithi130.stdout: "client_info": { 2024-08-17T18:21:11.384 INFO:teuthology.orchestra.run.smithi130.stdout: "client": "client.47132", 2024-08-17T18:21:11.384 INFO:teuthology.orchestra.run.smithi130.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:21:11.384 INFO:teuthology.orchestra.run.smithi130.stdout: "tid": 246827 2024-08-17T18:21:11.384 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.384 INFO:teuthology.orchestra.run.smithi130.stdout: "events": [ 2024-08-17T18:21:11.384 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.384 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "initiated", 2024-08-17T18:21:11.384 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:08.469835+0000", 2024-08-17T18:21:11.384 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.385 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.385 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.385 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "header_read", 2024-08-17T18:21:11.385 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:08.469835+0000", 2024-08-17T18:21:11.385 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0 2024-08-17T18:21:11.385 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.385 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.385 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "throttled", 2024-08-17T18:21:11.385 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:08.469837+0000", 2024-08-17T18:21:11.385 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.4360000000000001e-06 2024-08-17T18:21:11.385 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.385 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.385 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "all_read", 2024-08-17T18:21:11.386 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:08.469843+0000", 2024-08-17T18:21:11.386 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 6.5799999999999997e-06 2024-08-17T18:21:11.386 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.386 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.386 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "dispatched", 2024-08-17T18:21:11.386 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:08.469859+0000", 2024-08-17T18:21:11.386 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 1.5164000000000001e-05 2024-08-17T18:21:11.386 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.386 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.386 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "queued_for_pg", 2024-08-17T18:21:11.386 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:08.469867+0000", 2024-08-17T18:21:11.386 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 8.5820000000000002e-06 2024-08-17T18:21:11.386 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.387 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.387 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.387 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:08.500634+0000", 2024-08-17T18:21:11.387 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.030767129000000001 2024-08-17T18:21:11.387 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.387 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.387 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for missing object", 2024-08-17T18:21:11.387 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:08.503046+0000", 2024-08-17T18:21:11.387 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.002412048 2024-08-17T18:21:11.387 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.387 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.387 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.387 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:08.553325+0000", 2024-08-17T18:21:11.387 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.050278940000000001 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for rw locks", 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:08.556164+0000", 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0028390770000000002 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "reached_pg", 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:09.191976+0000", 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.63581194600000002 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "started", 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:09.194397+0000", 2024-08-17T18:21:11.388 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0024209259999999999 2024-08-17T18:21:11.389 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.389 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.389 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "waiting for subops from 0,3,6", 2024-08-17T18:21:11.389 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:09.199455+0000", 2024-08-17T18:21:11.389 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.005058178 2024-08-17T18:21:11.389 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.389 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.389 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "op_commit", 2024-08-17T18:21:11.389 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:09.223275+0000", 2024-08-17T18:21:11.389 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.023819801000000002 2024-08-17T18:21:11.389 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.389 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.389 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.390 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:09.223930+0000", 2024-08-17T18:21:11.390 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.00065503899999999995 2024-08-17T18:21:11.390 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.390 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.390 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.390 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:09.224770+0000", 2024-08-17T18:21:11.390 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.000840232 2024-08-17T18:21:11.390 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.390 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.390 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:21:11.390 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:09.225994+0000", 2024-08-17T18:21:11.390 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0012237159999999999 2024-08-17T18:21:11.390 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.391 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.391 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "commit_sent", 2024-08-17T18:21:11.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:09.227825+0000", 2024-08-17T18:21:11.391 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0018305350000000001 2024-08-17T18:21:11.391 INFO:teuthology.orchestra.run.smithi130.stdout: }, 2024-08-17T18:21:11.391 INFO:teuthology.orchestra.run.smithi130.stdout: { 2024-08-17T18:21:11.391 INFO:teuthology.orchestra.run.smithi130.stdout: "event": "done", 2024-08-17T18:21:11.391 INFO:teuthology.orchestra.run.smithi130.stdout: "time": "2024-08-17T18:21:09.232069+0000", 2024-08-17T18:21:11.391 INFO:teuthology.orchestra.run.smithi130.stdout: "duration": 0.0042442679999999998 2024-08-17T18:21:11.391 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.391 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.391 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.391 INFO:teuthology.orchestra.run.smithi130.stdout: } 2024-08-17T18:21:11.391 INFO:teuthology.orchestra.run.smithi130.stdout: ] 2024-08-17T18:21:11.392 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:21:11.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:11.392 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:11.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:11.421 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:21:11.493 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:11.517 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:11.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:11.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:11.619 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:11.643 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:11.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:11.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:11.746 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:11.768 INFO:teuthology.orchestra.run.smithi130.stderr:marked out osd.5. 2024-08-17T18:21:11.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:11.775 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:11.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:11.876 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:11.900 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:11.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:11.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:12.002 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:12.014 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:11 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/535329516' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-17T18:21:12.015 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:11 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-17T18:21:12.015 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:11 smithi130 ceph-mon[105521]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-17T18:21:12.015 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 17708/130644 objects degraded (13.554%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:21:12.015 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:11 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/535329516' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-17T18:21:12.015 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:11 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-17T18:21:12.015 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:11 smithi130 ceph-mon[110068]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-17T18:21:12.015 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 17708/130644 objects degraded (13.554%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:21:12.029 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:12.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:12.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:11 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/535329516' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-17T18:21:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:11 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["5"]} : dispatch 2024-08-17T18:21:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:11 smithi179 ceph-mon[93519]: Client client.admin marked osd.5 out, while it was still marked up 2024-08-17T18:21:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 17708/130644 objects degraded (13.554%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T18:21:12.131 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:12.155 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:12.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:12.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:12.256 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:12.279 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:12.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:12.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:12.305 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T18:21:12.381 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:12.406 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:12.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:12.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:12.508 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:12.534 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:12.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:12.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:12.636 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:12.659 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:12.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:12.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:12.761 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:12.802 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:12.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:12.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:12.904 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:12.938 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:12.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:12.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:12.942 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:12 smithi130 ceph-mon[105521]: pgmap v2874: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 17708/130644 objects degraded (13.554%); 36905/130644 objects misplaced (28.249%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T18:21:12.942 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:12 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-17T18:21:12.942 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:12 smithi130 ceph-mon[105521]: osdmap e1477: 8 total, 8 up, 6 in 2024-08-17T18:21:12.942 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:12 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:12.942 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:21:12.942 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:12 smithi130 ceph-mon[110068]: pgmap v2874: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 17708/130644 objects degraded (13.554%); 36905/130644 objects misplaced (28.249%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T18:21:12.942 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:12 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-17T18:21:12.942 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:12 smithi130 ceph-mon[110068]: osdmap e1477: 8 total, 8 up, 6 in 2024-08-17T18:21:12.942 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:12 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:12.943 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:21:13.040 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:13.063 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:13.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:13.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:13.080 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:21:13.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:12 smithi179 ceph-mon[93519]: pgmap v2874: 99 pgs: 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 10 active+undersized+degraded+remapped+backfill_wait, 9 active+remapped+backfill_wait, 77 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 476 op/s; 17708/130644 objects degraded (13.554%); 36905/130644 objects misplaced (28.249%); 3.6 MiB/s, 57 objects/s recovering 2024-08-17T18:21:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:12 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2024-08-17T18:21:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:12 smithi179 ceph-mon[93519]: osdmap e1477: 8 total, 8 up, 6 in 2024-08-17T18:21:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:12 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:21:13.095 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:21:13.114 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:21:13.165 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:13.190 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:13.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:13.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:13.292 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:13.314 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:13.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:13.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:13.416 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:13.437 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:13.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:13.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:13.539 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:13.565 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:13.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:13.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:13.667 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:13.690 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:13.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:13.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:13.792 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:13.819 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:13.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:13.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:13.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:13.956 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:13.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:13.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:14.031 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: 50.4 continuing backfill to osd.2 from (1439'2075,1476'11222] MIN to 1476'11222 2024-08-17T18:21:14.031 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: 50.17 continuing backfill to osd.2 from (1439'2783,1476'11053] MIN to 1476'11053 2024-08-17T18:21:14.031 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: 50.17 starting backfill to osd.4 from (0'0,0'0] MAX to 1476'11053 2024-08-17T18:21:14.031 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: 50.10 continuing backfill to osd.2 from (1438'900,1476'11176] 50:08a65aa2:::benchmark_data_smithi130_398002_object116454:head to 1476'11176 2024-08-17T18:21:14.032 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: 50.10 continuing backfill to osd.4 from (1438'900,1476'11176] 50:09efc84d:::benchmark_data_smithi130_398002_object91543:head to 1476'11176 2024-08-17T18:21:14.032 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: osdmap e1478: 8 total, 8 up, 6 in 2024-08-17T18:21:14.032 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: 50.15 continuing backfill to osd.2 from (1440'5054,1476'11067] MIN to 1476'11067 2024-08-17T18:21:14.032 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: 50.15 starting backfill to osd.3 from (0'0,0'0] MAX to 1476'11067 2024-08-17T18:21:14.032 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: 50.2 continuing backfill to osd.4 from (1440'5439,1476'11221] 50:425d3f84:::benchmark_data_smithi130_398002_object143159:head to 1476'11221 2024-08-17T18:21:14.032 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: 50.5 continuing backfill to osd.2 from (1440'5294,1476'11139] MIN to 1476'11139 2024-08-17T18:21:14.032 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: 50.5 starting backfill to osd.3 from (0'0,0'0] MAX to 1476'11139 2024-08-17T18:21:14.032 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: 50.12 continuing backfill to osd.4 from (1440'5214,1476'11137] MIN to 1476'11137 2024-08-17T18:21:14.032 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: 50.7 continuing backfill to osd.2 from (1439'2582,1476'10992] MIN to 1476'10992 2024-08-17T18:21:14.032 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: 50.7 starting backfill to osd.4 from (0'0,0'0] MAX to 1476'10992 2024-08-17T18:21:14.032 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: 50.b continuing backfill to osd.4 from (1451'10478,1476'15689] MIN to 1476'15689 2024-08-17T18:21:14.032 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[105521]: osdmap e1479: 8 total, 8 up, 6 in 2024-08-17T18:21:14.033 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: 50.4 continuing backfill to osd.2 from (1439'2075,1476'11222] MIN to 1476'11222 2024-08-17T18:21:14.033 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: 50.17 continuing backfill to osd.2 from (1439'2783,1476'11053] MIN to 1476'11053 2024-08-17T18:21:14.033 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: 50.17 starting backfill to osd.4 from (0'0,0'0] MAX to 1476'11053 2024-08-17T18:21:14.033 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: 50.10 continuing backfill to osd.2 from (1438'900,1476'11176] 50:08a65aa2:::benchmark_data_smithi130_398002_object116454:head to 1476'11176 2024-08-17T18:21:14.033 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: 50.10 continuing backfill to osd.4 from (1438'900,1476'11176] 50:09efc84d:::benchmark_data_smithi130_398002_object91543:head to 1476'11176 2024-08-17T18:21:14.033 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: osdmap e1478: 8 total, 8 up, 6 in 2024-08-17T18:21:14.033 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: 50.15 continuing backfill to osd.2 from (1440'5054,1476'11067] MIN to 1476'11067 2024-08-17T18:21:14.033 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: 50.15 starting backfill to osd.3 from (0'0,0'0] MAX to 1476'11067 2024-08-17T18:21:14.033 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: 50.2 continuing backfill to osd.4 from (1440'5439,1476'11221] 50:425d3f84:::benchmark_data_smithi130_398002_object143159:head to 1476'11221 2024-08-17T18:21:14.033 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: 50.5 continuing backfill to osd.2 from (1440'5294,1476'11139] MIN to 1476'11139 2024-08-17T18:21:14.033 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: 50.5 starting backfill to osd.3 from (0'0,0'0] MAX to 1476'11139 2024-08-17T18:21:14.033 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: 50.12 continuing backfill to osd.4 from (1440'5214,1476'11137] MIN to 1476'11137 2024-08-17T18:21:14.034 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: 50.7 continuing backfill to osd.2 from (1439'2582,1476'10992] MIN to 1476'10992 2024-08-17T18:21:14.034 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: 50.7 starting backfill to osd.4 from (0'0,0'0] MAX to 1476'10992 2024-08-17T18:21:14.034 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: 50.b continuing backfill to osd.4 from (1451'10478,1476'15689] MIN to 1476'15689 2024-08-17T18:21:14.034 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:13 smithi130 ceph-mon[110068]: osdmap e1479: 8 total, 8 up, 6 in 2024-08-17T18:21:14.058 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:14.083 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:14.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:14.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: 50.4 continuing backfill to osd.2 from (1439'2075,1476'11222] MIN to 1476'11222 2024-08-17T18:21:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: 50.17 continuing backfill to osd.2 from (1439'2783,1476'11053] MIN to 1476'11053 2024-08-17T18:21:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: 50.17 starting backfill to osd.4 from (0'0,0'0] MAX to 1476'11053 2024-08-17T18:21:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: 50.10 continuing backfill to osd.2 from (1438'900,1476'11176] 50:08a65aa2:::benchmark_data_smithi130_398002_object116454:head to 1476'11176 2024-08-17T18:21:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: 50.10 continuing backfill to osd.4 from (1438'900,1476'11176] 50:09efc84d:::benchmark_data_smithi130_398002_object91543:head to 1476'11176 2024-08-17T18:21:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: osdmap e1478: 8 total, 8 up, 6 in 2024-08-17T18:21:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: 50.15 continuing backfill to osd.2 from (1440'5054,1476'11067] MIN to 1476'11067 2024-08-17T18:21:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: 50.15 starting backfill to osd.3 from (0'0,0'0] MAX to 1476'11067 2024-08-17T18:21:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: 50.2 continuing backfill to osd.4 from (1440'5439,1476'11221] 50:425d3f84:::benchmark_data_smithi130_398002_object143159:head to 1476'11221 2024-08-17T18:21:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: 50.5 continuing backfill to osd.2 from (1440'5294,1476'11139] MIN to 1476'11139 2024-08-17T18:21:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: 50.5 starting backfill to osd.3 from (0'0,0'0] MAX to 1476'11139 2024-08-17T18:21:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: 50.12 continuing backfill to osd.4 from (1440'5214,1476'11137] MIN to 1476'11137 2024-08-17T18:21:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: 50.7 continuing backfill to osd.2 from (1439'2582,1476'10992] MIN to 1476'10992 2024-08-17T18:21:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: 50.7 starting backfill to osd.4 from (0'0,0'0] MAX to 1476'10992 2024-08-17T18:21:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: 50.b continuing backfill to osd.4 from (1451'10478,1476'15689] MIN to 1476'15689 2024-08-17T18:21:14.094 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:13 smithi179 ceph-mon[93519]: osdmap e1479: 8 total, 8 up, 6 in 2024-08-17T18:21:14.186 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:14.215 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:14.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:14.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:14.317 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:14.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:14.366 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:14.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:14.467 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:14.493 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:14.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:14.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:14.595 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:14.622 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:14.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:14.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:14.724 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:14.751 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:14.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:14.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:14.853 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:14.879 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:14.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:14.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:14.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:14 smithi130 ceph-mon[110068]: pgmap v2877: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 3 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 8 remapped+peering, 7 active+undersized+degraded+remapped+backfill_wait, 4 active+remapped+backfill_wait, 72 active+clean; 2.6 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 515 op/s; 13675/125622 objects degraded (10.886%); 17532/125622 objects misplaced (13.956%); 13 MiB/s, 202 objects/s recovering 2024-08-17T18:21:14.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:14 smithi130 ceph-mon[110068]: 50.0 continuing backfill to osd.2 from (1438'900,1476'11087] 50:013c81a5:::benchmark_data_smithi130_398002_object38821:head to 1476'11087 2024-08-17T18:21:14.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:14 smithi130 ceph-mon[110068]: 50.14 starting backfill to osd.6 from (1439'1603,1472'11023] MIN to 1476'11225 2024-08-17T18:21:14.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:14 smithi130 ceph-mon[110068]: 50.d starting backfill to osd.2 from (0'0,0'0] MAX to 1476'15393 2024-08-17T18:21:14.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:14 smithi130 ceph-mon[105521]: pgmap v2877: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 3 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 8 remapped+peering, 7 active+undersized+degraded+remapped+backfill_wait, 4 active+remapped+backfill_wait, 72 active+clean; 2.6 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 515 op/s; 13675/125622 objects degraded (10.886%); 17532/125622 objects misplaced (13.956%); 13 MiB/s, 202 objects/s recovering 2024-08-17T18:21:14.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:14 smithi130 ceph-mon[105521]: 50.0 continuing backfill to osd.2 from (1438'900,1476'11087] 50:013c81a5:::benchmark_data_smithi130_398002_object38821:head to 1476'11087 2024-08-17T18:21:14.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:14 smithi130 ceph-mon[105521]: 50.14 starting backfill to osd.6 from (1439'1603,1472'11023] MIN to 1476'11225 2024-08-17T18:21:14.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:14 smithi130 ceph-mon[105521]: 50.d starting backfill to osd.2 from (0'0,0'0] MAX to 1476'15393 2024-08-17T18:21:14.981 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:15.020 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:15.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:15.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:15.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:14 smithi179 ceph-mon[93519]: pgmap v2877: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 3 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded+remapped, 8 remapped+peering, 7 active+undersized+degraded+remapped+backfill_wait, 4 active+remapped+backfill_wait, 72 active+clean; 2.6 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 515 op/s; 13675/125622 objects degraded (10.886%); 17532/125622 objects misplaced (13.956%); 13 MiB/s, 202 objects/s recovering 2024-08-17T18:21:15.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:14 smithi179 ceph-mon[93519]: 50.0 continuing backfill to osd.2 from (1438'900,1476'11087] 50:013c81a5:::benchmark_data_smithi130_398002_object38821:head to 1476'11087 2024-08-17T18:21:15.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:14 smithi179 ceph-mon[93519]: 50.14 starting backfill to osd.6 from (1439'1603,1472'11023] MIN to 1476'11225 2024-08-17T18:21:15.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:14 smithi179 ceph-mon[93519]: 50.d starting backfill to osd.2 from (0'0,0'0] MAX to 1476'15393 2024-08-17T18:21:15.122 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:15.321 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:15.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:15.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:15.423 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:15.454 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:15.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:15.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:15.556 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:15.578 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:15.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:15.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:15.680 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:15.707 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:15.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:15.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:15.809 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:15.834 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:15.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:15.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:15.936 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:15.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:15.978 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:15.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:16.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:16.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:15 smithi179 ceph-mon[93519]: osdmap e1480: 8 total, 8 up, 6 in 2024-08-17T18:21:16.114 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:16.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:16.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:16.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:15 smithi130 ceph-mon[105521]: osdmap e1480: 8 total, 8 up, 6 in 2024-08-17T18:21:16.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:15 smithi130 ceph-mon[110068]: osdmap e1480: 8 total, 8 up, 6 in 2024-08-17T18:21:16.216 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:16.247 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:16.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:16.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:16.349 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:16.370 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:16.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:16.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:16.472 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:16.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:16.494 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:16.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:16.595 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:16.616 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:16.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:16.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:16.717 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:16.745 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:16.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:16.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:16.847 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:16.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:16.895 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:16.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:16.902 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:16 smithi130 ceph-mon[105521]: pgmap v2880: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 3 peering, 1 active+undersized+degraded+remapped+backfilling, 8 remapped+peering, 7 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 71 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 445 op/s; 13541/125301 objects degraded (10.807%); 17634/125301 objects misplaced (14.073%); 26 MiB/s, 415 objects/s recovering 2024-08-17T18:21:16.902 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:16 smithi130 ceph-mon[105521]: osdmap e1481: 8 total, 8 up, 6 in 2024-08-17T18:21:16.902 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 13541/125301 objects degraded (10.807%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T18:21:16.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:16 smithi130 ceph-mon[110068]: pgmap v2880: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 3 peering, 1 active+undersized+degraded+remapped+backfilling, 8 remapped+peering, 7 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 71 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 445 op/s; 13541/125301 objects degraded (10.807%); 17634/125301 objects misplaced (14.073%); 26 MiB/s, 415 objects/s recovering 2024-08-17T18:21:16.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:16 smithi130 ceph-mon[110068]: osdmap e1481: 8 total, 8 up, 6 in 2024-08-17T18:21:16.903 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 13541/125301 objects degraded (10.807%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T18:21:16.995 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:17.044 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:17.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:17.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:17.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:16 smithi179 ceph-mon[93519]: pgmap v2880: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 3 peering, 1 active+undersized+degraded+remapped+backfilling, 8 remapped+peering, 7 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 71 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 445 op/s; 13541/125301 objects degraded (10.807%); 17634/125301 objects misplaced (14.073%); 26 MiB/s, 415 objects/s recovering 2024-08-17T18:21:17.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:16 smithi179 ceph-mon[93519]: osdmap e1481: 8 total, 8 up, 6 in 2024-08-17T18:21:17.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 13541/125301 objects degraded (10.807%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T18:21:17.145 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:17.194 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:17.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:17.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:17.297 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:17.341 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:17.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:17.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:17.443 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:17.469 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:17.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:17.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:17.571 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:17.623 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:17.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:17.681 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:21:17.681 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:21:17.681 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:21:17.681 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:21:17.682 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:21:17.682 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:21:17.682 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:21:17.682 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:21:17.682 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:21:17.682 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:21:17.682 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:21:17.682 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:21:17.682 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:21:17.682 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:21:17.698 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:21:17.698 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:21:17.723 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:17.762 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:17.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:17.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:17.864 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:17.897 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:17.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:17.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:17.999 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:18.025 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:18.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:18.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:18.044 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 7, 1, 6, 4] out_osds: [0, 5] dead_osds: [] live_osds: [1, 4, 5, 6, 7, 0, 3, 2] 2024-08-17T18:21:18.044 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:21:18.044 INFO:tasks.thrashosds.thrasher:Adding osd 0 2024-08-17T18:21:18.044 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd in 0 2024-08-17T18:21:18.088 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:17 smithi130 ceph-mon[110068]: osdmap e1482: 8 total, 8 up, 6 in 2024-08-17T18:21:18.088 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:17 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1277092415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:21:18.089 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:17 smithi130 ceph-mon[105521]: osdmap e1482: 8 total, 8 up, 6 in 2024-08-17T18:21:18.089 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:17 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1277092415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:21:18.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:17 smithi179 ceph-mon[93519]: osdmap e1482: 8 total, 8 up, 6 in 2024-08-17T18:21:18.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:17 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1277092415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:21:18.128 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:18.153 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:18.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:18.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:18.255 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:18.277 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:18.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:18.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:18.379 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:18.409 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:18.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:18.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:18.511 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:18.534 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:18.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:18.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:18.635 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:18.662 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:18.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:18.764 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:18.785 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:18.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:18.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:18.832 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:21:18.887 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:18.922 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:18.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:18.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:19.024 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:19.051 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:19.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:19.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:18 smithi179 ceph-mon[93519]: pgmap v2883: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 1 active+clean+remapped, 1 active+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 remapped+peering, 9 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 74 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 16047/124509 objects degraded (12.888%); 23939/124509 objects misplaced (19.227%); 17 MiB/s, 269 objects/s recovering 2024-08-17T18:21:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:18 smithi179 ceph-mon[93519]: osdmap e1483: 8 total, 8 up, 6 in 2024-08-17T18:21:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4232645076' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:21:19.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:18 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:21:19.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:18 smithi130 ceph-mon[105521]: pgmap v2883: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 1 active+clean+remapped, 1 active+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 remapped+peering, 9 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 74 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 16047/124509 objects degraded (12.888%); 23939/124509 objects misplaced (19.227%); 17 MiB/s, 269 objects/s recovering 2024-08-17T18:21:19.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:18 smithi130 ceph-mon[105521]: osdmap e1483: 8 total, 8 up, 6 in 2024-08-17T18:21:19.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4232645076' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:21:19.146 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:18 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:21:19.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:18 smithi130 ceph-mon[110068]: pgmap v2883: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 1 active+clean+remapped, 1 active+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 3 remapped+peering, 9 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 74 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 16047/124509 objects degraded (12.888%); 23939/124509 objects misplaced (19.227%); 17 MiB/s, 269 objects/s recovering 2024-08-17T18:21:19.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:18 smithi130 ceph-mon[110068]: osdmap e1483: 8 total, 8 up, 6 in 2024-08-17T18:21:19.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4232645076' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:21:19.146 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:18 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:21:19.153 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:19.177 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:19.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:19.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:19.280 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:19.314 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:19.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:19.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:19.416 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:19.443 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:19.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:19.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:19.545 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:19.585 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:19.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:19.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:19.687 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:19.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:19.724 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:19.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:19.825 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:19.851 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:19.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:19.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:19.952 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:19.978 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:19.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:19.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:20.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:20.081 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:19 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:21:20.081 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:19 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:21:20.081 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:19 smithi130 ceph-mon[110068]: osdmap e1484: 8 total, 8 up, 6 in 2024-08-17T18:21:20.081 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:21:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:21:19] "GET /metrics HTTP/1.1" 200 36809 "" "Prometheus/2.51.0" 2024-08-17T18:21:20.082 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:19 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:21:20.082 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:19 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:21:20.082 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:19 smithi130 ceph-mon[105521]: osdmap e1484: 8 total, 8 up, 6 in 2024-08-17T18:21:20.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:20.121 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:20.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:20.213 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:19 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:21:20.214 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:19 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:21:20.214 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:19 smithi179 ceph-mon[93519]: osdmap e1484: 8 total, 8 up, 6 in 2024-08-17T18:21:20.222 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:20.252 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:20.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:20.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:20.354 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:20.378 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:20.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:20.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:20.424 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:21:20.424 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:21:20.480 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:20.504 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:20.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:20.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:20.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:21:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:21:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:21:20.606 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:20.632 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:20.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:20.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:20.733 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:20.760 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:20.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:20.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:20.862 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:20.890 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:20.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:20.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:20.992 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:21.021 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:21.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:21.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:21.122 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:21.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:20 smithi130 ceph-mon[105521]: pgmap v2886: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 14089/119220 objects degraded (11.818%); 27640/119220 objects misplaced (23.184%); 43 MiB/s, 690 objects/s recovering 2024-08-17T18:21:21.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:20 smithi130 ceph-mon[105521]: osdmap e1485: 8 total, 8 up, 6 in 2024-08-17T18:21:21.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:20 smithi130 ceph-mon[110068]: pgmap v2886: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 14089/119220 objects degraded (11.818%); 27640/119220 objects misplaced (23.184%); 43 MiB/s, 690 objects/s recovering 2024-08-17T18:21:21.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:20 smithi130 ceph-mon[110068]: osdmap e1485: 8 total, 8 up, 6 in 2024-08-17T18:21:21.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:21.168 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:21.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:21.190 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:21:21.269 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:21.295 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:21.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:21.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:21.320 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T18:21:21.325 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:20 smithi179 ceph-mon[93519]: pgmap v2886: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 506 op/s; 14089/119220 objects degraded (11.818%); 27640/119220 objects misplaced (23.184%); 43 MiB/s, 690 objects/s recovering 2024-08-17T18:21:21.325 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:20 smithi179 ceph-mon[93519]: osdmap e1485: 8 total, 8 up, 6 in 2024-08-17T18:21:21.397 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:21.426 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:21.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:21.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:21.527 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:21.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:21.567 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:21.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:21.668 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:21.693 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:21.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:21.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:21.761 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:21:21.795 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:21.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:21.846 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:21.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:21.859 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:21 smithi130 ceph-mon[105521]: osdmap e1486: 8 total, 8 up, 6 in 2024-08-17T18:21:21.947 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:21.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:21.986 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:21.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:22.087 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:22.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:21:22.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:22.131 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:22.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:22.169 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 14089/119220 objects degraded (11.818%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:21:22.169 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:21 smithi130 ceph-mon[110068]: osdmap e1486: 8 total, 8 up, 6 in 2024-08-17T18:21:22.170 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 14089/119220 objects degraded (11.818%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:21:22.194 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:21:22.232 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:22.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:22.290 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:22.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:22.338 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:21 smithi179 ceph-mon[93519]: osdmap e1486: 8 total, 8 up, 6 in 2024-08-17T18:21:22.338 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 14089/119220 objects degraded (11.818%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:21:22.391 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:22.414 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:22.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:22.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:22.516 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:22.538 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:22.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:22.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:22.639 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:22.662 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:22.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:22.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:22.764 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:22.786 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:22.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:22.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:22.887 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:22.929 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:22.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:22.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:23.031 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:23.073 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:23.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:23.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:23.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:22 smithi130 ceph-mon[105521]: pgmap v2889: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 440 op/s; 14089/119220 objects degraded (11.818%); 27640/119220 objects misplaced (23.184%); 29 MiB/s, 456 objects/s recovering 2024-08-17T18:21:23.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:22 smithi130 ceph-mon[105521]: osdmap e1487: 8 total, 8 up, 6 in 2024-08-17T18:21:23.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:22 smithi130 ceph-mon[110068]: pgmap v2889: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 440 op/s; 14089/119220 objects degraded (11.818%); 27640/119220 objects misplaced (23.184%); 29 MiB/s, 456 objects/s recovering 2024-08-17T18:21:23.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:22 smithi130 ceph-mon[110068]: osdmap e1487: 8 total, 8 up, 6 in 2024-08-17T18:21:23.174 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:23.214 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:23.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:23.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:23.315 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:22 smithi179 ceph-mon[93519]: pgmap v2889: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 76 active+clean; 2.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 440 op/s; 14089/119220 objects degraded (11.818%); 27640/119220 objects misplaced (23.184%); 29 MiB/s, 456 objects/s recovering 2024-08-17T18:21:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:22 smithi179 ceph-mon[93519]: osdmap e1487: 8 total, 8 up, 6 in 2024-08-17T18:21:23.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:23.344 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:23.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:23.445 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:23.485 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:23.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:23.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:23.587 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:23.616 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:23.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:23.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:23.718 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:23.749 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:23.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:23.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:23.850 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:23.894 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:23.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:23.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:23.996 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:24.021 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:24.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:24.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:24.123 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:24.144 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:23 smithi130 ceph-mon[105521]: osdmap e1488: 8 total, 8 up, 6 in 2024-08-17T18:21:24.144 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:23 smithi130 ceph-mon[110068]: osdmap e1488: 8 total, 8 up, 6 in 2024-08-17T18:21:24.144 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:24.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:24.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:24.246 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:24.278 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:24.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:24.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:24.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:23 smithi179 ceph-mon[93519]: osdmap e1488: 8 total, 8 up, 6 in 2024-08-17T18:21:24.380 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:24.402 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:24.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:24.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:24.503 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:24.524 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:24.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:24.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:24.626 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:24.647 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:24.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:24.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:24.749 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:24.773 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:24.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:24.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:24.875 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:24.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:24.907 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:24.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:25.008 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:25.037 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:25.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:25.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:25.139 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:25.161 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:25.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:25.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:25.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:24 smithi130 ceph-mon[105521]: pgmap v2892: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 78 active+clean; 2.3 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 661 op/s; 12617/111282 objects degraded (11.338%); 25897/111282 objects misplaced (23.272%); 10 MiB/s, 163 objects/s recovering 2024-08-17T18:21:25.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:24 smithi130 ceph-mon[110068]: pgmap v2892: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 78 active+clean; 2.3 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 661 op/s; 12617/111282 objects degraded (11.338%); 25897/111282 objects misplaced (23.272%); 10 MiB/s, 163 objects/s recovering 2024-08-17T18:21:25.263 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:25.287 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:25.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:25.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:25.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:24 smithi179 ceph-mon[93519]: pgmap v2892: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 78 active+clean; 2.3 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 661 op/s; 12617/111282 objects degraded (11.338%); 25897/111282 objects misplaced (23.272%); 10 MiB/s, 163 objects/s recovering 2024-08-17T18:21:25.389 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:25.412 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:25.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:25.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:25.514 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:25.536 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:25.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:25.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:25.638 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:25.675 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:25.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:25.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:25.777 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:25.802 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:25.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:25.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:25.904 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:25.927 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:25.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:25.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:26.028 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:26.055 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:26.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:26.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:26.157 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:26.188 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:26.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:26.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:26.289 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:26.315 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:26.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:26.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:26.416 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:26.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:26 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:26.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:26 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:26.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 12617/111282 objects degraded (11.338%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:26.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:26 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:26.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:26 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:26.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 12617/111282 objects degraded (11.338%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:26.439 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:26.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:26.541 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:26.570 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:26.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:26.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:26.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:26 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:26 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 12617/111282 objects degraded (11.338%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:26.671 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:26.697 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:26.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:26.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:26.799 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:26.824 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:26.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:26.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:26.925 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:26.946 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:26.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:26.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:27.048 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:27.087 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:27.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:27.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:27.192 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:27.216 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:27.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:27.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:27.318 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:27.341 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:27.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:27.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:27.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:27 smithi130 ceph-mon[105521]: pgmap v2893: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 79 active+clean; 2.2 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 505 op/s; 12083/110337 objects degraded (10.951%); 25866/110337 objects misplaced (23.443%); 11 MiB/s, 177 objects/s recovering 2024-08-17T18:21:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:21:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:21:27.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:27 smithi130 ceph-mon[110068]: pgmap v2893: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 79 active+clean; 2.2 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 505 op/s; 12083/110337 objects degraded (10.951%); 25866/110337 objects misplaced (23.443%); 11 MiB/s, 177 objects/s recovering 2024-08-17T18:21:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:21:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:21:27.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:27.443 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:27.466 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:27.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:27.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:27.568 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:27.589 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:27.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:27.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:27.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:27 smithi179 ceph-mon[93519]: pgmap v2893: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 79 active+clean; 2.2 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 505 op/s; 12083/110337 objects degraded (10.951%); 25866/110337 objects misplaced (23.443%); 11 MiB/s, 177 objects/s recovering 2024-08-17T18:21:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:21:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:21:27.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:27.691 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:27.714 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:27.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:27.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:27.816 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:27.837 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:27.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:27.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:27.938 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:27.968 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:27.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:27.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:28.069 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:28.097 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:28.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:28.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:28.199 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:28.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:28.233 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:28.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:28.334 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:28.356 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:28.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:28.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:28.457 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:28.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:28.500 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:28.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:28.525 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:21:28.601 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:28.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:28.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:21:28.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:28.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:21:28.626 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:28.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:28.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:28.728 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:28.758 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:28.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:28.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:28.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:28.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:21:28.860 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:28.884 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:28.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:28.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:28.986 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:29.010 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:29.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:29.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:29.112 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:29.134 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:29.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:29.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:29.236 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:29.260 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:29.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:29.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:29.362 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:29.385 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:29.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:29.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:29.487 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:29.508 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:29.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:29.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:29.610 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:29.643 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:29.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:29.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:29.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:29 smithi130 ceph-mon[105521]: pgmap v2894: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 79 active+clean; 2.2 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 499 op/s; 11902/108948 objects degraded (10.924%); 25403/108948 objects misplaced (23.317%); 9.5 MiB/s, 151 objects/s recovering 2024-08-17T18:21:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:29 smithi130 ceph-mon[110068]: pgmap v2894: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 79 active+clean; 2.2 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 499 op/s; 11902/108948 objects degraded (10.924%); 25403/108948 objects misplaced (23.317%); 9.5 MiB/s, 151 objects/s recovering 2024-08-17T18:21:29.745 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:29.781 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:29.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:29.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:29.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:29 smithi179 ceph-mon[93519]: pgmap v2894: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 79 active+clean; 2.2 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 499 op/s; 11902/108948 objects degraded (10.924%); 25403/108948 objects misplaced (23.317%); 9.5 MiB/s, 151 objects/s recovering 2024-08-17T18:21:29.883 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:29.904 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:29.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:29.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:30.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:30.028 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:30.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:30.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:30.130 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:30.154 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:30.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:30.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:21:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:21:29] "GET /metrics HTTP/1.1" 200 36809 "" "Prometheus/2.51.0" 2024-08-17T18:21:30.256 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:30.282 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:30.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:30.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:30.384 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:30.411 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:30.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:30.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:30.513 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:30.538 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:30.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:30.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:21:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:21:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:21:30.640 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:30.672 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:30.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:30.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:30.773 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:30.808 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:30.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:30.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:30.909 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:30.933 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:30.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:30.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:31.035 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:31.069 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:31.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:31.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:31.171 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:31.193 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:31.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:31.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:31.295 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:31.317 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:31.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:31.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:31.419 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:31.444 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:31.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:31.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:31.546 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:31.567 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:31.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:31.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:31.662 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:31 smithi179 ceph-mon[93519]: pgmap v2895: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 11240/102606 objects degraded (10.955%); 24124/102606 objects misplaced (23.511%); 9.4 MiB/s, 150 objects/s recovering 2024-08-17T18:21:31.663 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:31 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/381886881' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:21:31.663 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:31 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 11240/102606 objects degraded (10.955%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:21:31.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:31 smithi130 ceph-mon[110068]: pgmap v2895: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 11240/102606 objects degraded (10.955%); 24124/102606 objects misplaced (23.511%); 9.4 MiB/s, 150 objects/s recovering 2024-08-17T18:21:31.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:31 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/381886881' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:21:31.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:31 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 11240/102606 objects degraded (10.955%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:21:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:31 smithi130 ceph-mon[105521]: pgmap v2895: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 11240/102606 objects degraded (10.955%); 24124/102606 objects misplaced (23.511%); 9.4 MiB/s, 150 objects/s recovering 2024-08-17T18:21:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:31 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/381886881' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:21:31.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:31 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 11240/102606 objects degraded (10.955%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:21:31.669 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:31.697 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:31.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:31.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:31.799 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:31.821 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:31.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:31.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:31.922 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:31.944 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:31.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:31.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:32.003 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:21:32.045 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:32.074 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:32.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:32.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:32.176 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:32.203 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:32.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:32.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:32.305 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:32.333 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:32.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:32.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:32.373 INFO:teuthology.orchestra.run.smithi130.stderr:marked in osd.0. 2024-08-17T18:21:32.435 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:32.460 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:32.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:32.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:32.505 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:21:32.505 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:21:32.505 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:21:32.505 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:21:32.505 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:21:32.506 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:21:32.506 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:21:32.506 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:21:32.507 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:21:32.507 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:21:32.507 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:21:32.507 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:21:32.507 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:21:32.507 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:21:32.508 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:21:32.508 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:21:32.508 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2795733560' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T18:21:32.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:32 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T18:21:32.242+0000 7fa856f19640 -1 mon.a@0(leader).osd e1488 definitely_dead 0 2024-08-17T18:21:32.508 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2795733560' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T18:21:32.562 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:32.584 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:32.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:32.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:32.686 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:32.709 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:32.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:32.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:32.810 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:32.834 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:32.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:32.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:32.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2795733560' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T18:21:32.936 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:32.958 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:32.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:32.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:33.062 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:33.085 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:33.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:33.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:33.187 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:33.210 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:33.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:33.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:33.312 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:33.334 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:33.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:33.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:33.435 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:33.459 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:33.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:33.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:33.561 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:33.584 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:33.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:33.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:33.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:33 smithi130 ceph-mon[110068]: pgmap v2896: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 563 op/s; 11240/102606 objects degraded (10.955%); 24124/102606 objects misplaced (23.511%); 7.6 MiB/s, 122 objects/s recovering 2024-08-17T18:21:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:33 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2795733560' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-08-17T18:21:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:33 smithi130 ceph-mon[110068]: osdmap e1489: 8 total, 8 up, 7 in 2024-08-17T18:21:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:33 smithi130 ceph-mon[110068]: 50.3 scrub starts 2024-08-17T18:21:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:33 smithi130 ceph-mon[105521]: pgmap v2896: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 563 op/s; 11240/102606 objects degraded (10.955%); 24124/102606 objects misplaced (23.511%); 7.6 MiB/s, 122 objects/s recovering 2024-08-17T18:21:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:33 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2795733560' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-08-17T18:21:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:33 smithi130 ceph-mon[105521]: osdmap e1489: 8 total, 8 up, 7 in 2024-08-17T18:21:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:33 smithi130 ceph-mon[105521]: 50.3 scrub starts 2024-08-17T18:21:33.686 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:33.708 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:33.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:33.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:33.810 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:33.836 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:33.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:33.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:33.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:33 smithi179 ceph-mon[93519]: pgmap v2896: 99 pgs: 1 active+recovering+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovery_wait+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 8 active+remapped+backfill_wait, 79 active+clean; 2.1 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 563 op/s; 11240/102606 objects degraded (10.955%); 24124/102606 objects misplaced (23.511%); 7.6 MiB/s, 122 objects/s recovering 2024-08-17T18:21:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:33 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2795733560' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-08-17T18:21:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:33 smithi179 ceph-mon[93519]: osdmap e1489: 8 total, 8 up, 7 in 2024-08-17T18:21:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:33 smithi179 ceph-mon[93519]: 50.3 scrub starts 2024-08-17T18:21:33.855 INFO:tasks.thrashosds.thrasher:Added osd 0 2024-08-17T18:21:33.938 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:33.961 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:33.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:33.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:34.063 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:34.086 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:34.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:34.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:34.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:34.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:34.211 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:34.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:34.312 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:34.334 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:34.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:34.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:34.436 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:34.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:34.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:34.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:34.484 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[110068]: 50.3 scrub ok 2024-08-17T18:21:34.484 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[110068]: 50.d starting backfill to osd.0 from (1441'6510,1468'14949] MIN to 1488'15979 2024-08-17T18:21:34.484 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[110068]: 50.5 continuing backfill to osd.2 from (1441'6545,1488'11457] MIN to 1488'11457 2024-08-17T18:21:34.484 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[110068]: 50.15 continuing backfill to osd.2 from (1441'6271,1488'11349] MIN to 1488'11349 2024-08-17T18:21:34.484 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[110068]: 50.10 continuing backfill to osd.2 from (1438'900,1488'11465] 50:08ed6208:::benchmark_data_smithi130_398002_object114558:head to 1488'11465 2024-08-17T18:21:34.484 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[110068]: 50.10 continuing backfill to osd.4 from (1438'900,1488'11465] 50:09efc84d:::benchmark_data_smithi130_398002_object91543:head to 1488'11465 2024-08-17T18:21:34.485 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[110068]: osdmap e1490: 8 total, 8 up, 7 in 2024-08-17T18:21:34.485 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[110068]: 50.17 continuing backfill to osd.2 from (1440'3901,1488'11339] MIN to 1488'11339 2024-08-17T18:21:34.485 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[110068]: 50.7 continuing backfill to osd.2 from (1440'3629,1488'11268] MIN to 1488'11268 2024-08-17T18:21:34.485 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[110068]: 50.e continuing backfill to osd.2 from (1450'9541,1488'15827] MIN to 1488'15827 2024-08-17T18:21:34.486 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[105521]: 50.3 scrub ok 2024-08-17T18:21:34.486 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[105521]: 50.d starting backfill to osd.0 from (1441'6510,1468'14949] MIN to 1488'15979 2024-08-17T18:21:34.486 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[105521]: 50.5 continuing backfill to osd.2 from (1441'6545,1488'11457] MIN to 1488'11457 2024-08-17T18:21:34.486 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[105521]: 50.15 continuing backfill to osd.2 from (1441'6271,1488'11349] MIN to 1488'11349 2024-08-17T18:21:34.486 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[105521]: 50.10 continuing backfill to osd.2 from (1438'900,1488'11465] 50:08ed6208:::benchmark_data_smithi130_398002_object114558:head to 1488'11465 2024-08-17T18:21:34.486 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[105521]: 50.10 continuing backfill to osd.4 from (1438'900,1488'11465] 50:09efc84d:::benchmark_data_smithi130_398002_object91543:head to 1488'11465 2024-08-17T18:21:34.486 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[105521]: osdmap e1490: 8 total, 8 up, 7 in 2024-08-17T18:21:34.486 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[105521]: 50.17 continuing backfill to osd.2 from (1440'3901,1488'11339] MIN to 1488'11339 2024-08-17T18:21:34.486 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[105521]: 50.7 continuing backfill to osd.2 from (1440'3629,1488'11268] MIN to 1488'11268 2024-08-17T18:21:34.486 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:34 smithi130 ceph-mon[105521]: 50.e continuing backfill to osd.2 from (1450'9541,1488'15827] MIN to 1488'15827 2024-08-17T18:21:34.564 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:34.590 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:34.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:34.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:34.692 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:34.722 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:34.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:34.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:34.823 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:34 smithi179 ceph-mon[93519]: 50.3 scrub ok 2024-08-17T18:21:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:34 smithi179 ceph-mon[93519]: 50.d starting backfill to osd.0 from (1441'6510,1468'14949] MIN to 1488'15979 2024-08-17T18:21:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:34 smithi179 ceph-mon[93519]: 50.5 continuing backfill to osd.2 from (1441'6545,1488'11457] MIN to 1488'11457 2024-08-17T18:21:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:34 smithi179 ceph-mon[93519]: 50.15 continuing backfill to osd.2 from (1441'6271,1488'11349] MIN to 1488'11349 2024-08-17T18:21:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:34 smithi179 ceph-mon[93519]: 50.10 continuing backfill to osd.2 from (1438'900,1488'11465] 50:08ed6208:::benchmark_data_smithi130_398002_object114558:head to 1488'11465 2024-08-17T18:21:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:34 smithi179 ceph-mon[93519]: 50.10 continuing backfill to osd.4 from (1438'900,1488'11465] 50:09efc84d:::benchmark_data_smithi130_398002_object91543:head to 1488'11465 2024-08-17T18:21:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:34 smithi179 ceph-mon[93519]: osdmap e1490: 8 total, 8 up, 7 in 2024-08-17T18:21:34.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:34 smithi179 ceph-mon[93519]: 50.17 continuing backfill to osd.2 from (1440'3901,1488'11339] MIN to 1488'11339 2024-08-17T18:21:34.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:34 smithi179 ceph-mon[93519]: 50.7 continuing backfill to osd.2 from (1440'3629,1488'11268] MIN to 1488'11268 2024-08-17T18:21:34.844 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:34 smithi179 ceph-mon[93519]: 50.e continuing backfill to osd.2 from (1450'9541,1488'15827] MIN to 1488'15827 2024-08-17T18:21:34.863 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:34.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:34.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:34.965 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:35.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:35.005 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:35.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:35.106 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:35.147 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:35.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:35.249 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:35.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:35.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:35.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:35.376 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:35.408 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:35.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:35.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:35.509 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:35.552 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:35.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:35.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:35.654 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:35.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:35 smithi130 ceph-mon[105521]: pgmap v2899: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 2 active+undersized+degraded+remapped+backfilling, 23 peering, 1 active+recovery_wait+degraded+remapped, 6 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 55 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 7539/98085 objects degraded (7.686%); 12646/98085 objects misplaced (12.893%); 11 MiB/s, 170 objects/s recovering 2024-08-17T18:21:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:35 smithi130 ceph-mon[105521]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-08-17T18:21:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:35 smithi130 ceph-mon[105521]: 50.0 continuing backfill to osd.2 from (1439'3257,1488'11384] 50:013c81a5:::benchmark_data_smithi130_398002_object38821:head to 1488'11384 2024-08-17T18:21:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:35 smithi130 ceph-mon[105521]: 50.a starting backfill to osd.0 from (1444'6738,1479'15655] MIN to 1488'16188 2024-08-17T18:21:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:35 smithi130 ceph-mon[105521]: 50.19 starting backfill to osd.0 from (1439'1419,1469'10849] MIN to 1488'11395 2024-08-17T18:21:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:35 smithi130 ceph-mon[105521]: 50.9 starting backfill to osd.0 from (1439'1389,1473'10868] MIN to 1488'11349 2024-08-17T18:21:35.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:35 smithi130 ceph-mon[105521]: osdmap e1491: 8 total, 8 up, 7 in 2024-08-17T18:21:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:35 smithi130 ceph-mon[110068]: pgmap v2899: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 2 active+undersized+degraded+remapped+backfilling, 23 peering, 1 active+recovery_wait+degraded+remapped, 6 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 55 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 7539/98085 objects degraded (7.686%); 12646/98085 objects misplaced (12.893%); 11 MiB/s, 170 objects/s recovering 2024-08-17T18:21:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:35 smithi130 ceph-mon[110068]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-08-17T18:21:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:35 smithi130 ceph-mon[110068]: 50.0 continuing backfill to osd.2 from (1439'3257,1488'11384] 50:013c81a5:::benchmark_data_smithi130_398002_object38821:head to 1488'11384 2024-08-17T18:21:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:35 smithi130 ceph-mon[110068]: 50.a starting backfill to osd.0 from (1444'6738,1479'15655] MIN to 1488'16188 2024-08-17T18:21:35.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:35 smithi130 ceph-mon[110068]: 50.19 starting backfill to osd.0 from (1439'1419,1469'10849] MIN to 1488'11395 2024-08-17T18:21:35.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:35 smithi130 ceph-mon[110068]: 50.9 starting backfill to osd.0 from (1439'1389,1473'10868] MIN to 1488'11349 2024-08-17T18:21:35.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:35 smithi130 ceph-mon[110068]: osdmap e1491: 8 total, 8 up, 7 in 2024-08-17T18:21:35.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:35.693 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:35.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:35.794 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:35.831 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:35.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:35.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:35 smithi179 ceph-mon[93519]: pgmap v2899: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 2 active+undersized+degraded+remapped+backfilling, 23 peering, 1 active+recovery_wait+degraded+remapped, 6 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 55 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 7539/98085 objects degraded (7.686%); 12646/98085 objects misplaced (12.893%); 11 MiB/s, 170 objects/s recovering 2024-08-17T18:21:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:35 smithi179 ceph-mon[93519]: Health check failed: Reduced data availability: 6 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-08-17T18:21:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:35 smithi179 ceph-mon[93519]: 50.0 continuing backfill to osd.2 from (1439'3257,1488'11384] 50:013c81a5:::benchmark_data_smithi130_398002_object38821:head to 1488'11384 2024-08-17T18:21:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:35 smithi179 ceph-mon[93519]: 50.a starting backfill to osd.0 from (1444'6738,1479'15655] MIN to 1488'16188 2024-08-17T18:21:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:35 smithi179 ceph-mon[93519]: 50.19 starting backfill to osd.0 from (1439'1419,1469'10849] MIN to 1488'11395 2024-08-17T18:21:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:35 smithi179 ceph-mon[93519]: 50.9 starting backfill to osd.0 from (1439'1389,1473'10868] MIN to 1488'11349 2024-08-17T18:21:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:35 smithi179 ceph-mon[93519]: osdmap e1491: 8 total, 8 up, 7 in 2024-08-17T18:21:35.933 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:35.958 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:35.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:35.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:36.059 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:36.085 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:36.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:36.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:36.186 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:36.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:36.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:36.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:36.321 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:36.348 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:36.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:36.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:36.450 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:36.476 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:36.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:36.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:36.525 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:21:36.577 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:36.602 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:36.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:36.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:36.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:36 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 7539/98085 objects degraded (7.686%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:36.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:36 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 7539/98085 objects degraded (7.686%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:36.703 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:36.728 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:36.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:36.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:36.829 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:36.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:36 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 7539/98085 objects degraded (7.686%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:36.859 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:36.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:36.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:36.961 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:36.990 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:36.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:36.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:37.092 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:37.142 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:37.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:37.246 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:37.278 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:37.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:37.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:37.381 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:37.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:37.423 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:37.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:37.524 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:37.563 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:37.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:37.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:37.592 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:37 smithi130 ceph-mon[105521]: pgmap v2901: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 2 active+undersized+degraded+remapped+backfilling, 23 peering, 1 active+recovery_wait+degraded+remapped, 6 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 56 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 7538/97839 objects degraded (7.704%); 12646/97839 objects misplaced (12.925%) 2024-08-17T18:21:37.593 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:37 smithi130 ceph-mon[110068]: pgmap v2901: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 2 active+undersized+degraded+remapped+backfilling, 23 peering, 1 active+recovery_wait+degraded+remapped, 6 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 56 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 7538/97839 objects degraded (7.704%); 12646/97839 objects misplaced (12.925%) 2024-08-17T18:21:37.665 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:37.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:37.701 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:37.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:37.802 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:37 smithi179 ceph-mon[93519]: pgmap v2901: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+degraded+remapped, 2 active+undersized+degraded+remapped+backfilling, 23 peering, 1 active+recovery_wait+degraded+remapped, 6 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 56 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 7538/97839 objects degraded (7.704%); 12646/97839 objects misplaced (12.925%) 2024-08-17T18:21:37.847 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:37.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:37.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:37.949 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:37.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:37.988 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:37.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:38.089 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:38.110 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:38.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:38.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:38.212 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:38.250 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:38.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:38.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:38.352 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:38.387 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:38.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:38.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:38.489 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:38.533 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:38.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:38.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:38.635 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:38.664 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:38.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:38.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:38.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:38 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3587220966' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:21:38.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:38 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3587220966' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:21:38.766 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:38.807 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:38.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:38.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:38.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:38 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3587220966' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:21:38.856 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 7, 1, 6, 4, 0] out_osds: [5] dead_osds: [] live_osds: [1, 4, 5, 6, 7, 0, 3, 2] 2024-08-17T18:21:38.856 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:21:38.856 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [2, 3, 7, 1, 6, 4, 0] 2024-08-17T18:21:38.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd out 1 2024-08-17T18:21:38.857 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:21:38.909 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:38.940 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:38.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:38.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:39.042 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:39.106 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:39.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:39.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:39.208 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:39.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:39.256 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:39.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:39.357 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:39.400 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:39.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:39.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:39.502 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:39.531 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:39.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:39.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:39.632 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:39.658 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:39.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:39.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:39 smithi130 ceph-mon[105521]: pgmap v2902: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+undersized+degraded+remapped+backfilling, 23 peering, 1 active+recovery_wait+degraded+remapped, 6 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 56 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 7403/96822 objects degraded (7.646%); 9548/96822 objects misplaced (9.861%); 9.0 MiB/s, 144 objects/s recovering 2024-08-17T18:21:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:39 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:21:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:39 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3587220966' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:21:39.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:39 smithi130 ceph-mon[105521]: osdmap e1492: 8 total, 8 up, 7 in 2024-08-17T18:21:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:39 smithi130 ceph-mon[110068]: pgmap v2902: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+undersized+degraded+remapped+backfilling, 23 peering, 1 active+recovery_wait+degraded+remapped, 6 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 56 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 7403/96822 objects degraded (7.646%); 9548/96822 objects misplaced (9.861%); 9.0 MiB/s, 144 objects/s recovering 2024-08-17T18:21:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:39 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:21:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:39 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3587220966' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:21:39.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:39 smithi130 ceph-mon[110068]: osdmap e1492: 8 total, 8 up, 7 in 2024-08-17T18:21:39.760 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:39.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:39.818 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:39.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:39 smithi179 ceph-mon[93519]: pgmap v2902: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 2 active+undersized+degraded+remapped+backfilling, 23 peering, 1 active+recovery_wait+degraded+remapped, 6 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 56 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 7403/96822 objects degraded (7.646%); 9548/96822 objects misplaced (9.861%); 9.0 MiB/s, 144 objects/s recovering 2024-08-17T18:21:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:39 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:21:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:39 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3587220966' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:21:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:39 smithi179 ceph-mon[93519]: osdmap e1492: 8 total, 8 up, 7 in 2024-08-17T18:21:39.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:39.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:39.947 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:39.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:40.047 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:40.085 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:40.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:40.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:40.167 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:21:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:21:39] "GET /metrics HTTP/1.1" 200 36809 "" "Prometheus/2.51.0" 2024-08-17T18:21:40.187 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:40.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:40.220 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:40.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:40.321 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:40.347 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:40.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:40.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:40.449 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:40.465 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:21:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:21:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:21:40.490 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:40.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:40.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:40.495 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:21:40.495 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T18:21:40.495 INFO:teuthology.orchestra.run.smithi130.stdout: "complaint_time": 30, 2024-08-17T18:21:40.495 INFO:teuthology.orchestra.run.smithi130.stdout: "num_blocked_ops": 0 2024-08-17T18:21:40.495 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:21:40.593 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:40.642 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:40.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:40.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:40.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:40 smithi130 ceph-mon[105521]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-08-17T18:21:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:40 smithi130 ceph-mon[110068]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-08-17T18:21:40.744 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:40.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:40.805 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:40.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:40.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:40 smithi179 ceph-mon[93519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-08-17T18:21:40.906 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:40.934 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:40.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:40.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:41.036 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:41.064 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:41.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:41.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:41.166 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:41.194 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:41.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:41.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:41.290 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:21:41.296 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:41.307 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T18:21:41.322 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:41.322 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:21:41.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:41.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:41.424 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:41.450 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:41.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:41.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:41.552 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:41.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:41.593 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:41.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:41.694 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:41.717 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:41.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:41.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:41.819 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:41.831 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:41 smithi130 ceph-mon[105521]: pgmap v2904: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 79 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 10170/87999 objects degraded (11.557%); 10561/87999 objects misplaced (12.001%); 13 MiB/s, 210 objects/s recovering 2024-08-17T18:21:41.831 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:41 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 10170/87999 objects degraded (11.557%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:41.831 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:41 smithi130 ceph-mon[105521]: osdmap e1493: 8 total, 8 up, 7 in 2024-08-17T18:21:41.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:41 smithi130 ceph-mon[110068]: pgmap v2904: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 79 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 10170/87999 objects degraded (11.557%); 10561/87999 objects misplaced (12.001%); 13 MiB/s, 210 objects/s recovering 2024-08-17T18:21:41.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:41 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 10170/87999 objects degraded (11.557%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:41.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:41 smithi130 ceph-mon[110068]: osdmap e1493: 8 total, 8 up, 7 in 2024-08-17T18:21:41.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:41 smithi179 ceph-mon[93519]: pgmap v2904: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 79 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 10170/87999 objects degraded (11.557%); 10561/87999 objects misplaced (12.001%); 13 MiB/s, 210 objects/s recovering 2024-08-17T18:21:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:41 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 10170/87999 objects degraded (11.557%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:41 smithi179 ceph-mon[93519]: osdmap e1493: 8 total, 8 up, 7 in 2024-08-17T18:21:41.844 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:41.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:41.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:41.946 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:41.976 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:41.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:41.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:42.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:42.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:42.101 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:42.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:42.202 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:42.224 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:42.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:42.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:42.326 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:42.354 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:42.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:42.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:42.359 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:21:42.456 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:42.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:42.482 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:42.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:42.583 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:42.608 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:42.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:42.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:42.710 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:42.736 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:42.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:42.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:42.838 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:42.881 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:42.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:42.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:42.983 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:43.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:43.011 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:43.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:43.112 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:43.158 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:43.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:43.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:43.260 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:43.300 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:43.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:43.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:43.402 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:43.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:43 smithi130 ceph-mon[105521]: pgmap v2906: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 79 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 10170/87999 objects degraded (11.557%); 10561/87999 objects misplaced (12.001%); 16 MiB/s, 262 objects/s recovering 2024-08-17T18:21:43.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:43 smithi130 ceph-mon[105521]: osdmap e1494: 8 total, 8 up, 7 in 2024-08-17T18:21:43.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:43.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:21:43.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:43 smithi130 ceph-mon[110068]: pgmap v2906: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 79 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 10170/87999 objects degraded (11.557%); 10561/87999 objects misplaced (12.001%); 16 MiB/s, 262 objects/s recovering 2024-08-17T18:21:43.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:43 smithi130 ceph-mon[110068]: osdmap e1494: 8 total, 8 up, 7 in 2024-08-17T18:21:43.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:43.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:21:43.436 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:43.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:43.538 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:43.577 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:43.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:43.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:43 smithi179 ceph-mon[93519]: pgmap v2906: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 79 active+clean; 1.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 10170/87999 objects degraded (11.557%); 10561/87999 objects misplaced (12.001%); 16 MiB/s, 262 objects/s recovering 2024-08-17T18:21:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:43 smithi179 ceph-mon[93519]: osdmap e1494: 8 total, 8 up, 7 in 2024-08-17T18:21:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:21:43.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:21:43.678 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:43.716 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:43.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:43.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:43.817 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:43.855 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:43.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:43.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:43.957 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:43.981 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:43.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:43.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:44.083 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:44.122 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:44.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:44.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:44.224 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:44.245 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:44.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:44.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:44.347 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:44.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:44.394 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:44.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:44.495 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:44.529 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:44.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:44.630 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:44.681 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:44.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:44.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:44.783 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:44.810 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:44.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:44.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:44.912 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:44.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:44.940 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:44.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:45.041 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:45.062 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:45.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:45.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:45.164 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:45.204 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:45.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:45.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:45.306 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:45.340 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:45.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:45.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:45.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:45 smithi130 ceph-mon[105521]: pgmap v2908: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 8980/76992 objects degraded (11.664%); 8424/76992 objects misplaced (10.941%); 22 MiB/s, 349 objects/s recovering 2024-08-17T18:21:45.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:45 smithi130 ceph-mon[110068]: pgmap v2908: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 8980/76992 objects degraded (11.664%); 8424/76992 objects misplaced (10.941%); 22 MiB/s, 349 objects/s recovering 2024-08-17T18:21:45.442 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:45.470 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:45.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:45.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:45.572 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:45.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:45 smithi179 ceph-mon[93519]: pgmap v2908: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 8980/76992 objects degraded (11.664%); 8424/76992 objects misplaced (10.941%); 22 MiB/s, 349 objects/s recovering 2024-08-17T18:21:45.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:45.604 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:45.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:45.705 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:45.742 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:45.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:45.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:45.844 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:45.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:45.871 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:45.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:45.974 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:46.020 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:46.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:46.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:46.122 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:46.154 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:46.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:46.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:46.256 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:46.287 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:46.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:46.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:46.389 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:46.411 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:46.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:46.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:46.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:46 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 8980/76992 objects degraded (11.664%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:46.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:46 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 8980/76992 objects degraded (11.664%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:46.513 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:46.540 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:46.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:46.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:46.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:46 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 8980/76992 objects degraded (11.664%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:46.642 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:46.681 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:46.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:46.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:46.782 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:46.827 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:46.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:46.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:46.929 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:46.951 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:46.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:46.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:47.052 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:47.075 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:47.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:47.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:47.177 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:47.221 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:47.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:47.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:47.323 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:47.348 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:47.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:47.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:47.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:47 smithi130 ceph-mon[105521]: pgmap v2909: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 8980/76647 objects degraded (11.716%); 8424/76647 objects misplaced (10.991%); 15 MiB/s, 232 objects/s recovering 2024-08-17T18:21:47.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:47 smithi130 ceph-mon[110068]: pgmap v2909: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 8980/76647 objects degraded (11.716%); 8424/76647 objects misplaced (10.991%); 15 MiB/s, 232 objects/s recovering 2024-08-17T18:21:47.451 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:47.478 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:47.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:47.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:47.580 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:47.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:47 smithi179 ceph-mon[93519]: pgmap v2909: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 8980/76647 objects degraded (11.716%); 8424/76647 objects misplaced (10.991%); 15 MiB/s, 232 objects/s recovering 2024-08-17T18:21:47.621 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:47.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:47.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:47.723 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:47.767 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:47.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:47.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:47.869 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:47.890 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:47.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:47.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:47.992 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:48.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:48.029 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:48.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:48.130 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:48.168 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:48.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:48.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:48.270 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:48.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:48.304 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:48.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:48.347 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:21:48.405 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:48.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:48.443 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:48.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:48.544 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:48.565 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:48.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:48.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:48.667 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:48.689 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:48.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:48.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:48.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:48.812 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:48.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:48.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:48.914 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:48.935 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:48.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:48.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:49.037 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:49.060 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:49.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:49.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:49.162 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:49.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:49.189 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:49.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:49.290 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:49.327 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:49.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:49.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:49.358 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:21:49.429 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:49.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:49.489 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:49.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:49.569 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:49 smithi130 ceph-mon[105521]: pgmap v2910: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 8754/75351 objects degraded (11.618%); 7596/75351 objects misplaced (10.081%); 15 MiB/s, 237 objects/s recovering 2024-08-17T18:21:49.569 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:49 smithi130 ceph-mon[105521]: osdmap e1495: 8 total, 8 up, 7 in 2024-08-17T18:21:49.569 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:49 smithi130 ceph-mon[110068]: pgmap v2910: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 8754/75351 objects degraded (11.618%); 7596/75351 objects misplaced (10.081%); 15 MiB/s, 237 objects/s recovering 2024-08-17T18:21:49.569 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:49 smithi130 ceph-mon[110068]: osdmap e1495: 8 total, 8 up, 7 in 2024-08-17T18:21:49.590 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:49 smithi179 ceph-mon[93519]: pgmap v2910: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 8754/75351 objects degraded (11.618%); 7596/75351 objects misplaced (10.081%); 15 MiB/s, 237 objects/s recovering 2024-08-17T18:21:49.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:49 smithi179 ceph-mon[93519]: osdmap e1495: 8 total, 8 up, 7 in 2024-08-17T18:21:49.617 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:49.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:49.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:49.719 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:49.743 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:49.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:49.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:49.778 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:21:49.779 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:21:49.811 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:21:49.811 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:21:49.845 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:49.848 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:21:49.848 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:21:49.890 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:49.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:49.891 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:21:49.891 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:21:49.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:49.918 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:21:49.919 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:21:49.947 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:21:49.947 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:21:49.978 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:21:49.979 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:21:49.993 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:50.012 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:21:50.012 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:21:50.024 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:50.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:50.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:50.126 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:50.165 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:50.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:50.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:21:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:21:49] "GET /metrics HTTP/1.1" 200 36819 "" "Prometheus/2.51.0" 2024-08-17T18:21:50.267 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:50.289 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:50.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:50.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:50.391 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:50.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:50.439 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:50.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:50.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:50 smithi179 ceph-mon[93519]: osdmap e1496: 8 total, 8 up, 7 in 2024-08-17T18:21:50.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:50 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/601141618' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:21:50.494 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:21:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:21:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:21:50.512 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:50 smithi130 ceph-mon[105521]: osdmap e1496: 8 total, 8 up, 7 in 2024-08-17T18:21:50.512 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:50 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/601141618' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:21:50.512 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:50 smithi130 ceph-mon[110068]: osdmap e1496: 8 total, 8 up, 7 in 2024-08-17T18:21:50.512 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:50 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/601141618' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:21:50.540 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:50.581 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:50.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:50.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:50.683 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:50.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:50.721 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:50.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:50.822 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:50.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:50.853 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:50.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:50.954 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:50.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:50.991 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:50.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:51.092 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:51.127 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:51.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:51.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:51.229 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:51.261 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:51.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:51.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:51.363 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:51.398 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:51.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:51.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:51.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:51 smithi130 ceph-mon[105521]: pgmap v2913: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 940 op/s; 7751/65289 objects degraded (11.872%); 6444/65289 objects misplaced (9.870%); 16 MiB/s, 259 objects/s recovering 2024-08-17T18:21:51.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:51 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 7751/65289 objects degraded (11.872%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:51.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:51 smithi130 ceph-mon[110068]: pgmap v2913: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 940 op/s; 7751/65289 objects degraded (11.872%); 6444/65289 objects misplaced (9.870%); 16 MiB/s, 259 objects/s recovering 2024-08-17T18:21:51.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:51 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 7751/65289 objects degraded (11.872%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:51.466 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:51 smithi179 ceph-mon[93519]: pgmap v2913: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 940 op/s; 7751/65289 objects degraded (11.872%); 6444/65289 objects misplaced (9.870%); 16 MiB/s, 259 objects/s recovering 2024-08-17T18:21:51.466 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:51 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 7751/65289 objects degraded (11.872%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:51.499 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:51.539 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:51.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:51.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:51.641 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:51.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:51.675 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:51.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:51.776 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:51.812 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:51.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:51.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:51.914 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:51.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:51.957 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:51.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:52.058 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:52.101 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:52.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:52.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:52.203 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:52.239 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:52.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:52.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:52.340 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:52.366 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:52.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:52.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:52.468 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:52.504 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:52.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:52.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:52.605 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:52.645 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:52.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:52.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:52.746 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:52.779 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:52.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:52.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:52.880 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:52.911 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:52.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:52.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:53.013 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:53.043 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:53.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:53.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:53.144 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:53.174 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:53.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:53.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:53.276 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:53.313 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:53.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:53.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:53.415 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:53.450 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:53.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:53.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:53.552 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:53.581 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:53.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:53.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:53.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:53 smithi179 ceph-mon[93519]: pgmap v2914: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 7751/65289 objects degraded (11.872%); 6444/65289 objects misplaced (9.870%); 8.5 MiB/s, 135 objects/s recovering 2024-08-17T18:21:53.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:53 smithi130 ceph-mon[105521]: pgmap v2914: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 7751/65289 objects degraded (11.872%); 6444/65289 objects misplaced (9.870%); 8.5 MiB/s, 135 objects/s recovering 2024-08-17T18:21:53.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:53 smithi130 ceph-mon[110068]: pgmap v2914: 99 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 80 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 487 op/s; 7751/65289 objects degraded (11.872%); 6444/65289 objects misplaced (9.870%); 8.5 MiB/s, 135 objects/s recovering 2024-08-17T18:21:53.683 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:53.711 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:53.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:53.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:53.813 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:53.839 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:53.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:53.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:53.941 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:53.978 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:53.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:53.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:54.080 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:54.124 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:54.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:54.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:54.226 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:54.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:54.251 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:54.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:54.351 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:54.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:54.401 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:54.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:54.502 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:54.544 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:54.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:54.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:54.646 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:54.686 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:21:54.688 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:54.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:54.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:54.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:54.799 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:21:54.799 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:21:54.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:54.818 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:54.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:54.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:54.945 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:54.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:54.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:55.046 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:55.087 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:55.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:55.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:55.188 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:55.215 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:55.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:55.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:55.239 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:21:55.317 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:55.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:55.346 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:55.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:55 smithi130 ceph-mon[105521]: pgmap v2915: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 6459/54123 objects degraded (11.934%); 5240/54123 objects misplaced (9.682%); 12 MiB/s, 199 objects/s recovering 2024-08-17T18:21:55.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:55 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3813680779' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:21:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:55 smithi130 ceph-mon[110068]: pgmap v2915: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 6459/54123 objects degraded (11.934%); 5240/54123 objects misplaced (9.682%); 12 MiB/s, 199 objects/s recovering 2024-08-17T18:21:55.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:55 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3813680779' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:21:55.447 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:55.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:55.474 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:55.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:55.575 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:55.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:55 smithi179 ceph-mon[93519]: pgmap v2915: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 6459/54123 objects degraded (11.934%); 5240/54123 objects misplaced (9.682%); 12 MiB/s, 199 objects/s recovering 2024-08-17T18:21:55.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:55 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3813680779' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:21:55.617 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:55.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:55.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:55.629 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T18:21:55.719 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:55.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:55.759 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:55.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:55.860 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:55.903 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:55.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:55.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:56.005 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:56.033 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:21:56.045 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:56.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:56.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:56.147 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:56.177 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:56.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:56.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:56.278 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:56.331 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:56.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:56.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:56.433 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:56.462 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:56.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:56.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:56.518 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:56 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:21:56.518 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:56 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3813680779' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:21:56.518 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:56 smithi179 ceph-mon[93519]: osdmap e1497: 8 total, 8 up, 7 in 2024-08-17T18:21:56.518 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:56 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 6459/54123 objects degraded (11.934%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:56.564 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:56.593 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:56.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:56.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:56.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:56 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:21:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:56 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3813680779' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:21:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:56 smithi130 ceph-mon[110068]: osdmap e1497: 8 total, 8 up, 7 in 2024-08-17T18:21:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:56 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 6459/54123 objects degraded (11.934%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:56 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:21:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:56 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3813680779' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:21:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:56 smithi130 ceph-mon[105521]: osdmap e1497: 8 total, 8 up, 7 in 2024-08-17T18:21:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:56 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 6459/54123 objects degraded (11.934%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T18:21:56.695 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:21:56.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:56.723 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:56.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:21:56.806 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:21:56.824 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:56.850 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:56.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:56.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:56.952 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:57.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:57.002 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:57.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:57.103 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:57.121 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:56 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T18:21:56.840+0000 7fa856f19640 -1 mon.a@0(leader).osd e1497 definitely_dead 0 2024-08-17T18:21:57.139 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:57.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:57.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:57.241 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:57.246 INFO:teuthology.orchestra.run.smithi130.stderr:marked out osd.1. 2024-08-17T18:21:57.269 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:57 smithi130 ceph-mon[105521]: pgmap v2917: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 6459/53649 objects degraded (12.039%); 5240/53649 objects misplaced (9.767%); 6.8 MiB/s, 109 objects/s recovering 2024-08-17T18:21:57.269 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:57 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/904767724' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T18:21:57.269 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:57 smithi130 ceph-mon[110068]: pgmap v2917: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 6459/53649 objects degraded (12.039%); 5240/53649 objects misplaced (9.767%); 6.8 MiB/s, 109 objects/s recovering 2024-08-17T18:21:57.269 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:57 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/904767724' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T18:21:57.269 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:57 smithi130 ceph-mon[110068]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T18:21:57.291 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:57.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:57.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:57.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:57 smithi179 ceph-mon[93519]: pgmap v2917: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 882 op/s; 6459/53649 objects degraded (12.039%); 5240/53649 objects misplaced (9.767%); 6.8 MiB/s, 109 objects/s recovering 2024-08-17T18:21:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:57 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/904767724' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T18:21:57.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:57 smithi179 ceph-mon[93519]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T18:21:57.393 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:57.406 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:57 smithi130 ceph-mon[105521]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T18:21:57.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:57.419 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:57.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:57.520 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:57.542 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:57.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:57.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:57.643 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:57.665 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:57.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:57.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:57.767 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:57.788 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:57.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:57.890 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:57.913 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:57.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:57.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:58.014 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:58.037 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:58.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:58.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:58.075 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:21:58.075 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:21:58.139 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:58.160 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:58.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:58.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:58.262 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:58.302 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:58.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:58.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:58.404 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:21:58.426 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:58.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:58.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:21:58.527 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:58.551 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:58.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:58.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:58.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:58 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/904767724' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T18:21:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:58 smithi179 ceph-mon[93519]: osdmap e1498: 8 total, 8 up, 6 in 2024-08-17T18:21:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:58 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:21:58.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:58 smithi179 ceph-mon[93519]: osdmap e1499: 8 total, 8 up, 6 in 2024-08-17T18:21:58.648 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:58 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/904767724' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T18:21:58.648 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:58 smithi130 ceph-mon[105521]: osdmap e1498: 8 total, 8 up, 6 in 2024-08-17T18:21:58.648 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:58 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:21:58.649 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:58 smithi130 ceph-mon[105521]: osdmap e1499: 8 total, 8 up, 6 in 2024-08-17T18:21:58.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:58 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/904767724' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T18:21:58.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:58 smithi130 ceph-mon[110068]: osdmap e1498: 8 total, 8 up, 6 in 2024-08-17T18:21:58.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:58 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:21:58.649 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:58 smithi130 ceph-mon[110068]: osdmap e1499: 8 total, 8 up, 6 in 2024-08-17T18:21:58.654 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:58.675 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:58.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:58.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:58.776 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:21:58.800 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:58.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:58.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:21:58.834 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:21:58.903 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:58.926 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:58.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:58.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:58.996 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T18:21:59.027 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:21:59.059 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:59.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:59.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:21:59.161 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:21:59.184 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:59.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:59.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:21:59.278 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:21:59.286 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:59.326 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:59.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:59.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:59.427 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:21:59.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:59.461 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:21:59.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:21:59.562 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:59.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:59 smithi179 ceph-mon[93519]: pgmap v2919: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 6154/52734 objects degraded (11.670%); 5240/52734 objects misplaced (9.937%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T18:21:59.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:21:59 smithi179 ceph-mon[93519]: osdmap e1500: 8 total, 8 up, 6 in 2024-08-17T18:21:59.597 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:59.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:59.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:59.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:59 smithi130 ceph-mon[105521]: pgmap v2919: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 6154/52734 objects degraded (11.670%); 5240/52734 objects misplaced (9.937%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T18:21:59.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:21:59 smithi130 ceph-mon[105521]: osdmap e1500: 8 total, 8 up, 6 in 2024-08-17T18:21:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:59 smithi130 ceph-mon[110068]: pgmap v2919: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 81 active+clean; 1.1 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 6154/52734 objects degraded (11.670%); 5240/52734 objects misplaced (9.937%); 4.0 MiB/s, 64 objects/s recovering 2024-08-17T18:21:59.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:21:59 smithi130 ceph-mon[110068]: osdmap e1500: 8 total, 8 up, 6 in 2024-08-17T18:21:59.699 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:21:59.725 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:59.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:59.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:21:59.827 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:59.850 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:59.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:59.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:59.914 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:21:59.914 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:21:59.934 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:21:59.935 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:21:59.952 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:21:59.956 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:21:59.956 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:21:59.974 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:21:59.975 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:21:59.975 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:21:59.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:21:59.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:21:59.991 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:21:59.991 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:22:00.007 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:22:00.007 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:22:00.023 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:22:00.023 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:22:00.041 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:22:00.042 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:22:00.051 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:21:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:21:59] "GET /metrics HTTP/1.1" 200 36805 "" "Prometheus/2.51.0" 2024-08-17T18:22:00.078 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:00.101 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:00.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:00.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:00.202 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:00.225 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:00.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:00.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:00.327 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:00.361 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:00.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:00.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:00.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[105521]: 50.f continuing backfill to osd.4 from (1452'10483,1497'17171] 50:fa95eb3d:::benchmark_data_smithi130_398002_object138511:head to 1497'17171 2024-08-17T18:22:00.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[105521]: 50.2 starting backfill to osd.7 from (1441'6214,1482'11284] MIN to 1497'12110 2024-08-17T18:22:00.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[105521]: 50.5 continuing backfill to osd.2 from (1441'6545,1497'12019] MIN to 1497'12019 2024-08-17T18:22:00.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[105521]: 50.5 starting backfill to osd.3 from (1441'6545,1488'11457] MIN to 1497'12019 2024-08-17T18:22:00.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[105521]: 50.c starting backfill to osd.2 from (0'0,0'0] MAX to 1497'17311 2024-08-17T18:22:00.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[105521]: 50.b starting backfill to osd.3 from (0'0,0'0] MAX to 1497'17367 2024-08-17T18:22:00.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[105521]: 50.15 continuing backfill to osd.2 from (1441'6271,1497'11909] MIN to 1497'11909 2024-08-17T18:22:00.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[105521]: 50.15 starting backfill to osd.3 from (1441'6271,1488'11349] MIN to 1497'11909 2024-08-17T18:22:00.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3685916325' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:00.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[105521]: osdmap e1501: 8 total, 8 up, 6 in 2024-08-17T18:22:00.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[110068]: 50.f continuing backfill to osd.4 from (1452'10483,1497'17171] 50:fa95eb3d:::benchmark_data_smithi130_398002_object138511:head to 1497'17171 2024-08-17T18:22:00.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[110068]: 50.2 starting backfill to osd.7 from (1441'6214,1482'11284] MIN to 1497'12110 2024-08-17T18:22:00.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[110068]: 50.5 continuing backfill to osd.2 from (1441'6545,1497'12019] MIN to 1497'12019 2024-08-17T18:22:00.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[110068]: 50.5 starting backfill to osd.3 from (1441'6545,1488'11457] MIN to 1497'12019 2024-08-17T18:22:00.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[110068]: 50.c starting backfill to osd.2 from (0'0,0'0] MAX to 1497'17311 2024-08-17T18:22:00.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[110068]: 50.b starting backfill to osd.3 from (0'0,0'0] MAX to 1497'17367 2024-08-17T18:22:00.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[110068]: 50.15 continuing backfill to osd.2 from (1441'6271,1497'11909] MIN to 1497'11909 2024-08-17T18:22:00.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[110068]: 50.15 starting backfill to osd.3 from (1441'6271,1488'11349] MIN to 1497'11909 2024-08-17T18:22:00.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3685916325' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:00.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:00 smithi130 ceph-mon[110068]: osdmap e1501: 8 total, 8 up, 6 in 2024-08-17T18:22:00.464 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:00.496 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:00.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:00.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:00 smithi179 ceph-mon[93519]: 50.f continuing backfill to osd.4 from (1452'10483,1497'17171] 50:fa95eb3d:::benchmark_data_smithi130_398002_object138511:head to 1497'17171 2024-08-17T18:22:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:00 smithi179 ceph-mon[93519]: 50.2 starting backfill to osd.7 from (1441'6214,1482'11284] MIN to 1497'12110 2024-08-17T18:22:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:00 smithi179 ceph-mon[93519]: 50.5 continuing backfill to osd.2 from (1441'6545,1497'12019] MIN to 1497'12019 2024-08-17T18:22:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:00 smithi179 ceph-mon[93519]: 50.5 starting backfill to osd.3 from (1441'6545,1488'11457] MIN to 1497'12019 2024-08-17T18:22:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:00 smithi179 ceph-mon[93519]: 50.c starting backfill to osd.2 from (0'0,0'0] MAX to 1497'17311 2024-08-17T18:22:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:00 smithi179 ceph-mon[93519]: 50.b starting backfill to osd.3 from (0'0,0'0] MAX to 1497'17367 2024-08-17T18:22:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:00 smithi179 ceph-mon[93519]: 50.15 continuing backfill to osd.2 from (1441'6271,1497'11909] MIN to 1497'11909 2024-08-17T18:22:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:00 smithi179 ceph-mon[93519]: 50.15 starting backfill to osd.3 from (1441'6271,1488'11349] MIN to 1497'11909 2024-08-17T18:22:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:00 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3685916325' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:00.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:00 smithi179 ceph-mon[93519]: osdmap e1501: 8 total, 8 up, 6 in 2024-08-17T18:22:00.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:22:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:22:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:22:00.598 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:00.633 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:00.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:00.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:00.735 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:00.760 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:00.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:00.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:00.862 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:00.889 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:00.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:00.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:00.990 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:01.014 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:01.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:01.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:01.116 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:01.139 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:01.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:01.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:01.240 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:01.270 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:01.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:01.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:01.372 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:01.396 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:01.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:01.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:01.400 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:01.401 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:22:01.401 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:22:01.401 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:22:01.401 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:22:01.401 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:22:01.497 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:01.520 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:01.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:01.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:01.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:01 smithi130 ceph-mon[110068]: pgmap v2922: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 activating, 22 peering, 3 remapped+peering, 5 active+undersized+degraded+remapped+backfill_wait, 6 active+remapped+backfill_wait, 57 active+clean; 893 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 802 op/s; 5459/42900 objects degraded (12.725%); 3893/42900 objects misplaced (9.075%); 4.6 MiB/s, 114 objects/s recovering 2024-08-17T18:22:01.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:01 smithi130 ceph-mon[110068]: Health check failed: Reduced data availability: 5 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T18:22:01.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:01 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 5459/42900 objects degraded (12.725%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:22:01.528 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:01 smithi130 ceph-mon[110068]: osdmap e1502: 8 total, 8 up, 6 in 2024-08-17T18:22:01.528 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:01 smithi130 ceph-mon[105521]: pgmap v2922: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 activating, 22 peering, 3 remapped+peering, 5 active+undersized+degraded+remapped+backfill_wait, 6 active+remapped+backfill_wait, 57 active+clean; 893 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 802 op/s; 5459/42900 objects degraded (12.725%); 3893/42900 objects misplaced (9.075%); 4.6 MiB/s, 114 objects/s recovering 2024-08-17T18:22:01.528 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:01 smithi130 ceph-mon[105521]: Health check failed: Reduced data availability: 5 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T18:22:01.528 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:01 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 5459/42900 objects degraded (12.725%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:22:01.528 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:01 smithi130 ceph-mon[105521]: osdmap e1502: 8 total, 8 up, 6 in 2024-08-17T18:22:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:01 smithi179 ceph-mon[93519]: pgmap v2922: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 activating, 22 peering, 3 remapped+peering, 5 active+undersized+degraded+remapped+backfill_wait, 6 active+remapped+backfill_wait, 57 active+clean; 893 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 802 op/s; 5459/42900 objects degraded (12.725%); 3893/42900 objects misplaced (9.075%); 4.6 MiB/s, 114 objects/s recovering 2024-08-17T18:22:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:01 smithi179 ceph-mon[93519]: Health check failed: Reduced data availability: 5 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T18:22:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:01 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 5459/42900 objects degraded (12.725%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:22:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:01 smithi179 ceph-mon[93519]: osdmap e1502: 8 total, 8 up, 6 in 2024-08-17T18:22:01.622 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:01.644 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:01.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:01.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:01.746 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:01.772 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:01.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:01.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:01.873 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:01.898 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:01.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:01.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:02.000 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:02.021 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:02.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:02.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:02.123 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:02.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:02.148 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:02.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:02.187 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T18:22:02.249 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:02.282 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:02.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:02.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:02.385 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:02.412 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:02.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:02.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:02.513 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:02.539 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:22:02.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:02.539 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:02.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:02.640 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:02.664 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:02.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:02.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:02.766 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:02.788 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:02.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:02.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:02.890 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:02.911 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:02.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:02.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:03.013 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:03.035 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:03.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:03.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:03.137 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:03.163 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:03.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:03.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:03.265 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:03.293 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:03.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:03.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:03.304 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:22:03.395 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:03.397 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:03 smithi130 ceph-mon[105521]: pgmap v2925: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 activating, 22 peering, 3 remapped+peering, 5 active+undersized+degraded+remapped+backfill_wait, 6 active+remapped+backfill_wait, 57 active+clean; 893 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 819 op/s; 5459/42900 objects degraded (12.725%); 3893/42900 objects misplaced (9.075%); 5.2 MiB/s, 127 objects/s recovering 2024-08-17T18:22:03.398 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:03 smithi130 ceph-mon[105521]: osdmap e1503: 8 total, 8 up, 6 in 2024-08-17T18:22:03.398 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:03 smithi130 ceph-mon[110068]: pgmap v2925: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 activating, 22 peering, 3 remapped+peering, 5 active+undersized+degraded+remapped+backfill_wait, 6 active+remapped+backfill_wait, 57 active+clean; 893 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 819 op/s; 5459/42900 objects degraded (12.725%); 3893/42900 objects misplaced (9.075%); 5.2 MiB/s, 127 objects/s recovering 2024-08-17T18:22:03.398 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:03 smithi130 ceph-mon[110068]: osdmap e1503: 8 total, 8 up, 6 in 2024-08-17T18:22:03.418 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:03.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:03.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:03.479 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:03 smithi179 ceph-mon[93519]: pgmap v2925: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 2 activating, 22 peering, 3 remapped+peering, 5 active+undersized+degraded+remapped+backfill_wait, 6 active+remapped+backfill_wait, 57 active+clean; 893 MiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 819 op/s; 5459/42900 objects degraded (12.725%); 3893/42900 objects misplaced (9.075%); 5.2 MiB/s, 127 objects/s recovering 2024-08-17T18:22:03.479 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:03 smithi179 ceph-mon[93519]: osdmap e1503: 8 total, 8 up, 6 in 2024-08-17T18:22:03.519 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:03.543 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:03.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:03.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:03.618 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:03.644 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:03.667 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:03.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:03.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:03.769 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:03.791 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:03.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:03.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:03.892 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:03.913 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:03.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:03.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:04.015 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:04.038 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:04.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:04.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:04.140 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:04.166 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:22:04.166 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:04.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:04.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:04.266 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:04 smithi179 ceph-mon[93519]: osdmap e1504: 8 total, 8 up, 6 in 2024-08-17T18:22:04.266 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:04 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4011011665' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:22:04.267 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:04 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:22:04.272 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:04.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:04.312 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:04.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:04.409 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:04 smithi130 ceph-mon[105521]: osdmap e1504: 8 total, 8 up, 6 in 2024-08-17T18:22:04.409 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:04 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4011011665' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:22:04.409 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:04 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:22:04.410 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:04 smithi130 ceph-mon[110068]: osdmap e1504: 8 total, 8 up, 6 in 2024-08-17T18:22:04.410 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:04 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4011011665' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:22:04.410 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:04 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:22:04.413 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:04.442 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:04.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:04.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:04.544 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:04.574 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:04.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:04.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:04.680 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:04.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:04.734 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:04.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:04.835 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:04.867 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:04.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:04.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:04.969 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:04.973 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:22:04.973 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T18:22:04.973 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T18:22:04.973 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T18:22:04.974 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.974 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:131689 50.a 50.f2e7e73a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1446)", 2024-08-17T18:22:04.974 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:49.528457+0000", 2024-08-17T18:22:04.974 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 195.44107020199999, 2024-08-17T18:22:04.974 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97764823599999995, 2024-08-17T18:22:04.974 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:04.974 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:22:04.974 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:04.974 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:04.974 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:04.974 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 131689 2024-08-17T18:22:04.975 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.975 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:04.975 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.975 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:04.975 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.528457+0000", 2024-08-17T18:22:04.975 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:04.975 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.975 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.975 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:04.975 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.528457+0000", 2024-08-17T18:22:04.975 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:04.975 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.975 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.976 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:04.976 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.528455+0000", 2024-08-17T18:22:04.976 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:04.976 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.976 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.976 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:04.976 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.528643+0000", 2024-08-17T18:22:04.976 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000187126 2024-08-17T18:22:04.976 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.976 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.976 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:04.976 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.528671+0000", 2024-08-17T18:22:04.976 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.8381000000000002e-05 2024-08-17T18:22:04.976 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.977 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.977 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:04.977 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.528690+0000", 2024-08-17T18:22:04.977 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.8844000000000001e-05 2024-08-17T18:22:04.977 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.977 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.977 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:04.977 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.528769+0000", 2024-08-17T18:22:04.977 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9396999999999999e-05 2024-08-17T18:22:04.977 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.977 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.977 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:04.978 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.528778+0000", 2024-08-17T18:22:04.978 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.4400000000000005e-06 2024-08-17T18:22:04.978 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.978 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.978 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:04.978 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:50.506099+0000", 2024-08-17T18:22:04.978 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97732155499999995 2024-08-17T18:22:04.978 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.978 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.978 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:04.978 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:50.506105+0000", 2024-08-17T18:22:04.978 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0430000000000004e-06 2024-08-17T18:22:04.978 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:04.979 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:04.979 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:04.979 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.979 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.979 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:131700 50.a 50.e14add6a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1446)", 2024-08-17T18:22:04.979 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:49.529014+0000", 2024-08-17T18:22:04.979 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 195.44051282800001, 2024-08-17T18:22:04.979 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97711819600000005, 2024-08-17T18:22:04.979 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:04.979 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:22:04.979 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:04.979 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:04.979 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:04.980 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 131700 2024-08-17T18:22:04.980 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.980 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:04.980 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.980 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:04.980 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.529014+0000", 2024-08-17T18:22:04.980 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:04.980 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.981 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.981 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:04.981 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.529014+0000", 2024-08-17T18:22:04.981 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:04.981 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.981 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.981 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:04.981 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.529014+0000", 2024-08-17T18:22:04.981 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:22:04.981 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.981 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.981 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:04.981 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.529078+0000", 2024-08-17T18:22:04.982 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4210999999999999e-05 2024-08-17T18:22:04.982 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.982 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.982 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:04.982 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.529088+0000", 2024-08-17T18:22:04.982 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0199e-05 2024-08-17T18:22:04.982 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.982 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.982 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:04.982 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.529094+0000", 2024-08-17T18:22:04.982 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5589999999999996e-06 2024-08-17T18:22:04.982 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.983 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.983 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:04.983 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.529336+0000", 2024-08-17T18:22:04.983 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000242692 2024-08-17T18:22:04.983 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.983 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.983 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:04.983 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:49.529342+0000", 2024-08-17T18:22:04.983 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2719999999999997e-06 2024-08-17T18:22:04.983 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.983 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.983 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:04.984 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:50.506126+0000", 2024-08-17T18:22:04.984 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97678425300000005 2024-08-17T18:22:04.984 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.984 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.984 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:04.984 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:50.506133+0000", 2024-08-17T18:22:04.984 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7039999999999997e-06 2024-08-17T18:22:04.984 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:04.984 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:04.984 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:04.984 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.985 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.985 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:248334 50.0 50:027b6e78:::benchmark_data_smithi130_398002_object103268:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1477)", 2024-08-17T18:22:04.985 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:11.910990+0000", 2024-08-17T18:22:04.985 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 53.058537158, 2024-08-17T18:22:04.985 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.905535985, 2024-08-17T18:22:04.985 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:04.985 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:04.985 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:04.985 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:04.985 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:04.985 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 248334 2024-08-17T18:22:04.985 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.985 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:04.986 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.986 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:04.986 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:11.910990+0000", 2024-08-17T18:22:04.986 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:04.986 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.986 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.986 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:04.986 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:11.910990+0000", 2024-08-17T18:22:04.986 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:04.986 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.986 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.986 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:04.990 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:11.910988+0000", 2024-08-17T18:22:04.990 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:04.990 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.990 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.991 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:04.991 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:11.910996+0000", 2024-08-17T18:22:04.991 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.5050000000000001e-06 2024-08-17T18:22:04.991 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.991 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.991 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:04.991 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:11.911011+0000", 2024-08-17T18:22:04.991 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4819e-05 2024-08-17T18:22:04.991 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.991 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.991 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:04.991 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:11.911017+0000", 2024-08-17T18:22:04.992 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9989999999999997e-06 2024-08-17T18:22:04.992 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.992 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.992 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:04.992 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:11.911056+0000", 2024-08-17T18:22:04.992 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9428999999999998e-05 2024-08-17T18:22:04.992 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.992 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.992 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:04.992 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:11.911063+0000", 2024-08-17T18:22:04.992 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5459999999999997e-06 2024-08-17T18:22:04.992 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.992 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.992 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:04.993 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:12.763717+0000", 2024-08-17T18:22:04.993 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.85265399799999997 2024-08-17T18:22:04.993 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.993 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.994 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:04.994 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:12.763726+0000", 2024-08-17T18:22:04.994 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.1269999999999992e-06 2024-08-17T18:22:04.994 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.994 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.994 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:04.994 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:13.800605+0000", 2024-08-17T18:22:04.994 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.036878991 2024-08-17T18:22:04.994 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.994 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.995 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:04.995 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:13.800737+0000", 2024-08-17T18:22:04.995 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013246899999999999 2024-08-17T18:22:04.995 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.995 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.995 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2,6", 2024-08-17T18:22:04.995 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:13.801348+0000", 2024-08-17T18:22:04.995 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00061030800000000005 2024-08-17T18:22:04.995 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.995 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.995 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:04.995 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:13.816236+0000", 2024-08-17T18:22:04.995 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.014888492999999999 2024-08-17T18:22:04.995 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.996 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.996 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:04.996 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:13.816246+0000", 2024-08-17T18:22:04.996 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.8260000000000005e-06 2024-08-17T18:22:04.996 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.996 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.996 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:04.996 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:13.816293+0000", 2024-08-17T18:22:04.996 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.672e-05 2024-08-17T18:22:04.996 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.996 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.996 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:04.997 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:13.816339+0000", 2024-08-17T18:22:04.997 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6016000000000001e-05 2024-08-17T18:22:04.997 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.997 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.997 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:04.997 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:13.816397+0000", 2024-08-17T18:22:04.997 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8683e-05 2024-08-17T18:22:04.997 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.997 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.997 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:04.997 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:13.816526+0000", 2024-08-17T18:22:04.997 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012867600000000001 2024-08-17T18:22:04.997 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:04.997 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:04.998 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:04.998 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.998 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.998 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:249047 50.a 50:5bc7b0ce:::benchmark_data_smithi130_398002_object103981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1480)", 2024-08-17T18:22:04.998 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:14.820102+0000", 2024-08-17T18:22:04.998 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 50.149425262000001, 2024-08-17T18:22:04.998 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.047594125, 2024-08-17T18:22:04.998 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:04.998 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:04.998 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:04.998 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:04.998 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:04.999 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 249047 2024-08-17T18:22:04.999 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.999 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:04.999 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.999 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:04.999 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.820102+0000", 2024-08-17T18:22:04.999 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:04.999 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:04.999 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:04.999 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:04.999 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.820102+0000", 2024-08-17T18:22:05.000 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.000 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.000 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.000 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.000 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.820100+0000", 2024-08-17T18:22:05.000 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:05.000 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.000 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.000 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.000 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.820111+0000", 2024-08-17T18:22:05.000 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.013e-05 2024-08-17T18:22:05.000 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.000 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.820125+0000", 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4506e-05 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.820131+0000", 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.314e-06 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.820191+0000", 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9808000000000001e-05 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.001 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.002 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.002 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.820202+0000", 2024-08-17T18:22:05.002 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0329999999999999e-05 2024-08-17T18:22:05.002 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.002 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.002 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.002 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.846359+0000", 2024-08-17T18:22:05.002 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.026157282 2024-08-17T18:22:05.002 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.002 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.002 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:05.002 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.846494+0000", 2024-08-17T18:22:05.003 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000134859 2024-08-17T18:22:05.003 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.003 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.003 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 6,7", 2024-08-17T18:22:05.003 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.847095+0000", 2024-08-17T18:22:05.003 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00060159199999999997 2024-08-17T18:22:05.003 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.003 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.003 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:05.003 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.867413+0000", 2024-08-17T18:22:05.003 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.020317983000000001 2024-08-17T18:22:05.003 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.003 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.004 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.004 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.867543+0000", 2024-08-17T18:22:05.004 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012943699999999999 2024-08-17T18:22:05.004 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.004 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.004 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.004 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.867606+0000", 2024-08-17T18:22:05.004 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3487999999999999e-05 2024-08-17T18:22:05.004 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.004 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.004 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:05.004 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.867658+0000", 2024-08-17T18:22:05.005 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2169999999999997e-05 2024-08-17T18:22:05.005 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.005 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.005 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.005 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.867696+0000", 2024-08-17T18:22:05.005 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7777000000000001e-05 2024-08-17T18:22:05.005 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.005 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.005 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.005 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.005 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.006 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:249084 50.a 50:50422086:::benchmark_data_smithi130_398002_object104018:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1480)", 2024-08-17T18:22:05.006 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:14.866505+0000", 2024-08-17T18:22:05.006 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 50.103022265, 2024-08-17T18:22:05.006 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.001307199, 2024-08-17T18:22:05.006 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.006 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:05.006 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.006 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.006 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.006 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 249084 2024-08-17T18:22:05.006 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.006 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.006 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.007 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.007 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.866505+0000", 2024-08-17T18:22:05.007 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.007 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.007 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.007 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.007 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.866505+0000", 2024-08-17T18:22:05.007 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.007 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.007 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.007 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.007 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.866504+0000", 2024-08-17T18:22:05.007 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:05.008 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.008 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.008 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.008 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.866510+0000", 2024-08-17T18:22:05.008 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2650000000000002e-06 2024-08-17T18:22:05.008 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.008 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.008 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.008 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.866520+0000", 2024-08-17T18:22:05.008 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.042e-05 2024-08-17T18:22:05.008 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.008 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.008 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.008 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.866526+0000", 2024-08-17T18:22:05.009 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8490000000000002e-06 2024-08-17T18:22:05.009 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.009 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.009 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.009 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.866575+0000", 2024-08-17T18:22:05.009 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8538000000000002e-05 2024-08-17T18:22:05.009 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.009 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.009 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.009 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.866584+0000", 2024-08-17T18:22:05.009 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.9760000000000001e-06 2024-08-17T18:22:05.009 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.009 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.010 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.010 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.858795+0000", 2024-08-17T18:22:05.010 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99221094499999996 2024-08-17T18:22:05.010 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.010 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.010 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:05.010 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.858925+0000", 2024-08-17T18:22:05.010 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013046800000000001 2024-08-17T18:22:05.010 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.010 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.010 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 6,7", 2024-08-17T18:22:05.010 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.859056+0000", 2024-08-17T18:22:05.010 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013120699999999999 2024-08-17T18:22:05.011 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.011 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.011 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.011 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.867578+0000", 2024-08-17T18:22:05.011 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0085219670000000001 2024-08-17T18:22:05.011 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.011 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.011 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:05.011 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.867583+0000", 2024-08-17T18:22:05.011 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3139999999999997e-06 2024-08-17T18:22:05.011 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.011 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.011 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.012 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.867755+0000", 2024-08-17T18:22:05.012 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000172714 2024-08-17T18:22:05.012 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.012 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.012 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:05.012 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.867789+0000", 2024-08-17T18:22:05.012 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3210000000000002e-05 2024-08-17T18:22:05.012 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.012 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.012 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.012 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.867812+0000", 2024-08-17T18:22:05.012 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3629e-05 2024-08-17T18:22:05.012 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.012 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.013 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.013 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.013 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.013 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:249098 50.a 50:52c44813:::benchmark_data_smithi130_398002_object104032:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1480)", 2024-08-17T18:22:05.013 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:14.888029+0000", 2024-08-17T18:22:05.013 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 50.081498318999998, 2024-08-17T18:22:05.013 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98575658399999999, 2024-08-17T18:22:05.013 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.013 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:05.013 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.013 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.013 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.013 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 249098 2024-08-17T18:22:05.013 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.014 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.014 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.014 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.014 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.888029+0000", 2024-08-17T18:22:05.014 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.014 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.014 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.014 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.014 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.888029+0000", 2024-08-17T18:22:05.014 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.014 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.014 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.014 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.015 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.888027+0000", 2024-08-17T18:22:05.015 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:22:05.015 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.015 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.015 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.015 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.888038+0000", 2024-08-17T18:22:05.015 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0997e-05 2024-08-17T18:22:05.015 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.015 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.015 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.016 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.888056+0000", 2024-08-17T18:22:05.016 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.7859e-05 2024-08-17T18:22:05.016 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.016 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.016 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.016 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.888063+0000", 2024-08-17T18:22:05.016 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3139999999999998e-06 2024-08-17T18:22:05.016 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.016 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.016 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.016 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.888108+0000", 2024-08-17T18:22:05.016 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5195999999999999e-05 2024-08-17T18:22:05.017 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.017 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.017 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.017 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:14.888120+0000", 2024-08-17T18:22:05.017 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1714e-05 2024-08-17T18:22:05.017 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.017 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.017 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.017 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.862149+0000", 2024-08-17T18:22:05.017 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.974028915 2024-08-17T18:22:05.017 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.017 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.018 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:05.018 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.862259+0000", 2024-08-17T18:22:05.018 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000109554 2024-08-17T18:22:05.018 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.018 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.018 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 6,7", 2024-08-17T18:22:05.018 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.862375+0000", 2024-08-17T18:22:05.018 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00011660800000000001 2024-08-17T18:22:05.018 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.018 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.018 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:05.018 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.867709+0000", 2024-08-17T18:22:05.018 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0053335819999999999 2024-08-17T18:22:05.019 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.019 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.019 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.019 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.867726+0000", 2024-08-17T18:22:05.019 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.7470999999999999e-05 2024-08-17T18:22:05.019 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.019 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.019 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.019 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.873698+0000", 2024-08-17T18:22:05.019 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0059717290000000003 2024-08-17T18:22:05.019 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.019 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.020 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:05.020 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.873743+0000", 2024-08-17T18:22:05.020 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5127999999999999e-05 2024-08-17T18:22:05.020 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.020 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.020 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.020 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:15.873785+0000", 2024-08-17T18:22:05.020 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2465000000000001e-05 2024-08-17T18:22:05.020 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.020 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.020 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.020 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.020 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.020 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:257742 50.a 50:5e8da67b:::benchmark_data_smithi130_398002_object112676:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1490)", 2024-08-17T18:22:05.021 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:33.479247+0000", 2024-08-17T18:22:05.021 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 31.490280636000001, 2024-08-17T18:22:05.021 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.140117077, 2024-08-17T18:22:05.021 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.021 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:05.021 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.021 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.021 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.021 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 257742 2024-08-17T18:22:05.021 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.021 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.021 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.021 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.021 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.479247+0000", 2024-08-17T18:22:05.022 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.022 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.022 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.022 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.022 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.479247+0000", 2024-08-17T18:22:05.022 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.022 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.022 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.022 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.022 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.479245+0000", 2024-08-17T18:22:05.022 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:22:05.022 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.022 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.023 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.023 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.479251+0000", 2024-08-17T18:22:05.023 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7320000000000003e-06 2024-08-17T18:22:05.023 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.023 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.023 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.023 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.479261+0000", 2024-08-17T18:22:05.023 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0142999999999999e-05 2024-08-17T18:22:05.023 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.023 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.023 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.023 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.479267+0000", 2024-08-17T18:22:05.024 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.463e-06 2024-08-17T18:22:05.024 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.024 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.024 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.024 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.479317+0000", 2024-08-17T18:22:05.024 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.9672000000000001e-05 2024-08-17T18:22:05.024 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.024 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.024 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.024 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.479323+0000", 2024-08-17T18:22:05.024 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.8340000000000001e-06 2024-08-17T18:22:05.024 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.024 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.024 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.025 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.605877+0000", 2024-08-17T18:22:05.025 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1265539920000001 2024-08-17T18:22:05.025 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.025 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.025 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:05.025 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.605958+0000", 2024-08-17T18:22:05.025 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.0394000000000003e-05 2024-08-17T18:22:05.025 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.025 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.025 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,6,7", 2024-08-17T18:22:05.025 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.606031+0000", 2024-08-17T18:22:05.025 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3102000000000006e-05 2024-08-17T18:22:05.025 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.026 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.026 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:05.026 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.614849+0000", 2024-08-17T18:22:05.026 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0088179150000000008 2024-08-17T18:22:05.026 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.026 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.026 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.026 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.619119+0000", 2024-08-17T18:22:05.026 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0042703719999999997 2024-08-17T18:22:05.026 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.026 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.026 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.027 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.619197+0000", 2024-08-17T18:22:05.027 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7799000000000005e-05 2024-08-17T18:22:05.027 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.027 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.027 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.027 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.619252+0000", 2024-08-17T18:22:05.027 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5090999999999998e-05 2024-08-17T18:22:05.027 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.027 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.027 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:05.027 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.619286+0000", 2024-08-17T18:22:05.027 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3506000000000001e-05 2024-08-17T18:22:05.027 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.028 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.028 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.028 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.619364+0000", 2024-08-17T18:22:05.028 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.8153999999999997e-05 2024-08-17T18:22:05.028 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.028 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.028 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.028 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.028 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.028 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:263420 50.12 50:4b5cd8ea:::benchmark_data_smithi130_398002_object118354:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1493)", 2024-08-17T18:22:05.028 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:41.148856+0000", 2024-08-17T18:22:05.028 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 23.820671165, 2024-08-17T18:22:05.029 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.032365961, 2024-08-17T18:22:05.029 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.029 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:22:05.029 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.029 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.029 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.029 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 263420 2024-08-17T18:22:05.029 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.029 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.029 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.029 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.029 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:41.148856+0000", 2024-08-17T18:22:05.029 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.029 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.030 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.030 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.030 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:41.148856+0000", 2024-08-17T18:22:05.030 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.030 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.030 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.030 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.030 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:41.148856+0000", 2024-08-17T18:22:05.030 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:22:05.030 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.030 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.030 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.031 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:41.148859+0000", 2024-08-17T18:22:05.031 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4379999999999999e-06 2024-08-17T18:22:05.031 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.031 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.031 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.031 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:41.148866+0000", 2024-08-17T18:22:05.031 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.5399999999999998e-06 2024-08-17T18:22:05.031 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.031 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.031 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.031 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:41.148870+0000", 2024-08-17T18:22:05.032 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7100000000000001e-06 2024-08-17T18:22:05.032 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.032 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.032 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.032 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:41.148913+0000", 2024-08-17T18:22:05.032 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2301e-05 2024-08-17T18:22:05.032 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.032 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.032 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.032 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:41.148918+0000", 2024-08-17T18:22:05.032 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7690000000000002e-06 2024-08-17T18:22:05.032 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.032 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.033 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.033 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:42.181155+0000", 2024-08-17T18:22:05.033 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.032236476 2024-08-17T18:22:05.033 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.033 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.033 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.033 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:42.181222+0000", 2024-08-17T18:22:05.033 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7275999999999998e-05 2024-08-17T18:22:05.033 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.033 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.033 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.033 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.033 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.033 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:268388 50.b 50:d1e6af7a:::benchmark_data_smithi130_398002_object123322:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1495)", 2024-08-17T18:22:05.034 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:48.181080+0000", 2024-08-17T18:22:05.034 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 16.788447687000001, 2024-08-17T18:22:05.034 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0366244710000001, 2024-08-17T18:22:05.034 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.034 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:22:05.034 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.034 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.034 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.034 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 268388 2024-08-17T18:22:05.034 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.034 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.034 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.034 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.035 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.181080+0000", 2024-08-17T18:22:05.035 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.035 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.035 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.035 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.035 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.181080+0000", 2024-08-17T18:22:05.035 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.035 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.035 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.036 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.036 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.181079+0000", 2024-08-17T18:22:05.036 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:22:05.036 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.036 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.036 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.036 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.181083+0000", 2024-08-17T18:22:05.036 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0559999999999998e-06 2024-08-17T18:22:05.036 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.036 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.036 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.036 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.181092+0000", 2024-08-17T18:22:05.037 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.2289999999999992e-06 2024-08-17T18:22:05.037 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.037 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.037 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.037 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.181097+0000", 2024-08-17T18:22:05.037 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1449999999999999e-06 2024-08-17T18:22:05.037 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.037 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.037 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.037 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.182228+0000", 2024-08-17T18:22:05.037 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001131197 2024-08-17T18:22:05.037 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.037 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.038 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.038 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.182234+0000", 2024-08-17T18:22:05.038 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7899999999999996e-06 2024-08-17T18:22:05.038 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.038 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.038 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.038 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.217617+0000", 2024-08-17T18:22:05.038 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.035383108 2024-08-17T18:22:05.038 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.038 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.038 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.038 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.217704+0000", 2024-08-17T18:22:05.038 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6687000000000005e-05 2024-08-17T18:22:05.039 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.039 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.039 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.039 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.039 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.039 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:268392 50.b 50:de4ffd8f:::benchmark_data_smithi130_398002_object123326:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1495)", 2024-08-17T18:22:05.039 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:48.181135+0000", 2024-08-17T18:22:05.040 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 16.788392130999998, 2024-08-17T18:22:05.040 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0370198980000001, 2024-08-17T18:22:05.040 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.040 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:22:05.040 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.040 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.040 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.040 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 268392 2024-08-17T18:22:05.040 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.040 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.040 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.040 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.040 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.181135+0000", 2024-08-17T18:22:05.041 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.041 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.041 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.041 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.041 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.181135+0000", 2024-08-17T18:22:05.041 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.041 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.041 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.041 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.041 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.181135+0000", 2024-08-17T18:22:05.041 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:22:05.041 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.041 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.041 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.042 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.181137+0000", 2024-08-17T18:22:05.042 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5639999999999999e-06 2024-08-17T18:22:05.042 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.042 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.042 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.042 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.181143+0000", 2024-08-17T18:22:05.042 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9800000000000003e-06 2024-08-17T18:22:05.042 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.042 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.042 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.042 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.181147+0000", 2024-08-17T18:22:05.042 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6950000000000002e-06 2024-08-17T18:22:05.042 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.042 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.043 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.043 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.182951+0000", 2024-08-17T18:22:05.043 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0018039200000000001 2024-08-17T18:22:05.043 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.043 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.043 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.043 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.182957+0000", 2024-08-17T18:22:05.043 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.404e-06 2024-08-17T18:22:05.043 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.043 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.043 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.043 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.218090+0000", 2024-08-17T18:22:05.043 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0351329579999999 2024-08-17T18:22:05.043 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.044 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.044 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.044 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.218155+0000", 2024-08-17T18:22:05.044 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4808999999999996e-05 2024-08-17T18:22:05.044 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.044 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.044 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.044 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.044 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.044 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:268402 50.b 50:dafb1d5a:::benchmark_data_smithi130_398002_object123336:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1495)", 2024-08-17T18:22:05.044 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:48.189081+0000", 2024-08-17T18:22:05.044 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 16.780446583, 2024-08-17T18:22:05.045 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.038614366, 2024-08-17T18:22:05.045 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.045 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:05.045 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.045 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.045 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.045 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 268402 2024-08-17T18:22:05.045 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.045 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.045 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.045 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.045 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.189081+0000", 2024-08-17T18:22:05.045 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.045 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.046 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.046 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.046 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.189081+0000", 2024-08-17T18:22:05.046 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.046 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.046 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.046 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.046 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.189079+0000", 2024-08-17T18:22:05.046 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:05.046 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.046 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.046 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.046 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.189085+0000", 2024-08-17T18:22:05.047 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5550000000000002e-06 2024-08-17T18:22:05.047 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.047 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.047 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.047 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.189096+0000", 2024-08-17T18:22:05.047 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1498000000000001e-05 2024-08-17T18:22:05.047 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.047 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.047 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.047 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.189101+0000", 2024-08-17T18:22:05.047 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6299999999999997e-06 2024-08-17T18:22:05.047 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.047 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.048 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.048 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.189134+0000", 2024-08-17T18:22:05.048 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3105999999999998e-05 2024-08-17T18:22:05.048 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.048 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.048 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.048 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.189141+0000", 2024-08-17T18:22:05.048 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6540000000000002e-06 2024-08-17T18:22:05.048 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.048 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.048 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.048 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.218188+0000", 2024-08-17T18:22:05.048 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0290469229999999 2024-08-17T18:22:05.049 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.049 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.049 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:05.049 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.218266+0000", 2024-08-17T18:22:05.049 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7933000000000001e-05 2024-08-17T18:22:05.049 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.049 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.049 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:22:05.049 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.218368+0000", 2024-08-17T18:22:05.049 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000101827 2024-08-17T18:22:05.049 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.049 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.049 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.050 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.227570+0000", 2024-08-17T18:22:05.050 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0092024059999999998 2024-08-17T18:22:05.050 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.050 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.050 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:05.050 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.227576+0000", 2024-08-17T18:22:05.050 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4080000000000003e-06 2024-08-17T18:22:05.050 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.050 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.050 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.050 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.227606+0000", 2024-08-17T18:22:05.050 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9566000000000001e-05 2024-08-17T18:22:05.050 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.051 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.051 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:05.051 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.227651+0000", 2024-08-17T18:22:05.051 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4963000000000003e-05 2024-08-17T18:22:05.051 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.051 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.051 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.051 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.227695+0000", 2024-08-17T18:22:05.051 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4146000000000001e-05 2024-08-17T18:22:05.051 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.051 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.051 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.052 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.052 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.052 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:268422 50.b 50:d6ff7492:::benchmark_data_smithi130_398002_object123356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1495)", 2024-08-17T18:22:05.052 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:48.211751+0000", 2024-08-17T18:22:05.052 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 16.757776097000001, 2024-08-17T18:22:05.052 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.016082551, 2024-08-17T18:22:05.052 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.052 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:05.052 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.052 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.052 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.052 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 268422 2024-08-17T18:22:05.052 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.053 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.053 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.053 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.053 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.211751+0000", 2024-08-17T18:22:05.053 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.053 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.053 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.053 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.053 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.211751+0000", 2024-08-17T18:22:05.053 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.053 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.053 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.053 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:05.055 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:05.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:05.056 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.211750+0000", 2024-08-17T18:22:05.056 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:22:05.056 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.056 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.056 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.056 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.211756+0000", 2024-08-17T18:22:05.056 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3689999999999998e-06 2024-08-17T18:22:05.056 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.056 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.056 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.056 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.211763+0000", 2024-08-17T18:22:05.057 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9170000000000006e-06 2024-08-17T18:22:05.057 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.057 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.057 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.057 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.211768+0000", 2024-08-17T18:22:05.057 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8099999999999997e-06 2024-08-17T18:22:05.057 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.057 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.057 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.057 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.211820+0000", 2024-08-17T18:22:05.057 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1276000000000002e-05 2024-08-17T18:22:05.058 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.058 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.058 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.058 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.211827+0000", 2024-08-17T18:22:05.058 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.058e-06 2024-08-17T18:22:05.058 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.058 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.058 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.058 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.220514+0000", 2024-08-17T18:22:05.058 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0086876499999999 2024-08-17T18:22:05.058 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.058 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.059 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:05.059 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.220624+0000", 2024-08-17T18:22:05.059 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000109906 2024-08-17T18:22:05.059 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.059 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.059 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:22:05.059 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.220734+0000", 2024-08-17T18:22:05.059 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000109517 2024-08-17T18:22:05.059 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.059 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.059 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:05.059 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.227708+0000", 2024-08-17T18:22:05.059 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0069740280000000002 2024-08-17T18:22:05.060 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.060 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.060 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.060 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.227733+0000", 2024-08-17T18:22:05.060 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5520999999999998e-05 2024-08-17T18:22:05.060 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.060 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.060 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.060 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.227765+0000", 2024-08-17T18:22:05.060 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2305000000000003e-05 2024-08-17T18:22:05.060 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.060 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.060 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:05.060 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.227800+0000", 2024-08-17T18:22:05.061 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4739000000000001e-05 2024-08-17T18:22:05.061 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.061 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.061 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.061 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.227834+0000", 2024-08-17T18:22:05.061 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.3448000000000002e-05 2024-08-17T18:22:05.061 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.061 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.061 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.061 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.061 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.061 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:268441 50.b 50:df88baa1:::benchmark_data_smithi130_398002_object123375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1495)", 2024-08-17T18:22:05.061 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:48.232227+0000", 2024-08-17T18:22:05.062 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 16.737300620999999, 2024-08-17T18:22:05.062 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0008471750000001, 2024-08-17T18:22:05.062 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.062 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:05.062 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.062 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.062 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.062 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 268441 2024-08-17T18:22:05.062 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.062 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.062 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.062 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.062 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.232227+0000", 2024-08-17T18:22:05.063 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.063 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.063 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.063 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.063 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.232227+0000", 2024-08-17T18:22:05.063 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.063 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.063 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.063 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.063 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.232225+0000", 2024-08-17T18:22:05.063 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:22:05.063 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.063 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.063 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.232234+0000", 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.098e-06 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.232253+0000", 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.9167000000000001e-05 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.232260+0000", 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7610000000000002e-06 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.064 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.065 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.232318+0000", 2024-08-17T18:22:05.065 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.842e-05 2024-08-17T18:22:05.065 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.065 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.065 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.065 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.232328+0000", 2024-08-17T18:22:05.065 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.7180000000000008e-06 2024-08-17T18:22:05.065 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.065 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.065 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.065 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.223079+0000", 2024-08-17T18:22:05.065 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.990750982 2024-08-17T18:22:05.066 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.066 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.066 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:05.066 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.223171+0000", 2024-08-17T18:22:05.066 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.1738999999999997e-05 2024-08-17T18:22:05.066 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.066 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.066 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:22:05.066 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.223266+0000", 2024-08-17T18:22:05.066 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5029000000000001e-05 2024-08-17T18:22:05.066 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.066 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.066 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:05.067 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.227841+0000", 2024-08-17T18:22:05.067 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.004575132 2024-08-17T18:22:05.067 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.067 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.067 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.067 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.232918+0000", 2024-08-17T18:22:05.067 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0050772830000000001 2024-08-17T18:22:05.067 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.067 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.067 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.067 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.232977+0000", 2024-08-17T18:22:05.067 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8692e-05 2024-08-17T18:22:05.067 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.067 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.068 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:05.068 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.233030+0000", 2024-08-17T18:22:05.068 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3115999999999998e-05 2024-08-17T18:22:05.068 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.068 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.068 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.068 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.233074+0000", 2024-08-17T18:22:05.068 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4023000000000002e-05 2024-08-17T18:22:05.068 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.068 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.068 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.068 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.068 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.068 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:268445 50.b 50:db15065c:::benchmark_data_smithi130_398002_object123379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1495)", 2024-08-17T18:22:05.069 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:48.233962+0000", 2024-08-17T18:22:05.069 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 16.735565565000002, 2024-08-17T18:22:05.069 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99922783900000001, 2024-08-17T18:22:05.069 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.069 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:05.069 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.069 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.069 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.069 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 268445 2024-08-17T18:22:05.069 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.069 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.069 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.070 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.070 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.233962+0000", 2024-08-17T18:22:05.070 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.070 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.070 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.070 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.070 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.233962+0000", 2024-08-17T18:22:05.070 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.070 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.070 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.070 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.070 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.233960+0000", 2024-08-17T18:22:05.070 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:05.071 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.071 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.071 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.071 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.233970+0000", 2024-08-17T18:22:05.071 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0118999999999999e-05 2024-08-17T18:22:05.071 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.071 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.071 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.071 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.233989+0000", 2024-08-17T18:22:05.071 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.8419000000000001e-05 2024-08-17T18:22:05.071 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.071 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.071 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.233995+0000", 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5849999999999996e-06 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.234051+0000", 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5991e-05 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.234063+0000", 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1335000000000001e-05 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.072 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.073 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.225134+0000", 2024-08-17T18:22:05.073 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99107193699999996 2024-08-17T18:22:05.073 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.073 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.073 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:05.073 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.225247+0000", 2024-08-17T18:22:05.073 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000112693 2024-08-17T18:22:05.073 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.073 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.073 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:22:05.073 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.225348+0000", 2024-08-17T18:22:05.073 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000100812 2024-08-17T18:22:05.073 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.073 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.074 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:05.074 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.227843+0000", 2024-08-17T18:22:05.074 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024953839999999998 2024-08-17T18:22:05.074 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.074 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.074 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.074 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.232949+0000", 2024-08-17T18:22:05.074 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0051055229999999998 2024-08-17T18:22:05.074 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.074 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.074 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.074 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.233112+0000", 2024-08-17T18:22:05.074 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00016296 2024-08-17T18:22:05.075 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.075 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.075 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:05.075 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.233146+0000", 2024-08-17T18:22:05.075 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4118000000000001e-05 2024-08-17T18:22:05.075 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.075 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.075 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.075 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.233190+0000", 2024-08-17T18:22:05.075 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3529000000000003e-05 2024-08-17T18:22:05.075 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.075 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.076 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.076 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.076 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.076 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:268447 50.b 50:de41f0dd:::benchmark_data_smithi130_398002_object123381:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1495)", 2024-08-17T18:22:05.076 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:48.234421+0000", 2024-08-17T18:22:05.076 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 16.735106244000001, 2024-08-17T18:22:05.076 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0123331170000001, 2024-08-17T18:22:05.076 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.076 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:05.076 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.076 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.076 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.076 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 268447 2024-08-17T18:22:05.077 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.077 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.077 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.077 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.077 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.234421+0000", 2024-08-17T18:22:05.077 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.077 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.077 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.077 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.077 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.234421+0000", 2024-08-17T18:22:05.077 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.077 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.078 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.078 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.078 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.234420+0000", 2024-08-17T18:22:05.078 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999995 2024-08-17T18:22:05.078 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.078 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.078 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.078 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.234425+0000", 2024-08-17T18:22:05.078 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2860000000000001e-06 2024-08-17T18:22:05.078 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.078 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.078 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.078 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.234434+0000", 2024-08-17T18:22:05.079 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0289999999999994e-06 2024-08-17T18:22:05.079 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.079 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.079 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.079 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.234439+0000", 2024-08-17T18:22:05.079 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2689999999999999e-06 2024-08-17T18:22:05.079 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.079 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.079 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.079 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.234479+0000", 2024-08-17T18:22:05.079 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0181e-05 2024-08-17T18:22:05.079 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.079 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.080 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.080 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.234487+0000", 2024-08-17T18:22:05.080 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.6329999999999996e-06 2024-08-17T18:22:05.080 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.080 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.080 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.080 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.227875+0000", 2024-08-17T18:22:05.080 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99338844500000001 2024-08-17T18:22:05.080 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.080 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.080 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:05.080 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.227950+0000", 2024-08-17T18:22:05.081 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.5136999999999996e-05 2024-08-17T18:22:05.081 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.081 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.081 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:22:05.081 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.228041+0000", 2024-08-17T18:22:05.081 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.0519000000000005e-05 2024-08-17T18:22:05.081 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.081 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.081 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.081 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.238995+0000", 2024-08-17T18:22:05.081 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.010954317 2024-08-17T18:22:05.081 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.081 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.082 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:05.082 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.246621+0000", 2024-08-17T18:22:05.082 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0076260130000000001 2024-08-17T18:22:05.082 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.082 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.082 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.082 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.246672+0000", 2024-08-17T18:22:05.082 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0748999999999998e-05 2024-08-17T18:22:05.082 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.082 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.082 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:05.082 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.246718+0000", 2024-08-17T18:22:05.082 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6221999999999999e-05 2024-08-17T18:22:05.083 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.083 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.083 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.083 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.246754+0000", 2024-08-17T18:22:05.083 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6004000000000003e-05 2024-08-17T18:22:05.083 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.083 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.083 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.083 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.083 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.083 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:268470 50.b 50:d653fb18:::benchmark_data_smithi130_398002_object123404:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1495)", 2024-08-17T18:22:05.083 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:48.262744+0000", 2024-08-17T18:22:05.084 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 16.706782753999999, 2024-08-17T18:22:05.084 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98413915100000005, 2024-08-17T18:22:05.084 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.084 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:05.084 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.084 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.084 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.084 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 268470 2024-08-17T18:22:05.084 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.084 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.084 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.084 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.084 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.262744+0000", 2024-08-17T18:22:05.085 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.085 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.085 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.085 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.085 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.262744+0000", 2024-08-17T18:22:05.085 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.085 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.085 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.085 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.085 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.262744+0000", 2024-08-17T18:22:05.085 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:22:05.085 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.085 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.086 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.086 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.262749+0000", 2024-08-17T18:22:05.086 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3800000000000002e-06 2024-08-17T18:22:05.086 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.086 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.086 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.086 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.262760+0000", 2024-08-17T18:22:05.086 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1049e-05 2024-08-17T18:22:05.086 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.086 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.086 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.086 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.262766+0000", 2024-08-17T18:22:05.086 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.716e-06 2024-08-17T18:22:05.087 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.087 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.087 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.087 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.262822+0000", 2024-08-17T18:22:05.087 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6514000000000001e-05 2024-08-17T18:22:05.087 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.087 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.087 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.087 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:48.262831+0000", 2024-08-17T18:22:05.087 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6000000000000007e-06 2024-08-17T18:22:05.087 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.087 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.087 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.088 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.230119+0000", 2024-08-17T18:22:05.088 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.96728840199999999 2024-08-17T18:22:05.088 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.088 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.088 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:05.088 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.230223+0000", 2024-08-17T18:22:05.088 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000103649 2024-08-17T18:22:05.088 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.088 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.088 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 1,7", 2024-08-17T18:22:05.088 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.230328+0000", 2024-08-17T18:22:05.088 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105337 2024-08-17T18:22:05.088 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.089 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.089 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.089 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.239023+0000", 2024-08-17T18:22:05.089 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0086947440000000008 2024-08-17T18:22:05.089 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.089 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.089 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:05.089 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.246626+0000", 2024-08-17T18:22:05.089 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0076034830000000003 2024-08-17T18:22:05.089 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.089 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.089 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.089 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.246822+0000", 2024-08-17T18:22:05.089 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000195801 2024-08-17T18:22:05.090 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.090 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.090 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:05.090 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.246854+0000", 2024-08-17T18:22:05.090 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2249000000000002e-05 2024-08-17T18:22:05.090 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.090 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.090 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.090 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:49.246884+0000", 2024-08-17T18:22:05.090 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9135000000000001e-05 2024-08-17T18:22:05.090 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.090 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.090 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.091 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.091 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.091 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:276564 50.12 50:4af798a4:::benchmark_data_smithi130_398002_object131498:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1502)", 2024-08-17T18:22:05.091 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:22:01.201938+0000", 2024-08-17T18:22:05.091 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 3.7675892069999999, 2024-08-17T18:22:05.091 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98482448600000005, 2024-08-17T18:22:05.091 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.091 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:05.091 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.091 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.091 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.091 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 276564 2024-08-17T18:22:05.091 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.091 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.092 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.092 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.092 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.201938+0000", 2024-08-17T18:22:05.092 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.092 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.092 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.092 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.092 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.201938+0000", 2024-08-17T18:22:05.092 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.092 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.092 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.092 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.092 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.201936+0000", 2024-08-17T18:22:05.092 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:22:05.093 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.093 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.093 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.093 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.201947+0000", 2024-08-17T18:22:05.093 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1134999999999999e-05 2024-08-17T18:22:05.093 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.093 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.093 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.093 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.201960+0000", 2024-08-17T18:22:05.093 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2614e-05 2024-08-17T18:22:05.093 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.093 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.093 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.093 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.201965+0000", 2024-08-17T18:22:05.094 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3380000000000004e-06 2024-08-17T18:22:05.094 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.094 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.094 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.094 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.202189+0000", 2024-08-17T18:22:05.094 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00022333800000000001 2024-08-17T18:22:05.094 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.094 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.094 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.094 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.202198+0000", 2024-08-17T18:22:05.095 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.9379999999999998e-06 2024-08-17T18:22:05.095 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.095 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.095 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.095 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:02.179268+0000", 2024-08-17T18:22:05.095 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97707002300000001 2024-08-17T18:22:05.095 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.095 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.095 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:05.095 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:02.179397+0000", 2024-08-17T18:22:05.095 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012995699999999999 2024-08-17T18:22:05.095 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.096 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.096 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:22:05.096 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:02.179526+0000", 2024-08-17T18:22:05.096 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012880699999999999 2024-08-17T18:22:05.096 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.096 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.096 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.096 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:02.186649+0000", 2024-08-17T18:22:05.096 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0071225250000000002 2024-08-17T18:22:05.096 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.096 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.096 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.096 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:02.186692+0000", 2024-08-17T18:22:05.097 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2719999999999998e-05 2024-08-17T18:22:05.097 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.097 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.097 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:05.097 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:02.186697+0000", 2024-08-17T18:22:05.097 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.4090000000000002e-06 2024-08-17T18:22:05.097 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.097 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.097 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:05.097 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:02.186730+0000", 2024-08-17T18:22:05.097 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2854000000000001e-05 2024-08-17T18:22:05.097 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.097 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.098 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.098 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:02.186763+0000", 2024-08-17T18:22:05.098 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.2706999999999997e-05 2024-08-17T18:22:05.098 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.098 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.098 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.098 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.098 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.098 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:276916 50.b 50:d4db09bd:::benchmark_data_smithi130_398002_object131850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1504)", 2024-08-17T18:22:05.098 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:22:03.197047+0000", 2024-08-17T18:22:05.098 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 1.7724804679999999, 2024-08-17T18:22:05.098 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0077679260000001, 2024-08-17T18:22:05.098 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.099 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:05.099 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.099 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.099 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.099 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 276916 2024-08-17T18:22:05.099 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.099 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.099 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.099 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.099 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.197047+0000", 2024-08-17T18:22:05.099 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.100 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.100 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.100 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.100 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.197047+0000", 2024-08-17T18:22:05.100 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.100 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.100 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.100 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.100 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.197045+0000", 2024-08-17T18:22:05.100 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:22:05.100 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.100 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.100 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.101 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.197052+0000", 2024-08-17T18:22:05.101 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.922e-06 2024-08-17T18:22:05.101 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.101 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.101 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.101 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.197066+0000", 2024-08-17T18:22:05.101 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4362000000000001e-05 2024-08-17T18:22:05.101 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.101 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.101 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.101 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.197072+0000", 2024-08-17T18:22:05.101 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5860000000000004e-06 2024-08-17T18:22:05.101 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.101 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.102 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.102 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.197122+0000", 2024-08-17T18:22:05.102 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0139000000000002e-05 2024-08-17T18:22:05.102 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.102 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.102 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.102 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.197130+0000", 2024-08-17T18:22:05.102 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6139999999999992e-06 2024-08-17T18:22:05.102 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.102 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.102 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.102 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.198429+0000", 2024-08-17T18:22:05.103 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.001298308 2024-08-17T18:22:05.103 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.103 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.103 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:05.103 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.198517+0000", 2024-08-17T18:22:05.103 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7985999999999994e-05 2024-08-17T18:22:05.103 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.103 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.103 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:22:05.103 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.198590+0000", 2024-08-17T18:22:05.103 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3607000000000002e-05 2024-08-17T18:22:05.103 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.103 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.103 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.104 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.204648+0000", 2024-08-17T18:22:05.104 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0060578999999999997 2024-08-17T18:22:05.104 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.104 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.104 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:05.104 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.204656+0000", 2024-08-17T18:22:05.104 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3380000000000005e-06 2024-08-17T18:22:05.104 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.104 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.104 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.104 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.204752+0000", 2024-08-17T18:22:05.104 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5805999999999997e-05 2024-08-17T18:22:05.104 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.105 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.105 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:05.105 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.204789+0000", 2024-08-17T18:22:05.105 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6704e-05 2024-08-17T18:22:05.105 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.105 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.105 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.105 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.204815+0000", 2024-08-17T18:22:05.105 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.5724e-05 2024-08-17T18:22:05.105 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.105 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.105 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.105 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.106 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.106 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:276934 50.b 50:d3b26e74:::benchmark_data_smithi130_398002_object131868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1504)", 2024-08-17T18:22:05.106 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:22:03.215835+0000", 2024-08-17T18:22:05.106 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 1.753692402, 2024-08-17T18:22:05.106 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.99277999800000005, 2024-08-17T18:22:05.106 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.106 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:05.106 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.106 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.106 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.106 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 276934 2024-08-17T18:22:05.106 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.106 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.107 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.107 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.107 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.215835+0000", 2024-08-17T18:22:05.107 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.107 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.107 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.107 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.107 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.215835+0000", 2024-08-17T18:22:05.107 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.107 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.107 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.107 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.107 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.215833+0000", 2024-08-17T18:22:05.108 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:05.108 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.108 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.108 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.108 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.215841+0000", 2024-08-17T18:22:05.108 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.4529999999999996e-06 2024-08-17T18:22:05.108 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.108 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.108 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.108 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.215848+0000", 2024-08-17T18:22:05.108 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.4950000000000002e-06 2024-08-17T18:22:05.108 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.108 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.109 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.109 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.215853+0000", 2024-08-17T18:22:05.109 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5240000000000002e-06 2024-08-17T18:22:05.109 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.109 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.109 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.109 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.215897+0000", 2024-08-17T18:22:05.109 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3838999999999998e-05 2024-08-17T18:22:05.109 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.109 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.109 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.109 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.215903+0000", 2024-08-17T18:22:05.109 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.4359999999999999e-06 2024-08-17T18:22:05.109 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.110 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.110 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.110 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.201013+0000", 2024-08-17T18:22:05.110 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98511002599999997 2024-08-17T18:22:05.110 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.110 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.110 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:05.110 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.201098+0000", 2024-08-17T18:22:05.110 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.4913000000000005e-05 2024-08-17T18:22:05.110 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.110 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.110 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:22:05.111 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.201159+0000", 2024-08-17T18:22:05.111 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0643000000000002e-05 2024-08-17T18:22:05.111 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.111 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.111 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.111 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.204874+0000", 2024-08-17T18:22:05.111 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037150059999999999 2024-08-17T18:22:05.111 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.111 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.111 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:05.111 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.204882+0000", 2024-08-17T18:22:05.111 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.5259999999999993e-06 2024-08-17T18:22:05.112 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.112 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.112 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.112 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.208528+0000", 2024-08-17T18:22:05.112 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0036456169999999999 2024-08-17T18:22:05.112 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.112 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.112 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:05.112 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.208576+0000", 2024-08-17T18:22:05.112 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8059999999999997e-05 2024-08-17T18:22:05.112 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.113 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.113 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.113 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.208615+0000", 2024-08-17T18:22:05.113 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8800999999999997e-05 2024-08-17T18:22:05.113 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.113 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.113 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.113 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.113 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.113 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:276952 50.b 50:dcd3275d:::benchmark_data_smithi130_398002_object131886:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1504)", 2024-08-17T18:22:05.113 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:22:03.227462+0000", 2024-08-17T18:22:05.113 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 1.74206555, 2024-08-17T18:22:05.113 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98126887299999999, 2024-08-17T18:22:05.114 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:05.114 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:05.114 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:05.114 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:05.114 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:05.114 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 276952 2024-08-17T18:22:05.114 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.114 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:05.114 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.115 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:05.115 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.227462+0000", 2024-08-17T18:22:05.115 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.115 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.115 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.115 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:05.115 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.227462+0000", 2024-08-17T18:22:05.115 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:05.115 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.115 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.115 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:05.115 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.227461+0000", 2024-08-17T18:22:05.115 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:22:05.116 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.116 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.116 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:05.116 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.227466+0000", 2024-08-17T18:22:05.116 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1710000000000001e-06 2024-08-17T18:22:05.116 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.116 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.116 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:05.116 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.227477+0000", 2024-08-17T18:22:05.116 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1637000000000001e-05 2024-08-17T18:22:05.116 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.116 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.116 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:05.117 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.227482+0000", 2024-08-17T18:22:05.117 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0699999999999997e-06 2024-08-17T18:22:05.117 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.117 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.117 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.117 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.227526+0000", 2024-08-17T18:22:05.117 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3213000000000001e-05 2024-08-17T18:22:05.117 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.117 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.117 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:05.117 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:03.227533+0000", 2024-08-17T18:22:05.117 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3359999999999997e-06 2024-08-17T18:22:05.117 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.117 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.118 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:05.118 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.202887+0000", 2024-08-17T18:22:05.118 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97535408499999998 2024-08-17T18:22:05.118 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.118 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.118 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:05.118 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.202982+0000", 2024-08-17T18:22:05.118 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5006000000000005e-05 2024-08-17T18:22:05.118 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.118 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.118 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 3,7", 2024-08-17T18:22:05.118 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.203068+0000", 2024-08-17T18:22:05.118 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6188999999999998e-05 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.204891+0000", 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001822288 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.204911+0000", 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.0655000000000001e-05 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.208654+0000", 2024-08-17T18:22:05.119 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037429500000000001 2024-08-17T18:22:05.120 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.120 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.120 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:05.120 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.208695+0000", 2024-08-17T18:22:05.120 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0355999999999998e-05 2024-08-17T18:22:05.120 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:05.120 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:05.120 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:05.120 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:04.208731+0000", 2024-08-17T18:22:05.120 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6016000000000002e-05 2024-08-17T18:22:05.120 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.120 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.120 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.120 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:05.121 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:05.121 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:22:05.156 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:05.178 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:05.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:05.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:05.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:05.314 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:05.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:05.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:05.416 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:05.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:05 smithi130 ceph-mon[110068]: pgmap v2928: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 7 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 824 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 272 op/s; 4244/39624 objects degraded (10.711%); 3982/39624 objects misplaced (10.049%); 35 MiB/s, 657 objects/s recovering 2024-08-17T18:22:05.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:05 smithi130 ceph-mon[110068]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 4 pgs peering) 2024-08-17T18:22:05.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:05 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:22:05.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:05 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:22:05.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:05 smithi130 ceph-mon[110068]: osdmap e1505: 8 total, 8 up, 6 in 2024-08-17T18:22:05.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:05 smithi130 ceph-mon[105521]: pgmap v2928: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 7 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 824 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 272 op/s; 4244/39624 objects degraded (10.711%); 3982/39624 objects misplaced (10.049%); 35 MiB/s, 657 objects/s recovering 2024-08-17T18:22:05.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:05 smithi130 ceph-mon[105521]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 4 pgs peering) 2024-08-17T18:22:05.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:05 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:22:05.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:05 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:22:05.418 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:05 smithi130 ceph-mon[105521]: osdmap e1505: 8 total, 8 up, 6 in 2024-08-17T18:22:05.443 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:05.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:05.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:05.544 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:05.579 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:05.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:05.592 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 7, 6, 4, 0] out_osds: [5, 1] dead_osds: [] live_osds: [1, 4, 5, 6, 7, 0, 3, 2] 2024-08-17T18:22:05.592 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:22:05.593 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2024-08-17T18:22:05.593 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:22:05.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:05 smithi179 ceph-mon[93519]: pgmap v2928: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 7 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 824 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 272 op/s; 4244/39624 objects degraded (10.711%); 3982/39624 objects misplaced (10.049%); 35 MiB/s, 657 objects/s recovering 2024-08-17T18:22:05.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:05 smithi179 ceph-mon[93519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 4 pgs peering) 2024-08-17T18:22:05.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:05 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:22:05.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:05 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:22:05.594 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:05 smithi179 ceph-mon[93519]: osdmap e1505: 8 total, 8 up, 6 in 2024-08-17T18:22:05.681 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:05.715 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:05.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:05.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:05.818 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:05.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:05.839 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:05.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:05.940 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:05.999 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:05.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:06.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:06.100 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:06.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:06.139 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:06.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:06.240 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:06.276 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:06.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:06.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:06.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:06 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 4244/39624 objects degraded (10.711%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:22:06.378 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:06.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:06.407 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:06.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:06.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:06 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 4244/39624 objects degraded (10.711%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:22:06.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:06 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 4244/39624 objects degraded (10.711%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:22:06.488 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T18:22:06.508 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:06.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:06.575 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:06.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:06.676 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:06.713 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:06.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:06.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:06.815 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:06.846 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:06.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:06.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:06.866 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:22:06.948 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:06.983 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:06.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:06.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:07.085 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:07.120 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:07.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:07.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:07.222 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:07.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:07.259 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:07.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:07.274 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:07 smithi179 ceph-mon[93519]: pgmap v2930: 99 pgs: 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped, 1 active+clean+remapped, 1 peering, 7 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 820 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 255 op/s; 3132/39405 objects degraded (7.948%); 3982/39405 objects misplaced (10.105%); 31 MiB/s, 725 objects/s recovering 2024-08-17T18:22:07.360 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:07.396 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:07.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:07.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:07.498 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:07.542 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:07.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:07.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:07.644 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:07.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:07 smithi130 ceph-mon[105521]: pgmap v2930: 99 pgs: 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped, 1 active+clean+remapped, 1 peering, 7 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 820 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 255 op/s; 3132/39405 objects degraded (7.948%); 3982/39405 objects misplaced (10.105%); 31 MiB/s, 725 objects/s recovering 2024-08-17T18:22:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:07 smithi130 ceph-mon[110068]: pgmap v2930: 99 pgs: 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped, 1 active+clean+remapped, 1 peering, 7 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 820 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 255 op/s; 3132/39405 objects degraded (7.948%); 3982/39405 objects misplaced (10.105%); 31 MiB/s, 725 objects/s recovering 2024-08-17T18:22:07.680 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:07.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:07.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:07.782 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:07.810 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:07.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:07.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:07.912 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:07.933 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:07.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:07.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:08.034 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:08.071 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:08.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:08.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:08.172 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:08.204 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:08.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:08.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:08.306 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:08.337 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:08.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:08.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:08.439 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:08.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:08.472 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:08.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:08.573 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:08.619 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:08.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:08.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:08.721 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:08.759 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:08.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:08.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:08.860 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:08.896 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:08.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:08.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:08.999 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:09.020 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:09.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:09.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:09.122 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:09.160 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:09.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:09.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:09.261 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:09.289 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:09.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:09.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:09.391 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:09.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:09 smithi130 ceph-mon[105521]: pgmap v2931: 99 pgs: 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped, 1 active+clean+remapped, 1 peering, 7 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 820 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 3132/39405 objects degraded (7.948%); 3982/39405 objects misplaced (10.105%); 23 MiB/s, 552 objects/s recovering 2024-08-17T18:22:09.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:09 smithi130 ceph-mon[110068]: pgmap v2931: 99 pgs: 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped, 1 active+clean+remapped, 1 peering, 7 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 820 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 3132/39405 objects degraded (7.948%); 3982/39405 objects misplaced (10.105%); 23 MiB/s, 552 objects/s recovering 2024-08-17T18:22:09.421 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:09.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:09.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:09.523 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:09.550 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:09.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:09.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:09.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:09 smithi179 ceph-mon[93519]: pgmap v2931: 99 pgs: 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped, 1 active+clean+remapped, 1 peering, 7 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 82 active+clean; 820 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 194 op/s; 3132/39405 objects degraded (7.948%); 3982/39405 objects misplaced (10.105%); 23 MiB/s, 552 objects/s recovering 2024-08-17T18:22:09.652 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:09.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:09.685 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:09.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:09.786 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:09.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:09.820 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:09.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:09.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:09.964 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:09.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:09.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:10.066 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:10.096 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:10.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:10.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:22:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:22:09] "GET /metrics HTTP/1.1" 200 36805 "" "Prometheus/2.51.0" 2024-08-17T18:22:10.197 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:10.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:10.236 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:10.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:10.337 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:10.357 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:10.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:10.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:10.459 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:10.491 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:10.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:10.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:22:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:22:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:22:10.593 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:10.619 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:10.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:10.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:10.720 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:10.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:10.764 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:10.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:10.865 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:10.886 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:10.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:10.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:10.988 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:11.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:11.026 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:11.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:11.101 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:22:11.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:11.181 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:11.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:11.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:11.283 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:11.310 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:11.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:11.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:11.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:11 smithi179 ceph-mon[93519]: pgmap v2932: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 559 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 1948/26904 objects degraded (7.241%); 2767/26904 objects misplaced (10.285%); 20 MiB/s, 492 objects/s recovering 2024-08-17T18:22:11.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:11 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 1948/26904 objects degraded (7.241%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:22:11.395 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:22:11.395 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T18:22:11.396 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.396 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.47132.0:283917 50.d e1505/1489 50:ba55ccc6:::benchmark_data_smithi130_398002_object138851:head v 1505'17602, mlcod=1505'17598)", 2024-08-17T18:22:11.396 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:22:11.390006+0000", 2024-08-17T18:22:11.396 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.004749924, 2024-08-17T18:22:11.396 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0047854179999999996, 2024-08-17T18:22:11.396 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:11.396 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "started", 2024-08-17T18:22:11.396 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:11.396 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.396 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:11.396 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.390006+0000", 2024-08-17T18:22:11.397 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:11.397 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.397 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.397 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:11.397 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.390006+0000", 2024-08-17T18:22:11.397 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:11.397 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.397 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.397 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:11.397 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.390005+0000", 2024-08-17T18:22:11.397 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:11.397 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.397 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.397 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:11.398 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.390019+0000", 2024-08-17T18:22:11.398 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4131000000000001e-05 2024-08-17T18:22:11.398 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.398 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.398 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:11.398 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.390042+0000", 2024-08-17T18:22:11.398 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.3506000000000002e-05 2024-08-17T18:22:11.398 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.398 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.398 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:11.398 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.390050+0000", 2024-08-17T18:22:11.398 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.2609999999999996e-06 2024-08-17T18:22:11.398 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.399 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.399 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:11.399 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.390095+0000", 2024-08-17T18:22:11.399 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5908000000000002e-05 2024-08-17T18:22:11.399 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.399 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.399 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:11.399 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.390128+0000", 2024-08-17T18:22:11.399 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.277e-05 2024-08-17T18:22:11.399 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:11.399 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:11.399 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:11.399 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.400 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.400 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_repop(client.47132.0:283920 50.d e1505/1489 50:bfbde15e:::benchmark_data_smithi130_398002_object138854:head v 1505'17603, mlcod=1505'17598)", 2024-08-17T18:22:11.400 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:22:11.392396+0000", 2024-08-17T18:22:11.400 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 0.0023600470000000001, 2024-08-17T18:22:11.400 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.002462779, 2024-08-17T18:22:11.400 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:11.400 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "started", 2024-08-17T18:22:11.400 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:11.400 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.400 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:11.400 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.392396+0000", 2024-08-17T18:22:11.400 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:11.400 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.401 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.401 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:11.401 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.392396+0000", 2024-08-17T18:22:11.401 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:11.401 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.401 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.401 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:11.401 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.392395+0000", 2024-08-17T18:22:11.401 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:22:11.401 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.401 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.401 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:11.401 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.392406+0000", 2024-08-17T18:22:11.401 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1246e-05 2024-08-17T18:22:11.402 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.402 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.402 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:11.402 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.392422+0000", 2024-08-17T18:22:11.402 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5557e-05 2024-08-17T18:22:11.402 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.402 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.402 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:11.402 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.392428+0000", 2024-08-17T18:22:11.402 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6229999999999999e-06 2024-08-17T18:22:11.402 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.402 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.402 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:11.403 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.392714+0000", 2024-08-17T18:22:11.403 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00028561999999999999 2024-08-17T18:22:11.403 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:11.403 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:11.403 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:11.403 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:11.392736+0000", 2024-08-17T18:22:11.403 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.2467e-05 2024-08-17T18:22:11.403 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:11.403 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:11.403 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:11.403 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:11.403 INFO:teuthology.orchestra.run.smithi179.stdout: ], 2024-08-17T18:22:11.403 INFO:teuthology.orchestra.run.smithi179.stdout: "num_ops": 2 2024-08-17T18:22:11.403 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:22:11.412 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:11.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:11 smithi130 ceph-mon[105521]: pgmap v2932: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 559 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 1948/26904 objects degraded (7.241%); 2767/26904 objects misplaced (10.285%); 20 MiB/s, 492 objects/s recovering 2024-08-17T18:22:11.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:11 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 1948/26904 objects degraded (7.241%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:22:11.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:11 smithi130 ceph-mon[110068]: pgmap v2932: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 559 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 1948/26904 objects degraded (7.241%); 2767/26904 objects misplaced (10.285%); 20 MiB/s, 492 objects/s recovering 2024-08-17T18:22:11.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:11 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 1948/26904 objects degraded (7.241%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T18:22:11.461 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:11.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:11.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:11.563 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:11.598 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:11.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:11.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:11.700 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:11.731 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:11.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:11.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:11.833 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:11.874 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:11.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:11.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:11.977 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:12.009 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:12.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:12.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:12.111 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:12.143 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:12.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:12.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:12.234 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T18:22:12.244 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:12.286 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:12.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:12.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:12.388 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:12.423 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:12.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:12.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:12.525 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:12.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:12.571 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:12.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:12.643 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:22:12.672 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:12.715 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:12.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:12.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:12.817 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:12.853 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:12.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:12.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:12.954 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:12.987 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:12.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:12.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:13.089 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:13.113 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:13.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:13.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:13.154 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:13.196 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:13.215 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:13.256 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:13.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:13.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:13.358 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:13.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:13.381 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:13.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:13.415 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:13 smithi130 ceph-mon[105521]: pgmap v2933: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 559 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 594 op/s; 1948/26904 objects degraded (7.241%); 2767/26904 objects misplaced (10.285%); 7.6 MiB/s, 223 objects/s recovering 2024-08-17T18:22:13.415 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:13 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:22:13.415 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:13 smithi130 ceph-mon[110068]: pgmap v2933: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 559 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 594 op/s; 1948/26904 objects degraded (7.241%); 2767/26904 objects misplaced (10.285%); 7.6 MiB/s, 223 objects/s recovering 2024-08-17T18:22:13.415 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:13 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:22:13.482 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:13.509 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:13.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:13.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:13 smithi179 ceph-mon[93519]: pgmap v2933: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 559 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 594 op/s; 1948/26904 objects degraded (7.241%); 2767/26904 objects misplaced (10.285%); 7.6 MiB/s, 223 objects/s recovering 2024-08-17T18:22:13.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:13 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:22:13.611 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:13.659 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:13.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:13.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:13.761 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:13.799 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:13.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:13.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:13.901 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:13.930 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:13.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:13.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:14.032 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:14.053 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:14.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:14.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:14.154 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:14.184 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:14.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:14.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:14.286 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:14.317 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:14.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:14.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:14.419 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:14.444 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:14.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:14.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:14.546 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:14.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:14.593 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:14.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:14.694 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:14.725 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:14.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:14.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:14.827 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:14.835 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:22:14.835 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:22:14.835 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:22:14.835 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:22:14.835 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:22:14.835 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:22:14.835 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:22:14.835 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:22:14.835 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:22:14.835 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:22:14.836 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:22:14.836 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:22:14.836 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:22:14.836 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:22:14.836 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:22:14.836 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:22:14.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:14.864 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:14.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:14.965 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:14.986 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:14.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:14.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:15.088 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:15.117 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:15.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:15.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:15.218 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:15.250 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:15.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:15.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:15.352 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:15.373 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:15.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:15.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:15.475 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:15.512 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:15.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:15.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:15.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:15 smithi179 ceph-mon[93519]: pgmap v2934: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 280 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 871 op/s; 844/13470 objects degraded (6.266%); 1284/13470 objects misplaced (9.532%); 1.4 MiB/s, 140 objects/s recovering 2024-08-17T18:22:15.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:15 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3545154733' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:15.614 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:15.649 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:15.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:15.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:15.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:15 smithi130 ceph-mon[105521]: pgmap v2934: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 280 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 871 op/s; 844/13470 objects degraded (6.266%); 1284/13470 objects misplaced (9.532%); 1.4 MiB/s, 140 objects/s recovering 2024-08-17T18:22:15.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:15 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3545154733' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:15 smithi130 ceph-mon[110068]: pgmap v2934: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 280 MiB data, 6.1 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 871 op/s; 844/13470 objects degraded (6.266%); 1284/13470 objects misplaced (9.532%); 1.4 MiB/s, 140 objects/s recovering 2024-08-17T18:22:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:15 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3545154733' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:15.751 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:15.791 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:15.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:15.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:15.893 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:15.917 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:15.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:15.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:16.019 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:16.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:16.042 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:16.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:16.143 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:16.189 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:16.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:16.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:16.291 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:16.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:16.313 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:16.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:16.315 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:16 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 844/13470 objects degraded (6.266%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T18:22:16.315 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:16 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 844/13470 objects degraded (6.266%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T18:22:16.414 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:16.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:16.459 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:16.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:16.560 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:16.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:16 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 844/13470 objects degraded (6.266%), 8 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T18:22:16.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:16.601 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:16.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:16.702 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:16.740 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:16.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:16.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:16.842 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:16.880 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:16.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:16.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:16.982 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:17.012 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:17.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:17.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:17.114 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:17.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:17.158 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:17.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:17.259 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:17.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:17.297 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:17.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:17.398 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:17.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:17 smithi130 ceph-mon[105521]: pgmap v2935: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 261 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 774 op/s; 844/12564 objects degraded (6.718%); 1284/12564 objects misplaced (10.220%); 1.2 MiB/s, 62 objects/s recovering 2024-08-17T18:22:17.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:17 smithi130 ceph-mon[110068]: pgmap v2935: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 261 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 774 op/s; 844/12564 objects degraded (6.718%); 1284/12564 objects misplaced (10.220%); 1.2 MiB/s, 62 objects/s recovering 2024-08-17T18:22:17.421 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:17.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:17.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:17.523 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:17.551 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:17.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:17.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:17.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:17 smithi179 ceph-mon[93519]: pgmap v2935: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 261 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 774 op/s; 844/12564 objects degraded (6.718%); 1284/12564 objects misplaced (10.220%); 1.2 MiB/s, 62 objects/s recovering 2024-08-17T18:22:17.653 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:17.678 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:17.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:17.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:17.780 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:17.817 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:17.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:17.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:17.919 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:17.921 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:22:17.948 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:17.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:17.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:18.050 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:18.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:18.077 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:18.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:18.178 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:18.229 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:18.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:18.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:18.331 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:18.340 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Removed 145063 objects 2024-08-17T18:22:18.340 INFO:tasks.radosbench.radosbench.0.smithi130.stdout:Clean up completed and total clean up time :196.179 2024-08-17T18:22:18.352 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:18.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:18.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:18.453 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:18.477 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:18.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:18.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:18.579 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:18.603 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:18.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:18.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:18.705 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:18.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:18.727 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:18.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:18.828 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:18.850 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:18.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:18.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:18.952 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:18.972 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:18.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:18.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:19.074 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:19.095 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:19.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:19.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:19.197 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:19.219 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:19.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:19.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:19.321 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:19.342 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:19.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:19.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:19.428 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:19 smithi179 ceph-mon[93519]: pgmap v2936: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 261 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 745 op/s; 844/12564 objects degraded (6.718%); 1284/12564 objects misplaced (10.220%); 1.1 MiB/s, 60 objects/s recovering 2024-08-17T18:22:19.444 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:19.465 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:19.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:19.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:19.516 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:19 smithi130 ceph-mon[105521]: pgmap v2936: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 261 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 745 op/s; 844/12564 objects degraded (6.718%); 1284/12564 objects misplaced (10.220%); 1.1 MiB/s, 60 objects/s recovering 2024-08-17T18:22:19.517 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:19 smithi130 ceph-mon[110068]: pgmap v2936: 99 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 7 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 261 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 745 op/s; 844/12564 objects degraded (6.718%); 1284/12564 objects misplaced (10.220%); 1.1 MiB/s, 60 objects/s recovering 2024-08-17T18:22:19.567 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:19.588 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:19.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:19.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:19.604 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:22:19.604 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:22:19.691 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:19.712 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:19.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:19.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:19.814 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:19.835 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:19.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:19.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:19.937 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:19.959 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:19.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:19.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:20.061 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:20.082 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:20.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:20.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:20.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:22:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:22:19] "GET /metrics HTTP/1.1" 200 36802 "" "Prometheus/2.51.0" 2024-08-17T18:22:20.185 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:20.206 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:20.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:20.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:20.308 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:20.329 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:20.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:20.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:20.399 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:22:20.433 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:20.455 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:20.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:20.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:20.557 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:20.578 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:20.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:20.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:20.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2698360289' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:22:20.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:20 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:22:20.592 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:22:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:22:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:22:20.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2698360289' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:22:20.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:20 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:22:20.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2698360289' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:22:20.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:20 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:22:20.680 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:20.702 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:20.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:20.803 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:20.827 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:20.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:20.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:20.928 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:20.945 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T18:22:20.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:20.954 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:20.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:21.055 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:21.079 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:21.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:21.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:21.181 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:21.210 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:21.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:21.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:21.300 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:22:21.311 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:21.333 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:21.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:21.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:21.435 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:21.459 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:21.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:21.559 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:21.561 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:21 smithi179 ceph-mon[93519]: pgmap v2937: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 21 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 77/1041 objects degraded (7.397%); 141/1041 objects misplaced (13.545%); 1.1 MiB/s, 99 objects/s recovering 2024-08-17T18:22:21.561 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:21 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:22:21.561 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:21 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:22:21.561 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:21 smithi179 ceph-mon[93519]: osdmap e1506: 8 total, 8 up, 6 in 2024-08-17T18:22:21.561 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:21 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 77/1041 objects degraded (7.397%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T18:22:21.561 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:21 smithi179 ceph-mon[93519]: osdmap e1507: 8 total, 8 up, 6 in 2024-08-17T18:22:21.583 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:21.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:21.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:21 smithi130 ceph-mon[105521]: pgmap v2937: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 21 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 77/1041 objects degraded (7.397%); 141/1041 objects misplaced (13.545%); 1.1 MiB/s, 99 objects/s recovering 2024-08-17T18:22:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:21 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:22:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:21 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:22:21.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:21 smithi130 ceph-mon[105521]: osdmap e1506: 8 total, 8 up, 6 in 2024-08-17T18:22:21.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:21 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 77/1041 objects degraded (7.397%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T18:22:21.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:21 smithi130 ceph-mon[105521]: osdmap e1507: 8 total, 8 up, 6 in 2024-08-17T18:22:21.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:21 smithi130 ceph-mon[110068]: pgmap v2937: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 21 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 77/1041 objects degraded (7.397%); 141/1041 objects misplaced (13.545%); 1.1 MiB/s, 99 objects/s recovering 2024-08-17T18:22:21.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:21 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:22:21.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:21 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:22:21.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:21 smithi130 ceph-mon[110068]: osdmap e1506: 8 total, 8 up, 6 in 2024-08-17T18:22:21.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:21 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 77/1041 objects degraded (7.397%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T18:22:21.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:21 smithi130 ceph-mon[110068]: osdmap e1507: 8 total, 8 up, 6 in 2024-08-17T18:22:21.685 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:21.708 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:21.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:21.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:21.810 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:21.831 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:21.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:21.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:21.933 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:21.954 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:21.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:21.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:22.056 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:22.077 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:22.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:22.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:22.179 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:22.203 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:22.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:22.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:22.305 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:22.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:22.328 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:22.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:22.429 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:22.450 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:22.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:22.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:22.497 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:22 smithi179 ceph-mon[93519]: 50.18 scrub starts 2024-08-17T18:22:22.497 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:22 smithi179 ceph-mon[93519]: 50.18 scrub ok 2024-08-17T18:22:22.497 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:22 smithi179 ceph-mon[93519]: osdmap e1508: 8 total, 8 up, 6 in 2024-08-17T18:22:22.552 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:22.557 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:22 smithi130 ceph-mon[105521]: 50.18 scrub starts 2024-08-17T18:22:22.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:22 smithi130 ceph-mon[105521]: 50.18 scrub ok 2024-08-17T18:22:22.558 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:22 smithi130 ceph-mon[105521]: osdmap e1508: 8 total, 8 up, 6 in 2024-08-17T18:22:22.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:22 smithi130 ceph-mon[110068]: 50.18 scrub starts 2024-08-17T18:22:22.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:22 smithi130 ceph-mon[110068]: 50.18 scrub ok 2024-08-17T18:22:22.558 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:22 smithi130 ceph-mon[110068]: osdmap e1508: 8 total, 8 up, 6 in 2024-08-17T18:22:22.574 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:22.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:22.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:22.676 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:22.696 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:22.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:22.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:22.734 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:22:22.735 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:22:22.798 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:22.820 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:22.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:22.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:22.921 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:22.943 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:22.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:22.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:23.045 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:23.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:23.068 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:23.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:23.162 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:22:23.162 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:22:23.170 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:23.191 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:23.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:23.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:23.290 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:23.292 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:23.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:23.317 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:23.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:23.342 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:23 smithi179 ceph-mon[93519]: pgmap v2940: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 21 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 555 op/s; 77/1041 objects degraded (7.397%); 141/1041 objects misplaced (13.545%); 0 B/s, 59 objects/s recovering 2024-08-17T18:22:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:23 smithi179 ceph-mon[93519]: 50.13 scrub starts 2024-08-17T18:22:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:23 smithi179 ceph-mon[93519]: 50.13 scrub ok 2024-08-17T18:22:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:23 smithi179 ceph-mon[93519]: 50.15 scrub starts 2024-08-17T18:22:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:23 smithi179 ceph-mon[93519]: 50.15 scrub ok 2024-08-17T18:22:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:23 smithi179 ceph-mon[93519]: 50.6 scrub starts 2024-08-17T18:22:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:23 smithi179 ceph-mon[93519]: 50.6 scrub ok 2024-08-17T18:22:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:23 smithi179 ceph-mon[93519]: 50.14 scrub starts 2024-08-17T18:22:23.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:23 smithi179 ceph-mon[93519]: 50.14 scrub ok 2024-08-17T18:22:23.418 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:23.438 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:23.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:23.540 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:23.562 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:23.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:23.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:23.664 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[105521]: pgmap v2940: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 21 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 555 op/s; 77/1041 objects degraded (7.397%); 141/1041 objects misplaced (13.545%); 0 B/s, 59 objects/s recovering 2024-08-17T18:22:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[105521]: 50.13 scrub starts 2024-08-17T18:22:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[105521]: 50.13 scrub ok 2024-08-17T18:22:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[105521]: 50.15 scrub starts 2024-08-17T18:22:23.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[105521]: 50.15 scrub ok 2024-08-17T18:22:23.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[105521]: 50.6 scrub starts 2024-08-17T18:22:23.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[105521]: 50.6 scrub ok 2024-08-17T18:22:23.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[105521]: 50.14 scrub starts 2024-08-17T18:22:23.668 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[105521]: 50.14 scrub ok 2024-08-17T18:22:23.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[110068]: pgmap v2940: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 4 active+undersized+remapped+backfill_wait, 3 active+undersized+degraded+remapped+backfill_wait, 5 active+remapped+backfill_wait, 85 active+clean; 21 MiB data, 5.4 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 555 op/s; 77/1041 objects degraded (7.397%); 141/1041 objects misplaced (13.545%); 0 B/s, 59 objects/s recovering 2024-08-17T18:22:23.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[110068]: 50.13 scrub starts 2024-08-17T18:22:23.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[110068]: 50.13 scrub ok 2024-08-17T18:22:23.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[110068]: 50.15 scrub starts 2024-08-17T18:22:23.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[110068]: 50.15 scrub ok 2024-08-17T18:22:23.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[110068]: 50.6 scrub starts 2024-08-17T18:22:23.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[110068]: 50.6 scrub ok 2024-08-17T18:22:23.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[110068]: 50.14 scrub starts 2024-08-17T18:22:23.669 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:23 smithi130 ceph-mon[110068]: 50.14 scrub ok 2024-08-17T18:22:23.686 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:23.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:23.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:23.787 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:23.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T18:22:23.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:23.809 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:23.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:23.907 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2941,"stamp":"2024-08-17T18:22:22.152185+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":21600229,"num_objects":347,"num_object_clones":0,"num_object_copies":1041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":77,"num_objects_misplaced":141,"num_objects_unfound":0,"num_objects_dirty":347,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":259922,"num_write_kb":7560208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45987,"num_bytes_recovered":2790867557,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156731,"ondisk_log_size":156731,"up":297,"acting":288,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":304,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":5659508,"kb_used_data":444164,"kb_used_omap":348,"kb_used_meta":5214819,"kb_avail":556811404,"statfs":{"total":575970213888,"available":570174877696,"internally_reserved":0,"allocated":454823936,"data_stored":421982175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":357179,"internal_metadata":5339974853},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-251691008,"num_objects":-3841,"num_object_clones":0,"num_object_copies":-11523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-767,"num_objects_misplaced":-1143,"num_objects_unfound":0,"num_objects_dirty":-3841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4141,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":475,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.435890"},"pg_stats":[{"pgid":"50.19","version":"1505'12384","reported_seq":15752,"reported_epoch":1505,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:22:18.211337+0000","last_change":"2024-08-17T18:21:34.642936+0000","last_active":"2024-08-17T18:22:18.211337+0000","last_peered":"2024-08-17T18:22:18.211337+0000","last_clean":"2024-08-17T18:21:32.363148+0000","last_became_active":"2024-08-17T18:21:34.603808+0000","last_became_peered":"2024-08-17T18:21:34.603808+0000","last_unstale":"2024-08-17T18:22:18.211337+0000","last_undegraded":"2024-08-17T18:22:18.211337+0000","last_fullsized":"2024-08-17T18:22:18.211337+0000","mapping_epoch":1490,"log_start":"1442'6654","ondisk_log_start":"1442'6654","created":1443,"last_epoch_clean":1472,"parent":"0.0","parent_split_bits":5,"last_scrub":"1445'7151","last_scrub_stamp":"2024-08-17T18:18:42.434245+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:18:42.434245+0000","objects_scrubbed":3820,"log_size":5730,"log_dups_size":0,"ondisk_log_size":5730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:09.673917+0000","scrub_duration":1.241929928,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196608,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5255,"num_write_kb":46016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1540,"num_bytes_recovered":100925440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,7,1],"avail_no_missing":["6","1","7"],"object_location_counts":[{"shards":"1,6,7","objects":3}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"50.18","version":"1505'12256","reported_seq":20422,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:18.189114+0000","last_change":"2024-08-17T18:22:15.790105+0000","last_active":"2024-08-17T18:22:18.189114+0000","last_peered":"2024-08-17T18:22:18.189114+0000","last_clean":"2024-08-17T18:22:18.189114+0000","last_became_active":"2024-08-17T18:22:03.156816+0000","last_became_peered":"2024-08-17T18:22:03.156816+0000","last_unstale":"2024-08-17T18:22:18.189114+0000","last_undegraded":"2024-08-17T18:22:18.189114+0000","last_fullsized":"2024-08-17T18:22:18.189114+0000","mapping_epoch":1503,"log_start":"1441'5971","ondisk_log_start":"1441'5971","created":1443,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":6285,"log_dups_size":0,"ondisk_log_size":6285,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T18:22:20.724473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5373952,"num_objects":82,"num_object_clones":0,"num_object_copies":246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8952,"num_write_kb":289088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3999,"num_bytes_recovered":204996608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"50.17","version":"1505'12360","reported_seq":13691,"reported_epoch":1505,"state":"active+undersized+remapped+backfill_wait","last_fresh":"2024-08-17T18:22:18.200004+0000","last_change":"2024-08-17T18:22:14.242855+0000","last_active":"2024-08-17T18:22:18.200004+0000","last_peered":"2024-08-17T18:22:18.200004+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:21:33.419896+0000","last_became_peered":"2024-08-17T18:21:33.419896+0000","last_unstale":"2024-08-17T18:22:18.200004+0000","last_undegraded":"2024-08-17T18:22:18.200004+0000","last_fullsized":"2024-08-17T18:21:33.390320+0000","mapping_epoch":1489,"log_start":"1441'5893","ondisk_log_start":"1441'5893","created":1443,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":6467,"log_dups_size":0,"ondisk_log_size":6467,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9064,"num_write_kb":288672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":602,"num_bytes_recovered":39452672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0],"avail_no_missing":["7","0","2","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"50.16","version":"1505'12204","reported_seq":14704,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:18.303589+0000","last_change":"2024-08-17T18:21:22.888754+0000","last_active":"2024-08-17T18:22:18.303589+0000","last_peered":"2024-08-17T18:22:18.303589+0000","last_clean":"2024-08-17T18:22:18.303589+0000","last_became_active":"2024-08-17T18:21:22.869545+0000","last_became_peered":"2024-08-17T18:21:22.869545+0000","last_unstale":"2024-08-17T18:22:18.303589+0000","last_undegraded":"2024-08-17T18:22:18.303589+0000","last_fullsized":"2024-08-17T18:22:18.303589+0000","mapping_epoch":1487,"log_start":"1440'5525","ondisk_log_start":"1440'5525","created":1443,"last_epoch_clean":1488,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":6679,"log_dups_size":0,"ondisk_log_size":6679,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1114112,"num_objects":17,"num_object_clones":0,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8899,"num_write_kb":285312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1195,"num_bytes_recovered":78315520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"50.15","version":"1505'12325","reported_seq":12509,"reported_epoch":1505,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:22:18.284563+0000","last_change":"2024-08-17T18:21:59.442987+0000","last_active":"2024-08-17T18:22:18.284563+0000","last_peered":"2024-08-17T18:22:18.284563+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:21:59.394625+0000","last_became_peered":"2024-08-17T18:21:59.394625+0000","last_unstale":"2024-08-17T18:22:18.284563+0000","last_undegraded":"2024-08-17T18:21:59.277355+0000","last_fullsized":"2024-08-17T18:21:59.264665+0000","mapping_epoch":1499,"log_start":"1444'6689","ondisk_log_start":"1444'6689","created":1443,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5636,"log_dups_size":0,"ondisk_log_size":5636,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1933312,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30,"num_objects_misplaced":30,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8986,"num_write_kb":288480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":30}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"50.14","version":"1505'12555","reported_seq":17348,"reported_epoch":1505,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:22:18.258954+0000","last_change":"2024-08-17T18:21:13.943087+0000","last_active":"2024-08-17T18:22:18.258954+0000","last_peered":"2024-08-17T18:22:18.258954+0000","last_clean":"2024-08-17T18:21:11.673508+0000","last_became_active":"2024-08-17T18:21:13.901970+0000","last_became_peered":"2024-08-17T18:21:13.901970+0000","last_unstale":"2024-08-17T18:22:18.258954+0000","last_undegraded":"2024-08-17T18:22:18.258954+0000","last_fullsized":"2024-08-17T18:22:18.258954+0000","mapping_epoch":1478,"log_start":"1445'7168","ondisk_log_start":"1445'7168","created":1443,"last_epoch_clean":1474,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5387,"log_dups_size":0,"ondisk_log_size":5387,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2555904,"num_objects":39,"num_object_clones":0,"num_object_copies":117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":39,"num_objects_unfound":0,"num_objects_dirty":39,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9175,"num_write_kb":294848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1538,"num_bytes_recovered":100794368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,5],"avail_no_missing":["3","2","5"],"object_location_counts":[{"shards":"2,3,5","objects":39}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"50.13","version":"1505'12404","reported_seq":13754,"reported_epoch":1505,"state":"active+undersized+remapped+backfill_wait","last_fresh":"2024-08-17T18:22:18.206304+0000","last_change":"2024-08-17T18:22:15.047564+0000","last_active":"2024-08-17T18:22:18.206304+0000","last_peered":"2024-08-17T18:22:18.206304+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:20:53.842047+0000","last_became_peered":"2024-08-17T18:20:53.842047+0000","last_unstale":"2024-08-17T18:22:18.206304+0000","last_undegraded":"2024-08-17T18:22:18.206304+0000","last_fullsized":"2024-08-17T18:20:53.839369+0000","mapping_epoch":1468,"log_start":"1445'6965","ondisk_log_start":"1445'6965","created":1443,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5439,"log_dups_size":0,"ondisk_log_size":5439,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9104,"num_write_kb":290112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":591,"num_bytes_recovered":38731776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[3,6],"avail_no_missing":["3","2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":3,"purged_snaps":[]},{"pgid":"50.12","version":"1505'12500","reported_seq":15503,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:17.972971+0000","last_change":"2024-08-17T18:22:02.181858+0000","last_active":"2024-08-17T18:22:17.972971+0000","last_peered":"2024-08-17T18:22:17.972971+0000","last_clean":"2024-08-17T18:22:17.972971+0000","last_became_active":"2024-08-17T18:22:02.153285+0000","last_became_peered":"2024-08-17T18:22:02.153285+0000","last_unstale":"2024-08-17T18:22:17.972971+0000","last_undegraded":"2024-08-17T18:22:17.972971+0000","last_fullsized":"2024-08-17T18:22:17.972971+0000","mapping_epoch":1502,"log_start":"1441'6530","ondisk_log_start":"1441'6530","created":1443,"last_epoch_clean":1503,"parent":"0.0","parent_split_bits":5,"last_scrub":"1445'7193","last_scrub_stamp":"2024-08-17T18:18:42.157479+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:18:42.157479+0000","objects_scrubbed":3780,"log_size":5970,"log_dups_size":0,"ondisk_log_size":5970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:43:01.638265+0000","scrub_duration":984,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5325,"num_write_kb":49600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1368,"num_bytes_recovered":63700992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"50.11","version":"1505'12338","reported_seq":12540,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:18.287459+0000","last_change":"2024-08-17T18:21:34.424344+0000","last_active":"2024-08-17T18:22:18.287459+0000","last_peered":"2024-08-17T18:22:18.287459+0000","last_clean":"2024-08-17T18:22:18.287459+0000","last_became_active":"2024-08-17T18:21:34.392349+0000","last_became_peered":"2024-08-17T18:21:34.392349+0000","last_unstale":"2024-08-17T18:22:18.287459+0000","last_undegraded":"2024-08-17T18:22:18.287459+0000","last_fullsized":"2024-08-17T18:22:18.287459+0000","mapping_epoch":1490,"log_start":"1444'6770","ondisk_log_start":"1444'6770","created":1443,"last_epoch_clean":1491,"parent":"0.0","parent_split_bits":5,"last_scrub":"1445'7058","last_scrub_stamp":"2024-08-17T18:18:40.173968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:18:40.173968+0000","objects_scrubbed":3697,"log_size":5568,"log_dups_size":0,"ondisk_log_size":5568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:23:17.108834+0000","scrub_duration":1311,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5297,"num_write_kb":51200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"50.10","version":"1505'12475","reported_seq":14984,"reported_epoch":1505,"state":"active+undersized+remapped","last_fresh":"2024-08-17T18:22:18.286908+0000","last_change":"2024-08-17T18:22:15.355816+0000","last_active":"2024-08-17T18:22:18.286908+0000","last_peered":"2024-08-17T18:22:18.286908+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:21:33.442090+0000","last_became_peered":"2024-08-17T18:21:33.442090+0000","last_unstale":"2024-08-17T18:22:18.286908+0000","last_undegraded":"2024-08-17T18:22:18.286908+0000","last_fullsized":"2024-08-17T18:21:33.389066+0000","mapping_epoch":1489,"log_start":"1444'6889","ondisk_log_start":"1444'6889","created":1443,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5586,"log_dups_size":0,"ondisk_log_size":5586,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9130,"num_write_kb":290336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1103,"num_bytes_recovered":72286208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[0,6],"avail_no_missing":["0","2","4","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"50.f","version":"1505'18038","reported_seq":22505,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:18.338590+0000","last_change":"2024-08-17T18:22:02.204956+0000","last_active":"2024-08-17T18:22:18.338590+0000","last_peered":"2024-08-17T18:22:18.338590+0000","last_clean":"2024-08-17T18:22:18.338590+0000","last_became_active":"2024-08-17T18:22:02.152634+0000","last_became_peered":"2024-08-17T18:22:02.152634+0000","last_unstale":"2024-08-17T18:22:18.338590+0000","last_undegraded":"2024-08-17T18:22:18.338590+0000","last_fullsized":"2024-08-17T18:22:18.338590+0000","mapping_epoch":1502,"log_start":"1457'11849","ondisk_log_start":"1457'11849","created":1434,"last_epoch_clean":1503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":6189,"log_dups_size":0,"ondisk_log_size":6189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:41.678629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18038,"num_write_kb":577216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2164,"num_bytes_recovered":141819904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,3],"acting":[7,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"50.e","version":"1505'17799","reported_seq":17911,"reported_epoch":1505,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:22:18.266670+0000","last_change":"2024-08-17T18:21:58.289512+0000","last_active":"2024-08-17T18:22:18.266670+0000","last_peered":"2024-08-17T18:22:18.266670+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:21:33.420857+0000","last_became_peered":"2024-08-17T18:21:33.420857+0000","last_unstale":"2024-08-17T18:22:18.266670+0000","last_undegraded":"2024-08-17T18:21:33.392241+0000","last_fullsized":"2024-08-17T18:21:33.392096+0000","mapping_epoch":1489,"log_start":"1459'12097","ondisk_log_start":"1459'12097","created":1434,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5702,"log_dups_size":0,"ondisk_log_size":5702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:22:49.281199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17799,"num_write_kb":569728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,0],"avail_no_missing":["6","0"],"object_location_counts":[{"shards":"0,6","objects":5}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"50.d","version":"1505'17982","reported_seq":39303,"reported_epoch":1505,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:22:18.282766+0000","last_change":"2024-08-17T18:21:33.557459+0000","last_active":"2024-08-17T18:22:18.282766+0000","last_peered":"2024-08-17T18:22:18.282766+0000","last_clean":"2024-08-17T18:21:11.750495+0000","last_became_active":"2024-08-17T18:21:33.525065+0000","last_became_peered":"2024-08-17T18:21:33.525065+0000","last_unstale":"2024-08-17T18:22:18.282766+0000","last_undegraded":"2024-08-17T18:22:18.282766+0000","last_fullsized":"2024-08-17T18:22:18.282766+0000","mapping_epoch":1489,"log_start":"1460'12622","ondisk_log_start":"1460'12622","created":1434,"last_epoch_clean":1470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327704,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":17982,"num_write_kb":575553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10108,"num_bytes_recovered":589168688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[3,4,5],"avail_no_missing":["3","4","5"],"object_location_counts":[{"shards":"3,4,5","objects":6}],"blocked_by":[],"up_primary":0,"acting_primary":3,"purged_snaps":[]},{"pgid":"50.c","version":"1505'18189","reported_seq":18299,"reported_epoch":1505,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:22:18.202315+0000","last_change":"2024-08-17T18:21:59.444218+0000","last_active":"2024-08-17T18:22:18.202315+0000","last_peered":"2024-08-17T18:22:18.202315+0000","last_clean":"2024-08-17T18:21:57.247951+0000","last_became_active":"2024-08-17T18:21:59.394648+0000","last_became_peered":"2024-08-17T18:21:59.394648+0000","last_unstale":"2024-08-17T18:22:18.202315+0000","last_undegraded":"2024-08-17T18:22:18.202315+0000","last_fullsized":"2024-08-17T18:22:18.202315+0000","mapping_epoch":1499,"log_start":"1459'12451","ondisk_log_start":"1459'12451","created":1434,"last_epoch_clean":1491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5738,"log_dups_size":0,"ondisk_log_size":5738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:29:59.577894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196608,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18189,"num_write_kb":582144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,1],"avail_no_missing":["6","0","1"],"object_location_counts":[{"shards":"0,1,6","objects":3}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"50.b","version":"1505'18242","reported_seq":23456,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:18.111882+0000","last_change":"2024-08-17T18:22:04.200928+0000","last_active":"2024-08-17T18:22:18.111882+0000","last_peered":"2024-08-17T18:22:18.111882+0000","last_clean":"2024-08-17T18:22:18.111882+0000","last_became_active":"2024-08-17T18:22:04.164564+0000","last_became_peered":"2024-08-17T18:22:04.164564+0000","last_unstale":"2024-08-17T18:22:18.111882+0000","last_undegraded":"2024-08-17T18:22:18.111882+0000","last_fullsized":"2024-08-17T18:22:18.111882+0000","mapping_epoch":1504,"log_start":"1457'11851","ondisk_log_start":"1457'11851","created":1434,"last_epoch_clean":1505,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":6391,"log_dups_size":0,"ondisk_log_size":6391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:31:42.682668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":786432,"num_objects":12,"num_object_clones":0,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18242,"num_write_kb":584128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2506,"num_bytes_recovered":164102144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"50.a","version":"1505'18219","reported_seq":28110,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:18.077617+0000","last_change":"2024-08-17T18:22:00.296875+0000","last_active":"2024-08-17T18:22:18.077617+0000","last_peered":"2024-08-17T18:22:18.077617+0000","last_clean":"2024-08-17T18:22:18.077617+0000","last_became_active":"2024-08-17T18:22:00.256903+0000","last_became_peered":"2024-08-17T18:22:00.256903+0000","last_unstale":"2024-08-17T18:22:18.077617+0000","last_undegraded":"2024-08-17T18:22:18.077617+0000","last_fullsized":"2024-08-17T18:22:18.077617+0000","mapping_epoch":1500,"log_start":"1457'11746","ondisk_log_start":"1457'11746","created":1434,"last_epoch_clean":1501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":6473,"log_dups_size":0,"ondisk_log_size":6473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:27:51.329604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":589824,"num_objects":9,"num_object_clones":0,"num_object_copies":27,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18219,"num_write_kb":583296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4851,"num_bytes_recovered":317915136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"50.9","version":"1505'12343","reported_seq":16164,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:18.217662+0000","last_change":"2024-08-17T18:22:01.173669+0000","last_active":"2024-08-17T18:22:18.217662+0000","last_peered":"2024-08-17T18:22:18.217662+0000","last_clean":"2024-08-17T18:22:18.217662+0000","last_became_active":"2024-08-17T18:22:01.150152+0000","last_became_peered":"2024-08-17T18:22:01.150152+0000","last_unstale":"2024-08-17T18:22:18.217662+0000","last_undegraded":"2024-08-17T18:22:18.217662+0000","last_fullsized":"2024-08-17T18:22:18.217662+0000","mapping_epoch":1501,"log_start":"1442'6657","ondisk_log_start":"1442'6657","created":1434,"last_epoch_clean":1502,"parent":"0.0","parent_split_bits":0,"last_scrub":"1445'7042","last_scrub_stamp":"2024-08-17T18:18:39.939010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:18:39.939010+0000","objects_scrubbed":3657,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:14:26.955813+0000","scrub_duration":1.7012776270000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196608,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5324,"num_write_kb":53440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1761,"num_bytes_recovered":115408896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"50.8","version":"1505'12488","reported_seq":20777,"reported_epoch":1505,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T18:22:18.324556+0000","last_change":"2024-08-17T18:22:04.696799+0000","last_active":"2024-08-17T18:22:18.324556+0000","last_peered":"2024-08-17T18:22:18.324556+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:21:34.483826+0000","last_became_peered":"2024-08-17T18:21:34.483826+0000","last_unstale":"2024-08-17T18:22:18.324556+0000","last_undegraded":"2024-08-17T18:22:18.324556+0000","last_fullsized":"2024-08-17T18:21:34.387738+0000","mapping_epoch":1490,"log_start":"1444'6840","ondisk_log_start":"1444'6840","created":1434,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5648,"log_dups_size":0,"ondisk_log_size":5648,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9184,"num_write_kb":291264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4065,"num_bytes_recovered":201916416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[0,2],"avail_no_missing":["0","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"50.7","version":"1505'12325","reported_seq":12455,"reported_epoch":1505,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T18:22:18.322484+0000","last_change":"2024-08-17T18:22:01.840251+0000","last_active":"2024-08-17T18:22:18.322484+0000","last_peered":"2024-08-17T18:22:18.322484+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:21:33.420493+0000","last_became_peered":"2024-08-17T18:21:33.420493+0000","last_unstale":"2024-08-17T18:22:18.322484+0000","last_undegraded":"2024-08-17T18:21:33.391749+0000","last_fullsized":"2024-08-17T18:21:33.391559+0000","mapping_epoch":1489,"log_start":"1441'5802","ondisk_log_start":"1441'5802","created":1434,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":6523,"log_dups_size":0,"ondisk_log_size":6523,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2785280,"num_objects":42,"num_object_clones":0,"num_object_copies":126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":42,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9028,"num_write_kb":290272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":42}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"50.6","version":"1505'12444","reported_seq":16109,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:18.277950+0000","last_change":"2024-08-17T18:21:16.842976+0000","last_active":"2024-08-17T18:22:18.277950+0000","last_peered":"2024-08-17T18:22:18.277950+0000","last_clean":"2024-08-17T18:22:18.277950+0000","last_became_active":"2024-08-17T18:21:16.814551+0000","last_became_peered":"2024-08-17T18:21:16.814551+0000","last_unstale":"2024-08-17T18:22:18.277950+0000","last_undegraded":"2024-08-17T18:22:18.277950+0000","last_fullsized":"2024-08-17T18:22:18.277950+0000","mapping_epoch":1481,"log_start":"1441'6074","ondisk_log_start":"1441'6074","created":1434,"last_epoch_clean":1482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":6370,"log_dups_size":0,"ondisk_log_size":6370,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9139,"num_write_kb":291904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1777,"num_bytes_recovered":116457472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"50.5","version":"1505'12473","reported_seq":12658,"reported_epoch":1505,"state":"active+undersized+remapped+backfill_wait","last_fresh":"2024-08-17T18:22:18.318919+0000","last_change":"2024-08-17T18:22:17.115802+0000","last_active":"2024-08-17T18:22:18.318919+0000","last_peered":"2024-08-17T18:22:18.318919+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:21:59.406143+0000","last_became_peered":"2024-08-17T18:21:59.406143+0000","last_unstale":"2024-08-17T18:22:18.318919+0000","last_undegraded":"2024-08-17T18:22:18.318919+0000","last_fullsized":"2024-08-17T18:21:59.253290+0000","mapping_epoch":1499,"log_start":"1444'6829","ondisk_log_start":"1444'6829","created":1434,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5644,"log_dups_size":0,"ondisk_log_size":5644,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9133,"num_write_kb":291360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,1],"avail_no_missing":["0","1","2","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"50.4","version":"1505'12464","reported_seq":12639,"reported_epoch":1505,"state":"active+undersized+remapped+backfill_wait","last_fresh":"2024-08-17T18:22:18.236097+0000","last_change":"2024-08-17T18:22:16.884233+0000","last_active":"2024-08-17T18:22:18.236097+0000","last_peered":"2024-08-17T18:22:18.236097+0000","last_clean":"2024-08-17T18:17:18.356409+0000","last_became_active":"2024-08-17T18:21:12.775372+0000","last_became_peered":"2024-08-17T18:21:12.775372+0000","last_unstale":"2024-08-17T18:22:18.236097+0000","last_undegraded":"2024-08-17T18:22:18.236097+0000","last_fullsized":"2024-08-17T18:21:12.759258+0000","mapping_epoch":1477,"log_start":"1445'7063","ondisk_log_start":"1445'7063","created":1434,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":5401,"log_dups_size":0,"ondisk_log_size":5401,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9084,"num_write_kb":289536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,6],"avail_no_missing":["3","2","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"50.3","version":"1505'12321","reported_seq":15099,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:18.286256+0000","last_change":"2024-08-17T18:21:32.540960+0000","last_active":"2024-08-17T18:22:18.286256+0000","last_peered":"2024-08-17T18:22:18.286256+0000","last_clean":"2024-08-17T18:22:18.286256+0000","last_became_active":"2024-08-17T18:21:20.863224+0000","last_became_peered":"2024-08-17T18:21:20.863224+0000","last_unstale":"2024-08-17T18:22:18.286256+0000","last_undegraded":"2024-08-17T18:22:18.286256+0000","last_fullsized":"2024-08-17T18:22:18.286256+0000","mapping_epoch":1485,"log_start":"1440'4702","ondisk_log_start":"1440'4702","created":1434,"last_epoch_clean":1486,"parent":"0.0","parent_split_bits":0,"last_scrub":"1488'11281","last_scrub_stamp":"2024-08-17T18:21:32.540854+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:21:32.540854+0000","objects_scrubbed":1040,"log_size":7619,"log_dups_size":0,"ondisk_log_size":7619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:00:41.788943+0000","scrub_duration":113,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1040,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"50.2","version":"1505'12568","reported_seq":17263,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:18.050870+0000","last_change":"2024-08-17T18:22:03.189070+0000","last_active":"2024-08-17T18:22:18.050870+0000","last_peered":"2024-08-17T18:22:18.050870+0000","last_clean":"2024-08-17T18:22:18.050870+0000","last_became_active":"2024-08-17T18:22:03.153962+0000","last_became_peered":"2024-08-17T18:22:03.153962+0000","last_unstale":"2024-08-17T18:22:18.050870+0000","last_undegraded":"2024-08-17T18:22:18.050870+0000","last_fullsized":"2024-08-17T18:22:18.050870+0000","mapping_epoch":1503,"log_start":"1441'6214","ondisk_log_start":"1441'6214","created":1434,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"1445'7124","last_scrub_stamp":"2024-08-17T18:18:40.375960+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:18:40.375960+0000","objects_scrubbed":3811,"log_size":6354,"log_dups_size":0,"ondisk_log_size":6354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:49:16.080252+0000","scrub_duration":1246,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5468,"num_write_kb":53184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1926,"num_bytes_recovered":126222336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"50.1","version":"1505'12557","reported_seq":12759,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:18.272977+0000","last_change":"2024-08-17T18:21:34.424346+0000","last_active":"2024-08-17T18:22:18.272977+0000","last_peered":"2024-08-17T18:22:18.272977+0000","last_clean":"2024-08-17T18:22:18.272977+0000","last_became_active":"2024-08-17T18:21:34.394170+0000","last_became_peered":"2024-08-17T18:21:34.394170+0000","last_unstale":"2024-08-17T18:22:18.272977+0000","last_undegraded":"2024-08-17T18:22:18.272977+0000","last_fullsized":"2024-08-17T18:22:18.272977+0000","mapping_epoch":1490,"log_start":"1445'6982","ondisk_log_start":"1445'6982","created":1434,"last_epoch_clean":1491,"parent":"0.0","parent_split_bits":0,"last_scrub":"1445'7036","last_scrub_stamp":"2024-08-17T18:18:39.012646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:18:39.012646+0000","objects_scrubbed":3678,"log_size":5575,"log_dups_size":0,"ondisk_log_size":5575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:23.332319+0000","scrub_duration":1112,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5544,"num_write_kb":59712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"50.0","version":"1505'12390","reported_seq":22114,"reported_epoch":1505,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T18:22:18.051364+0000","last_change":"2024-08-17T18:21:34.506321+0000","last_active":"2024-08-17T18:22:18.051364+0000","last_peered":"2024-08-17T18:22:18.051364+0000","last_clean":"2024-08-17T18:19:25.008527+0000","last_became_active":"2024-08-17T18:21:34.487601+0000","last_became_peered":"2024-08-17T18:21:34.487601+0000","last_unstale":"2024-08-17T18:22:18.051364+0000","last_undegraded":"2024-08-17T18:22:18.051364+0000","last_fullsized":"2024-08-17T18:22:18.051364+0000","mapping_epoch":1490,"log_start":"1440'5286","ondisk_log_start":"1440'5286","created":1434,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:17:18.356409+0000","last_clean_scrub_stamp":"2024-08-17T18:17:18.356409+0000","objects_scrubbed":0,"log_size":7104,"log_dups_size":0,"ondisk_log_size":7104,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3964928,"num_objects":60,"num_object_clones":0,"num_object_copies":180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":60,"num_objects_unfound":0,"num_objects_dirty":60,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9044,"num_write_kb":291360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4780,"num_bytes_recovered":313262080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[4,0,6],"avail_no_missing":["4","0","6"],"object_location_counts":[{"shards":"0,4,6","objects":60}],"blocked_by":[],"up_primary":2,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":48,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.165124+0000","last_change":"2024-08-17T18:21:33.450755+0000","last_active":"2024-08-17T18:22:04.165124+0000","last_peered":"2024-08-17T18:22:04.165124+0000","last_clean":"2024-08-17T18:22:04.165124+0000","last_became_active":"2024-08-17T18:21:33.427306+0000","last_became_peered":"2024-08-17T18:21:33.427306+0000","last_unstale":"2024-08-17T18:22:04.165124+0000","last_undegraded":"2024-08-17T18:22:04.165124+0000","last_fullsized":"2024-08-17T18:22:04.165124+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:55:45.822004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":78,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.150315+0000","last_change":"2024-08-17T18:21:58.262211+0000","last_active":"2024-08-17T18:22:03.150315+0000","last_peered":"2024-08-17T18:22:03.150315+0000","last_clean":"2024-08-17T18:22:03.150315+0000","last_became_active":"2024-08-17T18:21:58.261636+0000","last_became_peered":"2024-08-17T18:21:58.261636+0000","last_unstale":"2024-08-17T18:22:03.150315+0000","last_undegraded":"2024-08-17T18:22:03.150315+0000","last_fullsized":"2024-08-17T18:22:03.150315+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:22:53.662404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":2317,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.161425+0000","last_change":"2024-08-17T18:19:27.023211+0000","last_active":"2024-08-17T18:22:04.161425+0000","last_peered":"2024-08-17T18:22:04.161425+0000","last_clean":"2024-08-17T18:22:04.161425+0000","last_became_active":"2024-08-17T18:19:27.022827+0000","last_became_peered":"2024-08-17T18:19:27.022827+0000","last_unstale":"2024-08-17T18:22:04.161425+0000","last_undegraded":"2024-08-17T18:22:04.161425+0000","last_fullsized":"2024-08-17T18:22:04.161425+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:25:50.310320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2382,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.160921+0000","last_change":"2024-08-17T18:21:33.421541+0000","last_active":"2024-08-17T18:22:04.160921+0000","last_peered":"2024-08-17T18:22:04.160921+0000","last_clean":"2024-08-17T18:22:04.160921+0000","last_became_active":"2024-08-17T18:21:33.420932+0000","last_became_peered":"2024-08-17T18:21:33.420932+0000","last_unstale":"2024-08-17T18:22:04.160921+0000","last_undegraded":"2024-08-17T18:22:04.160921+0000","last_fullsized":"2024-08-17T18:22:04.160921+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:23.366908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":81,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.150819+0000","last_change":"2024-08-17T18:21:58.283162+0000","last_active":"2024-08-17T18:22:03.150819+0000","last_peered":"2024-08-17T18:22:03.150819+0000","last_clean":"2024-08-17T18:22:03.150819+0000","last_became_active":"2024-08-17T18:21:58.265637+0000","last_became_peered":"2024-08-17T18:21:58.265637+0000","last_unstale":"2024-08-17T18:22:03.150819+0000","last_undegraded":"2024-08-17T18:22:03.150819+0000","last_fullsized":"2024-08-17T18:22:03.150819+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:10.574052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":88,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.150685+0000","last_change":"2024-08-17T18:21:33.404642+0000","last_active":"2024-08-17T18:22:03.150685+0000","last_peered":"2024-08-17T18:22:03.150685+0000","last_clean":"2024-08-17T18:22:03.150685+0000","last_became_active":"2024-08-17T18:21:33.404374+0000","last_became_peered":"2024-08-17T18:21:33.404374+0000","last_unstale":"2024-08-17T18:22:03.150685+0000","last_undegraded":"2024-08-17T18:22:03.150685+0000","last_fullsized":"2024-08-17T18:22:03.150685+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:49:34.531843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":81,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.150349+0000","last_change":"2024-08-17T18:21:58.264821+0000","last_active":"2024-08-17T18:22:03.150349+0000","last_peered":"2024-08-17T18:22:03.150349+0000","last_clean":"2024-08-17T18:22:03.150349+0000","last_became_active":"2024-08-17T18:21:58.260477+0000","last_became_peered":"2024-08-17T18:21:58.260477+0000","last_unstale":"2024-08-17T18:22:03.150349+0000","last_undegraded":"2024-08-17T18:22:03.150349+0000","last_fullsized":"2024-08-17T18:22:03.150349+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:20.960079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":78,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.150927+0000","last_change":"2024-08-17T18:21:58.272053+0000","last_active":"2024-08-17T18:22:03.150927+0000","last_peered":"2024-08-17T18:22:03.150927+0000","last_clean":"2024-08-17T18:22:03.150927+0000","last_became_active":"2024-08-17T18:21:58.271727+0000","last_became_peered":"2024-08-17T18:21:58.271727+0000","last_unstale":"2024-08-17T18:22:03.150927+0000","last_undegraded":"2024-08-17T18:22:03.150927+0000","last_fullsized":"2024-08-17T18:22:03.150927+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:51.678694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":42,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.163908+0000","last_change":"2024-08-17T18:21:33.424299+0000","last_active":"2024-08-17T18:22:04.163908+0000","last_peered":"2024-08-17T18:22:04.163908+0000","last_clean":"2024-08-17T18:22:04.163908+0000","last_became_active":"2024-08-17T18:21:33.424016+0000","last_became_peered":"2024-08-17T18:21:33.424016+0000","last_unstale":"2024-08-17T18:22:04.163908+0000","last_undegraded":"2024-08-17T18:22:04.163908+0000","last_fullsized":"2024-08-17T18:22:04.163908+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:15.128415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2034,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.170349+0000","last_change":"2024-08-17T18:21:58.281079+0000","last_active":"2024-08-17T18:22:04.170349+0000","last_peered":"2024-08-17T18:22:04.170349+0000","last_clean":"2024-08-17T18:22:04.170349+0000","last_became_active":"2024-08-17T18:21:58.280727+0000","last_became_peered":"2024-08-17T18:21:58.280727+0000","last_unstale":"2024-08-17T18:22:04.170349+0000","last_undegraded":"2024-08-17T18:22:04.170349+0000","last_fullsized":"2024-08-17T18:22:04.170349+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:22:33.690910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":92,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.150767+0000","last_change":"2024-08-17T18:21:58.278189+0000","last_active":"2024-08-17T18:22:03.150767+0000","last_peered":"2024-08-17T18:22:03.150767+0000","last_clean":"2024-08-17T18:22:03.150767+0000","last_became_active":"2024-08-17T18:21:58.277795+0000","last_became_peered":"2024-08-17T18:21:58.277795+0000","last_unstale":"2024-08-17T18:22:03.150767+0000","last_undegraded":"2024-08-17T18:22:03.150767+0000","last_fullsized":"2024-08-17T18:22:03.150767+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:28.692252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2337,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.161374+0000","last_change":"2024-08-17T18:21:58.273212+0000","last_active":"2024-08-17T18:22:04.161374+0000","last_peered":"2024-08-17T18:22:04.161374+0000","last_clean":"2024-08-17T18:22:04.161374+0000","last_became_active":"2024-08-17T18:21:58.270393+0000","last_became_peered":"2024-08-17T18:21:58.270393+0000","last_unstale":"2024-08-17T18:22:04.161374+0000","last_undegraded":"2024-08-17T18:22:04.161374+0000","last_fullsized":"2024-08-17T18:22:04.161374+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:21.423509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2010,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.160724+0000","last_change":"2024-08-17T18:21:58.275572+0000","last_active":"2024-08-17T18:22:04.160724+0000","last_peered":"2024-08-17T18:22:04.160724+0000","last_clean":"2024-08-17T18:22:04.160724+0000","last_became_active":"2024-08-17T18:21:58.275094+0000","last_became_peered":"2024-08-17T18:21:58.275094+0000","last_unstale":"2024-08-17T18:22:04.160724+0000","last_undegraded":"2024-08-17T18:22:04.160724+0000","last_fullsized":"2024-08-17T18:22:04.160724+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:02.844716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":42,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.164271+0000","last_change":"2024-08-17T18:21:33.408805+0000","last_active":"2024-08-17T18:22:04.164271+0000","last_peered":"2024-08-17T18:22:04.164271+0000","last_clean":"2024-08-17T18:22:04.164271+0000","last_became_active":"2024-08-17T18:21:33.408216+0000","last_became_peered":"2024-08-17T18:21:33.408216+0000","last_unstale":"2024-08-17T18:22:04.164271+0000","last_undegraded":"2024-08-17T18:22:04.164271+0000","last_fullsized":"2024-08-17T18:22:04.164271+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:45.784110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":24,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.159943+0000","last_change":"2024-08-17T18:21:58.271246+0000","last_active":"2024-08-17T18:22:04.159943+0000","last_peered":"2024-08-17T18:22:04.159943+0000","last_clean":"2024-08-17T18:22:04.159943+0000","last_became_active":"2024-08-17T18:21:58.269527+0000","last_became_peered":"2024-08-17T18:21:58.269527+0000","last_unstale":"2024-08-17T18:22:04.159943+0000","last_undegraded":"2024-08-17T18:22:04.159943+0000","last_fullsized":"2024-08-17T18:22:04.159943+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:06.117680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":133,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.148139+0000","last_change":"2024-08-17T18:21:58.268560+0000","last_active":"2024-08-17T18:22:03.148139+0000","last_peered":"2024-08-17T18:22:03.148139+0000","last_clean":"2024-08-17T18:22:03.148139+0000","last_became_active":"2024-08-17T18:21:58.268263+0000","last_became_peered":"2024-08-17T18:21:58.268263+0000","last_unstale":"2024-08-17T18:22:03.148139+0000","last_undegraded":"2024-08-17T18:22:03.148139+0000","last_fullsized":"2024-08-17T18:22:03.148139+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:41.676163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2324,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.160177+0000","last_change":"2024-08-17T18:21:58.271897+0000","last_active":"2024-08-17T18:22:04.160177+0000","last_peered":"2024-08-17T18:22:04.160177+0000","last_clean":"2024-08-17T18:22:04.160177+0000","last_became_active":"2024-08-17T18:21:58.270598+0000","last_became_peered":"2024-08-17T18:21:58.270598+0000","last_unstale":"2024-08-17T18:22:04.160177+0000","last_undegraded":"2024-08-17T18:22:04.160177+0000","last_fullsized":"2024-08-17T18:22:04.160177+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:39:09.775610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":42,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.164169+0000","last_change":"2024-08-17T18:21:33.428841+0000","last_active":"2024-08-17T18:22:04.164169+0000","last_peered":"2024-08-17T18:22:04.164169+0000","last_clean":"2024-08-17T18:22:04.164169+0000","last_became_active":"2024-08-17T18:21:33.427865+0000","last_became_peered":"2024-08-17T18:21:33.427865+0000","last_unstale":"2024-08-17T18:22:04.164169+0000","last_undegraded":"2024-08-17T18:22:04.164169+0000","last_fullsized":"2024-08-17T18:22:04.164169+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:33.010010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2265,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.160584+0000","last_change":"2024-08-17T18:16:34.644366+0000","last_active":"2024-08-17T18:22:04.160584+0000","last_peered":"2024-08-17T18:22:04.160584+0000","last_clean":"2024-08-17T18:22:04.160584+0000","last_became_active":"2024-08-17T18:16:34.643994+0000","last_became_peered":"2024-08-17T18:16:34.643994+0000","last_unstale":"2024-08-17T18:22:04.160584+0000","last_undegraded":"2024-08-17T18:22:04.160584+0000","last_fullsized":"2024-08-17T18:22:04.160584+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:51.953343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":840,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.152810+0000","last_change":"2024-08-17T18:21:58.265707+0000","last_active":"2024-08-17T18:22:03.152810+0000","last_peered":"2024-08-17T18:22:03.152810+0000","last_clean":"2024-08-17T18:22:03.152810+0000","last_became_active":"2024-08-17T18:21:58.265367+0000","last_became_peered":"2024-08-17T18:21:58.265367+0000","last_unstale":"2024-08-17T18:22:03.152810+0000","last_undegraded":"2024-08-17T18:22:03.152810+0000","last_fullsized":"2024-08-17T18:22:03.152810+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:23:03.119320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1508,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.152086+0000","last_change":"2024-08-17T18:18:50.525363+0000","last_active":"2024-08-17T18:22:03.152086+0000","last_peered":"2024-08-17T18:22:03.152086+0000","last_clean":"2024-08-17T18:22:03.152086+0000","last_became_active":"2024-08-17T18:18:50.511676+0000","last_became_peered":"2024-08-17T18:18:50.511676+0000","last_unstale":"2024-08-17T18:22:03.152086+0000","last_undegraded":"2024-08-17T18:22:03.152086+0000","last_fullsized":"2024-08-17T18:22:03.152086+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:22.530382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":158,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.147792+0000","last_change":"2024-08-17T18:21:58.266640+0000","last_active":"2024-08-17T18:22:03.147792+0000","last_peered":"2024-08-17T18:22:03.147792+0000","last_clean":"2024-08-17T18:22:03.147792+0000","last_became_active":"2024-08-17T18:21:58.266135+0000","last_became_peered":"2024-08-17T18:21:58.266135+0000","last_unstale":"2024-08-17T18:22:03.147792+0000","last_undegraded":"2024-08-17T18:22:03.147792+0000","last_fullsized":"2024-08-17T18:22:03.147792+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:35.330096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":77,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.150244+0000","last_change":"2024-08-17T18:18:51.515962+0000","last_active":"2024-08-17T18:22:03.150244+0000","last_peered":"2024-08-17T18:22:03.150244+0000","last_clean":"2024-08-17T18:22:03.150244+0000","last_became_active":"2024-08-17T18:18:51.505508+0000","last_became_peered":"2024-08-17T18:18:51.505508+0000","last_unstale":"2024-08-17T18:22:03.150244+0000","last_undegraded":"2024-08-17T18:22:03.150244+0000","last_fullsized":"2024-08-17T18:22:03.150244+0000","mapping_epoch":1447,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:16.518470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":77,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.150778+0000","last_change":"2024-08-17T18:21:58.277259+0000","last_active":"2024-08-17T18:22:03.150778+0000","last_peered":"2024-08-17T18:22:03.150778+0000","last_clean":"2024-08-17T18:22:03.150778+0000","last_became_active":"2024-08-17T18:21:58.276909+0000","last_became_peered":"2024-08-17T18:21:58.276909+0000","last_unstale":"2024-08-17T18:22:03.150778+0000","last_undegraded":"2024-08-17T18:22:03.150778+0000","last_fullsized":"2024-08-17T18:22:03.150778+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:04:17.083826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":98,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.150063+0000","last_change":"2024-08-17T18:21:33.403392+0000","last_active":"2024-08-17T18:22:03.150063+0000","last_peered":"2024-08-17T18:22:03.150063+0000","last_clean":"2024-08-17T18:22:03.150063+0000","last_became_active":"2024-08-17T18:21:33.403017+0000","last_became_peered":"2024-08-17T18:21:33.403017+0000","last_unstale":"2024-08-17T18:22:03.150063+0000","last_undegraded":"2024-08-17T18:22:03.150063+0000","last_fullsized":"2024-08-17T18:22:03.150063+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:50:06.621109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":107,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.148212+0000","last_change":"2024-08-17T18:19:27.055728+0000","last_active":"2024-08-17T18:22:03.148212+0000","last_peered":"2024-08-17T18:22:03.148212+0000","last_clean":"2024-08-17T18:22:03.148212+0000","last_became_active":"2024-08-17T18:19:27.055425+0000","last_became_peered":"2024-08-17T18:19:27.055425+0000","last_unstale":"2024-08-17T18:22:03.148212+0000","last_undegraded":"2024-08-17T18:22:03.148212+0000","last_fullsized":"2024-08-17T18:22:03.148212+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:47.920145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":57,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.165191+0000","last_change":"2024-08-17T18:21:58.268968+0000","last_active":"2024-08-17T18:22:04.165191+0000","last_peered":"2024-08-17T18:22:04.165191+0000","last_clean":"2024-08-17T18:22:04.165191+0000","last_became_active":"2024-08-17T18:21:58.261371+0000","last_became_peered":"2024-08-17T18:21:58.261371+0000","last_unstale":"2024-08-17T18:22:04.165191+0000","last_undegraded":"2024-08-17T18:22:04.165191+0000","last_fullsized":"2024-08-17T18:22:04.165191+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:35:20.986571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2333,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.160223+0000","last_change":"2024-08-17T18:21:33.421768+0000","last_active":"2024-08-17T18:22:04.160223+0000","last_peered":"2024-08-17T18:22:04.160223+0000","last_clean":"2024-08-17T18:22:04.160223+0000","last_became_active":"2024-08-17T18:21:33.421467+0000","last_became_peered":"2024-08-17T18:21:33.421467+0000","last_unstale":"2024-08-17T18:22:04.160223+0000","last_undegraded":"2024-08-17T18:22:04.160223+0000","last_fullsized":"2024-08-17T18:22:04.160223+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:38.751105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":138,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.148286+0000","last_change":"2024-08-17T18:21:58.268916+0000","last_active":"2024-08-17T18:22:03.148286+0000","last_peered":"2024-08-17T18:22:03.148286+0000","last_clean":"2024-08-17T18:22:03.148286+0000","last_became_active":"2024-08-17T18:21:58.268521+0000","last_became_peered":"2024-08-17T18:21:58.268521+0000","last_unstale":"2024-08-17T18:22:03.148286+0000","last_undegraded":"2024-08-17T18:22:03.148286+0000","last_fullsized":"2024-08-17T18:22:03.148286+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:13.343838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":2353,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.161739+0000","last_change":"2024-08-17T18:21:33.497158+0000","last_active":"2024-08-17T18:22:04.161739+0000","last_peered":"2024-08-17T18:22:04.161739+0000","last_clean":"2024-08-17T18:22:04.161739+0000","last_became_active":"2024-08-17T18:21:33.421674+0000","last_became_peered":"2024-08-17T18:21:33.421674+0000","last_unstale":"2024-08-17T18:22:04.161739+0000","last_undegraded":"2024-08-17T18:22:04.161739+0000","last_fullsized":"2024-08-17T18:22:04.161739+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:58:57.198125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":49,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.164817+0000","last_change":"2024-08-17T18:21:58.273991+0000","last_active":"2024-08-17T18:22:04.164817+0000","last_peered":"2024-08-17T18:22:04.164817+0000","last_clean":"2024-08-17T18:22:04.164817+0000","last_became_active":"2024-08-17T18:21:58.273667+0000","last_became_peered":"2024-08-17T18:21:58.273667+0000","last_unstale":"2024-08-17T18:22:04.164817+0000","last_undegraded":"2024-08-17T18:22:04.164817+0000","last_fullsized":"2024-08-17T18:22:04.164817+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:28.587836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1545,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.152243+0000","last_change":"2024-08-17T18:21:33.420079+0000","last_active":"2024-08-17T18:22:03.152243+0000","last_peered":"2024-08-17T18:22:03.152243+0000","last_clean":"2024-08-17T18:22:03.152243+0000","last_became_active":"2024-08-17T18:21:33.419265+0000","last_became_peered":"2024-08-17T18:21:33.419265+0000","last_unstale":"2024-08-17T18:22:03.152243+0000","last_undegraded":"2024-08-17T18:22:03.152243+0000","last_fullsized":"2024-08-17T18:22:03.152243+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:33:44.811302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1823,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.170450+0000","last_change":"2024-08-17T18:21:58.277263+0000","last_active":"2024-08-17T18:22:04.170450+0000","last_peered":"2024-08-17T18:22:04.170450+0000","last_clean":"2024-08-17T18:22:04.170450+0000","last_became_active":"2024-08-17T18:21:58.276900+0000","last_became_peered":"2024-08-17T18:21:58.276900+0000","last_unstale":"2024-08-17T18:22:04.170450+0000","last_undegraded":"2024-08-17T18:22:04.170450+0000","last_fullsized":"2024-08-17T18:22:04.170450+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:05.316231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":244,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.150503+0000","last_change":"2024-08-17T18:21:58.278092+0000","last_active":"2024-08-17T18:22:03.150503+0000","last_peered":"2024-08-17T18:22:03.150503+0000","last_clean":"2024-08-17T18:22:03.150503+0000","last_became_active":"2024-08-17T18:21:58.277734+0000","last_became_peered":"2024-08-17T18:21:58.277734+0000","last_unstale":"2024-08-17T18:22:03.150503+0000","last_undegraded":"2024-08-17T18:22:03.150503+0000","last_fullsized":"2024-08-17T18:22:03.150503+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:50.326896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":232,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.163957+0000","last_change":"2024-08-17T18:21:58.275684+0000","last_active":"2024-08-17T18:22:04.163957+0000","last_peered":"2024-08-17T18:22:04.163957+0000","last_clean":"2024-08-17T18:22:04.163957+0000","last_became_active":"2024-08-17T18:21:58.275147+0000","last_became_peered":"2024-08-17T18:21:58.275147+0000","last_unstale":"2024-08-17T18:22:04.163957+0000","last_undegraded":"2024-08-17T18:22:04.163957+0000","last_fullsized":"2024-08-17T18:22:04.163957+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:03:12.960867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1110,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.161280+0000","last_change":"2024-08-17T18:21:12.774945+0000","last_active":"2024-08-17T18:22:04.161280+0000","last_peered":"2024-08-17T18:22:04.161280+0000","last_clean":"2024-08-17T18:22:04.161280+0000","last_became_active":"2024-08-17T18:21:12.774497+0000","last_became_peered":"2024-08-17T18:21:12.774497+0000","last_unstale":"2024-08-17T18:22:04.161280+0000","last_undegraded":"2024-08-17T18:22:04.161280+0000","last_fullsized":"2024-08-17T18:22:04.161280+0000","mapping_epoch":1477,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:36:52.456482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":605,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.170537+0000","last_change":"2024-08-17T18:21:58.275787+0000","last_active":"2024-08-17T18:22:04.170537+0000","last_peered":"2024-08-17T18:22:04.170537+0000","last_clean":"2024-08-17T18:22:04.170537+0000","last_became_active":"2024-08-17T18:21:58.275441+0000","last_became_peered":"2024-08-17T18:21:58.275441+0000","last_unstale":"2024-08-17T18:22:04.170537+0000","last_undegraded":"2024-08-17T18:22:04.170537+0000","last_fullsized":"2024-08-17T18:22:04.170537+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:44.334290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":88,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.151055+0000","last_change":"2024-08-17T18:21:33.403089+0000","last_active":"2024-08-17T18:22:03.151055+0000","last_peered":"2024-08-17T18:22:03.151055+0000","last_clean":"2024-08-17T18:22:03.151055+0000","last_became_active":"2024-08-17T18:21:33.402788+0000","last_became_peered":"2024-08-17T18:21:33.402788+0000","last_unstale":"2024-08-17T18:22:03.151055+0000","last_undegraded":"2024-08-17T18:22:03.151055+0000","last_fullsized":"2024-08-17T18:22:03.151055+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:25.909442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":91,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.147877+0000","last_change":"2024-08-17T18:21:58.286963+0000","last_active":"2024-08-17T18:22:03.147877+0000","last_peered":"2024-08-17T18:22:03.147877+0000","last_clean":"2024-08-17T18:22:03.147877+0000","last_became_active":"2024-08-17T18:21:58.269297+0000","last_became_peered":"2024-08-17T18:21:58.269297+0000","last_unstale":"2024-08-17T18:22:03.147877+0000","last_undegraded":"2024-08-17T18:22:03.147877+0000","last_fullsized":"2024-08-17T18:22:03.147877+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:53.779677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":245,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.164123+0000","last_change":"2024-08-17T18:21:58.267586+0000","last_active":"2024-08-17T18:22:04.164123+0000","last_peered":"2024-08-17T18:22:04.164123+0000","last_clean":"2024-08-17T18:22:04.164123+0000","last_became_active":"2024-08-17T18:21:58.267287+0000","last_became_peered":"2024-08-17T18:21:58.267287+0000","last_unstale":"2024-08-17T18:22:04.164123+0000","last_undegraded":"2024-08-17T18:22:04.164123+0000","last_fullsized":"2024-08-17T18:22:04.164123+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:51:06.593254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":159,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.147504+0000","last_change":"2024-08-17T18:21:58.288401+0000","last_active":"2024-08-17T18:22:03.147504+0000","last_peered":"2024-08-17T18:22:03.147504+0000","last_clean":"2024-08-17T18:22:03.147504+0000","last_became_active":"2024-08-17T18:21:58.288192+0000","last_became_peered":"2024-08-17T18:21:58.288192+0000","last_unstale":"2024-08-17T18:22:03.147504+0000","last_undegraded":"2024-08-17T18:22:03.147504+0000","last_fullsized":"2024-08-17T18:22:03.147504+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:07:54.845030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2257,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.168711+0000","last_change":"2024-08-17T18:21:33.419513+0000","last_active":"2024-08-17T18:22:04.168711+0000","last_peered":"2024-08-17T18:22:04.168711+0000","last_clean":"2024-08-17T18:22:04.168711+0000","last_became_active":"2024-08-17T18:21:33.419186+0000","last_became_peered":"2024-08-17T18:21:33.419186+0000","last_unstale":"2024-08-17T18:22:04.168711+0000","last_undegraded":"2024-08-17T18:22:04.168711+0000","last_fullsized":"2024-08-17T18:22:04.168711+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:27.395794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":49,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.164370+0000","last_change":"2024-08-17T18:21:58.274148+0000","last_active":"2024-08-17T18:22:04.164370+0000","last_peered":"2024-08-17T18:22:04.164370+0000","last_clean":"2024-08-17T18:22:04.164370+0000","last_became_active":"2024-08-17T18:21:58.272822+0000","last_became_peered":"2024-08-17T18:21:58.272822+0000","last_unstale":"2024-08-17T18:22:04.164370+0000","last_undegraded":"2024-08-17T18:22:04.164370+0000","last_fullsized":"2024-08-17T18:22:04.164370+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:45.155095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":42,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.163478+0000","last_change":"2024-08-17T18:21:33.407703+0000","last_active":"2024-08-17T18:22:04.163478+0000","last_peered":"2024-08-17T18:22:04.163478+0000","last_clean":"2024-08-17T18:22:04.163478+0000","last_became_active":"2024-08-17T18:21:33.407397+0000","last_became_peered":"2024-08-17T18:21:33.407397+0000","last_unstale":"2024-08-17T18:22:04.163478+0000","last_undegraded":"2024-08-17T18:22:04.163478+0000","last_fullsized":"2024-08-17T18:22:04.163478+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:06.725837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1566,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.153206+0000","last_change":"2024-08-17T18:21:58.270498+0000","last_active":"2024-08-17T18:22:03.153206+0000","last_peered":"2024-08-17T18:22:03.153206+0000","last_clean":"2024-08-17T18:22:03.153206+0000","last_became_active":"2024-08-17T18:21:58.270156+0000","last_became_peered":"2024-08-17T18:21:58.270156+0000","last_unstale":"2024-08-17T18:22:03.153206+0000","last_undegraded":"2024-08-17T18:22:03.153206+0000","last_fullsized":"2024-08-17T18:22:03.153206+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:19:20.221532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":103,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.147983+0000","last_change":"2024-08-17T18:18:50.516313+0000","last_active":"2024-08-17T18:22:03.147983+0000","last_peered":"2024-08-17T18:22:03.147983+0000","last_clean":"2024-08-17T18:22:03.147983+0000","last_became_active":"2024-08-17T18:18:50.515925+0000","last_became_peered":"2024-08-17T18:18:50.515925+0000","last_unstale":"2024-08-17T18:22:03.147983+0000","last_undegraded":"2024-08-17T18:22:03.147983+0000","last_fullsized":"2024-08-17T18:22:03.147983+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:14.611619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2305,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.160572+0000","last_change":"2024-08-17T18:21:12.770126+0000","last_active":"2024-08-17T18:22:04.160572+0000","last_peered":"2024-08-17T18:22:04.160572+0000","last_clean":"2024-08-17T18:22:04.160572+0000","last_became_active":"2024-08-17T18:21:12.769545+0000","last_became_peered":"2024-08-17T18:21:12.769545+0000","last_unstale":"2024-08-17T18:22:04.160572+0000","last_undegraded":"2024-08-17T18:22:04.160572+0000","last_fullsized":"2024-08-17T18:22:04.160572+0000","mapping_epoch":1477,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:21.660942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":280,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.170015+0000","last_change":"2024-08-17T18:21:32.421106+0000","last_active":"2024-08-17T18:22:04.170015+0000","last_peered":"2024-08-17T18:22:04.170015+0000","last_clean":"2024-08-17T18:22:04.170015+0000","last_became_active":"2024-08-17T18:21:12.794575+0000","last_became_peered":"2024-08-17T18:21:12.794575+0000","last_unstale":"2024-08-17T18:22:04.170015+0000","last_undegraded":"2024-08-17T18:22:04.170015+0000","last_fullsized":"2024-08-17T18:22:04.170015+0000","mapping_epoch":1477,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:53:46.665801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1976,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.153122+0000","last_change":"2024-08-17T18:22:00.258650+0000","last_active":"2024-08-17T18:22:03.153122+0000","last_peered":"2024-08-17T18:22:03.153122+0000","last_clean":"2024-08-17T18:22:03.153122+0000","last_became_active":"2024-08-17T18:22:00.258326+0000","last_became_peered":"2024-08-17T18:22:00.258326+0000","last_unstale":"2024-08-17T18:22:03.153122+0000","last_undegraded":"2024-08-17T18:22:03.153122+0000","last_fullsized":"2024-08-17T18:22:03.153122+0000","mapping_epoch":1500,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:02.578380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1948,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.161985+0000","last_change":"2024-08-17T18:21:58.270154+0000","last_active":"2024-08-17T18:22:04.161985+0000","last_peered":"2024-08-17T18:22:04.161985+0000","last_clean":"2024-08-17T18:22:04.161985+0000","last_became_active":"2024-08-17T18:21:58.269562+0000","last_became_peered":"2024-08-17T18:21:58.269562+0000","last_unstale":"2024-08-17T18:22:04.161985+0000","last_undegraded":"2024-08-17T18:22:04.161985+0000","last_fullsized":"2024-08-17T18:22:04.161985+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:21.136191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2361,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.160271+0000","last_change":"2024-08-17T18:21:58.271499+0000","last_active":"2024-08-17T18:22:04.160271+0000","last_peered":"2024-08-17T18:22:04.160271+0000","last_clean":"2024-08-17T18:22:04.160271+0000","last_became_active":"2024-08-17T18:21:58.270311+0000","last_became_peered":"2024-08-17T18:21:58.270311+0000","last_unstale":"2024-08-17T18:22:04.160271+0000","last_undegraded":"2024-08-17T18:22:04.160271+0000","last_fullsized":"2024-08-17T18:22:04.160271+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:10.795233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":78,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.150687+0000","last_change":"2024-08-17T18:21:58.266370+0000","last_active":"2024-08-17T18:22:03.150687+0000","last_peered":"2024-08-17T18:22:03.150687+0000","last_clean":"2024-08-17T18:22:03.150687+0000","last_became_active":"2024-08-17T18:21:58.266082+0000","last_became_peered":"2024-08-17T18:21:58.266082+0000","last_unstale":"2024-08-17T18:22:03.150687+0000","last_undegraded":"2024-08-17T18:22:03.150687+0000","last_fullsized":"2024-08-17T18:22:03.150687+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:02:20.218698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":82,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.149994+0000","last_change":"2024-08-17T18:19:27.038041+0000","last_active":"2024-08-17T18:22:03.149994+0000","last_peered":"2024-08-17T18:22:03.149994+0000","last_clean":"2024-08-17T18:22:03.149994+0000","last_became_active":"2024-08-17T18:19:27.037715+0000","last_became_peered":"2024-08-17T18:19:27.037715+0000","last_unstale":"2024-08-17T18:22:03.149994+0000","last_undegraded":"2024-08-17T18:22:03.149994+0000","last_fullsized":"2024-08-17T18:22:03.149994+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:06:39.726534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":89,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.150715+0000","last_change":"2024-08-17T18:21:58.278111+0000","last_active":"2024-08-17T18:22:03.150715+0000","last_peered":"2024-08-17T18:22:03.150715+0000","last_clean":"2024-08-17T18:22:03.150715+0000","last_became_active":"2024-08-17T18:21:58.277525+0000","last_became_peered":"2024-08-17T18:21:58.277525+0000","last_unstale":"2024-08-17T18:22:03.150715+0000","last_undegraded":"2024-08-17T18:22:03.150715+0000","last_fullsized":"2024-08-17T18:22:03.150715+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:37.907363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":113,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.147955+0000","last_change":"2024-08-17T18:21:58.268737+0000","last_active":"2024-08-17T18:22:03.147955+0000","last_peered":"2024-08-17T18:22:03.147955+0000","last_clean":"2024-08-17T18:22:03.147955+0000","last_became_active":"2024-08-17T18:21:58.268409+0000","last_became_peered":"2024-08-17T18:21:58.268409+0000","last_unstale":"2024-08-17T18:22:03.147955+0000","last_undegraded":"2024-08-17T18:22:03.147955+0000","last_fullsized":"2024-08-17T18:22:03.147955+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:56.962967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.150507+0000","last_change":"2024-08-17T18:18:50.515676+0000","last_active":"2024-08-17T18:22:03.150507+0000","last_peered":"2024-08-17T18:22:03.150507+0000","last_clean":"2024-08-17T18:22:03.150507+0000","last_became_active":"2024-08-17T18:18:50.515313+0000","last_became_peered":"2024-08-17T18:18:50.515313+0000","last_unstale":"2024-08-17T18:22:03.150507+0000","last_undegraded":"2024-08-17T18:22:03.150507+0000","last_fullsized":"2024-08-17T18:22:03.150507+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:36.955526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2392,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.161129+0000","last_change":"2024-08-17T18:21:33.493429+0000","last_active":"2024-08-17T18:22:04.161129+0000","last_peered":"2024-08-17T18:22:04.161129+0000","last_clean":"2024-08-17T18:22:04.161129+0000","last_became_active":"2024-08-17T18:21:33.421176+0000","last_became_peered":"2024-08-17T18:21:33.421176+0000","last_unstale":"2024-08-17T18:22:04.161129+0000","last_undegraded":"2024-08-17T18:22:04.161129+0000","last_fullsized":"2024-08-17T18:22:04.161129+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:42.334437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":49,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.163989+0000","last_change":"2024-08-17T18:21:58.268206+0000","last_active":"2024-08-17T18:22:04.163989+0000","last_peered":"2024-08-17T18:22:04.163989+0000","last_clean":"2024-08-17T18:22:04.163989+0000","last_became_active":"2024-08-17T18:21:58.267898+0000","last_became_peered":"2024-08-17T18:21:58.267898+0000","last_unstale":"2024-08-17T18:22:04.163989+0000","last_undegraded":"2024-08-17T18:22:04.163989+0000","last_fullsized":"2024-08-17T18:22:04.163989+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:41.843446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":108,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.148494+0000","last_change":"2024-08-17T18:21:58.267796+0000","last_active":"2024-08-17T18:22:03.148494+0000","last_peered":"2024-08-17T18:22:03.148494+0000","last_clean":"2024-08-17T18:22:03.148494+0000","last_became_active":"2024-08-17T18:21:58.267504+0000","last_became_peered":"2024-08-17T18:21:58.267504+0000","last_unstale":"2024-08-17T18:22:03.148494+0000","last_undegraded":"2024-08-17T18:22:03.148494+0000","last_fullsized":"2024-08-17T18:22:03.148494+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:55:35.703349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":103,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.147873+0000","last_change":"2024-08-17T18:21:58.266675+0000","last_active":"2024-08-17T18:22:03.147873+0000","last_peered":"2024-08-17T18:22:03.147873+0000","last_clean":"2024-08-17T18:22:03.147873+0000","last_became_active":"2024-08-17T18:21:58.266286+0000","last_became_peered":"2024-08-17T18:21:58.266286+0000","last_unstale":"2024-08-17T18:22:03.147873+0000","last_undegraded":"2024-08-17T18:22:03.147873+0000","last_fullsized":"2024-08-17T18:22:03.147873+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:08.506169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1527,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.152177+0000","last_change":"2024-08-17T18:21:12.791040+0000","last_active":"2024-08-17T18:22:03.152177+0000","last_peered":"2024-08-17T18:22:03.152177+0000","last_clean":"2024-08-17T18:22:03.152177+0000","last_became_active":"2024-08-17T18:21:12.773669+0000","last_became_peered":"2024-08-17T18:21:12.773669+0000","last_unstale":"2024-08-17T18:22:03.152177+0000","last_undegraded":"2024-08-17T18:22:03.152177+0000","last_fullsized":"2024-08-17T18:22:03.152177+0000","mapping_epoch":1477,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:12.823106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":78,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.150059+0000","last_change":"2024-08-17T18:21:58.269847+0000","last_active":"2024-08-17T18:22:03.150059+0000","last_peered":"2024-08-17T18:22:03.150059+0000","last_clean":"2024-08-17T18:22:03.150059+0000","last_became_active":"2024-08-17T18:21:58.269541+0000","last_became_peered":"2024-08-17T18:21:58.269541+0000","last_unstale":"2024-08-17T18:22:03.150059+0000","last_undegraded":"2024-08-17T18:22:03.150059+0000","last_fullsized":"2024-08-17T18:22:03.150059+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:13.259546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1542,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.152990+0000","last_change":"2024-08-17T18:21:58.277027+0000","last_active":"2024-08-17T18:22:03.152990+0000","last_peered":"2024-08-17T18:22:03.152990+0000","last_clean":"2024-08-17T18:22:03.152990+0000","last_became_active":"2024-08-17T18:21:58.263795+0000","last_became_peered":"2024-08-17T18:21:58.263795+0000","last_unstale":"2024-08-17T18:22:03.152990+0000","last_undegraded":"2024-08-17T18:22:03.152990+0000","last_fullsized":"2024-08-17T18:22:03.152990+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:45:12.859260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":418,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.151579+0000","last_change":"2024-08-17T18:21:12.772737+0000","last_active":"2024-08-17T18:22:03.151579+0000","last_peered":"2024-08-17T18:22:03.151579+0000","last_clean":"2024-08-17T18:22:03.151579+0000","last_became_active":"2024-08-17T18:21:12.772318+0000","last_became_peered":"2024-08-17T18:21:12.772318+0000","last_unstale":"2024-08-17T18:22:03.151579+0000","last_undegraded":"2024-08-17T18:22:03.151579+0000","last_fullsized":"2024-08-17T18:22:03.151579+0000","mapping_epoch":1477,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:20.974581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1572,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.153325+0000","last_change":"2024-08-17T18:21:33.495232+0000","last_active":"2024-08-17T18:22:03.153325+0000","last_peered":"2024-08-17T18:22:03.153325+0000","last_clean":"2024-08-17T18:22:03.153325+0000","last_became_active":"2024-08-17T18:21:33.438606+0000","last_became_peered":"2024-08-17T18:21:33.438606+0000","last_unstale":"2024-08-17T18:22:03.153325+0000","last_undegraded":"2024-08-17T18:22:03.153325+0000","last_fullsized":"2024-08-17T18:22:03.153325+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:14.007910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":183,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.147764+0000","last_change":"2024-08-17T18:21:33.491306+0000","last_active":"2024-08-17T18:22:03.147764+0000","last_peered":"2024-08-17T18:22:03.147764+0000","last_clean":"2024-08-17T18:22:03.147764+0000","last_became_active":"2024-08-17T18:21:33.419876+0000","last_became_peered":"2024-08-17T18:21:33.419876+0000","last_unstale":"2024-08-17T18:22:03.147764+0000","last_undegraded":"2024-08-17T18:22:03.147764+0000","last_fullsized":"2024-08-17T18:22:03.147764+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:33:46.580078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2359,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.162062+0000","last_change":"2024-08-17T18:21:58.269877+0000","last_active":"2024-08-17T18:22:04.162062+0000","last_peered":"2024-08-17T18:22:04.162062+0000","last_clean":"2024-08-17T18:22:04.162062+0000","last_became_active":"2024-08-17T18:21:58.269057+0000","last_became_peered":"2024-08-17T18:21:58.269057+0000","last_unstale":"2024-08-17T18:22:04.162062+0000","last_undegraded":"2024-08-17T18:22:04.162062+0000","last_fullsized":"2024-08-17T18:22:04.162062+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:56:33.384257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1496,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.152023+0000","last_change":"2024-08-17T18:18:50.515245+0000","last_active":"2024-08-17T18:22:03.152023+0000","last_peered":"2024-08-17T18:22:03.152023+0000","last_clean":"2024-08-17T18:22:03.152023+0000","last_became_active":"2024-08-17T18:18:50.514785+0000","last_became_peered":"2024-08-17T18:18:50.514785+0000","last_unstale":"2024-08-17T18:22:03.152023+0000","last_undegraded":"2024-08-17T18:22:03.152023+0000","last_fullsized":"2024-08-17T18:22:03.152023+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:08.696581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":42,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.165298+0000","last_change":"2024-08-17T18:21:33.408565+0000","last_active":"2024-08-17T18:22:04.165298+0000","last_peered":"2024-08-17T18:22:04.165298+0000","last_clean":"2024-08-17T18:22:04.165298+0000","last_became_active":"2024-08-17T18:21:33.407832+0000","last_became_peered":"2024-08-17T18:21:33.407832+0000","last_unstale":"2024-08-17T18:22:04.165298+0000","last_undegraded":"2024-08-17T18:22:04.165298+0000","last_fullsized":"2024-08-17T18:22:04.165298+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:09:06.558530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":42,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.163950+0000","last_change":"2024-08-17T18:21:33.428616+0000","last_active":"2024-08-17T18:22:04.163950+0000","last_peered":"2024-08-17T18:22:04.163950+0000","last_clean":"2024-08-17T18:22:04.163950+0000","last_became_active":"2024-08-17T18:21:33.428368+0000","last_became_peered":"2024-08-17T18:21:33.428368+0000","last_unstale":"2024-08-17T18:22:04.163950+0000","last_undegraded":"2024-08-17T18:22:04.163950+0000","last_fullsized":"2024-08-17T18:22:04.163950+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:14.945169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":904,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.151998+0000","last_change":"2024-08-17T18:21:12.777384+0000","last_active":"2024-08-17T18:22:03.151998+0000","last_peered":"2024-08-17T18:22:03.151998+0000","last_clean":"2024-08-17T18:22:03.151998+0000","last_became_active":"2024-08-17T18:21:12.776205+0000","last_became_peered":"2024-08-17T18:21:12.776205+0000","last_unstale":"2024-08-17T18:22:03.151998+0000","last_undegraded":"2024-08-17T18:22:03.151998+0000","last_fullsized":"2024-08-17T18:22:03.151998+0000","mapping_epoch":1477,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:02.449650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1545,"reported_epoch":1504,"state":"active+clean","last_fresh":"2024-08-17T18:22:03.152418+0000","last_change":"2024-08-17T18:19:27.037702+0000","last_active":"2024-08-17T18:22:03.152418+0000","last_peered":"2024-08-17T18:22:03.152418+0000","last_clean":"2024-08-17T18:22:03.152418+0000","last_became_active":"2024-08-17T18:19:27.036532+0000","last_became_peered":"2024-08-17T18:19:27.036532+0000","last_unstale":"2024-08-17T18:22:03.152418+0000","last_undegraded":"2024-08-17T18:22:03.152418+0000","last_fullsized":"2024-08-17T18:22:03.152418+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:24.777460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2359,"reported_epoch":1505,"state":"active+clean","last_fresh":"2024-08-17T18:22:04.161948+0000","last_change":"2024-08-17T18:21:58.274634+0000","last_active":"2024-08-17T18:22:04.161948+0000","last_peered":"2024-08-17T18:22:04.161948+0000","last_clean":"2024-08-17T18:22:04.161948+0000","last_became_active":"2024-08-17T18:21:58.274259+0000","last_became_peered":"2024-08-17T18:21:58.274259+0000","last_unstale":"2024-08-17T18:22:04.161948+0000","last_undegraded":"2024-08-17T18:22:04.161948+0000","last_fullsized":"2024-08-17T18:22:04.161948+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:15.278544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":50,"num_pg":26,"stat_sum":{"num_bytes":21004312,"num_objects":321,"num_object_clones":0,"num_object_copies":963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":77,"num_objects_misplaced":141,"num_objects_unfound":0,"num_objects_dirty":321,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":259644,"num_write_kb":7557761,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45876,"num_bytes_recovered":2785542192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":341852160,"data_stored":341835872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":156524,"ondisk_log_size":156524,"up":78,"acting":69,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":88,"num_bytes_recovered":11996,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234737,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1540200,"kb_used_data":50464,"kb_used_omap":1,"kb_used_meta":1489726,"kb_avail":92204952,"statfs":{"total":95995035648,"available":94417870848,"internally_reserved":0,"allocated":51675136,"data_stored":46193710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1525479768},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496372,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1018596,"kb_used_data":112924,"kb_used_omap":1,"kb_used_meta":905662,"kb_avail":92726556,"statfs":{"total":95995035648,"available":94951993344,"internally_reserved":0,"allocated":115634176,"data_stored":110157281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":927398232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208758010,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1343,"seq":5768141078683,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":780568,"kb_used_data":21456,"kb_used_omap":112,"kb_used_meta":758863,"kb_avail":92964584,"statfs":{"total":95995035648,"available":95195734016,"internally_reserved":0,"allocated":21970944,"data_stored":16500621,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":115373,"internal_metadata":777076051},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":1454,"seq":6244882448420,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":279504,"kb_used_data":12936,"kb_used_omap":111,"kb_used_meta":266448,"kb_avail":93465648,"statfs":{"total":95995035648,"available":95708823552,"internally_reserved":0,"allocated":13246464,"data_stored":7784726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":114548,"internal_metadata":272842892},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":0,"seq":2173253452398,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471475088,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1708508,"kb_used_data":219924,"kb_used_omap":119,"kb_used_meta":1488456,"kb_avail":92036644,"statfs":{"total":95995035648,"available":94245523456,"internally_reserved":0,"allocated":225202176,"data_stored":219728725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":1524179678},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019645,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":332132,"kb_used_data":26460,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93413020,"statfs":{"total":95995035648,"available":95654932480,"internally_reserved":0,"allocated":27095040,"data_stored":21617112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":50,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":195825664,"data_stored":195821592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":50,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":33751040,"data_stored":33751040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":50,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":50,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1118208,"data_stored":1114136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":50,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3870720,"data_stored":3866648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":50,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":50,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":77201408,"data_stored":77201408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":50,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":21692416,"data_stored":21692416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:22:23.912 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T18:22:23.912 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_5 pg_num 36 2024-08-17T18:22:23.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:23.935 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:23.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:23.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:24.037 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:24.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:24.059 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:24.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:24.061 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:22:24.161 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:24.182 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:24.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:24.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:24.283 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:24.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:24.306 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:24.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:24.407 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:24.418 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:24 smithi130 ceph-mon[110068]: from='client.47369 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:22:24.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:24 smithi130 ceph-mon[110068]: 50.8 scrub starts 2024-08-17T18:22:24.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:24 smithi130 ceph-mon[110068]: 50.8 scrub ok 2024-08-17T18:22:24.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:24 smithi130 ceph-mon[110068]: 50.5 deep-scrub starts 2024-08-17T18:22:24.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:24 smithi130 ceph-mon[110068]: 50.5 deep-scrub ok 2024-08-17T18:22:24.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:24 smithi130 ceph-mon[110068]: 50.4 deep-scrub starts 2024-08-17T18:22:24.419 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:24 smithi130 ceph-mon[110068]: 50.4 deep-scrub ok 2024-08-17T18:22:24.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:24 smithi130 ceph-mon[105521]: from='client.47369 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:22:24.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:24 smithi130 ceph-mon[105521]: 50.8 scrub starts 2024-08-17T18:22:24.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:24 smithi130 ceph-mon[105521]: 50.8 scrub ok 2024-08-17T18:22:24.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:24 smithi130 ceph-mon[105521]: 50.5 deep-scrub starts 2024-08-17T18:22:24.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:24 smithi130 ceph-mon[105521]: 50.5 deep-scrub ok 2024-08-17T18:22:24.419 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:24 smithi130 ceph-mon[105521]: 50.4 deep-scrub starts 2024-08-17T18:22:24.420 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:24 smithi130 ceph-mon[105521]: 50.4 deep-scrub ok 2024-08-17T18:22:24.429 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:24.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:24.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:24.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:24 smithi179 ceph-mon[93519]: from='client.47369 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:22:24.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:24 smithi179 ceph-mon[93519]: 50.8 scrub starts 2024-08-17T18:22:24.494 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:24 smithi179 ceph-mon[93519]: 50.8 scrub ok 2024-08-17T18:22:24.495 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:24 smithi179 ceph-mon[93519]: 50.5 deep-scrub starts 2024-08-17T18:22:24.495 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:24 smithi179 ceph-mon[93519]: 50.5 deep-scrub ok 2024-08-17T18:22:24.495 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:24 smithi179 ceph-mon[93519]: 50.4 deep-scrub starts 2024-08-17T18:22:24.495 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:24 smithi179 ceph-mon[93519]: 50.4 deep-scrub ok 2024-08-17T18:22:24.531 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:24.553 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:24.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:24.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:24.654 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:24.658 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:24.677 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:24.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:24.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:24.779 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:24.800 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:24.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:24.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:24.901 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:24.923 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:24.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:24.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:25.025 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:25.046 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:25.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:25.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:25.072 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T18:22:25.148 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:25.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:25.171 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:25.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:25.272 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:25.293 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:25.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:25.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:25.394 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:25.416 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:25.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:25 smithi130 ceph-mon[105521]: pgmap v2942: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+remapped+backfill_wait, 95 active+clean; 13 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 60/681 objects misplaced (8.811%) 2024-08-17T18:22:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:25 smithi130 ceph-mon[105521]: 50.0 scrub starts 2024-08-17T18:22:25.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:25 smithi130 ceph-mon[105521]: 50.0 scrub ok 2024-08-17T18:22:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:25 smithi130 ceph-mon[110068]: pgmap v2942: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+remapped+backfill_wait, 95 active+clean; 13 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 60/681 objects misplaced (8.811%) 2024-08-17T18:22:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:25 smithi130 ceph-mon[110068]: 50.0 scrub starts 2024-08-17T18:22:25.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:25 smithi130 ceph-mon[110068]: 50.0 scrub ok 2024-08-17T18:22:25.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:25.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:25.519 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:25.540 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:25.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:25.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:25.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:25 smithi179 ceph-mon[93519]: pgmap v2942: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 1 active+undersized+remapped+backfill_wait, 1 active+remapped+backfill_wait, 95 active+clean; 13 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 60/681 objects misplaced (8.811%) 2024-08-17T18:22:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:25 smithi179 ceph-mon[93519]: 50.0 scrub starts 2024-08-17T18:22:25.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:25 smithi179 ceph-mon[93519]: 50.0 scrub ok 2024-08-17T18:22:25.641 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:25.663 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:25.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:25.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:25.765 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:25.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:25.787 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:25.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:25.888 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:25.910 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:25.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:25.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:26.012 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:26.034 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:26.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:26.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:26.069 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:22:26.069 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:22:26.135 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:26.158 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:26.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:26.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:26.260 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:26.282 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:26.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:26.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:26.365 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:22:26.365 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:22:26.378 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:22:26.379 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:22:26.384 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:26.395 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:22:26.395 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:22:26.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:26.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:26.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:26.414 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:22:26.414 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:22:26.435 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:22:26.435 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:22:26.451 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:22:26.451 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:22:26.466 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:22:26.467 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:22:26.482 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:22:26.482 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:22:26.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:26.528 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:26.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:26.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:26.542 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:26 smithi130 ceph-mon[105521]: 50.16 scrub starts 2024-08-17T18:22:26.542 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:26 smithi130 ceph-mon[105521]: 50.16 scrub ok 2024-08-17T18:22:26.543 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:26 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T18:22:26.543 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:26 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T18:22:26.543 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:26 smithi130 ceph-mon[105521]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-08-17T18:22:26.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:26 smithi130 ceph-mon[110068]: 50.16 scrub starts 2024-08-17T18:22:26.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:26 smithi130 ceph-mon[110068]: 50.16 scrub ok 2024-08-17T18:22:26.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:26 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T18:22:26.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:26 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T18:22:26.543 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:26 smithi130 ceph-mon[110068]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-08-17T18:22:26.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:26 smithi179 ceph-mon[93519]: 50.16 scrub starts 2024-08-17T18:22:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:26 smithi179 ceph-mon[93519]: 50.16 scrub ok 2024-08-17T18:22:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:26 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T18:22:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:26 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-08-17T18:22:26.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:26 smithi179 ceph-mon[93519]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-08-17T18:22:26.630 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:26.652 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:26.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:26.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:26.750 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T18:22:26.754 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:26.813 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:26.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:26.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:26.914 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:26.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:26.937 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:26.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:27.038 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:27.047 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:22:27.060 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:27.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:27.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:27.161 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:27.182 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:27.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:27.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:27.284 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:27.306 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:27.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:27.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:27.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[110068]: pgmap v2943: 99 pgs: 1 active+undersized+remapped, 98 active+clean; 4.3 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:27.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[110068]: 50.7 scrub starts 2024-08-17T18:22:27.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[110068]: 50.7 scrub ok 2024-08-17T18:22:27.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[110068]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-08-17T18:22:27.356 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T18:22:27.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T18:22:27.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[110068]: osdmap e1509: 8 total, 8 up, 6 in 2024-08-17T18:22:27.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2883676815' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:27.357 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:22:27.357 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[105521]: pgmap v2943: 99 pgs: 1 active+undersized+remapped, 98 active+clean; 4.3 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:27.357 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[105521]: 50.7 scrub starts 2024-08-17T18:22:27.357 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[105521]: 50.7 scrub ok 2024-08-17T18:22:27.357 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[105521]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-08-17T18:22:27.357 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T18:22:27.357 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T18:22:27.357 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[105521]: osdmap e1509: 8 total, 8 up, 6 in 2024-08-17T18:22:27.358 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2883676815' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:27.358 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:22:27.408 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:27.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:27.430 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:27.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:27.502 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:27 smithi179 ceph-mon[93519]: pgmap v2943: 99 pgs: 1 active+undersized+remapped, 98 active+clean; 4.3 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:27.502 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:27 smithi179 ceph-mon[93519]: 50.7 scrub starts 2024-08-17T18:22:27.502 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:27 smithi179 ceph-mon[93519]: 50.7 scrub ok 2024-08-17T18:22:27.503 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:27 smithi179 ceph-mon[93519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-08-17T18:22:27.503 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:27 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T18:22:27.503 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-08-17T18:22:27.503 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:27 smithi179 ceph-mon[93519]: osdmap e1509: 8 total, 8 up, 6 in 2024-08-17T18:22:27.503 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:27 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2883676815' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:27.503 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:22:27.531 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:27.552 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:27.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:27.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:27.654 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:27.675 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:27.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:27.777 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:27.798 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:27.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:27.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:27.900 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:27.923 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:27.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:27.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:28.024 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:28.046 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:28.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:28.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:28.148 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:28.169 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:28.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:28.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:28.183 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:28.271 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:28.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:28.293 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:28.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:28.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[110068]: 50.17 scrub starts 2024-08-17T18:22:28.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[110068]: 50.17 scrub ok 2024-08-17T18:22:28.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[110068]: osdmap e1510: 8 total, 8 up, 6 in 2024-08-17T18:22:28.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[110068]: 50.10 continuing backfill to osd.2 from (1444'6889,1505'12475] 50:08ed6208:::benchmark_data_smithi130_398002_object114558:head to 1505'12475 2024-08-17T18:22:28.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[110068]: 50.10 starting backfill to osd.3 from (0'0,0'0] MAX to 1505'12475 2024-08-17T18:22:28.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:28.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:22:28.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T18:22:28.309 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T18:22:28.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[105521]: 50.17 scrub starts 2024-08-17T18:22:28.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[105521]: 50.17 scrub ok 2024-08-17T18:22:28.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[105521]: osdmap e1510: 8 total, 8 up, 6 in 2024-08-17T18:22:28.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[105521]: 50.10 continuing backfill to osd.2 from (1444'6889,1505'12475] 50:08ed6208:::benchmark_data_smithi130_398002_object114558:head to 1505'12475 2024-08-17T18:22:28.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[105521]: 50.10 starting backfill to osd.3 from (0'0,0'0] MAX to 1505'12475 2024-08-17T18:22:28.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:28.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:22:28.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T18:22:28.310 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:28 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T18:22:28.394 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:28.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:28.416 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:28.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:28.516 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:28.538 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:28.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:28.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:28.586 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:28 smithi179 ceph-mon[93519]: 50.17 scrub starts 2024-08-17T18:22:28.586 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:28 smithi179 ceph-mon[93519]: 50.17 scrub ok 2024-08-17T18:22:28.586 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:28 smithi179 ceph-mon[93519]: osdmap e1510: 8 total, 8 up, 6 in 2024-08-17T18:22:28.586 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:28 smithi179 ceph-mon[93519]: 50.10 continuing backfill to osd.2 from (1444'6889,1505'12475] 50:08ed6208:::benchmark_data_smithi130_398002_object114558:head to 1505'12475 2024-08-17T18:22:28.586 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:28 smithi179 ceph-mon[93519]: 50.10 starting backfill to osd.3 from (0'0,0'0] MAX to 1505'12475 2024-08-17T18:22:28.586 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:28.586 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:22:28.586 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:28 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T18:22:28.586 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:28 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-08-17T18:22:28.640 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:28.660 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:28.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:28.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:28.762 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:28.782 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:28.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:28.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:28.878 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:22:28.878 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [], 2024-08-17T18:22:28.879 INFO:teuthology.orchestra.run.smithi179.stdout: "complaint_time": 30, 2024-08-17T18:22:28.879 INFO:teuthology.orchestra.run.smithi179.stdout: "num_blocked_ops": 0 2024-08-17T18:22:28.879 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:22:28.884 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:28.906 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:28.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:28.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:29.008 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:29.029 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:29.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:29.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:29.131 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:29.154 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:29.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:29.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:29.256 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:29.261 INFO:teuthology.orchestra.run.smithi130.stderr:set pool 50 pg_num to 36 2024-08-17T18:22:29.286 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:29.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:29.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:29.388 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:29.409 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:29.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:29.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:29.483 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:29 smithi179 ceph-mon[93519]: pgmap v2946: 99 pgs: 1 active+undersized+remapped, 98 active+clean; 4.3 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:29.483 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:29 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T18:22:29.483 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:29 smithi179 ceph-mon[93519]: osdmap e1511: 8 total, 8 up, 6 in 2024-08-17T18:22:29.483 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:29 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3177211289' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T18:22:29.483 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:29 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T18:22:29.506 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T18:22:29.511 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:29.570 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:29.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:29.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:29.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:29 smithi130 ceph-mon[105521]: pgmap v2946: 99 pgs: 1 active+undersized+remapped, 98 active+clean; 4.3 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:29 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T18:22:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:29 smithi130 ceph-mon[105521]: osdmap e1511: 8 total, 8 up, 6 in 2024-08-17T18:22:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:29 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3177211289' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T18:22:29.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:29 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T18:22:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:29 smithi130 ceph-mon[110068]: pgmap v2946: 99 pgs: 1 active+undersized+remapped, 98 active+clean; 4.3 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:29 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-08-17T18:22:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:29 smithi130 ceph-mon[110068]: osdmap e1511: 8 total, 8 up, 6 in 2024-08-17T18:22:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:29 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3177211289' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T18:22:29.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:29 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T18:22:29.672 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:29.694 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:29.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:29.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:29.706 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:22:29.796 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:29.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:29.819 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:29.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:29.920 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:29.942 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:29.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:29.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:30.040 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:22:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:22:29] "GET /metrics HTTP/1.1" 200 36769 "" "Prometheus/2.51.0" 2024-08-17T18:22:30.044 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:30.066 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:30.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:30.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:30.167 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:30.188 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:30.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:30.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:30.290 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:30.311 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:30.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:30.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:30.413 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:30.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:30 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2024-08-17T18:22:30.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:30 smithi130 ceph-mon[105521]: osdmap e1512: 8 total, 8 up, 6 in 2024-08-17T18:22:30.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:30 smithi130 ceph-mon[105521]: osdmap e1513: 8 total, 8 up, 6 in 2024-08-17T18:22:30.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:30 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2024-08-17T18:22:30.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:30 smithi130 ceph-mon[110068]: osdmap e1512: 8 total, 8 up, 6 in 2024-08-17T18:22:30.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:30 smithi130 ceph-mon[110068]: osdmap e1513: 8 total, 8 up, 6 in 2024-08-17T18:22:30.434 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:30.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:30.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:30.536 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:30.557 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:30.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:30.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:30.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:30 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "36"}]': finished 2024-08-17T18:22:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:30 smithi179 ceph-mon[93519]: osdmap e1512: 8 total, 8 up, 6 in 2024-08-17T18:22:30.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:30 smithi179 ceph-mon[93519]: osdmap e1513: 8 total, 8 up, 6 in 2024-08-17T18:22:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:22:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:22:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:22:30.659 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:30.682 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:30.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:30.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:30.785 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:30.807 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:30.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:30.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:30.908 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:30.929 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:30.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:30.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:31.031 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:31.053 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:31.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:31.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:31.155 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:31.177 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:31.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:31.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:31.271 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-08-17T18:22:31.271 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-08-17T18:22:31.279 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:31.300 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:31.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:31.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:31.401 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:31.422 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:31.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:31.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:31.479 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:22:31.524 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:31.546 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:31.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:31.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:31.571 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:31 smithi130 ceph-mon[105521]: pgmap v2949: 99 pgs: 1 peering, 98 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:31.571 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:31 smithi130 ceph-mon[105521]: 50.10 scrub starts 2024-08-17T18:22:31.571 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:31 smithi130 ceph-mon[105521]: 50.10 scrub ok 2024-08-17T18:22:31.571 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:31 smithi130 ceph-mon[105521]: 50.11 starting backfill to osd.4 from (0'0,0'0] MAX to 1505'12338 2024-08-17T18:22:31.571 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:31 smithi130 ceph-mon[105521]: 50.11 starting backfill to osd.6 from (0'0,0'0] MAX to 1505'12338 2024-08-17T18:22:31.571 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:31 smithi130 ceph-mon[105521]: osdmap e1514: 8 total, 8 up, 6 in 2024-08-17T18:22:31.571 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:31 smithi130 ceph-mon[110068]: pgmap v2949: 99 pgs: 1 peering, 98 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:31.571 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:31 smithi130 ceph-mon[110068]: 50.10 scrub starts 2024-08-17T18:22:31.572 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:31 smithi130 ceph-mon[110068]: 50.10 scrub ok 2024-08-17T18:22:31.572 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:31 smithi130 ceph-mon[110068]: 50.11 starting backfill to osd.4 from (0'0,0'0] MAX to 1505'12338 2024-08-17T18:22:31.572 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:31 smithi130 ceph-mon[110068]: 50.11 starting backfill to osd.6 from (0'0,0'0] MAX to 1505'12338 2024-08-17T18:22:31.572 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:31 smithi130 ceph-mon[110068]: osdmap e1514: 8 total, 8 up, 6 in 2024-08-17T18:22:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:31 smithi179 ceph-mon[93519]: pgmap v2949: 99 pgs: 1 peering, 98 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:31 smithi179 ceph-mon[93519]: 50.10 scrub starts 2024-08-17T18:22:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:31 smithi179 ceph-mon[93519]: 50.10 scrub ok 2024-08-17T18:22:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:31 smithi179 ceph-mon[93519]: 50.11 starting backfill to osd.4 from (0'0,0'0] MAX to 1505'12338 2024-08-17T18:22:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:31 smithi179 ceph-mon[93519]: 50.11 starting backfill to osd.6 from (0'0,0'0] MAX to 1505'12338 2024-08-17T18:22:31.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:31 smithi179 ceph-mon[93519]: osdmap e1514: 8 total, 8 up, 6 in 2024-08-17T18:22:31.628 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:31.648 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:31.668 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:31.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:31.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:31.677 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:31.770 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:31.792 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:31.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:31.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:31.893 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:31.917 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:31.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:31.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:32.019 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:32.040 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:32.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:32.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:32.142 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:32.154 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is set 2024-08-17T18:22:32.164 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:32.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:32.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:32.266 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:32.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:32.291 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:32.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:32.392 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:32.414 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:32.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:32.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:32.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/300899064' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:22:32.416 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:32 smithi130 ceph-mon[105521]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:22:32.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:32 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/300899064' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:22:32.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:32 smithi130 ceph-mon[105521]: osdmap e1515: 8 total, 8 up, 6 in 2024-08-17T18:22:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/300899064' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:22:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:32 smithi130 ceph-mon[110068]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:22:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:32 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/300899064' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:22:32.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:32 smithi130 ceph-mon[110068]: osdmap e1515: 8 total, 8 up, 6 in 2024-08-17T18:22:32.516 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:32.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:32.538 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:32.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:32.558 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/300899064' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T18:22:32.558 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:32 smithi179 ceph-mon[93519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:22:32.559 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:32 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/300899064' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T18:22:32.559 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:32 smithi179 ceph-mon[93519]: osdmap e1515: 8 total, 8 up, 6 in 2024-08-17T18:22:32.639 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:32.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:32.662 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:32.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:32.763 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:32.784 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:32.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:32.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:32.886 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:32.908 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:32.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:32.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:33.010 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:33.032 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:33.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:33.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:33.134 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:33.156 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:33.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:33.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:33.258 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:33.281 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:33.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:33.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:33.313 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:33 smithi179 ceph-mon[93519]: pgmap v2952: 99 pgs: 1 peering, 98 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:33.383 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:33.406 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:33.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:33.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:33.507 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:33.529 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:33.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:33.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:33.541 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:33 smithi130 ceph-mon[105521]: pgmap v2952: 99 pgs: 1 peering, 98 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:33.541 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:33 smithi130 ceph-mon[110068]: pgmap v2952: 99 pgs: 1 peering, 98 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:33.631 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:33.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:33.654 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:33.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:33.756 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:33.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:33.780 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:33.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:33.881 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:33.903 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:33.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:33.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:34.005 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:34.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:34.027 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:34.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:34.128 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:34.149 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:34.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:34.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:34.251 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:34.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:34.274 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:34.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:34.375 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:34 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T18:22:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:34 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-08-17T18:22:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:34 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T18:22:34.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:34 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-08-17T18:22:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:34 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T18:22:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:34 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-08-17T18:22:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:34 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T18:22:34.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:34 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-08-17T18:22:34.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:34.436 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:34.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:34.471 INFO:teuthology.orchestra.run.smithi179.stdout:{ 2024-08-17T18:22:34.471 INFO:teuthology.orchestra.run.smithi179.stdout: "size": 20, 2024-08-17T18:22:34.472 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 600, 2024-08-17T18:22:34.472 INFO:teuthology.orchestra.run.smithi179.stdout: "ops": [ 2024-08-17T18:22:34.472 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.472 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36960.0:211976 49.0 49.eb210650 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1421)", 2024-08-17T18:22:34.472 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:16:33.650996+0000", 2024-08-17T18:22:34.472 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 359.95711533999997, 2024-08-17T18:22:34.472 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98308165800000002, 2024-08-17T18:22:34.472 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.472 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:22:34.472 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.472 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36960", 2024-08-17T18:22:34.473 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:22:34.473 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 211976 2024-08-17T18:22:34.473 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.473 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.473 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.473 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.473 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.650996+0000", 2024-08-17T18:22:34.473 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.473 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.473 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.473 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.473 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.650996+0000", 2024-08-17T18:22:34.473 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.473 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.474 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.474 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.474 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.650995+0000", 2024-08-17T18:22:34.474 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:34.474 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.474 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.474 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.474 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.650999+0000", 2024-08-17T18:22:34.474 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5879999999999999e-06 2024-08-17T18:22:34.474 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.474 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.474 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.474 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.651006+0000", 2024-08-17T18:22:34.475 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7700000000000004e-06 2024-08-17T18:22:34.475 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.475 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.475 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.475 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.651010+0000", 2024-08-17T18:22:34.475 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5539999999999999e-06 2024-08-17T18:22:34.475 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.475 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.475 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.475 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.651040+0000", 2024-08-17T18:22:34.475 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.9504000000000001e-05 2024-08-17T18:22:34.475 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.475 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.476 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.476 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.651046+0000", 2024-08-17T18:22:34.476 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5890000000000002e-06 2024-08-17T18:22:34.476 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.476 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.476 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.476 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:34.634065+0000", 2024-08-17T18:22:34.476 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98301972900000001 2024-08-17T18:22:34.476 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.476 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.476 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.476 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:34.634077+0000", 2024-08-17T18:22:34.476 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.22e-05 2024-08-17T18:22:34.476 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.36960.0:211995 49.0 49.89f3ff50 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1421)", 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:16:33.674686+0000", 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 359.93342552899998, 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.95975966000000001, 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.36960", 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/2730615239", 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 211995 2024-08-17T18:22:34.477 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.478 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.478 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.478 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.478 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674686+0000", 2024-08-17T18:22:34.478 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.478 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.478 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.478 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.478 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674686+0000", 2024-08-17T18:22:34.478 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.478 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.478 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.479 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.479 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674684+0000", 2024-08-17T18:22:34.479 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:34.479 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.479 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.479 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.479 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674690+0000", 2024-08-17T18:22:34.479 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3000000000000001e-06 2024-08-17T18:22:34.479 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.479 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.479 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.479 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674700+0000", 2024-08-17T18:22:34.479 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0628999999999999e-05 2024-08-17T18:22:34.479 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674706+0000", 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2730000000000002e-06 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674743+0000", 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.7249000000000002e-05 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:33.674750+0000", 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.8909999999999998e-06 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.480 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.481 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.481 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:34.634437+0000", 2024-08-17T18:22:34.481 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.95968738099999995 2024-08-17T18:22:34.481 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.481 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.481 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.481 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:16:34.634445+0000", 2024-08-17T18:22:34.481 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2169999999999994e-06 2024-08-17T18:22:34.481 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.481 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.481 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.481 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.481 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.481 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:106455 50.19 50:9b4a0a99:::benchmark_data_smithi130_398002_object106454:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1443)", 2024-08-17T18:22:34.482 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:18:30.541757+0000", 2024-08-17T18:22:34.482 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 243.0663539, 2024-08-17T18:22:34.482 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.91456280199999995, 2024-08-17T18:22:34.482 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.482 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:22:34.482 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.482 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.482 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.482 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 106455 2024-08-17T18:22:34.482 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.482 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.482 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.482 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.482 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.541757+0000", 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.541757+0000", 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.541755+0000", 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999976 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.541812+0000", 2024-08-17T18:22:34.483 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7299000000000001e-05 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.541830+0000", 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.8125999999999999e-05 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.541839+0000", 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.4509999999999992e-06 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.620408+0000", 2024-08-17T18:22:34.484 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.078569359000000005 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:30.620415+0000", 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.6440000000000003e-06 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:31.456262+0000", 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.83584671099999996 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:18:31.456320+0000", 2024-08-17T18:22:34.485 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8400000000000003e-05 2024-08-17T18:22:34.486 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.486 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.486 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.486 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.486 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.486 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:171228 50.8 50:13b9a0e7:::benchmark_data_smithi130_398002_object26162:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1453)", 2024-08-17T18:22:34.486 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:19:25.008820+0000", 2024-08-17T18:22:34.486 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 188.59929142499999, 2024-08-17T18:22:34.486 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.52510703, 2024-08-17T18:22:34.486 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.486 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:22:34.486 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.486 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.486 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 171228 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.008820+0000", 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.008820+0000", 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.487 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.488 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.008818+0000", 2024-08-17T18:22:34.488 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:34.488 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.488 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.488 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.488 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.008825+0000", 2024-08-17T18:22:34.488 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.9920000000000001e-06 2024-08-17T18:22:34.488 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.488 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.488 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.488 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.008834+0000", 2024-08-17T18:22:34.488 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.4179999999999997e-06 2024-08-17T18:22:34.488 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.488 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.008838+0000", 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2729999999999996e-06 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.009435+0000", 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00059668199999999996 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.009547+0000", 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000112531 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.489 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0", 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.010055+0000", 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00050719700000000003 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.015918+0000", 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0058633640000000002 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.035620+0000", 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.019701921000000001 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.490 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.035625+0000", 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3809999999999998e-06 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.022427+0000", 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98680159300000003 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.022435+0000", 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9540000000000004e-06 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.491 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.490134+0000", 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4676989499999999 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for missing object", 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.490266+0000", 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013211899999999999 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.533840+0000", 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.043573827000000002 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.492 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.533927+0000", 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.7044000000000002e-05 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:171235 50.8 50:1194ee85:::benchmark_data_smithi130_398002_object26169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1453)", 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:19:25.010475+0000", 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 188.59763618299999, 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.532957739, 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.493 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 171235 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.010475+0000", 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.010475+0000", 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.494 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.010474+0000", 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.010478+0000", 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.878e-06 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.010489+0000", 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1139e-05 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.495 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.010495+0000", 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0870000000000003e-06 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.012906+0000", 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0024104759999999999 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.012991+0000", 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.4858999999999995e-05 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.496 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0", 2024-08-17T18:22:34.497 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.013433+0000", 2024-08-17T18:22:34.497 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00044205500000000001 2024-08-17T18:22:34.497 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.497 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.497 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.497 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.018011+0000", 2024-08-17T18:22:34.497 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0045787270000000003 2024-08-17T18:22:34.497 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.497 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.497 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.497 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.035638+0000", 2024-08-17T18:22:34.497 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.017626869 2024-08-17T18:22:34.497 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.497 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.498 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.498 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.035641+0000", 2024-08-17T18:22:34.498 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.1030000000000002e-06 2024-08-17T18:22:34.498 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.498 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.498 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.498 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.022460+0000", 2024-08-17T18:22:34.498 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98681844299999999 2024-08-17T18:22:34.498 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.498 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.498 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.498 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.022466+0000", 2024-08-17T18:22:34.498 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0020000000000003e-06 2024-08-17T18:22:34.499 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.499 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.499 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.499 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.490354+0000", 2024-08-17T18:22:34.499 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4678876860000001 2024-08-17T18:22:34.499 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.499 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.499 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for missing object", 2024-08-17T18:22:34.499 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.490475+0000", 2024-08-17T18:22:34.499 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000121409 2024-08-17T18:22:34.499 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.499 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.499 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.499 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.543338+0000", 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.052862623999999997 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.543433+0000", 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.5105000000000005e-05 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:171255 50.8 50:15c2956d:::benchmark_data_smithi130_398002_object26189:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1454)", 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:19:25.050185+0000", 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 188.55792616599999, 2024-08-17T18:22:34.500 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4779792079999998, 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 171255 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.050185+0000", 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.501 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.050185+0000", 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.050183+0000", 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.050190+0000", 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.9450000000000004e-06 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.502 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.050207+0000", 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.6736999999999999e-05 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.050213+0000", 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0109999999999997e-06 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.050468+0000", 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00025471400000000001 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.503 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.050478+0000", 2024-08-17T18:22:34.504 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0129000000000001e-05 2024-08-17T18:22:34.504 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.504 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.504 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.504 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.022485+0000", 2024-08-17T18:22:34.504 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97200712199999995 2024-08-17T18:22:34.504 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.504 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.504 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.504 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.022491+0000", 2024-08-17T18:22:34.504 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.0909999999999998e-06 2024-08-17T18:22:34.504 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.504 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.504 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.490497+0000", 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4680065170000001 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.490633+0000", 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000135947 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.491116+0000", 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000482437 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.505 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.499245+0000", 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0081290649999999996 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.505370+0000", 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0061254229999999996 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.526572+0000", 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.021201587000000001 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:34.506 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.526630+0000", 2024-08-17T18:22:34.507 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8032999999999998e-05 2024-08-17T18:22:34.507 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.507 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.507 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.507 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.528164+0000", 2024-08-17T18:22:34.507 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0015342039999999999 2024-08-17T18:22:34.507 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.507 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.507 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.507 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.507 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.507 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:171262 50.8 50:146de0d6:::benchmark_data_smithi130_398002_object26196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1454)", 2024-08-17T18:22:34.507 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:19:25.060622+0000", 2024-08-17T18:22:34.508 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 188.54748899399999, 2024-08-17T18:22:34.508 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.4689575029999999, 2024-08-17T18:22:34.508 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.508 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:34.508 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.508 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.508 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.508 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 171262 2024-08-17T18:22:34.508 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.508 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.508 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.508 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.508 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.060622+0000", 2024-08-17T18:22:34.508 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.509 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.509 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.509 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.509 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.060622+0000", 2024-08-17T18:22:34.509 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.509 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.509 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.509 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.509 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.060621+0000", 2024-08-17T18:22:34.509 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:22:34.509 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.509 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.509 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.509 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.060626+0000", 2024-08-17T18:22:34.510 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4739999999999999e-06 2024-08-17T18:22:34.510 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.510 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.510 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.510 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.060635+0000", 2024-08-17T18:22:34.510 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.329e-06 2024-08-17T18:22:34.510 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.510 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.510 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.510 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.060639+0000", 2024-08-17T18:22:34.510 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.1459999999999998e-06 2024-08-17T18:22:34.510 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.511 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.511 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.511 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.060676+0000", 2024-08-17T18:22:34.511 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6545000000000003e-05 2024-08-17T18:22:34.511 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.511 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.511 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.511 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:25.060684+0000", 2024-08-17T18:22:34.511 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.7319999999999998e-06 2024-08-17T18:22:34.511 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.511 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.511 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.511 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.022511+0000", 2024-08-17T18:22:34.512 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.96182749099999998 2024-08-17T18:22:34.512 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.512 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.512 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.512 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.022517+0000", 2024-08-17T18:22:34.512 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8259999999999998e-06 2024-08-17T18:22:34.512 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.512 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.512 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.512 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.493274+0000", 2024-08-17T18:22:34.512 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4707574800000001 2024-08-17T18:22:34.513 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.513 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.513 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.513 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.493409+0000", 2024-08-17T18:22:34.513 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013458599999999999 2024-08-17T18:22:34.513 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.513 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.513 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:22:34.513 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.493834+0000", 2024-08-17T18:22:34.513 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00042470000000000002 2024-08-17T18:22:34.513 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.514 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.514 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.514 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.501344+0000", 2024-08-17T18:22:34.514 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0075098480000000004 2024-08-17T18:22:34.514 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.514 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.514 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.514 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.510148+0000", 2024-08-17T18:22:34.514 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0088041260000000007 2024-08-17T18:22:34.514 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.514 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.514 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.514 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.528212+0000", 2024-08-17T18:22:34.514 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.018064225 2024-08-17T18:22:34.515 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.515 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.515 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:34.515 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.528246+0000", 2024-08-17T18:22:34.515 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.4375000000000002e-05 2024-08-17T18:22:34.515 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.515 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.515 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.515 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.529580+0000", 2024-08-17T18:22:34.515 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0013334 2024-08-17T18:22:34.515 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.515 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.515 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.515 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.516 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.516 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:171276 50.18 50:1ee4b8e5:::benchmark_data_smithi130_398002_object26210:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1455)", 2024-08-17T18:22:34.516 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:19:26.140621+0000", 2024-08-17T18:22:34.516 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 187.46748978700001, 2024-08-17T18:22:34.516 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4048708750000001, 2024-08-17T18:22:34.516 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.516 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:34.516 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.516 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.516 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.516 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 171276 2024-08-17T18:22:34.516 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.516 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.516 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.140621+0000", 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.140621+0000", 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.140620+0000", 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.517 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.140626+0000", 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5200000000000003e-06 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.140640+0000", 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3485999999999999e-05 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.140647+0000", 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.3950000000000003e-06 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.518 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.519 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.519 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.140684+0000", 2024-08-17T18:22:34.519 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6470999999999997e-05 2024-08-17T18:22:34.519 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.519 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.519 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.519 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:26.140690+0000", 2024-08-17T18:22:34.519 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5540000000000003e-06 2024-08-17T18:22:34.519 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.519 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.519 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.519 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.503728+0000", 2024-08-17T18:22:34.519 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3630378249999999 2024-08-17T18:22:34.519 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.503861+0000", 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000132626 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2", 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.504287+0000", 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00042654100000000001 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.510103+0000", 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0058154590000000003 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.520 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.524167+0000", 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.014064465 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.543488+0000", 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.019321074000000001 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.543533+0000", 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4975999999999998e-05 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.521 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:19:27.545492+0000", 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0019590150000000001 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:223644 50.8 50:140ac24e:::benchmark_data_smithi130_398002_object78578:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1463)", 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:20:28.716871+0000", 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 124.891239873, 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.011436757, 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:34.522 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 223644 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.716871+0000", 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.716871+0000", 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.523 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.524 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.524 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.524 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.716870+0000", 2024-08-17T18:22:34.524 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:34.524 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.524 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.524 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.524 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.716877+0000", 2024-08-17T18:22:34.524 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.793e-06 2024-08-17T18:22:34.524 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.524 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.524 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.524 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.716888+0000", 2024-08-17T18:22:34.524 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1222e-05 2024-08-17T18:22:34.525 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.525 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.525 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.525 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.716893+0000", 2024-08-17T18:22:34.525 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1019999999999996e-06 2024-08-17T18:22:34.525 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.525 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.525 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.525 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.716936+0000", 2024-08-17T18:22:34.525 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.2861999999999997e-05 2024-08-17T18:22:34.525 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.525 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.525 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.525 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.716944+0000", 2024-08-17T18:22:34.526 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.3610000000000001e-06 2024-08-17T18:22:34.526 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.526 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.526 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.526 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.669583+0000", 2024-08-17T18:22:34.526 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.95263842799999998 2024-08-17T18:22:34.526 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.526 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.526 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.526 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.669715+0000", 2024-08-17T18:22:34.526 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000132578 2024-08-17T18:22:34.527 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.527 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.527 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T18:22:34.527 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.670309+0000", 2024-08-17T18:22:34.527 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00059361099999999997 2024-08-17T18:22:34.527 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.527 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.527 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.527 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.677092+0000", 2024-08-17T18:22:34.527 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.006782794 2024-08-17T18:22:34.527 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.527 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.527 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.528 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.677145+0000", 2024-08-17T18:22:34.528 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3689e-05 2024-08-17T18:22:34.528 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.528 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.528 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.528 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.680544+0000", 2024-08-17T18:22:34.528 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0033985310000000002 2024-08-17T18:22:34.528 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.528 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.528 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.528 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.726446+0000", 2024-08-17T18:22:34.528 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.045902500999999998 2024-08-17T18:22:34.528 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.528 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.726510+0000", 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3220999999999997e-05 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.728308+0000", 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0017984050000000001 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:223647 50.19 50:9ed39062:::benchmark_data_smithi130_398002_object78581:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1463)", 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:20:28.733150+0000", 2024-08-17T18:22:34.529 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 124.874961309, 2024-08-17T18:22:34.530 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.080769608, 2024-08-17T18:22:34.530 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.530 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:34.530 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.530 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.530 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.530 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 223647 2024-08-17T18:22:34.530 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.530 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.530 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.530 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.530 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.733150+0000", 2024-08-17T18:22:34.530 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.530 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.733150+0000", 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.733144+0000", 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999943 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.733158+0000", 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4018e-05 2024-08-17T18:22:34.531 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.733175+0000", 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.6722999999999999e-05 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.733183+0000", 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9999999999999996e-06 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.733242+0000", 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8786999999999998e-05 2024-08-17T18:22:34.532 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.733252+0000", 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0447e-05 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.758502+0000", 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0252495800000001 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.758637+0000", 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000134707 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.533 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.534 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1,7", 2024-08-17T18:22:34.534 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.758964+0000", 2024-08-17T18:22:34.534 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000327039 2024-08-17T18:22:34.534 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.534 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.534 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.534 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.765737+0000", 2024-08-17T18:22:34.534 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0067734290000000001 2024-08-17T18:22:34.534 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.534 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.534 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.534 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.765842+0000", 2024-08-17T18:22:34.534 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000105096 2024-08-17T18:22:34.534 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.793947+0000", 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.028105221 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.813772+0000", 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.019824424 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.813823+0000", 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1594000000000001e-05 2024-08-17T18:22:34.535 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.813919+0000", 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.6101999999999996e-05 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:223658 50.8 50:178275fa:::benchmark_data_smithi130_398002_object78592:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1463)", 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:20:28.757948+0000", 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 124.85016328899999, 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97213627499999999, 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.536 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 223658 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.757948+0000", 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.757948+0000", 2024-08-17T18:22:34.537 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.538 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.538 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.538 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.538 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.757946+0000", 2024-08-17T18:22:34.538 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:34.538 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.538 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.538 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.538 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.757953+0000", 2024-08-17T18:22:34.538 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.1559999999999998e-06 2024-08-17T18:22:34.538 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.538 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.757964+0000", 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0624e-05 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.757969+0000", 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.7190000000000001e-06 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.758030+0000", 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1285999999999996e-05 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.539 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.758040+0000", 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0348e-05 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.672673+0000", 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.91463213799999998 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.672758+0000", 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.5495999999999996e-05 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.540 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.673089+0000", 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00033113000000000001 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.677155+0000", 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0040661580000000003 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.678803+0000", 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0016480609999999999 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.541 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.542 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.680573+0000", 2024-08-17T18:22:34.542 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00176973 2024-08-17T18:22:34.542 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.542 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.542 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.542 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.728458+0000", 2024-08-17T18:22:34.542 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.047884692999999999 2024-08-17T18:22:34.542 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.542 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.542 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:34.542 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.728506+0000", 2024-08-17T18:22:34.542 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8615999999999997e-05 2024-08-17T18:22:34.542 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.542 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.543 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.543 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.730084+0000", 2024-08-17T18:22:34.543 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0015776880000000001 2024-08-17T18:22:34.543 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.543 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.543 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.543 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.543 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.543 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:223660 50.c 50:3ae53d5c:::benchmark_data_smithi130_398002_object78594:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1463)", 2024-08-17T18:22:34.543 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:20:28.758634+0000", 2024-08-17T18:22:34.544 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 124.84947730899999, 2024-08-17T18:22:34.544 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.90584708000000003, 2024-08-17T18:22:34.544 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.544 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:34.544 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.544 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.544 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.544 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 223660 2024-08-17T18:22:34.544 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.544 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.544 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.544 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.545 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.758634+0000", 2024-08-17T18:22:34.545 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.545 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.545 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.545 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.545 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.758634+0000", 2024-08-17T18:22:34.545 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.545 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.545 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.545 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.545 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.758633+0000", 2024-08-17T18:22:34.545 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:22:34.545 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.545 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.546 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.546 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.758639+0000", 2024-08-17T18:22:34.546 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7270000000000004e-06 2024-08-17T18:22:34.546 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.546 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.546 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.546 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.758653+0000", 2024-08-17T18:22:34.546 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.4609e-05 2024-08-17T18:22:34.546 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.546 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.546 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.546 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.758659+0000", 2024-08-17T18:22:34.547 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9370000000000001e-06 2024-08-17T18:22:34.547 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.547 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.547 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.547 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.758706+0000", 2024-08-17T18:22:34.547 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6474999999999998e-05 2024-08-17T18:22:34.547 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.547 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.547 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.547 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.758714+0000", 2024-08-17T18:22:34.547 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6249999999999996e-06 2024-08-17T18:22:34.547 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.547 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.548 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.548 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.655866+0000", 2024-08-17T18:22:34.548 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.89715191100000002 2024-08-17T18:22:34.548 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.548 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.548 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.548 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.655977+0000", 2024-08-17T18:22:34.548 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000111224 2024-08-17T18:22:34.548 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.548 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.548 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1,2", 2024-08-17T18:22:34.548 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.656107+0000", 2024-08-17T18:22:34.548 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000129438 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.664189+0000", 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0080819110000000006 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.664232+0000", 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3438e-05 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.664285+0000", 2024-08-17T18:22:34.549 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.2605000000000002e-05 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.664296+0000", 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0723000000000001e-05 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.664352+0000", 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6481999999999999e-05 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.664481+0000", 2024-08-17T18:22:34.550 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012891500000000001 2024-08-17T18:22:34.551 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.551 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.551 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.551 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.551 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.551 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:223661 50.c 50:3a2b870b:::benchmark_data_smithi130_398002_object78595:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1463)", 2024-08-17T18:22:34.551 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:20:28.759636+0000", 2024-08-17T18:22:34.551 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 124.84847469499999, 2024-08-17T18:22:34.551 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.905328099, 2024-08-17T18:22:34.551 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.551 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:34.551 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.551 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.552 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.552 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 223661 2024-08-17T18:22:34.552 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.552 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.552 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.552 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.552 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.759636+0000", 2024-08-17T18:22:34.552 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.552 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.552 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.552 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.552 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.759636+0000", 2024-08-17T18:22:34.552 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.552 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.553 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.553 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.553 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.759635+0000", 2024-08-17T18:22:34.553 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:22:34.553 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.553 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.553 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.553 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.759641+0000", 2024-08-17T18:22:34.553 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5659999999999997e-06 2024-08-17T18:22:34.553 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.553 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.553 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.553 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.759657+0000", 2024-08-17T18:22:34.553 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5531000000000001e-05 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.759663+0000", 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7780000000000002e-06 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.759707+0000", 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.3420000000000001e-05 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.759715+0000", 2024-08-17T18:22:34.554 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.8720000000000002e-06 2024-08-17T18:22:34.555 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.555 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.555 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.555 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.660334+0000", 2024-08-17T18:22:34.555 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.90061898900000004 2024-08-17T18:22:34.555 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.555 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.555 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.555 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.660504+0000", 2024-08-17T18:22:34.555 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017036699999999999 2024-08-17T18:22:34.555 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.555 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.555 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1,2", 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.660690+0000", 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00018630299999999999 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.664694+0000", 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0040037290000000001 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.664703+0000", 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 9.4029999999999996e-06 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.556 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.664754+0000", 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.0349000000000002e-05 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.664799+0000", 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.4916000000000002e-05 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.664850+0000", 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1252999999999999e-05 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.557 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.664965+0000", 2024-08-17T18:22:34.558 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000114608 2024-08-17T18:22:34.558 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.558 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.558 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.558 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.558 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.558 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:223668 50.19 50:9a3212e3:::benchmark_data_smithi130_398002_object78602:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1463)", 2024-08-17T18:22:34.558 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:20:28.780451+0000", 2024-08-17T18:22:34.558 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 124.827660233, 2024-08-17T18:22:34.558 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0368722459999999, 2024-08-17T18:22:34.558 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.558 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:34.558 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.559 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.559 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.559 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 223668 2024-08-17T18:22:34.559 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.559 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.559 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.559 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.559 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.780451+0000", 2024-08-17T18:22:34.559 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.559 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.559 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.559 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.559 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.780451+0000", 2024-08-17T18:22:34.559 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.560 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.560 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.560 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.560 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.780449+0000", 2024-08-17T18:22:34.560 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:34.560 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.560 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.560 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.560 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.780456+0000", 2024-08-17T18:22:34.560 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.7700000000000004e-06 2024-08-17T18:22:34.560 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.560 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.561 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.561 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.780468+0000", 2024-08-17T18:22:34.561 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2014000000000001e-05 2024-08-17T18:22:34.561 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.561 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.561 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.561 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.780473+0000", 2024-08-17T18:22:34.561 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3870000000000003e-06 2024-08-17T18:22:34.561 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.561 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.561 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.561 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.780525+0000", 2024-08-17T18:22:34.561 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1320000000000003e-05 2024-08-17T18:22:34.562 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.562 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.562 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.562 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.780532+0000", 2024-08-17T18:22:34.562 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.832e-06 2024-08-17T18:22:34.562 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.562 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.562 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.562 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.760212+0000", 2024-08-17T18:22:34.562 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.97968090500000005 2024-08-17T18:22:34.562 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.562 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.562 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.562 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.760326+0000", 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000113397 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1,7", 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.760685+0000", 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00035952 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.765794+0000", 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0051083980000000001 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.765873+0000", 2024-08-17T18:22:34.563 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.9085999999999995e-05 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.793968+0000", 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.028094798000000001 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.817170+0000", 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.023202698000000001 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.817222+0000", 2024-08-17T18:22:34.564 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.1384999999999997e-05 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.817323+0000", 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000101368 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:223677 50.8 50:17dda4b6:::benchmark_data_smithi130_398002_object78611:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1463)", 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:20:28.802870+0000", 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 124.805240955, 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.93191995800000005, 2024-08-17T18:22:34.565 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.566 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:34.566 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.566 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.566 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.566 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 223677 2024-08-17T18:22:34.566 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.566 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.566 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.566 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.566 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.802870+0000", 2024-08-17T18:22:34.566 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.566 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.566 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.566 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.802870+0000", 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.802869+0000", 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.802875+0000", 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.5610000000000004e-06 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.567 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.802886+0000", 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.116e-05 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.802892+0000", 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.7690000000000002e-06 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.802951+0000", 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9247999999999999e-05 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.568 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.569 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.802964+0000", 2024-08-17T18:22:34.569 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2225000000000001e-05 2024-08-17T18:22:34.569 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.569 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.569 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.569 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.674953+0000", 2024-08-17T18:22:34.569 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.87198962800000002 2024-08-17T18:22:34.569 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.569 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.569 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.569 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.675039+0000", 2024-08-17T18:22:34.569 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.6038000000000005e-05 2024-08-17T18:22:34.569 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.569 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.570 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,2,3", 2024-08-17T18:22:34.570 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.675367+0000", 2024-08-17T18:22:34.570 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00032768700000000001 2024-08-17T18:22:34.570 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.570 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.570 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.570 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.677163+0000", 2024-08-17T18:22:34.570 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001796387 2024-08-17T18:22:34.570 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.570 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.570 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.570 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.678929+0000", 2024-08-17T18:22:34.570 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0017660169999999999 2024-08-17T18:22:34.570 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.571 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.571 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.571 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.680832+0000", 2024-08-17T18:22:34.571 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001902883 2024-08-17T18:22:34.571 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.571 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.571 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.571 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.733056+0000", 2024-08-17T18:22:34.571 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.052223811000000002 2024-08-17T18:22:34.571 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.571 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.571 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:34.571 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.733116+0000", 2024-08-17T18:22:34.572 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.9753999999999997e-05 2024-08-17T18:22:34.572 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.572 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.572 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.572 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.734790+0000", 2024-08-17T18:22:34.572 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0016743369999999999 2024-08-17T18:22:34.572 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.572 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.572 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.572 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.572 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.572 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:223678 50.19 50:9d1c4a91:::benchmark_data_smithi130_398002_object78612:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1463)", 2024-08-17T18:22:34.572 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:20:28.806237+0000", 2024-08-17T18:22:34.572 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 124.801874224, 2024-08-17T18:22:34.573 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.016635255, 2024-08-17T18:22:34.573 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.573 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:34.573 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.573 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.573 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.573 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 223678 2024-08-17T18:22:34.573 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.573 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.573 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.573 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.573 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.806237+0000", 2024-08-17T18:22:34.573 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.573 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.806237+0000", 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.806236+0000", 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.806241+0000", 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.113e-06 2024-08-17T18:22:34.574 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.575 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.575 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.575 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.806252+0000", 2024-08-17T18:22:34.575 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0647e-05 2024-08-17T18:22:34.575 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.575 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.575 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.575 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.806258+0000", 2024-08-17T18:22:34.575 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.3119999999999999e-06 2024-08-17T18:22:34.575 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.575 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.575 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.575 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.806304+0000", 2024-08-17T18:22:34.575 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.6279000000000002e-05 2024-08-17T18:22:34.576 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.576 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.576 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.576 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:28.806312+0000", 2024-08-17T18:22:34.576 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.2969999999999998e-06 2024-08-17T18:22:34.576 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.576 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.576 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.576 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.762190+0000", 2024-08-17T18:22:34.576 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.95587829300000005 2024-08-17T18:22:34.576 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.576 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.576 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.576 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.762313+0000", 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012268199999999999 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1,7", 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.762626+0000", 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00031311199999999999 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.765801+0000", 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0031755049999999999 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.577 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.766174+0000", 2024-08-17T18:22:34.578 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000372524 2024-08-17T18:22:34.578 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.578 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.578 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.578 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.794001+0000", 2024-08-17T18:22:34.578 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.027827584999999998 2024-08-17T18:22:34.578 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.578 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.578 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.578 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.822589+0000", 2024-08-17T18:22:34.578 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.028587956000000001 2024-08-17T18:22:34.578 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.822672+0000", 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2847000000000001e-05 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:20:29.822872+0000", 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000200101 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.579 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:257738 50.e 50:7ce47edc:::benchmark_data_smithi130_398002_object112672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1488)", 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:32.376073+0000", 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 61.232038510000002, 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.0862740209999999, 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "reached pg", 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 257738 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:32.376073+0000", 2024-08-17T18:22:34.580 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:32.376073+0000", 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:32.376071+0000", 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999986 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:32.376077+0000", 2024-08-17T18:22:34.581 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.1600000000000003e-06 2024-08-17T18:22:34.582 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.582 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.582 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.582 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:32.376090+0000", 2024-08-17T18:22:34.582 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.2547e-05 2024-08-17T18:22:34.582 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.582 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.582 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.582 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:32.376095+0000", 2024-08-17T18:22:34.582 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.8199999999999996e-06 2024-08-17T18:22:34.582 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.582 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.582 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.583 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:32.376146+0000", 2024-08-17T18:22:34.583 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.113e-05 2024-08-17T18:22:34.583 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.583 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.583 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.583 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:32.376297+0000", 2024-08-17T18:22:34.583 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00015078800000000001 2024-08-17T18:22:34.583 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.583 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.583 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1,2", 2024-08-17T18:22:34.583 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:32.376447+0000", 2024-08-17T18:22:34.583 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00015028900000000001 2024-08-17T18:22:34.583 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.583 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.584 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.584 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:32.379760+0000", 2024-08-17T18:22:34.584 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003312789 2024-08-17T18:22:34.584 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.584 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.584 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.584 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:32.380312+0000", 2024-08-17T18:22:34.584 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000552243 2024-08-17T18:22:34.584 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.584 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.584 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.584 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:32.381580+0000", 2024-08-17T18:22:34.584 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.001268546 2024-08-17T18:22:34.584 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:32.392902+0000", 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.011321678999999999 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:32.393318+0000", 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000416012 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.462293+0000", 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.068974622 2024-08-17T18:22:34.585 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.462347+0000", 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3844000000000002e-05 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:257741 50.c 50:3e0f4230:::benchmark_data_smithi130_398002_object112675:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1490)", 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:21:33.475425+0000", 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 60.132685785, 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.98367757899999997, 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.586 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 257741 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.475425+0000", 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.475425+0000", 2024-08-17T18:22:34.587 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.588 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.588 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.588 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.588 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.475424+0000", 2024-08-17T18:22:34.588 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:22:34.588 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.588 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.588 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.588 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.475431+0000", 2024-08-17T18:22:34.588 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 7.2429999999999999e-06 2024-08-17T18:22:34.588 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.588 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.588 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.588 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.475447+0000", 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.5693e-05 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.475452+0000", 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.5770000000000001e-06 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.475498+0000", 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.5401000000000001e-05 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:33.475503+0000", 2024-08-17T18:22:34.589 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.8379999999999998e-06 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.439590+0000", 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.96408641900000003 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.439721+0000", 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00013139400000000001 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,1", 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.439856+0000", 2024-08-17T18:22:34.590 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000134392 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.444074+0000", 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0042180560000000004 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.445850+0000", 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0017762500000000001 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.459039+0000", 2024-08-17T18:22:34.591 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.013188617999999999 2024-08-17T18:22:34.592 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.592 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.592 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:34.592 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.459075+0000", 2024-08-17T18:22:34.592 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.6687999999999998e-05 2024-08-17T18:22:34.592 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.592 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.592 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.592 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:21:34.459103+0000", 2024-08-17T18:22:34.592 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 2.7696e-05 2024-08-17T18:22:34.592 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.592 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.592 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.592 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.593 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.593 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:276054 50.9 50:9507d341:::benchmark_data_smithi130_398002_object130988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1501)", 2024-08-17T18:22:34.593 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:22:00.308235+0000", 2024-08-17T18:22:34.593 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 33.299876454, 2024-08-17T18:22:34.593 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.87119294000000003, 2024-08-17T18:22:34.593 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.593 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:34.593 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.593 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.593 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.593 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 276054 2024-08-17T18:22:34.593 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.593 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.593 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.594 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.594 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.308235+0000", 2024-08-17T18:22:34.594 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.594 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.594 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.594 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.594 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.308235+0000", 2024-08-17T18:22:34.594 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.594 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.594 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.594 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.594 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.308233+0000", 2024-08-17T18:22:34.594 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.9999981 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.308241+0000", 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.1259999999999998e-06 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.308255+0000", 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.3536000000000001e-05 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.308261+0000", 2024-08-17T18:22:34.595 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.567e-06 2024-08-17T18:22:34.596 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.596 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.596 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.596 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.308314+0000", 2024-08-17T18:22:34.596 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3230999999999999e-05 2024-08-17T18:22:34.596 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.596 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.596 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.596 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.308323+0000", 2024-08-17T18:22:34.596 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.2949999999999994e-06 2024-08-17T18:22:34.596 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.596 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.596 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.596 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.170537+0000", 2024-08-17T18:22:34.597 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.86221447500000004 2024-08-17T18:22:34.597 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.597 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.597 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.597 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.170663+0000", 2024-08-17T18:22:34.597 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00012597699999999999 2024-08-17T18:22:34.597 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.597 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.597 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,7", 2024-08-17T18:22:34.597 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.171169+0000", 2024-08-17T18:22:34.597 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00050560000000000004 2024-08-17T18:22:34.597 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.597 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.598 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.598 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.179218+0000", 2024-08-17T18:22:34.598 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0080491610000000009 2024-08-17T18:22:34.598 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.598 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.598 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.598 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.179280+0000", 2024-08-17T18:22:34.598 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.2077999999999998e-05 2024-08-17T18:22:34.598 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.598 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.598 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.598 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.179319+0000", 2024-08-17T18:22:34.598 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.9549999999999999e-05 2024-08-17T18:22:34.599 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.599 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.599 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:34.599 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.179389+0000", 2024-08-17T18:22:34.599 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 6.9587999999999998e-05 2024-08-17T18:22:34.599 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.599 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.599 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.599 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.179428+0000", 2024-08-17T18:22:34.599 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 3.8559000000000001e-05 2024-08-17T18:22:34.599 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.599 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.599 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.599 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.600 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.600 INFO:teuthology.orchestra.run.smithi179.stdout: "description": "osd_op(client.47132.0:276063 50.9 50:90be8476:::benchmark_data_smithi130_398002_object130997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1501)", 2024-08-17T18:22:34.600 INFO:teuthology.orchestra.run.smithi179.stdout: "initiated_at": "2024-08-17T18:22:00.325265+0000", 2024-08-17T18:22:34.600 INFO:teuthology.orchestra.run.smithi179.stdout: "age": 33.282846061999997, 2024-08-17T18:22:34.600 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.86186045099999997, 2024-08-17T18:22:34.600 INFO:teuthology.orchestra.run.smithi179.stdout: "type_data": { 2024-08-17T18:22:34.600 INFO:teuthology.orchestra.run.smithi179.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T18:22:34.600 INFO:teuthology.orchestra.run.smithi179.stdout: "client_info": { 2024-08-17T18:22:34.600 INFO:teuthology.orchestra.run.smithi179.stdout: "client": "client.47132", 2024-08-17T18:22:34.600 INFO:teuthology.orchestra.run.smithi179.stdout: "client_addr": "172.21.15.130:0/1050487535", 2024-08-17T18:22:34.600 INFO:teuthology.orchestra.run.smithi179.stdout: "tid": 276063 2024-08-17T18:22:34.600 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.600 INFO:teuthology.orchestra.run.smithi179.stdout: "events": [ 2024-08-17T18:22:34.600 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.601 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "initiated", 2024-08-17T18:22:34.601 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.325265+0000", 2024-08-17T18:22:34.601 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.601 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.601 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.601 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "throttled", 2024-08-17T18:22:34.601 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.325265+0000", 2024-08-17T18:22:34.601 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0 2024-08-17T18:22:34.601 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.601 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.601 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "header_read", 2024-08-17T18:22:34.601 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.325264+0000", 2024-08-17T18:22:34.601 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4294967295.999999 2024-08-17T18:22:34.601 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.602 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.602 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "all_read", 2024-08-17T18:22:34.602 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.325269+0000", 2024-08-17T18:22:34.602 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.502e-06 2024-08-17T18:22:34.602 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.602 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.602 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "dispatched", 2024-08-17T18:22:34.602 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.325280+0000", 2024-08-17T18:22:34.602 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 1.1038e-05 2024-08-17T18:22:34.602 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.602 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.602 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "queued_for_pg", 2024-08-17T18:22:34.602 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.325286+0000", 2024-08-17T18:22:34.602 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.6899999999999997e-06 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.325459+0000", 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.00017238 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for peered", 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:00.325467+0000", 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 8.9199999999999993e-06 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "reached_pg", 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.173761+0000", 2024-08-17T18:22:34.603 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.84829309799999997 2024-08-17T18:22:34.604 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.604 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.604 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "started", 2024-08-17T18:22:34.604 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.173875+0000", 2024-08-17T18:22:34.604 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000114194 2024-08-17T18:22:34.604 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.604 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.604 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "waiting for subops from 0,7", 2024-08-17T18:22:34.604 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.174237+0000", 2024-08-17T18:22:34.604 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.000361775 2024-08-17T18:22:34.604 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.604 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.604 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.604 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.179479+0000", 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0052423010000000004 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "sub_op_commit_rec", 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.183234+0000", 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.003754987 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "op_commit", 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.187031+0000", 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 0.0037971120000000001 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "commit_sent", 2024-08-17T18:22:34.605 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.187085+0000", 2024-08-17T18:22:34.606 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 5.3736999999999997e-05 2024-08-17T18:22:34.606 INFO:teuthology.orchestra.run.smithi179.stdout: }, 2024-08-17T18:22:34.606 INFO:teuthology.orchestra.run.smithi179.stdout: { 2024-08-17T18:22:34.606 INFO:teuthology.orchestra.run.smithi179.stdout: "event": "done", 2024-08-17T18:22:34.606 INFO:teuthology.orchestra.run.smithi179.stdout: "time": "2024-08-17T18:22:01.187126+0000", 2024-08-17T18:22:34.606 INFO:teuthology.orchestra.run.smithi179.stdout: "duration": 4.0877999999999997e-05 2024-08-17T18:22:34.606 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.606 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.606 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.606 INFO:teuthology.orchestra.run.smithi179.stdout: } 2024-08-17T18:22:34.606 INFO:teuthology.orchestra.run.smithi179.stdout: ] 2024-08-17T18:22:34.606 INFO:teuthology.orchestra.run.smithi179.stdout:} 2024-08-17T18:22:34.607 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:34.607 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:34 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T18:22:34.608 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:34 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-08-17T18:22:34.608 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:34 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T18:22:34.608 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:34 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-08-17T18:22:34.628 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:34.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:34.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:34.730 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:34.752 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:34.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:34.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:34.853 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:34.875 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:34.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:34.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:34.977 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:34.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:35.000 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:35.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:35.101 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:35.123 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:35.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:35.225 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:35.248 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:35.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:35.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:35.333 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T18:22:35.350 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:35.380 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:35.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:35.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:35.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:35 smithi130 ceph-mon[110068]: pgmap v2954: 99 pgs: 99 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:35 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T18:22:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:35 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2024-08-17T18:22:35.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:35 smithi130 ceph-mon[110068]: osdmap e1516: 8 total, 8 up, 6 in 2024-08-17T18:22:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:35 smithi130 ceph-mon[105521]: pgmap v2954: 99 pgs: 99 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:35 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T18:22:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:35 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2024-08-17T18:22:35.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:35 smithi130 ceph-mon[105521]: osdmap e1516: 8 total, 8 up, 6 in 2024-08-17T18:22:35.482 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:35.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:35.504 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:35.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:35 smithi179 ceph-mon[93519]: pgmap v2954: 99 pgs: 99 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:35 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T18:22:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:35 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2024-08-17T18:22:35.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:35 smithi179 ceph-mon[93519]: osdmap e1516: 8 total, 8 up, 6 in 2024-08-17T18:22:35.605 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:35.627 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:35.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:35.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:35.630 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T18:22:35.729 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:35.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:35.751 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:35.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:35.852 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:35.874 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:35.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:35.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:35.976 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:35.997 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:35.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:35.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:36.099 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:36.122 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:36.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:36.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:36.151 INFO:teuthology.orchestra.run.smithi130.stderr:pool 'unique_pool_5' removed 2024-08-17T18:22:36.176 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:22:36.225 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:36.252 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:36.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:36.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:36.272 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 7, 6, 4, 0] out_osds: [5, 1] dead_osds: [] live_osds: [1, 4, 5, 6, 7, 0, 3, 2] 2024-08-17T18:22:36.272 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T18:22:36.354 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:36.375 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:36.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:36.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:36.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:36 smithi130 ceph-mon[110068]: osdmap e1517: 8 total, 8 up, 6 in 2024-08-17T18:22:36.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:36 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3826315282' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:22:36.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:36 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3826315282' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:22:36.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:36 smithi130 ceph-mon[110068]: osdmap e1518: 8 total, 8 up, 6 in 2024-08-17T18:22:36.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:36 smithi130 ceph-mon[105521]: osdmap e1517: 8 total, 8 up, 6 in 2024-08-17T18:22:36.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:36 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3826315282' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:22:36.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:36 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3826315282' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:22:36.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:36 smithi130 ceph-mon[105521]: osdmap e1518: 8 total, 8 up, 6 in 2024-08-17T18:22:36.477 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:36.498 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:36.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:36.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:36 smithi179 ceph-mon[93519]: osdmap e1517: 8 total, 8 up, 6 in 2024-08-17T18:22:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:36 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3826315282' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T18:22:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:36 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3826315282' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T18:22:36.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:36 smithi179 ceph-mon[93519]: osdmap e1518: 8 total, 8 up, 6 in 2024-08-17T18:22:36.600 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:36.625 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:36.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:36.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:36.727 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:36.747 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:36.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:36.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:36.849 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:36.871 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:36.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:36.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:36.973 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:36.994 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:36.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:36.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:37.096 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:37.118 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:37.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:37.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:37.219 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:37.241 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:37.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:37.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:37.267 INFO:teuthology.task.sequential:In sequential, running task print... 2024-08-17T18:22:37.267 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-08-17T18:22:37.267 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-08-17T18:22:37.268 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-08-17T18:22:37.268 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-08-17T18:22:37.269 DEBUG:teuthology.orchestra.run.smithi130:> stat -- /home/ubuntu/cephtest/mnt.0 2024-08-17T18:22:37.269 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-08-17T18:22:37.269 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:22:37.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:37.342 INFO:teuthology.orchestra.run.smithi130.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-08-17T18:22:37.342 DEBUG:teuthology.orchestra.run.smithi130:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-08-17T18:22:37.342 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:37.363 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:37.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:37.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:37.429 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-08-17T18:22:37.429 DEBUG:teuthology.orchestra.run.smithi130:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-08-17T18:22:37.445 INFO:tasks.workunit:timeout=3h 2024-08-17T18:22:37.445 INFO:tasks.workunit:cleanup=True 2024-08-17T18:22:37.446 DEBUG:teuthology.orchestra.run.smithi130:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-08-17T18:22:37.464 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:37.485 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:37.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:37.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:37.508 INFO:tasks.workunit.client.0.smithi130.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-08-17T18:22:37.587 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:37.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:37 smithi179 ceph-mon[93519]: pgmap v2957: 109 pgs: 10 unknown, 99 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:37.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:37 smithi179 ceph-mon[93519]: 50.0 scrub starts 2024-08-17T18:22:37.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:37 smithi179 ceph-mon[93519]: 50.1 scrub starts 2024-08-17T18:22:37.602 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:37 smithi130 ceph-mon[105521]: pgmap v2957: 109 pgs: 10 unknown, 99 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:37.602 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:37 smithi130 ceph-mon[105521]: 50.0 scrub starts 2024-08-17T18:22:37.602 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:37 smithi130 ceph-mon[105521]: 50.1 scrub starts 2024-08-17T18:22:37.602 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:37 smithi130 ceph-mon[110068]: pgmap v2957: 109 pgs: 10 unknown, 99 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:37.602 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:37 smithi130 ceph-mon[110068]: 50.0 scrub starts 2024-08-17T18:22:37.602 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:37 smithi130 ceph-mon[110068]: 50.1 scrub starts 2024-08-17T18:22:37.615 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:37.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:37.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:37.717 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:37.738 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:37.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:37.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:37.840 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:37.861 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:37.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:37.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:37.963 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:37.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:37.985 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:37.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:38.086 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:38.150 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:22:38.150 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:22:38.150 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:22:38.150 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:22:38.150 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:22:38.150 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:22:38.150 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:22:38.151 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:22:38.151 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:22:38.151 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:22:38.151 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:22:38.151 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:22:38.153 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:22:38.153 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:22:38.160 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:38.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:38.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:38.172 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:22:38.172 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:22:38.262 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:38.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:38.287 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:38.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:38.388 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:38.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:38.411 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:38.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:38.511 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:38.532 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:38.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:38.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:38.634 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:38.656 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:38.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:38.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:38.682 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:38.705 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:22:38.705 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:22:38.757 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:38.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:38.781 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:38.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:38.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:38 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:38.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:38 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:38.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:38 smithi130 ceph-mon[105521]: pgmap v2959: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:38.810 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:38 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1083066360' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:38.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:38 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:38.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:38 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:38.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:38 smithi130 ceph-mon[110068]: pgmap v2959: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:38.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:38 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1083066360' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:38.882 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:38.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:38.905 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:38.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:39.006 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:39.016 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:38 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:39.016 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:38 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:39.016 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:38 smithi179 ceph-mon[93519]: pgmap v2959: 73 pgs: 73 active+clean; 582 KiB data, 5.0 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:39.016 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:38 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1083066360' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:39.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:39.028 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:39.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:39.129 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:39.150 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:39.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:39.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:39.252 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:39.273 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:39.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:39.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:39.375 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:39.396 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:39.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:39.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:39.400 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T18:22:39.498 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:39.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:39.521 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:39.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:39.622 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:39.644 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:39.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:39.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:39.696 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:22:39.747 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:39.768 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:39.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:39.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:39.858 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:39.870 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:39.887 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:22:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:22:39] "GET /metrics HTTP/1.1" 200 36769 "" "Prometheus/2.51.0" 2024-08-17T18:22:39.891 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:39.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:39.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:39.993 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:40.015 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:40.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:40.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:40.116 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:40.137 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:40.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:40.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:40.239 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:40.260 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:40.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:40.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:40.362 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:40.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:40.385 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:40.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:40.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:40.508 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:40.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:40.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:22:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:22:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:22:40.610 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:40.631 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:40.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:40.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:40.733 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:40.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:40.755 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:40.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:40.856 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:40.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:40.879 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:40.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:40.981 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:41.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:41.004 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:41.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:41.106 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:41.127 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:41.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:41.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:41.229 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:41.251 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:41.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:41.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:41.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:41 smithi179 ceph-mon[93519]: pgmap v2960: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:41.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:41 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:41.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:41 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:41.353 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:41.375 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:41.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:41.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:41.378 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T18:22:41.459 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:41 smithi130 ceph-mon[105521]: pgmap v2960: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:41.459 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:41 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:41.459 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:41 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:41.459 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:41 smithi130 ceph-mon[110068]: pgmap v2960: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:41.460 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:41 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:41.460 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:41 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:41.477 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:41.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:41.499 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:41.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:41.600 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:41.621 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:41.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:41.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:41.652 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:22:41.652 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:22:41.722 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:41.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:41.745 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:41.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:41.846 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:41.870 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:41.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:41.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:41.973 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:41.994 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:41.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:41.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:42.096 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:42.118 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:42.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:42.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:42.220 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:42.241 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:42.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:42.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:42.322 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:22:42.323 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:22:42.323 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:42 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:42.343 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:42.347 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T18:22:42.367 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:42.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:42.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:42.469 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:42.491 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:42.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:42.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:42.593 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:42.602 INFO:teuthology.orchestra.run.smithi179.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T18:22:42.615 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:42.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:42.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:42.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:22:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:22:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:42 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:22:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:22:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:42 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:42.716 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:42.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:42.739 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:42.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:42.840 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:42.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:42.863 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:42.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:42.964 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:42.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:42.986 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:42.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:43.087 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:43.109 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:43.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:43.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:43.211 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:43.234 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:43.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:43.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:43.336 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:43.360 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:43.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:43.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:43.401 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is set 2024-08-17T18:22:43.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:43.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:43.485 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:43.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:43.586 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:43.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:43 smithi130 ceph-mon[105521]: pgmap v2961: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:43.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:43.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:43 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:22:43.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:43 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1915642898' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:22:43.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:43 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:22:43.590 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:43 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:43.590 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:43 smithi130 ceph-mon[110068]: pgmap v2961: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:43.590 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:43.590 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:43 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:22:43.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:43 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1915642898' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:22:43.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:43 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:22:43.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:43 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:43.607 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:43.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:43.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:43.638 DEBUG:teuthology.orchestra.run.smithi130:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-08-17T18:22:43.709 INFO:tasks.workunit.client.0.smithi130.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-08-17T18:22:43.709 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:43.711 INFO:tasks.workunit.client.0.smithi130.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-17T18:22:43.711 INFO:tasks.workunit.client.0.smithi130.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-08-17T18:22:43.731 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:43.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:43.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:43.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:43 smithi179 ceph-mon[93519]: pgmap v2961: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:43.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:43.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:43 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:22:43.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:43 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1915642898' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:22:43.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:43 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:22:43.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:43 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:22:43.833 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:43.854 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:43.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:43.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:43.956 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:43.977 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:43.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:43.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:44.024 INFO:tasks.workunit.client.0.smithi130.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-08-17T18:22:44.079 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:44.083 INFO:tasks.workunit.client.0.smithi130.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-08-17T18:22:44.101 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:44.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:44.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:44.123 INFO:tasks.workunit.client.0.smithi130.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-17T18:22:44.125 INFO:tasks.workunit.client.0.smithi130.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-17T18:22:44.125 INFO:tasks.workunit.client.0.smithi130.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-08-17T18:22:44.168 INFO:tasks.workunit.client.0.smithi130.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-17T18:22:44.192 DEBUG:teuthology.orchestra.run.smithi130:> set -ex 2024-08-17T18:22:44.192 DEBUG:teuthology.orchestra.run.smithi130:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-08-17T18:22:44.203 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:44.254 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:44.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:44.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:44.278 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-08-17T18:22:44.280 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-08-17T18:22:44.280 DEBUG:teuthology.orchestra.run.smithi130:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-08-17T18:22:44.356 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:44.375 INFO:teuthology.orchestra.run.smithi179.stdout:ERROR: (22) Invalid argument 2024-08-17T18:22:44.375 INFO:teuthology.orchestra.run.smithi179.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:22:44.383 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:44.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:44.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:44.485 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:44.506 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:44.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:44.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:44.533 INFO:tasks.workunit.client.0.smithi130.stdout:Running main() from gmock_main.cc 2024-08-17T18:22:44.533 INFO:tasks.workunit.client.0.smithi130.stdout:[==========] Running 70 tests from 1 test suite. 2024-08-17T18:22:44.533 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] Global test environment set-up. 2024-08-17T18:22:44.533 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 70 tests from TestClsRbd 2024-08-17T18:22:44.565 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:44.592 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:44 smithi179 ceph-mon[93519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:22:44.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:22:44.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:44 smithi179 ceph-mon[93519]: osdmap e1519: 8 total, 8 up, 6 in 2024-08-17T18:22:44.608 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:44.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:44.631 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:44.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:44.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:44 smithi130 ceph-mon[105521]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:22:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:22:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:44 smithi130 ceph-mon[105521]: osdmap e1519: 8 total, 8 up, 6 in 2024-08-17T18:22:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:44 smithi130 ceph-mon[110068]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:22:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T18:22:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:44 smithi130 ceph-mon[110068]: osdmap e1519: 8 total, 8 up, 6 in 2024-08-17T18:22:44.732 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:44.753 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:44.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:44.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:44.855 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:44.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:44.877 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:44.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:44.978 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:45.000 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:45.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:45.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:45.094 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T18:22:45.102 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:45.159 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:45.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:45.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:45.261 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:45.282 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:45.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:45.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:45.384 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:45.405 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:45.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:45.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:45.507 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:45.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:45.531 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:45.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:45.543 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:45 smithi130 ceph-mon[105521]: pgmap v2963: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:45.544 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:45 smithi130 ceph-mon[110068]: pgmap v2963: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:45.631 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:45.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:45.653 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:45.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:45.754 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:45.775 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:45.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:45.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:45.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:45 smithi179 ceph-mon[93519]: pgmap v2963: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:45.864 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:22:45.865 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:22:45.877 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:45.899 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:45.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:45.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:46.001 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:46.023 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:46.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:46.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:46.125 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:46.147 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:46.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:46.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:46.248 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:46.270 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:46.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:46.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:46.372 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:46.382 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.get_all_features 2024-08-17T18:22:46.401 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:46.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:46.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:46.407 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.get_all_features (25 ms) 2024-08-17T18:22:46.407 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.copyup 2024-08-17T18:22:46.504 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:46.526 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:46.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:46.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:46.550 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:22:46.573 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.copyup (166 ms) 2024-08-17T18:22:46.573 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-08-17T18:22:46.575 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:46 smithi130 ceph-mon[105521]: osdmap e1520: 8 total, 8 up, 6 in 2024-08-17T18:22:46.575 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:46 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3756203914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi130-433706-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T18:22:46.575 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:46 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi130-433706-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T18:22:46.575 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:46 smithi130 ceph-mon[110068]: osdmap e1520: 8 total, 8 up, 6 in 2024-08-17T18:22:46.575 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:46 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3756203914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi130-433706-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T18:22:46.576 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:46 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi130-433706-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T18:22:46.616 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.sparse_copyup (43 ms) 2024-08-17T18:22:46.616 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-08-17T18:22:46.623 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.get_and_set_id (8 ms) 2024-08-17T18:22:46.623 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-08-17T18:22:46.628 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:46.634 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.add_remove_child (10 ms) 2024-08-17T18:22:46.634 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.directory_methods 2024-08-17T18:22:46.649 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:46.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:46.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:46.662 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.directory_methods (28 ms) 2024-08-17T18:22:46.662 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.create 2024-08-17T18:22:46.674 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.create (12 ms) 2024-08-17T18:22:46.674 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.get_features 2024-08-17T18:22:46.675 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-08-17T18:22:46.676 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-08-17T18:22:46.677 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2024-08-17T18:22:46.677 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-08-17T18:22:46.679 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-08-17T18:22:46.679 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-08-17T18:22:46.680 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2024-08-17T18:22:46.680 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-08-17T18:22:46.681 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-08-17T18:22:46.681 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-08-17T18:22:46.684 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2024-08-17T18:22:46.684 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.get_size 2024-08-17T18:22:46.688 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.get_size (3 ms) 2024-08-17T18:22:46.688 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.set_size 2024-08-17T18:22:46.693 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2024-08-17T18:22:46.693 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.protection_status 2024-08-17T18:22:46.709 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.protection_status (17 ms) 2024-08-17T18:22:46.709 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-08-17T18:22:46.716 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.snapshot_limits (6 ms) 2024-08-17T18:22:46.716 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.parents_v1 2024-08-17T18:22:46.753 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:46.755 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.parents_v1 (38 ms) 2024-08-17T18:22:46.755 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.parents_v2 2024-08-17T18:22:46.775 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:46.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:46.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:46.782 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.parents_v2 (28 ms) 2024-08-17T18:22:46.782 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.snapshots 2024-08-17T18:22:46.800 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.snapshots (19 ms) 2024-08-17T18:22:46.800 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.snapid_race 2024-08-17T18:22:46.807 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.snapid_race (6 ms) 2024-08-17T18:22:46.807 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.stripingv2 2024-08-17T18:22:46.816 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2024-08-17T18:22:46.816 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.object_map_save 2024-08-17T18:22:46.817 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.object_map_save (1 ms) 2024-08-17T18:22:46.817 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-08-17T18:22:46.824 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2024-08-17T18:22:46.824 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.object_map_update 2024-08-17T18:22:46.830 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.object_map_update (5 ms) 2024-08-17T18:22:46.830 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-08-17T18:22:46.830 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-08-17T18:22:46.830 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-08-17T18:22:46.839 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.object_map_snap_add (9 ms) 2024-08-17T18:22:46.839 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-08-17T18:22:46.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:46 smithi179 ceph-mon[93519]: osdmap e1520: 8 total, 8 up, 6 in 2024-08-17T18:22:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:46 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3756203914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi130-433706-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T18:22:46.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:46 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi130-433706-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T18:22:46.848 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.object_map_snap_remove (10 ms) 2024-08-17T18:22:46.849 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.flags 2024-08-17T18:22:46.853 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.flags (4 ms) 2024-08-17T18:22:46.853 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.metadata 2024-08-17T18:22:46.876 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.metadata (23 ms) 2024-08-17T18:22:46.876 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.set_features 2024-08-17T18:22:46.876 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:46.882 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2024-08-17T18:22:46.882 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.mirror 2024-08-17T18:22:46.897 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:46.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:46.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:46.914 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.mirror (32 ms) 2024-08-17T18:22:46.914 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.mirror_image 2024-08-17T18:22:46.930 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.mirror_image (16 ms) 2024-08-17T18:22:46.930 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-08-17T18:22:46.997 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":2964,"stamp":"2024-08-17T18:22:45.371553+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":111,"num_bytes_recovered":5325365,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":5337628,"kb_used_data":80688,"kb_used_omap":348,"kb_used_meta":5256483,"kb_avail":557133284,"statfs":{"total":575970213888,"available":570504482816,"internally_reserved":0,"allocated":82624512,"data_stored":49197619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":357179,"internal_metadata":5382638789},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.219259"},"pg_stats":[{"pgid":"51.1f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.1e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.1d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.1c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.1b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.1a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.19","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.18","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.17","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.16","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.15","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.14","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.13","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.12","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.11","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.10","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"51.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T18:22:45.364727+0000","last_change":"2024-08-17T18:22:45.364727+0000","last_active":"2024-08-17T18:22:45.364727+0000","last_peered":"2024-08-17T18:22:45.364727+0000","last_clean":"2024-08-17T18:22:45.364727+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T18:22:45.364727+0000","last_undegraded":"2024-08-17T18:22:45.364727+0000","last_fullsized":"2024-08-17T18:22:45.364727+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T18:22:45.364727+0000","last_clean_scrub_stamp":"2024-08-17T18:22:45.364727+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":74,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.161300+0000","last_change":"2024-08-17T18:21:33.450755+0000","last_active":"2024-08-17T18:22:36.161300+0000","last_peered":"2024-08-17T18:22:36.161300+0000","last_clean":"2024-08-17T18:22:36.161300+0000","last_became_active":"2024-08-17T18:21:33.427306+0000","last_became_peered":"2024-08-17T18:21:33.427306+0000","last_unstale":"2024-08-17T18:22:36.161300+0000","last_undegraded":"2024-08-17T18:22:36.161300+0000","last_fullsized":"2024-08-17T18:22:36.161300+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:55:45.822004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1710,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":91,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.296839+0000","last_change":"2024-08-17T18:21:58.262211+0000","last_active":"2024-08-17T18:22:35.296839+0000","last_peered":"2024-08-17T18:22:35.296839+0000","last_clean":"2024-08-17T18:22:35.296839+0000","last_became_active":"2024-08-17T18:21:58.261636+0000","last_became_peered":"2024-08-17T18:21:58.261636+0000","last_unstale":"2024-08-17T18:22:35.296839+0000","last_undegraded":"2024-08-17T18:22:35.296839+0000","last_fullsized":"2024-08-17T18:22:35.296839+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:22:53.662404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":2343,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.155124+0000","last_change":"2024-08-17T18:19:27.023211+0000","last_active":"2024-08-17T18:22:36.155124+0000","last_peered":"2024-08-17T18:22:36.155124+0000","last_clean":"2024-08-17T18:22:36.155124+0000","last_became_active":"2024-08-17T18:19:27.022827+0000","last_became_peered":"2024-08-17T18:19:27.022827+0000","last_unstale":"2024-08-17T18:22:36.155124+0000","last_undegraded":"2024-08-17T18:22:36.155124+0000","last_fullsized":"2024-08-17T18:22:36.155124+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:25:50.310320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2408,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.155296+0000","last_change":"2024-08-17T18:21:33.421541+0000","last_active":"2024-08-17T18:22:36.155296+0000","last_peered":"2024-08-17T18:22:36.155296+0000","last_clean":"2024-08-17T18:22:36.155296+0000","last_became_active":"2024-08-17T18:21:33.420932+0000","last_became_peered":"2024-08-17T18:21:33.420932+0000","last_unstale":"2024-08-17T18:22:36.155296+0000","last_undegraded":"2024-08-17T18:22:36.155296+0000","last_fullsized":"2024-08-17T18:22:36.155296+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:23.366908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":94,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.296970+0000","last_change":"2024-08-17T18:21:58.283162+0000","last_active":"2024-08-17T18:22:35.296970+0000","last_peered":"2024-08-17T18:22:35.296970+0000","last_clean":"2024-08-17T18:22:35.296970+0000","last_became_active":"2024-08-17T18:21:58.265637+0000","last_became_peered":"2024-08-17T18:21:58.265637+0000","last_unstale":"2024-08-17T18:22:35.296970+0000","last_undegraded":"2024-08-17T18:22:35.296970+0000","last_fullsized":"2024-08-17T18:22:35.296970+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:59:10.574052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":101,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.297693+0000","last_change":"2024-08-17T18:21:33.404642+0000","last_active":"2024-08-17T18:22:35.297693+0000","last_peered":"2024-08-17T18:22:35.297693+0000","last_clean":"2024-08-17T18:22:35.297693+0000","last_became_active":"2024-08-17T18:21:33.404374+0000","last_became_peered":"2024-08-17T18:21:33.404374+0000","last_unstale":"2024-08-17T18:22:35.297693+0000","last_undegraded":"2024-08-17T18:22:35.297693+0000","last_fullsized":"2024-08-17T18:22:35.297693+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:49:34.531843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":94,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.296390+0000","last_change":"2024-08-17T18:21:58.264821+0000","last_active":"2024-08-17T18:22:35.296390+0000","last_peered":"2024-08-17T18:22:35.296390+0000","last_clean":"2024-08-17T18:22:35.296390+0000","last_became_active":"2024-08-17T18:21:58.260477+0000","last_became_peered":"2024-08-17T18:21:58.260477+0000","last_unstale":"2024-08-17T18:22:35.296390+0000","last_undegraded":"2024-08-17T18:22:35.296390+0000","last_fullsized":"2024-08-17T18:22:35.296390+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:20.960079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":91,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.296936+0000","last_change":"2024-08-17T18:21:58.272053+0000","last_active":"2024-08-17T18:22:35.296936+0000","last_peered":"2024-08-17T18:22:35.296936+0000","last_clean":"2024-08-17T18:22:35.296936+0000","last_became_active":"2024-08-17T18:21:58.271727+0000","last_became_peered":"2024-08-17T18:21:58.271727+0000","last_unstale":"2024-08-17T18:22:35.296936+0000","last_undegraded":"2024-08-17T18:22:35.296936+0000","last_fullsized":"2024-08-17T18:22:35.296936+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:51.678694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":68,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.157570+0000","last_change":"2024-08-17T18:21:33.424299+0000","last_active":"2024-08-17T18:22:36.157570+0000","last_peered":"2024-08-17T18:22:36.157570+0000","last_clean":"2024-08-17T18:22:36.157570+0000","last_became_active":"2024-08-17T18:21:33.424016+0000","last_became_peered":"2024-08-17T18:21:33.424016+0000","last_unstale":"2024-08-17T18:22:36.157570+0000","last_undegraded":"2024-08-17T18:22:36.157570+0000","last_fullsized":"2024-08-17T18:22:36.157570+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:15.128415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2060,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.159960+0000","last_change":"2024-08-17T18:21:58.281079+0000","last_active":"2024-08-17T18:22:36.159960+0000","last_peered":"2024-08-17T18:22:36.159960+0000","last_clean":"2024-08-17T18:22:36.159960+0000","last_became_active":"2024-08-17T18:21:58.280727+0000","last_became_peered":"2024-08-17T18:21:58.280727+0000","last_unstale":"2024-08-17T18:22:36.159960+0000","last_undegraded":"2024-08-17T18:22:36.159960+0000","last_fullsized":"2024-08-17T18:22:36.159960+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:22:33.690910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":105,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.297793+0000","last_change":"2024-08-17T18:21:58.278189+0000","last_active":"2024-08-17T18:22:35.297793+0000","last_peered":"2024-08-17T18:22:35.297793+0000","last_clean":"2024-08-17T18:22:35.297793+0000","last_became_active":"2024-08-17T18:21:58.277795+0000","last_became_peered":"2024-08-17T18:21:58.277795+0000","last_unstale":"2024-08-17T18:22:35.297793+0000","last_undegraded":"2024-08-17T18:22:35.297793+0000","last_fullsized":"2024-08-17T18:22:35.297793+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:28.692252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2363,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.156431+0000","last_change":"2024-08-17T18:21:58.273212+0000","last_active":"2024-08-17T18:22:36.156431+0000","last_peered":"2024-08-17T18:22:36.156431+0000","last_clean":"2024-08-17T18:22:36.156431+0000","last_became_active":"2024-08-17T18:21:58.270393+0000","last_became_peered":"2024-08-17T18:21:58.270393+0000","last_unstale":"2024-08-17T18:22:36.156431+0000","last_undegraded":"2024-08-17T18:22:36.156431+0000","last_fullsized":"2024-08-17T18:22:36.156431+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:21.423509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2036,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.155103+0000","last_change":"2024-08-17T18:21:58.275572+0000","last_active":"2024-08-17T18:22:36.155103+0000","last_peered":"2024-08-17T18:22:36.155103+0000","last_clean":"2024-08-17T18:22:36.155103+0000","last_became_active":"2024-08-17T18:21:58.275094+0000","last_became_peered":"2024-08-17T18:21:58.275094+0000","last_unstale":"2024-08-17T18:22:36.155103+0000","last_undegraded":"2024-08-17T18:22:36.155103+0000","last_fullsized":"2024-08-17T18:22:36.155103+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:02.844716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":68,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.160277+0000","last_change":"2024-08-17T18:21:33.408805+0000","last_active":"2024-08-17T18:22:36.160277+0000","last_peered":"2024-08-17T18:22:36.160277+0000","last_clean":"2024-08-17T18:22:36.160277+0000","last_became_active":"2024-08-17T18:21:33.408216+0000","last_became_peered":"2024-08-17T18:21:33.408216+0000","last_unstale":"2024-08-17T18:22:36.160277+0000","last_undegraded":"2024-08-17T18:22:36.160277+0000","last_fullsized":"2024-08-17T18:22:36.160277+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:45.784110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":50,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.153847+0000","last_change":"2024-08-17T18:21:58.271246+0000","last_active":"2024-08-17T18:22:36.153847+0000","last_peered":"2024-08-17T18:22:36.153847+0000","last_clean":"2024-08-17T18:22:36.153847+0000","last_became_active":"2024-08-17T18:21:58.269527+0000","last_became_peered":"2024-08-17T18:21:58.269527+0000","last_unstale":"2024-08-17T18:22:36.153847+0000","last_undegraded":"2024-08-17T18:22:36.153847+0000","last_fullsized":"2024-08-17T18:22:36.153847+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:26:06.117680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":146,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.301948+0000","last_change":"2024-08-17T18:21:58.268560+0000","last_active":"2024-08-17T18:22:35.301948+0000","last_peered":"2024-08-17T18:22:35.301948+0000","last_clean":"2024-08-17T18:22:35.301948+0000","last_became_active":"2024-08-17T18:21:58.268263+0000","last_became_peered":"2024-08-17T18:21:58.268263+0000","last_unstale":"2024-08-17T18:22:35.301948+0000","last_undegraded":"2024-08-17T18:22:35.301948+0000","last_fullsized":"2024-08-17T18:22:35.301948+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:41.676163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2350,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.154084+0000","last_change":"2024-08-17T18:21:58.271897+0000","last_active":"2024-08-17T18:22:36.154084+0000","last_peered":"2024-08-17T18:22:36.154084+0000","last_clean":"2024-08-17T18:22:36.154084+0000","last_became_active":"2024-08-17T18:21:58.270598+0000","last_became_peered":"2024-08-17T18:21:58.270598+0000","last_unstale":"2024-08-17T18:22:36.154084+0000","last_undegraded":"2024-08-17T18:22:36.154084+0000","last_fullsized":"2024-08-17T18:22:36.154084+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:39:09.775610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":68,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.161152+0000","last_change":"2024-08-17T18:21:33.428841+0000","last_active":"2024-08-17T18:22:36.161152+0000","last_peered":"2024-08-17T18:22:36.161152+0000","last_clean":"2024-08-17T18:22:36.161152+0000","last_became_active":"2024-08-17T18:21:33.427865+0000","last_became_peered":"2024-08-17T18:21:33.427865+0000","last_unstale":"2024-08-17T18:22:36.161152+0000","last_undegraded":"2024-08-17T18:22:36.161152+0000","last_fullsized":"2024-08-17T18:22:36.161152+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:33.010010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2291,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.154954+0000","last_change":"2024-08-17T18:16:34.644366+0000","last_active":"2024-08-17T18:22:36.154954+0000","last_peered":"2024-08-17T18:22:36.154954+0000","last_clean":"2024-08-17T18:22:36.154954+0000","last_became_active":"2024-08-17T18:16:34.643994+0000","last_became_peered":"2024-08-17T18:16:34.643994+0000","last_unstale":"2024-08-17T18:22:36.154954+0000","last_undegraded":"2024-08-17T18:22:36.154954+0000","last_fullsized":"2024-08-17T18:22:36.154954+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:51.953343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":854,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.617230+0000","last_change":"2024-08-17T18:21:58.265707+0000","last_active":"2024-08-17T18:22:36.617230+0000","last_peered":"2024-08-17T18:22:36.617230+0000","last_clean":"2024-08-17T18:22:36.617230+0000","last_became_active":"2024-08-17T18:21:58.265367+0000","last_became_peered":"2024-08-17T18:21:58.265367+0000","last_unstale":"2024-08-17T18:22:36.617230+0000","last_undegraded":"2024-08-17T18:22:36.617230+0000","last_fullsized":"2024-08-17T18:22:36.617230+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:23:03.119320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1522,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.616207+0000","last_change":"2024-08-17T18:18:50.525363+0000","last_active":"2024-08-17T18:22:36.616207+0000","last_peered":"2024-08-17T18:22:36.616207+0000","last_clean":"2024-08-17T18:22:36.616207+0000","last_became_active":"2024-08-17T18:18:50.511676+0000","last_became_peered":"2024-08-17T18:18:50.511676+0000","last_unstale":"2024-08-17T18:22:36.616207+0000","last_undegraded":"2024-08-17T18:22:36.616207+0000","last_fullsized":"2024-08-17T18:22:36.616207+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:22.530382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":171,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.299985+0000","last_change":"2024-08-17T18:21:58.266640+0000","last_active":"2024-08-17T18:22:35.299985+0000","last_peered":"2024-08-17T18:22:35.299985+0000","last_clean":"2024-08-17T18:22:35.299985+0000","last_became_active":"2024-08-17T18:21:58.266135+0000","last_became_peered":"2024-08-17T18:21:58.266135+0000","last_unstale":"2024-08-17T18:22:35.299985+0000","last_undegraded":"2024-08-17T18:22:35.299985+0000","last_fullsized":"2024-08-17T18:22:35.299985+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:35.330096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":90,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.296301+0000","last_change":"2024-08-17T18:18:51.515962+0000","last_active":"2024-08-17T18:22:35.296301+0000","last_peered":"2024-08-17T18:22:35.296301+0000","last_clean":"2024-08-17T18:22:35.296301+0000","last_became_active":"2024-08-17T18:18:51.505508+0000","last_became_peered":"2024-08-17T18:18:51.505508+0000","last_unstale":"2024-08-17T18:22:35.296301+0000","last_undegraded":"2024-08-17T18:22:35.296301+0000","last_fullsized":"2024-08-17T18:22:35.296301+0000","mapping_epoch":1447,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:16.518470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":90,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.296493+0000","last_change":"2024-08-17T18:21:58.277259+0000","last_active":"2024-08-17T18:22:35.296493+0000","last_peered":"2024-08-17T18:22:35.296493+0000","last_clean":"2024-08-17T18:22:35.296493+0000","last_became_active":"2024-08-17T18:21:58.276909+0000","last_became_peered":"2024-08-17T18:21:58.276909+0000","last_unstale":"2024-08-17T18:22:35.296493+0000","last_undegraded":"2024-08-17T18:22:35.296493+0000","last_fullsized":"2024-08-17T18:22:35.296493+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:04:17.083826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":111,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.296353+0000","last_change":"2024-08-17T18:21:33.403392+0000","last_active":"2024-08-17T18:22:35.296353+0000","last_peered":"2024-08-17T18:22:35.296353+0000","last_clean":"2024-08-17T18:22:35.296353+0000","last_became_active":"2024-08-17T18:21:33.403017+0000","last_became_peered":"2024-08-17T18:21:33.403017+0000","last_unstale":"2024-08-17T18:22:35.296353+0000","last_undegraded":"2024-08-17T18:22:35.296353+0000","last_fullsized":"2024-08-17T18:22:35.296353+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:50:06.621109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":120,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.302016+0000","last_change":"2024-08-17T18:19:27.055728+0000","last_active":"2024-08-17T18:22:35.302016+0000","last_peered":"2024-08-17T18:22:35.302016+0000","last_clean":"2024-08-17T18:22:35.302016+0000","last_became_active":"2024-08-17T18:19:27.055425+0000","last_became_peered":"2024-08-17T18:19:27.055425+0000","last_unstale":"2024-08-17T18:22:35.302016+0000","last_undegraded":"2024-08-17T18:22:35.302016+0000","last_fullsized":"2024-08-17T18:22:35.302016+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:47.920145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":83,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.162311+0000","last_change":"2024-08-17T18:21:58.268968+0000","last_active":"2024-08-17T18:22:36.162311+0000","last_peered":"2024-08-17T18:22:36.162311+0000","last_clean":"2024-08-17T18:22:36.162311+0000","last_became_active":"2024-08-17T18:21:58.261371+0000","last_became_peered":"2024-08-17T18:21:58.261371+0000","last_unstale":"2024-08-17T18:22:36.162311+0000","last_undegraded":"2024-08-17T18:22:36.162311+0000","last_fullsized":"2024-08-17T18:22:36.162311+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:35:20.986571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2359,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.154158+0000","last_change":"2024-08-17T18:21:33.421768+0000","last_active":"2024-08-17T18:22:36.154158+0000","last_peered":"2024-08-17T18:22:36.154158+0000","last_clean":"2024-08-17T18:22:36.154158+0000","last_became_active":"2024-08-17T18:21:33.421467+0000","last_became_peered":"2024-08-17T18:21:33.421467+0000","last_unstale":"2024-08-17T18:22:36.154158+0000","last_undegraded":"2024-08-17T18:22:36.154158+0000","last_fullsized":"2024-08-17T18:22:36.154158+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:38.751105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":151,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.302236+0000","last_change":"2024-08-17T18:21:58.268916+0000","last_active":"2024-08-17T18:22:35.302236+0000","last_peered":"2024-08-17T18:22:35.302236+0000","last_clean":"2024-08-17T18:22:35.302236+0000","last_became_active":"2024-08-17T18:21:58.268521+0000","last_became_peered":"2024-08-17T18:21:58.268521+0000","last_unstale":"2024-08-17T18:22:35.302236+0000","last_undegraded":"2024-08-17T18:22:35.302236+0000","last_fullsized":"2024-08-17T18:22:35.302236+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:13.343838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":2380,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.158151+0000","last_change":"2024-08-17T18:21:33.497158+0000","last_active":"2024-08-17T18:22:36.158151+0000","last_peered":"2024-08-17T18:22:36.158151+0000","last_clean":"2024-08-17T18:22:36.158151+0000","last_became_active":"2024-08-17T18:21:33.421674+0000","last_became_peered":"2024-08-17T18:21:33.421674+0000","last_unstale":"2024-08-17T18:22:36.158151+0000","last_undegraded":"2024-08-17T18:22:36.158151+0000","last_fullsized":"2024-08-17T18:22:36.158151+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:58:57.198125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":75,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.158108+0000","last_change":"2024-08-17T18:21:58.273991+0000","last_active":"2024-08-17T18:22:36.158108+0000","last_peered":"2024-08-17T18:22:36.158108+0000","last_clean":"2024-08-17T18:22:36.158108+0000","last_became_active":"2024-08-17T18:21:58.273667+0000","last_became_peered":"2024-08-17T18:21:58.273667+0000","last_unstale":"2024-08-17T18:22:36.158108+0000","last_undegraded":"2024-08-17T18:22:36.158108+0000","last_fullsized":"2024-08-17T18:22:36.158108+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:44:28.587836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1559,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.616428+0000","last_change":"2024-08-17T18:21:33.420079+0000","last_active":"2024-08-17T18:22:36.616428+0000","last_peered":"2024-08-17T18:22:36.616428+0000","last_clean":"2024-08-17T18:22:36.616428+0000","last_became_active":"2024-08-17T18:21:33.419265+0000","last_became_peered":"2024-08-17T18:21:33.419265+0000","last_unstale":"2024-08-17T18:22:36.616428+0000","last_undegraded":"2024-08-17T18:22:36.616428+0000","last_fullsized":"2024-08-17T18:22:36.616428+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:33:44.811302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1849,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.160505+0000","last_change":"2024-08-17T18:21:58.277263+0000","last_active":"2024-08-17T18:22:36.160505+0000","last_peered":"2024-08-17T18:22:36.160505+0000","last_clean":"2024-08-17T18:22:36.160505+0000","last_became_active":"2024-08-17T18:21:58.276900+0000","last_became_peered":"2024-08-17T18:21:58.276900+0000","last_unstale":"2024-08-17T18:22:36.160505+0000","last_undegraded":"2024-08-17T18:22:36.160505+0000","last_fullsized":"2024-08-17T18:22:36.160505+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:05.316231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":257,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.296963+0000","last_change":"2024-08-17T18:21:58.278092+0000","last_active":"2024-08-17T18:22:35.296963+0000","last_peered":"2024-08-17T18:22:35.296963+0000","last_clean":"2024-08-17T18:22:35.296963+0000","last_became_active":"2024-08-17T18:21:58.277734+0000","last_became_peered":"2024-08-17T18:21:58.277734+0000","last_unstale":"2024-08-17T18:22:35.296963+0000","last_undegraded":"2024-08-17T18:22:35.296963+0000","last_fullsized":"2024-08-17T18:22:35.296963+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:50.326896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":258,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.163074+0000","last_change":"2024-08-17T18:21:58.275684+0000","last_active":"2024-08-17T18:22:36.163074+0000","last_peered":"2024-08-17T18:22:36.163074+0000","last_clean":"2024-08-17T18:22:36.163074+0000","last_became_active":"2024-08-17T18:21:58.275147+0000","last_became_peered":"2024-08-17T18:21:58.275147+0000","last_unstale":"2024-08-17T18:22:36.163074+0000","last_undegraded":"2024-08-17T18:22:36.163074+0000","last_fullsized":"2024-08-17T18:22:36.163074+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:03:12.960867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1136,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.156327+0000","last_change":"2024-08-17T18:21:12.774945+0000","last_active":"2024-08-17T18:22:36.156327+0000","last_peered":"2024-08-17T18:22:36.156327+0000","last_clean":"2024-08-17T18:22:36.156327+0000","last_became_active":"2024-08-17T18:21:12.774497+0000","last_became_peered":"2024-08-17T18:21:12.774497+0000","last_unstale":"2024-08-17T18:22:36.156327+0000","last_undegraded":"2024-08-17T18:22:36.156327+0000","last_fullsized":"2024-08-17T18:22:36.156327+0000","mapping_epoch":1477,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:36:52.456482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":631,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.160604+0000","last_change":"2024-08-17T18:21:58.275787+0000","last_active":"2024-08-17T18:22:36.160604+0000","last_peered":"2024-08-17T18:22:36.160604+0000","last_clean":"2024-08-17T18:22:36.160604+0000","last_became_active":"2024-08-17T18:21:58.275441+0000","last_became_peered":"2024-08-17T18:21:58.275441+0000","last_unstale":"2024-08-17T18:22:36.160604+0000","last_undegraded":"2024-08-17T18:22:36.160604+0000","last_fullsized":"2024-08-17T18:22:36.160604+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:44.334290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":101,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.297021+0000","last_change":"2024-08-17T18:21:33.403089+0000","last_active":"2024-08-17T18:22:35.297021+0000","last_peered":"2024-08-17T18:22:35.297021+0000","last_clean":"2024-08-17T18:22:35.297021+0000","last_became_active":"2024-08-17T18:21:33.402788+0000","last_became_peered":"2024-08-17T18:21:33.402788+0000","last_unstale":"2024-08-17T18:22:35.297021+0000","last_undegraded":"2024-08-17T18:22:35.297021+0000","last_fullsized":"2024-08-17T18:22:35.297021+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:25.909442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":104,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.300119+0000","last_change":"2024-08-17T18:21:58.286963+0000","last_active":"2024-08-17T18:22:35.300119+0000","last_peered":"2024-08-17T18:22:35.300119+0000","last_clean":"2024-08-17T18:22:35.300119+0000","last_became_active":"2024-08-17T18:21:58.269297+0000","last_became_peered":"2024-08-17T18:21:58.269297+0000","last_unstale":"2024-08-17T18:22:35.300119+0000","last_undegraded":"2024-08-17T18:22:35.300119+0000","last_fullsized":"2024-08-17T18:22:35.300119+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:53.779677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":271,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.161062+0000","last_change":"2024-08-17T18:21:58.267586+0000","last_active":"2024-08-17T18:22:36.161062+0000","last_peered":"2024-08-17T18:22:36.161062+0000","last_clean":"2024-08-17T18:22:36.161062+0000","last_became_active":"2024-08-17T18:21:58.267287+0000","last_became_peered":"2024-08-17T18:21:58.267287+0000","last_unstale":"2024-08-17T18:22:36.161062+0000","last_undegraded":"2024-08-17T18:22:36.161062+0000","last_fullsized":"2024-08-17T18:22:36.161062+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:51:06.593254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":172,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.299495+0000","last_change":"2024-08-17T18:21:58.288401+0000","last_active":"2024-08-17T18:22:35.299495+0000","last_peered":"2024-08-17T18:22:35.299495+0000","last_clean":"2024-08-17T18:22:35.299495+0000","last_became_active":"2024-08-17T18:21:58.288192+0000","last_became_peered":"2024-08-17T18:21:58.288192+0000","last_unstale":"2024-08-17T18:22:35.299495+0000","last_undegraded":"2024-08-17T18:22:35.299495+0000","last_fullsized":"2024-08-17T18:22:35.299495+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:07:54.845030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2283,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.153547+0000","last_change":"2024-08-17T18:21:33.419513+0000","last_active":"2024-08-17T18:22:36.153547+0000","last_peered":"2024-08-17T18:22:36.153547+0000","last_clean":"2024-08-17T18:22:36.153547+0000","last_became_active":"2024-08-17T18:21:33.419186+0000","last_became_peered":"2024-08-17T18:21:33.419186+0000","last_unstale":"2024-08-17T18:22:36.153547+0000","last_undegraded":"2024-08-17T18:22:36.153547+0000","last_fullsized":"2024-08-17T18:22:36.153547+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:27.395794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":75,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.160369+0000","last_change":"2024-08-17T18:21:58.274148+0000","last_active":"2024-08-17T18:22:36.160369+0000","last_peered":"2024-08-17T18:22:36.160369+0000","last_clean":"2024-08-17T18:22:36.160369+0000","last_became_active":"2024-08-17T18:21:58.272822+0000","last_became_peered":"2024-08-17T18:21:58.272822+0000","last_unstale":"2024-08-17T18:22:36.160369+0000","last_undegraded":"2024-08-17T18:22:36.160369+0000","last_fullsized":"2024-08-17T18:22:36.160369+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:52:45.155095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":68,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.160743+0000","last_change":"2024-08-17T18:21:33.407703+0000","last_active":"2024-08-17T18:22:36.160743+0000","last_peered":"2024-08-17T18:22:36.160743+0000","last_clean":"2024-08-17T18:22:36.160743+0000","last_became_active":"2024-08-17T18:21:33.407397+0000","last_became_peered":"2024-08-17T18:21:33.407397+0000","last_unstale":"2024-08-17T18:22:36.160743+0000","last_undegraded":"2024-08-17T18:22:36.160743+0000","last_fullsized":"2024-08-17T18:22:36.160743+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:06.725837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1580,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.615687+0000","last_change":"2024-08-17T18:21:58.270498+0000","last_active":"2024-08-17T18:22:36.615687+0000","last_peered":"2024-08-17T18:22:36.615687+0000","last_clean":"2024-08-17T18:22:36.615687+0000","last_became_active":"2024-08-17T18:21:58.270156+0000","last_became_peered":"2024-08-17T18:21:58.270156+0000","last_unstale":"2024-08-17T18:22:36.615687+0000","last_undegraded":"2024-08-17T18:22:36.615687+0000","last_fullsized":"2024-08-17T18:22:36.615687+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:19:20.221532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":116,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.301665+0000","last_change":"2024-08-17T18:18:50.516313+0000","last_active":"2024-08-17T18:22:35.301665+0000","last_peered":"2024-08-17T18:22:35.301665+0000","last_clean":"2024-08-17T18:22:35.301665+0000","last_became_active":"2024-08-17T18:18:50.515925+0000","last_became_peered":"2024-08-17T18:18:50.515925+0000","last_unstale":"2024-08-17T18:22:35.301665+0000","last_undegraded":"2024-08-17T18:22:35.301665+0000","last_fullsized":"2024-08-17T18:22:35.301665+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:14.611619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2331,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.156205+0000","last_change":"2024-08-17T18:21:12.770126+0000","last_active":"2024-08-17T18:22:36.156205+0000","last_peered":"2024-08-17T18:22:36.156205+0000","last_clean":"2024-08-17T18:22:36.156205+0000","last_became_active":"2024-08-17T18:21:12.769545+0000","last_became_peered":"2024-08-17T18:21:12.769545+0000","last_unstale":"2024-08-17T18:22:36.156205+0000","last_undegraded":"2024-08-17T18:22:36.156205+0000","last_fullsized":"2024-08-17T18:22:36.156205+0000","mapping_epoch":1477,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:38:21.660942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":306,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.155648+0000","last_change":"2024-08-17T18:21:32.421106+0000","last_active":"2024-08-17T18:22:36.155648+0000","last_peered":"2024-08-17T18:22:36.155648+0000","last_clean":"2024-08-17T18:22:36.155648+0000","last_became_active":"2024-08-17T18:21:12.794575+0000","last_became_peered":"2024-08-17T18:21:12.794575+0000","last_unstale":"2024-08-17T18:22:36.155648+0000","last_undegraded":"2024-08-17T18:22:36.155648+0000","last_fullsized":"2024-08-17T18:22:36.155648+0000","mapping_epoch":1477,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:53:46.665801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":1990,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.615489+0000","last_change":"2024-08-17T18:22:00.258650+0000","last_active":"2024-08-17T18:22:36.615489+0000","last_peered":"2024-08-17T18:22:36.615489+0000","last_clean":"2024-08-17T18:22:36.615489+0000","last_became_active":"2024-08-17T18:22:00.258326+0000","last_became_peered":"2024-08-17T18:22:00.258326+0000","last_unstale":"2024-08-17T18:22:36.615489+0000","last_undegraded":"2024-08-17T18:22:36.615489+0000","last_fullsized":"2024-08-17T18:22:36.615489+0000","mapping_epoch":1500,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:02.578380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1974,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.158316+0000","last_change":"2024-08-17T18:21:58.270154+0000","last_active":"2024-08-17T18:22:36.158316+0000","last_peered":"2024-08-17T18:22:36.158316+0000","last_clean":"2024-08-17T18:22:36.158316+0000","last_became_active":"2024-08-17T18:21:58.269562+0000","last_became_peered":"2024-08-17T18:21:58.269562+0000","last_unstale":"2024-08-17T18:22:36.158316+0000","last_undegraded":"2024-08-17T18:22:36.158316+0000","last_fullsized":"2024-08-17T18:22:36.158316+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:21.136191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2388,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.154238+0000","last_change":"2024-08-17T18:21:58.271499+0000","last_active":"2024-08-17T18:22:36.154238+0000","last_peered":"2024-08-17T18:22:36.154238+0000","last_clean":"2024-08-17T18:22:36.154238+0000","last_became_active":"2024-08-17T18:21:58.270311+0000","last_became_peered":"2024-08-17T18:21:58.270311+0000","last_unstale":"2024-08-17T18:22:36.154238+0000","last_undegraded":"2024-08-17T18:22:36.154238+0000","last_fullsized":"2024-08-17T18:22:36.154238+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:10.795233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":91,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.296603+0000","last_change":"2024-08-17T18:21:58.266370+0000","last_active":"2024-08-17T18:22:35.296603+0000","last_peered":"2024-08-17T18:22:35.296603+0000","last_clean":"2024-08-17T18:22:35.296603+0000","last_became_active":"2024-08-17T18:21:58.266082+0000","last_became_peered":"2024-08-17T18:21:58.266082+0000","last_unstale":"2024-08-17T18:22:35.296603+0000","last_undegraded":"2024-08-17T18:22:35.296603+0000","last_fullsized":"2024-08-17T18:22:35.296603+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:02:20.218698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":95,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.296272+0000","last_change":"2024-08-17T18:19:27.038041+0000","last_active":"2024-08-17T18:22:35.296272+0000","last_peered":"2024-08-17T18:22:35.296272+0000","last_clean":"2024-08-17T18:22:35.296272+0000","last_became_active":"2024-08-17T18:19:27.037715+0000","last_became_peered":"2024-08-17T18:19:27.037715+0000","last_unstale":"2024-08-17T18:22:35.296272+0000","last_undegraded":"2024-08-17T18:22:35.296272+0000","last_fullsized":"2024-08-17T18:22:35.296272+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:06:39.726534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":102,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.296966+0000","last_change":"2024-08-17T18:21:58.278111+0000","last_active":"2024-08-17T18:22:35.296966+0000","last_peered":"2024-08-17T18:22:35.296966+0000","last_clean":"2024-08-17T18:22:35.296966+0000","last_became_active":"2024-08-17T18:21:58.277525+0000","last_became_peered":"2024-08-17T18:21:58.277525+0000","last_unstale":"2024-08-17T18:22:35.296966+0000","last_undegraded":"2024-08-17T18:22:35.296966+0000","last_fullsized":"2024-08-17T18:22:35.296966+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:13:37.907363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":126,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.300205+0000","last_change":"2024-08-17T18:21:58.268737+0000","last_active":"2024-08-17T18:22:35.300205+0000","last_peered":"2024-08-17T18:22:35.300205+0000","last_clean":"2024-08-17T18:22:35.300205+0000","last_became_active":"2024-08-17T18:21:58.268409+0000","last_became_peered":"2024-08-17T18:21:58.268409+0000","last_unstale":"2024-08-17T18:22:35.300205+0000","last_undegraded":"2024-08-17T18:22:35.300205+0000","last_fullsized":"2024-08-17T18:22:35.300205+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:56.962967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":81,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.296479+0000","last_change":"2024-08-17T18:18:50.515676+0000","last_active":"2024-08-17T18:22:35.296479+0000","last_peered":"2024-08-17T18:22:35.296479+0000","last_clean":"2024-08-17T18:22:35.296479+0000","last_became_active":"2024-08-17T18:18:50.515313+0000","last_became_peered":"2024-08-17T18:18:50.515313+0000","last_unstale":"2024-08-17T18:22:35.296479+0000","last_undegraded":"2024-08-17T18:22:35.296479+0000","last_fullsized":"2024-08-17T18:22:35.296479+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:36.955526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2418,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.157581+0000","last_change":"2024-08-17T18:21:33.493429+0000","last_active":"2024-08-17T18:22:36.157581+0000","last_peered":"2024-08-17T18:22:36.157581+0000","last_clean":"2024-08-17T18:22:36.157581+0000","last_became_active":"2024-08-17T18:21:33.421176+0000","last_became_peered":"2024-08-17T18:21:33.421176+0000","last_unstale":"2024-08-17T18:22:36.157581+0000","last_undegraded":"2024-08-17T18:22:36.157581+0000","last_fullsized":"2024-08-17T18:22:36.157581+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:42.334437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":75,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.157674+0000","last_change":"2024-08-17T18:21:58.268206+0000","last_active":"2024-08-17T18:22:36.157674+0000","last_peered":"2024-08-17T18:22:36.157674+0000","last_clean":"2024-08-17T18:22:36.157674+0000","last_became_active":"2024-08-17T18:21:58.267898+0000","last_became_peered":"2024-08-17T18:21:58.267898+0000","last_unstale":"2024-08-17T18:22:36.157674+0000","last_undegraded":"2024-08-17T18:22:36.157674+0000","last_fullsized":"2024-08-17T18:22:36.157674+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:41.843446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":121,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.302150+0000","last_change":"2024-08-17T18:21:58.267796+0000","last_active":"2024-08-17T18:22:35.302150+0000","last_peered":"2024-08-17T18:22:35.302150+0000","last_clean":"2024-08-17T18:22:35.302150+0000","last_became_active":"2024-08-17T18:21:58.267504+0000","last_became_peered":"2024-08-17T18:21:58.267504+0000","last_unstale":"2024-08-17T18:22:35.302150+0000","last_undegraded":"2024-08-17T18:22:35.302150+0000","last_fullsized":"2024-08-17T18:22:35.302150+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:55:35.703349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":116,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.300116+0000","last_change":"2024-08-17T18:21:58.266675+0000","last_active":"2024-08-17T18:22:35.300116+0000","last_peered":"2024-08-17T18:22:35.300116+0000","last_clean":"2024-08-17T18:22:35.300116+0000","last_became_active":"2024-08-17T18:21:58.266286+0000","last_became_peered":"2024-08-17T18:21:58.266286+0000","last_unstale":"2024-08-17T18:22:35.300116+0000","last_undegraded":"2024-08-17T18:22:35.300116+0000","last_fullsized":"2024-08-17T18:22:35.300116+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:08.506169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1541,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.616348+0000","last_change":"2024-08-17T18:21:12.791040+0000","last_active":"2024-08-17T18:22:36.616348+0000","last_peered":"2024-08-17T18:22:36.616348+0000","last_clean":"2024-08-17T18:22:36.616348+0000","last_became_active":"2024-08-17T18:21:12.773669+0000","last_became_peered":"2024-08-17T18:21:12.773669+0000","last_unstale":"2024-08-17T18:22:36.616348+0000","last_undegraded":"2024-08-17T18:22:36.616348+0000","last_fullsized":"2024-08-17T18:22:36.616348+0000","mapping_epoch":1477,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:12.823106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":91,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.296415+0000","last_change":"2024-08-17T18:21:58.269847+0000","last_active":"2024-08-17T18:22:35.296415+0000","last_peered":"2024-08-17T18:22:35.296415+0000","last_clean":"2024-08-17T18:22:35.296415+0000","last_became_active":"2024-08-17T18:21:58.269541+0000","last_became_peered":"2024-08-17T18:21:58.269541+0000","last_unstale":"2024-08-17T18:22:35.296415+0000","last_undegraded":"2024-08-17T18:22:35.296415+0000","last_fullsized":"2024-08-17T18:22:35.296415+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:13.259546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1556,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.617414+0000","last_change":"2024-08-17T18:21:58.277027+0000","last_active":"2024-08-17T18:22:36.617414+0000","last_peered":"2024-08-17T18:22:36.617414+0000","last_clean":"2024-08-17T18:22:36.617414+0000","last_became_active":"2024-08-17T18:21:58.263795+0000","last_became_peered":"2024-08-17T18:21:58.263795+0000","last_unstale":"2024-08-17T18:22:36.617414+0000","last_undegraded":"2024-08-17T18:22:36.617414+0000","last_fullsized":"2024-08-17T18:22:36.617414+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:45:12.859260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":432,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.615363+0000","last_change":"2024-08-17T18:21:12.772737+0000","last_active":"2024-08-17T18:22:36.615363+0000","last_peered":"2024-08-17T18:22:36.615363+0000","last_clean":"2024-08-17T18:22:36.615363+0000","last_became_active":"2024-08-17T18:21:12.772318+0000","last_became_peered":"2024-08-17T18:21:12.772318+0000","last_unstale":"2024-08-17T18:22:36.615363+0000","last_undegraded":"2024-08-17T18:22:36.615363+0000","last_fullsized":"2024-08-17T18:22:36.615363+0000","mapping_epoch":1477,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:20.974581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1586,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.618003+0000","last_change":"2024-08-17T18:21:33.495232+0000","last_active":"2024-08-17T18:22:36.618003+0000","last_peered":"2024-08-17T18:22:36.618003+0000","last_clean":"2024-08-17T18:22:36.618003+0000","last_became_active":"2024-08-17T18:21:33.438606+0000","last_became_peered":"2024-08-17T18:21:33.438606+0000","last_unstale":"2024-08-17T18:22:36.618003+0000","last_undegraded":"2024-08-17T18:22:36.618003+0000","last_fullsized":"2024-08-17T18:22:36.618003+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:14.007910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":196,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-08-17T18:22:35.299814+0000","last_change":"2024-08-17T18:21:33.491306+0000","last_active":"2024-08-17T18:22:35.299814+0000","last_peered":"2024-08-17T18:22:35.299814+0000","last_clean":"2024-08-17T18:22:35.299814+0000","last_became_active":"2024-08-17T18:21:33.419876+0000","last_became_peered":"2024-08-17T18:21:33.419876+0000","last_unstale":"2024-08-17T18:22:35.299814+0000","last_undegraded":"2024-08-17T18:22:35.299814+0000","last_fullsized":"2024-08-17T18:22:35.299814+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:33:46.580078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2385,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.158405+0000","last_change":"2024-08-17T18:21:58.269877+0000","last_active":"2024-08-17T18:22:36.158405+0000","last_peered":"2024-08-17T18:22:36.158405+0000","last_clean":"2024-08-17T18:22:36.158405+0000","last_became_active":"2024-08-17T18:21:58.269057+0000","last_became_peered":"2024-08-17T18:21:58.269057+0000","last_unstale":"2024-08-17T18:22:36.158405+0000","last_undegraded":"2024-08-17T18:22:36.158405+0000","last_fullsized":"2024-08-17T18:22:36.158405+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:56:33.384257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1510,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.616537+0000","last_change":"2024-08-17T18:18:50.515245+0000","last_active":"2024-08-17T18:22:36.616537+0000","last_peered":"2024-08-17T18:22:36.616537+0000","last_clean":"2024-08-17T18:22:36.616537+0000","last_became_active":"2024-08-17T18:18:50.514785+0000","last_became_peered":"2024-08-17T18:18:50.514785+0000","last_unstale":"2024-08-17T18:22:36.616537+0000","last_undegraded":"2024-08-17T18:22:36.616537+0000","last_fullsized":"2024-08-17T18:22:36.616537+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:08.696581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":68,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.162413+0000","last_change":"2024-08-17T18:21:33.408565+0000","last_active":"2024-08-17T18:22:36.162413+0000","last_peered":"2024-08-17T18:22:36.162413+0000","last_clean":"2024-08-17T18:22:36.162413+0000","last_became_active":"2024-08-17T18:21:33.407832+0000","last_became_peered":"2024-08-17T18:21:33.407832+0000","last_unstale":"2024-08-17T18:22:36.162413+0000","last_undegraded":"2024-08-17T18:22:36.162413+0000","last_fullsized":"2024-08-17T18:22:36.162413+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:09:06.558530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":68,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.158990+0000","last_change":"2024-08-17T18:21:33.428616+0000","last_active":"2024-08-17T18:22:36.158990+0000","last_peered":"2024-08-17T18:22:36.158990+0000","last_clean":"2024-08-17T18:22:36.158990+0000","last_became_active":"2024-08-17T18:21:33.428368+0000","last_became_peered":"2024-08-17T18:21:33.428368+0000","last_unstale":"2024-08-17T18:22:36.158990+0000","last_undegraded":"2024-08-17T18:22:36.158990+0000","last_fullsized":"2024-08-17T18:22:36.158990+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:14.945169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":918,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.616665+0000","last_change":"2024-08-17T18:21:12.777384+0000","last_active":"2024-08-17T18:22:36.616665+0000","last_peered":"2024-08-17T18:22:36.616665+0000","last_clean":"2024-08-17T18:22:36.616665+0000","last_became_active":"2024-08-17T18:21:12.776205+0000","last_became_peered":"2024-08-17T18:21:12.776205+0000","last_unstale":"2024-08-17T18:22:36.616665+0000","last_undegraded":"2024-08-17T18:22:36.616665+0000","last_fullsized":"2024-08-17T18:22:36.616665+0000","mapping_epoch":1477,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:12:02.449650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1559,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.616582+0000","last_change":"2024-08-17T18:19:27.037702+0000","last_active":"2024-08-17T18:22:36.616582+0000","last_peered":"2024-08-17T18:22:36.616582+0000","last_clean":"2024-08-17T18:22:36.616582+0000","last_became_active":"2024-08-17T18:19:27.036532+0000","last_became_peered":"2024-08-17T18:19:27.036532+0000","last_unstale":"2024-08-17T18:22:36.616582+0000","last_undegraded":"2024-08-17T18:22:36.616582+0000","last_fullsized":"2024-08-17T18:22:36.616582+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:24.777460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2385,"reported_epoch":1518,"state":"active+clean","last_fresh":"2024-08-17T18:22:36.156423+0000","last_change":"2024-08-17T18:21:58.274634+0000","last_active":"2024-08-17T18:22:36.156423+0000","last_peered":"2024-08-17T18:22:36.156423+0000","last_clean":"2024-08-17T18:22:36.156423+0000","last_became_active":"2024-08-17T18:21:58.274259+0000","last_became_peered":"2024-08-17T18:21:58.274259+0000","last_unstale":"2024-08-17T18:22:36.156423+0000","last_undegraded":"2024-08-17T18:22:36.156423+0000","last_fullsized":"2024-08-17T18:22:36.156423+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:15.278544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":51,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":88,"num_bytes_recovered":11996,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234742,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1503496,"kb_used_data":13748,"kb_used_omap":1,"kb_used_meta":1489726,"kb_avail":92241656,"statfs":{"total":95995035648,"available":94455455744,"internally_reserved":0,"allocated":14077952,"data_stored":8496968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1525479768},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496377,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":919428,"kb_used_data":13744,"kb_used_omap":1,"kb_used_meta":905662,"kb_avail":92825724,"statfs":{"total":95995035648,"available":95053541376,"internally_reserved":0,"allocated":14073856,"data_stored":8497403,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":927398232},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":0,"seq":163208758015,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1343,"seq":5768141078688,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":790192,"kb_used_data":13152,"kb_used_omap":112,"kb_used_meta":776911,"kb_avail":92954960,"statfs":{"total":95995035648,"available":95185879040,"internally_reserved":0,"allocated":13467648,"data_stored":7901839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":115373,"internal_metadata":795557203},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":1454,"seq":6244882448425,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":285296,"kb_used_data":13148,"kb_used_omap":111,"kb_used_meta":272016,"kb_avail":93459856,"statfs":{"total":95995035648,"available":95702892544,"internally_reserved":0,"allocated":13463552,"data_stored":7902256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":114548,"internal_metadata":278544524},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":0,"seq":2173253452402,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471475093,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1520372,"kb_used_data":13732,"kb_used_omap":119,"kb_used_meta":1506504,"kb_avail":92224780,"statfs":{"total":95995035648,"available":94438174720,"internally_reserved":0,"allocated":14061568,"data_stored":8492631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":1542660830},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019650,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":318844,"kb_used_data":13164,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93426308,"statfs":{"total":95995035648,"available":95668539392,"internally_reserved":0,"allocated":13479936,"data_stored":7906522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:22:47.002 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd pool set .mgr pgp_num 1 2024-08-17T18:22:47.002 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:47.023 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:47.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:47.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:47.125 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6 2024-08-17T18:22:47.146 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:47.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:47.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.6' 2024-08-17T18:22:47.247 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:47.273 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:47.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:47.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:47.319 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:47.376 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:47.398 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:47.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:47.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:47.500 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:47.527 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:47.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:47.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:47.591 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:47 smithi130 ceph-mon[105521]: from='client.37461 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:22:47.591 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:47 smithi130 ceph-mon[105521]: pgmap v2965: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:47.591 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:47 smithi130 ceph-mon[105521]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:22:47.591 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:47 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi130-433706-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T18:22:47.591 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:47 smithi130 ceph-mon[105521]: osdmap e1521: 8 total, 8 up, 6 in 2024-08-17T18:22:47.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:47 smithi130 ceph-mon[110068]: from='client.37461 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:22:47.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:47 smithi130 ceph-mon[110068]: pgmap v2965: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:47.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:47 smithi130 ceph-mon[110068]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:22:47.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:47 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi130-433706-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T18:22:47.592 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:47 smithi130 ceph-mon[110068]: osdmap e1521: 8 total, 8 up, 6 in 2024-08-17T18:22:47.616 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:22:47.616 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:22:47.616 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:22:47.616 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:22:47.625 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:47.630 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:47.637 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:22:47.637 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:22:47.660 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:47.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:47.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:47.663 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:22:47.664 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:22:47.674 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:47.685 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:22:47.685 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:22:47.709 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:22:47.709 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:22:47.730 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:22:47.730 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:22:47.748 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:22:47.749 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:22:47.762 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:47.783 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:47.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:47.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:47 smithi179 ceph-mon[93519]: from='client.37461 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:22:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:47 smithi179 ceph-mon[93519]: pgmap v2965: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:47 smithi179 ceph-mon[93519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T18:22:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:47 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi130-433706-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T18:22:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:47 smithi179 ceph-mon[93519]: osdmap e1521: 8 total, 8 up, 6 in 2024-08-17T18:22:47.885 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:47.910 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:47.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:47.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:48.012 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:48.036 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:48.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:48.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:48.138 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:48.161 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:48.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:48.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:48.263 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:48.284 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:48.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:48.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:48.386 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:48.410 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:48.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:48.512 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:48.533 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:48.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:48.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:48.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:48 smithi130 ceph-mon[105521]: osdmap e1522: 8 total, 8 up, 6 in 2024-08-17T18:22:48.555 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:48 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3712688090' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:48.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:48 smithi130 ceph-mon[110068]: osdmap e1522: 8 total, 8 up, 6 in 2024-08-17T18:22:48.556 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:48 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3712688090' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:48.634 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:48.655 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:48.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:48.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:48.756 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:48.779 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:48.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:48.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:48.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:48 smithi179 ceph-mon[93519]: osdmap e1522: 8 total, 8 up, 6 in 2024-08-17T18:22:48.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:48 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3712688090' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:48.881 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:48.904 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:48.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:48.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:49.006 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:49.031 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:49.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:49.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:49.133 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:49.153 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:49.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:49.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:49.255 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:49.278 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:49.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:49.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:49.380 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:49.401 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:49.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:49.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:49.442 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.mirror_image_status (2513 ms) 2024-08-17T18:22:49.443 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-08-17T18:22:49.502 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:49.519 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.mirror_image_map (76 ms) 2024-08-17T18:22:49.519 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-08-17T18:22:49.525 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:49.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:49.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:49.527 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.mirror_instances (7 ms) 2024-08-17T18:22:49.527 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-08-17T18:22:49.547 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.mirror_snapshot (21 ms) 2024-08-17T18:22:49.547 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-08-17T18:22:49.549 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-08-17T18:22:49.549 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-08-17T18:22:49.551 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-08-17T18:22:49.551 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-08-17T18:22:49.554 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-08-17T18:22:49.554 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-08-17T18:22:49.560 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2024-08-17T18:22:49.560 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-08-17T18:22:49.563 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-08-17T18:22:49.563 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-08-17T18:22:49.567 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2024-08-17T18:22:49.567 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_image_add 2024-08-17T18:22:49.569 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-08-17T18:22:49.569 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-08-17T18:22:49.571 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-08-17T18:22:49.571 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_image_list 2024-08-17T18:22:49.574 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_image_list (2 ms) 2024-08-17T18:22:49.574 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-08-17T18:22:49.577 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-08-17T18:22:49.577 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.image_group_add 2024-08-17T18:22:49.579 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2024-08-17T18:22:49.579 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-08-17T18:22:49.583 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-08-17T18:22:49.583 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.image_group_get 2024-08-17T18:22:49.585 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.image_group_get (3 ms) 2024-08-17T18:22:49.585 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-08-17T18:22:49.587 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (1 ms) 2024-08-17T18:22:49.587 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-08-17T18:22:49.589 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-08-17T18:22:49.589 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-08-17T18:22:49.591 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (2 ms) 2024-08-17T18:22:49.591 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-08-17T18:22:49.594 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2024-08-17T18:22:49.594 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-08-17T18:22:49.596 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2024-08-17T18:22:49.596 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-08-17T18:22:49.598 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2024-08-17T18:22:49.599 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-08-17T18:22:49.613 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (14 ms) 2024-08-17T18:22:49.613 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-08-17T18:22:49.627 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:49.650 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:49.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:49.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:49.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:49 smithi130 ceph-mon[105521]: pgmap v2968: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:49.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:49 smithi130 ceph-mon[105521]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:22:49.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:49 smithi130 ceph-mon[110068]: pgmap v2968: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:49.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:49 smithi130 ceph-mon[110068]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:22:49.752 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:49.774 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:49.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:49.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:49.788 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (176 ms) 2024-08-17T18:22:49.789 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-08-17T18:22:49.791 INFO:tasks.workunit.client.0.smithi130.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1404-gf2c39d4d/rpm/el9/BUILD/ceph-17.2.7-1404-gf2c39d4d/src/test/cls_rbd/test_cls_rbd.cc:2792: Failure 2024-08-17T18:22:49.791 INFO:tasks.workunit.client.0.smithi130.stdout:Expected equality of these values: 2024-08-17T18:22:49.791 INFO:tasks.workunit.client.0.smithi130.stdout: 1U 2024-08-17T18:22:49.791 INFO:tasks.workunit.client.0.smithi130.stdout: Which is: 1 2024-08-17T18:22:49.791 INFO:tasks.workunit.client.0.smithi130.stdout: keys.size() 2024-08-17T18:22:49.791 INFO:tasks.workunit.client.0.smithi130.stdout: Which is: 3 2024-08-17T18:22:49.791 INFO:tasks.workunit.client.0.smithi130.stdout:[ FAILED ] TestClsRbd.group_snap_remove (2 ms) 2024-08-17T18:22:49.792 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-08-17T18:22:49.793 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2024-08-17T18:22:49.793 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.trash_methods 2024-08-17T18:22:49.801 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.trash_methods (8 ms) 2024-08-17T18:22:49.801 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.op_features 2024-08-17T18:22:49.807 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.op_features (6 ms) 2024-08-17T18:22:49.807 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.clone_parent 2024-08-17T18:22:49.826 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2024-08-17T18:22:49.826 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-08-17T18:22:49.835 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2024-08-17T18:22:49.835 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.clone_child 2024-08-17T18:22:49.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:49 smithi179 ceph-mon[93519]: pgmap v2968: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:49.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:49 smithi179 ceph-mon[93519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T18:22:49.847 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.clone_child (12 ms) 2024-08-17T18:22:49.847 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-08-17T18:22:49.853 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.namespace_methods (6 ms) 2024-08-17T18:22:49.853 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.migration 2024-08-17T18:22:49.862 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.migration (9 ms) 2024-08-17T18:22:49.862 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.migration_v1 2024-08-17T18:22:49.870 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.migration_v1 (8 ms) 2024-08-17T18:22:49.870 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-08-17T18:22:49.876 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:49.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:49.897 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:49.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:49.998 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:50.019 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:50.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:50.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:50.121 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:50.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:50.143 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:50.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:50.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:22:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:22:49] "GET /metrics HTTP/1.1" 200 34378 "" "Prometheus/2.51.0" 2024-08-17T18:22:50.244 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:50.265 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:50.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:50.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:50.367 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:50.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:50.390 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:50.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:50.491 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:50.514 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:50.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:50.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:50.588 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T18:22:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:22:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:22:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:22:50.615 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:50.653 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:50.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:50.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:50.755 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:50.777 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:50.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:50.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:50.878 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:50.900 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:50.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:50.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:51.002 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:51.024 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:51.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:51.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:51.125 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1 2024-08-17T18:22:51.147 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:51.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:51.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.1' 2024-08-17T18:22:51.249 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:51.271 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:51.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:51.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:51.373 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7 2024-08-17T18:22:51.394 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:51.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:51.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.7' 2024-08-17T18:22:51.406 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1536 ms) 2024-08-17T18:22:51.407 INFO:tasks.workunit.client.0.smithi130.stdout:[ RUN ] TestClsRbd.sparsify 2024-08-17T18:22:51.450 INFO:tasks.workunit.client.0.smithi130.stdout:[ OK ] TestClsRbd.sparsify (44 ms) 2024-08-17T18:22:51.496 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:51.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:51.519 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:51.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:51.620 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:51.641 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:51.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:51.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:51.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:51 smithi130 ceph-mon[105521]: pgmap v2969: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 921 op/s 2024-08-17T18:22:51.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:51 smithi130 ceph-mon[105521]: osdmap e1523: 8 total, 8 up, 6 in 2024-08-17T18:22:51.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:51 smithi130 ceph-mon[110068]: pgmap v2969: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 921 op/s 2024-08-17T18:22:51.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:51 smithi130 ceph-mon[110068]: osdmap e1523: 8 total, 8 up, 6 in 2024-08-17T18:22:51.743 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:51.765 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:51.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:51.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:51.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:51 smithi179 ceph-mon[93519]: pgmap v2969: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 1.9 MiB/s rd, 1.0 MiB/s wr, 921 op/s 2024-08-17T18:22:51.831 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:51 smithi179 ceph-mon[93519]: osdmap e1523: 8 total, 8 up, 6 in 2024-08-17T18:22:51.867 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5 2024-08-17T18:22:51.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:51.890 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:51.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.5' 2024-08-17T18:22:51.991 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0 2024-08-17T18:22:52.015 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:52.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:52.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.0' 2024-08-17T18:22:52.116 DEBUG:teuthology.orchestra.run.smithi179:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4 2024-08-17T18:22:52.138 INFO:teuthology.orchestra.run.smithi179.stderr:sudo: docker: command not found 2024-08-17T18:22:52.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:52.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi179 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.4' 2024-08-17T18:22:52.240 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:52.261 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:52.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:52.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:52.363 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:52.384 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:52.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:52.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:52.409 INFO:teuthology.orchestra.run.smithi130.stderr:set pool 1 pgp_num to 1 2024-08-17T18:22:52.419 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] 70 tests from TestClsRbd (5068 ms total) 2024-08-17T18:22:52.419 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T18:22:52.419 INFO:tasks.workunit.client.0.smithi130.stdout:[----------] Global test environment tear-down 2024-08-17T18:22:52.419 INFO:tasks.workunit.client.0.smithi130.stdout:[==========] 70 tests from 1 test suite ran. (7886 ms total) 2024-08-17T18:22:52.419 INFO:tasks.workunit.client.0.smithi130.stdout:[ PASSED ] 69 tests. 2024-08-17T18:22:52.419 INFO:tasks.workunit.client.0.smithi130.stdout:[ FAILED ] 1 test, listed below: 2024-08-17T18:22:52.419 INFO:tasks.workunit.client.0.smithi130.stdout:[ FAILED ] TestClsRbd.group_snap_remove 2024-08-17T18:22:52.419 INFO:tasks.workunit.client.0.smithi130.stdout: 2024-08-17T18:22:52.420 INFO:tasks.workunit.client.0.smithi130.stdout: 1 FAILED TEST 2024-08-17T18:22:52.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:52.432 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-08-17T18:22:52.432 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-08-17T18:22:52.486 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3 2024-08-17T18:22:52.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:52.539 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:52.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.3' 2024-08-17T18:22:52.639 DEBUG:teuthology.orchestra.run.smithi130:> sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2 2024-08-17T18:22:52.661 INFO:teuthology.orchestra.run.smithi130.stderr:sudo: docker: command not found 2024-08-17T18:22:52.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:22:52.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi130 with status 1: 'sudo docker kill -s 1 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd.2' 2024-08-17T18:22:52.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:52 smithi130 ceph-mon[110068]: osdmap e1524: 8 total, 8 up, 6 in 2024-08-17T18:22:52.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:52 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2381405492' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T18:22:52.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:52 smithi130 ceph-mon[105521]: osdmap e1524: 8 total, 8 up, 6 in 2024-08-17T18:22:52.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:52 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2381405492' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T18:22:52.699 INFO:tasks.thrashosds:joining thrashosds 2024-08-17T18:22:52.747 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:52 smithi179 ceph-mon[93519]: osdmap e1524: 8 total, 8 up, 6 in 2024-08-17T18:22:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:52 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2381405492' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"} : dispatch 2024-08-17T18:22:53.599 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:53 smithi130 ceph-mon[105521]: pgmap v2972: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 2.1 MiB/s rd, 1.1 MiB/s wr, 1.03k op/s 2024-08-17T18:22:53.599 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:53 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2381405492' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T18:22:53.599 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:53 smithi130 ceph-mon[105521]: osdmap e1525: 8 total, 8 up, 6 in 2024-08-17T18:22:53.600 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:53 smithi130 ceph-mon[110068]: pgmap v2972: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 2.1 MiB/s rd, 1.1 MiB/s wr, 1.03k op/s 2024-08-17T18:22:53.600 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:53 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2381405492' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T18:22:53.600 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:53 smithi130 ceph-mon[110068]: osdmap e1525: 8 total, 8 up, 6 in 2024-08-17T18:22:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:53 smithi179 ceph-mon[93519]: pgmap v2972: 105 pgs: 105 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 2.1 MiB/s rd, 1.1 MiB/s wr, 1.03k op/s 2024-08-17T18:22:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:53 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2381405492' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2024-08-17T18:22:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:53 smithi179 ceph-mon[93519]: osdmap e1525: 8 total, 8 up, 6 in 2024-08-17T18:22:55.425 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:22:55.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:55 smithi130 ceph-mon[105521]: pgmap v2974: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:55 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/176820798' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:22:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:55 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:22:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:55 smithi130 ceph-mon[110068]: pgmap v2974: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:55 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/176820798' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:22:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:55 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:22:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:55 smithi179 ceph-mon[93519]: pgmap v2974: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:55 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/176820798' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:22:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:55 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:22:56.290 INFO:teuthology.orchestra.run.smithi130.stdout:{ 2024-08-17T18:22:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: "ops": [], 2024-08-17T18:22:56.290 INFO:teuthology.orchestra.run.smithi130.stdout: "num_ops": 0 2024-08-17T18:22:56.290 INFO:teuthology.orchestra.run.smithi130.stdout:} 2024-08-17T18:22:56.591 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:56 smithi130 ceph-mon[105521]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:22:56.591 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:56 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:22:56.591 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:56 smithi130 ceph-mon[105521]: osdmap e1526: 8 total, 8 up, 6 in 2024-08-17T18:22:56.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:56 smithi130 ceph-mon[110068]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:22:56.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:56 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:22:56.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:56 smithi130 ceph-mon[110068]: osdmap e1526: 8 total, 8 up, 6 in 2024-08-17T18:22:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:56 smithi179 ceph-mon[93519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T18:22:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:56 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:22:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:56 smithi179 ceph-mon[93519]: osdmap e1526: 8 total, 8 up, 6 in 2024-08-17T18:22:57.016 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T18:22:57.436 INFO:teuthology.orchestra.run.smithi130.stdout:osd.0: {} 2024-08-17T18:22:57.436 INFO:teuthology.orchestra.run.smithi130.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T18:22:57.453 INFO:teuthology.orchestra.run.smithi130.stdout:osd.1: {} 2024-08-17T18:22:57.453 INFO:teuthology.orchestra.run.smithi130.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T18:22:57.469 INFO:teuthology.orchestra.run.smithi130.stdout:osd.2: {} 2024-08-17T18:22:57.469 INFO:teuthology.orchestra.run.smithi130.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T18:22:57.485 INFO:teuthology.orchestra.run.smithi130.stdout:osd.3: {} 2024-08-17T18:22:57.486 INFO:teuthology.orchestra.run.smithi130.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T18:22:57.501 INFO:teuthology.orchestra.run.smithi130.stdout:osd.4: {} 2024-08-17T18:22:57.502 INFO:teuthology.orchestra.run.smithi130.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T18:22:57.517 INFO:teuthology.orchestra.run.smithi130.stdout:osd.5: {} 2024-08-17T18:22:57.518 INFO:teuthology.orchestra.run.smithi130.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T18:22:57.533 INFO:teuthology.orchestra.run.smithi130.stdout:osd.6: {} 2024-08-17T18:22:57.533 INFO:teuthology.orchestra.run.smithi130.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T18:22:57.549 INFO:teuthology.orchestra.run.smithi130.stdout:osd.7: {} 2024-08-17T18:22:57.549 INFO:teuthology.orchestra.run.smithi130.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T18:22:57.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:57 smithi130 ceph-mon[105521]: pgmap v2976: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:57.662 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:22:57.663 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:57 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2635422841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:57.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:57 smithi130 ceph-mon[110068]: pgmap v2976: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:57.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:22:57.663 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:57 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2635422841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:57 smithi179 ceph-mon[93519]: pgmap v2976: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:22:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:57 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2635422841' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T18:22:58.861 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:22:59.051 INFO:tasks.thrashosds.thrasher:inning osd 2024-08-17T18:22:59.052 INFO:tasks.thrashosds.thrasher:Adding osd 1 2024-08-17T18:22:59.052 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd in 1 2024-08-17T18:22:59.699 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:22:59 smithi130 ceph-mon[105521]: pgmap v2977: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:59.699 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:22:59 smithi130 ceph-mon[110068]: pgmap v2977: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:22:59 smithi179 ceph-mon[93519]: pgmap v2977: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:22:59.982 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:22:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:22:59] "GET /metrics HTTP/1.1" 200 34366 "" "Prometheus/2.51.0" 2024-08-17T18:23:00.281 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T18:23:00.559 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:23:00.559 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:23:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:23:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:23:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:23:01.021 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:01.536 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T18:23:01.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:01 smithi179 ceph-mon[93519]: pgmap v2978: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:23:01.682 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:01 smithi130 ceph-mon[105521]: pgmap v2978: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:23:01.682 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:01 smithi130 ceph-mon[110068]: pgmap v2978: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:23:02.265 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:02.301 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:03.686 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:03 smithi130 ceph-mon[105521]: pgmap v2979: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:23:03.713 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:03 smithi130 ceph-mon[110068]: pgmap v2979: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:23:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:03 smithi179 ceph-mon[93519]: pgmap v2979: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:23:04.440 INFO:teuthology.orchestra.run.smithi130.stderr:marked in osd.1. 2024-08-17T18:23:04.587 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:04 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2627921265' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T18:23:04.587 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:04 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T18:23:04.232+0000 7fa856f19640 -1 mon.a@0(leader).osd e1526 definitely_dead 0 2024-08-17T18:23:04.588 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:04 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2627921265' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T18:23:04.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:04 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2627921265' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T18:23:05.284 INFO:tasks.thrashosds.thrasher:Added osd 1 2024-08-17T18:23:05.285 INFO:tasks.thrashosds.thrasher:inning osd 2024-08-17T18:23:05.285 INFO:tasks.thrashosds.thrasher:Adding osd 5 2024-08-17T18:23:05.285 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd in 5 2024-08-17T18:23:05.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:05 smithi130 ceph-mon[105521]: pgmap v2980: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:23:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:05 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2627921265' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T18:23:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:05 smithi130 ceph-mon[105521]: osdmap e1527: 8 total, 8 up, 7 in 2024-08-17T18:23:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:05 smithi130 ceph-mon[110068]: pgmap v2980: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:23:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:05 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2627921265' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T18:23:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:05 smithi130 ceph-mon[110068]: osdmap e1527: 8 total, 8 up, 7 in 2024-08-17T18:23:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:05 smithi179 ceph-mon[93519]: pgmap v2980: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:23:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:05 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2627921265' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T18:23:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:05 smithi179 ceph-mon[93519]: osdmap e1527: 8 total, 8 up, 7 in 2024-08-17T18:23:06.234 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:06.793 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:06 smithi130 ceph-mon[105521]: osdmap e1528: 8 total, 8 up, 7 in 2024-08-17T18:23:06.793 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:06 smithi130 ceph-mon[110068]: osdmap e1528: 8 total, 8 up, 7 in 2024-08-17T18:23:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:06 smithi179 ceph-mon[93519]: osdmap e1528: 8 total, 8 up, 7 in 2024-08-17T18:23:06.905 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:23:06.905 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:23:07.708 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:07 smithi130 ceph-mon[105521]: pgmap v2983: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:23:07.708 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:07 smithi130 ceph-mon[110068]: pgmap v2983: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:23:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:07 smithi179 ceph-mon[93519]: pgmap v2983: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2024-08-17T18:23:08.464 INFO:teuthology.orchestra.run.smithi130.stderr:noscrub is unset 2024-08-17T18:23:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:08 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4102162079' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:23:08.850 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:08 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4102162079' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:23:08.850 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:08 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4102162079' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T18:23:09.171 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T18:23:09.589 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T18:23:09.623 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:09.741 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:09 smithi130 ceph-mon[110068]: pgmap v2984: 73 pgs: 73 active+clean; 582 KiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 4 B/s, 0 objects/s recovering 2024-08-17T18:23:09.741 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:09 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4102162079' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:23:09.741 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:09 smithi130 ceph-mon[110068]: osdmap e1529: 8 total, 8 up, 7 in 2024-08-17T18:23:09.741 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:09 smithi130 ceph-mon[105521]: pgmap v2984: 73 pgs: 73 active+clean; 582 KiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 4 B/s, 0 objects/s recovering 2024-08-17T18:23:09.741 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:09 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4102162079' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:23:09.741 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:09 smithi130 ceph-mon[105521]: osdmap e1529: 8 total, 8 up, 7 in 2024-08-17T18:23:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:09 smithi179 ceph-mon[93519]: pgmap v2984: 73 pgs: 73 active+clean; 582 KiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 4 B/s, 0 objects/s recovering 2024-08-17T18:23:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:09 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4102162079' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T18:23:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:09 smithi179 ceph-mon[93519]: osdmap e1529: 8 total, 8 up, 7 in 2024-08-17T18:23:09.998 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:23:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:23:09] "GET /metrics HTTP/1.1" 200 34366 "" "Prometheus/2.51.0" 2024-08-17T18:23:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:23:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:23:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:23:10.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:10 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T18:23:10.483+0000 7fa856f19640 -1 mon.a@0(leader).osd e1529 definitely_dead 0 2024-08-17T18:23:11.325 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:11.472 INFO:teuthology.orchestra.run.smithi130.stderr:marked in osd.5. 2024-08-17T18:23:11.568 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:11 smithi130 ceph-mon[105521]: pgmap v2986: 73 pgs: 73 active+clean; 582 KiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 6 B/s, 0 objects/s recovering 2024-08-17T18:23:11.568 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:11 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1054132641' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T18:23:11.568 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:11 smithi130 ceph-mon[110068]: pgmap v2986: 73 pgs: 73 active+clean; 582 KiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 6 B/s, 0 objects/s recovering 2024-08-17T18:23:11.568 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:11 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1054132641' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T18:23:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:11 smithi179 ceph-mon[93519]: pgmap v2986: 73 pgs: 73 active+clean; 582 KiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 6 B/s, 0 objects/s recovering 2024-08-17T18:23:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:11 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1054132641' entity='client.admin' cmd={"prefix": "osd in", "ids": ["5"]} : dispatch 2024-08-17T18:23:12.145 INFO:tasks.thrashosds.thrasher:Added osd 5 2024-08-17T18:23:12.145 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-08-17T18:23:12.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:12 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1054132641' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-17T18:23:12.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:12 smithi130 ceph-mon[105521]: osdmap e1530: 8 total, 8 up, 8 in 2024-08-17T18:23:12.791 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:23:12.791 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:12 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1054132641' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-17T18:23:12.791 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:12 smithi130 ceph-mon[110068]: osdmap e1530: 8 total, 8 up, 8 in 2024-08-17T18:23:12.791 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:23:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:12 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1054132641' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2024-08-17T18:23:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:12 smithi179 ceph-mon[93519]: osdmap e1530: 8 total, 8 up, 8 in 2024-08-17T18:23:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:23:13.103 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:13.490 INFO:teuthology.orchestra.run.smithi130.stderr:nodeep-scrub is unset 2024-08-17T18:23:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:13 smithi179 ceph-mon[93519]: pgmap v2988: 73 pgs: 73 active+clean; 582 KiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 6 B/s, 0 objects/s recovering 2024-08-17T18:23:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:13 smithi179 ceph-mon[93519]: osdmap e1531: 8 total, 8 up, 8 in 2024-08-17T18:23:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:13 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1542180871' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:23:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:13 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:23:13.865 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:13 smithi130 ceph-mon[105521]: pgmap v2988: 73 pgs: 73 active+clean; 582 KiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 6 B/s, 0 objects/s recovering 2024-08-17T18:23:13.865 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:13 smithi130 ceph-mon[105521]: osdmap e1531: 8 total, 8 up, 8 in 2024-08-17T18:23:13.865 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:13 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1542180871' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:23:13.865 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:13 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:23:13.865 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:13 smithi130 ceph-mon[110068]: pgmap v2988: 73 pgs: 73 active+clean; 582 KiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 6 B/s, 0 objects/s recovering 2024-08-17T18:23:13.865 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:13 smithi130 ceph-mon[110068]: osdmap e1531: 8 total, 8 up, 8 in 2024-08-17T18:23:13.865 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:13 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1542180871' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:23:13.866 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:13 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T18:23:14.807 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:14 smithi130 ceph-mon[105521]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:23:14.807 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:14 smithi130 ceph-mon[105521]: Cluster is now healthy 2024-08-17T18:23:14.807 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:14 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:23:14.807 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:14 smithi130 ceph-mon[105521]: osdmap e1532: 8 total, 8 up, 8 in 2024-08-17T18:23:14.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:14 smithi130 ceph-mon[110068]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:23:14.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:14 smithi130 ceph-mon[110068]: Cluster is now healthy 2024-08-17T18:23:14.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:14 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:23:14.808 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:14 smithi130 ceph-mon[110068]: osdmap e1532: 8 total, 8 up, 8 in 2024-08-17T18:23:14.818 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:23:14.819 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:23:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:14 smithi179 ceph-mon[93519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T18:23:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:14 smithi179 ceph-mon[93519]: Cluster is now healthy 2024-08-17T18:23:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:14 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T18:23:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:14 smithi179 ceph-mon[93519]: osdmap e1532: 8 total, 8 up, 8 in 2024-08-17T18:23:15.383 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T18:23:15.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:15 smithi130 ceph-mon[105521]: pgmap v2991: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 121 B/s, 1 objects/s recovering 2024-08-17T18:23:15.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:15 smithi130 ceph-mon[110068]: pgmap v2991: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 121 B/s, 1 objects/s recovering 2024-08-17T18:23:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:15 smithi179 ceph-mon[93519]: pgmap v2991: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 121 B/s, 1 objects/s recovering 2024-08-17T18:23:16.946 INFO:teuthology.orchestra.run.smithi130.stdout:mon.a: {} 2024-08-17T18:23:16.947 INFO:teuthology.orchestra.run.smithi130.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T18:23:16.957 INFO:teuthology.orchestra.run.smithi130.stdout:mon.b: {} 2024-08-17T18:23:16.957 INFO:teuthology.orchestra.run.smithi130.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T18:23:16.972 INFO:teuthology.orchestra.run.smithi130.stdout:mon.c: {} 2024-08-17T18:23:16.972 INFO:teuthology.orchestra.run.smithi130.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T18:23:16.983 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:17 smithi130 ceph-mon[105521]: pgmap v2992: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 106 B/s, 1 objects/s recovering 2024-08-17T18:23:17.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:17 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1344176913' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T18:23:17.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:17 smithi130 ceph-mon[110068]: pgmap v2992: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 106 B/s, 1 objects/s recovering 2024-08-17T18:23:17.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:17 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1344176913' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T18:23:17.740 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T18:23:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:17 smithi179 ceph-mon[93519]: pgmap v2992: 73 pgs: 3 peering, 70 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 106 B/s, 1 objects/s recovering 2024-08-17T18:23:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:17 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1344176913' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T18:23:18.460 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:19.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:19 smithi130 ceph-mon[105521]: pgmap v2993: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 102 B/s, 1 objects/s recovering 2024-08-17T18:23:19.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:19 smithi130 ceph-mon[110068]: pgmap v2993: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 102 B/s, 1 objects/s recovering 2024-08-17T18:23:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:19 smithi179 ceph-mon[93519]: pgmap v2993: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 102 B/s, 1 objects/s recovering 2024-08-17T18:23:20.134 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:23:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:23:19] "GET /metrics HTTP/1.1" 200 34367 "" "Prometheus/2.51.0" 2024-08-17T18:23:20.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:23:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:23:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:23:21.233 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:23:21.233 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:23:21.496 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T18:23:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:21 smithi179 ceph-mon[93519]: pgmap v2994: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 79 B/s, 0 objects/s recovering 2024-08-17T18:23:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:21 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3559083638' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:23:21.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:21 smithi130 ceph-mon[105521]: pgmap v2994: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 79 B/s, 0 objects/s recovering 2024-08-17T18:23:21.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:21 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3559083638' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:23:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:21 smithi130 ceph-mon[110068]: pgmap v2994: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 79 B/s, 0 objects/s recovering 2024-08-17T18:23:21.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:21 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3559083638' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:23:22.235 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T18:23:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:22 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3559083638' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T18:23:22.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:22 smithi179 ceph-mon[93519]: osdmap e1533: 8 total, 8 up, 8 in 2024-08-17T18:23:22.901 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:22 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3559083638' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T18:23:22.901 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:22 smithi130 ceph-mon[105521]: osdmap e1533: 8 total, 8 up, 8 in 2024-08-17T18:23:22.901 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:22 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3559083638' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T18:23:22.901 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:22 smithi130 ceph-mon[110068]: osdmap e1533: 8 total, 8 up, 8 in 2024-08-17T18:23:22.927 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:22.937 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 1 1 2024-08-17T18:23:23.685 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:23.808 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:23 smithi130 ceph-mon[105521]: pgmap v2996: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2024-08-17T18:23:23.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:23 smithi130 ceph-mon[110068]: pgmap v2996: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2024-08-17T18:23:23.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:23 smithi179 ceph-mon[93519]: pgmap v2996: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 8 B/s, 0 objects/s recovering 2024-08-17T18:23:24.794 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:23:24.794 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:23:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:25 smithi179 ceph-mon[93519]: pgmap v2997: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:25.911 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:25 smithi130 ceph-mon[105521]: pgmap v2997: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:25.911 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:25 smithi130 ceph-mon[110068]: pgmap v2997: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:25.942 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T18:23:26.546 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:27 smithi130 ceph-mon[105521]: pgmap v2998: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:27 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:23:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:27 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4008856248' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:23:27.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:27 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:23:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:27 smithi130 ceph-mon[110068]: pgmap v2998: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:27 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:27.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:23:27.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:27 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4008856248' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:23:27.668 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:27 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:23:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:27 smithi179 ceph-mon[93519]: pgmap v2998: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:27 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:23:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:27 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4008856248' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:23:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:27 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T18:23:28.362 INFO:teuthology.orchestra.run.smithi130.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-08-17T18:23:28.810 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:23:28.810 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:23:29.416 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:29 smithi130 ceph-mon[110068]: pgmap v2999: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:29.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:29 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-17T18:23:29.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:29 smithi130 ceph-mon[110068]: osdmap e1534: 8 total, 8 up, 8 in 2024-08-17T18:23:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:29 smithi130 ceph-mon[105521]: pgmap v2999: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:29 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-17T18:23:29.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:29 smithi130 ceph-mon[105521]: osdmap e1534: 8 total, 8 up, 8 in 2024-08-17T18:23:29.511 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T18:23:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:29 smithi179 ceph-mon[93519]: pgmap v2999: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:29 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-17T18:23:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:29 smithi179 ceph-mon[93519]: osdmap e1534: 8 total, 8 up, 8 in 2024-08-17T18:23:30.020 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T18:23:30.109 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:30.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:23:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:23:29] "GET /metrics HTTP/1.1" 200 34358 "" "Prometheus/2.51.0" 2024-08-17T18:23:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:23:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:23:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:23:31.644 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:31 smithi130 ceph-mon[105521]: pgmap v3001: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:31.644 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:31 smithi130 ceph-mon[110068]: pgmap v3001: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:31.836 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:31 smithi179 ceph-mon[93519]: pgmap v3001: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:31.844 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:23:31.844 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:23:32.514 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T18:23:33.262 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:33.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:33 smithi130 ceph-mon[105521]: pgmap v3002: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:33.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:33 smithi130 ceph-mon[110068]: pgmap v3002: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:33 smithi179 ceph-mon[93519]: pgmap v3002: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:35.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:35 smithi130 ceph-mon[105521]: pgmap v3003: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:35.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:35 smithi130 ceph-mon[110068]: pgmap v3003: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:35 smithi179 ceph-mon[93519]: pgmap v3003: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:36.065 INFO:teuthology.orchestra.run.smithi130.stdout:ERROR: (22) Invalid argument 2024-08-17T18:23:36.065 INFO:teuthology.orchestra.run.smithi130.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T18:23:36.375 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T18:23:36.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:36 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3818805477' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T18:23:36.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:36 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3818805477' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T18:23:36.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:36 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3818805477' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T18:23:37.364 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 4 1 2024-08-17T18:23:37.629 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:37.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:37 smithi130 ceph-mon[105521]: pgmap v3004: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:37.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:37 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3818805477' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T18:23:37.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:37 smithi130 ceph-mon[105521]: osdmap e1535: 8 total, 8 up, 8 in 2024-08-17T18:23:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:37 smithi130 ceph-mon[110068]: pgmap v3004: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:37 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3818805477' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T18:23:37.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:37 smithi130 ceph-mon[110068]: osdmap e1535: 8 total, 8 up, 8 in 2024-08-17T18:23:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:37 smithi179 ceph-mon[93519]: pgmap v3004: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:37 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3818805477' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T18:23:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:37 smithi179 ceph-mon[93519]: osdmap e1535: 8 total, 8 up, 8 in 2024-08-17T18:23:39.417 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:39 smithi130 ceph-mon[105521]: pgmap v3006: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:39.417 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:39 smithi130 ceph-mon[110068]: pgmap v3006: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:39.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:39 smithi179 ceph-mon[93519]: pgmap v3006: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:40.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:23:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:23:39] "GET /metrics HTTP/1.1" 200 34358 "" "Prometheus/2.51.0" 2024-08-17T18:23:40.390 INFO:teuthology.orchestra.run.smithi130.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-08-17T18:23:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:40 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3476947311' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T18:23:40.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:40 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T18:23:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:23:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:23:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:23:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:40 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3476947311' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T18:23:40.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:40 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T18:23:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:40 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3476947311' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T18:23:40.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:40 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T18:23:41.093 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T18:23:41.365 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:41.641 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:41 smithi130 ceph-mon[105521]: pgmap v3007: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:41.641 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:41 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T18:23:41.641 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:41 smithi130 ceph-mon[105521]: osdmap e1536: 8 total, 8 up, 8 in 2024-08-17T18:23:41.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:41 smithi130 ceph-mon[110068]: pgmap v3007: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:41.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:41 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T18:23:41.641 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:41 smithi130 ceph-mon[110068]: osdmap e1536: 8 total, 8 up, 8 in 2024-08-17T18:23:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:41 smithi179 ceph-mon[93519]: pgmap v3007: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:41 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T18:23:41.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:41 smithi179 ceph-mon[93519]: osdmap e1536: 8 total, 8 up, 8 in 2024-08-17T18:23:42.562 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:23:42.563 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:23:42.563 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:23:42.563 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:23:42.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:23:42.593 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:23:43.398 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T18:23:43.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:43 smithi130 ceph-mon[105521]: pgmap v3009: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:43 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1210181270' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T18:23:43.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:43 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T18:23:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:43 smithi130 ceph-mon[110068]: pgmap v3009: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:43 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1210181270' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T18:23:43.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:43 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T18:23:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:43 smithi179 ceph-mon[93519]: pgmap v3009: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:43 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1210181270' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T18:23:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:43 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T18:23:44.239 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 5 1 2024-08-17T18:23:44.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:44 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T18:23:44.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:44 smithi130 ceph-mon[105521]: osdmap e1537: 8 total, 8 up, 8 in 2024-08-17T18:23:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:44 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T18:23:44.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:44 smithi130 ceph-mon[110068]: osdmap e1537: 8 total, 8 up, 8 in 2024-08-17T18:23:44.782 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:44.783 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:44 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T18:23:44.783 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:44 smithi179 ceph-mon[93519]: osdmap e1537: 8 total, 8 up, 8 in 2024-08-17T18:23:45.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:45 smithi130 ceph-mon[105521]: pgmap v3011: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:45.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:45 smithi130 ceph-mon[110068]: pgmap v3011: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:45.670 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:45 smithi179 ceph-mon[93519]: pgmap v3011: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:47.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:46 smithi130 ceph-mon[105521]: pgmap v3012: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:47.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:46 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:47.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:46 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:47.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:46 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2487177660' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T18:23:47.167 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:46 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T18:23:47.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:47 smithi130 ceph-mon[110068]: pgmap v3012: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:47.167 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:47 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:47.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:47 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:47.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:47 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2487177660' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T18:23:47.168 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:47 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T18:23:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:46 smithi179 ceph-mon[93519]: pgmap v3012: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:46 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:46 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:46 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2487177660' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T18:23:47.343 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:46 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T18:23:47.417 INFO:teuthology.orchestra.run.smithi130.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-08-17T18:23:48.489 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T18:23:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:48 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T18:23:48.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:48 smithi179 ceph-mon[93519]: osdmap e1538: 8 total, 8 up, 8 in 2024-08-17T18:23:48.856 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:48.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:48 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T18:23:48.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:48 smithi130 ceph-mon[105521]: osdmap e1538: 8 total, 8 up, 8 in 2024-08-17T18:23:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:48 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T18:23:48.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:48 smithi130 ceph-mon[110068]: osdmap e1538: 8 total, 8 up, 8 in 2024-08-17T18:23:49.615 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:49 smithi130 ceph-mon[105521]: pgmap v3014: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:49.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:49 smithi130 ceph-mon[110068]: pgmap v3014: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:49.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:49 smithi179 ceph-mon[93519]: pgmap v3014: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:49.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:23:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:23:49] "GET /metrics HTTP/1.1" 200 34363 "" "Prometheus/2.51.0" 2024-08-17T18:23:50.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:23:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:23:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:23:51.598 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:51 smithi130 ceph-mon[105521]: pgmap v3015: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:51.598 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:51 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:51.598 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:51 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:51.598 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:51 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:23:51.598 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:51 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:23:51.598 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:51 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:51.599 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:51 smithi130 ceph-mon[110068]: pgmap v3015: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:51.599 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:51 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:51.599 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:51 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:51.599 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:51 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:23:51.599 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:51 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:23:51.599 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:51 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:51.794 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:51 smithi179 ceph-mon[93519]: pgmap v3015: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:51.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:51 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:51.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:51 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:51.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:51 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:23:51.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:51 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:23:51.795 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:51 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:23:52.360 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T18:23:52.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:52 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/131462384' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T18:23:52.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:52 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/131462384' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T18:23:52.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:52 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/131462384' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T18:23:53.090 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 6 1 2024-08-17T18:23:53.352 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:53.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:53 smithi130 ceph-mon[105521]: pgmap v3016: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:53.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:53 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/131462384' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T18:23:53.616 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:53 smithi130 ceph-mon[105521]: osdmap e1539: 8 total, 8 up, 8 in 2024-08-17T18:23:53.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:53 smithi130 ceph-mon[110068]: pgmap v3016: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:53.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:53 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/131462384' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T18:23:53.616 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:53 smithi130 ceph-mon[110068]: osdmap e1539: 8 total, 8 up, 8 in 2024-08-17T18:23:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:53 smithi179 ceph-mon[93519]: pgmap v3016: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:53 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/131462384' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T18:23:53.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:53 smithi179 ceph-mon[93519]: osdmap e1539: 8 total, 8 up, 8 in 2024-08-17T18:23:55.366 INFO:teuthology.orchestra.run.smithi130.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-08-17T18:23:55.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:55 smithi130 ceph-mon[105521]: pgmap v3018: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:55 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2608050482' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T18:23:55.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:55 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T18:23:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:55 smithi130 ceph-mon[110068]: pgmap v3018: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:55 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2608050482' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T18:23:55.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:55 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T18:23:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:55 smithi179 ceph-mon[93519]: pgmap v3018: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:55 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2608050482' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T18:23:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:55 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T18:23:56.043 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T18:23:56.299 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:56 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T18:23:56.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:56 smithi130 ceph-mon[105521]: osdmap e1540: 8 total, 8 up, 8 in 2024-08-17T18:23:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:56 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T18:23:56.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:56 smithi130 ceph-mon[110068]: osdmap e1540: 8 total, 8 up, 8 in 2024-08-17T18:23:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:56 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T18:23:56.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:56 smithi179 ceph-mon[93519]: osdmap e1540: 8 total, 8 up, 8 in 2024-08-17T18:23:57.599 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:57 smithi130 ceph-mon[105521]: pgmap v3020: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:57.599 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:23:57.599 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:57 smithi130 ceph-mon[110068]: pgmap v3020: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:57.599 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:23:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:57 smithi179 ceph-mon[93519]: pgmap v3020: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:23:58.387 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T18:23:58.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:58 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1231783129' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:23:58.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:58 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:23:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:58 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1231783129' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:23:58.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:58 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:23:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:58 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1231783129' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:23:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:58 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:23:59.209 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 7 1 2024-08-17T18:23:59.465 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:23:59.588 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:59 smithi130 ceph-mon[105521]: pgmap v3021: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:59.588 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:59 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:23:59.588 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:23:59 smithi130 ceph-mon[105521]: osdmap e1541: 8 total, 8 up, 8 in 2024-08-17T18:23:59.588 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:59 smithi130 ceph-mon[110068]: pgmap v3021: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:59.588 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:59 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:23:59.588 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:23:59 smithi130 ceph-mon[110068]: osdmap e1541: 8 total, 8 up, 8 in 2024-08-17T18:23:59.845 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:59 smithi179 ceph-mon[93519]: pgmap v3021: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:23:59.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:59 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:23:59.846 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:23:59 smithi179 ceph-mon[93519]: osdmap e1541: 8 total, 8 up, 8 in 2024-08-17T18:23:59.885 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:23:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:23:59] "GET /metrics HTTP/1.1" 200 34364 "" "Prometheus/2.51.0" 2024-08-17T18:24:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:24:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:24:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:24:01.385 INFO:teuthology.orchestra.run.smithi130.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-08-17T18:24:01.580 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:01 smithi179 ceph-mon[93519]: pgmap v3023: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:01.580 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:01 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4212130691' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:24:01.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:01 smithi130 ceph-mon[110068]: pgmap v3023: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:01 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4212130691' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:24:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:01 smithi130 ceph-mon[105521]: pgmap v3023: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:01 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4212130691' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T18:24:02.199 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T18:24:02.456 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:02.579 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:02 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4212130691' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:24:02.579 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:02 smithi130 ceph-mon[105521]: osdmap e1542: 8 total, 8 up, 8 in 2024-08-17T18:24:02.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:02 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4212130691' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:24:02.579 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:02 smithi130 ceph-mon[110068]: osdmap e1542: 8 total, 8 up, 8 in 2024-08-17T18:24:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:02 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4212130691' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-17T18:24:02.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:02 smithi179 ceph-mon[93519]: osdmap e1542: 8 total, 8 up, 8 in 2024-08-17T18:24:03.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:03 smithi130 ceph-mon[105521]: pgmap v3025: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:03.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:03 smithi130 ceph-mon[110068]: pgmap v3025: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:03 smithi179 ceph-mon[93519]: pgmap v3025: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:05.411 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T18:24:05.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:05 smithi130 ceph-mon[105521]: pgmap v3026: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:05.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:05 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2730922763' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:24:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:05 smithi130 ceph-mon[110068]: pgmap v3026: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:05.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:05 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2730922763' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:24:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:05 smithi179 ceph-mon[93519]: pgmap v3026: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:05 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2730922763' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:24:06.162 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 0 1 2024-08-17T18:24:06.425 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:06.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:06 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2730922763' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:24:06.683 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:06 smithi130 ceph-mon[105521]: osdmap e1543: 8 total, 8 up, 8 in 2024-08-17T18:24:06.683 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:06 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2730922763' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:24:06.683 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:06 smithi130 ceph-mon[110068]: osdmap e1543: 8 total, 8 up, 8 in 2024-08-17T18:24:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:06 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2730922763' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:24:06.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:06 smithi179 ceph-mon[93519]: osdmap e1543: 8 total, 8 up, 8 in 2024-08-17T18:24:07.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:07 smithi130 ceph-mon[110068]: pgmap v3028: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:07.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:07 smithi130 ceph-mon[105521]: pgmap v3028: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:07 smithi179 ceph-mon[93519]: pgmap v3028: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:08.429 INFO:teuthology.orchestra.run.smithi130.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-08-17T18:24:08.666 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:08 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3139192820' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:24:08.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:08 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:24:08.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:08 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3139192820' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:24:08.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:08 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:24:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:08 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3139192820' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:24:08.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:08 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T18:24:09.071 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T18:24:09.325 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:09.717 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:09 smithi130 ceph-mon[105521]: pgmap v3029: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:09.717 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:09 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:24:09.717 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:09 smithi130 ceph-mon[105521]: osdmap e1544: 8 total, 8 up, 8 in 2024-08-17T18:24:09.718 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:09 smithi130 ceph-mon[110068]: pgmap v3029: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:09.718 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:09 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:24:09.718 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:09 smithi130 ceph-mon[110068]: osdmap e1544: 8 total, 8 up, 8 in 2024-08-17T18:24:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:09 smithi179 ceph-mon[93519]: pgmap v3029: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:09 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-17T18:24:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:09 smithi179 ceph-mon[93519]: osdmap e1544: 8 total, 8 up, 8 in 2024-08-17T18:24:10.166 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:24:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:24:09] "GET /metrics HTTP/1.1" 200 34364 "" "Prometheus/2.51.0" 2024-08-17T18:24:10.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:24:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:24:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:24:11.440 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T18:24:11.745 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:11 smithi130 ceph-mon[105521]: pgmap v3031: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:11.746 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:11 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/409245721' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:24:11.746 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:11 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:24:11.746 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:11 smithi130 ceph-mon[110068]: pgmap v3031: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:11.746 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:11 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/409245721' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:24:11.746 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:11 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:24:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:11 smithi179 ceph-mon[93519]: pgmap v3031: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:11 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/409245721' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:24:11.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:11 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:24:12.105 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 3 1 2024-08-17T18:24:12.372 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:12.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:12 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:24:12.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:12 smithi130 ceph-mon[105521]: osdmap e1545: 8 total, 8 up, 8 in 2024-08-17T18:24:12.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:24:12.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:12 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:24:12.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:12 smithi130 ceph-mon[110068]: osdmap e1545: 8 total, 8 up, 8 in 2024-08-17T18:24:12.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:24:12.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:12 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:24:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:12 smithi179 ceph-mon[93519]: osdmap e1545: 8 total, 8 up, 8 in 2024-08-17T18:24:12.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:24:13.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:13 smithi130 ceph-mon[110068]: pgmap v3033: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:13.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:13 smithi130 ceph-mon[105521]: pgmap v3033: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:13.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:13 smithi179 ceph-mon[93519]: pgmap v3033: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:14.463 INFO:teuthology.orchestra.run.smithi130.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-08-17T18:24:14.778 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:14 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2145455103' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:24:14.778 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:14 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:24:14.778 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:14 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2145455103' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:24:14.778 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:14 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:24:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:14 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2145455103' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:24:14.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:14 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T18:24:15.103 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T18:24:15.359 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:15.765 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:15 smithi130 ceph-mon[105521]: pgmap v3034: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:15.765 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:15 smithi130 ceph-mon[105521]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:24:15.765 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:15 smithi130 ceph-mon[105521]: osdmap e1546: 8 total, 8 up, 8 in 2024-08-17T18:24:15.765 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:15 smithi130 ceph-mon[110068]: pgmap v3034: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:15.765 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:15 smithi130 ceph-mon[110068]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:24:15.765 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:15 smithi130 ceph-mon[110068]: osdmap e1546: 8 total, 8 up, 8 in 2024-08-17T18:24:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:15 smithi179 ceph-mon[93519]: pgmap v3034: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:15 smithi179 ceph-mon[93519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-17T18:24:15.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:15 smithi179 ceph-mon[93519]: osdmap e1546: 8 total, 8 up, 8 in 2024-08-17T18:24:16.771 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-08-17T18:24:16.771 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]} : dispatch 2024-08-17T18:24:16.771 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:16 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]} : dispatch 2024-08-17T18:24:16.771 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-08-17T18:24:16.771 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]} : dispatch 2024-08-17T18:24:16.771 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:16 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]} : dispatch 2024-08-17T18:24:16.772 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-08-17T18:24:16.772 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]} : dispatch 2024-08-17T18:24:16.772 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:16 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]} : dispatch 2024-08-17T18:24:16.772 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-08-17T18:24:16.772 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]} : dispatch 2024-08-17T18:24:16.772 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:16 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]} : dispatch 2024-08-17T18:24:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-08-17T18:24:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]} : dispatch 2024-08-17T18:24:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:16 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]} : dispatch 2024-08-17T18:24:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-08-17T18:24:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]} : dispatch 2024-08-17T18:24:16.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:16 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]} : dispatch 2024-08-17T18:24:17.463 INFO:teuthology.orchestra.run.smithi130.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T18:24:17.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:17 smithi130 ceph-mon[105521]: pgmap v3036: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:17.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:17 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-08-17T18:24:17.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:17 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-08-17T18:24:17.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:17 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2024-08-17T18:24:17.802 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:17 smithi130 ceph-mon[105521]: osdmap e1547: 8 total, 8 up, 8 in 2024-08-17T18:24:17.803 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:17 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2805465422' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:24:17.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:17 smithi130 ceph-mon[110068]: pgmap v3036: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:17.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:17 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-08-17T18:24:17.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:17 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-08-17T18:24:17.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:17 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2024-08-17T18:24:17.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:17 smithi130 ceph-mon[110068]: osdmap e1547: 8 total, 8 up, 8 in 2024-08-17T18:24:17.803 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:17 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2805465422' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:24:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:17 smithi179 ceph-mon[93519]: pgmap v3036: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:17 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-08-17T18:24:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:17 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-08-17T18:24:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:17 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2024-08-17T18:24:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:17 smithi179 ceph-mon[93519]: osdmap e1547: 8 total, 8 up, 8 in 2024-08-17T18:24:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:17 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2805465422' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:24:18.144 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 2 1 2024-08-17T18:24:18.400 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:18.733 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:18 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2805465422' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:24:18.734 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:18 smithi130 ceph-mon[105521]: osdmap e1548: 8 total, 8 up, 8 in 2024-08-17T18:24:18.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:18 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2805465422' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:24:18.734 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:18 smithi130 ceph-mon[110068]: osdmap e1548: 8 total, 8 up, 8 in 2024-08-17T18:24:18.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:18 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2805465422' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:24:18.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:18 smithi179 ceph-mon[93519]: osdmap e1548: 8 total, 8 up, 8 in 2024-08-17T18:24:19.651 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:19 smithi130 ceph-mon[105521]: pgmap v3039: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:19.652 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:19 smithi130 ceph-mon[110068]: pgmap v3039: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:19.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:19 smithi179 ceph-mon[93519]: pgmap v3039: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:19.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:24:19 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:24:19] "GET /metrics HTTP/1.1" 200 34362 "" "Prometheus/2.51.0" 2024-08-17T18:24:20.487 INFO:teuthology.orchestra.run.smithi130.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-08-17T18:24:20.492 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:24:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:24:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:24:20.501 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:20 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2282068266' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:24:20.501 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:20 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2282068266' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:24:20.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:20 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2282068266' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T18:24:21.236 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-08-17T18:24:21.236 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-08-17T18:24:21.236 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-08-17T18:24:21.236 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-08-17T18:24:21.236 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T18:24:21.237 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T18:24:21.495 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:21.756 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:21 smithi130 ceph-mon[105521]: pgmap v3040: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:21.756 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:21 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2282068266' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:24:21.756 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:21 smithi130 ceph-mon[105521]: osdmap e1549: 8 total, 8 up, 8 in 2024-08-17T18:24:21.756 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:21 smithi130 ceph-mon[110068]: pgmap v3040: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:21.757 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:21 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2282068266' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:24:21.757 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:21 smithi130 ceph-mon[110068]: osdmap e1549: 8 total, 8 up, 8 in 2024-08-17T18:24:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:21 smithi179 ceph-mon[93519]: pgmap v3040: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:21 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2282068266' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T18:24:21.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:21 smithi179 ceph-mon[93519]: osdmap e1549: 8 total, 8 up, 8 in 2024-08-17T18:24:23.497 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:24:23.498 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":1549,"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","created":"2024-08-17T17:15:09.488012+0000","modified":"2024-08-17T18:24:20.482202+0000","last_up_change":"2024-08-17T18:19:25.012493+0000","last_in_change":"2024-08-17T18:23:11.466153+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":51,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:18:10.192831+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1525","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:20:37.124673+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T17:21:51.081217+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T17:21:51.379206+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"30c31adb-2cda-466b-933e-a8e2cb714fab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":438,"last_clean_end":1163,"up_from":1178,"up_thru":1530,"down_at":1164,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":2575115774},{"type":"v1","addr":"172.21.15.130:6803","nonce":2575115774}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":2575115774},{"type":"v1","addr":"172.21.15.130:6805","nonce":2575115774}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":2575115774},{"type":"v1","addr":"172.21.15.130:6809","nonce":2575115774}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":2575115774},{"type":"v1","addr":"172.21.15.130:6807","nonce":2575115774}]},"public_addr":"172.21.15.130:6803/2575115774","cluster_addr":"172.21.15.130:6805/2575115774","heartbeat_back_addr":"172.21.15.130:6809/2575115774","heartbeat_front_addr":"172.21.15.130:6807/2575115774","state":["exists","up"]},{"osd":1,"uuid":"55049bb4-b976-42c6-8a5d-8b3e9ae20988","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":493,"up_from":506,"up_thru":1530,"down_at":494,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6811","nonce":790699819}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6813","nonce":790699819}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6817","nonce":790699819}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6815","nonce":790699819}]},"public_addr":"172.21.15.130:6811/790699819","cluster_addr":"172.21.15.130:6813/790699819","heartbeat_back_addr":"172.21.15.130:6817/790699819","heartbeat_front_addr":"172.21.15.130:6815/790699819","state":["exists","up"]},{"osd":2,"uuid":"3f40f088-27f7-4450-8f31-b449711365ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":538,"last_clean_end":1404,"up_from":1454,"up_thru":1547,"down_at":1405,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":950156771},{"type":"v1","addr":"172.21.15.130:6819","nonce":950156771}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":950156771},{"type":"v1","addr":"172.21.15.130:6821","nonce":950156771}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":950156771},{"type":"v1","addr":"172.21.15.130:6825","nonce":950156771}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":950156771},{"type":"v1","addr":"172.21.15.130:6823","nonce":950156771}]},"public_addr":"172.21.15.130:6819/950156771","cluster_addr":"172.21.15.130:6821/950156771","heartbeat_back_addr":"172.21.15.130:6825/950156771","heartbeat_front_addr":"172.21.15.130:6823/950156771","state":["exists","up"]},{"osd":3,"uuid":"7684c401-462a-4e35-bb9c-1be23dfe573b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":571,"last_clean_end":1316,"up_from":1343,"up_thru":1530,"down_at":1317,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":3141916061},{"type":"v1","addr":"172.21.15.130:6827","nonce":3141916061}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":3141916061},{"type":"v1","addr":"172.21.15.130:6829","nonce":3141916061}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3141916061},{"type":"v1","addr":"172.21.15.130:6833","nonce":3141916061}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":3141916061},{"type":"v1","addr":"172.21.15.130:6831","nonce":3141916061}]},"public_addr":"172.21.15.130:6827/3141916061","cluster_addr":"172.21.15.130:6829/3141916061","heartbeat_back_addr":"172.21.15.130:6833/3141916061","heartbeat_front_addr":"172.21.15.130:6831/3141916061","state":["exists","up"]},{"osd":4,"uuid":"7a3a331f-533a-49de-910d-5989f2f84b65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":1547,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6800","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6801","nonce":2107797845}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6802","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6803","nonce":2107797845}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6806","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6807","nonce":2107797845}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6804","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6805","nonce":2107797845}]},"public_addr":"172.21.15.179:6801/2107797845","cluster_addr":"172.21.15.179:6803/2107797845","heartbeat_back_addr":"172.21.15.179:6807/2107797845","heartbeat_front_addr":"172.21.15.179:6805/2107797845","state":["exists","up"]},{"osd":5,"uuid":"d53d65ee-bbb2-45ec-b557-07cc62a45b05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":1530,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6808","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6809","nonce":2392026049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6810","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6811","nonce":2392026049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6814","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6815","nonce":2392026049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6812","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6813","nonce":2392026049}]},"public_addr":"172.21.15.179:6809/2392026049","cluster_addr":"172.21.15.179:6811/2392026049","heartbeat_back_addr":"172.21.15.179:6815/2392026049","heartbeat_front_addr":"172.21.15.179:6813/2392026049","state":["exists","up"]},{"osd":6,"uuid":"b84229ad-2c88-4064-a070-33b98714ef96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":1530,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6816","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6817","nonce":139895003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6818","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6819","nonce":139895003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6822","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6823","nonce":139895003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6820","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6821","nonce":139895003}]},"public_addr":"172.21.15.179:6817/139895003","cluster_addr":"172.21.15.179:6819/139895003","heartbeat_back_addr":"172.21.15.179:6823/139895003","heartbeat_front_addr":"172.21.15.179:6821/139895003","state":["exists","up"]},{"osd":7,"uuid":"bba7978f-8b19-40e6-86bd-692a0e282429","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":1530,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6824","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6825","nonce":2533639201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6826","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6827","nonce":2533639201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6830","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6831","nonce":2533639201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6828","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6829","nonce":2533639201}]},"public_addr":"172.21.15.179:6825/2533639201","cluster_addr":"172.21.15.179:6827/2533639201","heartbeat_back_addr":"172.21.15.179:6831/2533639201","heartbeat_front_addr":"172.21.15.179:6829/2533639201","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:19.220776+0000","dead_epoch":1163},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:42.876683+0000","dead_epoch":493},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:07.028539+0000","dead_epoch":1404},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:31.259509+0000","dead_epoch":1316},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:52.172235+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:15.112109+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:37.781476+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:20:00.893216+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.c","mappings":[{"from":3,"to":2}]},{"pgid":"3.18","mappings":[{"from":3,"to":2}]},{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.e","mappings":[{"from":1,"to":5}]},{"pgid":"4.15","mappings":[{"from":3,"to":2}]},{"pgid":"4.1a","mappings":[{"from":3,"to":7}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/485089907":"2024-08-17T18:31:20.602503+0000","172.21.15.130:6801/3641086954":"2024-08-18T17:26:11.940622+0000","172.21.15.130:6800/3641086954":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/2447795507":"2024-08-17T18:31:19.611103+0000","172.21.15.130:0/851371227":"2024-08-18T17:20:33.507974+0000","172.21.15.179:0/393594060":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/3071443649":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2578011752":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/1457627030":"2024-08-18T17:24:24.724005+0000","172.21.15.130:0/2870283877":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/3943801420":"2024-08-18T17:26:11.940622+0000","172.21.15.179:0/2395764912":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/1299294781":"2024-08-18T17:24:24.724005+0000","172.21.15.179:0/1781739324":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/1690071627":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/2991152682":"2024-08-18T17:26:11.940622+0000","172.21.15.130:6801/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2810193027":"2024-08-18T17:26:11.940622+0000","172.21.15.179:0/56636976":"2024-08-18T17:25:30.337581+0000","172.21.15.130:6800/824081166":"2024-08-18T17:24:24.724005+0000","172.21.15.130:0/2589122337":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/1247136467":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/4037898622":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/3453310625":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/3471387839":"2024-08-18T17:15:57.894717+0000","172.21.15.130:6801/824081166":"2024-08-18T17:24:24.724005+0000","172.21.15.130:0/368845317":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/695423662":"2024-08-17T18:36:26.582385+0000","172.21.15.130:0/2999329360":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6800/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.179:0/196590495":"2024-08-18T17:25:30.337581+0000","172.21.15.130:6800/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/854839742":"2024-08-18T17:24:24.724005+0000","172.21.15.179:6834/3911752105":"2024-08-18T17:25:30.337581+0000","172.21.15.130:6801/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.179:0/2399909254":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/1843620043":"2024-08-18T17:24:24.724005+0000","172.21.15.130:6800/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6801/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/88320168":"2024-08-18T17:15:34.935061+0000","172.21.15.179:6835/3911752105":"2024-08-18T17:25:30.337581+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":14,"snaps":[{"begin":8,"length":1}]},{"pool":16,"snaps":[{"begin":35,"length":3},{"begin":44,"length":4}]},{"pool":17,"snaps":[{"begin":17,"length":2},{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]},{"pool":51,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T18:24:23.824 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:23 smithi130 ceph-mon[105521]: pgmap v3042: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:23.825 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:23 smithi130 ceph-mon[110068]: pgmap v3042: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:23.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:23 smithi179 ceph-mon[93519]: pgmap v3042: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:24.218 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T18:24:24.218 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T18:24:24.473 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:24.775 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:24 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/100492628' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T18:24:24.775 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:24 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/100492628' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T18:24:24.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:24 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/100492628' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T18:24:25.800 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:25 smithi130 ceph-mon[105521]: pgmap v3043: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:25.800 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:25 smithi130 ceph-mon[110068]: pgmap v3043: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:25.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:25 smithi179 ceph-mon[93519]: pgmap v3043: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:26.476 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:24:26.476 INFO:teuthology.orchestra.run.smithi130.stdout:{"epoch":1549,"fsid":"25127030-5cbc-11ef-bccf-c7b262605968","created":"2024-08-17T17:15:09.488012+0000","modified":"2024-08-17T18:24:20.482202+0000","last_up_change":"2024-08-17T18:19:25.012493+0000","last_in_change":"2024-08-17T18:23:11.466153+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":51,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T17:18:10.192831+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1525","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T17:20:37.124673+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T17:21:51.081217+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T17:21:51.379206+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"30c31adb-2cda-466b-933e-a8e2cb714fab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":438,"last_clean_end":1163,"up_from":1178,"up_thru":1530,"down_at":1164,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6802","nonce":2575115774},{"type":"v1","addr":"172.21.15.130:6803","nonce":2575115774}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6804","nonce":2575115774},{"type":"v1","addr":"172.21.15.130:6805","nonce":2575115774}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6808","nonce":2575115774},{"type":"v1","addr":"172.21.15.130:6809","nonce":2575115774}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6806","nonce":2575115774},{"type":"v1","addr":"172.21.15.130:6807","nonce":2575115774}]},"public_addr":"172.21.15.130:6803/2575115774","cluster_addr":"172.21.15.130:6805/2575115774","heartbeat_back_addr":"172.21.15.130:6809/2575115774","heartbeat_front_addr":"172.21.15.130:6807/2575115774","state":["exists","up"]},{"osd":1,"uuid":"55049bb4-b976-42c6-8a5d-8b3e9ae20988","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":493,"up_from":506,"up_thru":1530,"down_at":494,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6810","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6811","nonce":790699819}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6812","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6813","nonce":790699819}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6816","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6817","nonce":790699819}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6814","nonce":790699819},{"type":"v1","addr":"172.21.15.130:6815","nonce":790699819}]},"public_addr":"172.21.15.130:6811/790699819","cluster_addr":"172.21.15.130:6813/790699819","heartbeat_back_addr":"172.21.15.130:6817/790699819","heartbeat_front_addr":"172.21.15.130:6815/790699819","state":["exists","up"]},{"osd":2,"uuid":"3f40f088-27f7-4450-8f31-b449711365ab","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":538,"last_clean_end":1404,"up_from":1454,"up_thru":1547,"down_at":1405,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6818","nonce":950156771},{"type":"v1","addr":"172.21.15.130:6819","nonce":950156771}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6820","nonce":950156771},{"type":"v1","addr":"172.21.15.130:6821","nonce":950156771}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6824","nonce":950156771},{"type":"v1","addr":"172.21.15.130:6825","nonce":950156771}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6822","nonce":950156771},{"type":"v1","addr":"172.21.15.130:6823","nonce":950156771}]},"public_addr":"172.21.15.130:6819/950156771","cluster_addr":"172.21.15.130:6821/950156771","heartbeat_back_addr":"172.21.15.130:6825/950156771","heartbeat_front_addr":"172.21.15.130:6823/950156771","state":["exists","up"]},{"osd":3,"uuid":"7684c401-462a-4e35-bb9c-1be23dfe573b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":571,"last_clean_end":1316,"up_from":1343,"up_thru":1530,"down_at":1317,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6826","nonce":3141916061},{"type":"v1","addr":"172.21.15.130:6827","nonce":3141916061}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6828","nonce":3141916061},{"type":"v1","addr":"172.21.15.130:6829","nonce":3141916061}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6832","nonce":3141916061},{"type":"v1","addr":"172.21.15.130:6833","nonce":3141916061}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.130:6830","nonce":3141916061},{"type":"v1","addr":"172.21.15.130:6831","nonce":3141916061}]},"public_addr":"172.21.15.130:6827/3141916061","cluster_addr":"172.21.15.130:6829/3141916061","heartbeat_back_addr":"172.21.15.130:6833/3141916061","heartbeat_front_addr":"172.21.15.130:6831/3141916061","state":["exists","up"]},{"osd":4,"uuid":"7a3a331f-533a-49de-910d-5989f2f84b65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":1547,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6800","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6801","nonce":2107797845}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6802","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6803","nonce":2107797845}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6806","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6807","nonce":2107797845}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6804","nonce":2107797845},{"type":"v1","addr":"172.21.15.179:6805","nonce":2107797845}]},"public_addr":"172.21.15.179:6801/2107797845","cluster_addr":"172.21.15.179:6803/2107797845","heartbeat_back_addr":"172.21.15.179:6807/2107797845","heartbeat_front_addr":"172.21.15.179:6805/2107797845","state":["exists","up"]},{"osd":5,"uuid":"d53d65ee-bbb2-45ec-b557-07cc62a45b05","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":1530,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6808","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6809","nonce":2392026049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6810","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6811","nonce":2392026049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6814","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6815","nonce":2392026049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6812","nonce":2392026049},{"type":"v1","addr":"172.21.15.179:6813","nonce":2392026049}]},"public_addr":"172.21.15.179:6809/2392026049","cluster_addr":"172.21.15.179:6811/2392026049","heartbeat_back_addr":"172.21.15.179:6815/2392026049","heartbeat_front_addr":"172.21.15.179:6813/2392026049","state":["exists","up"]},{"osd":6,"uuid":"b84229ad-2c88-4064-a070-33b98714ef96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":1530,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6816","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6817","nonce":139895003}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6818","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6819","nonce":139895003}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6822","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6823","nonce":139895003}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6820","nonce":139895003},{"type":"v1","addr":"172.21.15.179:6821","nonce":139895003}]},"public_addr":"172.21.15.179:6817/139895003","cluster_addr":"172.21.15.179:6819/139895003","heartbeat_back_addr":"172.21.15.179:6823/139895003","heartbeat_front_addr":"172.21.15.179:6821/139895003","state":["exists","up"]},{"osd":7,"uuid":"bba7978f-8b19-40e6-86bd-692a0e282429","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":1530,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6824","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6825","nonce":2533639201}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6826","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6827","nonce":2533639201}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6830","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6831","nonce":2533639201}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.179:6828","nonce":2533639201},{"type":"v1","addr":"172.21.15.179:6829","nonce":2533639201}]},"public_addr":"172.21.15.179:6825/2533639201","cluster_addr":"172.21.15.179:6827/2533639201","heartbeat_back_addr":"172.21.15.179:6831/2533639201","heartbeat_front_addr":"172.21.15.179:6829/2533639201","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:19.220776+0000","dead_epoch":1163},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:17:42.876683+0000","dead_epoch":493},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:07.028539+0000","dead_epoch":1404},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:31.259509+0000","dead_epoch":1316},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:18:52.172235+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:15.112109+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:19:37.781476+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T17:20:00.893216+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.c","mappings":[{"from":3,"to":2}]},{"pgid":"3.18","mappings":[{"from":3,"to":2}]},{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.e","mappings":[{"from":1,"to":5}]},{"pgid":"4.15","mappings":[{"from":3,"to":2}]},{"pgid":"4.1a","mappings":[{"from":3,"to":7}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.130:0/485089907":"2024-08-17T18:31:20.602503+0000","172.21.15.130:6801/3641086954":"2024-08-18T17:26:11.940622+0000","172.21.15.130:6800/3641086954":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/2447795507":"2024-08-17T18:31:19.611103+0000","172.21.15.130:0/851371227":"2024-08-18T17:20:33.507974+0000","172.21.15.179:0/393594060":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/3071443649":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2578011752":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/1457627030":"2024-08-18T17:24:24.724005+0000","172.21.15.130:0/2870283877":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/3943801420":"2024-08-18T17:26:11.940622+0000","172.21.15.179:0/2395764912":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/1299294781":"2024-08-18T17:24:24.724005+0000","172.21.15.179:0/1781739324":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/1690071627":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/2991152682":"2024-08-18T17:26:11.940622+0000","172.21.15.130:6801/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/2810193027":"2024-08-18T17:26:11.940622+0000","172.21.15.179:0/56636976":"2024-08-18T17:25:30.337581+0000","172.21.15.130:6800/824081166":"2024-08-18T17:24:24.724005+0000","172.21.15.130:0/2589122337":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/1247136467":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/4037898622":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/3453310625":"2024-08-18T17:26:11.940622+0000","172.21.15.130:0/3471387839":"2024-08-18T17:15:57.894717+0000","172.21.15.130:6801/824081166":"2024-08-18T17:24:24.724005+0000","172.21.15.130:0/368845317":"2024-08-18T17:20:33.507974+0000","172.21.15.130:0/695423662":"2024-08-17T18:36:26.582385+0000","172.21.15.130:0/2999329360":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6800/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.179:0/196590495":"2024-08-18T17:25:30.337581+0000","172.21.15.130:6800/368286319":"2024-08-18T17:15:57.894717+0000","172.21.15.130:0/854839742":"2024-08-18T17:24:24.724005+0000","172.21.15.179:6834/3911752105":"2024-08-18T17:25:30.337581+0000","172.21.15.130:6801/3883563384":"2024-08-18T17:20:33.507974+0000","172.21.15.179:0/2399909254":"2024-08-18T17:25:30.337581+0000","172.21.15.130:0/1843620043":"2024-08-18T17:24:24.724005+0000","172.21.15.130:6800/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:6801/1239370657":"2024-08-18T17:15:34.935061+0000","172.21.15.130:0/88320168":"2024-08-18T17:15:34.935061+0000","172.21.15.179:6835/3911752105":"2024-08-18T17:25:30.337581+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":14,"snaps":[{"begin":8,"length":1}]},{"pool":16,"snaps":[{"begin":35,"length":3},{"begin":44,"length":4}]},{"pool":17,"snaps":[{"begin":17,"length":2},{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]},{"pool":51,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T18:24:26.815 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:26 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/81665522' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T18:24:26.815 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:26 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/81665522' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T18:24:26.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:26 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/81665522' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T18:24:27.126 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T18:24:27.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T18:24:27.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T18:24:27.127 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T18:24:27.128 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T18:24:27.128 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T18:24:27.129 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T18:24:27.129 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T18:24:27.689 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:27.690 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:27.691 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:27.691 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:27.692 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:27.696 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:27.700 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:27.704 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:27.831 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:27 smithi130 ceph-mon[105521]: pgmap v3044: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:27.832 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:27 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:24:27.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:27 smithi130 ceph-mon[110068]: pgmap v3044: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:27.832 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:27 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:24:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:27 smithi179 ceph-mon[93519]: pgmap v3044: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:27.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:27 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:24:29.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:29 smithi130 ceph-mon[105521]: pgmap v3045: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:29.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:29 smithi130 ceph-mon[110068]: pgmap v3045: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:29.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:29 smithi179 ceph-mon[93519]: pgmap v3045: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:30.144 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:24:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:24:29] "GET /metrics HTTP/1.1" 200 34366 "" "Prometheus/2.51.0" 2024-08-17T18:24:30.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:24:30 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:24:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:24:31.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:31 smithi179 ceph-mon[93519]: pgmap v3046: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:31.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:31 smithi130 ceph-mon[105521]: pgmap v3046: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:31.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:31 smithi130 ceph-mon[110068]: pgmap v3046: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:33.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:33 smithi179 ceph-mon[93519]: pgmap v3047: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:33.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:33 smithi130 ceph-mon[110068]: pgmap v3047: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:33.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:33 smithi130 ceph-mon[105521]: pgmap v3047: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:35.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:35 smithi179 ceph-mon[93519]: pgmap v3048: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:35.866 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:35 smithi130 ceph-mon[105521]: pgmap v3048: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:35.866 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:35 smithi130 ceph-mon[110068]: pgmap v3048: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:37.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:37 smithi179 ceph-mon[93519]: pgmap v3049: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:37.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:37 smithi130 ceph-mon[105521]: pgmap v3049: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:37.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:37 smithi130 ceph-mon[110068]: pgmap v3049: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:37.982 INFO:teuthology.orchestra.run.smithi130.stdout:6244882448448 2024-08-17T18:24:37.983 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T18:24:38.492 INFO:teuthology.orchestra.run.smithi130.stdout:5059471475116 2024-08-17T18:24:38.492 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T18:24:39.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:39 smithi130 ceph-mon[105521]: pgmap v3050: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:39.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:39 smithi130 ceph-mon[110068]: pgmap v3050: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:39.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:39 smithi179 ceph-mon[93519]: pgmap v3050: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:40.075 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:24:39 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:24:39] "GET /metrics HTTP/1.1" 200 34366 "" "Prometheus/2.51.0" 2024-08-17T18:24:40.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:24:40 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:24:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:24:40.615 INFO:teuthology.orchestra.run.smithi130.stdout:197568496401 2024-08-17T18:24:40.615 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T18:24:41.265 INFO:teuthology.orchestra.run.smithi130.stdout:231928234766 2024-08-17T18:24:41.266 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T18:24:41.799 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:41 smithi130 ceph-mon[105521]: pgmap v3051: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:41.799 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:41 smithi130 ceph-mon[110068]: pgmap v3051: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:41.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:41 smithi179 ceph-mon[93519]: pgmap v3051: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:42.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:42 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:24:42.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:42 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:24:42.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:42 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:24:43.357 INFO:teuthology.orchestra.run.smithi130.stdout:2173253452426 2024-08-17T18:24:43.357 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T18:24:43.747 INFO:teuthology.orchestra.run.smithi130.stdout:163208758039 2024-08-17T18:24:43.748 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T18:24:43.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:43 smithi179 ceph-mon[93519]: pgmap v3052: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:43.892 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:43 smithi130 ceph-mon[105521]: pgmap v3052: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:43.892 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:43 smithi130 ceph-mon[110068]: pgmap v3052: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:44.764 INFO:teuthology.orchestra.run.smithi130.stdout:5768141078713 2024-08-17T18:24:44.764 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T18:24:44.767 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:45.509 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:45.549 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:45.797 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:45 smithi130 ceph-mon[110068]: pgmap v3053: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:45.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:45 smithi130 ceph-mon[105521]: pgmap v3053: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:45.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:45 smithi179 ceph-mon[93519]: pgmap v3053: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:47.415 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:47.415 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:47.416 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:47.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:47 smithi179 ceph-mon[93519]: pgmap v3054: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:47.847 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:47 smithi130 ceph-mon[105521]: pgmap v3054: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:47.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:47 smithi130 ceph-mon[110068]: pgmap v3054: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:48.198 INFO:teuthology.orchestra.run.smithi130.stdout:128849019675 2024-08-17T18:24:48.198 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T18:24:49.820 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:49 smithi130 ceph-mon[105521]: pgmap v3055: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:49.820 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:49 smithi130 ceph-mon[110068]: pgmap v3055: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:49.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:49 smithi179 ceph-mon[93519]: pgmap v3055: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:50.090 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:24:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:24:49] "GET /metrics HTTP/1.1" 200 34365 "" "Prometheus/2.51.0" 2024-08-17T18:24:50.545 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:24:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:24:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:24:50.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:50 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/4178351866' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T18:24:50.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:50 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/4178351866' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T18:24:50.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:50 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/4178351866' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T18:24:51.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:51 smithi130 ceph-mon[105521]: pgmap v3056: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:51.809 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:51 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:24:51.809 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:51 smithi130 ceph-mon[110068]: pgmap v3056: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:51.810 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:51 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:24:51.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:51 smithi179 ceph-mon[93519]: pgmap v3056: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:51.830 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:51 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T18:24:51.943 INFO:teuthology.orchestra.run.smithi130.stdout:2173253452428 2024-08-17T18:24:52.593 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2173253452426 got 2173253452428 for osd.1 2024-08-17T18:24:52.593 DEBUG:teuthology.parallel:result is None 2024-08-17T18:24:53.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:53 smithi130 ceph-mon[105521]: pgmap v3057: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:53.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:53 smithi130 ceph-mon[110068]: pgmap v3057: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:53.788 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:53 smithi179 ceph-mon[93519]: pgmap v3057: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:53.994 INFO:teuthology.orchestra.run.smithi130.stdout:231928234768 2024-08-17T18:24:54.189 INFO:teuthology.orchestra.run.smithi130.stdout:163208758041 2024-08-17T18:24:54.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/2090435971' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T18:24:54.818 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:54 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3133143296' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T18:24:54.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/2090435971' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T18:24:54.818 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:54 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3133143296' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T18:24:54.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/2090435971' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T18:24:54.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:54 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3133143296' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T18:24:54.844 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208758039 got 163208758041 for osd.5 2024-08-17T18:24:54.844 DEBUG:teuthology.parallel:result is None 2024-08-17T18:24:54.860 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:55.591 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:55 smithi130 ceph-mon[105521]: pgmap v3058: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:55.591 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:55 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:24:55.591 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:55 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:24:55.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:55 smithi130 ceph-mon[110068]: pgmap v3058: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:55.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:55 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:24:55.591 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:55 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:24:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:55 smithi179 ceph-mon[93519]: pgmap v3058: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:55 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:24:55.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:55 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:24:57.740 INFO:teuthology.orchestra.run.smithi130.stdout:5768141078715 2024-08-17T18:24:57.740 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:57 smithi130 ceph-mon[105521]: pgmap v3059: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:57.740 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:57 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:24:57.741 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:57 smithi130 ceph-mon[110068]: pgmap v3059: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:57.741 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:57 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:24:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:57 smithi179 ceph-mon[93519]: pgmap v3059: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:57.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:57 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:24:58.285 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234766 got 231928234768 for osd.7 2024-08-17T18:24:58.285 DEBUG:teuthology.parallel:result is None 2024-08-17T18:24:58.662 INFO:teuthology.orchestra.run.smithi130.stdout:197568496404 2024-08-17T18:24:58.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:58 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1139060647' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T18:24:58.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:58 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1139060647' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T18:24:58.916 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:58 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1139060647' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T18:24:59.615 INFO:tasks.cephadm.ceph_manager.ceph:need seq 5768141078713 got 5768141078715 for osd.3 2024-08-17T18:24:59.615 DEBUG:teuthology.parallel:result is None 2024-08-17T18:24:59.801 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:24:59.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:59 smithi130 ceph-mon[105521]: pgmap v3060: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:59.819 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:24:59 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/963719866' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T18:24:59.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:59 smithi130 ceph-mon[110068]: pgmap v3060: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:59.819 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:24:59 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/963719866' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T18:24:59.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:59 smithi179 ceph-mon[93519]: pgmap v3060: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:24:59.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:24:59 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/963719866' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T18:25:00.090 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:24:59 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:24:59] "GET /metrics HTTP/1.1" 200 34365 "" "Prometheus/2.51.0" 2024-08-17T18:25:00.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:25:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:25:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:25:01.389 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568496401 got 197568496404 for osd.6 2024-08-17T18:25:01.389 DEBUG:teuthology.parallel:result is None 2024-08-17T18:25:01.541 INFO:teuthology.orchestra.run.smithi130.stdout:6244882448453 2024-08-17T18:25:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:01 smithi130 ceph-mon[110068]: pgmap v3061: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:01.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:01 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/1664098879' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T18:25:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:01 smithi130 ceph-mon[105521]: pgmap v3061: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:01.667 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:01 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/1664098879' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T18:25:01.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:01 smithi179 ceph-mon[93519]: pgmap v3061: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:01.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:01 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/1664098879' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T18:25:02.172 INFO:tasks.cephadm.ceph_manager.ceph:need seq 6244882448448 got 6244882448453 for osd.2 2024-08-17T18:25:02.172 DEBUG:teuthology.parallel:result is None 2024-08-17T18:25:03.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:03 smithi179 ceph-mon[93519]: pgmap v3062: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:03.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:03 smithi130 ceph-mon[105521]: pgmap v3062: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:03.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:03 smithi130 ceph-mon[110068]: pgmap v3062: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:04.044 INFO:teuthology.orchestra.run.smithi130.stdout:5059471475121 2024-08-17T18:25:04.666 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:04 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3475474401' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T18:25:04.667 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:04 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3475474401' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T18:25:04.811 INFO:tasks.cephadm.ceph_manager.ceph:need seq 5059471475116 got 5059471475121 for osd.0 2024-08-17T18:25:04.812 DEBUG:teuthology.parallel:result is None 2024-08-17T18:25:04.842 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:04 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3475474401' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T18:25:05.253 INFO:teuthology.orchestra.run.smithi130.stdout:128849019679 2024-08-17T18:25:05.797 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:05 smithi130 ceph-mon[105521]: pgmap v3063: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:05.798 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:05 smithi130 ceph-mon[105521]: from='client.? 172.21.15.130:0/3873504560' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T18:25:05.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:05 smithi130 ceph-mon[110068]: pgmap v3063: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:05.798 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:05 smithi130 ceph-mon[110068]: from='client.? 172.21.15.130:0/3873504560' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T18:25:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:05 smithi179 ceph-mon[93519]: pgmap v3063: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:05.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:05 smithi179 ceph-mon[93519]: from='client.? 172.21.15.130:0/3873504560' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T18:25:06.635 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019675 got 128849019679 for osd.4 2024-08-17T18:25:06.635 DEBUG:teuthology.parallel:result is None 2024-08-17T18:25:06.635 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T18:25:06.635 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:25:07.450 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:25:07.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:07 smithi179 ceph-mon[93519]: pgmap v3064: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:07.867 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:07 smithi130 ceph-mon[105521]: pgmap v3064: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:07.867 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:07 smithi130 ceph-mon[110068]: pgmap v3064: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:09 smithi179 ceph-mon[93519]: pgmap v3065: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:09 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:25:09.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:09 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:25:09.847 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:09 smithi130 ceph-mon[105521]: pgmap v3065: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:09.847 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:09 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:25:09.847 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:09 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:25:09.847 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:25:09 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: ::ffff:172.21.15.179 - - [17/Aug/2024:18:25:09] "GET /metrics HTTP/1.1" 200 34365 "" "Prometheus/2.51.0" 2024-08-17T18:25:09.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:09 smithi130 ceph-mon[110068]: pgmap v3065: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:09.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:09 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:25:09.848 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:09 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:25:10.116 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:25:10.117 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:25:10.591 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:25:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:25:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"25127030-5cbc-11ef-bccf-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-08-17T18:25:10.693 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:10 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:25:10.693 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:10 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:25:10.693 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:10 smithi130 ceph-mon[105521]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:25:10.694 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:10 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:25:10.694 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:10 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:25:10.694 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:10 smithi130 ceph-mon[110068]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:25:10.720 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":3066,"stamp":"2024-08-17T18:25:09.756236+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":125,"num_bytes_recovered":5326043,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6741752,"kb_used_data":71384,"kb_used_omap":405,"kb_used_meta":6669866,"kb_avail":743219464,"statfs":{"total":767960285184,"available":761056731136,"internally_reserved":0,"allocated":73097216,"data_stored":40958831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414805,"internal_metadata":6829943723},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003255"},"pg_stats":[{"pgid":"3.1f","version":"71'4","reported_seq":149,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.632053+0000","last_change":"2024-08-17T18:23:12.507440+0000","last_active":"2024-08-17T18:24:20.632053+0000","last_peered":"2024-08-17T18:24:20.632053+0000","last_clean":"2024-08-17T18:24:20.632053+0000","last_became_active":"2024-08-17T18:23:12.493772+0000","last_became_peered":"2024-08-17T18:23:12.493772+0000","last_unstale":"2024-08-17T18:24:20.632053+0000","last_undegraded":"2024-08-17T18:24:20.632053+0000","last_fullsized":"2024-08-17T18:24:20.632053+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:28.385234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2280,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":132,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468402+0000","last_change":"2024-08-17T18:23:05.464220+0000","last_active":"2024-08-17T18:24:17.468402+0000","last_peered":"2024-08-17T18:24:17.468402+0000","last_clean":"2024-08-17T18:24:17.468402+0000","last_became_active":"2024-08-17T18:23:05.463841+0000","last_became_peered":"2024-08-17T18:23:05.463841+0000","last_unstale":"2024-08-17T18:24:17.468402+0000","last_undegraded":"2024-08-17T18:24:17.468402+0000","last_fullsized":"2024-08-17T18:24:17.468402+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:47:48.738919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":2406,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493674+0000","last_change":"2024-08-17T18:19:27.023211+0000","last_active":"2024-08-17T18:24:20.493674+0000","last_peered":"2024-08-17T18:24:20.493674+0000","last_clean":"2024-08-17T18:24:20.493674+0000","last_became_active":"2024-08-17T18:19:27.022827+0000","last_became_peered":"2024-08-17T18:19:27.022827+0000","last_unstale":"2024-08-17T18:24:20.493674+0000","last_undegraded":"2024-08-17T18:24:20.493674+0000","last_fullsized":"2024-08-17T18:24:20.493674+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:25:50.310320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2471,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493345+0000","last_change":"2024-08-17T18:21:33.421541+0000","last_active":"2024-08-17T18:24:20.493345+0000","last_peered":"2024-08-17T18:24:20.493345+0000","last_clean":"2024-08-17T18:24:20.493345+0000","last_became_active":"2024-08-17T18:21:33.420932+0000","last_became_peered":"2024-08-17T18:21:33.420932+0000","last_unstale":"2024-08-17T18:24:20.493345+0000","last_undegraded":"2024-08-17T18:24:20.493345+0000","last_fullsized":"2024-08-17T18:24:20.493345+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:23.366908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":32,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467402+0000","last_change":"2024-08-17T18:23:12.502922+0000","last_active":"2024-08-17T18:24:17.467402+0000","last_peered":"2024-08-17T18:24:17.467402+0000","last_clean":"2024-08-17T18:24:17.467402+0000","last_became_active":"2024-08-17T18:23:12.494769+0000","last_became_peered":"2024-08-17T18:23:12.494769+0000","last_unstale":"2024-08-17T18:24:17.467402+0000","last_undegraded":"2024-08-17T18:24:17.467402+0000","last_fullsized":"2024-08-17T18:24:17.467402+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:50.746984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":132,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468944+0000","last_change":"2024-08-17T18:21:33.404642+0000","last_active":"2024-08-17T18:24:17.468944+0000","last_peered":"2024-08-17T18:24:17.468944+0000","last_clean":"2024-08-17T18:24:17.468944+0000","last_became_active":"2024-08-17T18:21:33.404374+0000","last_became_peered":"2024-08-17T18:21:33.404374+0000","last_unstale":"2024-08-17T18:24:17.468944+0000","last_undegraded":"2024-08-17T18:24:17.468944+0000","last_fullsized":"2024-08-17T18:24:17.468944+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:49:34.531843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":32,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467135+0000","last_change":"2024-08-17T18:23:12.500729+0000","last_active":"2024-08-17T18:24:17.467135+0000","last_peered":"2024-08-17T18:24:17.467135+0000","last_clean":"2024-08-17T18:24:17.467135+0000","last_became_active":"2024-08-17T18:23:12.492379+0000","last_became_peered":"2024-08-17T18:23:12.492379+0000","last_unstale":"2024-08-17T18:24:17.467135+0000","last_undegraded":"2024-08-17T18:24:17.467135+0000","last_fullsized":"2024-08-17T18:24:17.467135+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:15:18.000497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":132,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.469074+0000","last_change":"2024-08-17T18:23:05.473376+0000","last_active":"2024-08-17T18:24:17.469074+0000","last_peered":"2024-08-17T18:24:17.469074+0000","last_clean":"2024-08-17T18:24:17.469074+0000","last_became_active":"2024-08-17T18:23:05.472991+0000","last_became_peered":"2024-08-17T18:23:05.472991+0000","last_unstale":"2024-08-17T18:24:17.469074+0000","last_undegraded":"2024-08-17T18:24:17.469074+0000","last_fullsized":"2024-08-17T18:24:17.469074+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:40:40.533867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":130,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631378+0000","last_change":"2024-08-17T18:21:33.424299+0000","last_active":"2024-08-17T18:24:20.631378+0000","last_peered":"2024-08-17T18:24:20.631378+0000","last_clean":"2024-08-17T18:24:20.631378+0000","last_became_active":"2024-08-17T18:21:33.424016+0000","last_became_peered":"2024-08-17T18:21:33.424016+0000","last_unstale":"2024-08-17T18:24:20.631378+0000","last_undegraded":"2024-08-17T18:24:20.631378+0000","last_fullsized":"2024-08-17T18:24:20.631378+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:15.128415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2130,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.492706+0000","last_change":"2024-08-17T18:23:05.475948+0000","last_active":"2024-08-17T18:24:20.492706+0000","last_peered":"2024-08-17T18:24:20.492706+0000","last_clean":"2024-08-17T18:24:20.492706+0000","last_became_active":"2024-08-17T18:23:05.475137+0000","last_became_peered":"2024-08-17T18:23:05.475137+0000","last_unstale":"2024-08-17T18:24:20.492706+0000","last_undegraded":"2024-08-17T18:24:20.492706+0000","last_fullsized":"2024-08-17T18:24:20.492706+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:40.443880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":146,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468810+0000","last_change":"2024-08-17T18:23:05.473551+0000","last_active":"2024-08-17T18:24:17.468810+0000","last_peered":"2024-08-17T18:24:17.468810+0000","last_clean":"2024-08-17T18:24:17.468810+0000","last_became_active":"2024-08-17T18:23:05.473125+0000","last_became_peered":"2024-08-17T18:23:05.473125+0000","last_unstale":"2024-08-17T18:24:17.468810+0000","last_undegraded":"2024-08-17T18:24:17.468810+0000","last_fullsized":"2024-08-17T18:24:17.468810+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:55:43.580574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2433,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493551+0000","last_change":"2024-08-17T18:23:05.468151+0000","last_active":"2024-08-17T18:24:20.493551+0000","last_peered":"2024-08-17T18:24:20.493551+0000","last_clean":"2024-08-17T18:24:20.493551+0000","last_became_active":"2024-08-17T18:23:05.467820+0000","last_became_peered":"2024-08-17T18:23:05.467820+0000","last_unstale":"2024-08-17T18:24:20.493551+0000","last_undegraded":"2024-08-17T18:24:20.493551+0000","last_fullsized":"2024-08-17T18:24:20.493551+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:23:15.032777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631419+0000","last_change":"2024-08-17T18:23:05.472428+0000","last_active":"2024-08-17T18:24:20.631419+0000","last_peered":"2024-08-17T18:24:20.631419+0000","last_clean":"2024-08-17T18:24:20.631419+0000","last_became_active":"2024-08-17T18:23:05.468270+0000","last_became_peered":"2024-08-17T18:23:05.468270+0000","last_unstale":"2024-08-17T18:24:20.631419+0000","last_undegraded":"2024-08-17T18:24:20.631419+0000","last_fullsized":"2024-08-17T18:24:20.631419+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:06.983782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":130,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631293+0000","last_change":"2024-08-17T18:21:33.408805+0000","last_active":"2024-08-17T18:24:20.631293+0000","last_peered":"2024-08-17T18:24:20.631293+0000","last_clean":"2024-08-17T18:24:20.631293+0000","last_became_active":"2024-08-17T18:21:33.408216+0000","last_became_peered":"2024-08-17T18:21:33.408216+0000","last_unstale":"2024-08-17T18:24:20.631293+0000","last_undegraded":"2024-08-17T18:24:20.631293+0000","last_fullsized":"2024-08-17T18:24:20.631293+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:45.784110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2148,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.492493+0000","last_change":"2024-08-17T18:23:05.469853+0000","last_active":"2024-08-17T18:24:20.492493+0000","last_peered":"2024-08-17T18:24:20.492493+0000","last_clean":"2024-08-17T18:24:20.492493+0000","last_became_active":"2024-08-17T18:23:05.469535+0000","last_became_peered":"2024-08-17T18:23:05.469535+0000","last_unstale":"2024-08-17T18:24:20.492493+0000","last_undegraded":"2024-08-17T18:24:20.492493+0000","last_fullsized":"2024-08-17T18:24:20.492493+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:21.036841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":197,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467313+0000","last_change":"2024-08-17T18:23:12.491476+0000","last_active":"2024-08-17T18:24:17.467313+0000","last_peered":"2024-08-17T18:24:17.467313+0000","last_clean":"2024-08-17T18:24:17.467313+0000","last_became_active":"2024-08-17T18:23:12.491162+0000","last_became_peered":"2024-08-17T18:23:12.491162+0000","last_unstale":"2024-08-17T18:24:17.467313+0000","last_undegraded":"2024-08-17T18:24:17.467313+0000","last_fullsized":"2024-08-17T18:24:17.467313+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:20.885678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2420,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493091+0000","last_change":"2024-08-17T18:23:05.468384+0000","last_active":"2024-08-17T18:24:20.493091+0000","last_peered":"2024-08-17T18:24:20.493091+0000","last_clean":"2024-08-17T18:24:20.493091+0000","last_became_active":"2024-08-17T18:23:05.467843+0000","last_became_peered":"2024-08-17T18:23:05.467843+0000","last_unstale":"2024-08-17T18:24:20.493091+0000","last_undegraded":"2024-08-17T18:24:20.493091+0000","last_fullsized":"2024-08-17T18:24:20.493091+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:18:23.218603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":137,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631955+0000","last_change":"2024-08-17T18:23:12.491873+0000","last_active":"2024-08-17T18:24:20.631955+0000","last_peered":"2024-08-17T18:24:20.631955+0000","last_clean":"2024-08-17T18:24:20.631955+0000","last_became_active":"2024-08-17T18:23:12.491406+0000","last_became_peered":"2024-08-17T18:23:12.491406+0000","last_unstale":"2024-08-17T18:24:20.631955+0000","last_undegraded":"2024-08-17T18:24:20.631955+0000","last_fullsized":"2024-08-17T18:24:20.631955+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:28.454580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2354,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493132+0000","last_change":"2024-08-17T18:16:34.644366+0000","last_active":"2024-08-17T18:24:20.493132+0000","last_peered":"2024-08-17T18:24:20.493132+0000","last_clean":"2024-08-17T18:24:20.493132+0000","last_became_active":"2024-08-17T18:16:34.643994+0000","last_became_peered":"2024-08-17T18:16:34.643994+0000","last_unstale":"2024-08-17T18:24:20.493132+0000","last_undegraded":"2024-08-17T18:24:20.493132+0000","last_fullsized":"2024-08-17T18:24:20.493132+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:51.953343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":28,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.466987+0000","last_change":"2024-08-17T18:23:12.497021+0000","last_active":"2024-08-17T18:24:17.466987+0000","last_peered":"2024-08-17T18:24:17.466987+0000","last_clean":"2024-08-17T18:24:17.466987+0000","last_became_active":"2024-08-17T18:23:12.496093+0000","last_became_peered":"2024-08-17T18:23:12.496093+0000","last_unstale":"2024-08-17T18:24:17.466987+0000","last_undegraded":"2024-08-17T18:24:17.466987+0000","last_fullsized":"2024-08-17T18:24:17.466987+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:43.945061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1552,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731702+0000","last_change":"2024-08-17T18:18:50.525363+0000","last_active":"2024-08-17T18:24:17.731702+0000","last_peered":"2024-08-17T18:24:17.731702+0000","last_clean":"2024-08-17T18:24:17.731702+0000","last_became_active":"2024-08-17T18:18:50.511676+0000","last_became_peered":"2024-08-17T18:18:50.511676+0000","last_unstale":"2024-08-17T18:24:17.731702+0000","last_undegraded":"2024-08-17T18:24:17.731702+0000","last_fullsized":"2024-08-17T18:24:17.731702+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:22.530382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631643+0000","last_change":"2024-08-17T18:23:05.472573+0000","last_active":"2024-08-17T18:24:20.631643+0000","last_peered":"2024-08-17T18:24:20.631643+0000","last_clean":"2024-08-17T18:24:20.631643+0000","last_became_active":"2024-08-17T18:23:05.471921+0000","last_became_peered":"2024-08-17T18:23:05.471921+0000","last_unstale":"2024-08-17T18:24:20.631643+0000","last_undegraded":"2024-08-17T18:24:20.631643+0000","last_fullsized":"2024-08-17T18:24:20.631643+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:10.207241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":121,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468618+0000","last_change":"2024-08-17T18:18:51.515962+0000","last_active":"2024-08-17T18:24:17.468618+0000","last_peered":"2024-08-17T18:24:17.468618+0000","last_clean":"2024-08-17T18:24:17.468618+0000","last_became_active":"2024-08-17T18:18:51.505508+0000","last_became_peered":"2024-08-17T18:18:51.505508+0000","last_unstale":"2024-08-17T18:24:17.468618+0000","last_undegraded":"2024-08-17T18:24:17.468618+0000","last_fullsized":"2024-08-17T18:24:17.468618+0000","mapping_epoch":1447,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:16.518470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":131,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468968+0000","last_change":"2024-08-17T18:23:05.473525+0000","last_active":"2024-08-17T18:24:17.468968+0000","last_peered":"2024-08-17T18:24:17.468968+0000","last_clean":"2024-08-17T18:24:17.468968+0000","last_became_active":"2024-08-17T18:23:05.473177+0000","last_became_peered":"2024-08-17T18:23:05.473177+0000","last_unstale":"2024-08-17T18:24:17.468968+0000","last_undegraded":"2024-08-17T18:24:17.468968+0000","last_fullsized":"2024-08-17T18:24:17.468968+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:20.885141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":152,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468740+0000","last_change":"2024-08-17T18:23:12.491602+0000","last_active":"2024-08-17T18:24:17.468740+0000","last_peered":"2024-08-17T18:24:17.468740+0000","last_clean":"2024-08-17T18:24:17.468740+0000","last_became_active":"2024-08-17T18:23:12.491282+0000","last_became_peered":"2024-08-17T18:23:12.491282+0000","last_unstale":"2024-08-17T18:24:17.468740+0000","last_undegraded":"2024-08-17T18:24:17.468740+0000","last_fullsized":"2024-08-17T18:24:17.468740+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:02:52.727878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":151,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467390+0000","last_change":"2024-08-17T18:19:27.055728+0000","last_active":"2024-08-17T18:24:17.467390+0000","last_peered":"2024-08-17T18:24:17.467390+0000","last_clean":"2024-08-17T18:24:17.467390+0000","last_became_active":"2024-08-17T18:19:27.055425+0000","last_became_peered":"2024-08-17T18:19:27.055425+0000","last_unstale":"2024-08-17T18:24:17.467390+0000","last_undegraded":"2024-08-17T18:24:17.467390+0000","last_fullsized":"2024-08-17T18:24:17.467390+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:47.920145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":156,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631759+0000","last_change":"2024-08-17T18:23:05.479998+0000","last_active":"2024-08-17T18:24:20.631759+0000","last_peered":"2024-08-17T18:24:20.631759+0000","last_clean":"2024-08-17T18:24:20.631759+0000","last_became_active":"2024-08-17T18:23:05.465166+0000","last_became_peered":"2024-08-17T18:23:05.465166+0000","last_unstale":"2024-08-17T18:24:20.631759+0000","last_undegraded":"2024-08-17T18:24:20.631759+0000","last_fullsized":"2024-08-17T18:24:20.631759+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:58:08.921362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2422,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493130+0000","last_change":"2024-08-17T18:21:33.421768+0000","last_active":"2024-08-17T18:24:20.493130+0000","last_peered":"2024-08-17T18:24:20.493130+0000","last_clean":"2024-08-17T18:24:20.493130+0000","last_became_active":"2024-08-17T18:21:33.421467+0000","last_became_peered":"2024-08-17T18:21:33.421467+0000","last_unstale":"2024-08-17T18:24:20.493130+0000","last_undegraded":"2024-08-17T18:24:20.493130+0000","last_fullsized":"2024-08-17T18:24:20.493130+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:38.751105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":54,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631591+0000","last_change":"2024-08-17T18:23:05.466442+0000","last_active":"2024-08-17T18:24:20.631591+0000","last_peered":"2024-08-17T18:24:20.631591+0000","last_clean":"2024-08-17T18:24:20.631591+0000","last_became_active":"2024-08-17T18:23:05.466154+0000","last_became_peered":"2024-08-17T18:23:05.466154+0000","last_unstale":"2024-08-17T18:24:20.631591+0000","last_undegraded":"2024-08-17T18:24:20.631591+0000","last_fullsized":"2024-08-17T18:24:20.631591+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:50:14.484586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":2443,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493727+0000","last_change":"2024-08-17T18:21:33.497158+0000","last_active":"2024-08-17T18:24:20.493727+0000","last_peered":"2024-08-17T18:24:20.493727+0000","last_clean":"2024-08-17T18:24:20.493727+0000","last_became_active":"2024-08-17T18:21:33.421674+0000","last_became_peered":"2024-08-17T18:21:33.421674+0000","last_unstale":"2024-08-17T18:24:20.493727+0000","last_undegraded":"2024-08-17T18:24:20.493727+0000","last_fullsized":"2024-08-17T18:24:20.493727+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:58:57.198125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":151,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631570+0000","last_change":"2024-08-17T18:23:12.495046+0000","last_active":"2024-08-17T18:24:20.631570+0000","last_peered":"2024-08-17T18:24:20.631570+0000","last_clean":"2024-08-17T18:24:20.631570+0000","last_became_active":"2024-08-17T18:23:12.494730+0000","last_became_peered":"2024-08-17T18:23:12.494730+0000","last_unstale":"2024-08-17T18:24:20.631570+0000","last_undegraded":"2024-08-17T18:24:20.631570+0000","last_fullsized":"2024-08-17T18:24:20.631570+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:57:24.534722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1589,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731838+0000","last_change":"2024-08-17T18:21:33.420079+0000","last_active":"2024-08-17T18:24:17.731838+0000","last_peered":"2024-08-17T18:24:17.731838+0000","last_clean":"2024-08-17T18:24:17.731838+0000","last_became_active":"2024-08-17T18:21:33.419265+0000","last_became_peered":"2024-08-17T18:21:33.419265+0000","last_unstale":"2024-08-17T18:24:17.731838+0000","last_undegraded":"2024-08-17T18:24:17.731838+0000","last_fullsized":"2024-08-17T18:24:17.731838+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:33:44.811302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":61,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631804+0000","last_change":"2024-08-17T18:23:12.494642+0000","last_active":"2024-08-17T18:24:20.631804+0000","last_peered":"2024-08-17T18:24:20.631804+0000","last_clean":"2024-08-17T18:24:20.631804+0000","last_became_active":"2024-08-17T18:23:12.494333+0000","last_became_peered":"2024-08-17T18:23:12.494333+0000","last_unstale":"2024-08-17T18:24:20.631804+0000","last_undegraded":"2024-08-17T18:24:20.631804+0000","last_fullsized":"2024-08-17T18:24:20.631804+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:11.129724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.492684+0000","last_change":"2024-08-17T18:24:17.471276+0000","last_active":"2024-08-17T18:24:20.492684+0000","last_peered":"2024-08-17T18:24:20.492684+0000","last_clean":"2024-08-17T18:24:20.492684+0000","last_became_active":"2024-08-17T18:24:17.470930+0000","last_became_peered":"2024-08-17T18:24:17.470930+0000","last_unstale":"2024-08-17T18:24:20.492684+0000","last_undegraded":"2024-08-17T18:24:20.492684+0000","last_fullsized":"2024-08-17T18:24:20.492684+0000","mapping_epoch":1547,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:28.633664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":54,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631771+0000","last_change":"2024-08-17T18:23:05.473157+0000","last_active":"2024-08-17T18:24:20.631771+0000","last_peered":"2024-08-17T18:24:20.631771+0000","last_clean":"2024-08-17T18:24:20.631771+0000","last_became_active":"2024-08-17T18:23:05.468199+0000","last_became_peered":"2024-08-17T18:23:05.468199+0000","last_unstale":"2024-08-17T18:24:20.631771+0000","last_undegraded":"2024-08-17T18:24:20.631771+0000","last_fullsized":"2024-08-17T18:24:20.631771+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:17:53.820423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":28,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467322+0000","last_change":"2024-08-17T18:23:12.495597+0000","last_active":"2024-08-17T18:24:17.467322+0000","last_peered":"2024-08-17T18:24:17.467322+0000","last_clean":"2024-08-17T18:24:17.467322+0000","last_became_active":"2024-08-17T18:23:12.493578+0000","last_became_peered":"2024-08-17T18:23:12.493578+0000","last_unstale":"2024-08-17T18:24:17.467322+0000","last_undegraded":"2024-08-17T18:24:17.467322+0000","last_fullsized":"2024-08-17T18:24:17.467322+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:17.580085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":54,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631824+0000","last_change":"2024-08-17T18:23:05.475522+0000","last_active":"2024-08-17T18:24:20.631824+0000","last_peered":"2024-08-17T18:24:20.631824+0000","last_clean":"2024-08-17T18:24:20.631824+0000","last_became_active":"2024-08-17T18:23:05.475176+0000","last_became_peered":"2024-08-17T18:23:05.475176+0000","last_unstale":"2024-08-17T18:24:20.631824+0000","last_undegraded":"2024-08-17T18:24:20.631824+0000","last_fullsized":"2024-08-17T18:24:20.631824+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:05.025690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":132,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.469205+0000","last_change":"2024-08-17T18:21:33.403089+0000","last_active":"2024-08-17T18:24:17.469205+0000","last_peered":"2024-08-17T18:24:17.469205+0000","last_clean":"2024-08-17T18:24:17.469205+0000","last_became_active":"2024-08-17T18:21:33.402788+0000","last_became_peered":"2024-08-17T18:21:33.402788+0000","last_unstale":"2024-08-17T18:24:17.469205+0000","last_undegraded":"2024-08-17T18:24:17.469205+0000","last_fullsized":"2024-08-17T18:24:17.469205+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:25.909442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":32,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467139+0000","last_change":"2024-08-17T18:23:12.506453+0000","last_active":"2024-08-17T18:24:17.467139+0000","last_peered":"2024-08-17T18:24:17.467139+0000","last_clean":"2024-08-17T18:24:17.467139+0000","last_became_active":"2024-08-17T18:23:12.493518+0000","last_became_peered":"2024-08-17T18:23:12.493518+0000","last_unstale":"2024-08-17T18:24:17.467139+0000","last_undegraded":"2024-08-17T18:24:17.467139+0000","last_fullsized":"2024-08-17T18:24:17.467139+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:36.173814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":61,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631662+0000","last_change":"2024-08-17T18:23:12.493978+0000","last_active":"2024-08-17T18:24:20.631662+0000","last_peered":"2024-08-17T18:24:20.631662+0000","last_clean":"2024-08-17T18:24:20.631662+0000","last_became_active":"2024-08-17T18:23:12.493667+0000","last_became_peered":"2024-08-17T18:23:12.493667+0000","last_unstale":"2024-08-17T18:24:20.631662+0000","last_undegraded":"2024-08-17T18:24:20.631662+0000","last_fullsized":"2024-08-17T18:24:20.631662+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:40.765437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631296+0000","last_change":"2024-08-17T18:23:05.466293+0000","last_active":"2024-08-17T18:24:20.631296+0000","last_peered":"2024-08-17T18:24:20.631296+0000","last_clean":"2024-08-17T18:24:20.631296+0000","last_became_active":"2024-08-17T18:23:05.466000+0000","last_became_peered":"2024-08-17T18:23:05.466000+0000","last_unstale":"2024-08-17T18:24:20.631296+0000","last_undegraded":"2024-08-17T18:24:20.631296+0000","last_fullsized":"2024-08-17T18:24:20.631296+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:04.835392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2347,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.492617+0000","last_change":"2024-08-17T18:21:33.419513+0000","last_active":"2024-08-17T18:24:20.492617+0000","last_peered":"2024-08-17T18:24:20.492617+0000","last_clean":"2024-08-17T18:24:20.492617+0000","last_became_active":"2024-08-17T18:21:33.419186+0000","last_became_peered":"2024-08-17T18:21:33.419186+0000","last_unstale":"2024-08-17T18:24:20.492617+0000","last_undegraded":"2024-08-17T18:24:20.492617+0000","last_fullsized":"2024-08-17T18:24:20.492617+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:27.395794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":144,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631381+0000","last_change":"2024-08-17T18:23:05.467307+0000","last_active":"2024-08-17T18:24:20.631381+0000","last_peered":"2024-08-17T18:24:20.631381+0000","last_clean":"2024-08-17T18:24:20.631381+0000","last_became_active":"2024-08-17T18:23:05.466928+0000","last_became_peered":"2024-08-17T18:23:05.466928+0000","last_unstale":"2024-08-17T18:24:20.631381+0000","last_undegraded":"2024-08-17T18:24:20.631381+0000","last_fullsized":"2024-08-17T18:24:20.631381+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:26:02.689664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":130,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631221+0000","last_change":"2024-08-17T18:21:33.407703+0000","last_active":"2024-08-17T18:24:20.631221+0000","last_peered":"2024-08-17T18:24:20.631221+0000","last_clean":"2024-08-17T18:24:20.631221+0000","last_became_active":"2024-08-17T18:21:33.407397+0000","last_became_peered":"2024-08-17T18:21:33.407397+0000","last_unstale":"2024-08-17T18:24:20.631221+0000","last_undegraded":"2024-08-17T18:24:20.631221+0000","last_fullsized":"2024-08-17T18:24:20.631221+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:06.725837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1620,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731319+0000","last_change":"2024-08-17T18:23:05.463133+0000","last_active":"2024-08-17T18:24:17.731319+0000","last_peered":"2024-08-17T18:24:17.731319+0000","last_clean":"2024-08-17T18:24:17.731319+0000","last_became_active":"2024-08-17T18:23:05.462900+0000","last_became_peered":"2024-08-17T18:23:05.462900+0000","last_unstale":"2024-08-17T18:24:17.731319+0000","last_undegraded":"2024-08-17T18:24:17.731319+0000","last_fullsized":"2024-08-17T18:24:17.731319+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:34.527149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":147,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467763+0000","last_change":"2024-08-17T18:18:50.516313+0000","last_active":"2024-08-17T18:24:17.467763+0000","last_peered":"2024-08-17T18:24:17.467763+0000","last_clean":"2024-08-17T18:24:17.467763+0000","last_became_active":"2024-08-17T18:18:50.515925+0000","last_became_peered":"2024-08-17T18:18:50.515925+0000","last_unstale":"2024-08-17T18:24:17.467763+0000","last_undegraded":"2024-08-17T18:24:17.467763+0000","last_fullsized":"2024-08-17T18:24:17.467763+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:14.611619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2401,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493304+0000","last_change":"2024-08-17T18:23:12.493173+0000","last_active":"2024-08-17T18:24:20.493304+0000","last_peered":"2024-08-17T18:24:20.493304+0000","last_clean":"2024-08-17T18:24:20.493304+0000","last_became_active":"2024-08-17T18:23:12.492862+0000","last_became_peered":"2024-08-17T18:23:12.492862+0000","last_unstale":"2024-08-17T18:24:20.493304+0000","last_undegraded":"2024-08-17T18:24:20.493304+0000","last_fullsized":"2024-08-17T18:24:20.493304+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:26.508877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":32,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467287+0000","last_change":"2024-08-17T18:23:12.509125+0000","last_active":"2024-08-17T18:24:17.467287+0000","last_peered":"2024-08-17T18:24:17.467287+0000","last_clean":"2024-08-17T18:24:17.467287+0000","last_became_active":"2024-08-17T18:23:12.495743+0000","last_became_peered":"2024-08-17T18:23:12.495743+0000","last_unstale":"2024-08-17T18:24:17.467287+0000","last_undegraded":"2024-08-17T18:24:17.467287+0000","last_fullsized":"2024-08-17T18:24:17.467287+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:30:20.834830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":2020,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731288+0000","last_change":"2024-08-17T18:22:00.258650+0000","last_active":"2024-08-17T18:24:17.731288+0000","last_peered":"2024-08-17T18:24:17.731288+0000","last_clean":"2024-08-17T18:24:17.731288+0000","last_became_active":"2024-08-17T18:22:00.258326+0000","last_became_peered":"2024-08-17T18:22:00.258326+0000","last_unstale":"2024-08-17T18:24:17.731288+0000","last_undegraded":"2024-08-17T18:24:17.731288+0000","last_fullsized":"2024-08-17T18:24:17.731288+0000","mapping_epoch":1500,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:02.578380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631566+0000","last_change":"2024-08-17T18:23:05.468372+0000","last_active":"2024-08-17T18:24:20.631566+0000","last_peered":"2024-08-17T18:24:20.631566+0000","last_clean":"2024-08-17T18:24:20.631566+0000","last_became_active":"2024-08-17T18:23:05.468085+0000","last_became_peered":"2024-08-17T18:23:05.468085+0000","last_unstale":"2024-08-17T18:24:20.631566+0000","last_undegraded":"2024-08-17T18:24:20.631566+0000","last_fullsized":"2024-08-17T18:24:20.631566+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:20:45.718917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2458,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493231+0000","last_change":"2024-08-17T18:23:05.468144+0000","last_active":"2024-08-17T18:24:20.493231+0000","last_peered":"2024-08-17T18:24:20.493231+0000","last_clean":"2024-08-17T18:24:20.493231+0000","last_became_active":"2024-08-17T18:23:05.467350+0000","last_became_peered":"2024-08-17T18:23:05.467350+0000","last_unstale":"2024-08-17T18:24:20.493231+0000","last_undegraded":"2024-08-17T18:24:20.493231+0000","last_fullsized":"2024-08-17T18:24:20.493231+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:04.364659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":142,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.474926+0000","last_change":"2024-08-17T18:24:17.472904+0000","last_active":"2024-08-17T18:24:17.474926+0000","last_peered":"2024-08-17T18:24:17.474926+0000","last_clean":"2024-08-17T18:24:17.474926+0000","last_became_active":"2024-08-17T18:24:17.472453+0000","last_became_peered":"2024-08-17T18:24:17.472453+0000","last_unstale":"2024-08-17T18:24:17.474926+0000","last_undegraded":"2024-08-17T18:24:17.474926+0000","last_fullsized":"2024-08-17T18:24:17.474926+0000","mapping_epoch":1547,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:15.975606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":126,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468695+0000","last_change":"2024-08-17T18:19:27.038041+0000","last_active":"2024-08-17T18:24:17.468695+0000","last_peered":"2024-08-17T18:24:17.468695+0000","last_clean":"2024-08-17T18:24:17.468695+0000","last_became_active":"2024-08-17T18:19:27.037715+0000","last_became_peered":"2024-08-17T18:19:27.037715+0000","last_unstale":"2024-08-17T18:24:17.468695+0000","last_undegraded":"2024-08-17T18:24:17.468695+0000","last_fullsized":"2024-08-17T18:24:17.468695+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:06:39.726534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":153,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.474917+0000","last_change":"2024-08-17T18:24:17.473333+0000","last_active":"2024-08-17T18:24:17.474917+0000","last_peered":"2024-08-17T18:24:17.474917+0000","last_clean":"2024-08-17T18:24:17.474917+0000","last_became_active":"2024-08-17T18:24:17.472908+0000","last_became_peered":"2024-08-17T18:24:17.472908+0000","last_unstale":"2024-08-17T18:24:17.474917+0000","last_undegraded":"2024-08-17T18:24:17.474917+0000","last_fullsized":"2024-08-17T18:24:17.474917+0000","mapping_epoch":1547,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:57.382675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":167,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468051+0000","last_change":"2024-08-17T18:23:05.476053+0000","last_active":"2024-08-17T18:24:17.468051+0000","last_peered":"2024-08-17T18:24:17.468051+0000","last_clean":"2024-08-17T18:24:17.468051+0000","last_became_active":"2024-08-17T18:23:05.475692+0000","last_became_peered":"2024-08-17T18:23:05.475692+0000","last_unstale":"2024-08-17T18:24:17.468051+0000","last_undegraded":"2024-08-17T18:24:17.468051+0000","last_fullsized":"2024-08-17T18:24:17.468051+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:29:41.007375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":112,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468829+0000","last_change":"2024-08-17T18:18:50.515676+0000","last_active":"2024-08-17T18:24:17.468829+0000","last_peered":"2024-08-17T18:24:17.468829+0000","last_clean":"2024-08-17T18:24:17.468829+0000","last_became_active":"2024-08-17T18:18:50.515313+0000","last_became_peered":"2024-08-17T18:18:50.515313+0000","last_unstale":"2024-08-17T18:24:17.468829+0000","last_undegraded":"2024-08-17T18:24:17.468829+0000","last_fullsized":"2024-08-17T18:24:17.468829+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:36.955526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2481,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493349+0000","last_change":"2024-08-17T18:21:33.493429+0000","last_active":"2024-08-17T18:24:20.493349+0000","last_peered":"2024-08-17T18:24:20.493349+0000","last_clean":"2024-08-17T18:24:20.493349+0000","last_became_active":"2024-08-17T18:21:33.421176+0000","last_became_peered":"2024-08-17T18:21:33.421176+0000","last_unstale":"2024-08-17T18:24:20.493349+0000","last_undegraded":"2024-08-17T18:24:20.493349+0000","last_fullsized":"2024-08-17T18:24:20.493349+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:42.334437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":144,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631488+0000","last_change":"2024-08-17T18:23:05.469853+0000","last_active":"2024-08-17T18:24:20.631488+0000","last_peered":"2024-08-17T18:24:20.631488+0000","last_clean":"2024-08-17T18:24:20.631488+0000","last_became_active":"2024-08-17T18:23:05.469532+0000","last_became_peered":"2024-08-17T18:23:05.469532+0000","last_unstale":"2024-08-17T18:24:20.631488+0000","last_undegraded":"2024-08-17T18:24:20.631488+0000","last_fullsized":"2024-08-17T18:24:20.631488+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:28.110068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":28,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467181+0000","last_change":"2024-08-17T18:23:12.493834+0000","last_active":"2024-08-17T18:24:17.467181+0000","last_peered":"2024-08-17T18:24:17.467181+0000","last_clean":"2024-08-17T18:24:17.467181+0000","last_became_active":"2024-08-17T18:23:12.493498+0000","last_became_peered":"2024-08-17T18:23:12.493498+0000","last_unstale":"2024-08-17T18:24:17.467181+0000","last_undegraded":"2024-08-17T18:24:17.467181+0000","last_fullsized":"2024-08-17T18:24:17.467181+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:05.457849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":157,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467996+0000","last_change":"2024-08-17T18:23:05.468008+0000","last_active":"2024-08-17T18:24:17.467996+0000","last_peered":"2024-08-17T18:24:17.467996+0000","last_clean":"2024-08-17T18:24:17.467996+0000","last_became_active":"2024-08-17T18:23:05.467317+0000","last_became_peered":"2024-08-17T18:23:05.467317+0000","last_unstale":"2024-08-17T18:24:17.467996+0000","last_undegraded":"2024-08-17T18:24:17.467996+0000","last_fullsized":"2024-08-17T18:24:17.467996+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:58.702296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1585,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731806+0000","last_change":"2024-08-17T18:23:12.504856+0000","last_active":"2024-08-17T18:24:17.731806+0000","last_peered":"2024-08-17T18:24:17.731806+0000","last_clean":"2024-08-17T18:24:17.731806+0000","last_became_active":"2024-08-17T18:23:12.497068+0000","last_became_peered":"2024-08-17T18:23:12.497068+0000","last_unstale":"2024-08-17T18:24:17.731806+0000","last_undegraded":"2024-08-17T18:24:17.731806+0000","last_fullsized":"2024-08-17T18:24:17.731806+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:45:02.864888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":132,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468829+0000","last_change":"2024-08-17T18:23:05.467377+0000","last_active":"2024-08-17T18:24:17.468829+0000","last_peered":"2024-08-17T18:24:17.468829+0000","last_clean":"2024-08-17T18:24:17.468829+0000","last_became_active":"2024-08-17T18:23:05.467049+0000","last_became_peered":"2024-08-17T18:23:05.467049+0000","last_unstale":"2024-08-17T18:24:17.468829+0000","last_undegraded":"2024-08-17T18:24:17.468829+0000","last_fullsized":"2024-08-17T18:24:17.468829+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:23.876632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1600,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731866+0000","last_change":"2024-08-17T18:23:05.475599+0000","last_active":"2024-08-17T18:24:17.731866+0000","last_peered":"2024-08-17T18:24:17.731866+0000","last_clean":"2024-08-17T18:24:17.731866+0000","last_became_active":"2024-08-17T18:23:05.464545+0000","last_became_peered":"2024-08-17T18:23:05.464545+0000","last_unstale":"2024-08-17T18:24:17.731866+0000","last_undegraded":"2024-08-17T18:24:17.731866+0000","last_fullsized":"2024-08-17T18:24:17.731866+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:54:53.309007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":28,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.466870+0000","last_change":"2024-08-17T18:23:12.495976+0000","last_active":"2024-08-17T18:24:17.466870+0000","last_peered":"2024-08-17T18:24:17.466870+0000","last_clean":"2024-08-17T18:24:17.466870+0000","last_became_active":"2024-08-17T18:23:12.495615+0000","last_became_peered":"2024-08-17T18:23:12.495615+0000","last_unstale":"2024-08-17T18:24:17.466870+0000","last_undegraded":"2024-08-17T18:24:17.466870+0000","last_fullsized":"2024-08-17T18:24:17.466870+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:03:37.648294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1616,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731840+0000","last_change":"2024-08-17T18:21:33.495232+0000","last_active":"2024-08-17T18:24:17.731840+0000","last_peered":"2024-08-17T18:24:17.731840+0000","last_clean":"2024-08-17T18:24:17.731840+0000","last_became_active":"2024-08-17T18:21:33.438606+0000","last_became_peered":"2024-08-17T18:21:33.438606+0000","last_unstale":"2024-08-17T18:24:17.731840+0000","last_undegraded":"2024-08-17T18:24:17.731840+0000","last_fullsized":"2024-08-17T18:24:17.731840+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:14.007910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":247,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468108+0000","last_change":"2024-08-17T18:23:12.502460+0000","last_active":"2024-08-17T18:24:17.468108+0000","last_peered":"2024-08-17T18:24:17.468108+0000","last_clean":"2024-08-17T18:24:17.468108+0000","last_became_active":"2024-08-17T18:23:12.490256+0000","last_became_peered":"2024-08-17T18:23:12.490256+0000","last_unstale":"2024-08-17T18:24:17.468108+0000","last_undegraded":"2024-08-17T18:24:17.468108+0000","last_fullsized":"2024-08-17T18:24:17.468108+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:58:12.083653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2455,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493781+0000","last_change":"2024-08-17T18:23:05.469169+0000","last_active":"2024-08-17T18:24:20.493781+0000","last_peered":"2024-08-17T18:24:20.493781+0000","last_clean":"2024-08-17T18:24:20.493781+0000","last_became_active":"2024-08-17T18:23:05.467765+0000","last_became_peered":"2024-08-17T18:23:05.467765+0000","last_unstale":"2024-08-17T18:24:20.493781+0000","last_undegraded":"2024-08-17T18:24:20.493781+0000","last_fullsized":"2024-08-17T18:24:20.493781+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:14:04.176083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1540,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731380+0000","last_change":"2024-08-17T18:18:50.515245+0000","last_active":"2024-08-17T18:24:17.731380+0000","last_peered":"2024-08-17T18:24:17.731380+0000","last_clean":"2024-08-17T18:24:17.731380+0000","last_became_active":"2024-08-17T18:18:50.514785+0000","last_became_peered":"2024-08-17T18:18:50.514785+0000","last_unstale":"2024-08-17T18:24:17.731380+0000","last_undegraded":"2024-08-17T18:24:17.731380+0000","last_fullsized":"2024-08-17T18:24:17.731380+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:08.696581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":130,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631772+0000","last_change":"2024-08-17T18:21:33.408565+0000","last_active":"2024-08-17T18:24:20.631772+0000","last_peered":"2024-08-17T18:24:20.631772+0000","last_clean":"2024-08-17T18:24:20.631772+0000","last_became_active":"2024-08-17T18:21:33.407832+0000","last_became_peered":"2024-08-17T18:21:33.407832+0000","last_unstale":"2024-08-17T18:24:20.631772+0000","last_undegraded":"2024-08-17T18:24:20.631772+0000","last_fullsized":"2024-08-17T18:24:20.631772+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:09:06.558530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":130,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631389+0000","last_change":"2024-08-17T18:21:33.428616+0000","last_active":"2024-08-17T18:24:20.631389+0000","last_peered":"2024-08-17T18:24:20.631389+0000","last_clean":"2024-08-17T18:24:20.631389+0000","last_became_active":"2024-08-17T18:21:33.428368+0000","last_became_peered":"2024-08-17T18:21:33.428368+0000","last_unstale":"2024-08-17T18:24:20.631389+0000","last_undegraded":"2024-08-17T18:24:20.631389+0000","last_fullsized":"2024-08-17T18:24:20.631389+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:14.945169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":28,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467240+0000","last_change":"2024-08-17T18:23:12.497684+0000","last_active":"2024-08-17T18:24:17.467240+0000","last_peered":"2024-08-17T18:24:17.467240+0000","last_clean":"2024-08-17T18:24:17.467240+0000","last_became_active":"2024-08-17T18:23:12.497197+0000","last_became_peered":"2024-08-17T18:23:12.497197+0000","last_unstale":"2024-08-17T18:24:17.467240+0000","last_undegraded":"2024-08-17T18:24:17.467240+0000","last_fullsized":"2024-08-17T18:24:17.467240+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:04:38.203244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1589,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731595+0000","last_change":"2024-08-17T18:19:27.037702+0000","last_active":"2024-08-17T18:24:17.731595+0000","last_peered":"2024-08-17T18:24:17.731595+0000","last_clean":"2024-08-17T18:24:17.731595+0000","last_became_active":"2024-08-17T18:19:27.036532+0000","last_became_peered":"2024-08-17T18:19:27.036532+0000","last_unstale":"2024-08-17T18:24:17.731595+0000","last_undegraded":"2024-08-17T18:24:17.731595+0000","last_fullsized":"2024-08-17T18:24:17.731595+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:24.777460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2455,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493539+0000","last_change":"2024-08-17T18:23:05.476037+0000","last_active":"2024-08-17T18:24:20.493539+0000","last_peered":"2024-08-17T18:24:20.493539+0000","last_clean":"2024-08-17T18:24:20.493539+0000","last_became_active":"2024-08-17T18:23:05.475721+0000","last_became_peered":"2024-08-17T18:23:05.475721+0000","last_unstale":"2024-08-17T18:24:20.493539+0000","last_undegraded":"2024-08-17T18:24:20.493539+0000","last_fullsized":"2024-08-17T18:24:20.493539+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:30:24.857288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":12636,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234772,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1503200,"kb_used_data":9300,"kb_used_omap":1,"kb_used_meta":1493886,"kb_avail":92241952,"statfs":{"total":95995035648,"available":94455758848,"internally_reserved":0,"allocated":9523200,"data_stored":5491709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1529739608},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496407,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902488,"kb_used_data":9292,"kb_used_omap":1,"kb_used_meta":893182,"kb_avail":92842664,"statfs":{"total":95995035648,"available":95070887936,"internally_reserved":0,"allocated":9515008,"data_stored":5491574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":914618712},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208758045,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":318540,"kb_used_data":8704,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93426612,"statfs":{"total":95995035648,"available":95668850688,"internally_reserved":0,"allocated":8912896,"data_stored":4897032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1343,"seq":5768141078718,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":781572,"kb_used_data":8696,"kb_used_omap":112,"kb_used_meta":772751,"kb_avail":92963580,"statfs":{"total":95995035648,"available":95194705920,"internally_reserved":0,"allocated":8904704,"data_stored":4896491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":115373,"internal_metadata":791297363},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":1454,"seq":6244882448455,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":285000,"kb_used_data":8700,"kb_used_omap":111,"kb_used_meta":276176,"kb_avail":93460152,"statfs":{"total":95995035648,"available":95703195648,"internally_reserved":0,"allocated":8908800,"data_stored":4896997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":114548,"internal_metadata":282804364},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452432,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1112580,"kb_used_data":8696,"kb_used_omap":54,"kb_used_meta":1103817,"kb_avail":92632572,"statfs":{"total":95995035648,"available":94855753728,"internally_reserved":0,"allocated":8904704,"data_stored":4896412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":1130309006},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471475123,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1528140,"kb_used_data":9280,"kb_used_omap":119,"kb_used_meta":1518728,"kb_avail":92217012,"statfs":{"total":95995035648,"available":94430220288,"internally_reserved":0,"allocated":9502720,"data_stored":5487353,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":1555178206},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019680,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310232,"kb_used_data":8716,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93434920,"statfs":{"total":95995035648,"available":95677358080,"internally_reserved":0,"allocated":8925184,"data_stored":4901263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:25:10.724 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 25127030-5cbc-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T18:25:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:10 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T18:25:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:10 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T18:25:10.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:10 smithi179 ceph-mon[93519]: from='mgr.25982 ' entity='mgr.y' 2024-08-17T18:25:10.983 INFO:teuthology.orchestra.run.smithi130.stderr:Inferring config /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T18:25:11.901 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:11 smithi130 ceph-mon[105521]: pgmap v3066: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:11.901 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:11 smithi130 ceph-mon[105521]: from='client.47687 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:25:11.901 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:11 smithi130 ceph-mon[110068]: pgmap v3066: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:11.901 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:11 smithi130 ceph-mon[110068]: from='client.47687 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:25:12.092 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:11 smithi179 ceph-mon[93519]: pgmap v3066: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:12.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:11 smithi179 ceph-mon[93519]: from='client.47687 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:25:12.730 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:12 smithi130 ceph-mon[105521]: pgmap v3067: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:12.730 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:12 smithi130 ceph-mon[105521]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:25:12.730 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:12 smithi130 ceph-mon[110068]: pgmap v3067: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:12.730 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:12 smithi130 ceph-mon[110068]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:25:12.906 INFO:teuthology.orchestra.run.smithi130.stdout: 2024-08-17T18:25:12.907 INFO:teuthology.orchestra.run.smithi130.stderr:dumped all 2024-08-17T18:25:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:12 smithi179 ceph-mon[93519]: pgmap v3067: 73 pgs: 73 active+clean; 582 KiB data, 6.4 GiB used, 709 GiB / 715 GiB avail 2024-08-17T18:25:13.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:12 smithi179 ceph-mon[93519]: from='mgr.25982 172.21.15.130:0/1382866148' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T18:25:13.527 INFO:teuthology.orchestra.run.smithi130.stdout:{"pg_ready":true,"pg_map":{"version":3067,"stamp":"2024-08-17T18:25:11.756892+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595917,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":278,"num_write_kb":2447,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":125,"num_bytes_recovered":5326043,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207,"ondisk_log_size":207,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6741752,"kb_used_data":71384,"kb_used_omap":405,"kb_used_meta":6669866,"kb_avail":743219464,"statfs":{"total":767960285184,"available":761056731136,"internally_reserved":0,"allocated":73097216,"data_stored":40958831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414805,"internal_metadata":6829943723},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003456"},"pg_stats":[{"pgid":"3.1f","version":"71'4","reported_seq":149,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.632053+0000","last_change":"2024-08-17T18:23:12.507440+0000","last_active":"2024-08-17T18:24:20.632053+0000","last_peered":"2024-08-17T18:24:20.632053+0000","last_clean":"2024-08-17T18:24:20.632053+0000","last_became_active":"2024-08-17T18:23:12.493772+0000","last_became_peered":"2024-08-17T18:23:12.493772+0000","last_unstale":"2024-08-17T18:24:20.632053+0000","last_undegraded":"2024-08-17T18:24:20.632053+0000","last_fullsized":"2024-08-17T18:24:20.632053+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:28.385234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2280,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":132,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468402+0000","last_change":"2024-08-17T18:23:05.464220+0000","last_active":"2024-08-17T18:24:17.468402+0000","last_peered":"2024-08-17T18:24:17.468402+0000","last_clean":"2024-08-17T18:24:17.468402+0000","last_became_active":"2024-08-17T18:23:05.463841+0000","last_became_peered":"2024-08-17T18:23:05.463841+0000","last_unstale":"2024-08-17T18:24:17.468402+0000","last_undegraded":"2024-08-17T18:24:17.468402+0000","last_fullsized":"2024-08-17T18:24:17.468402+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:47:48.738919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":2406,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493674+0000","last_change":"2024-08-17T18:19:27.023211+0000","last_active":"2024-08-17T18:24:20.493674+0000","last_peered":"2024-08-17T18:24:20.493674+0000","last_clean":"2024-08-17T18:24:20.493674+0000","last_became_active":"2024-08-17T18:19:27.022827+0000","last_became_peered":"2024-08-17T18:19:27.022827+0000","last_unstale":"2024-08-17T18:24:20.493674+0000","last_undegraded":"2024-08-17T18:24:20.493674+0000","last_fullsized":"2024-08-17T18:24:20.493674+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:25:50.310320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2471,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493345+0000","last_change":"2024-08-17T18:21:33.421541+0000","last_active":"2024-08-17T18:24:20.493345+0000","last_peered":"2024-08-17T18:24:20.493345+0000","last_clean":"2024-08-17T18:24:20.493345+0000","last_became_active":"2024-08-17T18:21:33.420932+0000","last_became_peered":"2024-08-17T18:21:33.420932+0000","last_unstale":"2024-08-17T18:24:20.493345+0000","last_undegraded":"2024-08-17T18:24:20.493345+0000","last_fullsized":"2024-08-17T18:24:20.493345+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:48:23.366908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":32,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467402+0000","last_change":"2024-08-17T18:23:12.502922+0000","last_active":"2024-08-17T18:24:17.467402+0000","last_peered":"2024-08-17T18:24:17.467402+0000","last_clean":"2024-08-17T18:24:17.467402+0000","last_became_active":"2024-08-17T18:23:12.494769+0000","last_became_peered":"2024-08-17T18:23:12.494769+0000","last_unstale":"2024-08-17T18:24:17.467402+0000","last_undegraded":"2024-08-17T18:24:17.467402+0000","last_fullsized":"2024-08-17T18:24:17.467402+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:50.746984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":132,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468944+0000","last_change":"2024-08-17T18:21:33.404642+0000","last_active":"2024-08-17T18:24:17.468944+0000","last_peered":"2024-08-17T18:24:17.468944+0000","last_clean":"2024-08-17T18:24:17.468944+0000","last_became_active":"2024-08-17T18:21:33.404374+0000","last_became_peered":"2024-08-17T18:21:33.404374+0000","last_unstale":"2024-08-17T18:24:17.468944+0000","last_undegraded":"2024-08-17T18:24:17.468944+0000","last_fullsized":"2024-08-17T18:24:17.468944+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:49:34.531843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":32,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467135+0000","last_change":"2024-08-17T18:23:12.500729+0000","last_active":"2024-08-17T18:24:17.467135+0000","last_peered":"2024-08-17T18:24:17.467135+0000","last_clean":"2024-08-17T18:24:17.467135+0000","last_became_active":"2024-08-17T18:23:12.492379+0000","last_became_peered":"2024-08-17T18:23:12.492379+0000","last_unstale":"2024-08-17T18:24:17.467135+0000","last_undegraded":"2024-08-17T18:24:17.467135+0000","last_fullsized":"2024-08-17T18:24:17.467135+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:15:18.000497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":132,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.469074+0000","last_change":"2024-08-17T18:23:05.473376+0000","last_active":"2024-08-17T18:24:17.469074+0000","last_peered":"2024-08-17T18:24:17.469074+0000","last_clean":"2024-08-17T18:24:17.469074+0000","last_became_active":"2024-08-17T18:23:05.472991+0000","last_became_peered":"2024-08-17T18:23:05.472991+0000","last_unstale":"2024-08-17T18:24:17.469074+0000","last_undegraded":"2024-08-17T18:24:17.469074+0000","last_fullsized":"2024-08-17T18:24:17.469074+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:40:40.533867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":130,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631378+0000","last_change":"2024-08-17T18:21:33.424299+0000","last_active":"2024-08-17T18:24:20.631378+0000","last_peered":"2024-08-17T18:24:20.631378+0000","last_clean":"2024-08-17T18:24:20.631378+0000","last_became_active":"2024-08-17T18:21:33.424016+0000","last_became_peered":"2024-08-17T18:21:33.424016+0000","last_unstale":"2024-08-17T18:24:20.631378+0000","last_undegraded":"2024-08-17T18:24:20.631378+0000","last_fullsized":"2024-08-17T18:24:20.631378+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:15.128415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2130,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.492706+0000","last_change":"2024-08-17T18:23:05.475948+0000","last_active":"2024-08-17T18:24:20.492706+0000","last_peered":"2024-08-17T18:24:20.492706+0000","last_clean":"2024-08-17T18:24:20.492706+0000","last_became_active":"2024-08-17T18:23:05.475137+0000","last_became_peered":"2024-08-17T18:23:05.475137+0000","last_unstale":"2024-08-17T18:24:20.492706+0000","last_undegraded":"2024-08-17T18:24:20.492706+0000","last_fullsized":"2024-08-17T18:24:20.492706+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:04:40.443880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":146,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468810+0000","last_change":"2024-08-17T18:23:05.473551+0000","last_active":"2024-08-17T18:24:17.468810+0000","last_peered":"2024-08-17T18:24:17.468810+0000","last_clean":"2024-08-17T18:24:17.468810+0000","last_became_active":"2024-08-17T18:23:05.473125+0000","last_became_peered":"2024-08-17T18:23:05.473125+0000","last_unstale":"2024-08-17T18:24:17.468810+0000","last_undegraded":"2024-08-17T18:24:17.468810+0000","last_fullsized":"2024-08-17T18:24:17.468810+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:55:43.580574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2433,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493551+0000","last_change":"2024-08-17T18:23:05.468151+0000","last_active":"2024-08-17T18:24:20.493551+0000","last_peered":"2024-08-17T18:24:20.493551+0000","last_clean":"2024-08-17T18:24:20.493551+0000","last_became_active":"2024-08-17T18:23:05.467820+0000","last_became_peered":"2024-08-17T18:23:05.467820+0000","last_unstale":"2024-08-17T18:24:20.493551+0000","last_undegraded":"2024-08-17T18:24:20.493551+0000","last_fullsized":"2024-08-17T18:24:20.493551+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:23:15.032777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631419+0000","last_change":"2024-08-17T18:23:05.472428+0000","last_active":"2024-08-17T18:24:20.631419+0000","last_peered":"2024-08-17T18:24:20.631419+0000","last_clean":"2024-08-17T18:24:20.631419+0000","last_became_active":"2024-08-17T18:23:05.468270+0000","last_became_peered":"2024-08-17T18:23:05.468270+0000","last_unstale":"2024-08-17T18:24:20.631419+0000","last_undegraded":"2024-08-17T18:24:20.631419+0000","last_fullsized":"2024-08-17T18:24:20.631419+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:06.983782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":130,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631293+0000","last_change":"2024-08-17T18:21:33.408805+0000","last_active":"2024-08-17T18:24:20.631293+0000","last_peered":"2024-08-17T18:24:20.631293+0000","last_clean":"2024-08-17T18:24:20.631293+0000","last_became_active":"2024-08-17T18:21:33.408216+0000","last_became_peered":"2024-08-17T18:21:33.408216+0000","last_unstale":"2024-08-17T18:24:20.631293+0000","last_undegraded":"2024-08-17T18:24:20.631293+0000","last_fullsized":"2024-08-17T18:24:20.631293+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:44:45.784110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2148,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.492493+0000","last_change":"2024-08-17T18:23:05.469853+0000","last_active":"2024-08-17T18:24:20.492493+0000","last_peered":"2024-08-17T18:24:20.492493+0000","last_clean":"2024-08-17T18:24:20.492493+0000","last_became_active":"2024-08-17T18:23:05.469535+0000","last_became_peered":"2024-08-17T18:23:05.469535+0000","last_unstale":"2024-08-17T18:24:20.492493+0000","last_undegraded":"2024-08-17T18:24:20.492493+0000","last_fullsized":"2024-08-17T18:24:20.492493+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:21.036841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":197,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467313+0000","last_change":"2024-08-17T18:23:12.491476+0000","last_active":"2024-08-17T18:24:17.467313+0000","last_peered":"2024-08-17T18:24:17.467313+0000","last_clean":"2024-08-17T18:24:17.467313+0000","last_became_active":"2024-08-17T18:23:12.491162+0000","last_became_peered":"2024-08-17T18:23:12.491162+0000","last_unstale":"2024-08-17T18:24:17.467313+0000","last_undegraded":"2024-08-17T18:24:17.467313+0000","last_fullsized":"2024-08-17T18:24:17.467313+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:20.885678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":2420,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493091+0000","last_change":"2024-08-17T18:23:05.468384+0000","last_active":"2024-08-17T18:24:20.493091+0000","last_peered":"2024-08-17T18:24:20.493091+0000","last_clean":"2024-08-17T18:24:20.493091+0000","last_became_active":"2024-08-17T18:23:05.467843+0000","last_became_peered":"2024-08-17T18:23:05.467843+0000","last_unstale":"2024-08-17T18:24:20.493091+0000","last_undegraded":"2024-08-17T18:24:20.493091+0000","last_fullsized":"2024-08-17T18:24:20.493091+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:18:23.218603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":137,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631955+0000","last_change":"2024-08-17T18:23:12.491873+0000","last_active":"2024-08-17T18:24:20.631955+0000","last_peered":"2024-08-17T18:24:20.631955+0000","last_clean":"2024-08-17T18:24:20.631955+0000","last_became_active":"2024-08-17T18:23:12.491406+0000","last_became_peered":"2024-08-17T18:23:12.491406+0000","last_unstale":"2024-08-17T18:24:20.631955+0000","last_undegraded":"2024-08-17T18:24:20.631955+0000","last_fullsized":"2024-08-17T18:24:20.631955+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:28.454580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2354,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493132+0000","last_change":"2024-08-17T18:16:34.644366+0000","last_active":"2024-08-17T18:24:20.493132+0000","last_peered":"2024-08-17T18:24:20.493132+0000","last_clean":"2024-08-17T18:24:20.493132+0000","last_became_active":"2024-08-17T18:16:34.643994+0000","last_became_peered":"2024-08-17T18:16:34.643994+0000","last_unstale":"2024-08-17T18:24:20.493132+0000","last_undegraded":"2024-08-17T18:24:20.493132+0000","last_fullsized":"2024-08-17T18:24:20.493132+0000","mapping_epoch":1421,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:51.953343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":28,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.466987+0000","last_change":"2024-08-17T18:23:12.497021+0000","last_active":"2024-08-17T18:24:17.466987+0000","last_peered":"2024-08-17T18:24:17.466987+0000","last_clean":"2024-08-17T18:24:17.466987+0000","last_became_active":"2024-08-17T18:23:12.496093+0000","last_became_peered":"2024-08-17T18:23:12.496093+0000","last_unstale":"2024-08-17T18:24:17.466987+0000","last_undegraded":"2024-08-17T18:24:17.466987+0000","last_fullsized":"2024-08-17T18:24:17.466987+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:56:43.945061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":1552,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731702+0000","last_change":"2024-08-17T18:18:50.525363+0000","last_active":"2024-08-17T18:24:17.731702+0000","last_peered":"2024-08-17T18:24:17.731702+0000","last_clean":"2024-08-17T18:24:17.731702+0000","last_became_active":"2024-08-17T18:18:50.511676+0000","last_became_peered":"2024-08-17T18:18:50.511676+0000","last_unstale":"2024-08-17T18:24:17.731702+0000","last_undegraded":"2024-08-17T18:24:17.731702+0000","last_fullsized":"2024-08-17T18:24:17.731702+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:22.530382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631643+0000","last_change":"2024-08-17T18:23:05.472573+0000","last_active":"2024-08-17T18:24:20.631643+0000","last_peered":"2024-08-17T18:24:20.631643+0000","last_clean":"2024-08-17T18:24:20.631643+0000","last_became_active":"2024-08-17T18:23:05.471921+0000","last_became_peered":"2024-08-17T18:23:05.471921+0000","last_unstale":"2024-08-17T18:24:20.631643+0000","last_undegraded":"2024-08-17T18:24:20.631643+0000","last_fullsized":"2024-08-17T18:24:20.631643+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:10.207241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"337'22","reported_seq":121,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468618+0000","last_change":"2024-08-17T18:18:51.515962+0000","last_active":"2024-08-17T18:24:17.468618+0000","last_peered":"2024-08-17T18:24:17.468618+0000","last_clean":"2024-08-17T18:24:17.468618+0000","last_became_active":"2024-08-17T18:18:51.505508+0000","last_became_peered":"2024-08-17T18:18:51.505508+0000","last_unstale":"2024-08-17T18:24:17.468618+0000","last_undegraded":"2024-08-17T18:24:17.468618+0000","last_fullsized":"2024-08-17T18:24:17.468618+0000","mapping_epoch":1447,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1448,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:16.518470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9488,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":131,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468968+0000","last_change":"2024-08-17T18:23:05.473525+0000","last_active":"2024-08-17T18:24:17.468968+0000","last_peered":"2024-08-17T18:24:17.468968+0000","last_clean":"2024-08-17T18:24:17.468968+0000","last_became_active":"2024-08-17T18:23:05.473177+0000","last_became_peered":"2024-08-17T18:23:05.473177+0000","last_unstale":"2024-08-17T18:24:17.468968+0000","last_undegraded":"2024-08-17T18:24:17.468968+0000","last_fullsized":"2024-08-17T18:24:17.468968+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:36:20.885141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":152,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468740+0000","last_change":"2024-08-17T18:23:12.491602+0000","last_active":"2024-08-17T18:24:17.468740+0000","last_peered":"2024-08-17T18:24:17.468740+0000","last_clean":"2024-08-17T18:24:17.468740+0000","last_became_active":"2024-08-17T18:23:12.491282+0000","last_became_peered":"2024-08-17T18:23:12.491282+0000","last_unstale":"2024-08-17T18:24:17.468740+0000","last_undegraded":"2024-08-17T18:24:17.468740+0000","last_fullsized":"2024-08-17T18:24:17.468740+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:02:52.727878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":151,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467390+0000","last_change":"2024-08-17T18:19:27.055728+0000","last_active":"2024-08-17T18:24:17.467390+0000","last_peered":"2024-08-17T18:24:17.467390+0000","last_clean":"2024-08-17T18:24:17.467390+0000","last_became_active":"2024-08-17T18:19:27.055425+0000","last_became_peered":"2024-08-17T18:19:27.055425+0000","last_unstale":"2024-08-17T18:24:17.467390+0000","last_undegraded":"2024-08-17T18:24:17.467390+0000","last_fullsized":"2024-08-17T18:24:17.467390+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:47.920145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035875399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":156,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631759+0000","last_change":"2024-08-17T18:23:05.479998+0000","last_active":"2024-08-17T18:24:20.631759+0000","last_peered":"2024-08-17T18:24:20.631759+0000","last_clean":"2024-08-17T18:24:20.631759+0000","last_became_active":"2024-08-17T18:23:05.465166+0000","last_became_peered":"2024-08-17T18:23:05.465166+0000","last_unstale":"2024-08-17T18:24:20.631759+0000","last_undegraded":"2024-08-17T18:24:20.631759+0000","last_fullsized":"2024-08-17T18:24:20.631759+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:58:08.921362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2422,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493130+0000","last_change":"2024-08-17T18:21:33.421768+0000","last_active":"2024-08-17T18:24:20.493130+0000","last_peered":"2024-08-17T18:24:20.493130+0000","last_clean":"2024-08-17T18:24:20.493130+0000","last_became_active":"2024-08-17T18:21:33.421467+0000","last_became_peered":"2024-08-17T18:21:33.421467+0000","last_unstale":"2024-08-17T18:24:20.493130+0000","last_undegraded":"2024-08-17T18:24:20.493130+0000","last_fullsized":"2024-08-17T18:24:20.493130+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:51:38.751105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":54,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631591+0000","last_change":"2024-08-17T18:23:05.466442+0000","last_active":"2024-08-17T18:24:20.631591+0000","last_peered":"2024-08-17T18:24:20.631591+0000","last_clean":"2024-08-17T18:24:20.631591+0000","last_became_active":"2024-08-17T18:23:05.466154+0000","last_became_peered":"2024-08-17T18:23:05.466154+0000","last_unstale":"2024-08-17T18:24:20.631591+0000","last_undegraded":"2024-08-17T18:24:20.631591+0000","last_fullsized":"2024-08-17T18:24:20.631591+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:50:14.484586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":2443,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493727+0000","last_change":"2024-08-17T18:21:33.497158+0000","last_active":"2024-08-17T18:24:20.493727+0000","last_peered":"2024-08-17T18:24:20.493727+0000","last_clean":"2024-08-17T18:24:20.493727+0000","last_became_active":"2024-08-17T18:21:33.421674+0000","last_became_peered":"2024-08-17T18:21:33.421674+0000","last_unstale":"2024-08-17T18:24:20.493727+0000","last_undegraded":"2024-08-17T18:24:20.493727+0000","last_fullsized":"2024-08-17T18:24:20.493727+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:58:57.198125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":151,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631570+0000","last_change":"2024-08-17T18:23:12.495046+0000","last_active":"2024-08-17T18:24:20.631570+0000","last_peered":"2024-08-17T18:24:20.631570+0000","last_clean":"2024-08-17T18:24:20.631570+0000","last_became_active":"2024-08-17T18:23:12.494730+0000","last_became_peered":"2024-08-17T18:23:12.494730+0000","last_unstale":"2024-08-17T18:24:20.631570+0000","last_undegraded":"2024-08-17T18:24:20.631570+0000","last_fullsized":"2024-08-17T18:24:20.631570+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:57:24.534722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1589,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731838+0000","last_change":"2024-08-17T18:21:33.420079+0000","last_active":"2024-08-17T18:24:17.731838+0000","last_peered":"2024-08-17T18:24:17.731838+0000","last_clean":"2024-08-17T18:24:17.731838+0000","last_became_active":"2024-08-17T18:21:33.419265+0000","last_became_peered":"2024-08-17T18:21:33.419265+0000","last_unstale":"2024-08-17T18:24:17.731838+0000","last_undegraded":"2024-08-17T18:24:17.731838+0000","last_fullsized":"2024-08-17T18:24:17.731838+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:33:44.811302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030236599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":61,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631804+0000","last_change":"2024-08-17T18:23:12.494642+0000","last_active":"2024-08-17T18:24:20.631804+0000","last_peered":"2024-08-17T18:24:20.631804+0000","last_clean":"2024-08-17T18:24:20.631804+0000","last_became_active":"2024-08-17T18:23:12.494333+0000","last_became_peered":"2024-08-17T18:23:12.494333+0000","last_unstale":"2024-08-17T18:24:20.631804+0000","last_undegraded":"2024-08-17T18:24:20.631804+0000","last_fullsized":"2024-08-17T18:24:20.631804+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:55:11.129724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.492684+0000","last_change":"2024-08-17T18:24:17.471276+0000","last_active":"2024-08-17T18:24:20.492684+0000","last_peered":"2024-08-17T18:24:20.492684+0000","last_clean":"2024-08-17T18:24:20.492684+0000","last_became_active":"2024-08-17T18:24:17.470930+0000","last_became_peered":"2024-08-17T18:24:17.470930+0000","last_unstale":"2024-08-17T18:24:20.492684+0000","last_undegraded":"2024-08-17T18:24:20.492684+0000","last_fullsized":"2024-08-17T18:24:20.492684+0000","mapping_epoch":1547,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:28.633664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":54,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631771+0000","last_change":"2024-08-17T18:23:05.473157+0000","last_active":"2024-08-17T18:24:20.631771+0000","last_peered":"2024-08-17T18:24:20.631771+0000","last_clean":"2024-08-17T18:24:20.631771+0000","last_became_active":"2024-08-17T18:23:05.468199+0000","last_became_peered":"2024-08-17T18:23:05.468199+0000","last_unstale":"2024-08-17T18:24:20.631771+0000","last_undegraded":"2024-08-17T18:24:20.631771+0000","last_fullsized":"2024-08-17T18:24:20.631771+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:17:53.820423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000227517,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":28,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467322+0000","last_change":"2024-08-17T18:23:12.495597+0000","last_active":"2024-08-17T18:24:17.467322+0000","last_peered":"2024-08-17T18:24:17.467322+0000","last_clean":"2024-08-17T18:24:17.467322+0000","last_became_active":"2024-08-17T18:23:12.493578+0000","last_became_peered":"2024-08-17T18:23:12.493578+0000","last_unstale":"2024-08-17T18:24:17.467322+0000","last_undegraded":"2024-08-17T18:24:17.467322+0000","last_fullsized":"2024-08-17T18:24:17.467322+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:17.580085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":54,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631824+0000","last_change":"2024-08-17T18:23:05.475522+0000","last_active":"2024-08-17T18:24:20.631824+0000","last_peered":"2024-08-17T18:24:20.631824+0000","last_clean":"2024-08-17T18:24:20.631824+0000","last_became_active":"2024-08-17T18:23:05.475176+0000","last_became_peered":"2024-08-17T18:23:05.475176+0000","last_unstale":"2024-08-17T18:24:20.631824+0000","last_undegraded":"2024-08-17T18:24:20.631824+0000","last_fullsized":"2024-08-17T18:24:20.631824+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:05.025690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":132,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.469205+0000","last_change":"2024-08-17T18:21:33.403089+0000","last_active":"2024-08-17T18:24:17.469205+0000","last_peered":"2024-08-17T18:24:17.469205+0000","last_clean":"2024-08-17T18:24:17.469205+0000","last_became_active":"2024-08-17T18:21:33.402788+0000","last_became_peered":"2024-08-17T18:21:33.402788+0000","last_unstale":"2024-08-17T18:24:17.469205+0000","last_undegraded":"2024-08-17T18:24:17.469205+0000","last_fullsized":"2024-08-17T18:24:17.469205+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:01:25.909442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":32,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467139+0000","last_change":"2024-08-17T18:23:12.506453+0000","last_active":"2024-08-17T18:24:17.467139+0000","last_peered":"2024-08-17T18:24:17.467139+0000","last_clean":"2024-08-17T18:24:17.467139+0000","last_became_active":"2024-08-17T18:23:12.493518+0000","last_became_peered":"2024-08-17T18:23:12.493518+0000","last_unstale":"2024-08-17T18:24:17.467139+0000","last_undegraded":"2024-08-17T18:24:17.467139+0000","last_fullsized":"2024-08-17T18:24:17.467139+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:36.173814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":61,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631662+0000","last_change":"2024-08-17T18:23:12.493978+0000","last_active":"2024-08-17T18:24:20.631662+0000","last_peered":"2024-08-17T18:24:20.631662+0000","last_clean":"2024-08-17T18:24:20.631662+0000","last_became_active":"2024-08-17T18:23:12.493667+0000","last_became_peered":"2024-08-17T18:23:12.493667+0000","last_unstale":"2024-08-17T18:24:20.631662+0000","last_undegraded":"2024-08-17T18:24:20.631662+0000","last_fullsized":"2024-08-17T18:24:20.631662+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:40.765437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":55,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631296+0000","last_change":"2024-08-17T18:23:05.466293+0000","last_active":"2024-08-17T18:24:20.631296+0000","last_peered":"2024-08-17T18:24:20.631296+0000","last_clean":"2024-08-17T18:24:20.631296+0000","last_became_active":"2024-08-17T18:23:05.466000+0000","last_became_peered":"2024-08-17T18:23:05.466000+0000","last_unstale":"2024-08-17T18:24:20.631296+0000","last_undegraded":"2024-08-17T18:24:20.631296+0000","last_fullsized":"2024-08-17T18:24:20.631296+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:04.835392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2347,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.492617+0000","last_change":"2024-08-17T18:21:33.419513+0000","last_active":"2024-08-17T18:24:20.492617+0000","last_peered":"2024-08-17T18:24:20.492617+0000","last_clean":"2024-08-17T18:24:20.492617+0000","last_became_active":"2024-08-17T18:21:33.419186+0000","last_became_peered":"2024-08-17T18:21:33.419186+0000","last_unstale":"2024-08-17T18:24:20.492617+0000","last_undegraded":"2024-08-17T18:24:20.492617+0000","last_fullsized":"2024-08-17T18:24:20.492617+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:27.395794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041855100000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":144,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631381+0000","last_change":"2024-08-17T18:23:05.467307+0000","last_active":"2024-08-17T18:24:20.631381+0000","last_peered":"2024-08-17T18:24:20.631381+0000","last_clean":"2024-08-17T18:24:20.631381+0000","last_became_active":"2024-08-17T18:23:05.466928+0000","last_became_peered":"2024-08-17T18:23:05.466928+0000","last_unstale":"2024-08-17T18:24:20.631381+0000","last_undegraded":"2024-08-17T18:24:20.631381+0000","last_fullsized":"2024-08-17T18:24:20.631381+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:26:02.689664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":130,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631221+0000","last_change":"2024-08-17T18:21:33.407703+0000","last_active":"2024-08-17T18:24:20.631221+0000","last_peered":"2024-08-17T18:24:20.631221+0000","last_clean":"2024-08-17T18:24:20.631221+0000","last_became_active":"2024-08-17T18:21:33.407397+0000","last_became_peered":"2024-08-17T18:21:33.407397+0000","last_unstale":"2024-08-17T18:24:20.631221+0000","last_undegraded":"2024-08-17T18:24:20.631221+0000","last_fullsized":"2024-08-17T18:24:20.631221+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:28:06.725837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1620,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731319+0000","last_change":"2024-08-17T18:23:05.463133+0000","last_active":"2024-08-17T18:24:17.731319+0000","last_peered":"2024-08-17T18:24:17.731319+0000","last_clean":"2024-08-17T18:24:17.731319+0000","last_became_active":"2024-08-17T18:23:05.462900+0000","last_became_peered":"2024-08-17T18:23:05.462900+0000","last_unstale":"2024-08-17T18:24:17.731319+0000","last_undegraded":"2024-08-17T18:24:17.731319+0000","last_fullsized":"2024-08-17T18:24:17.731319+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:34.527149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026511500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":147,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467763+0000","last_change":"2024-08-17T18:18:50.516313+0000","last_active":"2024-08-17T18:24:17.467763+0000","last_peered":"2024-08-17T18:24:17.467763+0000","last_clean":"2024-08-17T18:24:17.467763+0000","last_became_active":"2024-08-17T18:18:50.515925+0000","last_became_peered":"2024-08-17T18:18:50.515925+0000","last_unstale":"2024-08-17T18:24:17.467763+0000","last_undegraded":"2024-08-17T18:24:17.467763+0000","last_fullsized":"2024-08-17T18:24:17.467763+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:14.611619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2401,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493304+0000","last_change":"2024-08-17T18:23:12.493173+0000","last_active":"2024-08-17T18:24:20.493304+0000","last_peered":"2024-08-17T18:24:20.493304+0000","last_clean":"2024-08-17T18:24:20.493304+0000","last_became_active":"2024-08-17T18:23:12.492862+0000","last_became_peered":"2024-08-17T18:23:12.492862+0000","last_unstale":"2024-08-17T18:24:20.493304+0000","last_undegraded":"2024-08-17T18:24:20.493304+0000","last_fullsized":"2024-08-17T18:24:20.493304+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:26.508877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":32,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467287+0000","last_change":"2024-08-17T18:23:12.509125+0000","last_active":"2024-08-17T18:24:17.467287+0000","last_peered":"2024-08-17T18:24:17.467287+0000","last_clean":"2024-08-17T18:24:17.467287+0000","last_became_active":"2024-08-17T18:23:12.495743+0000","last_became_peered":"2024-08-17T18:23:12.495743+0000","last_unstale":"2024-08-17T18:24:17.467287+0000","last_undegraded":"2024-08-17T18:24:17.467287+0000","last_fullsized":"2024-08-17T18:24:17.467287+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:20:37.642246+0000","last_clean_scrub_stamp":"2024-08-17T17:20:37.642246+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:30:20.834830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"257'149","reported_seq":2020,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731288+0000","last_change":"2024-08-17T18:22:00.258650+0000","last_active":"2024-08-17T18:24:17.731288+0000","last_peered":"2024-08-17T18:24:17.731288+0000","last_clean":"2024-08-17T18:24:17.731288+0000","last_became_active":"2024-08-17T18:22:00.258326+0000","last_became_peered":"2024-08-17T18:22:00.258326+0000","last_unstale":"2024-08-17T18:24:17.731288+0000","last_undegraded":"2024-08-17T18:24:17.731288+0000","last_fullsized":"2024-08-17T18:24:17.731288+0000","mapping_epoch":1500,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":1501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:18:10.243013+0000","last_clean_scrub_stamp":"2024-08-17T17:18:10.243013+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:02.578380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631566+0000","last_change":"2024-08-17T18:23:05.468372+0000","last_active":"2024-08-17T18:24:20.631566+0000","last_peered":"2024-08-17T18:24:20.631566+0000","last_clean":"2024-08-17T18:24:20.631566+0000","last_became_active":"2024-08-17T18:23:05.468085+0000","last_became_peered":"2024-08-17T18:23:05.468085+0000","last_unstale":"2024-08-17T18:24:20.631566+0000","last_undegraded":"2024-08-17T18:24:20.631566+0000","last_fullsized":"2024-08-17T18:24:20.631566+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:20:45.718917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2458,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493231+0000","last_change":"2024-08-17T18:23:05.468144+0000","last_active":"2024-08-17T18:24:20.493231+0000","last_peered":"2024-08-17T18:24:20.493231+0000","last_clean":"2024-08-17T18:24:20.493231+0000","last_became_active":"2024-08-17T18:23:05.467350+0000","last_became_peered":"2024-08-17T18:23:05.467350+0000","last_unstale":"2024-08-17T18:24:20.493231+0000","last_undegraded":"2024-08-17T18:24:20.493231+0000","last_fullsized":"2024-08-17T18:24:20.493231+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:09:04.364659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":142,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.474926+0000","last_change":"2024-08-17T18:24:17.472904+0000","last_active":"2024-08-17T18:24:17.474926+0000","last_peered":"2024-08-17T18:24:17.474926+0000","last_clean":"2024-08-17T18:24:17.474926+0000","last_became_active":"2024-08-17T18:24:17.472453+0000","last_became_peered":"2024-08-17T18:24:17.472453+0000","last_unstale":"2024-08-17T18:24:17.474926+0000","last_undegraded":"2024-08-17T18:24:17.474926+0000","last_fullsized":"2024-08-17T18:24:17.474926+0000","mapping_epoch":1547,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:44:15.975606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":126,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468695+0000","last_change":"2024-08-17T18:19:27.038041+0000","last_active":"2024-08-17T18:24:17.468695+0000","last_peered":"2024-08-17T18:24:17.468695+0000","last_clean":"2024-08-17T18:24:17.468695+0000","last_became_active":"2024-08-17T18:19:27.037715+0000","last_became_peered":"2024-08-17T18:19:27.037715+0000","last_unstale":"2024-08-17T18:24:17.468695+0000","last_undegraded":"2024-08-17T18:24:17.468695+0000","last_fullsized":"2024-08-17T18:24:17.468695+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:06:39.726534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":153,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.474917+0000","last_change":"2024-08-17T18:24:17.473333+0000","last_active":"2024-08-17T18:24:17.474917+0000","last_peered":"2024-08-17T18:24:17.474917+0000","last_clean":"2024-08-17T18:24:17.474917+0000","last_became_active":"2024-08-17T18:24:17.472908+0000","last_became_peered":"2024-08-17T18:24:17.472908+0000","last_unstale":"2024-08-17T18:24:17.474917+0000","last_undegraded":"2024-08-17T18:24:17.474917+0000","last_fullsized":"2024-08-17T18:24:17.474917+0000","mapping_epoch":1547,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1548,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:46:57.382675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":167,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468051+0000","last_change":"2024-08-17T18:23:05.476053+0000","last_active":"2024-08-17T18:24:17.468051+0000","last_peered":"2024-08-17T18:24:17.468051+0000","last_clean":"2024-08-17T18:24:17.468051+0000","last_became_active":"2024-08-17T18:23:05.475692+0000","last_became_peered":"2024-08-17T18:23:05.475692+0000","last_unstale":"2024-08-17T18:24:17.468051+0000","last_undegraded":"2024-08-17T18:24:17.468051+0000","last_fullsized":"2024-08-17T18:24:17.468051+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:29:41.007375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":112,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468829+0000","last_change":"2024-08-17T18:18:50.515676+0000","last_active":"2024-08-17T18:24:17.468829+0000","last_peered":"2024-08-17T18:24:17.468829+0000","last_clean":"2024-08-17T18:24:17.468829+0000","last_became_active":"2024-08-17T18:18:50.515313+0000","last_became_peered":"2024-08-17T18:18:50.515313+0000","last_unstale":"2024-08-17T18:24:17.468829+0000","last_undegraded":"2024-08-17T18:24:17.468829+0000","last_fullsized":"2024-08-17T18:24:17.468829+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:36.955526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":2481,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493349+0000","last_change":"2024-08-17T18:21:33.493429+0000","last_active":"2024-08-17T18:24:20.493349+0000","last_peered":"2024-08-17T18:24:20.493349+0000","last_clean":"2024-08-17T18:24:20.493349+0000","last_became_active":"2024-08-17T18:21:33.421176+0000","last_became_peered":"2024-08-17T18:21:33.421176+0000","last_unstale":"2024-08-17T18:24:20.493349+0000","last_undegraded":"2024-08-17T18:24:20.493349+0000","last_fullsized":"2024-08-17T18:24:20.493349+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:42.334437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":144,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631488+0000","last_change":"2024-08-17T18:23:05.469853+0000","last_active":"2024-08-17T18:24:20.631488+0000","last_peered":"2024-08-17T18:24:20.631488+0000","last_clean":"2024-08-17T18:24:20.631488+0000","last_became_active":"2024-08-17T18:23:05.469532+0000","last_became_peered":"2024-08-17T18:23:05.469532+0000","last_unstale":"2024-08-17T18:24:20.631488+0000","last_undegraded":"2024-08-17T18:24:20.631488+0000","last_fullsized":"2024-08-17T18:24:20.631488+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:28.110068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":28,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467181+0000","last_change":"2024-08-17T18:23:12.493834+0000","last_active":"2024-08-17T18:24:17.467181+0000","last_peered":"2024-08-17T18:24:17.467181+0000","last_clean":"2024-08-17T18:24:17.467181+0000","last_became_active":"2024-08-17T18:23:12.493498+0000","last_became_peered":"2024-08-17T18:23:12.493498+0000","last_unstale":"2024-08-17T18:24:17.467181+0000","last_undegraded":"2024-08-17T18:24:17.467181+0000","last_fullsized":"2024-08-17T18:24:17.467181+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:05.457849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":157,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467996+0000","last_change":"2024-08-17T18:23:05.468008+0000","last_active":"2024-08-17T18:24:17.467996+0000","last_peered":"2024-08-17T18:24:17.467996+0000","last_clean":"2024-08-17T18:24:17.467996+0000","last_became_active":"2024-08-17T18:23:05.467317+0000","last_became_peered":"2024-08-17T18:23:05.467317+0000","last_unstale":"2024-08-17T18:24:17.467996+0000","last_undegraded":"2024-08-17T18:24:17.467996+0000","last_fullsized":"2024-08-17T18:24:17.467996+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:58.702296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":1585,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731806+0000","last_change":"2024-08-17T18:23:12.504856+0000","last_active":"2024-08-17T18:24:17.731806+0000","last_peered":"2024-08-17T18:24:17.731806+0000","last_clean":"2024-08-17T18:24:17.731806+0000","last_became_active":"2024-08-17T18:23:12.497068+0000","last_became_peered":"2024-08-17T18:23:12.497068+0000","last_unstale":"2024-08-17T18:24:17.731806+0000","last_undegraded":"2024-08-17T18:24:17.731806+0000","last_fullsized":"2024-08-17T18:24:17.731806+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:45:02.864888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":132,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468829+0000","last_change":"2024-08-17T18:23:05.467377+0000","last_active":"2024-08-17T18:24:17.468829+0000","last_peered":"2024-08-17T18:24:17.468829+0000","last_clean":"2024-08-17T18:24:17.468829+0000","last_became_active":"2024-08-17T18:23:05.467049+0000","last_became_peered":"2024-08-17T18:23:05.467049+0000","last_unstale":"2024-08-17T18:24:17.468829+0000","last_undegraded":"2024-08-17T18:24:17.468829+0000","last_fullsized":"2024-08-17T18:24:17.468829+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:23.876632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":1600,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731866+0000","last_change":"2024-08-17T18:23:05.475599+0000","last_active":"2024-08-17T18:24:17.731866+0000","last_peered":"2024-08-17T18:24:17.731866+0000","last_clean":"2024-08-17T18:24:17.731866+0000","last_became_active":"2024-08-17T18:23:05.464545+0000","last_became_peered":"2024-08-17T18:23:05.464545+0000","last_unstale":"2024-08-17T18:24:17.731866+0000","last_undegraded":"2024-08-17T18:24:17.731866+0000","last_fullsized":"2024-08-17T18:24:17.731866+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:54:53.309007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":28,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.466870+0000","last_change":"2024-08-17T18:23:12.495976+0000","last_active":"2024-08-17T18:24:17.466870+0000","last_peered":"2024-08-17T18:24:17.466870+0000","last_clean":"2024-08-17T18:24:17.466870+0000","last_became_active":"2024-08-17T18:23:12.495615+0000","last_became_peered":"2024-08-17T18:23:12.495615+0000","last_unstale":"2024-08-17T18:24:17.466870+0000","last_undegraded":"2024-08-17T18:24:17.466870+0000","last_fullsized":"2024-08-17T18:24:17.466870+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:03:37.648294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":1616,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731840+0000","last_change":"2024-08-17T18:21:33.495232+0000","last_active":"2024-08-17T18:24:17.731840+0000","last_peered":"2024-08-17T18:24:17.731840+0000","last_clean":"2024-08-17T18:24:17.731840+0000","last_became_active":"2024-08-17T18:21:33.438606+0000","last_became_peered":"2024-08-17T18:21:33.438606+0000","last_unstale":"2024-08-17T18:24:17.731840+0000","last_undegraded":"2024-08-17T18:24:17.731840+0000","last_fullsized":"2024-08-17T18:24:17.731840+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:14.007910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":247,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.468108+0000","last_change":"2024-08-17T18:23:12.502460+0000","last_active":"2024-08-17T18:24:17.468108+0000","last_peered":"2024-08-17T18:24:17.468108+0000","last_clean":"2024-08-17T18:24:17.468108+0000","last_became_active":"2024-08-17T18:23:12.490256+0000","last_became_peered":"2024-08-17T18:23:12.490256+0000","last_unstale":"2024-08-17T18:24:17.468108+0000","last_undegraded":"2024-08-17T18:24:17.468108+0000","last_fullsized":"2024-08-17T18:24:17.468108+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:58:12.083653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2455,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493781+0000","last_change":"2024-08-17T18:23:05.469169+0000","last_active":"2024-08-17T18:24:20.493781+0000","last_peered":"2024-08-17T18:24:20.493781+0000","last_clean":"2024-08-17T18:24:20.493781+0000","last_became_active":"2024-08-17T18:23:05.467765+0000","last_became_peered":"2024-08-17T18:23:05.467765+0000","last_unstale":"2024-08-17T18:24:20.493781+0000","last_undegraded":"2024-08-17T18:24:20.493781+0000","last_fullsized":"2024-08-17T18:24:20.493781+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:14:04.176083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1540,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731380+0000","last_change":"2024-08-17T18:18:50.515245+0000","last_active":"2024-08-17T18:24:17.731380+0000","last_peered":"2024-08-17T18:24:17.731380+0000","last_clean":"2024-08-17T18:24:17.731380+0000","last_became_active":"2024-08-17T18:18:50.514785+0000","last_became_peered":"2024-08-17T18:18:50.514785+0000","last_unstale":"2024-08-17T18:24:17.731380+0000","last_undegraded":"2024-08-17T18:24:17.731380+0000","last_fullsized":"2024-08-17T18:24:17.731380+0000","mapping_epoch":1446,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1447,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:08.696581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":130,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631772+0000","last_change":"2024-08-17T18:21:33.408565+0000","last_active":"2024-08-17T18:24:20.631772+0000","last_peered":"2024-08-17T18:24:20.631772+0000","last_clean":"2024-08-17T18:24:20.631772+0000","last_became_active":"2024-08-17T18:21:33.407832+0000","last_became_peered":"2024-08-17T18:21:33.407832+0000","last_unstale":"2024-08-17T18:24:20.631772+0000","last_undegraded":"2024-08-17T18:24:20.631772+0000","last_fullsized":"2024-08-17T18:24:20.631772+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:09:06.558530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":130,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.631389+0000","last_change":"2024-08-17T18:21:33.428616+0000","last_active":"2024-08-17T18:24:20.631389+0000","last_peered":"2024-08-17T18:24:20.631389+0000","last_clean":"2024-08-17T18:24:20.631389+0000","last_became_active":"2024-08-17T18:21:33.428368+0000","last_became_peered":"2024-08-17T18:21:33.428368+0000","last_unstale":"2024-08-17T18:24:20.631389+0000","last_undegraded":"2024-08-17T18:24:20.631389+0000","last_fullsized":"2024-08-17T18:24:20.631389+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:14.945169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":28,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.467240+0000","last_change":"2024-08-17T18:23:12.497684+0000","last_active":"2024-08-17T18:24:17.467240+0000","last_peered":"2024-08-17T18:24:17.467240+0000","last_clean":"2024-08-17T18:24:17.467240+0000","last_became_active":"2024-08-17T18:23:12.497197+0000","last_became_peered":"2024-08-17T18:23:12.497197+0000","last_unstale":"2024-08-17T18:24:17.467240+0000","last_undegraded":"2024-08-17T18:24:17.467240+0000","last_fullsized":"2024-08-17T18:24:17.467240+0000","mapping_epoch":1530,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1531,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:04:38.203244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":1589,"reported_epoch":1548,"state":"active+clean","last_fresh":"2024-08-17T18:24:17.731595+0000","last_change":"2024-08-17T18:19:27.037702+0000","last_active":"2024-08-17T18:24:17.731595+0000","last_peered":"2024-08-17T18:24:17.731595+0000","last_clean":"2024-08-17T18:24:17.731595+0000","last_became_active":"2024-08-17T18:19:27.036532+0000","last_became_peered":"2024-08-17T18:19:27.036532+0000","last_unstale":"2024-08-17T18:24:17.731595+0000","last_undegraded":"2024-08-17T18:24:17.731595+0000","last_fullsized":"2024-08-17T18:24:17.731595+0000","mapping_epoch":1455,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":1456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:51.131321+0000","last_clean_scrub_stamp":"2024-08-17T17:21:51.131321+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:24.777460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2455,"reported_epoch":1549,"state":"active+clean","last_fresh":"2024-08-17T18:24:20.493539+0000","last_change":"2024-08-17T18:23:05.476037+0000","last_active":"2024-08-17T18:24:20.493539+0000","last_peered":"2024-08-17T18:24:20.493539+0000","last_clean":"2024-08-17T18:24:20.493539+0000","last_became_active":"2024-08-17T18:23:05.475721+0000","last_became_peered":"2024-08-17T18:23:05.475721+0000","last_unstale":"2024-08-17T18:24:20.493539+0000","last_undegraded":"2024-08-17T18:24:20.493539+0000","last_fullsized":"2024-08-17T18:24:20.493539+0000","mapping_epoch":1527,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:21:52.134589+0000","last_clean_scrub_stamp":"2024-08-17T17:21:52.134589+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:30:24.857288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":95,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":99,"num_bytes_recovered":12636,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234772,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1503200,"kb_used_data":9300,"kb_used_omap":1,"kb_used_meta":1493886,"kb_avail":92241952,"statfs":{"total":95995035648,"available":94455758848,"internally_reserved":0,"allocated":9523200,"data_stored":5491709,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":1529739608},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496407,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":902488,"kb_used_data":9292,"kb_used_omap":1,"kb_used_meta":893182,"kb_avail":92842664,"statfs":{"total":95995035648,"available":95070887936,"internally_reserved":0,"allocated":9515008,"data_stored":5491574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":914618712},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208758045,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":318540,"kb_used_data":8704,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93426612,"statfs":{"total":95995035648,"available":95668850688,"internally_reserved":0,"allocated":8912896,"data_stored":4897032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":1343,"seq":5768141078718,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":781572,"kb_used_data":8696,"kb_used_omap":112,"kb_used_meta":772751,"kb_avail":92963580,"statfs":{"total":95995035648,"available":95194705920,"internally_reserved":0,"allocated":8904704,"data_stored":4896491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":115373,"internal_metadata":791297363},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":1454,"seq":6244882448455,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":285000,"kb_used_data":8700,"kb_used_omap":111,"kb_used_meta":276176,"kb_avail":93460152,"statfs":{"total":95995035648,"available":95703195648,"internally_reserved":0,"allocated":8908800,"data_stored":4896997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":114548,"internal_metadata":282804364},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":506,"seq":2173253452432,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1112580,"kb_used_data":8696,"kb_used_omap":54,"kb_used_meta":1103817,"kb_avail":92632572,"statfs":{"total":95995035648,"available":94855753728,"internally_reserved":0,"allocated":8904704,"data_stored":4896412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":55922,"internal_metadata":1130309006},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":1178,"seq":5059471475123,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1528140,"kb_used_data":9280,"kb_used_omap":119,"kb_used_meta":1518728,"kb_avail":92217012,"statfs":{"total":95995035648,"available":94430220288,"internally_reserved":0,"allocated":9502720,"data_stored":5487353,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":122146,"internal_metadata":1555178206},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019680,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":310232,"kb_used_data":8716,"kb_used_omap":1,"kb_used_meta":301502,"kb_avail":93434920,"statfs":{"total":95995035648,"available":95677358080,"internally_reserved":0,"allocated":8925184,"data_stored":4901263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308738392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2234,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T18:25:13.530 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T18:25:13.531 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi130 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-08-17T18:25:13.719 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=b640226155b54cc8908c032e6e210927 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi130 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-08-17T18:25:13.722 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-08-17T18:25:13.737 INFO:tasks.cephadm:Teardown begin 2024-08-17T18:25:13.737 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T18:25:13.752 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 18:25:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a[80002]: logger=cleanup t=2024-08-17T18:25:13.136963308Z level=info msg="Completed cleanup jobs" duration=15.311818ms 2024-08-17T18:25:13.775 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:13 smithi130 ceph-mon[105521]: from='client.37737 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:25:13.776 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:13 smithi130 ceph-mon[110068]: from='client.37737 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:25:13.790 DEBUG:teuthology.orchestra.run.smithi179:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T18:25:13.836 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-08-17T18:25:13.836 DEBUG:teuthology.orchestra.run.smithi130:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-08-17T18:25:13.852 DEBUG:teuthology.orchestra.run.smithi179:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-08-17T18:25:13.870 INFO:tasks.cephadm:Stopping all daemons... 2024-08-17T18:25:13.870 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-08-17T18:25:13.871 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.a 2024-08-17T18:25:14.093 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:13 smithi179 ceph-mon[93519]: from='client.37737 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T18:25:14.166 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:13 smithi130 systemd[1]: Stopping Ceph mon.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:25:14.653 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:14 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T18:25:14.215+0000 7fa85c724640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T18:25:14.654 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:14 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a[105499]: 2024-08-17T18:25:14.215+0000 7fa85c724640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-08-17T18:25:14.654 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:14 smithi130 podman[446810]: 2024-08-17 18:25:14.358587473 +0000 UTC m=+0.281546698 container died 79749aebcefa9827c6fada1f615d739fe273e8768e0aff218f6e74616b5bd5e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T18:25:14.916 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:14 smithi130 podman[446810]: 2024-08-17 18:25:14.792309731 +0000 UTC m=+0.715268955 container cleanup 79749aebcefa9827c6fada1f615d739fe273e8768e0aff218f6e74616b5bd5e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-08-17T18:25:14.917 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:14 smithi130 bash[446810]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a 2024-08-17T18:25:15.275 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.a.service' 2024-08-17T18:25:15.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:15 smithi130 podman[446827]: 2024-08-17 18:25:15.023698729 +0000 UTC m=+0.663915821 container remove 79749aebcefa9827c6fada1f615d739fe273e8768e0aff218f6e74616b5bd5e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-a, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-08-17T18:25:15.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:15 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.a.service: Deactivated successfully. 2024-08-17T18:25:15.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:15 smithi130 systemd[1]: Stopped Ceph mon.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T18:25:15.297 INFO:journalctl@ceph.mon.a.smithi130.stdout:Aug 17 18:25:15 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.a.service: Consumed 56.097s CPU time. 2024-08-17T18:25:15.312 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:25:15.313 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-08-17T18:25:15.313 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-08-17T18:25:15.313 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.c 2024-08-17T18:25:15.654 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:15 smithi130 systemd[1]: Stopping Ceph mon.c for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:25:15.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:25:15 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:18:25:15] ENGINE Bus STOPPING 2024-08-17T18:25:15.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:25:15 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:18:25:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T18:25:15.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:25:15 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:18:25:15] ENGINE Bus STOPPED 2024-08-17T18:25:15.917 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:25:15 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:18:25:15] ENGINE Bus STARTING 2024-08-17T18:25:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:15 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c[110045]: 2024-08-17T18:25:15.653+0000 7f82ab79f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T18:25:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:15 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c[110045]: 2024-08-17T18:25:15.653+0000 7f82ab79f640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-08-17T18:25:15.917 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:15 smithi130 podman[446942]: 2024-08-17 18:25:15.7261006 +0000 UTC m=+0.178611674 container died 4161e48dac506efae59e4ed2c12cfb1a94313f82cd32cac24d9a43b58a586648 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T18:25:16.387 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:25:15 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:18:25:15] ENGINE Serving on http://:::9283 2024-08-17T18:25:16.387 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:25:15 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y[95344]: [17/Aug/2024:18:25:15] ENGINE Bus STARTED 2024-08-17T18:25:16.387 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:16 smithi130 podman[446942]: 2024-08-17 18:25:16.143442736 +0000 UTC m=+0.595953797 container cleanup 4161e48dac506efae59e4ed2c12cfb1a94313f82cd32cac24d9a43b58a586648 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-08-17T18:25:16.388 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:16 smithi130 bash[446942]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c 2024-08-17T18:25:16.631 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.c.service' 2024-08-17T18:25:16.653 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:16 smithi130 podman[446961]: 2024-08-17 18:25:16.38667661 +0000 UTC m=+0.656111264 container remove 4161e48dac506efae59e4ed2c12cfb1a94313f82cd32cac24d9a43b58a586648 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-c, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-08-17T18:25:16.653 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:16 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.c.service: Deactivated successfully. 2024-08-17T18:25:16.654 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:16 smithi130 systemd[1]: Stopped Ceph mon.c for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T18:25:16.654 INFO:journalctl@ceph.mon.c.smithi130.stdout:Aug 17 18:25:16 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.c.service: Consumed 42.355s CPU time. 2024-08-17T18:25:16.666 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:25:16.666 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-08-17T18:25:16.666 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-08-17T18:25:16.666 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.b 2024-08-17T18:25:17.002 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:16 smithi179 systemd[1]: Stopping Ceph mon.b for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:25:17.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:17 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b[93496]: 2024-08-17T18:25:16.999+0000 7f4bb6931640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T18:25:17.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:17 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b[93496]: 2024-08-17T18:25:16.999+0000 7f4bb6931640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-08-17T18:25:17.346 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:17 smithi179 podman[302670]: 2024-08-17 18:25:17.174593855 +0000 UTC m=+0.303386566 container died d85c3c5c065e6a5ae87a14062c054cbf2b655411d64b7d52406952c31d908d64 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b, RELEASE=main-cd8bdbe, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-08-17T18:25:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:17 smithi179 podman[302670]: 2024-08-17 18:25:17.682943808 +0000 UTC m=+0.811736563 container cleanup d85c3c5c065e6a5ae87a14062c054cbf2b655411d64b7d52406952c31d908d64 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T18:25:17.843 INFO:journalctl@ceph.mon.b.smithi179.stdout:Aug 17 18:25:17 smithi179 bash[302670]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-mon-b 2024-08-17T18:25:18.179 DEBUG:teuthology.orchestra.run.smithi179:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@mon.b.service' 2024-08-17T18:25:18.226 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:25:18.227 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-08-17T18:25:18.227 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-08-17T18:25:18.227 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.y 2024-08-17T18:25:18.585 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:25:18 smithi130 systemd[1]: Stopping Ceph mgr.y for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:25:18.916 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:25:18 smithi130 podman[447084]: 2024-08-17 18:25:18.592658342 +0000 UTC m=+0.168719545 container died 584426b3875b678837089bbc161553d2e12545075efb0e11cffda8c96414307c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.36.0) 2024-08-17T18:25:19.416 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:25:19 smithi130 podman[447084]: 2024-08-17 18:25:19.274198205 +0000 UTC m=+0.850259414 container cleanup 584426b3875b678837089bbc161553d2e12545075efb0e11cffda8c96414307c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y, RELEASE=main-cd8bdbe, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-08-17T18:25:19.417 INFO:journalctl@ceph.mgr.y.smithi130.stdout:Aug 17 18:25:19 smithi130 bash[447084]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-y 2024-08-17T18:25:19.687 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.y.service' 2024-08-17T18:25:19.722 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:25:19.723 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-08-17T18:25:19.723 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-08-17T18:25:19.723 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.x 2024-08-17T18:25:20.063 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 18:25:19 smithi179 systemd[1]: Stopping Ceph mgr.x for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:25:20.333 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 18:25:20 smithi179 podman[302798]: 2024-08-17 18:25:20.063049479 +0000 UTC m=+0.147636654 container died cdd6dde7c7af3a41df5f582d18829d798ea8c6a7e1ae45b9befcfb7280d2e13f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-08-17T18:25:20.593 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 18:25:20 smithi179 podman[302798]: 2024-08-17 18:25:20.521063538 +0000 UTC m=+0.605650711 container cleanup cdd6dde7c7af3a41df5f582d18829d798ea8c6a7e1ae45b9befcfb7280d2e13f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD) 2024-08-17T18:25:20.593 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 18:25:20 smithi179 bash[302798]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x 2024-08-17T18:25:20.987 DEBUG:teuthology.orchestra.run.smithi179:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.x.service' 2024-08-17T18:25:21.010 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 18:25:20 smithi179 podman[302819]: 2024-08-17 18:25:20.739228464 +0000 UTC m=+0.632584587 container remove cdd6dde7c7af3a41df5f582d18829d798ea8c6a7e1ae45b9befcfb7280d2e13f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T18:25:21.010 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 18:25:20 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-17T18:25:21.011 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 18:25:20 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-17T18:25:21.011 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 18:25:20 smithi179 systemd[1]: Stopped Ceph mgr.x for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T18:25:21.011 INFO:journalctl@ceph.mgr.x.smithi179.stdout:Aug 17 18:25:20 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@mgr.x.service: Consumed 15.051s CPU time. 2024-08-17T18:25:21.025 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:25:21.026 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-08-17T18:25:21.026 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-08-17T18:25:21.026 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.0 2024-08-17T18:25:21.347 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 18:25:21 smithi130 systemd[1]: Stopping Ceph osd.0 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:25:21.666 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 18:25:21 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[177501]: 2024-08-17T18:25:21.346+0000 7fbba9614640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T18:25:21.667 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 18:25:21 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[177501]: 2024-08-17T18:25:21.346+0000 7fbba9614640 -1 osd.0 1549 *** Got signal Terminated *** 2024-08-17T18:25:21.667 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 18:25:21 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0[177501]: 2024-08-17T18:25:21.346+0000 7fbba9614640 -1 osd.0 1549 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T18:25:27.339 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 18:25:27 smithi130 podman[447216]: 2024-08-17 18:25:27.018942661 +0000 UTC m=+5.788236776 container died 691ff0a9846456e8efbe04563345e30cbcceeb592b170c12c38b6b79112a3637 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-08-17T18:25:27.666 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 18:25:27 smithi130 podman[447216]: 2024-08-17 18:25:27.553827628 +0000 UTC m=+6.323121733 container cleanup 691ff0a9846456e8efbe04563345e30cbcceeb592b170c12c38b6b79112a3637 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0) 2024-08-17T18:25:27.667 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 18:25:27 smithi130 bash[447216]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0 2024-08-17T18:25:28.166 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 18:25:27 smithi130 podman[447239]: 2024-08-17 18:25:27.805665523 +0000 UTC m=+0.767223584 container remove 691ff0a9846456e8efbe04563345e30cbcceeb592b170c12c38b6b79112a3637 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_CLEAN=True) 2024-08-17T18:25:28.542 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 18:25:28 smithi130 podman[447335]: 2024-08-17 18:25:28.22318788 +0000 UTC m=+0.114071124 container create cf0c7cd9b5e9f30f0f625503f67ecd48c21439a65c53ccd8bb4f1b2771600677 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-08-17T18:25:28.543 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 18:25:28 smithi130 podman[447335]: 2024-08-17 18:25:28.126254008 +0000 UTC m=+0.017137278 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T18:25:28.543 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 18:25:28 smithi130 podman[447335]: 2024-08-17 18:25:28.450691761 +0000 UTC m=+0.341575023 container init cf0c7cd9b5e9f30f0f625503f67ecd48c21439a65c53ccd8bb4f1b2771600677 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-08-17T18:25:28.543 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 18:25:28 smithi130 podman[447335]: 2024-08-17 18:25:28.456972046 +0000 UTC m=+0.347855294 container start cf0c7cd9b5e9f30f0f625503f67ecd48c21439a65c53ccd8bb4f1b2771600677 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, RELEASE=main-cd8bdbe, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-08-17T18:25:28.543 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 18:25:28 smithi130 podman[447335]: 2024-08-17 18:25:28.54235049 +0000 UTC m=+0.433233738 container attach cf0c7cd9b5e9f30f0f625503f67ecd48c21439a65c53ccd8bb4f1b2771600677 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-08-17T18:25:28.916 INFO:journalctl@ceph.osd.0.smithi130.stdout:Aug 17 18:25:28 smithi130 podman[447335]: 2024-08-17 18:25:28.68952599 +0000 UTC m=+0.580409243 container died cf0c7cd9b5e9f30f0f625503f67ecd48c21439a65c53ccd8bb4f1b2771600677 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T18:25:29.462 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.0.service' 2024-08-17T18:25:29.505 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:25:29.505 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-08-17T18:25:29.505 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-08-17T18:25:29.505 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.1 2024-08-17T18:25:29.844 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 18:25:29 smithi130 systemd[1]: Stopping Ceph osd.1 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:25:30.166 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 18:25:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1[124680]: 2024-08-17T18:25:29.842+0000 7fd495c7d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T18:25:30.167 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 18:25:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1[124680]: 2024-08-17T18:25:29.842+0000 7fd495c7d640 -1 osd.1 1549 *** Got signal Terminated *** 2024-08-17T18:25:30.167 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 18:25:29 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1[124680]: 2024-08-17T18:25:29.842+0000 7fd495c7d640 -1 osd.1 1549 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T18:25:35.666 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 18:25:35 smithi130 podman[447469]: 2024-08-17 18:25:35.348864526 +0000 UTC m=+5.613199225 container died 54f6b453c741e71860bf00acb41c5c844fea1150fb2705046969391e5edcf815 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-08-17T18:25:36.109 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 18:25:35 smithi130 podman[447469]: 2024-08-17 18:25:35.891373668 +0000 UTC m=+6.155708357 container cleanup 54f6b453c741e71860bf00acb41c5c844fea1150fb2705046969391e5edcf815 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1, ceph=True, GIT_CLEAN=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-17T18:25:36.110 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 18:25:35 smithi130 bash[447469]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1 2024-08-17T18:25:36.416 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 18:25:36 smithi130 podman[447489]: 2024-08-17 18:25:36.10944275 +0000 UTC m=+0.762168063 container remove 54f6b453c741e71860bf00acb41c5c844fea1150fb2705046969391e5edcf815 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-08-17T18:25:36.821 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 18:25:36 smithi130 podman[447586]: 2024-08-17 18:25:36.46208316 +0000 UTC m=+0.018329892 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T18:25:36.822 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 18:25:36 smithi130 podman[447586]: 2024-08-17 18:25:36.585522735 +0000 UTC m=+0.141769499 container create ef74ff41def470eb2f513728535f6b7507a6f947c32bb049f0e274852cbdaf91 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-08-17T18:25:36.822 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 18:25:36 smithi130 podman[447586]: 2024-08-17 18:25:36.82154765 +0000 UTC m=+0.377794380 container init ef74ff41def470eb2f513728535f6b7507a6f947c32bb049f0e274852cbdaf91 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-08-17T18:25:37.167 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 18:25:36 smithi130 podman[447586]: 2024-08-17 18:25:36.82645031 +0000 UTC m=+0.382697041 container start ef74ff41def470eb2f513728535f6b7507a6f947c32bb049f0e274852cbdaf91 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD) 2024-08-17T18:25:37.167 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 18:25:36 smithi130 podman[447586]: 2024-08-17 18:25:36.913207957 +0000 UTC m=+0.469454690 container attach ef74ff41def470eb2f513728535f6b7507a6f947c32bb049f0e274852cbdaf91 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-08-17T18:25:37.167 INFO:journalctl@ceph.osd.1.smithi130.stdout:Aug 17 18:25:37 smithi130 podman[447586]: 2024-08-17 18:25:37.015249008 +0000 UTC m=+0.571495739 container died ef74ff41def470eb2f513728535f6b7507a6f947c32bb049f0e274852cbdaf91 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-1-deactivate, ceph=True, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0) 2024-08-17T18:25:37.666 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.1.service' 2024-08-17T18:25:37.707 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:25:37.707 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-08-17T18:25:37.708 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-08-17T18:25:37.708 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.2 2024-08-17T18:25:37.916 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:25:37 smithi130 systemd[1]: Stopping Ceph osd.2 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:25:38.417 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:25:38 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[407557]: 2024-08-17T18:25:38.044+0000 7f7253c2f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T18:25:38.417 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:25:38 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[407557]: 2024-08-17T18:25:38.045+0000 7f7253c2f640 -1 osd.2 1549 *** Got signal Terminated *** 2024-08-17T18:25:38.417 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:25:38 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2[407557]: 2024-08-17T18:25:38.045+0000 7f7253c2f640 -1 osd.2 1549 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T18:25:43.762 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:25:43 smithi130 podman[447712]: 2024-08-17 18:25:43.447057707 +0000 UTC m=+5.510076655 container died 294e7a7bd278c7b3a13f0e1b4b9c0a793fc64bff3f7cecc3644635b586ac1f8e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0) 2024-08-17T18:25:44.166 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:25:43 smithi130 podman[447712]: 2024-08-17 18:25:43.986759474 +0000 UTC m=+6.049778487 container cleanup 294e7a7bd278c7b3a13f0e1b4b9c0a793fc64bff3f7cecc3644635b586ac1f8e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T18:25:44.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:25:43 smithi130 bash[447712]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2 2024-08-17T18:25:44.623 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:25:44 smithi130 podman[447736]: 2024-08-17 18:25:44.213448049 +0000 UTC m=+0.744556470 container remove 294e7a7bd278c7b3a13f0e1b4b9c0a793fc64bff3f7cecc3644635b586ac1f8e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-08-17T18:25:44.875 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:25:44 smithi130 podman[447828]: 2024-08-17 18:25:44.526163193 +0000 UTC m=+0.017188795 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T18:25:44.875 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:25:44 smithi130 podman[447828]: 2024-08-17 18:25:44.639253454 +0000 UTC m=+0.130279094 container create fb8331d9bd1e18d35b2747d049f1667cf0428cfc37e2a4f292fb45889c5384e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=main-cd8bdbe, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0) 2024-08-17T18:25:44.875 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:25:44 smithi130 podman[447828]: 2024-08-17 18:25:44.875195688 +0000 UTC m=+0.366221282 container init fb8331d9bd1e18d35b2747d049f1667cf0428cfc37e2a4f292fb45889c5384e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T18:25:45.166 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:25:44 smithi130 podman[447828]: 2024-08-17 18:25:44.880771444 +0000 UTC m=+0.371797049 container start fb8331d9bd1e18d35b2747d049f1667cf0428cfc37e2a4f292fb45889c5384e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-08-17T18:25:45.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:25:44 smithi130 podman[447828]: 2024-08-17 18:25:44.966533209 +0000 UTC m=+0.457558807 container attach fb8331d9bd1e18d35b2747d049f1667cf0428cfc37e2a4f292fb45889c5384e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-08-17T18:25:45.167 INFO:journalctl@ceph.osd.2.smithi130.stdout:Aug 17 18:25:45 smithi130 podman[447828]: 2024-08-17 18:25:45.074863504 +0000 UTC m=+0.565889099 container died fb8331d9bd1e18d35b2747d049f1667cf0428cfc37e2a4f292fb45889c5384e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-2-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.36.0) 2024-08-17T18:25:45.781 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.2.service' 2024-08-17T18:25:45.819 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:25:45.819 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-08-17T18:25:45.820 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-08-17T18:25:45.820 DEBUG:teuthology.orchestra.run.smithi130:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.3 2024-08-17T18:25:46.166 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:45 smithi130 systemd[1]: Stopping Ceph osd.3 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:25:46.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:25:46.324Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.130:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.130:8765: connect: connection refused" 2024-08-17T18:25:46.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:25:46.326Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.130:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.130:8765: connect: connection refused" 2024-08-17T18:25:46.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:25:46.328Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.130:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.130:8765: connect: connection refused" 2024-08-17T18:25:46.593 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:25:46.332Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.130:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.130:8765: connect: connection refused" 2024-08-17T18:25:46.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:25:46.334Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.130:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.130:8765: connect: connection refused" 2024-08-17T18:25:46.594 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:25:46 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:25:46.334Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.130:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.130:8765: connect: connection refused" 2024-08-17T18:25:46.666 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:46 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[337468]: 2024-08-17T18:25:46.175+0000 7f4e53b8a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T18:25:46.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:46 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[337468]: 2024-08-17T18:25:46.175+0000 7f4e53b8a640 -1 osd.3 1549 *** Got signal Terminated *** 2024-08-17T18:25:46.667 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:46 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[337468]: 2024-08-17T18:25:46.175+0000 7f4e53b8a640 -1 osd.3 1549 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T18:25:48.593 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:48 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:48.241+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:25:49.416 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:48 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[337468]: 2024-08-17T18:25:48.942+0000 7f4e4c197640 -1 osd.3 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:23.389753+0000 front 2024-08-17T18:25:23.389913+0000 (oldest deadline 2024-08-17T18:25:48.089743+0000) 2024-08-17T18:25:49.592 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:49 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:49.233+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:25:50.416 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:49 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[337468]: 2024-08-17T18:25:49.974+0000 7f4e4c197640 -1 osd.3 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:23.389753+0000 front 2024-08-17T18:25:23.389913+0000 (oldest deadline 2024-08-17T18:25:48.089743+0000) 2024-08-17T18:25:50.593 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:50 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:50.267+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:25:51.417 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:51 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3[337468]: 2024-08-17T18:25:51.015+0000 7f4e4c197640 -1 osd.3 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:23.389753+0000 front 2024-08-17T18:25:23.389913+0000 (oldest deadline 2024-08-17T18:25:48.089743+0000) 2024-08-17T18:25:51.593 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:51 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:51.271+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:25:51.593 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:25:51 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:25:51.151+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:25:51.974 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:51 smithi130 podman[447968]: 2024-08-17 18:25:51.678705829 +0000 UTC m=+5.618457471 container died bafb5a2dcf5301fcbd2bdfc34d145a50270c8e08197f3314a2a300f8781c8199 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, RELEASE=main-cd8bdbe, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-08-17T18:25:52.405 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:25:52 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:25:52.113+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:25:52.406 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:52 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:52.311+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:25:52.416 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:52 smithi130 podman[447968]: 2024-08-17 18:25:52.156926639 +0000 UTC m=+6.096678273 container cleanup bafb5a2dcf5301fcbd2bdfc34d145a50270c8e08197f3314a2a300f8781c8199 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-08-17T18:25:52.417 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:52 smithi130 bash[447968]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3 2024-08-17T18:25:52.760 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:52 smithi130 podman[447992]: 2024-08-17 18:25:52.441582499 +0000 UTC m=+0.750260735 container remove bafb5a2dcf5301fcbd2bdfc34d145a50270c8e08197f3314a2a300f8781c8199 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0) 2024-08-17T18:25:52.842 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:25:52 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:25:52.405+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:25:53.070 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:52 smithi130 podman[448084]: 2024-08-17 18:25:52.662440637 +0000 UTC m=+0.017402304 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T18:25:53.071 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:52 smithi130 podman[448084]: 2024-08-17 18:25:52.784484253 +0000 UTC m=+0.139445909 container create 456832f74ff58c6bd62900a9683b9328d5d77d37cd1fd924348e0086cec66071 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, ceph=True, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-08-17T18:25:53.071 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:52 smithi130 podman[448084]: 2024-08-17 18:25:52.995334022 +0000 UTC m=+0.350295691 container init 456832f74ff58c6bd62900a9683b9328d5d77d37cd1fd924348e0086cec66071 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-08-17T18:25:53.071 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:53 smithi130 podman[448084]: 2024-08-17 18:25:53.000007148 +0000 UTC m=+0.354968816 container start 456832f74ff58c6bd62900a9683b9328d5d77d37cd1fd924348e0086cec66071 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe) 2024-08-17T18:25:53.071 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:53 smithi130 podman[448084]: 2024-08-17 18:25:53.069971629 +0000 UTC m=+0.424933286 container attach 456832f74ff58c6bd62900a9683b9328d5d77d37cd1fd924348e0086cec66071 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-08-17T18:25:53.343 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:53 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:53.328+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:25:53.343 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:25:53 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:25:53.083+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:25:53.343 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:25:53 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4[40234]: 2024-08-17T18:25:53.304+0000 7f543bd25640 -1 osd.4 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.343927+0000 front 2024-08-17T18:25:25.343816+0000 (oldest deadline 2024-08-17T18:25:52.344213+0000) 2024-08-17T18:25:53.416 INFO:journalctl@ceph.osd.3.smithi130.stdout:Aug 17 18:25:53 smithi130 podman[448084]: 2024-08-17 18:25:53.195302718 +0000 UTC m=+0.550264375 container died 456832f74ff58c6bd62900a9683b9328d5d77d37cd1fd924348e0086cec66071 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T18:25:53.843 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:25:53 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:25:53.390+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:25:53.926 DEBUG:teuthology.orchestra.run.smithi130:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.3.service' 2024-08-17T18:25:53.965 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:25:53.965 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-08-17T18:25:53.965 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-08-17T18:25:53.965 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.4 2024-08-17T18:25:54.280 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:25:54 smithi179 systemd[1]: Stopping Ceph osd.4 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:25:54.280 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:25:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:25:54.099+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:25:54.593 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:25:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:25:54.430+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:25:54.593 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:54.336+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:25:54.593 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:25:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4[40234]: 2024-08-17T18:25:54.278+0000 7f543bd25640 -1 osd.4 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.343927+0000 front 2024-08-17T18:25:25.343816+0000 (oldest deadline 2024-08-17T18:25:52.344213+0000) 2024-08-17T18:25:54.594 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:25:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4[40234]: 2024-08-17T18:25:54.391+0000 7f544070f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T18:25:54.594 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:25:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4[40234]: 2024-08-17T18:25:54.391+0000 7f544070f640 -1 osd.4 1549 *** Got signal Terminated *** 2024-08-17T18:25:54.594 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:25:54 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4[40234]: 2024-08-17T18:25:54.391+0000 7f544070f640 -1 osd.4 1549 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T18:25:55.399 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:25:55 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4[40234]: 2024-08-17T18:25:55.316+0000 7f543bd25640 -1 osd.4 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.343927+0000 front 2024-08-17T18:25:25.343816+0000 (oldest deadline 2024-08-17T18:25:52.344213+0000) 2024-08-17T18:25:55.400 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:25:55 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:25:55.142+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:25:55.400 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:55 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:55.338+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:25:55.842 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:25:55 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:25:55.399+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:25:56.593 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:25:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:25:56.419+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:25:56.593 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:25:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:25:56.419+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.305537+0000 front 2024-08-17T18:25:32.305571+0000 (oldest deadline 2024-08-17T18:25:55.805478+0000) 2024-08-17T18:25:56.593 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:25:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:25:56.185+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:25:56.593 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:25:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:25:56.185+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:25:56.594 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:25:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4[40234]: 2024-08-17T18:25:56.350+0000 7f543bd25640 -1 osd.4 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.343927+0000 front 2024-08-17T18:25:25.343816+0000 (oldest deadline 2024-08-17T18:25:52.344213+0000) 2024-08-17T18:25:56.594 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:56.362+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:25:56.594 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:56 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:56.362+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:25:57.430 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:25:57 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4[40234]: 2024-08-17T18:25:57.387+0000 7f543bd25640 -1 osd.4 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.343927+0000 front 2024-08-17T18:25:25.343816+0000 (oldest deadline 2024-08-17T18:25:52.344213+0000) 2024-08-17T18:25:57.430 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:25:57 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:25:57.148+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:25:57.431 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:25:57 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:25:57.148+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:25:57.431 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:57 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:57.367+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:25:57.431 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:57 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:57.367+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:25:57.843 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:25:57 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:25:57.429+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:25:57.843 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:25:57 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:25:57.429+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.305537+0000 front 2024-08-17T18:25:32.305571+0000 (oldest deadline 2024-08-17T18:25:55.805478+0000) 2024-08-17T18:25:58.416 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:25:58 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4[40234]: 2024-08-17T18:25:58.356+0000 7f543bd25640 -1 osd.4 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.343927+0000 front 2024-08-17T18:25:25.343816+0000 (oldest deadline 2024-08-17T18:25:52.344213+0000) 2024-08-17T18:25:58.416 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:25:58 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:25:58.155+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:25:58.416 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:25:58 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:25:58.155+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:25:58.843 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:25:58 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:25:58.431+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:25:58.843 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:25:58 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:25:58.431+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.305537+0000 front 2024-08-17T18:25:32.305571+0000 (oldest deadline 2024-08-17T18:25:55.805478+0000) 2024-08-17T18:25:58.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:58 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:58.415+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:25:58.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:58 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:58.415+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:25:59.403 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:25:59 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4[40234]: 2024-08-17T18:25:59.329+0000 7f543bd25640 -1 osd.4 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.343927+0000 front 2024-08-17T18:25:25.343816+0000 (oldest deadline 2024-08-17T18:25:52.344213+0000) 2024-08-17T18:25:59.404 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:25:59 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:25:59.125+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:25:59.404 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:25:59 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:25:59.125+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:25:59.843 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:25:59 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:25:59.410+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:25:59.843 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:25:59 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:25:59.411+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.305537+0000 front 2024-08-17T18:25:32.305571+0000 (oldest deadline 2024-08-17T18:25:55.805478+0000) 2024-08-17T18:25:59.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:59 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:59.402+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:25:59.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:25:59 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:25:59.402+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:00.308 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:26:00 smithi179 podman[302957]: 2024-08-17 18:26:00.007924072 +0000 UTC m=+5.846970939 container died 254dc07634207468630c402ee2039c7520fed46cba93df95e0a39657d718b6ea (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.36.0) 2024-08-17T18:26:00.309 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:00.172+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:00.309 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:00.172+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:00.593 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:00.451+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:00.593 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:00.451+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:00.593 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:26:00 smithi179 podman[302957]: 2024-08-17 18:26:00.390280045 +0000 UTC m=+6.229326902 container cleanup 254dc07634207468630c402ee2039c7520fed46cba93df95e0a39657d718b6ea (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True) 2024-08-17T18:26:00.594 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:26:00 smithi179 bash[302957]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4 2024-08-17T18:26:00.594 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:00.369+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:26:00.594 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:00 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:00.369+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.305537+0000 front 2024-08-17T18:25:32.305571+0000 (oldest deadline 2024-08-17T18:25:55.805478+0000) 2024-08-17T18:26:01.019 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:26:00 smithi179 podman[302979]: 2024-08-17 18:26:00.700618629 +0000 UTC m=+0.684656745 container remove 254dc07634207468630c402ee2039c7520fed46cba93df95e0a39657d718b6ea (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=quincy-f2c39d4, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-08-17T18:26:01.333 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:01 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:01.134+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:01.333 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:01 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:01.134+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:01.334 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:26:01 smithi179 podman[303076]: 2024-08-17 18:26:00.92221193 +0000 UTC m=+0.017071998 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T18:26:01.334 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:26:01 smithi179 podman[303076]: 2024-08-17 18:26:01.034849875 +0000 UTC m=+0.129709975 container create 6ceaf4e8e604ab2c90bcc5e22638e824b05f2594dd88bc74f4024f8fb003b82d (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0) 2024-08-17T18:26:01.334 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:26:01 smithi179 podman[303076]: 2024-08-17 18:26:01.262679822 +0000 UTC m=+0.357539876 container init 6ceaf4e8e604ab2c90bcc5e22638e824b05f2594dd88bc74f4024f8fb003b82d (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-f2c39d4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-08-17T18:26:01.334 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:26:01 smithi179 podman[303076]: 2024-08-17 18:26:01.267223616 +0000 UTC m=+0.362083682 container start 6ceaf4e8e604ab2c90bcc5e22638e824b05f2594dd88bc74f4024f8fb003b82d (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-08-17T18:26:01.588 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:26:01 smithi179 podman[303076]: 2024-08-17 18:26:01.395915906 +0000 UTC m=+0.490776042 container attach 6ceaf4e8e604ab2c90bcc5e22638e824b05f2594dd88bc74f4024f8fb003b82d (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=quincy-f2c39d4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-08-17T18:26:01.589 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:01 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:01.331+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:26:01.589 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:01 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:01.331+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.305537+0000 front 2024-08-17T18:25:32.305571+0000 (oldest deadline 2024-08-17T18:25:55.805478+0000) 2024-08-17T18:26:01.589 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:01 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:01.471+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:01.590 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:01 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:01.471+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:01.843 INFO:journalctl@ceph.osd.4.smithi179.stdout:Aug 17 18:26:01 smithi179 podman[303076]: 2024-08-17 18:26:01.588452261 +0000 UTC m=+0.683312321 container died 6ceaf4e8e604ab2c90bcc5e22638e824b05f2594dd88bc74f4024f8fb003b82d (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-4-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4) 2024-08-17T18:26:02.200 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:02.167+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:02.200 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:02.167+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:02.223 DEBUG:teuthology.orchestra.run.smithi179:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.4.service' 2024-08-17T18:26:02.262 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:26:02.262 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-08-17T18:26:02.262 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-08-17T18:26:02.262 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.5 2024-08-17T18:26:02.472 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:02.341+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:26:02.472 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:02.341+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.305537+0000 front 2024-08-17T18:25:32.305571+0000 (oldest deadline 2024-08-17T18:25:55.805478+0000) 2024-08-17T18:26:02.472 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:02 smithi179 systemd[1]: Stopping Ceph osd.5 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:26:02.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:02.469+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:02.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:02.469+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:02.843 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:02.611+0000 7efc594b0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T18:26:02.843 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:02.611+0000 7efc594b0640 -1 osd.5 1549 *** Got signal Terminated *** 2024-08-17T18:26:02.843 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:02 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:02.611+0000 7efc594b0640 -1 osd.5 1549 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T18:26:03.481 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:03.336+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:26:03.482 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:03.336+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.305537+0000 front 2024-08-17T18:25:32.305571+0000 (oldest deadline 2024-08-17T18:25:55.805478+0000) 2024-08-17T18:26:03.482 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:03.123+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:03.482 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:03.123+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:03.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:03.480+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:03.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:03 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:03.480+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:04.439 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:04.126+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:04.439 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:04.126+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:04.439 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:04.361+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:26:04.439 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:04.361+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.305537+0000 front 2024-08-17T18:25:32.305571+0000 (oldest deadline 2024-08-17T18:25:55.805478+0000) 2024-08-17T18:26:04.846 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:04.437+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:04.846 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:04 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:04.437+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:05.593 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:05.172+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:05.593 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:05.172+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:05.593 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:05.172+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.341077+0000 front 2024-08-17T18:25:42.341184+0000 (oldest deadline 2024-08-17T18:26:04.641143+0000) 2024-08-17T18:26:05.594 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:05.393+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:05.594 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:05.393+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:05.594 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:05.393+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:05.594 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:05.372+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:26:05.595 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:05 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:05.372+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.305537+0000 front 2024-08-17T18:25:32.305571+0000 (oldest deadline 2024-08-17T18:25:55.805478+0000) 2024-08-17T18:26:06.433 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:06 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:06.335+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:26:06.433 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:06 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:06.335+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.305537+0000 front 2024-08-17T18:25:32.305571+0000 (oldest deadline 2024-08-17T18:25:55.805478+0000) 2024-08-17T18:26:06.434 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:06 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:06.179+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:06.434 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:06 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:06.179+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:06.434 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:06 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:06.179+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.341077+0000 front 2024-08-17T18:25:42.341184+0000 (oldest deadline 2024-08-17T18:26:04.641143+0000) 2024-08-17T18:26:06.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:06 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:06.432+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:06.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:06 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:06.432+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:06.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:06 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:06.432+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:07.431 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:07.141+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:07.431 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:07.141+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:07.431 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:07.141+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.341077+0000 front 2024-08-17T18:25:42.341184+0000 (oldest deadline 2024-08-17T18:26:04.641143+0000) 2024-08-17T18:26:07.431 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:07.356+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:22.304677+0000 front 2024-08-17T18:25:22.304546+0000 (oldest deadline 2024-08-17T18:25:52.304974+0000) 2024-08-17T18:26:07.432 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:07.356+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.305537+0000 front 2024-08-17T18:25:32.305571+0000 (oldest deadline 2024-08-17T18:25:55.805478+0000) 2024-08-17T18:26:07.432 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5[46254]: 2024-08-17T18:26:07.356+0000 7efc552c7640 -1 osd.5 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:41.007036+0000 front 2024-08-17T18:25:41.007090+0000 (oldest deadline 2024-08-17T18:26:06.906893+0000) 2024-08-17T18:26:07.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:07.428+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:07.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:07.428+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:07.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:07 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:07.428+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:08.379 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:08 smithi179 podman[303204]: 2024-08-17 18:26:08.103355698 +0000 UTC m=+5.612809976 container died 078704aadcd7bc5fabea23fb6e1475a3d150b19d90ebf642eb03a24ecc27d52a (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5, ceph=True, org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T18:26:08.520 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:08 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:08.145+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:08.521 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:08 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:08.145+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:08.521 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:08 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:08.145+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.341077+0000 front 2024-08-17T18:25:42.341184+0000 (oldest deadline 2024-08-17T18:26:04.641143+0000) 2024-08-17T18:26:08.729 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:08 smithi179 podman[303204]: 2024-08-17 18:26:08.477623384 +0000 UTC m=+5.987077653 container cleanup 078704aadcd7bc5fabea23fb6e1475a3d150b19d90ebf642eb03a24ecc27d52a (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5, RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0) 2024-08-17T18:26:08.729 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:08 smithi179 bash[303204]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5 2024-08-17T18:26:08.730 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:08 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:08.386+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:08.730 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:08 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:08.386+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:08.730 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:08 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:08.386+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:09.046 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:08 smithi179 podman[303227]: 2024-08-17 18:26:08.729077212 +0000 UTC m=+0.629009997 container remove 078704aadcd7bc5fabea23fb6e1475a3d150b19d90ebf642eb03a24ecc27d52a (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, org.label-schema.build-date=20240731) 2024-08-17T18:26:09.315 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:09 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:09.157+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:09.316 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:09 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:09.157+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:09.316 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:09 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:09.157+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.341077+0000 front 2024-08-17T18:25:42.341184+0000 (oldest deadline 2024-08-17T18:26:04.641143+0000) 2024-08-17T18:26:09.316 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:09 smithi179 podman[303317]: 2024-08-17 18:26:09.046468872 +0000 UTC m=+0.111739911 container create 343aac254b9b2b67bdd755c918c1c934057aa37ea3b04d7e0ea51d86f8d7034b (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T18:26:09.316 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:09 smithi179 podman[303317]: 2024-08-17 18:26:08.951647178 +0000 UTC m=+0.016918187 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T18:26:09.316 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:09 smithi179 podman[303317]: 2024-08-17 18:26:09.232206037 +0000 UTC m=+0.297477042 container init 343aac254b9b2b67bdd755c918c1c934057aa37ea3b04d7e0ea51d86f8d7034b (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5-deactivate, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T18:26:09.317 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:09 smithi179 podman[303317]: 2024-08-17 18:26:09.237167682 +0000 UTC m=+0.302438680 container start 343aac254b9b2b67bdd755c918c1c934057aa37ea3b04d7e0ea51d86f8d7034b (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5-deactivate, RELEASE=quincy-f2c39d4, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-08-17T18:26:09.317 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:09 smithi179 podman[303317]: 2024-08-17 18:26:09.31559414 +0000 UTC m=+0.380865140 container attach 343aac254b9b2b67bdd755c918c1c934057aa37ea3b04d7e0ea51d86f8d7034b (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5-deactivate, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T18:26:09.593 INFO:journalctl@ceph.osd.5.smithi179.stdout:Aug 17 18:26:09 smithi179 podman[303317]: 2024-08-17 18:26:09.408600616 +0000 UTC m=+0.473871610 container died 343aac254b9b2b67bdd755c918c1c934057aa37ea3b04d7e0ea51d86f8d7034b (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=quincy-f2c39d4, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-08-17T18:26:09.593 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:09 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:09.430+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:09.593 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:09 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:09.430+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:09.593 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:09 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:09.430+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:09.978 DEBUG:teuthology.orchestra.run.smithi179:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.5.service' 2024-08-17T18:26:10.017 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:26:10.018 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-08-17T18:26:10.018 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-08-17T18:26:10.018 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.6 2024-08-17T18:26:10.343 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:10 smithi179 systemd[1]: Stopping Ceph osd.6 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:26:10.343 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:10.171+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:10.343 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:10.171+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:10.343 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:10.171+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.341077+0000 front 2024-08-17T18:25:42.341184+0000 (oldest deadline 2024-08-17T18:26:04.641143+0000) 2024-08-17T18:26:10.843 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:10.368+0000 7f91159f5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T18:26:10.843 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:10.368+0000 7f91159f5640 -1 osd.6 1549 *** Got signal Terminated *** 2024-08-17T18:26:10.843 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:10.368+0000 7f91159f5640 -1 osd.6 1549 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T18:26:10.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:10.474+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:10.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:10.474+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:10.844 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:10 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:10.474+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:11.474 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:11.141+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:11.474 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:11.141+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:11.474 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:11.141+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.341077+0000 front 2024-08-17T18:25:42.341184+0000 (oldest deadline 2024-08-17T18:26:04.641143+0000) 2024-08-17T18:26:11.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:11.473+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:11.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:11.473+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:11.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:11 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:11.473+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:12.517 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:12 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:12.125+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:12.517 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:12 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:12.125+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:12.517 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:12 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:12.125+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.341077+0000 front 2024-08-17T18:25:42.341184+0000 (oldest deadline 2024-08-17T18:26:04.641143+0000) 2024-08-17T18:26:12.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:12 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:12.516+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:12.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:12 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:12.516+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:12.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:12 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:12.516+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:13.565 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:13.118+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:13.566 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:13.118+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:13.566 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:13.118+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.341077+0000 front 2024-08-17T18:25:42.341184+0000 (oldest deadline 2024-08-17T18:26:04.641143+0000) 2024-08-17T18:26:13.566 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:13.118+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-08-17T18:25:50.442722+0000 front 2024-08-17T18:25:50.442708+0000 (oldest deadline 2024-08-17T18:26:12.142649+0000) 2024-08-17T18:26:13.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:13.564+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:13.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:13.564+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:13.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:13.564+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:13.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:13 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:13.564+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-08-17T18:25:49.533146+0000 front 2024-08-17T18:25:49.533193+0000 (oldest deadline 2024-08-17T18:26:13.033087+0000) 2024-08-17T18:26:14.593 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:14 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:14.163+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:14.593 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:14 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:14.163+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:14.593 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:14 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:14.163+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.341077+0000 front 2024-08-17T18:25:42.341184+0000 (oldest deadline 2024-08-17T18:26:04.641143+0000) 2024-08-17T18:26:14.593 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:14 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:14.163+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-08-17T18:25:50.442722+0000 front 2024-08-17T18:25:50.442708+0000 (oldest deadline 2024-08-17T18:26:12.142649+0000) 2024-08-17T18:26:15.093 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:14 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:14.594+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:15.093 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:14 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:14.594+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:15.093 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:14 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:14.594+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:15.093 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:14 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:14.594+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-08-17T18:25:49.533146+0000 front 2024-08-17T18:25:49.533193+0000 (oldest deadline 2024-08-17T18:26:13.033087+0000) 2024-08-17T18:26:15.573 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:15.154+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:26.137759+0000 front 2024-08-17T18:25:26.137800+0000 (oldest deadline 2024-08-17T18:25:50.237692+0000) 2024-08-17T18:26:15.574 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:15.154+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:32.439254+0000 front 2024-08-17T18:25:32.439372+0000 (oldest deadline 2024-08-17T18:25:55.939403+0000) 2024-08-17T18:26:15.574 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:15.154+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.341077+0000 front 2024-08-17T18:25:42.341184+0000 (oldest deadline 2024-08-17T18:26:04.641143+0000) 2024-08-17T18:26:15.574 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6[52350]: 2024-08-17T18:26:15.154+0000 7f911300f640 -1 osd.6 1549 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-08-17T18:25:50.442722+0000 front 2024-08-17T18:25:50.442708+0000 (oldest deadline 2024-08-17T18:26:12.142649+0000) 2024-08-17T18:26:15.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:15.572+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:15.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:15.572+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:15.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:15.572+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:15.844 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:15 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:15.572+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-08-17T18:25:49.533146+0000 front 2024-08-17T18:25:49.533193+0000 (oldest deadline 2024-08-17T18:26:13.033087+0000) 2024-08-17T18:26:16.157 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:15 smithi179 podman[303443]: 2024-08-17 18:26:15.900726712 +0000 UTC m=+5.651564135 container died ab570ba67ae71cdd89756d26360c50dfa60e2c8eccb99af26dc6d811fdca6562 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=quincy-f2c39d4, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T18:26:16.440 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:16 smithi179 podman[303443]: 2024-08-17 18:26:16.246764613 +0000 UTC m=+5.997602026 container cleanup ab570ba67ae71cdd89756d26360c50dfa60e2c8eccb99af26dc6d811fdca6562 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, RELEASE=quincy-f2c39d4, org.label-schema.build-date=20240731) 2024-08-17T18:26:16.440 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:16 smithi179 bash[303443]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6 2024-08-17T18:26:16.843 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:16 smithi179 podman[303466]: 2024-08-17 18:26:16.439924267 +0000 UTC m=+0.536145407 container remove ab570ba67ae71cdd89756d26360c50dfa60e2c8eccb99af26dc6d811fdca6562 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T18:26:16.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:16.617+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:16.844 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:16.617+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:16.844 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:16.617+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:16.844 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:16 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:16.617+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-08-17T18:25:49.533146+0000 front 2024-08-17T18:25:49.533193+0000 (oldest deadline 2024-08-17T18:26:13.033087+0000) 2024-08-17T18:26:17.160 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:16 smithi179 podman[303561]: 2024-08-17 18:26:16.763513719 +0000 UTC m=+0.017248816 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T18:26:17.160 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:16 smithi179 podman[303561]: 2024-08-17 18:26:16.874055803 +0000 UTC m=+0.127790891 container create fd9f10c3375343c2f84a09a918db2bfa20c8b910c5659dac228c2fc5a0eb2f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-f2c39d4) 2024-08-17T18:26:17.160 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:17 smithi179 podman[303561]: 2024-08-17 18:26:17.068045387 +0000 UTC m=+0.321780476 container init fd9f10c3375343c2f84a09a918db2bfa20c8b910c5659dac228c2fc5a0eb2f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=quincy-f2c39d4, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-08-17T18:26:17.160 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:17 smithi179 podman[303561]: 2024-08-17 18:26:17.072638089 +0000 UTC m=+0.326373177 container start fd9f10c3375343c2f84a09a918db2bfa20c8b910c5659dac228c2fc5a0eb2f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6-deactivate, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, RELEASE=quincy-f2c39d4, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-08-17T18:26:17.160 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:17 smithi179 podman[303561]: 2024-08-17 18:26:17.159628138 +0000 UTC m=+0.413363235 container attach fd9f10c3375343c2f84a09a918db2bfa20c8b910c5659dac228c2fc5a0eb2f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-08-17T18:26:17.546 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:17 smithi179 podman[303561]: 2024-08-17 18:26:17.245668078 +0000 UTC m=+0.499403182 container died fd9f10c3375343c2f84a09a918db2bfa20c8b910c5659dac228c2fc5a0eb2f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-08-17T18:26:17.791 DEBUG:teuthology.orchestra.run.smithi179:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.6.service' 2024-08-17T18:26:17.814 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:17 smithi179 podman[303603]: 2024-08-17 18:26:17.771378429 +0000 UTC m=+0.519450340 container remove fd9f10c3375343c2f84a09a918db2bfa20c8b910c5659dac228c2fc5a0eb2f6c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=quincy-f2c39d4, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-08-17T18:26:17.815 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:17 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.6.service: Deactivated successfully. 2024-08-17T18:26:17.815 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:17 smithi179 systemd[1]: Stopped Ceph osd.6 for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T18:26:17.815 INFO:journalctl@ceph.osd.6.smithi179.stdout:Aug 17 18:26:17 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.6.service: Consumed 20min 3.434s CPU time. 2024-08-17T18:26:17.815 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:17 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:17.606+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:17.815 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:17 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:17.606+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:17.816 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:17 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:17.606+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:17.816 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:17 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:17.606+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-08-17T18:25:49.533146+0000 front 2024-08-17T18:25:49.533193+0000 (oldest deadline 2024-08-17T18:26:13.033087+0000) 2024-08-17T18:26:17.832 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:26:17.832 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-08-17T18:26:17.832 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-08-17T18:26:17.833 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.7 2024-08-17T18:26:18.093 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:17 smithi179 systemd[1]: Stopping Ceph osd.7 for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:26:18.593 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:18 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:18.184+0000 7f9ceee68640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T18:26:18.593 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:18 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:18.184+0000 7f9ceee68640 -1 osd.7 1549 *** Got signal Terminated *** 2024-08-17T18:26:18.593 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:18 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:18.184+0000 7f9ceee68640 -1 osd.7 1549 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T18:26:19.093 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:18 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:18.603+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:19.093 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:18 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:18.603+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:19.093 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:18 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:18.603+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:19.093 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:18 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:18.603+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-08-17T18:25:49.533146+0000 front 2024-08-17T18:25:49.533193+0000 (oldest deadline 2024-08-17T18:26:13.033087+0000) 2024-08-17T18:26:20.093 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:19 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:19.637+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:20.093 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:19 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:19.637+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:20.093 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:19 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:19.637+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:20.093 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:19 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:19.637+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-08-17T18:25:49.533146+0000 front 2024-08-17T18:25:49.533193+0000 (oldest deadline 2024-08-17T18:26:13.033087+0000) 2024-08-17T18:26:20.994 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:20.620+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:20.995 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:20.620+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:20.995 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:20.620+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:20.995 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:20 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:20.620+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-08-17T18:25:49.533146+0000 front 2024-08-17T18:25:49.533193+0000 (oldest deadline 2024-08-17T18:26:13.033087+0000) 2024-08-17T18:26:22.093 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:21 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:21.598+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:22.094 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:21 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:21.598+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:22.094 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:21 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:21.598+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:22.094 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:21 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:21.598+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-08-17T18:25:49.533146+0000 front 2024-08-17T18:25:49.533193+0000 (oldest deadline 2024-08-17T18:26:13.033087+0000) 2024-08-17T18:26:22.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:22 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:22.550+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6806 osd.0 since back 2024-08-17T18:25:25.229476+0000 front 2024-08-17T18:25:25.229573+0000 (oldest deadline 2024-08-17T18:25:47.529369+0000) 2024-08-17T18:26:22.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:22 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:22.550+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6814 osd.1 since back 2024-08-17T18:25:31.030227+0000 front 2024-08-17T18:25:31.030357+0000 (oldest deadline 2024-08-17T18:25:56.230512+0000) 2024-08-17T18:26:22.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:22 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:22.550+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6822 osd.2 since back 2024-08-17T18:25:42.632092+0000 front 2024-08-17T18:25:42.632145+0000 (oldest deadline 2024-08-17T18:26:04.931988+0000) 2024-08-17T18:26:22.843 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:22 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7[58830]: 2024-08-17T18:26:22.550+0000 7f9cec482640 -1 osd.7 1549 heartbeat_check: no reply from 172.21.15.130:6830 osd.3 since back 2024-08-17T18:25:49.533146+0000 front 2024-08-17T18:25:49.533193+0000 (oldest deadline 2024-08-17T18:26:13.033087+0000) 2024-08-17T18:26:24.175 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:23 smithi179 podman[303687]: 2024-08-17 18:26:23.885458137 +0000 UTC m=+5.827878973 container died 99ae6a861fa0880ee995b1452b49d668a1e1b55c5232a3f3117273e7680b260c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7, RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-08-17T18:26:24.542 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:24 smithi179 podman[303687]: 2024-08-17 18:26:24.265775975 +0000 UTC m=+6.208196801 container cleanup 99ae6a861fa0880ee995b1452b49d668a1e1b55c5232a3f3117273e7680b260c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=quincy-f2c39d4, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-08-17T18:26:24.543 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:24 smithi179 bash[303687]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7 2024-08-17T18:26:24.842 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:24 smithi179 podman[303719]: 2024-08-17 18:26:24.542166247 +0000 UTC m=+0.653711667 container remove 99ae6a861fa0880ee995b1452b49d668a1e1b55c5232a3f3117273e7680b260c (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7, org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-08-17T18:26:25.220 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:24 smithi179 podman[303810]: 2024-08-17 18:26:24.763563669 +0000 UTC m=+0.017445203 image pull f69471c56189b063ba7a049adea817a12ba1a40187d11cab5cbef8f3aac57cce quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251 2024-08-17T18:26:25.220 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:24 smithi179 podman[303810]: 2024-08-17 18:26:24.884656617 +0000 UTC m=+0.138538183 container create a19f34591f05ea4ceb94d638edfece4e560ae6c4a9a4f1b858b741847926a4d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7-deactivate, io.buildah.version=1.36.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=quincy-f2c39d4, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-08-17T18:26:25.220 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:25 smithi179 podman[303810]: 2024-08-17 18:26:25.086192974 +0000 UTC m=+0.340074495 container init a19f34591f05ea4ceb94d638edfece4e560ae6c4a9a4f1b858b741847926a4d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7-deactivate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=quincy-f2c39d4, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.36.0) 2024-08-17T18:26:25.221 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:25 smithi179 podman[303810]: 2024-08-17 18:26:25.090952716 +0000 UTC m=+0.344834245 container start a19f34591f05ea4ceb94d638edfece4e560ae6c4a9a4f1b858b741847926a4d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7-deactivate, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-f2c39d4, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-08-17T18:26:25.221 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:25 smithi179 podman[303810]: 2024-08-17 18:26:25.219986088 +0000 UTC m=+0.473867620 container attach a19f34591f05ea4ceb94d638edfece4e560ae6c4a9a4f1b858b741847926a4d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-f2c39d4, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T18:26:25.565 INFO:journalctl@ceph.osd.7.smithi179.stdout:Aug 17 18:26:25 smithi179 podman[303810]: 2024-08-17 18:26:25.272921201 +0000 UTC m=+0.526802735 container died a19f34591f05ea4ceb94d638edfece4e560ae6c4a9a4f1b858b741847926a4d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8be70e7cb4b008d7f8b5e1c4866ccf4eb46a6d6ad871f778e5b065881df65251, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, RELEASE=quincy-f2c39d4) 2024-08-17T18:26:25.869 DEBUG:teuthology.orchestra.run.smithi179:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@osd.7.service' 2024-08-17T18:26:25.913 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:26:25.913 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-08-17T18:26:25.913 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-08-17T18:26:25.913 DEBUG:teuthology.orchestra.run.smithi179:> sudo systemctl stop ceph-25127030-5cbc-11ef-bccf-c7b262605968@prometheus.a 2024-08-17T18:26:26.253 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:25 smithi179 systemd[1]: Stopping Ceph prometheus.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:26:26.525 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:26:26.253Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T18:26:26.525 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:26:26.253Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-08-17T18:26:26.525 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:26:26.253Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-08-17T18:26:26.525 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:26:26.253Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T18:26:26.526 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:26:26.253Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-08-17T18:26:26.526 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:26:26.253Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-08-17T18:26:26.526 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:26:26.253Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T18:26:26.526 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:26:26.253Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-08-17T18:26:26.526 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:26:26.253Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-08-17T18:26:26.526 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:26:26.256Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T18:26:26.526 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:26:26.256Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-08-17T18:26:26.527 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a[83132]: ts=2024-08-17T18:26:26.256Z caller=main.go:1273 level=info msg="See you next time!" 2024-08-17T18:26:26.527 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 podman[303936]: 2024-08-17 18:26:26.263996819 +0000 UTC m=+0.118378503 container died 145f77b6fc933e25c44dde4b744667240605594338e857c8de6b5dac9f44f7d0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T18:26:26.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 podman[303936]: 2024-08-17 18:26:26.609616356 +0000 UTC m=+0.463998041 container cleanup 145f77b6fc933e25c44dde4b744667240605594338e857c8de6b5dac9f44f7d0 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T18:26:26.786 INFO:journalctl@ceph.prometheus.a.smithi179.stdout:Aug 17 18:26:26 smithi179 bash[303936]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T18:26:27.009 DEBUG:teuthology.orchestra.run.smithi179:> sudo pkill -f 'journalctl -f -n 0 -u ceph-25127030-5cbc-11ef-bccf-c7b262605968@prometheus.a.service' 2024-08-17T18:26:27.043 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T18:26:27.043 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-08-17T18:26:27.043 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 25127030-5cbc-11ef-bccf-c7b262605968 --force --keep-logs 2024-08-17T18:26:28.917 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 18:26:28 smithi130 systemd[1]: Stopping Ceph alertmanager.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:26:29.263 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 18:26:28 smithi130 ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a[93533]: ts=2024-08-17T18:26:28.936Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-17T18:26:29.264 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 18:26:28 smithi130 podman[448511]: 2024-08-17 18:26:28.988486984 +0000 UTC m=+0.144501069 container died 73ea4ebcf68b62371b6f3a9b602f4da8a4a5d8caf3bf52b41612736d8edb2256 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T18:26:29.666 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 18:26:29 smithi130 podman[448511]: 2024-08-17 18:26:29.459892679 +0000 UTC m=+0.615906756 container cleanup 73ea4ebcf68b62371b6f3a9b602f4da8a4a5d8caf3bf52b41612736d8edb2256 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T18:26:29.667 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 18:26:29 smithi130 bash[448511]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a 2024-08-17T18:26:30.145 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 18:26:29 smithi130 podman[448529]: 2024-08-17 18:26:29.752775732 +0000 UTC m=+0.762708694 container remove 73ea4ebcf68b62371b6f3a9b602f4da8a4a5d8caf3bf52b41612736d8edb2256 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T18:26:30.145 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 18:26:29 smithi130 podman[448529]: 2024-08-17 18:26:29.837481912 +0000 UTC m=+0.847414874 volume remove 1a974e46d1a594913a189ebbcebde784db90732e8d8be8a5f4d9580f03fa0bcf 2024-08-17T18:26:30.145 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 18:26:29 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-17T18:26:30.146 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 18:26:29 smithi130 systemd[1]: Stopped Ceph alertmanager.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T18:26:30.146 INFO:journalctl@ceph.alertmanager.a.smithi130.stdout:Aug 17 18:26:29 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@alertmanager.a.service: Consumed 5.093s CPU time. 2024-08-17T18:26:30.410 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 18:26:30 smithi130 systemd[1]: Stopping Ceph node-exporter.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:26:30.666 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 18:26:30 smithi130 podman[448640]: 2024-08-17 18:26:30.410399307 +0000 UTC m=+0.123650847 container died fd7392d87597789984f045c0c3d7a948955a4b151d8280d745ff2da3cbe23f5a (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T18:26:31.049 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 18:26:30 smithi130 podman[448640]: 2024-08-17 18:26:30.797863775 +0000 UTC m=+0.511115313 container cleanup fd7392d87597789984f045c0c3d7a948955a4b151d8280d745ff2da3cbe23f5a (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T18:26:31.049 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 18:26:30 smithi130 bash[448640]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a 2024-08-17T18:26:31.401 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 18:26:31 smithi130 podman[448659]: 2024-08-17 18:26:31.048982084 +0000 UTC m=+0.642262911 container remove fd7392d87597789984f045c0c3d7a948955a4b151d8280d745ff2da3cbe23f5a (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T18:26:31.401 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 18:26:31 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-08-17T18:26:31.401 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 18:26:31 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-08-17T18:26:31.401 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 18:26:31 smithi130 systemd[1]: Stopped Ceph node-exporter.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T18:26:31.401 INFO:journalctl@ceph.node-exporter.a.smithi130.stdout:Aug 17 18:26:31 smithi130 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@node-exporter.a.service: Consumed 10.362s CPU time. 2024-08-17T18:26:39.133 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 25127030-5cbc-11ef-bccf-c7b262605968 --force --keep-logs 2024-08-17T18:26:41.010 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 18:26:40 smithi179 systemd[1]: Stopping Ceph grafana.a for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:26:41.265 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 18:26:41 smithi179 podman[304355]: 2024-08-17 18:26:41.009842485 +0000 UTC m=+0.111230096 container died 399f89cc0037a839cab9284c7043df281fd15a93122b75ef9312b690844c92c2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, io.openshift.expose-services=, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, version=8.5, io.buildah.version=1.32.0, name=ubi8, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, description=Ceph Grafana Container) 2024-08-17T18:26:41.548 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 18:26:41 smithi179 podman[304355]: 2024-08-17 18:26:41.338330898 +0000 UTC m=+0.439718508 container cleanup 399f89cc0037a839cab9284c7043df281fd15a93122b75ef9312b690844c92c2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, release=239.1651231664, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, io.openshift.tags=base rhel8, vcs-type=git, distribution-scope=public, description=Ceph Grafana Container, com.redhat.component=ubi8-container) 2024-08-17T18:26:41.548 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 18:26:41 smithi179 bash[304355]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a 2024-08-17T18:26:41.842 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 18:26:41 smithi179 podman[304373]: 2024-08-17 18:26:41.547817088 +0000 UTC m=+0.537136479 container remove 399f89cc0037a839cab9284c7043df281fd15a93122b75ef9312b690844c92c2 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-grafana-a, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, release=239.1651231664, version=8.5, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, io.openshift.expose-services=, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-08-17T18:26:41.843 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 18:26:41 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-08-17T18:26:41.843 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 18:26:41 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-08-17T18:26:41.843 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 18:26:41 smithi179 systemd[1]: Stopped Ceph grafana.a for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T18:26:41.843 INFO:journalctl@ceph.grafana.a.smithi179.stdout:Aug 17 18:26:41 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@grafana.a.service: Consumed 37.498s CPU time. 2024-08-17T18:26:42.205 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 18:26:41 smithi179 systemd[1]: Stopping Ceph node-exporter.b for 25127030-5cbc-11ef-bccf-c7b262605968... 2024-08-17T18:26:42.491 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 18:26:42 smithi179 podman[304486]: 2024-08-17 18:26:42.205124217 +0000 UTC m=+0.108088870 container died c10588f17ed83aab23e10a7e4dabc369e00fe54a6f76dd2b3fbfb21b4526ff9f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T18:26:42.492 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 18:26:42 smithi179 podman[304486]: 2024-08-17 18:26:42.491852099 +0000 UTC m=+0.394816752 container cleanup c10588f17ed83aab23e10a7e4dabc369e00fe54a6f76dd2b3fbfb21b4526ff9f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T18:26:42.814 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 18:26:42 smithi179 bash[304486]: ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b 2024-08-17T18:26:42.815 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 18:26:42 smithi179 podman[304503]: 2024-08-17 18:26:42.701490568 +0000 UTC m=+0.498039897 container remove c10588f17ed83aab23e10a7e4dabc369e00fe54a6f76dd2b3fbfb21b4526ff9f (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-25127030-5cbc-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T18:26:42.815 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 18:26:42 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-08-17T18:26:43.092 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 18:26:42 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-08-17T18:26:43.093 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 18:26:42 smithi179 systemd[1]: Stopped Ceph node-exporter.b for 25127030-5cbc-11ef-bccf-c7b262605968. 2024-08-17T18:26:43.093 INFO:journalctl@ceph.node-exporter.b.smithi179.stdout:Aug 17 18:26:42 smithi179 systemd[1]: ceph-25127030-5cbc-11ef-bccf-c7b262605968@node-exporter.b.service: Consumed 9.040s CPU time. 2024-08-17T18:26:48.904 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T18:26:48.938 DEBUG:teuthology.orchestra.run.smithi179:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T18:26:48.965 INFO:tasks.cephadm:Archiving crash dumps... 2024-08-17T18:26:48.967 DEBUG:teuthology.misc:Transferring archived files from smithi130:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858336/remote/smithi130/crash 2024-08-17T18:26:48.968 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/crash -- . 2024-08-17T18:26:49.018 INFO:teuthology.orchestra.run.smithi130.stderr:tar: /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/crash: Cannot open: No such file or directory 2024-08-17T18:26:49.019 INFO:teuthology.orchestra.run.smithi130.stderr:tar: Error is not recoverable: exiting now 2024-08-17T18:26:49.020 DEBUG:teuthology.misc:Transferring archived files from smithi179:/var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858336/remote/smithi179/crash 2024-08-17T18:26:49.021 DEBUG:teuthology.orchestra.run.smithi179:> sudo tar c -f - -C /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/crash -- . 2024-08-17T18:26:49.058 INFO:teuthology.orchestra.run.smithi179.stderr:tar: /var/lib/ceph/25127030-5cbc-11ef-bccf-c7b262605968/crash: Cannot open: No such file or directory 2024-08-17T18:26:49.059 INFO:teuthology.orchestra.run.smithi179.stderr:tar: Error is not recoverable: exiting now 2024-08-17T18:26:49.059 INFO:tasks.cephadm:Checking cluster log for badness... 2024-08-17T18:26:49.059 DEBUG:teuthology.orchestra.run.smithi130:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v OSD_DOWN | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v OSDMAP_FLAGS | egrep -v PG_AVAILABILITY | head -n 1 2024-08-17T18:26:49.150 INFO:teuthology.orchestra.run.smithi130.stdout:2024-08-17T17:26:06.724584+0000 mon.a (mon.0) 15 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-08-17T18:26:49.154 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-08-17T18:26:49.155 INFO:tasks.cephadm:Compressing logs... 2024-08-17T18:26:49.155 DEBUG:teuthology.orchestra.run.smithi130:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-08-17T18:26:49.157 DEBUG:teuthology.orchestra.run.smithi179:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-08-17T18:26:49.192 INFO:teuthology.orchestra.run.smithi130.stderr:find: ‘/var/log/rbd-target-api’gzip: No such file or directory -5 2024-08-17T18:26:49.192 INFO:teuthology.orchestra.run.smithi130.stderr: --verbose -- /var/log/ceph/cephadm.log 2024-08-17T18:26:49.192 INFO:teuthology.orchestra.run.smithi130.stderr:gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mds.foo.smithi130.jmaidg.log 2024-08-17T18:26:49.192 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.log 2024-08-17T18:26:49.192 INFO:teuthology.orchestra.run.smithi130.stderr:gzip/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mds.foo.smithi130.jmaidg.log: -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.2.log 2024-08-17T18:26:49.193 INFO:teuthology.orchestra.run.smithi130.stderr: 82.1% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mds.foo.smithi130.jmaidg.log.gz 2024-08-17T18:26:49.193 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.audit.log 2024-08-17T18:26:49.193 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.3.log 2024-08-17T18:26:49.193 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.1.log 2024-08-17T18:26:49.193 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mgr.y.log 2024-08-17T18:26:49.193 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.0.log 2024-08-17T18:26:49.195 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mon.c.log 2024-08-17T18:26:49.196 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mon.a.log 2024-08-17T18:26:49.196 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.cephadm.log 2024-08-17T18:26:49.196 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-volume.log 2024-08-17T18:26:49.198 INFO:teuthology.orchestra.run.smithi179.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose --: No such file or directory /var/log/ceph/cephadm.log 2024-08-17T18:26:49.198 INFO:teuthology.orchestra.run.smithi179.stderr: 2024-08-17T18:26:49.202 INFO:teuthology.orchestra.run.smithi179.stderr:gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.log 2024-08-17T18:26:49.202 INFO:teuthology.orchestra.run.smithi179.stderr:gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.audit.log 2024-08-17T18:26:49.202 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.7.log 2024-08-17T18:26:49.202 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.log: /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mds.foo.smithi179.zkkvno.log 2024-08-17T18:26:49.202 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.5.log 2024-08-17T18:26:49.203 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mds.foo.smithi179.zkkvno.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.4.log 2024-08-17T18:26:49.203 INFO:teuthology.orchestra.run.smithi179.stderr: 62.1% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mds.foo.smithi179.zkkvno.log.gz 2024-08-17T18:26:49.203 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mgr.x.log 2024-08-17T18:26:49.203 INFO:teuthology.orchestra.run.smithi179.stderr:gzip/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.4.log: -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.6.log 2024-08-17T18:26:49.203 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.cephadm.log 2024-08-17T18:26:49.203 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-volume.log 2024-08-17T18:26:49.203 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mon.b.log 2024-08-17T18:26:49.206 INFO:teuthology.orchestra.run.smithi130.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.cephadm.log: /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-volume.log: 92.7% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.log.gz 2024-08-17T18:26:49.295 INFO:teuthology.orchestra.run.smithi130.stderr: 95.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-08-17T18:26:49.306 INFO:teuthology.orchestra.run.smithi179.stderr:/var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-volume.log: /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mon.b.log: 87.1% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.log.gz 2024-08-17T18:26:49.375 INFO:teuthology.orchestra.run.smithi179.stderr: 89.9% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.audit.log.gz 2024-08-17T18:26:49.444 INFO:teuthology.orchestra.run.smithi130.stderr: 93.9% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.audit.log.gz 2024-08-17T18:26:49.448 INFO:teuthology.orchestra.run.smithi130.stderr: 90.9% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.cephadm.log.gz 2024-08-17T18:26:49.449 INFO:teuthology.orchestra.run.smithi179.stderr: 84.1% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph.cephadm.log.gz 2024-08-17T18:26:49.482 INFO:teuthology.orchestra.run.smithi179.stderr: 92.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-08-17T18:26:49.545 INFO:teuthology.orchestra.run.smithi179.stderr: 91.3% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-volume.log.gz 2024-08-17T18:26:49.677 INFO:teuthology.orchestra.run.smithi130.stderr: 91.2% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-volume.log.gz 2024-08-17T18:26:49.956 INFO:teuthology.orchestra.run.smithi179.stderr: 90.2% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mgr.x.log.gz 2024-08-17T18:26:58.718 INFO:teuthology.orchestra.run.smithi179.stderr: 92.2% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mon.b.log.gz 2024-08-17T18:26:58.822 INFO:teuthology.orchestra.run.smithi130.stderr: 89.6% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mgr.y.log.gz 2024-08-17T18:27:22.512 INFO:teuthology.orchestra.run.smithi130.stderr: 91.4% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mon.c.log.gz 2024-08-17T18:27:34.064 INFO:teuthology.orchestra.run.smithi130.stderr: 90.5% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-mon.a.log.gz 2024-08-17T18:27:45.165 INFO:teuthology.orchestra.run.smithi179.stderr: 94.4% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.5.log.gz 2024-08-17T18:27:52.096 INFO:teuthology.orchestra.run.smithi179.stderr: 93.4% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.4.log.gz 2024-08-17T18:30:24.198 INFO:teuthology.orchestra.run.smithi179.stderr: 93.0% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.6.log.gz 2024-08-17T18:31:42.753 INFO:teuthology.orchestra.run.smithi179.stderr: 93.1% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.7.log.gz 2024-08-17T18:31:42.755 INFO:teuthology.orchestra.run.smithi179.stderr: 2024-08-17T18:31:42.755 INFO:teuthology.orchestra.run.smithi179.stderr:real 4m53.582s 2024-08-17T18:31:42.755 INFO:teuthology.orchestra.run.smithi179.stderr:user 4m47.404s 2024-08-17T18:31:42.755 INFO:teuthology.orchestra.run.smithi179.stderr:sys 0m11.685s 2024-08-17T18:49:38.539 INFO:teuthology.orchestra.run.smithi130.stderr: 93.4% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.2.log.gz 2024-08-17T18:52:03.072 INFO:teuthology.orchestra.run.smithi130.stderr: 93.7% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.1.log.gz 2024-08-17T18:56:49.194 INFO:teuthology.orchestra.run.smithi130.stderr: 93.2% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.0.log.gz 2024-08-17T18:57:10.974 INFO:teuthology.orchestra.run.smithi130.stderr: 92.9% -- replaced with /var/log/ceph/25127030-5cbc-11ef-bccf-c7b262605968/ceph-osd.3.log.gz 2024-08-17T18:57:11.015 INFO:teuthology.orchestra.run.smithi130.stderr: 2024-08-17T18:57:11.016 INFO:teuthology.orchestra.run.smithi130.stderr:real 30m21.845s 2024-08-17T18:57:11.016 INFO:teuthology.orchestra.run.smithi130.stderr:user 10m2.247s 2024-08-17T18:57:11.016 INFO:teuthology.orchestra.run.smithi130.stderr:sys 0m31.513s 2024-08-17T18:57:11.016 INFO:tasks.cephadm:Archiving logs... 2024-08-17T18:57:11.018 DEBUG:teuthology.misc:Transferring archived files from smithi130:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858336/remote/smithi130/log 2024-08-17T18:57:11.019 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /var/log/ceph -- . 2024-08-17T18:58:42.566 DEBUG:teuthology.misc:Transferring archived files from smithi179:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858336/remote/smithi179/log 2024-08-17T18:58:42.568 DEBUG:teuthology.orchestra.run.smithi179:> sudo tar c -f - -C /var/log/ceph -- . 2024-08-17T18:59:28.153 INFO:tasks.cephadm:Removing cluster... 2024-08-17T18:59:28.154 DEBUG:teuthology.orchestra.run.smithi130:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 25127030-5cbc-11ef-bccf-c7b262605968 --force 2024-08-17T18:59:30.361 DEBUG:teuthology.orchestra.run.smithi179:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 25127030-5cbc-11ef-bccf-c7b262605968 --force 2024-08-17T18:59:31.313 INFO:tasks.cephadm:Removing cephadm ... 2024-08-17T18:59:31.313 DEBUG:teuthology.orchestra.run.smithi130:> rm -rf /home/ubuntu/cephtest/cephadm 2024-08-17T18:59:31.351 DEBUG:teuthology.orchestra.run.smithi179:> rm -rf /home/ubuntu/cephtest/cephadm 2024-08-17T18:59:31.366 INFO:tasks.cephadm:Teardown complete 2024-08-17T18:59:31.367 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-08-17T18:59:31.379 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-08-17T18:59:31.380 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-08-17T18:59:31.394 DEBUG:teuthology.orchestra.run.smithi179:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-08-17T18:59:31.541 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-08-17T18:59:31.551 INFO:teuthology.task.clock:Checking final clock skew... 2024-08-17T18:59:31.551 DEBUG:teuthology.orchestra.run.smithi130:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T18:59:31.554 DEBUG:teuthology.orchestra.run.smithi179:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T18:59:31.569 INFO:teuthology.orchestra.run.smithi179.stderr:bash: line 1: ntpq: command not found 2024-08-17T18:59:31.569 INFO:teuthology.orchestra.run.smithi130.stderr:bash: line 1: ntpq: command not found 2024-08-17T18:59:31.601 INFO:teuthology.orchestra.run.smithi179.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T18:59:31.601 INFO:teuthology.orchestra.run.smithi179.stdout:=============================================================================== 2024-08-17T18:59:31.601 INFO:teuthology.orchestra.run.smithi179.stdout:^* hv01.front.sepia.ceph.com 3 6 377 29 +264us[ +262us] +/- 59ms 2024-08-17T18:59:31.601 INFO:teuthology.orchestra.run.smithi179.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 32 -449us[ -450us] +/- 80ms 2024-08-17T18:59:31.601 INFO:teuthology.orchestra.run.smithi179.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 36 -33us[ -34us] +/- 62ms 2024-08-17T18:59:31.602 INFO:teuthology.orchestra.run.smithi179.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T18:59:31.625 INFO:teuthology.orchestra.run.smithi130.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T18:59:31.625 INFO:teuthology.orchestra.run.smithi130.stdout:=============================================================================== 2024-08-17T18:59:31.625 INFO:teuthology.orchestra.run.smithi130.stdout:^* hv01.front.sepia.ceph.com 3 6 377 34 +268us[ +268us] +/- 59ms 2024-08-17T18:59:31.625 INFO:teuthology.orchestra.run.smithi130.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 32 -456us[ -456us] +/- 80ms 2024-08-17T18:59:31.625 INFO:teuthology.orchestra.run.smithi130.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 31 +46us[ +46us] +/- 62ms 2024-08-17T18:59:31.625 INFO:teuthology.orchestra.run.smithi130.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T18:59:31.651 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-08-17T18:59:31.662 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-08-17T18:59:31.663 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-08-17T18:59:31.673 DEBUG:teuthology.orchestra.run.smithi130:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-08-17T18:59:31.877 DEBUG:teuthology.orchestra.run.smithi179:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-08-17T18:59:32.018 DEBUG:teuthology.orchestra.run.smithi130:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-08-17T18:59:32.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:59:32.051 DEBUG:teuthology.orchestra.run.smithi179:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-08-17T18:59:32.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:59:32.082 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-08-17T18:59:32.092 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-08-17T18:59:32.101 INFO:teuthology.task.internal:Duration was 6718.138511 seconds 2024-08-17T18:59:32.102 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-08-17T18:59:32.111 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-08-17T18:59:32.111 DEBUG:teuthology.orchestra.run.smithi130:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-08-17T18:59:32.114 DEBUG:teuthology.orchestra.run.smithi179:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-08-17T18:59:32.179 INFO:teuthology.orchestra.run.smithi179.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T18:59:32.181 INFO:teuthology.orchestra.run.smithi130.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T18:59:32.519 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-08-17T18:59:32.519 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi130.front.sepia.ceph.com 2024-08-17T18:59:32.519 DEBUG:teuthology.orchestra.run.smithi130:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-08-17T18:59:32.546 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi179.front.sepia.ceph.com 2024-08-17T18:59:32.547 DEBUG:teuthology.orchestra.run.smithi179:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-08-17T18:59:32.576 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-08-17T18:59:32.576 DEBUG:teuthology.orchestra.run.smithi130:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-08-17T18:59:32.589 DEBUG:teuthology.orchestra.run.smithi179:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-08-17T18:59:32.644 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-08-17T18:59:32.644 DEBUG:teuthology.orchestra.run.smithi130:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-08-17T18:59:32.665 DEBUG:teuthology.orchestra.run.smithi179:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-08-17T18:59:34.902 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-08-17T18:59:34.912 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-08-17T18:59:34.913 DEBUG:teuthology.orchestra.run.smithi130:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-08-17T18:59:34.941 DEBUG:teuthology.orchestra.run.smithi179:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-08-17T18:59:34.973 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-08-17T18:59:34.986 DEBUG:teuthology.orchestra.run.smithi130:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:59:34.989 DEBUG:teuthology.orchestra.run.smithi179:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:59:35.025 INFO:teuthology.orchestra.run.smithi130.stdout:kernel.core_pattern = core 2024-08-17T18:59:35.053 INFO:teuthology.orchestra.run.smithi179.stdout:kernel.core_pattern = core 2024-08-17T18:59:35.077 DEBUG:teuthology.orchestra.run.smithi130:> test -e /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:59:35.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:59:35.105 DEBUG:teuthology.orchestra.run.smithi179:> test -e /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:59:35.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:59:35.134 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-08-17T18:59:35.144 INFO:teuthology.task.internal:Transferring archived files... 2024-08-17T18:59:35.145 DEBUG:teuthology.misc:Transferring archived files from smithi130:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858336/remote/smithi130 2024-08-17T18:59:35.145 DEBUG:teuthology.orchestra.run.smithi130:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-08-17T18:59:35.234 DEBUG:teuthology.misc:Transferring archived files from smithi179:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858336/remote/smithi179 2024-08-17T18:59:35.234 DEBUG:teuthology.orchestra.run.smithi179:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-08-17T18:59:35.288 INFO:teuthology.task.internal:Removing archive directory... 2024-08-17T18:59:35.289 DEBUG:teuthology.orchestra.run.smithi130:> rm -rf -- /home/ubuntu/cephtest/archive 2024-08-17T18:59:35.291 DEBUG:teuthology.orchestra.run.smithi179:> rm -rf -- /home/ubuntu/cephtest/archive 2024-08-17T18:59:35.334 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-08-17T18:59:35.345 INFO:teuthology.task.internal:Not uploading archives. 2024-08-17T18:59:35.345 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-08-17T18:59:35.356 INFO:teuthology.task.internal:Tidying up after the test... 2024-08-17T18:59:35.357 DEBUG:teuthology.orchestra.run.smithi130:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-08-17T18:59:35.359 DEBUG:teuthology.orchestra.run.smithi179:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-08-17T18:59:35.382 INFO:teuthology.orchestra.run.smithi130.stdout: 262165 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 17 18:59 /home/ubuntu/cephtest 2024-08-17T18:59:35.382 INFO:teuthology.orchestra.run.smithi130.stdout: 5242891 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 17 18:22 /home/ubuntu/cephtest/mnt.0 2024-08-17T18:59:35.383 INFO:teuthology.orchestra.run.smithi130.stdout: 5242893 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 17 18:22 /home/ubuntu/cephtest/mnt.0/client.0 2024-08-17T18:59:35.383 INFO:teuthology.orchestra.run.smithi130.stdout: 5256381 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Aug 17 18:22 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-08-17T18:59:35.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:59:35.384 INFO:teuthology.orchestra.run.smithi130.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-08-17T18:59:35.384 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi130 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-08-17T18:59:35.391 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-08-17T18:59:35.402 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-08-17T18:59:35.414 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi130 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-08-17T18:59:35.420 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 6718.138511180878 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi130 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=b640226155b54cc8908c032e6e210927 status: fail success: false 2024-08-17T18:59:35.421 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-08-17T18:59:35.450 INFO:teuthology.orchestra.run.smithi179.stdout: 262165 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Aug 17 18:59 /home/ubuntu/cephtest 2024-08-17T18:59:35.533 INFO:teuthology.run:FAIL